From e0b9cb2a8c9c69822526908aa2841adc7f1322bb Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Thu, 19 May 2016 17:46:34 +0100 Subject: [PATCH 0001/1301] support Python 3.5 Signed-off-by: Thomas Grainger --- .travis.yml | 3 +- docker/utils/utils.py | 10 ++---- setup.py | 4 +++ test-requirements.txt | 4 +-- tests/unit/utils_test.py | 70 +++++++++++----------------------------- tox.ini | 4 +-- 6 files changed, 31 insertions(+), 64 deletions(-) diff --git a/.travis.yml b/.travis.yml index abbb578295..fb62a3493a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,12 +1,13 @@ sudo: false language: python python: - - "2.7" + - "3.5" env: - TOX_ENV=py26 - TOX_ENV=py27 - TOX_ENV=py33 - TOX_ENV=py34 + - TOX_ENV=py35 - TOX_ENV=flake8 install: - pip install tox diff --git a/docker/utils/utils.py b/docker/utils/utils.py index 98cbd61f19..caa98314ea 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -546,12 +546,6 @@ def datetime_to_timestamp(dt): return delta.seconds + delta.days * 24 * 3600 -def longint(n): - if six.PY3: - return int(n) - return long(n) - - def parse_bytes(s): if isinstance(s, six.integer_types + (float,)): return s @@ -574,7 +568,7 @@ def parse_bytes(s): if suffix in units.keys() or suffix.isdigit(): try: - digits = longint(digits_part) + digits = int(digits_part) except ValueError: raise errors.DockerException( 'Failed converting the string value for memory ({0}) to' @@ -582,7 +576,7 @@ def parse_bytes(s): ) # Reconvert to long for the final result - s = longint(digits * units[suffix]) + s = int(digits * units[suffix]) else: raise errors.DockerException( 'The specified value for memory ({0}) should specify the' diff --git a/setup.py b/setup.py index fac5129fa5..854271102b 100644 --- a/setup.py +++ b/setup.py @@ -16,6 +16,7 @@ ':python_version < "3.3"': 'ipaddress >= 1.0.16', } +version = None exec(open('docker/version.py').read()) with open('./test-requirements.txt') as test_reqs_txt: @@ -42,10 +43,13 @@ 'Intended Audience :: Developers', 'Operating System :: OS Independent', 'Programming Language :: Python', + 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', 'Topic :: Utilities', 'License :: OSI Approved :: Apache Software License', ], diff --git a/test-requirements.txt b/test-requirements.txt index be4998803f..460db10734 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,5 +1,5 @@ mock==1.0.1 -pytest==2.7.2 +pytest==2.9.1 coverage==3.7.1 pytest-cov==2.1.0 -flake8==2.4.1 \ No newline at end of file +flake8==2.4.1 diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index 47c43ee262..ef927d3676 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -299,56 +299,30 @@ def test_convert_volume_binds_no_mode(self): self.assertEqual(convert_volume_binds(data), ['/mnt/vol1:/data:rw']) def test_convert_volume_binds_unicode_bytes_input(self): - if six.PY2: - expected = [unicode('/mnt/지연:/unicode/박:rw', 'utf-8')] - - data = { - '/mnt/지연': { - 'bind': '/unicode/박', - 'mode': 'rw' - } - } - self.assertEqual( - convert_volume_binds(data), expected - ) - else: - expected = ['/mnt/지연:/unicode/박:rw'] + expected = [u'/mnt/지연:/unicode/박:rw'] - data = { - bytes('/mnt/지연', 'utf-8'): { - 'bind': bytes('/unicode/박', 'utf-8'), - 'mode': 'rw' - } + data = { + u'/mnt/지연'.encode('utf-8'): { + 'bind': u'/unicode/박'.encode('utf-8'), + 'mode': 'rw' } - self.assertEqual( - convert_volume_binds(data), expected - ) + } + self.assertEqual( + convert_volume_binds(data), expected + ) def test_convert_volume_binds_unicode_unicode_input(self): - if six.PY2: - expected = [unicode('/mnt/지연:/unicode/박:rw', 'utf-8')] - - data = { - unicode('/mnt/지연', 'utf-8'): { - 'bind': unicode('/unicode/박', 'utf-8'), - 'mode': 'rw' - } - } - self.assertEqual( - convert_volume_binds(data), expected - ) - else: - expected = ['/mnt/지연:/unicode/박:rw'] + expected = [u'/mnt/지연:/unicode/박:rw'] - data = { - '/mnt/지연': { - 'bind': '/unicode/박', - 'mode': 'rw' - } + data = { + u'/mnt/지연': { + 'bind': u'/unicode/박', + 'mode': 'rw' } - self.assertEqual( - convert_volume_binds(data), expected - ) + } + self.assertEqual( + convert_volume_binds(data), expected + ) class ParseEnvFileTest(base.BaseTestCase): @@ -612,13 +586,7 @@ def test_create_ipam_config(self): class SplitCommandTest(base.BaseTestCase): def test_split_command_with_unicode(self): - if six.PY2: - self.assertEqual( - split_command(unicode('echo μμ', 'utf-8')), - ['echo', 'μμ'] - ) - else: - self.assertEqual(split_command('echo μμ'), ['echo', 'μμ']) + self.assertEqual(split_command(u'echo μμ'), ['echo', 'μμ']) @pytest.mark.skipif(six.PY3, reason="shlex doesn't support bytes in py3") def test_split_command_with_bytes(self): diff --git a/tox.ini b/tox.ini index 40e46fafbb..be4508e42d 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py26, py27, py33, py34, flake8 +envlist = py26, py27, py33, py34, py35, flake8 skipsdist=True [testenv] @@ -11,5 +11,5 @@ deps = -r{toxinidir}/requirements.txt [testenv:flake8] -commands = flake8 docker tests +commands = flake8 docker tests setup.py deps = flake8 From 7e2430493446ec0060e6504452f7ab90f302efab Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Thu, 19 May 2016 18:04:36 +0100 Subject: [PATCH 0002/1301] update docs to py3.5 Signed-off-by: Thomas Grainger --- Dockerfile-py3 | 2 +- docs/api.md | 4 ++-- docs/tls.md | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile-py3 b/Dockerfile-py3 index 4372c055d3..21e713bb39 100644 --- a/Dockerfile-py3 +++ b/Dockerfile-py3 @@ -1,4 +1,4 @@ -FROM python:3.4 +FROM python:3.5 MAINTAINER Joffrey F RUN mkdir /home/docker-py diff --git a/docs/api.md b/docs/api.md index 08131fc201..6e1cd25d63 100644 --- a/docs/api.md +++ b/docs/api.md @@ -109,7 +109,7 @@ correct value (e.g `gzip`). ``` **Raises:** [TypeError]( -https://docs.python.org/3.4/library/exceptions.html#TypeError) if `path` nor +https://docs.python.org/3.5/library/exceptions.html#TypeError) if `path` nor `fileobj` are specified ## commit @@ -207,7 +207,7 @@ of the created container in bytes) or a string with a units identification char character, bytes are assumed as an intended unit. `volumes_from` and `dns` arguments raise [TypeError]( -https://docs.python.org/3.4/library/exceptions.html#TypeError) exception if +https://docs.python.org/3.5/library/exceptions.html#TypeError) exception if they are used against v1.10 and above of the Docker remote API. Those arguments should be passed as part of the `host_config` dictionary. diff --git a/docs/tls.md b/docs/tls.md index 85a22ee357..147e674f76 100644 --- a/docs/tls.md +++ b/docs/tls.md @@ -12,7 +12,7 @@ first.* * ca_cert (str): Path to CA cert file * verify (bool or str): This can be `False` or a path to a CA Cert file * ssl_version (int): A valid [SSL version]( -https://docs.python.org/3.4/library/ssl.html#ssl.PROTOCOL_TLSv1) +https://docs.python.org/3.5/library/ssl.html#ssl.PROTOCOL_TLSv1) * assert_hostname (bool): Verify hostname of docker daemon ### configure_client From 98093544a78878b86d815fb65d15585bf3b972e1 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Thu, 19 May 2016 18:16:04 +0100 Subject: [PATCH 0003/1301] support new Py3.5 Enum signals Signed-off-by: Thomas Grainger --- docker/api/container.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/api/container.py b/docker/api/container.py index da4ac14e3d..b591b17312 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -187,7 +187,7 @@ def kill(self, container, signal=None): url = self._url("/containers/{0}/kill", container) params = {} if signal is not None: - params['signal'] = signal + params['signal'] = int(signal) res = self._post(url, params=params) self._raise_for_status(res) From a34e0cbfaaa72d045a049848f6d55eebabc9a156 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 2 Jun 2016 16:28:52 -0700 Subject: [PATCH 0004/1301] Experimental npipe:// support Signed-off-by: Joffrey F --- docker/client.py | 12 ++ docker/constants.py | 4 + docker/npipeconn/__init__.py | 1 + docker/npipeconn/npipeconn.py | 80 ++++++++++ docker/npipeconn/npipesocket.py | 194 ++++++++++++++++++++++++ docker/npipeconn/test_npipe_echoserv.py | 37 +++++ docker/utils/utils.py | 5 +- 7 files changed, 332 insertions(+), 1 deletion(-) create mode 100644 docker/npipeconn/__init__.py create mode 100644 docker/npipeconn/npipeconn.py create mode 100644 docker/npipeconn/npipesocket.py create mode 100644 docker/npipeconn/test_npipe_echoserv.py diff --git a/docker/client.py b/docker/client.py index de3cb3ca55..ad325071f5 100644 --- a/docker/client.py +++ b/docker/client.py @@ -27,6 +27,10 @@ from . import errors from .auth import auth from .unixconn import unixconn +try: + from .npipeconn import npipeconn +except ImportError: + pass from .ssladapter import ssladapter from .utils import utils, check_resource, update_headers, kwargs_from_env from .tls import TLSConfig @@ -64,6 +68,14 @@ def __init__(self, base_url=None, version=None, self._custom_adapter = unixconn.UnixAdapter(base_url, timeout) self.mount('http+docker://', self._custom_adapter) self.base_url = 'http+docker://localunixsocket' + elif base_url.startswith('npipe://'): + if not constants.IS_WINDOWS_PLATFORM: + raise errors.DockerException( + 'The npipe:// protocol is only supported on Windows' + ) + self._custom_adapter = npipeconn.NpipeAdapter(base_url, timeout) + self.mount('http+docker://', self._custom_adapter) + self.base_url = 'http+docker://localnpipe' else: # Use SSLAdapter for the ability to specify SSL version if isinstance(tls, TLSConfig): diff --git a/docker/constants.py b/docker/constants.py index 6c381de3be..0388f705a7 100644 --- a/docker/constants.py +++ b/docker/constants.py @@ -1,3 +1,5 @@ +import sys + DEFAULT_DOCKER_API_VERSION = '1.22' DEFAULT_TIMEOUT_SECONDS = 60 STREAM_HEADER_SIZE_BYTES = 8 @@ -8,3 +10,5 @@ INSECURE_REGISTRY_DEPRECATION_WARNING = \ 'The `insecure_registry` argument to {} ' \ 'is deprecated and non-functional. Please remove it.' + +IS_WINDOWS_PLATFORM = (sys.platform == 'win32') diff --git a/docker/npipeconn/__init__.py b/docker/npipeconn/__init__.py new file mode 100644 index 0000000000..d04bc85284 --- /dev/null +++ b/docker/npipeconn/__init__.py @@ -0,0 +1 @@ +from .npipeconn import NpipeAdapter # flake8: noqa diff --git a/docker/npipeconn/npipeconn.py b/docker/npipeconn/npipeconn.py new file mode 100644 index 0000000000..736ddf675c --- /dev/null +++ b/docker/npipeconn/npipeconn.py @@ -0,0 +1,80 @@ +import six +import requests.adapters + +from .npipesocket import NpipeSocket + +if six.PY3: + import http.client as httplib +else: + import httplib + +try: + import requests.packages.urllib3 as urllib3 +except ImportError: + import urllib3 + + +RecentlyUsedContainer = urllib3._collections.RecentlyUsedContainer + + +class NpipeHTTPConnection(httplib.HTTPConnection, object): + def __init__(self, npipe_path, timeout=60): + super(NpipeHTTPConnection, self).__init__( + 'localhost', timeout=timeout + ) + self.npipe_path = npipe_path + self.timeout = timeout + + def connect(self): + sock = NpipeSocket() + sock.settimeout(self.timeout) + sock.connect(self.npipe_path) + self.sock = sock + + +class NpipeHTTPConnectionPool(urllib3.connectionpool.HTTPConnectionPool): + def __init__(self, npipe_path, timeout=60): + super(NpipeHTTPConnectionPool, self).__init__( + 'localhost', timeout=timeout + ) + self.npipe_path = npipe_path + self.timeout = timeout + + def _new_conn(self): + return NpipeHTTPConnection( + self.npipe_path, self.timeout + ) + + +class NpipeAdapter(requests.adapters.HTTPAdapter): + def __init__(self, base_url, timeout=60): + self.npipe_path = base_url.replace('npipe://', '') + self.timeout = timeout + self.pools = RecentlyUsedContainer( + 10, dispose_func=lambda p: p.close() + ) + super(NpipeAdapter, self).__init__() + + def get_connection(self, url, proxies=None): + with self.pools.lock: + pool = self.pools.get(url) + if pool: + return pool + + pool = NpipeHTTPConnectionPool( + self.npipe_path, self.timeout + ) + self.pools[url] = pool + + return pool + + def request_url(self, request, proxies): + # The select_proxy utility in requests errors out when the provided URL + # doesn't have a hostname, like is the case when using a UNIX socket. + # Since proxies are an irrelevant notion in the case of UNIX sockets + # anyway, we simply return the path URL directly. + # See also: https://github.com/docker/docker-py/issues/811 + return request.path_url + + def close(self): + self.pools.clear() diff --git a/docker/npipeconn/npipesocket.py b/docker/npipeconn/npipesocket.py new file mode 100644 index 0000000000..a4469f9895 --- /dev/null +++ b/docker/npipeconn/npipesocket.py @@ -0,0 +1,194 @@ +import functools +import io + +import win32file +import win32pipe + +cSECURITY_SQOS_PRESENT = 0x100000 +cSECURITY_ANONYMOUS = 0 +cPIPE_READMODE_MESSAGE = 2 + + +def check_closed(f): + @functools.wraps(f) + def wrapped(self, *args, **kwargs): + if self._closed: + raise RuntimeError( + 'Can not reuse socket after connection was closed.' + ) + return f(self, *args, **kwargs) + return wrapped + + +class NpipeSocket(object): + """ Partial implementation of the socket API over windows named pipes. + This implementation is only designed to be used as a client socket, + and server-specific methods (bind, listen, accept...) are not + implemented. + """ + def __init__(self, handle=None): + self._timeout = win32pipe.NMPWAIT_USE_DEFAULT_WAIT + self._handle = handle + self._closed = False + + def accept(self): + raise NotImplementedError() + + def bind(self, address): + raise NotImplementedError() + + def close(self): + self._handle.Close() + self._closed = True + + @check_closed + def connect(self, address): + win32pipe.WaitNamedPipe(address, self._timeout) + handle = win32file.CreateFile( + address, + win32file.GENERIC_READ | win32file.GENERIC_WRITE, + 0, + None, + win32file.OPEN_EXISTING, + cSECURITY_ANONYMOUS | cSECURITY_SQOS_PRESENT, + 0 + ) + self.flags = win32pipe.GetNamedPipeInfo(handle)[0] + # self.state = win32pipe.GetNamedPipeHandleState(handle)[0] + + # if self.state & cPIPE_READMODE_MESSAGE != 0: + # raise RuntimeError("message readmode pipes not supported") + self._handle = handle + self._address = address + + @check_closed + def connect_ex(self, address): + return self.connect(address) + + @check_closed + def detach(self): + self._closed = True + return self._handle + + @check_closed + def dup(self): + return NpipeSocket(self._handle) + + @check_closed + def fileno(self): + return int(self._handle) + + def getpeername(self): + return self._address + + def getsockname(self): + return self._address + + def getsockopt(self, level, optname, buflen=None): + raise NotImplementedError() + + def ioctl(self, control, option): + raise NotImplementedError() + + def listen(self, backlog): + raise NotImplementedError() + + def makefile(self, mode=None, bufsize=None): + if mode.strip('b') != 'r': + raise NotImplementedError() + rawio = NpipeFileIOBase(self) + if bufsize is None: + bufsize = io.DEFAULT_BUFFER_SIZE + return io.BufferedReader(rawio, buffer_size=bufsize) + + @check_closed + def recv(self, bufsize, flags=0): + err, data = win32file.ReadFile(self._handle, bufsize) + return data + + @check_closed + def recvfrom(self, bufsize, flags=0): + data = self.recv(bufsize, flags) + return (data, self._address) + + @check_closed + def recvfrom_into(self, buf, nbytes=0, flags=0): + return self.recv_into(buf, nbytes, flags), self._address + + @check_closed + def recv_into(self, buf, nbytes=0): + readbuf = buf + if not isinstance(buf, memoryview): + readbuf = memoryview(buf) + + err, data = win32file.ReadFile( + self._handle, + readbuf[:nbytes] if nbytes else readbuf + ) + return len(data) + + @check_closed + def send(self, string, flags=0): + err, nbytes = win32file.WriteFile(self._handle, string) + return nbytes + + @check_closed + def sendall(self, string, flags=0): + return self.send(string, flags) + + @check_closed + def sendto(self, string, address): + self.connect(address) + return self.send(string) + + def setblocking(self, flag): + if flag: + return self.settimeout(None) + return self.settimeout(0) + + def settimeout(self, value): + if value is None: + self._timeout = win32pipe.NMPWAIT_NOWAIT + elif not isinstance(value, (float, int)) or value < 0: + raise ValueError('Timeout value out of range') + elif value == 0: + self._timeout = win32pipe.NMPWAIT_USE_DEFAULT_WAIT + else: + self._timeout = value + + def gettimeout(self): + return self._timeout + + def setsockopt(self, level, optname, value): + raise NotImplementedError() + + @check_closed + def shutdown(self, how): + return self.close() + + +class NpipeFileIOBase(io.RawIOBase): + def __init__(self, npipe_socket): + self.sock = npipe_socket + + def close(self): + super(NpipeFileIOBase, self).close() + self.sock = None + + def fileno(self): + return self.sock.fileno() + + def isatty(self): + return False + + def readable(self): + return True + + def readinto(self, buf): + return self.sock.recv_into(buf) + + def seekable(self): + return False + + def writable(self): + return False diff --git a/docker/npipeconn/test_npipe_echoserv.py b/docker/npipeconn/test_npipe_echoserv.py new file mode 100644 index 0000000000..6d8382bebb --- /dev/null +++ b/docker/npipeconn/test_npipe_echoserv.py @@ -0,0 +1,37 @@ +import win32pipe +import win32file + +import random + +def pipe_name(): + return 'testpipe{}'.format(random.randint(0, 4096)) + +def create_pipe(name): + handle = win32pipe.CreateNamedPipe( + '//./pipe/{}'.format(name), + win32pipe.PIPE_ACCESS_DUPLEX, + win32pipe.PIPE_TYPE_BYTE | win32pipe.PIPE_READMODE_BYTE | win32pipe.PIPE_WAIT, + 128, 4096, 4096, 300, None + ) + return handle + +def rw_loop(pipe): + err = win32pipe.ConnectNamedPipe(pipe, None) + if err != 0: + raise RuntimeError('Error code: {}'.format(err)) + while True: + err, data = win32file.ReadFile(pipe, 4096, None) + if err != 0: + raise RuntimeError('Error code: {}'.format(err)) + print('Data received: ', data, len(data)) + win32file.WriteFile(pipe, b'ACK', None) + + +def __main__(): + name = pipe_name() + print('Initializing pipe {}'.format(name)) + pipe = create_pipe(name) + print('Pipe created, entering server loop.') + rw_loop(pipe) + +__main__() \ No newline at end of file diff --git a/docker/utils/utils.py b/docker/utils/utils.py index caa98314ea..ab0fdf94a6 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -413,6 +413,9 @@ def parse_host(addr, platform=None, tls=False): elif addr.startswith('https://'): proto = "https" addr = addr[8:] + elif addr.startswith('npipe://'): + proto = 'npipe' + addr = addr[8:] elif addr.startswith('fd://'): raise errors.DockerException("fd protocol is not implemented") else: @@ -448,7 +451,7 @@ def parse_host(addr, platform=None, tls=False): else: host = addr - if proto == "http+unix": + if proto == "http+unix" or proto == 'npipe': return "{0}://{1}".format(proto, host) return "{0}://{1}:{2}{3}".format(proto, host, port, path) From 4a8832ca8b98fb80397831422660dc7d525d0188 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 2 Jun 2016 17:21:43 -0700 Subject: [PATCH 0005/1301] pypiwin32 added to requirements Signed-off-by: Joffrey F --- setup.py | 6 ++++++ win32-requirements.txt | 2 ++ 2 files changed, 8 insertions(+) create mode 100644 win32-requirements.txt diff --git a/setup.py b/setup.py index 854271102b..101e45dc20 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,10 @@ #!/usr/bin/env python import os +import sys + from setuptools import setup + ROOT_DIR = os.path.dirname(__file__) SOURCE_DIR = os.path.join(ROOT_DIR) @@ -11,6 +14,9 @@ 'websocket-client >= 0.32.0', ] +if sys.platform == 'win32': + requirements.append('pypiwin32 >= 219') + extras_require = { ':python_version < "3.5"': 'backports.ssl_match_hostname >= 3.5', ':python_version < "3.3"': 'ipaddress >= 1.0.16', diff --git a/win32-requirements.txt b/win32-requirements.txt new file mode 100644 index 0000000000..e77c3d90f8 --- /dev/null +++ b/win32-requirements.txt @@ -0,0 +1,2 @@ +-r requirements.txt +pypiwin32==219 \ No newline at end of file From b5d9312f9a2a175f497d00c38047486e261a55d9 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 2 Jun 2016 17:45:20 -0700 Subject: [PATCH 0006/1301] npipe support cleanup Signed-off-by: Joffrey F --- docker/client.py | 19 ++++++---- docker/npipeconn/test_npipe_echoserv.py | 37 ------------------- docker/transport/__init__.py | 3 ++ .../npipe}/__init__.py | 0 .../npipe}/npipeconn.py | 0 .../npipe}/npipesocket.py | 0 docker/{ => transport}/unixconn/__init__.py | 0 docker/{ => transport}/unixconn/unixconn.py | 0 setup.py | 2 +- 9 files changed, 16 insertions(+), 45 deletions(-) delete mode 100644 docker/npipeconn/test_npipe_echoserv.py create mode 100644 docker/transport/__init__.py rename docker/{npipeconn => transport/npipe}/__init__.py (100%) rename docker/{npipeconn => transport/npipe}/npipeconn.py (100%) rename docker/{npipeconn => transport/npipe}/npipesocket.py (100%) rename docker/{ => transport}/unixconn/__init__.py (100%) rename docker/{ => transport}/unixconn/unixconn.py (100%) diff --git a/docker/client.py b/docker/client.py index ad325071f5..2da355a559 100644 --- a/docker/client.py +++ b/docker/client.py @@ -26,14 +26,14 @@ from . import constants from . import errors from .auth import auth -from .unixconn import unixconn +from .ssladapter import ssladapter +from .tls import TLSConfig +from .transport import UnixAdapter +from .utils import utils, check_resource, update_headers, kwargs_from_env try: - from .npipeconn import npipeconn + from .transport import NpipeAdapter except ImportError: pass -from .ssladapter import ssladapter -from .utils import utils, check_resource, update_headers, kwargs_from_env -from .tls import TLSConfig def from_env(**kwargs): @@ -65,7 +65,7 @@ def __init__(self, base_url=None, version=None, base_url = utils.parse_host(base_url, sys.platform, tls=bool(tls)) if base_url.startswith('http+unix://'): - self._custom_adapter = unixconn.UnixAdapter(base_url, timeout) + self._custom_adapter = UnixAdapter(base_url, timeout) self.mount('http+docker://', self._custom_adapter) self.base_url = 'http+docker://localunixsocket' elif base_url.startswith('npipe://'): @@ -73,7 +73,12 @@ def __init__(self, base_url=None, version=None, raise errors.DockerException( 'The npipe:// protocol is only supported on Windows' ) - self._custom_adapter = npipeconn.NpipeAdapter(base_url, timeout) + try: + self._custom_adapter = NpipeAdapter(base_url, timeout) + except NameError: + raise errors.DockerException( + 'Install pypiwin32 package to enable npipe:// support' + ) self.mount('http+docker://', self._custom_adapter) self.base_url = 'http+docker://localnpipe' else: diff --git a/docker/npipeconn/test_npipe_echoserv.py b/docker/npipeconn/test_npipe_echoserv.py deleted file mode 100644 index 6d8382bebb..0000000000 --- a/docker/npipeconn/test_npipe_echoserv.py +++ /dev/null @@ -1,37 +0,0 @@ -import win32pipe -import win32file - -import random - -def pipe_name(): - return 'testpipe{}'.format(random.randint(0, 4096)) - -def create_pipe(name): - handle = win32pipe.CreateNamedPipe( - '//./pipe/{}'.format(name), - win32pipe.PIPE_ACCESS_DUPLEX, - win32pipe.PIPE_TYPE_BYTE | win32pipe.PIPE_READMODE_BYTE | win32pipe.PIPE_WAIT, - 128, 4096, 4096, 300, None - ) - return handle - -def rw_loop(pipe): - err = win32pipe.ConnectNamedPipe(pipe, None) - if err != 0: - raise RuntimeError('Error code: {}'.format(err)) - while True: - err, data = win32file.ReadFile(pipe, 4096, None) - if err != 0: - raise RuntimeError('Error code: {}'.format(err)) - print('Data received: ', data, len(data)) - win32file.WriteFile(pipe, b'ACK', None) - - -def __main__(): - name = pipe_name() - print('Initializing pipe {}'.format(name)) - pipe = create_pipe(name) - print('Pipe created, entering server loop.') - rw_loop(pipe) - -__main__() \ No newline at end of file diff --git a/docker/transport/__init__.py b/docker/transport/__init__.py new file mode 100644 index 0000000000..ed26f728cc --- /dev/null +++ b/docker/transport/__init__.py @@ -0,0 +1,3 @@ +# flake8: noqa +from .npipe import NpipeAdapter +from .unixconn import UnixAdapter diff --git a/docker/npipeconn/__init__.py b/docker/transport/npipe/__init__.py similarity index 100% rename from docker/npipeconn/__init__.py rename to docker/transport/npipe/__init__.py diff --git a/docker/npipeconn/npipeconn.py b/docker/transport/npipe/npipeconn.py similarity index 100% rename from docker/npipeconn/npipeconn.py rename to docker/transport/npipe/npipeconn.py diff --git a/docker/npipeconn/npipesocket.py b/docker/transport/npipe/npipesocket.py similarity index 100% rename from docker/npipeconn/npipesocket.py rename to docker/transport/npipe/npipesocket.py diff --git a/docker/unixconn/__init__.py b/docker/transport/unixconn/__init__.py similarity index 100% rename from docker/unixconn/__init__.py rename to docker/transport/unixconn/__init__.py diff --git a/docker/unixconn/unixconn.py b/docker/transport/unixconn/unixconn.py similarity index 100% rename from docker/unixconn/unixconn.py rename to docker/transport/unixconn/unixconn.py diff --git a/setup.py b/setup.py index 101e45dc20..ac58b1f94c 100644 --- a/setup.py +++ b/setup.py @@ -35,7 +35,7 @@ description="Python client for Docker.", url='https://github.com/docker/docker-py/', packages=[ - 'docker', 'docker.api', 'docker.auth', 'docker.unixconn', + 'docker', 'docker.api', 'docker.auth', 'docker.transport', 'docker.utils', 'docker.utils.ports', 'docker.ssladapter' ], install_requires=requirements, From d9227139238a824fcbab2229ea62e1dab64150d1 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 2 Jun 2016 17:52:32 -0700 Subject: [PATCH 0007/1301] Reorganize docker.transport package Signed-off-by: Joffrey F --- docker/transport/__init__.py | 5 ++++- docker/transport/npipe/__init__.py | 1 - docker/transport/{npipe => }/npipeconn.py | 0 docker/transport/{npipe => }/npipesocket.py | 0 docker/transport/{unixconn => }/unixconn.py | 0 docker/transport/unixconn/__init__.py | 1 - 6 files changed, 4 insertions(+), 3 deletions(-) delete mode 100644 docker/transport/npipe/__init__.py rename docker/transport/{npipe => }/npipeconn.py (100%) rename docker/transport/{npipe => }/npipesocket.py (100%) rename docker/transport/{unixconn => }/unixconn.py (100%) delete mode 100644 docker/transport/unixconn/__init__.py diff --git a/docker/transport/__init__.py b/docker/transport/__init__.py index ed26f728cc..d647483e2a 100644 --- a/docker/transport/__init__.py +++ b/docker/transport/__init__.py @@ -1,3 +1,6 @@ # flake8: noqa -from .npipe import NpipeAdapter from .unixconn import UnixAdapter +try: + from .npipeconn import NpipeAdapter +except ImportError: + pass \ No newline at end of file diff --git a/docker/transport/npipe/__init__.py b/docker/transport/npipe/__init__.py deleted file mode 100644 index d04bc85284..0000000000 --- a/docker/transport/npipe/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .npipeconn import NpipeAdapter # flake8: noqa diff --git a/docker/transport/npipe/npipeconn.py b/docker/transport/npipeconn.py similarity index 100% rename from docker/transport/npipe/npipeconn.py rename to docker/transport/npipeconn.py diff --git a/docker/transport/npipe/npipesocket.py b/docker/transport/npipesocket.py similarity index 100% rename from docker/transport/npipe/npipesocket.py rename to docker/transport/npipesocket.py diff --git a/docker/transport/unixconn/unixconn.py b/docker/transport/unixconn.py similarity index 100% rename from docker/transport/unixconn/unixconn.py rename to docker/transport/unixconn.py diff --git a/docker/transport/unixconn/__init__.py b/docker/transport/unixconn/__init__.py deleted file mode 100644 index 53711fc6d8..0000000000 --- a/docker/transport/unixconn/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .unixconn import UnixAdapter # flake8: noqa From 0176fa171f9bb5a4453ceb2abe1cdff1696bee59 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 2 Jun 2016 18:21:29 -0700 Subject: [PATCH 0008/1301] Update parse_host and tests Signed-off-by: Joffrey F --- docker/client.py | 5 +++-- docker/utils/utils.py | 4 ++-- tests/unit/utils_test.py | 7 +++---- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docker/client.py b/docker/client.py index 2da355a559..b96a78ce4e 100644 --- a/docker/client.py +++ b/docker/client.py @@ -14,7 +14,6 @@ import json import struct -import sys import requests import requests.exceptions @@ -63,7 +62,9 @@ def __init__(self, base_url=None, version=None, self._auth_configs = auth.load_config() - base_url = utils.parse_host(base_url, sys.platform, tls=bool(tls)) + base_url = utils.parse_host( + base_url, constants.IS_WINDOWS_PLATFORM, tls=bool(tls) + ) if base_url.startswith('http+unix://'): self._custom_adapter = UnixAdapter(base_url, timeout) self.mount('http+docker://', self._custom_adapter) diff --git a/docker/utils/utils.py b/docker/utils/utils.py index ab0fdf94a6..4a56829d17 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -383,13 +383,13 @@ def parse_repository_tag(repo_name): # fd:// protocol unsupported (for obvious reasons) # Added support for http and https # Protocol translation: tcp -> http, unix -> http+unix -def parse_host(addr, platform=None, tls=False): +def parse_host(addr, is_win32=False, tls=False): proto = "http+unix" host = DEFAULT_HTTP_HOST port = None path = '' - if not addr and platform == 'win32': + if not addr and is_win32: addr = '{0}:{1}'.format(DEFAULT_HTTP_HOST, 2375) if not addr or addr.strip() == 'unix://': diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index ef927d3676..ae821fd35e 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -388,6 +388,7 @@ def test_parse_host(self): 'somehost.net:80/service/swarm': ( 'http://somehost.net:80/service/swarm' ), + 'npipe:////./pipe/docker_engine': 'npipe:////./pipe/docker_engine', } for host in invalid_hosts: @@ -402,10 +403,8 @@ def test_parse_host_empty_value(self): tcp_port = 'http://127.0.0.1:2375' for val in [None, '']: - for platform in ['darwin', 'linux2', None]: - assert parse_host(val, platform) == unix_socket - - assert parse_host(val, 'win32') == tcp_port + assert parse_host(val, is_win32=False) == unix_socket + assert parse_host(val, is_win32=True) == tcp_port def test_parse_host_tls(self): host_value = 'myhost.docker.net:3348' From 896d36ea1d2496beafaca5805f2a869a5fcace50 Mon Sep 17 00:00:00 2001 From: yunzhu-li Date: Tue, 24 May 2016 19:04:10 -0400 Subject: [PATCH 0009/1301] Add support for Block IO constraints in HostConfig This adds support for Block IO constraint options: - blkio-weight - blkio-weight-device - device-read-bps - device-write-bps - device-read-iops - device-write-iops Signed-off-by: yunzhu-li --- docker/utils/utils.py | 60 ++++++++++++++++++++++++++++++++++++++-- docs/hostconfig.md | 8 ++++++ tests/unit/utils_test.py | 19 +++++++++++++ 3 files changed, 85 insertions(+), 2 deletions(-) diff --git a/docker/utils/utils.py b/docker/utils/utils.py index caa98314ea..840ede817b 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -612,8 +612,12 @@ def create_host_config(binds=None, port_bindings=None, lxc_conf=None, security_opt=None, ulimits=None, log_config=None, mem_limit=None, memswap_limit=None, mem_swappiness=None, cgroup_parent=None, group_add=None, cpu_quota=None, - cpu_period=None, oom_kill_disable=False, shm_size=None, - version=None, tmpfs=None, oom_score_adj=None): + cpu_period=None, blkio_weight=None, + blkio_weight_device=None, device_read_bps=None, + device_write_bps=None, device_read_iops=None, + device_write_iops=None, oom_kill_disable=False, + shm_size=None, version=None, tmpfs=None, + oom_score_adj=None): host_config = {} @@ -789,6 +793,58 @@ def create_host_config(binds=None, port_bindings=None, lxc_conf=None, host_config['CpuPeriod'] = cpu_period + if blkio_weight: + if not isinstance(blkio_weight, int): + raise host_config_type_error('blkio_weight', blkio_weight, 'int') + if version_lt(version, '1.22'): + raise host_config_version_error('blkio_weight', '1.22') + host_config["BlkioWeight"] = blkio_weight + + if blkio_weight_device: + if not isinstance(blkio_weight_device, list): + raise host_config_type_error( + 'blkio_weight_device', blkio_weight_device, 'list' + ) + if version_lt(version, '1.22'): + raise host_config_version_error('blkio_weight_device', '1.22') + host_config["BlkioWeightDevice"] = blkio_weight_device + + if device_read_bps: + if not isinstance(device_read_bps, list): + raise host_config_type_error( + 'device_read_bps', device_read_bps, 'list' + ) + if version_lt(version, '1.22'): + raise host_config_version_error('device_read_bps', '1.22') + host_config["BlkioDeviceReadBps"] = device_read_bps + + if device_write_bps: + if not isinstance(device_write_bps, list): + raise host_config_type_error( + 'device_write_bps', device_write_bps, 'list' + ) + if version_lt(version, '1.22'): + raise host_config_version_error('device_write_bps', '1.22') + host_config["BlkioDeviceWriteBps"] = device_write_bps + + if device_read_iops: + if not isinstance(device_read_iops, list): + raise host_config_type_error( + 'device_read_iops', device_read_iops, 'list' + ) + if version_lt(version, '1.22'): + raise host_config_version_error('device_read_iops', '1.22') + host_config["BlkioDeviceReadIOps"] = device_read_iops + + if device_write_iops: + if not isinstance(device_write_iops, list): + raise host_config_type_error( + 'device_write_iops', device_write_iops, 'list' + ) + if version_lt(version, '1.22'): + raise host_config_version_error('device_write_iops', '1.22') + host_config["BlkioDeviceWriteIOps"] = device_write_iops + if tmpfs: if version_lt(version, '1.22'): raise host_config_version_error('tmpfs', '1.22') diff --git a/docs/hostconfig.md b/docs/hostconfig.md index cd96433e0f..c1e23533a5 100644 --- a/docs/hostconfig.md +++ b/docs/hostconfig.md @@ -109,6 +109,14 @@ for example: * cpu_group (int): The length of a CPU period in microseconds. * cpu_period (int): Microseconds of CPU time that the container can get in a CPU period. +* blkio_weight: Block IO weight (relative weight), accepts a weight value between 10 and 1000. +* blkio_weight_device: Block IO weight (relative device weight) in the form of: + `[{"Path": "device_path", "Weight": weight}]` +* device_read_bps: Limit read rate (bytes per second) from a device in the form of: + `[{"Path": "device_path", "Rate": rate}]` +* device_write_bps: Limit write rate (bytes per second) from a device. +* device_read_iops: Limit read rate (IO per second) from a device. +* device_write_iops: Limit write rate (IO per second) from a device. * group_add (list): List of additional group names and/or IDs that the container process will run as. * devices (list): Host device bindings. See [host devices](host-devices.md) diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index ef927d3676..50092bb73b 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -64,6 +64,25 @@ def test_create_host_config_with_cpu_period(self): config = create_host_config(version='1.20', cpu_period=1999) self.assertEqual(config.get('CpuPeriod'), 1999) + def test_create_host_config_with_blkio_constraints(self): + blkio_rate = [{"Path": "/dev/sda", "Rate": 1000}] + config = create_host_config(version='1.22', + blkio_weight=1999, + blkio_weight_device=blkio_rate, + device_read_bps=blkio_rate, + device_write_bps=blkio_rate, + device_read_iops=blkio_rate, + device_write_iops=blkio_rate) + + self.assertEqual(config.get('BlkioWeight'), 1999) + self.assertTrue(config.get('BlkioWeightDevice') is blkio_rate) + self.assertTrue(config.get('BlkioDeviceReadBps') is blkio_rate) + self.assertTrue(config.get('BlkioDeviceWriteBps') is blkio_rate) + self.assertTrue(config.get('BlkioDeviceReadIOps') is blkio_rate) + self.assertTrue(config.get('BlkioDeviceWriteIOps') is blkio_rate) + self.assertEqual(blkio_rate[0]['Path'], "/dev/sda") + self.assertEqual(blkio_rate[0]['Rate'], 1000) + def test_create_host_config_with_shm_size(self): config = create_host_config(version='1.22', shm_size=67108864) self.assertEqual(config.get('ShmSize'), 67108864) From 86d1b8fb83fe1e74109cdd25847208e0538ebf5d Mon Sep 17 00:00:00 2001 From: Massimiliano Pippi Date: Mon, 6 Jun 2016 19:29:09 -0400 Subject: [PATCH 0010/1301] invoke self._result with json=True if decode=True Signed-off-by: Massimiliano Pippi --- docker/client.py | 2 +- tests/unit/api_test.py | 42 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/docker/client.py b/docker/client.py index de3cb3ca55..a7e84a7d85 100644 --- a/docker/client.py +++ b/docker/client.py @@ -235,7 +235,7 @@ def _stream_helper(self, response, decode=False): else: # Response isn't chunked, meaning we probably # encountered an error immediately - yield self._result(response) + yield self._result(response, json=decode) def _multiplexed_buffer_helper(self, response): """A generator of multiplexed data blocks read from a buffered diff --git a/tests/unit/api_test.py b/tests/unit/api_test.py index 23fd191346..263cd693aa 100644 --- a/tests/unit/api_test.py +++ b/tests/unit/api_test.py @@ -22,9 +22,11 @@ import tempfile import threading import time +import io import docker import requests +from requests.packages import urllib3 import six from .. import base @@ -42,7 +44,7 @@ def response(status_code=200, content='', headers=None, reason=None, elapsed=0, - request=None): + request=None, raw=None): res = requests.Response() res.status_code = status_code if not isinstance(content, six.binary_type): @@ -52,6 +54,7 @@ def response(status_code=200, content='', headers=None, reason=None, elapsed=0, res.reason = reason res.elapsed = datetime.timedelta(elapsed) res.request = request + res.raw = raw return res @@ -317,6 +320,43 @@ def test_create_host_config_secopt(self): TypeError, self.client.create_host_config, security_opt='wrong' ) + def test_stream_helper_decoding(self): + status_code, content = fake_api.fake_responses[url_prefix + 'events']() + content_str = json.dumps(content) + if six.PY3: + content_str = content_str.encode('utf-8') + body = io.BytesIO(content_str) + + # mock a stream interface + raw_resp = urllib3.HTTPResponse(body=body) + setattr(raw_resp._fp, 'chunked', True) + setattr(raw_resp._fp, 'chunk_left', len(body.getvalue())-1) + + # pass `decode=False` to the helper + raw_resp._fp.seek(0) + resp = response(status_code=status_code, content=content, raw=raw_resp) + result = next(self.client._stream_helper(resp)) + self.assertEqual(result, content_str) + + # pass `decode=True` to the helper + raw_resp._fp.seek(0) + resp = response(status_code=status_code, content=content, raw=raw_resp) + result = next(self.client._stream_helper(resp, decode=True)) + self.assertEqual(result, content) + + # non-chunked response, pass `decode=False` to the helper + setattr(raw_resp._fp, 'chunked', False) + raw_resp._fp.seek(0) + resp = response(status_code=status_code, content=content, raw=raw_resp) + result = next(self.client._stream_helper(resp)) + self.assertEqual(result, content_str.decode('utf-8')) + + # non-chunked response, pass `decode=True` to the helper + raw_resp._fp.seek(0) + resp = response(status_code=status_code, content=content, raw=raw_resp) + result = next(self.client._stream_helper(resp, decode=True)) + self.assertEqual(result, content) + class StreamTest(base.Cleanup, base.BaseTestCase): def setUp(self): From 7f255cd295668df3521a8583651da28a9fdb224b Mon Sep 17 00:00:00 2001 From: Adam Chainz Date: Wed, 8 Jun 2016 03:46:56 +0100 Subject: [PATCH 0011/1301] Convert readthedocs links for their .org -> .io migration for hosted projects MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit As per [their blog post of the 27th April](https://blog.readthedocs.com/securing-subdomains/) ‘Securing subdomains’: > Starting today, Read the Docs will start hosting projects from subdomains on the domain readthedocs.io, instead of on readthedocs.org. This change addresses some security concerns around site cookies while hosting user generated data on the same domain as our dashboard. Test Plan: Manually visited all the links I’ve modified. Signed-off-by: Adam Chainz --- CONTRIBUTING.md | 2 +- README.md | 2 +- docs/change_log.md | 4 ++-- mkdocs.yml | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 75365c8825..1bd8d42699 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -20,7 +20,7 @@ pip freeze | grep docker-py && python --version && docker version - If possible, steps or a code snippet to reproduce the issue To save yourself time, please be sure to check our -[documentation](http://docker-py.readthedocs.org/) and use the +[documentation](https://docker-py.readthedocs.io/) and use the [search function](https://github.com/docker/docker-py/search) to find out if it has already been addressed, or is currently being looked at. diff --git a/README.md b/README.md index 0a70968da0..bdec785418 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ Documentation [![Documentation Status](https://readthedocs.org/projects/docker-py/badge/?version=latest)](https://readthedocs.org/projects/docker-py/?badge=latest) -[Read the full documentation here](http://docker-py.readthedocs.org/en/latest/). +[Read the full documentation here](https://docker-py.readthedocs.io/en/latest/). The source is available in the `docs/` directory. diff --git a/docs/change_log.md b/docs/change_log.md index be818df56d..6db720ab44 100644 --- a/docs/change_log.md +++ b/docs/change_log.md @@ -213,7 +213,7 @@ Change Log ### Features * Added `utils.parse_env_file` to support env-files. - See [docs](http://docker-py.readthedocs.org/en/latest/api/#create_container) + See [docs](https://docker-py.readthedocs.io/en/latest/api/#create_container) for usage. * Added support for arbitrary log drivers * Added support for URL paths in the docker host URL (`base_url`) @@ -577,7 +577,7 @@ Change Log ### Documentation * Added new MkDocs documentation. Currently hosted on - [ReadTheDocs](http://docker-py.readthedocs.org/en/latest/) + [ReadTheDocs](https://docker-py.readthedocs.io/en/latest/) ### Miscellaneous diff --git a/mkdocs.yml b/mkdocs.yml index dba8fdbd34..67b40893cf 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,7 +1,7 @@ site_name: docker-py Documentation site_description: An API client for Docker written in Python site_favicon: favicon_whale.png -site_url: http://docker-py.readthedocs.org +site_url: https://docker-py.readthedocs.io repo_url: https://github.com/docker/docker-py/ theme: readthedocs pages: From a8746f7a99907f8657698ac42afba41823066386 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 8 Jun 2016 17:02:33 -0700 Subject: [PATCH 0012/1301] Remove obsolete, commented out code Signed-off-by: Joffrey F --- docker/transport/npipesocket.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/docker/transport/npipesocket.py b/docker/transport/npipesocket.py index a4469f9895..35418ef170 100644 --- a/docker/transport/npipesocket.py +++ b/docker/transport/npipesocket.py @@ -54,10 +54,7 @@ def connect(self, address): 0 ) self.flags = win32pipe.GetNamedPipeInfo(handle)[0] - # self.state = win32pipe.GetNamedPipeHandleState(handle)[0] - # if self.state & cPIPE_READMODE_MESSAGE != 0: - # raise RuntimeError("message readmode pipes not supported") self._handle = handle self._address = address From b0505442376fe372be8cb8f302822af166515d0e Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 9 Jun 2016 13:01:29 -0700 Subject: [PATCH 0013/1301] Fix param name for inspect_image in API docs Signed-off-by: Joffrey F --- docs/api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/api.md b/docs/api.md index 6e1cd25d63..e7ab80f705 100644 --- a/docs/api.md +++ b/docs/api.md @@ -606,7 +606,7 @@ Identical to the `docker inspect` command, but only for images. **Params**: -* image_id (str): The image to inspect +* image (str): The image to inspect **Returns** (dict): Nearly the same output as `docker inspect`, just as a single dict From b6fa9862936c6c933280a3bca864d95fb7b906f5 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 7 Jun 2016 19:14:41 -0700 Subject: [PATCH 0014/1301] Add ipv[46]_address params to create_endpoint_config. Update networks documentation with exhaustive API docs Signed-off-by: Joffrey F --- docker/utils/utils.py | 21 +++- docs/api.md | 1 + docs/networks.md | 154 +++++++++++++++++++++++++++++- tests/integration/network_test.py | 61 +++++++++++- 4 files changed, 226 insertions(+), 11 deletions(-) diff --git a/docker/utils/utils.py b/docker/utils/utils.py index caa98314ea..ee48bbac34 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -813,19 +813,30 @@ def create_networking_config(endpoints_config=None): return networking_config -def create_endpoint_config(version, aliases=None, links=None): +def create_endpoint_config(version, aliases=None, links=None, + ipv4_address=None, ipv6_address=None): + if version_lt(version, '1.22'): + raise errors.InvalidVersion( + 'Endpoint config is not supported for API version < 1.22' + ) endpoint_config = {} if aliases: - if version_lt(version, '1.22'): - raise host_config_version_error('endpoint_config.aliases', '1.22') endpoint_config["Aliases"] = aliases if links: - if version_lt(version, '1.22'): - raise host_config_version_error('endpoint_config.links', '1.22') endpoint_config["Links"] = normalize_links(links) + ipam_config = {} + if ipv4_address: + ipam_config['IPv4Address'] = ipv4_address + + if ipv6_address: + ipam_config['IPv6Address'] = ipv6_address + + if ipam_config: + endpoint_config['IPAMConfig'] = ipam_config + return endpoint_config diff --git a/docs/api.md b/docs/api.md index 6e1cd25d63..12967a9b95 100644 --- a/docs/api.md +++ b/docs/api.md @@ -242,6 +242,7 @@ from. Optionally a single string joining container id's with commas * labels (dict or list): A dictionary of name-value labels (e.g. `{"label1": "value1", "label2": "value2"}`) or a list of names of labels to set with empty values (e.g. `["label1", "label2"]`) * volume_driver (str): The name of a volume driver/plugin. * stop_signal (str): The stop signal to use to stop the container (e.g. `SIGINT`). +* networking_config (dict): A [NetworkingConfig](networks.md) dictionary **Returns** (dict): A dictionary with an image 'Id' key and a 'Warnings' key. diff --git a/docs/networks.md b/docs/networks.md index 5a14d38d76..a805c6c73f 100644 --- a/docs/networks.md +++ b/docs/networks.md @@ -1,27 +1,171 @@ # Using Networks +## Network creation + With the release of Docker 1.9 you can now manage custom networks. -Here you can see how to create a network named ```network1``` using the ```bridge``` driver +Here you can see how to create a network named `network1` using +the `bridge` driver ```python docker_client.create_network("network1", driver="bridge") ``` -You can also create more advanced networks with custom IPAM configurations. For example, -setting the subnet to ```192.168.52.0/24``` and gateway to ```192.168.52.254``` +You can also create more advanced networks with custom IPAM configurations. +For example, setting the subnet to `192.168.52.0/24` and gateway address +to `192.168.52.254` ```python -ipam_config = docker.utils.create_ipam_config(subnet='192.168.52.0/24', gateway='192.168.52.254') +ipam_config = docker.utils.create_ipam_config( + subnet='192.168.52.0/24', + gateway='192.168.52.254' +) docker_client.create_network("network1", driver="bridge", ipam=ipam_config) ``` -With Docker 1.10 you can now also create internal networks +By default, when you connect a container to an overlay network, Docker also +connects a bridge network to it to provide external connectivity. If you want +to create an externally isolated overlay network, with Docker 1.10 you can +create an internal network. ```python docker_client.create_network("network1", driver="bridge", internal=True) ``` + +## Container network configuration + +In order to specify which network(s) a container will be connected to and +additional configuration, use the `networking_config` parameter in +`Client.create_container` + +```python +networking_config = docker_client.create_networking_config({ + 'network1': docker_client.create_endpoint_config( + ipv4_address='172.28.0.124', + aliases=['foo', 'bar'], + links=['container2'] + ) +}) + +ctnr = docker_client.create_container( + img, command, networking_config=networking_config +) + +``` + +## Network API documentation + +### Client.create_networking_config + +Create a networking config dictionary to be used as the `networking_config` +parameter in `Client.create_container_config` + +**Params**: + +* endpoints_config (dict): A dictionary of `network_name -> endpoint_config` + relationships. Values should be endpoint config dictionaries created by + `Client.create_endpoint_config`. Defaults to `None` (default config). + +**Returns** A networking config dictionary. + +```python + +docker_client.create_network('network1') + +networking_config = docker_client.create_networking_config({ + 'network1': docker_client.create_endpoint_config() +}) + +container = docker_client.create_container( + img, command, networking_config=networking_config +) +``` + + +### Client.create_endpoint_config + +Create an endpoint config dictionary to be used with +`Client.create_networking_config`. + +**Params**: + +* aliases (list): A list of aliases for this endpoint. Names in that list can + be used within the network to reach the container. Defaults to `None`. +* links (list): A list of links for this endpoint. Containers declared in this + list will be [linked](https://docs.docker.com/engine/userguide/networking/work-with-networks/#linking-containers-in-user-defined-networks) + to this container. Defaults to `None`. +* ipv4_address (str): The IP address of this container on the network, + using the IPv4 protocol. Defaults to `None`. +* ipv6_address (str): The IP address of this container on the network, + using the IPv6 protocol. Defaults to `None`. + +**Returns** An endpoint config dictionary. + +```python +endpoint_config = docker_client.create_endpoint_config( + aliases=['web', 'app'], + links=['app_db'], + ipv4_address='132.65.0.123' +) + +docker_client.create_network('network1') +networking_config = docker_client.create_networking_config({ + 'network1': endpoint_config +}) +container = docker_client.create_container( + img, command, networking_config=networking_config +) +``` +### docker.utils.create_ipam_config + +Create an IPAM (IP Address Management) config dictionary to be used with +`Client.create_network`. + + +**Params**: + +* driver (str): The IPAM driver to use. Defaults to `'default'`. +* pool_configs (list): A list of pool configuration dictionaries as created + by `docker.utils.create_ipam_pool`. Defaults to empty list. + +**Returns** An IPAM config dictionary + +```python +ipam_config = docker.utils.create_ipam_config(driver='default') +network = docker_client.create_network('network1', ipam=ipam_config) +``` + +### docker.utils.create_ipam_pool + +Create an IPAM pool config dictionary to be added to the `pool_configs` param +in `docker.utils.create_ipam_config`. + +**Params**: + +* subnet (str): Custom subnet for this IPAM pool using the CIDR notation. + Defaults to `None`. +* iprange (str): Custom IP range for endpoints in this IPAM pool using the + CIDR notation. Defaults to `None`. +* gateway (str): Custom IP address for the pool's gateway. +* aux_addresses (dict): A dictionary of `key -> ip_address` relationships + specifying auxiliary addresses that need to be allocated by the + IPAM driver. + +**Returns** An IPAM pool config dictionary + +```python +ipam_pool = docker.utils.create_ipam_pool( + subnet='124.42.0.0/16', + iprange='124.42.0.0/8', + gateway='124.42.0.254', + aux_addresses={ + 'reserved1': '124.42.1.1' + } +) +ipam_config = docker.utils.create_ipam_config(pool_configs=[ipam_pool]) +network = docker_client.create_network('network1', ipam=ipam_config) +``` diff --git a/tests/integration/network_test.py b/tests/integration/network_test.py index 179ae88e66..26d27a5a83 100644 --- a/tests/integration/network_test.py +++ b/tests/integration/network_test.py @@ -185,7 +185,66 @@ def test_create_with_aliases(self): container_data = self.client.inspect_container(container) self.assertEqual( container_data['NetworkSettings']['Networks'][net_name]['Aliases'], - ['foo', 'bar']) + ['foo', 'bar'] + ) + + @requires_api_version('1.22') + def test_create_with_ipv4_address(self): + net_name, net_id = self.create_network( + ipam=create_ipam_config( + driver='default', + pool_configs=[create_ipam_pool(subnet="132.124.0.0/16")], + ), + ) + container = self.client.create_container( + image='busybox', command='top', + host_config=self.client.create_host_config(network_mode=net_name), + networking_config=self.client.create_networking_config({ + net_name: self.client.create_endpoint_config( + ipv4_address='132.124.0.23' + ) + }) + ) + self.tmp_containers.append(container) + self.client.start(container) + + container_data = self.client.inspect_container(container) + self.assertEqual( + container_data[ + 'NetworkSettings']['Networks'][net_name]['IPAMConfig'][ + 'IPv4Address' + ], + '132.124.0.23' + ) + + @requires_api_version('1.22') + def test_create_with_ipv6_address(self): + net_name, net_id = self.create_network( + ipam=create_ipam_config( + driver='default', + pool_configs=[create_ipam_pool(subnet="2001:389::1/64")], + ), + ) + container = self.client.create_container( + image='busybox', command='top', + host_config=self.client.create_host_config(network_mode=net_name), + networking_config=self.client.create_networking_config({ + net_name: self.client.create_endpoint_config( + ipv6_address='2001:389::f00d' + ) + }) + ) + self.tmp_containers.append(container) + self.client.start(container) + + container_data = self.client.inspect_container(container) + self.assertEqual( + container_data[ + 'NetworkSettings']['Networks'][net_name]['IPAMConfig'][ + 'IPv6Address' + ], + '2001:389::f00d' + ) @requires_api_version('1.22') def test_create_with_links(self): From fc4bfde0d68c8d3106bdab707fb780b6aa8501bf Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 8 Jun 2016 16:43:51 -0700 Subject: [PATCH 0015/1301] Unify endpoint config creation when using connect_container_to_network Signed-off-by: Joffrey F --- docker/api/network.py | 24 +++++------------------- docs/networks.md | 8 +++++--- 2 files changed, 10 insertions(+), 22 deletions(-) diff --git a/docker/api/network.py b/docker/api/network.py index 4dec3f52ee..a35f0a405b 100644 --- a/docker/api/network.py +++ b/docker/api/network.py @@ -1,7 +1,7 @@ import json from ..errors import InvalidVersion -from ..utils import check_resource, minimum_version, normalize_links +from ..utils import check_resource, minimum_version from ..utils import version_lt @@ -63,26 +63,12 @@ def connect_container_to_network(self, container, net_id, aliases=None, links=None): data = { "Container": container, - "EndpointConfig": { - "Aliases": aliases, - "Links": normalize_links(links) if links else None, - }, + "EndpointConfig": self.create_endpoint_config( + aliases=aliases, links=links, ipv4_address=ipv4_address, + ipv6_address=ipv6_address + ), } - # IPv4 or IPv6 or neither: - if ipv4_address or ipv6_address: - if version_lt(self._version, '1.22'): - raise InvalidVersion('IP address assignment is not ' - 'supported in API version < 1.22') - - data['EndpointConfig']['IPAMConfig'] = dict() - if ipv4_address: - data['EndpointConfig']['IPAMConfig']['IPv4Address'] = \ - ipv4_address - if ipv6_address: - data['EndpointConfig']['IPAMConfig']['IPv6Address'] = \ - ipv6_address - url = self._url("/networks/{0}/connect", net_id) res = self._post_json(url, data=data) self._raise_for_status(res) diff --git a/docs/networks.md b/docs/networks.md index a805c6c73f..df28a8e267 100644 --- a/docs/networks.md +++ b/docs/networks.md @@ -17,11 +17,13 @@ For example, setting the subnet to `192.168.52.0/24` and gateway address to `192.168.52.254` ```python - -ipam_config = docker.utils.create_ipam_config( +ipam_pool = docker.utils.create_ipam_pool( subnet='192.168.52.0/24', gateway='192.168.52.254' ) +ipam_config = docker.utils.create_ipam_config( + pool_configs=[ipam_pool] +) docker_client.create_network("network1", driver="bridge", ipam=ipam_config) ``` @@ -160,7 +162,7 @@ in `docker.utils.create_ipam_config`. ```python ipam_pool = docker.utils.create_ipam_pool( subnet='124.42.0.0/16', - iprange='124.42.0.0/8', + iprange='124.42.0.0/24', gateway='124.42.0.254', aux_addresses={ 'reserved1': '124.42.1.1' From d991db5b45d15c36bd2d240943c7b03d71cf0d3e Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 13 Jun 2016 17:37:54 -0700 Subject: [PATCH 0016/1301] Expand on the 1-network limit in create_container Signed-off-by: Joffrey F --- docs/networks.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/networks.md b/docs/networks.md index df28a8e267..ec45e1c5fc 100644 --- a/docs/networks.md +++ b/docs/networks.md @@ -40,9 +40,12 @@ docker_client.create_network("network1", driver="bridge", internal=True) ## Container network configuration -In order to specify which network(s) a container will be connected to and +In order to specify which network a container will be connected to, and additional configuration, use the `networking_config` parameter in -`Client.create_container` +`Client.create_container`. Note that at the time of creation, you can +only connect a container to a single network. Later on, you may create more +connections using `Client.connect_container_to_network`. + ```python networking_config = docker_client.create_networking_config({ From a2160145cf5fca580daf5579f87f2d42f03d4e14 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 16 Jun 2016 12:46:46 -0700 Subject: [PATCH 0017/1301] 1.9.0 RC1 Signed-off-by: Joffrey F --- docker/version.py | 2 +- docs/change_log.md | 24 +++++++++++++++++++++++- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/docker/version.py b/docker/version.py index 4b7b86c181..a80bee1ea2 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "1.8.1" +version = "1.9.0-rc1" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) diff --git a/docs/change_log.md b/docs/change_log.md index 6db720ab44..2d80022549 100644 --- a/docs/change_log.md +++ b/docs/change_log.md @@ -1,10 +1,32 @@ Change Log ========== +1.9.0 +----- + +[List of PRs / issues for this release](https://github.com/docker/docker-py/issues?q=milestone%3A1.9.0+is%3Aclosed) + +### Features + +* Added **experimental** support for Windows named pipes (`npipe://` protocol). +* Added support for Block IO constraints in `Client.create_host_config`. This + includes parameters `blkio_weight`, `blkio_weight_device`, `device_read_bps`, + `device_write_bps`, `device_read_iops` and `device_write_iops`. +* Added support for the `internal` param in `Client.create_network`. +* Added support for `ipv4_address` and `ipv6_address` in utils function + `create_endpoint_config`. + +### Bugfixes + +* Fixed an issue where the HTTP timeout on streaming responses would sometimes + be set incorrectly. +* Fixed an issue where explicit relative paths in `.dockerignore` files were + not being recognized. + 1.8.1 ----- -[List of PRs / issues for this release](https://github.com/docker/docker-py/issues?q=milestone%3A1.8.0+is%3Aclosed) +[List of PRs / issues for this release](https://github.com/docker/docker-py/issues?q=milestone%3A1.8.1+is%3Aclosed) ### Bugfixes From 7d9bb6d209480ac8a48d1361ea08456542f5f865 Mon Sep 17 00:00:00 2001 From: Aiden Luo Date: Fri, 17 Jun 2016 11:20:39 +0800 Subject: [PATCH 0018/1301] fix #1094, support PidsLimit in host config Signed-off-by: Aiden Luo --- docker/utils/utils.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docker/utils/utils.py b/docker/utils/utils.py index 2ef8ef0db1..8cf21277f0 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -620,7 +620,7 @@ def create_host_config(binds=None, port_bindings=None, lxc_conf=None, device_write_bps=None, device_read_iops=None, device_write_iops=None, oom_kill_disable=False, shm_size=None, version=None, tmpfs=None, - oom_score_adj=None): + oom_score_adj=None, pids_limit=None,): host_config = {} @@ -853,6 +853,13 @@ def create_host_config(binds=None, port_bindings=None, lxc_conf=None, raise host_config_version_error('tmpfs', '1.22') host_config["Tmpfs"] = convert_tmpfs_mounts(tmpfs) + if pids_limit: + if not isinstance(pids_limit, int): + raise host_config_type_error('pids_limit', pids_limit, 'int') + if version_lt(version, '1.23'): + raise host_config_version_error('pids_limit', '1.23') + host_config["PidsLimit"] = pids_limit + return host_config From 0de366da3de451939ee05ef636506333e4f1ca70 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 24 Jun 2016 15:17:58 -0700 Subject: [PATCH 0019/1301] Add support for link-local IPs in endpoint config Signed-off-by: Joffrey F --- docker/api/network.py | 5 +++-- docker/utils/utils.py | 10 +++++++++- docs/api.md | 10 ++++++++++ docs/networks.md | 1 + 4 files changed, 23 insertions(+), 3 deletions(-) diff --git a/docker/api/network.py b/docker/api/network.py index a35f0a405b..34cd8987a1 100644 --- a/docker/api/network.py +++ b/docker/api/network.py @@ -60,12 +60,13 @@ def inspect_network(self, net_id): @minimum_version('1.21') def connect_container_to_network(self, container, net_id, ipv4_address=None, ipv6_address=None, - aliases=None, links=None): + aliases=None, links=None, + link_local_ips=None): data = { "Container": container, "EndpointConfig": self.create_endpoint_config( aliases=aliases, links=links, ipv4_address=ipv4_address, - ipv6_address=ipv6_address + ipv6_address=ipv6_address, link_local_ips=link_local_ips ), } diff --git a/docker/utils/utils.py b/docker/utils/utils.py index 2ef8ef0db1..b38cda47b3 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -873,7 +873,8 @@ def create_networking_config(endpoints_config=None): def create_endpoint_config(version, aliases=None, links=None, - ipv4_address=None, ipv6_address=None): + ipv4_address=None, ipv6_address=None, + link_local_ips=None): if version_lt(version, '1.22'): raise errors.InvalidVersion( 'Endpoint config is not supported for API version < 1.22' @@ -896,6 +897,13 @@ def create_endpoint_config(version, aliases=None, links=None, if ipam_config: endpoint_config['IPAMConfig'] = ipam_config + if link_local_ips is not None: + if version_lt(version, '1.24'): + raise errors.InvalidVersion( + 'link_local_ips is not supported for API version < 1.24' + ) + endpoint_config['LinkLocalIPs'] = link_local_ips + return endpoint_config diff --git a/docs/api.md b/docs/api.md index 51b6e2716a..5b8ef22b4d 100644 --- a/docs/api.md +++ b/docs/api.md @@ -179,6 +179,16 @@ Connect a container to a network. * container (str): container-id/name to be connected to the network * net_id (str): network id +* aliases (list): A list of aliases for this endpoint. Names in that list can + be used within the network to reach the container. Defaults to `None`. +* links (list): A list of links for this endpoint. Containers declared in this + list will be [linked](https://docs.docker.com/engine/userguide/networking/work-with-networks/#linking-containers-in-user-defined-networks) + to this container. Defaults to `None`. +* ipv4_address (str): The IP address of this container on the network, + using the IPv4 protocol. Defaults to `None`. +* ipv6_address (str): The IP address of this container on the network, + using the IPv6 protocol. Defaults to `None`. +* link_local_ips (list): A list of link-local (IPv4/IPv6) addresses. ## copy Identical to the `docker cp` command. Get files/folders from the container. diff --git a/docs/networks.md b/docs/networks.md index ec45e1c5fc..fb0e9f420c 100644 --- a/docs/networks.md +++ b/docs/networks.md @@ -107,6 +107,7 @@ Create an endpoint config dictionary to be used with using the IPv4 protocol. Defaults to `None`. * ipv6_address (str): The IP address of this container on the network, using the IPv6 protocol. Defaults to `None`. +* link_local_ips (list): A list of link-local (IPv4/IPv6) addresses. **Returns** An endpoint config dictionary. From 5480493662df912f13b2d31ee217c425bef003e0 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 27 Jun 2016 18:00:55 -0700 Subject: [PATCH 0020/1301] signal in Client.kill can be a string containing the signal's name Signed-off-by: Joffrey F --- docker/api/container.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker/api/container.py b/docker/api/container.py index b591b17312..9cc14dbd34 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -187,7 +187,9 @@ def kill(self, container, signal=None): url = self._url("/containers/{0}/kill", container) params = {} if signal is not None: - params['signal'] = int(signal) + if not isinstance(signal, six.string_types): + signal = int(signal) + params['signal'] = signal res = self._post(url, params=params) self._raise_for_status(res) From 1132368be19e39cbf2c3ab0ee073949ebb434815 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 28 Jun 2016 11:05:01 -0700 Subject: [PATCH 0021/1301] Fix network aliases test with Engine 1.12 Signed-off-by: Joffrey F --- tests/integration/network_test.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/tests/integration/network_test.py b/tests/integration/network_test.py index 26d27a5a83..f719fea485 100644 --- a/tests/integration/network_test.py +++ b/tests/integration/network_test.py @@ -138,9 +138,11 @@ def test_connect_with_aliases(self): self.client.connect_container_to_network( container, net_id, aliases=['foo', 'bar']) container_data = self.client.inspect_container(container) - self.assertEqual( - container_data['NetworkSettings']['Networks'][net_name]['Aliases'], - ['foo', 'bar']) + aliases = ( + container_data['NetworkSettings']['Networks'][net_name]['Aliases'] + ) + assert 'foo' in aliases + assert 'bar' in aliases @requires_api_version('1.21') def test_connect_on_container_create(self): @@ -183,10 +185,11 @@ def test_create_with_aliases(self): self.client.start(container) container_data = self.client.inspect_container(container) - self.assertEqual( - container_data['NetworkSettings']['Networks'][net_name]['Aliases'], - ['foo', 'bar'] + aliases = ( + container_data['NetworkSettings']['Networks'][net_name]['Aliases'] ) + assert 'foo' in aliases + assert 'bar' in aliases @requires_api_version('1.22') def test_create_with_ipv4_address(self): From d96d848bb44a958941043d808efc78a39e27ca33 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 28 Jun 2016 11:55:13 -0700 Subject: [PATCH 0022/1301] Add integration tests for different types of kill signals Signed-off-by: Joffrey F --- tests/integration/container_test.py | 30 +++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/tests/integration/container_test.py b/tests/integration/container_test.py index 940e5b83ff..56b648a3cc 100644 --- a/tests/integration/container_test.py +++ b/tests/integration/container_test.py @@ -840,6 +840,36 @@ def test_kill_with_signal(self): self.assertIn('Running', state) self.assertEqual(state['Running'], False, state) + def test_kill_with_signal_name(self): + id = self.client.create_container(BUSYBOX, ['sleep', '60']) + self.client.start(id) + self.tmp_containers.append(id) + self.client.kill(id, signal='SIGKILL') + exitcode = self.client.wait(id) + self.assertNotEqual(exitcode, 0) + container_info = self.client.inspect_container(id) + self.assertIn('State', container_info) + state = container_info['State'] + self.assertIn('ExitCode', state) + self.assertNotEqual(state['ExitCode'], 0) + self.assertIn('Running', state) + self.assertEqual(state['Running'], False, state) + + def test_kill_with_signal_integer(self): + id = self.client.create_container(BUSYBOX, ['sleep', '60']) + self.client.start(id) + self.tmp_containers.append(id) + self.client.kill(id, signal=9) + exitcode = self.client.wait(id) + self.assertNotEqual(exitcode, 0) + container_info = self.client.inspect_container(id) + self.assertIn('State', container_info) + state = container_info['State'] + self.assertIn('ExitCode', state) + self.assertNotEqual(state['ExitCode'], 0) + self.assertIn('Running', state) + self.assertEqual(state['Running'], False, state) + class PortTest(helpers.BaseTestCase): def test_port(self): From 900703ef2fa0d7f33f922bb007d4a0cc384e5214 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 27 Jun 2016 18:00:55 -0700 Subject: [PATCH 0023/1301] signal in Client.kill can be a string containing the signal's name Signed-off-by: Joffrey F --- docker/api/container.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker/api/container.py b/docker/api/container.py index b591b17312..9cc14dbd34 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -187,7 +187,9 @@ def kill(self, container, signal=None): url = self._url("/containers/{0}/kill", container) params = {} if signal is not None: - params['signal'] = int(signal) + if not isinstance(signal, six.string_types): + signal = int(signal) + params['signal'] = signal res = self._post(url, params=params) self._raise_for_status(res) From 4ed90bc8b8d1887704c533ed029852d2b6aa9acd Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 28 Jun 2016 11:05:01 -0700 Subject: [PATCH 0024/1301] Fix network aliases test with Engine 1.12 Signed-off-by: Joffrey F --- tests/integration/network_test.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/tests/integration/network_test.py b/tests/integration/network_test.py index 26d27a5a83..f719fea485 100644 --- a/tests/integration/network_test.py +++ b/tests/integration/network_test.py @@ -138,9 +138,11 @@ def test_connect_with_aliases(self): self.client.connect_container_to_network( container, net_id, aliases=['foo', 'bar']) container_data = self.client.inspect_container(container) - self.assertEqual( - container_data['NetworkSettings']['Networks'][net_name]['Aliases'], - ['foo', 'bar']) + aliases = ( + container_data['NetworkSettings']['Networks'][net_name]['Aliases'] + ) + assert 'foo' in aliases + assert 'bar' in aliases @requires_api_version('1.21') def test_connect_on_container_create(self): @@ -183,10 +185,11 @@ def test_create_with_aliases(self): self.client.start(container) container_data = self.client.inspect_container(container) - self.assertEqual( - container_data['NetworkSettings']['Networks'][net_name]['Aliases'], - ['foo', 'bar'] + aliases = ( + container_data['NetworkSettings']['Networks'][net_name]['Aliases'] ) + assert 'foo' in aliases + assert 'bar' in aliases @requires_api_version('1.22') def test_create_with_ipv4_address(self): From 0d8624b49b8c8296785df6746e16432d787bd4cc Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 28 Jun 2016 11:55:13 -0700 Subject: [PATCH 0025/1301] Add integration tests for different types of kill signals Signed-off-by: Joffrey F --- tests/integration/container_test.py | 30 +++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/tests/integration/container_test.py b/tests/integration/container_test.py index 940e5b83ff..56b648a3cc 100644 --- a/tests/integration/container_test.py +++ b/tests/integration/container_test.py @@ -840,6 +840,36 @@ def test_kill_with_signal(self): self.assertIn('Running', state) self.assertEqual(state['Running'], False, state) + def test_kill_with_signal_name(self): + id = self.client.create_container(BUSYBOX, ['sleep', '60']) + self.client.start(id) + self.tmp_containers.append(id) + self.client.kill(id, signal='SIGKILL') + exitcode = self.client.wait(id) + self.assertNotEqual(exitcode, 0) + container_info = self.client.inspect_container(id) + self.assertIn('State', container_info) + state = container_info['State'] + self.assertIn('ExitCode', state) + self.assertNotEqual(state['ExitCode'], 0) + self.assertIn('Running', state) + self.assertEqual(state['Running'], False, state) + + def test_kill_with_signal_integer(self): + id = self.client.create_container(BUSYBOX, ['sleep', '60']) + self.client.start(id) + self.tmp_containers.append(id) + self.client.kill(id, signal=9) + exitcode = self.client.wait(id) + self.assertNotEqual(exitcode, 0) + container_info = self.client.inspect_container(id) + self.assertIn('State', container_info) + state = container_info['State'] + self.assertIn('ExitCode', state) + self.assertNotEqual(state['ExitCode'], 0) + self.assertIn('Running', state) + self.assertEqual(state['Running'], False, state) + class PortTest(helpers.BaseTestCase): def test_port(self): From 0f15c6599df8cdfb64954595926a63c19c05be80 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 28 Jun 2016 14:54:37 -0700 Subject: [PATCH 0026/1301] Bump to 1.9.0-rc2 Signed-off-by: Joffrey F --- docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index a80bee1ea2..a09ddc8104 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "1.9.0-rc1" +version = "1.9.0-rc2" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) From b5d3556bce4190c9092551a441c1725f36176aec Mon Sep 17 00:00:00 2001 From: Srikalyan Swayampakula Date: Wed, 29 Jun 2016 21:00:19 -0700 Subject: [PATCH 0027/1301] Added support for user namespace. Signed-off-by: Srikalyan Swayampakula --- docker/utils/utils.py | 10 +++++++++- docs/hostconfig.md | 2 ++ tests/unit/utils_test.py | 11 ++++++++++- 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/docker/utils/utils.py b/docker/utils/utils.py index b38cda47b3..6d9e721830 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -620,7 +620,7 @@ def create_host_config(binds=None, port_bindings=None, lxc_conf=None, device_write_bps=None, device_read_iops=None, device_write_iops=None, oom_kill_disable=False, shm_size=None, version=None, tmpfs=None, - oom_score_adj=None): + oom_score_adj=None, userns_mode=None): host_config = {} @@ -853,6 +853,14 @@ def create_host_config(binds=None, port_bindings=None, lxc_conf=None, raise host_config_version_error('tmpfs', '1.22') host_config["Tmpfs"] = convert_tmpfs_mounts(tmpfs) + if userns_mode: + if version_lt(version, '1.23'): + raise host_config_version_error('userns_mode', '1.23') + + if userns_mode != "host": + raise host_config_value_error("userns_mode", userns_mode) + host_config['UsernsMode'] = userns_mode + return host_config diff --git a/docs/hostconfig.md b/docs/hostconfig.md index c1e23533a5..e996a75b4c 100644 --- a/docs/hostconfig.md +++ b/docs/hostconfig.md @@ -123,6 +123,8 @@ for example: for more information. * tmpfs: Temporary filesystems to mouunt. See [Using tmpfs](tmpfs.md) for more information. +* userns_mode: Sets the user namespace mode for the container when user namespace remapping option + is enabled. supported values are: host **Returns** (dict) HostConfig dictionary diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index 128778f17e..61d87b73ff 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -98,6 +98,16 @@ def test_create_host_config_with_oom_kill_disable(self): InvalidVersion, lambda: create_host_config(version='1.18.3', oom_kill_disable=True)) + def test_create_host_config_with_userns_mode(self): + config = create_host_config(version='1.23', userns_mode='host') + self.assertEqual(config.get('UsernsMode'), 'host') + self.assertRaises( + InvalidVersion, lambda: create_host_config(version='1.22', + userns_mode='host')) + self.assertRaises( + ValueError, lambda: create_host_config(version='1.23', + userns_mode='host12')) + def test_create_host_config_with_oom_score_adj(self): config = create_host_config(version='1.22', oom_score_adj=100) self.assertEqual(config.get('OomScoreAdj'), 100) @@ -602,7 +612,6 @@ def test_create_ipam_config(self): class SplitCommandTest(base.BaseTestCase): - def test_split_command_with_unicode(self): self.assertEqual(split_command(u'echo μμ'), ['echo', 'μμ']) From 6d347cd8948d83545379703bbb501d45f203c63b Mon Sep 17 00:00:00 2001 From: Faylixe Date: Fri, 8 Jul 2016 10:41:39 +0200 Subject: [PATCH 0028/1301] Update api.md Added ``buildargs`` parameter to ``build`` documentation Signed-off-by: Faylixe --- docs/api.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/api.md b/docs/api.md index 5b8ef22b4d..07d7508879 100644 --- a/docs/api.md +++ b/docs/api.md @@ -46,6 +46,8 @@ already, pass a readable file-like object to `fileobj` and also pass `custom_context=True`. If the stream is compressed also, set `encoding` to the correct value (e.g `gzip`). +Build argument can also be pass a a Python dict through ``buildargs`` parameter. + **Params**: * path (str): Path to the directory containing the Dockerfile @@ -65,6 +67,7 @@ correct value (e.g `gzip`). * pull (bool): Downloads any updates to the FROM image in Dockerfiles * forcerm (bool): Always remove intermediate containers, even after unsuccessful builds * dockerfile (str): path within the build context to the Dockerfile +* buildargs (dict): A dictionary of build arguments * container_limits (dict): A dictionary of limits applied to each container created by the build process. Valid keys: - memory (int): set memory limit for build From c8c6f0073ee0d8e3de9e6f3c13e9eb805661949e Mon Sep 17 00:00:00 2001 From: Faylixe Date: Fri, 8 Jul 2016 21:43:23 +0200 Subject: [PATCH 0029/1301] Removed superfluous sentence MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Faylixe Signed-off-by: Félix Voituret --- docs/api.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/api.md b/docs/api.md index 07d7508879..4d2fe10365 100644 --- a/docs/api.md +++ b/docs/api.md @@ -46,8 +46,6 @@ already, pass a readable file-like object to `fileobj` and also pass `custom_context=True`. If the stream is compressed also, set `encoding` to the correct value (e.g `gzip`). -Build argument can also be pass a a Python dict through ``buildargs`` parameter. - **Params**: * path (str): Path to the directory containing the Dockerfile From 66e7af93532890dcb8e43d8a701ca3b3eae51d4e Mon Sep 17 00:00:00 2001 From: Justin Michalicek Date: Wed, 6 Jul 2016 16:10:16 -0400 Subject: [PATCH 0030/1301] Pass X-Registry-Auth when building an image * Initialize headers variable in BuildApiMixin.build() as a dict rather than as None. This way the correct object gets passed to _set_auth_headers() even if no headers were set in build() * Changing object from None to {} in BuildApiMixin._set_auth_headers() removed because it changes the object reference, so has no effect on calling code. Signed-off-by: Justin Michalicek --- docker/api/build.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/api/build.py b/docker/api/build.py index 971a50edbc..74037167e7 100644 --- a/docker/api/build.py +++ b/docker/api/build.py @@ -18,7 +18,8 @@ def build(self, path=None, tag=None, quiet=False, fileobj=None, custom_context=False, encoding=None, pull=False, forcerm=False, dockerfile=None, container_limits=None, decode=False, buildargs=None, gzip=False): - remote = context = headers = None + remote = context = None + headers = {} container_limits = container_limits or {} if path is None and fileobj is None: raise TypeError("Either path or fileobj needs to be provided.") @@ -134,8 +135,7 @@ def _set_auth_headers(self, headers): ', '.join(repr(k) for k in self._auth_configs.keys()) ) ) - if headers is None: - headers = {} + if utils.compare_version('1.19', self._version) >= 0: headers['X-Registry-Config'] = auth.encode_header( self._auth_configs From f7807bdb52513dcbdab486a633ad5edeefa09e66 Mon Sep 17 00:00:00 2001 From: Justin Michalicek Date: Thu, 7 Jul 2016 17:20:31 -0400 Subject: [PATCH 0031/1301] Update build unit tests * Test that the request from build when the client has auth configs contains the correct X-Registry-Config header * Test that BuildApiMixin._set_auth_headers() updates the passed in headers dict with auth data from the client * Test that BuildApiMixin._set_auth_headers() leaves headers dict intact when there is no _auth_config on the client. Signed-off-by: Justin Michalicek --- tests/unit/build_test.py | 61 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/tests/unit/build_test.py b/tests/unit/build_test.py index 414153ed52..8bd626bdfe 100644 --- a/tests/unit/build_test.py +++ b/tests/unit/build_test.py @@ -2,8 +2,9 @@ import io import docker +from docker import auth -from .api_test import DockerClientTest +from .api_test import DockerClientTest, fake_request, url_prefix class BuildTest(DockerClientTest): @@ -83,8 +84,25 @@ def test_build_remote_with_registry_auth(self): } } + expected_params = {'t': None, 'q': False, 'dockerfile': None, + 'rm': False, 'nocache': False, 'pull': False, + 'forcerm': False, + 'remote': 'https://github.com/docker-library/mongo'} + expected_headers = { + 'X-Registry-Config': auth.encode_header(self.client._auth_configs)} + self.client.build(path='https://github.com/docker-library/mongo') + fake_request.assert_called_with( + 'POST', + url_prefix + 'build', + stream=True, + data=None, + headers=expected_headers, + params=expected_params, + timeout=None + ) + def test_build_container_with_named_dockerfile(self): self.client.build('.', dockerfile='nameddockerfile') @@ -103,3 +121,44 @@ def test_build_container_invalid_container_limits(self): 'foo': 'bar' }) ) + + def test__set_auth_headers_with_empty_dict_and_auth_configs(self): + self.client._auth_configs = { + 'https://example.com': { + 'user': 'example', + 'password': 'example', + 'email': 'example@example.com' + } + } + + headers = {} + expected_headers = { + 'X-Registry-Config': auth.encode_header(self.client._auth_configs)} + self.client._set_auth_headers(headers) + self.assertEqual(headers, expected_headers) + + def test__set_auth_headers_with_dict_and_auth_configs(self): + self.client._auth_configs = { + 'https://example.com': { + 'user': 'example', + 'password': 'example', + 'email': 'example@example.com' + } + } + + headers = {'foo': 'bar'} + expected_headers = { + 'foo': 'bar', + 'X-Registry-Config': auth.encode_header(self.client._auth_configs)} + + self.client._set_auth_headers(headers) + self.assertEqual(headers, expected_headers) + + def test__set_auth_headers_with_dict_and_no_auth_configs(self): + headers = {'foo': 'bar'} + expected_headers = { + 'foo': 'bar' + } + + self.client._set_auth_headers(headers) + self.assertEqual(headers, expected_headers) From e8ea79dfdb7b722801113131bfe90e88c141dc09 Mon Sep 17 00:00:00 2001 From: Justin Michalicek Date: Tue, 12 Jul 2016 10:04:37 -0400 Subject: [PATCH 0032/1301] Change double underscore in test case names for _set_auth_headers * Change test__set_auth_headers_* methods to test_set_auth_headers_* Signed-off-by: Justin Michalicek --- tests/unit/build_test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unit/build_test.py b/tests/unit/build_test.py index 8bd626bdfe..b2705eb2f1 100644 --- a/tests/unit/build_test.py +++ b/tests/unit/build_test.py @@ -122,7 +122,7 @@ def test_build_container_invalid_container_limits(self): }) ) - def test__set_auth_headers_with_empty_dict_and_auth_configs(self): + def test_set_auth_headers_with_empty_dict_and_auth_configs(self): self.client._auth_configs = { 'https://example.com': { 'user': 'example', @@ -137,7 +137,7 @@ def test__set_auth_headers_with_empty_dict_and_auth_configs(self): self.client._set_auth_headers(headers) self.assertEqual(headers, expected_headers) - def test__set_auth_headers_with_dict_and_auth_configs(self): + def test_set_auth_headers_with_dict_and_auth_configs(self): self.client._auth_configs = { 'https://example.com': { 'user': 'example', @@ -154,7 +154,7 @@ def test__set_auth_headers_with_dict_and_auth_configs(self): self.client._set_auth_headers(headers) self.assertEqual(headers, expected_headers) - def test__set_auth_headers_with_dict_and_no_auth_configs(self): + def test_set_auth_headers_with_dict_and_no_auth_configs(self): headers = {'foo': 'bar'} expected_headers = { 'foo': 'bar' From 8f8a3d0ed2dd1f1bc3ae68c2af186070dc3007a8 Mon Sep 17 00:00:00 2001 From: Tomas Tomecek Date: Thu, 30 Jun 2016 10:26:30 +0200 Subject: [PATCH 0033/1301] volumes,create: support adding labels Fixes #1102 Signed-off-by: Tomas Tomecek --- docker/api/volume.py | 13 ++++++++++++- docs/api.md | 11 +++++++++-- tests/unit/fake_api.py | 5 ++++- tests/unit/volume_test.py | 16 ++++++++++++++++ 4 files changed, 41 insertions(+), 4 deletions(-) diff --git a/docker/api/volume.py b/docker/api/volume.py index bb8b39b316..afc72cbbc8 100644 --- a/docker/api/volume.py +++ b/docker/api/volume.py @@ -1,3 +1,4 @@ +from .. import errors from .. import utils @@ -11,7 +12,7 @@ def volumes(self, filters=None): return self._result(self._get(url, params=params), True) @utils.minimum_version('1.21') - def create_volume(self, name, driver=None, driver_opts=None): + def create_volume(self, name, driver=None, driver_opts=None, labels=None): url = self._url('/volumes/create') if driver_opts is not None and not isinstance(driver_opts, dict): raise TypeError('driver_opts must be a dictionary') @@ -21,6 +22,16 @@ def create_volume(self, name, driver=None, driver_opts=None): 'Driver': driver, 'DriverOpts': driver_opts, } + + if labels is not None: + if utils.compare_version('1.23', self._version) < 0: + raise errors.InvalidVersion( + 'volume labels were introduced in API 1.23' + ) + if not isinstance(labels, dict): + raise TypeError('labels must be a dictionary') + data["Labels"] = labels + return self._result(self._post_json(url, data=data), True) @utils.minimum_version('1.21') diff --git a/docs/api.md b/docs/api.md index 4d2fe10365..41c5e6cf09 100644 --- a/docs/api.md +++ b/docs/api.md @@ -310,6 +310,7 @@ Create and register a named volume * name (str): Name of the volume * driver (str): Name of the driver used to create the volume * driver_opts (dict): Driver options as a key-value dictionary +* labels (dict): Labels to set on the volume **Returns** (dict): The created volume reference object @@ -317,10 +318,16 @@ Create and register a named volume >>> from docker import Client >>> cli = Client() >>> volume = cli.create_volume( - name='foobar', driver='local', driver_opts={'foo': 'bar', 'baz': 'false'} + name='foobar', driver='local', driver_opts={'foo': 'bar', 'baz': 'false'}, + labels={"key": "value"} ) >>> print(volume) -{u'Mountpoint': u'/var/lib/docker/volumes/foobar/_data', u'Driver': u'local', u'Name': u'foobar'} +{ + u'Mountpoint': u'/var/lib/docker/volumes/foobar/_data', + u'Driver': u'local', + u'Name': u'foobar', + u'Labels': {u'key': u'value'} +} ``` ## diff diff --git a/tests/unit/fake_api.py b/tests/unit/fake_api.py index 99525956d0..835d73f2a4 100644 --- a/tests/unit/fake_api.py +++ b/tests/unit/fake_api.py @@ -433,7 +433,10 @@ def get_fake_volume(): response = { 'Name': 'perfectcherryblossom', 'Driver': 'local', - 'Mountpoint': '/var/lib/docker/volumes/perfectcherryblossom' + 'Mountpoint': '/var/lib/docker/volumes/perfectcherryblossom', + 'Labels': { + 'com.example.some-label': 'some-value' + } } return status_code, response diff --git a/tests/unit/volume_test.py b/tests/unit/volume_test.py index 5b1823a4ca..136d11afc1 100644 --- a/tests/unit/volume_test.py +++ b/tests/unit/volume_test.py @@ -43,6 +43,22 @@ def test_create_volume(self): self.assertEqual(args[0][1], url_prefix + 'volumes/create') self.assertEqual(json.loads(args[1]['data']), {'Name': name}) + @base.requires_api_version('1.23') + def test_create_volume_with_labels(self): + name = 'perfectcherryblossom' + result = self.client.create_volume(name, labels={ + 'com.example.some-label': 'some-value'}) + self.assertEqual( + result["Labels"], + {'com.example.some-label': 'some-value'} + ) + + @base.requires_api_version('1.23') + def test_create_volume_with_invalid_labels(self): + name = 'perfectcherryblossom' + with pytest.raises(TypeError): + self.client.create_volume(name, labels=1) + @base.requires_api_version('1.21') def test_create_volume_with_driver(self): name = 'perfectcherryblossom' From 6dec639a1ada87f924bc5230718386b8a8a98206 Mon Sep 17 00:00:00 2001 From: David Gageot Date: Fri, 8 Jul 2016 12:11:40 +0200 Subject: [PATCH 0034/1301] Add hijack hints for attach api calls Signed-off-by: David Gageot --- docker/api/container.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/docker/api/container.py b/docker/api/container.py index 9cc14dbd34..eec25802f5 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -15,10 +15,16 @@ def attach(self, container, stdout=True, stderr=True, 'logs': logs and 1 or 0, 'stdout': stdout and 1 or 0, 'stderr': stderr and 1 or 0, - 'stream': stream and 1 or 0, + 'stream': stream and 1 or 0 } + + headers = { + 'Connection': 'Upgrade', + 'Upgrade': 'tcp' + } + u = self._url("/containers/{0}/attach", container) - response = self._post(u, params=params, stream=stream) + response = self._post(u, headers=headers, params=params, stream=stream) return self._get_result(container, stream, response) From 5464cf2bea9f232923c74cec17ef59b43b3613ef Mon Sep 17 00:00:00 2001 From: David Gageot Date: Fri, 8 Jul 2016 12:12:23 +0200 Subject: [PATCH 0035/1301] Add hijack hints for non-detached exec api calls Signed-off-by: David Gageot --- docker/api/exec_api.py | 10 +++++++++- tests/unit/exec_test.py | 32 ++++++++++++++++++++++++++++++-- 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/docker/api/exec_api.py b/docker/api/exec_api.py index f0e4afa6f9..ad2cd33118 100644 --- a/docker/api/exec_api.py +++ b/docker/api/exec_api.py @@ -66,8 +66,16 @@ def exec_start(self, exec_id, detach=False, tty=False, stream=False, 'Detach': detach } + headers = {} if detach else { + 'Connection': 'Upgrade', + 'Upgrade': 'tcp' + } + res = self._post_json( - self._url('/exec/{0}/start', exec_id), data=data, stream=stream + self._url('/exec/{0}/start', exec_id), + headers=headers, + data=data, + stream=stream ) if socket: diff --git a/tests/unit/exec_test.py b/tests/unit/exec_test.py index 3007799cb5..6ba2a3ddf6 100644 --- a/tests/unit/exec_test.py +++ b/tests/unit/exec_test.py @@ -51,8 +51,36 @@ def test_exec_start(self): } ) - self.assertEqual(args[1]['headers'], - {'Content-Type': 'application/json'}) + self.assertEqual( + args[1]['headers'], { + 'Content-Type': 'application/json', + 'Connection': 'Upgrade', + 'Upgrade': 'tcp' + } + ) + + def test_exec_start_detached(self): + self.client.exec_start(fake_api.FAKE_EXEC_ID, detach=True) + + args = fake_request.call_args + self.assertEqual( + args[0][1], url_prefix + 'exec/{0}/start'.format( + fake_api.FAKE_EXEC_ID + ) + ) + + self.assertEqual( + json.loads(args[1]['data']), { + 'Tty': False, + 'Detach': True + } + ) + + self.assertEqual( + args[1]['headers'], { + 'Content-Type': 'application/json' + } + ) def test_exec_inspect(self): self.client.exec_inspect(fake_api.FAKE_EXEC_ID) From 76ed9c37cdd532a0efa0b07b2f23d024dd8a3ab4 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 8 Jul 2016 15:58:50 -0700 Subject: [PATCH 0036/1301] Read from socket after sending TCP upgrade headers. Signed-off-by: Joffrey F --- docker/api/container.py | 15 ++++++++++--- docker/api/exec_api.py | 6 ++--- docker/client.py | 50 +++++++++++++++++++++++++++++++++++++++++ tests/helpers.py | 4 ++-- 4 files changed, 66 insertions(+), 9 deletions(-) diff --git a/docker/api/container.py b/docker/api/container.py index eec25802f5..b8507d85a7 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -26,7 +26,7 @@ def attach(self, container, stdout=True, stderr=True, u = self._url("/containers/{0}/attach", container) response = self._post(u, headers=headers, params=params, stream=stream) - return self._get_result(container, stream, response) + return self._read_from_socket(response, stream) @utils.check_resource def attach_socket(self, container, params=None, ws=False): @@ -40,9 +40,18 @@ def attach_socket(self, container, params=None, ws=False): if ws: return self._attach_websocket(container, params) + headers = { + 'Connection': 'Upgrade', + 'Upgrade': 'tcp' + } + u = self._url("/containers/{0}/attach", container) - return self._get_raw_response_socket(self.post( - u, None, params=self._attach_params(params), stream=True)) + return self._get_raw_response_socket( + self.post( + u, None, params=self._attach_params(params), stream=True, + headers=headers + ) + ) @utils.check_resource def commit(self, container, repository=None, tag=None, message=None, diff --git a/docker/api/exec_api.py b/docker/api/exec_api.py index ad2cd33118..6e49996046 100644 --- a/docker/api/exec_api.py +++ b/docker/api/exec_api.py @@ -56,8 +56,6 @@ def exec_resize(self, exec_id, height=None, width=None): def exec_start(self, exec_id, detach=False, tty=False, stream=False, socket=False): # we want opened socket if socket == True - if socket: - stream = True if isinstance(exec_id, dict): exec_id = exec_id.get('Id') @@ -75,9 +73,9 @@ def exec_start(self, exec_id, detach=False, tty=False, stream=False, self._url('/exec/{0}/start', exec_id), headers=headers, data=data, - stream=stream + stream=True ) if socket: return self._get_raw_response_socket(res) - return self._get_result_tty(stream, res, tty) + return self._read_from_socket(res, stream) diff --git a/docker/client.py b/docker/client.py index b96a78ce4e..dbbfb06c72 100644 --- a/docker/client.py +++ b/docker/client.py @@ -12,7 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +import errno import json +import os +import select import struct import requests @@ -305,6 +308,53 @@ def _stream_raw_result(self, response): for out in response.iter_content(chunk_size=1, decode_unicode=True): yield out + def _read_from_socket(self, response, stream): + def read_socket(socket, n=4096): + recoverable_errors = ( + errno.EINTR, errno.EDEADLK, errno.EWOULDBLOCK + ) + + # wait for data to become available + select.select([socket], [], []) + + try: + if hasattr(socket, 'recv'): + return socket.recv(n) + return os.read(socket.fileno(), n) + except EnvironmentError as e: + if e.errno not in recoverable_errors: + raise + + def next_packet_size(socket): + data = six.binary_type() + while len(data) < 8: + next_data = read_socket(socket, 8 - len(data)) + if not next_data: + return 0 + data = data + next_data + + if data is None: + return 0 + + if len(data) == 8: + _, actual = struct.unpack('>BxxxL', data) + return actual + + def read_loop(socket): + n = next_packet_size(socket) + while n > 0: + yield read_socket(socket, n) + n = next_packet_size(socket) + + socket = self._get_raw_response_socket(response) + if stream: + return read_loop(socket) + else: + data = six.binary_type() + for d in read_loop(socket): + data += d + return data + def _disable_socket_timeout(self, socket): """ Depending on the combination of python version and whether we're connecting over http or https, we might need to access _sock, which diff --git a/tests/helpers.py b/tests/helpers.py index 21036acead..70be803c68 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -54,7 +54,7 @@ def exec_driver_is_native(): c = docker_client() EXEC_DRIVER = c.info()['ExecutionDriver'] c.close() - return EXEC_DRIVER.startswith('native') + return EXEC_DRIVER.startswith('native') or EXEC_DRIVER == '' def docker_client(**kwargs): @@ -105,7 +105,7 @@ def read_data(socket, packet_size): while len(data) < packet_size: next_data = read_socket(socket, packet_size - len(data)) if not next_data: - assert False, "Failed trying to read in the dataz" + assert False, "Failed trying to read in the data" data += next_data return data From e64ba8f2b96ad1bbdaed8a65e538b5ab6129f0ba Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 8 Jul 2016 16:21:35 -0700 Subject: [PATCH 0037/1301] Mock read_from_socket method Signed-off-by: Joffrey F --- tests/unit/api_test.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/unit/api_test.py b/tests/unit/api_test.py index 23fd191346..34bf14f646 100644 --- a/tests/unit/api_test.py +++ b/tests/unit/api_test.py @@ -93,6 +93,10 @@ def fake_put(self, url, *args, **kwargs): def fake_delete(self, url, *args, **kwargs): return fake_request('DELETE', url, *args, **kwargs) + +def fake_read_from_socket(self, response, stream): + return six.binary_type() + url_base = 'http+docker://localunixsocket/' url_prefix = '{0}v{1}/'.format( url_base, @@ -103,7 +107,8 @@ class DockerClientTest(base.Cleanup, base.BaseTestCase): def setUp(self): self.patcher = mock.patch.multiple( 'docker.Client', get=fake_get, post=fake_post, put=fake_put, - delete=fake_delete + delete=fake_delete, + _read_from_socket=fake_read_from_socket ) self.patcher.start() self.client = docker.Client() From 73f06e3335bc9d2bc5569dc9bdfeeab2a78fcdb8 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Tue, 12 Jul 2016 16:23:13 -0400 Subject: [PATCH 0038/1301] Move socket-reading test helpers into docker.utils.socket Signed-off-by: Aanand Prasad --- docker/utils/socket.py | 49 +++++++++++++++++++++++++++++ tests/helpers.py | 46 --------------------------- tests/integration/container_test.py | 6 ++-- tests/integration/exec_test.py | 7 +++-- 4 files changed, 58 insertions(+), 50 deletions(-) create mode 100644 docker/utils/socket.py diff --git a/docker/utils/socket.py b/docker/utils/socket.py new file mode 100644 index 0000000000..f81d2f5d57 --- /dev/null +++ b/docker/utils/socket.py @@ -0,0 +1,49 @@ +import errno +import os +import select +import struct + +import six + + +def read_socket(socket, n=4096): + """ Code stolen from dockerpty to read the socket """ + recoverable_errors = (errno.EINTR, errno.EDEADLK, errno.EWOULDBLOCK) + + # wait for data to become available + select.select([socket], [], []) + + try: + if hasattr(socket, 'recv'): + return socket.recv(n) + return os.read(socket.fileno(), n) + except EnvironmentError as e: + if e.errno not in recoverable_errors: + raise + + +def next_packet_size(socket): + """ Code stolen from dockerpty to get the next packet size """ + data = six.binary_type() + while len(data) < 8: + next_data = read_socket(socket, 8 - len(data)) + if not next_data: + return 0 + data = data + next_data + + if data is None: + return 0 + + if len(data) == 8: + _, actual = struct.unpack('>BxxxL', data) + return actual + + +def read_data(socket, packet_size): + data = six.binary_type() + while len(data) < packet_size: + next_data = read_socket(socket, packet_size - len(data)) + if not next_data: + assert False, "Failed trying to read in the data" + data += next_data + return data diff --git a/tests/helpers.py b/tests/helpers.py index 70be803c68..94ea3887a8 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -1,9 +1,6 @@ -import errno import os import os.path -import select import shutil -import struct import tarfile import tempfile import unittest @@ -67,49 +64,6 @@ def docker_client_kwargs(**kwargs): return client_kwargs -def read_socket(socket, n=4096): - """ Code stolen from dockerpty to read the socket """ - recoverable_errors = (errno.EINTR, errno.EDEADLK, errno.EWOULDBLOCK) - - # wait for data to become available - select.select([socket], [], []) - - try: - if hasattr(socket, 'recv'): - return socket.recv(n) - return os.read(socket.fileno(), n) - except EnvironmentError as e: - if e.errno not in recoverable_errors: - raise - - -def next_packet_size(socket): - """ Code stolen from dockerpty to get the next packet size """ - data = six.binary_type() - while len(data) < 8: - next_data = read_socket(socket, 8 - len(data)) - if not next_data: - return 0 - data = data + next_data - - if data is None: - return 0 - - if len(data) == 8: - _, actual = struct.unpack('>BxxxL', data) - return actual - - -def read_data(socket, packet_size): - data = six.binary_type() - while len(data) < packet_size: - next_data = read_socket(socket, packet_size - len(data)) - if not next_data: - assert False, "Failed trying to read in the data" - data += next_data - return data - - class BaseTestCase(unittest.TestCase): tmp_imgs = [] tmp_containers = [] diff --git a/tests/integration/container_test.py b/tests/integration/container_test.py index 56b648a3cc..594aaa32ab 100644 --- a/tests/integration/container_test.py +++ b/tests/integration/container_test.py @@ -3,6 +3,8 @@ import tempfile import docker +from docker.utils.socket import next_packet_size +from docker.utils.socket import read_data import pytest import six @@ -1025,9 +1027,9 @@ def test_run_container_reading_socket(self): self.client.start(ident) - next_size = helpers.next_packet_size(pty_stdout) + next_size = next_packet_size(pty_stdout) self.assertEqual(next_size, len(line)) - data = helpers.read_data(pty_stdout, next_size) + data = read_data(pty_stdout, next_size) self.assertEqual(data.decode('utf-8'), line) diff --git a/tests/integration/exec_test.py b/tests/integration/exec_test.py index 9f5480808b..d0c8c9bf95 100644 --- a/tests/integration/exec_test.py +++ b/tests/integration/exec_test.py @@ -1,5 +1,8 @@ import pytest +from docker.utils.socket import next_packet_size +from docker.utils.socket import read_data + from .. import helpers BUSYBOX = helpers.BUSYBOX @@ -107,9 +110,9 @@ def test_exec_start_socket(self): socket = self.client.exec_start(exec_id, socket=True) self.addCleanup(socket.close) - next_size = helpers.next_packet_size(socket) + next_size = next_packet_size(socket) self.assertEqual(next_size, len(line)) - data = helpers.read_data(socket, next_size) + data = read_data(socket, next_size) self.assertEqual(data.decode('utf-8'), line) def test_exec_inspect(self): From b100666a3c4ce97d9957003a0cd77741058ef752 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Wed, 13 Jul 2016 15:45:37 -0400 Subject: [PATCH 0039/1301] Remove duplicated methods from container.py Signed-off-by: Aanand Prasad --- docker/client.py | 32 +------------------------------- 1 file changed, 1 insertion(+), 31 deletions(-) diff --git a/docker/client.py b/docker/client.py index dbbfb06c72..53cc5c29c9 100644 --- a/docker/client.py +++ b/docker/client.py @@ -32,6 +32,7 @@ from .tls import TLSConfig from .transport import UnixAdapter from .utils import utils, check_resource, update_headers, kwargs_from_env +from .utils.socket import read_socket, next_packet_size try: from .transport import NpipeAdapter except ImportError: @@ -309,37 +310,6 @@ def _stream_raw_result(self, response): yield out def _read_from_socket(self, response, stream): - def read_socket(socket, n=4096): - recoverable_errors = ( - errno.EINTR, errno.EDEADLK, errno.EWOULDBLOCK - ) - - # wait for data to become available - select.select([socket], [], []) - - try: - if hasattr(socket, 'recv'): - return socket.recv(n) - return os.read(socket.fileno(), n) - except EnvironmentError as e: - if e.errno not in recoverable_errors: - raise - - def next_packet_size(socket): - data = six.binary_type() - while len(data) < 8: - next_data = read_socket(socket, 8 - len(data)) - if not next_data: - return 0 - data = data + next_data - - if data is None: - return 0 - - if len(data) == 8: - _, actual = struct.unpack('>BxxxL', data) - return actual - def read_loop(socket): n = next_packet_size(socket) while n > 0: From 43158cfe3fd9299c4c47536cefd4d683d627d6a1 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Wed, 13 Jul 2016 16:06:41 -0400 Subject: [PATCH 0040/1301] Move read_loop() into docker.utils.socket.read_iter() Signed-off-by: Aanand Prasad --- docker/client.py | 16 ++++------------ docker/utils/socket.py | 7 +++++++ 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/docker/client.py b/docker/client.py index 53cc5c29c9..7df587c7f4 100644 --- a/docker/client.py +++ b/docker/client.py @@ -32,7 +32,7 @@ from .tls import TLSConfig from .transport import UnixAdapter from .utils import utils, check_resource, update_headers, kwargs_from_env -from .utils.socket import read_socket, next_packet_size +from .utils.socket import read_socket, next_packet_size, read_iter try: from .transport import NpipeAdapter except ImportError: @@ -310,20 +310,12 @@ def _stream_raw_result(self, response): yield out def _read_from_socket(self, response, stream): - def read_loop(socket): - n = next_packet_size(socket) - while n > 0: - yield read_socket(socket, n) - n = next_packet_size(socket) - socket = self._get_raw_response_socket(response) + if stream: - return read_loop(socket) + return read_iter(socket) else: - data = six.binary_type() - for d in read_loop(socket): - data += d - return data + return six.binary_type().join(read_iter(socket)) def _disable_socket_timeout(self, socket): """ Depending on the combination of python version and whether we're diff --git a/docker/utils/socket.py b/docker/utils/socket.py index f81d2f5d57..2fb1180d48 100644 --- a/docker/utils/socket.py +++ b/docker/utils/socket.py @@ -47,3 +47,10 @@ def read_data(socket, packet_size): assert False, "Failed trying to read in the data" data += next_data return data + + +def read_iter(socket): + n = next_packet_size(socket) + while n > 0: + yield read_socket(socket, n) + n = next_packet_size(socket) From 3e2f4a61424c434949a4a080657506ee4eaaa776 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Wed, 13 Jul 2016 16:50:16 -0400 Subject: [PATCH 0041/1301] Refactors - `read_data()` raises an exception instead of asserting `False` - `next_packet_size()` uses `read_data()` - Renamed `packet_size` arg to `n` for consistency Signed-off-by: Aanand Prasad --- docker/utils/socket.py | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/docker/utils/socket.py b/docker/utils/socket.py index 2fb1180d48..fbbf1e62c6 100644 --- a/docker/utils/socket.py +++ b/docker/utils/socket.py @@ -6,6 +6,10 @@ import six +class SocketError(Exception): + pass + + def read_socket(socket, n=4096): """ Code stolen from dockerpty to read the socket """ recoverable_errors = (errno.EINTR, errno.EDEADLK, errno.EWOULDBLOCK) @@ -24,27 +28,22 @@ def read_socket(socket, n=4096): def next_packet_size(socket): """ Code stolen from dockerpty to get the next packet size """ - data = six.binary_type() - while len(data) < 8: - next_data = read_socket(socket, 8 - len(data)) - if not next_data: - return 0 - data = data + next_data - if data is None: + try: + data = read_data(socket, 8) + except SocketError: return 0 - if len(data) == 8: - _, actual = struct.unpack('>BxxxL', data) - return actual + _, actual = struct.unpack('>BxxxL', data) + return actual -def read_data(socket, packet_size): +def read_data(socket, n): data = six.binary_type() - while len(data) < packet_size: - next_data = read_socket(socket, packet_size - len(data)) + while len(data) < n: + next_data = read_socket(socket, n - len(data)) if not next_data: - assert False, "Failed trying to read in the data" + raise SocketError("Unexpected EOF") data += next_data return data From ce2b60ecf6aa56de73ee3b8ab6c67ee10905e0ac Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Wed, 13 Jul 2016 16:52:15 -0400 Subject: [PATCH 0042/1301] Document all socket utility methods Signed-off-by: Aanand Prasad --- docker/utils/socket.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/docker/utils/socket.py b/docker/utils/socket.py index fbbf1e62c6..47b2320f83 100644 --- a/docker/utils/socket.py +++ b/docker/utils/socket.py @@ -11,7 +11,9 @@ class SocketError(Exception): def read_socket(socket, n=4096): - """ Code stolen from dockerpty to read the socket """ + """ + Reads at most n bytes from socket + """ recoverable_errors = (errno.EINTR, errno.EDEADLK, errno.EWOULDBLOCK) # wait for data to become available @@ -27,8 +29,12 @@ def read_socket(socket, n=4096): def next_packet_size(socket): - """ Code stolen from dockerpty to get the next packet size """ + """ + Returns the size of the next frame of data waiting to be read from socket, + according to the protocol defined here: + https://docs.docker.com/engine/reference/api/docker_remote_api_v1.24/#/attach-to-a-container + """ try: data = read_data(socket, 8) except SocketError: @@ -39,6 +45,9 @@ def next_packet_size(socket): def read_data(socket, n): + """ + Reads exactly n bytes from socket + """ data = six.binary_type() while len(data) < n: next_data = read_socket(socket, n - len(data)) @@ -49,6 +58,9 @@ def read_data(socket, n): def read_iter(socket): + """ + Returns a generator of frames read from socket + """ n = next_packet_size(socket) while n > 0: yield read_socket(socket, n) From 472a7ffce8032a92dbcc264b9178a3e92695f8ae Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Wed, 13 Jul 2016 16:52:58 -0400 Subject: [PATCH 0043/1301] Remove unused imports Signed-off-by: Aanand Prasad --- docker/client.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/docker/client.py b/docker/client.py index 7df587c7f4..e131829ff7 100644 --- a/docker/client.py +++ b/docker/client.py @@ -12,10 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import errno import json -import os -import select import struct import requests @@ -32,7 +29,7 @@ from .tls import TLSConfig from .transport import UnixAdapter from .utils import utils, check_resource, update_headers, kwargs_from_env -from .utils.socket import read_socket, next_packet_size, read_iter +from .utils.socket import read_iter try: from .transport import NpipeAdapter except ImportError: From 456bfa1c1d2bbca68eb91343d210f55f645c5a33 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Wed, 13 Jul 2016 16:53:41 -0400 Subject: [PATCH 0044/1301] Reorder socket.py methods Signed-off-by: Aanand Prasad --- docker/utils/socket.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/docker/utils/socket.py b/docker/utils/socket.py index 47b2320f83..0174d5f525 100644 --- a/docker/utils/socket.py +++ b/docker/utils/socket.py @@ -28,6 +28,19 @@ def read_socket(socket, n=4096): raise +def read_data(socket, n): + """ + Reads exactly n bytes from socket + """ + data = six.binary_type() + while len(data) < n: + next_data = read_socket(socket, n - len(data)) + if not next_data: + raise SocketError("Unexpected EOF") + data += next_data + return data + + def next_packet_size(socket): """ Returns the size of the next frame of data waiting to be read from socket, @@ -44,19 +57,6 @@ def next_packet_size(socket): return actual -def read_data(socket, n): - """ - Reads exactly n bytes from socket - """ - data = six.binary_type() - while len(data) < n: - next_data = read_socket(socket, n - len(data)) - if not next_data: - raise SocketError("Unexpected EOF") - data += next_data - return data - - def read_iter(socket): """ Returns a generator of frames read from socket From 9fb2caecb9e58b287fc56f84a8135848e30e1e01 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Wed, 13 Jul 2016 16:54:37 -0400 Subject: [PATCH 0045/1301] Rename next_packet_size to next_frame_size Signed-off-by: Aanand Prasad --- docker/utils/socket.py | 6 +++--- tests/integration/container_test.py | 4 ++-- tests/integration/exec_test.py | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docker/utils/socket.py b/docker/utils/socket.py index 0174d5f525..fb099b3eae 100644 --- a/docker/utils/socket.py +++ b/docker/utils/socket.py @@ -41,7 +41,7 @@ def read_data(socket, n): return data -def next_packet_size(socket): +def next_frame_size(socket): """ Returns the size of the next frame of data waiting to be read from socket, according to the protocol defined here: @@ -61,7 +61,7 @@ def read_iter(socket): """ Returns a generator of frames read from socket """ - n = next_packet_size(socket) + n = next_frame_size(socket) while n > 0: yield read_socket(socket, n) - n = next_packet_size(socket) + n = next_frame_size(socket) diff --git a/tests/integration/container_test.py b/tests/integration/container_test.py index 594aaa32ab..b2f0e511c2 100644 --- a/tests/integration/container_test.py +++ b/tests/integration/container_test.py @@ -3,7 +3,7 @@ import tempfile import docker -from docker.utils.socket import next_packet_size +from docker.utils.socket import next_frame_size from docker.utils.socket import read_data import pytest import six @@ -1027,7 +1027,7 @@ def test_run_container_reading_socket(self): self.client.start(ident) - next_size = next_packet_size(pty_stdout) + next_size = next_frame_size(pty_stdout) self.assertEqual(next_size, len(line)) data = read_data(pty_stdout, next_size) self.assertEqual(data.decode('utf-8'), line) diff --git a/tests/integration/exec_test.py b/tests/integration/exec_test.py index d0c8c9bf95..2debe3066c 100644 --- a/tests/integration/exec_test.py +++ b/tests/integration/exec_test.py @@ -1,6 +1,6 @@ import pytest -from docker.utils.socket import next_packet_size +from docker.utils.socket import next_frame_size from docker.utils.socket import read_data from .. import helpers @@ -110,7 +110,7 @@ def test_exec_start_socket(self): socket = self.client.exec_start(exec_id, socket=True) self.addCleanup(socket.close) - next_size = next_packet_size(socket) + next_size = next_frame_size(socket) self.assertEqual(next_size, len(line)) data = read_data(socket, next_size) self.assertEqual(data.decode('utf-8'), line) From 69832627f89bfc7810aebbe6560a1d8f8d19feb4 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Wed, 13 Jul 2016 17:41:59 -0400 Subject: [PATCH 0046/1301] Rename read_iter() to frames_iter() This makes it more clearly high-level and distinct from the raw data-reading functions Signed-off-by: Aanand Prasad --- docker/client.py | 6 +++--- docker/utils/socket.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docker/client.py b/docker/client.py index e131829ff7..6ca9e57ae6 100644 --- a/docker/client.py +++ b/docker/client.py @@ -29,7 +29,7 @@ from .tls import TLSConfig from .transport import UnixAdapter from .utils import utils, check_resource, update_headers, kwargs_from_env -from .utils.socket import read_iter +from .utils.socket import frames_iter try: from .transport import NpipeAdapter except ImportError: @@ -310,9 +310,9 @@ def _read_from_socket(self, response, stream): socket = self._get_raw_response_socket(response) if stream: - return read_iter(socket) + return frames_iter(socket) else: - return six.binary_type().join(read_iter(socket)) + return six.binary_type().join(frames_iter(socket)) def _disable_socket_timeout(self, socket): """ Depending on the combination of python version and whether we're diff --git a/docker/utils/socket.py b/docker/utils/socket.py index fb099b3eae..610271de72 100644 --- a/docker/utils/socket.py +++ b/docker/utils/socket.py @@ -57,7 +57,7 @@ def next_frame_size(socket): return actual -def read_iter(socket): +def frames_iter(socket): """ Returns a generator of frames read from socket """ From 267021e4535606165887760b88e75ad79542ae99 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Wed, 13 Jul 2016 18:58:57 -0400 Subject: [PATCH 0047/1301] Rename read methods for clarity read_socket() is now just read(), because its behaviour is consistent with `os.read` et al. Signed-off-by: Aanand Prasad --- docker/utils/socket.py | 11 ++++++----- tests/integration/container_test.py | 4 ++-- tests/integration/exec_test.py | 4 ++-- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/docker/utils/socket.py b/docker/utils/socket.py index 610271de72..ed343507d8 100644 --- a/docker/utils/socket.py +++ b/docker/utils/socket.py @@ -10,7 +10,7 @@ class SocketError(Exception): pass -def read_socket(socket, n=4096): +def read(socket, n=4096): """ Reads at most n bytes from socket """ @@ -28,13 +28,14 @@ def read_socket(socket, n=4096): raise -def read_data(socket, n): +def read_exactly(socket, n): """ Reads exactly n bytes from socket + Raises SocketError if there isn't enough data """ data = six.binary_type() while len(data) < n: - next_data = read_socket(socket, n - len(data)) + next_data = read(socket, n - len(data)) if not next_data: raise SocketError("Unexpected EOF") data += next_data @@ -49,7 +50,7 @@ def next_frame_size(socket): https://docs.docker.com/engine/reference/api/docker_remote_api_v1.24/#/attach-to-a-container """ try: - data = read_data(socket, 8) + data = read_exactly(socket, 8) except SocketError: return 0 @@ -63,5 +64,5 @@ def frames_iter(socket): """ n = next_frame_size(socket) while n > 0: - yield read_socket(socket, n) + yield read(socket, n) n = next_frame_size(socket) diff --git a/tests/integration/container_test.py b/tests/integration/container_test.py index b2f0e511c2..61b33983bc 100644 --- a/tests/integration/container_test.py +++ b/tests/integration/container_test.py @@ -4,7 +4,7 @@ import docker from docker.utils.socket import next_frame_size -from docker.utils.socket import read_data +from docker.utils.socket import read_exactly import pytest import six @@ -1029,7 +1029,7 @@ def test_run_container_reading_socket(self): next_size = next_frame_size(pty_stdout) self.assertEqual(next_size, len(line)) - data = read_data(pty_stdout, next_size) + data = read_exactly(pty_stdout, next_size) self.assertEqual(data.decode('utf-8'), line) diff --git a/tests/integration/exec_test.py b/tests/integration/exec_test.py index 2debe3066c..8bf2762a91 100644 --- a/tests/integration/exec_test.py +++ b/tests/integration/exec_test.py @@ -1,7 +1,7 @@ import pytest from docker.utils.socket import next_frame_size -from docker.utils.socket import read_data +from docker.utils.socket import read_exactly from .. import helpers @@ -112,7 +112,7 @@ def test_exec_start_socket(self): next_size = next_frame_size(socket) self.assertEqual(next_size, len(line)) - data = read_data(socket, next_size) + data = read_exactly(socket, next_size) self.assertEqual(data.decode('utf-8'), line) def test_exec_inspect(self): From bd73225e14265dae4e2de1b15ad4a0c7fbc3e5ba Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Fri, 8 Jul 2016 13:50:50 +0100 Subject: [PATCH 0048/1301] Set custom user agent on client Signed-off-by: Ben Firshman --- docker/client.py | 4 +++- docker/constants.py | 3 +++ docs/api.md | 1 + tests/unit/api_test.py | 30 ++++++++++++++++++++++++++++++ 4 files changed, 37 insertions(+), 1 deletion(-) diff --git a/docker/client.py b/docker/client.py index 6ca9e57ae6..c3e5874eb0 100644 --- a/docker/client.py +++ b/docker/client.py @@ -50,7 +50,8 @@ class Client( api.VolumeApiMixin, api.NetworkApiMixin): def __init__(self, base_url=None, version=None, - timeout=constants.DEFAULT_TIMEOUT_SECONDS, tls=False): + timeout=constants.DEFAULT_TIMEOUT_SECONDS, tls=False, + user_agent=constants.DEFAULT_USER_AGENT): super(Client, self).__init__() if tls and not base_url: @@ -60,6 +61,7 @@ def __init__(self, base_url=None, version=None, self.base_url = base_url self.timeout = timeout + self.headers['User-Agent'] = user_agent self._auth_configs = auth.load_config() diff --git a/docker/constants.py b/docker/constants.py index 0388f705a7..904d50ea5e 100644 --- a/docker/constants.py +++ b/docker/constants.py @@ -1,4 +1,5 @@ import sys +from .version import version DEFAULT_DOCKER_API_VERSION = '1.22' DEFAULT_TIMEOUT_SECONDS = 60 @@ -12,3 +13,5 @@ 'is deprecated and non-functional. Please remove it.' IS_WINDOWS_PLATFORM = (sys.platform == 'win32') + +DEFAULT_USER_AGENT = "docker-py/{0}".format(version) diff --git a/docs/api.md b/docs/api.md index 41c5e6cf09..e058deb759 100644 --- a/docs/api.md +++ b/docs/api.md @@ -16,6 +16,7 @@ is hosted. to use the API version provided by the server. * timeout (int): The HTTP request timeout, in seconds. * tls (bool or [TLSConfig](tls.md#TLSConfig)): Equivalent CLI options: `docker --tls ...` +* user_agent (str): Set a custom user agent for requests to the server. **** diff --git a/tests/unit/api_test.py b/tests/unit/api_test.py index 34bf14f646..696c073914 100644 --- a/tests/unit/api_test.py +++ b/tests/unit/api_test.py @@ -420,3 +420,33 @@ def test_early_stream_response(self): self.assertEqual(list(stream), [ str(i).encode() for i in range(50)]) + + +class UserAgentTest(base.BaseTestCase): + def setUp(self): + self.patcher = mock.patch.object( + docker.Client, + 'send', + return_value=fake_resp("GET", "%s/version" % fake_api.prefix) + ) + self.mock_send = self.patcher.start() + + def tearDown(self): + self.patcher.stop() + + def test_default_user_agent(self): + client = docker.Client() + client.version() + + self.assertEqual(self.mock_send.call_count, 1) + headers = self.mock_send.call_args[0][0].headers + expected = 'docker-py/%s' % docker.__version__ + self.assertEqual(headers['User-Agent'], expected) + + def test_custom_user_agent(self): + client = docker.Client(user_agent='foo/bar') + client.version() + + self.assertEqual(self.mock_send.call_count, 1) + headers = self.mock_send.call_args[0][0].headers + self.assertEqual(headers['User-Agent'], 'foo/bar') From 9b63bed6a0b5185b043e85df8c49d86d2c048aa1 Mon Sep 17 00:00:00 2001 From: Keerthan Reddy Mala Date: Thu, 14 Jul 2016 22:43:33 -0600 Subject: [PATCH 0049/1301] Add optional auth config to docker push Signed-off-by: Keerthan Reddy Mala --- docker/api/image.py | 27 +++++++++++++++++---------- docs/api.md | 2 ++ 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/docker/api/image.py b/docker/api/image.py index 3e66347e42..2bdbce83ab 100644 --- a/docker/api/image.py +++ b/docker/api/image.py @@ -205,7 +205,7 @@ def pull(self, repository, tag=None, stream=False, return self._result(response) def push(self, repository, tag=None, stream=False, - insecure_registry=False, decode=False): + insecure_registry=False, auth_config=None, decode=False): if insecure_registry: warnings.warn( INSECURE_REGISTRY_DEPRECATION_WARNING.format('push()'), @@ -224,15 +224,22 @@ def push(self, repository, tag=None, stream=False, if utils.compare_version('1.5', self._version) >= 0: # If we don't have any auth data so far, try reloading the config # file one more time in case anything showed up in there. - if not self._auth_configs: - self._auth_configs = auth.load_config() - authcfg = auth.resolve_authconfig(self._auth_configs, registry) - - # Do not fail here if no authentication exists for this specific - # registry as we can have a readonly pull. Just put the header if - # we can. - if authcfg: - headers['X-Registry-Auth'] = auth.encode_header(authcfg) + if auth_config is None: + log.debug('Looking for auth config') + if not self._auth_configs: + log.debug( + "No auth config in memory - loading from filesystem" + ) + self._auth_configs = auth.load_config() + authcfg = auth.resolve_authconfig(self._auth_configs, registry) + # Do not fail here if no authentication exists for this + # specific registry as we can have a readonly pull. Just + # put the header if we can. + if authcfg: + headers['X-Registry-Auth'] = auth.encode_header(authcfg) + else: + log.debug('Sending supplied auth config') + headers['X-Registry-Auth'] = auth.encode_header(auth_config) response = self._post_json( u, None, headers=headers, stream=stream, params=params diff --git a/docs/api.md b/docs/api.md index e058deb759..9b3a7265e7 100644 --- a/docs/api.md +++ b/docs/api.md @@ -801,6 +801,8 @@ command. * tag (str): An optional tag to push * stream (bool): Stream the output as a blocking generator * insecure_registry (bool): Use `http://` to connect to the registry +* auth_config (dict): Override the credentials that Client.login has set for this request + `auth_config` should contain the `username` and `password` keys to be valid. **Returns** (generator or str): The output of the upload From 1294d3c4103fc33949edc146be9bc91fd1a05c4d Mon Sep 17 00:00:00 2001 From: Keerthan Reddy Mala Date: Thu, 21 Jul 2016 11:01:03 -0600 Subject: [PATCH 0050/1301] Add unit tests Signed-off-by: Keerthan Reddy Mala --- tests/unit/image_test.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/unit/image_test.py b/tests/unit/image_test.py index 8fd894cc07..b2b1dd6d90 100644 --- a/tests/unit/image_test.py +++ b/tests/unit/image_test.py @@ -2,6 +2,7 @@ import pytest from . import fake_api +from docker import auth from .api_test import ( DockerClientTest, fake_request, DEFAULT_TIMEOUT_SECONDS, url_prefix, fake_resolve_authconfig @@ -262,6 +263,31 @@ def test_push_image_with_tag(self): timeout=DEFAULT_TIMEOUT_SECONDS ) + def test_push_image_with_auth(self): + auth_config = { + 'username': "test_user", + 'password': "test_password", + 'serveraddress': "test_server", + } + encoded_auth = auth.encode_header(auth_config) + self.client.push( + fake_api.FAKE_IMAGE_NAME, tag=fake_api.FAKE_TAG_NAME, + auth_config=auth_config + ) + + fake_request.assert_called_with( + 'POST', + url_prefix + 'images/test_image/push', + params={ + 'tag': fake_api.FAKE_TAG_NAME, + }, + data='{}', + headers={'Content-Type': 'application/json', + 'X-Registry-Auth': encoded_auth}, + stream=False, + timeout=DEFAULT_TIMEOUT_SECONDS + ) + def test_push_image_stream(self): with mock.patch('docker.auth.auth.resolve_authconfig', fake_resolve_authconfig): From cea73760863182035ddbf1c336b388df283c7431 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 25 Jul 2016 15:04:04 -0700 Subject: [PATCH 0051/1301] Send LinkLocalIPs as part of IPAMConfig dictionary Signed-off-by: Joffrey F --- docker/utils/utils.py | 8 ++++---- tests/integration/network_test.py | 21 +++++++++++++++++++++ 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/docker/utils/utils.py b/docker/utils/utils.py index b38cda47b3..4e48fc7e0a 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -894,15 +894,15 @@ def create_endpoint_config(version, aliases=None, links=None, if ipv6_address: ipam_config['IPv6Address'] = ipv6_address - if ipam_config: - endpoint_config['IPAMConfig'] = ipam_config - if link_local_ips is not None: if version_lt(version, '1.24'): raise errors.InvalidVersion( 'link_local_ips is not supported for API version < 1.24' ) - endpoint_config['LinkLocalIPs'] = link_local_ips + ipam_config['LinkLocalIPs'] = link_local_ips + + if ipam_config: + endpoint_config['IPAMConfig'] = ipam_config return endpoint_config diff --git a/tests/integration/network_test.py b/tests/integration/network_test.py index f719fea485..27e1b14dec 100644 --- a/tests/integration/network_test.py +++ b/tests/integration/network_test.py @@ -249,6 +249,27 @@ def test_create_with_ipv6_address(self): '2001:389::f00d' ) + @requires_api_version('1.24') + def test_create_with_linklocal_ips(self): + container = self.client.create_container( + 'busybox', 'top', + networking_config=self.client.create_networking_config( + { + 'bridge': self.client.create_endpoint_config( + link_local_ips=['169.254.8.8'] + ) + } + ), + host_config=self.client.create_host_config(network_mode='bridge') + ) + self.tmp_containers.append(container) + self.client.start(container) + container_data = self.client.inspect_container(container) + net_cfg = container_data['NetworkSettings']['Networks']['bridge'] + assert 'IPAMConfig' in net_cfg + assert 'LinkLocalIPs' in net_cfg['IPAMConfig'] + assert net_cfg['IPAMConfig']['LinkLocalIPs'] == ['169.254.8.8'] + @requires_api_version('1.22') def test_create_with_links(self): net_name, net_id = self.create_network() From 0e68b0a42989fa8f072acde2f947055ae7d64d7a Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Mon, 25 Jul 2016 14:08:25 +0100 Subject: [PATCH 0052/1301] Default to npipe address on Windows Signed-off-by: Aanand Prasad --- docker/utils/utils.py | 4 +++- tests/unit/utils_test.py | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docker/utils/utils.py b/docker/utils/utils.py index 4e48fc7e0a..4d218692e6 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -36,6 +36,8 @@ DEFAULT_HTTP_HOST = "127.0.0.1" DEFAULT_UNIX_SOCKET = "http+unix://var/run/docker.sock" +DEFAULT_NPIPE = 'npipe:////./pipe/docker_engine' + BYTE_UNITS = { 'b': 1, 'k': 1024, @@ -390,7 +392,7 @@ def parse_host(addr, is_win32=False, tls=False): path = '' if not addr and is_win32: - addr = '{0}:{1}'.format(DEFAULT_HTTP_HOST, 2375) + addr = DEFAULT_NPIPE if not addr or addr.strip() == 'unix://': return DEFAULT_UNIX_SOCKET diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index 128778f17e..68484fe592 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -419,11 +419,11 @@ def test_parse_host(self): def test_parse_host_empty_value(self): unix_socket = 'http+unix://var/run/docker.sock' - tcp_port = 'http://127.0.0.1:2375' + npipe = 'npipe:////./pipe/docker_engine' for val in [None, '']: assert parse_host(val, is_win32=False) == unix_socket - assert parse_host(val, is_win32=True) == tcp_port + assert parse_host(val, is_win32=True) == npipe def test_parse_host_tls(self): host_value = 'myhost.docker.net:3348' From ae86949188589ae35ef49f36c8091b6862365ff2 Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Fri, 8 Jul 2016 13:50:50 +0100 Subject: [PATCH 0053/1301] Set custom user agent on client Signed-off-by: Ben Firshman --- docker/client.py | 4 +++- docker/constants.py | 3 +++ docs/api.md | 1 + tests/unit/api_test.py | 30 ++++++++++++++++++++++++++++++ 4 files changed, 37 insertions(+), 1 deletion(-) diff --git a/docker/client.py b/docker/client.py index b96a78ce4e..81e9de9e32 100644 --- a/docker/client.py +++ b/docker/client.py @@ -49,7 +49,8 @@ class Client( api.VolumeApiMixin, api.NetworkApiMixin): def __init__(self, base_url=None, version=None, - timeout=constants.DEFAULT_TIMEOUT_SECONDS, tls=False): + timeout=constants.DEFAULT_TIMEOUT_SECONDS, tls=False, + user_agent=constants.DEFAULT_USER_AGENT): super(Client, self).__init__() if tls and not base_url: @@ -59,6 +60,7 @@ def __init__(self, base_url=None, version=None, self.base_url = base_url self.timeout = timeout + self.headers['User-Agent'] = user_agent self._auth_configs = auth.load_config() diff --git a/docker/constants.py b/docker/constants.py index 0388f705a7..904d50ea5e 100644 --- a/docker/constants.py +++ b/docker/constants.py @@ -1,4 +1,5 @@ import sys +from .version import version DEFAULT_DOCKER_API_VERSION = '1.22' DEFAULT_TIMEOUT_SECONDS = 60 @@ -12,3 +13,5 @@ 'is deprecated and non-functional. Please remove it.' IS_WINDOWS_PLATFORM = (sys.platform == 'win32') + +DEFAULT_USER_AGENT = "docker-py/{0}".format(version) diff --git a/docs/api.md b/docs/api.md index 51b6e2716a..de0ad743a7 100644 --- a/docs/api.md +++ b/docs/api.md @@ -16,6 +16,7 @@ is hosted. to use the API version provided by the server. * timeout (int): The HTTP request timeout, in seconds. * tls (bool or [TLSConfig](tls.md#TLSConfig)): Equivalent CLI options: `docker --tls ...` +* user_agent (str): Set a custom user agent for requests to the server. **** diff --git a/tests/unit/api_test.py b/tests/unit/api_test.py index 23fd191346..bfe196cfcf 100644 --- a/tests/unit/api_test.py +++ b/tests/unit/api_test.py @@ -415,3 +415,33 @@ def test_early_stream_response(self): self.assertEqual(list(stream), [ str(i).encode() for i in range(50)]) + + +class UserAgentTest(base.BaseTestCase): + def setUp(self): + self.patcher = mock.patch.object( + docker.Client, + 'send', + return_value=fake_resp("GET", "%s/version" % fake_api.prefix) + ) + self.mock_send = self.patcher.start() + + def tearDown(self): + self.patcher.stop() + + def test_default_user_agent(self): + client = docker.Client() + client.version() + + self.assertEqual(self.mock_send.call_count, 1) + headers = self.mock_send.call_args[0][0].headers + expected = 'docker-py/%s' % docker.__version__ + self.assertEqual(headers['User-Agent'], expected) + + def test_custom_user_agent(self): + client = docker.Client(user_agent='foo/bar') + client.version() + + self.assertEqual(self.mock_send.call_count, 1) + headers = self.mock_send.call_args[0][0].headers + self.assertEqual(headers['User-Agent'], 'foo/bar') From 5b1e5564740b50cbbaf3223e9d573df25e7eea83 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 26 Jul 2016 12:11:32 -0700 Subject: [PATCH 0054/1301] Bump to 1.9.0 Signed-off-by: Joffrey F --- docker/version.py | 2 +- docs/change_log.md | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index a09ddc8104..95405c7446 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "1.9.0-rc2" +version = "1.9.0" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) diff --git a/docs/change_log.md b/docs/change_log.md index 2d80022549..089c003461 100644 --- a/docs/change_log.md +++ b/docs/change_log.md @@ -15,6 +15,8 @@ Change Log * Added support for the `internal` param in `Client.create_network`. * Added support for `ipv4_address` and `ipv6_address` in utils function `create_endpoint_config`. +* Added support for custom user agent setting in the `Client` constructor. + By default, docker-py now also declares itself in the `User-Agent` header. ### Bugfixes From 2d3bda84de39a75e560fc79512143d43e5d61226 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 26 Jul 2016 15:48:29 -0700 Subject: [PATCH 0055/1301] dev version Signed-off-by: Joffrey F --- docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index 95405c7446..dea7b7cb01 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "1.9.0" +version = "1.10.0-dev" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) From 723d144db528ff8defce7c6172ab11a4aa67f54c Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 27 Jul 2016 18:42:14 -0700 Subject: [PATCH 0056/1301] Add support for IPv6 docker host connections. Signed-off-by: Joffrey F --- docker/utils/utils.py | 38 ++++++++++++++++++-------------------- tests/unit/utils_test.py | 10 +++++++++- 2 files changed, 27 insertions(+), 21 deletions(-) diff --git a/docker/utils/utils.py b/docker/utils/utils.py index 4d218692e6..1cfc8acc2d 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -22,8 +22,8 @@ import tempfile import warnings from distutils.version import StrictVersion -from fnmatch import fnmatch from datetime import datetime +from fnmatch import fnmatch import requests import six @@ -33,6 +33,10 @@ from .. import tls from .types import Ulimit, LogConfig +if six.PY2: + from urllib import splitnport +else: + from urllib.parse import splitnport DEFAULT_HTTP_HOST = "127.0.0.1" DEFAULT_UNIX_SOCKET = "http+unix://var/run/docker.sock" @@ -387,7 +391,6 @@ def parse_repository_tag(repo_name): # Protocol translation: tcp -> http, unix -> http+unix def parse_host(addr, is_win32=False, tls=False): proto = "http+unix" - host = DEFAULT_HTTP_HOST port = None path = '' @@ -427,32 +430,27 @@ def parse_host(addr, is_win32=False, tls=False): ) proto = "https" if tls else "http" - if proto != "http+unix" and ":" in addr: - host_parts = addr.split(':') - if len(host_parts) != 2: - raise errors.DockerException( - "Invalid bind address format: {0}".format(addr) - ) - if host_parts[0]: - host = host_parts[0] + if proto in ("http", "https"): + address_parts = addr.split('/', 1) + host = address_parts[0] + if len(address_parts) == 2: + path = '/' + address_parts[1] + host, port = splitnport(host) - port = host_parts[1] - if '/' in port: - port, path = port.split('/', 1) - path = '/{0}'.format(path) - try: - port = int(port) - except Exception: + if port is None: raise errors.DockerException( "Invalid port: {0}".format(addr) ) - elif proto in ("http", "https") and ':' not in addr: - raise errors.DockerException( - "Bind address needs a port: {0}".format(addr)) + if not host: + host = DEFAULT_HTTP_HOST else: host = addr + if proto in ("http", "https") and port == -1: + raise errors.DockerException( + "Bind address needs a port: {0}".format(addr)) + if proto == "http+unix" or proto == 'npipe': return "{0}://{1}".format(proto, host) return "{0}://{1}:{2}{3}".format(proto, host, port, path) diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index 68484fe592..0f7a58c9ec 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -404,10 +404,18 @@ def test_parse_host(self): 'https://kokia.jp:2375': 'https://kokia.jp:2375', 'unix:///var/run/docker.sock': 'http+unix:///var/run/docker.sock', 'unix://': 'http+unix://var/run/docker.sock', + '12.234.45.127:2375/docker/engine': ( + 'http://12.234.45.127:2375/docker/engine' + ), 'somehost.net:80/service/swarm': ( 'http://somehost.net:80/service/swarm' ), 'npipe:////./pipe/docker_engine': 'npipe:////./pipe/docker_engine', + '[fd12::82d1]:2375': 'http://[fd12::82d1]:2375', + 'https://[fd12:5672::12aa]:1090': 'https://[fd12:5672::12aa]:1090', + '[fd12::82d1]:2375/docker/engine': ( + 'http://[fd12::82d1]:2375/docker/engine' + ), } for host in invalid_hosts: @@ -415,7 +423,7 @@ def test_parse_host(self): parse_host(host, None) for host, expected in valid_hosts.items(): - self.assertEqual(parse_host(host, None), expected, msg=host) + assert parse_host(host, None) == expected def test_parse_host_empty_value(self): unix_socket = 'http+unix://var/run/docker.sock' From f006da6a43e39a1126ccca1651e9ff4abda41aaa Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Mon, 1 Aug 2016 13:51:58 +0100 Subject: [PATCH 0057/1301] More explicit debug for config path logic Signed-off-by: Aanand Prasad --- docker/auth/auth.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/docker/auth/auth.py b/docker/auth/auth.py index d23e6f3cee..b61a8d09e6 100644 --- a/docker/auth/auth.py +++ b/docker/auth/auth.py @@ -160,18 +160,24 @@ def find_config_file(config_path=None): os.path.basename(DOCKER_CONFIG_FILENAME) ) if os.environ.get('DOCKER_CONFIG') else None - paths = [ + paths = filter(None, [ config_path, # 1 environment_path, # 2 os.path.join(os.path.expanduser('~'), DOCKER_CONFIG_FILENAME), # 3 os.path.join( os.path.expanduser('~'), LEGACY_DOCKER_CONFIG_FILENAME ) # 4 - ] + ]) + + log.debug("Trying paths: {0}".format(repr(paths))) for path in paths: - if path and os.path.exists(path): + if os.path.exists(path): + log.debug("Found file at path: {0}".format(path)) return path + + log.debug("No config file found") + return None @@ -186,7 +192,6 @@ def load_config(config_path=None): config_file = find_config_file(config_path) if not config_file: - log.debug("File doesn't exist") return {} try: From f8b843b127a99dc329b9da7da4bedc050be36ebf Mon Sep 17 00:00:00 2001 From: Tristan Escalada Date: Thu, 26 May 2016 21:56:34 -0400 Subject: [PATCH 0058/1301] 1059-Fixing a bug with multiple json objects This splits the text by CRLF and then json.loads each part independently instead of attempting the parse the whole string. Signed-off-by: Tristan Escalada --- docker/client.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/docker/client.py b/docker/client.py index c3e5874eb0..de67dbe47a 100644 --- a/docker/client.py +++ b/docker/client.py @@ -251,8 +251,16 @@ def _stream_helper(self, response, decode=False): if decode: if six.PY3: data = data.decode('utf-8') - data = json.loads(data) - yield data + # remove the trailing newline + data = data.strip() + # split the data at any newlines + data_list = data.split("\r\n") + # load and yield each line seperately + for data in data_list: + data = json.loads(data) + yield data + else: + yield data else: # Response isn't chunked, meaning we probably # encountered an error immediately From dec29e1c10be3fbba239891e3bf47dc6b40ee567 Mon Sep 17 00:00:00 2001 From: Jari Takkala Date: Thu, 28 Jul 2016 22:57:35 -0400 Subject: [PATCH 0059/1301] Add support for sysctl when creating container Closes #1144 Signed-off-by: Jari Takkala --- docker/utils/utils.py | 9 ++++++++- docs/hostconfig.md | 1 + tests/unit/container_test.py | 27 +++++++++++++++++++++++++++ 3 files changed, 36 insertions(+), 1 deletion(-) diff --git a/docker/utils/utils.py b/docker/utils/utils.py index 1cfc8acc2d..00a7af14fe 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -619,7 +619,7 @@ def create_host_config(binds=None, port_bindings=None, lxc_conf=None, blkio_weight_device=None, device_read_bps=None, device_write_bps=None, device_read_iops=None, device_write_iops=None, oom_kill_disable=False, - shm_size=None, version=None, tmpfs=None, + shm_size=None, sysctls=None, version=None, tmpfs=None, oom_score_adj=None): host_config = {} @@ -725,6 +725,13 @@ def create_host_config(binds=None, port_bindings=None, lxc_conf=None, host_config['SecurityOpt'] = security_opt + if sysctls: + if not isinstance(sysctls, dict): + raise host_config_type_error('sysctls', sysctls, 'dict') + host_config['Sysctls'] = {} + for k, v in six.iteritems(sysctls): + host_config['Sysctls'][k] = six.text_type(v) + if volumes_from is not None: if isinstance(volumes_from, six.string_types): volumes_from = volumes_from.split(',') diff --git a/docs/hostconfig.md b/docs/hostconfig.md index c1e23533a5..01c4625f62 100644 --- a/docs/hostconfig.md +++ b/docs/hostconfig.md @@ -123,6 +123,7 @@ for example: for more information. * tmpfs: Temporary filesystems to mouunt. See [Using tmpfs](tmpfs.md) for more information. +* sysctls (dict): Kernel parameters to set in the container. **Returns** (dict) HostConfig dictionary diff --git a/tests/unit/container_test.py b/tests/unit/container_test.py index 2a72c17936..4c94c84428 100644 --- a/tests/unit/container_test.py +++ b/tests/unit/container_test.py @@ -1074,6 +1074,33 @@ def test_create_container_with_tmpfs_dict(self): DEFAULT_TIMEOUT_SECONDS ) + @requires_api_version('1.24') + def test_create_container_with_sysctl(self): + self.client.create_container( + 'busybox', 'true', + host_config=self.client.create_host_config( + sysctls={ + 'net.core.somaxconn': 1024, + 'net.ipv4.tcp_syncookies': '0', + } + ) + ) + + args = fake_request.call_args + self.assertEqual(args[0][1], url_prefix + 'containers/create') + expected_payload = self.base_create_payload() + expected_payload['HostConfig'] = self.client.create_host_config() + expected_payload['HostConfig']['Sysctls'] = { + 'net.core.somaxconn': '1024', 'net.ipv4.tcp_syncookies': '0', + } + self.assertEqual(json.loads(args[1]['data']), expected_payload) + self.assertEqual( + args[1]['headers'], {'Content-Type': 'application/json'} + ) + self.assertEqual( + args[1]['timeout'], DEFAULT_TIMEOUT_SECONDS + ) + class ContainerTest(DockerClientTest): def test_list_containers(self): From ae7cb4b99f45ec88616da5c4c04129a78a8c0c46 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 2 Aug 2016 17:25:50 -0700 Subject: [PATCH 0060/1301] Avoid crashing in update_headers decorator when headers kwarg is None Signed-off-by: Joffrey F --- docker/utils/decorators.py | 2 +- tests/unit/utils_test.py | 35 ++++++++++++++++++++++++++++++++++- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/docker/utils/decorators.py b/docker/utils/decorators.py index 7c41a5f805..46c28a8092 100644 --- a/docker/utils/decorators.py +++ b/docker/utils/decorators.py @@ -40,7 +40,7 @@ def wrapper(self, *args, **kwargs): def update_headers(f): def inner(self, *args, **kwargs): if 'HttpHeaders' in self._auth_configs: - if 'headers' not in kwargs: + if not kwargs.get('headers'): kwargs['headers'] = self._auth_configs['HttpHeaders'] else: kwargs['headers'].update(self._auth_configs['HttpHeaders']) diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index 0f7a58c9ec..47ced433b6 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -20,9 +20,11 @@ create_host_config, Ulimit, LogConfig, parse_bytes, parse_env_file, exclude_paths, convert_volume_binds, decode_json_header, tar, split_command, create_ipam_config, create_ipam_pool, parse_devices, + update_headers, ) -from docker.utils.utils import create_endpoint_config + from docker.utils.ports import build_port_bindings, split_port +from docker.utils.utils import create_endpoint_config from .. import base from ..helpers import make_tree @@ -34,6 +36,37 @@ ) +class DecoratorsTest(base.BaseTestCase): + def test_update_headers(self): + sample_headers = { + 'X-Docker-Locale': 'en-US', + } + + def f(self, headers=None): + return headers + + client = Client() + client._auth_configs = {} + + g = update_headers(f) + assert g(client, headers=None) is None + assert g(client, headers={}) == {} + assert g(client, headers={'Content-type': 'application/json'}) == { + 'Content-type': 'application/json', + } + + client._auth_configs = { + 'HttpHeaders': sample_headers + } + + assert g(client, headers=None) == sample_headers + assert g(client, headers={}) == sample_headers + assert g(client, headers={'Content-type': 'application/json'}) == { + 'Content-type': 'application/json', + 'X-Docker-Locale': 'en-US', + } + + class HostConfigTest(base.BaseTestCase): def test_create_host_config_no_options(self): config = create_host_config(version='1.19') From 9d48b4f60323db22c45437892e450bf8e545d3ef Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 3 Aug 2016 16:48:41 -0700 Subject: [PATCH 0061/1301] Test fixes and updated Makefile for 1.12 testing Signed-off-by: Joffrey F --- Makefile | 23 +++++++++++++++++------ docker/utils/__init__.py | 13 +++++++------ tests/integration/container_test.py | 2 +- 3 files changed, 25 insertions(+), 13 deletions(-) diff --git a/Makefile b/Makefile index c8c7213018..a635edfad5 100644 --- a/Makefile +++ b/Makefile @@ -32,16 +32,27 @@ integration-test-py3: build-py3 integration-dind: build build-py3 docker rm -vf dpy-dind || : - docker run -d --name dpy-dind --privileged dockerswarm/dind:1.10.3 docker daemon -H tcp://0.0.0.0:2375 - docker run --env="DOCKER_HOST=tcp://docker:2375" --link=dpy-dind:docker docker-py py.test tests/integration - docker run --env="DOCKER_HOST=tcp://docker:2375" --link=dpy-dind:docker docker-py3 py.test tests/integration + docker run -d --name dpy-dind --privileged dockerswarm/dind:1.12.0 docker daemon\ + -H tcp://0.0.0.0:2375 + docker run --env="DOCKER_HOST=tcp://docker:2375" --link=dpy-dind:docker docker-py\ + py.test tests/integration + docker run --env="DOCKER_HOST=tcp://docker:2375" --link=dpy-dind:docker docker-py3\ + py.test tests/integration docker rm -vf dpy-dind integration-dind-ssl: build-dind-certs build build-py3 docker run -d --name dpy-dind-certs dpy-dind-certs - docker run -d --env="DOCKER_HOST=tcp://localhost:2375" --env="DOCKER_TLS_VERIFY=1" --env="DOCKER_CERT_PATH=/certs" --volumes-from dpy-dind-certs --name dpy-dind-ssl -v /tmp --privileged dockerswarm/dind:1.10.3 docker daemon --tlsverify --tlscacert=/certs/ca.pem --tlscert=/certs/server-cert.pem --tlskey=/certs/server-key.pem -H tcp://0.0.0.0:2375 - docker run --volumes-from dpy-dind-ssl --env="DOCKER_HOST=tcp://docker:2375" --env="DOCKER_TLS_VERIFY=1" --env="DOCKER_CERT_PATH=/certs" --link=dpy-dind-ssl:docker docker-py py.test tests/integration - docker run --volumes-from dpy-dind-ssl --env="DOCKER_HOST=tcp://docker:2375" --env="DOCKER_TLS_VERIFY=1" --env="DOCKER_CERT_PATH=/certs" --link=dpy-dind-ssl:docker docker-py3 py.test tests/integration + docker run -d --env="DOCKER_HOST=tcp://localhost:2375" --env="DOCKER_TLS_VERIFY=1"\ + --env="DOCKER_CERT_PATH=/certs" --volumes-from dpy-dind-certs --name dpy-dind-ssl\ + -v /tmp --privileged dockerswarm/dind:1.12.0 docker daemon --tlsverify\ + --tlscacert=/certs/ca.pem --tlscert=/certs/server-cert.pem\ + --tlskey=/certs/server-key.pem -H tcp://0.0.0.0:2375 + docker run --volumes-from dpy-dind-ssl --env="DOCKER_HOST=tcp://docker:2375"\ + --env="DOCKER_TLS_VERIFY=1" --env="DOCKER_CERT_PATH=/certs"\ + --link=dpy-dind-ssl:docker docker-py py.test tests/integration + docker run --volumes-from dpy-dind-ssl --env="DOCKER_HOST=tcp://docker:2375"\ + --env="DOCKER_TLS_VERIFY=1" --env="DOCKER_CERT_PATH=/certs"\ + --link=dpy-dind-ssl:docker docker-py3 py.test tests/integration docker rm -vf dpy-dind-ssl dpy-dind-certs flake8: build diff --git a/docker/utils/__init__.py b/docker/utils/__init__.py index ccc38191ee..41df0047cb 100644 --- a/docker/utils/__init__.py +++ b/docker/utils/__init__.py @@ -1,11 +1,12 @@ +# flake8: noqa from .utils import ( compare_version, convert_port_bindings, convert_volume_binds, mkbuildcontext, tar, exclude_paths, parse_repository_tag, parse_host, - kwargs_from_env, convert_filters, datetime_to_timestamp, create_host_config, - create_container_config, parse_bytes, ping_registry, parse_env_file, - version_lt, version_gte, decode_json_header, split_command, + kwargs_from_env, convert_filters, datetime_to_timestamp, + create_host_config, create_container_config, parse_bytes, ping_registry, + parse_env_file, version_lt, version_gte, decode_json_header, split_command, create_ipam_config, create_ipam_pool, parse_devices, normalize_links, -) # flake8: noqa +) -from .types import Ulimit, LogConfig # flake8: noqa -from .decorators import check_resource, minimum_version, update_headers #flake8: noqa +from .types import Ulimit, LogConfig +from .decorators import check_resource, minimum_version, update_headers diff --git a/tests/integration/container_test.py b/tests/integration/container_test.py index 61b33983bc..f347c12a98 100644 --- a/tests/integration/container_test.py +++ b/tests/integration/container_test.py @@ -292,7 +292,7 @@ def test_invalid_log_driver_raises_exception(self): ) self.client.start(container) - assert expected_msg in str(excinfo.value) + assert six.b(expected_msg) in excinfo.value.explanation def test_valid_no_log_driver_specified(self): log_config = docker.utils.LogConfig( From 44868fa0faf26a597703bf49114c54fe1a064066 Mon Sep 17 00:00:00 2001 From: minzhang Date: Wed, 29 Jun 2016 23:51:58 -0700 Subject: [PATCH 0062/1301] Added support for docker swarm api version 1.24. 3 API are added swarm_init() swarm_leave() swarm_join() Signed-off-by: Min Zhang Signed-off-by: Min Zhang --- docker/api/__init__.py | 1 + docker/api/swarm.py | 41 +++++++++++++++++++++++++++++++++++++++++ docker/client.py | 3 ++- docs/swarm.md | 35 +++++++++++++++++++++++++++++++++++ 4 files changed, 79 insertions(+), 1 deletion(-) create mode 100644 docker/api/swarm.py create mode 100644 docs/swarm.md diff --git a/docker/api/__init__.py b/docker/api/__init__.py index 9e7442890c..b0d60878f7 100644 --- a/docker/api/__init__.py +++ b/docker/api/__init__.py @@ -6,3 +6,4 @@ from .image import ImageApiMixin from .volume import VolumeApiMixin from .network import NetworkApiMixin +from .swarm import SwarmApiMixin diff --git a/docker/api/swarm.py b/docker/api/swarm.py new file mode 100644 index 0000000000..1b6f3429ee --- /dev/null +++ b/docker/api/swarm.py @@ -0,0 +1,41 @@ +from .. import utils +import logging +log = logging.getLogger(__name__) + + +class SwarmApiMixin(object): + @utils.minimum_version('1.24') + def swarm(self): + url = self._url('/swarm') + return self._result(self._get(url), True) + + @utils.minimum_version('1.24') + def swarm_init(self, listen_addr, force_new_cluster=False, + swarm_opts=None): + url = self._url('/swarm/init') + if swarm_opts is not None and not isinstance(swarm_opts, dict): + raise TypeError('swarm_opts must be a dictionary') + data = { + 'ListenAddr': listen_addr, + 'ForceNewCluster': force_new_cluster, + 'Spec': swarm_opts + } + return self._result(self._post_json(url, data=data), True) + + @utils.minimum_version('1.24') + def swarm_join(self, remote_address, listen_address=None, + secret=None, ca_cert_hash=None, manager=False): + data ={ + "RemoteAddr": remote_address, + "ListenAddr": listen_address, + "Secret": secret, + "CACertHash": ca_cert_hash, + "Manager": manager + } + url = self._url('/swarm/join', ) + return self._result(self._post_json(url, data=data), True) + + @utils.minimum_version('1.24') + def swarm_leave(self): + url = self._url('/swarm/leave') + return self._result(self._post(url)) diff --git a/docker/client.py b/docker/client.py index c3e5874eb0..1b5420e9cf 100644 --- a/docker/client.py +++ b/docker/client.py @@ -48,7 +48,8 @@ class Client( api.ExecApiMixin, api.ImageApiMixin, api.VolumeApiMixin, - api.NetworkApiMixin): + api.NetworkApiMixin, + api.SwarmApiMixin): def __init__(self, base_url=None, version=None, timeout=constants.DEFAULT_TIMEOUT_SECONDS, tls=False, user_agent=constants.DEFAULT_USER_AGENT): diff --git a/docs/swarm.md b/docs/swarm.md new file mode 100644 index 0000000000..e3a1cd1208 --- /dev/null +++ b/docs/swarm.md @@ -0,0 +1,35 @@ +# Using swarm for API version 1.24 or higher + +Swarm initialization is done in two parts. Provide a listen_addr and `force_new_cluster` (OPTIONAL) to +the `Client().swarm_init()` method, and declare mappings in the +`swarm_opts` section. + +```python +swarm_id = cli.swarm_init(listen_addr="0.0.0.0:4500", +swarm_opts={ + "AcceptancePolicy": { + "Policies": [ + { + "Role": "MANAGER", + "Autoaccept": True + } + ] + } +}) +``` + +Join another swarm, by providing the remote_address, listen_address(optional), +secret(optional), ca_cert_hash(optional, manager(optional) +```python +cli.swarm_join( + remote_address="swarm-master:2377", + manager=True +) +``` + + +Leave swarm + +```python +cli.swarm_leave() +``` From 9fdc8d476dfad2dad20ded9a1a471a225dc398aa Mon Sep 17 00:00:00 2001 From: minzhang Date: Wed, 29 Jun 2016 23:51:58 -0700 Subject: [PATCH 0063/1301] Added support for docker swarm api version 1.24. 3 API are added swarm_init() swarm_leave() swarm_join() Signed-off-by: Min Zhang Signed-off-by: Min Zhang --- docker/api/swarm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/api/swarm.py b/docker/api/swarm.py index 1b6f3429ee..be3eae41b8 100644 --- a/docker/api/swarm.py +++ b/docker/api/swarm.py @@ -25,7 +25,7 @@ def swarm_init(self, listen_addr, force_new_cluster=False, @utils.minimum_version('1.24') def swarm_join(self, remote_address, listen_address=None, secret=None, ca_cert_hash=None, manager=False): - data ={ + data = { "RemoteAddr": remote_address, "ListenAddr": listen_address, "Secret": secret, From 07563cfe3f565b57b955455d2ce2b350ed34883b Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 1 Aug 2016 13:59:52 -0700 Subject: [PATCH 0064/1301] Update swarm methods to include newly added parameters Rename swarm methods to be more explicit Utility methods / types to create swarm spec objects Integration tests Signed-off-by: Joffrey F --- docker/api/swarm.py | 43 ++++++++++++++++---------- docker/constants.py | 2 +- docker/utils/__init__.py | 5 ++- docker/utils/types.py | 49 +++++++++++++++++++++++++++++ tests/integration/swarm_test.py | 55 +++++++++++++++++++++++++++++++++ 5 files changed, 136 insertions(+), 18 deletions(-) create mode 100644 tests/integration/swarm_test.py diff --git a/docker/api/swarm.py b/docker/api/swarm.py index be3eae41b8..bc2179cc2b 100644 --- a/docker/api/swarm.py +++ b/docker/api/swarm.py @@ -4,38 +4,49 @@ class SwarmApiMixin(object): - @utils.minimum_version('1.24') - def swarm(self): - url = self._url('/swarm') - return self._result(self._get(url), True) + + def create_swarm_spec(self, *args, **kwargs): + return utils.SwarmSpec(*args, **kwargs) @utils.minimum_version('1.24') - def swarm_init(self, listen_addr, force_new_cluster=False, - swarm_opts=None): + def init_swarm(self, advertise_addr, listen_addr='0.0.0.0:2377', + force_new_cluster=False, swarm_spec=None): url = self._url('/swarm/init') - if swarm_opts is not None and not isinstance(swarm_opts, dict): - raise TypeError('swarm_opts must be a dictionary') + if swarm_spec is not None and not isinstance(swarm_spec, dict): + raise TypeError('swarm_spec must be a dictionary') data = { + 'AdvertiseAddr': advertise_addr, 'ListenAddr': listen_addr, 'ForceNewCluster': force_new_cluster, - 'Spec': swarm_opts + 'Spec': swarm_spec, } - return self._result(self._post_json(url, data=data), True) + response = self._post_json(url, data=data) + self._raise_for_status(response) + return True + + @utils.minimum_version('1.24') + def inspect_swarm(self): + url = self._url('/swarm') + return self._result(self._get(url), True) @utils.minimum_version('1.24') - def swarm_join(self, remote_address, listen_address=None, + def join_swarm(self, remote_addresses, listen_address=None, secret=None, ca_cert_hash=None, manager=False): data = { - "RemoteAddr": remote_address, + "RemoteAddrs": remote_addresses, "ListenAddr": listen_address, "Secret": secret, "CACertHash": ca_cert_hash, "Manager": manager } - url = self._url('/swarm/join', ) - return self._result(self._post_json(url, data=data), True) + url = self._url('/swarm/join') + response = self._post_json(url, data=data) + self._raise_for_status(response) + return True @utils.minimum_version('1.24') - def swarm_leave(self): + def leave_swarm(self, force=False): url = self._url('/swarm/leave') - return self._result(self._post(url)) + response = self._post(url, params={'force': force}) + self._raise_for_status(response) + return True diff --git a/docker/constants.py b/docker/constants.py index 904d50ea5e..cf5a39acdd 100644 --- a/docker/constants.py +++ b/docker/constants.py @@ -1,7 +1,7 @@ import sys from .version import version -DEFAULT_DOCKER_API_VERSION = '1.22' +DEFAULT_DOCKER_API_VERSION = '1.24' DEFAULT_TIMEOUT_SECONDS = 60 STREAM_HEADER_SIZE_BYTES = 8 CONTAINER_LIMITS_KEYS = [ diff --git a/docker/utils/__init__.py b/docker/utils/__init__.py index 41df0047cb..c02adea1ee 100644 --- a/docker/utils/__init__.py +++ b/docker/utils/__init__.py @@ -8,5 +8,8 @@ create_ipam_config, create_ipam_pool, parse_devices, normalize_links, ) -from .types import Ulimit, LogConfig +from .types import LogConfig, Ulimit +from .types import ( + SwarmAcceptancePolicy, SwarmExternalCA, SwarmSpec, +) from .decorators import check_resource, minimum_version, update_headers diff --git a/docker/utils/types.py b/docker/utils/types.py index ea9f06d549..b970114efe 100644 --- a/docker/utils/types.py +++ b/docker/utils/types.py @@ -94,3 +94,52 @@ def hard(self): @hard.setter def hard(self, value): self['Hard'] = value + + +class SwarmSpec(DictType): + def __init__(self, policies=None, task_history_retention_limit=None, + snapshot_interval=None, keep_old_snapshots=None, + log_entries_for_slow_followers=None, heartbeat_tick=None, + election_tick=None, dispatcher_heartbeat_period=None, + node_cert_expiry=None, external_ca=None): + if policies is not None: + self['AcceptancePolicy'] = {'Policies': policies} + if task_history_retention_limit is not None: + self['Orchestration'] = { + 'TaskHistoryRetentionLimit': task_history_retention_limit + } + if any(snapshot_interval, keep_old_snapshots, + log_entries_for_slow_followers, heartbeat_tick, election_tick): + self['Raft'] = { + 'SnapshotInterval': snapshot_interval, + 'KeepOldSnapshots': keep_old_snapshots, + 'LogEntriesForSlowFollowers': log_entries_for_slow_followers, + 'HeartbeatTick': heartbeat_tick, + 'ElectionTick': election_tick + } + + if dispatcher_heartbeat_period: + self['Dispatcher'] = { + 'HeartbeatPeriod': dispatcher_heartbeat_period + } + + if node_cert_expiry or external_ca: + self['CAConfig'] = { + 'NodeCertExpiry': node_cert_expiry, + 'ExternalCA': external_ca + } + + +class SwarmAcceptancePolicy(DictType): + def __init__(self, role, auto_accept=False, secret=None): + self['Role'] = role.upper() + self['Autoaccept'] = auto_accept + if secret is not None: + self['Secret'] = secret + + +class SwarmExternalCA(DictType): + def __init__(self, url, protocol=None, options=None): + self['URL'] = url + self['Protocol'] = protocol + self['Options'] = options diff --git a/tests/integration/swarm_test.py b/tests/integration/swarm_test.py new file mode 100644 index 0000000000..734d470174 --- /dev/null +++ b/tests/integration/swarm_test.py @@ -0,0 +1,55 @@ +import docker +import pytest + +from ..base import requires_api_version +from .. import helpers + + +BUSYBOX = helpers.BUSYBOX + + +class SwarmTest(helpers.BaseTestCase): + def setUp(self): + super(SwarmTest, self).setUp() + try: + self.client.leave_swarm(force=True) + except docker.errors.APIError: + pass + + def tearDown(self): + super(SwarmTest, self).tearDown() + try: + self.client.leave_swarm(force=True) + except docker.errors.APIError: + pass + + @requires_api_version('1.24') + def test_init_swarm_simple(self): + assert self.client.init_swarm('eth0') + + @requires_api_version('1.24') + def test_init_swarm_force_new_cluster(self): + pytest.skip('Test stalls the engine on 1.12') + + assert self.client.init_swarm('eth0') + version_1 = self.client.inspect_swarm()['Version']['Index'] + assert self.client.init_swarm('eth0', force_new_cluster=True) + version_2 = self.client.inspect_swarm()['Version']['Index'] + assert version_2 != version_1 + + @requires_api_version('1.24') + def test_init_already_in_cluster(self): + assert self.client.init_swarm('eth0') + with pytest.raises(docker.errors.APIError): + self.client.init_swarm('eth0') + + @requires_api_version('1.24') + def test_leave_swarm(self): + assert self.client.init_swarm('eth0') + with pytest.raises(docker.errors.APIError) as exc_info: + self.client.leave_swarm() + exc_info.value.response.status_code == 500 + assert self.client.leave_swarm(force=True) + with pytest.raises(docker.errors.APIError) as exc_info: + self.client.inspect_swarm() + exc_info.value.response.status_code == 406 From 1f055796a8992da041280401025792cc8fb22336 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 3 Aug 2016 18:00:29 -0700 Subject: [PATCH 0065/1301] Add new init_swarm test with custom spec Signed-off-by: Joffrey F --- docker/utils/types.py | 4 ++-- tests/integration/swarm_test.py | 14 +++++++++++++- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/docker/utils/types.py b/docker/utils/types.py index b970114efe..725c8c8100 100644 --- a/docker/utils/types.py +++ b/docker/utils/types.py @@ -108,8 +108,8 @@ def __init__(self, policies=None, task_history_retention_limit=None, self['Orchestration'] = { 'TaskHistoryRetentionLimit': task_history_retention_limit } - if any(snapshot_interval, keep_old_snapshots, - log_entries_for_slow_followers, heartbeat_tick, election_tick): + if any([snapshot_interval, keep_old_snapshots, + log_entries_for_slow_followers, heartbeat_tick, election_tick]): self['Raft'] = { 'SnapshotInterval': snapshot_interval, 'KeepOldSnapshots': keep_old_snapshots, diff --git a/tests/integration/swarm_test.py b/tests/integration/swarm_test.py index 734d470174..969e05ef7a 100644 --- a/tests/integration/swarm_test.py +++ b/tests/integration/swarm_test.py @@ -29,7 +29,7 @@ def test_init_swarm_simple(self): @requires_api_version('1.24') def test_init_swarm_force_new_cluster(self): - pytest.skip('Test stalls the engine on 1.12') + pytest.skip('Test stalls the engine on 1.12.0') assert self.client.init_swarm('eth0') version_1 = self.client.inspect_swarm()['Version']['Index'] @@ -43,6 +43,18 @@ def test_init_already_in_cluster(self): with pytest.raises(docker.errors.APIError): self.client.init_swarm('eth0') + @requires_api_version('1.24') + def test_init_swarm_custom_raft_spec(self): + spec = self.client.create_swarm_spec( + snapshot_interval=5000, log_entries_for_slow_followers=1200 + ) + assert self.client.init_swarm( + advertise_addr='eth0', swarm_spec=spec + ) + swarm_info = self.client.inspect_swarm() + assert swarm_info['Spec']['Raft']['SnapshotInterval'] == 5000 + assert swarm_info['Spec']['Raft']['LogEntriesForSlowFollowers'] == 1200 + @requires_api_version('1.24') def test_leave_swarm(self): assert self.client.init_swarm('eth0') From df31f9a8ce9c43f4e4b23d81d711fe76a2b7f696 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 3 Aug 2016 18:00:52 -0700 Subject: [PATCH 0066/1301] Update Swarm documentation Signed-off-by: Joffrey F --- docs/swarm.md | 83 +++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 61 insertions(+), 22 deletions(-) diff --git a/docs/swarm.md b/docs/swarm.md index e3a1cd1208..44a855b1be 100644 --- a/docs/swarm.md +++ b/docs/swarm.md @@ -1,35 +1,74 @@ -# Using swarm for API version 1.24 or higher +# Swarm management -Swarm initialization is done in two parts. Provide a listen_addr and `force_new_cluster` (OPTIONAL) to -the `Client().swarm_init()` method, and declare mappings in the -`swarm_opts` section. +Starting with Engine version 1.12 (API 1.24), it is possible to manage the +engine's associated Swarm cluster using the API. + +## Initializing a new Swarm + +You can initialize a new Swarm by calling `Client.init_swarm`. An advertising +address needs to be provided, usually simply by indicating which network +interface needs to be used. Advanced options are provided using the +`swarm_spec` parameter, which can easily be created using +`Client.create_swarm_spec`. ```python -swarm_id = cli.swarm_init(listen_addr="0.0.0.0:4500", -swarm_opts={ - "AcceptancePolicy": { - "Policies": [ - { - "Role": "MANAGER", - "Autoaccept": True - } - ] - } -}) +spec = client.create_swarm_spec( + snapshot_interval=5000, log_entries_for_slow_followers=1200 +) +client.init_swarm( + advertise_addr='eth0', listen_addr='0.0.0.0:5000', force_new_cluster=False, + swarm_spec=spec +) ``` -Join another swarm, by providing the remote_address, listen_address(optional), -secret(optional), ca_cert_hash(optional, manager(optional) +## Joining an existing Swarm + +If you're looking to have the engine your client is connected to joining an +existing Swarm, this ca be accomplished by using the `Client.join_swarm` +method. You will need to provide a list of at least one remote address +corresponding to other machines already part of the swarm. In most cases, +a `listen_address` for your node, as well as the `secret` token are required +to join too. + ```python -cli.swarm_join( - remote_address="swarm-master:2377", - manager=True +client.join_swarm( + remote_addresses=['192.168.14.221:2377'], secret='SWMTKN-1-redacted', + listen_address='0.0.0.0:5000', manager=True ) ``` +## Leaving the Swarm + +To leave the swarm you are currently a member of, simply use +`Client.leave_swarm`. Note that if your engine is the Swarm's manager, +you will need to specify `force=True` to be able to leave. + +```python +client.leave_swarm(force=False) +``` + + +## Retrieving Swarm status -Leave swarm +You can retrieve information about your current Swarm status by calling +`Client.inspect_swarm`. This method takes no arguments. ```python -cli.swarm_leave() +client.inspect_swarm() ``` + +## Swarm API documentation + +### Client.init_swarm + +#### Client.create_swarm_spec + +#### docker.utils.SwarmAcceptancePolicy + +#### docker.utils.SwarmExternalCA + +### Client.inspect_swarm + +### Client.join_swarm + +### CLient.leave_swarm \ No newline at end of file From 25db440c967e7be96b431f5f744e9250ab438a36 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 4 Aug 2016 15:11:13 -0700 Subject: [PATCH 0067/1301] Update arguments for several Swarm API methods Add Client.update_swarm method Add test for Client.update_swarm Signed-off-by: Joffrey F --- docker/api/swarm.py | 27 +++++++++++++++++++-------- docker/utils/__init__.py | 2 +- docker/utils/types.py | 12 +----------- tests/integration/swarm_test.py | 31 +++++++++++++++++++++++++++++++ 4 files changed, 52 insertions(+), 20 deletions(-) diff --git a/docker/api/swarm.py b/docker/api/swarm.py index bc2179cc2b..28f9336a51 100644 --- a/docker/api/swarm.py +++ b/docker/api/swarm.py @@ -9,7 +9,7 @@ def create_swarm_spec(self, *args, **kwargs): return utils.SwarmSpec(*args, **kwargs) @utils.minimum_version('1.24') - def init_swarm(self, advertise_addr, listen_addr='0.0.0.0:2377', + def init_swarm(self, advertise_addr=None, listen_addr='0.0.0.0:2377', force_new_cluster=False, swarm_spec=None): url = self._url('/swarm/init') if swarm_spec is not None and not isinstance(swarm_spec, dict): @@ -30,14 +30,13 @@ def inspect_swarm(self): return self._result(self._get(url), True) @utils.minimum_version('1.24') - def join_swarm(self, remote_addresses, listen_address=None, - secret=None, ca_cert_hash=None, manager=False): + def join_swarm(self, remote_addrs, join_token, listen_addr=None, + advertise_addr=None): data = { - "RemoteAddrs": remote_addresses, - "ListenAddr": listen_address, - "Secret": secret, - "CACertHash": ca_cert_hash, - "Manager": manager + "RemoteAddrs": remote_addrs, + "ListenAddr": listen_addr, + "JoinToken": join_token, + "AdvertiseAddr": advertise_addr, } url = self._url('/swarm/join') response = self._post_json(url, data=data) @@ -50,3 +49,15 @@ def leave_swarm(self, force=False): response = self._post(url, params={'force': force}) self._raise_for_status(response) return True + + @utils.minimum_version('1.24') + def update_swarm(self, version, swarm_spec=None, rotate_worker_token=False, + rotate_manager_token=False): + url = self._url('/swarm/update') + response = self._post_json(url, data=swarm_spec, params={ + 'rotateWorkerToken': rotate_worker_token, + 'rotateManagerToken': rotate_manager_token, + 'version': version + }) + self._raise_for_status(response) + return True diff --git a/docker/utils/__init__.py b/docker/utils/__init__.py index c02adea1ee..35acc77911 100644 --- a/docker/utils/__init__.py +++ b/docker/utils/__init__.py @@ -10,6 +10,6 @@ from .types import LogConfig, Ulimit from .types import ( - SwarmAcceptancePolicy, SwarmExternalCA, SwarmSpec, + SwarmExternalCA, SwarmSpec, ) from .decorators import check_resource, minimum_version, update_headers diff --git a/docker/utils/types.py b/docker/utils/types.py index 725c8c8100..92faaa8a02 100644 --- a/docker/utils/types.py +++ b/docker/utils/types.py @@ -97,13 +97,11 @@ def hard(self, value): class SwarmSpec(DictType): - def __init__(self, policies=None, task_history_retention_limit=None, + def __init__(self, task_history_retention_limit=None, snapshot_interval=None, keep_old_snapshots=None, log_entries_for_slow_followers=None, heartbeat_tick=None, election_tick=None, dispatcher_heartbeat_period=None, node_cert_expiry=None, external_ca=None): - if policies is not None: - self['AcceptancePolicy'] = {'Policies': policies} if task_history_retention_limit is not None: self['Orchestration'] = { 'TaskHistoryRetentionLimit': task_history_retention_limit @@ -130,14 +128,6 @@ def __init__(self, policies=None, task_history_retention_limit=None, } -class SwarmAcceptancePolicy(DictType): - def __init__(self, role, auto_accept=False, secret=None): - self['Role'] = role.upper() - self['Autoaccept'] = auto_accept - if secret is not None: - self['Secret'] = secret - - class SwarmExternalCA(DictType): def __init__(self, url, protocol=None, options=None): self['URL'] = url diff --git a/tests/integration/swarm_test.py b/tests/integration/swarm_test.py index 969e05ef7a..226689baa4 100644 --- a/tests/integration/swarm_test.py +++ b/tests/integration/swarm_test.py @@ -65,3 +65,34 @@ def test_leave_swarm(self): with pytest.raises(docker.errors.APIError) as exc_info: self.client.inspect_swarm() exc_info.value.response.status_code == 406 + + @requires_api_version('1.24') + def test_update_swarm(self): + assert self.client.init_swarm('eth0') + swarm_info_1 = self.client.inspect_swarm() + spec = self.client.create_swarm_spec( + snapshot_interval=5000, log_entries_for_slow_followers=1200, + node_cert_expiry=7776000000000000 + ) + assert self.client.update_swarm( + version=swarm_info_1['Version']['Index'], + swarm_spec=spec, rotate_worker_token=True + ) + swarm_info_2 = self.client.inspect_swarm() + + assert ( + swarm_info_1['Version']['Index'] != + swarm_info_2['Version']['Index'] + ) + assert swarm_info_2['Spec']['Raft']['SnapshotInterval'] == 5000 + assert ( + swarm_info_2['Spec']['Raft']['LogEntriesForSlowFollowers'] == 1200 + ) + assert ( + swarm_info_1['JoinTokens']['Manager'] == + swarm_info_2['JoinTokens']['Manager'] + ) + assert ( + swarm_info_1['JoinTokens']['Worker'] != + swarm_info_2['JoinTokens']['Worker'] + ) From fdfe582b764ab5a194f147b9d8efa04e7ae43f1c Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 4 Aug 2016 15:12:43 -0700 Subject: [PATCH 0068/1301] Update Swarm API docs Signed-off-by: Joffrey F --- docs/swarm.md | 140 ++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 131 insertions(+), 9 deletions(-) diff --git a/docs/swarm.md b/docs/swarm.md index 44a855b1be..2c87702c2b 100644 --- a/docs/swarm.md +++ b/docs/swarm.md @@ -23,17 +23,17 @@ client.init_swarm( ## Joining an existing Swarm -If you're looking to have the engine your client is connected to joining an -existing Swarm, this ca be accomplished by using the `Client.join_swarm` +If you're looking to have the engine your client is connected to join an +existing Swarm, this can be accomplished by using the `Client.join_swarm` method. You will need to provide a list of at least one remote address -corresponding to other machines already part of the swarm. In most cases, -a `listen_address` for your node, as well as the `secret` token are required -to join too. +corresponding to other machines already part of the swarm as well as the +`join_token`. In most cases, a `listen_addr` and `advertise_addr` for your +node are also required. ```python client.join_swarm( - remote_addresses=['192.168.14.221:2377'], secret='SWMTKN-1-redacted', - listen_address='0.0.0.0:5000', manager=True + remote_addrs=['192.168.14.221:2377'], join_token='SWMTKN-1-redacted', + listen_addr='0.0.0.0:5000', advertise_addr='eth0:5000' ) ``` @@ -61,14 +61,136 @@ client.inspect_swarm() ### Client.init_swarm +Initialize a new Swarm using the current connected engine as the first node. + +**Params:** + +* advertise_addr (string): Externally reachable address advertised to other + nodes. This can either be an address/port combination in the form + `192.168.1.1:4567`, or an interface followed by a port number, like + `eth0:4567`. If the port number is omitted, the port number from the listen + address is used. If `advertise_addr` is not specified, it will be + automatically detected when possible. Default: None +* listen_addr (string): Listen address used for inter-manager communication, + as well as determining the networking interface used for the VXLAN Tunnel + Endpoint (VTEP). This can either be an address/port combination in the form + `192.168.1.1:4567`, or an interface followed by a port number, like + `eth0:4567`. If the port number is omitted, the default swarm listening port + is used. Default: '0.0.0.0:2377' +* force_new_cluster (bool): Force creating a new Swarm, even if already part of + one. Default: False +* swarm_spec (dict): Configuration settings of the new Swarm. Use + `Client.create_swarm_spec` to generate a valid configuration. Default: None + +**Returns:** `True` if the request went through. Raises an `APIError` if it + fails. + #### Client.create_swarm_spec -#### docker.utils.SwarmAcceptancePolicy +Create a `docker.utils.SwarmSpec` instance that can be used as the `swarm_spec` +argument in `Client.init_swarm`. + +**Params:** + +* task_history_retention_limit (int): Maximum number of tasks history stored. +* snapshot_interval (int): Number of logs entries between snapshot. +* keep_old_snapshots (int): Number of snapshots to keep beyond the current + snapshot. +* log_entries_for_slow_followers (int): Number of log entries to keep around + to sync up slow followers after a snapshot is created. +* heartbeat_tick (int): Amount of ticks (in seconds) between each heartbeat. +* election_tick (int): Amount of ticks (in seconds) needed without a leader to + trigger a new election. +* dispatcher_heartbeat_period (int): The delay for an agent to send a + heartbeat to the dispatcher. +* node_cert_expiry (int): Automatic expiry for nodes certificates. +* external_ca (dict): Configuration for forwarding signing requests to an + external certificate authority. Use `docker.utils.SwarmExternalCA`. + +**Returns:** `docker.utils.SwarmSpec` instance. #### docker.utils.SwarmExternalCA +Create a configuration dictionary for the `external_ca` argument in a +`SwarmSpec`. + +**Params:** + +* protocol (string): Protocol for communication with the external CA (currently + only “cfssl” is supported). +* url (string): URL where certificate signing requests should be sent. +* options (dict): An object with key/value pairs that are interpreted as + protocol-specific options for the external CA driver. + ### Client.inspect_swarm +Retrieve information about the current Swarm. + +**Returns:** A dictionary containing information about the Swarm. See sample + below. + +```python +{u'CreatedAt': u'2016-08-04T21:26:18.779800579Z', + u'ID': u'8hk6e9wh4iq214qtbgvbp84a9', + u'JoinTokens': {u'Manager': u'SWMTKN-1-redacted-1', + u'Worker': u'SWMTKN-1-redacted-2'}, + u'Spec': {u'CAConfig': {u'NodeCertExpiry': 7776000000000000}, + u'Dispatcher': {u'HeartbeatPeriod': 5000000000}, + u'Name': u'default', + u'Orchestration': {u'TaskHistoryRetentionLimit': 10}, + u'Raft': {u'ElectionTick': 3, + u'HeartbeatTick': 1, + u'LogEntriesForSlowFollowers': 500, + u'SnapshotInterval': 10000}, + u'TaskDefaults': {}}, + u'UpdatedAt': u'2016-08-04T21:26:19.391623265Z', + u'Version': {u'Index': 11}} +``` + ### Client.join_swarm -### CLient.leave_swarm \ No newline at end of file +Join an existing Swarm. + +**Params:** + +* remote_addrs (list): Addresses of one or more manager nodes already + participating in the Swarm to join. +* join_token (string): Secret token for joining this Swarm. +* listen_addr (string): Listen address used for inter-manager communication + if the node gets promoted to manager, as well as determining the networking + interface used for the VXLAN Tunnel Endpoint (VTEP). Default: `None` +* advertise_addr (string): Externally reachable address advertised to other + nodes. This can either be an address/port combination in the form + `192.168.1.1:4567`, or an interface followed by a port number, like + `eth0:4567`. If the port number is omitted, the port number from the listen + address is used. If AdvertiseAddr is not specified, it will be automatically + detected when possible. Default: `None` + +**Returns:** `True` if the request went through. Raises an `APIError` if it + fails. + +### Client.leave_swarm + +Leave a Swarm. + +**Params:** + +* force (bool): Leave the Swarm even if this node is a manager. + Default: `False` + +**Returns:** `True` if the request went through. Raises an `APIError` if it + fails. + +### Client.update_swarm + +Update the Swarm's configuration + +**Params:** + +* version (int): The version number of the swarm object being updated. This + is required to avoid conflicting writes. +* swarm_spec (dict): Configuration settings to update. Use + `Client.create_swarm_spec` to generate a valid configuration. + Default: `None`. +* rotate_worker_token (bool): Rotate the worker join token. Default: `False`. +* rotate_manager_token (bool): Rotate the manager join token. Default: `False`. From 0f70b6a38b80133e6df29c9cb007bc2c709db8ec Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 4 Aug 2016 16:48:21 -0700 Subject: [PATCH 0069/1301] Add support for custom name in SwarmSpec Signed-off-by: Joffrey F --- docker/utils/types.py | 5 ++++- docs/swarm.md | 4 ++++ tests/integration/swarm_test.py | 18 ++++++++++++++++++ 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/docker/utils/types.py b/docker/utils/types.py index 92faaa8a02..d778b90dc8 100644 --- a/docker/utils/types.py +++ b/docker/utils/types.py @@ -101,7 +101,7 @@ def __init__(self, task_history_retention_limit=None, snapshot_interval=None, keep_old_snapshots=None, log_entries_for_slow_followers=None, heartbeat_tick=None, election_tick=None, dispatcher_heartbeat_period=None, - node_cert_expiry=None, external_ca=None): + node_cert_expiry=None, external_ca=None, name=None): if task_history_retention_limit is not None: self['Orchestration'] = { 'TaskHistoryRetentionLimit': task_history_retention_limit @@ -127,6 +127,9 @@ def __init__(self, task_history_retention_limit=None, 'ExternalCA': external_ca } + if name is not None: + self['Name'] = name + class SwarmExternalCA(DictType): def __init__(self, url, protocol=None, options=None): diff --git a/docs/swarm.md b/docs/swarm.md index 2c87702c2b..a9a1d1f501 100644 --- a/docs/swarm.md +++ b/docs/swarm.md @@ -106,6 +106,7 @@ argument in `Client.init_swarm`. * node_cert_expiry (int): Automatic expiry for nodes certificates. * external_ca (dict): Configuration for forwarding signing requests to an external certificate authority. Use `docker.utils.SwarmExternalCA`. +* name (string): Swarm's name **Returns:** `docker.utils.SwarmSpec` instance. @@ -194,3 +195,6 @@ Update the Swarm's configuration Default: `None`. * rotate_worker_token (bool): Rotate the worker join token. Default: `False`. * rotate_manager_token (bool): Rotate the manager join token. Default: `False`. + +**Returns:** `True` if the request went through. Raises an `APIError` if it + fails. diff --git a/tests/integration/swarm_test.py b/tests/integration/swarm_test.py index 226689baa4..b73f81c419 100644 --- a/tests/integration/swarm_test.py +++ b/tests/integration/swarm_test.py @@ -96,3 +96,21 @@ def test_update_swarm(self): swarm_info_1['JoinTokens']['Worker'] != swarm_info_2['JoinTokens']['Worker'] ) + + @requires_api_version('1.24') + def test_update_swarm_name(self): + assert self.client.init_swarm('eth0') + swarm_info_1 = self.client.inspect_swarm() + spec = self.client.create_swarm_spec( + node_cert_expiry=7776000000000000, name='reimuhakurei' + ) + assert self.client.update_swarm( + version=swarm_info_1['Version']['Index'], swarm_spec=spec + ) + swarm_info_2 = self.client.inspect_swarm() + + assert ( + swarm_info_1['Version']['Index'] != + swarm_info_2['Version']['Index'] + ) + assert swarm_info_2['Spec']['Name'] == 'reimuhakurei' From e1774c4c5b8ae8d28dfeef90236be75a8f54e88f Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 4 Aug 2016 17:03:02 -0700 Subject: [PATCH 0070/1301] Reference swarm methods in api.md file. Signed-off-by: Joffrey F --- docs/api.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/docs/api.md b/docs/api.md index 9b3a7265e7..7748254af2 100644 --- a/docs/api.md +++ b/docs/api.md @@ -606,6 +606,11 @@ Display system-wide information. Identical to the `docker info` command. 'SwapLimit': 1} ``` +## init_swarm + +Initialize a new Swarm using the current connected engine as the first node. +See the [Swarm documentation](swarm.md#clientinit_swarm). + ## insert *DEPRECATED* @@ -641,6 +646,11 @@ Retrieve network info by id. **Returns** (dict): Network information dictionary +## inspect_swarm + +Retrieve information about the current Swarm. +See the [Swarm documentation](swarm.md#clientinspect_swarm). + ## inspect_volume Retrieve volume info by name. @@ -656,6 +666,11 @@ Retrieve volume info by name. {u'Mountpoint': u'/var/lib/docker/volumes/foobar/_data', u'Driver': u'local', u'Name': u'foobar'} ``` +## join_swarm + +Join an existing Swarm. +See the [Swarm documentation](swarm.md#clientjoin_swarm). + ## kill Kill a container or send a signal to a container. @@ -665,6 +680,11 @@ Kill a container or send a signal to a container. * container (str): The container to kill * signal (str or int): The signal to send. Defaults to `SIGKILL` +## leave_swarm + +Leave the current Swarm. +See the [Swarm documentation](swarm.md#clientleave_swarm). + ## load_image Load an image that was previously saved using `Client.get_image` @@ -1054,6 +1074,11 @@ Update resource configs of one or more containers. **Returns** (dict): Dictionary containing a `Warnings` key. +## update_swarm + +Update the current Swarm. +See the [Swarm documentation](swarm.md#clientupdate_swarm). + ## version Nearly identical to the `docker version` command. From 93b4b4134e2c046433649c5e86d9c65ffd84f106 Mon Sep 17 00:00:00 2001 From: George Lester Date: Wed, 13 Jul 2016 21:36:38 -0700 Subject: [PATCH 0071/1301] Implemented dns_opt support (from api 1.21) Signed-off-by: George Lester --- docker/utils/utils.py | 8 +++++++- docs/api.md | 1 + tests/unit/utils_test.py | 13 +++++++++++++ 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/docker/utils/utils.py b/docker/utils/utils.py index 00a7af14fe..78457161bd 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -620,7 +620,7 @@ def create_host_config(binds=None, port_bindings=None, lxc_conf=None, device_write_bps=None, device_read_iops=None, device_write_iops=None, oom_kill_disable=False, shm_size=None, sysctls=None, version=None, tmpfs=None, - oom_score_adj=None): + oom_score_adj=None, dns_opt=None): host_config = {} @@ -719,6 +719,12 @@ def create_host_config(binds=None, port_bindings=None, lxc_conf=None, if dns is not None: host_config['Dns'] = dns + if dns_opt is not None: + if version_lt(version, '1.21'): + raise host_config_version_error('dns_opt', '1.21') + + host_config['DnsOptions'] = dns_opt + if security_opt is not None: if not isinstance(security_opt, list): raise host_config_type_error('security_opt', security_opt, 'list') diff --git a/docs/api.md b/docs/api.md index 9b3a7265e7..1810d5e1f9 100644 --- a/docs/api.md +++ b/docs/api.md @@ -239,6 +239,7 @@ where unit = b, k, m, or g) * environment (dict or list): A dictionary or a list of strings in the following format `["PASSWORD=xxx"]` or `{"PASSWORD": "xxx"}`. * dns (list): DNS name servers +* dns_opt (list): Additional options to be added to the container's `resolv.conf` file * volumes (str or list): * volumes_from (str or list): List of container names or Ids to get volumes from. Optionally a single string joining container id's with commas diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index 47ced433b6..537c5cfa19 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -141,6 +141,19 @@ def test_create_host_config_with_oom_score_adj(self): TypeError, lambda: create_host_config(version='1.22', oom_score_adj='100')) + def test_create_host_config_with_dns_opt(self): + + tested_opts = ['use-vc', 'no-tld-query'] + config = create_host_config(version='1.21', dns_opt=tested_opts) + dns_opts = config.get('DnsOptions') + + self.assertTrue('use-vc' in dns_opts) + self.assertTrue('no-tld-query' in dns_opts) + + self.assertRaises( + InvalidVersion, lambda: create_host_config(version='1.20', + dns_opt=tested_opts)) + def test_create_endpoint_config_with_aliases(self): config = create_endpoint_config(version='1.22', aliases=['foo', 'bar']) assert config == {'Aliases': ['foo', 'bar']} From a28a0d235593704f42db4462e5dc4ee7257c6ea3 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 9 Aug 2016 13:20:17 -0700 Subject: [PATCH 0072/1301] Exclude requests 2.11 from setup.py to work around unicode bug Signed-off-by: Joffrey F --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ac58b1f94c..85a4499422 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ SOURCE_DIR = os.path.join(ROOT_DIR) requirements = [ - 'requests >= 2.5.2', + 'requests >= 2.5.2, < 2.11', 'six >= 1.4.0', 'websocket-client >= 0.32.0', ] From 08b284ab399e9bf19296c020e158968ba3fb800b Mon Sep 17 00:00:00 2001 From: Tomas Tomecek Date: Wed, 27 Jul 2016 10:26:16 +0200 Subject: [PATCH 0073/1301] docker client consistency: don't quote ':/' E.g. docker client `/v1.21/images/localhost:5000/busybox/push?tag=` docker-py `/v1.21/images/localhost%3A5000%2Fbusybox/push` Signed-off-by: Tomas Tomecek --- docker/client.py | 4 +++- tests/unit/api_test.py | 10 ++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/docker/client.py b/docker/client.py index c3e5874eb0..771412e29d 100644 --- a/docker/client.py +++ b/docker/client.py @@ -14,6 +14,7 @@ import json import struct +from functools import partial import requests import requests.exceptions @@ -156,7 +157,8 @@ def _url(self, pathfmt, *args, **kwargs): 'instead'.format(arg, type(arg)) ) - args = map(six.moves.urllib.parse.quote_plus, args) + quote_f = partial(six.moves.urllib.parse.quote_plus, safe="/:") + args = map(quote_f, args) if kwargs.get('versioned_api', True): return '{0}/v{1}{2}'.format( diff --git a/tests/unit/api_test.py b/tests/unit/api_test.py index 696c073914..712f57e0be 100644 --- a/tests/unit/api_test.py +++ b/tests/unit/api_test.py @@ -159,9 +159,15 @@ def test_url_valid_resource(self): '{0}{1}'.format(url_prefix, 'hello/somename/world/someothername') ) - url = self.client._url('/hello/{0}/world', '/some?name') + url = self.client._url('/hello/{0}/world', 'some?name') self.assertEqual( - url, '{0}{1}'.format(url_prefix, 'hello/%2Fsome%3Fname/world') + url, '{0}{1}'.format(url_prefix, 'hello/some%3Fname/world') + ) + + url = self.client._url("/images/{0}/push", "localhost:5000/image") + self.assertEqual( + url, + '{0}{1}'.format(url_prefix, 'images/localhost:5000/image/push') ) def test_url_invalid_resource(self): From a75553b3ca1a8c1d94a49f328f96ef9a1b634c70 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 11 Aug 2016 17:16:41 -0700 Subject: [PATCH 0074/1301] Add `nodes` and `inspect_node` methods Signed-off-by: Joffrey F --- docker/api/swarm.py | 15 ++++++++++ docs/api.md | 9 ++++++ docs/swarm.md | 52 ++++++++++++++++++++++++++++++++- tests/integration/swarm_test.py | 29 ++++++++++++++++++ 4 files changed, 104 insertions(+), 1 deletion(-) diff --git a/docker/api/swarm.py b/docker/api/swarm.py index 28f9336a51..d099364552 100644 --- a/docker/api/swarm.py +++ b/docker/api/swarm.py @@ -29,6 +29,12 @@ def inspect_swarm(self): url = self._url('/swarm') return self._result(self._get(url), True) + @utils.check_resource + @utils.minimum_version('1.24') + def inspect_node(self, node_id): + url = self._url('/nodes/{0}', node_id) + return self._result(self._get(url), True) + @utils.minimum_version('1.24') def join_swarm(self, remote_addrs, join_token, listen_addr=None, advertise_addr=None): @@ -50,6 +56,15 @@ def leave_swarm(self, force=False): self._raise_for_status(response) return True + @utils.minimum_version('1.24') + def nodes(self, filters=None): + url = self._url('/nodes') + params = {} + if filters: + params['filters'] = utils.convert_filters(filters) + + return self._result(self._get(url, params=params), True) + @utils.minimum_version('1.24') def update_swarm(self, version, swarm_spec=None, rotate_worker_token=False, rotate_manager_token=False): diff --git a/docs/api.md b/docs/api.md index 7748254af2..ddfaffeb98 100644 --- a/docs/api.md +++ b/docs/api.md @@ -646,6 +646,11 @@ Retrieve network info by id. **Returns** (dict): Network information dictionary +## inspect_node + +Retrieve low-level information about a Swarm node. +See the [Swarm documentation](swarm.md#clientinspect_node). + ## inspect_swarm Retrieve information about the current Swarm. @@ -742,6 +747,10 @@ The above are combined to create a filters dict. **Returns** (dict): List of network objects. +## nodes + +List Swarm nodes. See the [Swarm documentation](swarm.md#clientnodes). + ## pause Pauses all processes within a container. diff --git a/docs/swarm.md b/docs/swarm.md index a9a1d1f501..0cd015a0eb 100644 --- a/docs/swarm.md +++ b/docs/swarm.md @@ -47,7 +47,6 @@ you will need to specify `force=True` to be able to leave. client.leave_swarm(force=False) ``` - ## Retrieving Swarm status You can retrieve information about your current Swarm status by calling @@ -57,6 +56,15 @@ You can retrieve information about your current Swarm status by calling client.inspect_swarm() ``` +## Listing Swarm nodes + +List all nodes that are part of the current Swarm using `Client.nodes`. +The `filters` argument allows to filter the results. + +```python +client.nodes(filters={'role': 'manager'}) +``` + ## Swarm API documentation ### Client.init_swarm @@ -123,6 +131,37 @@ Create a configuration dictionary for the `external_ca` argument in a * options (dict): An object with key/value pairs that are interpreted as protocol-specific options for the external CA driver. +### Client.inspect_node + +Retrieve low-level information about a Swarm node + +**Params:** + +* node_id (string): ID of the node to be inspected. + +**Returns:** A dictionary containing data about this node. See sample below. + +```python +{u'CreatedAt': u'2016-08-11T23:28:39.695834296Z', + u'Description': {u'Engine': {u'EngineVersion': u'1.12.0', + u'Plugins': [{u'Name': u'bridge', u'Type': u'Network'}, + {u'Name': u'host', u'Type': u'Network'}, + {u'Name': u'null', u'Type': u'Network'}, + {u'Name': u'overlay', u'Type': u'Network'}, + {u'Name': u'local', u'Type': u'Volume'}]}, + u'Hostname': u'dockerserv-1.local.net', + u'Platform': {u'Architecture': u'x86_64', u'OS': u'linux'}, + u'Resources': {u'MemoryBytes': 8052109312, u'NanoCPUs': 4000000000}}, + u'ID': u'1kqami616p23dz4hd7km35w63', + u'ManagerStatus': {u'Addr': u'10.0.131.127:2377', + u'Leader': True, + u'Reachability': u'reachable'}, + u'Spec': {u'Availability': u'active', u'Role': u'manager'}, + u'Status': {u'State': u'ready'}, + u'UpdatedAt': u'2016-08-11T23:28:39.979829529Z', + u'Version': {u'Index': 9}} + ``` + ### Client.inspect_swarm Retrieve information about the current Swarm. @@ -182,6 +221,17 @@ Leave a Swarm. **Returns:** `True` if the request went through. Raises an `APIError` if it fails. +### Client.nodes + +List Swarm nodes + +**Params:** + +* filters (dict): Filters to process on the nodes list. Valid filters: + `id`, `name`, `membership` and `role`. Default: `None` + +**Returns:** A list of dictionaries containing data about each swarm node. + ### Client.update_swarm Update the Swarm's configuration diff --git a/tests/integration/swarm_test.py b/tests/integration/swarm_test.py index b73f81c419..128628e618 100644 --- a/tests/integration/swarm_test.py +++ b/tests/integration/swarm_test.py @@ -114,3 +114,32 @@ def test_update_swarm_name(self): swarm_info_2['Version']['Index'] ) assert swarm_info_2['Spec']['Name'] == 'reimuhakurei' + + @requires_api_version('1.24') + def test_list_nodes(self): + assert self.client.init_swarm('eth0') + nodes_list = self.client.nodes() + assert len(nodes_list) == 1 + node = nodes_list[0] + assert 'ID' in node + assert 'Spec' in node + assert node['Spec']['Role'] == 'manager' + + filtered_list = self.client.nodes(filters={ + 'id': node['ID'] + }) + assert len(filtered_list) == 1 + filtered_list = self.client.nodes(filters={ + 'role': 'worker' + }) + assert len(filtered_list) == 0 + + @requires_api_version('1.24') + def test_inspect_node(self): + assert self.client.init_swarm('eth0') + nodes_list = self.client.nodes() + assert len(nodes_list) == 1 + node = nodes_list[0] + node_data = self.client.inspect_node(node['ID']) + assert node['ID'] == node_data['ID'] + assert node['Version'] == node_data['Version'] From 7d147c8ca18ad6c8dfd15f9f06f2892fc57372bb Mon Sep 17 00:00:00 2001 From: Josh Purvis Date: Mon, 15 Aug 2016 14:35:36 -0400 Subject: [PATCH 0075/1301] Move cpu_shares and cpuset_cpu to HostConfig when API >= 1.18 Signed-off-by: Josh Purvis --- docker/utils/utils.py | 26 +++++++++++++- docs/api.md | 1 - docs/hostconfig.md | 2 ++ tests/integration/container_test.py | 31 +++++++++++++++-- tests/unit/container_test.py | 54 +++++++++++++++++++++++++++++ 5 files changed, 110 insertions(+), 4 deletions(-) diff --git a/docker/utils/utils.py b/docker/utils/utils.py index 00a7af14fe..65f5dd9acd 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -620,7 +620,7 @@ def create_host_config(binds=None, port_bindings=None, lxc_conf=None, device_write_bps=None, device_read_iops=None, device_write_iops=None, oom_kill_disable=False, shm_size=None, sysctls=None, version=None, tmpfs=None, - oom_score_adj=None): + oom_score_adj=None, cpu_shares=None, cpuset_cpus=None): host_config = {} @@ -803,6 +803,21 @@ def create_host_config(binds=None, port_bindings=None, lxc_conf=None, host_config['CpuPeriod'] = cpu_period + if cpu_shares: + if version_lt(version, '1.18'): + raise host_config_version_error('cpu_shares', '1.18') + + if not isinstance(cpu_shares, int): + raise host_config_type_error('cpu_shares', cpu_shares, 'int') + + host_config['CpuShares'] = cpu_shares + + if cpuset_cpus: + if version_lt(version, '1.18'): + raise host_config_version_error('cpuset_cpus', '1.18') + + host_config['CpuSetCpus'] = cpuset_cpus + if blkio_weight: if not isinstance(blkio_weight, int): raise host_config_type_error('blkio_weight', blkio_weight, 'int') @@ -975,6 +990,14 @@ def create_container_config( 'labels were only introduced in API version 1.18' ) + if cpuset is not None or cpu_shares is not None: + if version_gte(version, '1.18'): + warnings.warn( + 'The cpuset_cpus and cpu_shares options have been moved to ' + 'host_config in API version 1.18, and will be removed', + DeprecationWarning + ) + if stop_signal is not None and compare_version('1.21', version) < 0: raise errors.InvalidVersion( 'stop_signal was only introduced in API version 1.21' @@ -1004,6 +1027,7 @@ def create_container_config( if mem_limit is not None: mem_limit = parse_bytes(mem_limit) + if memswap_limit is not None: memswap_limit = parse_bytes(memswap_limit) diff --git a/docs/api.md b/docs/api.md index 9b3a7265e7..960a673b60 100644 --- a/docs/api.md +++ b/docs/api.md @@ -245,7 +245,6 @@ from. Optionally a single string joining container id's with commas * network_disabled (bool): Disable networking * name (str): A name for the container * entrypoint (str or list): An entrypoint -* cpu_shares (int): CPU shares (relative weight) * working_dir (str): Path to the working directory * domainname (str or list): Set custom DNS search domains * memswap_limit (int): diff --git a/docs/hostconfig.md b/docs/hostconfig.md index 01c4625f62..229a28c0b4 100644 --- a/docs/hostconfig.md +++ b/docs/hostconfig.md @@ -109,6 +109,8 @@ for example: * cpu_group (int): The length of a CPU period in microseconds. * cpu_period (int): Microseconds of CPU time that the container can get in a CPU period. +* cpu_shares (int): CPU shares (relative weight) +* cpuset_cpus (str): CPUs in which to allow execution (0-3, 0,1) * blkio_weight: Block IO weight (relative weight), accepts a weight value between 10 and 1000. * blkio_weight_device: Block IO weight (relative device weight) in the form of: `[{"Path": "device_path", "Weight": weight}]` diff --git a/tests/integration/container_test.py b/tests/integration/container_test.py index f347c12a98..2d5b636760 100644 --- a/tests/integration/container_test.py +++ b/tests/integration/container_test.py @@ -1101,11 +1101,38 @@ def test_update_container(self): container = self.client.create_container( BUSYBOX, 'top', host_config=self.client.create_host_config( mem_limit=old_mem_limit - ), cpu_shares=102 + ) ) self.tmp_containers.append(container) self.client.start(container) self.client.update_container(container, mem_limit=new_mem_limit) inspect_data = self.client.inspect_container(container) self.assertEqual(inspect_data['HostConfig']['Memory'], new_mem_limit) - self.assertEqual(inspect_data['HostConfig']['CpuShares'], 102) + + +class ContainerCPUTest(helpers.BaseTestCase): + @requires_api_version('1.18') + def test_container_cpu_shares(self): + cpu_shares = 512 + container = self.client.create_container( + BUSYBOX, 'ls', host_config=self.client.create_host_config( + cpu_shares=cpu_shares + ) + ) + self.tmp_containers.append(container) + self.client.start(container) + inspect_data = self.client.inspect_container(container) + self.assertEqual(inspect_data['HostConfig']['CpuShares'], 512) + + @requires_api_version('1.18') + def test_container_cpuset(self): + cpuset_cpus = "0,1" + container = self.client.create_container( + BUSYBOX, 'ls', host_config=self.client.create_host_config( + cpuset_cpus=cpuset_cpus + ) + ) + self.tmp_containers.append(container) + self.client.start(container) + inspect_data = self.client.inspect_container(container) + self.assertEqual(inspect_data['HostConfig']['CpusetCpus'], cpuset_cpus) diff --git a/tests/unit/container_test.py b/tests/unit/container_test.py index 4c94c84428..c480462f2f 100644 --- a/tests/unit/container_test.py +++ b/tests/unit/container_test.py @@ -286,6 +286,33 @@ def test_create_container_with_cpu_shares(self): self.assertEqual(args[1]['headers'], {'Content-Type': 'application/json'}) + @requires_api_version('1.18') + def test_create_container_with_host_config_cpu_shares(self): + self.client.create_container( + 'busybox', 'ls', host_config=self.client.create_host_config( + cpu_shares=512 + ) + ) + + args = fake_request.call_args + self.assertEqual(args[0][1], + url_prefix + 'containers/create') + + self.assertEqual(json.loads(args[1]['data']), + json.loads(''' + {"Tty": false, "Image": "busybox", + "Cmd": ["ls"], "AttachStdin": false, + "AttachStderr": true, + "AttachStdout": true, "OpenStdin": false, + "StdinOnce": false, + "NetworkDisabled": false, + "HostConfig": { + "CpuShares": 512, + "NetworkMode": "default" + }}''')) + self.assertEqual(args[1]['headers'], + {'Content-Type': 'application/json'}) + def test_create_container_with_cpuset(self): self.client.create_container('busybox', 'ls', cpuset='0,1') @@ -306,6 +333,33 @@ def test_create_container_with_cpuset(self): self.assertEqual(args[1]['headers'], {'Content-Type': 'application/json'}) + @requires_api_version('1.18') + def test_create_container_with_host_config_cpuset(self): + self.client.create_container( + 'busybox', 'ls', host_config=self.client.create_host_config( + cpuset_cpus='0,1' + ) + ) + + args = fake_request.call_args + self.assertEqual(args[0][1], + url_prefix + 'containers/create') + + self.assertEqual(json.loads(args[1]['data']), + json.loads(''' + {"Tty": false, "Image": "busybox", + "Cmd": ["ls"], "AttachStdin": false, + "AttachStderr": true, + "AttachStdout": true, "OpenStdin": false, + "StdinOnce": false, + "NetworkDisabled": false, + "HostConfig": { + "CpuSetCpus": "0,1", + "NetworkMode": "default" + }}''')) + self.assertEqual(args[1]['headers'], + {'Content-Type': 'application/json'}) + def test_create_container_with_cgroup_parent(self): self.client.create_container( 'busybox', 'ls', host_config=self.client.create_host_config( From 0416338bae879574f74537a6ca2eab7a1f87f8a9 Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Tue, 16 Aug 2016 13:36:42 -0400 Subject: [PATCH 0076/1301] Remove references to "ExecutionDriver" Docker no longer has an `ExecutionDriver` as of Docker 1.11. The field in the `docker info` API will not be present in 1.13. Found this while working on docker/docker#25721 Signed-off-by: Brian Goff --- tests/helpers.py | 9 --------- tests/integration/container_test.py | 9 ++------- tests/integration/exec_test.py | 23 ----------------------- 3 files changed, 2 insertions(+), 39 deletions(-) diff --git a/tests/helpers.py b/tests/helpers.py index 94ea3887a8..40baef9cdc 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -45,15 +45,6 @@ def untar_file(tardata, filename): return result -def exec_driver_is_native(): - global EXEC_DRIVER - if not EXEC_DRIVER: - c = docker_client() - EXEC_DRIVER = c.info()['ExecutionDriver'] - c.close() - return EXEC_DRIVER.startswith('native') or EXEC_DRIVER == '' - - def docker_client(**kwargs): return docker.Client(**docker_client_kwargs(**kwargs)) diff --git a/tests/integration/container_test.py b/tests/integration/container_test.py index f347c12a98..334c81db2c 100644 --- a/tests/integration/container_test.py +++ b/tests/integration/container_test.py @@ -159,9 +159,6 @@ def test_create_container_with_volumes_from(self): self.assertCountEqual(info['HostConfig']['VolumesFrom'], vol_names) def create_container_readonly_fs(self): - if not helpers.exec_driver_is_native(): - pytest.skip('Exec driver not native') - ctnr = self.client.create_container( BUSYBOX, ['mkdir', '/shrine'], host_config=self.client.create_host_config( @@ -806,8 +803,7 @@ def test_kill(self): self.assertIn('State', container_info) state = container_info['State'] self.assertIn('ExitCode', state) - if helpers.exec_driver_is_native(): - self.assertNotEqual(state['ExitCode'], 0) + self.assertNotEqual(state['ExitCode'], 0) self.assertIn('Running', state) self.assertEqual(state['Running'], False) @@ -821,8 +817,7 @@ def test_kill_with_dict_instead_of_id(self): self.assertIn('State', container_info) state = container_info['State'] self.assertIn('ExitCode', state) - if helpers.exec_driver_is_native(): - self.assertNotEqual(state['ExitCode'], 0) + self.assertNotEqual(state['ExitCode'], 0) self.assertIn('Running', state) self.assertEqual(state['Running'], False) diff --git a/tests/integration/exec_test.py b/tests/integration/exec_test.py index 8bf2762a91..f377e09228 100644 --- a/tests/integration/exec_test.py +++ b/tests/integration/exec_test.py @@ -1,5 +1,3 @@ -import pytest - from docker.utils.socket import next_frame_size from docker.utils.socket import read_exactly @@ -10,9 +8,6 @@ class ExecTest(helpers.BaseTestCase): def test_execute_command(self): - if not helpers.exec_driver_is_native(): - pytest.skip('Exec driver not native') - container = self.client.create_container(BUSYBOX, 'cat', detach=True, stdin_open=True) id = container['Id'] @@ -26,9 +21,6 @@ def test_execute_command(self): self.assertEqual(exec_log, b'hello\n') def test_exec_command_string(self): - if not helpers.exec_driver_is_native(): - pytest.skip('Exec driver not native') - container = self.client.create_container(BUSYBOX, 'cat', detach=True, stdin_open=True) id = container['Id'] @@ -42,9 +34,6 @@ def test_exec_command_string(self): self.assertEqual(exec_log, b'hello world\n') def test_exec_command_as_user(self): - if not helpers.exec_driver_is_native(): - pytest.skip('Exec driver not native') - container = self.client.create_container(BUSYBOX, 'cat', detach=True, stdin_open=True) id = container['Id'] @@ -58,9 +47,6 @@ def test_exec_command_as_user(self): self.assertEqual(exec_log, b'default\n') def test_exec_command_as_root(self): - if not helpers.exec_driver_is_native(): - pytest.skip('Exec driver not native') - container = self.client.create_container(BUSYBOX, 'cat', detach=True, stdin_open=True) id = container['Id'] @@ -74,9 +60,6 @@ def test_exec_command_as_root(self): self.assertEqual(exec_log, b'root\n') def test_exec_command_streaming(self): - if not helpers.exec_driver_is_native(): - pytest.skip('Exec driver not native') - container = self.client.create_container(BUSYBOX, 'cat', detach=True, stdin_open=True) id = container['Id'] @@ -92,9 +75,6 @@ def test_exec_command_streaming(self): self.assertEqual(res, b'hello\nworld\n') def test_exec_start_socket(self): - if not helpers.exec_driver_is_native(): - pytest.skip('Exec driver not native') - container = self.client.create_container(BUSYBOX, 'cat', detach=True, stdin_open=True) container_id = container['Id'] @@ -116,9 +96,6 @@ def test_exec_start_socket(self): self.assertEqual(data.decode('utf-8'), line) def test_exec_inspect(self): - if not helpers.exec_driver_is_native(): - pytest.skip('Exec driver not native') - container = self.client.create_container(BUSYBOX, 'cat', detach=True, stdin_open=True) id = container['Id'] From 95d9306d2a1fd22dffb12a0548abf2d2f744ed9d Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 9 Aug 2016 13:20:17 -0700 Subject: [PATCH 0077/1301] Exclude requests 2.11 from setup.py to work around unicode bug Signed-off-by: Joffrey F --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ac58b1f94c..85a4499422 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ SOURCE_DIR = os.path.join(ROOT_DIR) requirements = [ - 'requests >= 2.5.2', + 'requests >= 2.5.2, < 2.11', 'six >= 1.4.0', 'websocket-client >= 0.32.0', ] From 3062ae4348ab916a9afd574cb70891b9131aff11 Mon Sep 17 00:00:00 2001 From: Tomas Tomecek Date: Wed, 27 Jul 2016 10:26:16 +0200 Subject: [PATCH 0078/1301] docker client consistency: don't quote ':/' E.g. docker client `/v1.21/images/localhost:5000/busybox/push?tag=` docker-py `/v1.21/images/localhost%3A5000%2Fbusybox/push` Signed-off-by: Tomas Tomecek --- docker/client.py | 4 +++- tests/unit/api_test.py | 10 ++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/docker/client.py b/docker/client.py index 1b5420e9cf..9f75ce73c7 100644 --- a/docker/client.py +++ b/docker/client.py @@ -14,6 +14,7 @@ import json import struct +from functools import partial import requests import requests.exceptions @@ -157,7 +158,8 @@ def _url(self, pathfmt, *args, **kwargs): 'instead'.format(arg, type(arg)) ) - args = map(six.moves.urllib.parse.quote_plus, args) + quote_f = partial(six.moves.urllib.parse.quote_plus, safe="/:") + args = map(quote_f, args) if kwargs.get('versioned_api', True): return '{0}/v{1}{2}'.format( diff --git a/tests/unit/api_test.py b/tests/unit/api_test.py index 696c073914..712f57e0be 100644 --- a/tests/unit/api_test.py +++ b/tests/unit/api_test.py @@ -159,9 +159,15 @@ def test_url_valid_resource(self): '{0}{1}'.format(url_prefix, 'hello/somename/world/someothername') ) - url = self.client._url('/hello/{0}/world', '/some?name') + url = self.client._url('/hello/{0}/world', 'some?name') self.assertEqual( - url, '{0}{1}'.format(url_prefix, 'hello/%2Fsome%3Fname/world') + url, '{0}{1}'.format(url_prefix, 'hello/some%3Fname/world') + ) + + url = self.client._url("/images/{0}/push", "localhost:5000/image") + self.assertEqual( + url, + '{0}{1}'.format(url_prefix, 'images/localhost:5000/image/push') ) def test_url_invalid_resource(self): From 0f47db7fcc4ae2b22500afdd6b029c557d86f5b1 Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Tue, 16 Aug 2016 13:36:42 -0400 Subject: [PATCH 0079/1301] Remove references to "ExecutionDriver" Docker no longer has an `ExecutionDriver` as of Docker 1.11. The field in the `docker info` API will not be present in 1.13. Found this while working on docker/docker#25721 Signed-off-by: Brian Goff --- tests/helpers.py | 9 --------- tests/integration/container_test.py | 9 ++------- tests/integration/exec_test.py | 23 ----------------------- 3 files changed, 2 insertions(+), 39 deletions(-) diff --git a/tests/helpers.py b/tests/helpers.py index 94ea3887a8..40baef9cdc 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -45,15 +45,6 @@ def untar_file(tardata, filename): return result -def exec_driver_is_native(): - global EXEC_DRIVER - if not EXEC_DRIVER: - c = docker_client() - EXEC_DRIVER = c.info()['ExecutionDriver'] - c.close() - return EXEC_DRIVER.startswith('native') or EXEC_DRIVER == '' - - def docker_client(**kwargs): return docker.Client(**docker_client_kwargs(**kwargs)) diff --git a/tests/integration/container_test.py b/tests/integration/container_test.py index f347c12a98..334c81db2c 100644 --- a/tests/integration/container_test.py +++ b/tests/integration/container_test.py @@ -159,9 +159,6 @@ def test_create_container_with_volumes_from(self): self.assertCountEqual(info['HostConfig']['VolumesFrom'], vol_names) def create_container_readonly_fs(self): - if not helpers.exec_driver_is_native(): - pytest.skip('Exec driver not native') - ctnr = self.client.create_container( BUSYBOX, ['mkdir', '/shrine'], host_config=self.client.create_host_config( @@ -806,8 +803,7 @@ def test_kill(self): self.assertIn('State', container_info) state = container_info['State'] self.assertIn('ExitCode', state) - if helpers.exec_driver_is_native(): - self.assertNotEqual(state['ExitCode'], 0) + self.assertNotEqual(state['ExitCode'], 0) self.assertIn('Running', state) self.assertEqual(state['Running'], False) @@ -821,8 +817,7 @@ def test_kill_with_dict_instead_of_id(self): self.assertIn('State', container_info) state = container_info['State'] self.assertIn('ExitCode', state) - if helpers.exec_driver_is_native(): - self.assertNotEqual(state['ExitCode'], 0) + self.assertNotEqual(state['ExitCode'], 0) self.assertIn('Running', state) self.assertEqual(state['Running'], False) diff --git a/tests/integration/exec_test.py b/tests/integration/exec_test.py index 8bf2762a91..f377e09228 100644 --- a/tests/integration/exec_test.py +++ b/tests/integration/exec_test.py @@ -1,5 +1,3 @@ -import pytest - from docker.utils.socket import next_frame_size from docker.utils.socket import read_exactly @@ -10,9 +8,6 @@ class ExecTest(helpers.BaseTestCase): def test_execute_command(self): - if not helpers.exec_driver_is_native(): - pytest.skip('Exec driver not native') - container = self.client.create_container(BUSYBOX, 'cat', detach=True, stdin_open=True) id = container['Id'] @@ -26,9 +21,6 @@ def test_execute_command(self): self.assertEqual(exec_log, b'hello\n') def test_exec_command_string(self): - if not helpers.exec_driver_is_native(): - pytest.skip('Exec driver not native') - container = self.client.create_container(BUSYBOX, 'cat', detach=True, stdin_open=True) id = container['Id'] @@ -42,9 +34,6 @@ def test_exec_command_string(self): self.assertEqual(exec_log, b'hello world\n') def test_exec_command_as_user(self): - if not helpers.exec_driver_is_native(): - pytest.skip('Exec driver not native') - container = self.client.create_container(BUSYBOX, 'cat', detach=True, stdin_open=True) id = container['Id'] @@ -58,9 +47,6 @@ def test_exec_command_as_user(self): self.assertEqual(exec_log, b'default\n') def test_exec_command_as_root(self): - if not helpers.exec_driver_is_native(): - pytest.skip('Exec driver not native') - container = self.client.create_container(BUSYBOX, 'cat', detach=True, stdin_open=True) id = container['Id'] @@ -74,9 +60,6 @@ def test_exec_command_as_root(self): self.assertEqual(exec_log, b'root\n') def test_exec_command_streaming(self): - if not helpers.exec_driver_is_native(): - pytest.skip('Exec driver not native') - container = self.client.create_container(BUSYBOX, 'cat', detach=True, stdin_open=True) id = container['Id'] @@ -92,9 +75,6 @@ def test_exec_command_streaming(self): self.assertEqual(res, b'hello\nworld\n') def test_exec_start_socket(self): - if not helpers.exec_driver_is_native(): - pytest.skip('Exec driver not native') - container = self.client.create_container(BUSYBOX, 'cat', detach=True, stdin_open=True) container_id = container['Id'] @@ -116,9 +96,6 @@ def test_exec_start_socket(self): self.assertEqual(data.decode('utf-8'), line) def test_exec_inspect(self): - if not helpers.exec_driver_is_native(): - pytest.skip('Exec driver not native') - container = self.client.create_container(BUSYBOX, 'cat', detach=True, stdin_open=True) id = container['Id'] From 172e95d52fe4f9bc6146828b34fbed567aec3945 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 24 Jun 2016 15:04:13 -0700 Subject: [PATCH 0080/1301] Swarm service API implementation Signed-off-by: Joffrey F --- docker/api/__init__.py | 6 +- docker/api/service.py | 232 +++++++++++++++++++++++++++++++++++++ docker/client.py | 5 +- docker/utils/decorators.py | 2 +- 4 files changed, 241 insertions(+), 4 deletions(-) create mode 100644 docker/api/service.py diff --git a/docker/api/__init__.py b/docker/api/__init__.py index b0d60878f7..3c74677dc0 100644 --- a/docker/api/__init__.py +++ b/docker/api/__init__.py @@ -4,6 +4,10 @@ from .daemon import DaemonApiMixin from .exec_api import ExecApiMixin from .image import ImageApiMixin -from .volume import VolumeApiMixin from .network import NetworkApiMixin +from .service import ( + ServiceApiMixin, TaskTemplate, ContainerSpec, Mount, Resources, + RestartPolicy, UpdateConfig +) from .swarm import SwarmApiMixin +from .volume import VolumeApiMixin diff --git a/docker/api/service.py b/docker/api/service.py new file mode 100644 index 0000000000..4be9624245 --- /dev/null +++ b/docker/api/service.py @@ -0,0 +1,232 @@ +import six + +from .. import errors +from .. import utils + + +class ServiceApiMixin(object): + @utils.minimum_version('1.24') + def services(self, filters=None): + params = { + 'filters': utils.convert_filters(filters) if filters else None + } + url = self._url('/services') + return self._result(self._get(url, params=params), True) + + @utils.minimum_version('1.24') + def create_service( + self, task_config, name=None, labels=None, mode=None, + update_config=None, networks=None, endpoint_config=None + ): + url = self._url('/services/create') + data = { + 'Name': name, + 'Labels': labels, + 'TaskTemplate': task_config, + 'Mode': mode, + 'UpdateConfig': update_config, + 'Networks': networks, + 'Endpoint': endpoint_config + } + return self._result(self._post_json(url, data=data), True) + + @utils.minimum_version('1.24') + @utils.check_resource + def inspect_service(self, service): + url = self._url('/services/{0}', service) + return self._result(self._get(url), True) + + @utils.minimum_version('1.24') + @utils.check_resource + def remove_service(self, service): + url = self._url('/services/{0}', service) + resp = self._delete(url) + self._raise_for_status(resp) + + @utils.minimum_version('1.24') + @utils.check_resource + def update_service(self, service, task_template=None, name=None, + labels=None, mode=None, update_config=None, + networks=None, endpoint_config=None): + url = self._url('/services/{0}/update', service) + data = {} + if name is not None: + data['Name'] = name + if labels is not None: + data['Labels'] = labels + if mode is not None: + data['Mode'] = mode + if task_template is not None: + data['TaskTemplate'] = task_template + if update_config is not None: + data['UpdateConfig'] = update_config + if networks is not None: + data['Networks'] = networks + if endpoint_config is not None: + data['Endpoint'] = endpoint_config + + return self._result(self._post_json(url, data=data), True) + + +class TaskTemplate(dict): + def __init__(self, container_spec, resources=None, restart_policy=None, + placement=None, log_driver=None): + self['ContainerSpec'] = container_spec + if resources: + self['Resources'] = resources + if restart_policy: + self['RestartPolicy'] = restart_policy + if placement: + self['Placement'] = placement + if log_driver: + self['LogDriver'] = log_driver + + @property + def container_spec(self): + return self.get('ContainerSpec') + + @property + def resources(self): + return self.get('Resources') + + @property + def restart_policy(self): + return self.get('RestartPolicy') + + @property + def placement(self): + return self.get('Placement') + + +class ContainerSpec(dict): + def __init__(self, image, command=None, args=None, env=None, workdir=None, + user=None, labels=None, mounts=None, stop_grace_period=None): + self['Image'] = image + self['Command'] = command + self['Args'] = args + + if env is not None: + self['Env'] = env + if workdir is not None: + self['Dir'] = workdir + if user is not None: + self['User'] = user + if labels is not None: + self['Labels'] = labels + if mounts is not None: + for mount in mounts: + if isinstance(mount, six.string_types): + mounts.append(Mount.parse_mount_string(mount)) + mounts.remove(mount) + self['Mounts'] = mounts + if stop_grace_period is not None: + self['StopGracePeriod'] = stop_grace_period + + +class Mount(dict): + def __init__(self, target, source, type='volume', read_only=False, + propagation=None, no_copy=False, labels=None, + driver_config=None): + self['Target'] = target + self['Source'] = source + if type not in ('bind', 'volume'): + raise errors.DockerError( + 'Only acceptable mount types are `bind` and `volume`.' + ) + self['Type'] = type + + if type == 'bind': + if propagation is not None: + self['BindOptions'] = { + 'Propagation': propagation + } + if any(labels, driver_config, no_copy): + raise errors.DockerError( + 'Mount type is binding but volume options have been ' + 'provided.' + ) + else: + volume_opts = {} + if no_copy: + volume_opts['NoCopy'] = True + if labels: + volume_opts['Labels'] = labels + if driver_config: + volume_opts['driver_config'] = driver_config + if volume_opts: + self['VolumeOptions'] = volume_opts + if propagation: + raise errors.DockerError( + 'Mount type is volume but `propagation` argument has been ' + 'provided.' + ) + + @classmethod + def parse_mount_string(cls, string): + parts = string.split(':') + if len(parts) > 3: + raise errors.DockerError( + 'Invalid mount format "{0}"'.format(string) + ) + if len(parts) == 1: + return cls(target=parts[0]) + else: + target = parts[1] + source = parts[0] + read_only = not (len(parts) == 3 or parts[2] == 'ro') + return cls(target, source, read_only=read_only) + + +class Resources(dict): + def __init__(self, cpu_limit=None, mem_limit=None, cpu_reservation=None, + mem_reservation=None): + limits = {} + reservation = {} + if cpu_limit is not None: + limits['NanoCPUs'] = cpu_limit + if mem_limit is not None: + limits['MemoryBytes'] = mem_limit + if cpu_reservation is not None: + reservation['NanoCPUs'] = cpu_reservation + if mem_reservation is not None: + reservation['MemoryBytes'] = mem_reservation + + self['Limits'] = limits + self['Reservation'] = reservation + + +class UpdateConfig(dict): + def __init__(self, parallelism=0, delay=None, failure_action='continue'): + self['Parallelism'] = parallelism + if delay is not None: + self['Delay'] = delay + if failure_action not in ('pause', 'continue'): + raise errors.DockerError( + 'failure_action must be either `pause` or `continue`.' + ) + self['FailureAction'] = failure_action + + +class RestartConditionTypesEnum(object): + _values = ( + 'none', + 'on_failure', + 'any', + ) + NONE, ON_FAILURE, ANY = _values + + +class RestartPolicy(dict): + condition_types = RestartConditionTypesEnum + + def __init__(self, condition=RestartConditionTypesEnum.NONE, delay=0, + attempts=0, window=0): + if condition not in self.condition_types._values: + raise TypeError( + 'Invalid RestartPolicy condition {0}'.format(condition) + ) + + self['Condition'] = condition + self['Delay'] = delay + self['Attempts'] = attempts + self['Window'] = window diff --git a/docker/client.py b/docker/client.py index 9f75ce73c7..6c1e189044 100644 --- a/docker/client.py +++ b/docker/client.py @@ -48,9 +48,10 @@ class Client( api.DaemonApiMixin, api.ExecApiMixin, api.ImageApiMixin, - api.VolumeApiMixin, api.NetworkApiMixin, - api.SwarmApiMixin): + api.ServiceApiMixin, + api.SwarmApiMixin, + api.VolumeApiMixin): def __init__(self, base_url=None, version=None, timeout=constants.DEFAULT_TIMEOUT_SECONDS, tls=False, user_agent=constants.DEFAULT_USER_AGENT): diff --git a/docker/utils/decorators.py b/docker/utils/decorators.py index 46c28a8092..2fe880c4a5 100644 --- a/docker/utils/decorators.py +++ b/docker/utils/decorators.py @@ -13,7 +13,7 @@ def wrapped(self, resource_id=None, *args, **kwargs): elif kwargs.get('image'): resource_id = kwargs.pop('image') if isinstance(resource_id, dict): - resource_id = resource_id.get('Id') + resource_id = resource_id.get('Id', resource_id.get('ID')) if not resource_id: raise errors.NullResource( 'image or container param is undefined' From 02e99e4967bebb116a7d9d650647df912c608297 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 17 Aug 2016 18:42:52 -0700 Subject: [PATCH 0081/1301] Service API integration tests Signed-off-by: Joffrey F --- docker/api/service.py | 7 ++- tests/integration/service_test.py | 100 ++++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+), 2 deletions(-) create mode 100644 tests/integration/service_test.py diff --git a/docker/api/service.py b/docker/api/service.py index 4be9624245..db19ae5376 100644 --- a/docker/api/service.py +++ b/docker/api/service.py @@ -42,10 +42,11 @@ def remove_service(self, service): url = self._url('/services/{0}', service) resp = self._delete(url) self._raise_for_status(resp) + return True @utils.minimum_version('1.24') @utils.check_resource - def update_service(self, service, task_template=None, name=None, + def update_service(self, service, version, task_template=None, name=None, labels=None, mode=None, update_config=None, networks=None, endpoint_config=None): url = self._url('/services/{0}/update', service) @@ -65,7 +66,9 @@ def update_service(self, service, task_template=None, name=None, if endpoint_config is not None: data['Endpoint'] = endpoint_config - return self._result(self._post_json(url, data=data), True) + resp = self._post_json(url, data=data, params={'version': version}) + self._raise_for_status(resp) + return True class TaskTemplate(dict): diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py new file mode 100644 index 0000000000..00109868ba --- /dev/null +++ b/tests/integration/service_test.py @@ -0,0 +1,100 @@ +import random + +import docker +# import pytest + +from ..base import requires_api_version +from .. import helpers + + +BUSYBOX = helpers.BUSYBOX + + +class ServiceTest(helpers.BaseTestCase): + def setUp(self): + super(ServiceTest, self).setUp() + try: + self.client.leave_swarm(force=True) + except docker.errors.APIError: + pass + self.client.init_swarm('eth0') + + def tearDown(self): + super(ServiceTest, self).tearDown() + for service in self.client.services(filters={'name': 'dockerpytest_'}): + try: + self.client.remove_service(service['ID']) + except docker.errors.APIError: + pass + try: + self.client.leave_swarm(force=True) + except docker.errors.APIError: + pass + + def get_service_name(self): + return 'dockerpytest_{0:x}'.format(random.getrandbits(64)) + + def create_simple_service(self, name=None): + if name: + name = 'dockerpytest_{0}'.format(name) + else: + name = self.get_service_name() + + container_spec = docker.api.ContainerSpec('busybox', ['echo', 'hello']) + task_tmpl = docker.api.TaskTemplate(container_spec) + return name, self.client.create_service(task_tmpl, name=name) + + @requires_api_version('1.24') + def test_list_services(self): + services = self.client.services() + assert isinstance(services, list) + + test_services = self.client.services(filters={'name': 'dockerpytest_'}) + assert len(test_services) == 0 + self.create_simple_service() + test_services = self.client.services(filters={'name': 'dockerpytest_'}) + assert len(test_services) == 1 + assert 'dockerpytest_' in test_services[0]['Spec']['Name'] + + def test_inspect_service_by_id(self): + svc_name, svc_id = self.create_simple_service() + svc_info = self.client.inspect_service(svc_id) + assert 'ID' in svc_info + assert svc_info['ID'] == svc_id['ID'] + + def test_inspect_service_by_name(self): + svc_name, svc_id = self.create_simple_service() + svc_info = self.client.inspect_service(svc_name) + assert 'ID' in svc_info + assert svc_info['ID'] == svc_id['ID'] + + def test_remove_service_by_id(self): + svc_name, svc_id = self.create_simple_service() + assert self.client.remove_service(svc_id) + test_services = self.client.services(filters={'name': 'dockerpytest_'}) + assert len(test_services) == 0 + + def test_rempve_service_by_name(self): + svc_name, svc_id = self.create_simple_service() + assert self.client.remove_service(svc_name) + test_services = self.client.services(filters={'name': 'dockerpytest_'}) + assert len(test_services) == 0 + + def test_create_service_simple(self): + name, svc_id = self.create_simple_service() + assert self.client.inspect_service(svc_id) + services = self.client.services(filters={'name': name}) + assert len(services) == 1 + assert services[0]['ID'] == svc_id['ID'] + + def test_update_service_name(self): + name, svc_id = self.create_simple_service() + svc_info = self.client.inspect_service(svc_id) + svc_version = svc_info['Version']['Index'] + new_name = self.get_service_name() + assert self.client.update_service( + svc_id, svc_version, name=new_name, + task_template=svc_info['Spec']['TaskTemplate'] + ) + svc_info = self.client.inspect_service(svc_id) + assert svc_info['Spec']['Name'] == new_name From 97094e4ea303b59ce05869132cf639305a6d947a Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 19 Aug 2016 16:46:21 -0700 Subject: [PATCH 0082/1301] New docker.types subpackage containing advanced config dictionary types Tests and docs updated to match docker.utils.types has been moved to docker.types Signed-off-by: Joffrey F --- docker/api/__init__.py | 5 +- docker/api/service.py | 187 +----------------- docker/types/__init__.py | 7 + docker/types/base.py | 7 + .../{utils/types.py => types/containers.py} | 50 +---- docker/types/services.py | 176 +++++++++++++++++ docker/types/swarm.py | 40 ++++ docker/utils/__init__.py | 6 +- docker/utils/utils.py | 2 +- docs/swarm.md | 8 +- setup.py | 3 +- tests/integration/service_test.py | 96 ++++++++- 12 files changed, 345 insertions(+), 242 deletions(-) create mode 100644 docker/types/__init__.py create mode 100644 docker/types/base.py rename docker/{utils/types.py => types/containers.py} (55%) create mode 100644 docker/types/services.py create mode 100644 docker/types/swarm.py diff --git a/docker/api/__init__.py b/docker/api/__init__.py index 3c74677dc0..bc7e93ceee 100644 --- a/docker/api/__init__.py +++ b/docker/api/__init__.py @@ -5,9 +5,6 @@ from .exec_api import ExecApiMixin from .image import ImageApiMixin from .network import NetworkApiMixin -from .service import ( - ServiceApiMixin, TaskTemplate, ContainerSpec, Mount, Resources, - RestartPolicy, UpdateConfig -) +from .service import ServiceApiMixin from .swarm import SwarmApiMixin from .volume import VolumeApiMixin diff --git a/docker/api/service.py b/docker/api/service.py index db19ae5376..c62e4946b8 100644 --- a/docker/api/service.py +++ b/docker/api/service.py @@ -1,28 +1,17 @@ -import six - -from .. import errors from .. import utils class ServiceApiMixin(object): - @utils.minimum_version('1.24') - def services(self, filters=None): - params = { - 'filters': utils.convert_filters(filters) if filters else None - } - url = self._url('/services') - return self._result(self._get(url, params=params), True) - @utils.minimum_version('1.24') def create_service( - self, task_config, name=None, labels=None, mode=None, + self, task_template, name=None, labels=None, mode=None, update_config=None, networks=None, endpoint_config=None ): url = self._url('/services/create') data = { 'Name': name, 'Labels': labels, - 'TaskTemplate': task_config, + 'TaskTemplate': task_template, 'Mode': mode, 'UpdateConfig': update_config, 'Networks': networks, @@ -44,6 +33,14 @@ def remove_service(self, service): self._raise_for_status(resp) return True + @utils.minimum_version('1.24') + def services(self, filters=None): + params = { + 'filters': utils.convert_filters(filters) if filters else None + } + url = self._url('/services') + return self._result(self._get(url, params=params), True) + @utils.minimum_version('1.24') @utils.check_resource def update_service(self, service, version, task_template=None, name=None, @@ -69,167 +66,3 @@ def update_service(self, service, version, task_template=None, name=None, resp = self._post_json(url, data=data, params={'version': version}) self._raise_for_status(resp) return True - - -class TaskTemplate(dict): - def __init__(self, container_spec, resources=None, restart_policy=None, - placement=None, log_driver=None): - self['ContainerSpec'] = container_spec - if resources: - self['Resources'] = resources - if restart_policy: - self['RestartPolicy'] = restart_policy - if placement: - self['Placement'] = placement - if log_driver: - self['LogDriver'] = log_driver - - @property - def container_spec(self): - return self.get('ContainerSpec') - - @property - def resources(self): - return self.get('Resources') - - @property - def restart_policy(self): - return self.get('RestartPolicy') - - @property - def placement(self): - return self.get('Placement') - - -class ContainerSpec(dict): - def __init__(self, image, command=None, args=None, env=None, workdir=None, - user=None, labels=None, mounts=None, stop_grace_period=None): - self['Image'] = image - self['Command'] = command - self['Args'] = args - - if env is not None: - self['Env'] = env - if workdir is not None: - self['Dir'] = workdir - if user is not None: - self['User'] = user - if labels is not None: - self['Labels'] = labels - if mounts is not None: - for mount in mounts: - if isinstance(mount, six.string_types): - mounts.append(Mount.parse_mount_string(mount)) - mounts.remove(mount) - self['Mounts'] = mounts - if stop_grace_period is not None: - self['StopGracePeriod'] = stop_grace_period - - -class Mount(dict): - def __init__(self, target, source, type='volume', read_only=False, - propagation=None, no_copy=False, labels=None, - driver_config=None): - self['Target'] = target - self['Source'] = source - if type not in ('bind', 'volume'): - raise errors.DockerError( - 'Only acceptable mount types are `bind` and `volume`.' - ) - self['Type'] = type - - if type == 'bind': - if propagation is not None: - self['BindOptions'] = { - 'Propagation': propagation - } - if any(labels, driver_config, no_copy): - raise errors.DockerError( - 'Mount type is binding but volume options have been ' - 'provided.' - ) - else: - volume_opts = {} - if no_copy: - volume_opts['NoCopy'] = True - if labels: - volume_opts['Labels'] = labels - if driver_config: - volume_opts['driver_config'] = driver_config - if volume_opts: - self['VolumeOptions'] = volume_opts - if propagation: - raise errors.DockerError( - 'Mount type is volume but `propagation` argument has been ' - 'provided.' - ) - - @classmethod - def parse_mount_string(cls, string): - parts = string.split(':') - if len(parts) > 3: - raise errors.DockerError( - 'Invalid mount format "{0}"'.format(string) - ) - if len(parts) == 1: - return cls(target=parts[0]) - else: - target = parts[1] - source = parts[0] - read_only = not (len(parts) == 3 or parts[2] == 'ro') - return cls(target, source, read_only=read_only) - - -class Resources(dict): - def __init__(self, cpu_limit=None, mem_limit=None, cpu_reservation=None, - mem_reservation=None): - limits = {} - reservation = {} - if cpu_limit is not None: - limits['NanoCPUs'] = cpu_limit - if mem_limit is not None: - limits['MemoryBytes'] = mem_limit - if cpu_reservation is not None: - reservation['NanoCPUs'] = cpu_reservation - if mem_reservation is not None: - reservation['MemoryBytes'] = mem_reservation - - self['Limits'] = limits - self['Reservation'] = reservation - - -class UpdateConfig(dict): - def __init__(self, parallelism=0, delay=None, failure_action='continue'): - self['Parallelism'] = parallelism - if delay is not None: - self['Delay'] = delay - if failure_action not in ('pause', 'continue'): - raise errors.DockerError( - 'failure_action must be either `pause` or `continue`.' - ) - self['FailureAction'] = failure_action - - -class RestartConditionTypesEnum(object): - _values = ( - 'none', - 'on_failure', - 'any', - ) - NONE, ON_FAILURE, ANY = _values - - -class RestartPolicy(dict): - condition_types = RestartConditionTypesEnum - - def __init__(self, condition=RestartConditionTypesEnum.NONE, delay=0, - attempts=0, window=0): - if condition not in self.condition_types._values: - raise TypeError( - 'Invalid RestartPolicy condition {0}'.format(condition) - ) - - self['Condition'] = condition - self['Delay'] = delay - self['Attempts'] = attempts - self['Window'] = window diff --git a/docker/types/__init__.py b/docker/types/__init__.py new file mode 100644 index 0000000000..46f10d86a4 --- /dev/null +++ b/docker/types/__init__.py @@ -0,0 +1,7 @@ +# flake8: noqa +from .containers import LogConfig, Ulimit +from .services import ( + ContainerSpec, LogDriver, Mount, Resources, RestartPolicy, TaskTemplate, + UpdateConfig +) +from .swarm import SwarmSpec, SwarmExternalCA diff --git a/docker/types/base.py b/docker/types/base.py new file mode 100644 index 0000000000..6891062313 --- /dev/null +++ b/docker/types/base.py @@ -0,0 +1,7 @@ +import six + + +class DictType(dict): + def __init__(self, init): + for k, v in six.iteritems(init): + self[k] = v diff --git a/docker/utils/types.py b/docker/types/containers.py similarity index 55% rename from docker/utils/types.py rename to docker/types/containers.py index d778b90dc8..40a44caf09 100644 --- a/docker/utils/types.py +++ b/docker/types/containers.py @@ -1,5 +1,7 @@ import six +from .base import DictType + class LogConfigTypesEnum(object): _values = ( @@ -13,12 +15,6 @@ class LogConfigTypesEnum(object): JSON, SYSLOG, JOURNALD, GELF, FLUENTD, NONE = _values -class DictType(dict): - def __init__(self, init): - for k, v in six.iteritems(init): - self[k] = v - - class LogConfig(DictType): types = LogConfigTypesEnum @@ -94,45 +90,3 @@ def hard(self): @hard.setter def hard(self, value): self['Hard'] = value - - -class SwarmSpec(DictType): - def __init__(self, task_history_retention_limit=None, - snapshot_interval=None, keep_old_snapshots=None, - log_entries_for_slow_followers=None, heartbeat_tick=None, - election_tick=None, dispatcher_heartbeat_period=None, - node_cert_expiry=None, external_ca=None, name=None): - if task_history_retention_limit is not None: - self['Orchestration'] = { - 'TaskHistoryRetentionLimit': task_history_retention_limit - } - if any([snapshot_interval, keep_old_snapshots, - log_entries_for_slow_followers, heartbeat_tick, election_tick]): - self['Raft'] = { - 'SnapshotInterval': snapshot_interval, - 'KeepOldSnapshots': keep_old_snapshots, - 'LogEntriesForSlowFollowers': log_entries_for_slow_followers, - 'HeartbeatTick': heartbeat_tick, - 'ElectionTick': election_tick - } - - if dispatcher_heartbeat_period: - self['Dispatcher'] = { - 'HeartbeatPeriod': dispatcher_heartbeat_period - } - - if node_cert_expiry or external_ca: - self['CAConfig'] = { - 'NodeCertExpiry': node_cert_expiry, - 'ExternalCA': external_ca - } - - if name is not None: - self['Name'] = name - - -class SwarmExternalCA(DictType): - def __init__(self, url, protocol=None, options=None): - self['URL'] = url - self['Protocol'] = protocol - self['Options'] = options diff --git a/docker/types/services.py b/docker/types/services.py new file mode 100644 index 0000000000..6a17e93f13 --- /dev/null +++ b/docker/types/services.py @@ -0,0 +1,176 @@ +import six + +from .. import errors + + +class TaskTemplate(dict): + def __init__(self, container_spec, resources=None, restart_policy=None, + placement=None, log_driver=None): + self['ContainerSpec'] = container_spec + if resources: + self['Resources'] = resources + if restart_policy: + self['RestartPolicy'] = restart_policy + if placement: + self['Placement'] = placement + if log_driver: + self['LogDriver'] = log_driver + + @property + def container_spec(self): + return self.get('ContainerSpec') + + @property + def resources(self): + return self.get('Resources') + + @property + def restart_policy(self): + return self.get('RestartPolicy') + + @property + def placement(self): + return self.get('Placement') + + +class ContainerSpec(dict): + def __init__(self, image, command=None, args=None, env=None, workdir=None, + user=None, labels=None, mounts=None, stop_grace_period=None): + self['Image'] = image + self['Command'] = command + self['Args'] = args + + if env is not None: + self['Env'] = env + if workdir is not None: + self['Dir'] = workdir + if user is not None: + self['User'] = user + if labels is not None: + self['Labels'] = labels + if mounts is not None: + for mount in mounts: + if isinstance(mount, six.string_types): + mounts.append(Mount.parse_mount_string(mount)) + mounts.remove(mount) + self['Mounts'] = mounts + if stop_grace_period is not None: + self['StopGracePeriod'] = stop_grace_period + + +class Mount(dict): + def __init__(self, target, source, type='volume', read_only=False, + propagation=None, no_copy=False, labels=None, + driver_config=None): + self['Target'] = target + self['Source'] = source + if type not in ('bind', 'volume'): + raise errors.DockerError( + 'Only acceptable mount types are `bind` and `volume`.' + ) + self['Type'] = type + + if type == 'bind': + if propagation is not None: + self['BindOptions'] = { + 'Propagation': propagation + } + if any(labels, driver_config, no_copy): + raise errors.DockerError( + 'Mount type is binding but volume options have been ' + 'provided.' + ) + else: + volume_opts = {} + if no_copy: + volume_opts['NoCopy'] = True + if labels: + volume_opts['Labels'] = labels + if driver_config: + volume_opts['driver_config'] = driver_config + if volume_opts: + self['VolumeOptions'] = volume_opts + if propagation: + raise errors.DockerError( + 'Mount type is volume but `propagation` argument has been ' + 'provided.' + ) + + @classmethod + def parse_mount_string(cls, string): + parts = string.split(':') + if len(parts) > 3: + raise errors.DockerError( + 'Invalid mount format "{0}"'.format(string) + ) + if len(parts) == 1: + return cls(target=parts[0]) + else: + target = parts[1] + source = parts[0] + read_only = not (len(parts) == 3 or parts[2] == 'ro') + return cls(target, source, read_only=read_only) + + +class Resources(dict): + def __init__(self, cpu_limit=None, mem_limit=None, cpu_reservation=None, + mem_reservation=None): + limits = {} + reservation = {} + if cpu_limit is not None: + limits['NanoCPUs'] = cpu_limit + if mem_limit is not None: + limits['MemoryBytes'] = mem_limit + if cpu_reservation is not None: + reservation['NanoCPUs'] = cpu_reservation + if mem_reservation is not None: + reservation['MemoryBytes'] = mem_reservation + + if limits: + self['Limits'] = limits + if reservation: + self['Reservations'] = reservation + + +class UpdateConfig(dict): + def __init__(self, parallelism=0, delay=None, failure_action='continue'): + self['Parallelism'] = parallelism + if delay is not None: + self['Delay'] = delay + if failure_action not in ('pause', 'continue'): + raise errors.DockerError( + 'failure_action must be either `pause` or `continue`.' + ) + self['FailureAction'] = failure_action + + +class RestartConditionTypesEnum(object): + _values = ( + 'none', + 'on_failure', + 'any', + ) + NONE, ON_FAILURE, ANY = _values + + +class RestartPolicy(dict): + condition_types = RestartConditionTypesEnum + + def __init__(self, condition=RestartConditionTypesEnum.NONE, delay=0, + max_attempts=0, window=0): + if condition not in self.condition_types._values: + raise TypeError( + 'Invalid RestartPolicy condition {0}'.format(condition) + ) + + self['Condition'] = condition + self['Delay'] = delay + self['MaxAttempts'] = max_attempts + self['Window'] = window + + +class LogDriver(dict): + def __init__(self, name, options=None): + self['Name'] = name + if options: + self['Options'] = options diff --git a/docker/types/swarm.py b/docker/types/swarm.py new file mode 100644 index 0000000000..865fde6203 --- /dev/null +++ b/docker/types/swarm.py @@ -0,0 +1,40 @@ +class SwarmSpec(dict): + def __init__(self, task_history_retention_limit=None, + snapshot_interval=None, keep_old_snapshots=None, + log_entries_for_slow_followers=None, heartbeat_tick=None, + election_tick=None, dispatcher_heartbeat_period=None, + node_cert_expiry=None, external_ca=None, name=None): + if task_history_retention_limit is not None: + self['Orchestration'] = { + 'TaskHistoryRetentionLimit': task_history_retention_limit + } + if any([snapshot_interval, keep_old_snapshots, + log_entries_for_slow_followers, heartbeat_tick, election_tick]): + self['Raft'] = { + 'SnapshotInterval': snapshot_interval, + 'KeepOldSnapshots': keep_old_snapshots, + 'LogEntriesForSlowFollowers': log_entries_for_slow_followers, + 'HeartbeatTick': heartbeat_tick, + 'ElectionTick': election_tick + } + + if dispatcher_heartbeat_period: + self['Dispatcher'] = { + 'HeartbeatPeriod': dispatcher_heartbeat_period + } + + if node_cert_expiry or external_ca: + self['CAConfig'] = { + 'NodeCertExpiry': node_cert_expiry, + 'ExternalCA': external_ca + } + + if name is not None: + self['Name'] = name + + +class SwarmExternalCA(dict): + def __init__(self, url, protocol=None, options=None): + self['URL'] = url + self['Protocol'] = protocol + self['Options'] = options diff --git a/docker/utils/__init__.py b/docker/utils/__init__.py index 35acc77911..4bb3876e65 100644 --- a/docker/utils/__init__.py +++ b/docker/utils/__init__.py @@ -8,8 +8,6 @@ create_ipam_config, create_ipam_pool, parse_devices, normalize_links, ) -from .types import LogConfig, Ulimit -from .types import ( - SwarmExternalCA, SwarmSpec, -) +from ..types import LogConfig, Ulimit +from ..types import SwarmExternalCA, SwarmSpec from .decorators import check_resource, minimum_version, update_headers diff --git a/docker/utils/utils.py b/docker/utils/utils.py index 00a7af14fe..bea436a3db 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -31,7 +31,7 @@ from .. import constants from .. import errors from .. import tls -from .types import Ulimit, LogConfig +from ..types import Ulimit, LogConfig if six.PY2: from urllib import splitnport diff --git a/docs/swarm.md b/docs/swarm.md index 0cd015a0eb..3cc44f8741 100644 --- a/docs/swarm.md +++ b/docs/swarm.md @@ -95,7 +95,7 @@ Initialize a new Swarm using the current connected engine as the first node. #### Client.create_swarm_spec -Create a `docker.utils.SwarmSpec` instance that can be used as the `swarm_spec` +Create a `docker.types.SwarmSpec` instance that can be used as the `swarm_spec` argument in `Client.init_swarm`. **Params:** @@ -113,12 +113,12 @@ argument in `Client.init_swarm`. heartbeat to the dispatcher. * node_cert_expiry (int): Automatic expiry for nodes certificates. * external_ca (dict): Configuration for forwarding signing requests to an - external certificate authority. Use `docker.utils.SwarmExternalCA`. + external certificate authority. Use `docker.types.SwarmExternalCA`. * name (string): Swarm's name -**Returns:** `docker.utils.SwarmSpec` instance. +**Returns:** `docker.types.SwarmSpec` instance. -#### docker.utils.SwarmExternalCA +#### docker.types.SwarmExternalCA Create a configuration dictionary for the `external_ca` argument in a `SwarmSpec`. diff --git a/setup.py b/setup.py index 85a4499422..c809321e7d 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,8 @@ url='https://github.com/docker/docker-py/', packages=[ 'docker', 'docker.api', 'docker.auth', 'docker.transport', - 'docker.utils', 'docker.utils.ports', 'docker.ssladapter' + 'docker.utils', 'docker.utils.ports', 'docker.ssladapter', + 'docker.types', ], install_requires=requirements, tests_require=test_requirements, diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 00109868ba..fda62b3562 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -40,8 +40,10 @@ def create_simple_service(self, name=None): else: name = self.get_service_name() - container_spec = docker.api.ContainerSpec('busybox', ['echo', 'hello']) - task_tmpl = docker.api.TaskTemplate(container_spec) + container_spec = docker.types.ContainerSpec( + 'busybox', ['echo', 'hello'] + ) + task_tmpl = docker.types.TaskTemplate(container_spec) return name, self.client.create_service(task_tmpl, name=name) @requires_api_version('1.24') @@ -74,7 +76,7 @@ def test_remove_service_by_id(self): test_services = self.client.services(filters={'name': 'dockerpytest_'}) assert len(test_services) == 0 - def test_rempve_service_by_name(self): + def test_remove_service_by_name(self): svc_name, svc_id = self.create_simple_service() assert self.client.remove_service(svc_name) test_services = self.client.services(filters={'name': 'dockerpytest_'}) @@ -87,6 +89,94 @@ def test_create_service_simple(self): assert len(services) == 1 assert services[0]['ID'] == svc_id['ID'] + def test_create_service_custom_log_driver(self): + container_spec = docker.types.ContainerSpec( + 'busybox', ['echo', 'hello'] + ) + log_cfg = docker.types.LogDriver('none') + task_tmpl = docker.types.TaskTemplate( + container_spec, log_driver=log_cfg + ) + name = self.get_service_name() + svc_id = self.client.create_service(task_tmpl, name=name) + svc_info = self.client.inspect_service(svc_id) + assert 'TaskTemplate' in svc_info['Spec'] + res_template = svc_info['Spec']['TaskTemplate'] + assert 'LogDriver' in res_template + assert 'Name' in res_template['LogDriver'] + assert res_template['LogDriver']['Name'] == 'none' + + def test_create_service_with_volume_mount(self): + vol_name = self.get_service_name() + container_spec = docker.types.ContainerSpec( + 'busybox', ['ls'], + mounts=[ + docker.types.Mount(target='/test', source=vol_name) + ] + ) + self.tmp_volumes.append(vol_name) + task_tmpl = docker.types.TaskTemplate(container_spec) + name = self.get_service_name() + svc_id = self.client.create_service(task_tmpl, name=name) + svc_info = self.client.inspect_service(svc_id) + assert 'ContainerSpec' in svc_info['Spec']['TaskTemplate'] + cspec = svc_info['Spec']['TaskTemplate']['ContainerSpec'] + assert 'Mounts' in cspec + assert len(cspec['Mounts']) == 1 + mount = cspec['Mounts'][0] + assert mount['Target'] == '/test' + assert mount['Source'] == vol_name + assert mount['Type'] == 'volume' + + def test_create_service_with_resources_constraints(self): + container_spec = docker.types.ContainerSpec('busybox', ['true']) + resources = docker.types.Resources( + cpu_limit=4000000, mem_limit=3 * 1024 * 1024 * 1024, + cpu_reservation=3500000, mem_reservation=2 * 1024 * 1024 * 1024 + ) + task_tmpl = docker.types.TaskTemplate( + container_spec, resources=resources + ) + name = self.get_service_name() + svc_id = self.client.create_service(task_tmpl, name=name) + svc_info = self.client.inspect_service(svc_id) + assert 'TaskTemplate' in svc_info['Spec'] + res_template = svc_info['Spec']['TaskTemplate'] + assert 'Resources' in res_template + assert res_template['Resources']['Limits'] == resources['Limits'] + assert res_template['Resources']['Reservations'] == resources[ + 'Reservations' + ] + + def test_create_service_with_update_config(self): + container_spec = docker.types.ContainerSpec('busybox', ['true']) + task_tmpl = docker.types.TaskTemplate(container_spec) + update_config = docker.types.UpdateConfig( + parallelism=10, delay=5, failure_action='pause' + ) + name = self.get_service_name() + svc_id = self.client.create_service( + task_tmpl, update_config=update_config, name=name + ) + svc_info = self.client.inspect_service(svc_id) + assert 'UpdateConfig' in svc_info['Spec'] + assert update_config == svc_info['Spec']['UpdateConfig'] + + def test_create_service_with_restart_policy(self): + container_spec = docker.types.ContainerSpec('busybox', ['true']) + policy = docker.types.RestartPolicy( + docker.types.RestartPolicy.condition_types.ANY, + delay=5, max_attempts=5 + ) + task_tmpl = docker.types.TaskTemplate( + container_spec, restart_policy=policy + ) + name = self.get_service_name() + svc_id = self.client.create_service(task_tmpl, name=name) + svc_info = self.client.inspect_service(svc_id) + assert 'RestartPolicy' in svc_info['Spec']['TaskTemplate'] + assert policy == svc_info['Spec']['TaskTemplate']['RestartPolicy'] + def test_update_service_name(self): name, svc_id = self.create_simple_service() svc_info = self.client.inspect_service(svc_id) From 8e97cb785758653116d5383094ef923f86b11ea9 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 19 Aug 2016 16:51:39 -0700 Subject: [PATCH 0083/1301] Services API documentation (WIP) Signed-off-by: Joffrey F --- docs/api.md | 25 +++++++++++++ docs/services.md | 93 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 118 insertions(+) create mode 100644 docs/services.md diff --git a/docs/api.md b/docs/api.md index ddfaffeb98..12467edacc 100644 --- a/docs/api.md +++ b/docs/api.md @@ -302,6 +302,11 @@ Create a network, similar to the `docker network create` command. **Returns** (dict): The created network reference object +## create_service + +Create a service, similar to the `docker service create` command. See the +[services documentation](services.md#Clientcreate_service) for details. + ## create_volume Create and register a named volume @@ -651,6 +656,11 @@ Retrieve network info by id. Retrieve low-level information about a Swarm node. See the [Swarm documentation](swarm.md#clientinspect_node). +## inspect_service + +Create a service, similar to the `docker service create` command. See the +[services documentation](services.md#clientinspect_service) for details. + ## inspect_swarm Retrieve information about the current Swarm. @@ -895,6 +905,11 @@ Remove a network. Similar to the `docker network rm` command. Failure to remove will raise a `docker.errors.APIError` exception. +## remove_service + +Remove a service, similar to the `docker service rm` command. See the +[services documentation](services.md#clientremove_service) for details. + ## remove_volume Remove a volume. Similar to the `docker volume rm` command. @@ -963,6 +978,11 @@ Identical to the `docker search` command. ... ``` +## services + +List services, similar to the `docker service ls` command. See the +[services documentation](services.md#clientservices) for details. + ## start Similar to the `docker start` command, but doesn't support attach options. Use @@ -1083,6 +1103,11 @@ Update resource configs of one or more containers. **Returns** (dict): Dictionary containing a `Warnings` key. +## update_service + +Update a service, similar to the `docker service update` command. See the +[services documentation](services.md#clientupdate_service) for details. + ## update_swarm Update the current Swarm. diff --git a/docs/services.md b/docs/services.md new file mode 100644 index 0000000000..f9cd428ecf --- /dev/null +++ b/docs/services.md @@ -0,0 +1,93 @@ +# Swarm services + +Starting with Engine version 1.12 (API 1.24), it is possible to manage services +using the Docker Engine API. Note that the engine needs to be part of a +[Swarm cluster](swarm.md) before you can use the service-related methods. + +## Creating a service + +The `Client.create_service` method lets you create a new service inside the +cluster. The method takes several arguments, `task_template` being mandatory. +This dictionary of values is most easily produced by instantiating a +`TaskTemplate` object. + +```python +container_spec = docker.types.ContainerSpec( + image='busybox', command=['echo', 'hello'] +) +task_tmpl = docker.types.TaskTemplate(container_spec) +service_id = client.create_service(task_tmpl, name=name) +``` + +## Listing services + +List all existing services using the `Client.services` method. + +```python +client.services(filters={'name': 'mysql'}) +``` + +## Retrieving service configuration + +To retrieve detailed information and configuration for a specific service, you +may use the `Client.inspect_service` method using the service's ID or name. + +```python +client.inspect_service(service='my_service_name') +``` + +## Updating service configuration + +The `Client.update_service` method lets you update a service's configuration. +The mandatory `version` argument (used to prevent concurrent writes) can be +retrieved using `Client.inspect_service`. + +```python +container_spec = docker.types.ContainerSpec( + image='busybox', command=['echo', 'hello world'] +) +task_tmpl = docker.types.TaskTemplate(container_spec) + +svc_version = client.inspect_service(svc_id)['Version']['Index'] + +client.update_service( + svc_id, svc_version, name='new_name', task_template=task_tmpl +) +``` + +## Removing a service + +A service may be removed simply using the `Client.remove_service` method. +Either the service name or service ID can be used as argument. + +```python +client.remove_service('my_service_name') +``` + +## Service API documentation + +### Client.create_service + +### Client.inspect_service + +### Client.remove_service + +### Client.services + +### Client.update_service + +### Configuration objects (`docker.types`) + +#### ContainerSpec + +#### LogDriver + +#### Mount + +#### Resources + +#### RestartPolicy + +#### TaskTemplate + +#### UpdateConfig From 1e2c58de9efdffc23dea78c9853fa08adb9109f3 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 19 Aug 2016 17:02:33 -0700 Subject: [PATCH 0084/1301] Add new pages to mkdocs index Signed-off-by: Joffrey F --- mkdocs.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mkdocs.yml b/mkdocs.yml index 67b40893cf..6cfaa543bc 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -13,6 +13,8 @@ pages: - Host devices: host-devices.md - Host configuration: hostconfig.md - Network configuration: networks.md +- Swarm management: swarm.md +- Swarm services: services.md - Using tmpfs: tmpfs.md - Using with Docker Machine: machine.md - Change Log: change_log.md From f53cdc3a0704e501170b326048d8b90d90e6a4ed Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 22 Aug 2016 16:11:48 -0700 Subject: [PATCH 0085/1301] Rename LogDriver to DriverConfig for genericity The class can be used for both log driver and volume driver specs. Use a name that reflects this. Signed-off-by: Joffrey F --- docker/types/__init__.py | 2 +- docker/types/services.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/types/__init__.py b/docker/types/__init__.py index 46f10d86a4..3609581d92 100644 --- a/docker/types/__init__.py +++ b/docker/types/__init__.py @@ -1,7 +1,7 @@ # flake8: noqa from .containers import LogConfig, Ulimit from .services import ( - ContainerSpec, LogDriver, Mount, Resources, RestartPolicy, TaskTemplate, + ContainerSpec, DriverConfig, Mount, Resources, RestartPolicy, TaskTemplate, UpdateConfig ) from .swarm import SwarmSpec, SwarmExternalCA diff --git a/docker/types/services.py b/docker/types/services.py index 6a17e93f13..dcc84f9c77 100644 --- a/docker/types/services.py +++ b/docker/types/services.py @@ -169,7 +169,7 @@ def __init__(self, condition=RestartConditionTypesEnum.NONE, delay=0, self['Window'] = window -class LogDriver(dict): +class DriverConfig(dict): def __init__(self, name, options=None): self['Name'] = name if options: From 7d5a1eeb7a46f17136aaf1041660d043a85051fc Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 22 Aug 2016 16:13:06 -0700 Subject: [PATCH 0086/1301] Add services documentation Signed-off-by: Joffrey F --- docs/services.md | 163 +++++++++++++++++++++++++++++- tests/integration/service_test.py | 2 +- 2 files changed, 163 insertions(+), 2 deletions(-) diff --git a/docs/services.md b/docs/services.md index f9cd428ecf..a6bb7d63c0 100644 --- a/docs/services.md +++ b/docs/services.md @@ -68,26 +68,187 @@ client.remove_service('my_service_name') ### Client.create_service +Create a service. + +**Params:** + +* task_template (dict): Specification of the task to start as part of the new + service. See the [TaskTemplate class](#TaskTemplate) for details. +* name (string): User-defined name for the service. Optional. +* labels (dict): A map of labels to associate with the service. Optional. +* mode (string): Scheduling mode for the service (`replicated` or `global`). + Defaults to `replicated`. +* update_config (dict): Specification for the update strategy of the service. + See the [UpdateConfig class](#UpdateConfig) for details. Default: `None`. +* networks (list): List of network names or IDs to attach the service to. + Default: `None`. +* endpoint_config (dict): Properties that can be configured to access and load + balance a service. Default: `None`. + +**Returns:** A dictionary containing an `ID` key for the newly created service. + ### Client.inspect_service +Return information on a service. + +**Params:** + +* service (string): A service identifier (either its name or service ID) + +**Returns:** `True` if successful. Raises an `APIError` otherwise. + ### Client.remove_service +Stop and remove a service. + +**Params:** + +* service (string): A service identifier (either its name or service ID) + +**Returns:** `True` if successful. Raises an `APIError` otherwise. + ### Client.services +List services. + +**Params:** + +* filters (dict): Filters to process on the nodes list. Valid filters: + `id` and `name`. Default: `None`. + +**Returns:** A list of dictionaries containing data about each service. + ### Client.update_service +Update a service. + +**Params:** + +* service (string): A service identifier (either its name or service ID). +* version (int): The version number of the service object being updated. This + is required to avoid conflicting writes. +* task_template (dict): Specification of the updated task to start as part of + the service. See the [TaskTemplate class](#TaskTemplate) for details. +* name (string): New name for the service. Optional. +* labels (dict): A map of labels to associate with the service. Optional. +* mode (string): Scheduling mode for the service (`replicated` or `global`). + Defaults to `replicated`. +* update_config (dict): Specification for the update strategy of the service. + See the [UpdateConfig class](#UpdateConfig) for details. Default: `None`. +* networks (list): List of network names or IDs to attach the service to. + Default: `None`. +* endpoint_config (dict): Properties that can be configured to access and load + balance a service. Default: `None`. + +**Returns:** `True` if successful. Raises an `APIError` otherwise. + ### Configuration objects (`docker.types`) #### ContainerSpec -#### LogDriver +A `ContainerSpec` object describes the behavior of containers that are part +of a task, and is used when declaring a `TaskTemplate`. + +**Params:** + +* image (string): The image name to use for the container. +* command (string or list): The command to be run in the image. +* args (list): Arguments to the command. +* env (dict): Environment variables. +* dir (string): The working directory for commands to run in. +* user (string): The user inside the container. +* labels (dict): A map of labels to associate with the service. +* mounts (list): A list of specifications for mounts to be added to containers + created as part of the service. See the [Mount class](#Mount) for details. +* stop_grace_period (int): Amount of time to wait for the container to + terminate before forcefully killing it. + +#### DriverConfig + +A `LogDriver` object indicates which driver to use, as well as its +configuration. It can be used for the `log_driver` in a `ContainerSpec`, +and for the `driver_config` in a volume `Mount`. + +**Params:** + +* name (string): Name of the logging driver to use. +* options (dict): Driver-specific options. Default: `None`. #### Mount +A `Mount` object describes a mounted folder's configuration inside a +container. A list of `Mount`s would be used as part of a `ContainerSpec`. + +* target (string): Container path. +* source (string): Mount source (e.g. a volume name or a host path). +* type (string): The mount type (`bind` or `volume`). Default: `volume`. +* read_only (bool): Whether the mount should be read-only. +* propagation (string): A propagation mode with the value `[r]private`, + `[r]shared`, or `[r]slave`. Only valid for the `bind` type. +* no_copy (bool): False if the volume should be populated with the data from + the target. Default: `False`. Only valid for the `volume` type. +* labels (dict): User-defined name and labels for the volume. Only valid for + the `volume` type. +* driver_config (dict): Volume driver configuration. + See the [DriverConfig class](#DriverConfig) for details. Only valid for the + `volume` type. + #### Resources +A `Resources` object configures resource allocation for containers when +made part of a `ContainerSpec`. + +**Params:** + +* cpu_limit (int): CPU limit in units of 10^9 CPU shares. +* mem_limit (int): Memory limit in Bytes. +* cpu_reservation (int): CPU reservation in units of 10^9 CPU shares. +* mem_reservation (int): Memory reservation in Bytes. + #### RestartPolicy +A `RestartPolicy` object is used when creating a `ContainerSpec`. It dictates +whether a container should restart after stopping or failing. + +* condition (string): Condition for restart (`none`, `on-failure`, or `any`). + Default: `none`. +* delay (int): Delay between restart attempts. Default: 0 +* attempts (int): Maximum attempts to restart a given container before giving + up. Default value is 0, which is ignored. +* window (int): Time window used to evaluate the restart policy. Default value + is 0, which is unbounded. + + #### TaskTemplate +A `TaskTemplate` object can be used to describe the task specification to be +used when creating or updating a service. + +**Params:** + +* container_spec (dict): Container settings for containers started as part of + this task. See the [ContainerSpec class](#ContainerSpec) for details. +* log_driver (dict): Log configuration for containers created as part of the + service. See the [DriverConfig class](#DriverConfig) for details. +* resources (dict): Resource requirements which apply to each individual + container created as part of the service. See the + [Resources class](#Resources) for details. +* restart_policy (dict): Specification for the restart policy which applies + to containers created as part of this service. See the + [RestartPolicy class](#RestartPolicy) for details. +* placement (list): A list of constraints. + + #### UpdateConfig + +An `UpdateConfig` object can be used to specify the way container updates +should be performed by a service. + +**Params:** + +* parallelism (int): Maximum number of tasks to be updated in one iteration + (0 means unlimited parallelism). Default: 0. +* delay (int): Amount of time between updates. +* failure_action (string): Action to take if an updated task fails to run, or + stops running during the update. Acceptable values are `continue` and + `pause`. Default: `continue` diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index fda62b3562..3113df18d1 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -93,7 +93,7 @@ def test_create_service_custom_log_driver(self): container_spec = docker.types.ContainerSpec( 'busybox', ['echo', 'hello'] ) - log_cfg = docker.types.LogDriver('none') + log_cfg = docker.types.DriverConfig('none') task_tmpl = docker.types.TaskTemplate( container_spec, log_driver=log_cfg ) From 775b581c04dfa5f7d421ad74f969f25869fa8151 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 22 Aug 2016 19:12:27 -0700 Subject: [PATCH 0087/1301] Private images support in create_service / update_service Refactor auth header computation Add tasks methods and documentation. Signed-off-by: Joffrey F --- docker/api/image.py | 41 +++++-------------------------- docker/api/service.py | 41 +++++++++++++++++++++++++++++-- docker/auth/auth.py | 20 +++++++++++++++ docker/types/services.py | 5 ++++ docs/api.md | 21 ++++++++++++++++ tests/integration/service_test.py | 1 - 6 files changed, 91 insertions(+), 38 deletions(-) diff --git a/docker/api/image.py b/docker/api/image.py index 2bdbce83ab..4d6561e52d 100644 --- a/docker/api/image.py +++ b/docker/api/image.py @@ -166,28 +166,10 @@ def pull(self, repository, tag=None, stream=False, headers = {} if utils.compare_version('1.5', self._version) >= 0: - # If we don't have any auth data so far, try reloading the config - # file one more time in case anything showed up in there. if auth_config is None: - log.debug('Looking for auth config') - if not self._auth_configs: - log.debug( - "No auth config in memory - loading from filesystem" - ) - self._auth_configs = auth.load_config() - authcfg = auth.resolve_authconfig(self._auth_configs, registry) - # Do not fail here if no authentication exists for this - # specific registry as we can have a readonly pull. Just - # put the header if we can. - if authcfg: - log.debug('Found auth config') - # auth_config needs to be a dict in the format used by - # auth.py username , password, serveraddress, email - headers['X-Registry-Auth'] = auth.encode_header( - authcfg - ) - else: - log.debug('No auth config found') + header = auth.get_config_header(self, registry) + if header: + headers['X-Registry-Auth'] = header else: log.debug('Sending supplied auth config') headers['X-Registry-Auth'] = auth.encode_header(auth_config) @@ -222,21 +204,10 @@ def push(self, repository, tag=None, stream=False, headers = {} if utils.compare_version('1.5', self._version) >= 0: - # If we don't have any auth data so far, try reloading the config - # file one more time in case anything showed up in there. if auth_config is None: - log.debug('Looking for auth config') - if not self._auth_configs: - log.debug( - "No auth config in memory - loading from filesystem" - ) - self._auth_configs = auth.load_config() - authcfg = auth.resolve_authconfig(self._auth_configs, registry) - # Do not fail here if no authentication exists for this - # specific registry as we can have a readonly pull. Just - # put the header if we can. - if authcfg: - headers['X-Registry-Auth'] = auth.encode_header(authcfg) + header = auth.get_config_header(self, registry) + if header: + headers['X-Registry-Auth'] = header else: log.debug('Sending supplied auth config') headers['X-Registry-Auth'] = auth.encode_header(auth_config) diff --git a/docker/api/service.py b/docker/api/service.py index c62e4946b8..baebbadfe4 100644 --- a/docker/api/service.py +++ b/docker/api/service.py @@ -1,4 +1,6 @@ +from .. import errors from .. import utils +from ..auth import auth class ServiceApiMixin(object): @@ -8,6 +10,16 @@ def create_service( update_config=None, networks=None, endpoint_config=None ): url = self._url('/services/create') + headers = {} + image = task_template.get('ContainerSpec', {}).get('Image', None) + if image is None: + raise errors.DockerException( + 'Missing mandatory Image key in ContainerSpec' + ) + registry, repo_name = auth.resolve_repository_name(image) + auth_header = auth.get_config_header(self, registry) + if auth_header: + headers['X-Registry-Auth'] = auth_header data = { 'Name': name, 'Labels': labels, @@ -17,7 +29,9 @@ def create_service( 'Networks': networks, 'Endpoint': endpoint_config } - return self._result(self._post_json(url, data=data), True) + return self._result( + self._post_json(url, data=data, headers=headers), True + ) @utils.minimum_version('1.24') @utils.check_resource @@ -25,6 +39,12 @@ def inspect_service(self, service): url = self._url('/services/{0}', service) return self._result(self._get(url), True) + @utils.minimum_version('1.24') + @utils.check_resource + def inspect_task(self, task): + url = self._url('/tasks/{0}', task) + return self._result(self._get(url), True) + @utils.minimum_version('1.24') @utils.check_resource def remove_service(self, service): @@ -41,6 +61,14 @@ def services(self, filters=None): url = self._url('/services') return self._result(self._get(url, params=params), True) + @utils.minimum_version('1.24') + def tasks(self, filters=None): + params = { + 'filters': utils.convert_filters(filters) if filters else None + } + url = self._url('/tasks') + return self._result(self._get(url, params=params), True) + @utils.minimum_version('1.24') @utils.check_resource def update_service(self, service, version, task_template=None, name=None, @@ -48,6 +76,7 @@ def update_service(self, service, version, task_template=None, name=None, networks=None, endpoint_config=None): url = self._url('/services/{0}/update', service) data = {} + headers = {} if name is not None: data['Name'] = name if labels is not None: @@ -55,6 +84,12 @@ def update_service(self, service, version, task_template=None, name=None, if mode is not None: data['Mode'] = mode if task_template is not None: + image = task_template.get('ContainerSpec', {}).get('Image', None) + if image is not None: + registry, repo_name = auth.resolve_repository_name(image) + auth_header = auth.get_config_header(self, registry) + if auth_header: + headers['X-Registry-Auth'] = auth_header data['TaskTemplate'] = task_template if update_config is not None: data['UpdateConfig'] = update_config @@ -63,6 +98,8 @@ def update_service(self, service, version, task_template=None, name=None, if endpoint_config is not None: data['Endpoint'] = endpoint_config - resp = self._post_json(url, data=data, params={'version': version}) + resp = self._post_json( + url, data=data, params={'version': version}, headers=headers + ) self._raise_for_status(resp) return True diff --git a/docker/auth/auth.py b/docker/auth/auth.py index b61a8d09e6..7195f56ad4 100644 --- a/docker/auth/auth.py +++ b/docker/auth/auth.py @@ -51,6 +51,26 @@ def resolve_index_name(index_name): return index_name +def get_config_header(client, registry): + log.debug('Looking for auth config') + if not client._auth_configs: + log.debug( + "No auth config in memory - loading from filesystem" + ) + client._auth_configs = load_config() + authcfg = resolve_authconfig(client._auth_configs, registry) + # Do not fail here if no authentication exists for this + # specific registry as we can have a readonly pull. Just + # put the header if we can. + if authcfg: + log.debug('Found auth config') + # auth_config needs to be a dict in the format used by + # auth.py username , password, serveraddress, email + return encode_header(authcfg) + log.debug('No auth config found') + return None + + def split_repo_name(repo_name): parts = repo_name.split('/', 1) if len(parts) == 1 or ( diff --git a/docker/types/services.py b/docker/types/services.py index dcc84f9c77..2c1a830c3b 100644 --- a/docker/types/services.py +++ b/docker/types/services.py @@ -36,7 +36,12 @@ def placement(self): class ContainerSpec(dict): def __init__(self, image, command=None, args=None, env=None, workdir=None, user=None, labels=None, mounts=None, stop_grace_period=None): + from ..utils import split_command # FIXME: circular import + self['Image'] = image + + if isinstance(command, six.string_types): + command = split_command(command) self['Command'] = command self['Args'] = args diff --git a/docs/api.md b/docs/api.md index 12467edacc..5857892fef 100644 --- a/docs/api.md +++ b/docs/api.md @@ -666,6 +666,16 @@ Create a service, similar to the `docker service create` command. See the Retrieve information about the current Swarm. See the [Swarm documentation](swarm.md#clientinspect_swarm). +## inspect_task + +Retrieve information about a task. + +**Params**: + +* task (str): Task identifier + +**Returns** (dict): Task information dictionary + ## inspect_volume Retrieve volume info by name. @@ -1055,6 +1065,17 @@ Tag an image into a repository. Identical to the `docker tag` command. **Returns** (bool): True if successful +## tasks + +Retrieve a list of tasks. + +**Params**: + +* filters (dict): A map of filters to process on the tasks list. Valid filters: + `id`, `name`, `service`, `node`, `label` and `desired-state`. + +**Returns** (list): List of task dictionaries. + ## top Display the running processes of a container. diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 3113df18d1..2b99316bae 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -1,7 +1,6 @@ import random import docker -# import pytest from ..base import requires_api_version from .. import helpers From fc72ac66e99551c10375398ee7afa552fbe867f2 Mon Sep 17 00:00:00 2001 From: Kay Yan Date: Wed, 13 Jul 2016 14:55:33 +0800 Subject: [PATCH 0088/1301] support MemoryReservation and KernelMemory Signed-off-by: Kay Yan --- docker/utils/utils.py | 18 ++++++++++++++++-- tests/unit/utils_test.py | 14 ++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/docker/utils/utils.py b/docker/utils/utils.py index 082bd9b0ad..a5fbe0bace 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -613,8 +613,10 @@ def create_host_config(binds=None, port_bindings=None, lxc_conf=None, cap_drop=None, devices=None, extra_hosts=None, read_only=None, pid_mode=None, ipc_mode=None, security_opt=None, ulimits=None, log_config=None, - mem_limit=None, memswap_limit=None, mem_swappiness=None, - cgroup_parent=None, group_add=None, cpu_quota=None, + mem_limit=None, memswap_limit=None, + mem_reservation=None, kernel_memory=None, + mem_swappiness=None, cgroup_parent=None, + group_add=None, cpu_quota=None, cpu_period=None, blkio_weight=None, blkio_weight_device=None, device_read_bps=None, device_write_bps=None, device_read_iops=None, @@ -638,6 +640,18 @@ def create_host_config(binds=None, port_bindings=None, lxc_conf=None, if memswap_limit is not None: host_config['MemorySwap'] = parse_bytes(memswap_limit) + if mem_reservation: + if version_lt(version, '1.21'): + raise host_config_version_error('mem_reservation', '1.21') + + host_config['MemoryReservation'] = parse_bytes(mem_reservation) + + if kernel_memory: + if version_lt(version, '1.21'): + raise host_config_version_error('kernel_memory', '1.21') + + host_config['KernelMemory'] = parse_bytes(kernel_memory) + if mem_swappiness is not None: if version_lt(version, '1.20'): raise host_config_version_error('mem_swappiness', '1.20') diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index b37739147e..3476f0418a 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -171,6 +171,20 @@ def test_create_endpoint_config_with_aliases(self): with pytest.raises(InvalidVersion): create_endpoint_config(version='1.21', aliases=['foo', 'bar']) + def test_create_host_config_with_mem_reservation(self): + config = create_host_config(version='1.21', mem_reservation=67108864) + self.assertEqual(config.get('MemoryReservation'), 67108864) + self.assertRaises( + InvalidVersion, lambda: create_host_config( + version='1.20', mem_reservation=67108864)) + + def test_create_host_config_with_kernel_memory(self): + config = create_host_config(version='1.21', kernel_memory=67108864) + self.assertEqual(config.get('KernelMemory'), 67108864) + self.assertRaises( + InvalidVersion, lambda: create_host_config( + version='1.20', kernel_memory=67108864)) + class UlimitTest(base.BaseTestCase): def test_create_host_config_dict_ulimit(self): From 902c7a76ccf23e2e210e982cc832a2770cfc99f4 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 23 Aug 2016 17:05:08 -0700 Subject: [PATCH 0089/1301] Docs and tests for pids_limit. Signed-off-by: Joffrey F --- docs/hostconfig.md | 8 +++++--- tests/unit/utils_test.py | 9 +++++++++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/docs/hostconfig.md b/docs/hostconfig.md index 6645bd1f35..008d5cf210 100644 --- a/docs/hostconfig.md +++ b/docs/hostconfig.md @@ -111,11 +111,12 @@ for example: CPU period. * cpu_shares (int): CPU shares (relative weight) * cpuset_cpus (str): CPUs in which to allow execution (0-3, 0,1) -* blkio_weight: Block IO weight (relative weight), accepts a weight value between 10 and 1000. +* blkio_weight: Block IO weight (relative weight), accepts a weight value + between 10 and 1000. * blkio_weight_device: Block IO weight (relative device weight) in the form of: `[{"Path": "device_path", "Weight": weight}]` -* device_read_bps: Limit read rate (bytes per second) from a device in the form of: - `[{"Path": "device_path", "Rate": rate}]` +* device_read_bps: Limit read rate (bytes per second) from a device in the + form of: `[{"Path": "device_path", "Rate": rate}]` * device_write_bps: Limit write rate (bytes per second) from a device. * device_read_iops: Limit read rate (IO per second) from a device. * device_write_iops: Limit write rate (IO per second) from a device. @@ -128,6 +129,7 @@ for example: * sysctls (dict): Kernel parameters to set in the container. * userns_mode (str): Sets the user namespace mode for the container when user namespace remapping option is enabled. Supported values are: `host` +* pids_limit (int): Tune a container’s pids limit. Set -1 for unlimited. **Returns** (dict) HostConfig dictionary diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index 3476f0418a..2a2759d033 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -185,6 +185,15 @@ def test_create_host_config_with_kernel_memory(self): InvalidVersion, lambda: create_host_config( version='1.20', kernel_memory=67108864)) + def test_create_host_config_with_pids_limit(self): + config = create_host_config(version='1.23', pids_limit=1024) + self.assertEqual(config.get('PidsLimit'), 1024) + + with pytest.raises(InvalidVersion): + create_host_config(version='1.22', pids_limit=1024) + with pytest.raises(TypeError): + create_host_config(version='1.22', pids_limit='1024') + class UlimitTest(base.BaseTestCase): def test_create_host_config_dict_ulimit(self): From 5bedd32a6942e89eacd4f63298551404856be5fc Mon Sep 17 00:00:00 2001 From: fermayo Date: Thu, 25 Aug 2016 13:34:54 +0200 Subject: [PATCH 0090/1301] Fix creating containers with env vars with unicode characters Signed-off-by: Fernando Mayo --- docker/utils/utils.py | 2 +- tests/unit/container_test.py | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/docker/utils/utils.py b/docker/utils/utils.py index a5fbe0bace..8385a76000 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -993,7 +993,7 @@ def format_environment(environment): def format_env(key, value): if value is None: return key - return '{key}={value}'.format(key=key, value=value) + return u'{key}={value}'.format(key=key, value=value) return [format_env(*var) for var in six.iteritems(environment)] diff --git a/tests/unit/container_test.py b/tests/unit/container_test.py index c480462f2f..3cea42fbd4 100644 --- a/tests/unit/container_test.py +++ b/tests/unit/container_test.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- + import datetime import json import signal @@ -1155,6 +1157,24 @@ def test_create_container_with_sysctl(self): args[1]['timeout'], DEFAULT_TIMEOUT_SECONDS ) + def test_create_container_with_unicode_envvars(self): + envvars_dict = { + 'foo': u'☃', + } + + expected = [ + u'foo=☃' + ] + + self.client.create_container( + 'busybox', 'true', + environment=envvars_dict, + ) + + args = fake_request.call_args + self.assertEqual(args[0][1], url_prefix + 'containers/create') + self.assertEqual(json.loads(args[1]['data'])['Env'], expected) + class ContainerTest(DockerClientTest): def test_list_containers(self): From 764d7b38c484f8dd45eafb47d0add602de5d3ada Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 23 Aug 2016 16:51:52 -0700 Subject: [PATCH 0091/1301] Support version parameter in `Client.from_env` Signed-off-by: Joffrey F --- docker/client.py | 3 ++- tests/unit/client_test.py | 8 ++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/docker/client.py b/docker/client.py index 758675369d..dc28ac46c1 100644 --- a/docker/client.py +++ b/docker/client.py @@ -114,7 +114,8 @@ def __init__(self, base_url=None, version=None, @classmethod def from_env(cls, **kwargs): - return cls(**kwargs_from_env(**kwargs)) + version = kwargs.pop('version', None) + return cls(version=version, **kwargs_from_env(**kwargs)) def _retrieve_server_version(self): try: diff --git a/tests/unit/client_test.py b/tests/unit/client_test.py index b21f1d6ae4..6ceb8cbbc0 100644 --- a/tests/unit/client_test.py +++ b/tests/unit/client_test.py @@ -25,6 +25,14 @@ def test_from_env(self): client = Client.from_env() self.assertEqual(client.base_url, "https://192.168.59.103:2376") + def test_from_env_with_version(self): + os.environ.update(DOCKER_HOST='tcp://192.168.59.103:2376', + DOCKER_CERT_PATH=TEST_CERT_DIR, + DOCKER_TLS_VERIFY='1') + client = Client.from_env(version='2.32') + self.assertEqual(client.base_url, "https://192.168.59.103:2376") + self.assertEqual(client._version, '2.32') + class DisableSocketTest(base.BaseTestCase): class DummySocket(object): From a665dfb3750058aaaa074799d5262876cb821884 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 31 Aug 2016 18:26:16 -0700 Subject: [PATCH 0092/1301] Add support for labels and enable_ipv6 in create_network Tests + docs Signed-off-by: Joffrey F --- docker/api/network.py | 19 ++++++++++++++++++- docs/api.md | 15 +++++++++------ tests/integration/network_test.py | 27 ++++++++++++++++++++++++++- 3 files changed, 53 insertions(+), 8 deletions(-) diff --git a/docker/api/network.py b/docker/api/network.py index 34cd8987a1..c4f48c20f0 100644 --- a/docker/api/network.py +++ b/docker/api/network.py @@ -22,7 +22,8 @@ def networks(self, names=None, ids=None): @minimum_version('1.21') def create_network(self, name, driver=None, options=None, ipam=None, - check_duplicate=None, internal=False): + check_duplicate=None, internal=False, labels=None, + enable_ipv6=False): if options is not None and not isinstance(options, dict): raise TypeError('options must be a dictionary') @@ -34,6 +35,22 @@ def create_network(self, name, driver=None, options=None, ipam=None, 'CheckDuplicate': check_duplicate } + if labels is not None: + if version_lt(self._version, '1.23'): + raise InvalidVersion( + 'network labels were introduced in API 1.23' + ) + if not isinstance(labels, dict): + raise TypeError('labels must be a dictionary') + data["Labels"] = labels + + if enable_ipv6: + if version_lt(self._version, '1.23'): + raise InvalidVersion( + 'enable_ipv6 was introduced in API 1.23' + ) + data['EnableIPv6'] = True + if internal: if version_lt(self._version, '1.22'): raise InvalidVersion('Internal networks are not ' diff --git a/docs/api.md b/docs/api.md index 895d7d4574..6af330afd6 100644 --- a/docs/api.md +++ b/docs/api.md @@ -283,22 +283,25 @@ The utility can be used as follows: ```python >>> import docker.utils >>> my_envs = docker.utils.parse_env_file('/path/to/file') ->>> docker.utils.create_container_config('1.18', '_mongodb', 'foobar', environment=my_envs) +>>> client.create_container('myimage', 'command', environment=my_envs) ``` -You can now use this with 'environment' for `create_container`. - - ## create_network -Create a network, similar to the `docker network create` command. +Create a network, similar to the `docker network create` command. See the +[networks documentation](networks.md) for details. **Params**: * name (str): Name of the network * driver (str): Name of the driver used to create the network - * options (dict): Driver options as a key-value dictionary +* ipam (dict): Optional custom IP scheme for the network +* check_duplicate (bool): Request daemon to check for networks with same name. + Default: `True`. +* internal (bool): Restrict external access to the network. Default `False`. +* labels (dict): Map of labels to set on the network. Default `None`. +* enable_ipv6 (bool): Enable IPv6 on the network. Default `False`. **Returns** (dict): The created network reference object diff --git a/tests/integration/network_test.py b/tests/integration/network_test.py index 27e1b14dec..70dff06075 100644 --- a/tests/integration/network_test.py +++ b/tests/integration/network_test.py @@ -300,7 +300,8 @@ def test_create_check_duplicate(self): net_name, net_id = self.create_network() with self.assertRaises(docker.errors.APIError): self.client.create_network(net_name, check_duplicate=True) - self.client.create_network(net_name, check_duplicate=False) + net_id = self.client.create_network(net_name, check_duplicate=False) + self.tmp_networks.append(net_id['Id']) @requires_api_version('1.22') def test_connect_with_links(self): @@ -387,3 +388,27 @@ def test_create_internal_networks(self): _, net_id = self.create_network(internal=True) net = self.client.inspect_network(net_id) assert net['Internal'] is True + + @requires_api_version('1.23') + def test_create_network_with_labels(self): + _, net_id = self.create_network(labels={ + 'com.docker.py.test': 'label' + }) + + net = self.client.inspect_network(net_id) + assert 'Labels' in net + assert len(net['Labels']) == 1 + assert net['Labels'] == { + 'com.docker.py.test': 'label' + } + + @requires_api_version('1.23') + def test_create_network_with_labels_wrong_type(self): + with pytest.raises(TypeError): + self.create_network(labels=['com.docker.py.test=label', ]) + + @requires_api_version('1.23') + def test_create_network_ipv6_enabled(self): + _, net_id = self.create_network(enable_ipv6=True) + net = self.client.inspect_network(net_id) + assert net['EnableIPv6'] is True From 6552076856bed2925b1611326630b341f27f41b2 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 31 Aug 2016 18:41:17 -0700 Subject: [PATCH 0093/1301] Add support for force disconnect Signed-off-by: Joffrey F --- docker/api/network.py | 11 +++++++++-- docs/api.md | 2 ++ tests/integration/network_test.py | 30 +++++++++++++++++++++++++++++- tests/unit/network_test.py | 2 +- 4 files changed, 41 insertions(+), 4 deletions(-) diff --git a/docker/api/network.py b/docker/api/network.py index c4f48c20f0..0ee0dab6ea 100644 --- a/docker/api/network.py +++ b/docker/api/network.py @@ -93,8 +93,15 @@ def connect_container_to_network(self, container, net_id, @check_resource @minimum_version('1.21') - def disconnect_container_from_network(self, container, net_id): - data = {"container": container} + def disconnect_container_from_network(self, container, net_id, + force=False): + data = {"Container": container} + if force: + if version_lt(self._version, '1.22'): + raise InvalidVersion( + 'Forced disconnect was introduced in API 1.22' + ) + data['Force'] = force url = self._url("/networks/{0}/disconnect", net_id) res = self._post_json(url, data=data) self._raise_for_status(res) diff --git a/docs/api.md b/docs/api.md index 6af330afd6..1699344a66 100644 --- a/docs/api.md +++ b/docs/api.md @@ -355,6 +355,8 @@ Inspect changes on a container's filesystem. * container (str): container-id/name to be disconnected from a network * net_id (str): network id +* force (bool): Force the container to disconnect from a network. + Default: `False` ## events diff --git a/tests/integration/network_test.py b/tests/integration/network_test.py index 70dff06075..6726db4b49 100644 --- a/tests/integration/network_test.py +++ b/tests/integration/network_test.py @@ -115,7 +115,8 @@ def test_connect_and_disconnect_container(self): network_data = self.client.inspect_network(net_id) self.assertEqual( list(network_data['Containers'].keys()), - [container['Id']]) + [container['Id']] + ) with pytest.raises(docker.errors.APIError): self.client.connect_container_to_network(container, net_id) @@ -127,6 +128,33 @@ def test_connect_and_disconnect_container(self): with pytest.raises(docker.errors.APIError): self.client.disconnect_container_from_network(container, net_id) + @requires_api_version('1.22') + def test_connect_and_force_disconnect_container(self): + net_name, net_id = self.create_network() + + container = self.client.create_container('busybox', 'top') + self.tmp_containers.append(container) + self.client.start(container) + + network_data = self.client.inspect_network(net_id) + self.assertFalse(network_data.get('Containers')) + + self.client.connect_container_to_network(container, net_id) + network_data = self.client.inspect_network(net_id) + self.assertEqual( + list(network_data['Containers'].keys()), + [container['Id']] + ) + + self.client.disconnect_container_from_network(container, net_id, True) + network_data = self.client.inspect_network(net_id) + self.assertFalse(network_data.get('Containers')) + + with pytest.raises(docker.errors.APIError): + self.client.disconnect_container_from_network( + container, net_id, force=True + ) + @requires_api_version('1.22') def test_connect_with_aliases(self): net_name, net_id = self.create_network() diff --git a/tests/unit/network_test.py b/tests/unit/network_test.py index 5bba9db230..2521688de8 100644 --- a/tests/unit/network_test.py +++ b/tests/unit/network_test.py @@ -184,4 +184,4 @@ def test_disconnect_container_from_network(self): self.assertEqual( json.loads(post.call_args[1]['data']), - {'container': container_id}) + {'Container': container_id}) From 9799c2d69b593c5f5dce8ff44a477207dc118e46 Mon Sep 17 00:00:00 2001 From: Joel Martin Date: Fri, 2 Sep 2016 15:29:00 -0500 Subject: [PATCH 0094/1301] Fix Mount bind type sanity check any() expects a single collection argument, not a list of arguments. Signed-off-by: Joel Martin --- docker/types/services.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/types/services.py b/docker/types/services.py index 2c1a830c3b..8488d6e2bc 100644 --- a/docker/types/services.py +++ b/docker/types/services.py @@ -80,7 +80,7 @@ def __init__(self, target, source, type='volume', read_only=False, self['BindOptions'] = { 'Propagation': propagation } - if any(labels, driver_config, no_copy): + if any([labels, driver_config, no_copy]): raise errors.DockerError( 'Mount type is binding but volume options have been ' 'provided.' From 3769c089e8dbac13ea6a65373f87d3b8ee539c5f Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Mon, 5 Sep 2016 17:48:09 +0200 Subject: [PATCH 0095/1301] Fix licenses * Complete main LICENSE * Remove unnecessary licenses from individual files Signed-off-by: Ben Firshman --- LICENSE | 13 +------------ docker/__init__.py | 14 -------------- docker/auth/auth.py | 14 -------------- docker/client.py | 14 -------------- docker/errors.py | 13 ------------- docker/transport/unixconn.py | 13 ------------- docker/utils/utils.py | 14 -------------- tests/unit/api_test.py | 14 -------------- tests/unit/fake_api.py | 14 -------------- 9 files changed, 1 insertion(+), 122 deletions(-) diff --git a/LICENSE b/LICENSE index d645695673..75191a4dc7 100644 --- a/LICENSE +++ b/LICENSE @@ -176,18 +176,7 @@ END OF TERMS AND CONDITIONS - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] + Copyright 2016 Docker, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/docker/__init__.py b/docker/__init__.py index 84d0734f51..ad53805e7f 100644 --- a/docker/__init__.py +++ b/docker/__init__.py @@ -1,17 +1,3 @@ -# Copyright 2013 dotCloud inc. - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at - -# http://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - from .version import version, version_info __version__ = version diff --git a/docker/auth/auth.py b/docker/auth/auth.py index 7195f56ad4..50f86f63c8 100644 --- a/docker/auth/auth.py +++ b/docker/auth/auth.py @@ -1,17 +1,3 @@ -# Copyright 2013 dotCloud inc. - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at - -# http://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - import base64 import json import logging diff --git a/docker/client.py b/docker/client.py index ef718a7207..b811d36c96 100644 --- a/docker/client.py +++ b/docker/client.py @@ -1,17 +1,3 @@ -# Copyright 2013 dotCloud inc. - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at - -# http://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - import json import struct from functools import partial diff --git a/docker/errors.py b/docker/errors.py index e85910cdfc..97be802d0a 100644 --- a/docker/errors.py +++ b/docker/errors.py @@ -1,16 +1,3 @@ -# Copyright 2014 dotCloud inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - import requests diff --git a/docker/transport/unixconn.py b/docker/transport/unixconn.py index f4d83ef309..e09b6bfa2c 100644 --- a/docker/transport/unixconn.py +++ b/docker/transport/unixconn.py @@ -1,16 +1,3 @@ -# Copyright 2013 dotCloud inc. - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at - -# http://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. import six import requests.adapters import socket diff --git a/docker/utils/utils.py b/docker/utils/utils.py index c108a83576..d46f8fcdbb 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -1,17 +1,3 @@ -# Copyright 2013 dotCloud inc. - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at - -# http://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - import base64 import io import os diff --git a/tests/unit/api_test.py b/tests/unit/api_test.py index 5850afa2fb..389b5f5360 100644 --- a/tests/unit/api_test.py +++ b/tests/unit/api_test.py @@ -1,17 +1,3 @@ -# Copyright 2013 dotCloud inc. - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at - -# http://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - import datetime import json import os diff --git a/tests/unit/fake_api.py b/tests/unit/fake_api.py index 835d73f2a4..54d5566b22 100644 --- a/tests/unit/fake_api.py +++ b/tests/unit/fake_api.py @@ -1,17 +1,3 @@ -# Copyright 2013 dotCloud inc. - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at - -# http://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - from . import fake_stat from docker import constants From 291470146f7264148997bda2a57124352fd04769 Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Mon, 5 Sep 2016 19:21:09 +0200 Subject: [PATCH 0096/1301] Add make docs command for building docs Signed-off-by: Ben Firshman --- Dockerfile-docs | 9 +++++++++ Makefile | 6 ++++++ docs-requirements.txt | 2 +- 3 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 Dockerfile-docs diff --git a/Dockerfile-docs b/Dockerfile-docs new file mode 100644 index 0000000000..1103ffd179 --- /dev/null +++ b/Dockerfile-docs @@ -0,0 +1,9 @@ +FROM python:2.7 + +RUN mkdir /home/docker-py +WORKDIR /home/docker-py + +COPY docs-requirements.txt /home/docker-py/docs-requirements.txt +RUN pip install -r docs-requirements.txt + +COPY . /home/docker-py diff --git a/Makefile b/Makefile index a635edfad5..3a8f5e8c3f 100644 --- a/Makefile +++ b/Makefile @@ -13,6 +13,9 @@ build: build-py3: docker build -t docker-py3 -f Dockerfile-py3 . +build-docs: + docker build -t docker-py-docs -f Dockerfile-docs . + build-dind-certs: docker build -t dpy-dind-certs -f tests/Dockerfile-dind-certs . @@ -57,3 +60,6 @@ integration-dind-ssl: build-dind-certs build build-py3 flake8: build docker run docker-py flake8 docker tests + +docs: build-docs + docker run -v `pwd`/docs:/home/docker-py/docs/ -p 8000:8000 docker-py-docs mkdocs serve -a 0.0.0.0:8000 diff --git a/docs-requirements.txt b/docs-requirements.txt index abc8d72db6..aede1cbadf 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -1 +1 @@ -mkdocs==0.9 +mkdocs==0.15.3 From fbe1686e629804fb47dabea1eda5c6d664f0a6b7 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 2 Sep 2016 19:59:47 -0700 Subject: [PATCH 0097/1301] Add credentials store support Signed-off-by: Joffrey F --- docker/auth/auth.py | 37 +++++++++++++++++++++++++++++++++++++ requirements.txt | 3 ++- setup.py | 1 + 3 files changed, 40 insertions(+), 1 deletion(-) diff --git a/docker/auth/auth.py b/docker/auth/auth.py index 50f86f63c8..fdaa7b5f97 100644 --- a/docker/auth/auth.py +++ b/docker/auth/auth.py @@ -3,6 +3,7 @@ import logging import os +import dockerpycreds import six from .. import errors @@ -11,6 +12,7 @@ INDEX_URL = 'https://{0}/v1/'.format(INDEX_NAME) DOCKER_CONFIG_FILENAME = os.path.join('.docker', 'config.json') LEGACY_DOCKER_CONFIG_FILENAME = '.dockercfg' +TOKEN_USERNAME = '' log = logging.getLogger(__name__) @@ -74,6 +76,13 @@ def resolve_authconfig(authconfig, registry=None): with full URLs are stripped down to hostnames before checking for a match. Returns None if no match was found. """ + if 'credsStore' in authconfig: + log.debug( + 'Using credentials store "{0}"'.format(authconfig['credsStore']) + ) + return _resolve_authconfig_credstore( + authconfig, registry, authconfig['credsStore'] + ) # Default to the public index server registry = resolve_index_name(registry) if registry else INDEX_NAME log.debug("Looking for auth entry for {0}".format(repr(registry))) @@ -91,6 +100,34 @@ def resolve_authconfig(authconfig, registry=None): return None +def _resolve_authconfig_credstore(authconfig, registry, credstore_name): + if not registry or registry == INDEX_NAME: + # The ecosystem is a little schizophrenic with index.docker.io VS + # docker.io - in that case, it seems the full URL is necessary. + registry = 'https://index.docker.io/v1/' + log.debug("Looking for auth entry for {0}".format(repr(registry))) + if registry not in authconfig: + log.debug("No entry found") + store = dockerpycreds.Store(credstore_name) + try: + data = store.get(registry) + res = { + 'ServerAddress': registry, + } + if data['Username'] == TOKEN_USERNAME: + res['IdentityToken'] = data['Secret'] + else: + res.update({ + 'Username': data['Username'], + 'Password': data['Secret'], + }) + return res + except dockerpycreds.StoreError as e: + log.error('Credentials store error: {0}'.format(repr(e))) + + return None + + def convert_to_hostname(url): return url.replace('http://', '').replace('https://', '').split('/', 1)[0] diff --git a/requirements.txt b/requirements.txt index a79b7bf8a6..078163afca 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,5 @@ requests==2.5.3 six>=1.4.0 websocket-client==0.32.0 backports.ssl_match_hostname>=3.5 ; python_version < '3.5' -ipaddress==1.0.16 ; python_version < '3.3' \ No newline at end of file +ipaddress==1.0.16 ; python_version < '3.3' +docker-pycreds==0.1.0 \ No newline at end of file diff --git a/setup.py b/setup.py index c809321e7d..3877e96650 100644 --- a/setup.py +++ b/setup.py @@ -12,6 +12,7 @@ 'requests >= 2.5.2, < 2.11', 'six >= 1.4.0', 'websocket-client >= 0.32.0', + 'docker-pycreds >= 0.1.0' ] if sys.platform == 'win32': From 219a8699f9c92de9cb4634d482ce700bb1ec76fc Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 6 Sep 2016 14:58:41 -0700 Subject: [PATCH 0098/1301] Better credentials store error handling in resolve_authconfig Signed-off-by: Joffrey F --- docker/auth/auth.py | 11 ++++++----- requirements.txt | 2 +- setup.py | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/docker/auth/auth.py b/docker/auth/auth.py index fdaa7b5f97..ea15def80a 100644 --- a/docker/auth/auth.py +++ b/docker/auth/auth.py @@ -106,8 +106,6 @@ def _resolve_authconfig_credstore(authconfig, registry, credstore_name): # docker.io - in that case, it seems the full URL is necessary. registry = 'https://index.docker.io/v1/' log.debug("Looking for auth entry for {0}".format(repr(registry))) - if registry not in authconfig: - log.debug("No entry found") store = dockerpycreds.Store(credstore_name) try: data = store.get(registry) @@ -122,10 +120,13 @@ def _resolve_authconfig_credstore(authconfig, registry, credstore_name): 'Password': data['Secret'], }) return res + except dockerpycreds.CredentialsNotFound as e: + log.debug('No entry found') + return None except dockerpycreds.StoreError as e: - log.error('Credentials store error: {0}'.format(repr(e))) - - return None + raise errors.DockerException( + 'Credentials store error: {0}'.format(repr(e)) + ) def convert_to_hostname(url): diff --git a/requirements.txt b/requirements.txt index 078163afca..4c0d5c203a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,4 @@ six>=1.4.0 websocket-client==0.32.0 backports.ssl_match_hostname>=3.5 ; python_version < '3.5' ipaddress==1.0.16 ; python_version < '3.3' -docker-pycreds==0.1.0 \ No newline at end of file +docker-pycreds==0.2.0 \ No newline at end of file diff --git a/setup.py b/setup.py index 3877e96650..1afd873b50 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ 'requests >= 2.5.2, < 2.11', 'six >= 1.4.0', 'websocket-client >= 0.32.0', - 'docker-pycreds >= 0.1.0' + 'docker-pycreds >= 0.2.0' ] if sys.platform == 'win32': From 65fb5be4cd30bb98f968986789bad00072b6001a Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 15 Jun 2016 15:54:09 -0700 Subject: [PATCH 0099/1301] Add support for changes param in import_image* methods Reduce code duplication in import_image* methods Signed-off-by: Joffrey F --- docker/api/image.py | 172 +++++++++++++++++++++++++------------------- 1 file changed, 97 insertions(+), 75 deletions(-) diff --git a/docker/api/image.py b/docker/api/image.py index 4d6561e52d..7f25f9d971 100644 --- a/docker/api/image.py +++ b/docker/api/image.py @@ -1,4 +1,5 @@ import logging +import os import six import warnings @@ -42,87 +43,79 @@ def images(self, name=None, quiet=False, all=False, viz=False, return [x['Id'] for x in res] return res - def import_image(self, src=None, repository=None, tag=None, image=None): - if src: - if isinstance(src, six.string_types): - try: - result = self.import_image_from_file( - src, repository=repository, tag=tag) - except IOError: - result = self.import_image_from_url( - src, repository=repository, tag=tag) - else: - result = self.import_image_from_data( - src, repository=repository, tag=tag) - elif image: - result = self.import_image_from_image( - image, repository=repository, tag=tag) - else: - raise Exception("Must specify a src or image") - - return result - - def import_image_from_data(self, data, repository=None, tag=None): - u = self._url("/images/create") - params = { - 'fromSrc': '-', - 'repo': repository, - 'tag': tag - } - headers = { - 'Content-Type': 'application/tar', - } - return self._result( - self._post(u, data=data, params=params, headers=headers)) + def import_image(self, src=None, repository=None, tag=None, image=None, + changes=None, stream_src=False): + if not (src or image): + raise errors.DockerException( + 'Must specify src or image to import from' + ) + u = self._url('/images/create') - def import_image_from_file(self, filename, repository=None, tag=None): - u = self._url("/images/create") - params = { - 'fromSrc': '-', - 'repo': repository, - 'tag': tag - } - headers = { - 'Content-Type': 'application/tar', - } - with open(filename, 'rb') as f: + params = _import_image_params( + repository, tag, image, + src=(src if isinstance(src, six.string_types) else None), + changes=changes + ) + headers = {'Content-Type': 'application/tar'} + + if image or params.get('fromSrc') != '-': # from image or URL return self._result( - self._post(u, data=f, params=params, headers=headers, - timeout=None)) + self._post(u, data=None, params=params) + ) + elif isinstance(src, six.string_types): # from file path + with open(src, 'rb') as f: + return self._result( + self._post( + u, data=f, params=params, headers=headers, timeout=None + ) + ) + else: # from raw data + if stream_src: + headers['Transfer-Encoding'] = 'chunked' + return self._result( + self._post(u, data=src, params=params, headers=headers) + ) - def import_image_from_stream(self, stream, repository=None, tag=None): - u = self._url("/images/create") - params = { - 'fromSrc': '-', - 'repo': repository, - 'tag': tag - } - headers = { - 'Content-Type': 'application/tar', - 'Transfer-Encoding': 'chunked', - } + def import_image_from_data(self, data, repository=None, tag=None, + changes=None): + u = self._url('/images/create') + params = _import_image_params( + repository, tag, src='-', changes=changes + ) + headers = {'Content-Type': 'application/tar'} return self._result( - self._post(u, data=stream, params=params, headers=headers)) + self._post( + u, data=data, params=params, headers=headers, timeout=None + ) + ) + return self.import_image( + src=data, repository=repository, tag=tag, changes=changes + ) - def import_image_from_url(self, url, repository=None, tag=None): - u = self._url("/images/create") - params = { - 'fromSrc': url, - 'repo': repository, - 'tag': tag - } - return self._result( - self._post(u, data=None, params=params)) + def import_image_from_file(self, filename, repository=None, tag=None, + changes=None): + return self.import_image( + src=filename, repository=repository, tag=tag, changes=changes + ) - def import_image_from_image(self, image, repository=None, tag=None): - u = self._url("/images/create") - params = { - 'fromImage': image, - 'repo': repository, - 'tag': tag - } - return self._result( - self._post(u, data=None, params=params)) + def import_image_from_stream(self, stream, repository=None, tag=None, + changes=None): + return self.import_image( + src=stream, stream_src=True, repository=repository, tag=tag, + changes=changes + ) + + def import_image_from_url(self, url, repository=None, tag=None, + changes=None): + return self.import_image( + src=url, repository=repository, tag=tag, changes=changes + ) + + def import_image_from_image(self, image, repository=None, tag=None, + changes=None): + return self.import_image( + image=image, repository=repository, tag=tag, changes=changes + ) @utils.check_resource def insert(self, image, url, path): @@ -246,3 +239,32 @@ def tag(self, image, repository, tag=None, force=False): res = self._post(url, params=params) self._raise_for_status(res) return res.status_code == 201 + + +def is_file(src): + try: + return ( + isinstance(src, six.string_types) and + os.path.isfile(src) + ) + except TypeError: # a data string will make isfile() raise a TypeError + return False + + +def _import_image_params(repo, tag, image=None, src=None, + changes=None): + params = { + 'repo': repo, + 'tag': tag, + } + if image: + params['fromImage'] = image + elif src and not is_file(src): + params['fromSrc'] = src + else: + params['fromSrc'] = '-' + + if changes: + params['changes'] = changes + + return params From 75497e07528a3d84a6ddd343aa66837ebb304e2a Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 15 Jun 2016 16:39:24 -0700 Subject: [PATCH 0100/1301] Add test for import_image with changes param Signed-off-by: Joffrey F --- Makefile | 5 ++-- tests/integration/image_test.py | 42 +++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 3a8f5e8c3f..5083b79ca2 100644 --- a/Makefile +++ b/Makefile @@ -3,9 +3,8 @@ all: test clean: - rm -rf tests/__pycache__ - rm -rf tests/*/__pycache__ - docker rm -vf dpy-dind + -docker rm -vf dpy-dind + find -name "__pycache__" | xargs rm -rf build: docker build -t docker-py . diff --git a/tests/integration/image_test.py b/tests/integration/image_test.py index 9f38366578..a61b58aeb7 100644 --- a/tests/integration/image_test.py +++ b/tests/integration/image_test.py @@ -208,6 +208,48 @@ def test_import_from_stream(self): img_id = result['status'] self.tmp_imgs.append(img_id) + def test_import_image_from_data_with_changes(self): + with self.dummy_tar_stream(n_bytes=500) as f: + content = f.read() + + statuses = self.client.import_image_from_data( + content, repository='test/import-from-bytes', + changes=['USER foobar', 'CMD ["echo"]'] + ) + + result_text = statuses.splitlines()[-1] + result = json.loads(result_text) + + assert 'error' not in result + + img_id = result['status'] + self.tmp_imgs.append(img_id) + + img_data = self.client.inspect_image(img_id) + assert img_data is not None + assert img_data['Config']['Cmd'] == ['echo'] + assert img_data['Config']['User'] == 'foobar' + + def test_import_image_with_changes(self): + with self.dummy_tar_file(n_bytes=self.TAR_SIZE) as tar_filename: + statuses = self.client.import_image( + src=tar_filename, repository='test/import-from-file', + changes=['USER foobar', 'CMD ["echo"]'] + ) + + result_text = statuses.splitlines()[-1] + result = json.loads(result_text) + + assert 'error' not in result + + img_id = result['status'] + self.tmp_imgs.append(img_id) + + img_data = self.client.inspect_image(img_id) + assert img_data is not None + assert img_data['Config']['Cmd'] == ['echo'] + assert img_data['Config']['User'] == 'foobar' + @contextlib.contextmanager def temporary_http_file_server(self, stream): '''Serve data from an IO stream over HTTP.''' From 0cdf7376253499923746f160106a757611988341 Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Wed, 7 Sep 2016 10:47:06 +0200 Subject: [PATCH 0101/1301] Fix unit test which doesn't do anything It also overrode the fake API inspect endpoint with a broken response. Signed-off-by: Ben Firshman --- tests/unit/container_test.py | 16 ++++++++++------ tests/unit/fake_api.py | 31 ------------------------------- 2 files changed, 10 insertions(+), 37 deletions(-) diff --git a/tests/unit/container_test.py b/tests/unit/container_test.py index 3cea42fbd4..8871b85452 100644 --- a/tests/unit/container_test.py +++ b/tests/unit/container_test.py @@ -751,14 +751,18 @@ def test_create_container_with_port_binds(self): ) def test_create_container_with_mac_address(self): - mac_address_expected = "02:42:ac:11:00:0a" + expected = "02:42:ac:11:00:0a" - container = self.client.create_container( - 'busybox', ['sleep', '60'], mac_address=mac_address_expected) + self.client.create_container( + 'busybox', + ['sleep', '60'], + mac_address=expected + ) - res = self.client.inspect_container(container['Id']) - self.assertEqual(mac_address_expected, - res['NetworkSettings']['MacAddress']) + args = fake_request.call_args + self.assertEqual(args[0][1], url_prefix + 'containers/create') + data = json.loads(args[1]['data']) + assert data['MacAddress'] == expected def test_create_container_with_links(self): link_path = 'path' diff --git a/tests/unit/fake_api.py b/tests/unit/fake_api.py index 54d5566b22..1e9d318df5 100644 --- a/tests/unit/fake_api.py +++ b/tests/unit/fake_api.py @@ -169,35 +169,6 @@ def get_fake_inspect_image(): return status_code, response -def get_fake_port(): - status_code = 200 - response = { - 'HostConfig': { - 'Binds': None, - 'ContainerIDFile': '', - 'Links': None, - 'LxcConf': None, - 'PortBindings': { - '1111': None, - '1111/tcp': [{'HostIp': '127.0.0.1', 'HostPort': '4567'}], - '2222': None - }, - 'Privileged': False, - 'PublishAllPorts': False - }, - 'NetworkSettings': { - 'Bridge': 'docker0', - 'PortMapping': None, - 'Ports': { - '1111': None, - '1111/tcp': [{'HostIp': '127.0.0.1', 'HostPort': '4567'}], - '2222': None}, - 'MacAddress': '02:42:ac:11:00:0a' - } - } - return status_code, response - - def get_fake_insert_image(): status_code = 200 response = {'StatusCode': 0} @@ -495,8 +466,6 @@ def post_fake_update_container(): post_fake_pause_container, '{1}/{0}/containers/3cc2351ab11b/unpause'.format(CURRENT_VERSION, prefix): post_fake_unpause_container, - '{1}/{0}/containers/3cc2351ab11b/json'.format(CURRENT_VERSION, prefix): - get_fake_port, '{1}/{0}/containers/3cc2351ab11b/restart'.format(CURRENT_VERSION, prefix): post_fake_restart_container, '{1}/{0}/containers/3cc2351ab11b'.format(CURRENT_VERSION, prefix): From 0430d00f2f3a5f3328cd1646d8ff6542b5ba6ff6 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 7 Sep 2016 17:49:07 -0700 Subject: [PATCH 0102/1301] Handle bufsize < 0 in makefile() as a substitute for default Signed-off-by: Joffrey F --- docker/transport/npipesocket.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/transport/npipesocket.py b/docker/transport/npipesocket.py index 35418ef170..9010cebe13 100644 --- a/docker/transport/npipesocket.py +++ b/docker/transport/npipesocket.py @@ -94,7 +94,7 @@ def makefile(self, mode=None, bufsize=None): if mode.strip('b') != 'r': raise NotImplementedError() rawio = NpipeFileIOBase(self) - if bufsize is None: + if bufsize is None or bufsize < 0: bufsize = io.DEFAULT_BUFFER_SIZE return io.BufferedReader(rawio, buffer_size=bufsize) From 06489235964470c9fdb12ec3a30e82aaa9586a28 Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Thu, 8 Sep 2016 09:10:27 +0100 Subject: [PATCH 0103/1301] Add .PHONY for each makefile instruction Makes it much easier to keep this maintained properly. See also: http://clarkgrubb.com/makefile-style-guide#phony-targets Replaces #1164 Signed-off-by: Ben Firshman --- Makefile | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 5083b79ca2..1fb21d44ed 100644 --- a/Makefile +++ b/Makefile @@ -1,37 +1,47 @@ -.PHONY: all build test integration-test unit-test build-py3 unit-test-py3 integration-test-py3 - +.PHONY: all all: test +.PHONY: clean clean: -docker rm -vf dpy-dind find -name "__pycache__" | xargs rm -rf +.PHONY: build build: docker build -t docker-py . +.PHONY: build-py3 build-py3: docker build -t docker-py3 -f Dockerfile-py3 . +.PHONY: build-docs build-docs: docker build -t docker-py-docs -f Dockerfile-docs . +.PHONY: build-dind-certs build-dind-certs: docker build -t dpy-dind-certs -f tests/Dockerfile-dind-certs . +.PHONY: test test: flake8 unit-test unit-test-py3 integration-dind integration-dind-ssl +.PHONY: unit-test unit-test: build docker run docker-py py.test tests/unit +.PHONY: unit-test-py3 unit-test-py3: build-py3 docker run docker-py3 py.test tests/unit +.PHONY: integration-test integration-test: build docker run -v /var/run/docker.sock:/var/run/docker.sock docker-py py.test tests/integration +.PHONY: integration-test-py3 integration-test-py3: build-py3 docker run -v /var/run/docker.sock:/var/run/docker.sock docker-py3 py.test tests/integration +.PHONY: integration-dind integration-dind: build build-py3 docker rm -vf dpy-dind || : docker run -d --name dpy-dind --privileged dockerswarm/dind:1.12.0 docker daemon\ @@ -42,6 +52,7 @@ integration-dind: build build-py3 py.test tests/integration docker rm -vf dpy-dind +.PHONY: integration-dind-ssl integration-dind-ssl: build-dind-certs build build-py3 docker run -d --name dpy-dind-certs dpy-dind-certs docker run -d --env="DOCKER_HOST=tcp://localhost:2375" --env="DOCKER_TLS_VERIFY=1"\ @@ -57,8 +68,10 @@ integration-dind-ssl: build-dind-certs build build-py3 --link=dpy-dind-ssl:docker docker-py3 py.test tests/integration docker rm -vf dpy-dind-ssl dpy-dind-certs +.PHONY: flake8 flake8: build docker run docker-py flake8 docker tests +.PHONY: docs docs: build-docs docker run -v `pwd`/docs:/home/docker-py/docs/ -p 8000:8000 docker-py-docs mkdocs serve -a 0.0.0.0:8000 From e6601e2e55fefa4216bcbf5d1f6c8dbbe7053f54 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 8 Jun 2016 17:27:25 -0700 Subject: [PATCH 0104/1301] Remove default adapters when connecting through a unix socket. Signed-off-by: Joffrey F --- docker/client.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docker/client.py b/docker/client.py index b811d36c96..6e8b2789d4 100644 --- a/docker/client.py +++ b/docker/client.py @@ -60,6 +60,7 @@ def __init__(self, base_url=None, version=None, if base_url.startswith('http+unix://'): self._custom_adapter = UnixAdapter(base_url, timeout) self.mount('http+docker://', self._custom_adapter) + self._unmount('http://', 'https://') self.base_url = 'http+docker://localunixsocket' elif base_url.startswith('npipe://'): if not constants.IS_WINDOWS_PLATFORM: @@ -368,6 +369,10 @@ def _get_result_tty(self, stream, res, is_tty): [x for x in self._multiplexed_buffer_helper(res)] ) + def _unmount(self, *args): + for proto in args: + self.adapters.pop(proto) + def get_adapter(self, url): try: return super(Client, self).get_adapter(url) From 3eb93f666289a1e4b5099a83e1c18950e8dd9d78 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 7 Sep 2016 17:42:45 -0700 Subject: [PATCH 0105/1301] Bump version Signed-off-by: Joffrey F --- docker/version.py | 2 +- docs/change_log.md | 43 +++++++++++++++++++++++++++++++++++++++++++ setup.cfg | 4 +++- 3 files changed, 47 insertions(+), 2 deletions(-) diff --git a/docker/version.py b/docker/version.py index dea7b7cb01..40accf1388 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "1.10.0-dev" +version = "1.10.0" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) diff --git a/docs/change_log.md b/docs/change_log.md index 089c003461..d37e48fd91 100644 --- a/docs/change_log.md +++ b/docs/change_log.md @@ -1,6 +1,49 @@ Change Log ========== +1.10.0 +------ + +[List of PRs / issues for this release](https://github.com/docker/docker-py/issues?q=milestone%3A1.10.0+is%3Aclosed) + +### Features + +* Added swarm mode and service management methods. See the documentation for + details. +* Added support for IPv6 Docker host addresses in the `Client` constructor. +* Added (read-only) support for the Docker credentials store. +* Added support for custom `auth_config` in `Client.push`. +* Added support for `labels` in `Client.create_volume`. +* Added support for `labels` and `enable_ipv6` in `Client.create_network`. +* Added support for `force` param in + `Client.disconnect_container_from_network`. +* Added support for `pids_limit`, `sysctls`, `userns_mode`, `cpuset_cpus`, + `cpu_shares`, `mem_reservation` and `kernel_memory` parameters in + `Client.create_host_config`. +* Added support for `link_local_ips` in `create_endpoint_config`. +* Added support for a `changes` parameter in `Client.import_image`. +* Added support for a `version` parameter in `Client.from_env`. + +### Bugfixes + +* Fixed a bug where `Client.build` would crash if the `config.json` file + contained a `HttpHeaders` entry. +* Fixed a bug where passing `decode=True` in some streaming methods would + crash when the daemon's response had an unexpected format. +* Fixed a bug where `environment` values with unicode characters weren't + handled properly in `create_container`. +* Fixed a bug where using the `npipe` protocol would sometimes break with + `ValueError: buffer size must be strictly positive`. + +### Miscellaneous + +* Fixed an issue where URL-quoting in docker-py was inconsistent with the + quoting done by the Docker CLI client. +* The client now sends TCP upgrade headers to hint potential proxies about + connection hijacking. +* The client now defaults to using the `npipe` protocol on Windows. + + 1.9.0 ----- diff --git a/setup.cfg b/setup.cfg index ccc93cfcbe..ad388d24d3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,3 +1,5 @@ [bdist_wheel] - universal = 1 + +[metadata] +description_file = README.md From 72e7afe17a9aa8b002a726377ce86cc9e13f6f35 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 9 Sep 2016 16:43:54 -0700 Subject: [PATCH 0106/1301] dev version Signed-off-by: Joffrey F --- docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index 40accf1388..3bbd804566 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "1.10.0" +version = "1.11.0-dev" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) From a6fb7a2064fbb2fd9b5302048ab758d44bbed358 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 9 Sep 2016 16:44:55 -0700 Subject: [PATCH 0107/1301] Re-add docker.utils.types module for backwards compatibility Signed-off-by: Joffrey F --- docker/utils/types.py | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 docker/utils/types.py diff --git a/docker/utils/types.py b/docker/utils/types.py new file mode 100644 index 0000000000..8098c470f8 --- /dev/null +++ b/docker/utils/types.py @@ -0,0 +1,7 @@ +# Compatibility module. See https://github.com/docker/docker-py/issues/1196 + +import warnings + +from ..types import Ulimit, LogConfig # flake8: noqa + +warnings.warn('docker.utils.types is now docker.types', ImportWarning) From 7167189486cb9bbb8148b80ee6fa5e53e655fca4 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 9 Sep 2016 16:55:10 -0700 Subject: [PATCH 0108/1301] Bump version + Update Changelog Signed-off-by: Joffrey F --- docker/version.py | 2 +- docs/change_log.md | 11 +++++++++++ setup.cfg | 2 +- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/docker/version.py b/docker/version.py index 40accf1388..1bc9811310 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "1.10.0" +version = "1.10.1" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) diff --git a/docs/change_log.md b/docs/change_log.md index d37e48fd91..0ef2787e72 100644 --- a/docs/change_log.md +++ b/docs/change_log.md @@ -1,6 +1,17 @@ Change Log ========== +1.10.1 +------ + +[List of PRs / issues for this release](https://github.com/docker/docker-py/issues?q=milestone%3A1.10.0+is%3Aclosed) + +### Bugfixes + +* The docker.utils.types module was removed in favor of docker.types, but some + applications imported it explicitly. It has been re-added with an import + warning advising to use the new module path. + 1.10.0 ------ diff --git a/setup.cfg b/setup.cfg index ad388d24d3..00b8f37d4a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -2,4 +2,4 @@ universal = 1 [metadata] -description_file = README.md +description_file = README.rst From 8abb8eecfca921977042a2c812b85f0da909ca2e Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 9 Sep 2016 16:44:55 -0700 Subject: [PATCH 0109/1301] Re-add docker.utils.types module for backwards compatibility Signed-off-by: Joffrey F --- docker/utils/types.py | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 docker/utils/types.py diff --git a/docker/utils/types.py b/docker/utils/types.py new file mode 100644 index 0000000000..8098c470f8 --- /dev/null +++ b/docker/utils/types.py @@ -0,0 +1,7 @@ +# Compatibility module. See https://github.com/docker/docker-py/issues/1196 + +import warnings + +from ..types import Ulimit, LogConfig # flake8: noqa + +warnings.warn('docker.utils.types is now docker.types', ImportWarning) From 2ad403c78b05ecae9f518282dc87fd4440f23cf8 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Sun, 11 Sep 2016 19:01:39 -0700 Subject: [PATCH 0110/1301] Bump docker-pycreds dependency ; bump patch number Signed-off-by: Joffrey F --- docker/version.py | 2 +- docs/change_log.md | 11 +++++++++++ requirements.txt | 2 +- setup.py | 2 +- 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/docker/version.py b/docker/version.py index 1bc9811310..730a834ef6 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "1.10.1" +version = "1.10.2" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) diff --git a/docs/change_log.md b/docs/change_log.md index 0ef2787e72..237770ff82 100644 --- a/docs/change_log.md +++ b/docs/change_log.md @@ -1,6 +1,17 @@ Change Log ========== +1.10.2 +------ + +[List of PRs / issues for this release](https://github.com/docker/docker-py/issues?q=milestone%3A1.10.0+is%3Aclosed) + +### Bugfixes + +* Updated the docker-pycreds dependency as it was causing issues for some + users with dependency resolution in applications using docker-py. + + 1.10.1 ------ diff --git a/requirements.txt b/requirements.txt index 4c0d5c203a..1e5284600f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,4 @@ six>=1.4.0 websocket-client==0.32.0 backports.ssl_match_hostname>=3.5 ; python_version < '3.5' ipaddress==1.0.16 ; python_version < '3.3' -docker-pycreds==0.2.0 \ No newline at end of file +docker-pycreds==0.2.1 \ No newline at end of file diff --git a/setup.py b/setup.py index 1afd873b50..9233ac2a8a 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ 'requests >= 2.5.2, < 2.11', 'six >= 1.4.0', 'websocket-client >= 0.32.0', - 'docker-pycreds >= 0.2.0' + 'docker-pycreds >= 0.2.1' ] if sys.platform == 'win32': From be7d0f01844d5c08ee157446ce96f5bc6381507c Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 12 Sep 2016 17:43:50 -0700 Subject: [PATCH 0111/1301] Number of pools in adapter is configurable Default increased from 10 to 25 Signed-off-by: Joffrey F --- docker/client.py | 15 +++++++++++---- docker/constants.py | 1 + docker/transport/npipeconn.py | 10 ++++++---- docker/transport/unixconn.py | 20 +++++++++++++------- 4 files changed, 31 insertions(+), 15 deletions(-) diff --git a/docker/client.py b/docker/client.py index 6e8b2789d4..47ad09e9a5 100644 --- a/docker/client.py +++ b/docker/client.py @@ -40,7 +40,8 @@ class Client( api.VolumeApiMixin): def __init__(self, base_url=None, version=None, timeout=constants.DEFAULT_TIMEOUT_SECONDS, tls=False, - user_agent=constants.DEFAULT_USER_AGENT): + user_agent=constants.DEFAULT_USER_AGENT, + num_pools=constants.DEFAULT_NUM_POOLS): super(Client, self).__init__() if tls and not base_url: @@ -58,7 +59,9 @@ def __init__(self, base_url=None, version=None, base_url, constants.IS_WINDOWS_PLATFORM, tls=bool(tls) ) if base_url.startswith('http+unix://'): - self._custom_adapter = UnixAdapter(base_url, timeout) + self._custom_adapter = UnixAdapter( + base_url, timeout, num_pools=num_pools + ) self.mount('http+docker://', self._custom_adapter) self._unmount('http://', 'https://') self.base_url = 'http+docker://localunixsocket' @@ -68,7 +71,9 @@ def __init__(self, base_url=None, version=None, 'The npipe:// protocol is only supported on Windows' ) try: - self._custom_adapter = NpipeAdapter(base_url, timeout) + self._custom_adapter = NpipeAdapter( + base_url, timeout, num_pools=num_pools + ) except NameError: raise errors.DockerException( 'Install pypiwin32 package to enable npipe:// support' @@ -80,7 +85,9 @@ def __init__(self, base_url=None, version=None, if isinstance(tls, TLSConfig): tls.configure_client(self) elif tls: - self._custom_adapter = ssladapter.SSLAdapter() + self._custom_adapter = ssladapter.SSLAdapter( + num_pools=num_pools + ) self.mount('https://', self._custom_adapter) self.base_url = base_url diff --git a/docker/constants.py b/docker/constants.py index cf5a39acdd..0c9a0205c2 100644 --- a/docker/constants.py +++ b/docker/constants.py @@ -15,3 +15,4 @@ IS_WINDOWS_PLATFORM = (sys.platform == 'win32') DEFAULT_USER_AGENT = "docker-py/{0}".format(version) +DEFAULT_NUM_POOLS = 25 diff --git a/docker/transport/npipeconn.py b/docker/transport/npipeconn.py index 736ddf675c..917fa8b3bf 100644 --- a/docker/transport/npipeconn.py +++ b/docker/transport/npipeconn.py @@ -1,6 +1,7 @@ import six import requests.adapters +from .. import constants from .npipesocket import NpipeSocket if six.PY3: @@ -33,9 +34,9 @@ def connect(self): class NpipeHTTPConnectionPool(urllib3.connectionpool.HTTPConnectionPool): - def __init__(self, npipe_path, timeout=60): + def __init__(self, npipe_path, timeout=60, maxsize=10): super(NpipeHTTPConnectionPool, self).__init__( - 'localhost', timeout=timeout + 'localhost', timeout=timeout, maxsize=maxsize ) self.npipe_path = npipe_path self.timeout = timeout @@ -47,11 +48,12 @@ def _new_conn(self): class NpipeAdapter(requests.adapters.HTTPAdapter): - def __init__(self, base_url, timeout=60): + def __init__(self, base_url, timeout=60, + num_pools=constants.DEFAULT_NUM_POOLS): self.npipe_path = base_url.replace('npipe://', '') self.timeout = timeout self.pools = RecentlyUsedContainer( - 10, dispose_func=lambda p: p.close() + num_pools, dispose_func=lambda p: p.close() ) super(NpipeAdapter, self).__init__() diff --git a/docker/transport/unixconn.py b/docker/transport/unixconn.py index e09b6bfa2c..b7905a042e 100644 --- a/docker/transport/unixconn.py +++ b/docker/transport/unixconn.py @@ -2,6 +2,8 @@ import requests.adapters import socket +from .. import constants + if six.PY3: import http.client as httplib else: @@ -12,6 +14,7 @@ except ImportError: import urllib3 + RecentlyUsedContainer = urllib3._collections.RecentlyUsedContainer @@ -32,28 +35,31 @@ def connect(self): class UnixHTTPConnectionPool(urllib3.connectionpool.HTTPConnectionPool): - def __init__(self, base_url, socket_path, timeout=60): + def __init__(self, base_url, socket_path, timeout=60, maxsize=10): super(UnixHTTPConnectionPool, self).__init__( - 'localhost', timeout=timeout + 'localhost', timeout=timeout, maxsize=maxsize ) self.base_url = base_url self.socket_path = socket_path self.timeout = timeout def _new_conn(self): - return UnixHTTPConnection(self.base_url, self.socket_path, - self.timeout) + return UnixHTTPConnection( + self.base_url, self.socket_path, self.timeout + ) class UnixAdapter(requests.adapters.HTTPAdapter): - def __init__(self, socket_url, timeout=60): + def __init__(self, socket_url, timeout=60, + num_pools=constants.DEFAULT_NUM_POOLS): socket_path = socket_url.replace('http+unix://', '') if not socket_path.startswith('/'): socket_path = '/' + socket_path self.socket_path = socket_path self.timeout = timeout - self.pools = RecentlyUsedContainer(10, - dispose_func=lambda p: p.close()) + self.pools = RecentlyUsedContainer( + num_pools, dispose_func=lambda p: p.close() + ) super(UnixAdapter, self).__init__() def get_connection(self, url, proxies=None): From dcd01f0f48525e4d5bb4953f1f57a7a65e424561 Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Wed, 14 Sep 2016 14:53:04 +0100 Subject: [PATCH 0112/1301] Parse JSON API errors Signed-off-by: Ben Firshman --- docker/errors.py | 5 ++++- tests/integration/container_test.py | 4 ++-- tests/integration/errors_test.py | 17 +++++++++++++++++ 3 files changed, 23 insertions(+), 3 deletions(-) create mode 100644 tests/integration/errors_test.py diff --git a/docker/errors.py b/docker/errors.py index 97be802d0a..df18d57830 100644 --- a/docker/errors.py +++ b/docker/errors.py @@ -11,7 +11,10 @@ def __init__(self, message, response, explanation=None): self.explanation = explanation if self.explanation is None and response.content: - self.explanation = response.content.strip() + try: + self.explanation = response.json()['message'] + except ValueError: + self.explanation = response.content.strip() def __str__(self): message = super(APIError, self).__str__() diff --git a/tests/integration/container_test.py b/tests/integration/container_test.py index 27d3046bbc..a7267efb14 100644 --- a/tests/integration/container_test.py +++ b/tests/integration/container_test.py @@ -118,7 +118,7 @@ def test_create_with_restart_policy(self): self.client.wait(id) with self.assertRaises(docker.errors.APIError) as exc: self.client.remove_container(id) - err = exc.exception.response.text + err = exc.exception.explanation self.assertIn( 'You cannot remove a running container', err ) @@ -289,7 +289,7 @@ def test_invalid_log_driver_raises_exception(self): ) self.client.start(container) - assert six.b(expected_msg) in excinfo.value.explanation + assert excinfo.value.explanation == expected_msg def test_valid_no_log_driver_specified(self): log_config = docker.utils.LogConfig( diff --git a/tests/integration/errors_test.py b/tests/integration/errors_test.py new file mode 100644 index 0000000000..42fbae4af3 --- /dev/null +++ b/tests/integration/errors_test.py @@ -0,0 +1,17 @@ +from docker.errors import APIError +from .. import helpers + + +class ErrorsTest(helpers.BaseTestCase): + def test_api_error_parses_json(self): + container = self.client.create_container( + helpers.BUSYBOX, + ['sleep', '10'] + ) + self.client.start(container['Id']) + with self.assertRaises(APIError) as cm: + self.client.remove_container(container['Id']) + explanation = cm.exception.explanation + assert 'You cannot remove a running container' in explanation + assert '{"message":' not in explanation + self.client.remove_container(container['Id'], force=True) From ca51ad29a504f664d673e92f9ab8e7d94b50d111 Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Wed, 14 Sep 2016 14:25:30 +0100 Subject: [PATCH 0113/1301] Ignore not in swarm error when force leaving Real errors were getting swallowed in these tests, producing other confusing cascading errors. This makes it much easier to make sure a node is not in a Swarm, while also handling other errors correctly. Signed-off-by: Ben Firshman --- docker/api/swarm.py | 6 +++++- tests/integration/service_test.py | 10 ++-------- tests/integration/swarm_test.py | 11 +++-------- 3 files changed, 10 insertions(+), 17 deletions(-) diff --git a/docker/api/swarm.py b/docker/api/swarm.py index d099364552..7481c67532 100644 --- a/docker/api/swarm.py +++ b/docker/api/swarm.py @@ -1,5 +1,6 @@ -from .. import utils import logging +from six.moves import http_client +from .. import utils log = logging.getLogger(__name__) @@ -53,6 +54,9 @@ def join_swarm(self, remote_addrs, join_token, listen_addr=None, def leave_swarm(self, force=False): url = self._url('/swarm/leave') response = self._post(url, params={'force': force}) + # Ignore "this node is not part of a swarm" error + if force and response.status_code == http_client.NOT_ACCEPTABLE: + return True self._raise_for_status(response) return True diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 2b99316bae..35438940b1 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -12,10 +12,7 @@ class ServiceTest(helpers.BaseTestCase): def setUp(self): super(ServiceTest, self).setUp() - try: - self.client.leave_swarm(force=True) - except docker.errors.APIError: - pass + self.client.leave_swarm(force=True) self.client.init_swarm('eth0') def tearDown(self): @@ -25,10 +22,7 @@ def tearDown(self): self.client.remove_service(service['ID']) except docker.errors.APIError: pass - try: - self.client.leave_swarm(force=True) - except docker.errors.APIError: - pass + self.client.leave_swarm(force=True) def get_service_name(self): return 'dockerpytest_{0:x}'.format(random.getrandbits(64)) diff --git a/tests/integration/swarm_test.py b/tests/integration/swarm_test.py index 128628e618..8c62f2ec06 100644 --- a/tests/integration/swarm_test.py +++ b/tests/integration/swarm_test.py @@ -11,17 +11,11 @@ class SwarmTest(helpers.BaseTestCase): def setUp(self): super(SwarmTest, self).setUp() - try: - self.client.leave_swarm(force=True) - except docker.errors.APIError: - pass + self.client.leave_swarm(force=True) def tearDown(self): super(SwarmTest, self).tearDown() - try: - self.client.leave_swarm(force=True) - except docker.errors.APIError: - pass + self.client.leave_swarm(force=True) @requires_api_version('1.24') def test_init_swarm_simple(self): @@ -65,6 +59,7 @@ def test_leave_swarm(self): with pytest.raises(docker.errors.APIError) as exc_info: self.client.inspect_swarm() exc_info.value.response.status_code == 406 + assert self.client.leave_swarm(force=True) @requires_api_version('1.24') def test_update_swarm(self): From 6220636536963680a5808dca2943cbc62a2fd409 Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Wed, 14 Sep 2016 16:54:54 +0100 Subject: [PATCH 0114/1301] Fix a few pep8 issues autopep8 --in-place --recursive --experimental -aaa --ignore E309 . Signed-off-by: Ben Firshman --- docker/__init__.py | 2 +- docker/auth/__init__.py | 2 +- docker/ssladapter/__init__.py | 2 +- docker/ssladapter/ssladapter.py | 1 + docker/tls.py | 2 +- docker/transport/__init__.py | 2 +- docker/transport/npipesocket.py | 1 + docker/types/swarm.py | 7 +++++-- docker/utils/ports/__init__.py | 2 +- tests/unit/api_test.py | 2 +- tests/unit/image_test.py | 6 +++--- 11 files changed, 17 insertions(+), 12 deletions(-) diff --git a/docker/__init__.py b/docker/__init__.py index ad53805e7f..0f4c8ecd8f 100644 --- a/docker/__init__.py +++ b/docker/__init__.py @@ -3,4 +3,4 @@ __version__ = version __title__ = 'docker-py' -from .client import Client, AutoVersionClient, from_env # flake8: noqa +from .client import Client, AutoVersionClient, from_env # flake8: noqa diff --git a/docker/auth/__init__.py b/docker/auth/__init__.py index 6fc83f83c9..50127fac10 100644 --- a/docker/auth/__init__.py +++ b/docker/auth/__init__.py @@ -5,4 +5,4 @@ load_config, resolve_authconfig, resolve_repository_name, -) # flake8: noqa \ No newline at end of file +) # flake8: noqa diff --git a/docker/ssladapter/__init__.py b/docker/ssladapter/__init__.py index 1a5e1bb6d4..31b8966b01 100644 --- a/docker/ssladapter/__init__.py +++ b/docker/ssladapter/__init__.py @@ -1 +1 @@ -from .ssladapter import SSLAdapter # flake8: noqa +from .ssladapter import SSLAdapter # flake8: noqa diff --git a/docker/ssladapter/ssladapter.py b/docker/ssladapter/ssladapter.py index e17dfad976..31f45fc459 100644 --- a/docker/ssladapter/ssladapter.py +++ b/docker/ssladapter/ssladapter.py @@ -24,6 +24,7 @@ class SSLAdapter(HTTPAdapter): '''An HTTPS Transport Adapter that uses an arbitrary SSL version.''' + def __init__(self, ssl_version=None, assert_hostname=None, assert_fingerprint=None, **kwargs): self.ssl_version = ssl_version diff --git a/docker/tls.py b/docker/tls.py index 7abfa60e1d..18c725987c 100644 --- a/docker/tls.py +++ b/docker/tls.py @@ -42,7 +42,7 @@ def __init__(self, client_cert=None, ca_cert=None, verify=None, ) if not (tls_cert and tls_key) or (not os.path.isfile(tls_cert) or - not os.path.isfile(tls_key)): + not os.path.isfile(tls_key)): raise errors.TLSParameterError( 'Path to a certificate and key files must be provided' ' through the client_config param' diff --git a/docker/transport/__init__.py b/docker/transport/__init__.py index d647483e2a..04a46d9883 100644 --- a/docker/transport/__init__.py +++ b/docker/transport/__init__.py @@ -3,4 +3,4 @@ try: from .npipeconn import NpipeAdapter except ImportError: - pass \ No newline at end of file + pass diff --git a/docker/transport/npipesocket.py b/docker/transport/npipesocket.py index 9010cebe13..d3847c3f56 100644 --- a/docker/transport/npipesocket.py +++ b/docker/transport/npipesocket.py @@ -26,6 +26,7 @@ class NpipeSocket(object): and server-specific methods (bind, listen, accept...) are not implemented. """ + def __init__(self, handle=None): self._timeout = win32pipe.NMPWAIT_USE_DEFAULT_WAIT self._handle = handle diff --git a/docker/types/swarm.py b/docker/types/swarm.py index 865fde6203..49beaa11f7 100644 --- a/docker/types/swarm.py +++ b/docker/types/swarm.py @@ -8,8 +8,11 @@ def __init__(self, task_history_retention_limit=None, self['Orchestration'] = { 'TaskHistoryRetentionLimit': task_history_retention_limit } - if any([snapshot_interval, keep_old_snapshots, - log_entries_for_slow_followers, heartbeat_tick, election_tick]): + if any([snapshot_interval, + keep_old_snapshots, + log_entries_for_slow_followers, + heartbeat_tick, + election_tick]): self['Raft'] = { 'SnapshotInterval': snapshot_interval, 'KeepOldSnapshots': keep_old_snapshots, diff --git a/docker/utils/ports/__init__.py b/docker/utils/ports/__init__.py index 1dbfa3a709..485feec06e 100644 --- a/docker/utils/ports/__init__.py +++ b/docker/utils/ports/__init__.py @@ -1,4 +1,4 @@ from .ports import ( split_port, build_port_bindings -) # flake8: noqa +) # flake8: noqa diff --git a/tests/unit/api_test.py b/tests/unit/api_test.py index 389b5f5360..c9706fbb4c 100644 --- a/tests/unit/api_test.py +++ b/tests/unit/api_test.py @@ -327,7 +327,7 @@ def test_stream_helper_decoding(self): # mock a stream interface raw_resp = urllib3.HTTPResponse(body=body) setattr(raw_resp._fp, 'chunked', True) - setattr(raw_resp._fp, 'chunk_left', len(body.getvalue())-1) + setattr(raw_resp._fp, 'chunk_left', len(body.getvalue()) - 1) # pass `decode=False` to the helper raw_resp._fp.seek(0) diff --git a/tests/unit/image_test.py b/tests/unit/image_test.py index b2b1dd6d90..cca519e38e 100644 --- a/tests/unit/image_test.py +++ b/tests/unit/image_test.py @@ -271,9 +271,9 @@ def test_push_image_with_auth(self): } encoded_auth = auth.encode_header(auth_config) self.client.push( - fake_api.FAKE_IMAGE_NAME, tag=fake_api.FAKE_TAG_NAME, - auth_config=auth_config - ) + fake_api.FAKE_IMAGE_NAME, tag=fake_api.FAKE_TAG_NAME, + auth_config=auth_config + ) fake_request.assert_called_with( 'POST', From 1a57f8800e4ef435392123321512ea57ebee5846 Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Wed, 14 Sep 2016 12:37:26 +0100 Subject: [PATCH 0115/1301] Add file arg to integration tests make integration-test file=models_services_test.py Signed-off-by: Ben Firshman --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 1fb21d44ed..db19709328 100644 --- a/Makefile +++ b/Makefile @@ -35,11 +35,11 @@ unit-test-py3: build-py3 .PHONY: integration-test integration-test: build - docker run -v /var/run/docker.sock:/var/run/docker.sock docker-py py.test tests/integration + docker run -v /var/run/docker.sock:/var/run/docker.sock docker-py py.test tests/integration/${file} .PHONY: integration-test-py3 integration-test-py3: build-py3 - docker run -v /var/run/docker.sock:/var/run/docker.sock docker-py3 py.test tests/integration + docker run -v /var/run/docker.sock:/var/run/docker.sock docker-py3 py.test tests/integration/${file} .PHONY: integration-dind integration-dind: build build-py3 From 71b0b7761a42464663d798a09b6f6adf6ad1a52d Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Wed, 14 Sep 2016 12:38:04 +0100 Subject: [PATCH 0116/1301] Add make shell to open a Python shell Signed-off-by: Ben Firshman --- Makefile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Makefile b/Makefile index db19709328..b997722c7a 100644 --- a/Makefile +++ b/Makefile @@ -75,3 +75,7 @@ flake8: build .PHONY: docs docs: build-docs docker run -v `pwd`/docs:/home/docker-py/docs/ -p 8000:8000 docker-py-docs mkdocs serve -a 0.0.0.0:8000 + +.PHONY: shell +shell: build + docker run -it -v /var/run/docker.sock:/var/run/docker.sock docker-py python From d731a4315c44974954eef503fd23c512b912296d Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 13 Sep 2016 17:24:02 -0700 Subject: [PATCH 0117/1301] Add support for identity tokens in config file. Signed-off-by: Joffrey F --- docker/auth/auth.py | 13 ++++++++++++- tests/unit/auth_test.py | 26 +++++++++++++++++++++++++- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/docker/auth/auth.py b/docker/auth/auth.py index ea15def80a..dc0baea80f 100644 --- a/docker/auth/auth.py +++ b/docker/auth/auth.py @@ -174,6 +174,15 @@ def parse_auth(entries, raise_on_error=False): 'Invalid configuration for registry {0}'.format(registry) ) return {} + if 'identitytoken' in entry: + log.debug('Found an IdentityToken entry for registry {0}'.format( + registry + )) + conf[registry] = { + 'IdentityToken': entry['identitytoken'] + } + continue # Other values are irrelevant if we have a token, skip. + if 'auth' not in entry: # Starting with engine v1.11 (API 1.23), an empty dictionary is # a valid value in the auths config. @@ -182,13 +191,15 @@ def parse_auth(entries, raise_on_error=False): 'Auth data for {0} is absent. Client might be using a ' 'credentials store instead.' ) - return {} + conf[registry] = {} + continue username, password = decode_auth(entry['auth']) log.debug( 'Found entry (registry={0}, username={1})' .format(repr(registry), repr(username)) ) + conf[registry] = { 'username': username, 'password': password, diff --git a/tests/unit/auth_test.py b/tests/unit/auth_test.py index 4ea40477ce..f3951335f4 100644 --- a/tests/unit/auth_test.py +++ b/tests/unit/auth_test.py @@ -460,4 +460,28 @@ def test_load_config_invalid_auth_dict(self): json.dump(config, f) cfg = auth.load_config(dockercfg_path) - assert cfg == {} + assert cfg == {'scarlet.net': {}} + + def test_load_config_identity_token(self): + folder = tempfile.mkdtemp() + registry = 'scarlet.net' + token = '1ce1cebb-503e-7043-11aa-7feb8bd4a1ce' + self.addCleanup(shutil.rmtree, folder) + dockercfg_path = os.path.join(folder, 'config.json') + auth_entry = encode_auth({'username': 'sakuya'}).decode('ascii') + config = { + 'auths': { + registry: { + 'auth': auth_entry, + 'identitytoken': token + } + } + } + with open(dockercfg_path, 'w') as f: + json.dump(config, f) + + cfg = auth.load_config(dockercfg_path) + assert registry in cfg + cfg = cfg[registry] + assert 'IdentityToken' in cfg + assert cfg['IdentityToken'] == token From e4cf97bc7b436072e61c21ee2494c23a1400ec7c Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 16 Sep 2016 11:54:06 -0700 Subject: [PATCH 0118/1301] Bump version Update Changelog Signed-off-by: Joffrey F --- docker/version.py | 2 +- docs/change_log.md | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index 730a834ef6..2bf8436d36 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "1.10.2" +version = "1.10.3" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) diff --git a/docs/change_log.md b/docs/change_log.md index 237770ff82..e32df1e974 100644 --- a/docs/change_log.md +++ b/docs/change_log.md @@ -1,6 +1,23 @@ Change Log ========== +1.10.3 +------ + +[List of PRs / issues for this release](https://github.com/docker/docker-py/issues?q=milestone%3A1.10.3+is%3Aclosed) + +### Bugfixes + +* Fixed an issue where identity tokens in configuration files weren't handled + by the library. + +### Miscellaneous + +* Increased the default number of connection pools from 10 to 25. This number + can now be configured using the `num_pools` parameter in the `Client` + constructor. + + 1.10.2 ------ From 06b6a62faab560ce7f07084d63294261db8bbca8 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 13 Sep 2016 17:24:02 -0700 Subject: [PATCH 0119/1301] Add support for identity tokens in config file. Signed-off-by: Joffrey F --- docker/auth/auth.py | 13 ++++++++++++- tests/unit/auth_test.py | 26 +++++++++++++++++++++++++- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/docker/auth/auth.py b/docker/auth/auth.py index ea15def80a..dc0baea80f 100644 --- a/docker/auth/auth.py +++ b/docker/auth/auth.py @@ -174,6 +174,15 @@ def parse_auth(entries, raise_on_error=False): 'Invalid configuration for registry {0}'.format(registry) ) return {} + if 'identitytoken' in entry: + log.debug('Found an IdentityToken entry for registry {0}'.format( + registry + )) + conf[registry] = { + 'IdentityToken': entry['identitytoken'] + } + continue # Other values are irrelevant if we have a token, skip. + if 'auth' not in entry: # Starting with engine v1.11 (API 1.23), an empty dictionary is # a valid value in the auths config. @@ -182,13 +191,15 @@ def parse_auth(entries, raise_on_error=False): 'Auth data for {0} is absent. Client might be using a ' 'credentials store instead.' ) - return {} + conf[registry] = {} + continue username, password = decode_auth(entry['auth']) log.debug( 'Found entry (registry={0}, username={1})' .format(repr(registry), repr(username)) ) + conf[registry] = { 'username': username, 'password': password, diff --git a/tests/unit/auth_test.py b/tests/unit/auth_test.py index 4ea40477ce..f3951335f4 100644 --- a/tests/unit/auth_test.py +++ b/tests/unit/auth_test.py @@ -460,4 +460,28 @@ def test_load_config_invalid_auth_dict(self): json.dump(config, f) cfg = auth.load_config(dockercfg_path) - assert cfg == {} + assert cfg == {'scarlet.net': {}} + + def test_load_config_identity_token(self): + folder = tempfile.mkdtemp() + registry = 'scarlet.net' + token = '1ce1cebb-503e-7043-11aa-7feb8bd4a1ce' + self.addCleanup(shutil.rmtree, folder) + dockercfg_path = os.path.join(folder, 'config.json') + auth_entry = encode_auth({'username': 'sakuya'}).decode('ascii') + config = { + 'auths': { + registry: { + 'auth': auth_entry, + 'identitytoken': token + } + } + } + with open(dockercfg_path, 'w') as f: + json.dump(config, f) + + cfg = auth.load_config(dockercfg_path) + assert registry in cfg + cfg = cfg[registry] + assert 'IdentityToken' in cfg + assert cfg['IdentityToken'] == token From 64fba723ddd1f186548a3a7f49ca952265ac1121 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 12 Sep 2016 17:43:50 -0700 Subject: [PATCH 0120/1301] Number of pools in adapter is configurable Default increased from 10 to 25 Signed-off-by: Joffrey F --- docker/client.py | 15 +++++++++++---- docker/constants.py | 1 + docker/transport/npipeconn.py | 10 ++++++---- docker/transport/unixconn.py | 20 +++++++++++++------- 4 files changed, 31 insertions(+), 15 deletions(-) diff --git a/docker/client.py b/docker/client.py index 6e8b2789d4..47ad09e9a5 100644 --- a/docker/client.py +++ b/docker/client.py @@ -40,7 +40,8 @@ class Client( api.VolumeApiMixin): def __init__(self, base_url=None, version=None, timeout=constants.DEFAULT_TIMEOUT_SECONDS, tls=False, - user_agent=constants.DEFAULT_USER_AGENT): + user_agent=constants.DEFAULT_USER_AGENT, + num_pools=constants.DEFAULT_NUM_POOLS): super(Client, self).__init__() if tls and not base_url: @@ -58,7 +59,9 @@ def __init__(self, base_url=None, version=None, base_url, constants.IS_WINDOWS_PLATFORM, tls=bool(tls) ) if base_url.startswith('http+unix://'): - self._custom_adapter = UnixAdapter(base_url, timeout) + self._custom_adapter = UnixAdapter( + base_url, timeout, num_pools=num_pools + ) self.mount('http+docker://', self._custom_adapter) self._unmount('http://', 'https://') self.base_url = 'http+docker://localunixsocket' @@ -68,7 +71,9 @@ def __init__(self, base_url=None, version=None, 'The npipe:// protocol is only supported on Windows' ) try: - self._custom_adapter = NpipeAdapter(base_url, timeout) + self._custom_adapter = NpipeAdapter( + base_url, timeout, num_pools=num_pools + ) except NameError: raise errors.DockerException( 'Install pypiwin32 package to enable npipe:// support' @@ -80,7 +85,9 @@ def __init__(self, base_url=None, version=None, if isinstance(tls, TLSConfig): tls.configure_client(self) elif tls: - self._custom_adapter = ssladapter.SSLAdapter() + self._custom_adapter = ssladapter.SSLAdapter( + num_pools=num_pools + ) self.mount('https://', self._custom_adapter) self.base_url = base_url diff --git a/docker/constants.py b/docker/constants.py index cf5a39acdd..0c9a0205c2 100644 --- a/docker/constants.py +++ b/docker/constants.py @@ -15,3 +15,4 @@ IS_WINDOWS_PLATFORM = (sys.platform == 'win32') DEFAULT_USER_AGENT = "docker-py/{0}".format(version) +DEFAULT_NUM_POOLS = 25 diff --git a/docker/transport/npipeconn.py b/docker/transport/npipeconn.py index 736ddf675c..917fa8b3bf 100644 --- a/docker/transport/npipeconn.py +++ b/docker/transport/npipeconn.py @@ -1,6 +1,7 @@ import six import requests.adapters +from .. import constants from .npipesocket import NpipeSocket if six.PY3: @@ -33,9 +34,9 @@ def connect(self): class NpipeHTTPConnectionPool(urllib3.connectionpool.HTTPConnectionPool): - def __init__(self, npipe_path, timeout=60): + def __init__(self, npipe_path, timeout=60, maxsize=10): super(NpipeHTTPConnectionPool, self).__init__( - 'localhost', timeout=timeout + 'localhost', timeout=timeout, maxsize=maxsize ) self.npipe_path = npipe_path self.timeout = timeout @@ -47,11 +48,12 @@ def _new_conn(self): class NpipeAdapter(requests.adapters.HTTPAdapter): - def __init__(self, base_url, timeout=60): + def __init__(self, base_url, timeout=60, + num_pools=constants.DEFAULT_NUM_POOLS): self.npipe_path = base_url.replace('npipe://', '') self.timeout = timeout self.pools = RecentlyUsedContainer( - 10, dispose_func=lambda p: p.close() + num_pools, dispose_func=lambda p: p.close() ) super(NpipeAdapter, self).__init__() diff --git a/docker/transport/unixconn.py b/docker/transport/unixconn.py index e09b6bfa2c..b7905a042e 100644 --- a/docker/transport/unixconn.py +++ b/docker/transport/unixconn.py @@ -2,6 +2,8 @@ import requests.adapters import socket +from .. import constants + if six.PY3: import http.client as httplib else: @@ -12,6 +14,7 @@ except ImportError: import urllib3 + RecentlyUsedContainer = urllib3._collections.RecentlyUsedContainer @@ -32,28 +35,31 @@ def connect(self): class UnixHTTPConnectionPool(urllib3.connectionpool.HTTPConnectionPool): - def __init__(self, base_url, socket_path, timeout=60): + def __init__(self, base_url, socket_path, timeout=60, maxsize=10): super(UnixHTTPConnectionPool, self).__init__( - 'localhost', timeout=timeout + 'localhost', timeout=timeout, maxsize=maxsize ) self.base_url = base_url self.socket_path = socket_path self.timeout = timeout def _new_conn(self): - return UnixHTTPConnection(self.base_url, self.socket_path, - self.timeout) + return UnixHTTPConnection( + self.base_url, self.socket_path, self.timeout + ) class UnixAdapter(requests.adapters.HTTPAdapter): - def __init__(self, socket_url, timeout=60): + def __init__(self, socket_url, timeout=60, + num_pools=constants.DEFAULT_NUM_POOLS): socket_path = socket_url.replace('http+unix://', '') if not socket_path.startswith('/'): socket_path = '/' + socket_path self.socket_path = socket_path self.timeout = timeout - self.pools = RecentlyUsedContainer(10, - dispose_func=lambda p: p.close()) + self.pools = RecentlyUsedContainer( + num_pools, dispose_func=lambda p: p.close() + ) super(UnixAdapter, self).__init__() def get_connection(self, url, proxies=None): From cbd2ba52af076219198a002533ac75fcd75a1ca3 Mon Sep 17 00:00:00 2001 From: Sebastian Schwarz Date: Wed, 10 Feb 2016 13:31:46 +0100 Subject: [PATCH 0121/1301] Synthesize executable bit on Windows The build context is tarred up on the client and then sent to the Docker daemon. However Windows permissions don't match the Unix ones. Therefore we have to mark all files as executable when creating a build context on Windows, like `docker build` already does: https://github.com/docker/docker/issues/11047. Signed-off-by: Sebastian Schwarz --- docker/utils/utils.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docker/utils/utils.py b/docker/utils/utils.py index d46f8fcdbb..62e06a2d58 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -4,6 +4,7 @@ import os.path import json import shlex +import sys import tarfile import tempfile import warnings @@ -92,7 +93,10 @@ def tar(path, exclude=None, dockerfile=None, fileobj=None, gzip=False): exclude = exclude or [] for path in sorted(exclude_paths(root, exclude, dockerfile=dockerfile)): - t.add(os.path.join(root, path), arcname=path, recursive=False) + i = t.gettarinfo(os.path.join(root, path), arcname=path) + if sys.platform == 'win32': + i.mode = i.mode & 0o755 | 0o111 + t.addfile(i) t.close() fileobj.seek(0) From 6ef14932d0f33583bd5de82affd64d639a6fd5e3 Mon Sep 17 00:00:00 2001 From: Nathan Shirlberg Date: Fri, 23 Sep 2016 15:07:59 -0500 Subject: [PATCH 0122/1301] enable setting of node labels #1225 Added update_node function to enable setting labels on nodes. This exposes the Update a Node function from the Docker API and should enable promoting/demoting manager nodes inside a swarm. Signed-off-by: Nathan Shirlberg --- docker/api/swarm.py | 7 +++++++ docs/api.md | 5 +++++ docs/swarm.md | 24 ++++++++++++++++++++++++ tests/integration/swarm_test.py | 24 ++++++++++++++++++++++++ tests/unit/fake_api.py | 9 +++++++++ tests/unit/swarm_test.py | 32 ++++++++++++++++++++++++++++++++ 6 files changed, 101 insertions(+) create mode 100644 tests/unit/swarm_test.py diff --git a/docker/api/swarm.py b/docker/api/swarm.py index 7481c67532..2fc877448a 100644 --- a/docker/api/swarm.py +++ b/docker/api/swarm.py @@ -69,6 +69,13 @@ def nodes(self, filters=None): return self._result(self._get(url, params=params), True) + @utils.minimum_version('1.24') + def update_node(self, node_id, version, node_spec=None): + url = self._url('/nodes/{0}/update?version={1}', node_id, str(version)) + res = self._post_json(url, data=node_spec) + self._raise_for_status(res) + return True + @utils.minimum_version('1.24') def update_swarm(self, version, swarm_spec=None, rotate_worker_token=False, rotate_manager_token=False): diff --git a/docs/api.md b/docs/api.md index 1699344a66..5cadb83081 100644 --- a/docs/api.md +++ b/docs/api.md @@ -1129,6 +1129,11 @@ Update resource configs of one or more containers. **Returns** (dict): Dictionary containing a `Warnings` key. +## update_node + +Update a node. +See the [Swarm documentation](swarm.md#clientupdate_node). + ## update_service Update a service, similar to the `docker service update` command. See the diff --git a/docs/swarm.md b/docs/swarm.md index 3cc44f8741..20c3945352 100644 --- a/docs/swarm.md +++ b/docs/swarm.md @@ -232,6 +232,30 @@ List Swarm nodes **Returns:** A list of dictionaries containing data about each swarm node. +### Client.update_node + +Update the Node's configuration + +**Params:** + +* version (int): The version number of the node object being updated. This + is required to avoid conflicting writes. +* node_spec (dict): Configuration settings to update. Any values not provided + will be removed. See the official [Docker API documentation](https://docs.docker.com/engine/reference/api/docker_remote_api_v1.24/#/update-a-node) for more details. + Default: `None`. + +**Returns:** `True` if the request went through. Raises an `APIError` if it + fails. + +```python +node_spec = {'Availability': 'active', + 'Name': 'node-name', + 'Role': 'manager', + 'Labels': {'foo': 'bar'} + } +client.update_node(node_id='24ifsmvkjbyhk', version=8, node_spec=node_spec) +``` + ### Client.update_swarm Update the Swarm's configuration diff --git a/tests/integration/swarm_test.py b/tests/integration/swarm_test.py index 8c62f2ec06..7f02c71170 100644 --- a/tests/integration/swarm_test.py +++ b/tests/integration/swarm_test.py @@ -1,3 +1,4 @@ +import copy import docker import pytest @@ -138,3 +139,26 @@ def test_inspect_node(self): node_data = self.client.inspect_node(node['ID']) assert node['ID'] == node_data['ID'] assert node['Version'] == node_data['Version'] + + @requires_api_version('1.24') + def test_update_node(self): + assert self.client.init_swarm('eth0') + nodes_list = self.client.nodes() + node = nodes_list[0] + orig_spec = node['Spec'] + + # add a new label + new_spec = copy.deepcopy(orig_spec) + new_spec['Labels'] = {'new.label': 'new value'} + self.client.update_node(node_id=node['ID'], + version=node['Version']['Index'], + node_spec=new_spec) + updated_node = self.client.inspect_node(node['ID']) + assert new_spec == updated_node['Spec'] + + # Revert the changes + self.client.update_node(node_id=node['ID'], + version=updated_node['Version']['Index'], + node_spec=orig_spec) + reverted_node = self.client.inspect_node(node['ID']) + assert orig_spec == reverted_node['Spec'] diff --git a/tests/unit/fake_api.py b/tests/unit/fake_api.py index 1e9d318df5..cfe6ef777f 100644 --- a/tests/unit/fake_api.py +++ b/tests/unit/fake_api.py @@ -14,6 +14,7 @@ FAKE_URL = 'myurl' FAKE_PATH = '/path' FAKE_VOLUME_NAME = 'perfectcherryblossom' +FAKE_NODE_ID = '24ifsmvkjbyhk' # Each method is prefixed with HTTP method (get, post...) # for clarity and readability @@ -406,6 +407,10 @@ def post_fake_update_container(): return 200, {'Warnings': []} +def post_fake_update_node(): + return 200, None + + # Maps real api url to fake response callback prefix = 'http+docker://localunixsocket' fake_responses = { @@ -504,4 +509,8 @@ def post_fake_update_container(): CURRENT_VERSION, prefix, FAKE_VOLUME_NAME ), 'DELETE'): fake_remove_volume, + ('{1}/{0}/nodes/{2}/update?version=1'.format( + CURRENT_VERSION, prefix, FAKE_NODE_ID + ), 'POST'): + post_fake_update_node, } diff --git a/tests/unit/swarm_test.py b/tests/unit/swarm_test.py new file mode 100644 index 0000000000..5580383406 --- /dev/null +++ b/tests/unit/swarm_test.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- + +import json + +from . import fake_api +from ..base import requires_api_version +from .api_test import (DockerClientTest, url_prefix, fake_request) + + +class SwarmTest(DockerClientTest): + @requires_api_version('1.24') + def test_node_update(self): + node_spec = { + 'Availability': 'active', + 'Name': 'node-name', + 'Role': 'manager', + 'Labels': {'foo': 'bar'} + } + + self.client.update_node( + node_id=fake_api.FAKE_NODE_ID, version=1, node_spec=node_spec + ) + args = fake_request.call_args + self.assertEqual( + args[0][1], url_prefix + 'nodes/24ifsmvkjbyhk/update?version=1' + ) + self.assertEqual( + json.loads(args[1]['data']), node_spec + ) + self.assertEqual( + args[1]['headers']['Content-Type'], 'application/json' + ) From a718ab690e3b97c5c9ae4a9697ed2511f4c6f7dd Mon Sep 17 00:00:00 2001 From: Christian Bundy Date: Tue, 27 Sep 2016 00:56:12 +0000 Subject: [PATCH 0123/1301] Pass file object to Tarfile.addfile() This resolves an issue where TarFile.gettarinfo() doesn't include the file object, meaning that TarFile.addfile(TarFile.gettarinfo()) doesn't pass the test suite. Instead, this uses an open() within a try...except block to include a file object for each file without passing a file object when the path is a directory. Signed-off-by: Christian Bundy --- docker/utils/utils.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/docker/utils/utils.py b/docker/utils/utils.py index 62e06a2d58..b565732d6e 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -94,9 +94,20 @@ def tar(path, exclude=None, dockerfile=None, fileobj=None, gzip=False): for path in sorted(exclude_paths(root, exclude, dockerfile=dockerfile)): i = t.gettarinfo(os.path.join(root, path), arcname=path) + if sys.platform == 'win32': + # Windows doesn't keep track of the execute bit, so we make files + # and directories executable by default. i.mode = i.mode & 0o755 | 0o111 - t.addfile(i) + + try: + # We open the file object in binary mode for Windows support. + f = open(os.path.join(root, path), 'rb') + except IOError: + # When we encounter a directory the file object is set to None. + f = None + + t.addfile(i, f) t.close() fileobj.seek(0) From a665c8c4431cc3af6eb5510aff1014b06be6b4b2 Mon Sep 17 00:00:00 2001 From: Maxime Feron Date: Tue, 20 Sep 2016 10:19:16 +0200 Subject: [PATCH 0124/1301] Add support for restart policy update Signed-off-by: Maxime Feron --- docker/api/container.py | 10 +++++++++- docs/api.md | 1 + tests/integration/container_test.py | 30 +++++++++++++++++++++++++++++ 3 files changed, 40 insertions(+), 1 deletion(-) diff --git a/docker/api/container.py b/docker/api/container.py index b8507d85a7..d71d17ad8e 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -421,7 +421,8 @@ def unpause(self, container): def update_container( self, container, blkio_weight=None, cpu_period=None, cpu_quota=None, cpu_shares=None, cpuset_cpus=None, cpuset_mems=None, mem_limit=None, - mem_reservation=None, memswap_limit=None, kernel_memory=None + mem_reservation=None, memswap_limit=None, kernel_memory=None, + restart_policy=None ): url = self._url('/containers/{0}/update', container) data = {} @@ -445,6 +446,13 @@ def update_container( data['MemorySwap'] = utils.parse_bytes(memswap_limit) if kernel_memory: data['KernelMemory'] = utils.parse_bytes(kernel_memory) + if restart_policy: + if utils.version_lt(self._version, '1.23'): + raise errors.InvalidVersion( + 'restart policy update is not supported ' + 'for API version < 1.23' + ) + data['RestartPolicy'] = restart_policy res = self._post_json(url, data=data) return self._result(res, True) diff --git a/docs/api.md b/docs/api.md index 1699344a66..579057aeb4 100644 --- a/docs/api.md +++ b/docs/api.md @@ -1126,6 +1126,7 @@ Update resource configs of one or more containers. * mem_reservation (int or str): Memory soft limit * memswap_limit (int or str): Total memory (memory + swap), -1 to disable swap * kernel_memory (int or str): Kernel memory limit +* restart_policy (dict): Restart policy dictionary **Returns** (dict): Dictionary containing a `Warnings` key. diff --git a/tests/integration/container_test.py b/tests/integration/container_test.py index a7267efb14..4bb78bf19d 100644 --- a/tests/integration/container_test.py +++ b/tests/integration/container_test.py @@ -1104,6 +1104,36 @@ def test_update_container(self): inspect_data = self.client.inspect_container(container) self.assertEqual(inspect_data['HostConfig']['Memory'], new_mem_limit) + @requires_api_version('1.23') + def test_restart_policy_update(self): + old_restart_policy = { + 'MaximumRetryCount': 0, + 'Name': 'always' + } + new_restart_policy = { + 'MaximumRetryCount': 42, + 'Name': 'on-failure' + } + container = self.client.create_container( + BUSYBOX, ['sleep', '60'], + host_config=self.client.create_host_config( + restart_policy=old_restart_policy + ) + ) + self.tmp_containers.append(container) + self.client.start(container) + self.client.update_container(container, + restart_policy=new_restart_policy) + inspect_data = self.client.inspect_container(container) + self.assertEqual( + inspect_data['HostConfig']['RestartPolicy']['MaximumRetryCount'], + new_restart_policy['MaximumRetryCount'] + ) + self.assertEqual( + inspect_data['HostConfig']['RestartPolicy']['Name'], + new_restart_policy['Name'] + ) + class ContainerCPUTest(helpers.BaseTestCase): @requires_api_version('1.18') From 65ad1545e82bab846d319e72671ec64d1784c63a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BE=90=E4=BF=8A=E6=9D=B0?= Date: Wed, 28 Sep 2016 11:01:32 +0800 Subject: [PATCH 0125/1301] replace on_failure with on-failure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 徐俊杰 Signed-off-by: XuPaco --- docker/types/services.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/types/services.py b/docker/types/services.py index 8488d6e2bc..063779cd88 100644 --- a/docker/types/services.py +++ b/docker/types/services.py @@ -152,7 +152,7 @@ def __init__(self, parallelism=0, delay=None, failure_action='continue'): class RestartConditionTypesEnum(object): _values = ( 'none', - 'on_failure', + 'on-failure', 'any', ) NONE, ON_FAILURE, ANY = _values From a864059b83f1e4411db4cd247a25679063062709 Mon Sep 17 00:00:00 2001 From: Alessandro Boch Date: Thu, 29 Sep 2016 09:46:07 -0700 Subject: [PATCH 0126/1301] Adjust test_create_network_with_ipam_config - to account for API change: IPAM class will now also include a Data class, besides Config. Signed-off-by: Alessandro Boch --- tests/integration/network_test.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/integration/network_test.py b/tests/integration/network_test.py index 6726db4b49..bacfd8139f 100644 --- a/tests/integration/network_test.py +++ b/tests/integration/network_test.py @@ -69,9 +69,9 @@ def test_create_network_with_ipam_config(self): assert ipam.pop('Options', None) is None - assert ipam == { - 'Driver': 'default', - 'Config': [{ + assert ipam['Driver'] == 'default' + + assert ipam['Config'] == [{ 'Subnet': "172.28.0.0/16", 'IPRange': "172.28.5.0/24", 'Gateway': "172.28.5.254", @@ -80,8 +80,7 @@ def test_create_network_with_ipam_config(self): "b": "172.28.1.6", "c": "172.28.1.7", }, - }], - } + }] @requires_api_version('1.21') def test_create_network_with_host_driver_fails(self): From 49997d040ba8e35f0d73bb0846b5b90cfa00b5d7 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 29 Sep 2016 16:30:51 -0700 Subject: [PATCH 0127/1301] Add support for isolation param in host config Signed-off-by: Joffrey F --- docker/utils/utils.py | 10 +++++++++- docs/hostconfig.md | 1 + tests/integration/container_test.py | 11 +++++++++++ tests/unit/utils_test.py | 13 ++++++++++++- 4 files changed, 33 insertions(+), 2 deletions(-) diff --git a/docker/utils/utils.py b/docker/utils/utils.py index b565732d6e..a3a8be858f 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -624,7 +624,8 @@ def create_host_config(binds=None, port_bindings=None, lxc_conf=None, device_write_iops=None, oom_kill_disable=False, shm_size=None, sysctls=None, version=None, tmpfs=None, oom_score_adj=None, dns_opt=None, cpu_shares=None, - cpuset_cpus=None, userns_mode=None, pids_limit=None): + cpuset_cpus=None, userns_mode=None, pids_limit=None, + isolation=None): host_config = {} @@ -912,6 +913,13 @@ def create_host_config(binds=None, port_bindings=None, lxc_conf=None, raise host_config_version_error('pids_limit', '1.23') host_config["PidsLimit"] = pids_limit + if isolation: + if not isinstance(isolation, six.string_types): + raise host_config_type_error('isolation', isolation, 'string') + if version_lt(version, '1.24'): + raise host_config_version_error('isolation', '1.24') + host_config['Isolation'] = isolation + return host_config diff --git a/docs/hostconfig.md b/docs/hostconfig.md index 008d5cf210..f989c7d6e0 100644 --- a/docs/hostconfig.md +++ b/docs/hostconfig.md @@ -130,6 +130,7 @@ for example: * userns_mode (str): Sets the user namespace mode for the container when user namespace remapping option is enabled. Supported values are: `host` * pids_limit (int): Tune a container’s pids limit. Set -1 for unlimited. +* isolation (str): Isolation technology to use. Default: `None`. **Returns** (dict) HostConfig dictionary diff --git a/tests/integration/container_test.py b/tests/integration/container_test.py index 4bb78bf19d..c8e5eff8e1 100644 --- a/tests/integration/container_test.py +++ b/tests/integration/container_test.py @@ -397,6 +397,17 @@ def test_create_with_tmpfs(self): config = self.client.inspect_container(container) assert config['HostConfig']['Tmpfs'] == tmpfs + @requires_api_version('1.24') + def test_create_with_isolation(self): + container = self.client.create_container( + BUSYBOX, ['echo'], host_config=self.client.create_host_config( + isolation='default' + ) + ) + self.tmp_containers.append(container['Id']) + config = self.client.inspect_container(container) + assert config['HostConfig']['Isolation'] == 'default' + class VolumeBindTest(helpers.BaseTestCase): def setUp(self): diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index 2a2759d033..83bda33d1e 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -192,7 +192,18 @@ def test_create_host_config_with_pids_limit(self): with pytest.raises(InvalidVersion): create_host_config(version='1.22', pids_limit=1024) with pytest.raises(TypeError): - create_host_config(version='1.22', pids_limit='1024') + create_host_config(version='1.23', pids_limit='1024') + + def test_create_host_config_with_isolation(self): + config = create_host_config(version='1.24', isolation='hyperv') + self.assertEqual(config.get('Isolation'), 'hyperv') + + with pytest.raises(InvalidVersion): + create_host_config(version='1.23', isolation='hyperv') + with pytest.raises(TypeError): + create_host_config( + version='1.24', isolation={'isolation': 'hyperv'} + ) class UlimitTest(base.BaseTestCase): From 8cb186b62392c16ff7bf4890a202220ef4203f8c Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Tue, 27 Sep 2016 12:17:28 +0100 Subject: [PATCH 0128/1301] Add timeout to from_env Signed-off-by: Ben Firshman --- docker/client.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker/client.py b/docker/client.py index 47ad09e9a5..23eb529882 100644 --- a/docker/client.py +++ b/docker/client.py @@ -108,8 +108,10 @@ def __init__(self, base_url=None, version=None, @classmethod def from_env(cls, **kwargs): + timeout = kwargs.pop('timeout', None) version = kwargs.pop('version', None) - return cls(version=version, **kwargs_from_env(**kwargs)) + return cls(timeout=timeout, version=version, + **kwargs_from_env(**kwargs)) def _retrieve_server_version(self): try: From 7339d7704346005e270de6765f2d54187372d8c7 Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Thu, 15 Sep 2016 12:38:09 +0100 Subject: [PATCH 0129/1301] Move BaseTestCase to BaseIntegrationTest Because two things called `BaseTestCase` is quite confusing. I haven't bothered refactoring the other `BaseTestCase` because that disappears anyway when we drop Python 2.6 support. Signed-off-by: Ben Firshman --- tests/helpers.py | 92 ---------------------------- tests/integration/api_test.py | 27 ++++---- tests/integration/base.py | 88 ++++++++++++++++++++++++++ tests/integration/build_test.py | 4 +- tests/integration/conftest.py | 5 +- tests/integration/container_test.py | 43 +++++++------ tests/integration/errors_test.py | 9 +-- tests/integration/exec_test.py | 6 +- tests/integration/image_test.py | 14 ++--- tests/integration/network_test.py | 4 +- tests/integration/regression_test.py | 6 +- tests/integration/service_test.py | 7 +-- tests/integration/swarm_test.py | 7 +-- tests/integration/volume_test.py | 3 +- 14 files changed, 147 insertions(+), 168 deletions(-) create mode 100644 tests/integration/base.py diff --git a/tests/helpers.py b/tests/helpers.py index 40baef9cdc..c7b8634b41 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -1,15 +1,8 @@ import os import os.path -import shutil import tarfile import tempfile -import unittest -import docker -import six - -BUSYBOX = 'busybox:buildroot-2014.02' -EXEC_DRIVER = [] def make_tree(dirs, files): @@ -43,88 +36,3 @@ def untar_file(tardata, filename): result = f.read() f.close() return result - - -def docker_client(**kwargs): - return docker.Client(**docker_client_kwargs(**kwargs)) - - -def docker_client_kwargs(**kwargs): - client_kwargs = docker.utils.kwargs_from_env(assert_hostname=False) - client_kwargs.update(kwargs) - return client_kwargs - - -class BaseTestCase(unittest.TestCase): - tmp_imgs = [] - tmp_containers = [] - tmp_folders = [] - tmp_volumes = [] - - def setUp(self): - if six.PY2: - self.assertRegex = self.assertRegexpMatches - self.assertCountEqual = self.assertItemsEqual - self.client = docker_client(timeout=60) - self.tmp_imgs = [] - self.tmp_containers = [] - self.tmp_folders = [] - self.tmp_volumes = [] - self.tmp_networks = [] - - def tearDown(self): - for img in self.tmp_imgs: - try: - self.client.remove_image(img) - except docker.errors.APIError: - pass - for container in self.tmp_containers: - try: - self.client.stop(container, timeout=1) - self.client.remove_container(container) - except docker.errors.APIError: - pass - for network in self.tmp_networks: - try: - self.client.remove_network(network) - except docker.errors.APIError: - pass - for folder in self.tmp_folders: - shutil.rmtree(folder) - - for volume in self.tmp_volumes: - try: - self.client.remove_volume(volume) - except docker.errors.APIError: - pass - - self.client.close() - - def run_container(self, *args, **kwargs): - container = self.client.create_container(*args, **kwargs) - self.tmp_containers.append(container) - self.client.start(container) - exitcode = self.client.wait(container) - - if exitcode != 0: - output = self.client.logs(container) - raise Exception( - "Container exited with code {}:\n{}" - .format(exitcode, output)) - - return container - - def create_and_start(self, image='busybox', command='top', **kwargs): - container = self.client.create_container( - image=image, command=command, **kwargs) - self.tmp_containers.append(container) - self.client.start(container) - return container - - def execute(self, container, cmd, exit_code=0, **kwargs): - exc = self.client.exec_create(container, cmd, **kwargs) - output = self.client.exec_start(exc) - actual_exit_code = self.client.exec_inspect(exc)['ExitCode'] - msg = "Expected `{}` to exit with code {} but returned {}:\n{}".format( - " ".join(cmd), exit_code, actual_exit_code, output) - assert actual_exit_code == exit_code, msg diff --git a/tests/integration/api_test.py b/tests/integration/api_test.py index 67ed068703..f20d30b10f 100644 --- a/tests/integration/api_test.py +++ b/tests/integration/api_test.py @@ -6,11 +6,12 @@ import warnings import docker +from docker.utils import kwargs_from_env -from .. import helpers +from .base import BaseIntegrationTest, BUSYBOX -class InformationTest(helpers.BaseTestCase): +class InformationTest(BaseIntegrationTest): def test_version(self): res = self.client.version() self.assertIn('GoVersion', res) @@ -24,19 +25,19 @@ def test_info(self): self.assertIn('Debug', res) def test_search(self): - self.client = helpers.docker_client(timeout=10) - res = self.client.search('busybox') + client = docker.from_env(timeout=10) + res = client.search('busybox') self.assertTrue(len(res) >= 1) base_img = [x for x in res if x['name'] == 'busybox'] self.assertEqual(len(base_img), 1) self.assertIn('description', base_img[0]) -class LinkTest(helpers.BaseTestCase): +class LinkTest(BaseIntegrationTest): def test_remove_link(self): # Create containers container1 = self.client.create_container( - helpers.BUSYBOX, 'cat', detach=True, stdin_open=True + BUSYBOX, 'cat', detach=True, stdin_open=True ) container1_id = container1['Id'] self.tmp_containers.append(container1_id) @@ -48,7 +49,7 @@ def test_remove_link(self): link_alias = 'mylink' container2 = self.client.create_container( - helpers.BUSYBOX, 'cat', host_config=self.client.create_host_config( + BUSYBOX, 'cat', host_config=self.client.create_host_config( links={link_path: link_alias} ) ) @@ -74,7 +75,7 @@ def test_remove_link(self): self.assertEqual(len(retrieved), 2) -class LoadConfigTest(helpers.BaseTestCase): +class LoadConfigTest(BaseIntegrationTest): def test_load_legacy_config(self): folder = tempfile.mkdtemp() self.tmp_folders.append(folder) @@ -113,7 +114,7 @@ def test_load_json_config(self): class AutoDetectVersionTest(unittest.TestCase): def test_client_init(self): - client = helpers.docker_client(version='auto') + client = docker.from_env(version='auto') client_version = client._version api_version = client.version(api_version=False)['ApiVersion'] self.assertEqual(client_version, api_version) @@ -122,7 +123,7 @@ def test_client_init(self): client.close() def test_auto_client(self): - client = docker.AutoVersionClient(**helpers.docker_client_kwargs()) + client = docker.AutoVersionClient(**kwargs_from_env()) client_version = client._version api_version = client.version(api_version=False)['ApiVersion'] self.assertEqual(client_version, api_version) @@ -130,9 +131,7 @@ def test_auto_client(self): self.assertEqual(client_version, api_version_2) client.close() with self.assertRaises(docker.errors.DockerException): - docker.AutoVersionClient( - **helpers.docker_client_kwargs(version='1.11') - ) + docker.AutoVersionClient(version='1.11', **kwargs_from_env()) class ConnectionTimeoutTest(unittest.TestCase): @@ -167,7 +166,7 @@ def test_resource_warnings(self): with warnings.catch_warnings(record=True) as w: warnings.simplefilter('always') - client = helpers.docker_client() + client = docker.from_env() client.images() client.close() del client diff --git a/tests/integration/base.py b/tests/integration/base.py new file mode 100644 index 0000000000..3fb25b52e3 --- /dev/null +++ b/tests/integration/base.py @@ -0,0 +1,88 @@ +import shutil +import unittest + +import docker +import six + + +BUSYBOX = 'busybox:buildroot-2014.02' + + +class BaseIntegrationTest(unittest.TestCase): + """ + A base class for integration test cases. + + It sets up a Docker client and cleans up the Docker server after itself. + """ + tmp_imgs = [] + tmp_containers = [] + tmp_folders = [] + tmp_volumes = [] + + def setUp(self): + if six.PY2: + self.assertRegex = self.assertRegexpMatches + self.assertCountEqual = self.assertItemsEqual + self.client = docker.from_env(timeout=60) + self.tmp_imgs = [] + self.tmp_containers = [] + self.tmp_folders = [] + self.tmp_volumes = [] + self.tmp_networks = [] + + def tearDown(self): + for img in self.tmp_imgs: + try: + self.client.remove_image(img) + except docker.errors.APIError: + pass + for container in self.tmp_containers: + try: + self.client.stop(container, timeout=1) + self.client.remove_container(container) + except docker.errors.APIError: + pass + for network in self.tmp_networks: + try: + self.client.remove_network(network) + except docker.errors.APIError: + pass + for folder in self.tmp_folders: + shutil.rmtree(folder) + + for volume in self.tmp_volumes: + try: + self.client.remove_volume(volume) + except docker.errors.APIError: + pass + + self.client.close() + + def run_container(self, *args, **kwargs): + container = self.client.create_container(*args, **kwargs) + self.tmp_containers.append(container) + self.client.start(container) + exitcode = self.client.wait(container) + + if exitcode != 0: + output = self.client.logs(container) + raise Exception( + "Container exited with code {}:\n{}" + .format(exitcode, output)) + + return container + + def create_and_start(self, image='busybox', command='top', **kwargs): + container = self.client.create_container( + image=image, command=command, **kwargs) + self.tmp_containers.append(container) + self.client.start(container) + return container + + def execute(self, container, cmd, exit_code=0, **kwargs): + exc = self.client.exec_create(container, cmd, **kwargs) + output = self.client.exec_start(exc) + actual_exit_code = self.client.exec_inspect(exc)['ExitCode'] + msg = "Expected `{}` to exit with code {} but returned {}:\n{}".format( + " ".join(cmd), exit_code, actual_exit_code, output) + assert actual_exit_code == exit_code, msg diff --git a/tests/integration/build_test.py b/tests/integration/build_test.py index cc8a8626de..db5ad14161 100644 --- a/tests/integration/build_test.py +++ b/tests/integration/build_test.py @@ -8,11 +8,11 @@ from docker import errors -from .. import helpers from ..base import requires_api_version +from .base import BaseIntegrationTest -class BuildTest(helpers.BaseTestCase): +class BuildTest(BaseIntegrationTest): def test_build_streaming(self): script = io.BytesIO('\n'.join([ 'FROM busybox', diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index b17419504e..b0be9665fd 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -7,14 +7,13 @@ import docker.errors import pytest -from ..helpers import BUSYBOX -from ..helpers import docker_client +from .base import BUSYBOX @pytest.fixture(autouse=True, scope='session') def setup_test_session(): warnings.simplefilter('error') - c = docker_client() + c = docker.from_env() try: c.inspect_image(BUSYBOX) except docker.errors.NotFound: diff --git a/tests/integration/container_test.py b/tests/integration/container_test.py index 4bb78bf19d..88ed0f8629 100644 --- a/tests/integration/container_test.py +++ b/tests/integration/container_test.py @@ -10,11 +10,10 @@ from ..base import requires_api_version from .. import helpers +from .base import BaseIntegrationTest, BUSYBOX -BUSYBOX = helpers.BUSYBOX - -class ListContainersTest(helpers.BaseTestCase): +class ListContainersTest(BaseIntegrationTest): def test_list_containers(self): res0 = self.client.containers(all=True) size = len(res0) @@ -34,7 +33,7 @@ def test_list_containers(self): self.assertIn('Status', retrieved) -class CreateContainerTest(helpers.BaseTestCase): +class CreateContainerTest(BaseIntegrationTest): def test_create(self): res = self.client.create_container(BUSYBOX, 'true') @@ -398,7 +397,7 @@ def test_create_with_tmpfs(self): assert config['HostConfig']['Tmpfs'] == tmpfs -class VolumeBindTest(helpers.BaseTestCase): +class VolumeBindTest(BaseIntegrationTest): def setUp(self): super(VolumeBindTest, self).setUp() @@ -487,7 +486,7 @@ def run_with_volume(self, ro, *args, **kwargs): @requires_api_version('1.20') -class ArchiveTest(helpers.BaseTestCase): +class ArchiveTest(BaseIntegrationTest): def test_get_file_archive_from_container(self): data = 'The Maid and the Pocket Watch of Blood' ctnr = self.client.create_container( @@ -567,7 +566,7 @@ def test_copy_directory_to_container(self): self.assertIn('bar/', results) -class RenameContainerTest(helpers.BaseTestCase): +class RenameContainerTest(BaseIntegrationTest): def test_rename_container(self): version = self.client.version()['Version'] name = 'hong_meiling' @@ -583,7 +582,7 @@ def test_rename_container(self): self.assertEqual('/{0}'.format(name), inspect['Name']) -class StartContainerTest(helpers.BaseTestCase): +class StartContainerTest(BaseIntegrationTest): def test_start_container(self): res = self.client.create_container(BUSYBOX, 'true') self.assertIn('Id', res) @@ -637,7 +636,7 @@ def test_run_shlex_commands(self): self.assertEqual(exitcode, 0, msg=cmd) -class WaitTest(helpers.BaseTestCase): +class WaitTest(BaseIntegrationTest): def test_wait(self): res = self.client.create_container(BUSYBOX, ['sleep', '3']) id = res['Id'] @@ -665,7 +664,7 @@ def test_wait_with_dict_instead_of_id(self): self.assertEqual(inspect['State']['ExitCode'], exitcode) -class LogsTest(helpers.BaseTestCase): +class LogsTest(BaseIntegrationTest): def test_logs(self): snippet = 'Flowering Nights (Sakuya Iyazoi)' container = self.client.create_container( @@ -737,7 +736,7 @@ def test_logs_with_tail_0(self): self.assertEqual(logs, ''.encode(encoding='ascii')) -class DiffTest(helpers.BaseTestCase): +class DiffTest(BaseIntegrationTest): def test_diff(self): container = self.client.create_container(BUSYBOX, ['touch', '/test']) id = container['Id'] @@ -765,7 +764,7 @@ def test_diff_with_dict_instead_of_id(self): self.assertEqual(test_diff[0]['Kind'], 1) -class StopTest(helpers.BaseTestCase): +class StopTest(BaseIntegrationTest): def test_stop(self): container = self.client.create_container(BUSYBOX, ['sleep', '9999']) id = container['Id'] @@ -792,7 +791,7 @@ def test_stop_with_dict_instead_of_id(self): self.assertEqual(state['Running'], False) -class KillTest(helpers.BaseTestCase): +class KillTest(BaseIntegrationTest): def test_kill(self): container = self.client.create_container(BUSYBOX, ['sleep', '9999']) id = container['Id'] @@ -868,7 +867,7 @@ def test_kill_with_signal_integer(self): self.assertEqual(state['Running'], False, state) -class PortTest(helpers.BaseTestCase): +class PortTest(BaseIntegrationTest): def test_port(self): port_bindings = { @@ -899,7 +898,7 @@ def test_port(self): self.client.kill(id) -class ContainerTopTest(helpers.BaseTestCase): +class ContainerTopTest(BaseIntegrationTest): def test_top(self): container = self.client.create_container( BUSYBOX, ['sleep', '60']) @@ -934,7 +933,7 @@ def test_top_with_psargs(self): self.client.kill(id) -class RestartContainerTest(helpers.BaseTestCase): +class RestartContainerTest(BaseIntegrationTest): def test_restart(self): container = self.client.create_container(BUSYBOX, ['sleep', '9999']) id = container['Id'] @@ -975,7 +974,7 @@ def test_restart_with_dict_instead_of_id(self): self.client.kill(id) -class RemoveContainerTest(helpers.BaseTestCase): +class RemoveContainerTest(BaseIntegrationTest): def test_remove(self): container = self.client.create_container(BUSYBOX, ['true']) id = container['Id'] @@ -997,7 +996,7 @@ def test_remove_with_dict_instead_of_id(self): self.assertEqual(len(res), 0) -class AttachContainerTest(helpers.BaseTestCase): +class AttachContainerTest(BaseIntegrationTest): def test_run_container_streaming(self): container = self.client.create_container(BUSYBOX, '/bin/sh', detach=True, stdin_open=True) @@ -1028,7 +1027,7 @@ def test_run_container_reading_socket(self): self.assertEqual(data.decode('utf-8'), line) -class PauseTest(helpers.BaseTestCase): +class PauseTest(BaseIntegrationTest): def test_pause_unpause(self): container = self.client.create_container(BUSYBOX, ['sleep', '9999']) id = container['Id'] @@ -1057,7 +1056,7 @@ def test_pause_unpause(self): self.assertEqual(state['Paused'], False) -class GetContainerStatsTest(helpers.BaseTestCase): +class GetContainerStatsTest(BaseIntegrationTest): @requires_api_version('1.19') def test_get_container_stats_no_stream(self): container = self.client.create_container( @@ -1088,7 +1087,7 @@ def test_get_container_stats_stream(self): self.assertIn(key, chunk) -class ContainerUpdateTest(helpers.BaseTestCase): +class ContainerUpdateTest(BaseIntegrationTest): @requires_api_version('1.22') def test_update_container(self): old_mem_limit = 400 * 1024 * 1024 @@ -1135,7 +1134,7 @@ def test_restart_policy_update(self): ) -class ContainerCPUTest(helpers.BaseTestCase): +class ContainerCPUTest(BaseIntegrationTest): @requires_api_version('1.18') def test_container_cpu_shares(self): cpu_shares = 512 diff --git a/tests/integration/errors_test.py b/tests/integration/errors_test.py index 42fbae4af3..4adfa32ffd 100644 --- a/tests/integration/errors_test.py +++ b/tests/integration/errors_test.py @@ -1,13 +1,10 @@ from docker.errors import APIError -from .. import helpers +from .base import BaseIntegrationTest, BUSYBOX -class ErrorsTest(helpers.BaseTestCase): +class ErrorsTest(BaseIntegrationTest): def test_api_error_parses_json(self): - container = self.client.create_container( - helpers.BUSYBOX, - ['sleep', '10'] - ) + container = self.client.create_container(BUSYBOX, ['sleep', '10']) self.client.start(container['Id']) with self.assertRaises(APIError) as cm: self.client.remove_container(container['Id']) diff --git a/tests/integration/exec_test.py b/tests/integration/exec_test.py index f377e09228..f2a8b1f553 100644 --- a/tests/integration/exec_test.py +++ b/tests/integration/exec_test.py @@ -1,12 +1,10 @@ from docker.utils.socket import next_frame_size from docker.utils.socket import read_exactly -from .. import helpers +from .base import BaseIntegrationTest, BUSYBOX -BUSYBOX = helpers.BUSYBOX - -class ExecTest(helpers.BaseTestCase): +class ExecTest(BaseIntegrationTest): def test_execute_command(self): container = self.client.create_container(BUSYBOX, 'cat', detach=True, stdin_open=True) diff --git a/tests/integration/image_test.py b/tests/integration/image_test.py index a61b58aeb7..84ddb4faab 100644 --- a/tests/integration/image_test.py +++ b/tests/integration/image_test.py @@ -14,12 +14,10 @@ import docker -from .. import helpers +from .base import BaseIntegrationTest, BUSYBOX -BUSYBOX = helpers.BUSYBOX - -class ListImagesTest(helpers.BaseTestCase): +class ListImagesTest(BaseIntegrationTest): def test_images(self): res1 = self.client.images(all=True) self.assertIn('Id', res1[0]) @@ -37,7 +35,7 @@ def test_images_quiet(self): self.assertEqual(type(res1[0]), six.text_type) -class PullImageTest(helpers.BaseTestCase): +class PullImageTest(BaseIntegrationTest): def test_pull(self): try: self.client.remove_image('hello-world') @@ -70,7 +68,7 @@ def test_pull_streaming(self): self.assertIn('Id', img_info) -class CommitTest(helpers.BaseTestCase): +class CommitTest(BaseIntegrationTest): def test_commit(self): container = self.client.create_container(BUSYBOX, ['touch', '/test']) id = container['Id'] @@ -105,7 +103,7 @@ def test_commit_with_changes(self): assert img['Config']['Cmd'] == ['bash'] -class RemoveImageTest(helpers.BaseTestCase): +class RemoveImageTest(BaseIntegrationTest): def test_remove(self): container = self.client.create_container(BUSYBOX, ['touch', '/test']) id = container['Id'] @@ -121,7 +119,7 @@ def test_remove(self): self.assertEqual(len(res), 0) -class ImportImageTest(helpers.BaseTestCase): +class ImportImageTest(BaseIntegrationTest): '''Base class for `docker import` test cases.''' TAR_SIZE = 512 * 1024 diff --git a/tests/integration/network_test.py b/tests/integration/network_test.py index bacfd8139f..f0f44f0273 100644 --- a/tests/integration/network_test.py +++ b/tests/integration/network_test.py @@ -5,11 +5,11 @@ from docker.utils import create_ipam_pool import pytest -from .. import helpers from ..base import requires_api_version +from .base import BaseIntegrationTest -class TestNetworks(helpers.BaseTestCase): +class TestNetworks(BaseIntegrationTest): def create_network(self, *args, **kwargs): net_name = u'dockerpy{}'.format(random.getrandbits(24))[:14] net_id = self.client.create_network(net_name, *args, **kwargs)['Id'] diff --git a/tests/integration/regression_test.py b/tests/integration/regression_test.py index 8b321cf5d7..0672c4fad5 100644 --- a/tests/integration/regression_test.py +++ b/tests/integration/regression_test.py @@ -4,12 +4,10 @@ import docker import six -from .. import helpers +from .base import BaseIntegrationTest, BUSYBOX -BUSYBOX = helpers.BUSYBOX - -class TestRegressions(helpers.BaseTestCase): +class TestRegressions(BaseIntegrationTest): def test_443_handle_nonchunked_response_in_stream(self): dfile = io.BytesIO() with self.assertRaises(docker.errors.APIError) as exc: diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 35438940b1..2c80035c95 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -3,13 +3,10 @@ import docker from ..base import requires_api_version -from .. import helpers +from .base import BaseIntegrationTest -BUSYBOX = helpers.BUSYBOX - - -class ServiceTest(helpers.BaseTestCase): +class ServiceTest(BaseIntegrationTest): def setUp(self): super(ServiceTest, self).setUp() self.client.leave_swarm(force=True) diff --git a/tests/integration/swarm_test.py b/tests/integration/swarm_test.py index 8c62f2ec06..fe011841b1 100644 --- a/tests/integration/swarm_test.py +++ b/tests/integration/swarm_test.py @@ -2,13 +2,10 @@ import pytest from ..base import requires_api_version -from .. import helpers +from .base import BaseIntegrationTest -BUSYBOX = helpers.BUSYBOX - - -class SwarmTest(helpers.BaseTestCase): +class SwarmTest(BaseIntegrationTest): def setUp(self): super(SwarmTest, self).setUp() self.client.leave_swarm(force=True) diff --git a/tests/integration/volume_test.py b/tests/integration/volume_test.py index 8fa2dab53f..7e3ba8c0c9 100644 --- a/tests/integration/volume_test.py +++ b/tests/integration/volume_test.py @@ -3,10 +3,11 @@ from .. import helpers from ..base import requires_api_version +from .base import BaseIntegrationTest @requires_api_version('1.21') -class TestVolumes(helpers.BaseTestCase): +class TestVolumes(BaseIntegrationTest): def test_create_volume(self): name = 'perfectcherryblossom' self.tmp_volumes.append(name) From 69e992ec48cdb7f7bcb41ed0a66c1dde5fc99f14 Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Thu, 15 Sep 2016 12:47:25 +0100 Subject: [PATCH 0130/1301] Move requires_api_version to helpers Because it's a helper, not a base thing. In preparation for moving/deleting the unit test base classes. Signed-off-by: Ben Firshman --- tests/base.py | 12 ------------ tests/helpers.py | 11 +++++++++++ tests/integration/build_test.py | 2 +- tests/integration/container_test.py | 2 +- tests/integration/network_test.py | 2 +- tests/integration/service_test.py | 2 +- tests/integration/swarm_test.py | 2 +- tests/integration/volume_test.py | 3 +-- tests/unit/container_test.py | 2 +- tests/unit/network_test.py | 14 +++++++------- tests/unit/volume_test.py | 20 ++++++++++---------- 11 files changed, 35 insertions(+), 37 deletions(-) diff --git a/tests/base.py b/tests/base.py index a2c01fc2d8..cac65fd9f5 100644 --- a/tests/base.py +++ b/tests/base.py @@ -1,11 +1,8 @@ import sys import unittest -import pytest import six -import docker - class BaseTestCase(unittest.TestCase): def assertIn(self, object, collection): @@ -14,15 +11,6 @@ def assertIn(self, object, collection): return super(BaseTestCase, self).assertIn(object, collection) -def requires_api_version(version): - return pytest.mark.skipif( - docker.utils.version_lt( - docker.constants.DEFAULT_DOCKER_API_VERSION, version - ), - reason="API version is too low (< {0})".format(version) - ) - - class Cleanup(object): if sys.version_info < (2, 7): # Provide a basic implementation of addCleanup for Python < 2.7 diff --git a/tests/helpers.py b/tests/helpers.py index c7b8634b41..529b727a3e 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -3,6 +3,8 @@ import tarfile import tempfile +import docker +import pytest def make_tree(dirs, files): @@ -36,3 +38,12 @@ def untar_file(tardata, filename): result = f.read() f.close() return result + + +def requires_api_version(version): + return pytest.mark.skipif( + docker.utils.version_lt( + docker.constants.DEFAULT_DOCKER_API_VERSION, version + ), + reason="API version is too low (< {0})".format(version) + ) diff --git a/tests/integration/build_test.py b/tests/integration/build_test.py index db5ad14161..8dcbd57181 100644 --- a/tests/integration/build_test.py +++ b/tests/integration/build_test.py @@ -8,7 +8,7 @@ from docker import errors -from ..base import requires_api_version +from ..helpers import requires_api_version from .base import BaseIntegrationTest diff --git a/tests/integration/container_test.py b/tests/integration/container_test.py index 88ed0f8629..e703f1802e 100644 --- a/tests/integration/container_test.py +++ b/tests/integration/container_test.py @@ -8,7 +8,7 @@ import pytest import six -from ..base import requires_api_version +from ..helpers import requires_api_version from .. import helpers from .base import BaseIntegrationTest, BUSYBOX diff --git a/tests/integration/network_test.py b/tests/integration/network_test.py index f0f44f0273..ea5db06f74 100644 --- a/tests/integration/network_test.py +++ b/tests/integration/network_test.py @@ -5,7 +5,7 @@ from docker.utils import create_ipam_pool import pytest -from ..base import requires_api_version +from ..helpers import requires_api_version from .base import BaseIntegrationTest diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 2c80035c95..960098ace5 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -2,7 +2,7 @@ import docker -from ..base import requires_api_version +from ..helpers import requires_api_version from .base import BaseIntegrationTest diff --git a/tests/integration/swarm_test.py b/tests/integration/swarm_test.py index fe011841b1..e877ef2868 100644 --- a/tests/integration/swarm_test.py +++ b/tests/integration/swarm_test.py @@ -1,7 +1,7 @@ import docker import pytest -from ..base import requires_api_version +from ..helpers import requires_api_version from .base import BaseIntegrationTest diff --git a/tests/integration/volume_test.py b/tests/integration/volume_test.py index 7e3ba8c0c9..329b4e0d96 100644 --- a/tests/integration/volume_test.py +++ b/tests/integration/volume_test.py @@ -1,8 +1,7 @@ import docker import pytest -from .. import helpers -from ..base import requires_api_version +from ..helpers import requires_api_version from .base import BaseIntegrationTest diff --git a/tests/unit/container_test.py b/tests/unit/container_test.py index 8871b85452..779ed69972 100644 --- a/tests/unit/container_test.py +++ b/tests/unit/container_test.py @@ -9,7 +9,7 @@ import six from . import fake_api -from ..base import requires_api_version +from ..helpers import requires_api_version from .api_test import ( DockerClientTest, url_prefix, fake_request, DEFAULT_TIMEOUT_SECONDS, fake_inspect_container diff --git a/tests/unit/network_test.py b/tests/unit/network_test.py index 2521688de8..93f03da4bc 100644 --- a/tests/unit/network_test.py +++ b/tests/unit/network_test.py @@ -2,7 +2,7 @@ import six -from .. import base +from ..helpers import requires_api_version from .api_test import DockerClientTest, url_prefix, response from docker.utils import create_ipam_config, create_ipam_pool @@ -13,7 +13,7 @@ class NetworkTest(DockerClientTest): - @base.requires_api_version('1.21') + @requires_api_version('1.21') def test_list_networks(self): networks = [ { @@ -49,7 +49,7 @@ def test_list_networks(self): filters = json.loads(get.call_args[1]['params']['filters']) self.assertEqual(filters, {'id': ['123']}) - @base.requires_api_version('1.21') + @requires_api_version('1.21') def test_create_network(self): network_data = { "id": 'abc12345', @@ -104,7 +104,7 @@ def test_create_network(self): } }) - @base.requires_api_version('1.21') + @requires_api_version('1.21') def test_remove_network(self): network_id = 'abc12345' delete = mock.Mock(return_value=response(status_code=200)) @@ -116,7 +116,7 @@ def test_remove_network(self): self.assertEqual(args[0][0], url_prefix + 'networks/{0}'.format(network_id)) - @base.requires_api_version('1.21') + @requires_api_version('1.21') def test_inspect_network(self): network_id = 'abc12345' network_name = 'foo' @@ -138,7 +138,7 @@ def test_inspect_network(self): self.assertEqual(args[0][0], url_prefix + 'networks/{0}'.format(network_id)) - @base.requires_api_version('1.21') + @requires_api_version('1.21') def test_connect_container_to_network(self): network_id = 'abc12345' container_id = 'def45678' @@ -167,7 +167,7 @@ def test_connect_container_to_network(self): }, }) - @base.requires_api_version('1.21') + @requires_api_version('1.21') def test_disconnect_container_from_network(self): network_id = 'abc12345' container_id = 'def45678' diff --git a/tests/unit/volume_test.py b/tests/unit/volume_test.py index 136d11afc1..3909977165 100644 --- a/tests/unit/volume_test.py +++ b/tests/unit/volume_test.py @@ -2,12 +2,12 @@ import pytest -from .. import base +from ..helpers import requires_api_version from .api_test import DockerClientTest, url_prefix, fake_request class VolumeTest(DockerClientTest): - @base.requires_api_version('1.21') + @requires_api_version('1.21') def test_list_volumes(self): volumes = self.client.volumes() self.assertIn('Volumes', volumes) @@ -17,7 +17,7 @@ def test_list_volumes(self): self.assertEqual(args[0][0], 'GET') self.assertEqual(args[0][1], url_prefix + 'volumes') - @base.requires_api_version('1.21') + @requires_api_version('1.21') def test_list_volumes_and_filters(self): volumes = self.client.volumes(filters={'dangling': True}) assert 'Volumes' in volumes @@ -29,7 +29,7 @@ def test_list_volumes_and_filters(self): assert args[1] == {'params': {'filters': '{"dangling": ["true"]}'}, 'timeout': 60} - @base.requires_api_version('1.21') + @requires_api_version('1.21') def test_create_volume(self): name = 'perfectcherryblossom' result = self.client.create_volume(name) @@ -43,7 +43,7 @@ def test_create_volume(self): self.assertEqual(args[0][1], url_prefix + 'volumes/create') self.assertEqual(json.loads(args[1]['data']), {'Name': name}) - @base.requires_api_version('1.23') + @requires_api_version('1.23') def test_create_volume_with_labels(self): name = 'perfectcherryblossom' result = self.client.create_volume(name, labels={ @@ -53,13 +53,13 @@ def test_create_volume_with_labels(self): {'com.example.some-label': 'some-value'} ) - @base.requires_api_version('1.23') + @requires_api_version('1.23') def test_create_volume_with_invalid_labels(self): name = 'perfectcherryblossom' with pytest.raises(TypeError): self.client.create_volume(name, labels=1) - @base.requires_api_version('1.21') + @requires_api_version('1.21') def test_create_volume_with_driver(self): name = 'perfectcherryblossom' driver_name = 'sshfs' @@ -72,7 +72,7 @@ def test_create_volume_with_driver(self): self.assertIn('Driver', data) self.assertEqual(data['Driver'], driver_name) - @base.requires_api_version('1.21') + @requires_api_version('1.21') def test_create_volume_invalid_opts_type(self): with pytest.raises(TypeError): self.client.create_volume( @@ -89,7 +89,7 @@ def test_create_volume_invalid_opts_type(self): 'perfectcherryblossom', driver_opts='' ) - @base.requires_api_version('1.21') + @requires_api_version('1.21') def test_inspect_volume(self): name = 'perfectcherryblossom' result = self.client.inspect_volume(name) @@ -102,7 +102,7 @@ def test_inspect_volume(self): self.assertEqual(args[0][0], 'GET') self.assertEqual(args[0][1], '{0}volumes/{1}'.format(url_prefix, name)) - @base.requires_api_version('1.21') + @requires_api_version('1.21') def test_remove_volume(self): name = 'perfectcherryblossom' self.client.remove_volume(name) From b00e321b589c1c9ade22a670c3517cc68f892cfc Mon Sep 17 00:00:00 2001 From: Yuriy Taraday Date: Mon, 3 Oct 2016 18:34:49 +0300 Subject: [PATCH 0131/1301] Support requests versions from 2.11.1 onwards Bug #1155 has been fixed starting with requests 2.11.1 and excluding it from dependencies causes failures when using latest versions of both libs together in our project. Signed-off-by: Yuriy Taraday --- requirements.txt | 4 ++-- setup.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 1e5284600f..375413122b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ -requests==2.5.3 +requests==2.11.1 six>=1.4.0 websocket-client==0.32.0 backports.ssl_match_hostname>=3.5 ; python_version < '3.5' ipaddress==1.0.16 ; python_version < '3.3' -docker-pycreds==0.2.1 \ No newline at end of file +docker-pycreds==0.2.1 diff --git a/setup.py b/setup.py index 9233ac2a8a..00e1febdc7 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ SOURCE_DIR = os.path.join(ROOT_DIR) requirements = [ - 'requests >= 2.5.2, < 2.11', + 'requests >= 2.5.2, != 2.11.0', 'six >= 1.4.0', 'websocket-client >= 0.32.0', 'docker-pycreds >= 0.2.1' From daec55d4e15d632c9507f9327ae2cea600ffd95c Mon Sep 17 00:00:00 2001 From: mattjegan Date: Tue, 4 Oct 2016 13:38:01 +1100 Subject: [PATCH 0132/1301] Fix syntax in README required by RST Before the readme is converted to rst it required an extra character to convert with no errors. Signed-off-by: Matthew Egan --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index bdec785418..876ed02636 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ The latest stable version is always available on PyPi. pip install docker-py Documentation ------------- +------------- [![Documentation Status](https://readthedocs.org/projects/docker-py/badge/?version=latest)](https://readthedocs.org/projects/docker-py/?badge=latest) From 8239032463899af665b690aa0890941c1909f399 Mon Sep 17 00:00:00 2001 From: Pierre Tardy Date: Tue, 4 Oct 2016 13:13:04 +0200 Subject: [PATCH 0133/1301] fix for got an unexpected keyword argument 'num_pools' requests's HTTPAdapter API is pool_connections for number of connection of the pool Signed-off-by: Pierre Tardy --- docker/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/client.py b/docker/client.py index 23eb529882..c16f314a80 100644 --- a/docker/client.py +++ b/docker/client.py @@ -86,7 +86,7 @@ def __init__(self, base_url=None, version=None, tls.configure_client(self) elif tls: self._custom_adapter = ssladapter.SSLAdapter( - num_pools=num_pools + pool_connections=num_pools ) self.mount('https://', self._custom_adapter) self.base_url = base_url From b65de73afea947cb65cc79c58acf414604ea6b16 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 4 Oct 2016 12:19:33 -0700 Subject: [PATCH 0134/1301] Update adapters to use pool_connections instead of num_pools Signed-off-by: Joffrey F --- docker/client.py | 4 ++-- docker/transport/npipeconn.py | 4 ++-- docker/transport/unixconn.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docker/client.py b/docker/client.py index c16f314a80..aba066ba35 100644 --- a/docker/client.py +++ b/docker/client.py @@ -60,7 +60,7 @@ def __init__(self, base_url=None, version=None, ) if base_url.startswith('http+unix://'): self._custom_adapter = UnixAdapter( - base_url, timeout, num_pools=num_pools + base_url, timeout, pool_connections=num_pools ) self.mount('http+docker://', self._custom_adapter) self._unmount('http://', 'https://') @@ -72,7 +72,7 @@ def __init__(self, base_url=None, version=None, ) try: self._custom_adapter = NpipeAdapter( - base_url, timeout, num_pools=num_pools + base_url, timeout, pool_connections=num_pools ) except NameError: raise errors.DockerException( diff --git a/docker/transport/npipeconn.py b/docker/transport/npipeconn.py index 917fa8b3bf..984049c720 100644 --- a/docker/transport/npipeconn.py +++ b/docker/transport/npipeconn.py @@ -49,11 +49,11 @@ def _new_conn(self): class NpipeAdapter(requests.adapters.HTTPAdapter): def __init__(self, base_url, timeout=60, - num_pools=constants.DEFAULT_NUM_POOLS): + pool_connections=constants.DEFAULT_NUM_POOLS): self.npipe_path = base_url.replace('npipe://', '') self.timeout = timeout self.pools = RecentlyUsedContainer( - num_pools, dispose_func=lambda p: p.close() + pool_connections, dispose_func=lambda p: p.close() ) super(NpipeAdapter, self).__init__() diff --git a/docker/transport/unixconn.py b/docker/transport/unixconn.py index b7905a042e..978c87a1bf 100644 --- a/docker/transport/unixconn.py +++ b/docker/transport/unixconn.py @@ -51,14 +51,14 @@ def _new_conn(self): class UnixAdapter(requests.adapters.HTTPAdapter): def __init__(self, socket_url, timeout=60, - num_pools=constants.DEFAULT_NUM_POOLS): + pool_connections=constants.DEFAULT_NUM_POOLS): socket_path = socket_url.replace('http+unix://', '') if not socket_path.startswith('/'): socket_path = '/' + socket_path self.socket_path = socket_path self.timeout = timeout self.pools = RecentlyUsedContainer( - num_pools, dispose_func=lambda p: p.close() + pool_connections, dispose_func=lambda p: p.close() ) super(UnixAdapter, self).__init__() From aa2bd80ebb1ab17e535ab7df5533bc16ae07daf7 Mon Sep 17 00:00:00 2001 From: Tomas Tomecek Date: Tue, 27 Sep 2016 08:45:41 +0200 Subject: [PATCH 0135/1301] document requirement for ipaddress module Signed-off-by: Tomas Tomecek --- setup.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup.py b/setup.py index 9233ac2a8a..5eda544539 100644 --- a/setup.py +++ b/setup.py @@ -20,6 +20,9 @@ extras_require = { ':python_version < "3.5"': 'backports.ssl_match_hostname >= 3.5', + # While not imported explicitly, the ipaddress module is required for + # ssl_match_hostname to verify hosts match with certificates via + # ServerAltname: https://pypi.python.org/pypi/backports.ssl_match_hostname ':python_version < "3.3"': 'ipaddress >= 1.0.16', } From 790d7525f0138184c7c4709b0ebd3e4ff73f4ce5 Mon Sep 17 00:00:00 2001 From: Tomas Tomecek Date: Mon, 11 Jul 2016 10:28:19 +0200 Subject: [PATCH 0136/1301] document how to recover from api version mismatch Signed-off-by: Tomas Tomecek --- docs/api.md | 28 ++++++++++++++++++++++++++++ mkdocs.yml | 1 + 2 files changed, 29 insertions(+) diff --git a/docs/api.md b/docs/api.md index 579057aeb4..738606d433 100644 --- a/docs/api.md +++ b/docs/api.md @@ -18,6 +18,34 @@ is hosted. * tls (bool or [TLSConfig](tls.md#TLSConfig)): Equivalent CLI options: `docker --tls ...` * user_agent (str): Set a custom user agent for requests to the server. + +## Version mismatch + +You may encounter an error like this: + +```text +client is newer than server (client API version: 1.24, server API version: 1.23) +``` + +To fix this, you have to either supply exact version to `Client` which you know that server supports: + +```python +client = docker.Client(version="1.23") +``` + +or let client pick the newest version server supports: + +```python +client = docker.Client(version="auto") +``` + +or even + +```python +client = docker.AutoVersionClient() +``` + + **** ## attach diff --git a/mkdocs.yml b/mkdocs.yml index 6cfaa543bc..abc68e548f 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -7,6 +7,7 @@ theme: readthedocs pages: - Home: index.md - Client API: api.md +- FAQ: faq.md - Port Bindings: port-bindings.md - Using Volumes: volumes.md - Using TLS: tls.md From 230d8c7efe21fd2480ab8cd3a2182773149086a0 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 5 Oct 2016 11:15:09 -0700 Subject: [PATCH 0137/1301] Removed non-existent FAQ page from mkdocs.yml Signed-off-by: Joffrey F --- mkdocs.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/mkdocs.yml b/mkdocs.yml index abc68e548f..6cfaa543bc 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -7,7 +7,6 @@ theme: readthedocs pages: - Home: index.md - Client API: api.md -- FAQ: faq.md - Port Bindings: port-bindings.md - Using Volumes: volumes.md - Using TLS: tls.md From f4cb91eb02001791b73d75039f67a8460a4307b2 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 5 Oct 2016 15:27:36 -0700 Subject: [PATCH 0138/1301] Add missing long_description and maintainer fields to setup.py Signed-off-by: Joffrey F --- setup.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/setup.py b/setup.py index 5eda544539..1e22f51383 100644 --- a/setup.py +++ b/setup.py @@ -33,10 +33,20 @@ test_requirements = [line for line in test_reqs_txt] +long_description = '' +try: + with open('./README.rst') as readme_rst: + long_description = readme_rst.read() +except IOError: + # README.rst is only generated on release. Its absence should not prevent + # setup.py from working properly. + pass + setup( name="docker-py", version=version, description="Python client for Docker.", + long_description=long_description, url='https://github.com/docker/docker-py/', packages=[ 'docker', 'docker.api', 'docker.auth', 'docker.transport', @@ -64,4 +74,6 @@ 'Topic :: Utilities', 'License :: OSI Approved :: Apache Software License', ], + maintainer='Joffrey F', + maintainer_email='joffrey@docker.com', ) From 6f7392ea09751be65821a0d539f6834e3f6ce31d Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 11 Oct 2016 15:16:43 -0700 Subject: [PATCH 0139/1301] Do not allow bufsize to be 0 in NpipeSocket.makefile() Signed-off-by: Joffrey F --- docker/transport/npipesocket.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/transport/npipesocket.py b/docker/transport/npipesocket.py index d3847c3f56..6dfc2f2d16 100644 --- a/docker/transport/npipesocket.py +++ b/docker/transport/npipesocket.py @@ -95,7 +95,7 @@ def makefile(self, mode=None, bufsize=None): if mode.strip('b') != 'r': raise NotImplementedError() rawio = NpipeFileIOBase(self) - if bufsize is None or bufsize < 0: + if bufsize is None or bufsize <= 0: bufsize = io.DEFAULT_BUFFER_SIZE return io.BufferedReader(rawio, buffer_size=bufsize) From c76ec15d9b58299aabb55b2c6632f08eb51d520c Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 11 Oct 2016 17:19:20 -0700 Subject: [PATCH 0140/1301] Several fixes to npipe support - Fix _get_raw_response_socket to always return the NpipeSocket object - Override NpipeHTTPConnectionPool._get_conn to avoid crash in urllib3 - Fix NpipeSocket.recv_into for Python 2 - Do not call select() on NpipeSocket objects Signed-off-by: Joffrey F --- docker/client.py | 4 +++- docker/transport/__init__.py | 1 + docker/transport/npipeconn.py | 23 ++++++++++++++++++++++- docker/transport/npipesocket.py | 10 ++++++++++ docker/utils/socket.py | 9 ++++++++- 5 files changed, 44 insertions(+), 3 deletions(-) diff --git a/docker/client.py b/docker/client.py index aba066ba35..aec78c8a6a 100644 --- a/docker/client.py +++ b/docker/client.py @@ -220,7 +220,9 @@ def _create_websocket_connection(self, url): def _get_raw_response_socket(self, response): self._raise_for_status(response) - if six.PY3: + if self.base_url == "http+docker://localnpipe": + sock = response.raw._fp.fp.raw.sock + elif six.PY3: sock = response.raw._fp.fp.raw if self.base_url.startswith("https://"): sock = sock._sock diff --git a/docker/transport/__init__.py b/docker/transport/__init__.py index 04a46d9883..d5560b63e2 100644 --- a/docker/transport/__init__.py +++ b/docker/transport/__init__.py @@ -2,5 +2,6 @@ from .unixconn import UnixAdapter try: from .npipeconn import NpipeAdapter + from .npipesocket import NpipeSocket except ImportError: pass diff --git a/docker/transport/npipeconn.py b/docker/transport/npipeconn.py index 984049c720..3054037f3a 100644 --- a/docker/transport/npipeconn.py +++ b/docker/transport/npipeconn.py @@ -14,7 +14,6 @@ except ImportError: import urllib3 - RecentlyUsedContainer = urllib3._collections.RecentlyUsedContainer @@ -46,6 +45,28 @@ def _new_conn(self): self.npipe_path, self.timeout ) + # When re-using connections, urllib3 tries to call select() on our + # NpipeSocket instance, causing a crash. To circumvent this, we override + # _get_conn, where that check happens. + def _get_conn(self, timeout): + conn = None + try: + conn = self.pool.get(block=self.block, timeout=timeout) + + except AttributeError: # self.pool is None + raise urllib3.exceptions.ClosedPoolError(self, "Pool is closed.") + + except six.moves.queue.Empty: + if self.block: + raise urllib3.exceptions.EmptyPoolError( + self, + "Pool reached maximum size and no more " + "connections are allowed." + ) + pass # Oh well, we'll create a new connection then + + return conn or self._new_conn() + class NpipeAdapter(requests.adapters.HTTPAdapter): def __init__(self, base_url, timeout=60, diff --git a/docker/transport/npipesocket.py b/docker/transport/npipesocket.py index 6dfc2f2d16..a9bf0ccba5 100644 --- a/docker/transport/npipesocket.py +++ b/docker/transport/npipesocket.py @@ -1,6 +1,7 @@ import functools import io +import six import win32file import win32pipe @@ -115,6 +116,9 @@ def recvfrom_into(self, buf, nbytes=0, flags=0): @check_closed def recv_into(self, buf, nbytes=0): + if six.PY2: + return self._recv_into_py2(buf, nbytes) + readbuf = buf if not isinstance(buf, memoryview): readbuf = memoryview(buf) @@ -125,6 +129,12 @@ def recv_into(self, buf, nbytes=0): ) return len(data) + def _recv_into_py2(self, buf, nbytes): + err, data = win32file.ReadFile(self._handle, nbytes or len(buf)) + n = len(data) + buf[:n] = data + return n + @check_closed def send(self, string, flags=0): err, nbytes = win32file.WriteFile(self._handle, string) diff --git a/docker/utils/socket.py b/docker/utils/socket.py index ed343507d8..164b845afc 100644 --- a/docker/utils/socket.py +++ b/docker/utils/socket.py @@ -5,6 +5,11 @@ import six +try: + from ..transport import NpipeSocket +except ImportError: + NpipeSocket = type(None) + class SocketError(Exception): pass @@ -14,10 +19,12 @@ def read(socket, n=4096): """ Reads at most n bytes from socket """ + recoverable_errors = (errno.EINTR, errno.EDEADLK, errno.EWOULDBLOCK) # wait for data to become available - select.select([socket], [], []) + if not isinstance(socket, NpipeSocket): + select.select([socket], [], []) try: if hasattr(socket, 'recv'): From 05f1060824e6567629351618bf1989df5826fda7 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 12 Oct 2016 16:06:43 -0700 Subject: [PATCH 0141/1301] Remove trailing slashes in result of utils.parse_host Signed-off-by: Joffrey F --- docker/utils/utils.py | 4 ++-- tests/unit/utils_test.py | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/docker/utils/utils.py b/docker/utils/utils.py index b565732d6e..e1c7ad0ceb 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -453,8 +453,8 @@ def parse_host(addr, is_win32=False, tls=False): "Bind address needs a port: {0}".format(addr)) if proto == "http+unix" or proto == 'npipe': - return "{0}://{1}".format(proto, host) - return "{0}://{1}:{2}{3}".format(proto, host, port, path) + return "{0}://{1}".format(proto, host).rstrip('/') + return "{0}://{1}:{2}{3}".format(proto, host, port, path).rstrip('/') def parse_devices(devices): diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index 2a2759d033..059c82d3bb 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -522,6 +522,11 @@ def test_parse_host_tls_tcp_proto(self): expected_result = 'https://myhost.docker.net:3348' assert parse_host(host_value, tls=True) == expected_result + def test_parse_host_trailing_slash(self): + host_value = 'tcp://myhost.docker.net:2376/' + expected_result = 'http://myhost.docker.net:2376' + assert parse_host(host_value) == expected_result + class ParseRepositoryTagTest(base.BaseTestCase): sha = 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855' From 059f61bf5a7b3252c1980342befe47ec6ea8f6bd Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 12 Oct 2016 17:19:08 -0700 Subject: [PATCH 0142/1301] Do not break when calling format_environment with unicode values Signed-off-by: Joffrey F --- docker/utils/utils.py | 3 +++ tests/unit/utils_test.py | 19 +++++++++++++++++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/docker/utils/utils.py b/docker/utils/utils.py index b565732d6e..10b9338d0b 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -1001,6 +1001,9 @@ def format_environment(environment): def format_env(key, value): if value is None: return key + if isinstance(value, six.binary_type): + value = value.decode('utf-8') + return u'{key}={value}'.format(key=key, value=value) return [format_env(*var) for var in six.iteritems(environment)] diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index 2a2759d033..8fc7ab4d2a 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -20,11 +20,11 @@ create_host_config, Ulimit, LogConfig, parse_bytes, parse_env_file, exclude_paths, convert_volume_binds, decode_json_header, tar, split_command, create_ipam_config, create_ipam_pool, parse_devices, - update_headers, + update_headers ) from docker.utils.ports import build_port_bindings, split_port -from docker.utils.utils import create_endpoint_config +from docker.utils.utils import create_endpoint_config, format_environment from .. import base from ..helpers import make_tree @@ -1042,3 +1042,18 @@ def test_tar_with_directory_symlinks(self): self.assertEqual( sorted(tar_data.getnames()), ['bar', 'bar/foo', 'foo'] ) + + +class FormatEnvironmentTest(base.BaseTestCase): + def test_format_env_binary_unicode_value(self): + env_dict = { + 'ARTIST_NAME': b'\xec\x86\xa1\xec\xa7\x80\xec\x9d\x80' + } + assert format_environment(env_dict) == [u'ARTIST_NAME=송지은'] + + def test_format_env_no_value(self): + env_dict = { + 'FOO': None, + 'BAR': '', + } + assert sorted(format_environment(env_dict)) == ['BAR=', 'FOO'] From 6768477edf7f92da84c9f3eb2fb45ad00ee88f4d Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Thu, 13 Oct 2016 10:48:32 +0200 Subject: [PATCH 0143/1301] Remove dead code in import_image_from_data Signed-off-by: Ben Firshman --- docker/api/image.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/docker/api/image.py b/docker/api/image.py index 7f25f9d971..262910cd60 100644 --- a/docker/api/image.py +++ b/docker/api/image.py @@ -88,9 +88,6 @@ def import_image_from_data(self, data, repository=None, tag=None, u, data=data, params=params, headers=headers, timeout=None ) ) - return self.import_image( - src=data, repository=repository, tag=tag, changes=changes - ) def import_image_from_file(self, filename, repository=None, tag=None, changes=None): From cec3fe7c318fd22386ff5b18519609ab72db696b Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 13 Oct 2016 16:40:55 -0700 Subject: [PATCH 0144/1301] Update tests to avoid failures on Windows platforms Signed-off-by: Joffrey F --- tests/integration/build_test.py | 12 +-- tests/integration/conftest.py | 4 +- tests/integration/container_test.py | 45 +++++---- tests/integration/image_test.py | 8 +- tests/integration/network_test.py | 18 ++-- tests/unit/api_test.py | 5 +- tests/unit/container_test.py | 8 +- tests/unit/fake_api.py | 3 + tests/unit/utils_test.py | 140 +++++++++++++++++----------- 9 files changed, 141 insertions(+), 102 deletions(-) diff --git a/tests/integration/build_test.py b/tests/integration/build_test.py index 8dcbd57181..699345fcce 100644 --- a/tests/integration/build_test.py +++ b/tests/integration/build_test.py @@ -1,5 +1,4 @@ import io -import json import os import shutil import tempfile @@ -22,14 +21,11 @@ def test_build_streaming(self): 'ADD https://dl.dropboxusercontent.com/u/20637798/silence.tar.gz' ' /tmp/silence.tar.gz' ]).encode('ascii')) - stream = self.client.build(fileobj=script, stream=True) - logs = '' + stream = self.client.build(fileobj=script, stream=True, decode=True) + logs = [] for chunk in stream: - if six.PY3: - chunk = chunk.decode('utf-8') - json.loads(chunk) # ensure chunk is a single, valid JSON blob - logs += chunk - self.assertNotEqual(logs, '') + logs.append(chunk) + assert len(logs) > 0 def test_build_from_stringio(self): if six.PY3: diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index b0be9665fd..c488f90672 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -1,6 +1,5 @@ from __future__ import print_function -import json import sys import warnings @@ -18,8 +17,7 @@ def setup_test_session(): c.inspect_image(BUSYBOX) except docker.errors.NotFound: print("\npulling {0}".format(BUSYBOX), file=sys.stderr) - for data in c.pull(BUSYBOX, stream=True): - data = json.loads(data.decode('utf-8')) + for data in c.pull(BUSYBOX, stream=True, decode=True): status = data.get("status") progress = data.get("progress") detail = "{0} - {1}".format(status, progress) diff --git a/tests/integration/container_test.py b/tests/integration/container_test.py index e703f1802e..5ced082da7 100644 --- a/tests/integration/container_test.py +++ b/tests/integration/container_test.py @@ -3,6 +3,7 @@ import tempfile import docker +from docker.constants import IS_WINDOWS_PLATFORM from docker.utils.socket import next_frame_size from docker.utils.socket import read_exactly import pytest @@ -523,13 +524,13 @@ def test_get_file_stat_from_container(self): def test_copy_file_to_container(self): data = b'Deaf To All But The Song' - with tempfile.NamedTemporaryFile() as test_file: + with tempfile.NamedTemporaryFile(delete=False) as test_file: test_file.write(data) test_file.seek(0) ctnr = self.client.create_container( BUSYBOX, 'cat {0}'.format( - os.path.join('/vol1', os.path.basename(test_file.name)) + os.path.join('/vol1/', os.path.basename(test_file.name)) ), volumes=['/vol1'] ) @@ -821,11 +822,12 @@ def test_kill_with_dict_instead_of_id(self): self.assertEqual(state['Running'], False) def test_kill_with_signal(self): - container = self.client.create_container(BUSYBOX, ['sleep', '60']) - id = container['Id'] - self.client.start(id) + id = self.client.create_container(BUSYBOX, ['sleep', '60']) self.tmp_containers.append(id) - self.client.kill(id, signal=signal.SIGKILL) + self.client.start(id) + self.client.kill( + id, signal=signal.SIGKILL if not IS_WINDOWS_PLATFORM else 9 + ) exitcode = self.client.wait(id) self.assertNotEqual(exitcode, 0) container_info = self.client.inspect_container(id) @@ -901,28 +903,34 @@ def test_port(self): class ContainerTopTest(BaseIntegrationTest): def test_top(self): container = self.client.create_container( - BUSYBOX, ['sleep', '60']) + BUSYBOX, ['sleep', '60'] + ) - id = container['Id'] + self.tmp_containers.append(container) self.client.start(container) - res = self.client.top(container['Id']) - self.assertEqual( - res['Titles'], - ['UID', 'PID', 'PPID', 'C', 'STIME', 'TTY', 'TIME', 'CMD'] - ) - self.assertEqual(len(res['Processes']), 1) - self.assertEqual(res['Processes'][0][7], 'sleep 60') - self.client.kill(id) + res = self.client.top(container) + if IS_WINDOWS_PLATFORM: + assert res['Titles'] == ['PID', 'USER', 'TIME', 'COMMAND'] + else: + assert res['Titles'] == [ + 'UID', 'PID', 'PPID', 'C', 'STIME', 'TTY', 'TIME', 'CMD' + ] + assert len(res['Processes']) == 1 + assert res['Processes'][0][-1] == 'sleep 60' + self.client.kill(container) + @pytest.mark.skipif( + IS_WINDOWS_PLATFORM, reason='No psargs support on windows' + ) def test_top_with_psargs(self): container = self.client.create_container( BUSYBOX, ['sleep', '60']) - id = container['Id'] + self.tmp_containers.append(container) self.client.start(container) - res = self.client.top(container['Id'], 'waux') + res = self.client.top(container, 'waux') self.assertEqual( res['Titles'], ['USER', 'PID', '%CPU', '%MEM', 'VSZ', 'RSS', @@ -930,7 +938,6 @@ def test_top_with_psargs(self): ) self.assertEqual(len(res['Processes']), 1) self.assertEqual(res['Processes'][0][10], 'sleep 60') - self.client.kill(id) class RestartContainerTest(BaseIntegrationTest): diff --git a/tests/integration/image_test.py b/tests/integration/image_test.py index 84ddb4faab..31d2218b13 100644 --- a/tests/integration/image_test.py +++ b/tests/integration/image_test.py @@ -55,12 +55,10 @@ def test_pull_streaming(self): self.client.remove_image('hello-world') except docker.errors.APIError: pass - stream = self.client.pull('hello-world', stream=True) + stream = self.client.pull('hello-world', stream=True, decode=True) self.tmp_imgs.append('hello-world') for chunk in stream: - if six.PY3: - chunk = chunk.decode('utf-8') - json.loads(chunk) # ensure chunk is a single, valid JSON blob + assert isinstance(chunk, dict) self.assertGreaterEqual( len(self.client.images('hello-world')), 1 ) @@ -150,7 +148,7 @@ def dummy_tar_stream(self, n_bytes): @contextlib.contextmanager def dummy_tar_file(self, n_bytes): '''Yields the name of a valid tar file of size n_bytes.''' - with tempfile.NamedTemporaryFile() as tar_file: + with tempfile.NamedTemporaryFile(delete=False) as tar_file: self.write_dummy_tar_content(n_bytes, tar_file) tar_file.seek(0) yield tar_file.name diff --git a/tests/integration/network_test.py b/tests/integration/network_test.py index ea5db06f74..2ff5f029b7 100644 --- a/tests/integration/network_test.py +++ b/tests/integration/network_test.py @@ -72,15 +72,15 @@ def test_create_network_with_ipam_config(self): assert ipam['Driver'] == 'default' assert ipam['Config'] == [{ - 'Subnet': "172.28.0.0/16", - 'IPRange': "172.28.5.0/24", - 'Gateway': "172.28.5.254", - 'AuxiliaryAddresses': { - "a": "172.28.1.5", - "b": "172.28.1.6", - "c": "172.28.1.7", - }, - }] + 'Subnet': "172.28.0.0/16", + 'IPRange': "172.28.5.0/24", + 'Gateway': "172.28.5.254", + 'AuxiliaryAddresses': { + "a": "172.28.1.5", + "b": "172.28.1.6", + "c": "172.28.1.7", + }, + }] @requires_api_version('1.21') def test_create_network_with_host_driver_fails(self): diff --git a/tests/unit/api_test.py b/tests/unit/api_test.py index c9706fbb4c..94092dd220 100644 --- a/tests/unit/api_test.py +++ b/tests/unit/api_test.py @@ -86,7 +86,7 @@ def fake_delete(self, url, *args, **kwargs): def fake_read_from_socket(self, response, stream): return six.binary_type() -url_base = 'http+docker://localunixsocket/' +url_base = '{0}/'.format(fake_api.prefix) url_prefix = '{0}v{1}/'.format( url_base, docker.constants.DEFAULT_DOCKER_API_VERSION) @@ -422,6 +422,9 @@ def early_response_sending_handler(self, connection): data += connection.recv(2048) + @pytest.mark.skipif( + docker.constants.IS_WINDOWS_PLATFORM, reason='Unix only' + ) def test_early_stream_response(self): self.request_handler = self.early_response_sending_handler lines = [] diff --git a/tests/unit/container_test.py b/tests/unit/container_test.py index 779ed69972..51e8cbba9c 100644 --- a/tests/unit/container_test.py +++ b/tests/unit/container_test.py @@ -270,8 +270,8 @@ def test_create_container_with_entrypoint(self): {'Content-Type': 'application/json'}) def test_create_container_with_cpu_shares(self): - self.client.create_container('busybox', 'ls', - cpu_shares=5) + with pytest.deprecated_call(): + self.client.create_container('busybox', 'ls', cpu_shares=5) args = fake_request.call_args self.assertEqual(args[0][1], @@ -316,8 +316,8 @@ def test_create_container_with_host_config_cpu_shares(self): {'Content-Type': 'application/json'}) def test_create_container_with_cpuset(self): - self.client.create_container('busybox', 'ls', - cpuset='0,1') + with pytest.deprecated_call(): + self.client.create_container('busybox', 'ls', cpuset='0,1') args = fake_request.call_args self.assertEqual(args[0][1], diff --git a/tests/unit/fake_api.py b/tests/unit/fake_api.py index 1e9d318df5..65a8c42447 100644 --- a/tests/unit/fake_api.py +++ b/tests/unit/fake_api.py @@ -408,6 +408,9 @@ def post_fake_update_container(): # Maps real api url to fake response callback prefix = 'http+docker://localunixsocket' +if constants.IS_WINDOWS_PLATFORM: + prefix = 'http+docker://localnpipe' + fake_responses = { '{0}/version'.format(prefix): get_fake_raw_version, diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index a06cbea320..290874f416 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -13,7 +13,9 @@ import six from docker.client import Client -from docker.constants import DEFAULT_DOCKER_API_VERSION +from docker.constants import ( + DEFAULT_DOCKER_API_VERSION, IS_WINDOWS_PLATFORM +) from docker.errors import DockerException, InvalidVersion from docker.utils import ( parse_repository_tag, parse_host, convert_filters, kwargs_from_env, @@ -809,6 +811,12 @@ def test_build_port_bindings_with_nonmatching_internal_port_ranges(self): self.assertEqual(port_bindings["2000"], [("127.0.0.1", "2000")]) +def convert_paths(collection): + if not IS_WINDOWS_PLATFORM: + return collection + return set(map(lambda x: x.replace('/', '\\'), collection)) + + class ExcludePathsTest(base.BaseTestCase): dirs = [ 'foo', @@ -843,7 +851,7 @@ def exclude(self, patterns, dockerfile=None): return set(exclude_paths(self.base, patterns, dockerfile=dockerfile)) def test_no_excludes(self): - assert self.exclude(['']) == self.all_paths + assert self.exclude(['']) == convert_paths(self.all_paths) def test_no_dupes(self): paths = exclude_paths(self.base, ['!a.py']) @@ -858,7 +866,9 @@ def test_exclude_dockerfile_dockerignore(self): Dockerfile and/or .dockerignore, don't exclude them from the actual tar file. """ - assert self.exclude(['Dockerfile', '.dockerignore']) == self.all_paths + assert self.exclude(['Dockerfile', '.dockerignore']) == convert_paths( + self.all_paths + ) def test_exclude_custom_dockerfile(self): """ @@ -877,94 +887,116 @@ def test_exclude_dockerfile_child(self): assert 'foo/a.py' not in includes def test_single_filename(self): - assert self.exclude(['a.py']) == self.all_paths - set(['a.py']) + assert self.exclude(['a.py']) == convert_paths( + self.all_paths - set(['a.py']) + ) def test_single_filename_leading_dot_slash(self): - assert self.exclude(['./a.py']) == self.all_paths - set(['a.py']) + assert self.exclude(['./a.py']) == convert_paths( + self.all_paths - set(['a.py']) + ) # As odd as it sounds, a filename pattern with a trailing slash on the # end *will* result in that file being excluded. def test_single_filename_trailing_slash(self): - assert self.exclude(['a.py/']) == self.all_paths - set(['a.py']) + assert self.exclude(['a.py/']) == convert_paths( + self.all_paths - set(['a.py']) + ) def test_wildcard_filename_start(self): - assert self.exclude(['*.py']) == self.all_paths - set([ - 'a.py', 'b.py', 'cde.py', - ]) + assert self.exclude(['*.py']) == convert_paths( + self.all_paths - set(['a.py', 'b.py', 'cde.py']) + ) def test_wildcard_with_exception(self): - assert self.exclude(['*.py', '!b.py']) == self.all_paths - set([ - 'a.py', 'cde.py', - ]) + assert self.exclude(['*.py', '!b.py']) == convert_paths( + self.all_paths - set(['a.py', 'cde.py']) + ) def test_wildcard_with_wildcard_exception(self): - assert self.exclude(['*.*', '!*.go']) == self.all_paths - set([ - 'a.py', 'b.py', 'cde.py', 'Dockerfile.alt', - ]) + assert self.exclude(['*.*', '!*.go']) == convert_paths( + self.all_paths - set([ + 'a.py', 'b.py', 'cde.py', 'Dockerfile.alt', + ]) + ) def test_wildcard_filename_end(self): - assert self.exclude(['a.*']) == self.all_paths - set(['a.py', 'a.go']) + assert self.exclude(['a.*']) == convert_paths( + self.all_paths - set(['a.py', 'a.go']) + ) def test_question_mark(self): - assert self.exclude(['?.py']) == self.all_paths - set(['a.py', 'b.py']) + assert self.exclude(['?.py']) == convert_paths( + self.all_paths - set(['a.py', 'b.py']) + ) def test_single_subdir_single_filename(self): - assert self.exclude(['foo/a.py']) == self.all_paths - set(['foo/a.py']) + assert self.exclude(['foo/a.py']) == convert_paths( + self.all_paths - set(['foo/a.py']) + ) def test_single_subdir_with_path_traversal(self): - assert self.exclude(['foo/whoops/../a.py']) == self.all_paths - set([ - 'foo/a.py', - ]) + assert self.exclude(['foo/whoops/../a.py']) == convert_paths( + self.all_paths - set(['foo/a.py']) + ) def test_single_subdir_wildcard_filename(self): - assert self.exclude(['foo/*.py']) == self.all_paths - set([ - 'foo/a.py', 'foo/b.py', - ]) + assert self.exclude(['foo/*.py']) == convert_paths( + self.all_paths - set(['foo/a.py', 'foo/b.py']) + ) def test_wildcard_subdir_single_filename(self): - assert self.exclude(['*/a.py']) == self.all_paths - set([ - 'foo/a.py', 'bar/a.py', - ]) + assert self.exclude(['*/a.py']) == convert_paths( + self.all_paths - set(['foo/a.py', 'bar/a.py']) + ) def test_wildcard_subdir_wildcard_filename(self): - assert self.exclude(['*/*.py']) == self.all_paths - set([ - 'foo/a.py', 'foo/b.py', 'bar/a.py', - ]) + assert self.exclude(['*/*.py']) == convert_paths( + self.all_paths - set(['foo/a.py', 'foo/b.py', 'bar/a.py']) + ) def test_directory(self): - assert self.exclude(['foo']) == self.all_paths - set([ - 'foo', 'foo/a.py', 'foo/b.py', - 'foo/bar', 'foo/bar/a.py', 'foo/Dockerfile3' - ]) + assert self.exclude(['foo']) == convert_paths( + self.all_paths - set([ + 'foo', 'foo/a.py', 'foo/b.py', 'foo/bar', 'foo/bar/a.py', + 'foo/Dockerfile3' + ]) + ) def test_directory_with_trailing_slash(self): - assert self.exclude(['foo']) == self.all_paths - set([ - 'foo', 'foo/a.py', 'foo/b.py', - 'foo/bar', 'foo/bar/a.py', 'foo/Dockerfile3' - ]) + assert self.exclude(['foo']) == convert_paths( + self.all_paths - set([ + 'foo', 'foo/a.py', 'foo/b.py', + 'foo/bar', 'foo/bar/a.py', 'foo/Dockerfile3' + ]) + ) def test_directory_with_single_exception(self): - assert self.exclude(['foo', '!foo/bar/a.py']) == self.all_paths - set([ - 'foo/a.py', 'foo/b.py', 'foo', 'foo/bar', - 'foo/Dockerfile3' - ]) + assert self.exclude(['foo', '!foo/bar/a.py']) == convert_paths( + self.all_paths - set([ + 'foo/a.py', 'foo/b.py', 'foo', 'foo/bar', + 'foo/Dockerfile3' + ]) + ) def test_directory_with_subdir_exception(self): - assert self.exclude(['foo', '!foo/bar']) == self.all_paths - set([ - 'foo/a.py', 'foo/b.py', 'foo', - 'foo/Dockerfile3' - ]) + assert self.exclude(['foo', '!foo/bar']) == convert_paths( + self.all_paths - set([ + 'foo/a.py', 'foo/b.py', 'foo', 'foo/Dockerfile3' + ]) + ) def test_directory_with_wildcard_exception(self): - assert self.exclude(['foo', '!foo/*.py']) == self.all_paths - set([ - 'foo/bar', 'foo/bar/a.py', 'foo', - 'foo/Dockerfile3' - ]) + assert self.exclude(['foo', '!foo/*.py']) == convert_paths( + self.all_paths - set([ + 'foo/bar', 'foo/bar/a.py', 'foo', 'foo/Dockerfile3' + ]) + ) def test_subdirectory(self): - assert self.exclude(['foo/bar']) == self.all_paths - set([ - 'foo/bar', 'foo/bar/a.py', - ]) + assert self.exclude(['foo/bar']) == convert_paths( + self.all_paths - set(['foo/bar', 'foo/bar/a.py']) + ) class TarTest(base.Cleanup, base.BaseTestCase): @@ -1023,6 +1055,7 @@ def test_tar_with_empty_directory(self): tar_data = tarfile.open(fileobj=archive) self.assertEqual(sorted(tar_data.getnames()), ['bar', 'foo']) + @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='No symlinks on Windows') def test_tar_with_file_symlinks(self): base = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, base) @@ -1036,6 +1069,7 @@ def test_tar_with_file_symlinks(self): sorted(tar_data.getnames()), ['bar', 'bar/foo', 'foo'] ) + @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='No symlinks on Windows') def test_tar_with_directory_symlinks(self): base = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, base) From 9b35c74f0e7626e360dfa2f00202d23b8d08b7a5 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 13 Oct 2016 16:45:01 -0700 Subject: [PATCH 0145/1301] Fix dockerignore exclusion logic on Windows Signed-off-by: Joffrey F --- docker/utils/utils.py | 4 ++-- tests/integration/container_test.py | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/docker/utils/utils.py b/docker/utils/utils.py index f2f7c2686a..d89aecf30e 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -209,8 +209,8 @@ def match_path(path, pattern): if pattern: pattern = os.path.relpath(pattern) - pattern_components = pattern.split('/') - path_components = path.split('/')[:len(pattern_components)] + pattern_components = pattern.split(os.path.sep) + path_components = path.split(os.path.sep)[:len(pattern_components)] return fnmatch('/'.join(path_components), pattern) diff --git a/tests/integration/container_test.py b/tests/integration/container_test.py index 5ced082da7..838ec3649f 100644 --- a/tests/integration/container_test.py +++ b/tests/integration/container_test.py @@ -414,6 +414,9 @@ def setUp(self): ['touch', os.path.join(self.mount_dest, self.filename)], ) + @pytest.mark.xfail( + IS_WINDOWS_PLATFORM, reason='Test not designed for Windows platform' + ) def test_create_with_binds_rw(self): container = self.run_with_volume( @@ -429,6 +432,9 @@ def test_create_with_binds_rw(self): inspect_data = self.client.inspect_container(container) self.check_container_data(inspect_data, True) + @pytest.mark.xfail( + IS_WINDOWS_PLATFORM, reason='Test not designed for Windows platform' + ) def test_create_with_binds_ro(self): self.run_with_volume( False, From a3981f891d9ca52b145ba875c4c0e6d72c36a716 Mon Sep 17 00:00:00 2001 From: Walker Lee Date: Thu, 27 Oct 2016 00:12:56 +0800 Subject: [PATCH 0146/1301] Add docker network IPAM options parameter Signed-off-by: Walker Lee --- docker/utils/utils.py | 8 ++++++-- docs/networks.md | 1 + tests/unit/network_test.py | 3 ++- tests/unit/utils_test.py | 3 ++- 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/docker/utils/utils.py b/docker/utils/utils.py index d89aecf30e..13254b9ebf 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -47,10 +47,14 @@ def create_ipam_pool(subnet=None, iprange=None, gateway=None, } -def create_ipam_config(driver='default', pool_configs=None): +def create_ipam_config(driver='default', pool_configs=None, options=None): + if options is not None and not isinstance(options, dict): + raise TypeError('IPAM options must be a dictionary') + return { 'Driver': driver, - 'Config': pool_configs or [] + 'Config': pool_configs or [], + 'Options': options } diff --git a/docs/networks.md b/docs/networks.md index fb0e9f420c..d9829909bf 100644 --- a/docs/networks.md +++ b/docs/networks.md @@ -137,6 +137,7 @@ Create an IPAM (IP Address Management) config dictionary to be used with * driver (str): The IPAM driver to use. Defaults to `'default'`. * pool_configs (list): A list of pool configuration dictionaries as created by `docker.utils.create_ipam_pool`. Defaults to empty list. +* options (dict): Driver options as a key-value dictionary. Defaults to `None`. **Returns** An IPAM config dictionary diff --git a/tests/unit/network_test.py b/tests/unit/network_test.py index 93f03da4bc..a79132eddd 100644 --- a/tests/unit/network_test.py +++ b/tests/unit/network_test.py @@ -100,7 +100,8 @@ def test_create_network(self): "Gateway": "192.168.52.254", "Subnet": "192.168.52.0/24", "AuxiliaryAddresses": None, - }] + }], + "Options": None } }) diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index 290874f416..a87293deb1 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -691,7 +691,8 @@ def test_create_ipam_config(self): 'Gateway': '192.168.52.254', 'AuxiliaryAddresses': None, 'IPRange': None, - }] + }], + 'Options': None }) From 163a1ce371f8a661e2e170293d8cd9312121da35 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 26 Oct 2016 18:04:25 -0700 Subject: [PATCH 0147/1301] Implement retry logic when the npipe open procedure fails with ERROR_PIPE_BUSY Signed-off-by: Joffrey F --- docker/transport/npipesocket.py | 34 +++++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/docker/transport/npipesocket.py b/docker/transport/npipesocket.py index a9bf0ccba5..509cabfb37 100644 --- a/docker/transport/npipesocket.py +++ b/docker/transport/npipesocket.py @@ -5,9 +5,11 @@ import win32file import win32pipe +cERROR_PIPE_BUSY = 0xe7 cSECURITY_SQOS_PRESENT = 0x100000 cSECURITY_ANONYMOUS = 0 -cPIPE_READMODE_MESSAGE = 2 + +RETRY_WAIT_TIMEOUT = 10000 def check_closed(f): @@ -46,15 +48,27 @@ def close(self): @check_closed def connect(self, address): win32pipe.WaitNamedPipe(address, self._timeout) - handle = win32file.CreateFile( - address, - win32file.GENERIC_READ | win32file.GENERIC_WRITE, - 0, - None, - win32file.OPEN_EXISTING, - cSECURITY_ANONYMOUS | cSECURITY_SQOS_PRESENT, - 0 - ) + try: + handle = win32file.CreateFile( + address, + win32file.GENERIC_READ | win32file.GENERIC_WRITE, + 0, + None, + win32file.OPEN_EXISTING, + cSECURITY_ANONYMOUS | cSECURITY_SQOS_PRESENT, + 0 + ) + except win32pipe.error as e: + # See Remarks: + # https://msdn.microsoft.com/en-us/library/aa365800.aspx + if e.winerror == cERROR_PIPE_BUSY: + # Another program or thread has grabbed our pipe instance + # before we got to it. Wait for availability and attempt to + # connect again. + win32pipe.WaitNamedPipe(address, RETRY_WAIT_TIMEOUT) + return self.connect(address) + raise e + self.flags = win32pipe.GetNamedPipeInfo(handle)[0] self._handle = handle From 54586053b1f72573b4881dc1982b4aed494b6827 Mon Sep 17 00:00:00 2001 From: Alessandro Boch Date: Tue, 1 Nov 2016 16:55:39 -0700 Subject: [PATCH 0148/1301] Changes for service tests to pass in docker master - update config structure has new members - service name update is no longer supported Signed-off-by: Alessandro Boch --- tests/integration/service_test.py | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 960098ace5..68510c536f 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -150,7 +150,10 @@ def test_create_service_with_update_config(self): ) svc_info = self.client.inspect_service(svc_id) assert 'UpdateConfig' in svc_info['Spec'] - assert update_config == svc_info['Spec']['UpdateConfig'] + uc = svc_info['Spec']['UpdateConfig'] + assert update_config['Parallelism'] == uc['Parallelism'] + assert update_config['Delay'] == uc['Delay'] + assert update_config['FailureAction'] == uc['FailureAction'] def test_create_service_with_restart_policy(self): container_spec = docker.types.ContainerSpec('busybox', ['true']) @@ -166,15 +169,3 @@ def test_create_service_with_restart_policy(self): svc_info = self.client.inspect_service(svc_id) assert 'RestartPolicy' in svc_info['Spec']['TaskTemplate'] assert policy == svc_info['Spec']['TaskTemplate']['RestartPolicy'] - - def test_update_service_name(self): - name, svc_id = self.create_simple_service() - svc_info = self.client.inspect_service(svc_id) - svc_version = svc_info['Version']['Index'] - new_name = self.get_service_name() - assert self.client.update_service( - svc_id, svc_version, name=new_name, - task_template=svc_info['Spec']['TaskTemplate'] - ) - svc_info = self.client.inspect_service(svc_id) - assert svc_info['Spec']['Name'] == new_name From 422bc2beb2d3c7dd46541f2e6ca3b982e5091520 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 2 Nov 2016 16:28:23 -0700 Subject: [PATCH 0149/1301] Fix NpipeSocket.settimeout to match expected behavior Signed-off-by: Joffrey F --- docker/transport/npipesocket.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/docker/transport/npipesocket.py b/docker/transport/npipesocket.py index 509cabfb37..c04b39dd66 100644 --- a/docker/transport/npipesocket.py +++ b/docker/transport/npipesocket.py @@ -170,13 +170,16 @@ def setblocking(self, flag): def settimeout(self, value): if value is None: - self._timeout = win32pipe.NMPWAIT_NOWAIT + # Blocking mode + self._timeout = win32pipe.NMPWAIT_WAIT_FOREVER elif not isinstance(value, (float, int)) or value < 0: raise ValueError('Timeout value out of range') elif value == 0: - self._timeout = win32pipe.NMPWAIT_USE_DEFAULT_WAIT + # Non-blocking mode + self._timeout = win32pipe.NMPWAIT_NO_WAIT else: - self._timeout = value + # Timeout mode - Value converted to milliseconds + self._timeout = value * 1000 def gettimeout(self): return self._timeout From be2ae8df36271cd49cde308d1bb86609624d3b2c Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 29 Sep 2016 17:58:54 -0700 Subject: [PATCH 0150/1301] WIP Signed-off-by: Joffrey F --- docker/api/service.py | 16 ++++++-- docker/types/__init__.py | 4 +- docker/types/services.py | 36 ++++++++++++++++++ docker/utils/__init__.py | 1 + docker/utils/utils.py | 14 +++++++ docs/services.md | 16 +++++++- tests/integration/service_test.py | 62 +++++++++++++++++++++++++++++++ 7 files changed, 143 insertions(+), 6 deletions(-) diff --git a/docker/api/service.py b/docker/api/service.py index baebbadfe4..39a23e4f08 100644 --- a/docker/api/service.py +++ b/docker/api/service.py @@ -1,3 +1,5 @@ +import warnings + from .. import errors from .. import utils from ..auth import auth @@ -7,8 +9,16 @@ class ServiceApiMixin(object): @utils.minimum_version('1.24') def create_service( self, task_template, name=None, labels=None, mode=None, - update_config=None, networks=None, endpoint_config=None + update_config=None, networks=None, endpoint_config=None, + endpoint_spec=None ): + if endpoint_config is not None: + warnings.warn( + 'endpoint_config has been renamed to endpoint_spec.', + DeprecationWarning + ) + endpoint_spec = endpoint_config + url = self._url('/services/create') headers = {} image = task_template.get('ContainerSpec', {}).get('Image', None) @@ -26,8 +36,8 @@ def create_service( 'TaskTemplate': task_template, 'Mode': mode, 'UpdateConfig': update_config, - 'Networks': networks, - 'Endpoint': endpoint_config + 'Networks': utils.convert_service_networks(networks), + 'EndpointSpec': endpoint_spec } return self._result( self._post_json(url, data=data, headers=headers), True diff --git a/docker/types/__init__.py b/docker/types/__init__.py index 3609581d92..71c0c97403 100644 --- a/docker/types/__init__.py +++ b/docker/types/__init__.py @@ -1,7 +1,7 @@ # flake8: noqa from .containers import LogConfig, Ulimit from .services import ( - ContainerSpec, DriverConfig, Mount, Resources, RestartPolicy, TaskTemplate, - UpdateConfig + ContainerSpec, DriverConfig, EndpointSpec, Mount, Resources, RestartPolicy, + TaskTemplate, UpdateConfig ) from .swarm import SwarmSpec, SwarmExternalCA diff --git a/docker/types/services.py b/docker/types/services.py index 063779cd88..2ac47ebd08 100644 --- a/docker/types/services.py +++ b/docker/types/services.py @@ -12,6 +12,8 @@ def __init__(self, container_spec, resources=None, restart_policy=None, if restart_policy: self['RestartPolicy'] = restart_policy if placement: + if isinstance(placement, list): + placement = {'Constraints': placement} self['Placement'] = placement if log_driver: self['LogDriver'] = log_driver @@ -179,3 +181,37 @@ def __init__(self, name, options=None): self['Name'] = name if options: self['Options'] = options + + +class EndpointSpec(dict): + def __init__(self, mode=None, ports=None): + if ports: + self['Ports'] = convert_service_ports(ports) + if mode: + self['Mode'] = mode + + +def convert_service_ports(ports): + if isinstance(ports, list): + return ports + if not isinstance(ports, dict): + raise TypeError( + 'Invalid type for ports, expected dict or list' + ) + + result = [] + for k, v in six.iteritems(ports): + port_spec = { + 'Protocol': 'tcp', + 'PublishedPort': k + } + + if isinstance(v, tuple): + port_spec['TargetPort'] = v[0] + if len(v) == 2: + port_spec['Protocol'] = v[1] + else: + port_spec['TargetPort'] = v + + result.append(port_spec) + return result diff --git a/docker/utils/__init__.py b/docker/utils/__init__.py index 4bb3876e65..5bd69b4d19 100644 --- a/docker/utils/__init__.py +++ b/docker/utils/__init__.py @@ -6,6 +6,7 @@ create_host_config, create_container_config, parse_bytes, ping_registry, parse_env_file, version_lt, version_gte, decode_json_header, split_command, create_ipam_config, create_ipam_pool, parse_devices, normalize_links, + convert_service_networks, ) from ..types import LogConfig, Ulimit diff --git a/docker/utils/utils.py b/docker/utils/utils.py index d89aecf30e..97261cdc79 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -376,6 +376,20 @@ def convert_tmpfs_mounts(tmpfs): return result +def convert_service_networks(networks): + if not networks: + return networks + if not isinstance(networks, list): + raise TypeError('networks parameter must be a list.') + + result = [] + for n in networks: + if isinstance(n, six.string_types): + n = {'Target': n} + result.append(n) + return result + + def parse_repository_tag(repo_name): parts = repo_name.rsplit('@', 1) if len(parts) == 2: diff --git a/docs/services.md b/docs/services.md index a6bb7d63c0..3935205a8e 100644 --- a/docs/services.md +++ b/docs/services.md @@ -82,7 +82,7 @@ Create a service. See the [UpdateConfig class](#UpdateConfig) for details. Default: `None`. * networks (list): List of network names or IDs to attach the service to. Default: `None`. -* endpoint_config (dict): Properties that can be configured to access and load +* endpoint_spec (dict): Properties that can be configured to access and load balance a service. Default: `None`. **Returns:** A dictionary containing an `ID` key for the newly created service. @@ -174,6 +174,20 @@ and for the `driver_config` in a volume `Mount`. * name (string): Name of the logging driver to use. * options (dict): Driver-specific options. Default: `None`. +#### EndpointSpec + +An `EndpointSpec` object describes properties to access and load-balance a +service. + +**Params:** + +* mode (string): The mode of resolution to use for internal load balancing + between tasks (`'vip'` or `'dnsrr'`). Defaults to `'vip'` if not provided. +* ports (dict): Exposed ports that this service is accessible on from the + outside, in the form of `{ target_port: published_port }` or + `{ target_port: (published_port, protocol) }`. Ports can only be provided if + the `vip` resolution mode is used. + #### Mount A `Mount` object describes a mounted folder's configuration inside a diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 68510c536f..636f507942 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -169,3 +169,65 @@ def test_create_service_with_restart_policy(self): svc_info = self.client.inspect_service(svc_id) assert 'RestartPolicy' in svc_info['Spec']['TaskTemplate'] assert policy == svc_info['Spec']['TaskTemplate']['RestartPolicy'] + + def test_create_service_with_custom_networks(self): + net1 = self.client.create_network( + 'dockerpytest_1', driver='overlay', ipam={'Driver': 'default'} + ) + self.tmp_networks.append(net1['Id']) + net2 = self.client.create_network( + 'dockerpytest_2', driver='overlay', ipam={'Driver': 'default'} + ) + self.tmp_networks.append(net2['Id']) + container_spec = docker.types.ContainerSpec('busybox', ['true']) + task_tmpl = docker.types.TaskTemplate(container_spec) + name = self.get_service_name() + svc_id = self.client.create_service( + task_tmpl, name=name, networks=[ + 'dockerpytest_1', {'Target': 'dockerpytest_2'} + ] + ) + svc_info = self.client.inspect_service(svc_id) + assert 'Networks' in svc_info['Spec'] + assert svc_info['Spec']['Networks'] == [ + {'Target': net1['Id']}, {'Target': net2['Id']} + ] + + def test_create_service_with_placement(self): + node_id = self.client.nodes()[0]['ID'] + container_spec = docker.types.ContainerSpec('busybox', ['true']) + task_tmpl = docker.types.TaskTemplate( + container_spec, placement=['node.id=={}'.format(node_id)] + ) + name = self.get_service_name() + svc_id = self.client.create_service(task_tmpl, name=name) + svc_info = self.client.inspect_service(svc_id) + assert 'Placement' in svc_info['Spec']['TaskTemplate'] + assert (svc_info['Spec']['TaskTemplate']['Placement'] == + {'Constraints': ['node.id=={}'.format(node_id)]}) + + def test_create_service_with_endpoint_spec(self): + container_spec = docker.types.ContainerSpec('busybox', ['true']) + task_tmpl = docker.types.TaskTemplate(container_spec) + name = self.get_service_name() + endpoint_spec = docker.types.EndpointSpec(ports={ + 12357: (1990, 'udp'), + 12562: (678,), + 53243: 8080, + }) + svc_id = self.client.create_service( + task_tmpl, name=name, endpoint_spec=endpoint_spec + ) + svc_info = self.client.inspect_service(svc_id) + print(svc_info) + ports = svc_info['Spec']['EndpointSpec']['Ports'] + assert { + 'PublishedPort': 12562, 'TargetPort': 678, 'Protocol': 'tcp' + } in ports + assert { + 'PublishedPort': 53243, 'TargetPort': 8080, 'Protocol': 'tcp' + } in ports + assert { + 'PublishedPort': 12357, 'TargetPort': 1990, 'Protocol': 'udp' + } in ports + assert len(ports) == 3 From 3ac73a285b2f370f6aa300d8a55c5af55660d0f4 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 30 Sep 2016 17:46:58 -0700 Subject: [PATCH 0151/1301] Fix endpoint spec and networks params in update_service Signed-off-by: Joffrey F --- docker/api/service.py | 17 +++++++++++++---- docs/services.md | 2 +- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/docker/api/service.py b/docker/api/service.py index 39a23e4f08..2e41b7cd37 100644 --- a/docker/api/service.py +++ b/docker/api/service.py @@ -83,7 +83,16 @@ def tasks(self, filters=None): @utils.check_resource def update_service(self, service, version, task_template=None, name=None, labels=None, mode=None, update_config=None, - networks=None, endpoint_config=None): + networks=None, endpoint_config=None, + endpoint_spec=None): + + if endpoint_config is not None: + warnings.warn( + 'endpoint_config has been renamed to endpoint_spec.', + DeprecationWarning + ) + endpoint_spec = endpoint_config + url = self._url('/services/{0}/update', service) data = {} headers = {} @@ -104,9 +113,9 @@ def update_service(self, service, version, task_template=None, name=None, if update_config is not None: data['UpdateConfig'] = update_config if networks is not None: - data['Networks'] = networks - if endpoint_config is not None: - data['Endpoint'] = endpoint_config + data['Networks'] = utils.convert_service_networks(networks) + if endpoint_spec is not None: + data['EndpointSpec'] = endpoint_spec resp = self._post_json( url, data=data, params={'version': version}, headers=headers diff --git a/docs/services.md b/docs/services.md index 3935205a8e..69e0649054 100644 --- a/docs/services.md +++ b/docs/services.md @@ -137,7 +137,7 @@ Update a service. See the [UpdateConfig class](#UpdateConfig) for details. Default: `None`. * networks (list): List of network names or IDs to attach the service to. Default: `None`. -* endpoint_config (dict): Properties that can be configured to access and load +* endpoint_spec (dict): Properties that can be configured to access and load balance a service. Default: `None`. **Returns:** `True` if successful. Raises an `APIError` otherwise. From 3c7c2319837235ccee258ca9604bd7f763f4c86b Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Mon, 7 Nov 2016 12:20:20 -0800 Subject: [PATCH 0152/1301] Fix broken unit test Introduced by https://github.com/docker/docker-py/pull/1230 Signed-off-by: Ben Firshman --- tests/unit/swarm_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/swarm_test.py b/tests/unit/swarm_test.py index 5580383406..39d4ec46fc 100644 --- a/tests/unit/swarm_test.py +++ b/tests/unit/swarm_test.py @@ -3,7 +3,7 @@ import json from . import fake_api -from ..base import requires_api_version +from ..helpers import requires_api_version from .api_test import (DockerClientTest, url_prefix, fake_request) From 98e2e1fcd629c161c09be9abd67dd0f4194feb38 Mon Sep 17 00:00:00 2001 From: Ryan Belgrave Date: Mon, 17 Oct 2016 15:26:45 -0500 Subject: [PATCH 0153/1301] Add labels and shmsize arguments to the image build Signed-off-by: Ryan Belgrave --- docker/api/build.py | 19 ++++++++++++++++- docs/api.md | 3 +++ tests/integration/build_test.py | 38 +++++++++++++++++++++++++++++++++ 3 files changed, 59 insertions(+), 1 deletion(-) diff --git a/docker/api/build.py b/docker/api/build.py index 74037167e7..68aa9621d2 100644 --- a/docker/api/build.py +++ b/docker/api/build.py @@ -17,7 +17,8 @@ def build(self, path=None, tag=None, quiet=False, fileobj=None, nocache=False, rm=False, stream=False, timeout=None, custom_context=False, encoding=None, pull=False, forcerm=False, dockerfile=None, container_limits=None, - decode=False, buildargs=None, gzip=False): + decode=False, buildargs=None, gzip=False, shmsize=None, + labels=None): remote = context = None headers = {} container_limits = container_limits or {} @@ -88,6 +89,22 @@ def build(self, path=None, tag=None, quiet=False, fileobj=None, 'buildargs was only introduced in API version 1.21' ) + if shmsize: + if utils.version_gte(self._version, '1.22'): + params.update({'shmsize': shmsize}) + else: + raise errors.InvalidVersion( + 'shmsize was only introduced in API version 1.22' + ) + + if labels: + if utils.version_gte(self._version, '1.23'): + params.update({'labels': json.dumps(labels)}) + else: + raise errors.InvalidVersion( + 'labels was only introduced in API version 1.23' + ) + if context is not None: headers = {'Content-Type': 'application/tar'} if encoding: diff --git a/docs/api.md b/docs/api.md index 4fa63b3f4c..fdf3e278ab 100644 --- a/docs/api.md +++ b/docs/api.md @@ -76,6 +76,9 @@ correct value (e.g `gzip`). - cpusetcpus (str): CPUs in which to allow execution, e.g., `"0-3"`, `"0,1"` * decode (bool): If set to `True`, the returned stream will be decoded into dicts on the fly. Default `False`. +* shmsize (int): Size of /dev/shm in bytes. The size must be greater + than 0. If omitted the system uses 64MB. +* labels (dict): A dictionary of labels to set on the image **Returns** (generator): A generator for the build output diff --git a/tests/integration/build_test.py b/tests/integration/build_test.py index 699345fcce..2695b92aa2 100644 --- a/tests/integration/build_test.py +++ b/tests/integration/build_test.py @@ -118,6 +118,44 @@ def test_build_with_buildargs(self): info = self.client.inspect_image('buildargs') self.assertEqual(info['Config']['User'], 'OK') + @requires_api_version('1.22') + def test_build_shmsize(self): + script = io.BytesIO('\n'.join([ + 'FROM scratch', + 'CMD sh -c "echo \'Hello, World!\'"', + ]).encode('ascii')) + + tag = 'shmsize' + shmsize = 134217728 + + stream = self.client.build( + fileobj=script, tag=tag, shmsize=shmsize + ) + self.tmp_imgs.append(tag) + for chunk in stream: + pass + + # There is currently no way to get the shmsize + # that was used to build the image + + @requires_api_version('1.23') + def test_build_labels(self): + script = io.BytesIO('\n'.join([ + 'FROM scratch', + ]).encode('ascii')) + + labels = {'test': 'OK'} + + stream = self.client.build( + fileobj=script, tag='labels', labels=labels + ) + self.tmp_imgs.append('labels') + for chunk in stream: + pass + + info = self.client.inspect_image('labels') + self.assertEqual(info['Config']['Labels'], labels) + def test_build_stderr_data(self): control_chars = ['\x1b[91m', '\x1b[0m'] snippet = 'Ancient Temple (Mystic Oriental Dream ~ Ancient Temple)' From d6ffe9aa0db75bc3f340bd394a40bfcb4a8be875 Mon Sep 17 00:00:00 2001 From: bin liu Date: Fri, 11 Nov 2016 20:18:08 +0800 Subject: [PATCH 0154/1301] fix JSON key typo, it should not be underscores, but should be camelCase with first letter capital Signed-off-by: bin liu --- docker/types/services.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/types/services.py b/docker/types/services.py index 2ac47ebd08..a95e0f2ce3 100644 --- a/docker/types/services.py +++ b/docker/types/services.py @@ -94,7 +94,7 @@ def __init__(self, target, source, type='volume', read_only=False, if labels: volume_opts['Labels'] = labels if driver_config: - volume_opts['driver_config'] = driver_config + volume_opts['DriverConfig'] = driver_config if volume_opts: self['VolumeOptions'] = volume_opts if propagation: From 6fad0855b5abdde54c1c6993893c74277dd03aa8 Mon Sep 17 00:00:00 2001 From: Simon Li Date: Fri, 11 Nov 2016 16:11:18 +0000 Subject: [PATCH 0155/1301] Use format_environment to convert env in ContainerSpec Signed-off-by: Simon Li --- docker/types/services.py | 6 +++++- docker/utils/__init__.py | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/docker/types/services.py b/docker/types/services.py index 2ac47ebd08..0ede7762da 100644 --- a/docker/types/services.py +++ b/docker/types/services.py @@ -39,6 +39,7 @@ class ContainerSpec(dict): def __init__(self, image, command=None, args=None, env=None, workdir=None, user=None, labels=None, mounts=None, stop_grace_period=None): from ..utils import split_command # FIXME: circular import + from ..utils import format_environment # FIXME: circular import self['Image'] = image @@ -48,7 +49,10 @@ def __init__(self, image, command=None, args=None, env=None, workdir=None, self['Args'] = args if env is not None: - self['Env'] = env + if isinstance(env, dict): + self['Env'] = format_environment(env) + else: + self['Env'] = env if workdir is not None: self['Dir'] = workdir if user is not None: diff --git a/docker/utils/__init__.py b/docker/utils/__init__.py index 5bd69b4d19..e834505b4b 100644 --- a/docker/utils/__init__.py +++ b/docker/utils/__init__.py @@ -6,7 +6,7 @@ create_host_config, create_container_config, parse_bytes, ping_registry, parse_env_file, version_lt, version_gte, decode_json_header, split_command, create_ipam_config, create_ipam_pool, parse_devices, normalize_links, - convert_service_networks, + convert_service_networks, format_environment, ) from ..types import LogConfig, Ulimit From cb967ef6825d62d10d2bebeda5ab89be8056a014 Mon Sep 17 00:00:00 2001 From: Simon Li Date: Fri, 11 Nov 2016 16:29:43 +0000 Subject: [PATCH 0156/1301] Add test for creating service with env Signed-off-by: Simon Li --- tests/integration/service_test.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 636f507942..072566315e 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -231,3 +231,19 @@ def test_create_service_with_endpoint_spec(self): 'PublishedPort': 12357, 'TargetPort': 1990, 'Protocol': 'udp' } in ports assert len(ports) == 3 + + def test_create_service_with_env(self): + container_spec = docker.types.ContainerSpec( + 'busybox', ['true'], env={'DOCKER_PY_TEST': 1} + ) + task_tmpl = docker.types.TaskTemplate( + container_spec, + ) + name = self.get_service_name() + svc_id = self.client.create_service(task_tmpl, name=name) + svc_info = self.client.inspect_service(svc_id) + assert 'TaskTemplate' in svc_info['Spec'] + assert 'ContainerSpec' in svc_info['Spec']['TaskTemplate'] + con_spec = svc_info['Spec']['TaskTemplate']['ContainerSpec'] + assert 'Env' in con_spec + assert con_spec['Env'] == ['DOCKER_PY_TEST=1'] From 9a485b30ee465d66ef9e6bb17ca721a1f9c218a7 Mon Sep 17 00:00:00 2001 From: Tomas Tomecek Date: Tue, 1 Nov 2016 11:52:25 +0100 Subject: [PATCH 0157/1301] ssl,test: OpenSSL may not support ssl2 Fixes #1265 Signed-off-by: Tomas Tomecek --- tests/unit/ssladapter_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/unit/ssladapter_test.py b/tests/unit/ssladapter_test.py index 2ad1cad177..8c905cf589 100644 --- a/tests/unit/ssladapter_test.py +++ b/tests/unit/ssladapter_test.py @@ -24,7 +24,8 @@ def test_only_uses_tls(self): ssl_context = ssladapter.urllib3.util.ssl_.create_urllib3_context() assert ssl_context.options & OP_NO_SSLv3 - assert ssl_context.options & OP_NO_SSLv2 + # if OpenSSL is compiled without SSL2 support, OP_NO_SSLv2 will be 0 + assert not bool(OP_NO_SSLv2) or ssl_context.options & OP_NO_SSLv2 assert not ssl_context.options & OP_NO_TLSv1 From c66c2d6fa563c5ee7e7df63cea2f9ac79ea7eda2 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Mon, 14 Nov 2016 18:20:13 +0000 Subject: [PATCH 0158/1301] Fix linting error This seems to have been ignored by older versions of flake8, and fixed in version 3.1.0 or 3.1.1. Signed-off-by: Aanand Prasad --- tests/unit/api_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit/api_test.py b/tests/unit/api_test.py index 94092dd220..aa19c90058 100644 --- a/tests/unit/api_test.py +++ b/tests/unit/api_test.py @@ -86,6 +86,7 @@ def fake_delete(self, url, *args, **kwargs): def fake_read_from_socket(self, response, stream): return six.binary_type() + url_base = '{0}/'.format(fake_api.prefix) url_prefix = '{0}v{1}/'.format( url_base, From b4f2b5fa70a513a89df08a100cce63a9727b6580 Mon Sep 17 00:00:00 2001 From: Jamie Greeff Date: Thu, 4 Aug 2016 15:17:06 +0100 Subject: [PATCH 0159/1301] Add support for passing healthcheck to create_container Signed-off-by: Jamie Greeff --- docker/api/container.py | 7 ++-- docker/types/__init__.py | 1 + docker/types/healthcheck.py | 47 +++++++++++++++++++++++++++ docker/utils/utils.py | 9 ++++- tests/integration/healthcheck_test.py | 35 ++++++++++++++++++++ 5 files changed, 95 insertions(+), 4 deletions(-) create mode 100644 docker/types/healthcheck.py create mode 100644 tests/integration/healthcheck_test.py diff --git a/docker/api/container.py b/docker/api/container.py index d71d17ad8e..338b79fe77 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -115,7 +115,8 @@ def create_container(self, image, command=None, hostname=None, user=None, cpu_shares=None, working_dir=None, domainname=None, memswap_limit=None, cpuset=None, host_config=None, mac_address=None, labels=None, volume_driver=None, - stop_signal=None, networking_config=None): + stop_signal=None, networking_config=None, + healthcheck=None): if isinstance(volumes, six.string_types): volumes = [volumes, ] @@ -130,7 +131,7 @@ def create_container(self, image, command=None, hostname=None, user=None, tty, mem_limit, ports, environment, dns, volumes, volumes_from, network_disabled, entrypoint, cpu_shares, working_dir, domainname, memswap_limit, cpuset, host_config, mac_address, labels, - volume_driver, stop_signal, networking_config, + volume_driver, stop_signal, networking_config, healthcheck, ) return self.create_container_from_config(config, name) @@ -365,7 +366,7 @@ def start(self, container, binds=None, port_bindings=None, lxc_conf=None, cap_drop=cap_drop, volumes_from=volumes_from, devices=devices, network_mode=network_mode, restart_policy=restart_policy, extra_hosts=extra_hosts, read_only=read_only, pid_mode=pid_mode, - ipc_mode=ipc_mode, security_opt=security_opt, ulimits=ulimits + ipc_mode=ipc_mode, security_opt=security_opt, ulimits=ulimits, ) start_config = None diff --git a/docker/types/__init__.py b/docker/types/__init__.py index 71c0c97403..a7c3a56b53 100644 --- a/docker/types/__init__.py +++ b/docker/types/__init__.py @@ -4,4 +4,5 @@ ContainerSpec, DriverConfig, EndpointSpec, Mount, Resources, RestartPolicy, TaskTemplate, UpdateConfig ) +from .healthcheck import Healthcheck from .swarm import SwarmSpec, SwarmExternalCA diff --git a/docker/types/healthcheck.py b/docker/types/healthcheck.py new file mode 100644 index 0000000000..8405869ee2 --- /dev/null +++ b/docker/types/healthcheck.py @@ -0,0 +1,47 @@ +from .base import DictType + + +class Healthcheck(DictType): + def __init__(self, **kwargs): + test = kwargs.get('test', kwargs.get('Test')) + interval = kwargs.get('interval', kwargs.get('Interval')) + timeout = kwargs.get('timeout', kwargs.get('Timeout')) + retries = kwargs.get('retries', kwargs.get('Retries')) + super(Healthcheck, self).__init__({ + 'Test': test, + 'Interval': interval, + 'Timeout': timeout, + 'Retries': retries + }) + + @property + def test(self): + return self['Test'] + + @test.setter + def test(self, value): + self['Test'] = value + + @property + def interval(self): + return self['Interval'] + + @interval.setter + def interval(self, value): + self['Interval'] = value + + @property + def timeout(self): + return self['Timeout'] + + @timeout.setter + def timeout(self, value): + self['Timeout'] = value + + @property + def retries(self): + return self['Retries'] + + @retries.setter + def retries(self, value): + self['Retries'] = value diff --git a/docker/utils/utils.py b/docker/utils/utils.py index 97261cdc79..ee324db5fb 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -1029,6 +1029,7 @@ def create_container_config( entrypoint=None, cpu_shares=None, working_dir=None, domainname=None, memswap_limit=None, cpuset=None, host_config=None, mac_address=None, labels=None, volume_driver=None, stop_signal=None, networking_config=None, + healthcheck=None, ): if isinstance(command, six.string_types): command = split_command(command) @@ -1057,6 +1058,11 @@ def create_container_config( 'stop_signal was only introduced in API version 1.21' ) + if healthcheck is not None and version_lt(version, '1.24'): + raise errors.InvalidVersion( + 'Health options were only introduced in API version 1.24' + ) + if compare_version('1.19', version) < 0: if volume_driver is not None: raise errors.InvalidVersion( @@ -1164,5 +1170,6 @@ def create_container_config( 'MacAddress': mac_address, 'Labels': labels, 'VolumeDriver': volume_driver, - 'StopSignal': stop_signal + 'StopSignal': stop_signal, + 'Healthcheck': healthcheck, } diff --git a/tests/integration/healthcheck_test.py b/tests/integration/healthcheck_test.py new file mode 100644 index 0000000000..b1a7e4ac36 --- /dev/null +++ b/tests/integration/healthcheck_test.py @@ -0,0 +1,35 @@ +import time +import docker + +from ..base import requires_api_version +from .. import helpers + + +class HealthcheckTest(helpers.BaseTestCase): + + @requires_api_version('1.21') + def test_healthcheck(self): + healthcheck = docker.types.Healthcheck( + test=["CMD-SHELL", + "foo.txt || (/bin/usleep 10000 && touch foo.txt)"], + interval=500000, + timeout=1000000000, + retries=1 + ) + container = self.client.create_container(helpers.BUSYBOX, 'cat', + detach=True, stdin_open=True, + healthcheck=healthcheck) + id = container['Id'] + self.client.start(id) + self.tmp_containers.append(id) + res1 = self.client.inspect_container(id) + self.assertIn('State', res1) + self.assertIn('Health', res1['State']) + self.assertIn('Status', res1['State']['Health']) + self.assertEqual(res1['State']['Health']['Status'], "starting") + time.sleep(0.5) + res2 = self.client.inspect_container(id) + self.assertIn('State', res2) + self.assertIn('Health', res2['State']) + self.assertIn('Status', res2['State']['Health']) + self.assertEqual(res2['State']['Health']['Status'], "healthy") From 6bb7844ab395ffa8d9ed9d5206871849d6a6f319 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Fri, 11 Nov 2016 12:59:47 +0000 Subject: [PATCH 0160/1301] Rework healthcheck integration test Signed-off-by: Aanand Prasad --- tests/helpers.py | 9 ++++ tests/integration/healthcheck_test.py | 75 +++++++++++++++++---------- 2 files changed, 57 insertions(+), 27 deletions(-) diff --git a/tests/helpers.py b/tests/helpers.py index 529b727a3e..f8b3e61383 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -2,6 +2,7 @@ import os.path import tarfile import tempfile +import time import docker import pytest @@ -47,3 +48,11 @@ def requires_api_version(version): ), reason="API version is too low (< {0})".format(version) ) + + +def wait_on_condition(condition, delay=0.1, timeout=40): + start_time = time.time() + while not condition(): + if time.time() - start_time > timeout: + raise AssertionError("Timeout: %s" % condition) + time.sleep(delay) diff --git a/tests/integration/healthcheck_test.py b/tests/integration/healthcheck_test.py index b1a7e4ac36..9df8cd76af 100644 --- a/tests/integration/healthcheck_test.py +++ b/tests/integration/healthcheck_test.py @@ -1,35 +1,56 @@ -import time import docker -from ..base import requires_api_version +from .base import BaseIntegrationTest +from .base import BUSYBOX from .. import helpers +SECOND = 1000000000 -class HealthcheckTest(helpers.BaseTestCase): - @requires_api_version('1.21') - def test_healthcheck(self): +class HealthcheckTest(BaseIntegrationTest): + + @helpers.requires_api_version('1.24') + def test_healthcheck_passes(self): healthcheck = docker.types.Healthcheck( - test=["CMD-SHELL", - "foo.txt || (/bin/usleep 10000 && touch foo.txt)"], - interval=500000, - timeout=1000000000, - retries=1 + test=["CMD-SHELL", "true"], + interval=1*SECOND, + timeout=1*SECOND, + retries=1, ) - container = self.client.create_container(helpers.BUSYBOX, 'cat', - detach=True, stdin_open=True, - healthcheck=healthcheck) - id = container['Id'] - self.client.start(id) - self.tmp_containers.append(id) - res1 = self.client.inspect_container(id) - self.assertIn('State', res1) - self.assertIn('Health', res1['State']) - self.assertIn('Status', res1['State']['Health']) - self.assertEqual(res1['State']['Health']['Status'], "starting") - time.sleep(0.5) - res2 = self.client.inspect_container(id) - self.assertIn('State', res2) - self.assertIn('Health', res2['State']) - self.assertIn('Status', res2['State']['Health']) - self.assertEqual(res2['State']['Health']['Status'], "healthy") + container = self.client.create_container( + BUSYBOX, 'top', healthcheck=healthcheck) + self.tmp_containers.append(container) + + res = self.client.inspect_container(container) + assert res['Config']['Healthcheck'] == { + "Test": ["CMD-SHELL", "true"], + "Interval": 1*SECOND, + "Timeout": 1*SECOND, + "Retries": 1, + } + + def condition(): + res = self.client.inspect_container(container) + return res['State']['Health']['Status'] == "healthy" + + self.client.start(container) + helpers.wait_on_condition(condition) + + @helpers.requires_api_version('1.24') + def test_healthcheck_fails(self): + healthcheck = docker.types.Healthcheck( + test=["CMD-SHELL", "false"], + interval=1*SECOND, + timeout=1*SECOND, + retries=1, + ) + container = self.client.create_container( + BUSYBOX, 'top', healthcheck=healthcheck) + self.tmp_containers.append(container) + + def condition(): + res = self.client.inspect_container(container) + return res['State']['Health']['Status'] == "unhealthy" + + self.client.start(container) + helpers.wait_on_condition(condition) From e4b6d0dca6d667c6defd10a9d507f7b4c11e6eb2 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Fri, 11 Nov 2016 17:32:42 +0000 Subject: [PATCH 0161/1301] Convert dicts to Healthcheck objects, string commands to CMD-SHELL lists Signed-off-by: Aanand Prasad --- docker/types/healthcheck.py | 6 +++ docker/utils/utils.py | 5 ++- tests/integration/healthcheck_test.py | 63 ++++++++++++--------------- 3 files changed, 39 insertions(+), 35 deletions(-) diff --git a/docker/types/healthcheck.py b/docker/types/healthcheck.py index 8405869ee2..ba63d21ed6 100644 --- a/docker/types/healthcheck.py +++ b/docker/types/healthcheck.py @@ -1,12 +1,18 @@ from .base import DictType +import six + class Healthcheck(DictType): def __init__(self, **kwargs): test = kwargs.get('test', kwargs.get('Test')) + if isinstance(test, six.string_types): + test = ["CMD-SHELL", test] + interval = kwargs.get('interval', kwargs.get('Interval')) timeout = kwargs.get('timeout', kwargs.get('Timeout')) retries = kwargs.get('retries', kwargs.get('Retries')) + super(Healthcheck, self).__init__({ 'Test': test, 'Interval': interval, diff --git a/docker/utils/utils.py b/docker/utils/utils.py index ee324db5fb..f61b5dd5cb 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -18,7 +18,7 @@ from .. import constants from .. import errors from .. import tls -from ..types import Ulimit, LogConfig +from ..types import Ulimit, LogConfig, Healthcheck if six.PY2: from urllib import splitnport @@ -1119,6 +1119,9 @@ def create_container_config( # Force None, an empty list or dict causes client.start to fail volumes_from = None + if healthcheck and isinstance(healthcheck, dict): + healthcheck = Healthcheck(**healthcheck) + attach_stdin = False attach_stdout = False attach_stderr = False diff --git a/tests/integration/healthcheck_test.py b/tests/integration/healthcheck_test.py index 9df8cd76af..9c0f39802d 100644 --- a/tests/integration/healthcheck_test.py +++ b/tests/integration/healthcheck_test.py @@ -1,5 +1,3 @@ -import docker - from .base import BaseIntegrationTest from .base import BUSYBOX from .. import helpers @@ -7,50 +5,47 @@ SECOND = 1000000000 +def wait_on_health_status(client, container, status): + def condition(): + res = client.inspect_container(container) + return res['State']['Health']['Status'] == status + return helpers.wait_on_condition(condition) + + class HealthcheckTest(BaseIntegrationTest): @helpers.requires_api_version('1.24') - def test_healthcheck_passes(self): - healthcheck = docker.types.Healthcheck( - test=["CMD-SHELL", "true"], - interval=1*SECOND, - timeout=1*SECOND, - retries=1, - ) + def test_healthcheck_shell_command(self): container = self.client.create_container( - BUSYBOX, 'top', healthcheck=healthcheck) + BUSYBOX, 'top', healthcheck=dict(test='echo "hello world"')) self.tmp_containers.append(container) res = self.client.inspect_container(container) - assert res['Config']['Healthcheck'] == { - "Test": ["CMD-SHELL", "true"], - "Interval": 1*SECOND, - "Timeout": 1*SECOND, - "Retries": 1, - } - - def condition(): - res = self.client.inspect_container(container) - return res['State']['Health']['Status'] == "healthy" + assert res['Config']['Healthcheck']['Test'] == \ + ['CMD-SHELL', 'echo "hello world"'] + @helpers.requires_api_version('1.24') + def test_healthcheck_passes(self): + container = self.client.create_container( + BUSYBOX, 'top', healthcheck=dict( + test="true", + interval=1*SECOND, + timeout=1*SECOND, + retries=1, + )) + self.tmp_containers.append(container) self.client.start(container) - helpers.wait_on_condition(condition) + wait_on_health_status(self.client, container, "healthy") @helpers.requires_api_version('1.24') def test_healthcheck_fails(self): - healthcheck = docker.types.Healthcheck( - test=["CMD-SHELL", "false"], - interval=1*SECOND, - timeout=1*SECOND, - retries=1, - ) container = self.client.create_container( - BUSYBOX, 'top', healthcheck=healthcheck) + BUSYBOX, 'top', healthcheck=dict( + test="false", + interval=1*SECOND, + timeout=1*SECOND, + retries=1, + )) self.tmp_containers.append(container) - - def condition(): - res = self.client.inspect_container(container) - return res['State']['Health']['Status'] == "unhealthy" - self.client.start(container) - helpers.wait_on_condition(condition) + wait_on_health_status(self.client, container, "unhealthy") From 087a049b065faaa0b28abf6018711e4bdfea4306 Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Tue, 18 Oct 2016 14:47:21 +0200 Subject: [PATCH 0162/1301] Add --rm to docker run in Makefile So tests don't leave containers all over the place. The downside is this makes it a bit harder to debug a test's filesystem - you'll have to remove the "--rm" and run the test again. Signed-off-by: Ben Firshman --- Makefile | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index b997722c7a..2e9ecf8e7f 100644 --- a/Makefile +++ b/Makefile @@ -27,28 +27,28 @@ test: flake8 unit-test unit-test-py3 integration-dind integration-dind-ssl .PHONY: unit-test unit-test: build - docker run docker-py py.test tests/unit + docker run --rm docker-py py.test tests/unit .PHONY: unit-test-py3 unit-test-py3: build-py3 - docker run docker-py3 py.test tests/unit + docker run --rm docker-py3 py.test tests/unit .PHONY: integration-test integration-test: build - docker run -v /var/run/docker.sock:/var/run/docker.sock docker-py py.test tests/integration/${file} + docker run --rm -v /var/run/docker.sock:/var/run/docker.sock docker-py py.test tests/integration/${file} .PHONY: integration-test-py3 integration-test-py3: build-py3 - docker run -v /var/run/docker.sock:/var/run/docker.sock docker-py3 py.test tests/integration/${file} + docker run --rm -v /var/run/docker.sock:/var/run/docker.sock docker-py3 py.test tests/integration/${file} .PHONY: integration-dind integration-dind: build build-py3 docker rm -vf dpy-dind || : docker run -d --name dpy-dind --privileged dockerswarm/dind:1.12.0 docker daemon\ -H tcp://0.0.0.0:2375 - docker run --env="DOCKER_HOST=tcp://docker:2375" --link=dpy-dind:docker docker-py\ + docker run --rm --env="DOCKER_HOST=tcp://docker:2375" --link=dpy-dind:docker docker-py\ py.test tests/integration - docker run --env="DOCKER_HOST=tcp://docker:2375" --link=dpy-dind:docker docker-py3\ + docker run --rm --env="DOCKER_HOST=tcp://docker:2375" --link=dpy-dind:docker docker-py3\ py.test tests/integration docker rm -vf dpy-dind @@ -60,17 +60,17 @@ integration-dind-ssl: build-dind-certs build build-py3 -v /tmp --privileged dockerswarm/dind:1.12.0 docker daemon --tlsverify\ --tlscacert=/certs/ca.pem --tlscert=/certs/server-cert.pem\ --tlskey=/certs/server-key.pem -H tcp://0.0.0.0:2375 - docker run --volumes-from dpy-dind-ssl --env="DOCKER_HOST=tcp://docker:2375"\ + docker run --rm --volumes-from dpy-dind-ssl --env="DOCKER_HOST=tcp://docker:2375"\ --env="DOCKER_TLS_VERIFY=1" --env="DOCKER_CERT_PATH=/certs"\ --link=dpy-dind-ssl:docker docker-py py.test tests/integration - docker run --volumes-from dpy-dind-ssl --env="DOCKER_HOST=tcp://docker:2375"\ + docker run --rm --volumes-from dpy-dind-ssl --env="DOCKER_HOST=tcp://docker:2375"\ --env="DOCKER_TLS_VERIFY=1" --env="DOCKER_CERT_PATH=/certs"\ --link=dpy-dind-ssl:docker docker-py3 py.test tests/integration docker rm -vf dpy-dind-ssl dpy-dind-certs .PHONY: flake8 flake8: build - docker run docker-py flake8 docker tests + docker run --rm docker-py flake8 docker tests .PHONY: docs docs: build-docs From 81dfc475b361c22ca9608639fc88ff1093dcc1c8 Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Thu, 17 Nov 2016 12:16:04 +0000 Subject: [PATCH 0163/1301] Drop support for API versions <1.24 Implemented as just a warning. Actual removal of code will follow sometime in the future. Signed-off-by: Ben Firshman --- docker/client.py | 8 ++++++++ docker/constants.py | 1 + 2 files changed, 9 insertions(+) diff --git a/docker/client.py b/docker/client.py index aec78c8a6a..751791c794 100644 --- a/docker/client.py +++ b/docker/client.py @@ -1,5 +1,6 @@ import json import struct +import warnings from functools import partial import requests @@ -105,6 +106,13 @@ def __init__(self, base_url=None, version=None, type(version).__name__ ) ) + if utils.version_lt(self._version, constants.MINIMUM_DOCKER_API_VERSION): + warnings.warn( + 'The minimum API version supported is {}, but you are using ' + 'version {}. It is recommended you either upgrade Docker ' + 'Engine or use an older version of docker-py.'.format( + constants.MINIMUM_DOCKER_API_VERSION, self._version) + ) @classmethod def from_env(cls, **kwargs): diff --git a/docker/constants.py b/docker/constants.py index 0c9a0205c2..c3048cb7ad 100644 --- a/docker/constants.py +++ b/docker/constants.py @@ -2,6 +2,7 @@ from .version import version DEFAULT_DOCKER_API_VERSION = '1.24' +MINIMUM_DOCKER_API_VERSION = '1.24' DEFAULT_TIMEOUT_SECONDS = 60 STREAM_HEADER_SIZE_BYTES = 8 CONTAINER_LIMITS_KEYS = [ From 2c9d1110f0e976cbaa06fe4cf52061a72c526d45 Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Thu, 15 Sep 2016 14:25:32 +0100 Subject: [PATCH 0164/1301] Make docker.auth a single file Signed-off-by: Ben Firshman --- docker/api/daemon.py | 3 +-- docker/api/image.py | 7 +++---- docker/api/service.py | 5 +---- docker/{auth => }/auth.py | 2 +- docker/auth/__init__.py | 8 -------- docker/client.py | 5 +---- setup.py | 5 ++--- tests/unit/auth_test.py | 6 ++---- tests/unit/image_test.py | 6 +++--- 9 files changed, 14 insertions(+), 33 deletions(-) rename docker/{auth => }/auth.py (99%) delete mode 100644 docker/auth/__init__.py diff --git a/docker/api/daemon.py b/docker/api/daemon.py index 9ebe73c0a7..980ae56876 100644 --- a/docker/api/daemon.py +++ b/docker/api/daemon.py @@ -2,9 +2,8 @@ import warnings from datetime import datetime -from ..auth import auth +from .. import auth, utils from ..constants import INSECURE_REGISTRY_DEPRECATION_WARNING -from ..utils import utils class DaemonApiMixin(object): diff --git a/docker/api/image.py b/docker/api/image.py index 262910cd60..978a0c14bb 100644 --- a/docker/api/image.py +++ b/docker/api/image.py @@ -1,12 +1,11 @@ import logging import os -import six import warnings -from ..auth import auth +import six + +from .. import auth, errors, utils from ..constants import INSECURE_REGISTRY_DEPRECATION_WARNING -from .. import utils -from .. import errors log = logging.getLogger(__name__) diff --git a/docker/api/service.py b/docker/api/service.py index 2e41b7cd37..ec2a303966 100644 --- a/docker/api/service.py +++ b/docker/api/service.py @@ -1,8 +1,5 @@ import warnings - -from .. import errors -from .. import utils -from ..auth import auth +from .. import auth, errors, utils class ServiceApiMixin(object): diff --git a/docker/auth/auth.py b/docker/auth.py similarity index 99% rename from docker/auth/auth.py rename to docker/auth.py index dc0baea80f..0a2eda1ebe 100644 --- a/docker/auth/auth.py +++ b/docker/auth.py @@ -6,7 +6,7 @@ import dockerpycreds import six -from .. import errors +from . import errors INDEX_NAME = 'docker.io' INDEX_URL = 'https://{0}/v1/'.format(INDEX_NAME) diff --git a/docker/auth/__init__.py b/docker/auth/__init__.py deleted file mode 100644 index 50127fac10..0000000000 --- a/docker/auth/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -from .auth import ( - INDEX_NAME, - INDEX_URL, - encode_header, - load_config, - resolve_authconfig, - resolve_repository_name, -) # flake8: noqa diff --git a/docker/client.py b/docker/client.py index 751791c794..19801e32fb 100644 --- a/docker/client.py +++ b/docker/client.py @@ -9,10 +9,7 @@ import websocket -from . import api -from . import constants -from . import errors -from .auth import auth +from . import api, auth, constants, errors from .ssladapter import ssladapter from .tls import TLSConfig from .transport import UnixAdapter diff --git a/setup.py b/setup.py index 96bce6a5d2..ff80e3a8fc 100644 --- a/setup.py +++ b/setup.py @@ -49,9 +49,8 @@ long_description=long_description, url='https://github.com/docker/docker-py/', packages=[ - 'docker', 'docker.api', 'docker.auth', 'docker.transport', - 'docker.utils', 'docker.utils.ports', 'docker.ssladapter', - 'docker.types', + 'docker', 'docker.api', 'docker.transport', 'docker.utils', + 'docker.utils.ports', 'docker.ssladapter', 'docker.types', ], install_requires=requirements, tests_require=test_requirements, diff --git a/tests/unit/auth_test.py b/tests/unit/auth_test.py index f3951335f4..89f70fcd24 100644 --- a/tests/unit/auth_test.py +++ b/tests/unit/auth_test.py @@ -8,9 +8,7 @@ import shutil import tempfile -from docker import auth -from docker.auth.auth import parse_auth -from docker import errors +from docker import auth, errors from .. import base @@ -122,7 +120,7 @@ class ResolveAuthTest(base.BaseTestCase): private_config = {'auth': encode_auth({'username': 'privateuser'})} legacy_config = {'auth': encode_auth({'username': 'legacyauth'})} - auth_config = parse_auth({ + auth_config = auth.parse_auth({ 'https://index.docker.io/v1/': index_config, 'my.registry.net': private_config, 'http://legacy.registry.url/v1/': legacy_config, diff --git a/tests/unit/image_test.py b/tests/unit/image_test.py index cca519e38e..be9d574c18 100644 --- a/tests/unit/image_test.py +++ b/tests/unit/image_test.py @@ -228,7 +228,7 @@ def test_insert_image(self): ) def test_push_image(self): - with mock.patch('docker.auth.auth.resolve_authconfig', + with mock.patch('docker.auth.resolve_authconfig', fake_resolve_authconfig): self.client.push(fake_api.FAKE_IMAGE_NAME) @@ -245,7 +245,7 @@ def test_push_image(self): ) def test_push_image_with_tag(self): - with mock.patch('docker.auth.auth.resolve_authconfig', + with mock.patch('docker.auth.resolve_authconfig', fake_resolve_authconfig): self.client.push( fake_api.FAKE_IMAGE_NAME, tag=fake_api.FAKE_TAG_NAME @@ -289,7 +289,7 @@ def test_push_image_with_auth(self): ) def test_push_image_stream(self): - with mock.patch('docker.auth.auth.resolve_authconfig', + with mock.patch('docker.auth.resolve_authconfig', fake_resolve_authconfig): self.client.push(fake_api.FAKE_IMAGE_NAME, stream=True) From 19eefcf70579b74881b63d1a82e00598d5a9cc09 Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Thu, 15 Sep 2016 14:26:33 +0100 Subject: [PATCH 0165/1301] Make docker.ssladaptor a single file Signed-off-by: Ben Firshman --- docker/client.py | 3 +-- docker/{ssladapter => }/ssladapter.py | 0 docker/ssladapter/__init__.py | 1 - docker/tls.py | 3 +-- setup.py | 2 +- tests/unit/ssladapter_test.py | 2 +- 6 files changed, 4 insertions(+), 7 deletions(-) rename docker/{ssladapter => }/ssladapter.py (100%) delete mode 100644 docker/ssladapter/__init__.py diff --git a/docker/client.py b/docker/client.py index 19801e32fb..35f45327dc 100644 --- a/docker/client.py +++ b/docker/client.py @@ -9,8 +9,7 @@ import websocket -from . import api, auth, constants, errors -from .ssladapter import ssladapter +from . import api, auth, constants, errors, ssladapter from .tls import TLSConfig from .transport import UnixAdapter from .utils import utils, check_resource, update_headers, kwargs_from_env diff --git a/docker/ssladapter/ssladapter.py b/docker/ssladapter.py similarity index 100% rename from docker/ssladapter/ssladapter.py rename to docker/ssladapter.py diff --git a/docker/ssladapter/__init__.py b/docker/ssladapter/__init__.py deleted file mode 100644 index 31b8966b01..0000000000 --- a/docker/ssladapter/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .ssladapter import SSLAdapter # flake8: noqa diff --git a/docker/tls.py b/docker/tls.py index 18c725987c..7c3a2ca35f 100644 --- a/docker/tls.py +++ b/docker/tls.py @@ -1,8 +1,7 @@ import os import ssl -from . import errors -from .ssladapter import ssladapter +from . import errors, ssladapter class TLSConfig(object): diff --git a/setup.py b/setup.py index ff80e3a8fc..120e05e0de 100644 --- a/setup.py +++ b/setup.py @@ -50,7 +50,7 @@ url='https://github.com/docker/docker-py/', packages=[ 'docker', 'docker.api', 'docker.transport', 'docker.utils', - 'docker.utils.ports', 'docker.ssladapter', 'docker.types', + 'docker.utils.ports', 'docker.types', ], install_requires=requirements, tests_require=test_requirements, diff --git a/tests/unit/ssladapter_test.py b/tests/unit/ssladapter_test.py index 8c905cf589..6af436bd7c 100644 --- a/tests/unit/ssladapter_test.py +++ b/tests/unit/ssladapter_test.py @@ -1,4 +1,4 @@ -from docker.ssladapter import ssladapter +from docker import ssladapter try: from backports.ssl_match_hostname import ( From b49cacced0683c8cfe9f2d2bf3dda2b1ce62609d Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Thu, 15 Sep 2016 14:31:50 +0100 Subject: [PATCH 0166/1301] Make docker.utils.ports a single file Signed-off-by: Ben Firshman --- docker/utils/{ports => }/ports.py | 0 docker/utils/ports/__init__.py | 4 ---- setup.py | 2 +- 3 files changed, 1 insertion(+), 5 deletions(-) rename docker/utils/{ports => }/ports.py (100%) delete mode 100644 docker/utils/ports/__init__.py diff --git a/docker/utils/ports/ports.py b/docker/utils/ports.py similarity index 100% rename from docker/utils/ports/ports.py rename to docker/utils/ports.py diff --git a/docker/utils/ports/__init__.py b/docker/utils/ports/__init__.py deleted file mode 100644 index 485feec06e..0000000000 --- a/docker/utils/ports/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -from .ports import ( - split_port, - build_port_bindings -) # flake8: noqa diff --git a/setup.py b/setup.py index 120e05e0de..a32c7c9bc4 100644 --- a/setup.py +++ b/setup.py @@ -50,7 +50,7 @@ url='https://github.com/docker/docker-py/', packages=[ 'docker', 'docker.api', 'docker.transport', 'docker.utils', - 'docker.utils.ports', 'docker.types', + 'docker.types', ], install_requires=requirements, tests_require=test_requirements, From d5bc7dc99acf7a06c54723dfd262f5a00c6288f1 Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Thu, 8 Sep 2016 12:14:38 +0100 Subject: [PATCH 0167/1301] Drop support for Python 2.6 Because it's ancient. If you're still using it, you can use an old version of docker-py. Signed-off-by: Ben Firshman --- .travis.yml | 1 - setup.py | 1 - tests/base.py | 36 ----------------------------------- tests/unit/api_test.py | 16 +++++----------- tests/unit/auth_test.py | 11 +++++------ tests/unit/client_test.py | 7 ++++--- tests/unit/network_test.py | 2 +- tests/unit/ssladapter_test.py | 7 +++---- tests/unit/utils_test.py | 35 +++++++++++++++++----------------- tox.ini | 2 +- 10 files changed, 36 insertions(+), 82 deletions(-) delete mode 100644 tests/base.py diff --git a/.travis.yml b/.travis.yml index fb62a3493a..6b48142f72 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,6 @@ language: python python: - "3.5" env: - - TOX_ENV=py26 - TOX_ENV=py27 - TOX_ENV=py33 - TOX_ENV=py34 diff --git a/setup.py b/setup.py index a32c7c9bc4..89c97c87be 100644 --- a/setup.py +++ b/setup.py @@ -64,7 +64,6 @@ 'Operating System :: OS Independent', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.3', diff --git a/tests/base.py b/tests/base.py deleted file mode 100644 index cac65fd9f5..0000000000 --- a/tests/base.py +++ /dev/null @@ -1,36 +0,0 @@ -import sys -import unittest - -import six - - -class BaseTestCase(unittest.TestCase): - def assertIn(self, object, collection): - if six.PY2 and sys.version_info[1] <= 6: - return self.assertTrue(object in collection) - return super(BaseTestCase, self).assertIn(object, collection) - - -class Cleanup(object): - if sys.version_info < (2, 7): - # Provide a basic implementation of addCleanup for Python < 2.7 - def __init__(self, *args, **kwargs): - super(Cleanup, self).__init__(*args, **kwargs) - self._cleanups = [] - - def tearDown(self): - super(Cleanup, self).tearDown() - ok = True - while self._cleanups: - fn, args, kwargs = self._cleanups.pop(-1) - try: - fn(*args, **kwargs) - except KeyboardInterrupt: - raise - except: - ok = False - if not ok: - raise - - def addCleanup(self, function, *args, **kwargs): - self._cleanups.append((function, args, kwargs)) diff --git a/tests/unit/api_test.py b/tests/unit/api_test.py index aa19c90058..5777ab9af5 100644 --- a/tests/unit/api_test.py +++ b/tests/unit/api_test.py @@ -1,21 +1,20 @@ import datetime import json +import io import os import re import shutil import socket -import sys import tempfile import threading import time -import io +import unittest import docker import requests from requests.packages import urllib3 import six -from .. import base from . import fake_api import pytest @@ -93,7 +92,7 @@ def fake_read_from_socket(self, response, stream): docker.constants.DEFAULT_DOCKER_API_VERSION) -class DockerClientTest(base.Cleanup, base.BaseTestCase): +class DockerClientTest(unittest.TestCase): def setUp(self): self.patcher = mock.patch.multiple( 'docker.Client', get=fake_get, post=fake_post, put=fake_put, @@ -109,11 +108,6 @@ def tearDown(self): self.client.close() self.patcher.stop() - def assertIn(self, object, collection): - if six.PY2 and sys.version_info[1] <= 6: - return self.assertTrue(object in collection) - return super(DockerClientTest, self).assertIn(object, collection) - def base_create_payload(self, img='busybox', cmd=None): if not cmd: cmd = ['true'] @@ -356,7 +350,7 @@ def test_stream_helper_decoding(self): self.assertEqual(result, content) -class StreamTest(base.Cleanup, base.BaseTestCase): +class StreamTest(unittest.TestCase): def setUp(self): socket_dir = tempfile.mkdtemp() self.build_context = tempfile.mkdtemp() @@ -458,7 +452,7 @@ def test_early_stream_response(self): str(i).encode() for i in range(50)]) -class UserAgentTest(base.BaseTestCase): +class UserAgentTest(unittest.TestCase): def setUp(self): self.patcher = mock.patch.object( docker.Client, diff --git a/tests/unit/auth_test.py b/tests/unit/auth_test.py index 89f70fcd24..e4c93b78d9 100644 --- a/tests/unit/auth_test.py +++ b/tests/unit/auth_test.py @@ -7,18 +7,17 @@ import random import shutil import tempfile +import unittest from docker import auth, errors -from .. import base - try: from unittest import mock except ImportError: import mock -class RegressionTest(base.BaseTestCase): +class RegressionTest(unittest.TestCase): def test_803_urlsafe_encode(self): auth_data = { 'username': 'root', @@ -29,7 +28,7 @@ def test_803_urlsafe_encode(self): assert b'_' in encoded -class ResolveRepositoryNameTest(base.BaseTestCase): +class ResolveRepositoryNameTest(unittest.TestCase): def test_resolve_repository_name_hub_library_image(self): self.assertEqual( auth.resolve_repository_name('image'), @@ -115,7 +114,7 @@ def encode_auth(auth_info): auth_info.get('password', '').encode('utf-8')) -class ResolveAuthTest(base.BaseTestCase): +class ResolveAuthTest(unittest.TestCase): index_config = {'auth': encode_auth({'username': 'indexuser'})} private_config = {'auth': encode_auth({'username': 'privateuser'})} legacy_config = {'auth': encode_auth({'username': 'legacyauth'})} @@ -270,7 +269,7 @@ def test_resolve_registry_and_auth_unauthenticated_registry(self): ) -class LoadConfigTest(base.Cleanup, base.BaseTestCase): +class LoadConfigTest(unittest.TestCase): def test_load_config_no_file(self): folder = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, folder) diff --git a/tests/unit/client_test.py b/tests/unit/client_test.py index 6ceb8cbbc0..2a5124b2d6 100644 --- a/tests/unit/client_test.py +++ b/tests/unit/client_test.py @@ -1,6 +1,7 @@ import os +import unittest + from docker.client import Client -from .. import base TEST_CERT_DIR = os.path.join( os.path.dirname(__file__), @@ -8,7 +9,7 @@ ) -class ClientTest(base.BaseTestCase): +class ClientTest(unittest.TestCase): def setUp(self): self.os_environ = os.environ.copy() @@ -34,7 +35,7 @@ def test_from_env_with_version(self): self.assertEqual(client._version, '2.32') -class DisableSocketTest(base.BaseTestCase): +class DisableSocketTest(unittest.TestCase): class DummySocket(object): def __init__(self, timeout=60): self.timeout = timeout diff --git a/tests/unit/network_test.py b/tests/unit/network_test.py index 93f03da4bc..5d1f4392d3 100644 --- a/tests/unit/network_test.py +++ b/tests/unit/network_test.py @@ -2,8 +2,8 @@ import six -from ..helpers import requires_api_version from .api_test import DockerClientTest, url_prefix, response +from ..helpers import requires_api_version from docker.utils import create_ipam_config, create_ipam_pool try: diff --git a/tests/unit/ssladapter_test.py b/tests/unit/ssladapter_test.py index 6af436bd7c..90d4c3202c 100644 --- a/tests/unit/ssladapter_test.py +++ b/tests/unit/ssladapter_test.py @@ -1,3 +1,4 @@ +import unittest from docker import ssladapter try: @@ -16,10 +17,8 @@ OP_NO_SSLv3 = 0x2000000 OP_NO_TLSv1 = 0x4000000 -from .. import base - -class SSLAdapterTest(base.BaseTestCase): +class SSLAdapterTest(unittest.TestCase): def test_only_uses_tls(self): ssl_context = ssladapter.urllib3.util.ssl_.create_urllib3_context() @@ -29,7 +28,7 @@ def test_only_uses_tls(self): assert not ssl_context.options & OP_NO_TLSv1 -class MatchHostnameTest(base.BaseTestCase): +class MatchHostnameTest(unittest.TestCase): cert = { 'issuer': ( (('countryName', u'US'),), diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index 25aef7379f..80d156f5be 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -8,6 +8,7 @@ import sys import tarfile import tempfile +import unittest import pytest import six @@ -28,7 +29,6 @@ from docker.utils.ports import build_port_bindings, split_port from docker.utils.utils import create_endpoint_config, format_environment -from .. import base from ..helpers import make_tree @@ -38,7 +38,7 @@ ) -class DecoratorsTest(base.BaseTestCase): +class DecoratorsTest(unittest.TestCase): def test_update_headers(self): sample_headers = { 'X-Docker-Locale': 'en-US', @@ -69,7 +69,7 @@ def f(self, headers=None): } -class HostConfigTest(base.BaseTestCase): +class HostConfigTest(unittest.TestCase): def test_create_host_config_no_options(self): config = create_host_config(version='1.19') self.assertFalse('NetworkMode' in config) @@ -208,7 +208,7 @@ def test_create_host_config_with_isolation(self): ) -class UlimitTest(base.BaseTestCase): +class UlimitTest(unittest.TestCase): def test_create_host_config_dict_ulimit(self): ulimit_dct = {'name': 'nofile', 'soft': 8096} config = create_host_config( @@ -253,7 +253,7 @@ def test_ulimit_invalid_type(self): self.assertRaises(ValueError, lambda: Ulimit(name='hello', hard='456')) -class LogConfigTest(base.BaseTestCase): +class LogConfigTest(unittest.TestCase): def test_create_host_config_dict_logconfig(self): dct = {'type': LogConfig.types.SYSLOG, 'config': {'key1': 'val1'}} config = create_host_config( @@ -277,7 +277,7 @@ def test_logconfig_invalid_config_type(self): LogConfig(type=LogConfig.types.JSON, config='helloworld') -class KwargsFromEnvTest(base.BaseTestCase): +class KwargsFromEnvTest(unittest.TestCase): def setUp(self): self.os_environ = os.environ.copy() @@ -377,7 +377,7 @@ def test_kwargs_from_env_alternate_env(self): assert 'tls' not in kwargs -class ConverVolumeBindsTest(base.BaseTestCase): +class ConverVolumeBindsTest(unittest.TestCase): def test_convert_volume_binds_empty(self): self.assertEqual(convert_volume_binds({}), []) self.assertEqual(convert_volume_binds([]), []) @@ -436,7 +436,7 @@ def test_convert_volume_binds_unicode_unicode_input(self): ) -class ParseEnvFileTest(base.BaseTestCase): +class ParseEnvFileTest(unittest.TestCase): def generate_tempfile(self, file_content=None): """ Generates a temporary file for tests with the content @@ -479,7 +479,7 @@ def test_parse_env_file_invalid_line(self): os.unlink(env_file) -class ParseHostTest(base.BaseTestCase): +class ParseHostTest(unittest.TestCase): def test_parse_host(self): invalid_hosts = [ '0.0.0.0', @@ -541,7 +541,7 @@ def test_parse_host_trailing_slash(self): assert parse_host(host_value) == expected_result -class ParseRepositoryTagTest(base.BaseTestCase): +class ParseRepositoryTagTest(unittest.TestCase): sha = 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855' def test_index_image_no_tag(self): @@ -587,7 +587,7 @@ def test_private_reg_image_sha(self): ) -class ParseDeviceTest(base.BaseTestCase): +class ParseDeviceTest(unittest.TestCase): def test_dict(self): devices = parse_devices([{ 'PathOnHost': '/dev/sda1', @@ -646,7 +646,7 @@ def test_hybrid_list(self): }) -class ParseBytesTest(base.BaseTestCase): +class ParseBytesTest(unittest.TestCase): def test_parse_bytes_valid(self): self.assertEqual(parse_bytes("512MB"), 536870912) self.assertEqual(parse_bytes("512M"), 536870912) @@ -666,7 +666,7 @@ def test_parse_bytes_maxint(self): ) -class UtilsTest(base.BaseTestCase): +class UtilsTest(unittest.TestCase): longMessage = True def test_convert_filters(self): @@ -706,7 +706,7 @@ def test_create_ipam_config(self): }) -class SplitCommandTest(base.BaseTestCase): +class SplitCommandTest(unittest.TestCase): def test_split_command_with_unicode(self): self.assertEqual(split_command(u'echo μμ'), ['echo', 'μμ']) @@ -715,7 +715,7 @@ def test_split_command_with_bytes(self): self.assertEqual(split_command('echo μμ'), ['echo', 'μμ']) -class PortsTest(base.BaseTestCase): +class PortsTest(unittest.TestCase): def test_split_port_with_host_ip(self): internal_port, external_port = split_port("127.0.0.1:1000:2000") self.assertEqual(internal_port, ["2000"]) @@ -821,14 +821,13 @@ def test_build_port_bindings_with_nonmatching_internal_port_ranges(self): self.assertEqual(port_bindings["1000"], [("127.0.0.1", "1000")]) self.assertEqual(port_bindings["2000"], [("127.0.0.1", "2000")]) - def convert_paths(collection): if not IS_WINDOWS_PLATFORM: return collection return set(map(lambda x: x.replace('/', '\\'), collection)) -class ExcludePathsTest(base.BaseTestCase): +class ExcludePathsTest(unittest.TestCase): dirs = [ 'foo', 'foo/bar', @@ -1010,7 +1009,7 @@ def test_subdirectory(self): ) -class TarTest(base.Cleanup, base.BaseTestCase): +class TarTest(unittest.TestCase): def test_tar_with_excludes(self): dirs = [ 'foo', diff --git a/tox.ini b/tox.ini index be4508e42d..1a41c6edac 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py26, py27, py33, py34, py35, flake8 +envlist = py27, py33, py34, py35, flake8 skipsdist=True [testenv] From c7903f084eb3a7525e21974390b48e2a0dc03732 Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Tue, 18 Oct 2016 15:36:06 +0200 Subject: [PATCH 0168/1301] Remove AutoVersionClient Just do Client(version='auto'). Signed-off-by: Ben Firshman --- docker/__init__.py | 2 +- docker/client.py | 10 ---------- tests/integration/api_test.py | 11 ----------- 3 files changed, 1 insertion(+), 22 deletions(-) diff --git a/docker/__init__.py b/docker/__init__.py index 0f4c8ecd8f..8ad1e71321 100644 --- a/docker/__init__.py +++ b/docker/__init__.py @@ -3,4 +3,4 @@ __version__ = version __title__ = 'docker-py' -from .client import Client, AutoVersionClient, from_env # flake8: noqa +from .client import Client, from_env # flake8: noqa diff --git a/docker/client.py b/docker/client.py index 35f45327dc..cc6b48ef3c 100644 --- a/docker/client.py +++ b/docker/client.py @@ -400,13 +400,3 @@ def get_adapter(self, url): @property def api_version(self): return self._version - - -class AutoVersionClient(Client): - def __init__(self, *args, **kwargs): - if 'version' in kwargs and kwargs['version']: - raise errors.DockerException( - 'Can not specify version for AutoVersionClient' - ) - kwargs['version'] = 'auto' - super(AutoVersionClient, self).__init__(*args, **kwargs) diff --git a/tests/integration/api_test.py b/tests/integration/api_test.py index f20d30b10f..6b54aa937c 100644 --- a/tests/integration/api_test.py +++ b/tests/integration/api_test.py @@ -122,17 +122,6 @@ def test_client_init(self): self.assertEqual(client_version, api_version_2) client.close() - def test_auto_client(self): - client = docker.AutoVersionClient(**kwargs_from_env()) - client_version = client._version - api_version = client.version(api_version=False)['ApiVersion'] - self.assertEqual(client_version, api_version) - api_version_2 = client.version()['ApiVersion'] - self.assertEqual(client_version, api_version_2) - client.close() - with self.assertRaises(docker.errors.DockerException): - docker.AutoVersionClient(version='1.11', **kwargs_from_env()) - class ConnectionTimeoutTest(unittest.TestCase): def setUp(self): From 9daa320454ec0c19035a09b436a41e13c3fb71ad Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Mon, 5 Sep 2016 16:00:14 +0200 Subject: [PATCH 0169/1301] Rename Client to APIClient Signed-off-by: Ben Firshman --- docker/__init__.py | 4 +- docker/api/__init__.py | 10 +- docker/{ => api}/client.py | 94 +++++++++++-------- .../{build_test.py => api_build_test.py} | 0 .../{api_test.py => api_client_test.py} | 8 +- ...ontainer_test.py => api_container_test.py} | 0 .../{exec_test.py => api_exec_test.py} | 0 ...hcheck_test.py => api_healthcheck_test.py} | 5 +- .../{image_test.py => api_image_test.py} | 0 .../{network_test.py => api_network_test.py} | 0 .../{service_test.py => api_service_test.py} | 0 .../{swarm_test.py => api_swarm_test.py} | 0 .../{volume_test.py => api_volume_test.py} | 0 tests/integration/base.py | 3 +- tests/integration/conftest.py | 3 +- .../unit/{build_test.py => api_build_test.py} | 0 .../{client_test.py => api_client_test.py} | 8 +- ...ontainer_test.py => api_container_test.py} | 22 ++--- tests/unit/{exec_test.py => api_exec_test.py} | 0 .../unit/{image_test.py => api_image_test.py} | 0 .../{network_test.py => api_network_test.py} | 12 +-- tests/unit/api_test.py | 30 +++--- .../{volume_test.py => api_volume_test.py} | 0 tests/unit/utils_test.py | 15 ++- 24 files changed, 114 insertions(+), 100 deletions(-) rename docker/{ => api}/client.py (84%) rename tests/integration/{build_test.py => api_build_test.py} (100%) rename tests/integration/{api_test.py => api_client_test.py} (95%) rename tests/integration/{container_test.py => api_container_test.py} (100%) rename tests/integration/{exec_test.py => api_exec_test.py} (100%) rename tests/integration/{healthcheck_test.py => api_healthcheck_test.py} (93%) rename tests/integration/{image_test.py => api_image_test.py} (100%) rename tests/integration/{network_test.py => api_network_test.py} (100%) rename tests/integration/{service_test.py => api_service_test.py} (100%) rename tests/integration/{swarm_test.py => api_swarm_test.py} (100%) rename tests/integration/{volume_test.py => api_volume_test.py} (100%) rename tests/unit/{build_test.py => api_build_test.py} (100%) rename tests/unit/{client_test.py => api_client_test.py} (93%) rename tests/unit/{container_test.py => api_container_test.py} (98%) rename tests/unit/{exec_test.py => api_exec_test.py} (100%) rename tests/unit/{image_test.py => api_image_test.py} (100%) rename tests/unit/{network_test.py => api_network_test.py} (93%) rename tests/unit/{volume_test.py => api_volume_test.py} (100%) diff --git a/docker/__init__.py b/docker/__init__.py index 8ad1e71321..95edb6b1bf 100644 --- a/docker/__init__.py +++ b/docker/__init__.py @@ -1,6 +1,6 @@ +# flake8: noqa +from .api import APIClient from .version import version, version_info __version__ = version __title__ = 'docker-py' - -from .client import Client, from_env # flake8: noqa diff --git a/docker/api/__init__.py b/docker/api/__init__.py index bc7e93ceee..ff5184414b 100644 --- a/docker/api/__init__.py +++ b/docker/api/__init__.py @@ -1,10 +1,2 @@ # flake8: noqa -from .build import BuildApiMixin -from .container import ContainerApiMixin -from .daemon import DaemonApiMixin -from .exec_api import ExecApiMixin -from .image import ImageApiMixin -from .network import NetworkApiMixin -from .service import ServiceApiMixin -from .swarm import SwarmApiMixin -from .volume import VolumeApiMixin +from .client import APIClient diff --git a/docker/client.py b/docker/api/client.py similarity index 84% rename from docker/client.py rename to docker/api/client.py index cc6b48ef3c..2fc2ef09b6 100644 --- a/docker/client.py +++ b/docker/api/client.py @@ -8,41 +8,59 @@ import six import websocket - -from . import api, auth, constants, errors, ssladapter -from .tls import TLSConfig -from .transport import UnixAdapter -from .utils import utils, check_resource, update_headers, kwargs_from_env -from .utils.socket import frames_iter +from .build import BuildApiMixin +from .container import ContainerApiMixin +from .daemon import DaemonApiMixin +from .exec_api import ExecApiMixin +from .image import ImageApiMixin +from .network import NetworkApiMixin +from .service import ServiceApiMixin +from .swarm import SwarmApiMixin +from .volume import VolumeApiMixin +from .. import auth, ssladapter +from ..constants import (DEFAULT_TIMEOUT_SECONDS, DEFAULT_USER_AGENT, + IS_WINDOWS_PLATFORM, DEFAULT_DOCKER_API_VERSION, + STREAM_HEADER_SIZE_BYTES, DEFAULT_NUM_POOLS, + MINIMUM_DOCKER_API_VERSION) +from ..errors import DockerException, APIError, TLSParameterError, NotFound +from ..tls import TLSConfig +from ..transport import UnixAdapter +from ..utils import utils, check_resource, update_headers, kwargs_from_env +from ..utils.socket import frames_iter try: - from .transport import NpipeAdapter + from ..transport import NpipeAdapter except ImportError: pass def from_env(**kwargs): - return Client.from_env(**kwargs) + return APIClient.from_env(**kwargs) -class Client( +class APIClient( requests.Session, - api.BuildApiMixin, - api.ContainerApiMixin, - api.DaemonApiMixin, - api.ExecApiMixin, - api.ImageApiMixin, - api.NetworkApiMixin, - api.ServiceApiMixin, - api.SwarmApiMixin, - api.VolumeApiMixin): + BuildApiMixin, + ContainerApiMixin, + DaemonApiMixin, + ExecApiMixin, + ImageApiMixin, + NetworkApiMixin, + ServiceApiMixin, + SwarmApiMixin, + VolumeApiMixin): + """ + A low-level client for the Docker Remote API. + + Each method maps one-to-one with a REST API endpoint, so calling each + method results in a single API call. + """ def __init__(self, base_url=None, version=None, - timeout=constants.DEFAULT_TIMEOUT_SECONDS, tls=False, - user_agent=constants.DEFAULT_USER_AGENT, - num_pools=constants.DEFAULT_NUM_POOLS): - super(Client, self).__init__() + timeout=DEFAULT_TIMEOUT_SECONDS, tls=False, + user_agent=DEFAULT_USER_AGENT, num_pools=DEFAULT_NUM_POOLS): + super(APIClient, self).__init__() if tls and not base_url: - raise errors.TLSParameterError( + raise TLSParameterError( 'If using TLS, the base_url argument must be provided.' ) @@ -53,7 +71,7 @@ def __init__(self, base_url=None, version=None, self._auth_configs = auth.load_config() base_url = utils.parse_host( - base_url, constants.IS_WINDOWS_PLATFORM, tls=bool(tls) + base_url, IS_WINDOWS_PLATFORM, tls=bool(tls) ) if base_url.startswith('http+unix://'): self._custom_adapter = UnixAdapter( @@ -63,8 +81,8 @@ def __init__(self, base_url=None, version=None, self._unmount('http://', 'https://') self.base_url = 'http+docker://localunixsocket' elif base_url.startswith('npipe://'): - if not constants.IS_WINDOWS_PLATFORM: - raise errors.DockerException( + if not IS_WINDOWS_PLATFORM: + raise DockerException( 'The npipe:// protocol is only supported on Windows' ) try: @@ -72,7 +90,7 @@ def __init__(self, base_url=None, version=None, base_url, timeout, pool_connections=num_pools ) except NameError: - raise errors.DockerException( + raise DockerException( 'Install pypiwin32 package to enable npipe:// support' ) self.mount('http+docker://', self._custom_adapter) @@ -90,24 +108,24 @@ def __init__(self, base_url=None, version=None, # version detection needs to be after unix adapter mounting if version is None: - self._version = constants.DEFAULT_DOCKER_API_VERSION + self._version = DEFAULT_DOCKER_API_VERSION elif isinstance(version, six.string_types): if version.lower() == 'auto': self._version = self._retrieve_server_version() else: self._version = version else: - raise errors.DockerException( + raise DockerException( 'Version parameter must be a string or None. Found {0}'.format( type(version).__name__ ) ) - if utils.version_lt(self._version, constants.MINIMUM_DOCKER_API_VERSION): + if utils.version_lt(self._version, MINIMUM_DOCKER_API_VERSION): warnings.warn( 'The minimum API version supported is {}, but you are using ' 'version {}. It is recommended you either upgrade Docker ' 'Engine or use an older version of docker-py.'.format( - constants.MINIMUM_DOCKER_API_VERSION, self._version) + MINIMUM_DOCKER_API_VERSION, self._version) ) @classmethod @@ -121,12 +139,12 @@ def _retrieve_server_version(self): try: return self.version(api_version=False)["ApiVersion"] except KeyError: - raise errors.DockerException( + raise DockerException( 'Invalid response from docker daemon: key "ApiVersion"' ' is missing.' ) except Exception as e: - raise errors.DockerException( + raise DockerException( 'Error while fetching server API version: {0}'.format(e) ) @@ -176,8 +194,8 @@ def _raise_for_status(self, response, explanation=None): response.raise_for_status() except requests.exceptions.HTTPError as e: if e.response.status_code == 404: - raise errors.NotFound(e, response, explanation=explanation) - raise errors.APIError(e, response, explanation=explanation) + raise NotFound(e, response, explanation=explanation) + raise APIError(e, response, explanation=explanation) def _result(self, response, json=False, binary=False): assert not (json and binary) @@ -282,7 +300,7 @@ def _multiplexed_buffer_helper(self, response): if len(buf[walker:]) < 8: break _, length = struct.unpack_from('>BxxxL', buf[walker:]) - start = walker + constants.STREAM_HEADER_SIZE_BYTES + start = walker + STREAM_HEADER_SIZE_BYTES end = start + length walker = end yield buf[start:end] @@ -297,7 +315,7 @@ def _multiplexed_response_stream_helper(self, response): self._disable_socket_timeout(socket) while True: - header = response.raw.read(constants.STREAM_HEADER_SIZE_BYTES) + header = response.raw.read(STREAM_HEADER_SIZE_BYTES) if not header: break _, length = struct.unpack('>BxxxL', header) @@ -390,7 +408,7 @@ def _unmount(self, *args): def get_adapter(self, url): try: - return super(Client, self).get_adapter(url) + return super(APIClient, self).get_adapter(url) except requests.exceptions.InvalidSchema as e: if self._custom_adapter: return self._custom_adapter diff --git a/tests/integration/build_test.py b/tests/integration/api_build_test.py similarity index 100% rename from tests/integration/build_test.py rename to tests/integration/api_build_test.py diff --git a/tests/integration/api_test.py b/tests/integration/api_client_test.py similarity index 95% rename from tests/integration/api_test.py rename to tests/integration/api_client_test.py index 6b54aa937c..270bc3dd51 100644 --- a/tests/integration/api_test.py +++ b/tests/integration/api_client_test.py @@ -25,7 +25,7 @@ def test_info(self): self.assertIn('Debug', res) def test_search(self): - client = docker.from_env(timeout=10) + client = docker.APIClient(timeout=10, **kwargs_from_env()) res = client.search('busybox') self.assertTrue(len(res) >= 1) base_img = [x for x in res if x['name'] == 'busybox'] @@ -114,7 +114,7 @@ def test_load_json_config(self): class AutoDetectVersionTest(unittest.TestCase): def test_client_init(self): - client = docker.from_env(version='auto') + client = docker.APIClient(version='auto', **kwargs_from_env()) client_version = client._version api_version = client.version(api_version=False)['ApiVersion'] self.assertEqual(client_version, api_version) @@ -126,7 +126,7 @@ def test_client_init(self): class ConnectionTimeoutTest(unittest.TestCase): def setUp(self): self.timeout = 0.5 - self.client = docker.client.Client(base_url='http://192.168.10.2:4243', + self.client = docker.api.APIClient(base_url='http://192.168.10.2:4243', timeout=self.timeout) def test_timeout(self): @@ -155,7 +155,7 @@ def test_resource_warnings(self): with warnings.catch_warnings(record=True) as w: warnings.simplefilter('always') - client = docker.from_env() + client = docker.APIClient(**kwargs_from_env()) client.images() client.close() del client diff --git a/tests/integration/container_test.py b/tests/integration/api_container_test.py similarity index 100% rename from tests/integration/container_test.py rename to tests/integration/api_container_test.py diff --git a/tests/integration/exec_test.py b/tests/integration/api_exec_test.py similarity index 100% rename from tests/integration/exec_test.py rename to tests/integration/api_exec_test.py diff --git a/tests/integration/healthcheck_test.py b/tests/integration/api_healthcheck_test.py similarity index 93% rename from tests/integration/healthcheck_test.py rename to tests/integration/api_healthcheck_test.py index 9c0f39802d..afe1dea21c 100644 --- a/tests/integration/healthcheck_test.py +++ b/tests/integration/api_healthcheck_test.py @@ -1,5 +1,4 @@ -from .base import BaseIntegrationTest -from .base import BUSYBOX +from .base import BaseAPIIntegrationTest, BUSYBOX from .. import helpers SECOND = 1000000000 @@ -12,7 +11,7 @@ def condition(): return helpers.wait_on_condition(condition) -class HealthcheckTest(BaseIntegrationTest): +class HealthcheckTest(BaseAPIIntegrationTest): @helpers.requires_api_version('1.24') def test_healthcheck_shell_command(self): diff --git a/tests/integration/image_test.py b/tests/integration/api_image_test.py similarity index 100% rename from tests/integration/image_test.py rename to tests/integration/api_image_test.py diff --git a/tests/integration/network_test.py b/tests/integration/api_network_test.py similarity index 100% rename from tests/integration/network_test.py rename to tests/integration/api_network_test.py diff --git a/tests/integration/service_test.py b/tests/integration/api_service_test.py similarity index 100% rename from tests/integration/service_test.py rename to tests/integration/api_service_test.py diff --git a/tests/integration/swarm_test.py b/tests/integration/api_swarm_test.py similarity index 100% rename from tests/integration/swarm_test.py rename to tests/integration/api_swarm_test.py diff --git a/tests/integration/volume_test.py b/tests/integration/api_volume_test.py similarity index 100% rename from tests/integration/volume_test.py rename to tests/integration/api_volume_test.py diff --git a/tests/integration/base.py b/tests/integration/base.py index 3fb25b52e3..2c7c05299b 100644 --- a/tests/integration/base.py +++ b/tests/integration/base.py @@ -2,6 +2,7 @@ import unittest import docker +from docker.utils import kwargs_from_env import six @@ -23,7 +24,7 @@ def setUp(self): if six.PY2: self.assertRegex = self.assertRegexpMatches self.assertCountEqual = self.assertItemsEqual - self.client = docker.from_env(timeout=60) + self.client = docker.APIClient(timeout=60, **kwargs_from_env()) self.tmp_imgs = [] self.tmp_containers = [] self.tmp_folders = [] diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index c488f90672..7217fe07a3 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -4,6 +4,7 @@ import warnings import docker.errors +from docker.utils import kwargs_from_env import pytest from .base import BUSYBOX @@ -12,7 +13,7 @@ @pytest.fixture(autouse=True, scope='session') def setup_test_session(): warnings.simplefilter('error') - c = docker.from_env() + c = docker.APIClient(**kwargs_from_env()) try: c.inspect_image(BUSYBOX) except docker.errors.NotFound: diff --git a/tests/unit/build_test.py b/tests/unit/api_build_test.py similarity index 100% rename from tests/unit/build_test.py rename to tests/unit/api_build_test.py diff --git a/tests/unit/client_test.py b/tests/unit/api_client_test.py similarity index 93% rename from tests/unit/client_test.py rename to tests/unit/api_client_test.py index 2a5124b2d6..6fc8202537 100644 --- a/tests/unit/client_test.py +++ b/tests/unit/api_client_test.py @@ -1,7 +1,7 @@ import os import unittest -from docker.client import Client +from docker.api import APIClient TEST_CERT_DIR = os.path.join( os.path.dirname(__file__), @@ -23,14 +23,14 @@ def test_from_env(self): os.environ.update(DOCKER_HOST='tcp://192.168.59.103:2376', DOCKER_CERT_PATH=TEST_CERT_DIR, DOCKER_TLS_VERIFY='1') - client = Client.from_env() + client = APIClient.from_env() self.assertEqual(client.base_url, "https://192.168.59.103:2376") def test_from_env_with_version(self): os.environ.update(DOCKER_HOST='tcp://192.168.59.103:2376', DOCKER_CERT_PATH=TEST_CERT_DIR, DOCKER_TLS_VERIFY='1') - client = Client.from_env(version='2.32') + client = APIClient.from_env(version='2.32') self.assertEqual(client.base_url, "https://192.168.59.103:2376") self.assertEqual(client._version, '2.32') @@ -47,7 +47,7 @@ def gettimeout(self): return self.timeout def setUp(self): - self.client = Client() + self.client = APIClient() def test_disable_socket_timeout(self): """Test that the timeout is disabled on a generic socket object.""" diff --git a/tests/unit/container_test.py b/tests/unit/api_container_test.py similarity index 98% rename from tests/unit/container_test.py rename to tests/unit/api_container_test.py index 51e8cbba9c..5eb6c5375f 100644 --- a/tests/unit/container_test.py +++ b/tests/unit/api_container_test.py @@ -1244,7 +1244,7 @@ def test_wait_with_dict_instead_of_id(self): ) def test_logs(self): - with mock.patch('docker.Client.inspect_container', + with mock.patch('docker.api.client.APIClient.inspect_container', fake_inspect_container): logs = self.client.logs(fake_api.FAKE_CONTAINER_ID) @@ -1263,7 +1263,7 @@ def test_logs(self): ) def test_logs_with_dict_instead_of_id(self): - with mock.patch('docker.Client.inspect_container', + with mock.patch('docker.api.client.APIClient.inspect_container', fake_inspect_container): logs = self.client.logs({'Id': fake_api.FAKE_CONTAINER_ID}) @@ -1282,7 +1282,7 @@ def test_logs_with_dict_instead_of_id(self): ) def test_log_streaming(self): - with mock.patch('docker.Client.inspect_container', + with mock.patch('docker.api.client.APIClient.inspect_container', fake_inspect_container): self.client.logs(fake_api.FAKE_CONTAINER_ID, stream=True, follow=False) @@ -1297,7 +1297,7 @@ def test_log_streaming(self): ) def test_log_following(self): - with mock.patch('docker.Client.inspect_container', + with mock.patch('docker.api.client.APIClient.inspect_container', fake_inspect_container): self.client.logs(fake_api.FAKE_CONTAINER_ID, stream=False, follow=True) @@ -1312,7 +1312,7 @@ def test_log_following(self): ) def test_log_following_backwards(self): - with mock.patch('docker.Client.inspect_container', + with mock.patch('docker.api.client.APIClient.inspect_container', fake_inspect_container): self.client.logs(fake_api.FAKE_CONTAINER_ID, stream=True) @@ -1326,7 +1326,7 @@ def test_log_following_backwards(self): ) def test_log_streaming_and_following(self): - with mock.patch('docker.Client.inspect_container', + with mock.patch('docker.api.client.APIClient.inspect_container', fake_inspect_container): self.client.logs(fake_api.FAKE_CONTAINER_ID, stream=True, follow=True) @@ -1342,7 +1342,7 @@ def test_log_streaming_and_following(self): def test_log_tail(self): - with mock.patch('docker.Client.inspect_container', + with mock.patch('docker.api.client.APIClient.inspect_container', fake_inspect_container): self.client.logs(fake_api.FAKE_CONTAINER_ID, stream=False, follow=False, tail=10) @@ -1358,7 +1358,7 @@ def test_log_tail(self): def test_log_since(self): ts = 809222400 - with mock.patch('docker.Client.inspect_container', + with mock.patch('docker.api.client.APIClient.inspect_container', fake_inspect_container): self.client.logs(fake_api.FAKE_CONTAINER_ID, stream=False, follow=False, since=ts) @@ -1375,7 +1375,7 @@ def test_log_since(self): def test_log_since_with_datetime(self): ts = 809222400 time = datetime.datetime.utcfromtimestamp(ts) - with mock.patch('docker.Client.inspect_container', + with mock.patch('docker.api.client.APIClient.inspect_container', fake_inspect_container): self.client.logs(fake_api.FAKE_CONTAINER_ID, stream=False, follow=False, since=time) @@ -1391,9 +1391,9 @@ def test_log_since_with_datetime(self): def test_log_tty(self): m = mock.Mock() - with mock.patch('docker.Client.inspect_container', + with mock.patch('docker.api.client.APIClient.inspect_container', fake_inspect_container_tty): - with mock.patch('docker.Client._stream_raw_result', + with mock.patch('docker.api.client.APIClient._stream_raw_result', m): self.client.logs(fake_api.FAKE_CONTAINER_ID, follow=True, stream=True) diff --git a/tests/unit/exec_test.py b/tests/unit/api_exec_test.py similarity index 100% rename from tests/unit/exec_test.py rename to tests/unit/api_exec_test.py diff --git a/tests/unit/image_test.py b/tests/unit/api_image_test.py similarity index 100% rename from tests/unit/image_test.py rename to tests/unit/api_image_test.py diff --git a/tests/unit/network_test.py b/tests/unit/api_network_test.py similarity index 93% rename from tests/unit/network_test.py rename to tests/unit/api_network_test.py index 5d1f4392d3..083b2e6590 100644 --- a/tests/unit/network_test.py +++ b/tests/unit/api_network_test.py @@ -33,7 +33,7 @@ def test_list_networks(self): get = mock.Mock(return_value=response( status_code=200, content=json.dumps(networks).encode('utf-8'))) - with mock.patch('docker.Client.get', get): + with mock.patch('docker.api.client.APIClient.get', get): self.assertEqual(self.client.networks(), networks) self.assertEqual(get.call_args[0][0], url_prefix + 'networks') @@ -59,7 +59,7 @@ def test_create_network(self): network_response = response(status_code=200, content=network_data) post = mock.Mock(return_value=network_response) - with mock.patch('docker.Client.post', post): + with mock.patch('docker.api.client.APIClient.post', post): result = self.client.create_network('foo') self.assertEqual(result, network_data) @@ -109,7 +109,7 @@ def test_remove_network(self): network_id = 'abc12345' delete = mock.Mock(return_value=response(status_code=200)) - with mock.patch('docker.Client.delete', delete): + with mock.patch('docker.api.client.APIClient.delete', delete): self.client.remove_network(network_id) args = delete.call_args @@ -130,7 +130,7 @@ def test_inspect_network(self): network_response = response(status_code=200, content=network_data) get = mock.Mock(return_value=network_response) - with mock.patch('docker.Client.get', get): + with mock.patch('docker.api.client.APIClient.get', get): result = self.client.inspect_network(network_id) self.assertEqual(result, network_data) @@ -145,7 +145,7 @@ def test_connect_container_to_network(self): post = mock.Mock(return_value=response(status_code=201)) - with mock.patch('docker.Client.post', post): + with mock.patch('docker.api.client.APIClient.post', post): self.client.connect_container_to_network( {'Id': container_id}, network_id, @@ -174,7 +174,7 @@ def test_disconnect_container_from_network(self): post = mock.Mock(return_value=response(status_code=201)) - with mock.patch('docker.Client.post', post): + with mock.patch('docker.api.client.APIClient.post', post): self.client.disconnect_container_from_network( {'Id': container_id}, network_id) diff --git a/tests/unit/api_test.py b/tests/unit/api_test.py index 5777ab9af5..3ab500a080 100644 --- a/tests/unit/api_test.py +++ b/tests/unit/api_test.py @@ -11,6 +11,7 @@ import unittest import docker +from docker.api import APIClient import requests from requests.packages import urllib3 import six @@ -95,12 +96,15 @@ def fake_read_from_socket(self, response, stream): class DockerClientTest(unittest.TestCase): def setUp(self): self.patcher = mock.patch.multiple( - 'docker.Client', get=fake_get, post=fake_post, put=fake_put, + 'docker.api.client.APIClient', + get=fake_get, + post=fake_post, + put=fake_put, delete=fake_delete, _read_from_socket=fake_read_from_socket ) self.patcher.start() - self.client = docker.Client() + self.client = APIClient() # Force-clear authconfig to avoid tampering with the tests self.client._cfg = {'Configs': {}} @@ -122,7 +126,7 @@ def base_create_payload(self, img='busybox', cmd=None): class DockerApiTest(DockerClientTest): def test_ctor(self): with pytest.raises(docker.errors.DockerException) as excinfo: - docker.Client(version=1.12) + APIClient(version=1.12) self.assertEqual( str(excinfo.value), @@ -189,7 +193,7 @@ def test_version_no_api_version(self): ) def test_retrieve_server_version(self): - client = docker.Client(version="auto") + client = APIClient(version="auto") self.assertTrue(isinstance(client._version, six.string_types)) self.assertFalse(client._version == "auto") client.close() @@ -269,27 +273,27 @@ def _socket_path_for_client_session(self, client): return socket_adapter.socket_path def test_url_compatibility_unix(self): - c = docker.Client(base_url="unix://socket") + c = APIClient(base_url="unix://socket") assert self._socket_path_for_client_session(c) == '/socket' def test_url_compatibility_unix_triple_slash(self): - c = docker.Client(base_url="unix:///socket") + c = APIClient(base_url="unix:///socket") assert self._socket_path_for_client_session(c) == '/socket' def test_url_compatibility_http_unix_triple_slash(self): - c = docker.Client(base_url="http+unix:///socket") + c = APIClient(base_url="http+unix:///socket") assert self._socket_path_for_client_session(c) == '/socket' def test_url_compatibility_http(self): - c = docker.Client(base_url="http://hostname:1234") + c = APIClient(base_url="http://hostname:1234") assert c.base_url == "http://hostname:1234" def test_url_compatibility_tcp(self): - c = docker.Client(base_url="tcp://hostname:1234") + c = APIClient(base_url="tcp://hostname:1234") assert c.base_url == "http://hostname:1234" @@ -435,7 +439,7 @@ def test_early_stream_response(self): b'\r\n' ) + b'\r\n'.join(lines) - with docker.Client(base_url="http+unix://" + self.socket_file) \ + with APIClient(base_url="http+unix://" + self.socket_file) \ as client: for i in range(5): try: @@ -455,7 +459,7 @@ def test_early_stream_response(self): class UserAgentTest(unittest.TestCase): def setUp(self): self.patcher = mock.patch.object( - docker.Client, + APIClient, 'send', return_value=fake_resp("GET", "%s/version" % fake_api.prefix) ) @@ -465,7 +469,7 @@ def tearDown(self): self.patcher.stop() def test_default_user_agent(self): - client = docker.Client() + client = APIClient() client.version() self.assertEqual(self.mock_send.call_count, 1) @@ -474,7 +478,7 @@ def test_default_user_agent(self): self.assertEqual(headers['User-Agent'], expected) def test_custom_user_agent(self): - client = docker.Client(user_agent='foo/bar') + client = APIClient(user_agent='foo/bar') client.version() self.assertEqual(self.mock_send.call_count, 1) diff --git a/tests/unit/volume_test.py b/tests/unit/api_volume_test.py similarity index 100% rename from tests/unit/volume_test.py rename to tests/unit/api_volume_test.py diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index 80d156f5be..57aa226d84 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -13,10 +13,8 @@ import pytest import six -from docker.client import Client -from docker.constants import ( - DEFAULT_DOCKER_API_VERSION, IS_WINDOWS_PLATFORM -) +from docker.api.client import APIClient +from docker.constants import DEFAULT_DOCKER_API_VERSION, IS_WINDOWS_PLATFORM from docker.errors import DockerException, InvalidVersion from docker.utils import ( parse_repository_tag, parse_host, convert_filters, kwargs_from_env, @@ -47,7 +45,7 @@ def test_update_headers(self): def f(self, headers=None): return headers - client = Client() + client = APIClient() client._auth_configs = {} g = update_headers(f) @@ -305,7 +303,7 @@ def test_kwargs_from_env_tls(self): self.assertEqual(False, kwargs['tls'].assert_hostname) self.assertTrue(kwargs['tls'].verify) try: - client = Client(**kwargs) + client = APIClient(**kwargs) self.assertEqual(kwargs['base_url'], client.base_url) self.assertEqual(kwargs['tls'].ca_cert, client.verify) self.assertEqual(kwargs['tls'].cert, client.cert) @@ -324,7 +322,7 @@ def test_kwargs_from_env_tls_verify_false(self): self.assertEqual(True, kwargs['tls'].assert_hostname) self.assertEqual(False, kwargs['tls'].verify) try: - client = Client(**kwargs) + client = APIClient(**kwargs) self.assertEqual(kwargs['base_url'], client.base_url) self.assertEqual(kwargs['tls'].cert, client.cert) self.assertFalse(kwargs['tls'].verify) @@ -821,6 +819,7 @@ def test_build_port_bindings_with_nonmatching_internal_port_ranges(self): self.assertEqual(port_bindings["1000"], [("127.0.0.1", "1000")]) self.assertEqual(port_bindings["2000"], [("127.0.0.1", "2000")]) + def convert_paths(collection): if not IS_WINDOWS_PLATFORM: return collection @@ -1093,7 +1092,7 @@ def test_tar_with_directory_symlinks(self): ) -class FormatEnvironmentTest(base.BaseTestCase): +class FormatEnvironmentTest(unittest.TestCase): def test_format_env_binary_unicode_value(self): env_dict = { 'ARTIST_NAME': b'\xec\x86\xa1\xec\xa7\x80\xec\x9d\x80' From dac7174ff2408bf9e28b4c330e24e38c130d372c Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Tue, 18 Oct 2016 17:03:26 +0200 Subject: [PATCH 0170/1301] Make ping return bool instead of string Signed-off-by: Ben Firshman --- docker/api/daemon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/api/daemon.py b/docker/api/daemon.py index 980ae56876..2e87cf041f 100644 --- a/docker/api/daemon.py +++ b/docker/api/daemon.py @@ -68,7 +68,7 @@ def login(self, username, password=None, email=None, registry=None, return self._result(response, json=True) def ping(self): - return self._result(self._get(self._url('/_ping'))) + return self._result(self._get(self._url('/_ping'))) == 'OK' def version(self, api_version=True): url = self._url("/version", versioned_api=api_version) From e055a1c813768984c776a4d83b33ca2dbcd5f316 Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Tue, 6 Sep 2016 10:48:47 +0200 Subject: [PATCH 0171/1301] Rename DockerClientTest to BaseAPIClientTest Signed-off-by: Ben Firshman --- tests/unit/api_build_test.py | 4 ++-- tests/unit/api_container_test.py | 8 ++++---- tests/unit/api_exec_test.py | 4 ++-- tests/unit/api_image_test.py | 4 ++-- tests/unit/api_network_test.py | 4 ++-- tests/unit/api_test.py | 4 ++-- tests/unit/api_volume_test.py | 4 ++-- tests/unit/swarm_test.py | 4 ++-- 8 files changed, 18 insertions(+), 18 deletions(-) diff --git a/tests/unit/api_build_test.py b/tests/unit/api_build_test.py index b2705eb2f1..8146fee721 100644 --- a/tests/unit/api_build_test.py +++ b/tests/unit/api_build_test.py @@ -4,10 +4,10 @@ import docker from docker import auth -from .api_test import DockerClientTest, fake_request, url_prefix +from .api_test import BaseAPIClientTest, fake_request, url_prefix -class BuildTest(DockerClientTest): +class BuildTest(BaseAPIClientTest): def test_build_container(self): script = io.BytesIO('\n'.join([ 'FROM busybox', diff --git a/tests/unit/api_container_test.py b/tests/unit/api_container_test.py index 5eb6c5375f..6c08064179 100644 --- a/tests/unit/api_container_test.py +++ b/tests/unit/api_container_test.py @@ -11,7 +11,7 @@ from . import fake_api from ..helpers import requires_api_version from .api_test import ( - DockerClientTest, url_prefix, fake_request, DEFAULT_TIMEOUT_SECONDS, + BaseAPIClientTest, url_prefix, fake_request, DEFAULT_TIMEOUT_SECONDS, fake_inspect_container ) @@ -25,7 +25,7 @@ def fake_inspect_container_tty(self, container): return fake_inspect_container(self, container, tty=True) -class StartContainerTest(DockerClientTest): +class StartContainerTest(BaseAPIClientTest): def test_start_container(self): self.client.start(fake_api.FAKE_CONTAINER_ID) @@ -168,7 +168,7 @@ def test_start_container_with_dict_instead_of_id(self): ) -class CreateContainerTest(DockerClientTest): +class CreateContainerTest(BaseAPIClientTest): def test_create_container(self): self.client.create_container('busybox', 'true') @@ -1180,7 +1180,7 @@ def test_create_container_with_unicode_envvars(self): self.assertEqual(json.loads(args[1]['data'])['Env'], expected) -class ContainerTest(DockerClientTest): +class ContainerTest(BaseAPIClientTest): def test_list_containers(self): self.client.containers(all=True) diff --git a/tests/unit/api_exec_test.py b/tests/unit/api_exec_test.py index 6ba2a3ddf6..41ee940a8a 100644 --- a/tests/unit/api_exec_test.py +++ b/tests/unit/api_exec_test.py @@ -2,11 +2,11 @@ from . import fake_api from .api_test import ( - DockerClientTest, url_prefix, fake_request, DEFAULT_TIMEOUT_SECONDS, + BaseAPIClientTest, url_prefix, fake_request, DEFAULT_TIMEOUT_SECONDS, ) -class ExecTest(DockerClientTest): +class ExecTest(BaseAPIClientTest): def test_exec_create(self): self.client.exec_create(fake_api.FAKE_CONTAINER_ID, ['ls', '-1']) diff --git a/tests/unit/api_image_test.py b/tests/unit/api_image_test.py index be9d574c18..fbfb146bb7 100644 --- a/tests/unit/api_image_test.py +++ b/tests/unit/api_image_test.py @@ -4,7 +4,7 @@ from . import fake_api from docker import auth from .api_test import ( - DockerClientTest, fake_request, DEFAULT_TIMEOUT_SECONDS, url_prefix, + BaseAPIClientTest, fake_request, DEFAULT_TIMEOUT_SECONDS, url_prefix, fake_resolve_authconfig ) @@ -14,7 +14,7 @@ import mock -class ImageTest(DockerClientTest): +class ImageTest(BaseAPIClientTest): def test_image_viz(self): with pytest.raises(Exception): self.client.images('busybox', viz=True) diff --git a/tests/unit/api_network_test.py b/tests/unit/api_network_test.py index 083b2e6590..8e09c6756c 100644 --- a/tests/unit/api_network_test.py +++ b/tests/unit/api_network_test.py @@ -2,7 +2,7 @@ import six -from .api_test import DockerClientTest, url_prefix, response +from .api_test import BaseAPIClientTest, url_prefix, response from ..helpers import requires_api_version from docker.utils import create_ipam_config, create_ipam_pool @@ -12,7 +12,7 @@ import mock -class NetworkTest(DockerClientTest): +class NetworkTest(BaseAPIClientTest): @requires_api_version('1.21') def test_list_networks(self): networks = [ diff --git a/tests/unit/api_test.py b/tests/unit/api_test.py index 3ab500a080..f57865aef8 100644 --- a/tests/unit/api_test.py +++ b/tests/unit/api_test.py @@ -93,7 +93,7 @@ def fake_read_from_socket(self, response, stream): docker.constants.DEFAULT_DOCKER_API_VERSION) -class DockerClientTest(unittest.TestCase): +class BaseAPIClientTest(unittest.TestCase): def setUp(self): self.patcher = mock.patch.multiple( 'docker.api.client.APIClient', @@ -123,7 +123,7 @@ def base_create_payload(self, img='busybox', cmd=None): } -class DockerApiTest(DockerClientTest): +class DockerApiTest(BaseAPIClientTest): def test_ctor(self): with pytest.raises(docker.errors.DockerException) as excinfo: APIClient(version=1.12) diff --git a/tests/unit/api_volume_test.py b/tests/unit/api_volume_test.py index 3909977165..cb72cb2580 100644 --- a/tests/unit/api_volume_test.py +++ b/tests/unit/api_volume_test.py @@ -3,10 +3,10 @@ import pytest from ..helpers import requires_api_version -from .api_test import DockerClientTest, url_prefix, fake_request +from .api_test import BaseAPIClientTest, url_prefix, fake_request -class VolumeTest(DockerClientTest): +class VolumeTest(BaseAPIClientTest): @requires_api_version('1.21') def test_list_volumes(self): volumes = self.client.volumes() diff --git a/tests/unit/swarm_test.py b/tests/unit/swarm_test.py index 39d4ec46fc..374f8b2473 100644 --- a/tests/unit/swarm_test.py +++ b/tests/unit/swarm_test.py @@ -4,10 +4,10 @@ from . import fake_api from ..helpers import requires_api_version -from .api_test import (DockerClientTest, url_prefix, fake_request) +from .api_test import BaseAPIClientTest, url_prefix, fake_request -class SwarmTest(DockerClientTest): +class SwarmTest(BaseAPIClientTest): @requires_api_version('1.24') def test_node_update(self): node_spec = { From 39900c558c0d801641fc502745e1acd48cf07d4a Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Tue, 6 Sep 2016 10:53:31 +0200 Subject: [PATCH 0172/1301] Move APIClient tests into single file For some reason this was spread across two files. Signed-off-by: Ben Firshman --- tests/unit/api_client_test.py | 79 ----------------------------------- tests/unit/api_test.py | 71 +++++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+), 79 deletions(-) delete mode 100644 tests/unit/api_client_test.py diff --git a/tests/unit/api_client_test.py b/tests/unit/api_client_test.py deleted file mode 100644 index 6fc8202537..0000000000 --- a/tests/unit/api_client_test.py +++ /dev/null @@ -1,79 +0,0 @@ -import os -import unittest - -from docker.api import APIClient - -TEST_CERT_DIR = os.path.join( - os.path.dirname(__file__), - 'testdata/certs', -) - - -class ClientTest(unittest.TestCase): - def setUp(self): - self.os_environ = os.environ.copy() - - def tearDown(self): - os.environ = self.os_environ - - def test_from_env(self): - """Test that environment variables are passed through to - utils.kwargs_from_env(). KwargsFromEnvTest tests that environment - variables are parsed correctly.""" - os.environ.update(DOCKER_HOST='tcp://192.168.59.103:2376', - DOCKER_CERT_PATH=TEST_CERT_DIR, - DOCKER_TLS_VERIFY='1') - client = APIClient.from_env() - self.assertEqual(client.base_url, "https://192.168.59.103:2376") - - def test_from_env_with_version(self): - os.environ.update(DOCKER_HOST='tcp://192.168.59.103:2376', - DOCKER_CERT_PATH=TEST_CERT_DIR, - DOCKER_TLS_VERIFY='1') - client = APIClient.from_env(version='2.32') - self.assertEqual(client.base_url, "https://192.168.59.103:2376") - self.assertEqual(client._version, '2.32') - - -class DisableSocketTest(unittest.TestCase): - class DummySocket(object): - def __init__(self, timeout=60): - self.timeout = timeout - - def settimeout(self, timeout): - self.timeout = timeout - - def gettimeout(self): - return self.timeout - - def setUp(self): - self.client = APIClient() - - def test_disable_socket_timeout(self): - """Test that the timeout is disabled on a generic socket object.""" - socket = self.DummySocket() - - self.client._disable_socket_timeout(socket) - - self.assertEqual(socket.timeout, None) - - def test_disable_socket_timeout2(self): - """Test that the timeouts are disabled on a generic socket object - and it's _sock object if present.""" - socket = self.DummySocket() - socket._sock = self.DummySocket() - - self.client._disable_socket_timeout(socket) - - self.assertEqual(socket.timeout, None) - self.assertEqual(socket._sock.timeout, None) - - def test_disable_socket_timout_non_blocking(self): - """Test that a non-blocking socket does not get set to blocking.""" - socket = self.DummySocket() - socket._sock = self.DummySocket(0.0) - - self.client._disable_socket_timeout(socket) - - self.assertEqual(socket.timeout, None) - self.assertEqual(socket._sock.timeout, 0.0) diff --git a/tests/unit/api_test.py b/tests/unit/api_test.py index f57865aef8..dbd551df5f 100644 --- a/tests/unit/api_test.py +++ b/tests/unit/api_test.py @@ -27,6 +27,7 @@ DEFAULT_TIMEOUT_SECONDS = docker.constants.DEFAULT_TIMEOUT_SECONDS +TEST_CERT_DIR = os.path.join(os.path.dirname(__file__), 'testdata/certs') def response(status_code=200, content='', headers=None, reason=None, elapsed=0, @@ -484,3 +485,73 @@ def test_custom_user_agent(self): self.assertEqual(self.mock_send.call_count, 1) headers = self.mock_send.call_args[0][0].headers self.assertEqual(headers['User-Agent'], 'foo/bar') + + +class FromEnvTest(unittest.TestCase): + def setUp(self): + self.os_environ = os.environ.copy() + + def tearDown(self): + os.environ = self.os_environ + + def test_from_env(self): + """Test that environment variables are passed through to + utils.kwargs_from_env(). KwargsFromEnvTest tests that environment + variables are parsed correctly.""" + os.environ.update(DOCKER_HOST='tcp://192.168.59.103:2376', + DOCKER_CERT_PATH=TEST_CERT_DIR, + DOCKER_TLS_VERIFY='1') + client = APIClient.from_env() + self.assertEqual(client.base_url, "https://192.168.59.103:2376") + + def test_from_env_with_version(self): + os.environ.update(DOCKER_HOST='tcp://192.168.59.103:2376', + DOCKER_CERT_PATH=TEST_CERT_DIR, + DOCKER_TLS_VERIFY='1') + client = APIClient.from_env(version='2.32') + self.assertEqual(client.base_url, "https://192.168.59.103:2376") + self.assertEqual(client._version, '2.32') + + +class DisableSocketTest(unittest.TestCase): + class DummySocket(object): + def __init__(self, timeout=60): + self.timeout = timeout + + def settimeout(self, timeout): + self.timeout = timeout + + def gettimeout(self): + return self.timeout + + def setUp(self): + self.client = APIClient() + + def test_disable_socket_timeout(self): + """Test that the timeout is disabled on a generic socket object.""" + socket = self.DummySocket() + + self.client._disable_socket_timeout(socket) + + self.assertEqual(socket.timeout, None) + + def test_disable_socket_timeout2(self): + """Test that the timeouts are disabled on a generic socket object + and it's _sock object if present.""" + socket = self.DummySocket() + socket._sock = self.DummySocket() + + self.client._disable_socket_timeout(socket) + + self.assertEqual(socket.timeout, None) + self.assertEqual(socket._sock.timeout, None) + + def test_disable_socket_timout_non_blocking(self): + """Test that a non-blocking socket does not get set to blocking.""" + socket = self.DummySocket() + socket._sock = self.DummySocket(0.0) + + self.client._disable_socket_timeout(socket) + + self.assertEqual(socket.timeout, None) + self.assertEqual(socket._sock.timeout, 0.0) From 6334312e475572f42a87988e09b3f65f03c91f93 Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Thu, 15 Sep 2016 17:57:49 +0100 Subject: [PATCH 0173/1301] Split out base integration test for APIClient So the cleanup stuff can be reused for model tests. Signed-off-by: Ben Firshman --- tests/integration/api_build_test.py | 4 +-- tests/integration/api_client_test.py | 8 ++--- tests/integration/api_container_test.py | 42 ++++++++++++------------- tests/integration/api_exec_test.py | 4 +-- tests/integration/api_image_test.py | 12 +++---- tests/integration/api_network_test.py | 4 +-- tests/integration/api_service_test.py | 4 +-- tests/integration/api_swarm_test.py | 4 +-- tests/integration/api_volume_test.py | 4 +-- tests/integration/base.py | 37 ++++++++++++---------- tests/integration/errors_test.py | 4 +-- tests/integration/regression_test.py | 4 +-- 12 files changed, 67 insertions(+), 64 deletions(-) diff --git a/tests/integration/api_build_test.py b/tests/integration/api_build_test.py index 2695b92aa2..9ae74f4d44 100644 --- a/tests/integration/api_build_test.py +++ b/tests/integration/api_build_test.py @@ -8,10 +8,10 @@ from docker import errors from ..helpers import requires_api_version -from .base import BaseIntegrationTest +from .base import BaseAPIIntegrationTest -class BuildTest(BaseIntegrationTest): +class BuildTest(BaseAPIIntegrationTest): def test_build_streaming(self): script = io.BytesIO('\n'.join([ 'FROM busybox', diff --git a/tests/integration/api_client_test.py b/tests/integration/api_client_test.py index 270bc3dd51..dab8ddf382 100644 --- a/tests/integration/api_client_test.py +++ b/tests/integration/api_client_test.py @@ -8,10 +8,10 @@ import docker from docker.utils import kwargs_from_env -from .base import BaseIntegrationTest, BUSYBOX +from .base import BaseAPIIntegrationTest, BUSYBOX -class InformationTest(BaseIntegrationTest): +class InformationTest(BaseAPIIntegrationTest): def test_version(self): res = self.client.version() self.assertIn('GoVersion', res) @@ -33,7 +33,7 @@ def test_search(self): self.assertIn('description', base_img[0]) -class LinkTest(BaseIntegrationTest): +class LinkTest(BaseAPIIntegrationTest): def test_remove_link(self): # Create containers container1 = self.client.create_container( @@ -75,7 +75,7 @@ def test_remove_link(self): self.assertEqual(len(retrieved), 2) -class LoadConfigTest(BaseIntegrationTest): +class LoadConfigTest(BaseAPIIntegrationTest): def test_load_legacy_config(self): folder = tempfile.mkdtemp() self.tmp_folders.append(folder) diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index 71e192fde1..a5be6e765f 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -11,10 +11,10 @@ from ..helpers import requires_api_version from .. import helpers -from .base import BaseIntegrationTest, BUSYBOX +from .base import BaseAPIIntegrationTest, BUSYBOX -class ListContainersTest(BaseIntegrationTest): +class ListContainersTest(BaseAPIIntegrationTest): def test_list_containers(self): res0 = self.client.containers(all=True) size = len(res0) @@ -34,7 +34,7 @@ def test_list_containers(self): self.assertIn('Status', retrieved) -class CreateContainerTest(BaseIntegrationTest): +class CreateContainerTest(BaseAPIIntegrationTest): def test_create(self): res = self.client.create_container(BUSYBOX, 'true') @@ -409,7 +409,7 @@ def test_create_with_isolation(self): assert config['HostConfig']['Isolation'] == 'default' -class VolumeBindTest(BaseIntegrationTest): +class VolumeBindTest(BaseAPIIntegrationTest): def setUp(self): super(VolumeBindTest, self).setUp() @@ -504,7 +504,7 @@ def run_with_volume(self, ro, *args, **kwargs): @requires_api_version('1.20') -class ArchiveTest(BaseIntegrationTest): +class ArchiveTest(BaseAPIIntegrationTest): def test_get_file_archive_from_container(self): data = 'The Maid and the Pocket Watch of Blood' ctnr = self.client.create_container( @@ -584,7 +584,7 @@ def test_copy_directory_to_container(self): self.assertIn('bar/', results) -class RenameContainerTest(BaseIntegrationTest): +class RenameContainerTest(BaseAPIIntegrationTest): def test_rename_container(self): version = self.client.version()['Version'] name = 'hong_meiling' @@ -600,7 +600,7 @@ def test_rename_container(self): self.assertEqual('/{0}'.format(name), inspect['Name']) -class StartContainerTest(BaseIntegrationTest): +class StartContainerTest(BaseAPIIntegrationTest): def test_start_container(self): res = self.client.create_container(BUSYBOX, 'true') self.assertIn('Id', res) @@ -654,7 +654,7 @@ def test_run_shlex_commands(self): self.assertEqual(exitcode, 0, msg=cmd) -class WaitTest(BaseIntegrationTest): +class WaitTest(BaseAPIIntegrationTest): def test_wait(self): res = self.client.create_container(BUSYBOX, ['sleep', '3']) id = res['Id'] @@ -682,7 +682,7 @@ def test_wait_with_dict_instead_of_id(self): self.assertEqual(inspect['State']['ExitCode'], exitcode) -class LogsTest(BaseIntegrationTest): +class LogsTest(BaseAPIIntegrationTest): def test_logs(self): snippet = 'Flowering Nights (Sakuya Iyazoi)' container = self.client.create_container( @@ -754,7 +754,7 @@ def test_logs_with_tail_0(self): self.assertEqual(logs, ''.encode(encoding='ascii')) -class DiffTest(BaseIntegrationTest): +class DiffTest(BaseAPIIntegrationTest): def test_diff(self): container = self.client.create_container(BUSYBOX, ['touch', '/test']) id = container['Id'] @@ -782,7 +782,7 @@ def test_diff_with_dict_instead_of_id(self): self.assertEqual(test_diff[0]['Kind'], 1) -class StopTest(BaseIntegrationTest): +class StopTest(BaseAPIIntegrationTest): def test_stop(self): container = self.client.create_container(BUSYBOX, ['sleep', '9999']) id = container['Id'] @@ -809,7 +809,7 @@ def test_stop_with_dict_instead_of_id(self): self.assertEqual(state['Running'], False) -class KillTest(BaseIntegrationTest): +class KillTest(BaseAPIIntegrationTest): def test_kill(self): container = self.client.create_container(BUSYBOX, ['sleep', '9999']) id = container['Id'] @@ -886,7 +886,7 @@ def test_kill_with_signal_integer(self): self.assertEqual(state['Running'], False, state) -class PortTest(BaseIntegrationTest): +class PortTest(BaseAPIIntegrationTest): def test_port(self): port_bindings = { @@ -917,7 +917,7 @@ def test_port(self): self.client.kill(id) -class ContainerTopTest(BaseIntegrationTest): +class ContainerTopTest(BaseAPIIntegrationTest): def test_top(self): container = self.client.create_container( BUSYBOX, ['sleep', '60'] @@ -957,7 +957,7 @@ def test_top_with_psargs(self): self.assertEqual(res['Processes'][0][10], 'sleep 60') -class RestartContainerTest(BaseIntegrationTest): +class RestartContainerTest(BaseAPIIntegrationTest): def test_restart(self): container = self.client.create_container(BUSYBOX, ['sleep', '9999']) id = container['Id'] @@ -998,7 +998,7 @@ def test_restart_with_dict_instead_of_id(self): self.client.kill(id) -class RemoveContainerTest(BaseIntegrationTest): +class RemoveContainerTest(BaseAPIIntegrationTest): def test_remove(self): container = self.client.create_container(BUSYBOX, ['true']) id = container['Id'] @@ -1020,7 +1020,7 @@ def test_remove_with_dict_instead_of_id(self): self.assertEqual(len(res), 0) -class AttachContainerTest(BaseIntegrationTest): +class AttachContainerTest(BaseAPIIntegrationTest): def test_run_container_streaming(self): container = self.client.create_container(BUSYBOX, '/bin/sh', detach=True, stdin_open=True) @@ -1051,7 +1051,7 @@ def test_run_container_reading_socket(self): self.assertEqual(data.decode('utf-8'), line) -class PauseTest(BaseIntegrationTest): +class PauseTest(BaseAPIIntegrationTest): def test_pause_unpause(self): container = self.client.create_container(BUSYBOX, ['sleep', '9999']) id = container['Id'] @@ -1080,7 +1080,7 @@ def test_pause_unpause(self): self.assertEqual(state['Paused'], False) -class GetContainerStatsTest(BaseIntegrationTest): +class GetContainerStatsTest(BaseAPIIntegrationTest): @requires_api_version('1.19') def test_get_container_stats_no_stream(self): container = self.client.create_container( @@ -1111,7 +1111,7 @@ def test_get_container_stats_stream(self): self.assertIn(key, chunk) -class ContainerUpdateTest(BaseIntegrationTest): +class ContainerUpdateTest(BaseAPIIntegrationTest): @requires_api_version('1.22') def test_update_container(self): old_mem_limit = 400 * 1024 * 1024 @@ -1158,7 +1158,7 @@ def test_restart_policy_update(self): ) -class ContainerCPUTest(BaseIntegrationTest): +class ContainerCPUTest(BaseAPIIntegrationTest): @requires_api_version('1.18') def test_container_cpu_shares(self): cpu_shares = 512 diff --git a/tests/integration/api_exec_test.py b/tests/integration/api_exec_test.py index f2a8b1f553..0ceeefa9e1 100644 --- a/tests/integration/api_exec_test.py +++ b/tests/integration/api_exec_test.py @@ -1,10 +1,10 @@ from docker.utils.socket import next_frame_size from docker.utils.socket import read_exactly -from .base import BaseIntegrationTest, BUSYBOX +from .base import BaseAPIIntegrationTest, BUSYBOX -class ExecTest(BaseIntegrationTest): +class ExecTest(BaseAPIIntegrationTest): def test_execute_command(self): container = self.client.create_container(BUSYBOX, 'cat', detach=True, stdin_open=True) diff --git a/tests/integration/api_image_test.py b/tests/integration/api_image_test.py index 31d2218b13..135f115b1c 100644 --- a/tests/integration/api_image_test.py +++ b/tests/integration/api_image_test.py @@ -14,10 +14,10 @@ import docker -from .base import BaseIntegrationTest, BUSYBOX +from .base import BaseAPIIntegrationTest, BUSYBOX -class ListImagesTest(BaseIntegrationTest): +class ListImagesTest(BaseAPIIntegrationTest): def test_images(self): res1 = self.client.images(all=True) self.assertIn('Id', res1[0]) @@ -35,7 +35,7 @@ def test_images_quiet(self): self.assertEqual(type(res1[0]), six.text_type) -class PullImageTest(BaseIntegrationTest): +class PullImageTest(BaseAPIIntegrationTest): def test_pull(self): try: self.client.remove_image('hello-world') @@ -66,7 +66,7 @@ def test_pull_streaming(self): self.assertIn('Id', img_info) -class CommitTest(BaseIntegrationTest): +class CommitTest(BaseAPIIntegrationTest): def test_commit(self): container = self.client.create_container(BUSYBOX, ['touch', '/test']) id = container['Id'] @@ -101,7 +101,7 @@ def test_commit_with_changes(self): assert img['Config']['Cmd'] == ['bash'] -class RemoveImageTest(BaseIntegrationTest): +class RemoveImageTest(BaseAPIIntegrationTest): def test_remove(self): container = self.client.create_container(BUSYBOX, ['touch', '/test']) id = container['Id'] @@ -117,7 +117,7 @@ def test_remove(self): self.assertEqual(len(res), 0) -class ImportImageTest(BaseIntegrationTest): +class ImportImageTest(BaseAPIIntegrationTest): '''Base class for `docker import` test cases.''' TAR_SIZE = 512 * 1024 diff --git a/tests/integration/api_network_test.py b/tests/integration/api_network_test.py index 2ff5f029b7..f5d0ea95ef 100644 --- a/tests/integration/api_network_test.py +++ b/tests/integration/api_network_test.py @@ -6,10 +6,10 @@ import pytest from ..helpers import requires_api_version -from .base import BaseIntegrationTest +from .base import BaseAPIIntegrationTest -class TestNetworks(BaseIntegrationTest): +class TestNetworks(BaseAPIIntegrationTest): def create_network(self, *args, **kwargs): net_name = u'dockerpy{}'.format(random.getrandbits(24))[:14] net_id = self.client.create_network(net_name, *args, **kwargs)['Id'] diff --git a/tests/integration/api_service_test.py b/tests/integration/api_service_test.py index 636f507942..3581f99521 100644 --- a/tests/integration/api_service_test.py +++ b/tests/integration/api_service_test.py @@ -3,10 +3,10 @@ import docker from ..helpers import requires_api_version -from .base import BaseIntegrationTest +from .base import BaseAPIIntegrationTest -class ServiceTest(BaseIntegrationTest): +class ServiceTest(BaseAPIIntegrationTest): def setUp(self): super(ServiceTest, self).setUp() self.client.leave_swarm(force=True) diff --git a/tests/integration/api_swarm_test.py b/tests/integration/api_swarm_test.py index d623b83872..24c566f7e2 100644 --- a/tests/integration/api_swarm_test.py +++ b/tests/integration/api_swarm_test.py @@ -3,10 +3,10 @@ import pytest from ..helpers import requires_api_version -from .base import BaseIntegrationTest +from .base import BaseAPIIntegrationTest -class SwarmTest(BaseIntegrationTest): +class SwarmTest(BaseAPIIntegrationTest): def setUp(self): super(SwarmTest, self).setUp() self.client.leave_swarm(force=True) diff --git a/tests/integration/api_volume_test.py b/tests/integration/api_volume_test.py index 329b4e0d96..bc97f462e5 100644 --- a/tests/integration/api_volume_test.py +++ b/tests/integration/api_volume_test.py @@ -2,11 +2,11 @@ import pytest from ..helpers import requires_api_version -from .base import BaseIntegrationTest +from .base import BaseAPIIntegrationTest @requires_api_version('1.21') -class TestVolumes(BaseIntegrationTest): +class TestVolumes(BaseAPIIntegrationTest): def test_create_volume(self): name = 'perfectcherryblossom' self.tmp_volumes.append(name) diff --git a/tests/integration/base.py b/tests/integration/base.py index 2c7c05299b..ea43d056e5 100644 --- a/tests/integration/base.py +++ b/tests/integration/base.py @@ -11,20 +11,14 @@ class BaseIntegrationTest(unittest.TestCase): """ - A base class for integration test cases. - - It sets up a Docker client and cleans up the Docker server after itself. + A base class for integration test cases. It cleans up the Docker server + after itself. """ - tmp_imgs = [] - tmp_containers = [] - tmp_folders = [] - tmp_volumes = [] def setUp(self): if six.PY2: self.assertRegex = self.assertRegexpMatches self.assertCountEqual = self.assertItemsEqual - self.client = docker.APIClient(timeout=60, **kwargs_from_env()) self.tmp_imgs = [] self.tmp_containers = [] self.tmp_folders = [] @@ -32,32 +26,41 @@ def setUp(self): self.tmp_networks = [] def tearDown(self): + client = docker.from_env() for img in self.tmp_imgs: try: - self.client.remove_image(img) + client.api.remove_image(img) except docker.errors.APIError: pass for container in self.tmp_containers: try: - self.client.stop(container, timeout=1) - self.client.remove_container(container) + client.api.remove_container(container, force=True) except docker.errors.APIError: pass for network in self.tmp_networks: try: - self.client.remove_network(network) + client.api.remove_network(network) except docker.errors.APIError: pass - for folder in self.tmp_folders: - shutil.rmtree(folder) - for volume in self.tmp_volumes: try: - self.client.remove_volume(volume) + client.api.remove_volume(volume) except docker.errors.APIError: pass - self.client.close() + for folder in self.tmp_folders: + shutil.rmtree(folder) + + +class BaseAPIIntegrationTest(BaseIntegrationTest): + """ + A test case for `APIClient` integration tests. It sets up an `APIClient` + as `self.client`. + """ + + def setUp(self): + super(BaseAPIIntegrationTest, self).setUp() + self.client = docker.APIClient(timeout=60, **kwargs_from_env()) def run_container(self, *args, **kwargs): container = self.client.create_container(*args, **kwargs) diff --git a/tests/integration/errors_test.py b/tests/integration/errors_test.py index 4adfa32ffd..dc5cef4945 100644 --- a/tests/integration/errors_test.py +++ b/tests/integration/errors_test.py @@ -1,8 +1,8 @@ from docker.errors import APIError -from .base import BaseIntegrationTest, BUSYBOX +from .base import BaseAPIIntegrationTest, BUSYBOX -class ErrorsTest(BaseIntegrationTest): +class ErrorsTest(BaseAPIIntegrationTest): def test_api_error_parses_json(self): container = self.client.create_container(BUSYBOX, ['sleep', '10']) self.client.start(container['Id']) diff --git a/tests/integration/regression_test.py b/tests/integration/regression_test.py index 0672c4fad5..e3e6d9b7e3 100644 --- a/tests/integration/regression_test.py +++ b/tests/integration/regression_test.py @@ -4,10 +4,10 @@ import docker import six -from .base import BaseIntegrationTest, BUSYBOX +from .base import BaseAPIIntegrationTest, BUSYBOX -class TestRegressions(BaseIntegrationTest): +class TestRegressions(BaseAPIIntegrationTest): def test_443_handle_nonchunked_response_in_stream(self): dfile = io.BytesIO() with self.assertRaises(docker.errors.APIError) as exc: From d98a8790163d3e3cdc15a43c9f981d28af1003d2 Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Mon, 7 Nov 2016 17:55:32 -0800 Subject: [PATCH 0174/1301] Add random_name test helper Signed-off-by: Ben Firshman --- tests/helpers.py | 5 +++++ tests/integration/api_network_test.py | 10 +++------- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/helpers.py b/tests/helpers.py index f8b3e61383..09fb653222 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -1,5 +1,6 @@ import os import os.path +import random import tarfile import tempfile import time @@ -56,3 +57,7 @@ def wait_on_condition(condition, delay=0.1, timeout=40): if time.time() - start_time > timeout: raise AssertionError("Timeout: %s" % condition) time.sleep(delay) + + +def random_name(): + return u'dockerpytest_{0:x}'.format(random.getrandbits(64)) diff --git a/tests/integration/api_network_test.py b/tests/integration/api_network_test.py index f5d0ea95ef..092a12c137 100644 --- a/tests/integration/api_network_test.py +++ b/tests/integration/api_network_test.py @@ -1,17 +1,15 @@ -import random - import docker from docker.utils import create_ipam_config from docker.utils import create_ipam_pool import pytest -from ..helpers import requires_api_version +from ..helpers import random_name, requires_api_version from .base import BaseAPIIntegrationTest class TestNetworks(BaseAPIIntegrationTest): def create_network(self, *args, **kwargs): - net_name = u'dockerpy{}'.format(random.getrandbits(24))[:14] + net_name = random_name() net_id = self.client.create_network(net_name, *args, **kwargs)['Id'] self.tmp_networks.append(net_id) return (net_name, net_id) @@ -84,10 +82,8 @@ def test_create_network_with_ipam_config(self): @requires_api_version('1.21') def test_create_network_with_host_driver_fails(self): - net_name = 'dockerpy{}'.format(random.getrandbits(24))[:14] - with pytest.raises(docker.errors.APIError): - self.client.create_network(net_name, driver='host') + self.client.create_network(random_name(), driver='host') @requires_api_version('1.21') def test_remove_network(self): From ed959f2144ed4719113cc4a570cd58ac0cca25da Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Tue, 18 Oct 2016 18:50:21 +0200 Subject: [PATCH 0175/1301] Move contributing docs to CONTRIBUTING.md CONTRIBUTING.md is the place that GitHub expects it to be. Signed-off-by: Ben Firshman --- CONTRIBUTING.md | 34 ++++++++++++++++++++++++++++++++++ docs/contributing.md | 36 ------------------------------------ 2 files changed, 34 insertions(+), 36 deletions(-) delete mode 100644 docs/contributing.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1bd8d42699..dbc1c02a91 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,5 +1,8 @@ # Contributing guidelines +See the [Docker contributing guidelines](https://github.com/docker/docker/blob/master/CONTRIBUTING.md). +The following is specific to docker-py. + Thank you for your interest in the project. We look forward to your contribution. In order to make the process as fast and streamlined as possible, here is a set of guidelines we recommend you follow. @@ -100,3 +103,34 @@ here are the steps to get you started. 5. Run `python setup.py develop` to install the dev version of the project and required dependencies. We recommend you do so inside a [virtual environment](http://docs.python-guide.org/en/latest/dev/virtualenvs) + +## Running the tests & Code Quality + +To get the source source code and run the unit tests, run: +``` +$ git clone git://github.com/docker/docker-py.git +$ cd docker-py +$ pip install tox +$ tox +``` + +## Building the docs + +``` +$ make docs +$ open _build/index.html +``` + +## Release Checklist + +Before a new release, please go through the following checklist: + +* Bump version in docker/version.py +* Add a release note in docs/change_log.md +* Git tag the version +* Upload to pypi + +## Vulnerability Reporting +For any security issues, please do NOT file an issue or pull request on github! +Please contact [security@docker.com](mailto:security@docker.com) or read [the +Docker security page](https://www.docker.com/resources/security/). diff --git a/docs/contributing.md b/docs/contributing.md deleted file mode 100644 index e776458338..0000000000 --- a/docs/contributing.md +++ /dev/null @@ -1,36 +0,0 @@ -# Contributing -See the [Docker contributing guidelines](https://github.com/docker/docker/blob/master/CONTRIBUTING.md). -The following is specific to docker-py. - -## Running the tests & Code Quality - - -To get the source source code and run the unit tests, run: -``` -$ git clone git://github.com/docker/docker-py.git -$ cd docker-py -$ pip install tox -$ tox -``` - -## Building the docs -Docs are built with [MkDocs](http://www.mkdocs.org/). For development, you can -run the following in the project directory: -``` -$ pip install -r docs-requirements.txt -$ mkdocs serve -``` - -## Release Checklist - -Before a new release, please go through the following checklist: - -* Bump version in docker/version.py -* Add a release note in docs/change_log.md -* Git tag the version -* Upload to pypi - -## Vulnerability Reporting -For any security issues, please do NOT file an issue or pull request on github! -Please contact [security@docker.com](mailto:security@docker.com) or read [the -Docker security page](https://www.docker.com/resources/security/). From f32c0c170917518b7e224adf8627e95b5e620a91 Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Mon, 7 Nov 2016 18:06:31 -0800 Subject: [PATCH 0176/1301] Add docstrings to low-level API Signed-off-by: Ben Firshman --- docker/api/build.py | 83 +++++ docker/api/client.py | 28 +- docker/api/container.py | 794 ++++++++++++++++++++++++++++++++++++++++ docker/api/daemon.py | 84 +++++ docker/api/exec_api.py | 62 ++++ docker/api/image.py | 244 ++++++++++++ docker/api/network.py | 100 +++++ docker/api/service.py | 126 +++++++ docker/api/swarm.py | 204 +++++++++++ docker/api/volume.py | 85 +++++ docker/tls.py | 17 + docker/utils/utils.py | 51 +++ 12 files changed, 1876 insertions(+), 2 deletions(-) diff --git a/docker/api/build.py b/docker/api/build.py index 68aa9621d2..297c9e0dbe 100644 --- a/docker/api/build.py +++ b/docker/api/build.py @@ -19,6 +19,89 @@ def build(self, path=None, tag=None, quiet=False, fileobj=None, forcerm=False, dockerfile=None, container_limits=None, decode=False, buildargs=None, gzip=False, shmsize=None, labels=None): + """ + Similar to the ``docker build`` command. Either ``path`` or ``fileobj`` + needs to be set. ``path`` can be a local path (to a directory + containing a Dockerfile) or a remote URL. ``fileobj`` must be a + readable file-like object to a Dockerfile. + + If you have a tar file for the Docker build context (including a + Dockerfile) already, pass a readable file-like object to ``fileobj`` + and also pass ``custom_context=True``. If the stream is compressed + also, set ``encoding`` to the correct value (e.g ``gzip``). + + Example: + >>> from io import BytesIO + >>> from docker import Client + >>> dockerfile = ''' + ... # Shared Volume + ... FROM busybox:buildroot-2014.02 + ... VOLUME /data + ... CMD ["/bin/sh"] + ... ''' + >>> f = BytesIO(dockerfile.encode('utf-8')) + >>> cli = Client(base_url='tcp://127.0.0.1:2375') + >>> response = [line for line in cli.build( + ... fileobj=f, rm=True, tag='yourname/volume' + ... )] + >>> response + ['{"stream":" ---\\u003e a9eb17255234\\n"}', + '{"stream":"Step 1 : VOLUME /data\\n"}', + '{"stream":" ---\\u003e Running in abdc1e6896c6\\n"}', + '{"stream":" ---\\u003e 713bca62012e\\n"}', + '{"stream":"Removing intermediate container abdc1e6896c6\\n"}', + '{"stream":"Step 2 : CMD [\\"/bin/sh\\"]\\n"}', + '{"stream":" ---\\u003e Running in dba30f2a1a7e\\n"}', + '{"stream":" ---\\u003e 032b8b2855fc\\n"}', + '{"stream":"Removing intermediate container dba30f2a1a7e\\n"}', + '{"stream":"Successfully built 032b8b2855fc\\n"}'] + + Args: + path (str): Path to the directory containing the Dockerfile + fileobj: A file object to use as the Dockerfile. (Or a file-like + object) + tag (str): A tag to add to the final image + quiet (bool): Whether to return the status + nocache (bool): Don't use the cache when set to ``True`` + rm (bool): Remove intermediate containers. The ``docker build`` + command now defaults to ``--rm=true``, but we have kept the old + default of `False` to preserve backward compatibility + stream (bool): *Deprecated for API version > 1.8 (always True)*. + Return a blocking generator you can iterate over to retrieve + build output as it happens + timeout (int): HTTP timeout + custom_context (bool): Optional if using ``fileobj`` + encoding (str): The encoding for a stream. Set to ``gzip`` for + compressing + pull (bool): Downloads any updates to the FROM image in Dockerfiles + forcerm (bool): Always remove intermediate containers, even after + unsuccessful builds + dockerfile (str): path within the build context to the Dockerfile + buildargs (dict): A dictionary of build arguments + container_limits (dict): A dictionary of limits applied to each + container created by the build process. Valid keys: + + - memory (int): set memory limit for build + - memswap (int): Total memory (memory + swap), -1 to disable + swap + - cpushares (int): CPU shares (relative weight) + - cpusetcpus (str): CPUs in which to allow execution, e.g., + ``"0-3"``, ``"0,1"`` + decode (bool): If set to ``True``, the returned stream will be + decoded into dicts on the fly. Default ``False``. + shmsize (int): Size of `/dev/shm` in bytes. The size must be + greater than 0. If omitted the system uses 64MB. + labels (dict): A dictionary of labels to set on the image. + + Returns: + A generator for the build output. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + ``TypeError`` + If neither ``path`` nor ``fileobj`` is specified. + """ remote = context = None headers = {} container_limits = container_limits or {} diff --git a/docker/api/client.py b/docker/api/client.py index 2fc2ef09b6..5c26d63cb9 100644 --- a/docker/api/client.py +++ b/docker/api/client.py @@ -51,8 +51,32 @@ class APIClient( """ A low-level client for the Docker Remote API. - Each method maps one-to-one with a REST API endpoint, so calling each - method results in a single API call. + Example: + + >>> import docker + >>> client = docker.APIClient(base_url='unix://var/run/docker.sock') + >>> client.version() + {u'ApiVersion': u'1.24', + u'Arch': u'amd64', + u'BuildTime': u'2016-09-27T23:38:15.810178467+00:00', + u'Experimental': True, + u'GitCommit': u'45bed2c', + u'GoVersion': u'go1.6.3', + u'KernelVersion': u'4.4.22-moby', + u'Os': u'linux', + u'Version': u'1.12.2-rc1'} + + Args: + base_url (str): URL to the Docker server. For example, + ``unix:///var/run/docker.sock`` or ``tcp://127.0.0.1:1234``. + version (str): The version of the API to use. Set to ``auto`` to + automatically detect the server's version. Default: ``1.24`` + timeout (int): Default timeout for API calls, in seconds. + tls (bool or :py:class:`~docker.tls.TLSConfig`): Enable TLS. Pass + ``True`` to enable it with default options, or pass a + :py:class:`~docker.tls.TLSConfig` object to use custom + configuration. + user_agent (str): Set a custom user agent for requests to the server. """ def __init__(self, base_url=None, version=None, timeout=DEFAULT_TIMEOUT_SECONDS, tls=False, diff --git a/docker/api/container.py b/docker/api/container.py index 338b79fe77..e6d01e7293 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -11,6 +11,30 @@ class ContainerApiMixin(object): @utils.check_resource def attach(self, container, stdout=True, stderr=True, stream=False, logs=False): + """ + Attach to a container. + + The ``.logs()`` function is a wrapper around this method, which you can + use instead if you want to fetch/stream container output without first + retrieving the entire backlog. + + Args: + container (str): The container to attach to. + stdout (bool): Include stdout. + stderr (bool): Include stderr. + stream (bool): Return container output progressively as an iterator + of strings, rather than a single string. + logs (bool): Include the container's previous output. + + Returns: + By default, the container's output as a single string. + + If ``stream=True``, an iterator of output strings. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ params = { 'logs': logs and 1 or 0, 'stdout': stdout and 1 or 0, @@ -30,6 +54,20 @@ def attach(self, container, stdout=True, stderr=True, @utils.check_resource def attach_socket(self, container, params=None, ws=False): + """ + Like ``attach``, but returns the underlying socket-like object for the + HTTP request. + + Args: + container (str): The container to attach to. + params (dict): Dictionary of request parameters (e.g. ``stdout``, + ``stderr``, ``stream``). + ws (bool): Use websockets instead of raw HTTP. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ if params is None: params = { 'stdout': 1, @@ -56,6 +94,26 @@ def attach_socket(self, container, params=None, ws=False): @utils.check_resource def commit(self, container, repository=None, tag=None, message=None, author=None, changes=None, conf=None): + """ + Commit a container to an image. Similar to the ``docker commit`` + command. + + Args: + container (str): The image hash of the container + repository (str): The repository to push the image to + tag (str): The tag to push + message (str): A commit message + author (str): The name of the author + changes (str): Dockerfile instructions to apply while committing + conf (dict): The configuration for the container. See the + `Remote API documentation + `_ + for full details. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ params = { 'container': container, 'repo': repository, @@ -71,6 +129,50 @@ def commit(self, container, repository=None, tag=None, message=None, def containers(self, quiet=False, all=False, trunc=False, latest=False, since=None, before=None, limit=-1, size=False, filters=None): + """ + List containers. Similar to the ``docker ps`` command. + + Args: + quiet (bool): Only display numeric Ids + all (bool): Show all containers. Only running containers are shown + by default trunc (bool): Truncate output + latest (bool): Show only the latest created container, include + non-running ones. + since (str): Show only containers created since Id or Name, include + non-running ones + before (str): Show only container created before Id or Name, + include non-running ones + limit (int): Show `limit` last created containers, include + non-running ones + size (bool): Display sizes + filters (dict): Filters to be processed on the image list. + Available filters: + + - `exited` (int): Only containers with specified exit code + - `status` (str): One of ``restarting``, ``running``, + ``paused``, ``exited`` + - `label` (str): format either ``"key"`` or ``"key=value"`` + - `id` (str): The id of the container. + - `name` (str): The name of the container. + - `ancestor` (str): Filter by container ancestor. Format of + ``[:tag]``, ````, or + ````. + - `before` (str): Only containers created before a particular + container. Give the container name or id. + - `since` (str): Only containers created after a particular + container. Give container name or id. + + A comprehensive list can be found in the documentation for + `docker ps + `_. + + Returns: + A list of dicts, one per container + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ params = { 'limit': 1 if latest else limit, 'all': 1 if all else 0, @@ -93,6 +195,24 @@ def containers(self, quiet=False, all=False, trunc=False, latest=False, @utils.check_resource def copy(self, container, resource): + """ + Identical to the ``docker cp`` command. Get files/folders from the + container. + + **Deprecated for API version >= 1.20.** Use + :py:meth:`~ContainerApiMixin.get_archive` instead. + + Args: + container (str): The container to copy from + resource (str): The path within the container + + Returns: + The contents of the file as a string + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ if utils.version_gte(self._version, '1.20'): warnings.warn( 'Client.copy() is deprecated for API version >= 1.20, ' @@ -117,7 +237,190 @@ def create_container(self, image, command=None, hostname=None, user=None, mac_address=None, labels=None, volume_driver=None, stop_signal=None, networking_config=None, healthcheck=None): + """ + Creates a container. Parameters are similar to those for the ``docker + run`` command except it doesn't support the attach options (``-a``). + + The arguments that are passed directly to this function are + host-independent configuration options. Host-specific configuration + is passed with the `host_config` argument. You'll normally want to + use this method in combination with the :py:meth:`create_host_config` + method to generate ``host_config``. + + **Port bindings** + + Port binding is done in two parts: first, provide a list of ports to + open inside the container with the ``ports`` parameter, then declare + bindings with the ``host_config`` parameter. For example: + + .. code-block:: python + + container_id = cli.create_container( + 'busybox', 'ls', ports=[1111, 2222], + host_config=cli.create_host_config(port_bindings={ + 1111: 4567, + 2222: None + }) + ) + + + You can limit the host address on which the port will be exposed like + such: + + .. code-block:: python + + cli.create_host_config(port_bindings={1111: ('127.0.0.1', 4567)}) + + Or without host port assignment: + + .. code-block:: python + + cli.create_host_config(port_bindings={1111: ('127.0.0.1',)}) + + If you wish to use UDP instead of TCP (default), you need to declare + ports as such in both the config and host config: + + .. code-block:: python + + container_id = cli.create_container( + 'busybox', 'ls', ports=[(1111, 'udp'), 2222], + host_config=cli.create_host_config(port_bindings={ + '1111/udp': 4567, 2222: None + }) + ) + + To bind multiple host ports to a single container port, use the + following syntax: + + .. code-block:: python + cli.create_host_config(port_bindings={ + 1111: [1234, 4567] + }) + + You can also bind multiple IPs to a single container port: + + .. code-block:: python + + cli.create_host_config(port_bindings={ + 1111: [ + ('192.168.0.100', 1234), + ('192.168.0.101', 1234) + ] + }) + + **Using volumes** + + Volume declaration is done in two parts. Provide a list of mountpoints + to the with the ``volumes`` parameter, and declare mappings in the + ``host_config`` section. + + .. code-block:: python + + container_id = cli.create_container( + 'busybox', 'ls', volumes=['/mnt/vol1', '/mnt/vol2'], + host_config=cli.create_host_config(binds={ + '/home/user1/': { + 'bind': '/mnt/vol2', + 'mode': 'rw', + }, + '/var/www': { + 'bind': '/mnt/vol1', + 'mode': 'ro', + } + }) + ) + + You can alternatively specify binds as a list. This code is equivalent + to the example above: + + .. code-block:: python + + container_id = cli.create_container( + 'busybox', 'ls', volumes=['/mnt/vol1', '/mnt/vol2'], + host_config=cli.create_host_config(binds=[ + '/home/user1/:/mnt/vol2', + '/var/www:/mnt/vol1:ro', + ]) + ) + + **Networking** + + You can specify networks to connect the container to by using the + ``networking_config`` parameter. At the time of creation, you can + only connect a container to a single networking, but you + can create more connections by using + :py:meth:`~connect_container_to_network`. + + For example: + + .. code-block:: python + + networking_config = docker_client.create_networking_config({ + 'network1': docker_client.create_endpoint_config( + ipv4_address='172.28.0.124', + aliases=['foo', 'bar'], + links=['container2'] + ) + }) + + ctnr = docker_client.create_container( + img, command, networking_config=networking_config + ) + + Args: + image (str): The image to run + command (str or list): The command to be run in the container + hostname (str): Optional hostname for the container + user (str or int): Username or UID + detach (bool): Detached mode: run container in the background and + return container ID + stdin_open (bool): Keep STDIN open even if not attached + tty (bool): Allocate a pseudo-TTY + mem_limit (float or str): Memory limit. Accepts float values (which + represent the memory limit of the created container in bytes) + or a string with a units identification char (``100000b``, + ``1000k``, ``128m``, ``1g``). If a string is specified without + a units character, bytes are assumed as an intended unit. + ports (list of ints): A list of port numbers + environment (dict or list): A dictionary or a list of strings in + the following format ``["PASSWORD=xxx"]`` or + ``{"PASSWORD": "xxx"}``. + dns (list): DNS name servers. Deprecated since API version 1.10. + Use ``host_config`` instead. + dns_opt (list): Additional options to be added to the container's + ``resolv.conf`` file + volumes (str or list): + volumes_from (list): List of container names or Ids to get + volumes from. + network_disabled (bool): Disable networking + name (str): A name for the container + entrypoint (str or list): An entrypoint + working_dir (str): Path to the working directory + domainname (str or list): Set custom DNS search domains + memswap_limit (int): + host_config (dict): A dictionary created with + :py:meth:`create_host_config`. + mac_address (str): The Mac Address to assign the container + labels (dict or list): A dictionary of name-value labels (e.g. + ``{"label1": "value1", "label2": "value2"}``) or a list of + names of labels to set with empty values (e.g. + ``["label1", "label2"]``) + volume_driver (str): The name of a volume driver/plugin. + stop_signal (str): The stop signal to use to stop the container + (e.g. ``SIGINT``). + networking_config (dict): A networking configuration generated + by :py:meth:`create_networking_config`. + + Returns: + A dictionary with an image 'Id' key and a 'Warnings' key. + + Raises: + :py:class:`docker.errors.ImageNotFound` + If the specified image does not exist. + :py:class:`docker.errors.APIError` + If the server returns an error. + """ if isinstance(volumes, six.string_types): volumes = [volumes, ] @@ -147,6 +450,130 @@ def create_container_from_config(self, config, name=None): return self._result(res, True) def create_host_config(self, *args, **kwargs): + """ + Create a dictionary for the ``host_config`` argument to + :py:meth:`create_container`. + + Args: + binds (dict): Volumes to bind. See :py:meth:`create_container` + for more information. + blkio_weight_device: Block IO weight (relative device weight) in + the form of: ``[{"Path": "device_path", "Weight": weight}]``. + blkio_weight: Block IO weight (relative weight), accepts a weight + value between 10 and 1000. + cap_add (list of str): Add kernel capabilities. For example, + ``["SYS_ADMIN", "MKNOD"]``. + cap_drop (list of str): Drop kernel capabilities. + cpu_group (int): The length of a CPU period in microseconds. + cpu_period (int): Microseconds of CPU time that the container can + get in a CPU period. + cpu_shares (int): CPU shares (relative weight). + cpuset_cpus (str): CPUs in which to allow execution (``0-3``, + ``0,1``). + device_read_bps: Limit read rate (bytes per second) from a device + in the form of: `[{"Path": "device_path", "Rate": rate}]` + device_read_iops: Limit read rate (IO per second) from a device. + device_write_bps: Limit write rate (bytes per second) from a + device. + device_write_iops: Limit write rate (IO per second) from a device. + devices (list): Expose host devices to the container, as a list + of strings in the form + ``::``. + + For example, ``/dev/sda:/dev/xvda:rwm`` allows the container + to have read-write access to the host's ``/dev/sda`` via a + node named ``/dev/xvda`` inside the container. + dns (list): Set custom DNS servers. + dns_search (list): DNS search domains. + extra_hosts (dict): Addtional hostnames to resolve inside the + container, as a mapping of hostname to IP address. + group_add (list): List of additional group names and/or IDs that + the container process will run as. + ipc_mode (str): Set the IPC mode for the container. + isolation (str): Isolation technology to use. Default: `None`. + links (dict or list of tuples): Either a dictionary mapping name + to alias or as a list of ``(name, alias)`` tuples. + log_config (dict): Logging configuration, as a dictionary with + keys: + + - ``type`` The logging driver name. + - ``config`` A dictionary of configuration for the logging + driver. + + lxc_conf (dict): LXC config. + mem_limit (float or str): Memory limit. Accepts float values + (which represent the memory limit of the created container in + bytes) or a string with a units identification char + (``100000b``, ``1000k``, ``128m``, ``1g``). If a string is + specified without a units character, bytes are assumed as an + mem_swappiness (int): Tune a container's memory swappiness + behavior. Accepts number between 0 and 100. + memswap_limit (str or int): Maximum amount of memory + swap a + container is allowed to consume. + network_mode (str): One of: + + - ``bridge`` Create a new network stack for the container on + on the bridge network. + - ``none`` No networking for this container. + - ``container:`` Reuse another container's network + stack. + - ``host`` Use the host network stack. + oom_kill_disable (bool): Whether to disable OOM killer. + oom_score_adj (int): An integer value containing the score given + to the container in order to tune OOM killer preferences. + pid_mode (str): If set to ``host``, use the host PID namespace + inside the container. + pids_limit (int): Tune a container's pids limit. Set ``-1`` for + unlimited. + port_bindings (dict): See :py:meth:`create_container` + for more information. + privileged (bool): Give extended privileges to this container. + publish_all_ports (bool): Publish all ports to the host. + read_only (bool): Mount the container's root filesystem as read + only. + restart_policy (dict): Restart the container when it exits. + Configured as a dictionary with keys: + + - ``Name`` One of ``on-failure``, or ``always``. + - ``MaximumRetryCount`` Number of times to restart the + container on failure. + security_opt (list): A list of string values to customize labels + for MLS systems, such as SELinux. + shm_size (str or int): Size of /dev/shm (e.g. ``1G``). + sysctls (dict): Kernel parameters to set in the container. + tmpfs (dict): Temporary filesystems to mount, as a dictionary + mapping a path inside the container to options for that path. + + For example: + + .. code-block:: python + + { + '/mnt/vol2': '', + '/mnt/vol1': 'size=3G,uid=1000' + } + + ulimits (list): Ulimits to set inside the container, as a list of + dicts. + userns_mode (str): Sets the user namespace mode for the container + when user namespace remapping option is enabled. Supported + values are: ``host`` + volumes_from (list): List of container names or IDs to get + volumes from. + + + Returns: + (dict) A dictionary which can be passed to the ``host_config`` + argument to :py:meth:`create_container`. + + Example: + + >>> cli.create_host_config(privileged=True, cap_drop=['MKNOD'], + volumes_from=['nostalgic_newton']) + {'CapDrop': ['MKNOD'], 'LxcConf': None, 'Privileged': True, + 'VolumesFrom': ['nostalgic_newton'], 'PublishAllPorts': False} + +""" if not kwargs: kwargs = {} if 'version' in kwargs: @@ -158,19 +585,98 @@ def create_host_config(self, *args, **kwargs): return utils.create_host_config(*args, **kwargs) def create_networking_config(self, *args, **kwargs): + """ + Create a networking config dictionary to be used as the + ``networking_config`` parameter in :py:meth:`create_container`. + + Args: + endpoints_config (dict): A dictionary mapping network names to + endpoint configurations generated by + :py:meth:`create_endpoint_config`. + + Returns: + (dict) A networking config. + + Example: + + >>> docker_client.create_network('network1') + >>> networking_config = docker_client.create_networking_config({ + 'network1': docker_client.create_endpoint_config() + }) + >>> container = docker_client.create_container( + img, command, networking_config=networking_config + ) + + """ return create_networking_config(*args, **kwargs) def create_endpoint_config(self, *args, **kwargs): + """ + Create an endpoint config dictionary to be used with + :py:meth:`create_networking_config`. + + Args: + aliases (list): A list of aliases for this endpoint. Names in + that list can be used within the network to reach the + container. Defaults to ``None``. + links (list): A list of links for this endpoint. Containers + declared in this list will be linked to this container. + Defaults to ``None``. + ipv4_address (str): The IP address of this container on the + network, using the IPv4 protocol. Defaults to ``None``. + ipv6_address (str): The IP address of this container on the + network, using the IPv6 protocol. Defaults to ``None``. + link_local_ips (list): A list of link-local (IPv4/IPv6) + addresses. + + Returns: + (dict) An endpoint config. + + Example: + + >>> endpoint_config = client.create_endpoint_config( + aliases=['web', 'app'], + links=['app_db'], + ipv4_address='132.65.0.123' + ) + + """ return create_endpoint_config(self._version, *args, **kwargs) @utils.check_resource def diff(self, container): + """ + Inspect changes on a container's filesystem. + + Args: + container (str): The container to diff + + Returns: + (str) + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ return self._result( self._get(self._url("/containers/{0}/changes", container)), True ) @utils.check_resource def export(self, container): + """ + Export the contents of a filesystem as a tar archive. + + Args: + container (str): The container to export + + Returns: + (str): The filesystem tar archive + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ res = self._get( self._url("/containers/{0}/export", container), stream=True ) @@ -180,6 +686,22 @@ def export(self, container): @utils.check_resource @utils.minimum_version('1.20') def get_archive(self, container, path): + """ + Retrieve a file or folder from a container in the form of a tar + archive. + + Args: + container (str): The container where the file is located + path (str): Path to the file or folder to retrieve + + Returns: + (tuple): First element is a raw tar data stream. Second element is + a dict containing ``stat`` information on the specified ``path``. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ params = { 'path': path } @@ -194,12 +716,37 @@ def get_archive(self, container, path): @utils.check_resource def inspect_container(self, container): + """ + Identical to the `docker inspect` command, but only for containers. + + Args: + container (str): The container to inspect + + Returns: + (dict): Similar to the output of `docker inspect`, but as a + single dict + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ return self._result( self._get(self._url("/containers/{0}/json", container)), True ) @utils.check_resource def kill(self, container, signal=None): + """ + Kill a container or send a signal to a container. + + Args: + container (str): The container to kill + signal (str or int): The signal to send. Defaults to ``SIGKILL`` + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ url = self._url("/containers/{0}/kill", container) params = {} if signal is not None: @@ -213,6 +760,32 @@ def kill(self, container, signal=None): @utils.check_resource def logs(self, container, stdout=True, stderr=True, stream=False, timestamps=False, tail='all', since=None, follow=None): + """ + Get logs from a container. Similar to the ``docker logs`` command. + + The ``stream`` parameter makes the ``logs`` function return a blocking + generator you can iterate over to retrieve log output as it happens. + + Args: + container (str): The container to get logs from + stdout (bool): Get ``STDOUT`` + stderr (bool): Get ``STDERR`` + stream (bool): Stream the response + timestamps (bool): Show timestamps + tail (str or int): Output specified number of lines at the end of + logs. Either an integer of number of lines or the string + ``all``. Default ``all`` + since (datetime or int): Show logs since a given datetime or + integer epoch (in seconds) + follow (bool): Follow log output + + Returns: + (generator or str) + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ if utils.compare_version('1.11', self._version) >= 0: if follow is None: follow = stream @@ -249,12 +822,48 @@ def logs(self, container, stdout=True, stderr=True, stream=False, @utils.check_resource def pause(self, container): + """ + Pauses all processes within a container. + + Args: + container (str): The container to pause + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ url = self._url('/containers/{0}/pause', container) res = self._post(url) self._raise_for_status(res) @utils.check_resource def port(self, container, private_port): + """ + Lookup the public-facing port that is NAT-ed to ``private_port``. + Identical to the ``docker port`` command. + + Args: + container (str): The container to look up + private_port (int): The private port to inspect + + Returns: + (list of dict): The mapping for the host ports + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + + Example: + .. code-block:: bash + + $ docker run -d -p 80:80 ubuntu:14.04 /bin/sleep 30 + 7174d6347063a83f412fad6124c99cffd25ffe1a0807eb4b7f9cec76ac8cb43b + + .. code-block:: python + + >>> cli.port('7174d6347063', 80) + [{'HostIp': '0.0.0.0', 'HostPort': '80'}] + """ res = self._get(self._url("/containers/{0}/json", container)) self._raise_for_status(res) json_ = res.json() @@ -279,6 +888,26 @@ def port(self, container, private_port): @utils.check_resource @utils.minimum_version('1.20') def put_archive(self, container, path, data): + """ + Insert a file or folder in an existing container using a tar archive as + source. + + Args: + container (str): The container where the file(s) will be extracted + path (str): Path inside the container where the file(s) will be + extracted. Must exist. + data (bytes): tar data to be extracted + + Returns: + (bool): True if the call succeeds. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + + Raises: + :py:class:`~docker.errors.APIError` If an error occurs. + """ params = {'path': path} url = self._url('/containers/{0}/archive', container) res = self._put(url, params=params, data=data) @@ -287,6 +916,21 @@ def put_archive(self, container, path, data): @utils.check_resource def remove_container(self, container, v=False, link=False, force=False): + """ + Remove a container. Similar to the ``docker rm`` command. + + Args: + container (str): The container to remove + v (bool): Remove the volumes associated with the container + link (bool): Remove the specified link and not the underlying + container + force (bool): Force the removal of a running container (uses + ``SIGKILL``) + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ params = {'v': v, 'link': link, 'force': force} res = self._delete( self._url("/containers/{0}", container), params=params @@ -296,6 +940,17 @@ def remove_container(self, container, v=False, link=False, force=False): @utils.minimum_version('1.17') @utils.check_resource def rename(self, container, name): + """ + Rename a container. Similar to the ``docker rename`` command. + + Args: + container (str): ID of the container to rename + name (str): New name for the container + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ url = self._url("/containers/{0}/rename", container) params = {'name': name} res = self._post(url, params=params) @@ -303,6 +958,18 @@ def rename(self, container, name): @utils.check_resource def resize(self, container, height, width): + """ + Resize the tty session. + + Args: + container (str or dict): The container to resize + height (int): Height of tty session + width (int): Width of tty session + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ params = {'h': height, 'w': width} url = self._url("/containers/{0}/resize", container) res = self._post(url, params=params) @@ -310,6 +977,20 @@ def resize(self, container, height, width): @utils.check_resource def restart(self, container, timeout=10): + """ + Restart a container. Similar to the ``docker restart`` command. + + Args: + container (str or dict): The container to restart. If a dict, the + ``Id`` key is used. + timeout (int): Number of seconds to try to stop for before killing + the container. Once killed it will then be restarted. Default + is 10 seconds. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ params = {'t': timeout} url = self._url("/containers/{0}/restart", container) res = self._post(url, params=params) @@ -322,7 +1003,28 @@ def start(self, container, binds=None, port_bindings=None, lxc_conf=None, restart_policy=None, cap_add=None, cap_drop=None, devices=None, extra_hosts=None, read_only=None, pid_mode=None, ipc_mode=None, security_opt=None, ulimits=None): + """ + Start a container. Similar to the ``docker start`` command, but + doesn't support attach options. + + **Deprecation warning:** For API version > 1.15, it is highly + recommended to provide host config options in the ``host_config`` + parameter of :py:meth:`~ContainerApiMixin.create_container`. + Args: + container (str): The container to start + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + + Example: + + >>> container = cli.create_container( + ... image='busybox:latest', + ... command='/bin/sleep 30') + >>> cli.start(container=container.get('Id')) + """ if utils.compare_version('1.10', self._version) < 0: if dns is not None: raise errors.InvalidVersion( @@ -386,6 +1088,22 @@ def start(self, container, binds=None, port_bindings=None, lxc_conf=None, @utils.minimum_version('1.17') @utils.check_resource def stats(self, container, decode=None, stream=True): + """ + Stream statistics for a specific container. Similar to the + ``docker stats`` command. + + Args: + container (str): The container to stream statistics from + decode (bool): If set to true, stream will be decoded into dicts + on the fly. False by default. + stream (bool): If set to false, only the current stats will be + returned instead of a stream. True by default. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + + """ url = self._url("/containers/{0}/stats", container) if stream: return self._stream_helper(self._get(url, stream=True), @@ -396,6 +1114,18 @@ def stats(self, container, decode=None, stream=True): @utils.check_resource def stop(self, container, timeout=10): + """ + Stops a container. Similar to the ``docker stop`` command. + + Args: + container (str): The container to stop + timeout (int): Timeout in seconds to wait for the container to + stop before sending a ``SIGKILL``. Default: 10 + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ params = {'t': timeout} url = self._url("/containers/{0}/stop", container) @@ -405,6 +1135,20 @@ def stop(self, container, timeout=10): @utils.check_resource def top(self, container, ps_args=None): + """ + Display the running processes of a container. + + Args: + container (str): The container to inspect + ps_args (str): An optional arguments passed to ps (e.g. ``aux``) + + Returns: + (str): The output of the top + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ u = self._url("/containers/{0}/top", container) params = {} if ps_args is not None: @@ -413,6 +1157,12 @@ def top(self, container, ps_args=None): @utils.check_resource def unpause(self, container): + """ + Unpause all processes within a container. + + Args: + container (str): The container to unpause + """ url = self._url('/containers/{0}/unpause', container) res = self._post(url) self._raise_for_status(res) @@ -425,6 +1175,31 @@ def update_container( mem_reservation=None, memswap_limit=None, kernel_memory=None, restart_policy=None ): + """ + Update resource configs of one or more containers. + + Args: + container (str): The container to inspect + blkio_weight (int): Block IO (relative weight), between 10 and 1000 + cpu_period (int): Limit CPU CFS (Completely Fair Scheduler) period + cpu_quota (int): Limit CPU CFS (Completely Fair Scheduler) quota + cpu_shares (int): CPU shares (relative weight) + cpuset_cpus (str): CPUs in which to allow execution + cpuset_mems (str): MEMs in which to allow execution + mem_limit (int or str): Memory limit + mem_reservation (int or str): Memory soft limit + memswap_limit (int or str): Total memory (memory + swap), -1 to + disable swap + kernel_memory (int or str): Kernel memory limit + restart_policy (dict): Restart policy dictionary + + Returns: + (dict): Dictionary containing a ``Warnings`` key. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ url = self._url('/containers/{0}/update', container) data = {} if blkio_weight: @@ -460,6 +1235,25 @@ def update_container( @utils.check_resource def wait(self, container, timeout=None): + """ + Block until a container stops, then return its exit code. Similar to + the ``docker wait`` command. + + Args: + container (str or dict): The container to wait on. If a dict, the + ``Id`` key is used. + timeout (int): Request timeout + + Returns: + (int): The exit code of the container. Returns ``-1`` if the API + responds without a ``StatusCode`` attribute. + + Raises: + :py:class:`requests.exceptions.ReadTimeout` + If the timeout is exceeded. + :py:class:`docker.errors.APIError` + If the server returns an error. + """ url = self._url("/containers/{0}/wait", container) res = self._post(url, timeout=timeout) self._raise_for_status(res) diff --git a/docker/api/daemon.py b/docker/api/daemon.py index 2e87cf041f..d40631f59c 100644 --- a/docker/api/daemon.py +++ b/docker/api/daemon.py @@ -8,6 +8,36 @@ class DaemonApiMixin(object): def events(self, since=None, until=None, filters=None, decode=None): + """ + Get real-time events from the server. Similar to the ``docker events`` + command. + + Args: + since (UTC datetime or int): Get events from this point + until (UTC datetime or int): Get events until this point + filters (dict): Filter the events by event time, container or image + decode (bool): If set to true, stream will be decoded into dicts on + the fly. False by default. + + Returns: + (generator): A blocking generator you can iterate over to retrieve + events as they happen. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + + Example: + + >>> for event in client.events() + ... print event + {u'from': u'image/with:tag', + u'id': u'container-id', + u'status': u'start', + u'time': 1423339459} + ... + """ + if isinstance(since, datetime): since = utils.datetime_to_timestamp(since) @@ -29,10 +59,42 @@ def events(self, since=None, until=None, filters=None, decode=None): ) def info(self): + """ + Display system-wide information. Identical to the ``docker info`` + command. + + Returns: + (dict): The info as a dict + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ return self._result(self._get(self._url("/info")), True) def login(self, username, password=None, email=None, registry=None, reauth=False, insecure_registry=False, dockercfg_path=None): + """ + Authenticate with a registry. Similar to the ``docker login`` command. + + Args: + username (str): The registry username + password (str): The plaintext password + email (str): The email for the registry account + registry (str): URL to the registry. E.g. + ``https://index.docker.io/v1/`` + reauth (bool): Whether refresh existing authentication on the + Docker server. + dockercfg_path (str): Use a custom path for the ``.dockercfg`` file + (default ``$HOME/.dockercfg``) + + Returns: + (dict): The response from the login request + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ if insecure_registry: warnings.warn( INSECURE_REGISTRY_DEPRECATION_WARNING.format('login()'), @@ -68,8 +130,30 @@ def login(self, username, password=None, email=None, registry=None, return self._result(response, json=True) def ping(self): + """ + Checks the server is responsive. An exception will be raised if it + isn't responding. + + Returns: + (bool) The response from the server. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ return self._result(self._get(self._url('/_ping'))) == 'OK' def version(self, api_version=True): + """ + Returns version information from the server. Similar to the ``docker + version`` command. + + Returns: + (dict): The server version information + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ url = self._url("/version", versioned_api=api_version) return self._result(self._get(url), json=True) diff --git a/docker/api/exec_api.py b/docker/api/exec_api.py index 6e49996046..694b30a674 100644 --- a/docker/api/exec_api.py +++ b/docker/api/exec_api.py @@ -9,6 +9,28 @@ class ExecApiMixin(object): @utils.check_resource def exec_create(self, container, cmd, stdout=True, stderr=True, stdin=False, tty=False, privileged=False, user=''): + """ + Sets up an exec instance in a running container. + + Args: + container (str): Target container where exec instance will be + created + cmd (str or list): Command to be executed + stdout (bool): Attach to stdout. Default: ``True`` + stderr (bool): Attach to stderr. Default: ``True`` + stdin (bool): Attach to stdin. Default: ``False`` + tty (bool): Allocate a pseudo-TTY. Default: False + privileged (bool): Run as privileged. + user (str): User to execute command as. Default: root + + Returns: + (dict): A dictionary with an exec ``Id`` key. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + if privileged and utils.compare_version('1.19', self._version) < 0: raise errors.InvalidVersion( 'Privileged exec is not supported in API < 1.19' @@ -37,6 +59,19 @@ def exec_create(self, container, cmd, stdout=True, stderr=True, @utils.minimum_version('1.16') def exec_inspect(self, exec_id): + """ + Return low-level information about an exec command. + + Args: + exec_id (str): ID of the exec instance + + Returns: + (dict): Dictionary of values returned by the endpoint. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ if isinstance(exec_id, dict): exec_id = exec_id.get('Id') res = self._get(self._url("/exec/{0}/json", exec_id)) @@ -44,6 +79,15 @@ def exec_inspect(self, exec_id): @utils.minimum_version('1.15') def exec_resize(self, exec_id, height=None, width=None): + """ + Resize the tty session used by the specified exec command. + + Args: + exec_id (str): ID of the exec instance + height (int): Height of tty session + width (int): Width of tty session + """ + if isinstance(exec_id, dict): exec_id = exec_id.get('Id') @@ -55,6 +99,24 @@ def exec_resize(self, exec_id, height=None, width=None): @utils.minimum_version('1.15') def exec_start(self, exec_id, detach=False, tty=False, stream=False, socket=False): + """ + Start a previously set up exec instance. + + Args: + exec_id (str): ID of the exec instance + detach (bool): If true, detach from the exec command. + Default: False + tty (bool): Allocate a pseudo-TTY. Default: False + stream (bool): Stream response data. Default: False + + Returns: + (generator or str): If ``stream=True``, a generator yielding + response chunks. A string containing response data otherwise. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ # we want opened socket if socket == True if isinstance(exec_id, dict): exec_id = exec_id.get('Id') diff --git a/docker/api/image.py b/docker/api/image.py index 978a0c14bb..2c8cbb2315 100644 --- a/docker/api/image.py +++ b/docker/api/image.py @@ -14,17 +14,71 @@ class ImageApiMixin(object): @utils.check_resource def get_image(self, image): + """ + Get a tarball of an image. Similar to the ``docker save`` command. + + Args: + image (str): Image name to get + + Returns: + (urllib3.response.HTTPResponse object): The response from the + daemon. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + + Example: + + >>> image = cli.get_image("fedora:latest") + >>> f = open('/tmp/fedora-latest.tar', 'w') + >>> f.write(image.data) + >>> f.close() + """ res = self._get(self._url("/images/{0}/get", image), stream=True) self._raise_for_status(res) return res.raw @utils.check_resource def history(self, image): + """ + Show the history of an image. + + Args: + image (str): The image to show history for + + Returns: + (str): The history of the image + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ res = self._get(self._url("/images/{0}/history", image)) return self._result(res, True) def images(self, name=None, quiet=False, all=False, viz=False, filters=None): + """ + List images. Similar to the ``docker images`` command. + + Args: + name (str): Only show images belonging to the repository ``name`` + quiet (bool): Only return numeric IDs as a list. + all (bool): Show intermediate image layers. By default, these are + filtered out. + filters (dict): Filters to be processed on the image list. + Available filters: + - ``dangling`` (bool) + - ``label`` (str): format either ``key`` or ``key=value`` + + Returns: + (dict or list): A list if ``quiet=True``, otherwise a dict. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ if viz: if utils.compare_version('1.7', self._version) >= 0: raise Exception('Viz output is not supported in API >= 1.7!') @@ -44,6 +98,25 @@ def images(self, name=None, quiet=False, all=False, viz=False, def import_image(self, src=None, repository=None, tag=None, image=None, changes=None, stream_src=False): + """ + Import an image. Similar to the ``docker import`` command. + + If ``src`` is a string or unicode string, it will first be treated as a + path to a tarball on the local system. If there is an error reading + from that file, ``src`` will be treated as a URL instead to fetch the + image from. You can also pass an open file handle as ``src``, in which + case the data will be read from that file. + + If ``src`` is unset but ``image`` is set, the ``image`` parameter will + be taken as the name of an existing image to import from. + + Args: + src (str or file): Path to tarfile, URL, or file-like object + repository (str): The repository to create + tag (str): The tag to apply + image (str): Use another image like the ``FROM`` Dockerfile + parameter + """ if not (src or image): raise errors.DockerException( 'Must specify src or image to import from' @@ -77,6 +150,16 @@ def import_image(self, src=None, repository=None, tag=None, image=None, def import_image_from_data(self, data, repository=None, tag=None, changes=None): + """ + Like :py:meth:`~docker.api.image.ImageApiMixin.import_image`, but + allows importing in-memory bytes data. + + Args: + data (bytes collection): Bytes collection containing valid tar data + repository (str): The repository to create + tag (str): The tag to apply + """ + u = self._url('/images/create') params = _import_image_params( repository, tag, src='-', changes=changes @@ -90,6 +173,19 @@ def import_image_from_data(self, data, repository=None, tag=None, def import_image_from_file(self, filename, repository=None, tag=None, changes=None): + """ + Like :py:meth:`~docker.api.image.ImageApiMixin.import_image`, but only + supports importing from a tar file on disk. + + Args: + filename (str): Full path to a tar file. + repository (str): The repository to create + tag (str): The tag to apply + + Raises: + IOError: File does not exist. + """ + return self.import_image( src=filename, repository=repository, tag=tag, changes=changes ) @@ -103,12 +199,31 @@ def import_image_from_stream(self, stream, repository=None, tag=None, def import_image_from_url(self, url, repository=None, tag=None, changes=None): + """ + Like :py:meth:`~docker.api.image.ImageApiMixin.import_image`, but only + supports importing from a URL. + + Args: + url (str): A URL pointing to a tar file. + repository (str): The repository to create + tag (str): The tag to apply + """ return self.import_image( src=url, repository=repository, tag=tag, changes=changes ) def import_image_from_image(self, image, repository=None, tag=None, changes=None): + """ + Like :py:meth:`~docker.api.image.ImageApiMixin.import_image`, but only + supports importing from another image, like the ``FROM`` Dockerfile + parameter. + + Args: + image (str): Image name to import from + repository (str): The repository to create + tag (str): The tag to apply + """ return self.import_image( image=image, repository=repository, tag=tag, changes=changes ) @@ -128,16 +243,75 @@ def insert(self, image, url, path): @utils.check_resource def inspect_image(self, image): + """ + Get detailed information about an image. Similar to the ``docker + inspect`` command, but only for containers. + + Args: + container (str): The container to inspect + + Returns: + (dict): Similar to the output of ``docker inspect``, but as a + single dict + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ return self._result( self._get(self._url("/images/{0}/json", image)), True ) def load_image(self, data): + """ + Load an image that was previously saved using + :py:meth:`~docker.api.image.ImageApiMixin.get_image` (or ``docker + save``). Similar to ``docker load``. + + Args: + data (binary): Image data to be loaded. + """ res = self._post(self._url("/images/load"), data=data) self._raise_for_status(res) def pull(self, repository, tag=None, stream=False, insecure_registry=False, auth_config=None, decode=False): + """ + Pulls an image. Similar to the ``docker pull`` command. + + Args: + repository (str): The repository to pull + tag (str): The tag to pull + stream (bool): Stream the output as a generator + insecure_registry (bool): Use an insecure registry + auth_config (dict): Override the credentials that + :py:meth:`~docker.api.daemon.DaemonApiMixin.login` has set for + this request. ``auth_config`` should contain the ``username`` + and ``password`` keys to be valid. + + Returns: + (generator or str): The output + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + + Example: + + >>> for line in cli.pull('busybox', stream=True): + ... print(json.dumps(json.loads(line), indent=4)) + { + "status": "Pulling image (latest) from busybox", + "progressDetail": {}, + "id": "e72ac664f4f0" + } + { + "status": "Pulling image (latest) from busybox, endpoint: ...", + "progressDetail": {}, + "id": "e72ac664f4f0" + } + + """ if insecure_registry: warnings.warn( INSECURE_REGISTRY_DEPRECATION_WARNING.format('pull()'), @@ -177,6 +351,38 @@ def pull(self, repository, tag=None, stream=False, def push(self, repository, tag=None, stream=False, insecure_registry=False, auth_config=None, decode=False): + """ + Push an image or a repository to the registry. Similar to the ``docker + push`` command. + + Args: + repository (str): The repository to push to + tag (str): An optional tag to push + stream (bool): Stream the output as a blocking generator + insecure_registry (bool): Use ``http://`` to connect to the + registry + auth_config (dict): Override the credentials that + :py:meth:`~docker.api.daemon.DaemonApiMixin.login` has set for + this request. ``auth_config`` should contain the ``username`` + and ``password`` keys to be valid. + + Returns: + (generator or str): The output from the server. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + + Example: + >>> for line in cli.push('yourname/app', stream=True): + ... print line + {"status":"Pushing repository yourname/app (1 tags)"} + {"status":"Pushing","progressDetail":{},"id":"511136ea3c5a"} + {"status":"Image already pushed, skipping","progressDetail":{}, + "id":"511136ea3c5a"} + ... + + """ if insecure_registry: warnings.warn( INSECURE_REGISTRY_DEPRECATION_WARNING.format('push()'), @@ -214,11 +420,33 @@ def push(self, repository, tag=None, stream=False, @utils.check_resource def remove_image(self, image, force=False, noprune=False): + """ + Remove an image. Similar to the ``docker rmi`` command. + + Args: + image (str): The image to remove + force (bool): Force removal of the image + noprune (bool): Do not delete untagged parents + """ params = {'force': force, 'noprune': noprune} res = self._delete(self._url("/images/{0}", image), params=params) self._raise_for_status(res) def search(self, term): + """ + Search for images on Docker Hub. Similar to the ``docker search`` + command. + + Args: + term (str): A term to search for. + + Returns: + (list of dicts): The response of the search. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ return self._result( self._get(self._url("/images/search"), params={'term': term}), True @@ -226,6 +454,22 @@ def search(self, term): @utils.check_resource def tag(self, image, repository, tag=None, force=False): + """ + Tag an image into a repository. Similar to the ``docker tag`` command. + + Args: + image (str): The image to tag + repository (str): The repository to set for the tag + tag (str): The tag name + force (bool): Force + + Returns: + (bool): ``True`` if successful + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ params = { 'tag': tag, 'repo': repository, diff --git a/docker/api/network.py b/docker/api/network.py index 0ee0dab6ea..65aeb30525 100644 --- a/docker/api/network.py +++ b/docker/api/network.py @@ -8,6 +8,21 @@ class NetworkApiMixin(object): @minimum_version('1.21') def networks(self, names=None, ids=None): + """ + List networks. Similar to the ``docker networks ls`` command. + + Args: + names (list): List of names to filter by + ids (list): List of ids to filter by + + Returns: + (dict): List of network objects. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + filters = {} if names: filters['name'] = names @@ -24,6 +39,51 @@ def networks(self, names=None, ids=None): def create_network(self, name, driver=None, options=None, ipam=None, check_duplicate=None, internal=False, labels=None, enable_ipv6=False): + """ + Create a network. Similar to the ``docker network create``. + + Args: + name (str): Name of the network + driver (str): Name of the driver used to create the network + options (dict): Driver options as a key-value dictionary + ipam (dict): Optional custom IP scheme for the network. + Created with :py:meth:`~docker.utils.create_ipam_config`. + check_duplicate (bool): Request daemon to check for networks with + same name. Default: ``True``. + internal (bool): Restrict external access to the network. Default + ``False``. + labels (dict): Map of labels to set on the network. Default + ``None``. + enable_ipv6 (bool): Enable IPv6 on the network. Default ``False``. + + Returns: + (dict): The created network reference object + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + + Example: + A network using the bridge driver: + + >>> client.create_network("network1", driver="bridge") + + You can also create more advanced networks with custom IPAM + configurations. For example, setting the subnet to + ``192.168.52.0/24`` and gateway address to ``192.168.52.254``. + + .. code-block:: python + + >>> ipam_pool = docker.utils.create_ipam_pool( + subnet='192.168.52.0/24', + gateway='192.168.52.254' + ) + >>> ipam_config = docker.utils.create_ipam_config( + pool_configs=[ipam_pool] + ) + >>> docker_client.create_network("network1", driver="bridge", + ipam=ipam_config) + """ if options is not None and not isinstance(options, dict): raise TypeError('options must be a dictionary') @@ -63,12 +123,24 @@ def create_network(self, name, driver=None, options=None, ipam=None, @minimum_version('1.21') def remove_network(self, net_id): + """ + Remove a network. Similar to the ``docker network rm`` command. + + Args: + net_id (str): The network's id + """ url = self._url("/networks/{0}", net_id) res = self._delete(url) self._raise_for_status(res) @minimum_version('1.21') def inspect_network(self, net_id): + """ + Get detailed information about a network. + + Args: + net_id (str): ID of network + """ url = self._url("/networks/{0}", net_id) res = self._get(url) return self._result(res, json=True) @@ -79,6 +151,24 @@ def connect_container_to_network(self, container, net_id, ipv4_address=None, ipv6_address=None, aliases=None, links=None, link_local_ips=None): + """ + Connect a container to a network. + + Args: + container (str): container-id/name to be connected to the network + net_id (str): network id + aliases (list): A list of aliases for this endpoint. Names in that + list can be used within the network to reach the container. + Defaults to ``None``. + links (list): A list of links for this endpoint. Containers + declared in this list will be linkedto this container. + Defaults to ``None``. + ipv4_address (str): The IP address of this container on the + network, using the IPv4 protocol. Defaults to ``None``. + ipv6_address (str): The IP address of this container on the + network, using the IPv6 protocol. Defaults to ``None``. + link_local_ips (list): A list of link-local (IPv4/IPv6) addresses. + """ data = { "Container": container, "EndpointConfig": self.create_endpoint_config( @@ -95,6 +185,16 @@ def connect_container_to_network(self, container, net_id, @minimum_version('1.21') def disconnect_container_from_network(self, container, net_id, force=False): + """ + Disconnect a container from a network. + + Args: + container (str): container ID or name to be disconnected from the + network + net_id (str): network ID + force (bool): Force the container to disconnect from a network. + Default: ``False`` + """ data = {"Container": container} if force: if version_lt(self._version, '1.22'): diff --git a/docker/api/service.py b/docker/api/service.py index ec2a303966..7708b75274 100644 --- a/docker/api/service.py +++ b/docker/api/service.py @@ -9,6 +9,32 @@ def create_service( update_config=None, networks=None, endpoint_config=None, endpoint_spec=None ): + """ + Create a service. + + Args: + task_template (dict): Specification of the task to start as part + of the new service. + name (string): User-defined name for the service. Optional. + labels (dict): A map of labels to associate with the service. + Optional. + mode (string): Scheduling mode for the service (``replicated`` or + ``global``). Defaults to ``replicated``. + update_config (dict): Specification for the update strategy of the + service. Default: ``None`` + networks (list): List of network names or IDs to attach the + service to. Default: ``None``. + endpoint_config (dict): Properties that can be configured to + access and load balance a service. Default: ``None``. + + Returns: + A dictionary containing an ``ID`` key for the newly created + service. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ if endpoint_config is not None: warnings.warn( 'endpoint_config has been renamed to endpoint_spec.', @@ -43,18 +69,58 @@ def create_service( @utils.minimum_version('1.24') @utils.check_resource def inspect_service(self, service): + """ + Return information about a service. + + Args: + service (str): Service name or ID + + Returns: + ``True`` if successful. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ url = self._url('/services/{0}', service) return self._result(self._get(url), True) @utils.minimum_version('1.24') @utils.check_resource def inspect_task(self, task): + """ + Retrieve information about a task. + + Args: + task (str): Task ID + + Returns: + (dict): Information about the task. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ url = self._url('/tasks/{0}', task) return self._result(self._get(url), True) @utils.minimum_version('1.24') @utils.check_resource def remove_service(self, service): + """ + Stop and remove a service. + + Args: + service (str): Service name or ID + + Returns: + ``True`` if successful. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + url = self._url('/services/{0}', service) resp = self._delete(url) self._raise_for_status(resp) @@ -62,6 +128,20 @@ def remove_service(self, service): @utils.minimum_version('1.24') def services(self, filters=None): + """ + List services. + + Args: + filters (dict): Filters to process on the nodes list. Valid + filters: ``id`` and ``name``. Default: ``None``. + + Returns: + A list of dictionaries containing data about each service. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ params = { 'filters': utils.convert_filters(filters) if filters else None } @@ -70,6 +150,22 @@ def services(self, filters=None): @utils.minimum_version('1.24') def tasks(self, filters=None): + """ + Retrieve a list of tasks. + + Args: + filters (dict): A map of filters to process on the tasks list. + Valid filters: ``id``, ``name``, ``service``, ``node``, + ``label`` and ``desired-state``. + + Returns: + (list): List of task dictionaries. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + params = { 'filters': utils.convert_filters(filters) if filters else None } @@ -82,7 +178,37 @@ def update_service(self, service, version, task_template=None, name=None, labels=None, mode=None, update_config=None, networks=None, endpoint_config=None, endpoint_spec=None): + """ + Update a service. + + Args: + service (string): A service identifier (either its name or service + ID). + version (int): The version number of the service object being + updated. This is required to avoid conflicting writes. + task_template (dict): Specification of the updated task to start + as part of the service. See the [TaskTemplate + class](#TaskTemplate) for details. + name (string): New name for the service. Optional. + labels (dict): A map of labels to associate with the service. + Optional. + mode (string): Scheduling mode for the service (``replicated`` or + ``global``). Defaults to ``replicated``. + update_config (dict): Specification for the update strategy of the + service. See the [UpdateConfig class](#UpdateConfig) for + details. Default: ``None``. + networks (list): List of network names or IDs to attach the + service to. Default: ``None``. + endpoint_config (dict): Properties that can be configured to + access and load balance a service. Default: ``None``. + + Returns: + ``True`` if successful. + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ if endpoint_config is not None: warnings.warn( 'endpoint_config has been renamed to endpoint_spec.', diff --git a/docker/api/swarm.py b/docker/api/swarm.py index 2fc877448a..a4cb8dded4 100644 --- a/docker/api/swarm.py +++ b/docker/api/swarm.py @@ -7,11 +7,87 @@ class SwarmApiMixin(object): def create_swarm_spec(self, *args, **kwargs): + """ + Create a ``docker.types.SwarmSpec`` instance that can be used as the + ``swarm_spec`` argument in + :py:meth:`~docker.api.swarm.SwarmApiMixin.init_swarm`. + + Args: + task_history_retention_limit (int): Maximum number of tasks + history stored. + snapshot_interval (int): Number of logs entries between snapshot. + keep_old_snapshots (int): Number of snapshots to keep beyond the + current snapshot. + log_entries_for_slow_followers (int): Number of log entries to + keep around to sync up slow followers after a snapshot is + created. + heartbeat_tick (int): Amount of ticks (in seconds) between each + heartbeat. + election_tick (int): Amount of ticks (in seconds) needed without a + leader to trigger a new election. + dispatcher_heartbeat_period (int): The delay for an agent to send + a heartbeat to the dispatcher. + node_cert_expiry (int): Automatic expiry for nodes certificates. + external_ca (dict): Configuration for forwarding signing requests + to an external certificate authority. Use + ``docker.types.SwarmExternalCA``. + name (string): Swarm's name + + Returns: + ``docker.types.SwarmSpec`` instance. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + + Example: + + >>> spec = client.create_swarm_spec( + snapshot_interval=5000, log_entries_for_slow_followers=1200 + ) + >>> client.init_swarm( + advertise_addr='eth0', listen_addr='0.0.0.0:5000', + force_new_cluster=False, swarm_spec=spec + ) + """ return utils.SwarmSpec(*args, **kwargs) @utils.minimum_version('1.24') def init_swarm(self, advertise_addr=None, listen_addr='0.0.0.0:2377', force_new_cluster=False, swarm_spec=None): + """ + Initialize a new Swarm using the current connected engine as the first + node. + + Args: + advertise_addr (string): Externally reachable address advertised + to other nodes. This can either be an address/port combination + in the form ``192.168.1.1:4567``, or an interface followed by a + port number, like ``eth0:4567``. If the port number is omitted, + the port number from the listen address is used. If + ``advertise_addr`` is not specified, it will be automatically + detected when possible. Default: None + listen_addr (string): Listen address used for inter-manager + communication, as well as determining the networking interface + used for the VXLAN Tunnel Endpoint (VTEP). This can either be + an address/port combination in the form ``192.168.1.1:4567``, + or an interface followed by a port number, like ``eth0:4567``. + If the port number is omitted, the default swarm listening port + is used. Default: '0.0.0.0:2377' + force_new_cluster (bool): Force creating a new Swarm, even if + already part of one. Default: False + swarm_spec (dict): Configuration settings of the new Swarm. Use + ``Client.create_swarm_spec`` to generate a valid + configuration. Default: None + + Returns: + ``True`` if successful. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + url = self._url('/swarm/init') if swarm_spec is not None and not isinstance(swarm_spec, dict): raise TypeError('swarm_spec must be a dictionary') @@ -27,18 +103,67 @@ def init_swarm(self, advertise_addr=None, listen_addr='0.0.0.0:2377', @utils.minimum_version('1.24') def inspect_swarm(self): + """ + Retrieve low-level information about the current swarm. + + Returns: + A dictionary containing data about the swarm. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ url = self._url('/swarm') return self._result(self._get(url), True) @utils.check_resource @utils.minimum_version('1.24') def inspect_node(self, node_id): + """ + Retrieve low-level information about a swarm node + + Args: + node_id (string): ID of the node to be inspected. + + Returns: + A dictionary containing data about this node. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ url = self._url('/nodes/{0}', node_id) return self._result(self._get(url), True) @utils.minimum_version('1.24') def join_swarm(self, remote_addrs, join_token, listen_addr=None, advertise_addr=None): + """ + Make this Engine join a swarm that has already been created. + + Args: + remote_addrs (list): Addresses of one or more manager nodes already + participating in the Swarm to join. + join_token (string): Secret token for joining this Swarm. + listen_addr (string): Listen address used for inter-manager + communication if the node gets promoted to manager, as well as + determining the networking interface used for the VXLAN Tunnel + Endpoint (VTEP). Default: ``None`` + advertise_addr (string): Externally reachable address advertised + to other nodes. This can either be an address/port combination + in the form ``192.168.1.1:4567``, or an interface followed by a + port number, like ``eth0:4567``. If the port number is omitted, + the port number from the listen address is used. If + AdvertiseAddr is not specified, it will be automatically + detected when possible. Default: ``None`` + + Returns: + ``True`` if the request went through. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ data = { "RemoteAddrs": remote_addrs, "ListenAddr": listen_addr, @@ -52,6 +177,20 @@ def join_swarm(self, remote_addrs, join_token, listen_addr=None, @utils.minimum_version('1.24') def leave_swarm(self, force=False): + """ + Leave a swarm. + + Args: + force (bool): Leave the swarm even if this node is a manager. + Default: ``False`` + + Returns: + ``True`` if the request went through. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ url = self._url('/swarm/leave') response = self._post(url, params={'force': force}) # Ignore "this node is not part of a swarm" error @@ -62,6 +201,21 @@ def leave_swarm(self, force=False): @utils.minimum_version('1.24') def nodes(self, filters=None): + """ + List swarm nodes. + + Args: + filters (dict): Filters to process on the nodes list. Valid + filters: ``id``, ``name``, ``membership`` and ``role``. + Default: ``None`` + + Returns: + A list of dictionaries containing data about each swarm node. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ url = self._url('/nodes') params = {} if filters: @@ -71,6 +225,34 @@ def nodes(self, filters=None): @utils.minimum_version('1.24') def update_node(self, node_id, version, node_spec=None): + """ + Update the Node's configuration + + Args: + + version (int): The version number of the node object being + updated. This is required to avoid conflicting writes. + node_spec (dict): Configuration settings to update. Any values + not provided will be removed. Default: ``None`` + + Returns: + `True` if the request went through. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + + Example: + + >>> node_spec = {'Availability': 'active', + 'Name': 'node-name', + 'Role': 'manager', + 'Labels': {'foo': 'bar'} + } + >>> client.update_node(node_id='24ifsmvkjbyhk', version=8, + node_spec=node_spec) + + """ url = self._url('/nodes/{0}/update?version={1}', node_id, str(version)) res = self._post_json(url, data=node_spec) self._raise_for_status(res) @@ -79,6 +261,28 @@ def update_node(self, node_id, version, node_spec=None): @utils.minimum_version('1.24') def update_swarm(self, version, swarm_spec=None, rotate_worker_token=False, rotate_manager_token=False): + """ + Update the Swarm's configuration + + Args: + version (int): The version number of the swarm object being + updated. This is required to avoid conflicting writes. + swarm_spec (dict): Configuration settings to update. Use + :py:meth:`~docker.api.swarm.SwarmApiMixin.create_swarm_spec` to + generate a valid configuration. Default: ``None``. + rotate_worker_token (bool): Rotate the worker join token. Default: + ``False``. + rotate_manager_token (bool): Rotate the manager join token. + Default: ``False``. + + Returns: + ``True`` if the request went through. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + url = self._url('/swarm/update') response = self._post_json(url, data=swarm_spec, params={ 'rotateWorkerToken': rotate_worker_token, diff --git a/docker/api/volume.py b/docker/api/volume.py index afc72cbbc8..9c6d5f8351 100644 --- a/docker/api/volume.py +++ b/docker/api/volume.py @@ -5,6 +5,32 @@ class VolumeApiMixin(object): @utils.minimum_version('1.21') def volumes(self, filters=None): + """ + List volumes currently registered by the docker daemon. Similar to the + ``docker volume ls`` command. + + Args: + filters (dict): Server-side list filtering options. + + Returns: + (dict): Dictionary with list of volume objects as value of the + ``Volumes`` key. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + + Example: + + >>> cli.volumes() + {u'Volumes': [{u'Driver': u'local', + u'Mountpoint': u'/var/lib/docker/volumes/foobar/_data', + u'Name': u'foobar'}, + {u'Driver': u'local', + u'Mountpoint': u'/var/lib/docker/volumes/baz/_data', + u'Name': u'baz'}]} + """ + params = { 'filters': utils.convert_filters(filters) if filters else None } @@ -13,6 +39,34 @@ def volumes(self, filters=None): @utils.minimum_version('1.21') def create_volume(self, name, driver=None, driver_opts=None, labels=None): + """ + Create and register a named volume + + Args: + name (str): Name of the volume + driver (str): Name of the driver used to create the volume + driver_opts (dict): Driver options as a key-value dictionary + labels (dict): Labels to set on the volume + + Returns: + (dict): The created volume reference object + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + + Example: + + >>> volume = cli.create_volume(name='foobar', driver='local', + driver_opts={'foo': 'bar', 'baz': 'false'}, + labels={"key": "value"}) + >>> print(volume) + {u'Driver': u'local', + u'Labels': {u'key': u'value'}, + u'Mountpoint': u'/var/lib/docker/volumes/foobar/_data', + u'Name': u'foobar'} + + """ url = self._url('/volumes/create') if driver_opts is not None and not isinstance(driver_opts, dict): raise TypeError('driver_opts must be a dictionary') @@ -36,11 +90,42 @@ def create_volume(self, name, driver=None, driver_opts=None, labels=None): @utils.minimum_version('1.21') def inspect_volume(self, name): + """ + Retrieve volume info by name. + + Args: + name (str): volume name + + Returns: + (dict): Volume information dictionary + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + + Example: + + >>> cli.inspect_volume('foobar') + {u'Driver': u'local', + u'Mountpoint': u'/var/lib/docker/volumes/foobar/_data', + u'Name': u'foobar'} + + """ url = self._url('/volumes/{0}', name) return self._result(self._get(url), True) @utils.minimum_version('1.21') def remove_volume(self, name): + """ + Remove a volume. Similar to the ``docker volume rm`` command. + + Args: + name (str): The volume's name + + Raises: + + ``docker.errors.APIError``: If volume failed to remove. + """ url = self._url('/volumes/{0}', name) resp = self._delete(url) self._raise_for_status(resp) diff --git a/docker/tls.py b/docker/tls.py index 7c3a2ca35f..3a0827a25c 100644 --- a/docker/tls.py +++ b/docker/tls.py @@ -5,6 +5,20 @@ class TLSConfig(object): + """ + TLS configuration. + + Args: + client_cert (tuple of str): Path to client cert, path to client key. + ca_cert (str): Path to CA cert file. + verify (bool or str): This can be ``False`` or a path to a CA cert + file. + ssl_version (int): A valid `SSL version`_. + assert_hostname (bool): Verify the hostname of the server. + + .. _`SSL version`: + https://docs.python.org/3.5/library/ssl.html#ssl.PROTOCOL_TLSv1 + """ cert = None ca_cert = None verify = None @@ -57,6 +71,9 @@ def __init__(self, client_cert=None, ca_cert=None, verify=None, ) def configure_client(self, client): + """ + Configure a client with these TLS options. + """ client.ssl_version = self.ssl_version if self.verify and self.ca_cert: diff --git a/docker/utils/utils.py b/docker/utils/utils.py index b1db884a99..b107f22e97 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -39,6 +39,38 @@ def create_ipam_pool(subnet=None, iprange=None, gateway=None, aux_addresses=None): + """ + Create an IPAM pool config dictionary to be added to the + ``pool_configs`` parameter of + :py:meth:`~docker.utils.create_ipam_config`. + + Args: + + subnet (str): Custom subnet for this IPAM pool using the CIDR + notation. Defaults to ``None``. + iprange (str): Custom IP range for endpoints in this IPAM pool using + the CIDR notation. Defaults to ``None``. + gateway (str): Custom IP address for the pool's gateway. + aux_addresses (dict): A dictionary of ``key -> ip_address`` + relationships specifying auxiliary addresses that need to be + allocated by the IPAM driver. + + Returns: + (dict) An IPAM pool config + + Example: + + >>> ipam_pool = docker.utils.create_ipam_pool( + subnet='124.42.0.0/16', + iprange='124.42.0.0/24', + gateway='124.42.0.254', + aux_addresses={ + 'reserved1': '124.42.1.1' + } + ) + >>> ipam_config = docker.utils.create_ipam_config( + pool_configs=[ipam_pool]) + """ return { 'Subnet': subnet, 'IPRange': iprange, @@ -48,6 +80,25 @@ def create_ipam_pool(subnet=None, iprange=None, gateway=None, def create_ipam_config(driver='default', pool_configs=None): + """ + Create an IPAM (IP Address Management) config dictionary to be used with + :py:meth:`~docker.api.network.NetworkApiMixin.create_network`. + + Args: + driver (str): The IPAM driver to use. Defaults to ``default``. + pool_configs (list): A list of pool configuration dictionaries as + created by :py:meth:`~docker.utils.create_ipam_pool`. Defaults to + empty list. + + Returns: + (dict) An IPAM config. + + Example: + + >>> ipam_config = docker.utils.create_ipam_config(driver='default') + >>> network = client.create_network('network1', ipam=ipam_config) + + """ return { 'Driver': driver, 'Config': pool_configs or [] From 1984f68730512a1c07017118f4e229c7949ff8a8 Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Mon, 7 Nov 2016 17:56:02 -0800 Subject: [PATCH 0177/1301] Add new user-focused API See #1086 Signed-off-by: Ben Firshman --- README.md | 77 +- docker/__init__.py | 1 + docker/api/client.py | 22 +- docker/client.py | 157 ++++ docker/errors.py | 93 ++- docker/models/__init__.py | 0 docker/models/containers.py | 883 ++++++++++++++++++++ docker/models/images.py | 269 ++++++ docker/models/networks.py | 181 ++++ docker/models/nodes.py | 88 ++ docker/models/resource.py | 84 ++ docker/models/services.py | 240 ++++++ docker/models/swarm.py | 146 ++++ docker/models/volumes.py | 84 ++ docker/utils/json_stream.py | 79 ++ tests/helpers.py | 13 + tests/integration/client_test.py | 20 + tests/integration/models_containers_test.py | 204 +++++ tests/integration/models_images_test.py | 67 ++ tests/integration/models_networks_test.py | 64 ++ tests/integration/models_nodes_test.py | 34 + tests/integration/models_resources_test.py | 16 + tests/integration/models_services_test.py | 100 +++ tests/integration/models_swarm_test.py | 22 + tests/integration/models_volumes_test.py | 30 + tests/unit/api_test.py | 27 - tests/unit/client_test.py | 73 ++ tests/unit/errors_test.py | 22 + tests/unit/fake_api.py | 103 ++- tests/unit/fake_api_client.py | 61 ++ tests/unit/models_containers_test.py | 465 +++++++++++ tests/unit/models_images_test.py | 102 +++ tests/unit/models_networks_test.py | 64 ++ tests/unit/models_resources_test.py | 14 + tests/unit/models_services_test.py | 52 ++ tests/unit/utils_json_stream_test.py | 62 ++ 36 files changed, 3942 insertions(+), 77 deletions(-) create mode 100644 docker/client.py create mode 100644 docker/models/__init__.py create mode 100644 docker/models/containers.py create mode 100644 docker/models/images.py create mode 100644 docker/models/networks.py create mode 100644 docker/models/nodes.py create mode 100644 docker/models/resource.py create mode 100644 docker/models/services.py create mode 100644 docker/models/swarm.py create mode 100644 docker/models/volumes.py create mode 100644 docker/utils/json_stream.py create mode 100644 tests/integration/client_test.py create mode 100644 tests/integration/models_containers_test.py create mode 100644 tests/integration/models_images_test.py create mode 100644 tests/integration/models_networks_test.py create mode 100644 tests/integration/models_nodes_test.py create mode 100644 tests/integration/models_resources_test.py create mode 100644 tests/integration/models_services_test.py create mode 100644 tests/integration/models_swarm_test.py create mode 100644 tests/integration/models_volumes_test.py create mode 100644 tests/unit/client_test.py create mode 100644 tests/unit/errors_test.py create mode 100644 tests/unit/fake_api_client.py create mode 100644 tests/unit/models_containers_test.py create mode 100644 tests/unit/models_images_test.py create mode 100644 tests/unit/models_networks_test.py create mode 100644 tests/unit/models_resources_test.py create mode 100644 tests/unit/models_services_test.py create mode 100644 tests/unit/utils_json_stream_test.py diff --git a/README.md b/README.md index 876ed02636..094b13c8e3 100644 --- a/README.md +++ b/README.md @@ -1,26 +1,73 @@ -docker-py -========= +# Docker SDK for Python [![Build Status](https://travis-ci.org/docker/docker-py.png)](https://travis-ci.org/docker/docker-py) -A Python library for the Docker Remote API. It does everything the `docker` command does, but from within Python – run containers, manage them, pull/push images, etc. +A Python library for the Docker API. It lets you do anything the `docker` command does, but from within Python apps – run containers, manage containers, manage Swarms, etc. -Installation ------------- +## Installation -The latest stable version is always available on PyPi. +The latest stable version [is available on PyPi](https://pypi.python.org/pypi/docker/). Either add `docker` to your `requirements.txt` file or install with pip: - pip install docker-py + pip install docker -Documentation -------------- +## Usage -[![Documentation Status](https://readthedocs.org/projects/docker-py/badge/?version=latest)](https://readthedocs.org/projects/docker-py/?badge=latest) +Connect to Docker using the default socket or the configuration in your environment: -[Read the full documentation here](https://docker-py.readthedocs.io/en/latest/). -The source is available in the `docs/` directory. +```python +import docker +client = docker.from_env() +``` +You can run containers: -License -------- -Docker is licensed under the Apache License, Version 2.0. See LICENSE for full license text +```python +>>> client.containers.run("ubuntu", "echo hello world") +'hello world\n' +``` + +You can run containers in the background: + +```python +>>> client.containers.run("bfirsh/reticulate-splines", detach=True) + +``` + +You can manage containers: + +```python +>>> client.containers.list() +[, , ...] + +>>> container = client.containers.get('45e6d2de7c54') + +>>> container.attrs['Config']['Image'] +"bfirsh/reticulate-splines" + +>>> container.logs() +"Reticulating spline 1...\n" + +>>> container.stop() +``` + +You can stream logs: + +```python +>>> for line in container.logs(stream=True): +... print line.strip() +Reticulating spline 2... +Reticulating spline 3... +... +``` + +You can manage images: + +```python +>>> client.images.pull('nginx') + + +>>> client.images.list() +[, , ...] +``` + +[Read the full documentation](https://docs.docker.com/sdk/python/) to see everything you can do. diff --git a/docker/__init__.py b/docker/__init__.py index 95edb6b1bf..acf4b5566e 100644 --- a/docker/__init__.py +++ b/docker/__init__.py @@ -1,5 +1,6 @@ # flake8: noqa from .api import APIClient +from .client import Client, from_env from .version import version, version_info __version__ = version diff --git a/docker/api/client.py b/docker/api/client.py index 5c26d63cb9..23e239c66f 100644 --- a/docker/api/client.py +++ b/docker/api/client.py @@ -22,10 +22,11 @@ IS_WINDOWS_PLATFORM, DEFAULT_DOCKER_API_VERSION, STREAM_HEADER_SIZE_BYTES, DEFAULT_NUM_POOLS, MINIMUM_DOCKER_API_VERSION) -from ..errors import DockerException, APIError, TLSParameterError, NotFound +from ..errors import (DockerException, TLSParameterError, + create_api_error_from_http_exception) from ..tls import TLSConfig from ..transport import UnixAdapter -from ..utils import utils, check_resource, update_headers, kwargs_from_env +from ..utils import utils, check_resource, update_headers from ..utils.socket import frames_iter try: from ..transport import NpipeAdapter @@ -33,10 +34,6 @@ pass -def from_env(**kwargs): - return APIClient.from_env(**kwargs) - - class APIClient( requests.Session, BuildApiMixin, @@ -152,13 +149,6 @@ def __init__(self, base_url=None, version=None, MINIMUM_DOCKER_API_VERSION, self._version) ) - @classmethod - def from_env(cls, **kwargs): - timeout = kwargs.pop('timeout', None) - version = kwargs.pop('version', None) - return cls(timeout=timeout, version=version, - **kwargs_from_env(**kwargs)) - def _retrieve_server_version(self): try: return self.version(api_version=False)["ApiVersion"] @@ -212,14 +202,12 @@ def _url(self, pathfmt, *args, **kwargs): else: return '{0}{1}'.format(self.base_url, pathfmt.format(*args)) - def _raise_for_status(self, response, explanation=None): + def _raise_for_status(self, response): """Raises stored :class:`APIError`, if one occurred.""" try: response.raise_for_status() except requests.exceptions.HTTPError as e: - if e.response.status_code == 404: - raise NotFound(e, response, explanation=explanation) - raise APIError(e, response, explanation=explanation) + raise create_api_error_from_http_exception(e) def _result(self, response, json=False, binary=False): assert not (json and binary) diff --git a/docker/client.py b/docker/client.py new file mode 100644 index 0000000000..6c5ae4080d --- /dev/null +++ b/docker/client.py @@ -0,0 +1,157 @@ +from .api.client import APIClient +from .models.containers import ContainerCollection +from .models.images import ImageCollection +from .models.networks import NetworkCollection +from .models.nodes import NodeCollection +from .models.services import ServiceCollection +from .models.swarm import Swarm +from .models.volumes import VolumeCollection +from .utils import kwargs_from_env + + +class Client(object): + """ + A client for communicating with a Docker server. + + Example: + + >>> import docker + >>> client = Client(base_url='unix://var/run/docker.sock') + + Args: + base_url (str): URL to the Docker server. For example, + ``unix:///var/run/docker.sock`` or ``tcp://127.0.0.1:1234``. + version (str): The version of the API to use. Set to ``auto`` to + automatically detect the server's version. Default: ``1.24`` + timeout (int): Default timeout for API calls, in seconds. + tls (bool or :py:class:`~docker.tls.TLSConfig`): Enable TLS. Pass + ``True`` to enable it with default options, or pass a + :py:class:`~docker.tls.TLSConfig` object to use custom + configuration. + user_agent (str): Set a custom user agent for requests to the server. + """ + def __init__(self, *args, **kwargs): + self.api = APIClient(*args, **kwargs) + + @classmethod + def from_env(cls, **kwargs): + """ + Return a client configured from environment variables. + + The environment variables used are the same as those used by the + Docker command-line client. They are: + + .. envvar:: DOCKER_HOST + + The URL to the Docker host. + + .. envvar:: DOCKER_TLS_VERIFY + + Verify the host against a CA certificate. + + .. envvar:: DOCKER_CERT_PATH + + A path to a directory containing TLS certificates to use when + connecting to the Docker host. + + Args: + version (str): The version of the API to use. Set to ``auto`` to + automatically detect the server's version. Default: ``1.24`` + timeout (int): Default timeout for API calls, in seconds. + ssl_version (int): A valid `SSL version`_. + assert_hostname (bool): Verify the hostname of the server. + environment (dict): The environment to read environment variables + from. Default: the value of ``os.environ`` + + Example: + + >>> import docker + >>> client = docker.from_env() + + .. _`SSL version`: + https://docs.python.org/3.5/library/ssl.html#ssl.PROTOCOL_TLSv1 + """ + timeout = kwargs.pop('timeout', None) + version = kwargs.pop('version', None) + return cls(timeout=timeout, version=version, + **kwargs_from_env(**kwargs)) + + # Resources + @property + def containers(self): + """ + An object for managing containers on the server. See the + :doc:`containers documentation ` for full details. + """ + return ContainerCollection(client=self) + + @property + def images(self): + """ + An object for managing images on the server. See the + :doc:`images documentation ` for full details. + """ + return ImageCollection(client=self) + + @property + def networks(self): + """ + An object for managing networks on the server. See the + :doc:`networks documentation ` for full details. + """ + return NetworkCollection(client=self) + + @property + def nodes(self): + """ + An object for managing nodes on the server. See the + :doc:`nodes documentation ` for full details. + """ + return NodeCollection(client=self) + + @property + def services(self): + """ + An object for managing services on the server. See the + :doc:`services documentation ` for full details. + """ + return ServiceCollection(client=self) + + @property + def swarm(self): + """ + An object for managing a swarm on the server. See the + :doc:`swarm documentation ` for full details. + """ + return Swarm(client=self) + + @property + def volumes(self): + """ + An object for managing volumes on the server. See the + :doc:`volumes documentation ` for full details. + """ + return VolumeCollection(client=self) + + # Top-level methods + def events(self, *args, **kwargs): + return self.api.events(*args, **kwargs) + events.__doc__ = APIClient.events.__doc__ + + def info(self, *args, **kwargs): + return self.api.info(*args, **kwargs) + info.__doc__ = APIClient.info.__doc__ + + def login(self, *args, **kwargs): + return self.api.login(*args, **kwargs) + login.__doc__ = APIClient.login.__doc__ + + def ping(self, *args, **kwargs): + return self.api.ping(*args, **kwargs) + ping.__doc__ = APIClient.ping.__doc__ + + def version(self, *args, **kwargs): + return self.api.version(*args, **kwargs) + version.__doc__ = APIClient.version.__doc__ + +from_env = Client.from_env diff --git a/docker/errors.py b/docker/errors.py index df18d57830..8572007d42 100644 --- a/docker/errors.py +++ b/docker/errors.py @@ -1,21 +1,44 @@ import requests -class APIError(requests.exceptions.HTTPError): - def __init__(self, message, response, explanation=None): +class DockerException(Exception): + """ + A base class from which all other exceptions inherit. + + If you want to catch all errors that the Docker SDK might raise, + catch this base exception. + """ + + +def create_api_error_from_http_exception(e): + """ + Create a suitable APIError from requests.exceptions.HTTPError. + """ + response = e.response + try: + explanation = response.json()['message'] + except ValueError: + explanation = response.content.strip() + cls = APIError + if response.status_code == 404: + if explanation and 'No such image' in str(explanation): + cls = ImageNotFound + else: + cls = NotFound + raise cls(e, response=response, explanation=explanation) + + +class APIError(requests.exceptions.HTTPError, DockerException): + """ + An HTTP error from the API. + """ + def __init__(self, message, response=None, explanation=None): # requests 1.2 supports response as a keyword argument, but # requests 1.1 doesn't super(APIError, self).__init__(message) self.response = response - self.explanation = explanation - if self.explanation is None and response.content: - try: - self.explanation = response.json()['message'] - except ValueError: - self.explanation = response.content.strip() - def __str__(self): message = super(APIError, self).__str__() @@ -32,18 +55,27 @@ def __str__(self): return message + @property + def status_code(self): + if self.response: + return self.response.status_code + def is_client_error(self): - return 400 <= self.response.status_code < 500 + if self.status_code is None: + return False + return 400 <= self.status_code < 500 def is_server_error(self): - return 500 <= self.response.status_code < 600 + if self.status_code is None: + return False + return 500 <= self.status_code < 600 -class DockerException(Exception): +class NotFound(APIError): pass -class NotFound(APIError): +class ImageNotFound(NotFound): pass @@ -76,3 +108,38 @@ def __str__(self): class NullResource(DockerException, ValueError): pass + + +class ContainerError(DockerException): + """ + Represents a container that has exited with a non-zero exit code. + """ + def __init__(self, container, exit_status, command, image, stderr): + self.container = container + self.exit_status = exit_status + self.command = command + self.image = image + self.stderr = stderr + msg = ("Command '{}' in image '{}' returned non-zero exit status {}: " + "{}").format(command, image, exit_status, stderr) + super(ContainerError, self).__init__(msg) + + +class StreamParseError(RuntimeError): + def __init__(self, reason): + self.msg = reason + + +class BuildError(Exception): + pass + + +def create_unexpected_kwargs_error(name, kwargs): + quoted_kwargs = ["'{}'".format(k) for k in sorted(kwargs)] + text = ["{}() ".format(name)] + if len(quoted_kwargs) == 1: + text.append("got an unexpected keyword argument ") + else: + text.append("got unexpected keyword arguments ") + text.append(', '.join(quoted_kwargs)) + return TypeError(''.join(text)) diff --git a/docker/models/__init__.py b/docker/models/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docker/models/containers.py b/docker/models/containers.py new file mode 100644 index 0000000000..9682248a91 --- /dev/null +++ b/docker/models/containers.py @@ -0,0 +1,883 @@ +import copy + +from ..errors import (ContainerError, ImageNotFound, + create_unexpected_kwargs_error) +from ..utils import create_host_config +from .images import Image +from .resource import Collection, Model + + +class Container(Model): + + @property + def name(self): + """ + The name of the container. + """ + if self.attrs.get('Name') is not None: + return self.attrs['Name'].lstrip('/') + + @property + def status(self): + """ + The status of the container. For example, ``running``, or ``exited``. + """ + return self.attrs['State']['Status'] + + def attach(self, **kwargs): + """ + Attach to this container. + + :py:meth:`logs` is a wrapper around this method, which you can + use instead if you want to fetch/stream container output without first + retrieving the entire backlog. + + Args: + stdout (bool): Include stdout. + stderr (bool): Include stderr. + stream (bool): Return container output progressively as an iterator + of strings, rather than a single string. + logs (bool): Include the container's previous output. + + Returns: + By default, the container's output as a single string. + + If ``stream=True``, an iterator of output strings. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + return self.client.api.attach(self.id, **kwargs) + + def attach_socket(self, **kwargs): + """ + Like :py:meth:`attach`, but returns the underlying socket-like object + for the HTTP request. + + Args: + params (dict): Dictionary of request parameters (e.g. ``stdout``, + ``stderr``, ``stream``). + ws (bool): Use websockets instead of raw HTTP. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + return self.client.api.attach_socket(self.id, **kwargs) + + def commit(self, repository=None, tag=None, **kwargs): + """ + Commit a container to an image. Similar to the ``docker commit`` + command. + + Args: + repository (str): The repository to push the image to + tag (str): The tag to push + message (str): A commit message + author (str): The name of the author + changes (str): Dockerfile instructions to apply while committing + conf (dict): The configuration for the container. See the + `Remote API documentation + `_ + for full details. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + + resp = self.client.api.commit(self.id, repository=repository, tag=tag, + **kwargs) + return self.client.images.get(resp['Id']) + + def diff(self): + """ + Inspect changes on a container's filesystem. + + Returns: + (str) + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + return self.client.api.diff(self.id) + + def exec_run(self, cmd, stdout=True, stderr=True, stdin=False, tty=False, + privileged=False, user='', detach=False, stream=False, + socket=False): + """ + Run a command inside this container. Similar to + ``docker exec``. + + Args: + cmd (str or list): Command to be executed + stdout (bool): Attach to stdout. Default: ``True`` + stderr (bool): Attach to stderr. Default: ``True`` + stdin (bool): Attach to stdin. Default: ``False`` + tty (bool): Allocate a pseudo-TTY. Default: False + privileged (bool): Run as privileged. + user (str): User to execute command as. Default: root + detach (bool): If true, detach from the exec command. + Default: False + tty (bool): Allocate a pseudo-TTY. Default: False + stream (bool): Stream response data. Default: False + + Returns: + (generator or str): If ``stream=True``, a generator yielding + response chunks. A string containing response data otherwise. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + resp = self.client.api.exec_create( + self.id, cmd, stdout=stdout, stderr=stderr, stdin=stdin, tty=tty, + privileged=privileged, user=user + ) + return self.client.api.exec_start( + resp['Id'], detach=detach, tty=tty, stream=stream, socket=socket + ) + + def export(self): + """ + Export the contents of the container's filesystem as a tar archive. + + Returns: + (str): The filesystem tar archive + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + return self.client.api.export(self.id) + + def get_archive(self, path): + """ + Retrieve a file or folder from the container in the form of a tar + archive. + + Args: + path (str): Path to the file or folder to retrieve + + Returns: + (tuple): First element is a raw tar data stream. Second element is + a dict containing ``stat`` information on the specified ``path``. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + return self.client.api.get_archive(self.id, path) + + def kill(self, signal=None): + """ + Kill or send a signal to the container. + + Args: + signal (str or int): The signal to send. Defaults to ``SIGKILL`` + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + + return self.client.api.kill(self.id, signal=signal) + + def logs(self, **kwargs): + """ + Get logs from this container. Similar to the ``docker logs`` command. + + The ``stream`` parameter makes the ``logs`` function return a blocking + generator you can iterate over to retrieve log output as it happens. + + Args: + stdout (bool): Get ``STDOUT`` + stderr (bool): Get ``STDERR`` + stream (bool): Stream the response + timestamps (bool): Show timestamps + tail (str or int): Output specified number of lines at the end of + logs. Either an integer of number of lines or the string + ``all``. Default ``all`` + since (datetime or int): Show logs since a given datetime or + integer epoch (in seconds) + follow (bool): Follow log output + + Returns: + (generator or str): Logs from the container. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + return self.client.api.logs(self.id, **kwargs) + + def pause(self): + """ + Pauses all processes within this container. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + return self.client.api.pause(self.id) + + def put_archive(self, path, data): + """ + Insert a file or folder in this container using a tar archive as + source. + + Args: + path (str): Path inside the container where the file(s) will be + extracted. Must exist. + data (bytes): tar data to be extracted + + Returns: + (bool): True if the call succeeds. + + Raises: + :py:class:`~docker.errors.APIError` If an error occurs. + """ + return self.client.api.put_archive(self.id, path, data) + + def remove(self, **kwargs): + """ + Remove this container. Similar to the ``docker rm`` command. + + Args: + v (bool): Remove the volumes associated with the container + link (bool): Remove the specified link and not the underlying + container + force (bool): Force the removal of a running container (uses + ``SIGKILL``) + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + return self.client.api.remove_container(self.id, **kwargs) + + def rename(self, name): + """ + Rename this container. Similar to the ``docker rename`` command. + + Args: + name (str): New name for the container + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + return self.client.api.rename(self.id, name) + + def resize(self, height, width): + """ + Resize the tty session. + + Args: + height (int): Height of tty session + width (int): Width of tty session + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + return self.client.api.resize(self.id, height, width) + + def restart(self, **kwargs): + """ + Restart this container. Similar to the ``docker restart`` command. + + Args: + timeout (int): Number of seconds to try to stop for before killing + the container. Once killed it will then be restarted. Default + is 10 seconds. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + return self.client.api.restart(self.id, **kwargs) + + def start(self, **kwargs): + """ + Start this container. Similar to the ``docker start`` command, but + doesn't support attach options. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + return self.client.api.start(self.id, **kwargs) + + def stats(self, **kwargs): + """ + Stream statistics for this container. Similar to the + ``docker stats`` command. + + Args: + decode (bool): If set to true, stream will be decoded into dicts + on the fly. False by default. + stream (bool): If set to false, only the current stats will be + returned instead of a stream. True by default. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + return self.client.api.stats(self.id, **kwargs) + + def stop(self, **kwargs): + """ + Stops a container. Similar to the ``docker stop`` command. + + Args: + timeout (int): Timeout in seconds to wait for the container to + stop before sending a ``SIGKILL``. Default: 10 + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + return self.client.api.stop(self.id, **kwargs) + + def top(self, **kwargs): + """ + Display the running processes of the container. + + Args: + ps_args (str): An optional arguments passed to ps (e.g. ``aux``) + + Returns: + (str): The output of the top + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + return self.client.api.top(self.id, **kwargs) + + def unpause(self): + """ + Unpause all processes within the container. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + return self.client.api.unpause(self.id) + + def update(self, **kwargs): + """ + Update resource configuration of the containers. + + Args: + blkio_weight (int): Block IO (relative weight), between 10 and 1000 + cpu_period (int): Limit CPU CFS (Completely Fair Scheduler) period + cpu_quota (int): Limit CPU CFS (Completely Fair Scheduler) quota + cpu_shares (int): CPU shares (relative weight) + cpuset_cpus (str): CPUs in which to allow execution + cpuset_mems (str): MEMs in which to allow execution + mem_limit (int or str): Memory limit + mem_reservation (int or str): Memory soft limit + memswap_limit (int or str): Total memory (memory + swap), -1 to + disable swap + kernel_memory (int or str): Kernel memory limit + restart_policy (dict): Restart policy dictionary + + Returns: + (dict): Dictionary containing a ``Warnings`` key. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + return self.client.api.update_container(self.id, **kwargs) + + def wait(self, **kwargs): + """ + Block until the container stops, then return its exit code. Similar to + the ``docker wait`` command. + + Args: + timeout (int): Request timeout + + Returns: + (int): The exit code of the container. Returns ``-1`` if the API + responds without a ``StatusCode`` attribute. + + Raises: + :py:class:`requests.exceptions.ReadTimeout` + If the timeout is exceeded. + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + return self.client.api.wait(self.id, **kwargs) + + +class ContainerCollection(Collection): + model = Container + + def run(self, image, command=None, stdout=True, stderr=False, + remove=False, **kwargs): + """ + Run a container. By default, it will wait for the container to finish + and return its logs, similar to ``docker run``. + + If the ``detach`` argument is ``True``, it will start the container + and immediately return a :py:class:`Container` object, similar to + ``docker run -d``. + + Example: + Run a container and get its output: + + >>> import docker + >>> client = docker.from_env() + >>> client.containers.run('alpine', 'echo hello world') + b'hello world\\n' + + Run a container and detach: + + >>> container = client.containers.run('bfirsh/reticulate-splines', + detach=True) + >>> container.logs() + 'Reticulating spline 1...\\nReticulating spline 2...\\n' + + Args: + image (str): The image to run. + command (str or list): The command to run in the container. + blkio_weight_device: Block IO weight (relative device weight) in + the form of: ``[{"Path": "device_path", "Weight": weight}]``. + blkio_weight: Block IO weight (relative weight), accepts a weight + value between 10 and 1000. + cap_add (list of str): Add kernel capabilities. For example, + ``["SYS_ADMIN", "MKNOD"]``. + cap_drop (list of str): Drop kernel capabilities. + cpu_group (int): The length of a CPU period in microseconds. + cpu_period (int): Microseconds of CPU time that the container can + get in a CPU period. + cpu_shares (int): CPU shares (relative weight). + cpuset_cpus (str): CPUs in which to allow execution (``0-3``, + ``0,1``). + detach (bool): Run container in the background and return a + :py:class:`Container` object. + device_read_bps: Limit read rate (bytes per second) from a device + in the form of: `[{"Path": "device_path", "Rate": rate}]` + device_read_iops: Limit read rate (IO per second) from a device. + device_write_bps: Limit write rate (bytes per second) from a + device. + device_write_iops: Limit write rate (IO per second) from a device. + devices (list): Expose host devices to the container, as a list + of strings in the form + ``::``. + + For example, ``/dev/sda:/dev/xvda:rwm`` allows the container + to have read-write access to the host's ``/dev/sda`` via a + node named ``/dev/xvda`` inside the container. + dns (list): Set custom DNS servers. + dns_opt (list): Additional options to be added to the container's + ``resolv.conf`` file. + dns_search (list): DNS search domains. + domainname (str or list): Set custom DNS search domains. + entrypoint (str or list): The entrypoint for the container. + environment (dict or list): Environment variables to set inside + the container, as a dictionary or a list of strings in the + format ``["SOMEVARIABLE=xxx"]``. + extra_hosts (dict): Addtional hostnames to resolve inside the + container, as a mapping of hostname to IP address. + group_add (list): List of additional group names and/or IDs that + the container process will run as. + hostname (str): Optional hostname for the container. + ipc_mode (str): Set the IPC mode for the container. + isolation (str): Isolation technology to use. Default: `None`. + labels (dict or list): A dictionary of name-value labels (e.g. + ``{"label1": "value1", "label2": "value2"}``) or a list of + names of labels to set with empty values (e.g. + ``["label1", "label2"]``) + links (dict or list of tuples): Either a dictionary mapping name + to alias or as a list of ``(name, alias)`` tuples. + log_config (dict): Logging configuration, as a dictionary with + keys: + + - ``type`` The logging driver name. + - ``config`` A dictionary of configuration for the logging + driver. + + mac_address (str): MAC address to assign to the container. + mem_limit (float or str): Memory limit. Accepts float values + (which represent the memory limit of the created container in + bytes) or a string with a units identification char + (``100000b``, ``1000k``, ``128m``, ``1g``). If a string is + specified without a units character, bytes are assumed as an + intended unit. + mem_limit (str or int): Maximum amount of memory container is + allowed to consume. (e.g. ``1G``). + mem_swappiness (int): Tune a container's memory swappiness + behavior. Accepts number between 0 and 100. + memswap_limit (str or int): Maximum amount of memory + swap a + container is allowed to consume. + networks (list): A list of network names to connect this + container to. + name (str): The name for this container. + network_disabled (bool): Disable networking. + network_mode (str): One of: + + - ``bridge`` Create a new network stack for the container on + on the bridge network. + - ``none`` No networking for this container. + - ``container:`` Reuse another container's network + stack. + - ``host`` Use the host network stack. + oom_kill_disable (bool): Whether to disable OOM killer. + oom_score_adj (int): An integer value containing the score given + to the container in order to tune OOM killer preferences. + pid_mode (str): If set to ``host``, use the host PID namespace + inside the container. + pids_limit (int): Tune a container's pids limit. Set ``-1`` for + unlimited. + ports (dict): Ports to bind inside the container. + + The keys of the dictionary are the ports to bind inside the + container, either as an integer or a string in the form + ``port/protocol``, where the protocol is either ``tcp`` or + ``udp``. + + The values of the dictionary are the corresponding ports to + open on the host, which can be either: + + - The port number, as an integer. For example, + ``{'2222/tcp': 3333}`` will expose port 2222 inside the + container as port 3333 on the host. + - ``None``, to assign a random host port. For example, + ``{'2222/tcp': None}``. + - A tuple of ``(address, port)`` if you want to specify the + host interface. For example, + ``{'1111/tcp': ('127.0.0.1', 1111)}``. + - A list of integers, if you want to bind multiple host ports + to a single container port. For example, + ``{'1111/tcp': [1234, 4567]}``. + + privileged (bool): Give extended privileges to this container. + publish_all_ports (bool): Publish all ports to the host. + read_only (bool): Mount the container's root filesystem as read + only. + remove (bool): Remove the container when it has finished running. + Default: ``False``. + restart_policy (dict): Restart the container when it exits. + Configured as a dictionary with keys: + + - ``Name`` One of ``on-failure``, or ``always``. + - ``MaximumRetryCount`` Number of times to restart the + container on failure. + + For example: + ``{"Name": "on-failure", "MaximumRetryCount": 5}`` + + security_opt (list): A list of string values to customize labels + for MLS systems, such as SELinux. + shm_size (str or int): Size of /dev/shm (e.g. ``1G``). + stdin_open (bool): Keep ``STDIN`` open even if not attached. + stdout (bool): Return logs from ``STDOUT`` when ``detach=False``. + Default: ``True``. + stdout (bool): Return logs from ``STDERR`` when ``detach=False``. + Default: ``False``. + stop_signal (str): The stop signal to use to stop the container + (e.g. ``SIGINT``). + sysctls (dict): Kernel parameters to set in the container. + tmpfs (dict): Temporary filesystems to mount, as a dictionary + mapping a path inside the container to options for that path. + + For example: + + .. code-block:: python + + { + '/mnt/vol2': '', + '/mnt/vol1': 'size=3G,uid=1000' + } + + tty (bool): Allocate a pseudo-TTY. + ulimits (list): Ulimits to set inside the container, as a list of + dicts. + user (str or int): Username or UID to run commands as inside the + container. + userns_mode (str): Sets the user namespace mode for the container + when user namespace remapping option is enabled. Supported + values are: ``host`` + volume_driver (str): The name of a volume driver/plugin. + volumes (dict or list): A dictionary to configure volumes mounted + inside the container. The key is either the host path or a + volume name, and the value is a dictionary with the keys: + + - ``bind`` The path to mount the volume inside the container + - ``mode`` Either ``rw`` to mount the volume read/write, or + ``ro`` to mount it read-only. + + For example: + + .. code-block:: python + + {'/home/user1/': {'bind': '/mnt/vol2', 'mode': 'rw'}, + '/var/www': {'bind': '/mnt/vol1', 'mode': 'ro'}} + + volumes_from (list): List of container names or IDs to get + volumes from. + working_dir (str): Path to the working directory. + + Returns: + The container logs, either ``STDOUT``, ``STDERR``, or both, + depending on the value of the ``stdout`` and ``stderr`` arguments. + + If ``detach`` is ``True``, a :py:class:`Container` object is + returned instead. + + Raises: + :py:class:`docker.errors.ContainerError` + If the container exits with a non-zero exit code and + ``detach`` is ``False``. + :py:class:`docker.errors.ImageNotFound` + If the specified image does not exist. + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + if isinstance(image, Image): + image = image.id + detach = kwargs.pop("detach", False) + if detach and remove: + raise RuntimeError("The options 'detach' and 'remove' cannot be " + "used together.") + + try: + container = self.create(image=image, command=command, + detach=detach, **kwargs) + except ImageNotFound: + self.client.images.pull(image) + container = self.create(image=image, command=command, + detach=detach, **kwargs) + + container.start() + + if detach: + return container + + exit_status = container.wait() + if exit_status != 0: + stdout = False + stderr = True + out = container.logs(stdout=stdout, stderr=stderr) + if remove: + container.remove() + if exit_status != 0: + raise ContainerError(container, exit_status, command, image, out) + return out + + def create(self, image, command=None, **kwargs): + """ + Create a container without starting it. Similar to ``docker create``. + + Takes the same arguments as :py:meth:`run`, except for ``stdout``, + ``stderr``, and ``remove``. + + Returns: + A :py:class:`Container` object. + + Raises: + :py:class:`docker.errors.ImageNotFound` + If the specified image does not exist. + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + if isinstance(image, Image): + image = image.id + kwargs['image'] = image + kwargs['command'] = command + kwargs['version'] = self.client.api._version + create_kwargs = _create_container_args(kwargs) + resp = self.client.api.create_container(**create_kwargs) + return self.get(resp['Id']) + + def get(self, container_id): + """ + Get a container by name or ID. + + Args: + container_id (str): Container name or ID. + + Returns: + A :py:class:`Container` object. + + Raises: + :py:class:`docker.errors.NotFound` + If the container does not exist. + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + resp = self.client.api.inspect_container(container_id) + return self.prepare_model(resp) + + def list(self, all=False, before=None, filters=None, limit=-1, since=None): + """ + List containers. Similar to the ``docker ps`` command. + + Args: + all (bool): Show all containers. Only running containers are shown + by default trunc (bool): Truncate output + since (str): Show only containers created since Id or Name, include + non-running ones + before (str): Show only container created before Id or Name, + include non-running ones + limit (int): Show `limit` last created containers, include + non-running ones + filters (dict): Filters to be processed on the image list. + Available filters: + + - `exited` (int): Only containers with specified exit code + - `status` (str): One of ``restarting``, ``running``, + ``paused``, ``exited`` + - `label` (str): format either ``"key"`` or ``"key=value"`` + - `id` (str): The id of the container. + - `name` (str): The name of the container. + - `ancestor` (str): Filter by container ancestor. Format of + ``[:tag]``, ````, or + ````. + - `before` (str): Only containers created before a particular + container. Give the container name or id. + - `since` (str): Only containers created after a particular + container. Give container name or id. + + A comprehensive list can be found in the documentation for + `docker ps + `_. + + Returns: + (list of :py:class:`Container`) + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + resp = self.client.api.containers(all=all, before=before, + filters=filters, limit=limit, + since=since) + return [self.get(r['Id']) for r in resp] + + +# kwargs to copy straight from run to create +RUN_CREATE_KWARGS = [ + 'command', + 'detach', + 'domainname', + 'entrypoint', + 'environment', + 'healthcheck', + 'hostname', + 'image', + 'labels', + 'mac_address', + 'name', + 'network_disabled', + 'stdin_open', + 'stop_signal', + 'tty', + 'user', + 'volume_driver', + 'working_dir', +] + +# kwargs to copy straight from run to host_config +RUN_HOST_CONFIG_KWARGS = [ + 'blkio_weight_device', + 'blkio_weight', + 'cap_add', + 'cap_drop', + 'cgroup_parent', + 'cpu_period', + 'cpu_quota', + 'cpu_shares', + 'cpuset_cpus', + 'device_read_bps', + 'device_read_iops', + 'device_write_bps', + 'device_write_iops', + 'devices', + 'dns_opt', + 'dns_search', + 'dns', + 'extra_hosts', + 'group_add', + 'ipc_mode', + 'isolation', + 'kernel_memory', + 'links', + 'log_config', + 'lxc_conf', + 'mem_limit', + 'mem_reservation', + 'mem_swappiness', + 'memswap_limit', + 'network_mode', + 'oom_kill_disable', + 'oom_score_adj', + 'pid_mode', + 'pids_limit', + 'privileged', + 'publish_all_ports', + 'read_only', + 'restart_policy', + 'security_opt', + 'shm_size', + 'sysctls', + 'tmpfs', + 'ulimits', + 'userns_mode', + 'version', + 'volumes_from', +] + + +def _create_container_args(kwargs): + """ + Convert arguments to create() to arguments to create_container(). + """ + # Copy over kwargs which can be copied directly + create_kwargs = {} + for key in copy.copy(kwargs): + if key in RUN_CREATE_KWARGS: + create_kwargs[key] = kwargs.pop(key) + host_config_kwargs = {} + for key in copy.copy(kwargs): + if key in RUN_HOST_CONFIG_KWARGS: + host_config_kwargs[key] = kwargs.pop(key) + + # Process kwargs which are split over both create and host_config + ports = kwargs.pop('ports', {}) + if ports: + host_config_kwargs['port_bindings'] = ports + + volumes = kwargs.pop('volumes', {}) + if volumes: + host_config_kwargs['binds'] = volumes + + networks = kwargs.pop('networks', []) + if networks: + create_kwargs['networking_config'] = {network: None + for network in networks} + + # All kwargs should have been consumed by this point, so raise + # error if any are left + if kwargs: + raise create_unexpected_kwargs_error('run', kwargs) + + create_kwargs['host_config'] = create_host_config(**host_config_kwargs) + + # Fill in any kwargs which need processing by create_host_config first + port_bindings = create_kwargs['host_config'].get('PortBindings') + if port_bindings: + # sort to make consistent for tests + create_kwargs['ports'] = [tuple(p.split('/', 1)) + for p in sorted(port_bindings.keys())] + binds = create_kwargs['host_config'].get('Binds') + if binds: + create_kwargs['volumes'] = [v.split(':')[0] for v in binds] + return create_kwargs diff --git a/docker/models/images.py b/docker/models/images.py new file mode 100644 index 0000000000..e0ff1f42c4 --- /dev/null +++ b/docker/models/images.py @@ -0,0 +1,269 @@ +import re + +import six + +from ..api import APIClient +from ..errors import BuildError +from ..utils.json_stream import json_stream +from .resource import Collection, Model + + +class Image(Model): + """ + An image on the server. + """ + def __repr__(self): + return "<%s: '%s'>" % (self.__class__.__name__, "', '".join(self.tags)) + + @property + def short_id(self): + """ + The ID of the image truncated to 10 characters, plus the ``sha256:`` + prefix. + """ + if self.id.startswith('sha256:'): + return self.id[:17] + return self.id[:10] + + @property + def tags(self): + """ + The image's tags. + """ + return [ + tag for tag in self.attrs.get('RepoTags', []) + if tag != ':' + ] + + def history(self): + """ + Show the history of an image. + + Returns: + (str): The history of the image. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + return self.client.api.history(self.id) + + def save(self): + """ + Get a tarball of an image. Similar to the ``docker save`` command. + + Returns: + (urllib3.response.HTTPResponse object): The response from the + daemon. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + + Example: + + >>> image = cli.get("fedora:latest") + >>> resp = image.save() + >>> f = open('/tmp/fedora-latest.tar', 'w') + >>> f.write(resp.data) + >>> f.close() + """ + return self.client.api.get_image(self.id) + + def tag(self, repository, tag=None, **kwargs): + """ + Tag this image into a repository. Similar to the ``docker tag`` + command. + + Args: + repository (str): The repository to set for the tag + tag (str): The tag name + force (bool): Force + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + + Returns: + (bool): ``True`` if successful + """ + self.client.api.tag(self.id, repository, tag=tag, **kwargs) + + +class ImageCollection(Collection): + model = Image + + def build(self, **kwargs): + """ + Build an image and return it. Similar to the ``docker build`` + command. Either ``path`` or ``fileobj`` must be set. + + If you have a tar file for the Docker build context (including a + Dockerfile) already, pass a readable file-like object to ``fileobj`` + and also pass ``custom_context=True``. If the stream is compressed + also, set ``encoding`` to the correct value (e.g ``gzip``). + + If you want to get the raw output of the build, use the + :py:meth:`~docker.api.build.BuildApiMixin.build` method in the + low-level API. + + Args: + path (str): Path to the directory containing the Dockerfile + fileobj: A file object to use as the Dockerfile. (Or a file-like + object) + tag (str): A tag to add to the final image + quiet (bool): Whether to return the status + nocache (bool): Don't use the cache when set to ``True`` + rm (bool): Remove intermediate containers. The ``docker build`` + command now defaults to ``--rm=true``, but we have kept the old + default of `False` to preserve backward compatibility + stream (bool): *Deprecated for API version > 1.8 (always True)*. + Return a blocking generator you can iterate over to retrieve + build output as it happens + timeout (int): HTTP timeout + custom_context (bool): Optional if using ``fileobj`` + encoding (str): The encoding for a stream. Set to ``gzip`` for + compressing + pull (bool): Downloads any updates to the FROM image in Dockerfiles + forcerm (bool): Always remove intermediate containers, even after + unsuccessful builds + dockerfile (str): path within the build context to the Dockerfile + buildargs (dict): A dictionary of build arguments + container_limits (dict): A dictionary of limits applied to each + container created by the build process. Valid keys: + + - memory (int): set memory limit for build + - memswap (int): Total memory (memory + swap), -1 to disable + swap + - cpushares (int): CPU shares (relative weight) + - cpusetcpus (str): CPUs in which to allow execution, e.g., + ``"0-3"``, ``"0,1"`` + decode (bool): If set to ``True``, the returned stream will be + decoded into dicts on the fly. Default ``False``. + + Returns: + (:py:class:`Image`): The built image. + + Raises: + :py:class:`docker.errors.BuildError` + If there is an error during the build. + :py:class:`docker.errors.APIError` + If the server returns any other error. + ``TypeError`` + If neither ``path`` nor ``fileobj`` is specified. + """ + resp = self.client.api.build(**kwargs) + if isinstance(resp, six.string_types): + return self.get(resp) + events = list(json_stream(resp)) + if not events: + return BuildError('Unknown') + event = events[-1] + if 'stream' in event: + match = re.search(r'Successfully built ([0-9a-f]+)', + event.get('stream', '')) + if match: + image_id = match.group(1) + return self.get(image_id) + + raise BuildError(event.get('error') or event) + + def get(self, name): + """ + Gets an image. + + Args: + name (str): The name of the image. + + Returns: + (:py:class:`Image`): The image. + + Raises: + :py:class:`docker.errors.ImageNotFound` If the image does not + exist. + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + return self.prepare_model(self.client.api.inspect_image(name)) + + def list(self, name=None, all=False, filters=None): + """ + List images on the server. + + Args: + name (str): Only show images belonging to the repository ``name`` + all (bool): Show intermediate image layers. By default, these are + filtered out. + filters (dict): Filters to be processed on the image list. + Available filters: + - ``dangling`` (bool) + - ``label`` (str): format either ``key`` or ``key=value`` + + Returns: + (list of :py:class:`Image`): The images. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + resp = self.client.api.images(name=name, all=all, filters=filters) + return [self.prepare_model(r) for r in resp] + + def load(self, data): + """ + Load an image that was previously saved using + :py:meth:`~docker.models.images.Image.save` (or ``docker save``). + Similar to ``docker load``. + + Args: + data (binary): Image data to be loaded. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + return self.client.api.load_image(data) + + def pull(self, name, **kwargs): + """ + Pull an image of the given name and return it. Similar to the + ``docker pull`` command. + + If you want to get the raw pull output, use the + :py:meth:`~docker.api.image.ImageApiMixin.pull` method in the + low-level API. + + Args: + repository (str): The repository to pull + tag (str): The tag to pull + insecure_registry (bool): Use an insecure registry + auth_config (dict): Override the credentials that + :py:meth:`~docker.client.Client.login` has set for + this request. ``auth_config`` should contain the ``username`` + and ``password`` keys to be valid. + + Returns: + (:py:class:`Image`): The image that has been pulled. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + + Example: + + >>> image = client.images.pull('busybox') + """ + self.client.api.pull(name, **kwargs) + return self.get(name) + + def push(self, repository, tag=None, **kwargs): + return self.client.api.push(repository, tag=tag, **kwargs) + push.__doc__ = APIClient.push.__doc__ + + def remove(self, *args, **kwargs): + self.client.api.remove_image(*args, **kwargs) + remove.__doc__ = APIClient.remove_image.__doc__ + + def search(self, *args, **kwargs): + return self.client.api.search(*args, **kwargs) + search.__doc__ = APIClient.search.__doc__ diff --git a/docker/models/networks.py b/docker/models/networks.py new file mode 100644 index 0000000000..64af9ad9aa --- /dev/null +++ b/docker/models/networks.py @@ -0,0 +1,181 @@ +from .containers import Container +from .resource import Model, Collection + + +class Network(Model): + """ + A Docker network. + """ + @property + def name(self): + """ + The name of the network. + """ + return self.attrs.get('Name') + + @property + def containers(self): + """ + The containers that are connected to the network, as a list of + :py:class:`~docker.models.containers.Container` objects. + """ + return [ + self.client.containers.get(cid) for cid in + self.attrs.get('Containers', {}).keys() + ] + + def connect(self, container): + """ + Connect a container to this network. + + Args: + container (str): Container to connect to this network, as either + an ID, name, or :py:class:`~docker.models.containers.Container` + object. + aliases (list): A list of aliases for this endpoint. Names in that + list can be used within the network to reach the container. + Defaults to ``None``. + links (list): A list of links for this endpoint. Containers + declared in this list will be linkedto this container. + Defaults to ``None``. + ipv4_address (str): The IP address of this container on the + network, using the IPv4 protocol. Defaults to ``None``. + ipv6_address (str): The IP address of this container on the + network, using the IPv6 protocol. Defaults to ``None``. + link_local_ips (list): A list of link-local (IPv4/IPv6) addresses. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + if isinstance(container, Container): + container = container.id + return self.client.api.connect_container_to_network(container, self.id) + + def disconnect(self, container): + """ + Disconnect a container from this network. + + Args: + container (str): Container to disconnect from this network, as + either an ID, name, or + :py:class:`~docker.models.containers.Container` object. + force (bool): Force the container to disconnect from a network. + Default: ``False`` + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + if isinstance(container, Container): + container = container.id + return self.client.api.disconnect_container_from_network(container, + self.id) + + def remove(self): + """ + Remove this network. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + return self.client.api.remove_network(self.id) + + +class NetworkCollection(Collection): + """ + Networks on the Docker server. + """ + model = Network + + def create(self, name, *args, **kwargs): + """ + Create a network. Similar to the ``docker network create``. + + Args: + name (str): Name of the network + driver (str): Name of the driver used to create the network + options (dict): Driver options as a key-value dictionary + ipam (dict): Optional custom IP scheme for the network. + Created with :py:meth:`~docker.utils.create_ipam_config`. + check_duplicate (bool): Request daemon to check for networks with + same name. Default: ``True``. + internal (bool): Restrict external access to the network. Default + ``False``. + labels (dict): Map of labels to set on the network. Default + ``None``. + enable_ipv6 (bool): Enable IPv6 on the network. Default ``False``. + + Returns: + (:py:class:`Network`): The network that was created. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + + Example: + A network using the bridge driver: + + >>> client.networks.create("network1", driver="bridge") + + You can also create more advanced networks with custom IPAM + configurations. For example, setting the subnet to + ``192.168.52.0/24`` and gateway address to ``192.168.52.254``. + + .. code-block:: python + + >>> ipam_pool = docker.utils.create_ipam_pool( + subnet='192.168.52.0/24', + gateway='192.168.52.254' + ) + >>> ipam_config = docker.utils.create_ipam_config( + pool_configs=[ipam_pool] + ) + >>> client.networks.create( + "network1", + driver="bridge", + ipam=ipam_config + ) + + """ + resp = self.client.api.create_network(name, *args, **kwargs) + return self.get(resp['Id']) + + def get(self, network_id): + """ + Get a network by its ID. + + Args: + network_id (str): The ID of the network. + + Returns: + (:py:class:`Network`) The network. + + Raises: + :py:class:`docker.errors.NotFound` + If the network does not exist. + + :py:class:`docker.errors.APIError` + If the server returns an error. + + """ + return self.prepare_model(self.client.api.inspect_network(network_id)) + + def list(self, *args, **kwargs): + """ + List networks. Similar to the ``docker networks ls`` command. + + Args: + names (list): List of names to filter by. + ids (list): List of ids to filter by. + + Returns: + (list of :py:class:`Network`) The networks on the server. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + resp = self.client.api.networks(*args, **kwargs) + return [self.prepare_model(item) for item in resp] diff --git a/docker/models/nodes.py b/docker/models/nodes.py new file mode 100644 index 0000000000..0887f99c24 --- /dev/null +++ b/docker/models/nodes.py @@ -0,0 +1,88 @@ +from .resource import Model, Collection + + +class Node(Model): + """A node in a swarm.""" + id_attribute = 'ID' + + @property + def version(self): + """ + The version number of the service. If this is not the same as the + server, the :py:meth:`update` function will not work and you will + need to call :py:meth:`reload` before calling it again. + """ + return self.attrs.get('Version').get('Index') + + def update(self, node_spec): + """ + Update the node's configuration. + + Args: + node_spec (dict): Configuration settings to update. Any values + not provided will be removed. Default: ``None`` + + Returns: + `True` if the request went through. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + + Example: + + >>> node_spec = {'Availability': 'active', + 'Name': 'node-name', + 'Role': 'manager', + 'Labels': {'foo': 'bar'} + } + >>> node.update(node_spec) + + """ + return self.client.api.update_node(self.id, self.version, node_spec) + + +class NodeCollection(Collection): + """Nodes on the Docker server.""" + model = Node + + def get(self, node_id): + """ + Get a node. + + Args: + node_id (string): ID of the node to be inspected. + + Returns: + A :py:class:`Node` object. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + return self.prepare_model(self.client.api.inspect_node(node_id)) + + def list(self, *args, **kwargs): + """ + List swarm nodes. + + Args: + filters (dict): Filters to process on the nodes list. Valid + filters: ``id``, ``name``, ``membership`` and ``role``. + Default: ``None`` + + Returns: + A list of :py:class:`Node` objects. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + + Example: + + >>> client.nodes.list(filters={'role': 'manager'}) + """ + return [ + self.prepare_model(n) + for n in self.client.api.nodes(*args, **kwargs) + ] diff --git a/docker/models/resource.py b/docker/models/resource.py new file mode 100644 index 0000000000..9634a24f33 --- /dev/null +++ b/docker/models/resource.py @@ -0,0 +1,84 @@ + +class Model(object): + """ + A base class for representing a single object on the server. + """ + id_attribute = 'Id' + + def __init__(self, attrs=None, client=None, collection=None): + #: A client pointing at the server that this object is on. + self.client = client + + #: The collection that this model is part of. + self.collection = collection + + #: The raw representation of this object from the API + self.attrs = attrs + if self.attrs is None: + self.attrs = {} + + def __repr__(self): + return "<%s: %s>" % (self.__class__.__name__, self.short_id) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.id == other.id + + @property + def id(self): + """ + The ID of the object. + """ + return self.attrs.get(self.id_attribute) + + @property + def short_id(self): + """ + The ID of the object, truncated to 10 characters. + """ + return self.id[:10] + + def reload(self): + """ + Load this object from the server again and update ``attrs`` with the + new data. + """ + new_model = self.collection.get(self.id) + self.attrs = new_model.attrs + + +class Collection(object): + """ + A base class for representing all objects of a particular type on the + server. + """ + + #: The type of object this collection represents, set by subclasses + model = None + + def __init__(self, client=None): + #: The client pointing at the server that this collection of objects + #: is on. + self.client = client + + def list(self): + raise NotImplementedError + + def get(self, key): + raise NotImplementedError + + def create(self, attrs=None): + raise NotImplementedError + + def prepare_model(self, attrs): + """ + Create a model from a set of attributes. + """ + if isinstance(attrs, Model): + attrs.client = self.client + attrs.collection = self + return attrs + elif isinstance(attrs, dict): + return self.model(attrs=attrs, client=self.client, collection=self) + else: + raise Exception("Can't create %s from %s" % + (self.model.__name__, attrs)) diff --git a/docker/models/services.py b/docker/models/services.py new file mode 100644 index 0000000000..d70c9e7a08 --- /dev/null +++ b/docker/models/services.py @@ -0,0 +1,240 @@ +import copy +from docker.errors import create_unexpected_kwargs_error +from docker.types import TaskTemplate, ContainerSpec +from .resource import Model, Collection + + +class Service(Model): + """A service.""" + id_attribute = 'ID' + + @property + def name(self): + """The service's name.""" + return self.attrs['Spec']['Name'] + + @property + def version(self): + """ + The version number of the service. If this is not the same as the + server, the :py:meth:`update` function will not work and you will + need to call :py:meth:`reload` before calling it again. + """ + return self.attrs.get('Version').get('Index') + + def remove(self): + """ + Stop and remove the service. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + return self.client.api.remove_service(self.id) + + def tasks(self, filters=None): + """ + List the tasks in this service. + + Args: + filters (dict): A map of filters to process on the tasks list. + Valid filters: ``id``, ``name``, ``node``, + ``label``, and ``desired-state``. + + Returns: + (list): List of task dictionaries. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + if filters is None: + filters = {} + filters['service'] = self.id + return self.client.api.tasks(filters=filters) + + def update(self, **kwargs): + """ + Update a service's configuration. Similar to the ``docker service + update`` command. + + Takes the same parameters as :py:meth:`~ServiceCollection.create`. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + # Image is required, so if it hasn't been set, use current image + if 'image' not in kwargs: + spec = self.attrs['Spec']['TaskTemplate']['ContainerSpec'] + kwargs['image'] = spec['Image'] + + create_kwargs = _get_create_service_kwargs('update', kwargs) + + return self.client.api.update_service( + self.id, + self.version, + **create_kwargs + ) + + +class ServiceCollection(Collection): + """Services on the Docker server.""" + model = Service + + def create(self, image, command=None, **kwargs): + """ + Create a service. Similar to the ``docker service create`` command. + + Args: + image (str): The image name to use for the containers. + command (list of str or str): Command to run. + args (list of str): Arguments to the command. + constraints (list of str): Placement constraints. + container_labels (dict): Labels to apply to the container. + endpoint_spec (dict): Properties that can be configured to + access and load balance a service. Default: ``None``. + env (list of str): Environment variables, in the form + ``KEY=val``. + labels (dict): Labels to apply to the service. + log_driver (str): Log driver to use for containers. + log_driver_options (dict): Log driver options. + mode (string): Scheduling mode for the service (``replicated`` or + ``global``). Defaults to ``replicated``. + mounts (list of str): Mounts for the containers, in the form + ``source:target:options``, where options is either + ``ro`` or ``rw``. + name (str): Name to give to the service. + networks (list): List of network names or IDs to attach the + service to. Default: ``None``. + resources (dict): Resource limits and reservations. For the + format, see the Remote API documentation. + restart_policy (dict): Restart policy for containers. For the + format, see the Remote API documentation. + stop_grace_period (int): Amount of time to wait for + containers to terminate before forcefully killing them. + update_config (dict): Specification for the update strategy of the + service. Default: ``None`` + user (str): User to run commands as. + workdir (str): Working directory for commands to run. + + Returns: + (:py:class:`Service`) The created service. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + kwargs['image'] = image + kwargs['command'] = command + create_kwargs = _get_create_service_kwargs('create', kwargs) + service_id = self.client.api.create_service(**create_kwargs) + return self.get(service_id) + + def get(self, service_id): + """ + Get a service. + + Args: + service_id (str): The ID of the service. + + Returns: + (:py:class:`Service`): The service. + + Raises: + :py:class:`docker.errors.NotFound` + If the service does not exist. + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + return self.prepare_model(self.client.api.inspect_service(service_id)) + + def list(self, **kwargs): + """ + List services. + + Args: + filters (dict): Filters to process on the nodes list. Valid + filters: ``id`` and ``name``. Default: ``None``. + + Returns: + (list of :py:class:`Service`): The services. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + return [ + self.prepare_model(s) + for s in self.client.api.services(**kwargs) + ] + + +# kwargs to copy straight over to ContainerSpec +CONTAINER_SPEC_KWARGS = [ + 'image', + 'command', + 'args', + 'env', + 'workdir', + 'user', + 'labels', + 'mounts', + 'stop_grace_period', +] + +# kwargs to copy straight over to TaskTemplate +TASK_TEMPLATE_KWARGS = [ + 'resources', + 'restart_policy', +] + +# kwargs to copy straight over to create_service +CREATE_SERVICE_KWARGS = [ + 'name', + 'labels', + 'mode', + 'update_config', + 'networks', + 'endpoint_spec', +] + + +def _get_create_service_kwargs(func_name, kwargs): + # Copy over things which can be copied directly + create_kwargs = {} + for key in copy.copy(kwargs): + if key in CREATE_SERVICE_KWARGS: + create_kwargs[key] = kwargs.pop(key) + container_spec_kwargs = {} + for key in copy.copy(kwargs): + if key in CONTAINER_SPEC_KWARGS: + container_spec_kwargs[key] = kwargs.pop(key) + task_template_kwargs = {} + for key in copy.copy(kwargs): + if key in TASK_TEMPLATE_KWARGS: + task_template_kwargs[key] = kwargs.pop(key) + + if 'container_labels' in kwargs: + container_spec_kwargs['labels'] = kwargs.pop('container_labels') + + if 'constraints' in kwargs: + task_template_kwargs['placement'] = { + 'Constraints': kwargs.pop('constraints') + } + + if 'log_driver' in kwargs: + task_template_kwargs['log_driver'] = { + 'Name': kwargs.pop('log_driver'), + 'Options': kwargs.pop('log_driver_options', {}) + } + + # All kwargs should have been consumed by this point, so raise + # error if any are left + if kwargs: + raise create_unexpected_kwargs_error(func_name, kwargs) + + container_spec = ContainerSpec(**container_spec_kwargs) + task_template_kwargs['container_spec'] = container_spec + create_kwargs['task_template'] = TaskTemplate(**task_template_kwargs) + return create_kwargs diff --git a/docker/models/swarm.py b/docker/models/swarm.py new file mode 100644 index 0000000000..38c1e9f9c2 --- /dev/null +++ b/docker/models/swarm.py @@ -0,0 +1,146 @@ +from docker.api import APIClient +from docker.errors import APIError +from docker.types import SwarmSpec +from .resource import Model + + +class Swarm(Model): + """ + The server's Swarm state. This a singleton that must be reloaded to get + the current state of the Swarm. + """ + def __init__(self, *args, **kwargs): + super(Swarm, self).__init__(*args, **kwargs) + if self.client: + try: + self.reload() + except APIError as e: + if e.response.status_code != 406: + raise + + @property + def version(self): + """ + The version number of the swarm. If this is not the same as the + server, the :py:meth:`update` function will not work and you will + need to call :py:meth:`reload` before calling it again. + """ + return self.attrs.get('Version').get('Index') + + def init(self, advertise_addr=None, listen_addr='0.0.0.0:2377', + force_new_cluster=False, swarm_spec=None, **kwargs): + """ + Initialize a new swarm on this Engine. + + Args: + advertise_addr (str): Externally reachable address advertised to + other nodes. This can either be an address/port combination in + the form ``192.168.1.1:4567``, or an interface followed by a + port number, like ``eth0:4567``. If the port number is omitted, + the port number from the listen address is used. + + If not specified, it will be automatically detected when + possible. + listen_addr (str): Listen address used for inter-manager + communication, as well as determining the networking interface + used for the VXLAN Tunnel Endpoint (VTEP). This can either be + an address/port combination in the form ``192.168.1.1:4567``, + or an interface followed by a port number, like ``eth0:4567``. + If the port number is omitted, the default swarm listening port + is used. Default: ``0.0.0.0:2377`` + force_new_cluster (bool): Force creating a new Swarm, even if + already part of one. Default: False + task_history_retention_limit (int): Maximum number of tasks + history stored. + snapshot_interval (int): Number of logs entries between snapshot. + keep_old_snapshots (int): Number of snapshots to keep beyond the + current snapshot. + log_entries_for_slow_followers (int): Number of log entries to + keep around to sync up slow followers after a snapshot is + created. + heartbeat_tick (int): Amount of ticks (in seconds) between each + heartbeat. + election_tick (int): Amount of ticks (in seconds) needed without a + leader to trigger a new election. + dispatcher_heartbeat_period (int): The delay for an agent to send + a heartbeat to the dispatcher. + node_cert_expiry (int): Automatic expiry for nodes certificates. + external_ca (dict): Configuration for forwarding signing requests + to an external certificate authority. Use + ``docker.types.SwarmExternalCA``. + name (string): Swarm's name + + Returns: + ``True`` if the request went through. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + + Example: + + >>> client.swarm.init( + advertise_addr='eth0', listen_addr='0.0.0.0:5000', + force_new_cluster=False, snapshot_interval=5000, + log_entries_for_slow_followers=1200 + ) + + """ + init_kwargs = {} + for arg in ['advertise_addr', 'listen_addr', 'force_new_cluster']: + if arg in kwargs: + init_kwargs[arg] = kwargs[arg] + del kwargs[arg] + init_kwargs['swarm_spec'] = SwarmSpec(**kwargs) + self.client.api.init_swarm(**init_kwargs) + self.reload() + + def join(self, *args, **kwargs): + return self.client.api.join_swarm(*args, **kwargs) + join.__doc__ = APIClient.join_swarm.__doc__ + + def leave(self, *args, **kwargs): + return self.client.api.leave_swarm(*args, **kwargs) + leave.__doc__ = APIClient.leave_swarm.__doc__ + + def reload(self): + """ + Inspect the swarm on the server and store the response in + :py:attr:`attrs`. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + self.attrs = self.client.api.inspect_swarm() + + def update(self, rotate_worker_token=False, rotate_manager_token=False, + **kwargs): + """ + Update the swarm's configuration. + + It takes the same arguments as :py:meth:`init`, except + ``advertise_addr``, ``listen_addr``, and ``force_new_cluster``. In + addition, it takes these arguments: + + Args: + rotate_worker_token (bool): Rotate the worker join token. Default: + ``False``. + rotate_manager_token (bool): Rotate the manager join token. + Default: ``False``. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + + """ + # this seems to have to be set + if kwargs.get('node_cert_expiry') is None: + kwargs['node_cert_expiry'] = 7776000000000000 + + return self.client.api.update_swarm( + version=self.version, + swarm_spec=SwarmSpec(**kwargs), + rotate_worker_token=rotate_worker_token, + rotate_manager_token=rotate_manager_token + ) diff --git a/docker/models/volumes.py b/docker/models/volumes.py new file mode 100644 index 0000000000..5a31541260 --- /dev/null +++ b/docker/models/volumes.py @@ -0,0 +1,84 @@ +from .resource import Model, Collection + + +class Volume(Model): + """A volume.""" + id_attribute = 'Name' + + @property + def name(self): + """The name of the volume.""" + return self.attrs['Name'] + + def remove(self): + """Remove this volume.""" + return self.client.api.remove_volume(self.id) + + +class VolumeCollection(Collection): + """Volumes on the Docker server.""" + model = Volume + + def create(self, name, **kwargs): + """ + Create a volume. + + Args: + name (str): Name of the volume + driver (str): Name of the driver used to create the volume + driver_opts (dict): Driver options as a key-value dictionary + labels (dict): Labels to set on the volume + + Returns: + (:py:class:`Volume`): The volume created. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + + Example: + + >>> volume = client.volumes.create(name='foobar', driver='local', + driver_opts={'foo': 'bar', 'baz': 'false'}, + labels={"key": "value"}) + + """ + obj = self.client.api.create_volume(name, **kwargs) + return self.prepare_model(obj) + + def get(self, volume_id): + """ + Get a volume. + + Args: + volume_id (str): Volume name. + + Returns: + (:py:class:`Volume`): The volume. + + Raises: + :py:class:`docker.errors.NotFound` + If the volume does not exist. + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + return self.prepare_model(self.client.api.inspect_volume(volume_id)) + + def list(self, **kwargs): + """ + List volumes. Similar to the ``docker volume ls`` command. + + Args: + filters (dict): Server-side list filtering options. + + Returns: + (list of :py:class:`Volume`): The volumes. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + resp = self.client.api.volumes(**kwargs) + if not resp.get('Volumes'): + return [] + return [self.prepare_model(obj) for obj in resp['Volumes']] diff --git a/docker/utils/json_stream.py b/docker/utils/json_stream.py new file mode 100644 index 0000000000..f97ab9e296 --- /dev/null +++ b/docker/utils/json_stream.py @@ -0,0 +1,79 @@ +from __future__ import absolute_import +from __future__ import unicode_literals + +import json +import json.decoder + +import six + +from ..errors import StreamParseError + + +json_decoder = json.JSONDecoder() + + +def stream_as_text(stream): + """Given a stream of bytes or text, if any of the items in the stream + are bytes convert them to text. + This function can be removed once docker-py returns text streams instead + of byte streams. + """ + for data in stream: + if not isinstance(data, six.text_type): + data = data.decode('utf-8', 'replace') + yield data + + +def json_splitter(buffer): + """Attempt to parse a json object from a buffer. If there is at least one + object, return it and the rest of the buffer, otherwise return None. + """ + buffer = buffer.strip() + try: + obj, index = json_decoder.raw_decode(buffer) + rest = buffer[json.decoder.WHITESPACE.match(buffer, index).end():] + return obj, rest + except ValueError: + return None + + +def json_stream(stream): + """Given a stream of text, return a stream of json objects. + This handles streams which are inconsistently buffered (some entries may + be newline delimited, and others are not). + """ + return split_buffer(stream, json_splitter, json_decoder.decode) + + +def line_splitter(buffer, separator=u'\n'): + index = buffer.find(six.text_type(separator)) + if index == -1: + return None + return buffer[:index + 1], buffer[index + 1:] + + +def split_buffer(stream, splitter=None, decoder=lambda a: a): + """Given a generator which yields strings and a splitter function, + joins all input, splits on the separator and yields each chunk. + Unlike string.split(), each chunk includes the trailing + separator, except for the last one if none was found on the end + of the input. + """ + splitter = splitter or line_splitter + buffered = six.text_type('') + + for data in stream_as_text(stream): + buffered += data + while True: + buffer_split = splitter(buffered) + if buffer_split is None: + break + + item, buffered = buffer_split + yield item + + if buffered: + try: + yield decoder(buffered) + except Exception as e: + raise StreamParseError(e) diff --git a/tests/helpers.py b/tests/helpers.py index 09fb653222..1d24577a67 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -61,3 +61,16 @@ def wait_on_condition(condition, delay=0.1, timeout=40): def random_name(): return u'dockerpytest_{0:x}'.format(random.getrandbits(64)) + + +def force_leave_swarm(client): + """Actually force leave a Swarm. There seems to be a bug in Swarm that + occasionally throws "context deadline exceeded" errors when leaving.""" + while True: + try: + return client.swarm.leave(force=True) + except docker.errors.APIError as e: + if e.explanation == "context deadline exceeded": + continue + else: + raise diff --git a/tests/integration/client_test.py b/tests/integration/client_test.py new file mode 100644 index 0000000000..dfced9b66f --- /dev/null +++ b/tests/integration/client_test.py @@ -0,0 +1,20 @@ +import unittest + +import docker + + +class ClientTest(unittest.TestCase): + + def test_info(self): + client = docker.from_env() + info = client.info() + assert 'ID' in info + assert 'Name' in info + + def test_ping(self): + client = docker.from_env() + assert client.ping() is True + + def test_version(self): + client = docker.from_env() + assert 'Version' in client.version() diff --git a/tests/integration/models_containers_test.py b/tests/integration/models_containers_test.py new file mode 100644 index 0000000000..d8b4c62c35 --- /dev/null +++ b/tests/integration/models_containers_test.py @@ -0,0 +1,204 @@ +import docker +from .base import BaseIntegrationTest + + +class ContainerCollectionTest(BaseIntegrationTest): + + def test_run(self): + client = docker.from_env() + self.assertEqual( + client.containers.run("alpine", "echo hello world", remove=True), + b'hello world\n' + ) + + def test_run_detach(self): + client = docker.from_env() + container = client.containers.run("alpine", "sleep 300", detach=True) + self.tmp_containers.append(container.id) + assert container.attrs['Config']['Image'] == "alpine" + assert container.attrs['Config']['Cmd'] == ['sleep', '300'] + + def test_run_with_error(self): + client = docker.from_env() + with self.assertRaises(docker.errors.ContainerError) as cm: + client.containers.run("alpine", "cat /test", remove=True) + assert cm.exception.exit_status == 1 + assert "cat /test" in str(cm.exception) + assert "alpine" in str(cm.exception) + assert "No such file or directory" in str(cm.exception) + + def test_run_with_image_that_does_not_exist(self): + client = docker.from_env() + with self.assertRaises(docker.errors.ImageNotFound): + client.containers.run("dockerpytest_does_not_exist") + + def test_get(self): + client = docker.from_env() + container = client.containers.run("alpine", "sleep 300", detach=True) + self.tmp_containers.append(container.id) + assert client.containers.get(container.id).attrs[ + 'Config']['Image'] == "alpine" + + def test_list(self): + client = docker.from_env() + container_id = client.containers.run( + "alpine", "sleep 300", detach=True).id + self.tmp_containers.append(container_id) + containers = [c for c in client.containers.list() if c.id == + container_id] + assert len(containers) == 1 + + container = containers[0] + assert container.attrs['Config']['Image'] == 'alpine' + + container.kill() + container.remove() + assert container_id not in [c.id for c in client.containers.list()] + + +class ContainerTest(BaseIntegrationTest): + + def test_commit(self): + client = docker.from_env() + container = client.containers.run( + "alpine", "sh -c 'echo \"hello\" > /test'", + detach=True + ) + self.tmp_containers.append(container.id) + container.wait() + image = container.commit() + self.assertEqual( + client.containers.run(image.id, "cat /test", remove=True), + b"hello\n" + ) + + def test_diff(self): + client = docker.from_env() + container = client.containers.run("alpine", "touch /test", detach=True) + self.tmp_containers.append(container.id) + container.wait() + assert container.diff() == [{'Path': '/test', 'Kind': 1}] + + def test_exec_run(self): + client = docker.from_env() + container = client.containers.run( + "alpine", "sh -c 'echo \"hello\" > /test; sleep 60'", detach=True + ) + self.tmp_containers.append(container.id) + assert container.exec_run("cat /test") == b"hello\n" + + def test_kill(self): + client = docker.from_env() + container = client.containers.run("alpine", "sleep 300", detach=True) + self.tmp_containers.append(container.id) + while container.status != 'running': + container.reload() + assert container.status == 'running' + container.kill() + container.reload() + assert container.status == 'exited' + + def test_logs(self): + client = docker.from_env() + container = client.containers.run("alpine", "echo hello world", + detach=True) + self.tmp_containers.append(container.id) + container.wait() + assert container.logs() == b"hello world\n" + + def test_pause(self): + client = docker.from_env() + container = client.containers.run("alpine", "sleep 300", detach=True) + self.tmp_containers.append(container.id) + container.pause() + container.reload() + assert container.status == "paused" + container.unpause() + container.reload() + assert container.status == "running" + + def test_remove(self): + client = docker.from_env() + container = client.containers.run("alpine", "echo hello", detach=True) + self.tmp_containers.append(container.id) + assert container.id in [c.id for c in client.containers.list(all=True)] + container.wait() + container.remove() + containers = client.containers.list(all=True) + assert container.id not in [c.id for c in containers] + + def test_rename(self): + client = docker.from_env() + container = client.containers.run("alpine", "echo hello", name="test1", + detach=True) + self.tmp_containers.append(container.id) + assert container.name == "test1" + container.rename("test2") + container.reload() + assert container.name == "test2" + + def test_restart(self): + client = docker.from_env() + container = client.containers.run("alpine", "sleep 100", detach=True) + self.tmp_containers.append(container.id) + first_started_at = container.attrs['State']['StartedAt'] + container.restart() + container.reload() + second_started_at = container.attrs['State']['StartedAt'] + assert first_started_at != second_started_at + + def test_start(self): + client = docker.from_env() + container = client.containers.create("alpine", "sleep 50", detach=True) + self.tmp_containers.append(container.id) + assert container.status == "created" + container.start() + container.reload() + assert container.status == "running" + + def test_stats(self): + client = docker.from_env() + container = client.containers.run("alpine", "sleep 100", detach=True) + self.tmp_containers.append(container.id) + stats = container.stats(stream=False) + for key in ['read', 'networks', 'precpu_stats', 'cpu_stats', + 'memory_stats', 'blkio_stats']: + assert key in stats + + def test_stop(self): + client = docker.from_env() + container = client.containers.run("alpine", "top", detach=True) + self.tmp_containers.append(container.id) + assert container.status in ("running", "created") + container.stop(timeout=2) + container.reload() + assert container.status == "exited" + + def test_top(self): + client = docker.from_env() + container = client.containers.run("alpine", "sleep 60", detach=True) + self.tmp_containers.append(container.id) + top = container.top() + assert len(top['Processes']) == 1 + assert 'sleep 60' in top['Processes'][0] + + def test_update(self): + client = docker.from_env() + container = client.containers.run("alpine", "sleep 60", detach=True, + cpu_shares=2) + self.tmp_containers.append(container.id) + assert container.attrs['HostConfig']['CpuShares'] == 2 + container.update(cpu_shares=3) + container.reload() + assert container.attrs['HostConfig']['CpuShares'] == 3 + + def test_wait(self): + client = docker.from_env() + container = client.containers.run("alpine", "sh -c 'exit 0'", + detach=True) + self.tmp_containers.append(container.id) + assert container.wait() == 0 + container = client.containers.run("alpine", "sh -c 'exit 1'", + detach=True) + self.tmp_containers.append(container.id) + assert container.wait() == 1 diff --git a/tests/integration/models_images_test.py b/tests/integration/models_images_test.py new file mode 100644 index 0000000000..2be623252c --- /dev/null +++ b/tests/integration/models_images_test.py @@ -0,0 +1,67 @@ +import io +import docker +from .base import BaseIntegrationTest + + +class ImageCollectionTest(BaseIntegrationTest): + + def test_build(self): + client = docker.from_env() + image = client.images.build(fileobj=io.BytesIO( + "FROM alpine\n" + "CMD echo hello world".encode('ascii') + )) + self.tmp_imgs.append(image.id) + assert client.containers.run(image) == b"hello world\n" + + def test_build_with_error(self): + client = docker.from_env() + with self.assertRaises(docker.errors.BuildError) as cm: + client.images.build(fileobj=io.BytesIO( + "FROM alpine\n" + "NOTADOCKERFILECOMMAND".encode('ascii') + )) + assert str(cm.exception) == ("Unknown instruction: " + "NOTADOCKERFILECOMMAND") + + def test_list(self): + client = docker.from_env() + image = client.images.pull('alpine:latest') + assert image.id in get_ids(client.images.list()) + + def test_list_with_repository(self): + client = docker.from_env() + image = client.images.pull('alpine:latest') + assert image.id in get_ids(client.images.list('alpine')) + assert image.id in get_ids(client.images.list('alpine:latest')) + + def test_pull(self): + client = docker.from_env() + image = client.images.pull('alpine:latest') + assert 'alpine:latest' in image.attrs['RepoTags'] + + +class ImageTest(BaseIntegrationTest): + + def test_tag_and_remove(self): + repo = 'dockersdk.tests.images.test_tag' + tag = 'some-tag' + identifier = '{}:{}'.format(repo, tag) + + client = docker.from_env() + image = client.images.pull('alpine:latest') + + image.tag(repo, tag) + self.tmp_imgs.append(identifier) + assert image.id in get_ids(client.images.list(repo)) + assert image.id in get_ids(client.images.list(identifier)) + + client.images.remove(identifier) + assert image.id not in get_ids(client.images.list(repo)) + assert image.id not in get_ids(client.images.list(identifier)) + + assert image.id in get_ids(client.images.list('alpine:latest')) + + +def get_ids(images): + return [i.id for i in images] diff --git a/tests/integration/models_networks_test.py b/tests/integration/models_networks_test.py new file mode 100644 index 0000000000..771ee7d346 --- /dev/null +++ b/tests/integration/models_networks_test.py @@ -0,0 +1,64 @@ +import docker +from .. import helpers +from .base import BaseIntegrationTest + + +class ImageCollectionTest(BaseIntegrationTest): + + def test_create(self): + client = docker.from_env() + name = helpers.random_name() + network = client.networks.create(name, labels={'foo': 'bar'}) + self.tmp_networks.append(network.id) + assert network.name == name + assert network.attrs['Labels']['foo'] == "bar" + + def test_get(self): + client = docker.from_env() + name = helpers.random_name() + network_id = client.networks.create(name).id + self.tmp_networks.append(network_id) + network = client.networks.get(network_id) + assert network.name == name + + def test_list_remove(self): + client = docker.from_env() + name = helpers.random_name() + network = client.networks.create(name) + self.tmp_networks.append(network.id) + assert network.id in [n.id for n in client.networks.list()] + assert network.id not in [ + n.id for n in + client.networks.list(ids=["fdhjklfdfdshjkfds"]) + ] + assert network.id in [ + n.id for n in + client.networks.list(ids=[network.id]) + ] + assert network.id not in [ + n.id for n in + client.networks.list(names=["fdshjklfdsjhkl"]) + ] + assert network.id in [ + n.id for n in + client.networks.list(names=[name]) + ] + network.remove() + assert network.id not in [n.id for n in client.networks.list()] + + +class ImageTest(BaseIntegrationTest): + + def test_connect_disconnect(self): + client = docker.from_env() + network = client.networks.create(helpers.random_name()) + self.tmp_networks.append(network.id) + container = client.containers.create("alpine", "sleep 300") + self.tmp_containers.append(container.id) + assert network.containers == [] + network.connect(container) + container.start() + assert client.networks.get(network.id).containers == [container] + network.disconnect(container) + assert network.containers == [] + assert client.networks.get(network.id).containers == [] diff --git a/tests/integration/models_nodes_test.py b/tests/integration/models_nodes_test.py new file mode 100644 index 0000000000..0199d69303 --- /dev/null +++ b/tests/integration/models_nodes_test.py @@ -0,0 +1,34 @@ +import unittest +import docker +from .. import helpers + + +class NodesTest(unittest.TestCase): + def setUp(self): + helpers.force_leave_swarm(docker.from_env()) + + def tearDown(self): + helpers.force_leave_swarm(docker.from_env()) + + def test_list_get_update(self): + client = docker.from_env() + client.swarm.init() + nodes = client.nodes.list() + assert len(nodes) == 1 + assert nodes[0].attrs['Spec']['Role'] == 'manager' + + node = client.nodes.get(nodes[0].id) + assert node.id == nodes[0].id + assert node.attrs['Spec']['Role'] == 'manager' + assert node.version > 0 + + node = client.nodes.list()[0] + assert not node.attrs['Spec'].get('Labels') + node.update({ + 'Availability': 'active', + 'Name': 'node-name', + 'Role': 'manager', + 'Labels': {'foo': 'bar'} + }) + node.reload() + assert node.attrs['Spec']['Labels'] == {'foo': 'bar'} diff --git a/tests/integration/models_resources_test.py b/tests/integration/models_resources_test.py new file mode 100644 index 0000000000..b8eba81c6e --- /dev/null +++ b/tests/integration/models_resources_test.py @@ -0,0 +1,16 @@ +import docker +from .base import BaseIntegrationTest + + +class ModelTest(BaseIntegrationTest): + + def test_reload(self): + client = docker.from_env() + container = client.containers.run("alpine", "sleep 300", detach=True) + self.tmp_containers.append(container.id) + first_started_at = container.attrs['State']['StartedAt'] + container.kill() + container.start() + assert container.attrs['State']['StartedAt'] == first_started_at + container.reload() + assert container.attrs['State']['StartedAt'] != first_started_at diff --git a/tests/integration/models_services_test.py b/tests/integration/models_services_test.py new file mode 100644 index 0000000000..99cffc058b --- /dev/null +++ b/tests/integration/models_services_test.py @@ -0,0 +1,100 @@ +import unittest +import docker +from .. import helpers + + +class ServiceTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + client = docker.from_env() + helpers.force_leave_swarm(client) + client.swarm.init() + + @classmethod + def tearDownClass(cls): + helpers.force_leave_swarm(docker.from_env()) + + def test_create(self): + client = docker.from_env() + name = helpers.random_name() + service = client.services.create( + # create arguments + name=name, + labels={'foo': 'bar'}, + # ContainerSpec arguments + image="alpine", + command="sleep 300", + container_labels={'container': 'label'} + ) + assert service.name == name + assert service.attrs['Spec']['Labels']['foo'] == 'bar' + container_spec = service.attrs['Spec']['TaskTemplate']['ContainerSpec'] + assert container_spec['Image'] == "alpine" + assert container_spec['Labels'] == {'container': 'label'} + + def test_get(self): + client = docker.from_env() + name = helpers.random_name() + service = client.services.create( + name=name, + image="alpine", + command="sleep 300" + ) + service = client.services.get(service.id) + assert service.name == name + + def test_list_remove(self): + client = docker.from_env() + service = client.services.create( + name=helpers.random_name(), + image="alpine", + command="sleep 300" + ) + assert service in client.services.list() + service.remove() + assert service not in client.services.list() + + def test_tasks(self): + client = docker.from_env() + service1 = client.services.create( + name=helpers.random_name(), + image="alpine", + command="sleep 300" + ) + service2 = client.services.create( + name=helpers.random_name(), + image="alpine", + command="sleep 300" + ) + tasks = [] + while len(tasks) == 0: + tasks = service1.tasks() + assert len(tasks) == 1 + assert tasks[0]['ServiceID'] == service1.id + + tasks = [] + while len(tasks) == 0: + tasks = service2.tasks() + assert len(tasks) == 1 + assert tasks[0]['ServiceID'] == service2.id + + def test_update(self): + client = docker.from_env() + service = client.services.create( + # create arguments + name=helpers.random_name(), + # ContainerSpec arguments + image="alpine", + command="sleep 300" + ) + new_name = helpers.random_name() + service.update( + # create argument + name=new_name, + # ContainerSpec argument + command="sleep 600" + ) + service.reload() + assert service.name == new_name + container_spec = service.attrs['Spec']['TaskTemplate']['ContainerSpec'] + assert container_spec['Command'] == ["sleep", "600"] diff --git a/tests/integration/models_swarm_test.py b/tests/integration/models_swarm_test.py new file mode 100644 index 0000000000..abdff41ffa --- /dev/null +++ b/tests/integration/models_swarm_test.py @@ -0,0 +1,22 @@ +import unittest +import docker +from .. import helpers + + +class SwarmTest(unittest.TestCase): + def setUp(self): + helpers.force_leave_swarm(docker.from_env()) + + def tearDown(self): + helpers.force_leave_swarm(docker.from_env()) + + def test_init_update_leave(self): + client = docker.from_env() + client.swarm.init(snapshot_interval=5000) + assert client.swarm.attrs['Spec']['Raft']['SnapshotInterval'] == 5000 + client.swarm.update(snapshot_interval=10000) + assert client.swarm.attrs['Spec']['Raft']['SnapshotInterval'] == 10000 + assert client.swarm.leave(force=True) + with self.assertRaises(docker.errors.APIError) as cm: + client.swarm.reload() + assert cm.exception.response.status_code == 406 diff --git a/tests/integration/models_volumes_test.py b/tests/integration/models_volumes_test.py new file mode 100644 index 0000000000..094e68fadb --- /dev/null +++ b/tests/integration/models_volumes_test.py @@ -0,0 +1,30 @@ +import docker +from .base import BaseIntegrationTest + + +class VolumesTest(BaseIntegrationTest): + def test_create_get(self): + client = docker.from_env() + volume = client.volumes.create( + 'dockerpytest_1', + driver='local', + labels={'labelkey': 'labelvalue'} + ) + self.tmp_volumes.append(volume.id) + assert volume.id + assert volume.name == 'dockerpytest_1' + assert volume.attrs['Labels'] == {'labelkey': 'labelvalue'} + + volume = client.volumes.get(volume.id) + assert volume.name == 'dockerpytest_1' + + def test_list_remove(self): + client = docker.from_env() + volume = client.volumes.create('dockerpytest_1') + self.tmp_volumes.append(volume.id) + assert volume in client.volumes.list() + assert volume in client.volumes.list(filters={'name': 'dockerpytest_'}) + assert volume not in client.volumes.list(filters={'name': 'foobar'}) + + volume.remove() + assert volume not in client.volumes.list() diff --git a/tests/unit/api_test.py b/tests/unit/api_test.py index dbd551df5f..67373bac05 100644 --- a/tests/unit/api_test.py +++ b/tests/unit/api_test.py @@ -27,7 +27,6 @@ DEFAULT_TIMEOUT_SECONDS = docker.constants.DEFAULT_TIMEOUT_SECONDS -TEST_CERT_DIR = os.path.join(os.path.dirname(__file__), 'testdata/certs') def response(status_code=200, content='', headers=None, reason=None, elapsed=0, @@ -487,32 +486,6 @@ def test_custom_user_agent(self): self.assertEqual(headers['User-Agent'], 'foo/bar') -class FromEnvTest(unittest.TestCase): - def setUp(self): - self.os_environ = os.environ.copy() - - def tearDown(self): - os.environ = self.os_environ - - def test_from_env(self): - """Test that environment variables are passed through to - utils.kwargs_from_env(). KwargsFromEnvTest tests that environment - variables are parsed correctly.""" - os.environ.update(DOCKER_HOST='tcp://192.168.59.103:2376', - DOCKER_CERT_PATH=TEST_CERT_DIR, - DOCKER_TLS_VERIFY='1') - client = APIClient.from_env() - self.assertEqual(client.base_url, "https://192.168.59.103:2376") - - def test_from_env_with_version(self): - os.environ.update(DOCKER_HOST='tcp://192.168.59.103:2376', - DOCKER_CERT_PATH=TEST_CERT_DIR, - DOCKER_TLS_VERIFY='1') - client = APIClient.from_env(version='2.32') - self.assertEqual(client.base_url, "https://192.168.59.103:2376") - self.assertEqual(client._version, '2.32') - - class DisableSocketTest(unittest.TestCase): class DummySocket(object): def __init__(self, timeout=60): diff --git a/tests/unit/client_test.py b/tests/unit/client_test.py new file mode 100644 index 0000000000..e22983c7e2 --- /dev/null +++ b/tests/unit/client_test.py @@ -0,0 +1,73 @@ +import datetime +import docker +import os +import unittest + +from . import fake_api + +try: + from unittest import mock +except ImportError: + import mock + + +TEST_CERT_DIR = os.path.join(os.path.dirname(__file__), 'testdata/certs') + + +class ClientTest(unittest.TestCase): + + @mock.patch('docker.api.APIClient.events') + def test_events(self, mock_func): + since = datetime.datetime(2016, 1, 1, 0, 0) + mock_func.return_value = fake_api.get_fake_events()[1] + client = docker.from_env() + assert client.events(since=since) == mock_func.return_value + mock_func.assert_called_with(since=since) + + @mock.patch('docker.api.APIClient.info') + def test_info(self, mock_func): + mock_func.return_value = fake_api.get_fake_info()[1] + client = docker.from_env() + assert client.info() == mock_func.return_value + mock_func.assert_called_with() + + @mock.patch('docker.api.APIClient.ping') + def test_ping(self, mock_func): + mock_func.return_value = True + client = docker.from_env() + assert client.ping() is True + mock_func.assert_called_with() + + @mock.patch('docker.api.APIClient.version') + def test_version(self, mock_func): + mock_func.return_value = fake_api.get_fake_version()[1] + client = docker.from_env() + assert client.version() == mock_func.return_value + mock_func.assert_called_with() + + +class FromEnvTest(unittest.TestCase): + + def setUp(self): + self.os_environ = os.environ.copy() + + def tearDown(self): + os.environ = self.os_environ + + def test_from_env(self): + """Test that environment variables are passed through to + utils.kwargs_from_env(). KwargsFromEnvTest tests that environment + variables are parsed correctly.""" + os.environ.update(DOCKER_HOST='tcp://192.168.59.103:2376', + DOCKER_CERT_PATH=TEST_CERT_DIR, + DOCKER_TLS_VERIFY='1') + client = docker.from_env() + self.assertEqual(client.api.base_url, "https://192.168.59.103:2376") + + def test_from_env_with_version(self): + os.environ.update(DOCKER_HOST='tcp://192.168.59.103:2376', + DOCKER_CERT_PATH=TEST_CERT_DIR, + DOCKER_TLS_VERIFY='1') + client = docker.from_env(version='2.32') + self.assertEqual(client.api.base_url, "https://192.168.59.103:2376") + self.assertEqual(client.api._version, '2.32') diff --git a/tests/unit/errors_test.py b/tests/unit/errors_test.py new file mode 100644 index 0000000000..876ede3b5f --- /dev/null +++ b/tests/unit/errors_test.py @@ -0,0 +1,22 @@ +import unittest + +from docker.errors import (APIError, DockerException, + create_unexpected_kwargs_error) + + +class APIErrorTest(unittest.TestCase): + def test_api_error_is_caught_by_dockerexception(self): + try: + raise APIError("this should be caught by DockerException") + except DockerException: + pass + + +class CreateUnexpectedKwargsErrorTest(unittest.TestCase): + def test_create_unexpected_kwargs_error_single(self): + e = create_unexpected_kwargs_error('f', {'foo': 'bar'}) + assert str(e) == "f() got an unexpected keyword argument 'foo'" + + def test_create_unexpected_kwargs_error_multiple(self): + e = create_unexpected_kwargs_error('f', {'foo': 'bar', 'baz': 'bosh'}) + assert str(e) == "f() got unexpected keyword arguments 'baz', 'foo'" diff --git a/tests/unit/fake_api.py b/tests/unit/fake_api.py index a8fb60ba1d..cf3f7d7dd1 100644 --- a/tests/unit/fake_api.py +++ b/tests/unit/fake_api.py @@ -6,6 +6,7 @@ FAKE_CONTAINER_ID = '3cc2351ab11b' FAKE_IMAGE_ID = 'e9aa60c60128' FAKE_EXEC_ID = 'd5d177f121dc' +FAKE_NETWORK_ID = '33fb6a3462b8' FAKE_IMAGE_NAME = 'test_image' FAKE_TARBALL_PATH = '/path/to/tarball' FAKE_REPO_NAME = 'repo' @@ -46,6 +47,17 @@ def get_fake_info(): return status_code, response +def post_fake_auth(): + status_code = 200 + response = {'Status': 'Login Succeeded', + 'IdentityToken': '9cbaf023786cd7'} + return status_code, response + + +def get_fake_ping(): + return 200, "OK" + + def get_fake_search(): status_code = 200 response = [{'Name': 'busybox', 'Description': 'Fake Description'}] @@ -125,7 +137,9 @@ def get_fake_inspect_container(tty=False): 'Config': {'Privileged': True, 'Tty': tty}, 'ID': FAKE_CONTAINER_ID, 'Image': 'busybox:latest', + 'Name': 'foobar', "State": { + "Status": "running", "Running": True, "Pid": 0, "ExitCode": 0, @@ -140,11 +154,11 @@ def get_fake_inspect_container(tty=False): def get_fake_inspect_image(): status_code = 200 response = { - 'id': FAKE_IMAGE_ID, - 'parent': "27cf784147099545", - 'created': "2013-03-23T22:24:18.818426-07:00", - 'container': FAKE_CONTAINER_ID, - 'container_config': + 'Id': FAKE_IMAGE_ID, + 'Parent': "27cf784147099545", + 'Created': "2013-03-23T22:24:18.818426-07:00", + 'Container': FAKE_CONTAINER_ID, + 'ContainerConfig': { "Hostname": "", "User": "", @@ -411,6 +425,61 @@ def post_fake_update_node(): return 200, None +def get_fake_network_list(): + return 200, [{ + "Name": "bridge", + "Id": FAKE_NETWORK_ID, + "Scope": "local", + "Driver": "bridge", + "EnableIPv6": False, + "Internal": False, + "IPAM": { + "Driver": "default", + "Config": [ + { + "Subnet": "172.17.0.0/16" + } + ] + }, + "Containers": { + FAKE_CONTAINER_ID: { + "EndpointID": "ed2419a97c1d99", + "MacAddress": "02:42:ac:11:00:02", + "IPv4Address": "172.17.0.2/16", + "IPv6Address": "" + } + }, + "Options": { + "com.docker.network.bridge.default_bridge": "true", + "com.docker.network.bridge.enable_icc": "true", + "com.docker.network.bridge.enable_ip_masquerade": "true", + "com.docker.network.bridge.host_binding_ipv4": "0.0.0.0", + "com.docker.network.bridge.name": "docker0", + "com.docker.network.driver.mtu": "1500" + } + }] + + +def get_fake_network(): + return 200, get_fake_network_list()[1][0] + + +def post_fake_network(): + return 201, {"Id": FAKE_NETWORK_ID, "Warnings": []} + + +def delete_fake_network(): + return 204, None + + +def post_fake_network_connect(): + return 200, None + + +def post_fake_network_disconnect(): + return 200, None + + # Maps real api url to fake response callback prefix = 'http+docker://localunixsocket' if constants.IS_WINDOWS_PLATFORM: @@ -423,6 +492,10 @@ def post_fake_update_node(): get_fake_version, '{1}/{0}/info'.format(CURRENT_VERSION, prefix): get_fake_info, + '{1}/{0}/auth'.format(CURRENT_VERSION, prefix): + post_fake_auth, + '{1}/{0}/_ping'.format(CURRENT_VERSION, prefix): + get_fake_ping, '{1}/{0}/images/search'.format(CURRENT_VERSION, prefix): get_fake_search, '{1}/{0}/images/json'.format(CURRENT_VERSION, prefix): @@ -516,4 +589,24 @@ def post_fake_update_node(): CURRENT_VERSION, prefix, FAKE_NODE_ID ), 'POST'): post_fake_update_node, + ('{1}/{0}/networks'.format(CURRENT_VERSION, prefix), 'GET'): + get_fake_network_list, + ('{1}/{0}/networks/create'.format(CURRENT_VERSION, prefix), 'POST'): + post_fake_network, + ('{1}/{0}/networks/{2}'.format( + CURRENT_VERSION, prefix, FAKE_NETWORK_ID + ), 'GET'): + get_fake_network, + ('{1}/{0}/networks/{2}'.format( + CURRENT_VERSION, prefix, FAKE_NETWORK_ID + ), 'DELETE'): + delete_fake_network, + ('{1}/{0}/networks/{2}/connect'.format( + CURRENT_VERSION, prefix, FAKE_NETWORK_ID + ), 'POST'): + post_fake_network_connect, + ('{1}/{0}/networks/{2}/disconnect'.format( + CURRENT_VERSION, prefix, FAKE_NETWORK_ID + ), 'POST'): + post_fake_network_disconnect, } diff --git a/tests/unit/fake_api_client.py b/tests/unit/fake_api_client.py new file mode 100644 index 0000000000..84e1d9def3 --- /dev/null +++ b/tests/unit/fake_api_client.py @@ -0,0 +1,61 @@ +import copy +import docker + +from . import fake_api + +try: + from unittest import mock +except ImportError: + import mock + + +class CopyReturnMagicMock(mock.MagicMock): + """ + A MagicMock which deep copies every return value. + """ + def _mock_call(self, *args, **kwargs): + ret = super(CopyReturnMagicMock, self)._mock_call(*args, **kwargs) + if isinstance(ret, (dict, list)): + ret = copy.deepcopy(ret) + return ret + + +def make_fake_api_client(): + """ + Returns non-complete fake APIClient. + + This returns most of the default cases correctly, but most arguments that + change behaviour will not work. + """ + api_client = docker.APIClient() + mock_client = CopyReturnMagicMock(**{ + 'build.return_value': fake_api.FAKE_IMAGE_ID, + 'commit.return_value': fake_api.post_fake_commit()[1], + 'containers.return_value': fake_api.get_fake_containers()[1], + 'create_container.return_value': + fake_api.post_fake_create_container()[1], + 'create_host_config.side_effect': api_client.create_host_config, + 'create_network.return_value': fake_api.post_fake_network()[1], + 'exec_create.return_value': fake_api.post_fake_exec_create()[1], + 'exec_start.return_value': fake_api.post_fake_exec_start()[1], + 'images.return_value': fake_api.get_fake_images()[1], + 'inspect_container.return_value': + fake_api.get_fake_inspect_container()[1], + 'inspect_image.return_value': fake_api.get_fake_inspect_image()[1], + 'inspect_network.return_value': fake_api.get_fake_network()[1], + 'logs.return_value': 'hello world\n', + 'networks.return_value': fake_api.get_fake_network_list()[1], + 'start.return_value': None, + 'wait.return_value': 0, + }) + mock_client._version = docker.constants.DEFAULT_DOCKER_API_VERSION + return mock_client + + +def make_fake_client(): + """ + Returns a Client with a fake APIClient. + """ + client = docker.Client() + client.api = make_fake_api_client() + return client diff --git a/tests/unit/models_containers_test.py b/tests/unit/models_containers_test.py new file mode 100644 index 0000000000..c3086c629a --- /dev/null +++ b/tests/unit/models_containers_test.py @@ -0,0 +1,465 @@ +import docker +from docker.models.containers import Container, _create_container_args +from docker.models.images import Image +import unittest + +from .fake_api import FAKE_CONTAINER_ID, FAKE_IMAGE_ID, FAKE_EXEC_ID +from .fake_api_client import make_fake_client + + +class ContainerCollectionTest(unittest.TestCase): + def test_run(self): + client = make_fake_client() + out = client.containers.run("alpine", "echo hello world") + + assert out == 'hello world\n' + + client.api.create_container.assert_called_with( + image="alpine", + command="echo hello world", + detach=False, + host_config={'NetworkMode': 'default'} + ) + client.api.inspect_container.assert_called_with(FAKE_CONTAINER_ID) + client.api.start.assert_called_with(FAKE_CONTAINER_ID) + client.api.wait.assert_called_with(FAKE_CONTAINER_ID) + client.api.logs.assert_called_with( + FAKE_CONTAINER_ID, + stderr=False, + stdout=True + ) + + def test_create_container_args(self): + create_kwargs = _create_container_args(dict( + image='alpine', + command='echo hello world', + blkio_weight_device=[{'Path': 'foo', 'Weight': 3}], + blkio_weight=2, + cap_add=['foo'], + cap_drop=['bar'], + cgroup_parent='foobar', + cpu_period=1, + cpu_quota=2, + cpu_shares=5, + cpuset_cpus='0-3', + detach=False, + device_read_bps=[{'Path': 'foo', 'Rate': 3}], + device_read_iops=[{'Path': 'foo', 'Rate': 3}], + device_write_bps=[{'Path': 'foo', 'Rate': 3}], + device_write_iops=[{'Path': 'foo', 'Rate': 3}], + devices=['/dev/sda:/dev/xvda:rwm'], + dns=['8.8.8.8'], + domainname='example.com', + dns_opt=['foo'], + dns_search=['example.com'], + entrypoint='/bin/sh', + environment={'FOO': 'BAR'}, + extra_hosts={'foo': '1.2.3.4'}, + group_add=['blah'], + ipc_mode='foo', + kernel_memory=123, + labels={'key': 'value'}, + links={'foo': 'bar'}, + log_config={'Type': 'json-file', 'Config': {}}, + lxc_conf={'foo': 'bar'}, + healthcheck={'test': 'true'}, + hostname='somehost', + mac_address='abc123', + mem_limit=123, + mem_reservation=123, + mem_swappiness=2, + memswap_limit=456, + name='somename', + network_disabled=False, + network_mode='blah', + networks=['foo'], + oom_kill_disable=True, + oom_score_adj=5, + pid_mode='host', + pids_limit=500, + ports={ + 1111: 4567, + 2222: None + }, + privileged=True, + publish_all_ports=True, + read_only=True, + restart_policy={'Name': 'always'}, + security_opt=['blah'], + shm_size=123, + stdin_open=True, + stop_signal=9, + sysctls={'foo': 'bar'}, + tmpfs={'/blah': ''}, + tty=True, + ulimits=[{"Name": "nofile", "Soft": 1024, "Hard": 2048}], + user='bob', + userns_mode='host', + version='1.23', + volume_driver='some_driver', + volumes=[ + '/home/user1/:/mnt/vol2', + '/var/www:/mnt/vol1:ro', + ], + volumes_from=['container'], + working_dir='/code' + )) + + expected = dict( + image='alpine', + command='echo hello world', + domainname='example.com', + detach=False, + entrypoint='/bin/sh', + environment={'FOO': 'BAR'}, + host_config={ + 'Binds': [ + '/home/user1/:/mnt/vol2', + '/var/www:/mnt/vol1:ro', + ], + 'BlkioDeviceReadBps': [{'Path': 'foo', 'Rate': 3}], + 'BlkioDeviceReadIOps': [{'Path': 'foo', 'Rate': 3}], + 'BlkioDeviceWriteBps': [{'Path': 'foo', 'Rate': 3}], + 'BlkioDeviceWriteIOps': [{'Path': 'foo', 'Rate': 3}], + 'BlkioWeightDevice': [{'Path': 'foo', 'Weight': 3}], + 'BlkioWeight': 2, + 'CapAdd': ['foo'], + 'CapDrop': ['bar'], + 'CgroupParent': 'foobar', + 'CpuPeriod': 1, + 'CpuQuota': 2, + 'CpuShares': 5, + 'CpuSetCpus': '0-3', + 'Devices': [{'PathOnHost': '/dev/sda', + 'CgroupPermissions': 'rwm', + 'PathInContainer': '/dev/xvda'}], + 'Dns': ['8.8.8.8'], + 'DnsOptions': ['foo'], + 'DnsSearch': ['example.com'], + 'ExtraHosts': ['foo:1.2.3.4'], + 'GroupAdd': ['blah'], + 'IpcMode': 'foo', + 'KernelMemory': 123, + 'Links': ['foo:bar'], + 'LogConfig': {'Type': 'json-file', 'Config': {}}, + 'LxcConf': [{'Key': 'foo', 'Value': 'bar'}], + 'Memory': 123, + 'MemoryReservation': 123, + 'MemorySwap': 456, + 'MemorySwappiness': 2, + 'NetworkMode': 'blah', + 'OomKillDisable': True, + 'OomScoreAdj': 5, + 'PidMode': 'host', + 'PidsLimit': 500, + 'PortBindings': { + '1111/tcp': [{'HostIp': '', 'HostPort': '4567'}], + '2222/tcp': [{'HostIp': '', 'HostPort': ''}] + }, + 'Privileged': True, + 'PublishAllPorts': True, + 'ReadonlyRootfs': True, + 'RestartPolicy': {'Name': 'always'}, + 'SecurityOpt': ['blah'], + 'ShmSize': 123, + 'Sysctls': {'foo': 'bar'}, + 'Tmpfs': {'/blah': ''}, + 'Ulimits': [{"Name": "nofile", "Soft": 1024, "Hard": 2048}], + 'UsernsMode': 'host', + 'VolumesFrom': ['container'], + }, + healthcheck={'test': 'true'}, + hostname='somehost', + labels={'key': 'value'}, + mac_address='abc123', + name='somename', + network_disabled=False, + networking_config={'foo': None}, + ports=[('1111', 'tcp'), ('2222', 'tcp')], + stdin_open=True, + stop_signal=9, + tty=True, + user='bob', + volume_driver='some_driver', + volumes=['/home/user1/', '/var/www'], + working_dir='/code' + ) + + assert create_kwargs == expected + + def test_run_detach(self): + client = make_fake_client() + container = client.containers.run('alpine', 'sleep 300', detach=True) + assert isinstance(container, Container) + assert container.id == FAKE_CONTAINER_ID + client.api.create_container.assert_called_with( + image='alpine', + command='sleep 300', + detach=True, + host_config={ + 'NetworkMode': 'default', + } + ) + client.api.inspect_container.assert_called_with(FAKE_CONTAINER_ID) + client.api.start.assert_called_with(FAKE_CONTAINER_ID) + + def test_run_pull(self): + client = make_fake_client() + + # raise exception on first call, then return normal value + client.api.create_container.side_effect = [ + docker.errors.ImageNotFound(""), + client.api.create_container.return_value + ] + + container = client.containers.run('alpine', 'sleep 300', detach=True) + + assert container.id == FAKE_CONTAINER_ID + client.api.pull.assert_called_with('alpine') + + def test_run_with_error(self): + client = make_fake_client() + client.api.logs.return_value = "some error" + client.api.wait.return_value = 1 + + with self.assertRaises(docker.errors.ContainerError) as cm: + client.containers.run('alpine', 'echo hello world') + assert cm.exception.exit_status == 1 + assert "some error" in str(cm.exception) + + def test_run_with_image_object(self): + client = make_fake_client() + image = client.images.get(FAKE_IMAGE_ID) + client.containers.run(image) + client.api.create_container.assert_called_with( + image=image.id, + command=None, + detach=False, + host_config={ + 'NetworkMode': 'default', + } + ) + + def test_run_remove(self): + client = make_fake_client() + client.containers.run("alpine") + client.api.remove_container.assert_not_called() + + client = make_fake_client() + client.api.wait.return_value = 1 + with self.assertRaises(docker.errors.ContainerError): + client.containers.run("alpine") + client.api.remove_container.assert_not_called() + + client = make_fake_client() + client.containers.run("alpine", remove=True) + client.api.remove_container.assert_called_with(FAKE_CONTAINER_ID) + + client = make_fake_client() + client.api.wait.return_value = 1 + with self.assertRaises(docker.errors.ContainerError): + client.containers.run("alpine", remove=True) + client.api.remove_container.assert_called_with(FAKE_CONTAINER_ID) + + client = make_fake_client() + with self.assertRaises(RuntimeError): + client.containers.run("alpine", detach=True, remove=True) + + def test_create(self): + client = make_fake_client() + container = client.containers.create( + 'alpine', + 'echo hello world', + environment={'FOO': 'BAR'} + ) + assert isinstance(container, Container) + assert container.id == FAKE_CONTAINER_ID + client.api.create_container.assert_called_with( + image='alpine', + command='echo hello world', + environment={'FOO': 'BAR'}, + host_config={'NetworkMode': 'default'} + ) + client.api.inspect_container.assert_called_with(FAKE_CONTAINER_ID) + + def test_create_with_image_object(self): + client = make_fake_client() + image = client.images.get(FAKE_IMAGE_ID) + client.containers.create(image) + client.api.create_container.assert_called_with( + image=image.id, + command=None, + host_config={'NetworkMode': 'default'} + ) + + def test_get(self): + client = make_fake_client() + container = client.containers.get(FAKE_CONTAINER_ID) + assert isinstance(container, Container) + assert container.id == FAKE_CONTAINER_ID + client.api.inspect_container.assert_called_with(FAKE_CONTAINER_ID) + + def test_list(self): + client = make_fake_client() + containers = client.containers.list(all=True) + client.api.containers.assert_called_with( + all=True, + before=None, + filters=None, + limit=-1, + since=None + ) + client.api.inspect_container.assert_called_with(FAKE_CONTAINER_ID) + assert len(containers) == 1 + assert isinstance(containers[0], Container) + assert containers[0].id == FAKE_CONTAINER_ID + + +class ContainerTest(unittest.TestCase): + def test_name(self): + client = make_fake_client() + container = client.containers.get(FAKE_CONTAINER_ID) + assert container.name == 'foobar' + + def test_status(self): + client = make_fake_client() + container = client.containers.get(FAKE_CONTAINER_ID) + assert container.status == "running" + + def test_attach(self): + client = make_fake_client() + container = client.containers.get(FAKE_CONTAINER_ID) + container.attach(stream=True) + client.api.attach.assert_called_with(FAKE_CONTAINER_ID, stream=True) + + def test_commit(self): + client = make_fake_client() + container = client.containers.get(FAKE_CONTAINER_ID) + image = container.commit() + client.api.commit.assert_called_with(FAKE_CONTAINER_ID, + repository=None, + tag=None) + assert isinstance(image, Image) + assert image.id == FAKE_IMAGE_ID + + def test_diff(self): + client = make_fake_client() + container = client.containers.get(FAKE_CONTAINER_ID) + container.diff() + client.api.diff.assert_called_with(FAKE_CONTAINER_ID) + + def test_exec_run(self): + client = make_fake_client() + container = client.containers.get(FAKE_CONTAINER_ID) + container.exec_run("echo hello world", privileged=True, stream=True) + client.api.exec_create.assert_called_with( + FAKE_CONTAINER_ID, "echo hello world", stdout=True, stderr=True, + stdin=False, tty=False, privileged=True, user='' + ) + client.api.exec_start.assert_called_with( + FAKE_EXEC_ID, detach=False, tty=False, stream=True, socket=False + ) + + def test_export(self): + client = make_fake_client() + container = client.containers.get(FAKE_CONTAINER_ID) + container.export() + client.api.export.assert_called_with(FAKE_CONTAINER_ID) + + def test_get_archive(self): + client = make_fake_client() + container = client.containers.get(FAKE_CONTAINER_ID) + container.get_archive('foo') + client.api.get_archive.assert_called_with(FAKE_CONTAINER_ID, 'foo') + + def test_kill(self): + client = make_fake_client() + container = client.containers.get(FAKE_CONTAINER_ID) + container.kill(signal=5) + client.api.kill.assert_called_with(FAKE_CONTAINER_ID, signal=5) + + def test_logs(self): + client = make_fake_client() + container = client.containers.get(FAKE_CONTAINER_ID) + container.logs() + client.api.logs.assert_called_with(FAKE_CONTAINER_ID) + + def test_pause(self): + client = make_fake_client() + container = client.containers.get(FAKE_CONTAINER_ID) + container.pause() + client.api.pause.assert_called_with(FAKE_CONTAINER_ID) + + def test_put_archive(self): + client = make_fake_client() + container = client.containers.get(FAKE_CONTAINER_ID) + container.put_archive('path', 'foo') + client.api.put_archive.assert_called_with(FAKE_CONTAINER_ID, + 'path', 'foo') + + def test_remove(self): + client = make_fake_client() + container = client.containers.get(FAKE_CONTAINER_ID) + container.remove() + client.api.remove_container.assert_called_with(FAKE_CONTAINER_ID) + + def test_rename(self): + client = make_fake_client() + container = client.containers.get(FAKE_CONTAINER_ID) + container.rename("foo") + client.api.rename.assert_called_with(FAKE_CONTAINER_ID, "foo") + + def test_resize(self): + client = make_fake_client() + container = client.containers.get(FAKE_CONTAINER_ID) + container.resize(1, 2) + client.api.resize.assert_called_with(FAKE_CONTAINER_ID, 1, 2) + + def test_restart(self): + client = make_fake_client() + container = client.containers.get(FAKE_CONTAINER_ID) + container.restart() + client.api.restart.assert_called_with(FAKE_CONTAINER_ID) + + def test_start(self): + client = make_fake_client() + container = client.containers.get(FAKE_CONTAINER_ID) + container.start() + client.api.start.assert_called_with(FAKE_CONTAINER_ID) + + def test_stats(self): + client = make_fake_client() + container = client.containers.get(FAKE_CONTAINER_ID) + container.stats() + client.api.stats.assert_called_with(FAKE_CONTAINER_ID) + + def test_stop(self): + client = make_fake_client() + container = client.containers.get(FAKE_CONTAINER_ID) + container.stop() + client.api.stop.assert_called_with(FAKE_CONTAINER_ID) + + def test_top(self): + client = make_fake_client() + container = client.containers.get(FAKE_CONTAINER_ID) + container.top() + client.api.top.assert_called_with(FAKE_CONTAINER_ID) + + def test_unpause(self): + client = make_fake_client() + container = client.containers.get(FAKE_CONTAINER_ID) + container.unpause() + client.api.unpause.assert_called_with(FAKE_CONTAINER_ID) + + def test_update(self): + client = make_fake_client() + container = client.containers.get(FAKE_CONTAINER_ID) + container.update(cpu_shares=2) + client.api.update_container.assert_called_with(FAKE_CONTAINER_ID, + cpu_shares=2) + + def test_wait(self): + client = make_fake_client() + container = client.containers.get(FAKE_CONTAINER_ID) + container.wait() + client.api.wait.assert_called_with(FAKE_CONTAINER_ID) diff --git a/tests/unit/models_images_test.py b/tests/unit/models_images_test.py new file mode 100644 index 0000000000..392c58d79f --- /dev/null +++ b/tests/unit/models_images_test.py @@ -0,0 +1,102 @@ +from docker.models.images import Image +import unittest + +from .fake_api import FAKE_IMAGE_ID +from .fake_api_client import make_fake_client + + +class ImageCollectionTest(unittest.TestCase): + def test_build(self): + client = make_fake_client() + image = client.images.build() + client.api.build.assert_called_with() + client.api.inspect_image.assert_called_with(FAKE_IMAGE_ID) + assert isinstance(image, Image) + assert image.id == FAKE_IMAGE_ID + + def test_get(self): + client = make_fake_client() + image = client.images.get(FAKE_IMAGE_ID) + client.api.inspect_image.assert_called_with(FAKE_IMAGE_ID) + assert isinstance(image, Image) + assert image.id == FAKE_IMAGE_ID + + def test_list(self): + client = make_fake_client() + images = client.images.list(all=True) + client.api.images.assert_called_with(all=True, name=None, filters=None) + assert len(images) == 1 + assert isinstance(images[0], Image) + assert images[0].id == FAKE_IMAGE_ID + + def test_load(self): + client = make_fake_client() + client.images.load('byte stream') + client.api.load_image.assert_called_with('byte stream') + + def test_pull(self): + client = make_fake_client() + image = client.images.pull('test_image') + client.api.pull.assert_called_with('test_image') + client.api.inspect_image.assert_called_with('test_image') + assert isinstance(image, Image) + assert image.id == FAKE_IMAGE_ID + + def test_push(self): + client = make_fake_client() + client.images.push('foobar', insecure_registry=True) + client.api.push.assert_called_with( + 'foobar', + tag=None, + insecure_registry=True + ) + + def test_remove(self): + client = make_fake_client() + client.images.remove('test_image') + client.api.remove_image.assert_called_with('test_image') + + def test_search(self): + client = make_fake_client() + client.images.search('test') + client.api.search.assert_called_with('test') + + +class ImageTest(unittest.TestCase): + def test_short_id(self): + image = Image(attrs={'Id': 'sha256:b6846070672ce4e8f1f91564ea6782bd675' + 'f69d65a6f73ef6262057ad0a15dcd'}) + assert image.short_id == 'sha256:b684607067' + + image = Image(attrs={'Id': 'b6846070672ce4e8f1f91564ea6782bd675' + 'f69d65a6f73ef6262057ad0a15dcd'}) + assert image.short_id == 'b684607067' + + def test_tags(self): + image = Image(attrs={ + 'RepoTags': ['test_image:latest'] + }) + assert image.tags == ['test_image:latest'] + + image = Image(attrs={ + 'RepoTags': [':'] + }) + assert image.tags == [] + + def test_history(self): + client = make_fake_client() + image = client.images.get(FAKE_IMAGE_ID) + image.history() + client.api.history.assert_called_with(FAKE_IMAGE_ID) + + def test_save(self): + client = make_fake_client() + image = client.images.get(FAKE_IMAGE_ID) + image.save() + client.api.get_image.assert_called_with(FAKE_IMAGE_ID) + + def test_tag(self): + client = make_fake_client() + image = client.images.get(FAKE_IMAGE_ID) + image.tag('foo') + client.api.tag.assert_called_with(FAKE_IMAGE_ID, 'foo', tag=None) diff --git a/tests/unit/models_networks_test.py b/tests/unit/models_networks_test.py new file mode 100644 index 0000000000..943b904568 --- /dev/null +++ b/tests/unit/models_networks_test.py @@ -0,0 +1,64 @@ +import unittest + +from .fake_api import FAKE_NETWORK_ID, FAKE_CONTAINER_ID +from .fake_api_client import make_fake_client + + +class ImageCollectionTest(unittest.TestCase): + + def test_create(self): + client = make_fake_client() + network = client.networks.create("foobar", labels={'foo': 'bar'}) + assert network.id == FAKE_NETWORK_ID + assert client.api.inspect_network.called_once_with(FAKE_NETWORK_ID) + assert client.api.create_network.called_once_with( + "foobar", + labels={'foo': 'bar'} + ) + + def test_get(self): + client = make_fake_client() + network = client.networks.get(FAKE_NETWORK_ID) + assert network.id == FAKE_NETWORK_ID + assert client.api.inspect_network.called_once_with(FAKE_NETWORK_ID) + + def test_list(self): + client = make_fake_client() + networks = client.networks.list() + assert networks[0].id == FAKE_NETWORK_ID + assert client.api.networks.called_once_with() + + client = make_fake_client() + client.networks.list(ids=["abc"]) + assert client.api.networks.called_once_with(ids=["abc"]) + + client = make_fake_client() + client.networks.list(names=["foobar"]) + assert client.api.networks.called_once_with(names=["foobar"]) + + +class ImageTest(unittest.TestCase): + + def test_connect(self): + client = make_fake_client() + network = client.networks.get(FAKE_NETWORK_ID) + network.connect(FAKE_CONTAINER_ID) + assert client.api.connect_container_to_network.called_once_with( + FAKE_CONTAINER_ID, + FAKE_NETWORK_ID + ) + + def test_disconnect(self): + client = make_fake_client() + network = client.networks.get(FAKE_NETWORK_ID) + network.disconnect(FAKE_CONTAINER_ID) + assert client.api.disconnect_container_from_network.called_once_with( + FAKE_CONTAINER_ID, + FAKE_NETWORK_ID + ) + + def test_remove(self): + client = make_fake_client() + network = client.networks.get(FAKE_NETWORK_ID) + network.remove() + assert client.api.remove_network.called_once_with(FAKE_NETWORK_ID) diff --git a/tests/unit/models_resources_test.py b/tests/unit/models_resources_test.py new file mode 100644 index 0000000000..25c6a3ed0c --- /dev/null +++ b/tests/unit/models_resources_test.py @@ -0,0 +1,14 @@ +import unittest + +from .fake_api import FAKE_CONTAINER_ID +from .fake_api_client import make_fake_client + + +class ModelTest(unittest.TestCase): + def test_reload(self): + client = make_fake_client() + container = client.containers.get(FAKE_CONTAINER_ID) + container.attrs['Name'] = "oldname" + container.reload() + assert client.api.inspect_container.call_count == 2 + assert container.attrs['Name'] == "foobar" diff --git a/tests/unit/models_services_test.py b/tests/unit/models_services_test.py new file mode 100644 index 0000000000..c3b63ae087 --- /dev/null +++ b/tests/unit/models_services_test.py @@ -0,0 +1,52 @@ +import unittest +from docker.models.services import _get_create_service_kwargs + + +class CreateServiceKwargsTest(unittest.TestCase): + def test_get_create_service_kwargs(self): + kwargs = _get_create_service_kwargs('test', { + 'image': 'foo', + 'command': 'true', + 'name': 'somename', + 'labels': {'key': 'value'}, + 'mode': 'global', + 'update_config': {'update': 'config'}, + 'networks': ['somenet'], + 'endpoint_spec': {'blah': 'blah'}, + 'container_labels': {'containerkey': 'containervalue'}, + 'resources': {'foo': 'bar'}, + 'restart_policy': {'restart': 'policy'}, + 'log_driver': 'logdriver', + 'log_driver_options': {'foo': 'bar'}, + 'args': ['some', 'args'], + 'env': {'FOO': 'bar'}, + 'workdir': '/', + 'user': 'bob', + 'mounts': [{'some': 'mounts'}], + 'stop_grace_period': 5, + 'constraints': ['foo=bar'], + }) + + task_template = kwargs.pop('task_template') + + assert kwargs == { + 'name': 'somename', + 'labels': {'key': 'value'}, + 'mode': 'global', + 'update_config': {'update': 'config'}, + 'networks': ['somenet'], + 'endpoint_spec': {'blah': 'blah'}, + } + assert set(task_template.keys()) == set([ + 'ContainerSpec', 'Resources', 'RestartPolicy', 'Placement', + 'LogDriver' + ]) + assert task_template['Placement'] == {'Constraints': ['foo=bar']} + assert task_template['LogDriver'] == { + 'Name': 'logdriver', + 'Options': {'foo': 'bar'} + } + assert set(task_template['ContainerSpec'].keys()) == set([ + 'Image', 'Command', 'Args', 'Env', 'Dir', 'User', 'Labels', + 'Mounts', 'StopGracePeriod' + ]) diff --git a/tests/unit/utils_json_stream_test.py b/tests/unit/utils_json_stream_test.py new file mode 100644 index 0000000000..f7aefd0f18 --- /dev/null +++ b/tests/unit/utils_json_stream_test.py @@ -0,0 +1,62 @@ +# encoding: utf-8 +from __future__ import absolute_import +from __future__ import unicode_literals + +from docker.utils.json_stream import json_splitter, stream_as_text, json_stream + + +class TestJsonSplitter(object): + + def test_json_splitter_no_object(self): + data = '{"foo": "bar' + assert json_splitter(data) is None + + def test_json_splitter_with_object(self): + data = '{"foo": "bar"}\n \n{"next": "obj"}' + assert json_splitter(data) == ({'foo': 'bar'}, '{"next": "obj"}') + + def test_json_splitter_leading_whitespace(self): + data = '\n \r{"foo": "bar"}\n\n {"next": "obj"}' + assert json_splitter(data) == ({'foo': 'bar'}, '{"next": "obj"}') + + +class TestStreamAsText(object): + + def test_stream_with_non_utf_unicode_character(self): + stream = [b'\xed\xf3\xf3'] + output, = stream_as_text(stream) + assert output == '���' + + def test_stream_with_utf_character(self): + stream = ['ěĝ'.encode('utf-8')] + output, = stream_as_text(stream) + assert output == 'ěĝ' + + +class TestJsonStream(object): + + def test_with_falsy_entries(self): + stream = [ + '{"one": "two"}\n{}\n', + "[1, 2, 3]\n[]\n", + ] + output = list(json_stream(stream)) + assert output == [ + {'one': 'two'}, + {}, + [1, 2, 3], + [], + ] + + def test_with_leading_whitespace(self): + stream = [ + '\n \r\n {"one": "two"}{"x": 1}', + ' {"three": "four"}\t\t{"x": 2}' + ] + output = list(json_stream(stream)) + assert output == [ + {'one': 'two'}, + {'x': 1}, + {'three': 'four'}, + {'x': 2} + ] From c7a3aa7e446536e048bc02ca881ec14016f08675 Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Mon, 7 Nov 2016 18:07:00 -0800 Subject: [PATCH 0178/1301] Add new Sphinx documentation Initial work thanks to @aanand. Signed-off-by: Ben Firshman --- .gitignore | 2 +- Dockerfile-docs | 5 +- Makefile | 2 +- docs-requirements.txt | 3 +- docs/_static/custom.css | 3 + docs/_templates/page.html | 2 + docs/api.md | 1237 ------------------------- docs/api.rst | 109 +++ docs/{change_log.md => change-log.md} | 2 +- docs/client.rst | 30 + docs/conf.py | 365 ++++++++ docs/containers.rst | 51 + docs/host-devices.md | 29 - docs/hostconfig.md | 142 --- docs/images.rst | 39 + docs/index.md | 15 - docs/index.rst | 93 ++ docs/machine.md | 26 - docs/networks.md | 177 ---- docs/networks.rst | 33 + docs/nodes.rst | 30 + docs/port-bindings.md | 58 -- docs/services.rst | 36 + docs/swarm.md | 274 ------ docs/swarm.rst | 24 + docs/tls.md | 86 -- docs/tls.rst | 37 + docs/tmpfs.md | 33 - docs/volumes.md | 34 - docs/volumes.rst | 31 + mkdocs.yml | 21 - 31 files changed, 892 insertions(+), 2137 deletions(-) create mode 100644 docs/_static/custom.css create mode 100644 docs/_templates/page.html delete mode 100644 docs/api.md create mode 100644 docs/api.rst rename docs/{change_log.md => change-log.md} (99%) create mode 100644 docs/client.rst create mode 100644 docs/conf.py create mode 100644 docs/containers.rst delete mode 100644 docs/host-devices.md delete mode 100644 docs/hostconfig.md create mode 100644 docs/images.rst delete mode 100644 docs/index.md create mode 100644 docs/index.rst delete mode 100644 docs/machine.md delete mode 100644 docs/networks.md create mode 100644 docs/networks.rst create mode 100644 docs/nodes.rst delete mode 100644 docs/port-bindings.md create mode 100644 docs/services.rst delete mode 100644 docs/swarm.md create mode 100644 docs/swarm.rst delete mode 100644 docs/tls.md create mode 100644 docs/tls.rst delete mode 100644 docs/tmpfs.md delete mode 100644 docs/volumes.md create mode 100644 docs/volumes.rst delete mode 100644 mkdocs.yml diff --git a/.gitignore b/.gitignore index 34ccd387e4..e626dc6cef 100644 --- a/.gitignore +++ b/.gitignore @@ -10,7 +10,7 @@ dist html/* # Compiled Documentation -site/ +_build/ README.rst env/ diff --git a/Dockerfile-docs b/Dockerfile-docs index 1103ffd179..705649ff95 100644 --- a/Dockerfile-docs +++ b/Dockerfile-docs @@ -1,8 +1,11 @@ -FROM python:2.7 +FROM python:3.5 RUN mkdir /home/docker-py WORKDIR /home/docker-py +COPY requirements.txt /home/docker-py/requirements.txt +RUN pip install -r requirements.txt + COPY docs-requirements.txt /home/docker-py/docs-requirements.txt RUN pip install -r docs-requirements.txt diff --git a/Makefile b/Makefile index 2e9ecf8e7f..425fffd897 100644 --- a/Makefile +++ b/Makefile @@ -74,7 +74,7 @@ flake8: build .PHONY: docs docs: build-docs - docker run -v `pwd`/docs:/home/docker-py/docs/ -p 8000:8000 docker-py-docs mkdocs serve -a 0.0.0.0:8000 + docker run --rm -it -v `pwd`:/home/docker-py docker-py-docs sphinx-build docs ./_build .PHONY: shell shell: build diff --git a/docs-requirements.txt b/docs-requirements.txt index aede1cbadf..d69373d7c7 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -1 +1,2 @@ -mkdocs==0.15.3 +recommonmark==0.4.0 +Sphinx==1.4.6 diff --git a/docs/_static/custom.css b/docs/_static/custom.css new file mode 100644 index 0000000000..5d711eeffb --- /dev/null +++ b/docs/_static/custom.css @@ -0,0 +1,3 @@ +dl.hide-signature > dt { + display: none; +} diff --git a/docs/_templates/page.html b/docs/_templates/page.html new file mode 100644 index 0000000000..cf0264cf73 --- /dev/null +++ b/docs/_templates/page.html @@ -0,0 +1,2 @@ +{% extends "!page.html" %} +{% set css_files = css_files + ["_static/custom.css"] %} diff --git a/docs/api.md b/docs/api.md deleted file mode 100644 index fdf3e278ab..0000000000 --- a/docs/api.md +++ /dev/null @@ -1,1237 +0,0 @@ -# Client API - -To instantiate a `Client` class that will allow you to communicate with a -Docker daemon, simply do: - -```python ->>> from docker import Client ->>> cli = Client(base_url='unix://var/run/docker.sock') -``` - -**Params**: - -* base_url (str): Refers to the protocol+hostname+port where the Docker server -is hosted. -* version (str): The version of the API the client will use. Specify `'auto'` - to use the API version provided by the server. -* timeout (int): The HTTP request timeout, in seconds. -* tls (bool or [TLSConfig](tls.md#TLSConfig)): Equivalent CLI options: `docker --tls ...` -* user_agent (str): Set a custom user agent for requests to the server. - - -**** - -## attach - -The `.logs()` function is a wrapper around this method, which you can use -instead if you want to fetch/stream container output without first retrieving -the entire backlog. - -**Params**: - -* container (str): The container to attach to -* stdout (bool): Get STDOUT -* stderr (bool): Get STDERR -* stream (bool): Return an iterator -* logs (bool): Get all previous output - -**Returns** (generator or str): The logs or output for the image - -## build - -Similar to the `docker build` command. Either `path` or `fileobj` needs to be -set. `path` can be a local path (to a directory containing a Dockerfile) or a -remote URL. `fileobj` must be a readable file-like object to a Dockerfile. - -If you have a tar file for the Docker build context (including a Dockerfile) -already, pass a readable file-like object to `fileobj` and also pass -`custom_context=True`. If the stream is compressed also, set `encoding` to the -correct value (e.g `gzip`). - -**Params**: - -* path (str): Path to the directory containing the Dockerfile -* tag (str): A tag to add to the final image -* quiet (bool): Whether to return the status -* fileobj: A file object to use as the Dockerfile. (Or a file-like object) -* nocache (bool): Don't use the cache when set to `True` -* rm (bool): Remove intermediate containers. The `docker build` command now - defaults to ``--rm=true``, but we have kept the old default of `False` - to preserve backward compatibility -* stream (bool): *Deprecated for API version > 1.8 (always True)*. - Return a blocking generator you can iterate over to retrieve build output as - it happens -* timeout (int): HTTP timeout -* custom_context (bool): Optional if using `fileobj` -* encoding (str): The encoding for a stream. Set to `gzip` for compressing -* pull (bool): Downloads any updates to the FROM image in Dockerfiles -* forcerm (bool): Always remove intermediate containers, even after unsuccessful builds -* dockerfile (str): path within the build context to the Dockerfile -* buildargs (dict): A dictionary of build arguments -* container_limits (dict): A dictionary of limits applied to each container - created by the build process. Valid keys: - - memory (int): set memory limit for build - - memswap (int): Total memory (memory + swap), -1 to disable swap - - cpushares (int): CPU shares (relative weight) - - cpusetcpus (str): CPUs in which to allow execution, e.g., `"0-3"`, `"0,1"` -* decode (bool): If set to `True`, the returned stream will be decoded into - dicts on the fly. Default `False`. -* shmsize (int): Size of /dev/shm in bytes. The size must be greater - than 0. If omitted the system uses 64MB. -* labels (dict): A dictionary of labels to set on the image - -**Returns** (generator): A generator for the build output - -```python ->>> from io import BytesIO ->>> from docker import Client ->>> dockerfile = ''' -... # Shared Volume -... FROM busybox:buildroot-2014.02 -... MAINTAINER first last, first.last@yourdomain.com -... VOLUME /data -... CMD ["/bin/sh"] -... ''' ->>> f = BytesIO(dockerfile.encode('utf-8')) ->>> cli = Client(base_url='tcp://127.0.0.1:2375') ->>> response = [line for line in cli.build( -... fileobj=f, rm=True, tag='yourname/volume' -... )] ->>> response -['{"stream":" ---\\u003e a9eb17255234\\n"}', -'{"stream":"Step 1 : MAINTAINER first last, first.last@yourdomain.com\\n"}', -'{"stream":" ---\\u003e Running in 08787d0ee8b1\\n"}', -'{"stream":" ---\\u003e 23e5e66a4494\\n"}', -'{"stream":"Removing intermediate container 08787d0ee8b1\\n"}', -'{"stream":"Step 2 : VOLUME /data\\n"}', -'{"stream":" ---\\u003e Running in abdc1e6896c6\\n"}', -'{"stream":" ---\\u003e 713bca62012e\\n"}', -'{"stream":"Removing intermediate container abdc1e6896c6\\n"}', -'{"stream":"Step 3 : CMD [\\"/bin/sh\\"]\\n"}', -'{"stream":" ---\\u003e Running in dba30f2a1a7e\\n"}', -'{"stream":" ---\\u003e 032b8b2855fc\\n"}', -'{"stream":"Removing intermediate container dba30f2a1a7e\\n"}', -'{"stream":"Successfully built 032b8b2855fc\\n"}'] -``` - -**Raises:** [TypeError]( -https://docs.python.org/3.5/library/exceptions.html#TypeError) if `path` nor -`fileobj` are specified - -## commit - -Identical to the `docker commit` command. - -**Params**: - -* container (str): The image hash of the container -* repository (str): The repository to push the image to -* tag (str): The tag to push -* message (str): A commit message -* author (str): The name of the author -* changes (str): Dockerfile instructions to apply while committing -* conf (dict): The configuration for the container. See the [Docker remote api]( -https://docs.docker.com/reference/api/docker_remote_api/) for full details. - -## containers - -List containers. Identical to the `docker ps` command. - -**Params**: - -* quiet (bool): Only display numeric Ids -* all (bool): Show all containers. Only running containers are shown by default -* trunc (bool): Truncate output -* latest (bool): Show only the latest created container, include non-running -ones. -* since (str): Show only containers created since Id or Name, include -non-running ones -* before (str): Show only container created before Id or Name, include -non-running ones -* limit (int): Show `limit` last created containers, include non-running ones -* size (bool): Display sizes -* filters (dict): Filters to be processed on the image list. Available filters: - - `exited` (int): Only containers with specified exit code - - `status` (str): One of `restarting`, `running`, `paused`, `exited` - - `label` (str): format either `"key"` or `"key=value"` - - `id` (str): The id of the container. - - `name` (str): The name of the container. - - `ancestor` (str): Filter by container ancestor. Format of `[:tag]`, ``, or ``. - - `before` (str): Only containers created before a particular container. Give the container name or id. - - `since` (str): Only containers created after a particular container. Give container name or id. - - A comprehensive list can be found [here](https://docs.docker.com/engine/reference/commandline/ps/) - -**Returns** (dict): The system's containers - -```python ->>> from docker import Client ->>> cli = Client(base_url='tcp://127.0.0.1:2375') ->>> cli.containers() -[{'Command': '/bin/sleep 30', - 'Created': 1412574844, - 'Id': '6e276c9e6e5759e12a6a9214efec6439f80b4f37618e1a6547f28a3da34db07a', - 'Image': 'busybox:buildroot-2014.02', - 'Names': ['/grave_mayer'], - 'Ports': [], - 'Status': 'Up 1 seconds'}] -``` - -## connect_container_to_network - -Connect a container to a network. - -**Params**: - -* container (str): container-id/name to be connected to the network -* net_id (str): network id -* aliases (list): A list of aliases for this endpoint. Names in that list can - be used within the network to reach the container. Defaults to `None`. -* links (list): A list of links for this endpoint. Containers declared in this - list will be [linked](https://docs.docker.com/engine/userguide/networking/work-with-networks/#linking-containers-in-user-defined-networks) - to this container. Defaults to `None`. -* ipv4_address (str): The IP address of this container on the network, - using the IPv4 protocol. Defaults to `None`. -* ipv6_address (str): The IP address of this container on the network, - using the IPv6 protocol. Defaults to `None`. -* link_local_ips (list): A list of link-local (IPv4/IPv6) addresses. - -## copy -Identical to the `docker cp` command. Get files/folders from the container. -**Deprecated for API version >= 1.20** – Consider using -[`get_archive`](#get_archive) **instead.** - -**Params**: - -* container (str): The container to copy from -* resource (str): The path within the container - -**Returns** (str): The contents of the file as a string - -## create_container - -Creates a container that can then be `.start()` ed. Parameters are similar to -those for the `docker run` command except it doesn't support the attach -options (`-a`). - -See [Port bindings](port-bindings.md) and [Using volumes](volumes.md) for more -information on how to create port bindings and volume mappings. - -The `mem_limit` variable accepts float values (which represent the memory limit -of the created container in bytes) or a string with a units identification char -('100000b', '1000k', '128m', '1g'). If a string is specified without a units -character, bytes are assumed as an intended unit. - -`volumes_from` and `dns` arguments raise [TypeError]( -https://docs.python.org/3.5/library/exceptions.html#TypeError) exception if -they are used against v1.10 and above of the Docker remote API. Those -arguments should be passed as part of the `host_config` dictionary. - -**Params**: - -* image (str): The image to run -* command (str or list): The command to be run in the container -* hostname (str): Optional hostname for the container -* user (str or int): Username or UID -* detach (bool): Detached mode: run container in the background and print new -container Id -* stdin_open (bool): Keep STDIN open even if not attached -* tty (bool): Allocate a pseudo-TTY -* mem_limit (float or str): Memory limit (format: [number][optional unit], -where unit = b, k, m, or g) -* ports (list of ints): A list of port numbers -* environment (dict or list): A dictionary or a list of strings in the -following format `["PASSWORD=xxx"]` or `{"PASSWORD": "xxx"}`. -* dns (list): DNS name servers -* dns_opt (list): Additional options to be added to the container's `resolv.conf` file -* volumes (str or list): -* volumes_from (str or list): List of container names or Ids to get volumes -from. Optionally a single string joining container id's with commas -* network_disabled (bool): Disable networking -* name (str): A name for the container -* entrypoint (str or list): An entrypoint -* working_dir (str): Path to the working directory -* domainname (str or list): Set custom DNS search domains -* memswap_limit (int): -* host_config (dict): A [HostConfig](hostconfig.md) dictionary -* mac_address (str): The Mac Address to assign the container -* labels (dict or list): A dictionary of name-value labels (e.g. `{"label1": "value1", "label2": "value2"}`) or a list of names of labels to set with empty values (e.g. `["label1", "label2"]`) -* volume_driver (str): The name of a volume driver/plugin. -* stop_signal (str): The stop signal to use to stop the container (e.g. `SIGINT`). -* networking_config (dict): A [NetworkingConfig](networks.md) dictionary - -**Returns** (dict): A dictionary with an image 'Id' key and a 'Warnings' key. - -```python ->>> from docker import Client ->>> cli = Client(base_url='tcp://127.0.0.1:2375') ->>> container = cli.create_container(image='busybox:latest', command='/bin/sleep 30') ->>> print(container) -{'Id': '8a61192da2b3bb2d922875585e29b74ec0dc4e0117fcbf84c962204e97564cd7', - 'Warnings': None} -``` - -### docker.utils.parse_env_file - -A utility for parsing an environment file. - -The expected format of the file is as follows: - -``` -USERNAME=jdoe -PASSWORD=secret -``` - -The utility can be used as follows: - -```python ->>> import docker.utils ->>> my_envs = docker.utils.parse_env_file('/path/to/file') ->>> client.create_container('myimage', 'command', environment=my_envs) -``` - -## create_network - -Create a network, similar to the `docker network create` command. See the -[networks documentation](networks.md) for details. - -**Params**: - -* name (str): Name of the network -* driver (str): Name of the driver used to create the network -* options (dict): Driver options as a key-value dictionary -* ipam (dict): Optional custom IP scheme for the network -* check_duplicate (bool): Request daemon to check for networks with same name. - Default: `True`. -* internal (bool): Restrict external access to the network. Default `False`. -* labels (dict): Map of labels to set on the network. Default `None`. -* enable_ipv6 (bool): Enable IPv6 on the network. Default `False`. - -**Returns** (dict): The created network reference object - -## create_service - -Create a service, similar to the `docker service create` command. See the -[services documentation](services.md#Clientcreate_service) for details. - -## create_volume - -Create and register a named volume - -**Params**: - -* name (str): Name of the volume -* driver (str): Name of the driver used to create the volume -* driver_opts (dict): Driver options as a key-value dictionary -* labels (dict): Labels to set on the volume - -**Returns** (dict): The created volume reference object - -```python ->>> from docker import Client ->>> cli = Client() ->>> volume = cli.create_volume( - name='foobar', driver='local', driver_opts={'foo': 'bar', 'baz': 'false'}, - labels={"key": "value"} -) ->>> print(volume) -{ - u'Mountpoint': u'/var/lib/docker/volumes/foobar/_data', - u'Driver': u'local', - u'Name': u'foobar', - u'Labels': {u'key': u'value'} -} -``` - -## diff - -Inspect changes on a container's filesystem. - -**Params**: - -* container (str): The container to diff - -**Returns** (str): - -## disconnect_container_from_network - -**Params**: - -* container (str): container-id/name to be disconnected from a network -* net_id (str): network id -* force (bool): Force the container to disconnect from a network. - Default: `False` - -## events - -Identical to the `docker events` command: get real time events from the server. The `events` -function return a blocking generator you can iterate over to retrieve events as they happen. - -**Params**: - -* since (UTC datetime or int): get events from this point -* until (UTC datetime or int): get events until this point -* filters (dict): filter the events by event time, container or image -* decode (bool): If set to true, stream will be decoded into dicts on the - fly. False by default. - -**Returns** (generator): - -```python -{u'status': u'start', - u'from': u'image/with:tag', - u'id': u'container-id', - u'time': 1423339459} -``` - -## execute - -This command is deprecated for docker-py >= 1.2.0 ; use `exec_create` and -`exec_start` instead. - -## exec_create - -Sets up an exec instance in a running container. - -**Params**: - -* container (str): Target container where exec instance will be created -* cmd (str or list): Command to be executed -* stdout (bool): Attach to stdout of the exec command if true. Default: True -* stderr (bool): Attach to stderr of the exec command if true. Default: True -* since (UTC datetime or int): Output logs from this timestamp. Default: `None` (all logs are given) -* tty (bool): Allocate a pseudo-TTY. Default: False -* user (str): User to execute command as. Default: root - -**Returns** (dict): A dictionary with an exec 'Id' key. - - -## exec_inspect - -Return low-level information about an exec command. - -**Params**: - -* exec_id (str): ID of the exec instance - -**Returns** (dict): Dictionary of values returned by the endpoint. - - -## exec_resize - -Resize the tty session used by the specified exec command. - -**Params**: - -* exec_id (str): ID of the exec instance -* height (int): Height of tty session -* width (int): Width of tty session - -## exec_start - -Start a previously set up exec instance. - -**Params**: - -* exec_id (str): ID of the exec instance -* detach (bool): If true, detach from the exec command. Default: False -* tty (bool): Allocate a pseudo-TTY. Default: False -* stream (bool): Stream response data. Default: False - -**Returns** (generator or str): If `stream=True`, a generator yielding response -chunks. A string containing response data otherwise. - -## export - -Export the contents of a filesystem as a tar archive to STDOUT. - -**Params**: - -* container (str): The container to export - -**Returns** (str): The filesystem tar archive as a str - -## get_archive - -Retrieve a file or folder from a container in the form of a tar archive. - -**Params**: - -* container (str): The container where the file is located -* path (str): Path to the file or folder to retrieve - -**Returns** (tuple): First element is a raw tar data stream. Second element is -a dict containing `stat` information on the specified `path`. - -```python ->>> import docker ->>> cli = docker.Client() ->>> ctnr = cli.create_container('busybox', 'true') ->>> strm, stat = cli.get_archive(ctnr, '/bin/sh') ->>> print(stat) -{u'linkTarget': u'', u'mode': 493, u'mtime': u'2015-09-16T12:34:23-07:00', u'name': u'sh', u'size': 962860} -``` - -## get_image - -Get an image from the docker daemon. Similar to the `docker save` command. - -**Params**: - -* image (str): Image name to get - -**Returns** (urllib3.response.HTTPResponse object): The response from the docker daemon - -An example of how to get (save) an image to a file. -```python ->>> from docker import Client ->>> cli = Client(base_url='unix://var/run/docker.sock') ->>> image = cli.get_image(“fedora:latest”) ->>> image_tar = open(‘/tmp/fedora-latest.tar’,’w’) ->>> image_tar.write(image.data) ->>> image_tar.close() -``` - -## history - -Show the history of an image. - -**Params**: - -* image (str): The image to show history for - -**Returns** (str): The history of the image - -## images - -List images. Identical to the `docker images` command. - -**Params**: - -* name (str): Only show images belonging to the repository `name` -* quiet (bool): Only show numeric Ids. Returns a list -* all (bool): Show all images (by default filter out the intermediate image -layers) -* filters (dict): Filters to be processed on the image list. Available filters: - - `dangling` (bool) - - `label` (str): format either `"key"` or `"key=value"` - -**Returns** (dict or list): A list if `quiet=True`, otherwise a dict. - -```python -[{'Created': 1401926735, -'Id': 'a9eb172552348a9a49180694790b33a1097f546456d041b6e82e4d7716ddb721', -'ParentId': '120e218dd395ec314e7b6249f39d2853911b3d6def6ea164ae05722649f34b16', -'RepoTags': ['busybox:buildroot-2014.02', 'busybox:latest'], -'Size': 0, -'VirtualSize': 2433303}, -... -``` - -## import_image - -Similar to the `docker import` command. - -If `src` is a string or unicode string, it will first be treated as a path to -a tarball on the local system. If there is an error reading from that file, -src will be treated as a URL instead to fetch the image from. You can also pass -an open file handle as 'src', in which case the data will be read from that -file. - -If `src` is unset but `image` is set, the `image` parameter will be taken as -the name of an existing image to import from. - -**Params**: - -* src (str or file): Path to tarfile, URL, or file-like object -* repository (str): The repository to create -* tag (str): The tag to apply -* image (str): Use another image like the `FROM` Dockerfile parameter - -## import_image_from_data - -Like `.import_image()`, but allows importing in-memory bytes data. - -**Params**: - -* data (bytes collection): Bytes collection containing valid tar data -* repository (str): The repository to create -* tag (str): The tag to apply - -## import_image_from_file - -Like `.import_image()`, but only supports importing from a tar file on -disk. If the file doesn't exist it will raise `IOError`. - -**Params**: - -* filename (str): Full path to a tar file. -* repository (str): The repository to create -* tag (str): The tag to apply - -## import_image_from_url - -Like `.import_image()`, but only supports importing from a URL. - -**Params**: - -* url (str): A URL pointing to a tar file. -* repository (str): The repository to create -* tag (str): The tag to apply - -## import_image_from_image - -Like `.import_image()`, but only supports importing from another image, -like the `FROM` Dockerfile parameter. - -**Params**: - -* image (str): Image name to import from -* repository (str): The repository to create -* tag (str): The tag to apply - -## info - -Display system-wide information. Identical to the `docker info` command. - -**Returns** (dict): The info as a dict - -``` ->>> from docker import Client ->>> cli = Client(base_url='tcp://127.0.0.1:2375') ->>> cli.info() -{'Containers': 3, - 'Debug': 1, - 'Driver': 'aufs', - 'DriverStatus': [['Root Dir', '/mnt/sda1/var/lib/docker/aufs'], - ['Dirs', '225']], - 'ExecutionDriver': 'native-0.2', - 'IPv4Forwarding': 1, - 'Images': 219, - 'IndexServerAddress': 'https://index.docker.io/v1/', - 'InitPath': '/usr/local/bin/docker', - 'InitSha1': '', - 'KernelVersion': '3.16.1-tinycore64', - 'MemoryLimit': 1, - 'NEventsListener': 0, - 'NFd': 11, - 'NGoroutines': 12, - 'OperatingSystem': 'Boot2Docker 1.2.0 (TCL 5.3);', - 'SwapLimit': 1} -``` - -## init_swarm - -Initialize a new Swarm using the current connected engine as the first node. -See the [Swarm documentation](swarm.md#clientinit_swarm). - -## insert -*DEPRECATED* - -## inspect_container - -Identical to the `docker inspect` command, but only for containers. - -**Params**: - -* container (str): The container to inspect - -**Returns** (dict): Nearly the same output as `docker inspect`, just as a -single dict - -## inspect_image - -Identical to the `docker inspect` command, but only for images. - -**Params**: - -* image (str): The image to inspect - -**Returns** (dict): Nearly the same output as `docker inspect`, just as a -single dict - -## inspect_network - -Retrieve network info by id. - -**Params**: - -* net_id (str): network id - -**Returns** (dict): Network information dictionary - -## inspect_node - -Retrieve low-level information about a Swarm node. -See the [Swarm documentation](swarm.md#clientinspect_node). - -## inspect_service - -Create a service, similar to the `docker service create` command. See the -[services documentation](services.md#clientinspect_service) for details. - -## inspect_swarm - -Retrieve information about the current Swarm. -See the [Swarm documentation](swarm.md#clientinspect_swarm). - -## inspect_task - -Retrieve information about a task. - -**Params**: - -* task (str): Task identifier - -**Returns** (dict): Task information dictionary - -## inspect_volume - -Retrieve volume info by name. - -**Params**: - -* name (str): volume name - -**Returns** (dict): Volume information dictionary - -```python ->>> cli.inspect_volume('foobar') -{u'Mountpoint': u'/var/lib/docker/volumes/foobar/_data', u'Driver': u'local', u'Name': u'foobar'} -``` - -## join_swarm - -Join an existing Swarm. -See the [Swarm documentation](swarm.md#clientjoin_swarm). - -## kill - -Kill a container or send a signal to a container. - -**Params**: - -* container (str): The container to kill -* signal (str or int): The signal to send. Defaults to `SIGKILL` - -## leave_swarm - -Leave the current Swarm. -See the [Swarm documentation](swarm.md#clientleave_swarm). - -## load_image - -Load an image that was previously saved using `Client.get_image` -(or `docker save`). Similar to `docker load`. - -**Params**: - -* data (binary): Image data to be loaded - -## login - -Nearly identical to the `docker login` command, but non-interactive. - -**Params**: - -* username (str): The registry username -* password (str): The plaintext password -* email (str): The email for the registry account -* registry (str): URL to the registry. Ex:`https://index.docker.io/v1/` -* reauth (bool): Whether refresh existing authentication on the docker server. -* dockercfg_path (str): Use a custom path for the .dockercfg file - (default `$HOME/.dockercfg`) - -**Returns** (dict): The response from the login request - -## logs - -Identical to the `docker logs` command. The `stream` parameter makes the `logs` -function return a blocking generator you can iterate over to retrieve log -output as it happens. - -**Params**: - -* container (str): The container to get logs from -* stdout (bool): Get STDOUT -* stderr (bool): Get STDERR -* stream (bool): Stream the response -* timestamps (bool): Show timestamps -* tail (str or int): Output specified number of lines at the end of logs: `"all"` or `number`. Default `"all"` -* since (datetime or int): Show logs since a given datetime or integer epoch (in seconds) -* follow (bool): Follow log output - -**Returns** (generator or str): - -## networks - -List networks currently registered by the docker daemon. Similar to the `docker networks ls` command. - -**Params** - -* names (list): List of names to filter by -* ids (list): List of ids to filter by - -The above are combined to create a filters dict. - -**Returns** (dict): List of network objects. - -## nodes - -List Swarm nodes. See the [Swarm documentation](swarm.md#clientnodes). - -## pause - -Pauses all processes within a container. - -**Params**: - -* container (str): The container to pause - - -## ping - -Hits the `/_ping` endpoint of the remote API and returns the result. An -exception will be raised if the endpoint isn't responding. - -**Returns** (bool) - -## port -Lookup the public-facing port that is NAT-ed to `private_port`. Identical to -the `docker port` command. - -**Params**: - -* container (str): The container to look up -* private_port (int): The private port to inspect - -**Returns** (list of dict): The mapping for the host ports - -```bash -$ docker run -d -p 80:80 ubuntu:14.04 /bin/sleep 30 -7174d6347063a83f412fad6124c99cffd25ffe1a0807eb4b7f9cec76ac8cb43b -``` -```python ->>> cli.port('7174d6347063', 80) -[{'HostIp': '0.0.0.0', 'HostPort': '80'}] -``` - -## pull - -Identical to the `docker pull` command. - -**Params**: - -* repository (str): The repository to pull -* tag (str): The tag to pull -* stream (bool): Stream the output as a generator -* insecure_registry (bool): Use an insecure registry -* auth_config (dict): Override the credentials that Client.login has set for this request - `auth_config` should contain the `username` and `password` keys to be valid. - -**Returns** (generator or str): The output - -```python ->>> from docker import Client ->>> cli = Client(base_url='tcp://127.0.0.1:2375') ->>> for line in cli.pull('busybox', stream=True): -... print(json.dumps(json.loads(line), indent=4)) -{ - "status": "Pulling image (latest) from busybox", - "progressDetail": {}, - "id": "e72ac664f4f0" -} -{ - "status": "Pulling image (latest) from busybox, endpoint: ...", - "progressDetail": {}, - "id": "e72ac664f4f0" -} -``` - -## push - -Push an image or a repository to the registry. Identical to the `docker push` -command. - -**Params**: - -* repository (str): The repository to push to -* tag (str): An optional tag to push -* stream (bool): Stream the output as a blocking generator -* insecure_registry (bool): Use `http://` to connect to the registry -* auth_config (dict): Override the credentials that Client.login has set for this request - `auth_config` should contain the `username` and `password` keys to be valid. - -**Returns** (generator or str): The output of the upload - -```python ->>> from docker import Client ->>> cli = Client(base_url='tcp://127.0.0.1:2375') ->>> response = [line for line in cli.push('yourname/app', stream=True)] ->>> response -['{"status":"Pushing repository yourname/app (1 tags)"}\\n', - '{"status":"Pushing","progressDetail":{},"id":"511136ea3c5a"}\\n', - '{"status":"Image already pushed, skipping","progressDetail":{}, - "id":"511136ea3c5a"}\\n', - ... - '{"status":"Pushing tag for rev [918af568e6e5] on { - https://cdn-registry-1.docker.io/v1/repositories/ - yourname/app/tags/latest}"}\\n'] -``` - -## put_archive - -Insert a file or folder in an existing container using a tar archive as source. - -**Params**: - -* container (str): The container where the file(s) will be extracted -* path (str): Path inside the container where the file(s) will be extracted. - Must exist. -* data (bytes): tar data to be extracted - -**Returns** (bool): True if the call succeeds. `docker.errors.APIError` will -be raised if an error occurs. - -## remove_container - -Remove a container. Similar to the `docker rm` command. - -**Params**: - -* container (str): The container to remove -* v (bool): Remove the volumes associated with the container -* link (bool): Remove the specified link and not the underlying container -* force (bool): Force the removal of a running container (uses SIGKILL) - -## remove_image - -Remove an image. Similar to the `docker rmi` command. - -**Params**: - -* image (str): The image to remove -* force (bool): Force removal of the image -* noprune (bool): Do not delete untagged parents - -## remove_network - -Remove a network. Similar to the `docker network rm` command. - -**Params**: - -* net_id (str): The network's id - -Failure to remove will raise a `docker.errors.APIError` exception. - -## remove_service - -Remove a service, similar to the `docker service rm` command. See the -[services documentation](services.md#clientremove_service) for details. - -## remove_volume - -Remove a volume. Similar to the `docker volume rm` command. - -**Params**: - -* name (str): The volume's name - -Failure to remove will raise a `docker.errors.APIError` exception. - -## rename - -Rename a container. Similar to the `docker rename` command. - -**Params**: - -* container (str): ID of the container to rename -* name (str): New name for the container - -## resize - -Resize the tty session. - -**Params**: - -* container (str or dict): The container to resize -* height (int): Height of tty session -* width (int): Width of tty session - -## restart - -Restart a container. Similar to the `docker restart` command. - -If `container` a dict, the `Id` key is used. - -**Params**: - -* container (str or dict): The container to restart -* timeout (int): Number of seconds to try to stop for before killing the -container. Once killed it will then be restarted. Default is 10 seconds. - -## search -Identical to the `docker search` command. - -**Params**: - -* term (str): A term to search for - -**Returns** (list of dicts): The response of the search - -```python ->>> from docker import Client ->>> cli = Client(base_url='tcp://127.0.0.1:2375') ->>> response = cli.search('nginx') ->>> response[:2] -[{'description': 'Official build of Nginx.', - 'is_official': True, - 'is_trusted': False, - 'name': 'nginx', - 'star_count': 266}, - {'description': 'Trusted automated Nginx (http://nginx.org/) ...', - 'is_official': False, - 'is_trusted': True, - 'name': 'dockerfile/nginx', - 'star_count': 60}, - ... -``` - -## services - -List services, similar to the `docker service ls` command. See the -[services documentation](services.md#clientservices) for details. - -## start - -Similar to the `docker start` command, but doesn't support attach options. Use -`.logs()` to recover `stdout`/`stderr`. - -**Params**: - -* container (str): The container to start - -**Deprecation warning:** For API version > 1.15, it is highly recommended to - provide host config options in the - [`host_config` parameter of `create_container`](#create_container) - -```python ->>> from docker import Client ->>> cli = Client(base_url='tcp://127.0.0.1:2375') ->>> container = cli.create_container( -... image='busybox:latest', -... command='/bin/sleep 30') ->>> response = cli.start(container=container.get('Id')) ->>> print(response) -None -``` - -## stats - -The Docker API parallel to the `docker stats` command. -This will stream statistics for a specific container. - -**Params**: - -* container (str): The container to stream statistics for -* decode (bool): If set to true, stream will be decoded into dicts on the - fly. False by default. -* stream (bool): If set to false, only the current stats will be returned - instead of a stream. True by default. - -```python ->>> from docker import Client ->>> cli = Client(base_url='tcp://127.0.0.1:2375') ->>> stats_obj = cli.stats('elasticsearch') ->>> for stat in stats_obj: ->>> print(stat) -{"read":"2015-02-11T21:47:30.49388286+02:00","networks":{"eth0":{"rx_bytes":648,"rx_packets":8 ... -... -... -... -``` - -## stop - -Stops a container. Similar to the `docker stop` command. - -**Params**: - -* container (str): The container to stop -* timeout (int): Timeout in seconds to wait for the container to stop before -sending a `SIGKILL`. Default: 10 - -## tag - -Tag an image into a repository. Identical to the `docker tag` command. - -**Params**: - -* image (str): The image to tag -* repository (str): The repository to set for the tag -* tag (str): The tag name -* force (bool): Force - -**Returns** (bool): True if successful - -## tasks - -Retrieve a list of tasks. - -**Params**: - -* filters (dict): A map of filters to process on the tasks list. Valid filters: - `id`, `name`, `service`, `node`, `label` and `desired-state`. - -**Returns** (list): List of task dictionaries. - -## top -Display the running processes of a container. - -**Params**: - -* container (str): The container to inspect -* ps_args (str): An optional arguments passed to ps (e.g., aux) - -**Returns** (str): The output of the top - -```python ->>> from docker import Client ->>> cli = Client(base_url='tcp://127.0.0.1:2375') ->>> cli.create_container('busybox:latest', '/bin/sleep 30', name='sleeper') ->>> cli.start('sleeper') ->>> cli.top('sleeper') -{'Processes': [['952', 'root', '/bin/sleep 30']], - 'Titles': ['PID', 'USER', 'COMMAND']} -``` - -## unpause - -Unpause all processes within a container. - -**Params**: - -* container (str): The container to unpause - -## update_container - -Update resource configs of one or more containers. - -**Params**: - -* container (str): The container to inspect -* blkio_weight (int): Block IO (relative weight), between 10 and 1000 -* cpu_period (int): Limit CPU CFS (Completely Fair Scheduler) period -* cpu_quota (int): Limit CPU CFS (Completely Fair Scheduler) quota -* cpu_shares (int): CPU shares (relative weight) -* cpuset_cpus (str): CPUs in which to allow execution -* cpuset_mems (str): MEMs in which to allow execution -* mem_limit (int or str): Memory limit -* mem_reservation (int or str): Memory soft limit -* memswap_limit (int or str): Total memory (memory + swap), -1 to disable swap -* kernel_memory (int or str): Kernel memory limit -* restart_policy (dict): Restart policy dictionary - -**Returns** (dict): Dictionary containing a `Warnings` key. - -## update_node - -Update a node. -See the [Swarm documentation](swarm.md#clientupdate_node). - -## update_service - -Update a service, similar to the `docker service update` command. See the -[services documentation](services.md#clientupdate_service) for details. - -## update_swarm - -Update the current Swarm. -See the [Swarm documentation](swarm.md#clientupdate_swarm). - -## version - -Nearly identical to the `docker version` command. - -**Returns** (dict): The server version information - -```python ->>> from docker import Client ->>> cli = Client(base_url='tcp://127.0.0.1:2375') ->>> cli.version() -{ - "KernelVersion": "3.16.4-tinycore64", - "Arch": "amd64", - "ApiVersion": "1.15", - "Version": "1.3.0", - "GitCommit": "c78088f", - "Os": "linux", - "GoVersion": "go1.3.3" -} -``` - -## volumes - -List volumes currently registered by the docker daemon. Similar to the `docker volume ls` command. - -**Params** - -* filters (dict): Server-side list filtering options. - -**Returns** (dict): Dictionary with list of volume objects as value of the `Volumes` key. - -```python ->>> cli.volumes() -{u'Volumes': [ - {u'Mountpoint': u'/var/lib/docker/volumes/foobar/_data', u'Driver': u'local', u'Name': u'foobar'}, - {u'Mountpoint': u'/var/lib/docker/volumes/baz/_data', u'Driver': u'local', u'Name': u'baz'} -]} -``` - -## wait -Identical to the `docker wait` command. Block until a container stops, then -return its exit code. Returns the value `-1` if the API responds without a -`StatusCode` attribute. - -If `container` is a dict, the `Id` key is used. - -If the timeout value is exceeded, a `requests.exceptions.ReadTimeout` -exception will be raised. - -**Params**: - -* container (str or dict): The container to wait on -* timeout (int): Request timeout - -**Returns** (int): The exit code of the container - - - - -**** - -## Version mismatch - -You may encounter an error like this: - -```text -client is newer than server (client API version: 1.24, server API version: 1.23) -``` - -To fix this, you have to either supply the exact version your server supports -when instantiating the `Client`: - -```python -client = docker.Client(version="1.23") -``` - -or let the client automatically detect the newest version server supports: - -```python -client = docker.Client(version="auto") -``` diff --git a/docs/api.rst b/docs/api.rst new file mode 100644 index 0000000000..97db83945c --- /dev/null +++ b/docs/api.rst @@ -0,0 +1,109 @@ +Low-level API +============= + +The main object-orientated API is built on top of :py:class:`APIClient`. Each method on :py:class:`APIClient` maps one-to-one with a REST API endpoint, and returns the response that the API responds with. + +It's possible to use :py:class:`APIClient` directly. Some basic things (e.g. running a container) consist of several API calls and are complex to do with the low-level API, but it's useful if you need extra flexibility and power. + +.. py:module:: docker.api + +.. autoclass:: docker.api.client.APIClient + +Containers +---------- + +.. py:module:: docker.api.container + +.. rst-class:: hide-signature +.. autoclass:: ContainerApiMixin + :members: + :undoc-members: + +.. py:module:: docker.api.image + +Images +------ + +.. py:module:: docker.api.image + +.. rst-class:: hide-signature +.. autoclass:: ImageApiMixin + :members: + :undoc-members: + +Building images +--------------- + +.. py:module:: docker.api.build + +.. rst-class:: hide-signature +.. autoclass:: BuildApiMixin + :members: + :undoc-members: + +Networks +-------- + +.. rst-class:: hide-signature +.. autoclass:: docker.api.network.NetworkApiMixin + :members: + :undoc-members: + +Utilities +~~~~~~~~~ + +These functions are available under ``docker.utils`` to create arguments +for :py:meth:`create_network`: + +.. autofunction:: docker.utils.create_ipam_config +.. autofunction:: docker.utils.create_ipam_pool + +Volumes +------- + +.. py:module:: docker.api.volume + +.. rst-class:: hide-signature +.. autoclass:: VolumeApiMixin + :members: + :undoc-members: + +Executing commands in containers +-------------------------------- + +.. py:module:: docker.api.exec_api + +.. rst-class:: hide-signature +.. autoclass:: ExecApiMixin + :members: + :undoc-members: + +Swarms +------ + +.. py:module:: docker.api.swarm + +.. rst-class:: hide-signature +.. autoclass:: SwarmApiMixin + :members: + :undoc-members: + +Services +-------- + +.. py:module:: docker.api.service + +.. rst-class:: hide-signature +.. autoclass:: ServiceApiMixin + :members: + :undoc-members: + +The Docker daemon +----------------- + +.. py:module:: docker.api.daemon + +.. rst-class:: hide-signature +.. autoclass:: DaemonApiMixin + :members: + :undoc-members: diff --git a/docs/change_log.md b/docs/change-log.md similarity index 99% rename from docs/change_log.md rename to docs/change-log.md index e32df1e974..a7bb0b08ca 100644 --- a/docs/change_log.md +++ b/docs/change-log.md @@ -1,4 +1,4 @@ -Change Log +Change log ========== 1.10.3 diff --git a/docs/client.rst b/docs/client.rst new file mode 100644 index 0000000000..cd058fcc91 --- /dev/null +++ b/docs/client.rst @@ -0,0 +1,30 @@ +Client +====== +.. py:module:: docker.client + + +Creating a client +----------------- + +To communicate with the Docker daemon, you first need to instantiate a client. The easiest way to do that is by calling the function :py:func:`~docker.client.from_env`. It can also be configured manually by instantiating a :py:class:`~docker.client.Client` class. + +.. autofunction:: from_env() + +Client reference +---------------- + +.. autoclass:: Client() + + .. autoattribute:: containers + .. autoattribute:: images + .. autoattribute:: networks + .. autoattribute:: nodes + .. autoattribute:: services + .. autoattribute:: swarm + .. autoattribute:: volumes + + .. automethod:: events() + .. automethod:: info() + .. automethod:: login() + .. automethod:: ping() + .. automethod:: version() diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000000..4901279619 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,365 @@ +# -*- coding: utf-8 -*- +# +# docker-sdk-python documentation build configuration file, created by +# sphinx-quickstart on Wed Sep 14 15:48:58 2016. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import datetime +import os +import sys +sys.path.insert(0, os.path.abspath('..')) + + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.napoleon', +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + + +from recommonmark.parser import CommonMarkParser + +source_parsers = { + '.md': CommonMarkParser, +} + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +source_suffix = ['.rst', '.md'] +# source_suffix = '.md' + +# The encoding of source files. +# +# source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'Docker SDK for Python' +year = datetime.datetime.now().year +copyright = u'%d Docker Inc' % year +author = u'Docker Inc' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = u'2.0' +# The full version, including alpha/beta/rc tags. +release = u'2.0' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +# +# today = '' +# +# Else, today_fmt is used as the format for a strftime call. +# +# today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This patterns also effect to html_static_path and html_extra_path +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + +# The reST default role (used for this markup: `text`) to use for all +# documents. +# +# default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +# +# add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +# +add_module_names = False + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +# +# show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +# modindex_common_prefix = [] + +# If true, keep warnings as "system message" paragraphs in the built documents. +# keep_warnings = False + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = False + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'alabaster' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +html_theme_options = { + 'description': 'A Python library for the Docker Remote API', + 'fixed_sidebar': True, +} + +# Add any paths that contain custom themes here, relative to this directory. +# html_theme_path = [] + +# The name for this set of Sphinx documents. +# " v documentation" by default. +# +# html_title = u'docker-sdk-python v2.0' + +# A shorter title for the navigation bar. Default is the same as html_title. +# +# html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +# +# html_logo = None + +# The name of an image file (relative to this directory) to use as a favicon of +# the docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +# +# html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# Add any extra paths that contain custom files (such as robots.txt or +# .htaccess) here, relative to this directory. These files are copied +# directly to the root of the documentation. +# +# html_extra_path = [] + +# If not None, a 'Last updated on:' timestamp is inserted at every page +# bottom, using the given strftime format. +# The empty string is equivalent to '%b %d, %Y'. +# +# html_last_updated_fmt = None + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +# +# html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +# +html_sidebars = { + '**': [ + 'about.html', + 'navigation.html', + 'searchbox.html', + ] +} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +# +# html_additional_pages = {} + +# If false, no module index is generated. +# +# html_domain_indices = True + +# If false, no index is generated. +# +# html_use_index = True + +# If true, the index is split into individual pages for each letter. +# +# html_split_index = False + +# If true, links to the reST sources are added to the pages. +# +# html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +# +# html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +# +# html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +# +# html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +# html_file_suffix = None + +# Language to be used for generating the HTML full-text search index. +# Sphinx supports the following languages: +# 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja' +# 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr', 'zh' +# +# html_search_language = 'en' + +# A dictionary with options for the search language support, empty by default. +# 'ja' uses this config value. +# 'zh' user can custom change `jieba` dictionary path. +# +# html_search_options = {'type': 'default'} + +# The name of a javascript file (relative to the configuration directory) that +# implements a search results scorer. If empty, the default will be used. +# +# html_search_scorer = 'scorer.js' + +# Output file base name for HTML help builder. +htmlhelp_basename = 'docker-sdk-pythondoc' + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'docker-sdk-python.tex', u'docker-sdk-python Documentation', + u'Docker Inc.', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +# +# latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +# +# latex_use_parts = False + +# If true, show page references after internal links. +# +# latex_show_pagerefs = False + +# If true, show URL addresses after external links. +# +# latex_show_urls = False + +# Documents to append as an appendix to all manuals. +# +# latex_appendices = [] + +# It false, will not define \strong, \code, itleref, \crossref ... but only +# \sphinxstrong, ..., \sphinxtitleref, ... To help avoid clash with user added +# packages. +# +# latex_keep_old_macro_names = True + +# If false, no module index is generated. +# +# latex_domain_indices = True + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'docker-sdk-python', u'docker-sdk-python Documentation', + [author], 1) +] + +# If true, show URL addresses after external links. +# +# man_show_urls = False + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'docker-sdk-python', u'docker-sdk-python Documentation', + author, 'docker-sdk-python', 'One line description of project.', + 'Miscellaneous'), +] + +# Documents to append as an appendix to all manuals. +# +# texinfo_appendices = [] + +# If false, no module index is generated. +# +# texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +# +# texinfo_show_urls = 'footnote' + +# If true, do not generate a @detailmenu in the "Top" node's menu. +# +# texinfo_no_detailmenu = False + + +# Napoleon settings +napoleon_google_docstring = True +napoleon_numpy_docstring = False diff --git a/docs/containers.rst b/docs/containers.rst new file mode 100644 index 0000000000..eb51ae4c97 --- /dev/null +++ b/docs/containers.rst @@ -0,0 +1,51 @@ +Containers +========== + +.. py:module:: docker.models.containers + +Run and manage containers on the server. + +Methods available on ``client.containers``: + +.. rst-class:: hide-signature +.. autoclass:: ContainerCollection + + .. automethod:: run(image, command=None, **kwargs) + .. automethod:: create(image, command=None, **kwargs) + .. automethod:: get(id_or_name) + .. automethod:: list(**kwargs) + +Container objects +----------------- + +.. autoclass:: Container() + + .. autoattribute:: id + .. autoattribute:: short_id + .. autoattribute:: name + .. autoattribute:: status + .. py:attribute:: attrs + + The raw representation of this object from the server. + + .. automethod:: attach + .. automethod:: attach_socket + .. automethod:: commit + .. automethod:: diff + .. automethod:: exec_run + .. automethod:: export + .. automethod:: get_archive + .. automethod:: kill + .. automethod:: logs + .. automethod:: pause + .. automethod:: put_archive + .. automethod:: remove + .. automethod:: rename + .. automethod:: resize + .. automethod:: restart + .. automethod:: start + .. automethod:: stats + .. automethod:: stop + .. automethod:: top + .. automethod:: unpause + .. automethod:: update diff --git a/docs/host-devices.md b/docs/host-devices.md deleted file mode 100644 index 150a686255..0000000000 --- a/docs/host-devices.md +++ /dev/null @@ -1,29 +0,0 @@ -# Access to devices on the host - -If you need to directly expose some host devices to a container, you can use -the devices parameter in the `host_config` param in `Client.create_container` -as shown below: - -```python -cli.create_container( - 'busybox', 'true', host_config=cli.create_host_config(devices=[ - '/dev/sda:/dev/xvda:rwm' - ]) -) -``` - -Each string is a single mapping using the following format: -`::` -The above example allows the container to have read-write access to -the host's `/dev/sda` via a node named `/dev/xvda` inside the container. - -As a more verbose alternative, each host device definition can be specified as -a dictionary with the following keys: - -```python -{ - 'PathOnHost': '/dev/sda1', - 'PathInContainer': '/dev/xvda', - 'CgroupPermissions': 'rwm' -} -``` diff --git a/docs/hostconfig.md b/docs/hostconfig.md deleted file mode 100644 index f989c7d6e0..0000000000 --- a/docs/hostconfig.md +++ /dev/null @@ -1,142 +0,0 @@ -# HostConfig object - -The Docker Remote API introduced [support for HostConfig in version 1.15](http://docs.docker.com/reference/api/docker_remote_api_v1.15/#create-a-container). -This object contains all the parameters you could previously pass to `Client.start`. -*It is highly recommended that users pass the HostConfig in the `host_config`* -*param of `Client.create_container` instead of `Client.start`* - -## HostConfig helper - -### Client.create_host_config - -Creates a HostConfig dictionary to be used with `Client.create_container`. - -`binds` allows to bind a directory in the host to the container. See [Using -volumes](volumes.md) for more information. - -`port_bindings` exposes container ports to the host. -See [Port bindings](port-bindings.md) for more information. - -`lxc_conf` allows to pass LXC configuration options using a dictionary. - -`privileged` starts the container in privileged mode. - -[Links](http://docs.docker.io/en/latest/use/working_with_links_names/) can be -specified with the `links` argument. They can either be specified as a -dictionary mapping name to alias or as a list of `(name, alias)` tuples. - -`dns` and `volumes_from` are only available if they are used with version v1.10 -of docker remote API. Otherwise they are ignored. - -`network_mode` is available since v1.11 and sets the Network mode for the -container ('bridge': creates a new network stack for the container on the -Docker bridge, 'none': no networking for this container, 'container:[name|id]': -reuses another container network stack, 'host': use the host network stack -inside the container or any name that identifies an existing Docker network). - -`restart_policy` is available since v1.2.0 and sets the container's *RestartPolicy* -which defines the conditions under which a container should be restarted upon exit. -If no *RestartPolicy* is defined, the container will not be restarted when it exits. -The *RestartPolicy* is specified as a dict. For example, if the container -should always be restarted: -```python -{ - "MaximumRetryCount": 0, - "Name": "always" -} -``` - -It is possible to restart the container only on failure as well as limit the number -of restarts. For example: -```python -{ - "MaximumRetryCount": 5, - "Name": "on-failure" -} -``` - -`cap_add` and `cap_drop` are available since v1.2.0 and can be used to add or -drop certain capabilities. The user may specify the capabilities as an array -for example: -```python -[ - "SYS_ADMIN", - "MKNOD" -] -``` - - -**Params** - -* binds: Volumes to bind. See [Using volumes](volumes.md) for more information. -* port_bindings (dict): Port bindings. See [Port bindings](port-bindings.md) - for more information. -* lxc_conf (dict): LXC config -* oom_kill_disable (bool): Whether to disable OOM killer -* oom_score_adj (int): An integer value containing the score given to the - container in order to tune OOM killer preferences -* publish_all_ports (bool): Whether to publish all ports to the host -* links (dict or list of tuples): either as a dictionary mapping name to alias - or as a list of `(name, alias)` tuples -* privileged (bool): Give extended privileges to this container -* dns (list): Set custom DNS servers -* dns_search (list): DNS search domains -* volumes_from (str or list): List of container names or Ids to get volumes - from. Optionally a single string joining container id's with commas -* network_mode (str): One of `['bridge', 'none', 'container:', 'host']` -* restart_policy (dict): "Name" param must be one of - `['on-failure', 'always']` -* cap_add (list of str): Add kernel capabilities -* cap_drop (list of str): Drop kernel capabilities -* extra_hosts (dict): custom host-to-IP mappings (host:ip) -* read_only (bool): mount the container's root filesystem as read only -* pid_mode (str): if set to "host", use the host PID namespace inside the - container -* ipc_mode (str): Set the IPC mode for the container -* security_opt (list): A list of string values to customize labels for MLS - systems, such as SELinux. -* ulimits (list): A list of dicts or `docker.utils.Ulimit` objects. A list - of ulimits to be set in the container. -* log_config (`docker.utils.LogConfig` or dict): Logging configuration to - container -* mem_limit (str or int): Maximum amount of memory container is allowed to - consume. (e.g. `'1G'`) -* memswap_limit (str or int): Maximum amount of memory + swap a container is - allowed to consume. -* mem_swappiness (int): Tune a container's memory swappiness behavior. - Accepts number between 0 and 100. -* shm_size (str or int): Size of /dev/shm. (e.g. `'1G'`) -* cpu_group (int): The length of a CPU period in microseconds. -* cpu_period (int): Microseconds of CPU time that the container can get in a - CPU period. -* cpu_shares (int): CPU shares (relative weight) -* cpuset_cpus (str): CPUs in which to allow execution (0-3, 0,1) -* blkio_weight: Block IO weight (relative weight), accepts a weight value - between 10 and 1000. -* blkio_weight_device: Block IO weight (relative device weight) in the form of: - `[{"Path": "device_path", "Weight": weight}]` -* device_read_bps: Limit read rate (bytes per second) from a device in the - form of: `[{"Path": "device_path", "Rate": rate}]` -* device_write_bps: Limit write rate (bytes per second) from a device. -* device_read_iops: Limit read rate (IO per second) from a device. -* device_write_iops: Limit write rate (IO per second) from a device. -* group_add (list): List of additional group names and/or IDs that the - container process will run as. -* devices (list): Host device bindings. See [host devices](host-devices.md) - for more information. -* tmpfs: Temporary filesystems to mount. See [Using tmpfs](tmpfs.md) for more - information. -* sysctls (dict): Kernel parameters to set in the container. -* userns_mode (str): Sets the user namespace mode for the container when user - namespace remapping option is enabled. Supported values are: `host` -* pids_limit (int): Tune a container’s pids limit. Set -1 for unlimited. -* isolation (str): Isolation technology to use. Default: `None`. - -**Returns** (dict) HostConfig dictionary - -```python ->>> from docker import Client ->>> cli = Client() ->>> cli.create_host_config(privileged=True, cap_drop=['MKNOD'], volumes_from=['nostalgic_newton']) -{'CapDrop': ['MKNOD'], 'LxcConf': None, 'Privileged': True, 'VolumesFrom': ['nostalgic_newton'], 'PublishAllPorts': False} -``` diff --git a/docs/images.rst b/docs/images.rst new file mode 100644 index 0000000000..7572c2d6a5 --- /dev/null +++ b/docs/images.rst @@ -0,0 +1,39 @@ +Images +====== + +.. py:module:: docker.models.images + +Manage images on the server. + +Methods available on ``client.images``: + +.. rst-class:: hide-signature +.. py:class:: ImageCollection + + .. automethod:: build + .. automethod:: get + .. automethod:: list(**kwargs) + .. automethod:: load + .. automethod:: pull + .. automethod:: push + .. automethod:: remove + .. automethod:: search + + +Image objects +------------- + +.. autoclass:: Image() + + .. autoattribute:: id + .. autoattribute:: short_id + .. autoattribute:: tags + .. py:attribute:: attrs + + The raw representation of this object from the server. + + + .. automethod:: history + .. automethod:: reload + .. automethod:: save + .. automethod:: tag diff --git a/docs/index.md b/docs/index.md deleted file mode 100644 index 5b851f0a4b..0000000000 --- a/docs/index.md +++ /dev/null @@ -1,15 +0,0 @@ -# docker-py documentation - -An API client for docker written in Python - -## Installation - -Our latest stable is always available on PyPi. - - pip install docker-py - -## Documentation -Full documentation is available in the `/docs/` directory. - -## License -Docker is licensed under the Apache License, Version 2.0. See LICENSE for full license text diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000000..7eadf4c7e1 --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,93 @@ +Docker SDK for Python +===================== + +A Python library for the Docker Remote API. It lets you do anything the ``docker`` command does, but from within Python apps – run containers, manage containers, manage Swarms, etc. + +For more information about the Remote API, `see its documentation `_. + +Installation +------------ + +The latest stable version `is available on PyPi `_. Either add ``docker`` to your ``requirements.txt`` file or install with pip:: + + pip install docker + +Getting started +--------------- + +To talk to a Docker daemon, you first need to instantiate a client. You can use :py:func:`~docker.client.from_env` to connect using the default socket or the configuration in your environment: + +.. code-block:: python + + import docker + client = docker.from_env() + +You can now run containers: + +.. code-block:: python + + >>> client.containers.run("ubuntu", "echo hello world") + 'hello world\n' + +You can run containers in the background: + +.. code-block:: python + + >>> client.containers.run("bfirsh/reticulate-splines", detach=True) + + +You can manage containers: + +.. code-block:: python + + >>> client.containers.list() + [, , ...] + + >>> container = client.containers.get('45e6d2de7c54') + + >>> container.attrs['Config']['Image'] + "bfirsh/reticulate-splines" + + >>> container.logs() + "Reticulating spline 1...\n" + + >>> container.stop() + +You can stream logs: + +.. code-block:: python + + >>> for line in container.logs(stream=True): + ... print line.strip() + Reticulating spline 2... + Reticulating spline 3... + ... + +You can manage images: + +.. code-block:: python + + >>> client.images.pull('nginx') + + + >>> client.images.list() + [, , ...] + +That's just a taster of what you can do with the Docker SDK for Python. For more, :doc:`take a look at the reference `. + +.. toctree:: + :hidden: + :maxdepth: 2 + + Home + client + containers + images + networks + nodes + services + swarm + volumes + api + tls + change-log diff --git a/docs/machine.md b/docs/machine.md deleted file mode 100644 index 6c0bcbbe72..0000000000 --- a/docs/machine.md +++ /dev/null @@ -1,26 +0,0 @@ -# Using with Docker Toolbox and Machine - -In development, Docker recommends using -[Docker Toolbox](https://www.docker.com/products/docker-toolbox) to set up -Docker. It includes a tool called Machine which will create a VM running -Docker Engine and point your shell at it using environment variables. - -To configure docker-py with these environment variables - -First use Machine to set up the environment variables: -```bash -$ eval "$(docker-machine env)" -``` - -You can then use docker-py like this: -```python -import docker -client = docker.from_env(assert_hostname=False) -print client.version() -``` - -**Note:** This snippet is disabling TLS hostname checking with -`assert\_hostname=False`. Machine provides us with the exact certificate -the server is using so this is safe. If you are not using Machine and verifying -the host against a certificate authority, you'll want to enable hostname -verification. diff --git a/docs/networks.md b/docs/networks.md deleted file mode 100644 index fb0e9f420c..0000000000 --- a/docs/networks.md +++ /dev/null @@ -1,177 +0,0 @@ -# Using Networks - -## Network creation - -With the release of Docker 1.9 you can now manage custom networks. - - -Here you can see how to create a network named `network1` using -the `bridge` driver - -```python -docker_client.create_network("network1", driver="bridge") -``` - -You can also create more advanced networks with custom IPAM configurations. -For example, setting the subnet to `192.168.52.0/24` and gateway address -to `192.168.52.254` - -```python -ipam_pool = docker.utils.create_ipam_pool( - subnet='192.168.52.0/24', - gateway='192.168.52.254' -) -ipam_config = docker.utils.create_ipam_config( - pool_configs=[ipam_pool] -) - -docker_client.create_network("network1", driver="bridge", ipam=ipam_config) -``` - -By default, when you connect a container to an overlay network, Docker also -connects a bridge network to it to provide external connectivity. If you want -to create an externally isolated overlay network, with Docker 1.10 you can -create an internal network. - -```python - -docker_client.create_network("network1", driver="bridge", internal=True) -``` - -## Container network configuration - -In order to specify which network a container will be connected to, and -additional configuration, use the `networking_config` parameter in -`Client.create_container`. Note that at the time of creation, you can -only connect a container to a single network. Later on, you may create more -connections using `Client.connect_container_to_network`. - - -```python -networking_config = docker_client.create_networking_config({ - 'network1': docker_client.create_endpoint_config( - ipv4_address='172.28.0.124', - aliases=['foo', 'bar'], - links=['container2'] - ) -}) - -ctnr = docker_client.create_container( - img, command, networking_config=networking_config -) - -``` - -## Network API documentation - -### Client.create_networking_config - -Create a networking config dictionary to be used as the `networking_config` -parameter in `Client.create_container_config` - -**Params**: - -* endpoints_config (dict): A dictionary of `network_name -> endpoint_config` - relationships. Values should be endpoint config dictionaries created by - `Client.create_endpoint_config`. Defaults to `None` (default config). - -**Returns** A networking config dictionary. - -```python - -docker_client.create_network('network1') - -networking_config = docker_client.create_networking_config({ - 'network1': docker_client.create_endpoint_config() -}) - -container = docker_client.create_container( - img, command, networking_config=networking_config -) -``` - - -### Client.create_endpoint_config - -Create an endpoint config dictionary to be used with -`Client.create_networking_config`. - -**Params**: - -* aliases (list): A list of aliases for this endpoint. Names in that list can - be used within the network to reach the container. Defaults to `None`. -* links (list): A list of links for this endpoint. Containers declared in this - list will be [linked](https://docs.docker.com/engine/userguide/networking/work-with-networks/#linking-containers-in-user-defined-networks) - to this container. Defaults to `None`. -* ipv4_address (str): The IP address of this container on the network, - using the IPv4 protocol. Defaults to `None`. -* ipv6_address (str): The IP address of this container on the network, - using the IPv6 protocol. Defaults to `None`. -* link_local_ips (list): A list of link-local (IPv4/IPv6) addresses. - -**Returns** An endpoint config dictionary. - -```python -endpoint_config = docker_client.create_endpoint_config( - aliases=['web', 'app'], - links=['app_db'], - ipv4_address='132.65.0.123' -) - -docker_client.create_network('network1') -networking_config = docker_client.create_networking_config({ - 'network1': endpoint_config -}) -container = docker_client.create_container( - img, command, networking_config=networking_config -) -``` -### docker.utils.create_ipam_config - -Create an IPAM (IP Address Management) config dictionary to be used with -`Client.create_network`. - - -**Params**: - -* driver (str): The IPAM driver to use. Defaults to `'default'`. -* pool_configs (list): A list of pool configuration dictionaries as created - by `docker.utils.create_ipam_pool`. Defaults to empty list. - -**Returns** An IPAM config dictionary - -```python -ipam_config = docker.utils.create_ipam_config(driver='default') -network = docker_client.create_network('network1', ipam=ipam_config) -``` - -### docker.utils.create_ipam_pool - -Create an IPAM pool config dictionary to be added to the `pool_configs` param -in `docker.utils.create_ipam_config`. - -**Params**: - -* subnet (str): Custom subnet for this IPAM pool using the CIDR notation. - Defaults to `None`. -* iprange (str): Custom IP range for endpoints in this IPAM pool using the - CIDR notation. Defaults to `None`. -* gateway (str): Custom IP address for the pool's gateway. -* aux_addresses (dict): A dictionary of `key -> ip_address` relationships - specifying auxiliary addresses that need to be allocated by the - IPAM driver. - -**Returns** An IPAM pool config dictionary - -```python -ipam_pool = docker.utils.create_ipam_pool( - subnet='124.42.0.0/16', - iprange='124.42.0.0/24', - gateway='124.42.0.254', - aux_addresses={ - 'reserved1': '124.42.1.1' - } -) -ipam_config = docker.utils.create_ipam_config(pool_configs=[ipam_pool]) -network = docker_client.create_network('network1', ipam=ipam_config) -``` diff --git a/docs/networks.rst b/docs/networks.rst new file mode 100644 index 0000000000..f6de38bd71 --- /dev/null +++ b/docs/networks.rst @@ -0,0 +1,33 @@ +Networks +======== + +.. py:module:: docker.models.networks + +Create and manage networks on the server. For more information about networks, `see the Engine documentation `_. + +Methods available on ``client.networks``: + +.. rst-class:: hide-signature +.. py:class:: NetworkCollection + + .. automethod:: create + .. automethod:: get + .. automethod:: list + +Network objects +----------------- + +.. autoclass:: Network() + + .. autoattribute:: id + .. autoattribute:: short_id + .. autoattribute:: name + .. autoattribute:: containers + .. py:attribute:: attrs + + The raw representation of this object from the server. + + .. automethod:: connect + .. automethod:: disconnect + .. automethod:: reload + .. automethod:: remove diff --git a/docs/nodes.rst b/docs/nodes.rst new file mode 100644 index 0000000000..8ef1e20b29 --- /dev/null +++ b/docs/nodes.rst @@ -0,0 +1,30 @@ +Nodes +===== + +.. py:module:: docker.models.nodes + +Get and list nodes in a swarm. Before you can use these methods, you first need to :doc:`join or initialize a swarm `. + +Methods available on ``client.nodes``: + +.. rst-class:: hide-signature +.. py:class:: NodeCollection + + .. automethod:: get(id_or_name) + .. automethod:: list(**kwargs) + +Node objects +------------ + +.. autoclass:: Node() + + .. autoattribute:: id + .. autoattribute:: short_id + .. py:attribute:: attrs + + The raw representation of this object from the server. + + .. autoattribute:: version + + .. automethod:: reload + .. automethod:: update diff --git a/docs/port-bindings.md b/docs/port-bindings.md deleted file mode 100644 index d31760c38c..0000000000 --- a/docs/port-bindings.md +++ /dev/null @@ -1,58 +0,0 @@ -# Port bindings -Port bindings is done in two parts. Firstly, by providing a list of ports to -open inside the container in the `Client().create_container()` method. -Bindings are declared in the `host_config` parameter. - -```python -container_id = cli.create_container( - 'busybox', 'ls', ports=[1111, 2222], - host_config=cli.create_host_config(port_bindings={ - 1111: 4567, - 2222: None - }) -) -``` - - -You can limit the host address on which the port will be exposed like such: - -```python -cli.create_host_config(port_bindings={1111: ('127.0.0.1', 4567)}) -``` - -Or without host port assignment: - -```python -cli.create_host_config(port_bindings={1111: ('127.0.0.1',)}) -``` - -If you wish to use UDP instead of TCP (default), you need to declare ports -as such in both the config and host config: - -```python -container_id = cli.create_container( - 'busybox', 'ls', ports=[(1111, 'udp'), 2222], - host_config=cli.create_host_config(port_bindings={ - '1111/udp': 4567, 2222: None - }) -) -``` - -To bind multiple host ports to a single container port, use the following syntax: - -```python -cli.create_host_config(port_bindings={ - 1111: [1234, 4567] -}) -``` - -You can also bind multiple IPs to a single container port: - -```python -cli.create_host_config(port_bindings={ - 1111: [ - ('192.168.0.100', 1234), - ('192.168.0.101', 1234) - ] -}) -``` diff --git a/docs/services.rst b/docs/services.rst new file mode 100644 index 0000000000..d8e528545a --- /dev/null +++ b/docs/services.rst @@ -0,0 +1,36 @@ +Services +======== + +.. py:module:: docker.models.services + +Manage services on a swarm. For more information about services, `see the Engine documentation `_. + +Before you can use any of these methods, you first need to :doc:`join or initialize a swarm `. + +Methods available on ``client.services``: + +.. rst-class:: hide-signature +.. py:class:: ServiceCollection + + .. automethod:: create + .. automethod:: get + .. automethod:: list + +Service objects +--------------- + +.. autoclass:: Service() + + .. autoattribute:: id + .. autoattribute:: short_id + .. autoattribute:: name + .. autoattribute:: version + .. py:attribute:: attrs + + The raw representation of this object from the server. + + + .. automethod:: reload + .. automethod:: remove + .. automethod:: tasks + .. automethod:: update diff --git a/docs/swarm.md b/docs/swarm.md deleted file mode 100644 index 20c3945352..0000000000 --- a/docs/swarm.md +++ /dev/null @@ -1,274 +0,0 @@ -# Swarm management - -Starting with Engine version 1.12 (API 1.24), it is possible to manage the -engine's associated Swarm cluster using the API. - -## Initializing a new Swarm - -You can initialize a new Swarm by calling `Client.init_swarm`. An advertising -address needs to be provided, usually simply by indicating which network -interface needs to be used. Advanced options are provided using the -`swarm_spec` parameter, which can easily be created using -`Client.create_swarm_spec`. - -```python -spec = client.create_swarm_spec( - snapshot_interval=5000, log_entries_for_slow_followers=1200 -) -client.init_swarm( - advertise_addr='eth0', listen_addr='0.0.0.0:5000', force_new_cluster=False, - swarm_spec=spec -) -``` - -## Joining an existing Swarm - -If you're looking to have the engine your client is connected to join an -existing Swarm, this can be accomplished by using the `Client.join_swarm` -method. You will need to provide a list of at least one remote address -corresponding to other machines already part of the swarm as well as the -`join_token`. In most cases, a `listen_addr` and `advertise_addr` for your -node are also required. - -```python -client.join_swarm( - remote_addrs=['192.168.14.221:2377'], join_token='SWMTKN-1-redacted', - listen_addr='0.0.0.0:5000', advertise_addr='eth0:5000' -) -``` - -## Leaving the Swarm - -To leave the swarm you are currently a member of, simply use -`Client.leave_swarm`. Note that if your engine is the Swarm's manager, -you will need to specify `force=True` to be able to leave. - -```python -client.leave_swarm(force=False) -``` - -## Retrieving Swarm status - -You can retrieve information about your current Swarm status by calling -`Client.inspect_swarm`. This method takes no arguments. - -```python -client.inspect_swarm() -``` - -## Listing Swarm nodes - -List all nodes that are part of the current Swarm using `Client.nodes`. -The `filters` argument allows to filter the results. - -```python -client.nodes(filters={'role': 'manager'}) -``` - -## Swarm API documentation - -### Client.init_swarm - -Initialize a new Swarm using the current connected engine as the first node. - -**Params:** - -* advertise_addr (string): Externally reachable address advertised to other - nodes. This can either be an address/port combination in the form - `192.168.1.1:4567`, or an interface followed by a port number, like - `eth0:4567`. If the port number is omitted, the port number from the listen - address is used. If `advertise_addr` is not specified, it will be - automatically detected when possible. Default: None -* listen_addr (string): Listen address used for inter-manager communication, - as well as determining the networking interface used for the VXLAN Tunnel - Endpoint (VTEP). This can either be an address/port combination in the form - `192.168.1.1:4567`, or an interface followed by a port number, like - `eth0:4567`. If the port number is omitted, the default swarm listening port - is used. Default: '0.0.0.0:2377' -* force_new_cluster (bool): Force creating a new Swarm, even if already part of - one. Default: False -* swarm_spec (dict): Configuration settings of the new Swarm. Use - `Client.create_swarm_spec` to generate a valid configuration. Default: None - -**Returns:** `True` if the request went through. Raises an `APIError` if it - fails. - -#### Client.create_swarm_spec - -Create a `docker.types.SwarmSpec` instance that can be used as the `swarm_spec` -argument in `Client.init_swarm`. - -**Params:** - -* task_history_retention_limit (int): Maximum number of tasks history stored. -* snapshot_interval (int): Number of logs entries between snapshot. -* keep_old_snapshots (int): Number of snapshots to keep beyond the current - snapshot. -* log_entries_for_slow_followers (int): Number of log entries to keep around - to sync up slow followers after a snapshot is created. -* heartbeat_tick (int): Amount of ticks (in seconds) between each heartbeat. -* election_tick (int): Amount of ticks (in seconds) needed without a leader to - trigger a new election. -* dispatcher_heartbeat_period (int): The delay for an agent to send a - heartbeat to the dispatcher. -* node_cert_expiry (int): Automatic expiry for nodes certificates. -* external_ca (dict): Configuration for forwarding signing requests to an - external certificate authority. Use `docker.types.SwarmExternalCA`. -* name (string): Swarm's name - -**Returns:** `docker.types.SwarmSpec` instance. - -#### docker.types.SwarmExternalCA - -Create a configuration dictionary for the `external_ca` argument in a -`SwarmSpec`. - -**Params:** - -* protocol (string): Protocol for communication with the external CA (currently - only “cfssl” is supported). -* url (string): URL where certificate signing requests should be sent. -* options (dict): An object with key/value pairs that are interpreted as - protocol-specific options for the external CA driver. - -### Client.inspect_node - -Retrieve low-level information about a Swarm node - -**Params:** - -* node_id (string): ID of the node to be inspected. - -**Returns:** A dictionary containing data about this node. See sample below. - -```python -{u'CreatedAt': u'2016-08-11T23:28:39.695834296Z', - u'Description': {u'Engine': {u'EngineVersion': u'1.12.0', - u'Plugins': [{u'Name': u'bridge', u'Type': u'Network'}, - {u'Name': u'host', u'Type': u'Network'}, - {u'Name': u'null', u'Type': u'Network'}, - {u'Name': u'overlay', u'Type': u'Network'}, - {u'Name': u'local', u'Type': u'Volume'}]}, - u'Hostname': u'dockerserv-1.local.net', - u'Platform': {u'Architecture': u'x86_64', u'OS': u'linux'}, - u'Resources': {u'MemoryBytes': 8052109312, u'NanoCPUs': 4000000000}}, - u'ID': u'1kqami616p23dz4hd7km35w63', - u'ManagerStatus': {u'Addr': u'10.0.131.127:2377', - u'Leader': True, - u'Reachability': u'reachable'}, - u'Spec': {u'Availability': u'active', u'Role': u'manager'}, - u'Status': {u'State': u'ready'}, - u'UpdatedAt': u'2016-08-11T23:28:39.979829529Z', - u'Version': {u'Index': 9}} - ``` - -### Client.inspect_swarm - -Retrieve information about the current Swarm. - -**Returns:** A dictionary containing information about the Swarm. See sample - below. - -```python -{u'CreatedAt': u'2016-08-04T21:26:18.779800579Z', - u'ID': u'8hk6e9wh4iq214qtbgvbp84a9', - u'JoinTokens': {u'Manager': u'SWMTKN-1-redacted-1', - u'Worker': u'SWMTKN-1-redacted-2'}, - u'Spec': {u'CAConfig': {u'NodeCertExpiry': 7776000000000000}, - u'Dispatcher': {u'HeartbeatPeriod': 5000000000}, - u'Name': u'default', - u'Orchestration': {u'TaskHistoryRetentionLimit': 10}, - u'Raft': {u'ElectionTick': 3, - u'HeartbeatTick': 1, - u'LogEntriesForSlowFollowers': 500, - u'SnapshotInterval': 10000}, - u'TaskDefaults': {}}, - u'UpdatedAt': u'2016-08-04T21:26:19.391623265Z', - u'Version': {u'Index': 11}} -``` - -### Client.join_swarm - -Join an existing Swarm. - -**Params:** - -* remote_addrs (list): Addresses of one or more manager nodes already - participating in the Swarm to join. -* join_token (string): Secret token for joining this Swarm. -* listen_addr (string): Listen address used for inter-manager communication - if the node gets promoted to manager, as well as determining the networking - interface used for the VXLAN Tunnel Endpoint (VTEP). Default: `None` -* advertise_addr (string): Externally reachable address advertised to other - nodes. This can either be an address/port combination in the form - `192.168.1.1:4567`, or an interface followed by a port number, like - `eth0:4567`. If the port number is omitted, the port number from the listen - address is used. If AdvertiseAddr is not specified, it will be automatically - detected when possible. Default: `None` - -**Returns:** `True` if the request went through. Raises an `APIError` if it - fails. - -### Client.leave_swarm - -Leave a Swarm. - -**Params:** - -* force (bool): Leave the Swarm even if this node is a manager. - Default: `False` - -**Returns:** `True` if the request went through. Raises an `APIError` if it - fails. - -### Client.nodes - -List Swarm nodes - -**Params:** - -* filters (dict): Filters to process on the nodes list. Valid filters: - `id`, `name`, `membership` and `role`. Default: `None` - -**Returns:** A list of dictionaries containing data about each swarm node. - -### Client.update_node - -Update the Node's configuration - -**Params:** - -* version (int): The version number of the node object being updated. This - is required to avoid conflicting writes. -* node_spec (dict): Configuration settings to update. Any values not provided - will be removed. See the official [Docker API documentation](https://docs.docker.com/engine/reference/api/docker_remote_api_v1.24/#/update-a-node) for more details. - Default: `None`. - -**Returns:** `True` if the request went through. Raises an `APIError` if it - fails. - -```python -node_spec = {'Availability': 'active', - 'Name': 'node-name', - 'Role': 'manager', - 'Labels': {'foo': 'bar'} - } -client.update_node(node_id='24ifsmvkjbyhk', version=8, node_spec=node_spec) -``` - -### Client.update_swarm - -Update the Swarm's configuration - -**Params:** - -* version (int): The version number of the swarm object being updated. This - is required to avoid conflicting writes. -* swarm_spec (dict): Configuration settings to update. Use - `Client.create_swarm_spec` to generate a valid configuration. - Default: `None`. -* rotate_worker_token (bool): Rotate the worker join token. Default: `False`. -* rotate_manager_token (bool): Rotate the manager join token. Default: `False`. - -**Returns:** `True` if the request went through. Raises an `APIError` if it - fails. diff --git a/docs/swarm.rst b/docs/swarm.rst new file mode 100644 index 0000000000..0c21bae1ab --- /dev/null +++ b/docs/swarm.rst @@ -0,0 +1,24 @@ +Swarm +===== + +.. py:module:: docker.models.swarm + +Manage `Docker Engine's swarm mode `_. + +To use any swarm methods, you first need to make the Engine part of a swarm. This can be done by either initializing a new swarm with :py:meth:`~Swarm.init`, or joining an existing swarm with :py:meth:`~Swarm.join`. + +These methods are available on ``client.swarm``: + +.. rst-class:: hide-signature +.. py:class:: Swarm + + .. automethod:: init() + .. automethod:: join() + .. automethod:: leave() + .. automethod:: update() + .. automethod:: reload() + + .. autoattribute:: version + .. py:attribute:: attrs + + The raw representation of this object from the server. diff --git a/docs/tls.md b/docs/tls.md deleted file mode 100644 index 147e674f76..0000000000 --- a/docs/tls.md +++ /dev/null @@ -1,86 +0,0 @@ -## Connection to daemon using HTTPS - -**Note:** *These instructions are docker-py specific. Please refer to -[http://docs.docker.com/articles/https/](http://docs.docker.com/articles/https/) -first.* - -## TLSConfig - -**Params**: - -* client_cert (tuple of str): Path to client cert, path to client key -* ca_cert (str): Path to CA cert file -* verify (bool or str): This can be `False` or a path to a CA Cert file -* ssl_version (int): A valid [SSL version]( -https://docs.python.org/3.5/library/ssl.html#ssl.PROTOCOL_TLSv1) -* assert_hostname (bool): Verify hostname of docker daemon - -### configure_client - -**Params**: - -* client: ([Client](api.md#client-api)): A client to apply this config to - - -## Authenticate server based on public/default CA pool - -```python -client = docker.Client(base_url='', tls=True) -``` - -Equivalent CLI options: -```bash -docker --tls ... -``` - -If you want to use TLS but don't want to verify the server certificate -(for example when testing with a self-signed certificate): - -```python -tls_config = docker.tls.TLSConfig(verify=False) -client = docker.Client(base_url='', tls=tls_config) -``` - -## Authenticate server based on given CA - -```python -tls_config = docker.tls.TLSConfig(ca_cert='/path/to/ca.pem') -client = docker.Client(base_url='', tls=tls_config) -``` - -Equivalent CLI options: -```bash -docker --tlsverify --tlscacert /path/to/ca.pem ... -``` - -## Authenticate with client certificate, do not authenticate server based on given CA - -```python -tls_config = docker.tls.TLSConfig( - client_cert=('/path/to/client-cert.pem', '/path/to/client-key.pem') -) -client = docker.Client(base_url='', tls=tls_config) -``` - -Equivalent CLI options: -```bash -docker --tls --tlscert /path/to/client-cert.pem --tlskey /path/to/client-key.pem ... -``` - -## Authenticate with client certificate, authenticate server based on given CA - -```python -tls_config = docker.tls.TLSConfig( - client_cert=('/path/to/client-cert.pem', '/path/to/client-key.pem'), - verify='/path/to/ca.pem' -) -client = docker.Client(base_url='', tls=tls_config) -``` - -Equivalent CLI options: -```bash -docker --tlsverify \ - --tlscert /path/to/client-cert.pem \ - --tlskey /path/to/client-key.pem \ - --tlscacert /path/to/ca.pem ... -``` diff --git a/docs/tls.rst b/docs/tls.rst new file mode 100644 index 0000000000..0f318ff643 --- /dev/null +++ b/docs/tls.rst @@ -0,0 +1,37 @@ +Using TLS +========= + +.. py:module:: docker.tls + +Both the main :py:class:`~docker.client.Client` and low-level +:py:class:`~docker.api.client.APIClient` can connect to the Docker daemon with TLS. + +This is all configured automatically for you if you're using :py:func:`~docker.client.from_env`, but if you need some extra control it is possible to configure it manually by using a :py:class:`TLSConfig` object. + +Examples +-------- + +For example, to check the server against a specific CA certificate: + +.. code-block:: python + + tls_config = docker.tls.TLSConfig(ca_cert='/path/to/ca.pem') + client = docker.Client(base_url='', tls=tls_config) + +This is the equivalent of ``docker --tlsverify --tlscacert /path/to/ca.pem ...``. + +To authenticate with client certs: + +.. code-block:: python + + tls_config = docker.tls.TLSConfig( + client_cert=('/path/to/client-cert.pem', '/path/to/client-key.pem') + ) + client = docker.Client(base_url='', tls=tls_config) + +This is the equivalent of ``docker --tls --tlscert /path/to/client-cert.pem --tlskey /path/to/client-key.pem ...``. + +Reference +--------- + +.. autoclass:: TLSConfig() diff --git a/docs/tmpfs.md b/docs/tmpfs.md deleted file mode 100644 index d8be9b68c6..0000000000 --- a/docs/tmpfs.md +++ /dev/null @@ -1,33 +0,0 @@ -# Using tmpfs - -When creating a container, you can specify paths to be mounted with tmpfs using -the `tmpfs` argument to `create_host_config`, similarly to the `--tmpfs` -argument to `docker run`. - -This capability is supported in Docker Engine 1.10 and up. - -`tmpfs` can be either a list or a dictionary. If it's a list, each item is a -string specifying the path and (optionally) any configuration for the mount: - -```python -client.create_container( - 'busybox', 'ls', - host_config=client.create_host_config(tmpfs=[ - '/mnt/vol2', - '/mnt/vol1:size=3G,uid=1000' - ]) -) -``` - -Alternatively, if it's a dictionary, each key is a path and each value contains -the mount options: - -```python -client.create_container( - 'busybox', 'ls', - host_config=client.create_host_config(tmpfs={ - '/mnt/vol2': '', - '/mnt/vol1': 'size=3G,uid=1000' - }) -) -``` diff --git a/docs/volumes.md b/docs/volumes.md deleted file mode 100644 index 04273d805d..0000000000 --- a/docs/volumes.md +++ /dev/null @@ -1,34 +0,0 @@ -# Using volumes - -Volume declaration is done in two parts. Provide a list of mountpoints to -the `Client().create_container()` method, and declare mappings in the -`host_config` section. - -```python -container_id = cli.create_container( - 'busybox', 'ls', volumes=['/mnt/vol1', '/mnt/vol2'], - host_config=cli.create_host_config(binds={ - '/home/user1/': { - 'bind': '/mnt/vol2', - 'mode': 'rw', - }, - '/var/www': { - 'bind': '/mnt/vol1', - 'mode': 'ro', - } - }) -) -``` - -You can alternatively specify binds as a list. This code is equivalent to the -example above: - -```python -container_id = cli.create_container( - 'busybox', 'ls', volumes=['/mnt/vol1', '/mnt/vol2'], - host_config=cli.create_host_config(binds=[ - '/home/user1/:/mnt/vol2', - '/var/www:/mnt/vol1:ro', - ]) -) -``` diff --git a/docs/volumes.rst b/docs/volumes.rst new file mode 100644 index 0000000000..8c0574b562 --- /dev/null +++ b/docs/volumes.rst @@ -0,0 +1,31 @@ +Volumes +======= + +.. py:module:: docker.models.volumes + +Manage volumes on the server. + +Methods available on ``client.volumes``: + +.. rst-class:: hide-signature +.. py:class:: VolumeCollection + + .. automethod:: create + .. automethod:: get + .. automethod:: list + +Volume objects +-------------- + +.. autoclass:: Volume() + + .. autoattribute:: id + .. autoattribute:: short_id + .. autoattribute:: name + .. py:attribute:: attrs + + The raw representation of this object from the server. + + + .. automethod:: reload + .. automethod:: remove diff --git a/mkdocs.yml b/mkdocs.yml deleted file mode 100644 index 6cfaa543bc..0000000000 --- a/mkdocs.yml +++ /dev/null @@ -1,21 +0,0 @@ -site_name: docker-py Documentation -site_description: An API client for Docker written in Python -site_favicon: favicon_whale.png -site_url: https://docker-py.readthedocs.io -repo_url: https://github.com/docker/docker-py/ -theme: readthedocs -pages: -- Home: index.md -- Client API: api.md -- Port Bindings: port-bindings.md -- Using Volumes: volumes.md -- Using TLS: tls.md -- Host devices: host-devices.md -- Host configuration: hostconfig.md -- Network configuration: networks.md -- Swarm management: swarm.md -- Swarm services: services.md -- Using tmpfs: tmpfs.md -- Using with Docker Machine: machine.md -- Change Log: change_log.md -- Contributing: contributing.md From b5f7d380d02bdb121dfaf9aee8e14e2f7390790a Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Fri, 21 Oct 2016 17:14:17 +0200 Subject: [PATCH 0179/1301] Add helpful error for APIClient methods on Client Signed-off-by: Ben Firshman --- docker/client.py | 10 ++++++++++ tests/unit/client_test.py | 14 ++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/docker/client.py b/docker/client.py index 6c5ae4080d..b3b470042a 100644 --- a/docker/client.py +++ b/docker/client.py @@ -154,4 +154,14 @@ def version(self, *args, **kwargs): return self.api.version(*args, **kwargs) version.__doc__ = APIClient.version.__doc__ + def __getattr__(self, name): + s = ["'Client' object has no attribute '{}'".format(name)] + # If a user calls a method on APIClient, they + if hasattr(APIClient, name): + s.append("In docker-py 2.0, this method is now on the object " + "APIClient. See the low-level API section of the " + "documentation for more details.".format(name)) + raise AttributeError(' '.join(s)) + + from_env = Client.from_env diff --git a/tests/unit/client_test.py b/tests/unit/client_test.py index e22983c7e2..0a56b04d93 100644 --- a/tests/unit/client_test.py +++ b/tests/unit/client_test.py @@ -45,6 +45,20 @@ def test_version(self, mock_func): assert client.version() == mock_func.return_value mock_func.assert_called_with() + def test_call_api_client_method(self): + client = docker.from_env() + with self.assertRaises(AttributeError) as cm: + client.create_container() + s = str(cm.exception) + assert "'Client' object has no attribute 'create_container'" in s + assert "this method is now on the object APIClient" in s + + with self.assertRaises(AttributeError) as cm: + client.abcdef() + s = str(cm.exception) + assert "'Client' object has no attribute 'abcdef'" in s + assert "this method is now on the object APIClient" not in s + class FromEnvTest(unittest.TestCase): From a38644fc0ed91b7cfb1637cd87bd56e7996b3c4d Mon Sep 17 00:00:00 2001 From: biniambekele Date: Tue, 22 Nov 2016 15:17:45 -0500 Subject: [PATCH 0180/1301] Fix ContainerApiMixin.copy with dict container arg Signed-off-by: biniambekele --- docker/api/container.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/api/container.py b/docker/api/container.py index 338b79fe77..b436353250 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -100,7 +100,7 @@ def copy(self, container, resource): DeprecationWarning ) res = self._post_json( - self._url("/containers/{0}/copy".format(container)), + self._url("/containers/{0}/copy", container), data={"Resource": resource}, stream=True ) From c96848eb9c05f978a380ba3668605c51e811d4fb Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Wed, 23 Nov 2016 14:47:46 +0000 Subject: [PATCH 0181/1301] Fix documentation link in readme The rest of the readme is wrong until we release 2.0, but at least the documentation points to the right place. Closes #1302 Signed-off-by: Ben Firshman --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 094b13c8e3..6441931b52 100644 --- a/README.md +++ b/README.md @@ -70,4 +70,4 @@ You can manage images: [, , ...] ``` -[Read the full documentation](https://docs.docker.com/sdk/python/) to see everything you can do. +[Read the full documentation](https://docker-py.readthedocs.io) to see everything you can do. From afea2ca2698921d072f94f685f33606a6137c8cd Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Wed, 23 Nov 2016 15:21:56 +0000 Subject: [PATCH 0182/1301] Use SVG for build status badge Signed-off-by: Ben Firshman --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 094b13c8e3..0b0703715c 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Docker SDK for Python -[![Build Status](https://travis-ci.org/docker/docker-py.png)](https://travis-ci.org/docker/docker-py) +[![Build Status](https://travis-ci.org/docker/docker-py.svg?branch=master)](https://travis-ci.org/docker/docker-py) A Python library for the Docker API. It lets you do anything the `docker` command does, but from within Python apps – run containers, manage containers, manage Swarms, etc. From aafcf5a7fe2a584e50a3e32ea502fb1988d404e1 Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Wed, 23 Nov 2016 18:02:23 +0000 Subject: [PATCH 0183/1301] Change package in readme back to docker-py Signed-off-by: Ben Firshman --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6441931b52..41453fa80a 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ A Python library for the Docker API. It lets you do anything the `docker` comman The latest stable version [is available on PyPi](https://pypi.python.org/pypi/docker/). Either add `docker` to your `requirements.txt` file or install with pip: - pip install docker + pip install docker-py ## Usage From af67add6831825003c70f5e0de819b041fc71721 Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Wed, 23 Nov 2016 18:07:33 +0000 Subject: [PATCH 0184/1301] Add warning about development version to readme Signed-off-by: Ben Firshman --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 41453fa80a..fc0e3bc5f2 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ [![Build Status](https://travis-ci.org/docker/docker-py.png)](https://travis-ci.org/docker/docker-py) +**Warning:** This readme is for the development version of docker-py, which is significantly different to the stable version. [Documentation for the stable version is here.](https://docker-py.readthedocs.io/) + A Python library for the Docker API. It lets you do anything the `docker` command does, but from within Python apps – run containers, manage containers, manage Swarms, etc. ## Installation From 037ead974bbd1ad6ebe81bbd4578011d453f9dae Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 23 Nov 2016 11:37:33 -0800 Subject: [PATCH 0185/1301] Add Ben to MAINTAINERS Signed-off-by: Joffrey F --- MAINTAINERS | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/MAINTAINERS b/MAINTAINERS index ed93c01c70..1f46236e72 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -12,6 +12,7 @@ [Org."Core maintainers"] people = [ "aanand", + "bfirsh", "dnephin", "mnowster", "mpetazzoni", @@ -31,6 +32,11 @@ Email = "aanand@docker.com" GitHub = "aanand" + [people.bfirsh] + Name = "Ben Firshman" + Email = "b@fir.sh" + GitHub = "bfirsh" + [people.dnephin] Name = "Daniel Nephin" Email = "dnephin@gmail.com" From b4c02393b2773bddb212a24ded85d805bdca8334 Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Thu, 24 Nov 2016 13:32:20 +0000 Subject: [PATCH 0186/1301] Remove MAINTAINER from Dockerfiles It was deprecated in https://github.com/docker/docker/pull/25466 (Sorry @shin- ;) Signed-off-by: Ben Firshman --- Dockerfile | 1 - Dockerfile-py3 | 1 - tests/integration/api_build_test.py | 4 ---- tests/unit/api_build_test.py | 5 ----- 4 files changed, 11 deletions(-) diff --git a/Dockerfile b/Dockerfile index 012a1259b8..993ac012bf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,4 @@ FROM python:2.7 -MAINTAINER Joffrey F RUN mkdir /home/docker-py WORKDIR /home/docker-py diff --git a/Dockerfile-py3 b/Dockerfile-py3 index 21e713bb39..c746651772 100644 --- a/Dockerfile-py3 +++ b/Dockerfile-py3 @@ -1,5 +1,4 @@ FROM python:3.5 -MAINTAINER Joffrey F RUN mkdir /home/docker-py WORKDIR /home/docker-py diff --git a/tests/integration/api_build_test.py b/tests/integration/api_build_test.py index 9ae74f4d44..3dac0e932d 100644 --- a/tests/integration/api_build_test.py +++ b/tests/integration/api_build_test.py @@ -15,7 +15,6 @@ class BuildTest(BaseAPIIntegrationTest): def test_build_streaming(self): script = io.BytesIO('\n'.join([ 'FROM busybox', - 'MAINTAINER docker-py', 'RUN mkdir -p /tmp/test', 'EXPOSE 8080', 'ADD https://dl.dropboxusercontent.com/u/20637798/silence.tar.gz' @@ -32,7 +31,6 @@ def test_build_from_stringio(self): return script = io.StringIO(six.text_type('\n').join([ 'FROM busybox', - 'MAINTAINER docker-py', 'RUN mkdir -p /tmp/test', 'EXPOSE 8080', 'ADD https://dl.dropboxusercontent.com/u/20637798/silence.tar.gz' @@ -54,7 +52,6 @@ def test_build_with_dockerignore(self): with open(os.path.join(base_dir, 'Dockerfile'), 'w') as f: f.write("\n".join([ 'FROM busybox', - 'MAINTAINER docker-py', 'ADD . /test', ])) @@ -182,7 +179,6 @@ def test_build_gzip_encoding(self): with open(os.path.join(base_dir, 'Dockerfile'), 'w') as f: f.write("\n".join([ 'FROM busybox', - 'MAINTAINER docker-py', 'ADD . /test', ])) diff --git a/tests/unit/api_build_test.py b/tests/unit/api_build_test.py index 8146fee721..927aa9749e 100644 --- a/tests/unit/api_build_test.py +++ b/tests/unit/api_build_test.py @@ -11,7 +11,6 @@ class BuildTest(BaseAPIClientTest): def test_build_container(self): script = io.BytesIO('\n'.join([ 'FROM busybox', - 'MAINTAINER docker-py', 'RUN mkdir -p /tmp/test', 'EXPOSE 8080', 'ADD https://dl.dropboxusercontent.com/u/20637798/silence.tar.gz' @@ -23,7 +22,6 @@ def test_build_container(self): def test_build_container_pull(self): script = io.BytesIO('\n'.join([ 'FROM busybox', - 'MAINTAINER docker-py', 'RUN mkdir -p /tmp/test', 'EXPOSE 8080', 'ADD https://dl.dropboxusercontent.com/u/20637798/silence.tar.gz' @@ -35,7 +33,6 @@ def test_build_container_pull(self): def test_build_container_stream(self): script = io.BytesIO('\n'.join([ 'FROM busybox', - 'MAINTAINER docker-py', 'RUN mkdir -p /tmp/test', 'EXPOSE 8080', 'ADD https://dl.dropboxusercontent.com/u/20637798/silence.tar.gz' @@ -47,7 +44,6 @@ def test_build_container_stream(self): def test_build_container_custom_context(self): script = io.BytesIO('\n'.join([ 'FROM busybox', - 'MAINTAINER docker-py', 'RUN mkdir -p /tmp/test', 'EXPOSE 8080', 'ADD https://dl.dropboxusercontent.com/u/20637798/silence.tar.gz' @@ -60,7 +56,6 @@ def test_build_container_custom_context(self): def test_build_container_custom_context_gzip(self): script = io.BytesIO('\n'.join([ 'FROM busybox', - 'MAINTAINER docker-py', 'RUN mkdir -p /tmp/test', 'EXPOSE 8080', 'ADD https://dl.dropboxusercontent.com/u/20637798/silence.tar.gz' From e2eb4a3158ece4e9a1122a85accf95e23f6630dc Mon Sep 17 00:00:00 2001 From: Brandon Bodnar Date: Fri, 28 Oct 2016 00:06:22 -0400 Subject: [PATCH 0187/1301] Prevent traversing excluded directories with no possible dockerignore exceptions Fixes an issue where all files in a rather large excluded folder are traversed and examined when creating the build context for potential exception to the exclusion, even though the exclusion rule is for a completely unrelated folder. Signed-off-by: Brandon Bodnar --- docker/utils/utils.py | 51 +++++++++++++++++++++++++++---------------- 1 file changed, 32 insertions(+), 19 deletions(-) diff --git a/docker/utils/utils.py b/docker/utils/utils.py index b107f22e97..87401642a0 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -214,6 +214,31 @@ def should_include(path, exclude_patterns, include_patterns): return True +def should_check_directory(directory_path, exclude_patterns, include_patterns): + """ + Given a directory path, a list of exclude patterns, and a list of inclusion + patterns: + + 1. Returns True if the directory path should be included according to + should_include. + 2. Returns True if the directory path is the prefix for an inclusion + pattern + 3. Returns False otherwise + """ + + # To account for exception rules, check directories if their path is a + # a prefix to an inclusion pattern. This logic conforms with the current + # docker logic (2016-10-27): + # https://github.com/docker/docker/blob/bc52939b0455116ab8e0da67869ec81c1a1c3e2c/pkg/archive/archive.go#L640-L671 + + path_with_slash = directory_path + os.sep + possible_child_patterns = [pattern for pattern in include_patterns if + (pattern + os.sep).startswith(path_with_slash)] + directory_included = should_include(directory_path, exclude_patterns, + include_patterns) + return directory_included or len(possible_child_patterns) > 0 + + def get_paths(root, exclude_patterns, include_patterns, has_exceptions=False): paths = [] @@ -222,25 +247,13 @@ def get_paths(root, exclude_patterns, include_patterns, has_exceptions=False): if parent == '.': parent = '' - # If exception rules exist, we can't skip recursing into ignored - # directories, as we need to look for exceptions in them. - # - # It may be possible to optimize this further for exception patterns - # that *couldn't* match within ignored directores. - # - # This matches the current docker logic (as of 2015-11-24): - # https://github.com/docker/docker/blob/37ba67bf636b34dc5c0c0265d62a089d0492088f/pkg/archive/archive.go#L555-L557 - - if not has_exceptions: - - # Remove excluded patterns from the list of directories to traverse - # by mutating the dirs we're iterating over. - # This looks strange, but is considered the correct way to skip - # traversal. See https://docs.python.org/2/library/os.html#os.walk - - dirs[:] = [d for d in dirs if - should_include(os.path.join(parent, d), - exclude_patterns, include_patterns)] + # Remove excluded patterns from the list of directories to traverse + # by mutating the dirs we're iterating over. + # This looks strange, but is considered the correct way to skip + # traversal. See https://docs.python.org/2/library/os.html#os.walk + dirs[:] = [d for d in dirs if + should_check_directory(os.path.join(parent, d), + exclude_patterns, include_patterns)] for path in dirs: if should_include(os.path.join(parent, path), From 9fc8b3a730293b8cbee2feab04c355e753b20cdf Mon Sep 17 00:00:00 2001 From: Brandon Bodnar Date: Mon, 31 Oct 2016 19:56:02 -0400 Subject: [PATCH 0188/1301] Add unit tests for should_check_directory. Signed-off-by: Brandon Bodnar --- tests/unit/utils_test.py | 76 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 75 insertions(+), 1 deletion(-) diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index 57aa226d84..45f5914b76 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -25,7 +25,9 @@ ) from docker.utils.ports import build_port_bindings, split_port -from docker.utils.utils import create_endpoint_config, format_environment +from docker.utils.utils import ( + create_endpoint_config, format_environment, should_check_directory +) from ..helpers import make_tree @@ -1092,6 +1094,78 @@ def test_tar_with_directory_symlinks(self): ) +class ShouldCheckDirectoryTest(unittest.TestCase): + exclude_patterns = [ + 'exclude_rather_large_directory', + 'dir/with/subdir_excluded', + 'dir/with/exceptions' + ] + + include_patterns = [ + 'dir/with/exceptions/like_this_one', + 'dir/with/exceptions/in/descendents' + ] + + def test_should_check_directory_not_excluded(self): + self.assertTrue( + should_check_directory('not_excluded', self.exclude_patterns, + self.include_patterns) + ) + + self.assertTrue( + should_check_directory('dir/with', self.exclude_patterns, + self.include_patterns) + ) + + def test_shoud_check_parent_directories_of_excluded(self): + self.assertTrue( + should_check_directory('dir', self.exclude_patterns, + self.include_patterns) + ) + self.assertTrue( + should_check_directory('dir/with', self.exclude_patterns, + self.include_patterns) + ) + + def test_should_not_check_excluded_directories_with_no_exceptions(self): + self.assertFalse( + should_check_directory('exclude_rather_large_directory', + self.exclude_patterns, self.include_patterns + ) + ) + self.assertFalse( + should_check_directory('dir/with/subdir_excluded', + self.exclude_patterns, self.include_patterns + ) + ) + + def test_should_check_excluded_directory_with_exceptions(self): + self.assertTrue( + should_check_directory('dir/with/exceptions', + self.exclude_patterns, self.include_patterns + ) + ) + self.assertTrue( + should_check_directory('dir/with/exceptions/in', + self.exclude_patterns, self.include_patterns + ) + ) + + def test_should_not_check_siblings_of_exceptions(self): + self.assertFalse( + should_check_directory('dir/with/exceptions/but_not_here', + self.exclude_patterns, self.include_patterns + ) + ) + + def test_should_check_subdirectories_of_exceptions(self): + self.assertTrue( + should_check_directory('dir/with/exceptions/like_this_one/subdir', + self.exclude_patterns, self.include_patterns + ) + ) + + class FormatEnvironmentTest(unittest.TestCase): def test_format_env_binary_unicode_value(self): env_dict = { From a658e14e8b7f578d55102b981ce8828b906b432f Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Mon, 28 Nov 2016 14:37:34 +0000 Subject: [PATCH 0189/1301] Specify encoding when loading readme Loading readme fails when system locale is not utf-8. Potentially replaces #1313 Signed-off-by: Ben Firshman --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 89c97c87be..2d1bfdbb33 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,5 @@ #!/usr/bin/env python +import codecs import os import sys @@ -35,7 +36,7 @@ long_description = '' try: - with open('./README.rst') as readme_rst: + with codecs.open('./README.rst', encoding='utf-8') as readme_rst: long_description = readme_rst.read() except IOError: # README.rst is only generated on release. Its absence should not prevent From e04c4ad83f3a3da7a1c28d094cc865fe87f75763 Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Wed, 23 Nov 2016 14:49:56 +0000 Subject: [PATCH 0190/1301] Update readme to say "Docker Engine API" It's not the Docker API. Signed-off-by: Ben Firshman --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ab54d52852..11fcbad2fb 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ **Warning:** This readme is for the development version of docker-py, which is significantly different to the stable version. [Documentation for the stable version is here.](https://docker-py.readthedocs.io/) -A Python library for the Docker API. It lets you do anything the `docker` command does, but from within Python apps – run containers, manage containers, manage Swarms, etc. +A Python library for the Docker Engine API. It lets you do anything the `docker` command does, but from within Python apps – run containers, manage containers, manage Swarms, etc. ## Installation From 04e943798691ba2663b91eaedd8ae2410f016d2c Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Wed, 23 Nov 2016 14:52:59 +0000 Subject: [PATCH 0191/1301] Update setup.py details Move from "docker-py" package to "docker". Signed-off-by: Ben Firshman --- setup.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 89c97c87be..c3113701db 100644 --- a/setup.py +++ b/setup.py @@ -43,11 +43,11 @@ pass setup( - name="docker-py", + name="docker", version=version, - description="Python client for Docker.", + description="A Python library for the Docker Engine API.", long_description=long_description, - url='https://github.com/docker/docker-py/', + url='https://github.com/docker/docker-py', packages=[ 'docker', 'docker.api', 'docker.transport', 'docker.utils', 'docker.types', @@ -58,7 +58,7 @@ zip_safe=False, test_suite='tests', classifiers=[ - 'Development Status :: 4 - Beta', + 'Development Status :: 5 - Production/Stable', 'Environment :: Other Environment', 'Intended Audience :: Developers', 'Operating System :: OS Independent', From 6d770a65d7c05d708334a93ee3bdb099061447de Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Wed, 23 Nov 2016 15:02:59 +0000 Subject: [PATCH 0192/1301] Use find_packages in setup.py It was missing docker.models, and this will fix it forever more. Signed-off-by: Ben Firshman --- setup.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index c3113701db..edf4b0e50e 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ import os import sys -from setuptools import setup +from setuptools import setup, find_packages ROOT_DIR = os.path.dirname(__file__) @@ -48,10 +48,7 @@ description="A Python library for the Docker Engine API.", long_description=long_description, url='https://github.com/docker/docker-py', - packages=[ - 'docker', 'docker.api', 'docker.transport', 'docker.utils', - 'docker.types', - ], + packages=find_packages(exclude=["tests.*", "tests"]), install_requires=requirements, tests_require=test_requirements, extras_require=extras_require, From fc9f7e2b2feea99a52b75077046473e7ce15ea82 Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Wed, 23 Nov 2016 15:03:25 +0000 Subject: [PATCH 0193/1301] Bump version to 2.0.0-dev Signed-off-by: Ben Firshman --- docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index 3bbd804566..ab6838fb9e 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "1.11.0-dev" +version = "2.0.0-dev" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) From 94083f25acfa38d68e6dbb08c81d51021191c95b Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Mon, 28 Nov 2016 16:45:59 +0000 Subject: [PATCH 0194/1301] Start to rename docker-py to docker-sdk-python Pretty much everything except renaming the GitHub repo and documentation, which is not actually done yet. Nearer the release we can do a search/replace for all that stuff. Ref #1310 Signed-off-by: Ben Firshman --- Dockerfile | 10 +++++----- Dockerfile-docs | 10 +++++----- Dockerfile-py3 | 10 +++++----- Makefile | 28 ++++++++++++++-------------- docker/__init__.py | 2 +- docker/api/client.py | 4 ++-- docker/client.py | 6 +++--- docker/constants.py | 2 +- tests/unit/api_test.py | 2 +- 9 files changed, 37 insertions(+), 37 deletions(-) diff --git a/Dockerfile b/Dockerfile index 012a1259b8..fd932327b4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,14 +1,14 @@ FROM python:2.7 MAINTAINER Joffrey F -RUN mkdir /home/docker-py -WORKDIR /home/docker-py +RUN mkdir /src +WORKDIR /src -COPY requirements.txt /home/docker-py/requirements.txt +COPY requirements.txt /src/requirements.txt RUN pip install -r requirements.txt -COPY test-requirements.txt /home/docker-py/test-requirements.txt +COPY test-requirements.txt /src/test-requirements.txt RUN pip install -r test-requirements.txt -COPY . /home/docker-py +COPY . /src RUN pip install . diff --git a/Dockerfile-docs b/Dockerfile-docs index 705649ff95..6f4194009a 100644 --- a/Dockerfile-docs +++ b/Dockerfile-docs @@ -1,12 +1,12 @@ FROM python:3.5 -RUN mkdir /home/docker-py -WORKDIR /home/docker-py +RUN mkdir /src +WORKDIR /src -COPY requirements.txt /home/docker-py/requirements.txt +COPY requirements.txt /src/requirements.txt RUN pip install -r requirements.txt -COPY docs-requirements.txt /home/docker-py/docs-requirements.txt +COPY docs-requirements.txt /src/docs-requirements.txt RUN pip install -r docs-requirements.txt -COPY . /home/docker-py +COPY . /src diff --git a/Dockerfile-py3 b/Dockerfile-py3 index 21e713bb39..4fce014f2b 100644 --- a/Dockerfile-py3 +++ b/Dockerfile-py3 @@ -1,14 +1,14 @@ FROM python:3.5 MAINTAINER Joffrey F -RUN mkdir /home/docker-py -WORKDIR /home/docker-py +RUN mkdir /src +WORKDIR /src -COPY requirements.txt /home/docker-py/requirements.txt +COPY requirements.txt /src/requirements.txt RUN pip install -r requirements.txt -COPY test-requirements.txt /home/docker-py/test-requirements.txt +COPY test-requirements.txt /src/test-requirements.txt RUN pip install -r test-requirements.txt -COPY . /home/docker-py +COPY . /src RUN pip install . diff --git a/Makefile b/Makefile index 425fffd897..4c5cf0c67b 100644 --- a/Makefile +++ b/Makefile @@ -8,15 +8,15 @@ clean: .PHONY: build build: - docker build -t docker-py . + docker build -t docker-sdk-python . .PHONY: build-py3 build-py3: - docker build -t docker-py3 -f Dockerfile-py3 . + docker build -t docker-sdk-python3 -f Dockerfile-py3 . .PHONY: build-docs build-docs: - docker build -t docker-py-docs -f Dockerfile-docs . + docker build -t docker-sdk-python-docs -f Dockerfile-docs . .PHONY: build-dind-certs build-dind-certs: @@ -27,28 +27,28 @@ test: flake8 unit-test unit-test-py3 integration-dind integration-dind-ssl .PHONY: unit-test unit-test: build - docker run --rm docker-py py.test tests/unit + docker run --rm docker-sdk-python py.test tests/unit .PHONY: unit-test-py3 unit-test-py3: build-py3 - docker run --rm docker-py3 py.test tests/unit + docker run --rm docker-sdk-python3 py.test tests/unit .PHONY: integration-test integration-test: build - docker run --rm -v /var/run/docker.sock:/var/run/docker.sock docker-py py.test tests/integration/${file} + docker run --rm -v /var/run/docker.sock:/var/run/docker.sock docker-sdk-python py.test tests/integration/${file} .PHONY: integration-test-py3 integration-test-py3: build-py3 - docker run --rm -v /var/run/docker.sock:/var/run/docker.sock docker-py3 py.test tests/integration/${file} + docker run --rm -v /var/run/docker.sock:/var/run/docker.sock docker-sdk-python3 py.test tests/integration/${file} .PHONY: integration-dind integration-dind: build build-py3 docker rm -vf dpy-dind || : docker run -d --name dpy-dind --privileged dockerswarm/dind:1.12.0 docker daemon\ -H tcp://0.0.0.0:2375 - docker run --rm --env="DOCKER_HOST=tcp://docker:2375" --link=dpy-dind:docker docker-py\ + docker run --rm --env="DOCKER_HOST=tcp://docker:2375" --link=dpy-dind:docker docker-sdk-python\ py.test tests/integration - docker run --rm --env="DOCKER_HOST=tcp://docker:2375" --link=dpy-dind:docker docker-py3\ + docker run --rm --env="DOCKER_HOST=tcp://docker:2375" --link=dpy-dind:docker docker-sdk-python3\ py.test tests/integration docker rm -vf dpy-dind @@ -62,20 +62,20 @@ integration-dind-ssl: build-dind-certs build build-py3 --tlskey=/certs/server-key.pem -H tcp://0.0.0.0:2375 docker run --rm --volumes-from dpy-dind-ssl --env="DOCKER_HOST=tcp://docker:2375"\ --env="DOCKER_TLS_VERIFY=1" --env="DOCKER_CERT_PATH=/certs"\ - --link=dpy-dind-ssl:docker docker-py py.test tests/integration + --link=dpy-dind-ssl:docker docker-sdk-python py.test tests/integration docker run --rm --volumes-from dpy-dind-ssl --env="DOCKER_HOST=tcp://docker:2375"\ --env="DOCKER_TLS_VERIFY=1" --env="DOCKER_CERT_PATH=/certs"\ - --link=dpy-dind-ssl:docker docker-py3 py.test tests/integration + --link=dpy-dind-ssl:docker docker-sdk-python3 py.test tests/integration docker rm -vf dpy-dind-ssl dpy-dind-certs .PHONY: flake8 flake8: build - docker run --rm docker-py flake8 docker tests + docker run --rm docker-sdk-python flake8 docker tests .PHONY: docs docs: build-docs - docker run --rm -it -v `pwd`:/home/docker-py docker-py-docs sphinx-build docs ./_build + docker run --rm -it -v `pwd`:/code docker-sdk-python-docs sphinx-build docs ./_build .PHONY: shell shell: build - docker run -it -v /var/run/docker.sock:/var/run/docker.sock docker-py python + docker run -it -v /var/run/docker.sock:/var/run/docker.sock docker-sdk-python python diff --git a/docker/__init__.py b/docker/__init__.py index acf4b5566e..0ac269509b 100644 --- a/docker/__init__.py +++ b/docker/__init__.py @@ -4,4 +4,4 @@ from .version import version, version_info __version__ = version -__title__ = 'docker-py' +__title__ = 'docker' diff --git a/docker/api/client.py b/docker/api/client.py index 23e239c66f..3793299313 100644 --- a/docker/api/client.py +++ b/docker/api/client.py @@ -145,8 +145,8 @@ def __init__(self, base_url=None, version=None, warnings.warn( 'The minimum API version supported is {}, but you are using ' 'version {}. It is recommended you either upgrade Docker ' - 'Engine or use an older version of docker-py.'.format( - MINIMUM_DOCKER_API_VERSION, self._version) + 'Engine or use an older version of Docker SDK for ' + 'Python.'.format(MINIMUM_DOCKER_API_VERSION, self._version) ) def _retrieve_server_version(self): diff --git a/docker/client.py b/docker/client.py index b3b470042a..b5257ab005 100644 --- a/docker/client.py +++ b/docker/client.py @@ -158,9 +158,9 @@ def __getattr__(self, name): s = ["'Client' object has no attribute '{}'".format(name)] # If a user calls a method on APIClient, they if hasattr(APIClient, name): - s.append("In docker-py 2.0, this method is now on the object " - "APIClient. See the low-level API section of the " - "documentation for more details.".format(name)) + s.append("In Docker SDK for Python 2.0, this method is now on the " + "object APIClient. See the low-level API section of the " + "documentation for more details.") raise AttributeError(' '.join(s)) diff --git a/docker/constants.py b/docker/constants.py index c3048cb7ad..0bfc0b088a 100644 --- a/docker/constants.py +++ b/docker/constants.py @@ -15,5 +15,5 @@ IS_WINDOWS_PLATFORM = (sys.platform == 'win32') -DEFAULT_USER_AGENT = "docker-py/{0}".format(version) +DEFAULT_USER_AGENT = "docker-sdk-python/{0}".format(version) DEFAULT_NUM_POOLS = 25 diff --git a/tests/unit/api_test.py b/tests/unit/api_test.py index 67373bac05..15e4d7cc69 100644 --- a/tests/unit/api_test.py +++ b/tests/unit/api_test.py @@ -474,7 +474,7 @@ def test_default_user_agent(self): self.assertEqual(self.mock_send.call_count, 1) headers = self.mock_send.call_args[0][0].headers - expected = 'docker-py/%s' % docker.__version__ + expected = 'docker-sdk-python/%s' % docker.__version__ self.assertEqual(headers['User-Agent'], expected) def test_custom_user_agent(self): From 5eacb986d706226ccfd9a6b1021828b6eb67dbad Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 21 Oct 2015 15:43:02 -0700 Subject: [PATCH 0195/1301] Remove support for host_config in Client.start Any additional arguments passed to start will raise a DeprecatedMethod (DockerException) exception. Signed-off-by: Joffrey F --- docker/api/container.py | 81 ++++++-------------------------- tests/unit/api_container_test.py | 46 +++++------------- 2 files changed, 26 insertions(+), 101 deletions(-) diff --git a/docker/api/container.py b/docker/api/container.py index 72c5852dce..953a5f52fd 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -997,19 +997,16 @@ def restart(self, container, timeout=10): self._raise_for_status(res) @utils.check_resource - def start(self, container, binds=None, port_bindings=None, lxc_conf=None, - publish_all_ports=None, links=None, privileged=None, - dns=None, dns_search=None, volumes_from=None, network_mode=None, - restart_policy=None, cap_add=None, cap_drop=None, devices=None, - extra_hosts=None, read_only=None, pid_mode=None, ipc_mode=None, - security_opt=None, ulimits=None): + def start(self, container, *args, **kwargs): """ Start a container. Similar to the ``docker start`` command, but doesn't support attach options. - **Deprecation warning:** For API version > 1.15, it is highly - recommended to provide host config options in the ``host_config`` - parameter of :py:meth:`~ContainerApiMixin.create_container`. + **Deprecation warning:** Passing configuration options in ``start`` is + no longer supported. Users are expected to provide host config options + in the ``host_config`` parameter of + :py:meth:`~ContainerApiMixin.create_container`. + Args: container (str): The container to start @@ -1017,6 +1014,8 @@ def start(self, container, binds=None, port_bindings=None, lxc_conf=None, Raises: :py:class:`docker.errors.APIError` If the server returns an error. + :py:class:`docker.errors.DeprecatedMethod` + If any argument besides ``container`` are provided. Example: @@ -1025,64 +1024,14 @@ def start(self, container, binds=None, port_bindings=None, lxc_conf=None, ... command='/bin/sleep 30') >>> cli.start(container=container.get('Id')) """ - if utils.compare_version('1.10', self._version) < 0: - if dns is not None: - raise errors.InvalidVersion( - 'dns is only supported for API version >= 1.10' - ) - if volumes_from is not None: - raise errors.InvalidVersion( - 'volumes_from is only supported for API version >= 1.10' - ) - - if utils.compare_version('1.15', self._version) < 0: - if security_opt is not None: - raise errors.InvalidVersion( - 'security_opt is only supported for API version >= 1.15' - ) - if ipc_mode: - raise errors.InvalidVersion( - 'ipc_mode is only supported for API version >= 1.15' - ) - - if utils.compare_version('1.17', self._version) < 0: - if read_only is not None: - raise errors.InvalidVersion( - 'read_only is only supported for API version >= 1.17' - ) - if pid_mode is not None: - raise errors.InvalidVersion( - 'pid_mode is only supported for API version >= 1.17' - ) - - if utils.compare_version('1.18', self._version) < 0: - if ulimits is not None: - raise errors.InvalidVersion( - 'ulimits is only supported for API version >= 1.18' - ) - - start_config_kwargs = dict( - binds=binds, port_bindings=port_bindings, lxc_conf=lxc_conf, - publish_all_ports=publish_all_ports, links=links, dns=dns, - privileged=privileged, dns_search=dns_search, cap_add=cap_add, - cap_drop=cap_drop, volumes_from=volumes_from, devices=devices, - network_mode=network_mode, restart_policy=restart_policy, - extra_hosts=extra_hosts, read_only=read_only, pid_mode=pid_mode, - ipc_mode=ipc_mode, security_opt=security_opt, ulimits=ulimits, - ) - start_config = None - - if any(v is not None for v in start_config_kwargs.values()): - if utils.compare_version('1.15', self._version) > 0: - warnings.warn( - 'Passing host config parameters in start() is deprecated. ' - 'Please use host_config in create_container instead!', - DeprecationWarning - ) - start_config = self.create_host_config(**start_config_kwargs) - + if args or kwargs: + raise errors.DeprecatedMethod( + 'Providing configuration in the start() method is no longer ' + 'supported. Use the host_config param in create_container ' + 'instead.' + ) url = self._url("/containers/{0}/start", container) - res = self._post_json(url, data=start_config) + res = self._post(url) self._raise_for_status(res) @utils.minimum_version('1.17') diff --git a/tests/unit/api_container_test.py b/tests/unit/api_container_test.py index 6c08064179..abf3613885 100644 --- a/tests/unit/api_container_test.py +++ b/tests/unit/api_container_test.py @@ -34,10 +34,7 @@ def test_start_container(self): args[0][1], url_prefix + 'containers/3cc2351ab11b/start' ) - self.assertEqual(json.loads(args[1]['data']), {}) - self.assertEqual( - args[1]['headers'], {'Content-Type': 'application/json'} - ) + assert 'data' not in args[1] self.assertEqual( args[1]['timeout'], DEFAULT_TIMEOUT_SECONDS ) @@ -63,25 +60,21 @@ def test_start_container_regression_573(self): self.client.start(**{'container': fake_api.FAKE_CONTAINER_ID}) def test_start_container_with_lxc_conf(self): - def call_start(): + with pytest.raises(docker.errors.DeprecatedMethod): self.client.start( fake_api.FAKE_CONTAINER_ID, lxc_conf={'lxc.conf.k': 'lxc.conf.value'} ) - pytest.deprecated_call(call_start) - def test_start_container_with_lxc_conf_compat(self): - def call_start(): + with pytest.raises(docker.errors.DeprecatedMethod): self.client.start( fake_api.FAKE_CONTAINER_ID, lxc_conf=[{'Key': 'lxc.conf.k', 'Value': 'lxc.conf.value'}] ) - pytest.deprecated_call(call_start) - def test_start_container_with_binds_ro(self): - def call_start(): + with pytest.raises(docker.errors.DeprecatedMethod): self.client.start( fake_api.FAKE_CONTAINER_ID, binds={ '/tmp': { @@ -91,22 +84,18 @@ def call_start(): } ) - pytest.deprecated_call(call_start) - def test_start_container_with_binds_rw(self): - def call_start(): + with pytest.raises(docker.errors.DeprecatedMethod): self.client.start( fake_api.FAKE_CONTAINER_ID, binds={ '/tmp': {"bind": '/mnt', "ro": False} } ) - pytest.deprecated_call(call_start) - def test_start_container_with_port_binds(self): self.maxDiff = None - def call_start(): + with pytest.raises(docker.errors.DeprecatedMethod): self.client.start(fake_api.FAKE_CONTAINER_ID, port_bindings={ 1111: None, 2222: 2222, @@ -116,18 +105,14 @@ def call_start(): 6666: [('127.0.0.1',), ('192.168.0.1',)] }) - pytest.deprecated_call(call_start) - def test_start_container_with_links(self): - def call_start(): + with pytest.raises(docker.errors.DeprecatedMethod): self.client.start( fake_api.FAKE_CONTAINER_ID, links={'path': 'alias'} ) - pytest.deprecated_call(call_start) - def test_start_container_with_multiple_links(self): - def call_start(): + with pytest.raises(docker.errors.DeprecatedMethod): self.client.start( fake_api.FAKE_CONTAINER_ID, links={ @@ -136,21 +121,15 @@ def call_start(): } ) - pytest.deprecated_call(call_start) - def test_start_container_with_links_as_list_of_tuples(self): - def call_start(): + with pytest.raises(docker.errors.DeprecatedMethod): self.client.start(fake_api.FAKE_CONTAINER_ID, links=[('path', 'alias')]) - pytest.deprecated_call(call_start) - def test_start_container_privileged(self): - def call_start(): + with pytest.raises(docker.errors.DeprecatedMethod): self.client.start(fake_api.FAKE_CONTAINER_ID, privileged=True) - pytest.deprecated_call(call_start) - def test_start_container_with_dict_instead_of_id(self): self.client.start({'Id': fake_api.FAKE_CONTAINER_ID}) @@ -159,10 +138,7 @@ def test_start_container_with_dict_instead_of_id(self): args[0][1], url_prefix + 'containers/3cc2351ab11b/start' ) - self.assertEqual(json.loads(args[1]['data']), {}) - self.assertEqual( - args[1]['headers'], {'Content-Type': 'application/json'} - ) + assert 'data' not in args[1] self.assertEqual( args[1]['timeout'], DEFAULT_TIMEOUT_SECONDS ) From dbd704e68daa16fb91fa2228a9cc2a58219eeec3 Mon Sep 17 00:00:00 2001 From: Michael Merickel Date: Tue, 11 Oct 2016 02:38:20 -0500 Subject: [PATCH 0196/1301] do not assume that read will consume the number of bytes requested The issue is that ``os.read`` does not always read the expected number of bytes, and thus we are moving to the next frame too early causing drift in the byte stream. When the reading drifts, it starts reading garbage as the next frame size. The some examples of frame sizes were 4032897957 bytes, etc. Values this large were causing the exceptions from ``os.read``. fixes #1211 Signed-off-by: Michael Merickel --- docker/utils/socket.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/docker/utils/socket.py b/docker/utils/socket.py index 164b845afc..4080f253f5 100644 --- a/docker/utils/socket.py +++ b/docker/utils/socket.py @@ -69,7 +69,11 @@ def frames_iter(socket): """ Returns a generator of frames read from socket """ - n = next_frame_size(socket) - while n > 0: - yield read(socket, n) + while True: n = next_frame_size(socket) + if n == 0: + break + while n > 0: + result = read(socket, n) + n -= len(result) + yield result From 29215a0ad9d63b02a32eff4fb034a5f0a3c298a4 Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Mon, 28 Nov 2016 19:35:12 +0000 Subject: [PATCH 0197/1301] Add example for tag() Carrying #1120. Thanks @Faylixe! Signed-off-by: Ben Firshman --- docker/api/image.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docker/api/image.py b/docker/api/image.py index 2c8cbb2315..c1ebc69ca6 100644 --- a/docker/api/image.py +++ b/docker/api/image.py @@ -469,6 +469,11 @@ def tag(self, image, repository, tag=None, force=False): Raises: :py:class:`docker.errors.APIError` If the server returns an error. + + Example: + + >>> client.tag('ubuntu', 'localhost:5000/ubuntu', 'latest', + force=True) """ params = { 'tag': tag, From 4a7c772dafb4446fafbd599c8267e0522c138a76 Mon Sep 17 00:00:00 2001 From: Jon Cotton Date: Fri, 11 Nov 2016 16:24:18 -0800 Subject: [PATCH 0198/1301] Fix parsing for an environment file with newlines Fixes the new, purposely broken test added in the previous commit. Signed-off-by: Jon Cotton --- docker/utils/utils.py | 6 +++++- tests/unit/utils_test.py | 10 +++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/docker/utils/utils.py b/docker/utils/utils.py index b107f22e97..45adf3b9d3 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -1052,7 +1052,11 @@ def parse_env_file(env_file): if line[0] == '#': continue - parse_line = line.strip().split('=', 1) + line = line.strip() + if not line: + continue + + parse_line = line.split('=', 1) if len(parse_line) == 2: k, v = parse_line environment[k] = v diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index 57aa226d84..9166ed91ee 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -465,10 +465,18 @@ def test_parse_env_file_with_equals_character(self): def test_parse_env_file_commented_line(self): env_file = self.generate_tempfile( file_content='USER=jdoe\n#PASS=secret') - get_parse_env_file = parse_env_file((env_file)) + get_parse_env_file = parse_env_file(env_file) self.assertEqual(get_parse_env_file, {'USER': 'jdoe'}) os.unlink(env_file) + def test_parse_env_file_newline(self): + env_file = self.generate_tempfile( + file_content='\nUSER=jdoe\n\n\nPASS=secret') + get_parse_env_file = parse_env_file(env_file) + self.assertEqual(get_parse_env_file, + {'USER': 'jdoe', 'PASS': 'secret'}) + os.unlink(env_file) + def test_parse_env_file_invalid_line(self): env_file = self.generate_tempfile( file_content='USER jdoe') From 7ef48c3769cc42115b59157847275b2d072d7e5f Mon Sep 17 00:00:00 2001 From: Stepan Stipl Date: Wed, 31 Aug 2016 18:30:29 +0100 Subject: [PATCH 0199/1301] Allow custom PID mode for the container Docker added support for sharing PID namespaces with other containers since version 1.12 (see https://github.com/docker/docker/pull/22481). Signed-off-by: Stepan Stipl --- docker/utils/utils.py | 4 +--- tests/integration/api_container_test.py | 3 --- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/docker/utils/utils.py b/docker/utils/utils.py index b107f22e97..f4ad6f8769 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -735,9 +735,7 @@ def create_host_config(binds=None, port_bindings=None, lxc_conf=None, host_config['ShmSize'] = shm_size - if pid_mode not in (None, 'host'): - raise host_config_value_error('pid_mode', pid_mode) - elif pid_mode: + if pid_mode: host_config['PidMode'] = pid_mode if ipc_mode: diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index a5be6e765f..60c6ec0951 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -365,9 +365,6 @@ def test_create_host_config_exception_raising(self): self.assertRaises(TypeError, self.client.create_host_config, mem_swappiness='40') - self.assertRaises(ValueError, - self.client.create_host_config, pid_mode='40') - def test_create_with_environment_variable_no_value(self): container = self.client.create_container( BUSYBOX, From 8c27dd52335c55055a0e565ea4421d4e7c5ffdfb Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Wed, 23 Nov 2016 14:33:57 +0000 Subject: [PATCH 0200/1301] Show a helpful warning when people try to call `client.containers()` People upgrading to docker-py 2.0 without being aware of the new client API will likely try to call the old `containers()` method. This adds a helpful warning telling them to use APIClient to get the old API. Signed-off-by: Aanand Prasad --- docker/models/resource.py | 6 ++++++ tests/unit/client_test.py | 11 +++++++++++ 2 files changed, 17 insertions(+) diff --git a/docker/models/resource.py b/docker/models/resource.py index 9634a24f33..95712aefc6 100644 --- a/docker/models/resource.py +++ b/docker/models/resource.py @@ -60,6 +60,12 @@ def __init__(self, client=None): #: is on. self.client = client + def __call__(self, *args, **kwargs): + raise TypeError( + "'{}' object is not callable. You might be trying to use the old " + "(pre-2.0) API - use docker.APIClient if so." + .format(self.__class__.__name__)) + def list(self): raise NotImplementedError diff --git a/tests/unit/client_test.py b/tests/unit/client_test.py index 0a56b04d93..e40063f1de 100644 --- a/tests/unit/client_test.py +++ b/tests/unit/client_test.py @@ -1,5 +1,6 @@ import datetime import docker +from docker.utils import kwargs_from_env import os import unittest @@ -59,6 +60,16 @@ def test_call_api_client_method(self): assert "'Client' object has no attribute 'abcdef'" in s assert "this method is now on the object APIClient" not in s + def test_call_containers(self): + client = docker.Client(**kwargs_from_env()) + + with self.assertRaises(TypeError) as cm: + client.containers() + + s = str(cm.exception) + assert "'ContainerCollection' object is not callable" in s + assert "docker.APIClient" in s + class FromEnvTest(unittest.TestCase): From 44e57fb95d94145a789cc29884756b7fd4e03fe1 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 28 Nov 2016 12:01:25 -0800 Subject: [PATCH 0201/1301] Re-enable pid_mode checks for API < 1.24 Signed-off-by: Joffrey F --- docker/utils/utils.py | 2 ++ tests/integration/api_container_test.py | 4 ---- tests/unit/utils_test.py | 13 +++++++++++++ 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/docker/utils/utils.py b/docker/utils/utils.py index f4ad6f8769..d053feef4c 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -736,6 +736,8 @@ def create_host_config(binds=None, port_bindings=None, lxc_conf=None, host_config['ShmSize'] = shm_size if pid_mode: + if version_lt(version, '1.24') and pid_mode != 'host': + raise host_config_value_error('pid_mode', pid_mode) host_config['PidMode'] = pid_mode if ipc_mode: diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index 60c6ec0951..f09e75ad54 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -361,10 +361,6 @@ def test_create_with_memory_constraints_with_int(self): host_config = inspect['HostConfig'] self.assertIn('MemorySwappiness', host_config) - def test_create_host_config_exception_raising(self): - self.assertRaises(TypeError, - self.client.create_host_config, mem_swappiness='40') - def test_create_with_environment_variable_no_value(self): container = self.client.create_container( BUSYBOX, diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index 57aa226d84..c0eba60c60 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -205,6 +205,19 @@ def test_create_host_config_with_isolation(self): version='1.24', isolation={'isolation': 'hyperv'} ) + def test_create_host_config_pid_mode(self): + with pytest.raises(ValueError): + create_host_config(version='1.23', pid_mode='baccab125') + + config = create_host_config(version='1.23', pid_mode='host') + assert config.get('PidMode') == 'host' + config = create_host_config(version='1.24', pid_mode='baccab125') + assert config.get('PidMode') == 'baccab125' + + def test_create_host_config_invalid_mem_swappiness(self): + with pytest.raises(TypeError): + create_host_config(version='1.24', mem_swappiness='40') + class UlimitTest(unittest.TestCase): def test_create_host_config_dict_ulimit(self): From f5ac10c469fca252e69ae780749f4ec6fe369789 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 23 Nov 2016 15:15:58 -0800 Subject: [PATCH 0202/1301] Rename Client -> DockerClient Replace references to old Client with APIClient Moved contents of services.md to appropriate locations Signed-off-by: Joffrey F --- docker/__init__.py | 2 +- docker/api/build.py | 4 +- docker/api/container.py | 2 +- docker/api/swarm.py | 2 +- docker/client.py | 8 +- docker/models/images.py | 2 +- docker/types/services.py | 122 +++++++++++++ docker/utils/utils.py | 2 +- docs/client.rst | 4 +- docs/services.md | 268 ----------------------------- docs/tls.rst | 6 +- docs/user_guides/swarm_services.md | 65 +++++++ tests/unit/client_test.py | 4 +- tests/unit/fake_api_client.py | 2 +- 14 files changed, 206 insertions(+), 287 deletions(-) delete mode 100644 docs/services.md create mode 100644 docs/user_guides/swarm_services.md diff --git a/docker/__init__.py b/docker/__init__.py index acf4b5566e..96a9ef0955 100644 --- a/docker/__init__.py +++ b/docker/__init__.py @@ -1,6 +1,6 @@ # flake8: noqa from .api import APIClient -from .client import Client, from_env +from .client import DockerClient, from_env from .version import version, version_info __version__ = version diff --git a/docker/api/build.py b/docker/api/build.py index 297c9e0dbe..7cf4e0f77b 100644 --- a/docker/api/build.py +++ b/docker/api/build.py @@ -32,7 +32,7 @@ def build(self, path=None, tag=None, quiet=False, fileobj=None, Example: >>> from io import BytesIO - >>> from docker import Client + >>> from docker import APIClient >>> dockerfile = ''' ... # Shared Volume ... FROM busybox:buildroot-2014.02 @@ -40,7 +40,7 @@ def build(self, path=None, tag=None, quiet=False, fileobj=None, ... CMD ["/bin/sh"] ... ''' >>> f = BytesIO(dockerfile.encode('utf-8')) - >>> cli = Client(base_url='tcp://127.0.0.1:2375') + >>> cli = APIClient(base_url='tcp://127.0.0.1:2375') >>> response = [line for line in cli.build( ... fileobj=f, rm=True, tag='yourname/volume' ... )] diff --git a/docker/api/container.py b/docker/api/container.py index 72c5852dce..9b5ce59da0 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -215,7 +215,7 @@ def copy(self, container, resource): """ if utils.version_gte(self._version, '1.20'): warnings.warn( - 'Client.copy() is deprecated for API version >= 1.20, ' + 'APIClient.copy() is deprecated for API version >= 1.20, ' 'please use get_archive() instead', DeprecationWarning ) diff --git a/docker/api/swarm.py b/docker/api/swarm.py index a4cb8dded4..521076f4aa 100644 --- a/docker/api/swarm.py +++ b/docker/api/swarm.py @@ -77,7 +77,7 @@ def init_swarm(self, advertise_addr=None, listen_addr='0.0.0.0:2377', force_new_cluster (bool): Force creating a new Swarm, even if already part of one. Default: False swarm_spec (dict): Configuration settings of the new Swarm. Use - ``Client.create_swarm_spec`` to generate a valid + ``APIClient.create_swarm_spec`` to generate a valid configuration. Default: None Returns: diff --git a/docker/client.py b/docker/client.py index b3b470042a..b271eb7c1b 100644 --- a/docker/client.py +++ b/docker/client.py @@ -9,14 +9,14 @@ from .utils import kwargs_from_env -class Client(object): +class DockerClient(object): """ A client for communicating with a Docker server. Example: >>> import docker - >>> client = Client(base_url='unix://var/run/docker.sock') + >>> client = docker.DockerClient(base_url='unix://var/run/docker.sock') Args: base_url (str): URL to the Docker server. For example, @@ -155,7 +155,7 @@ def version(self, *args, **kwargs): version.__doc__ = APIClient.version.__doc__ def __getattr__(self, name): - s = ["'Client' object has no attribute '{}'".format(name)] + s = ["'DockerClient' object has no attribute '{}'".format(name)] # If a user calls a method on APIClient, they if hasattr(APIClient, name): s.append("In docker-py 2.0, this method is now on the object " @@ -164,4 +164,4 @@ def __getattr__(self, name): raise AttributeError(' '.join(s)) -from_env = Client.from_env +from_env = DockerClient.from_env diff --git a/docker/models/images.py b/docker/models/images.py index e0ff1f42c4..32068e6927 100644 --- a/docker/models/images.py +++ b/docker/models/images.py @@ -238,7 +238,7 @@ def pull(self, name, **kwargs): tag (str): The tag to pull insecure_registry (bool): Use an insecure registry auth_config (dict): Override the credentials that - :py:meth:`~docker.client.Client.login` has set for + :py:meth:`~docker.client.DockerClient.login` has set for this request. ``auth_config`` should contain the ``username`` and ``password`` keys to be valid. diff --git a/docker/types/services.py b/docker/types/services.py index a95e0f2ce3..9d5fa1b04b 100644 --- a/docker/types/services.py +++ b/docker/types/services.py @@ -4,6 +4,26 @@ class TaskTemplate(dict): + """ + Describe the task specification to be used when creating or updating a + service. + + Args: + + * container_spec (dict): Container settings for containers started as part + of this task. See the :py:class:`~docker.types.services.ContainerSpec` + for details. + * log_driver (dict): Log configuration for containers created as part of + the service. See the :py:class:`~docker.types.services.DriverConfig` + class for details. + * resources (dict): Resource requirements which apply to each individual + container created as part of the service. See the + :py:class:`~docker.types.services.Resources` class for details. + * restart_policy (dict): Specification for the restart policy which applies + to containers created as part of this service. See the + :py:class:`~docker.types.services.RestartPolicy` class for details. + * placement (list): A list of constraints. + """ def __init__(self, container_spec, resources=None, restart_policy=None, placement=None, log_driver=None): self['ContainerSpec'] = container_spec @@ -36,6 +56,25 @@ def placement(self): class ContainerSpec(dict): + """ + Describes the behavior of containers that are part of a task, and is used + when declaring a :py:class:`~docker.types.services.TaskTemplate`. + + Args: + + * image (string): The image name to use for the container. + * command (string or list): The command to be run in the image. + * args (list): Arguments to the command. + * env (dict): Environment variables. + * dir (string): The working directory for commands to run in. + * user (string): The user inside the container. + * labels (dict): A map of labels to associate with the service. + * mounts (list): A list of specifications for mounts to be added to + containers created as part of the service. See the + :py:class:`~docker.types.services.Mount` class for details. + * stop_grace_period (int): Amount of time to wait for the container to + terminate before forcefully killing it. + """ def __init__(self, image, command=None, args=None, env=None, workdir=None, user=None, labels=None, mounts=None, stop_grace_period=None): from ..utils import split_command # FIXME: circular import @@ -66,6 +105,28 @@ def __init__(self, image, command=None, args=None, env=None, workdir=None, class Mount(dict): + """ + Describes a mounted folder's configuration inside a container. A list of + ``Mount``s would be used as part of a + :py:class:`~docker.types.services.ContainerSpec`. + + Args: + + * target (string): Container path. + * source (string): Mount source (e.g. a volume name or a host path). + * type (string): The mount type (``bind`` or ``volume``). + Default: ``volume``. + * read_only (bool): Whether the mount should be read-only. + * propagation (string): A propagation mode with the value ``[r]private``, + ``[r]shared``, or ``[r]slave``. Only valid for the ``bind`` type. + * no_copy (bool): False if the volume should be populated with the data + from the target. Default: ``False``. Only valid for the ``volume`` type. + * labels (dict): User-defined name and labels for the volume. Only valid + for the ``volume`` type. + * driver_config (dict): Volume driver configuration. + See the :py:class:`~docker.types.services.DriverConfig` class for + details. Only valid for the ``volume`` type. + """ def __init__(self, target, source, type='volume', read_only=False, propagation=None, no_copy=False, labels=None, driver_config=None): @@ -120,6 +181,17 @@ def parse_mount_string(cls, string): class Resources(dict): + """ + Configures resource allocation for containers when made part of a + :py:class:`~docker.types.services.ContainerSpec`. + + Args: + + * cpu_limit (int): CPU limit in units of 10^9 CPU shares. + * mem_limit (int): Memory limit in Bytes. + * cpu_reservation (int): CPU reservation in units of 10^9 CPU shares. + * mem_reservation (int): Memory reservation in Bytes. + """ def __init__(self, cpu_limit=None, mem_limit=None, cpu_reservation=None, mem_reservation=None): limits = {} @@ -140,6 +212,19 @@ def __init__(self, cpu_limit=None, mem_limit=None, cpu_reservation=None, class UpdateConfig(dict): + """ + + Used to specify the way container updates should be performed by a service. + + Args: + + * parallelism (int): Maximum number of tasks to be updated in one iteration + (0 means unlimited parallelism). Default: 0. + * delay (int): Amount of time between updates. + * failure_action (string): Action to take if an updated task fails to run, + or stops running during the update. Acceptable values are ``continue`` + and ``pause``. Default: ``continue`` + """ def __init__(self, parallelism=0, delay=None, failure_action='continue'): self['Parallelism'] = parallelism if delay is not None: @@ -161,6 +246,19 @@ class RestartConditionTypesEnum(object): class RestartPolicy(dict): + """ + Used when creating a :py:class:`~docker.types.services.ContainerSpec`, + dictates whether a container should restart after stopping or failing. + + * condition (string): Condition for restart (``none``, ``on-failure``, + or ``any``). Default: `none`. + * delay (int): Delay between restart attempts. Default: 0 + * attempts (int): Maximum attempts to restart a given container before + giving up. Default value is 0, which is ignored. + * window (int): Time window used to evaluate the restart policy. Default + value is 0, which is unbounded. + """ + condition_types = RestartConditionTypesEnum def __init__(self, condition=RestartConditionTypesEnum.NONE, delay=0, @@ -177,6 +275,17 @@ def __init__(self, condition=RestartConditionTypesEnum.NONE, delay=0, class DriverConfig(dict): + """ + Indicates which driver to use, as well as its configuration. Can be used + as ``log_driver`` in a :py:class:`~docker.types.services.ContainerSpec`, + and for the `driver_config` in a volume + :py:class:`~docker.types.services.Mount`. + + Args: + + * name (string): Name of the driver to use. + * options (dict): Driver-specific options. Default: ``None``. + """ def __init__(self, name, options=None): self['Name'] = name if options: @@ -184,6 +293,19 @@ def __init__(self, name, options=None): class EndpointSpec(dict): + """ + Describes properties to access and load-balance a service. + + Args: + + * mode (string): The mode of resolution to use for internal load balancing + between tasks (``'vip'`` or ``'dnsrr'``). Defaults to ``'vip'`` if not + provided. + * ports (dict): Exposed ports that this service is accessible on from the + outside, in the form of ``{ target_port: published_port }`` or + ``{ target_port: (published_port, protocol) }``. Ports can only be + provided if the ``vip`` resolution mode is used. + """ def __init__(self, mode=None, ports=None): if ports: self['Ports'] = convert_service_ports(ports) diff --git a/docker/utils/utils.py b/docker/utils/utils.py index b107f22e97..26d1bf3f83 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -697,7 +697,7 @@ def create_host_config(binds=None, port_bindings=None, lxc_conf=None, if not version: warnings.warn( 'docker.utils.create_host_config() is deprecated. Please use ' - 'Client.create_host_config() instead.' + 'APIClient.create_host_config() instead.' ) version = constants.DEFAULT_DOCKER_API_VERSION diff --git a/docs/client.rst b/docs/client.rst index cd058fcc91..63bce2c875 100644 --- a/docs/client.rst +++ b/docs/client.rst @@ -6,14 +6,14 @@ Client Creating a client ----------------- -To communicate with the Docker daemon, you first need to instantiate a client. The easiest way to do that is by calling the function :py:func:`~docker.client.from_env`. It can also be configured manually by instantiating a :py:class:`~docker.client.Client` class. +To communicate with the Docker daemon, you first need to instantiate a client. The easiest way to do that is by calling the function :py:func:`~docker.client.from_env`. It can also be configured manually by instantiating a :py:class:`~docker.client.DockerClient` class. .. autofunction:: from_env() Client reference ---------------- -.. autoclass:: Client() +.. autoclass:: DockerClient() .. autoattribute:: containers .. autoattribute:: images diff --git a/docs/services.md b/docs/services.md deleted file mode 100644 index 69e0649054..0000000000 --- a/docs/services.md +++ /dev/null @@ -1,268 +0,0 @@ -# Swarm services - -Starting with Engine version 1.12 (API 1.24), it is possible to manage services -using the Docker Engine API. Note that the engine needs to be part of a -[Swarm cluster](swarm.md) before you can use the service-related methods. - -## Creating a service - -The `Client.create_service` method lets you create a new service inside the -cluster. The method takes several arguments, `task_template` being mandatory. -This dictionary of values is most easily produced by instantiating a -`TaskTemplate` object. - -```python -container_spec = docker.types.ContainerSpec( - image='busybox', command=['echo', 'hello'] -) -task_tmpl = docker.types.TaskTemplate(container_spec) -service_id = client.create_service(task_tmpl, name=name) -``` - -## Listing services - -List all existing services using the `Client.services` method. - -```python -client.services(filters={'name': 'mysql'}) -``` - -## Retrieving service configuration - -To retrieve detailed information and configuration for a specific service, you -may use the `Client.inspect_service` method using the service's ID or name. - -```python -client.inspect_service(service='my_service_name') -``` - -## Updating service configuration - -The `Client.update_service` method lets you update a service's configuration. -The mandatory `version` argument (used to prevent concurrent writes) can be -retrieved using `Client.inspect_service`. - -```python -container_spec = docker.types.ContainerSpec( - image='busybox', command=['echo', 'hello world'] -) -task_tmpl = docker.types.TaskTemplate(container_spec) - -svc_version = client.inspect_service(svc_id)['Version']['Index'] - -client.update_service( - svc_id, svc_version, name='new_name', task_template=task_tmpl -) -``` - -## Removing a service - -A service may be removed simply using the `Client.remove_service` method. -Either the service name or service ID can be used as argument. - -```python -client.remove_service('my_service_name') -``` - -## Service API documentation - -### Client.create_service - -Create a service. - -**Params:** - -* task_template (dict): Specification of the task to start as part of the new - service. See the [TaskTemplate class](#TaskTemplate) for details. -* name (string): User-defined name for the service. Optional. -* labels (dict): A map of labels to associate with the service. Optional. -* mode (string): Scheduling mode for the service (`replicated` or `global`). - Defaults to `replicated`. -* update_config (dict): Specification for the update strategy of the service. - See the [UpdateConfig class](#UpdateConfig) for details. Default: `None`. -* networks (list): List of network names or IDs to attach the service to. - Default: `None`. -* endpoint_spec (dict): Properties that can be configured to access and load - balance a service. Default: `None`. - -**Returns:** A dictionary containing an `ID` key for the newly created service. - -### Client.inspect_service - -Return information on a service. - -**Params:** - -* service (string): A service identifier (either its name or service ID) - -**Returns:** `True` if successful. Raises an `APIError` otherwise. - -### Client.remove_service - -Stop and remove a service. - -**Params:** - -* service (string): A service identifier (either its name or service ID) - -**Returns:** `True` if successful. Raises an `APIError` otherwise. - -### Client.services - -List services. - -**Params:** - -* filters (dict): Filters to process on the nodes list. Valid filters: - `id` and `name`. Default: `None`. - -**Returns:** A list of dictionaries containing data about each service. - -### Client.update_service - -Update a service. - -**Params:** - -* service (string): A service identifier (either its name or service ID). -* version (int): The version number of the service object being updated. This - is required to avoid conflicting writes. -* task_template (dict): Specification of the updated task to start as part of - the service. See the [TaskTemplate class](#TaskTemplate) for details. -* name (string): New name for the service. Optional. -* labels (dict): A map of labels to associate with the service. Optional. -* mode (string): Scheduling mode for the service (`replicated` or `global`). - Defaults to `replicated`. -* update_config (dict): Specification for the update strategy of the service. - See the [UpdateConfig class](#UpdateConfig) for details. Default: `None`. -* networks (list): List of network names or IDs to attach the service to. - Default: `None`. -* endpoint_spec (dict): Properties that can be configured to access and load - balance a service. Default: `None`. - -**Returns:** `True` if successful. Raises an `APIError` otherwise. - -### Configuration objects (`docker.types`) - -#### ContainerSpec - -A `ContainerSpec` object describes the behavior of containers that are part -of a task, and is used when declaring a `TaskTemplate`. - -**Params:** - -* image (string): The image name to use for the container. -* command (string or list): The command to be run in the image. -* args (list): Arguments to the command. -* env (dict): Environment variables. -* dir (string): The working directory for commands to run in. -* user (string): The user inside the container. -* labels (dict): A map of labels to associate with the service. -* mounts (list): A list of specifications for mounts to be added to containers - created as part of the service. See the [Mount class](#Mount) for details. -* stop_grace_period (int): Amount of time to wait for the container to - terminate before forcefully killing it. - -#### DriverConfig - -A `LogDriver` object indicates which driver to use, as well as its -configuration. It can be used for the `log_driver` in a `ContainerSpec`, -and for the `driver_config` in a volume `Mount`. - -**Params:** - -* name (string): Name of the logging driver to use. -* options (dict): Driver-specific options. Default: `None`. - -#### EndpointSpec - -An `EndpointSpec` object describes properties to access and load-balance a -service. - -**Params:** - -* mode (string): The mode of resolution to use for internal load balancing - between tasks (`'vip'` or `'dnsrr'`). Defaults to `'vip'` if not provided. -* ports (dict): Exposed ports that this service is accessible on from the - outside, in the form of `{ target_port: published_port }` or - `{ target_port: (published_port, protocol) }`. Ports can only be provided if - the `vip` resolution mode is used. - -#### Mount - -A `Mount` object describes a mounted folder's configuration inside a -container. A list of `Mount`s would be used as part of a `ContainerSpec`. - -* target (string): Container path. -* source (string): Mount source (e.g. a volume name or a host path). -* type (string): The mount type (`bind` or `volume`). Default: `volume`. -* read_only (bool): Whether the mount should be read-only. -* propagation (string): A propagation mode with the value `[r]private`, - `[r]shared`, or `[r]slave`. Only valid for the `bind` type. -* no_copy (bool): False if the volume should be populated with the data from - the target. Default: `False`. Only valid for the `volume` type. -* labels (dict): User-defined name and labels for the volume. Only valid for - the `volume` type. -* driver_config (dict): Volume driver configuration. - See the [DriverConfig class](#DriverConfig) for details. Only valid for the - `volume` type. - -#### Resources - -A `Resources` object configures resource allocation for containers when -made part of a `ContainerSpec`. - -**Params:** - -* cpu_limit (int): CPU limit in units of 10^9 CPU shares. -* mem_limit (int): Memory limit in Bytes. -* cpu_reservation (int): CPU reservation in units of 10^9 CPU shares. -* mem_reservation (int): Memory reservation in Bytes. - -#### RestartPolicy - -A `RestartPolicy` object is used when creating a `ContainerSpec`. It dictates -whether a container should restart after stopping or failing. - -* condition (string): Condition for restart (`none`, `on-failure`, or `any`). - Default: `none`. -* delay (int): Delay between restart attempts. Default: 0 -* attempts (int): Maximum attempts to restart a given container before giving - up. Default value is 0, which is ignored. -* window (int): Time window used to evaluate the restart policy. Default value - is 0, which is unbounded. - - -#### TaskTemplate - -A `TaskTemplate` object can be used to describe the task specification to be -used when creating or updating a service. - -**Params:** - -* container_spec (dict): Container settings for containers started as part of - this task. See the [ContainerSpec class](#ContainerSpec) for details. -* log_driver (dict): Log configuration for containers created as part of the - service. See the [DriverConfig class](#DriverConfig) for details. -* resources (dict): Resource requirements which apply to each individual - container created as part of the service. See the - [Resources class](#Resources) for details. -* restart_policy (dict): Specification for the restart policy which applies - to containers created as part of this service. See the - [RestartPolicy class](#RestartPolicy) for details. -* placement (list): A list of constraints. - - -#### UpdateConfig - -An `UpdateConfig` object can be used to specify the way container updates -should be performed by a service. - -**Params:** - -* parallelism (int): Maximum number of tasks to be updated in one iteration - (0 means unlimited parallelism). Default: 0. -* delay (int): Amount of time between updates. -* failure_action (string): Action to take if an updated task fails to run, or - stops running during the update. Acceptable values are `continue` and - `pause`. Default: `continue` diff --git a/docs/tls.rst b/docs/tls.rst index 0f318ff643..2e2f1ea94c 100644 --- a/docs/tls.rst +++ b/docs/tls.rst @@ -3,7 +3,7 @@ Using TLS .. py:module:: docker.tls -Both the main :py:class:`~docker.client.Client` and low-level +Both the main :py:class:`~docker.client.DockerClient` and low-level :py:class:`~docker.api.client.APIClient` can connect to the Docker daemon with TLS. This is all configured automatically for you if you're using :py:func:`~docker.client.from_env`, but if you need some extra control it is possible to configure it manually by using a :py:class:`TLSConfig` object. @@ -16,7 +16,7 @@ For example, to check the server against a specific CA certificate: .. code-block:: python tls_config = docker.tls.TLSConfig(ca_cert='/path/to/ca.pem') - client = docker.Client(base_url='', tls=tls_config) + client = docker.DockerClient(base_url='', tls=tls_config) This is the equivalent of ``docker --tlsverify --tlscacert /path/to/ca.pem ...``. @@ -27,7 +27,7 @@ To authenticate with client certs: tls_config = docker.tls.TLSConfig( client_cert=('/path/to/client-cert.pem', '/path/to/client-key.pem') ) - client = docker.Client(base_url='', tls=tls_config) + client = docker.DockerClient(base_url='', tls=tls_config) This is the equivalent of ``docker --tls --tlscert /path/to/client-cert.pem --tlskey /path/to/client-key.pem ...``. diff --git a/docs/user_guides/swarm_services.md b/docs/user_guides/swarm_services.md new file mode 100644 index 0000000000..9bd4dca3fb --- /dev/null +++ b/docs/user_guides/swarm_services.md @@ -0,0 +1,65 @@ +# Swarm services + +Starting with Engine version 1.12 (API 1.24), it is possible to manage services +using the Docker Engine API. Note that the engine needs to be part of a +[Swarm cluster](../swarm.rst) before you can use the service-related methods. + +## Creating a service + +The `APIClient.create_service` method lets you create a new service inside the +cluster. The method takes several arguments, `task_template` being mandatory. +This dictionary of values is most easily produced by instantiating a +`TaskTemplate` object. + +```python +container_spec = docker.types.ContainerSpec( + image='busybox', command=['echo', 'hello'] +) +task_tmpl = docker.types.TaskTemplate(container_spec) +service_id = client.create_service(task_tmpl, name=name) +``` + +## Listing services + +List all existing services using the `APIClient.services` method. + +```python +client.services(filters={'name': 'mysql'}) +``` + +## Retrieving service configuration + +To retrieve detailed information and configuration for a specific service, you +may use the `APIClient.inspect_service` method using the service's ID or name. + +```python +client.inspect_service(service='my_service_name') +``` + +## Updating service configuration + +The `APIClient.update_service` method lets you update a service's configuration. +The mandatory `version` argument (used to prevent concurrent writes) can be +retrieved using `APIClient.inspect_service`. + +```python +container_spec = docker.types.ContainerSpec( + image='busybox', command=['echo', 'hello world'] +) +task_tmpl = docker.types.TaskTemplate(container_spec) + +svc_version = client.inspect_service(svc_id)['Version']['Index'] + +client.update_service( + svc_id, svc_version, name='new_name', task_template=task_tmpl +) +``` + +## Removing a service + +A service may be removed simply using the `APIClient.remove_service` method. +Either the service name or service ID can be used as argument. + +```python +client.remove_service('my_service_name') +``` \ No newline at end of file diff --git a/tests/unit/client_test.py b/tests/unit/client_test.py index 0a56b04d93..aff6973aae 100644 --- a/tests/unit/client_test.py +++ b/tests/unit/client_test.py @@ -50,13 +50,13 @@ def test_call_api_client_method(self): with self.assertRaises(AttributeError) as cm: client.create_container() s = str(cm.exception) - assert "'Client' object has no attribute 'create_container'" in s + assert "'DockerClient' object has no attribute 'create_container'" in s assert "this method is now on the object APIClient" in s with self.assertRaises(AttributeError) as cm: client.abcdef() s = str(cm.exception) - assert "'Client' object has no attribute 'abcdef'" in s + assert "'DockerClient' object has no attribute 'abcdef'" in s assert "this method is now on the object APIClient" not in s diff --git a/tests/unit/fake_api_client.py b/tests/unit/fake_api_client.py index 84e1d9def3..47890ace91 100644 --- a/tests/unit/fake_api_client.py +++ b/tests/unit/fake_api_client.py @@ -56,6 +56,6 @@ def make_fake_client(): """ Returns a Client with a fake APIClient. """ - client = docker.Client() + client = docker.DockerClient() client.api = make_fake_api_client() return client From 515db1f6fd8db15edda33b72cd6f0fd955f8d777 Mon Sep 17 00:00:00 2001 From: Tomas Tomecek Date: Wed, 26 Oct 2016 16:58:27 +0200 Subject: [PATCH 0203/1301] exec: fix running with detach=True Fixes #1271 Signed-off-by: Tomas Tomecek --- docker/api/exec_api.py | 3 ++- tests/integration/api_exec_test.py | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/docker/api/exec_api.py b/docker/api/exec_api.py index 694b30a674..6c3e638338 100644 --- a/docker/api/exec_api.py +++ b/docker/api/exec_api.py @@ -137,7 +137,8 @@ def exec_start(self, exec_id, detach=False, tty=False, stream=False, data=data, stream=True ) - + if detach: + return self._result(res) if socket: return self._get_raw_response_socket(res) return self._read_from_socket(res, stream) diff --git a/tests/integration/api_exec_test.py b/tests/integration/api_exec_test.py index 0ceeefa9e1..55286e374e 100644 --- a/tests/integration/api_exec_test.py +++ b/tests/integration/api_exec_test.py @@ -93,6 +93,21 @@ def test_exec_start_socket(self): data = read_exactly(socket, next_size) self.assertEqual(data.decode('utf-8'), line) + def test_exec_start_detached(self): + container = self.client.create_container(BUSYBOX, 'cat', + detach=True, stdin_open=True) + container_id = container['Id'] + self.client.start(container_id) + self.tmp_containers.append(container_id) + + exec_id = self.client.exec_create( + container_id, ['printf', "asdqwe"]) + self.assertIn('Id', exec_id) + + response = self.client.exec_start(exec_id, detach=True) + + self.assertEqual(response, "") + def test_exec_inspect(self): container = self.client.create_container(BUSYBOX, 'cat', detach=True, stdin_open=True) From 176346bd95d4b62286eb6b7a1673f286cd128d97 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 30 Nov 2016 15:23:41 -0800 Subject: [PATCH 0204/1301] Exclude requests==2.12.2 from dependencies Signed-off-by: Joffrey F --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 3f2e3c4ace..b82a74f7d1 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ SOURCE_DIR = os.path.join(ROOT_DIR) requirements = [ - 'requests >= 2.5.2, != 2.11.0', + 'requests >= 2.5.2, != 2.11.0, != 2.12.2', 'six >= 1.4.0', 'websocket-client >= 0.32.0', 'docker-pycreds >= 0.2.1' From 993f298e85539751343048467004a679414426ec Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 29 Nov 2016 18:00:46 -0800 Subject: [PATCH 0205/1301] Move config type creation from docker.utils functions to classes in docker.types Signed-off-by: Joffrey F --- docker/api/container.py | 16 +- docker/api/network.py | 7 +- docker/api/swarm.py | 3 +- docker/models/containers.py | 4 +- docker/models/networks.py | 6 +- docker/types/__init__.py | 5 +- docker/types/containers.py | 490 +++++++++++++++++++ docker/types/networks.py | 104 ++++ docker/types/services.py | 143 +++--- docker/utils/__init__.py | 9 +- docker/utils/types.py | 7 - docker/utils/utils.py | 611 +----------------------- docs/api.rst | 25 +- tests/integration/api_container_test.py | 8 +- tests/integration/api_network_test.py | 23 +- tests/unit/api_network_test.py | 8 +- tests/unit/client_test.py | 2 +- tests/unit/dockertypes_test.py | 255 ++++++++++ tests/unit/utils_test.py | 248 +--------- 19 files changed, 1000 insertions(+), 974 deletions(-) create mode 100644 docker/types/networks.py delete mode 100644 docker/utils/types.py create mode 100644 tests/unit/dockertypes_test.py diff --git a/docker/api/container.py b/docker/api/container.py index 10d7ff481b..afe696ca19 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -4,7 +4,9 @@ from .. import errors from .. import utils -from ..utils.utils import create_networking_config, create_endpoint_config +from ..types import ( + ContainerConfig, EndpointConfig, HostConfig, NetworkingConfig +) class ContainerApiMixin(object): @@ -430,8 +432,8 @@ def create_container(self, image, command=None, hostname=None, user=None, ) config = self.create_container_config( - image, command, hostname, user, detach, stdin_open, - tty, mem_limit, ports, environment, dns, volumes, volumes_from, + image, command, hostname, user, detach, stdin_open, tty, mem_limit, + ports, dns, environment, volumes, volumes_from, network_disabled, entrypoint, cpu_shares, working_dir, domainname, memswap_limit, cpuset, host_config, mac_address, labels, volume_driver, stop_signal, networking_config, healthcheck, @@ -439,7 +441,7 @@ def create_container(self, image, command=None, hostname=None, user=None, return self.create_container_from_config(config, name) def create_container_config(self, *args, **kwargs): - return utils.create_container_config(self._version, *args, **kwargs) + return ContainerConfig(self._version, *args, **kwargs) def create_container_from_config(self, config, name=None): u = self._url("/containers/create") @@ -582,7 +584,7 @@ def create_host_config(self, *args, **kwargs): "keyword argument 'version'" ) kwargs['version'] = self._version - return utils.create_host_config(*args, **kwargs) + return HostConfig(*args, **kwargs) def create_networking_config(self, *args, **kwargs): """ @@ -608,7 +610,7 @@ def create_networking_config(self, *args, **kwargs): ) """ - return create_networking_config(*args, **kwargs) + return NetworkingConfig(*args, **kwargs) def create_endpoint_config(self, *args, **kwargs): """ @@ -641,7 +643,7 @@ def create_endpoint_config(self, *args, **kwargs): ) """ - return create_endpoint_config(self._version, *args, **kwargs) + return EndpointConfig(self._version, *args, **kwargs) @utils.check_resource def diff(self, container): diff --git a/docker/api/network.py b/docker/api/network.py index 65aeb30525..33da7eadfe 100644 --- a/docker/api/network.py +++ b/docker/api/network.py @@ -46,8 +46,7 @@ def create_network(self, name, driver=None, options=None, ipam=None, name (str): Name of the network driver (str): Name of the driver used to create the network options (dict): Driver options as a key-value dictionary - ipam (dict): Optional custom IP scheme for the network. - Created with :py:meth:`~docker.utils.create_ipam_config`. + ipam (IPAMConfig): Optional custom IP scheme for the network. check_duplicate (bool): Request daemon to check for networks with same name. Default: ``True``. internal (bool): Restrict external access to the network. Default @@ -74,11 +73,11 @@ def create_network(self, name, driver=None, options=None, ipam=None, .. code-block:: python - >>> ipam_pool = docker.utils.create_ipam_pool( + >>> ipam_pool = docker.types.IPAMPool( subnet='192.168.52.0/24', gateway='192.168.52.254' ) - >>> ipam_config = docker.utils.create_ipam_config( + >>> ipam_config = docker.types.IPAMConfig( pool_configs=[ipam_pool] ) >>> docker_client.create_network("network1", driver="bridge", diff --git a/docker/api/swarm.py b/docker/api/swarm.py index 521076f4aa..6a1b752fc5 100644 --- a/docker/api/swarm.py +++ b/docker/api/swarm.py @@ -1,5 +1,6 @@ import logging from six.moves import http_client +from .. import types from .. import utils log = logging.getLogger(__name__) @@ -50,7 +51,7 @@ def create_swarm_spec(self, *args, **kwargs): force_new_cluster=False, swarm_spec=spec ) """ - return utils.SwarmSpec(*args, **kwargs) + return types.SwarmSpec(*args, **kwargs) @utils.minimum_version('1.24') def init_swarm(self, advertise_addr=None, listen_addr='0.0.0.0:2377', diff --git a/docker/models/containers.py b/docker/models/containers.py index 9682248a91..ad1cb6139c 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -2,7 +2,7 @@ from ..errors import (ContainerError, ImageNotFound, create_unexpected_kwargs_error) -from ..utils import create_host_config +from ..types import HostConfig from .images import Image from .resource import Collection, Model @@ -869,7 +869,7 @@ def _create_container_args(kwargs): if kwargs: raise create_unexpected_kwargs_error('run', kwargs) - create_kwargs['host_config'] = create_host_config(**host_config_kwargs) + create_kwargs['host_config'] = HostConfig(**host_config_kwargs) # Fill in any kwargs which need processing by create_host_config first port_bindings = create_kwargs['host_config'].get('PortBindings') diff --git a/docker/models/networks.py b/docker/models/networks.py index 64af9ad9aa..d5e2097295 100644 --- a/docker/models/networks.py +++ b/docker/models/networks.py @@ -98,7 +98,7 @@ def create(self, name, *args, **kwargs): driver (str): Name of the driver used to create the network options (dict): Driver options as a key-value dictionary ipam (dict): Optional custom IP scheme for the network. - Created with :py:meth:`~docker.utils.create_ipam_config`. + Created with :py:class:`~docker.types.IPAMConfig`. check_duplicate (bool): Request daemon to check for networks with same name. Default: ``True``. internal (bool): Restrict external access to the network. Default @@ -125,11 +125,11 @@ def create(self, name, *args, **kwargs): .. code-block:: python - >>> ipam_pool = docker.utils.create_ipam_pool( + >>> ipam_pool = docker.types.IPAMPool( subnet='192.168.52.0/24', gateway='192.168.52.254' ) - >>> ipam_config = docker.utils.create_ipam_config( + >>> ipam_config = docker.types.IPAMConfig( pool_configs=[ipam_pool] ) >>> client.networks.create( diff --git a/docker/types/__init__.py b/docker/types/__init__.py index a7c3a56b53..7230723ee4 100644 --- a/docker/types/__init__.py +++ b/docker/types/__init__.py @@ -1,8 +1,9 @@ # flake8: noqa -from .containers import LogConfig, Ulimit +from .containers import ContainerConfig, HostConfig, LogConfig, Ulimit +from .healthcheck import Healthcheck +from .networks import EndpointConfig, IPAMConfig, IPAMPool, NetworkingConfig from .services import ( ContainerSpec, DriverConfig, EndpointSpec, Mount, Resources, RestartPolicy, TaskTemplate, UpdateConfig ) -from .healthcheck import Healthcheck from .swarm import SwarmSpec, SwarmExternalCA diff --git a/docker/types/containers.py b/docker/types/containers.py index 40a44caf09..8fdecb3e3d 100644 --- a/docker/types/containers.py +++ b/docker/types/containers.py @@ -1,6 +1,14 @@ import six +import warnings +from .. import errors +from ..utils.utils import ( + convert_port_bindings, convert_tmpfs_mounts, convert_volume_binds, + format_environment, normalize_links, parse_bytes, parse_devices, + split_command, version_gte, version_lt, +) from .base import DictType +from .healthcheck import Healthcheck class LogConfigTypesEnum(object): @@ -90,3 +98,485 @@ def hard(self): @hard.setter def hard(self, value): self['Hard'] = value + + +class HostConfig(dict): + def __init__(self, version, binds=None, port_bindings=None, + lxc_conf=None, publish_all_ports=False, links=None, + privileged=False, dns=None, dns_search=None, + volumes_from=None, network_mode=None, restart_policy=None, + cap_add=None, cap_drop=None, devices=None, extra_hosts=None, + read_only=None, pid_mode=None, ipc_mode=None, + security_opt=None, ulimits=None, log_config=None, + mem_limit=None, memswap_limit=None, mem_reservation=None, + kernel_memory=None, mem_swappiness=None, cgroup_parent=None, + group_add=None, cpu_quota=None, cpu_period=None, + blkio_weight=None, blkio_weight_device=None, + device_read_bps=None, device_write_bps=None, + device_read_iops=None, device_write_iops=None, + oom_kill_disable=False, shm_size=None, sysctls=None, + tmpfs=None, oom_score_adj=None, dns_opt=None, cpu_shares=None, + cpuset_cpus=None, userns_mode=None, pids_limit=None, + isolation=None): + + if mem_limit is not None: + self['Memory'] = parse_bytes(mem_limit) + + if memswap_limit is not None: + self['MemorySwap'] = parse_bytes(memswap_limit) + + if mem_reservation: + if version_lt(version, '1.21'): + raise host_config_version_error('mem_reservation', '1.21') + + self['MemoryReservation'] = parse_bytes(mem_reservation) + + if kernel_memory: + if version_lt(version, '1.21'): + raise host_config_version_error('kernel_memory', '1.21') + + self['KernelMemory'] = parse_bytes(kernel_memory) + + if mem_swappiness is not None: + if version_lt(version, '1.20'): + raise host_config_version_error('mem_swappiness', '1.20') + if not isinstance(mem_swappiness, int): + raise host_config_type_error( + 'mem_swappiness', mem_swappiness, 'int' + ) + + self['MemorySwappiness'] = mem_swappiness + + if shm_size is not None: + if isinstance(shm_size, six.string_types): + shm_size = parse_bytes(shm_size) + + self['ShmSize'] = shm_size + + if pid_mode: + if version_lt(version, '1.24') and pid_mode != 'host': + raise host_config_value_error('pid_mode', pid_mode) + self['PidMode'] = pid_mode + + if ipc_mode: + self['IpcMode'] = ipc_mode + + if privileged: + self['Privileged'] = privileged + + if oom_kill_disable: + if version_lt(version, '1.20'): + raise host_config_version_error('oom_kill_disable', '1.19') + + self['OomKillDisable'] = oom_kill_disable + + if oom_score_adj: + if version_lt(version, '1.22'): + raise host_config_version_error('oom_score_adj', '1.22') + if not isinstance(oom_score_adj, int): + raise host_config_type_error( + 'oom_score_adj', oom_score_adj, 'int' + ) + self['OomScoreAdj'] = oom_score_adj + + if publish_all_ports: + self['PublishAllPorts'] = publish_all_ports + + if read_only is not None: + self['ReadonlyRootfs'] = read_only + + if dns_search: + self['DnsSearch'] = dns_search + + if network_mode: + self['NetworkMode'] = network_mode + elif network_mode is None and version_gte(version, '1.20'): + self['NetworkMode'] = 'default' + + if restart_policy: + if not isinstance(restart_policy, dict): + raise host_config_type_error( + 'restart_policy', restart_policy, 'dict' + ) + + self['RestartPolicy'] = restart_policy + + if cap_add: + self['CapAdd'] = cap_add + + if cap_drop: + self['CapDrop'] = cap_drop + + if devices: + self['Devices'] = parse_devices(devices) + + if group_add: + if version_lt(version, '1.20'): + raise host_config_version_error('group_add', '1.20') + + self['GroupAdd'] = [six.text_type(grp) for grp in group_add] + + if dns is not None: + self['Dns'] = dns + + if dns_opt is not None: + if version_lt(version, '1.21'): + raise host_config_version_error('dns_opt', '1.21') + + self['DnsOptions'] = dns_opt + + if security_opt is not None: + if not isinstance(security_opt, list): + raise host_config_type_error( + 'security_opt', security_opt, 'list' + ) + + self['SecurityOpt'] = security_opt + + if sysctls: + if not isinstance(sysctls, dict): + raise host_config_type_error('sysctls', sysctls, 'dict') + self['Sysctls'] = {} + for k, v in six.iteritems(sysctls): + self['Sysctls'][k] = six.text_type(v) + + if volumes_from is not None: + if isinstance(volumes_from, six.string_types): + volumes_from = volumes_from.split(',') + + self['VolumesFrom'] = volumes_from + + if binds is not None: + self['Binds'] = convert_volume_binds(binds) + + if port_bindings is not None: + self['PortBindings'] = convert_port_bindings(port_bindings) + + if extra_hosts is not None: + if isinstance(extra_hosts, dict): + extra_hosts = [ + '{0}:{1}'.format(k, v) + for k, v in sorted(six.iteritems(extra_hosts)) + ] + + self['ExtraHosts'] = extra_hosts + + if links is not None: + self['Links'] = normalize_links(links) + + if isinstance(lxc_conf, dict): + formatted = [] + for k, v in six.iteritems(lxc_conf): + formatted.append({'Key': k, 'Value': str(v)}) + lxc_conf = formatted + + if lxc_conf is not None: + self['LxcConf'] = lxc_conf + + if cgroup_parent is not None: + self['CgroupParent'] = cgroup_parent + + if ulimits is not None: + if not isinstance(ulimits, list): + raise host_config_type_error('ulimits', ulimits, 'list') + self['Ulimits'] = [] + for l in ulimits: + if not isinstance(l, Ulimit): + l = Ulimit(**l) + self['Ulimits'].append(l) + + if log_config is not None: + if not isinstance(log_config, LogConfig): + if not isinstance(log_config, dict): + raise host_config_type_error( + 'log_config', log_config, 'LogConfig' + ) + log_config = LogConfig(**log_config) + + self['LogConfig'] = log_config + + if cpu_quota: + if not isinstance(cpu_quota, int): + raise host_config_type_error('cpu_quota', cpu_quota, 'int') + if version_lt(version, '1.19'): + raise host_config_version_error('cpu_quota', '1.19') + + self['CpuQuota'] = cpu_quota + + if cpu_period: + if not isinstance(cpu_period, int): + raise host_config_type_error('cpu_period', cpu_period, 'int') + if version_lt(version, '1.19'): + raise host_config_version_error('cpu_period', '1.19') + + self['CpuPeriod'] = cpu_period + + if cpu_shares: + if version_lt(version, '1.18'): + raise host_config_version_error('cpu_shares', '1.18') + + if not isinstance(cpu_shares, int): + raise host_config_type_error('cpu_shares', cpu_shares, 'int') + + self['CpuShares'] = cpu_shares + + if cpuset_cpus: + if version_lt(version, '1.18'): + raise host_config_version_error('cpuset_cpus', '1.18') + + self['CpuSetCpus'] = cpuset_cpus + + if blkio_weight: + if not isinstance(blkio_weight, int): + raise host_config_type_error( + 'blkio_weight', blkio_weight, 'int' + ) + if version_lt(version, '1.22'): + raise host_config_version_error('blkio_weight', '1.22') + self["BlkioWeight"] = blkio_weight + + if blkio_weight_device: + if not isinstance(blkio_weight_device, list): + raise host_config_type_error( + 'blkio_weight_device', blkio_weight_device, 'list' + ) + if version_lt(version, '1.22'): + raise host_config_version_error('blkio_weight_device', '1.22') + self["BlkioWeightDevice"] = blkio_weight_device + + if device_read_bps: + if not isinstance(device_read_bps, list): + raise host_config_type_error( + 'device_read_bps', device_read_bps, 'list' + ) + if version_lt(version, '1.22'): + raise host_config_version_error('device_read_bps', '1.22') + self["BlkioDeviceReadBps"] = device_read_bps + + if device_write_bps: + if not isinstance(device_write_bps, list): + raise host_config_type_error( + 'device_write_bps', device_write_bps, 'list' + ) + if version_lt(version, '1.22'): + raise host_config_version_error('device_write_bps', '1.22') + self["BlkioDeviceWriteBps"] = device_write_bps + + if device_read_iops: + if not isinstance(device_read_iops, list): + raise host_config_type_error( + 'device_read_iops', device_read_iops, 'list' + ) + if version_lt(version, '1.22'): + raise host_config_version_error('device_read_iops', '1.22') + self["BlkioDeviceReadIOps"] = device_read_iops + + if device_write_iops: + if not isinstance(device_write_iops, list): + raise host_config_type_error( + 'device_write_iops', device_write_iops, 'list' + ) + if version_lt(version, '1.22'): + raise host_config_version_error('device_write_iops', '1.22') + self["BlkioDeviceWriteIOps"] = device_write_iops + + if tmpfs: + if version_lt(version, '1.22'): + raise host_config_version_error('tmpfs', '1.22') + self["Tmpfs"] = convert_tmpfs_mounts(tmpfs) + + if userns_mode: + if version_lt(version, '1.23'): + raise host_config_version_error('userns_mode', '1.23') + + if userns_mode != "host": + raise host_config_value_error("userns_mode", userns_mode) + self['UsernsMode'] = userns_mode + + if pids_limit: + if not isinstance(pids_limit, int): + raise host_config_type_error('pids_limit', pids_limit, 'int') + if version_lt(version, '1.23'): + raise host_config_version_error('pids_limit', '1.23') + self["PidsLimit"] = pids_limit + + if isolation: + if not isinstance(isolation, six.string_types): + raise host_config_type_error('isolation', isolation, 'string') + if version_lt(version, '1.24'): + raise host_config_version_error('isolation', '1.24') + self['Isolation'] = isolation + + +def host_config_type_error(param, param_value, expected): + error_msg = 'Invalid type for {0} param: expected {1} but found {2}' + return TypeError(error_msg.format(param, expected, type(param_value))) + + +def host_config_version_error(param, version, less_than=True): + operator = '<' if less_than else '>' + error_msg = '{0} param is not supported in API versions {1} {2}' + return errors.InvalidVersion(error_msg.format(param, operator, version)) + + +def host_config_value_error(param, param_value): + error_msg = 'Invalid value for {0} param: {1}' + return ValueError(error_msg.format(param, param_value)) + + +class ContainerConfig(dict): + def __init__( + self, version, image, command, hostname=None, user=None, detach=False, + stdin_open=False, tty=False, mem_limit=None, ports=None, dns=None, + environment=None, volumes=None, volumes_from=None, + network_disabled=False, entrypoint=None, cpu_shares=None, + working_dir=None, domainname=None, memswap_limit=None, cpuset=None, + host_config=None, mac_address=None, labels=None, volume_driver=None, + stop_signal=None, networking_config=None, healthcheck=None, + ): + if isinstance(command, six.string_types): + command = split_command(command) + + if isinstance(entrypoint, six.string_types): + entrypoint = split_command(entrypoint) + + if isinstance(environment, dict): + environment = format_environment(environment) + + if labels is not None and version_lt(version, '1.18'): + raise errors.InvalidVersion( + 'labels were only introduced in API version 1.18' + ) + + if cpuset is not None or cpu_shares is not None: + if version_gte(version, '1.18'): + warnings.warn( + 'The cpuset_cpus and cpu_shares options have been moved to' + ' host_config in API version 1.18, and will be removed', + DeprecationWarning + ) + + if stop_signal is not None and version_lt(version, '1.21'): + raise errors.InvalidVersion( + 'stop_signal was only introduced in API version 1.21' + ) + + if healthcheck is not None and version_lt(version, '1.24'): + raise errors.InvalidVersion( + 'Health options were only introduced in API version 1.24' + ) + + if version_lt(version, '1.19'): + if volume_driver is not None: + raise errors.InvalidVersion( + 'Volume drivers were only introduced in API version 1.19' + ) + mem_limit = mem_limit if mem_limit is not None else 0 + memswap_limit = memswap_limit if memswap_limit is not None else 0 + else: + if mem_limit is not None: + raise errors.InvalidVersion( + 'mem_limit has been moved to host_config in API version' + ' 1.19' + ) + + if memswap_limit is not None: + raise errors.InvalidVersion( + 'memswap_limit has been moved to host_config in API ' + 'version 1.19' + ) + + if isinstance(labels, list): + labels = dict((lbl, six.text_type('')) for lbl in labels) + + if mem_limit is not None: + mem_limit = parse_bytes(mem_limit) + + if memswap_limit is not None: + memswap_limit = parse_bytes(memswap_limit) + + if isinstance(ports, list): + exposed_ports = {} + for port_definition in ports: + port = port_definition + proto = 'tcp' + if isinstance(port_definition, tuple): + if len(port_definition) == 2: + proto = port_definition[1] + port = port_definition[0] + exposed_ports['{0}/{1}'.format(port, proto)] = {} + ports = exposed_ports + + if isinstance(volumes, six.string_types): + volumes = [volumes, ] + + if isinstance(volumes, list): + volumes_dict = {} + for vol in volumes: + volumes_dict[vol] = {} + volumes = volumes_dict + + if volumes_from: + if not isinstance(volumes_from, six.string_types): + volumes_from = ','.join(volumes_from) + else: + # Force None, an empty list or dict causes client.start to fail + volumes_from = None + + if healthcheck and isinstance(healthcheck, dict): + healthcheck = Healthcheck(**healthcheck) + + attach_stdin = False + attach_stdout = False + attach_stderr = False + stdin_once = False + + if not detach: + attach_stdout = True + attach_stderr = True + + if stdin_open: + attach_stdin = True + stdin_once = True + + if version_gte(version, '1.10'): + message = ('{0!r} parameter has no effect on create_container().' + ' It has been moved to host_config') + if dns is not None: + raise errors.InvalidVersion(message.format('dns')) + if volumes_from is not None: + raise errors.InvalidVersion(message.format('volumes_from')) + + self.update({ + 'Hostname': hostname, + 'Domainname': domainname, + 'ExposedPorts': ports, + 'User': six.text_type(user) if user else None, + 'Tty': tty, + 'OpenStdin': stdin_open, + 'StdinOnce': stdin_once, + 'Memory': mem_limit, + 'AttachStdin': attach_stdin, + 'AttachStdout': attach_stdout, + 'AttachStderr': attach_stderr, + 'Env': environment, + 'Cmd': command, + 'Dns': dns, + 'Image': image, + 'Volumes': volumes, + 'VolumesFrom': volumes_from, + 'NetworkDisabled': network_disabled, + 'Entrypoint': entrypoint, + 'CpuShares': cpu_shares, + 'Cpuset': cpuset, + 'CpusetCpus': cpuset, + 'WorkingDir': working_dir, + 'MemorySwap': memswap_limit, + 'HostConfig': host_config, + 'NetworkingConfig': networking_config, + 'MacAddress': mac_address, + 'Labels': labels, + 'VolumeDriver': volume_driver, + 'StopSignal': stop_signal, + 'Healthcheck': healthcheck, + }) diff --git a/docker/types/networks.py b/docker/types/networks.py new file mode 100644 index 0000000000..a539ac00f6 --- /dev/null +++ b/docker/types/networks.py @@ -0,0 +1,104 @@ +from .. import errors +from ..utils import normalize_links, version_lt + + +class EndpointConfig(dict): + def __init__(self, version, aliases=None, links=None, ipv4_address=None, + ipv6_address=None, link_local_ips=None): + if version_lt(version, '1.22'): + raise errors.InvalidVersion( + 'Endpoint config is not supported for API version < 1.22' + ) + + if aliases: + self["Aliases"] = aliases + + if links: + self["Links"] = normalize_links(links) + + ipam_config = {} + if ipv4_address: + ipam_config['IPv4Address'] = ipv4_address + + if ipv6_address: + ipam_config['IPv6Address'] = ipv6_address + + if link_local_ips is not None: + if version_lt(version, '1.24'): + raise errors.InvalidVersion( + 'link_local_ips is not supported for API version < 1.24' + ) + ipam_config['LinkLocalIPs'] = link_local_ips + + if ipam_config: + self['IPAMConfig'] = ipam_config + + +class NetworkingConfig(dict): + def __init__(self, endpoints_config=None): + if endpoints_config: + self["EndpointsConfig"] = endpoints_config + + +class IPAMConfig(dict): + """ + Create an IPAM (IP Address Management) config dictionary to be used with + :py:meth:`~docker.api.network.NetworkApiMixin.create_network`. + + Args: + + driver (str): The IPAM driver to use. Defaults to ``default``. + pool_configs (list): A list of pool configurations + (:py:class:`~docker.types.IPAMPool`). Defaults to empty list. + + Example: + + >>> ipam_config = docker.types.IPAMConfig(driver='default') + >>> network = client.create_network('network1', ipam=ipam_config) + + """ + def __init__(self, driver='default', pool_configs=None): + self.update({ + 'Driver': driver, + 'Config': pool_configs or [] + }) + + +class IPAMPool(dict): + """ + Create an IPAM pool config dictionary to be added to the + ``pool_configs`` parameter of + :py:class:`~docker.types.IPAMConfig`. + + Args: + + subnet (str): Custom subnet for this IPAM pool using the CIDR + notation. Defaults to ``None``. + iprange (str): Custom IP range for endpoints in this IPAM pool using + the CIDR notation. Defaults to ``None``. + gateway (str): Custom IP address for the pool's gateway. + aux_addresses (dict): A dictionary of ``key -> ip_address`` + relationships specifying auxiliary addresses that need to be + allocated by the IPAM driver. + + Example: + + >>> ipam_pool = docker.types.IPAMPool( + subnet='124.42.0.0/16', + iprange='124.42.0.0/24', + gateway='124.42.0.254', + aux_addresses={ + 'reserved1': '124.42.1.1' + } + ) + >>> ipam_config = docker.types.IPAMConfig( + pool_configs=[ipam_pool]) + """ + def __init__(self, subnet=None, iprange=None, gateway=None, + aux_addresses=None): + self.update({ + 'Subnet': subnet, + 'IPRange': iprange, + 'Gateway': gateway, + 'AuxiliaryAddresses': aux_addresses + }) diff --git a/docker/types/services.py b/docker/types/services.py index 9d5fa1b04b..82d1e602bc 100644 --- a/docker/types/services.py +++ b/docker/types/services.py @@ -1,6 +1,7 @@ import six from .. import errors +from ..utils import split_command class TaskTemplate(dict): @@ -10,19 +11,15 @@ class TaskTemplate(dict): Args: - * container_spec (dict): Container settings for containers started as part - of this task. See the :py:class:`~docker.types.services.ContainerSpec` - for details. - * log_driver (dict): Log configuration for containers created as part of - the service. See the :py:class:`~docker.types.services.DriverConfig` - class for details. - * resources (dict): Resource requirements which apply to each individual - container created as part of the service. See the - :py:class:`~docker.types.services.Resources` class for details. - * restart_policy (dict): Specification for the restart policy which applies - to containers created as part of this service. See the - :py:class:`~docker.types.services.RestartPolicy` class for details. - * placement (list): A list of constraints. + container_spec (ContainerSpec): Container settings for containers + started as part of this task. + log_driver (DriverConfig): Log configuration for containers created as + part of the service. + resources (Resources): Resource requirements which apply to each + individual container created as part of the service. + restart_policy (RestartPolicy): Specification for the restart policy + which applies to containers created as part of this service. + placement (list): A list of constraints. """ def __init__(self, container_spec, resources=None, restart_policy=None, placement=None, log_driver=None): @@ -58,27 +55,25 @@ def placement(self): class ContainerSpec(dict): """ Describes the behavior of containers that are part of a task, and is used - when declaring a :py:class:`~docker.types.services.TaskTemplate`. + when declaring a :py:class:`~docker.types.TaskTemplate`. Args: - * image (string): The image name to use for the container. - * command (string or list): The command to be run in the image. - * args (list): Arguments to the command. - * env (dict): Environment variables. - * dir (string): The working directory for commands to run in. - * user (string): The user inside the container. - * labels (dict): A map of labels to associate with the service. - * mounts (list): A list of specifications for mounts to be added to - containers created as part of the service. See the - :py:class:`~docker.types.services.Mount` class for details. - * stop_grace_period (int): Amount of time to wait for the container to - terminate before forcefully killing it. + image (string): The image name to use for the container. + command (string or list): The command to be run in the image. + args (list): Arguments to the command. + env (dict): Environment variables. + dir (string): The working directory for commands to run in. + user (string): The user inside the container. + labels (dict): A map of labels to associate with the service. + mounts (list): A list of specifications for mounts to be added to + containers created as part of the service. See the + :py:class:`~docker.types.Mount` class for details. + stop_grace_period (int): Amount of time to wait for the container to + terminate before forcefully killing it. """ def __init__(self, image, command=None, args=None, env=None, workdir=None, user=None, labels=None, mounts=None, stop_grace_period=None): - from ..utils import split_command # FIXME: circular import - self['Image'] = image if isinstance(command, six.string_types): @@ -108,24 +103,24 @@ class Mount(dict): """ Describes a mounted folder's configuration inside a container. A list of ``Mount``s would be used as part of a - :py:class:`~docker.types.services.ContainerSpec`. + :py:class:`~docker.types.ContainerSpec`. Args: - * target (string): Container path. - * source (string): Mount source (e.g. a volume name or a host path). - * type (string): The mount type (``bind`` or ``volume``). - Default: ``volume``. - * read_only (bool): Whether the mount should be read-only. - * propagation (string): A propagation mode with the value ``[r]private``, - ``[r]shared``, or ``[r]slave``. Only valid for the ``bind`` type. - * no_copy (bool): False if the volume should be populated with the data - from the target. Default: ``False``. Only valid for the ``volume`` type. - * labels (dict): User-defined name and labels for the volume. Only valid - for the ``volume`` type. - * driver_config (dict): Volume driver configuration. - See the :py:class:`~docker.types.services.DriverConfig` class for - details. Only valid for the ``volume`` type. + target (string): Container path. + source (string): Mount source (e.g. a volume name or a host path). + type (string): The mount type (``bind`` or ``volume``). + Default: ``volume``. + read_only (bool): Whether the mount should be read-only. + propagation (string): A propagation mode with the value ``[r]private``, + ``[r]shared``, or ``[r]slave``. Only valid for the ``bind`` type. + no_copy (bool): False if the volume should be populated with the data + from the target. Default: ``False``. Only valid for the ``volume`` + type. + labels (dict): User-defined name and labels for the volume. Only valid + for the ``volume`` type. + driver_config (DriverConfig): Volume driver configuration. Only valid + for the ``volume`` type. """ def __init__(self, target, source, type='volume', read_only=False, propagation=None, no_copy=False, labels=None, @@ -183,14 +178,14 @@ def parse_mount_string(cls, string): class Resources(dict): """ Configures resource allocation for containers when made part of a - :py:class:`~docker.types.services.ContainerSpec`. + :py:class:`~docker.types.ContainerSpec`. Args: - * cpu_limit (int): CPU limit in units of 10^9 CPU shares. - * mem_limit (int): Memory limit in Bytes. - * cpu_reservation (int): CPU reservation in units of 10^9 CPU shares. - * mem_reservation (int): Memory reservation in Bytes. + cpu_limit (int): CPU limit in units of 10^9 CPU shares. + mem_limit (int): Memory limit in Bytes. + cpu_reservation (int): CPU reservation in units of 10^9 CPU shares. + mem_reservation (int): Memory reservation in Bytes. """ def __init__(self, cpu_limit=None, mem_limit=None, cpu_reservation=None, mem_reservation=None): @@ -218,12 +213,12 @@ class UpdateConfig(dict): Args: - * parallelism (int): Maximum number of tasks to be updated in one iteration - (0 means unlimited parallelism). Default: 0. - * delay (int): Amount of time between updates. - * failure_action (string): Action to take if an updated task fails to run, - or stops running during the update. Acceptable values are ``continue`` - and ``pause``. Default: ``continue`` + parallelism (int): Maximum number of tasks to be updated in one + iteration (0 means unlimited parallelism). Default: 0. + delay (int): Amount of time between updates. + failure_action (string): Action to take if an updated task fails to + run, or stops running during the update. Acceptable values are + ``continue`` and ``pause``. Default: ``continue`` """ def __init__(self, parallelism=0, delay=None, failure_action='continue'): self['Parallelism'] = parallelism @@ -247,16 +242,18 @@ class RestartConditionTypesEnum(object): class RestartPolicy(dict): """ - Used when creating a :py:class:`~docker.types.services.ContainerSpec`, + Used when creating a :py:class:`~docker.types.ContainerSpec`, dictates whether a container should restart after stopping or failing. - * condition (string): Condition for restart (``none``, ``on-failure``, - or ``any``). Default: `none`. - * delay (int): Delay between restart attempts. Default: 0 - * attempts (int): Maximum attempts to restart a given container before - giving up. Default value is 0, which is ignored. - * window (int): Time window used to evaluate the restart policy. Default - value is 0, which is unbounded. + Args: + + condition (string): Condition for restart (``none``, ``on-failure``, + or ``any``). Default: `none`. + delay (int): Delay between restart attempts. Default: 0 + attempts (int): Maximum attempts to restart a given container before + giving up. Default value is 0, which is ignored. + window (int): Time window used to evaluate the restart policy. Default + value is 0, which is unbounded. """ condition_types = RestartConditionTypesEnum @@ -277,14 +274,14 @@ def __init__(self, condition=RestartConditionTypesEnum.NONE, delay=0, class DriverConfig(dict): """ Indicates which driver to use, as well as its configuration. Can be used - as ``log_driver`` in a :py:class:`~docker.types.services.ContainerSpec`, + as ``log_driver`` in a :py:class:`~docker.types.ContainerSpec`, and for the `driver_config` in a volume - :py:class:`~docker.types.services.Mount`. + :py:class:`~docker.types.Mount`. Args: - * name (string): Name of the driver to use. - * options (dict): Driver-specific options. Default: ``None``. + name (string): Name of the driver to use. + options (dict): Driver-specific options. Default: ``None``. """ def __init__(self, name, options=None): self['Name'] = name @@ -298,13 +295,13 @@ class EndpointSpec(dict): Args: - * mode (string): The mode of resolution to use for internal load balancing - between tasks (``'vip'`` or ``'dnsrr'``). Defaults to ``'vip'`` if not - provided. - * ports (dict): Exposed ports that this service is accessible on from the - outside, in the form of ``{ target_port: published_port }`` or - ``{ target_port: (published_port, protocol) }``. Ports can only be - provided if the ``vip`` resolution mode is used. + mode (string): The mode of resolution to use for internal load + balancing between tasks (``'vip'`` or ``'dnsrr'``). Defaults to + ``'vip'`` if not provided. + ports (dict): Exposed ports that this service is accessible on from the + outside, in the form of ``{ target_port: published_port }`` or + ``{ target_port: (published_port, protocol) }``. Ports can only be + provided if the ``vip`` resolution mode is used. """ def __init__(self, mode=None, ports=None): if ports: diff --git a/docker/utils/__init__.py b/docker/utils/__init__.py index 5bd69b4d19..061c26e670 100644 --- a/docker/utils/__init__.py +++ b/docker/utils/__init__.py @@ -3,12 +3,9 @@ compare_version, convert_port_bindings, convert_volume_binds, mkbuildcontext, tar, exclude_paths, parse_repository_tag, parse_host, kwargs_from_env, convert_filters, datetime_to_timestamp, - create_host_config, create_container_config, parse_bytes, ping_registry, - parse_env_file, version_lt, version_gte, decode_json_header, split_command, - create_ipam_config, create_ipam_pool, parse_devices, normalize_links, - convert_service_networks, + create_host_config, parse_bytes, ping_registry, parse_env_file, version_lt, + version_gte, decode_json_header, split_command, create_ipam_config, + create_ipam_pool, parse_devices, normalize_links, convert_service_networks, ) -from ..types import LogConfig, Ulimit -from ..types import SwarmExternalCA, SwarmSpec from .decorators import check_resource, minimum_version, update_headers diff --git a/docker/utils/types.py b/docker/utils/types.py deleted file mode 100644 index 8098c470f8..0000000000 --- a/docker/utils/types.py +++ /dev/null @@ -1,7 +0,0 @@ -# Compatibility module. See https://github.com/docker/docker-py/issues/1196 - -import warnings - -from ..types import Ulimit, LogConfig # flake8: noqa - -warnings.warn('docker.utils.types is now docker.types', ImportWarning) diff --git a/docker/utils/utils.py b/docker/utils/utils.py index a6ebf0faf5..4e5f454906 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -15,10 +15,8 @@ import requests import six -from .. import constants from .. import errors from .. import tls -from ..types import Ulimit, LogConfig, Healthcheck if six.PY2: from urllib import splitnport @@ -37,72 +35,18 @@ } -def create_ipam_pool(subnet=None, iprange=None, gateway=None, - aux_addresses=None): - """ - Create an IPAM pool config dictionary to be added to the - ``pool_configs`` parameter of - :py:meth:`~docker.utils.create_ipam_config`. - - Args: - - subnet (str): Custom subnet for this IPAM pool using the CIDR - notation. Defaults to ``None``. - iprange (str): Custom IP range for endpoints in this IPAM pool using - the CIDR notation. Defaults to ``None``. - gateway (str): Custom IP address for the pool's gateway. - aux_addresses (dict): A dictionary of ``key -> ip_address`` - relationships specifying auxiliary addresses that need to be - allocated by the IPAM driver. - - Returns: - (dict) An IPAM pool config - - Example: - - >>> ipam_pool = docker.utils.create_ipam_pool( - subnet='124.42.0.0/16', - iprange='124.42.0.0/24', - gateway='124.42.0.254', - aux_addresses={ - 'reserved1': '124.42.1.1' - } - ) - >>> ipam_config = docker.utils.create_ipam_config( - pool_configs=[ipam_pool]) - """ - return { - 'Subnet': subnet, - 'IPRange': iprange, - 'Gateway': gateway, - 'AuxiliaryAddresses': aux_addresses - } - - -def create_ipam_config(driver='default', pool_configs=None): - """ - Create an IPAM (IP Address Management) config dictionary to be used with - :py:meth:`~docker.api.network.NetworkApiMixin.create_network`. - - Args: - driver (str): The IPAM driver to use. Defaults to ``default``. - pool_configs (list): A list of pool configuration dictionaries as - created by :py:meth:`~docker.utils.create_ipam_pool`. Defaults to - empty list. - - Returns: - (dict) An IPAM config. - - Example: +def create_ipam_pool(*args, **kwargs): + raise errors.DeprecatedMethod( + 'utils.create_ipam_pool has been removed. Please use a ' + 'docker.types.IPAMPool object instead.' + ) - >>> ipam_config = docker.utils.create_ipam_config(driver='default') - >>> network = client.create_network('network1', ipam=ipam_config) - """ - return { - 'Driver': driver, - 'Config': pool_configs or [] - } +def create_ipam_config(*args, **kwargs): + raise errors.DeprecatedMethod( + 'utils.create_ipam_config has been removed. Please use a ' + 'docker.types.IPAMConfig object instead.' + ) def mkbuildcontext(dockerfile): @@ -669,338 +613,6 @@ def parse_bytes(s): return s -def host_config_type_error(param, param_value, expected): - error_msg = 'Invalid type for {0} param: expected {1} but found {2}' - return TypeError(error_msg.format(param, expected, type(param_value))) - - -def host_config_version_error(param, version, less_than=True): - operator = '<' if less_than else '>' - error_msg = '{0} param is not supported in API versions {1} {2}' - return errors.InvalidVersion(error_msg.format(param, operator, version)) - - -def host_config_value_error(param, param_value): - error_msg = 'Invalid value for {0} param: {1}' - return ValueError(error_msg.format(param, param_value)) - - -def create_host_config(binds=None, port_bindings=None, lxc_conf=None, - publish_all_ports=False, links=None, privileged=False, - dns=None, dns_search=None, volumes_from=None, - network_mode=None, restart_policy=None, cap_add=None, - cap_drop=None, devices=None, extra_hosts=None, - read_only=None, pid_mode=None, ipc_mode=None, - security_opt=None, ulimits=None, log_config=None, - mem_limit=None, memswap_limit=None, - mem_reservation=None, kernel_memory=None, - mem_swappiness=None, cgroup_parent=None, - group_add=None, cpu_quota=None, - cpu_period=None, blkio_weight=None, - blkio_weight_device=None, device_read_bps=None, - device_write_bps=None, device_read_iops=None, - device_write_iops=None, oom_kill_disable=False, - shm_size=None, sysctls=None, version=None, tmpfs=None, - oom_score_adj=None, dns_opt=None, cpu_shares=None, - cpuset_cpus=None, userns_mode=None, pids_limit=None, - isolation=None): - - host_config = {} - - if not version: - warnings.warn( - 'docker.utils.create_host_config() is deprecated. Please use ' - 'APIClient.create_host_config() instead.' - ) - version = constants.DEFAULT_DOCKER_API_VERSION - - if mem_limit is not None: - host_config['Memory'] = parse_bytes(mem_limit) - - if memswap_limit is not None: - host_config['MemorySwap'] = parse_bytes(memswap_limit) - - if mem_reservation: - if version_lt(version, '1.21'): - raise host_config_version_error('mem_reservation', '1.21') - - host_config['MemoryReservation'] = parse_bytes(mem_reservation) - - if kernel_memory: - if version_lt(version, '1.21'): - raise host_config_version_error('kernel_memory', '1.21') - - host_config['KernelMemory'] = parse_bytes(kernel_memory) - - if mem_swappiness is not None: - if version_lt(version, '1.20'): - raise host_config_version_error('mem_swappiness', '1.20') - if not isinstance(mem_swappiness, int): - raise host_config_type_error( - 'mem_swappiness', mem_swappiness, 'int' - ) - - host_config['MemorySwappiness'] = mem_swappiness - - if shm_size is not None: - if isinstance(shm_size, six.string_types): - shm_size = parse_bytes(shm_size) - - host_config['ShmSize'] = shm_size - - if pid_mode: - if version_lt(version, '1.24') and pid_mode != 'host': - raise host_config_value_error('pid_mode', pid_mode) - host_config['PidMode'] = pid_mode - - if ipc_mode: - host_config['IpcMode'] = ipc_mode - - if privileged: - host_config['Privileged'] = privileged - - if oom_kill_disable: - if version_lt(version, '1.20'): - raise host_config_version_error('oom_kill_disable', '1.19') - - host_config['OomKillDisable'] = oom_kill_disable - - if oom_score_adj: - if version_lt(version, '1.22'): - raise host_config_version_error('oom_score_adj', '1.22') - if not isinstance(oom_score_adj, int): - raise host_config_type_error( - 'oom_score_adj', oom_score_adj, 'int' - ) - host_config['OomScoreAdj'] = oom_score_adj - - if publish_all_ports: - host_config['PublishAllPorts'] = publish_all_ports - - if read_only is not None: - host_config['ReadonlyRootfs'] = read_only - - if dns_search: - host_config['DnsSearch'] = dns_search - - if network_mode: - host_config['NetworkMode'] = network_mode - elif network_mode is None and compare_version('1.19', version) > 0: - host_config['NetworkMode'] = 'default' - - if restart_policy: - if not isinstance(restart_policy, dict): - raise host_config_type_error( - 'restart_policy', restart_policy, 'dict' - ) - - host_config['RestartPolicy'] = restart_policy - - if cap_add: - host_config['CapAdd'] = cap_add - - if cap_drop: - host_config['CapDrop'] = cap_drop - - if devices: - host_config['Devices'] = parse_devices(devices) - - if group_add: - if version_lt(version, '1.20'): - raise host_config_version_error('group_add', '1.20') - - host_config['GroupAdd'] = [six.text_type(grp) for grp in group_add] - - if dns is not None: - host_config['Dns'] = dns - - if dns_opt is not None: - if version_lt(version, '1.21'): - raise host_config_version_error('dns_opt', '1.21') - - host_config['DnsOptions'] = dns_opt - - if security_opt is not None: - if not isinstance(security_opt, list): - raise host_config_type_error('security_opt', security_opt, 'list') - - host_config['SecurityOpt'] = security_opt - - if sysctls: - if not isinstance(sysctls, dict): - raise host_config_type_error('sysctls', sysctls, 'dict') - host_config['Sysctls'] = {} - for k, v in six.iteritems(sysctls): - host_config['Sysctls'][k] = six.text_type(v) - - if volumes_from is not None: - if isinstance(volumes_from, six.string_types): - volumes_from = volumes_from.split(',') - - host_config['VolumesFrom'] = volumes_from - - if binds is not None: - host_config['Binds'] = convert_volume_binds(binds) - - if port_bindings is not None: - host_config['PortBindings'] = convert_port_bindings(port_bindings) - - if extra_hosts is not None: - if isinstance(extra_hosts, dict): - extra_hosts = [ - '{0}:{1}'.format(k, v) - for k, v in sorted(six.iteritems(extra_hosts)) - ] - - host_config['ExtraHosts'] = extra_hosts - - if links is not None: - host_config['Links'] = normalize_links(links) - - if isinstance(lxc_conf, dict): - formatted = [] - for k, v in six.iteritems(lxc_conf): - formatted.append({'Key': k, 'Value': str(v)}) - lxc_conf = formatted - - if lxc_conf is not None: - host_config['LxcConf'] = lxc_conf - - if cgroup_parent is not None: - host_config['CgroupParent'] = cgroup_parent - - if ulimits is not None: - if not isinstance(ulimits, list): - raise host_config_type_error('ulimits', ulimits, 'list') - host_config['Ulimits'] = [] - for l in ulimits: - if not isinstance(l, Ulimit): - l = Ulimit(**l) - host_config['Ulimits'].append(l) - - if log_config is not None: - if not isinstance(log_config, LogConfig): - if not isinstance(log_config, dict): - raise host_config_type_error( - 'log_config', log_config, 'LogConfig' - ) - log_config = LogConfig(**log_config) - - host_config['LogConfig'] = log_config - - if cpu_quota: - if not isinstance(cpu_quota, int): - raise host_config_type_error('cpu_quota', cpu_quota, 'int') - if version_lt(version, '1.19'): - raise host_config_version_error('cpu_quota', '1.19') - - host_config['CpuQuota'] = cpu_quota - - if cpu_period: - if not isinstance(cpu_period, int): - raise host_config_type_error('cpu_period', cpu_period, 'int') - if version_lt(version, '1.19'): - raise host_config_version_error('cpu_period', '1.19') - - host_config['CpuPeriod'] = cpu_period - - if cpu_shares: - if version_lt(version, '1.18'): - raise host_config_version_error('cpu_shares', '1.18') - - if not isinstance(cpu_shares, int): - raise host_config_type_error('cpu_shares', cpu_shares, 'int') - - host_config['CpuShares'] = cpu_shares - - if cpuset_cpus: - if version_lt(version, '1.18'): - raise host_config_version_error('cpuset_cpus', '1.18') - - host_config['CpuSetCpus'] = cpuset_cpus - - if blkio_weight: - if not isinstance(blkio_weight, int): - raise host_config_type_error('blkio_weight', blkio_weight, 'int') - if version_lt(version, '1.22'): - raise host_config_version_error('blkio_weight', '1.22') - host_config["BlkioWeight"] = blkio_weight - - if blkio_weight_device: - if not isinstance(blkio_weight_device, list): - raise host_config_type_error( - 'blkio_weight_device', blkio_weight_device, 'list' - ) - if version_lt(version, '1.22'): - raise host_config_version_error('blkio_weight_device', '1.22') - host_config["BlkioWeightDevice"] = blkio_weight_device - - if device_read_bps: - if not isinstance(device_read_bps, list): - raise host_config_type_error( - 'device_read_bps', device_read_bps, 'list' - ) - if version_lt(version, '1.22'): - raise host_config_version_error('device_read_bps', '1.22') - host_config["BlkioDeviceReadBps"] = device_read_bps - - if device_write_bps: - if not isinstance(device_write_bps, list): - raise host_config_type_error( - 'device_write_bps', device_write_bps, 'list' - ) - if version_lt(version, '1.22'): - raise host_config_version_error('device_write_bps', '1.22') - host_config["BlkioDeviceWriteBps"] = device_write_bps - - if device_read_iops: - if not isinstance(device_read_iops, list): - raise host_config_type_error( - 'device_read_iops', device_read_iops, 'list' - ) - if version_lt(version, '1.22'): - raise host_config_version_error('device_read_iops', '1.22') - host_config["BlkioDeviceReadIOps"] = device_read_iops - - if device_write_iops: - if not isinstance(device_write_iops, list): - raise host_config_type_error( - 'device_write_iops', device_write_iops, 'list' - ) - if version_lt(version, '1.22'): - raise host_config_version_error('device_write_iops', '1.22') - host_config["BlkioDeviceWriteIOps"] = device_write_iops - - if tmpfs: - if version_lt(version, '1.22'): - raise host_config_version_error('tmpfs', '1.22') - host_config["Tmpfs"] = convert_tmpfs_mounts(tmpfs) - - if userns_mode: - if version_lt(version, '1.23'): - raise host_config_version_error('userns_mode', '1.23') - - if userns_mode != "host": - raise host_config_value_error("userns_mode", userns_mode) - host_config['UsernsMode'] = userns_mode - - if pids_limit: - if not isinstance(pids_limit, int): - raise host_config_type_error('pids_limit', pids_limit, 'int') - if version_lt(version, '1.23'): - raise host_config_version_error('pids_limit', '1.23') - host_config["PidsLimit"] = pids_limit - - if isolation: - if not isinstance(isolation, six.string_types): - raise host_config_type_error('isolation', isolation, 'string') - if version_lt(version, '1.24'): - raise host_config_version_error('isolation', '1.24') - host_config['Isolation'] = isolation - - return host_config - - def normalize_links(links): if isinstance(links, dict): links = six.iteritems(links) @@ -1008,50 +620,6 @@ def normalize_links(links): return ['{0}:{1}'.format(k, v) for k, v in sorted(links)] -def create_networking_config(endpoints_config=None): - networking_config = {} - - if endpoints_config: - networking_config["EndpointsConfig"] = endpoints_config - - return networking_config - - -def create_endpoint_config(version, aliases=None, links=None, - ipv4_address=None, ipv6_address=None, - link_local_ips=None): - if version_lt(version, '1.22'): - raise errors.InvalidVersion( - 'Endpoint config is not supported for API version < 1.22' - ) - endpoint_config = {} - - if aliases: - endpoint_config["Aliases"] = aliases - - if links: - endpoint_config["Links"] = normalize_links(links) - - ipam_config = {} - if ipv4_address: - ipam_config['IPv4Address'] = ipv4_address - - if ipv6_address: - ipam_config['IPv6Address'] = ipv6_address - - if link_local_ips is not None: - if version_lt(version, '1.24'): - raise errors.InvalidVersion( - 'link_local_ips is not supported for API version < 1.24' - ) - ipam_config['LinkLocalIPs'] = link_local_ips - - if ipam_config: - endpoint_config['IPAMConfig'] = ipam_config - - return endpoint_config - - def parse_env_file(env_file): """ Reads a line-separated environment file. @@ -1098,157 +666,8 @@ def format_env(key, value): return [format_env(*var) for var in six.iteritems(environment)] -def create_container_config( - version, image, command, hostname=None, user=None, detach=False, - stdin_open=False, tty=False, mem_limit=None, ports=None, environment=None, - dns=None, volumes=None, volumes_from=None, network_disabled=False, - entrypoint=None, cpu_shares=None, working_dir=None, domainname=None, - memswap_limit=None, cpuset=None, host_config=None, mac_address=None, - labels=None, volume_driver=None, stop_signal=None, networking_config=None, - healthcheck=None, -): - if isinstance(command, six.string_types): - command = split_command(command) - - if isinstance(entrypoint, six.string_types): - entrypoint = split_command(entrypoint) - - if isinstance(environment, dict): - environment = format_environment(environment) - - if labels is not None and compare_version('1.18', version) < 0: - raise errors.InvalidVersion( - 'labels were only introduced in API version 1.18' - ) - - if cpuset is not None or cpu_shares is not None: - if version_gte(version, '1.18'): - warnings.warn( - 'The cpuset_cpus and cpu_shares options have been moved to ' - 'host_config in API version 1.18, and will be removed', - DeprecationWarning - ) - - if stop_signal is not None and compare_version('1.21', version) < 0: - raise errors.InvalidVersion( - 'stop_signal was only introduced in API version 1.21' - ) - - if healthcheck is not None and version_lt(version, '1.24'): - raise errors.InvalidVersion( - 'Health options were only introduced in API version 1.24' - ) - - if compare_version('1.19', version) < 0: - if volume_driver is not None: - raise errors.InvalidVersion( - 'Volume drivers were only introduced in API version 1.19' - ) - mem_limit = mem_limit if mem_limit is not None else 0 - memswap_limit = memswap_limit if memswap_limit is not None else 0 - else: - if mem_limit is not None: - raise errors.InvalidVersion( - 'mem_limit has been moved to host_config in API version 1.19' - ) - - if memswap_limit is not None: - raise errors.InvalidVersion( - 'memswap_limit has been moved to host_config in API ' - 'version 1.19' - ) - - if isinstance(labels, list): - labels = dict((lbl, six.text_type('')) for lbl in labels) - - if mem_limit is not None: - mem_limit = parse_bytes(mem_limit) - - if memswap_limit is not None: - memswap_limit = parse_bytes(memswap_limit) - - if isinstance(ports, list): - exposed_ports = {} - for port_definition in ports: - port = port_definition - proto = 'tcp' - if isinstance(port_definition, tuple): - if len(port_definition) == 2: - proto = port_definition[1] - port = port_definition[0] - exposed_ports['{0}/{1}'.format(port, proto)] = {} - ports = exposed_ports - - if isinstance(volumes, six.string_types): - volumes = [volumes, ] - - if isinstance(volumes, list): - volumes_dict = {} - for vol in volumes: - volumes_dict[vol] = {} - volumes = volumes_dict - - if volumes_from: - if not isinstance(volumes_from, six.string_types): - volumes_from = ','.join(volumes_from) - else: - # Force None, an empty list or dict causes client.start to fail - volumes_from = None - - if healthcheck and isinstance(healthcheck, dict): - healthcheck = Healthcheck(**healthcheck) - - attach_stdin = False - attach_stdout = False - attach_stderr = False - stdin_once = False - - if not detach: - attach_stdout = True - attach_stderr = True - - if stdin_open: - attach_stdin = True - stdin_once = True - - if compare_version('1.10', version) >= 0: - message = ('{0!r} parameter has no effect on create_container().' - ' It has been moved to host_config') - if dns is not None: - raise errors.InvalidVersion(message.format('dns')) - if volumes_from is not None: - raise errors.InvalidVersion(message.format('volumes_from')) - - return { - 'Hostname': hostname, - 'Domainname': domainname, - 'ExposedPorts': ports, - 'User': six.text_type(user) if user else None, - 'Tty': tty, - 'OpenStdin': stdin_open, - 'StdinOnce': stdin_once, - 'Memory': mem_limit, - 'AttachStdin': attach_stdin, - 'AttachStdout': attach_stdout, - 'AttachStderr': attach_stderr, - 'Env': environment, - 'Cmd': command, - 'Dns': dns, - 'Image': image, - 'Volumes': volumes, - 'VolumesFrom': volumes_from, - 'NetworkDisabled': network_disabled, - 'Entrypoint': entrypoint, - 'CpuShares': cpu_shares, - 'Cpuset': cpuset, - 'CpusetCpus': cpuset, - 'WorkingDir': working_dir, - 'MemorySwap': memswap_limit, - 'HostConfig': host_config, - 'NetworkingConfig': networking_config, - 'MacAddress': mac_address, - 'Labels': labels, - 'VolumeDriver': volume_driver, - 'StopSignal': stop_signal, - 'Healthcheck': healthcheck, - } +def create_host_config(self, *args, **kwargs): + raise errors.DeprecatedMethod( + 'utils.create_host_config has been removed. Please use a ' + 'docker.types.HostConfig object instead.' + ) diff --git a/docs/api.rst b/docs/api.rst index 97db83945c..5e59aa7ad1 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -49,15 +49,6 @@ Networks :members: :undoc-members: -Utilities -~~~~~~~~~ - -These functions are available under ``docker.utils`` to create arguments -for :py:meth:`create_network`: - -.. autofunction:: docker.utils.create_ipam_config -.. autofunction:: docker.utils.create_ipam_pool - Volumes ------- @@ -107,3 +98,19 @@ The Docker daemon .. autoclass:: DaemonApiMixin :members: :undoc-members: + +Configuration types +------------------- + +.. py:module:: docker.types + +.. autoclass:: IPAMConfig +.. autoclass:: IPAMPool +.. autoclass:: ContainerSpec +.. autoclass:: DriverConfig +.. autoclass:: EndpointSpec +.. autoclass:: Mount +.. autoclass:: Resources +.. autoclass:: RestartPolicy +.. autoclass:: TaskTemplate +.. autoclass:: UpdateConfig diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index f09e75ad54..bebadb71b5 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -255,7 +255,7 @@ def test_group_id_strings(self): self.assertIn('1001', groups) def test_valid_log_driver_and_log_opt(self): - log_config = docker.utils.LogConfig( + log_config = docker.types.LogConfig( type='json-file', config={'max-file': '100'} ) @@ -274,7 +274,7 @@ def test_valid_log_driver_and_log_opt(self): self.assertEqual(container_log_config['Config'], log_config.config) def test_invalid_log_driver_raises_exception(self): - log_config = docker.utils.LogConfig( + log_config = docker.types.LogConfig( type='asdf-nope', config={} ) @@ -292,7 +292,7 @@ def test_invalid_log_driver_raises_exception(self): assert excinfo.value.explanation == expected_msg def test_valid_no_log_driver_specified(self): - log_config = docker.utils.LogConfig( + log_config = docker.types.LogConfig( type="", config={'max-file': '100'} ) @@ -311,7 +311,7 @@ def test_valid_no_log_driver_specified(self): self.assertEqual(container_log_config['Config'], log_config.config) def test_valid_no_config_specified(self): - log_config = docker.utils.LogConfig( + log_config = docker.types.LogConfig( type="json-file", config=None ) diff --git a/tests/integration/api_network_test.py b/tests/integration/api_network_test.py index 092a12c137..b1ac52c494 100644 --- a/tests/integration/api_network_test.py +++ b/tests/integration/api_network_test.py @@ -1,6 +1,5 @@ import docker -from docker.utils import create_ipam_config -from docker.utils import create_ipam_pool +from docker.types import IPAMConfig, IPAMPool import pytest from ..helpers import random_name, requires_api_version @@ -45,10 +44,10 @@ def test_inspect_network(self): @requires_api_version('1.21') def test_create_network_with_ipam_config(self): _, net_id = self.create_network( - ipam=create_ipam_config( + ipam=IPAMConfig( driver='default', pool_configs=[ - create_ipam_pool( + IPAMPool( subnet="172.28.0.0/16", iprange="172.28.5.0/24", gateway="172.28.5.254", @@ -217,9 +216,9 @@ def test_create_with_aliases(self): @requires_api_version('1.22') def test_create_with_ipv4_address(self): net_name, net_id = self.create_network( - ipam=create_ipam_config( + ipam=IPAMConfig( driver='default', - pool_configs=[create_ipam_pool(subnet="132.124.0.0/16")], + pool_configs=[IPAMPool(subnet="132.124.0.0/16")], ), ) container = self.client.create_container( @@ -246,9 +245,9 @@ def test_create_with_ipv4_address(self): @requires_api_version('1.22') def test_create_with_ipv6_address(self): net_name, net_id = self.create_network( - ipam=create_ipam_config( + ipam=IPAMConfig( driver='default', - pool_configs=[create_ipam_pool(subnet="2001:389::1/64")], + pool_configs=[IPAMPool(subnet="2001:389::1/64")], ), ) container = self.client.create_container( @@ -353,10 +352,10 @@ def test_connect_with_links(self): @requires_api_version('1.22') def test_connect_with_ipv4_address(self): net_name, net_id = self.create_network( - ipam=create_ipam_config( + ipam=IPAMConfig( driver='default', pool_configs=[ - create_ipam_pool( + IPAMPool( subnet="172.28.0.0/16", iprange="172.28.5.0/24", gateway="172.28.5.254" ) @@ -381,10 +380,10 @@ def test_connect_with_ipv4_address(self): @requires_api_version('1.22') def test_connect_with_ipv6_address(self): net_name, net_id = self.create_network( - ipam=create_ipam_config( + ipam=IPAMConfig( driver='default', pool_configs=[ - create_ipam_pool( + IPAMPool( subnet="2001:389::1/64", iprange="2001:389::0/96", gateway="2001:389::ffff" ) diff --git a/tests/unit/api_network_test.py b/tests/unit/api_network_test.py index 8e09c6756c..037edb552d 100644 --- a/tests/unit/api_network_test.py +++ b/tests/unit/api_network_test.py @@ -4,7 +4,7 @@ from .api_test import BaseAPIClientTest, url_prefix, response from ..helpers import requires_api_version -from docker.utils import create_ipam_config, create_ipam_pool +from docker.types import IPAMConfig, IPAMPool try: from unittest import mock @@ -81,9 +81,9 @@ def test_create_network(self): json.loads(post.call_args[1]['data']), {"Name": "foo", "Driver": "bridge", "Options": opts}) - ipam_pool_config = create_ipam_pool(subnet="192.168.52.0/24", - gateway="192.168.52.254") - ipam_config = create_ipam_config(pool_configs=[ipam_pool_config]) + ipam_pool_config = IPAMPool(subnet="192.168.52.0/24", + gateway="192.168.52.254") + ipam_config = IPAMConfig(pool_configs=[ipam_pool_config]) self.client.create_network("bar", driver="bridge", ipam=ipam_config) diff --git a/tests/unit/client_test.py b/tests/unit/client_test.py index 716827a98a..b79c68e155 100644 --- a/tests/unit/client_test.py +++ b/tests/unit/client_test.py @@ -61,7 +61,7 @@ def test_call_api_client_method(self): assert "this method is now on the object APIClient" not in s def test_call_containers(self): - client = docker.Client(**kwargs_from_env()) + client = docker.DockerClient(**kwargs_from_env()) with self.assertRaises(TypeError) as cm: client.containers() diff --git a/tests/unit/dockertypes_test.py b/tests/unit/dockertypes_test.py new file mode 100644 index 0000000000..2480b9ef92 --- /dev/null +++ b/tests/unit/dockertypes_test.py @@ -0,0 +1,255 @@ +# -*- coding: utf-8 -*- + +import unittest + +import pytest + +from docker.constants import DEFAULT_DOCKER_API_VERSION +from docker.errors import InvalidVersion +from docker.types import ( + EndpointConfig, HostConfig, IPAMConfig, IPAMPool, LogConfig, Ulimit, +) + + +def create_host_config(*args, **kwargs): + return HostConfig(*args, **kwargs) + + +class HostConfigTest(unittest.TestCase): + def test_create_host_config_no_options(self): + config = create_host_config(version='1.19') + self.assertFalse('NetworkMode' in config) + + def test_create_host_config_no_options_newer_api_version(self): + config = create_host_config(version='1.20') + self.assertEqual(config['NetworkMode'], 'default') + + def test_create_host_config_invalid_cpu_cfs_types(self): + with pytest.raises(TypeError): + create_host_config(version='1.20', cpu_quota='0') + + with pytest.raises(TypeError): + create_host_config(version='1.20', cpu_period='0') + + with pytest.raises(TypeError): + create_host_config(version='1.20', cpu_quota=23.11) + + with pytest.raises(TypeError): + create_host_config(version='1.20', cpu_period=1999.0) + + def test_create_host_config_with_cpu_quota(self): + config = create_host_config(version='1.20', cpu_quota=1999) + self.assertEqual(config.get('CpuQuota'), 1999) + + def test_create_host_config_with_cpu_period(self): + config = create_host_config(version='1.20', cpu_period=1999) + self.assertEqual(config.get('CpuPeriod'), 1999) + + def test_create_host_config_with_blkio_constraints(self): + blkio_rate = [{"Path": "/dev/sda", "Rate": 1000}] + config = create_host_config(version='1.22', + blkio_weight=1999, + blkio_weight_device=blkio_rate, + device_read_bps=blkio_rate, + device_write_bps=blkio_rate, + device_read_iops=blkio_rate, + device_write_iops=blkio_rate) + + self.assertEqual(config.get('BlkioWeight'), 1999) + self.assertTrue(config.get('BlkioWeightDevice') is blkio_rate) + self.assertTrue(config.get('BlkioDeviceReadBps') is blkio_rate) + self.assertTrue(config.get('BlkioDeviceWriteBps') is blkio_rate) + self.assertTrue(config.get('BlkioDeviceReadIOps') is blkio_rate) + self.assertTrue(config.get('BlkioDeviceWriteIOps') is blkio_rate) + self.assertEqual(blkio_rate[0]['Path'], "/dev/sda") + self.assertEqual(blkio_rate[0]['Rate'], 1000) + + def test_create_host_config_with_shm_size(self): + config = create_host_config(version='1.22', shm_size=67108864) + self.assertEqual(config.get('ShmSize'), 67108864) + + def test_create_host_config_with_shm_size_in_mb(self): + config = create_host_config(version='1.22', shm_size='64M') + self.assertEqual(config.get('ShmSize'), 67108864) + + def test_create_host_config_with_oom_kill_disable(self): + config = create_host_config(version='1.20', oom_kill_disable=True) + self.assertEqual(config.get('OomKillDisable'), True) + self.assertRaises( + InvalidVersion, lambda: create_host_config(version='1.18.3', + oom_kill_disable=True)) + + def test_create_host_config_with_userns_mode(self): + config = create_host_config(version='1.23', userns_mode='host') + self.assertEqual(config.get('UsernsMode'), 'host') + self.assertRaises( + InvalidVersion, lambda: create_host_config(version='1.22', + userns_mode='host')) + self.assertRaises( + ValueError, lambda: create_host_config(version='1.23', + userns_mode='host12')) + + def test_create_host_config_with_oom_score_adj(self): + config = create_host_config(version='1.22', oom_score_adj=100) + self.assertEqual(config.get('OomScoreAdj'), 100) + self.assertRaises( + InvalidVersion, lambda: create_host_config(version='1.21', + oom_score_adj=100)) + self.assertRaises( + TypeError, lambda: create_host_config(version='1.22', + oom_score_adj='100')) + + def test_create_host_config_with_dns_opt(self): + + tested_opts = ['use-vc', 'no-tld-query'] + config = create_host_config(version='1.21', dns_opt=tested_opts) + dns_opts = config.get('DnsOptions') + + self.assertTrue('use-vc' in dns_opts) + self.assertTrue('no-tld-query' in dns_opts) + + self.assertRaises( + InvalidVersion, lambda: create_host_config(version='1.20', + dns_opt=tested_opts)) + + def test_create_host_config_with_mem_reservation(self): + config = create_host_config(version='1.21', mem_reservation=67108864) + self.assertEqual(config.get('MemoryReservation'), 67108864) + self.assertRaises( + InvalidVersion, lambda: create_host_config( + version='1.20', mem_reservation=67108864)) + + def test_create_host_config_with_kernel_memory(self): + config = create_host_config(version='1.21', kernel_memory=67108864) + self.assertEqual(config.get('KernelMemory'), 67108864) + self.assertRaises( + InvalidVersion, lambda: create_host_config( + version='1.20', kernel_memory=67108864)) + + def test_create_host_config_with_pids_limit(self): + config = create_host_config(version='1.23', pids_limit=1024) + self.assertEqual(config.get('PidsLimit'), 1024) + + with pytest.raises(InvalidVersion): + create_host_config(version='1.22', pids_limit=1024) + with pytest.raises(TypeError): + create_host_config(version='1.23', pids_limit='1024') + + def test_create_host_config_with_isolation(self): + config = create_host_config(version='1.24', isolation='hyperv') + self.assertEqual(config.get('Isolation'), 'hyperv') + + with pytest.raises(InvalidVersion): + create_host_config(version='1.23', isolation='hyperv') + with pytest.raises(TypeError): + create_host_config( + version='1.24', isolation={'isolation': 'hyperv'} + ) + + def test_create_host_config_pid_mode(self): + with pytest.raises(ValueError): + create_host_config(version='1.23', pid_mode='baccab125') + + config = create_host_config(version='1.23', pid_mode='host') + assert config.get('PidMode') == 'host' + config = create_host_config(version='1.24', pid_mode='baccab125') + assert config.get('PidMode') == 'baccab125' + + def test_create_host_config_invalid_mem_swappiness(self): + with pytest.raises(TypeError): + create_host_config(version='1.24', mem_swappiness='40') + + +class UlimitTest(unittest.TestCase): + def test_create_host_config_dict_ulimit(self): + ulimit_dct = {'name': 'nofile', 'soft': 8096} + config = create_host_config( + ulimits=[ulimit_dct], version=DEFAULT_DOCKER_API_VERSION + ) + self.assertIn('Ulimits', config) + self.assertEqual(len(config['Ulimits']), 1) + ulimit_obj = config['Ulimits'][0] + self.assertTrue(isinstance(ulimit_obj, Ulimit)) + self.assertEqual(ulimit_obj.name, ulimit_dct['name']) + self.assertEqual(ulimit_obj.soft, ulimit_dct['soft']) + self.assertEqual(ulimit_obj['Soft'], ulimit_obj.soft) + + def test_create_host_config_dict_ulimit_capitals(self): + ulimit_dct = {'Name': 'nofile', 'Soft': 8096, 'Hard': 8096 * 4} + config = create_host_config( + ulimits=[ulimit_dct], version=DEFAULT_DOCKER_API_VERSION + ) + self.assertIn('Ulimits', config) + self.assertEqual(len(config['Ulimits']), 1) + ulimit_obj = config['Ulimits'][0] + self.assertTrue(isinstance(ulimit_obj, Ulimit)) + self.assertEqual(ulimit_obj.name, ulimit_dct['Name']) + self.assertEqual(ulimit_obj.soft, ulimit_dct['Soft']) + self.assertEqual(ulimit_obj.hard, ulimit_dct['Hard']) + self.assertEqual(ulimit_obj['Soft'], ulimit_obj.soft) + + def test_create_host_config_obj_ulimit(self): + ulimit_dct = Ulimit(name='nofile', soft=8096) + config = create_host_config( + ulimits=[ulimit_dct], version=DEFAULT_DOCKER_API_VERSION + ) + self.assertIn('Ulimits', config) + self.assertEqual(len(config['Ulimits']), 1) + ulimit_obj = config['Ulimits'][0] + self.assertTrue(isinstance(ulimit_obj, Ulimit)) + self.assertEqual(ulimit_obj, ulimit_dct) + + def test_ulimit_invalid_type(self): + self.assertRaises(ValueError, lambda: Ulimit(name=None)) + self.assertRaises(ValueError, lambda: Ulimit(name='hello', soft='123')) + self.assertRaises(ValueError, lambda: Ulimit(name='hello', hard='456')) + + +class LogConfigTest(unittest.TestCase): + def test_create_host_config_dict_logconfig(self): + dct = {'type': LogConfig.types.SYSLOG, 'config': {'key1': 'val1'}} + config = create_host_config( + version=DEFAULT_DOCKER_API_VERSION, log_config=dct + ) + self.assertIn('LogConfig', config) + self.assertTrue(isinstance(config['LogConfig'], LogConfig)) + self.assertEqual(dct['type'], config['LogConfig'].type) + + def test_create_host_config_obj_logconfig(self): + obj = LogConfig(type=LogConfig.types.SYSLOG, config={'key1': 'val1'}) + config = create_host_config( + version=DEFAULT_DOCKER_API_VERSION, log_config=obj + ) + self.assertIn('LogConfig', config) + self.assertTrue(isinstance(config['LogConfig'], LogConfig)) + self.assertEqual(obj, config['LogConfig']) + + def test_logconfig_invalid_config_type(self): + with pytest.raises(ValueError): + LogConfig(type=LogConfig.types.JSON, config='helloworld') + + +class EndpointConfigTest(unittest.TestCase): + def test_create_endpoint_config_with_aliases(self): + config = EndpointConfig(version='1.22', aliases=['foo', 'bar']) + assert config == {'Aliases': ['foo', 'bar']} + + with pytest.raises(InvalidVersion): + EndpointConfig(version='1.21', aliases=['foo', 'bar']) + + +class IPAMConfigTest(unittest.TestCase): + def test_create_ipam_config(self): + ipam_pool = IPAMPool(subnet='192.168.52.0/24', + gateway='192.168.52.254') + + ipam_config = IPAMConfig(pool_configs=[ipam_pool]) + self.assertEqual(ipam_config, { + 'Driver': 'default', + 'Config': [{ + 'Subnet': '192.168.52.0/24', + 'Gateway': '192.168.52.254', + 'AuxiliaryAddresses': None, + 'IPRange': None, + }] + }) diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index f69c62cd72..743d076da3 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -14,19 +14,17 @@ import six from docker.api.client import APIClient -from docker.constants import DEFAULT_DOCKER_API_VERSION, IS_WINDOWS_PLATFORM -from docker.errors import DockerException, InvalidVersion +from docker.constants import IS_WINDOWS_PLATFORM +from docker.errors import DockerException from docker.utils import ( parse_repository_tag, parse_host, convert_filters, kwargs_from_env, - create_host_config, Ulimit, LogConfig, parse_bytes, parse_env_file, - exclude_paths, convert_volume_binds, decode_json_header, tar, - split_command, create_ipam_config, create_ipam_pool, parse_devices, - update_headers + parse_bytes, parse_env_file, exclude_paths, convert_volume_binds, + decode_json_header, tar, split_command, parse_devices, update_headers, ) from docker.utils.ports import build_port_bindings, split_port from docker.utils.utils import ( - create_endpoint_config, format_environment, should_check_directory + format_environment, should_check_directory ) from ..helpers import make_tree @@ -69,227 +67,6 @@ def f(self, headers=None): } -class HostConfigTest(unittest.TestCase): - def test_create_host_config_no_options(self): - config = create_host_config(version='1.19') - self.assertFalse('NetworkMode' in config) - - def test_create_host_config_no_options_newer_api_version(self): - config = create_host_config(version='1.20') - self.assertEqual(config['NetworkMode'], 'default') - - def test_create_host_config_invalid_cpu_cfs_types(self): - with pytest.raises(TypeError): - create_host_config(version='1.20', cpu_quota='0') - - with pytest.raises(TypeError): - create_host_config(version='1.20', cpu_period='0') - - with pytest.raises(TypeError): - create_host_config(version='1.20', cpu_quota=23.11) - - with pytest.raises(TypeError): - create_host_config(version='1.20', cpu_period=1999.0) - - def test_create_host_config_with_cpu_quota(self): - config = create_host_config(version='1.20', cpu_quota=1999) - self.assertEqual(config.get('CpuQuota'), 1999) - - def test_create_host_config_with_cpu_period(self): - config = create_host_config(version='1.20', cpu_period=1999) - self.assertEqual(config.get('CpuPeriod'), 1999) - - def test_create_host_config_with_blkio_constraints(self): - blkio_rate = [{"Path": "/dev/sda", "Rate": 1000}] - config = create_host_config(version='1.22', - blkio_weight=1999, - blkio_weight_device=blkio_rate, - device_read_bps=blkio_rate, - device_write_bps=blkio_rate, - device_read_iops=blkio_rate, - device_write_iops=blkio_rate) - - self.assertEqual(config.get('BlkioWeight'), 1999) - self.assertTrue(config.get('BlkioWeightDevice') is blkio_rate) - self.assertTrue(config.get('BlkioDeviceReadBps') is blkio_rate) - self.assertTrue(config.get('BlkioDeviceWriteBps') is blkio_rate) - self.assertTrue(config.get('BlkioDeviceReadIOps') is blkio_rate) - self.assertTrue(config.get('BlkioDeviceWriteIOps') is blkio_rate) - self.assertEqual(blkio_rate[0]['Path'], "/dev/sda") - self.assertEqual(blkio_rate[0]['Rate'], 1000) - - def test_create_host_config_with_shm_size(self): - config = create_host_config(version='1.22', shm_size=67108864) - self.assertEqual(config.get('ShmSize'), 67108864) - - def test_create_host_config_with_shm_size_in_mb(self): - config = create_host_config(version='1.22', shm_size='64M') - self.assertEqual(config.get('ShmSize'), 67108864) - - def test_create_host_config_with_oom_kill_disable(self): - config = create_host_config(version='1.20', oom_kill_disable=True) - self.assertEqual(config.get('OomKillDisable'), True) - self.assertRaises( - InvalidVersion, lambda: create_host_config(version='1.18.3', - oom_kill_disable=True)) - - def test_create_host_config_with_userns_mode(self): - config = create_host_config(version='1.23', userns_mode='host') - self.assertEqual(config.get('UsernsMode'), 'host') - self.assertRaises( - InvalidVersion, lambda: create_host_config(version='1.22', - userns_mode='host')) - self.assertRaises( - ValueError, lambda: create_host_config(version='1.23', - userns_mode='host12')) - - def test_create_host_config_with_oom_score_adj(self): - config = create_host_config(version='1.22', oom_score_adj=100) - self.assertEqual(config.get('OomScoreAdj'), 100) - self.assertRaises( - InvalidVersion, lambda: create_host_config(version='1.21', - oom_score_adj=100)) - self.assertRaises( - TypeError, lambda: create_host_config(version='1.22', - oom_score_adj='100')) - - def test_create_host_config_with_dns_opt(self): - - tested_opts = ['use-vc', 'no-tld-query'] - config = create_host_config(version='1.21', dns_opt=tested_opts) - dns_opts = config.get('DnsOptions') - - self.assertTrue('use-vc' in dns_opts) - self.assertTrue('no-tld-query' in dns_opts) - - self.assertRaises( - InvalidVersion, lambda: create_host_config(version='1.20', - dns_opt=tested_opts)) - - def test_create_endpoint_config_with_aliases(self): - config = create_endpoint_config(version='1.22', aliases=['foo', 'bar']) - assert config == {'Aliases': ['foo', 'bar']} - - with pytest.raises(InvalidVersion): - create_endpoint_config(version='1.21', aliases=['foo', 'bar']) - - def test_create_host_config_with_mem_reservation(self): - config = create_host_config(version='1.21', mem_reservation=67108864) - self.assertEqual(config.get('MemoryReservation'), 67108864) - self.assertRaises( - InvalidVersion, lambda: create_host_config( - version='1.20', mem_reservation=67108864)) - - def test_create_host_config_with_kernel_memory(self): - config = create_host_config(version='1.21', kernel_memory=67108864) - self.assertEqual(config.get('KernelMemory'), 67108864) - self.assertRaises( - InvalidVersion, lambda: create_host_config( - version='1.20', kernel_memory=67108864)) - - def test_create_host_config_with_pids_limit(self): - config = create_host_config(version='1.23', pids_limit=1024) - self.assertEqual(config.get('PidsLimit'), 1024) - - with pytest.raises(InvalidVersion): - create_host_config(version='1.22', pids_limit=1024) - with pytest.raises(TypeError): - create_host_config(version='1.23', pids_limit='1024') - - def test_create_host_config_with_isolation(self): - config = create_host_config(version='1.24', isolation='hyperv') - self.assertEqual(config.get('Isolation'), 'hyperv') - - with pytest.raises(InvalidVersion): - create_host_config(version='1.23', isolation='hyperv') - with pytest.raises(TypeError): - create_host_config( - version='1.24', isolation={'isolation': 'hyperv'} - ) - - def test_create_host_config_pid_mode(self): - with pytest.raises(ValueError): - create_host_config(version='1.23', pid_mode='baccab125') - - config = create_host_config(version='1.23', pid_mode='host') - assert config.get('PidMode') == 'host' - config = create_host_config(version='1.24', pid_mode='baccab125') - assert config.get('PidMode') == 'baccab125' - - def test_create_host_config_invalid_mem_swappiness(self): - with pytest.raises(TypeError): - create_host_config(version='1.24', mem_swappiness='40') - - -class UlimitTest(unittest.TestCase): - def test_create_host_config_dict_ulimit(self): - ulimit_dct = {'name': 'nofile', 'soft': 8096} - config = create_host_config( - ulimits=[ulimit_dct], version=DEFAULT_DOCKER_API_VERSION - ) - self.assertIn('Ulimits', config) - self.assertEqual(len(config['Ulimits']), 1) - ulimit_obj = config['Ulimits'][0] - self.assertTrue(isinstance(ulimit_obj, Ulimit)) - self.assertEqual(ulimit_obj.name, ulimit_dct['name']) - self.assertEqual(ulimit_obj.soft, ulimit_dct['soft']) - self.assertEqual(ulimit_obj['Soft'], ulimit_obj.soft) - - def test_create_host_config_dict_ulimit_capitals(self): - ulimit_dct = {'Name': 'nofile', 'Soft': 8096, 'Hard': 8096 * 4} - config = create_host_config( - ulimits=[ulimit_dct], version=DEFAULT_DOCKER_API_VERSION - ) - self.assertIn('Ulimits', config) - self.assertEqual(len(config['Ulimits']), 1) - ulimit_obj = config['Ulimits'][0] - self.assertTrue(isinstance(ulimit_obj, Ulimit)) - self.assertEqual(ulimit_obj.name, ulimit_dct['Name']) - self.assertEqual(ulimit_obj.soft, ulimit_dct['Soft']) - self.assertEqual(ulimit_obj.hard, ulimit_dct['Hard']) - self.assertEqual(ulimit_obj['Soft'], ulimit_obj.soft) - - def test_create_host_config_obj_ulimit(self): - ulimit_dct = Ulimit(name='nofile', soft=8096) - config = create_host_config( - ulimits=[ulimit_dct], version=DEFAULT_DOCKER_API_VERSION - ) - self.assertIn('Ulimits', config) - self.assertEqual(len(config['Ulimits']), 1) - ulimit_obj = config['Ulimits'][0] - self.assertTrue(isinstance(ulimit_obj, Ulimit)) - self.assertEqual(ulimit_obj, ulimit_dct) - - def test_ulimit_invalid_type(self): - self.assertRaises(ValueError, lambda: Ulimit(name=None)) - self.assertRaises(ValueError, lambda: Ulimit(name='hello', soft='123')) - self.assertRaises(ValueError, lambda: Ulimit(name='hello', hard='456')) - - -class LogConfigTest(unittest.TestCase): - def test_create_host_config_dict_logconfig(self): - dct = {'type': LogConfig.types.SYSLOG, 'config': {'key1': 'val1'}} - config = create_host_config( - version=DEFAULT_DOCKER_API_VERSION, log_config=dct - ) - self.assertIn('LogConfig', config) - self.assertTrue(isinstance(config['LogConfig'], LogConfig)) - self.assertEqual(dct['type'], config['LogConfig'].type) - - def test_create_host_config_obj_logconfig(self): - obj = LogConfig(type=LogConfig.types.SYSLOG, config={'key1': 'val1'}) - config = create_host_config( - version=DEFAULT_DOCKER_API_VERSION, log_config=obj - ) - self.assertIn('LogConfig', config) - self.assertTrue(isinstance(config['LogConfig'], LogConfig)) - self.assertEqual(obj, config['LogConfig']) - - def test_logconfig_invalid_config_type(self): - with pytest.raises(ValueError): - LogConfig(type=LogConfig.types.JSON, config='helloworld') - - class KwargsFromEnvTest(unittest.TestCase): def setUp(self): self.os_environ = os.environ.copy() @@ -711,21 +488,6 @@ def test_decode_json_header(self): decoded_data = decode_json_header(data) self.assertEqual(obj, decoded_data) - def test_create_ipam_config(self): - ipam_pool = create_ipam_pool(subnet='192.168.52.0/24', - gateway='192.168.52.254') - - ipam_config = create_ipam_config(pool_configs=[ipam_pool]) - self.assertEqual(ipam_config, { - 'Driver': 'default', - 'Config': [{ - 'Subnet': '192.168.52.0/24', - 'Gateway': '192.168.52.254', - 'AuxiliaryAddresses': None, - 'IPRange': None, - }] - }) - class SplitCommandTest(unittest.TestCase): def test_split_command_with_unicode(self): From 01c33c0f684868290249945f9db9e00d7f03ecd9 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 1 Dec 2016 18:02:16 -0800 Subject: [PATCH 0206/1301] Client -> DockerClient Signed-off-by: Joffrey F --- tests/unit/client_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/client_test.py b/tests/unit/client_test.py index 716827a98a..b79c68e155 100644 --- a/tests/unit/client_test.py +++ b/tests/unit/client_test.py @@ -61,7 +61,7 @@ def test_call_api_client_method(self): assert "this method is now on the object APIClient" not in s def test_call_containers(self): - client = docker.Client(**kwargs_from_env()) + client = docker.DockerClient(**kwargs_from_env()) with self.assertRaises(TypeError) as cm: client.containers() From d042c6aeda59f5bf7769085616eb56de8ee5fbe7 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 1 Dec 2016 17:50:10 -0800 Subject: [PATCH 0207/1301] Properly fill out auth headers in APIClient.build when using a credentials store Signed-off-by: Joffrey F --- docker/api/build.py | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/docker/api/build.py b/docker/api/build.py index 7cf4e0f77b..eb01bce389 100644 --- a/docker/api/build.py +++ b/docker/api/build.py @@ -230,19 +230,35 @@ def _set_auth_headers(self, headers): # Send the full auth configuration (if any exists), since the build # could use any (or all) of the registries. if self._auth_configs: + auth_data = {} + if self._auth_configs.get('credsStore'): + # Using a credentials store, we need to retrieve the + # credentials for each registry listed in the config.json file + # Matches CLI behavior: https://github.com/docker/docker/blob/ + # 67b85f9d26f1b0b2b240f2d794748fac0f45243c/cliconfig/ + # credentials/native_store.go#L68-L83 + for registry in self._auth_configs.keys(): + if registry == 'credsStore' or registry == 'HttpHeaders': + continue + auth_data[registry] = auth.resolve_authconfig( + self._auth_configs, registry + ) + else: + auth_data = self._auth_configs + log.debug( 'Sending auth config ({0})'.format( - ', '.join(repr(k) for k in self._auth_configs.keys()) + ', '.join(repr(k) for k in auth_data.keys()) ) ) if utils.compare_version('1.19', self._version) >= 0: headers['X-Registry-Config'] = auth.encode_header( - self._auth_configs + auth_data ) else: headers['X-Registry-Config'] = auth.encode_header({ - 'configs': self._auth_configs + 'configs': auth_data }) else: log.debug('No auth config found') From 453964466741a1c85fc420c8b40fb5710f40b017 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 1 Dec 2016 18:00:15 -0800 Subject: [PATCH 0208/1301] Move ssladapter to transport module Signed-off-by: Joffrey F --- docker/api/client.py | 8 +++----- docker/tls.py | 5 +++-- docker/transport/__init__.py | 1 + docker/{ => transport}/ssladapter.py | 0 tests/unit/ssladapter_test.py | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) rename docker/{ => transport}/ssladapter.py (100%) diff --git a/docker/api/client.py b/docker/api/client.py index 23e239c66f..0b4d1614c7 100644 --- a/docker/api/client.py +++ b/docker/api/client.py @@ -17,7 +17,7 @@ from .service import ServiceApiMixin from .swarm import SwarmApiMixin from .volume import VolumeApiMixin -from .. import auth, ssladapter +from .. import auth from ..constants import (DEFAULT_TIMEOUT_SECONDS, DEFAULT_USER_AGENT, IS_WINDOWS_PLATFORM, DEFAULT_DOCKER_API_VERSION, STREAM_HEADER_SIZE_BYTES, DEFAULT_NUM_POOLS, @@ -25,7 +25,7 @@ from ..errors import (DockerException, TLSParameterError, create_api_error_from_http_exception) from ..tls import TLSConfig -from ..transport import UnixAdapter +from ..transport import SSLAdapter, UnixAdapter from ..utils import utils, check_resource, update_headers from ..utils.socket import frames_iter try: @@ -121,9 +121,7 @@ def __init__(self, base_url=None, version=None, if isinstance(tls, TLSConfig): tls.configure_client(self) elif tls: - self._custom_adapter = ssladapter.SSLAdapter( - pool_connections=num_pools - ) + self._custom_adapter = SSLAdapter(pool_connections=num_pools) self.mount('https://', self._custom_adapter) self.base_url = base_url diff --git a/docker/tls.py b/docker/tls.py index 3a0827a25c..6488bbccc1 100644 --- a/docker/tls.py +++ b/docker/tls.py @@ -1,7 +1,8 @@ import os import ssl -from . import errors, ssladapter +from . import errors +from .transport import SSLAdapter class TLSConfig(object): @@ -84,7 +85,7 @@ def configure_client(self, client): if self.cert: client.cert = self.cert - client.mount('https://', ssladapter.SSLAdapter( + client.mount('https://', SSLAdapter( ssl_version=self.ssl_version, assert_hostname=self.assert_hostname, assert_fingerprint=self.assert_fingerprint, diff --git a/docker/transport/__init__.py b/docker/transport/__init__.py index d5560b63e2..abbee182fc 100644 --- a/docker/transport/__init__.py +++ b/docker/transport/__init__.py @@ -1,5 +1,6 @@ # flake8: noqa from .unixconn import UnixAdapter +from .ssladapter import SSLAdapter try: from .npipeconn import NpipeAdapter from .npipesocket import NpipeSocket diff --git a/docker/ssladapter.py b/docker/transport/ssladapter.py similarity index 100% rename from docker/ssladapter.py rename to docker/transport/ssladapter.py diff --git a/tests/unit/ssladapter_test.py b/tests/unit/ssladapter_test.py index 90d4c3202c..2b7ce52cb5 100644 --- a/tests/unit/ssladapter_test.py +++ b/tests/unit/ssladapter_test.py @@ -1,5 +1,5 @@ import unittest -from docker import ssladapter +from docker.transport import ssladapter try: from backports.ssl_match_hostname import ( From fb4969f74488cae4487e87bec67e7d7a65d22dfc Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Fri, 2 Dec 2016 17:58:14 +0000 Subject: [PATCH 0209/1301] Fix auth config path on Windows The Engine client looks *only* at the USERPROFILE environment variable on Windows, so we should do that too. Signed-off-by: Aanand Prasad --- docker/auth.py | 36 ++++++++++++++++++---------- tests/unit/auth_test.py | 53 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 12 deletions(-) diff --git a/docker/auth.py b/docker/auth.py index 0a2eda1ebe..7c1ce7618e 100644 --- a/docker/auth.py +++ b/docker/auth.py @@ -7,6 +7,7 @@ import six from . import errors +from .constants import IS_WINDOWS_PLATFORM INDEX_NAME = 'docker.io' INDEX_URL = 'https://{0}/v1/'.format(INDEX_NAME) @@ -210,19 +211,12 @@ def parse_auth(entries, raise_on_error=False): def find_config_file(config_path=None): - environment_path = os.path.join( - os.environ.get('DOCKER_CONFIG'), - os.path.basename(DOCKER_CONFIG_FILENAME) - ) if os.environ.get('DOCKER_CONFIG') else None - - paths = filter(None, [ + paths = list(filter(None, [ config_path, # 1 - environment_path, # 2 - os.path.join(os.path.expanduser('~'), DOCKER_CONFIG_FILENAME), # 3 - os.path.join( - os.path.expanduser('~'), LEGACY_DOCKER_CONFIG_FILENAME - ) # 4 - ]) + config_path_from_environment(), # 2 + os.path.join(home_dir(), DOCKER_CONFIG_FILENAME), # 3 + os.path.join(home_dir(), LEGACY_DOCKER_CONFIG_FILENAME), # 4 + ])) log.debug("Trying paths: {0}".format(repr(paths))) @@ -236,6 +230,24 @@ def find_config_file(config_path=None): return None +def config_path_from_environment(): + config_dir = os.environ.get('DOCKER_CONFIG') + if not config_dir: + return None + return os.path.join(config_dir, os.path.basename(DOCKER_CONFIG_FILENAME)) + + +def home_dir(): + """ + Get the user's home directory, using the same logic as the Docker Engine + client - use %USERPROFILE% on Windows, $HOME/getuid on POSIX. + """ + if IS_WINDOWS_PLATFORM: + return os.environ.get('USERPROFILE', '') + else: + return os.path.expanduser('~') + + def load_config(config_path=None): """ Loads authentication data from a Docker configuration file in the given diff --git a/tests/unit/auth_test.py b/tests/unit/auth_test.py index e4c93b78d9..f9f6fc1462 100644 --- a/tests/unit/auth_test.py +++ b/tests/unit/auth_test.py @@ -9,6 +9,9 @@ import tempfile import unittest +from py.test import ensuretemp +from pytest import mark + from docker import auth, errors try: @@ -269,6 +272,56 @@ def test_resolve_registry_and_auth_unauthenticated_registry(self): ) +class FindConfigFileTest(unittest.TestCase): + def tmpdir(self, name): + tmpdir = ensuretemp(name) + self.addCleanup(tmpdir.remove) + return tmpdir + + def test_find_config_fallback(self): + tmpdir = self.tmpdir('test_find_config_fallback') + + with mock.patch.dict(os.environ, {'HOME': str(tmpdir)}): + assert auth.find_config_file() is None + + def test_find_config_from_explicit_path(self): + tmpdir = self.tmpdir('test_find_config_from_explicit_path') + config_path = tmpdir.ensure('my-config-file.json') + + assert auth.find_config_file(str(config_path)) == str(config_path) + + def test_find_config_from_environment(self): + tmpdir = self.tmpdir('test_find_config_from_environment') + config_path = tmpdir.ensure('config.json') + + with mock.patch.dict(os.environ, {'DOCKER_CONFIG': str(tmpdir)}): + assert auth.find_config_file() == str(config_path) + + @mark.skipif("sys.platform == 'win32'") + def test_find_config_from_home_posix(self): + tmpdir = self.tmpdir('test_find_config_from_home_posix') + config_path = tmpdir.ensure('.docker', 'config.json') + + with mock.patch.dict(os.environ, {'HOME': str(tmpdir)}): + assert auth.find_config_file() == str(config_path) + + @mark.skipif("sys.platform == 'win32'") + def test_find_config_from_home_legacy_name(self): + tmpdir = self.tmpdir('test_find_config_from_home_legacy_name') + config_path = tmpdir.ensure('.dockercfg') + + with mock.patch.dict(os.environ, {'HOME': str(tmpdir)}): + assert auth.find_config_file() == str(config_path) + + @mark.skipif("sys.platform != 'win32'") + def test_find_config_from_home_windows(self): + tmpdir = self.tmpdir('test_find_config_from_home_windows') + config_path = tmpdir.ensure('.docker', 'config.json') + + with mock.patch.dict(os.environ, {'USERPROFILE': str(tmpdir)}): + assert auth.find_config_file() == str(config_path) + + class LoadConfigTest(unittest.TestCase): def test_load_config_no_file(self): folder = tempfile.mkdtemp() From 1d59aeca4b79132410877912af61eec5b5039cd4 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 2 Dec 2016 15:04:05 -0800 Subject: [PATCH 0210/1301] Add options to IPAMConfig Signed-off-by: Joffrey F --- docker/types/networks.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docker/types/networks.py b/docker/types/networks.py index a539ac00f6..628ea65ad2 100644 --- a/docker/types/networks.py +++ b/docker/types/networks.py @@ -50,6 +50,8 @@ class IPAMConfig(dict): driver (str): The IPAM driver to use. Defaults to ``default``. pool_configs (list): A list of pool configurations (:py:class:`~docker.types.IPAMPool`). Defaults to empty list. + options (dict): Driver options as a key-value dictionary. + Defaults to `None`. Example: @@ -57,12 +59,17 @@ class IPAMConfig(dict): >>> network = client.create_network('network1', ipam=ipam_config) """ - def __init__(self, driver='default', pool_configs=None): + def __init__(self, driver='default', pool_configs=None, options=None): self.update({ 'Driver': driver, 'Config': pool_configs or [] }) + if options: + if not isinstance(options, dict): + raise TypeError('IPAMConfig options must be a dictionary') + self['Options'] = options + class IPAMPool(dict): """ From 2b85fbf120fb2fa0a6afa9205f6a495d6b66d989 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Mon, 5 Dec 2016 16:00:53 -0500 Subject: [PATCH 0211/1301] Add attachable. Signed-off-by: Daniel Nephin --- docker/api/network.py | 13 +++++++++++-- tests/unit/api_network_test.py | 4 ++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/docker/api/network.py b/docker/api/network.py index 33da7eadfe..ca7cadf9a4 100644 --- a/docker/api/network.py +++ b/docker/api/network.py @@ -38,7 +38,7 @@ def networks(self, names=None, ids=None): @minimum_version('1.21') def create_network(self, name, driver=None, options=None, ipam=None, check_duplicate=None, internal=False, labels=None, - enable_ipv6=False): + enable_ipv6=False, attachable=None): """ Create a network. Similar to the ``docker network create``. @@ -54,6 +54,9 @@ def create_network(self, name, driver=None, options=None, ipam=None, labels (dict): Map of labels to set on the network. Default ``None``. enable_ipv6 (bool): Enable IPv6 on the network. Default ``False``. + attachable (bool): If enabled, and the network is in the global + scope, non-service containers on worker nodes will be able to + connect to the network. Returns: (dict): The created network reference object @@ -91,7 +94,7 @@ def create_network(self, name, driver=None, options=None, ipam=None, 'Driver': driver, 'Options': options, 'IPAM': ipam, - 'CheckDuplicate': check_duplicate + 'CheckDuplicate': check_duplicate, } if labels is not None: @@ -116,6 +119,12 @@ def create_network(self, name, driver=None, options=None, ipam=None, 'supported in API version < 1.22') data['Internal'] = True + if attachable is not None + if version_lt(self._version, '1.24'): + raise InvalidVersion('Attachable is not ' + 'supported in API version < 1.24') + data['Attachable'] = attachable + url = self._url("/networks/create") res = self._post_json(url, data=data) return self._result(res, json=True) diff --git a/tests/unit/api_network_test.py b/tests/unit/api_network_test.py index f997a1b829..7af531b357 100644 --- a/tests/unit/api_network_test.py +++ b/tests/unit/api_network_test.py @@ -104,6 +104,10 @@ def test_create_network(self): } }) + @requires_api_version('1.24') + def test_create_network_with_attachable(self): + pass + @requires_api_version('1.21') def test_remove_network(self): network_id = 'abc12345' From 993001bc49a679afceaa293de1e8bb48f7190d42 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 5 Dec 2016 16:53:21 -0800 Subject: [PATCH 0212/1301] Minimum version 1.24 -> 1.21 Signed-off-by: Joffrey F --- docker/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/constants.py b/docker/constants.py index 0bfc0b088a..4337fedced 100644 --- a/docker/constants.py +++ b/docker/constants.py @@ -2,7 +2,7 @@ from .version import version DEFAULT_DOCKER_API_VERSION = '1.24' -MINIMUM_DOCKER_API_VERSION = '1.24' +MINIMUM_DOCKER_API_VERSION = '1.21' DEFAULT_TIMEOUT_SECONDS = 60 STREAM_HEADER_SIZE_BYTES = 8 CONTAINER_LIMITS_KEYS = [ From 16d0f96bc577f1463f70517b70387a30f68d7690 Mon Sep 17 00:00:00 2001 From: Ryan Belgrave Date: Thu, 29 Sep 2016 12:33:18 -0500 Subject: [PATCH 0213/1301] Name is not required when creating a docker volume Signed-off-by: Ryan Belgrave --- docker/api/volume.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/api/volume.py b/docker/api/volume.py index 9c6d5f8351..e55c031222 100644 --- a/docker/api/volume.py +++ b/docker/api/volume.py @@ -38,7 +38,8 @@ def volumes(self, filters=None): return self._result(self._get(url, params=params), True) @utils.minimum_version('1.21') - def create_volume(self, name, driver=None, driver_opts=None, labels=None): + def create_volume(self, name=None, driver=None, driver_opts=None, + labels=None): """ Create and register a named volume From b71f34e948bdf986660989f3e8a052db7bb1335c Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 5 Dec 2016 16:52:22 -0800 Subject: [PATCH 0214/1301] Fix typo in create_network Signed-off-by: Joffrey F --- docker/api/network.py | 9 +++++---- tests/integration/api_network_test.py | 11 +++++++++++ 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/docker/api/network.py b/docker/api/network.py index ca7cadf9a4..c58ea6e502 100644 --- a/docker/api/network.py +++ b/docker/api/network.py @@ -38,7 +38,7 @@ def networks(self, names=None, ids=None): @minimum_version('1.21') def create_network(self, name, driver=None, options=None, ipam=None, check_duplicate=None, internal=False, labels=None, - enable_ipv6=False, attachable=None): + enable_ipv6=False, attachable=None, scope=None): """ Create a network. Similar to the ``docker network create``. @@ -119,10 +119,11 @@ def create_network(self, name, driver=None, options=None, ipam=None, 'supported in API version < 1.22') data['Internal'] = True - if attachable is not None + if attachable is not None: if version_lt(self._version, '1.24'): - raise InvalidVersion('Attachable is not ' - 'supported in API version < 1.24') + raise InvalidVersion( + 'attachable is not supported in API version < 1.24' + ) data['Attachable'] = attachable url = self._url("/networks/create") diff --git a/tests/integration/api_network_test.py b/tests/integration/api_network_test.py index b1ac52c494..e5a3801716 100644 --- a/tests/integration/api_network_test.py +++ b/tests/integration/api_network_test.py @@ -7,6 +7,10 @@ class TestNetworks(BaseAPIIntegrationTest): + def tearDown(self): + super(TestNetworks, self).tearDown() + self.client.leave_swarm(force=True) + def create_network(self, *args, **kwargs): net_name = random_name() net_id = self.client.create_network(net_name, *args, **kwargs)['Id'] @@ -434,3 +438,10 @@ def test_create_network_ipv6_enabled(self): _, net_id = self.create_network(enable_ipv6=True) net = self.client.inspect_network(net_id) assert net['EnableIPv6'] is True + + @requires_api_version('1.24') + def test_create_network_attachable(self): + assert self.client.init_swarm('eth0') + _, net_id = self.create_network(driver='overlay', attachable=True) + net = self.client.inspect_network(net_id) + assert net['Attachable'] is True From 2b88e9cddb8a9236e4aa363c649a03f97f209085 Mon Sep 17 00:00:00 2001 From: Pavel Sviderski Date: Wed, 7 Dec 2016 18:12:12 +0800 Subject: [PATCH 0215/1301] increase logs performance, do not copy bytes object Signed-off-by: Pavel Sviderski --- docker/api/client.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docker/api/client.py b/docker/api/client.py index a9fe7d0899..7fd080d3d4 100644 --- a/docker/api/client.py +++ b/docker/api/client.py @@ -305,11 +305,13 @@ def _multiplexed_buffer_helper(self, response): """A generator of multiplexed data blocks read from a buffered response.""" buf = self._result(response, binary=True) + buf_length = len(buf) walker = 0 while True: - if len(buf[walker:]) < 8: + if buf_length - walker < STREAM_HEADER_SIZE_BYTES: break - _, length = struct.unpack_from('>BxxxL', buf[walker:]) + header = buf[walker:walker + STREAM_HEADER_SIZE_BYTES] + _, length = struct.unpack_from('>BxxxL', header) start = walker + STREAM_HEADER_SIZE_BYTES end = start + length walker = end From c239e4050425394b20970998d3c33776621a90a7 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 5 Dec 2016 13:33:41 -0800 Subject: [PATCH 0216/1301] Implement swarm node removal Signed-off-by: Joffrey F --- docker/api/swarm.py | 28 ++++++++++++++++++++++++++++ docker/models/nodes.py | 19 +++++++++++++++++++ tests/integration/api_swarm_test.py | 17 +++++++++++++++++ 3 files changed, 64 insertions(+) diff --git a/docker/api/swarm.py b/docker/api/swarm.py index 6a1b752fc5..9a240a9c5f 100644 --- a/docker/api/swarm.py +++ b/docker/api/swarm.py @@ -224,6 +224,33 @@ def nodes(self, filters=None): return self._result(self._get(url, params=params), True) + @utils.check_resource + @utils.minimum_version('1.24') + def remove_node(self, node_id, force=False): + """ + Remove a node from the swarm. + + Args: + node_id (string): ID of the node to be removed. + force (bool): Force remove an active node. Default: `False` + + Raises: + :py:class:`docker.errors.NotFound` + If the node referenced doesn't exist in the swarm. + + :py:class:`docker.errors.APIError` + If the server returns an error. + Returns: + `True` if the request was successful. + """ + url = self._url('/nodes/{0}', node_id) + params = { + 'force': force + } + res = self._delete(url, params=params) + self._raise_for_status(res) + return True + @utils.minimum_version('1.24') def update_node(self, node_id, version, node_spec=None): """ @@ -231,6 +258,7 @@ def update_node(self, node_id, version, node_spec=None): Args: + node_id (string): ID of the node to be updated. version (int): The version number of the node object being updated. This is required to avoid conflicting writes. node_spec (dict): Configuration settings to update. Any values diff --git a/docker/models/nodes.py b/docker/models/nodes.py index 0887f99c24..8dd9350c02 100644 --- a/docker/models/nodes.py +++ b/docker/models/nodes.py @@ -41,6 +41,25 @@ def update(self, node_spec): """ return self.client.api.update_node(self.id, self.version, node_spec) + def remove(self, force=False): + """ + Remove this node from the swarm. + + Args: + force (bool): Force remove an active node. Default: `False` + + Returns: + `True` if the request was successful. + + Raises: + :py:class:`docker.errors.NotFound` + If the node doesn't exist in the swarm. + + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + return self.client.api.remove_node(self.id, force=force) + class NodeCollection(Collection): """Nodes on the Docker server.""" diff --git a/tests/integration/api_swarm_test.py b/tests/integration/api_swarm_test.py index 24c566f7e2..a10437b1e3 100644 --- a/tests/integration/api_swarm_test.py +++ b/tests/integration/api_swarm_test.py @@ -159,3 +159,20 @@ def test_update_node(self): node_spec=orig_spec) reverted_node = self.client.inspect_node(node['ID']) assert orig_spec == reverted_node['Spec'] + + @requires_api_version('1.24') + def test_remove_main_node(self): + assert self.client.init_swarm('eth0') + nodes_list = self.client.nodes() + node_id = nodes_list[0]['ID'] + with pytest.raises(docker.errors.NotFound): + self.client.remove_node('foobar01') + with pytest.raises(docker.errors.APIError) as e: + self.client.remove_node(node_id) + + assert e.value.response.status_code == 500 + + with pytest.raises(docker.errors.APIError) as e: + self.client.remove_node(node_id, True) + + assert e.value.response.status_code == 500 From 738cfdcdf96e7ff56f6bb3a4966e337187ba51c4 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 6 Dec 2016 16:58:07 -0800 Subject: [PATCH 0217/1301] Update code and tests for Engine 1.13 compatibility Makefile now runs tests against Docker 1.13 RC Signed-off-by: Joffrey F --- Makefile | 4 ++-- docker/api/swarm.py | 4 ++++ docker/errors.py | 4 +++- docker/models/swarm.py | 3 ++- tests/helpers.py | 2 +- tests/integration/api_network_test.py | 16 ++++++++++++---- tests/integration/api_service_test.py | 22 +++++++++++++--------- tests/integration/models_images_test.py | 4 ++++ tests/integration/models_services_test.py | 10 ++++++---- tests/integration/models_swarm_test.py | 7 ++++++- tests/unit/api_network_test.py | 4 ---- 11 files changed, 53 insertions(+), 27 deletions(-) diff --git a/Makefile b/Makefile index 4c5cf0c67b..8727ada4dc 100644 --- a/Makefile +++ b/Makefile @@ -44,7 +44,7 @@ integration-test-py3: build-py3 .PHONY: integration-dind integration-dind: build build-py3 docker rm -vf dpy-dind || : - docker run -d --name dpy-dind --privileged dockerswarm/dind:1.12.0 docker daemon\ + docker run -d --name dpy-dind --privileged dockerswarm/dind:1.13.0-rc3 docker daemon\ -H tcp://0.0.0.0:2375 docker run --rm --env="DOCKER_HOST=tcp://docker:2375" --link=dpy-dind:docker docker-sdk-python\ py.test tests/integration @@ -57,7 +57,7 @@ integration-dind-ssl: build-dind-certs build build-py3 docker run -d --name dpy-dind-certs dpy-dind-certs docker run -d --env="DOCKER_HOST=tcp://localhost:2375" --env="DOCKER_TLS_VERIFY=1"\ --env="DOCKER_CERT_PATH=/certs" --volumes-from dpy-dind-certs --name dpy-dind-ssl\ - -v /tmp --privileged dockerswarm/dind:1.12.0 docker daemon --tlsverify\ + -v /tmp --privileged dockerswarm/dind:1.13.0-rc3 docker daemon --tlsverify\ --tlscacert=/certs/ca.pem --tlscert=/certs/server-cert.pem\ --tlskey=/certs/server-key.pem -H tcp://0.0.0.0:2375 docker run --rm --volumes-from dpy-dind-ssl --env="DOCKER_HOST=tcp://docker:2375"\ diff --git a/docker/api/swarm.py b/docker/api/swarm.py index 6a1b752fc5..edc206fa77 100644 --- a/docker/api/swarm.py +++ b/docker/api/swarm.py @@ -197,6 +197,10 @@ def leave_swarm(self, force=False): # Ignore "this node is not part of a swarm" error if force and response.status_code == http_client.NOT_ACCEPTABLE: return True + # FIXME: Temporary workaround for 1.13.0-rc bug + # https://github.com/docker/docker/issues/29192 + if force and response.status_code == http_client.SERVICE_UNAVAILABLE: + return True self._raise_for_status(response) return True diff --git a/docker/errors.py b/docker/errors.py index 8572007d42..05f4cae5c1 100644 --- a/docker/errors.py +++ b/docker/errors.py @@ -21,7 +21,9 @@ def create_api_error_from_http_exception(e): explanation = response.content.strip() cls = APIError if response.status_code == 404: - if explanation and 'No such image' in str(explanation): + if explanation and ('No such image' in str(explanation) or + 'not found: does not exist or no read access' + in str(explanation)): cls = ImageNotFound else: cls = NotFound diff --git a/docker/models/swarm.py b/docker/models/swarm.py index 38c1e9f9c2..adfc51d920 100644 --- a/docker/models/swarm.py +++ b/docker/models/swarm.py @@ -15,7 +15,8 @@ def __init__(self, *args, **kwargs): try: self.reload() except APIError as e: - if e.response.status_code != 406: + # FIXME: https://github.com/docker/docker/issues/29192 + if e.response.status_code not in (406, 503): raise @property diff --git a/tests/helpers.py b/tests/helpers.py index 1d24577a67..53cf57ad70 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -73,4 +73,4 @@ def force_leave_swarm(client): if e.explanation == "context deadline exceeded": continue else: - raise + return diff --git a/tests/integration/api_network_test.py b/tests/integration/api_network_test.py index e5a3801716..2c297a00a6 100644 --- a/tests/integration/api_network_test.py +++ b/tests/integration/api_network_test.py @@ -20,12 +20,10 @@ def create_network(self, *args, **kwargs): @requires_api_version('1.21') def test_list_networks(self): networks = self.client.networks() - initial_size = len(networks) net_name, net_id = self.create_network() networks = self.client.networks() - self.assertEqual(len(networks), initial_size + 1) self.assertTrue(net_id in [n['Id'] for n in networks]) networks_by_name = self.client.networks(names=[net_name]) @@ -435,11 +433,21 @@ def test_create_network_with_labels_wrong_type(self): @requires_api_version('1.23') def test_create_network_ipv6_enabled(self): - _, net_id = self.create_network(enable_ipv6=True) + _, net_id = self.create_network( + enable_ipv6=True, ipam=IPAMConfig( + driver='default', + pool_configs=[ + IPAMPool( + subnet="2001:389::1/64", iprange="2001:389::0/96", + gateway="2001:389::ffff" + ) + ] + ) + ) net = self.client.inspect_network(net_id) assert net['EnableIPv6'] is True - @requires_api_version('1.24') + @requires_api_version('1.25') def test_create_network_attachable(self): assert self.client.init_swarm('eth0') _, net_id = self.create_network(driver='overlay', attachable=True) diff --git a/tests/integration/api_service_test.py b/tests/integration/api_service_test.py index bdf7c019aa..04f2fe0bcb 100644 --- a/tests/integration/api_service_test.py +++ b/tests/integration/api_service_test.py @@ -221,15 +221,19 @@ def test_create_service_with_endpoint_spec(self): svc_info = self.client.inspect_service(svc_id) print(svc_info) ports = svc_info['Spec']['EndpointSpec']['Ports'] - assert { - 'PublishedPort': 12562, 'TargetPort': 678, 'Protocol': 'tcp' - } in ports - assert { - 'PublishedPort': 53243, 'TargetPort': 8080, 'Protocol': 'tcp' - } in ports - assert { - 'PublishedPort': 12357, 'TargetPort': 1990, 'Protocol': 'udp' - } in ports + for port in ports: + if port['PublishedPort'] == 12562: + assert port['TargetPort'] == 678 + assert port['Protocol'] == 'tcp' + elif port['PublishedPort'] == 53243: + assert port['TargetPort'] == 8080 + assert port['Protocol'] == 'tcp' + elif port['PublishedPort'] == 12357: + assert port['TargetPort'] == 1990 + assert port['Protocol'] == 'udp' + else: + self.fail('Invalid port specification: {0}'.format(port)) + assert len(ports) == 3 def test_create_service_with_env(self): diff --git a/tests/integration/models_images_test.py b/tests/integration/models_images_test.py index 2be623252c..876ec292b6 100644 --- a/tests/integration/models_images_test.py +++ b/tests/integration/models_images_test.py @@ -1,5 +1,8 @@ import io + import docker +import pytest + from .base import BaseIntegrationTest @@ -14,6 +17,7 @@ def test_build(self): self.tmp_imgs.append(image.id) assert client.containers.run(image) == b"hello world\n" + @pytest.mark.xfail(reason='Engine 1.13 responds with status 500') def test_build_with_error(self): client = docker.from_env() with self.assertRaises(docker.errors.BuildError) as cm: diff --git a/tests/integration/models_services_test.py b/tests/integration/models_services_test.py index 99cffc058b..baa40a9120 100644 --- a/tests/integration/models_services_test.py +++ b/tests/integration/models_services_test.py @@ -1,5 +1,8 @@ import unittest + import docker +import pytest + from .. import helpers @@ -29,7 +32,7 @@ def test_create(self): assert service.name == name assert service.attrs['Spec']['Labels']['foo'] == 'bar' container_spec = service.attrs['Spec']['TaskTemplate']['ContainerSpec'] - assert container_spec['Image'] == "alpine" + assert "alpine" in container_spec['Image'] assert container_spec['Labels'] == {'container': 'label'} def test_get(self): @@ -78,6 +81,7 @@ def test_tasks(self): assert len(tasks) == 1 assert tasks[0]['ServiceID'] == service2.id + @pytest.mark.skip(reason="Makes Swarm unstable?") def test_update(self): client = docker.from_env() service = client.services.create( @@ -87,14 +91,12 @@ def test_update(self): image="alpine", command="sleep 300" ) - new_name = helpers.random_name() service.update( # create argument - name=new_name, + name=service.name, # ContainerSpec argument command="sleep 600" ) service.reload() - assert service.name == new_name container_spec = service.attrs['Spec']['TaskTemplate']['ContainerSpec'] assert container_spec['Command'] == ["sleep", "600"] diff --git a/tests/integration/models_swarm_test.py b/tests/integration/models_swarm_test.py index abdff41ffa..72bf9e5c92 100644 --- a/tests/integration/models_swarm_test.py +++ b/tests/integration/models_swarm_test.py @@ -19,4 +19,9 @@ def test_init_update_leave(self): assert client.swarm.leave(force=True) with self.assertRaises(docker.errors.APIError) as cm: client.swarm.reload() - assert cm.exception.response.status_code == 406 + assert ( + # FIXME: test for both until + # https://github.com/docker/docker/issues/29192 is resolved + cm.exception.response.status_code == 406 or + cm.exception.response.status_code == 503 + ) diff --git a/tests/unit/api_network_test.py b/tests/unit/api_network_test.py index 7af531b357..f997a1b829 100644 --- a/tests/unit/api_network_test.py +++ b/tests/unit/api_network_test.py @@ -104,10 +104,6 @@ def test_create_network(self): } }) - @requires_api_version('1.24') - def test_create_network_with_attachable(self): - pass - @requires_api_version('1.21') def test_remove_network(self): network_id = 'abc12345' From b9c48dca2c093c658867a4ecb13627124f9b5257 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BE=90=E4=BF=8A=E6=9D=B0?= Date: Tue, 6 Dec 2016 21:09:45 +0800 Subject: [PATCH 0218/1301] Scope is added in volume after docker 1.12 Signed-off-by: pacoxu add ut test for volume scope and no specified name create Signed-off-by: Paco Xu try to fix ut failure of volume creation Signed-off-by: Paco Xu try to fix ut failure of volume creation Signed-off-by: Paco Xu Scope is added in volume after docker 1.12 Signed-off-by: pacoxu Scope is added in volume after docker 1.12 Signed-off-by: pacoxu --- docker/api/volume.py | 3 ++- tests/unit/api_volume_test.py | 10 ++++++++++ tests/unit/fake_api.py | 9 ++++++--- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/docker/api/volume.py b/docker/api/volume.py index 9c6d5f8351..a73df18731 100644 --- a/docker/api/volume.py +++ b/docker/api/volume.py @@ -64,7 +64,8 @@ def create_volume(self, name, driver=None, driver_opts=None, labels=None): {u'Driver': u'local', u'Labels': {u'key': u'value'}, u'Mountpoint': u'/var/lib/docker/volumes/foobar/_data', - u'Name': u'foobar'} + u'Name': u'foobar', + u'Scope': u'local'} """ url = self._url('/volumes/create') diff --git a/tests/unit/api_volume_test.py b/tests/unit/api_volume_test.py index cb72cb2580..fc2a556d29 100644 --- a/tests/unit/api_volume_test.py +++ b/tests/unit/api_volume_test.py @@ -89,6 +89,16 @@ def test_create_volume_invalid_opts_type(self): 'perfectcherryblossom', driver_opts='' ) + @requires_api_version('1.24') + def test_create_volume_with_no_specified_name(self): + result = self.client.create_volume(name=None) + self.assertIn('Name', result) + self.assertNotEqual(result['Name'], None) + self.assertIn('Driver', result) + self.assertEqual(result['Driver'], 'local') + self.assertIn('Scope', result) + self.assertEqual(result['Scope'], 'local') + @requires_api_version('1.21') def test_inspect_volume(self): name = 'perfectcherryblossom' diff --git a/tests/unit/fake_api.py b/tests/unit/fake_api.py index cf3f7d7dd1..2d0a0b4541 100644 --- a/tests/unit/fake_api.py +++ b/tests/unit/fake_api.py @@ -389,11 +389,13 @@ def get_fake_volume_list(): { 'Name': 'perfectcherryblossom', 'Driver': 'local', - 'Mountpoint': '/var/lib/docker/volumes/perfectcherryblossom' + 'Mountpoint': '/var/lib/docker/volumes/perfectcherryblossom', + 'Scope': 'local' }, { 'Name': 'subterraneananimism', 'Driver': 'local', - 'Mountpoint': '/var/lib/docker/volumes/subterraneananimism' + 'Mountpoint': '/var/lib/docker/volumes/subterraneananimism', + 'Scope': 'local' } ] } @@ -408,7 +410,8 @@ def get_fake_volume(): 'Mountpoint': '/var/lib/docker/volumes/perfectcherryblossom', 'Labels': { 'com.example.some-label': 'some-value' - } + }, + 'Scope': 'local' } return status_code, response From 6f239fbf29109d39d3cfed0650c9ffe217c681a8 Mon Sep 17 00:00:00 2001 From: Flavio Curella Date: Thu, 8 Dec 2016 12:10:25 -0600 Subject: [PATCH 0219/1301] Make resources hashable, so that they can be added to `set`s Signed-off-by: Flavio Curella --- docker/models/resource.py | 3 +++ tests/unit/models_resources_test.py | 14 ++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/docker/models/resource.py b/docker/models/resource.py index 95712aefc6..ed3900af3a 100644 --- a/docker/models/resource.py +++ b/docker/models/resource.py @@ -23,6 +23,9 @@ def __repr__(self): def __eq__(self, other): return isinstance(other, self.__class__) and self.id == other.id + def __hash__(self): + return hash("%s:%s" % (self.__class__.__name__, self.id)) + @property def id(self): """ diff --git a/tests/unit/models_resources_test.py b/tests/unit/models_resources_test.py index 25c6a3ed0c..5af24ee69f 100644 --- a/tests/unit/models_resources_test.py +++ b/tests/unit/models_resources_test.py @@ -12,3 +12,17 @@ def test_reload(self): container.reload() assert client.api.inspect_container.call_count == 2 assert container.attrs['Name'] == "foobar" + + def test_hash(self): + client = make_fake_client() + container1 = client.containers.get(FAKE_CONTAINER_ID) + my_set = set([container1]) + assert len(my_set) == 1 + + container2 = client.containers.get(FAKE_CONTAINER_ID) + my_set.add(container2) + assert len(my_set) == 1 + + image1 = client.images.get(FAKE_CONTAINER_ID) + my_set.add(image1) + assert len(my_set) == 2 From 3fb48d111bd3c4e1a6a490fe73f76e6021847a5d Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 6 Dec 2016 16:59:34 -0800 Subject: [PATCH 0220/1301] Add Jenkinsfile for integration tests matrix Signed-off-by: Joffrey F --- Jenkinsfile | 82 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 Jenkinsfile diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 0000000000..3af2b1661e --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,82 @@ +#!groovy + +def imageNameBase = "dockerbuildbot/docker-py" +def imageNamePy2 +def imageNamePy3 +def images = [:] +def dockerVersions = ["1.12.3", "1.13.0-rc3"] + +def buildImage = { name, buildargs, pyTag -> + img = docker.image(name) + try { + img.pull() + } catch (Exception exc) { + img = docker.build(name, buildargs) + img.push() + } + images[pyTag] = img.id +} + +def buildImages = { -> + wrappedNode(label: "ubuntu && !zfs && amd64", cleanWorkspace: true) { + stage("build image") { + checkout(scm) + + imageNamePy2 = "${imageNameBase}:py2-${gitCommit()}" + imageNamePy3 = "${imageNameBase}:py3-${gitCommit()}" + + buildImage(imageNamePy2, ".", "py2.7") + buildImage(imageNamePy3, "-f Dockerfile-py3 .", "py3.5") + } + } +} + +def runTests = { Map settings -> + def dockerVersion = settings.get("dockerVersion", null) + def testImage = settings.get("testImage", null) + + if (!testImage) { + throw new Exception("Need test image object, e.g.: `runTests(testImage: img)`") + } + if (!dockerVersion) { + throw new Exception("Need Docker version to test, e.g.: `runTests(dockerVersion: '1.12.3')`") + } + + { -> + wrappedNode(label: "ubuntu && !zfs && amd64", cleanWorkspace: true) { + stage("test image=${testImage} / docker=${dockerVersion}") { + checkout(scm) + try { + sh """docker run -d --name dpy-dind-\$BUILD_NUMBER -v /tmp --privileged \\ + dockerswarm/dind:${dockerVersion} docker daemon -H tcp://0.0.0.0:2375 + """ + sh """docker run \\ + --name dpy-tests-\$BUILD_NUMBER --volumes-from dpy-dind-\$BUILD_NUMBER \\ + -e 'DOCKER_HOST=tcp://docker:2375' \\ + --link=dpy-dind-\$BUILD_NUMBER:docker \\ + ${testImage} \\ + py.test -rxs tests/integration + """ + } finally { + sh """ + docker stop dpy-tests-\$BUILD_NUMBER dpy-dind-\$BUILD_NUMBER + docker rm -vf dpy-tests-\$BUILD_NUMBER dpy-dind-\$BUILD_NUMBER + """ + } + } + } + } +} + + +buildImages() + +def testMatrix = [failFast: false] + +for (imgKey in new ArrayList(images.keySet())) { + for (version in dockerVersions) { + testMatrix["${imgKey}_${version}"] = runTests([testImage: images[imgKey], dockerVersion: version]) + } +} + +parallel(testMatrix) From 48c5cd82fc91b874fb1502149a58f5f72a026a91 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 8 Dec 2016 13:08:05 -0800 Subject: [PATCH 0221/1301] Prevent Swarm address conflicts Signed-off-by: Joffrey F --- Jenkinsfile | 24 +++++++++++-------- tests/helpers.py | 4 ++++ tests/integration/api_service_test.py | 2 +- tests/integration/api_swarm_test.py | 28 +++++++++++------------ tests/integration/base.py | 6 +++++ tests/integration/models_nodes_test.py | 4 +++- tests/integration/models_services_test.py | 2 +- tests/integration/models_swarm_test.py | 6 ++++- 8 files changed, 48 insertions(+), 28 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 3af2b1661e..b73a78eb95 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -4,7 +4,7 @@ def imageNameBase = "dockerbuildbot/docker-py" def imageNamePy2 def imageNamePy3 def images = [:] -def dockerVersions = ["1.12.3", "1.13.0-rc3"] +def dockerVersions = ["1.12.0", "1.13.0-rc3"] def buildImage = { name, buildargs, pyTag -> img = docker.image(name) @@ -33,6 +33,7 @@ def buildImages = { -> def runTests = { Map settings -> def dockerVersion = settings.get("dockerVersion", null) + def pythonVersion = settings.get("pythonVersion", null) def testImage = settings.get("testImage", null) if (!testImage) { @@ -41,26 +42,31 @@ def runTests = { Map settings -> if (!dockerVersion) { throw new Exception("Need Docker version to test, e.g.: `runTests(dockerVersion: '1.12.3')`") } + if (!pythonVersion) { + throw new Exception("Need Python version being tested, e.g.: `runTests(pythonVersion: 'py2.7')`") + } { -> wrappedNode(label: "ubuntu && !zfs && amd64", cleanWorkspace: true) { - stage("test image=${testImage} / docker=${dockerVersion}") { + stage("test python=${pythonVersion} / docker=${dockerVersion}") { checkout(scm) + def dindContainerName = "dpy-dind-\$BUILD_NUMBER-\$EXECUTOR_NUMBER" + def testContainerName = "dpy-tests-\$BUILD_NUMBER-\$EXECUTOR_NUMBER" try { - sh """docker run -d --name dpy-dind-\$BUILD_NUMBER -v /tmp --privileged \\ + sh """docker run -d --name ${dindContainerName} -v /tmp --privileged \\ dockerswarm/dind:${dockerVersion} docker daemon -H tcp://0.0.0.0:2375 """ sh """docker run \\ - --name dpy-tests-\$BUILD_NUMBER --volumes-from dpy-dind-\$BUILD_NUMBER \\ + --name ${testContainerName} --volumes-from ${dindContainerName} \\ -e 'DOCKER_HOST=tcp://docker:2375' \\ - --link=dpy-dind-\$BUILD_NUMBER:docker \\ + --link=${dindContainerName}:docker \\ ${testImage} \\ - py.test -rxs tests/integration + py.test -v -rxs tests/integration """ } finally { sh """ - docker stop dpy-tests-\$BUILD_NUMBER dpy-dind-\$BUILD_NUMBER - docker rm -vf dpy-tests-\$BUILD_NUMBER dpy-dind-\$BUILD_NUMBER + docker stop ${dindContainerName} ${testContainerName} + docker rm -vf ${dindContainerName} ${testContainerName} """ } } @@ -75,7 +81,7 @@ def testMatrix = [failFast: false] for (imgKey in new ArrayList(images.keySet())) { for (version in dockerVersions) { - testMatrix["${imgKey}_${version}"] = runTests([testImage: images[imgKey], dockerVersion: version]) + testMatrix["${imgKey}_${version}"] = runTests([testImage: images[imgKey], dockerVersion: version, pythonVersion: imgKey]) } } diff --git a/tests/helpers.py b/tests/helpers.py index 53cf57ad70..1e42363144 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -74,3 +74,7 @@ def force_leave_swarm(client): continue else: return + + +def swarm_listen_addr(): + return '0.0.0.0:{0}'.format(random.randrange(10000, 25000)) diff --git a/tests/integration/api_service_test.py b/tests/integration/api_service_test.py index 04f2fe0bcb..fc7940023f 100644 --- a/tests/integration/api_service_test.py +++ b/tests/integration/api_service_test.py @@ -10,7 +10,7 @@ class ServiceTest(BaseAPIIntegrationTest): def setUp(self): super(ServiceTest, self).setUp() self.client.leave_swarm(force=True) - self.client.init_swarm('eth0') + self.init_swarm() def tearDown(self): super(ServiceTest, self).tearDown() diff --git a/tests/integration/api_swarm_test.py b/tests/integration/api_swarm_test.py index a10437b1e3..a8f439c8b5 100644 --- a/tests/integration/api_swarm_test.py +++ b/tests/integration/api_swarm_test.py @@ -17,39 +17,37 @@ def tearDown(self): @requires_api_version('1.24') def test_init_swarm_simple(self): - assert self.client.init_swarm('eth0') + assert self.init_swarm() @requires_api_version('1.24') def test_init_swarm_force_new_cluster(self): pytest.skip('Test stalls the engine on 1.12.0') - assert self.client.init_swarm('eth0') + assert self.init_swarm() version_1 = self.client.inspect_swarm()['Version']['Index'] - assert self.client.init_swarm('eth0', force_new_cluster=True) + assert self.client.init_swarm(force_new_cluster=True) version_2 = self.client.inspect_swarm()['Version']['Index'] assert version_2 != version_1 @requires_api_version('1.24') def test_init_already_in_cluster(self): - assert self.client.init_swarm('eth0') + assert self.init_swarm() with pytest.raises(docker.errors.APIError): - self.client.init_swarm('eth0') + self.init_swarm() @requires_api_version('1.24') def test_init_swarm_custom_raft_spec(self): spec = self.client.create_swarm_spec( snapshot_interval=5000, log_entries_for_slow_followers=1200 ) - assert self.client.init_swarm( - advertise_addr='eth0', swarm_spec=spec - ) + assert self.init_swarm(swarm_spec=spec) swarm_info = self.client.inspect_swarm() assert swarm_info['Spec']['Raft']['SnapshotInterval'] == 5000 assert swarm_info['Spec']['Raft']['LogEntriesForSlowFollowers'] == 1200 @requires_api_version('1.24') def test_leave_swarm(self): - assert self.client.init_swarm('eth0') + assert self.init_swarm() with pytest.raises(docker.errors.APIError) as exc_info: self.client.leave_swarm() exc_info.value.response.status_code == 500 @@ -61,7 +59,7 @@ def test_leave_swarm(self): @requires_api_version('1.24') def test_update_swarm(self): - assert self.client.init_swarm('eth0') + assert self.init_swarm() swarm_info_1 = self.client.inspect_swarm() spec = self.client.create_swarm_spec( snapshot_interval=5000, log_entries_for_slow_followers=1200, @@ -92,7 +90,7 @@ def test_update_swarm(self): @requires_api_version('1.24') def test_update_swarm_name(self): - assert self.client.init_swarm('eth0') + assert self.init_swarm() swarm_info_1 = self.client.inspect_swarm() spec = self.client.create_swarm_spec( node_cert_expiry=7776000000000000, name='reimuhakurei' @@ -110,7 +108,7 @@ def test_update_swarm_name(self): @requires_api_version('1.24') def test_list_nodes(self): - assert self.client.init_swarm('eth0') + assert self.init_swarm() nodes_list = self.client.nodes() assert len(nodes_list) == 1 node = nodes_list[0] @@ -129,7 +127,7 @@ def test_list_nodes(self): @requires_api_version('1.24') def test_inspect_node(self): - assert self.client.init_swarm('eth0') + assert self.init_swarm() nodes_list = self.client.nodes() assert len(nodes_list) == 1 node = nodes_list[0] @@ -139,7 +137,7 @@ def test_inspect_node(self): @requires_api_version('1.24') def test_update_node(self): - assert self.client.init_swarm('eth0') + assert self.init_swarm() nodes_list = self.client.nodes() node = nodes_list[0] orig_spec = node['Spec'] @@ -162,7 +160,7 @@ def test_update_node(self): @requires_api_version('1.24') def test_remove_main_node(self): - assert self.client.init_swarm('eth0') + assert self.init_swarm() nodes_list = self.client.nodes() node_id = nodes_list[0]['ID'] with pytest.raises(docker.errors.NotFound): diff --git a/tests/integration/base.py b/tests/integration/base.py index ea43d056e5..4a41e6b81a 100644 --- a/tests/integration/base.py +++ b/tests/integration/base.py @@ -5,6 +5,7 @@ from docker.utils import kwargs_from_env import six +from .. import helpers BUSYBOX = 'busybox:buildroot-2014.02' @@ -90,3 +91,8 @@ def execute(self, container, cmd, exit_code=0, **kwargs): msg = "Expected `{}` to exit with code {} but returned {}:\n{}".format( " ".join(cmd), exit_code, actual_exit_code, output) assert actual_exit_code == exit_code, msg + + def init_swarm(self, **kwargs): + return self.client.init_swarm( + 'eth0', listen_addr=helpers.swarm_listen_addr(), **kwargs + ) diff --git a/tests/integration/models_nodes_test.py b/tests/integration/models_nodes_test.py index 0199d69303..9fd16593ac 100644 --- a/tests/integration/models_nodes_test.py +++ b/tests/integration/models_nodes_test.py @@ -1,5 +1,7 @@ import unittest + import docker + from .. import helpers @@ -12,7 +14,7 @@ def tearDown(self): def test_list_get_update(self): client = docker.from_env() - client.swarm.init() + client.swarm.init(listen_addr=helpers.swarm_listen_addr()) nodes = client.nodes.list() assert len(nodes) == 1 assert nodes[0].attrs['Spec']['Role'] == 'manager' diff --git a/tests/integration/models_services_test.py b/tests/integration/models_services_test.py index baa40a9120..a795df9841 100644 --- a/tests/integration/models_services_test.py +++ b/tests/integration/models_services_test.py @@ -11,7 +11,7 @@ class ServiceTest(unittest.TestCase): def setUpClass(cls): client = docker.from_env() helpers.force_leave_swarm(client) - client.swarm.init() + client.swarm.init(listen_addr=helpers.swarm_listen_addr()) @classmethod def tearDownClass(cls): diff --git a/tests/integration/models_swarm_test.py b/tests/integration/models_swarm_test.py index 72bf9e5c92..4f177f1005 100644 --- a/tests/integration/models_swarm_test.py +++ b/tests/integration/models_swarm_test.py @@ -1,5 +1,7 @@ import unittest + import docker + from .. import helpers @@ -12,7 +14,9 @@ def tearDown(self): def test_init_update_leave(self): client = docker.from_env() - client.swarm.init(snapshot_interval=5000) + client.swarm.init( + snapshot_interval=5000, listen_addr=helpers.swarm_listen_addr() + ) assert client.swarm.attrs['Spec']['Raft']['SnapshotInterval'] == 5000 client.swarm.update(snapshot_interval=10000) assert client.swarm.attrs['Spec']['Raft']['SnapshotInterval'] == 10000 From 769ca5a76a4dcd2f4c87248c12dc0ced5313ab75 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 2 Dec 2016 15:37:58 -0800 Subject: [PATCH 0222/1301] Rename non-URL occurrences of docker-py to "Docker SDK for Python" Signed-off-by: Joffrey F --- CONTRIBUTING.md | 29 ++++++++++++++--------------- MAINTAINERS | 2 +- README.md | 6 ++++-- docker/transport/npipeconn.py | 2 +- docker/utils/json_stream.py | 7 ++++--- 5 files changed, 24 insertions(+), 22 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index dbc1c02a91..861731188c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,7 +1,7 @@ # Contributing guidelines See the [Docker contributing guidelines](https://github.com/docker/docker/blob/master/CONTRIBUTING.md). -The following is specific to docker-py. +The following is specific to Docker SDK for Python. Thank you for your interest in the project. We look forward to your contribution. In order to make the process as fast and streamlined as possible, @@ -10,13 +10,14 @@ here is a set of guidelines we recommend you follow. ## Reporting issues We do our best to ensure bugs don't creep up in our releases, but some may -still slip through. If you encounter one while using docker-py, please create -an issue [in the tracker](https://github.com/docker/docker-py/issues/new) with +still slip through. If you encounter one while using the SDK, please +create an issue +[in the tracker](https://github.com/docker/docker-py/issues/new) with the following information: -- docker-py version, docker version and python version +- SDK version, Docker version and python version ``` -pip freeze | grep docker-py && python --version && docker version +pip freeze | grep docker && python --version && docker version ``` - OS, distribution and OS version - The issue you're encountering including a stacktrace if applicable @@ -24,14 +25,14 @@ pip freeze | grep docker-py && python --version && docker version To save yourself time, please be sure to check our [documentation](https://docker-py.readthedocs.io/) and use the -[search function](https://github.com/docker/docker-py/search) to find out if -it has already been addressed, or is currently being looked at. +[search function](https://github.com/docker/docker-py/search) to find +out if it has already been addressed, or is currently being looked at. ## Submitting pull requests Do you have a fix for an existing issue, or want to add a new functionality -to docker-py? We happily welcome pull requests. Here are a few tips to make -the review process easier on both the maintainers and yourself. +to the SDK? We happily welcome pull requests. Here are a few tips to +make the review process easier on both the maintainers and yourself. ### 1. Sign your commits @@ -87,11 +88,10 @@ to reach out and ask questions. We will do our best to answer and help out. ## Development environment -If you're looking contribute to docker-py but are new to the project or Python, -here are the steps to get you started. +If you're looking contribute to Docker SDK for Python but are new to the +project or Python, here are the steps to get you started. -1. Fork [https://github.com/docker/docker-py](https://github.com/docker/docker-py) - to your username. +1. Fork https://github.com/docker/docker-py to your username. 2. Clone your forked repository locally with `git clone git@github.com:yourusername/docker-py.git`. 3. Configure a @@ -110,8 +110,7 @@ To get the source source code and run the unit tests, run: ``` $ git clone git://github.com/docker/docker-py.git $ cd docker-py -$ pip install tox -$ tox +$ make test ``` ## Building the docs diff --git a/MAINTAINERS b/MAINTAINERS index 1f46236e72..76aafd8876 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1,4 +1,4 @@ -# Docker-py maintainers file +# Docker SDK for Python maintainers file # # This file describes who runs the docker/docker-py project and how. # This is a living document - if you see something out of date or missing, speak up! diff --git a/README.md b/README.md index 11fcbad2fb..4230f30960 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,9 @@ [![Build Status](https://travis-ci.org/docker/docker-py.svg?branch=master)](https://travis-ci.org/docker/docker-py) -**Warning:** This readme is for the development version of docker-py, which is significantly different to the stable version. [Documentation for the stable version is here.](https://docker-py.readthedocs.io/) +**Warning:** This README is for the development version of the Docker SDK for +Python, which is significantly different to the stable version. +[Documentation for the stable version is here.](https://docker-py.readthedocs.io/) A Python library for the Docker Engine API. It lets you do anything the `docker` command does, but from within Python apps – run containers, manage containers, manage Swarms, etc. @@ -10,7 +12,7 @@ A Python library for the Docker Engine API. It lets you do anything the `docker` The latest stable version [is available on PyPi](https://pypi.python.org/pypi/docker/). Either add `docker` to your `requirements.txt` file or install with pip: - pip install docker-py + pip install docker ## Usage diff --git a/docker/transport/npipeconn.py b/docker/transport/npipeconn.py index 3054037f3a..db059b445a 100644 --- a/docker/transport/npipeconn.py +++ b/docker/transport/npipeconn.py @@ -96,7 +96,7 @@ def request_url(self, request, proxies): # doesn't have a hostname, like is the case when using a UNIX socket. # Since proxies are an irrelevant notion in the case of UNIX sockets # anyway, we simply return the path URL directly. - # See also: https://github.com/docker/docker-py/issues/811 + # See also: https://github.com/docker/docker-sdk-python/issues/811 return request.path_url def close(self): diff --git a/docker/utils/json_stream.py b/docker/utils/json_stream.py index f97ab9e296..addffdf2fb 100644 --- a/docker/utils/json_stream.py +++ b/docker/utils/json_stream.py @@ -13,10 +13,11 @@ def stream_as_text(stream): - """Given a stream of bytes or text, if any of the items in the stream + """ + Given a stream of bytes or text, if any of the items in the stream are bytes convert them to text. - This function can be removed once docker-py returns text streams instead - of byte streams. + This function can be removed once we return text streams + instead of byte streams. """ for data in stream: if not isinstance(data, six.text_type): From 7b8809eb07d200b120029476d6a7d77042eb3337 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 6 Dec 2016 18:54:59 -0800 Subject: [PATCH 0223/1301] Bump version and update Changelog Signed-off-by: Joffrey F --- docker/version.py | 2 +- docs/change-log.md | 119 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 120 insertions(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index ab6838fb9e..dd7995059d 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "2.0.0-dev" +version = "2.0.0" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) diff --git a/docs/change-log.md b/docs/change-log.md index a7bb0b08ca..cb0e6130ba 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -1,6 +1,125 @@ Change log ========== + +2.0.0 +----- + +[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/22?closed=1) + +### Breaking changes + +* Dropped support for Python 2.6 +* `docker.Client` has been renamed to `docker.APIClient` +* `docker.from_env` now creates a `DockerClient` instance instead of an + `APIClient` instance. +* Removed HostConfig parameters from `APIClient.start` +* The minimum supported API version is now 1.21 (Engine version 1.9.0+) +* The name of the `pip` package is now `docker` (was: `docker-py`). New + versions of this library will only be published as `docker` from now on. +* `docker.ssladapter` is now `docker.transport.ssladapter` +* The package structure has been flattened in certain cases, which may affect + import for `docker.auth` and `docker.utils.ports` +* `docker.utils.types` has been moved to `docker.types` +* `create_host_config`, `create_ipam_pool` and `create_ipam_config` have been + removed from `docker.utils`. They have been replaced by the following classes + in `docker.types`: `HostConfig`, `IPAMPool` and `IPAMCOnfig`. + +### Features + +* Added a high-level, user-focused API as `docker.DockerClient`. See the + README and documentation for more information. +* Implemented `update_node` method in `APIClient`. +* Implemented `remove_node` method in `APIClient`. +* Added support for `restart_policy` in `update_container`. +* Added support for `labels` and `shmsize` in `build`. +* Added support for `attachable` in `create_network` +* Added support for `healthcheck` in `create_container`. +* Added support for `isolation` in `HostConfig`. +* Expanded support for `pid_mode` in `HostConfig` (now supports arbitrary + values for API version >= 1.24). +* Added support for `options` in `IPAMConfig` +* Added a `HealthCheck` class to `docker.types` to be used in + `create_container`. +* Added an `EndpointSpec` class to `docker.types` to be used in + `create_service` and `update_service`. + + +### Bugfixes + +* Fixed a bug where auth information would not be properly passed to the engine + during a `build` if the client used a credentials store. +* Fixed an issue with some exclusion patterns in `build`. +* Fixed an issue where context files were bundled with the wrong permissions + when calling `build` on Windows. +* Fixed an issue where auth info would not be retrieved from its default location + on Windows. +* Fixed an issue where lists of `networks` in `create_service` and + `update_service` wouldn't be properly converted for the engine. +* Fixed an issue where `endpoint_config` in `create_service` and + `update_service` would be ignored. +* `endpoint_config` in `create_service` and `update_service` has been + deprecated in favor of `endpoint_spec` +* Fixed a bug where `constraints` in a `TaskTemplate` object wouldn't be + properly converted for the engine. +* Fixed an issue where providing a dictionary for `env` in `ContainerSpec` + would provoke an `APIError` when sent to the engine. +* Fixed a bug where providing an `env_file` containing empty lines in + `create_container`would raise an exception. +* Fixed a bug where `detach` was being ignored by `exec_start`. + +### Documentation + +* Documentation for classes and methods is now included alongside the code as + docstrings. + +1.10.6 +------ + +[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/26?closed=1) + +### Bugfixes + +* Fixed an issue where setting a `NpipeSocket` instance to blocking mode would + put it in non-blocking mode and vice-versa. + + +1.10.5 +------ + +[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/25?closed=1) + +### Bugfixes + +* Fixed an issue where concurrent attempts to access to a named pipe by the + client would sometimes cause recoverable exceptions to be raised. + + +1.10.4 +------ + +[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/24?closed=1) + +### Bugfixes + +* Fixed an issue where `RestartPolicy.condition_types.ON_FAILURE` would yield + an invalid value. +* Fixed an issue where the SSL connection adapter would receive an invalid + argument. +* Fixed an issue that caused the Client to fail to reach API endpoints when + the provided `base_url` had a trailing slash. +* Fixed a bug where some `environment` values in `create_container` + containing unicode characters would raise an encoding error. +* Fixed a number of issues tied with named pipe transport on Windows. +* Fixed a bug where inclusion patterns in `.dockerignore` would cause some + excluded files to appear in the build context on Windows. + +### Miscellaneous + +* Adjusted version requirements for the `requests` library. +* It is now possible to run the docker-py test suite on Windows. + + 1.10.3 ------ From f008f344240b9e582f192ce20c9b122fcbeecf9a Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 12 Dec 2016 14:27:18 -0800 Subject: [PATCH 0224/1301] Remove development version warning from README Signed-off-by: Joffrey F --- README.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/README.md b/README.md index 4230f30960..d80d9307f0 100644 --- a/README.md +++ b/README.md @@ -2,10 +2,6 @@ [![Build Status](https://travis-ci.org/docker/docker-py.svg?branch=master)](https://travis-ci.org/docker/docker-py) -**Warning:** This README is for the development version of the Docker SDK for -Python, which is significantly different to the stable version. -[Documentation for the stable version is here.](https://docker-py.readthedocs.io/) - A Python library for the Docker Engine API. It lets you do anything the `docker` command does, but from within Python apps – run containers, manage containers, manage Swarms, etc. ## Installation From 4219ebc9a1d3d20239f4e467a52e9382193b8b76 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 12 Dec 2016 14:30:21 -0800 Subject: [PATCH 0225/1301] Bump version number to next dev Signed-off-by: Joffrey F --- docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index dd7995059d..957097006b 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "2.0.0" +version = "2.1.0-dev" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) From 59ccd8a782b64d91e3b4609b750c963b253fbb81 Mon Sep 17 00:00:00 2001 From: Dmitri Date: Wed, 28 Dec 2016 23:23:08 -0800 Subject: [PATCH 0226/1301] Fix readonly in mounts. Signed-off-by: Dmitri Zimine dz@stackstorm.com --- docker/types/services.py | 3 ++- tests/unit/types.py | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 tests/unit/types.py diff --git a/docker/types/services.py b/docker/types/services.py index 5041f89d92..d76561efd0 100644 --- a/docker/types/services.py +++ b/docker/types/services.py @@ -135,6 +135,7 @@ def __init__(self, target, source, type='volume', read_only=False, 'Only acceptable mount types are `bind` and `volume`.' ) self['Type'] = type + self['ReadOnly'] = read_only if type == 'bind': if propagation is not None: @@ -174,7 +175,7 @@ def parse_mount_string(cls, string): else: target = parts[1] source = parts[0] - read_only = not (len(parts) == 3 or parts[2] == 'ro') + read_only = not (len(parts) == 2 or parts[2] == 'rw') return cls(target, source, read_only=read_only) diff --git a/tests/unit/types.py b/tests/unit/types.py new file mode 100644 index 0000000000..5a7383748a --- /dev/null +++ b/tests/unit/types.py @@ -0,0 +1,16 @@ +import unittest +from docker.types.services import Mount + + +class TestMounts(unittest.TestCase): + def test_parse_mount_string_docker(self): + mount = Mount.parse_mount_string("foo/bar:/buz:ro") + self.assertEqual(mount['Source'], "foo/bar") + self.assertEqual(mount['Target'], "/buz") + self.assertEqual(mount['ReadOnly'], True) + + mount = Mount.parse_mount_string("foo/bar:/buz:rw") + self.assertEqual(mount['ReadOnly'], False) + + mount = Mount.parse_mount_string("foo/bar:/buz") + self.assertEqual(mount['ReadOnly'], False) From 9047263354114dd3cd4e6cb7e5cdbd02fc0e61d7 Mon Sep 17 00:00:00 2001 From: Lobsiinvok Date: Mon, 19 Dec 2016 15:07:29 +0800 Subject: [PATCH 0227/1301] Add filters option to NetworkApiMixin.networks Signed-off-by: Boik --- docker/api/network.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/docker/api/network.py b/docker/api/network.py index c58ea6e502..7ccda559d6 100644 --- a/docker/api/network.py +++ b/docker/api/network.py @@ -1,19 +1,23 @@ -import json - from ..errors import InvalidVersion from ..utils import check_resource, minimum_version from ..utils import version_lt +from .. import utils class NetworkApiMixin(object): @minimum_version('1.21') - def networks(self, names=None, ids=None): + def networks(self, names=None, ids=None, filters=None): """ List networks. Similar to the ``docker networks ls`` command. Args: names (list): List of names to filter by ids (list): List of ids to filter by + filters (dict): Filters to be processed on the network list. + Available filters: + - ``driver=[]`` Matches a network's driver. + - ``label=[]`` or ``label=[=]``. + - ``type=["custom"|"builtin"] `` Filters networks by type. Returns: (dict): List of network objects. @@ -23,14 +27,13 @@ def networks(self, names=None, ids=None): If the server returns an error. """ - filters = {} + if filters is None: + filters = {} if names: filters['name'] = names if ids: filters['id'] = ids - - params = {'filters': json.dumps(filters)} - + params = {'filters': utils.convert_filters(filters)} url = self._url("/networks") res = self._get(url, params=params) return self._result(res, json=True) From 2ba802dfbecff3a3ad347a4551925b28e3a942db Mon Sep 17 00:00:00 2001 From: realityone Date: Fri, 6 Jan 2017 11:29:56 +0800 Subject: [PATCH 0228/1301] provide best practice for Image.save Signed-off-by: realityone --- docker/models/images.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docker/models/images.py b/docker/models/images.py index 32068e6927..1afb0fd90a 100644 --- a/docker/models/images.py +++ b/docker/models/images.py @@ -62,10 +62,11 @@ def save(self): Example: - >>> image = cli.get("fedora:latest") + >>> image = cli.images.get("fedora:latest") >>> resp = image.save() >>> f = open('/tmp/fedora-latest.tar', 'w') - >>> f.write(resp.data) + >>> for chunk in resp.stream(): + >>> f.write(chunk) >>> f.close() """ return self.client.api.get_image(self.id) From 9450442c8c3e69d6ec82dc9610fe7f8ee31181f2 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 6 Jan 2017 16:37:15 -0800 Subject: [PATCH 0229/1301] Accept / as a path separator in dockerignore patterns on all platforms Signed-off-by: Joffrey F --- docker/utils/utils.py | 26 +++++++++++++++++--------- tests/unit/utils_test.py | 18 ++++++++++++++++++ 2 files changed, 35 insertions(+), 9 deletions(-) diff --git a/docker/utils/utils.py b/docker/utils/utils.py index 4e5f454906..e12fcf00dc 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -175,11 +175,17 @@ def should_check_directory(directory_path, exclude_patterns, include_patterns): # docker logic (2016-10-27): # https://github.com/docker/docker/blob/bc52939b0455116ab8e0da67869ec81c1a1c3e2c/pkg/archive/archive.go#L640-L671 - path_with_slash = directory_path + os.sep - possible_child_patterns = [pattern for pattern in include_patterns if - (pattern + os.sep).startswith(path_with_slash)] - directory_included = should_include(directory_path, exclude_patterns, - include_patterns) + def normalize_path(path): + return path.replace(os.path.sep, '/') + + path_with_slash = normalize_path(directory_path) + '/' + possible_child_patterns = [ + pattern for pattern in map(normalize_path, include_patterns) + if (pattern + '/').startswith(path_with_slash) + ] + directory_included = should_include( + directory_path, exclude_patterns, include_patterns + ) return directory_included or len(possible_child_patterns) > 0 @@ -195,9 +201,11 @@ def get_paths(root, exclude_patterns, include_patterns, has_exceptions=False): # by mutating the dirs we're iterating over. # This looks strange, but is considered the correct way to skip # traversal. See https://docs.python.org/2/library/os.html#os.walk - dirs[:] = [d for d in dirs if - should_check_directory(os.path.join(parent, d), - exclude_patterns, include_patterns)] + dirs[:] = [ + d for d in dirs if should_check_directory( + os.path.join(parent, d), exclude_patterns, include_patterns + ) + ] for path in dirs: if should_include(os.path.join(parent, path), @@ -213,7 +221,7 @@ def get_paths(root, exclude_patterns, include_patterns, has_exceptions=False): def match_path(path, pattern): - pattern = pattern.rstrip('/') + pattern = pattern.rstrip('/' + os.path.sep) if pattern: pattern = os.path.relpath(pattern) diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index 743d076da3..cf00616d3d 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -780,6 +780,16 @@ def test_directory_with_subdir_exception(self): ]) ) + @pytest.mark.skipif( + not IS_WINDOWS_PLATFORM, reason='Backslash patterns only on Windows' + ) + def test_directory_with_subdir_exception_win32_pathsep(self): + assert self.exclude(['foo', '!foo\\bar']) == convert_paths( + self.all_paths - set([ + 'foo/a.py', 'foo/b.py', 'foo', 'foo/Dockerfile3' + ]) + ) + def test_directory_with_wildcard_exception(self): assert self.exclude(['foo', '!foo/*.py']) == convert_paths( self.all_paths - set([ @@ -792,6 +802,14 @@ def test_subdirectory(self): self.all_paths - set(['foo/bar', 'foo/bar/a.py']) ) + @pytest.mark.skipif( + not IS_WINDOWS_PLATFORM, reason='Backslash patterns only on Windows' + ) + def test_subdirectory_win32_pathsep(self): + assert self.exclude(['foo\\bar']) == convert_paths( + self.all_paths - set(['foo/bar', 'foo/bar/a.py']) + ) + class TarTest(unittest.TestCase): def test_tar_with_excludes(self): From a96073199939c6f01894414a8a9eb78409ac4bb5 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 9 Jan 2017 14:33:58 -0800 Subject: [PATCH 0230/1301] Additional parse_mount_string tests Signed-off-by: Joffrey F --- tests/unit/dockertypes_test.py | 34 ++++++++++++++++++++++++++++++++-- tests/unit/types.py | 16 ---------------- 2 files changed, 32 insertions(+), 18 deletions(-) delete mode 100644 tests/unit/types.py diff --git a/tests/unit/dockertypes_test.py b/tests/unit/dockertypes_test.py index 2480b9ef92..5cf5f4e7b2 100644 --- a/tests/unit/dockertypes_test.py +++ b/tests/unit/dockertypes_test.py @@ -5,9 +5,9 @@ import pytest from docker.constants import DEFAULT_DOCKER_API_VERSION -from docker.errors import InvalidVersion +from docker.errors import InvalidArgument, InvalidVersion from docker.types import ( - EndpointConfig, HostConfig, IPAMConfig, IPAMPool, LogConfig, Ulimit, + EndpointConfig, HostConfig, IPAMConfig, IPAMPool, LogConfig, Mount, Ulimit, ) @@ -253,3 +253,33 @@ def test_create_ipam_config(self): 'IPRange': None, }] }) + + +class TestMounts(unittest.TestCase): + def test_parse_mount_string_ro(self): + mount = Mount.parse_mount_string("/foo/bar:/baz:ro") + self.assertEqual(mount['Source'], "/foo/bar") + self.assertEqual(mount['Target'], "/baz") + self.assertEqual(mount['ReadOnly'], True) + + def test_parse_mount_string_rw(self): + mount = Mount.parse_mount_string("/foo/bar:/baz:rw") + self.assertEqual(mount['Source'], "/foo/bar") + self.assertEqual(mount['Target'], "/baz") + self.assertEqual(mount['ReadOnly'], False) + + def test_parse_mount_string_short_form(self): + mount = Mount.parse_mount_string("/foo/bar:/baz") + self.assertEqual(mount['Source'], "/foo/bar") + self.assertEqual(mount['Target'], "/baz") + self.assertEqual(mount['ReadOnly'], False) + + def test_parse_mount_string_no_source(self): + mount = Mount.parse_mount_string("foo/bar") + self.assertEqual(mount['Source'], None) + self.assertEqual(mount['Target'], "foo/bar") + self.assertEqual(mount['ReadOnly'], False) + + def test_parse_mount_string_invalid(self): + with pytest.raises(InvalidArgument): + Mount.parse_mount_string("foo:bar:baz:rw") diff --git a/tests/unit/types.py b/tests/unit/types.py deleted file mode 100644 index 5a7383748a..0000000000 --- a/tests/unit/types.py +++ /dev/null @@ -1,16 +0,0 @@ -import unittest -from docker.types.services import Mount - - -class TestMounts(unittest.TestCase): - def test_parse_mount_string_docker(self): - mount = Mount.parse_mount_string("foo/bar:/buz:ro") - self.assertEqual(mount['Source'], "foo/bar") - self.assertEqual(mount['Target'], "/buz") - self.assertEqual(mount['ReadOnly'], True) - - mount = Mount.parse_mount_string("foo/bar:/buz:rw") - self.assertEqual(mount['ReadOnly'], False) - - mount = Mount.parse_mount_string("foo/bar:/buz") - self.assertEqual(mount['ReadOnly'], False) From 180dd6997489c7c5ccdd4c9bfbd213f0298143c4 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 9 Jan 2017 14:34:25 -0800 Subject: [PATCH 0231/1301] Raise InvalidArgument exception when invalid arguments are provided Signed-off-by: Joffrey F --- docker/errors.py | 4 ++++ docker/types/services.py | 12 ++++++------ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/docker/errors.py b/docker/errors.py index 05f4cae5c1..95c462b9d2 100644 --- a/docker/errors.py +++ b/docker/errors.py @@ -93,6 +93,10 @@ class InvalidConfigFile(DockerException): pass +class InvalidArgument(DockerException): + pass + + class DeprecatedMethod(DockerException): pass diff --git a/docker/types/services.py b/docker/types/services.py index d76561efd0..b52afd27df 100644 --- a/docker/types/services.py +++ b/docker/types/services.py @@ -131,7 +131,7 @@ def __init__(self, target, source, type='volume', read_only=False, self['Target'] = target self['Source'] = source if type not in ('bind', 'volume'): - raise errors.DockerError( + raise errors.InvalidArgument( 'Only acceptable mount types are `bind` and `volume`.' ) self['Type'] = type @@ -143,7 +143,7 @@ def __init__(self, target, source, type='volume', read_only=False, 'Propagation': propagation } if any([labels, driver_config, no_copy]): - raise errors.DockerError( + raise errors.InvalidArgument( 'Mount type is binding but volume options have been ' 'provided.' ) @@ -158,7 +158,7 @@ def __init__(self, target, source, type='volume', read_only=False, if volume_opts: self['VolumeOptions'] = volume_opts if propagation: - raise errors.DockerError( + raise errors.InvalidArgument( 'Mount type is volume but `propagation` argument has been ' 'provided.' ) @@ -167,11 +167,11 @@ def __init__(self, target, source, type='volume', read_only=False, def parse_mount_string(cls, string): parts = string.split(':') if len(parts) > 3: - raise errors.DockerError( + raise errors.InvalidArgument( 'Invalid mount format "{0}"'.format(string) ) if len(parts) == 1: - return cls(target=parts[0]) + return cls(target=parts[0], source=None) else: target = parts[1] source = parts[0] @@ -229,7 +229,7 @@ def __init__(self, parallelism=0, delay=None, failure_action='continue'): if delay is not None: self['Delay'] = delay if failure_action not in ('pause', 'continue'): - raise errors.DockerError( + raise errors.InvalidArgument( 'failure_action must be either `pause` or `continue`.' ) self['FailureAction'] = failure_action From 40089a781c3e12e628fe1cb8c6d60efa86402cf9 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 9 Jan 2017 15:13:09 -0800 Subject: [PATCH 0232/1301] Detect mount type in parse_mount_string Signed-off-by: Joffrey F --- docker/types/services.py | 12 ++++++++- tests/unit/dockertypes_test.py | 49 +++++++++++++++++++++++++--------- 2 files changed, 48 insertions(+), 13 deletions(-) diff --git a/docker/types/services.py b/docker/types/services.py index b52afd27df..93503dc054 100644 --- a/docker/types/services.py +++ b/docker/types/services.py @@ -1,6 +1,7 @@ import six from .. import errors +from ..constants import IS_WINDOWS_PLATFORM from ..utils import format_environment, split_command @@ -175,8 +176,17 @@ def parse_mount_string(cls, string): else: target = parts[1] source = parts[0] + mount_type = 'volume' + if source.startswith('/') or ( + IS_WINDOWS_PLATFORM and source[0].isalpha() and + source[1] == ':' + ): + # FIXME: That windows condition will fail earlier since we + # split on ':'. We should look into doing a smarter split + # if we detect we are on Windows. + mount_type = 'bind' read_only = not (len(parts) == 2 or parts[2] == 'rw') - return cls(target, source, read_only=read_only) + return cls(target, source, read_only=read_only, type=mount_type) class Resources(dict): diff --git a/tests/unit/dockertypes_test.py b/tests/unit/dockertypes_test.py index 5cf5f4e7b2..d11e4f03f0 100644 --- a/tests/unit/dockertypes_test.py +++ b/tests/unit/dockertypes_test.py @@ -10,6 +10,11 @@ EndpointConfig, HostConfig, IPAMConfig, IPAMPool, LogConfig, Mount, Ulimit, ) +try: + from unittest import mock +except: + import mock + def create_host_config(*args, **kwargs): return HostConfig(*args, **kwargs) @@ -258,28 +263,48 @@ def test_create_ipam_config(self): class TestMounts(unittest.TestCase): def test_parse_mount_string_ro(self): mount = Mount.parse_mount_string("/foo/bar:/baz:ro") - self.assertEqual(mount['Source'], "/foo/bar") - self.assertEqual(mount['Target'], "/baz") - self.assertEqual(mount['ReadOnly'], True) + assert mount['Source'] == "/foo/bar" + assert mount['Target'] == "/baz" + assert mount['ReadOnly'] is True def test_parse_mount_string_rw(self): mount = Mount.parse_mount_string("/foo/bar:/baz:rw") - self.assertEqual(mount['Source'], "/foo/bar") - self.assertEqual(mount['Target'], "/baz") - self.assertEqual(mount['ReadOnly'], False) + assert mount['Source'] == "/foo/bar" + assert mount['Target'] == "/baz" + assert not mount['ReadOnly'] def test_parse_mount_string_short_form(self): mount = Mount.parse_mount_string("/foo/bar:/baz") - self.assertEqual(mount['Source'], "/foo/bar") - self.assertEqual(mount['Target'], "/baz") - self.assertEqual(mount['ReadOnly'], False) + assert mount['Source'] == "/foo/bar" + assert mount['Target'] == "/baz" + assert not mount['ReadOnly'] def test_parse_mount_string_no_source(self): mount = Mount.parse_mount_string("foo/bar") - self.assertEqual(mount['Source'], None) - self.assertEqual(mount['Target'], "foo/bar") - self.assertEqual(mount['ReadOnly'], False) + assert mount['Source'] is None + assert mount['Target'] == "foo/bar" + assert not mount['ReadOnly'] def test_parse_mount_string_invalid(self): with pytest.raises(InvalidArgument): Mount.parse_mount_string("foo:bar:baz:rw") + + def test_parse_mount_named_volume(self): + mount = Mount.parse_mount_string("foobar:/baz") + assert mount['Source'] == 'foobar' + assert mount['Target'] == '/baz' + assert mount['Type'] == 'volume' + + def test_parse_mount_bind(self): + mount = Mount.parse_mount_string('/foo/bar:/baz') + assert mount['Source'] == "/foo/bar" + assert mount['Target'] == "/baz" + assert mount['Type'] == 'bind' + + @pytest.mark.xfail + def test_parse_mount_bind_windows(self): + with mock.patch('docker.types.services.IS_WINDOWS_PLATFORM', True): + mount = Mount.parse_mount_string('C:/foo/bar:/baz') + assert mount['Source'] == "C:/foo/bar" + assert mount['Target'] == "/baz" + assert mount['Type'] == 'bind' From 91a185d7a57464d2b8826d48495bde02098a0079 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 9 Jan 2017 15:22:22 -0800 Subject: [PATCH 0233/1301] Bump 2.0.1 Signed-off-by: Joffrey F --- docs/change-log.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/change-log.md b/docs/change-log.md index cb0e6130ba..91eafcc4dc 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -1,6 +1,19 @@ Change log ========== +2.0.1 +----- + +[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/28?closed=1) + +### Bugfixes + +* Fixed a bug where forward slashes in some .dockerignore patterns weren't + being parsed correctly on Windows +* Fixed a bug where `Mount.parse_mount_string` would never set the read_only + parameter on the resulting `Mount`. +* Fixed a bug where `Mount.parse_mount_string` would incorrectly mark host + binds as being of `volume` type. 2.0.0 ----- From fb6c9a82957def973f3029e9d53dd5b4753136f3 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 11 Jan 2017 17:24:50 -0800 Subject: [PATCH 0234/1301] Use json_stream function in decoded _stream_helper Signed-off-by: Joffrey F --- docker/api/client.py | 49 +++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 26 deletions(-) diff --git a/docker/api/client.py b/docker/api/client.py index a9fe7d0899..22c32b44d9 100644 --- a/docker/api/client.py +++ b/docker/api/client.py @@ -18,16 +18,20 @@ from .swarm import SwarmApiMixin from .volume import VolumeApiMixin from .. import auth -from ..constants import (DEFAULT_TIMEOUT_SECONDS, DEFAULT_USER_AGENT, - IS_WINDOWS_PLATFORM, DEFAULT_DOCKER_API_VERSION, - STREAM_HEADER_SIZE_BYTES, DEFAULT_NUM_POOLS, - MINIMUM_DOCKER_API_VERSION) -from ..errors import (DockerException, TLSParameterError, - create_api_error_from_http_exception) +from ..constants import ( + DEFAULT_TIMEOUT_SECONDS, DEFAULT_USER_AGENT, IS_WINDOWS_PLATFORM, + DEFAULT_DOCKER_API_VERSION, STREAM_HEADER_SIZE_BYTES, DEFAULT_NUM_POOLS, + MINIMUM_DOCKER_API_VERSION +) +from ..errors import ( + DockerException, TLSParameterError, + create_api_error_from_http_exception +) from ..tls import TLSConfig from ..transport import SSLAdapter, UnixAdapter from ..utils import utils, check_resource, update_headers from ..utils.socket import frames_iter +from ..utils.json_stream import json_stream try: from ..transport import NpipeAdapter except ImportError: @@ -274,27 +278,20 @@ def _get_raw_response_socket(self, response): def _stream_helper(self, response, decode=False): """Generator for data coming from a chunked-encoded HTTP response.""" + if response.raw._fp.chunked: - reader = response.raw - while not reader.closed: - # this read call will block until we get a chunk - data = reader.read(1) - if not data: - break - if reader._fp.chunk_left: - data += reader.read(reader._fp.chunk_left) - if decode: - if six.PY3: - data = data.decode('utf-8') - # remove the trailing newline - data = data.strip() - # split the data at any newlines - data_list = data.split("\r\n") - # load and yield each line seperately - for data in data_list: - data = json.loads(data) - yield data - else: + if decode: + for chunk in json_stream(self._stream_helper(response, False)): + yield chunk + else: + reader = response.raw + while not reader.closed: + # this read call will block until we get a chunk + data = reader.read(1) + if not data: + break + if reader._fp.chunk_left: + data += reader.read(reader._fp.chunk_left) yield data else: # Response isn't chunked, meaning we probably From f0ceca47130ff940998e0e1d93098996f16b9431 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Fri, 13 Jan 2017 09:59:47 +0000 Subject: [PATCH 0235/1301] case PyPI correctly Signed-off-by: Thomas Grainger --- README.md | 2 +- docs/index.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d80d9307f0..38963b325c 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ A Python library for the Docker Engine API. It lets you do anything the `docker` ## Installation -The latest stable version [is available on PyPi](https://pypi.python.org/pypi/docker/). Either add `docker` to your `requirements.txt` file or install with pip: +The latest stable version [is available on PyPI](https://pypi.python.org/pypi/docker/). Either add `docker` to your `requirements.txt` file or install with pip: pip install docker diff --git a/docs/index.rst b/docs/index.rst index 7eadf4c7e1..67dd33f2f7 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -8,7 +8,7 @@ For more information about the Remote API, `see its documentation `_. Either add ``docker`` to your ``requirements.txt`` file or install with pip:: +The latest stable version `is available on PyPI `_. Either add ``docker`` to your ``requirements.txt`` file or install with pip:: pip install docker From 95b6fddd14b9b3556d6a929bbf66affc1a5daa61 Mon Sep 17 00:00:00 2001 From: "Alejandro E. Brito Monedero" Date: Mon, 16 Jan 2017 08:48:41 +0100 Subject: [PATCH 0236/1301] Fix #1351 * Fix TypeError when getting the tags property from an image that has no tags. Ex: An image pulled by cryptohash. It is handled like when the image doesn't have defined the RepoTags member. Signed-off-by: Alejandro E. Brito Monedero --- docker/models/images.py | 8 ++++---- tests/unit/models_images_test.py | 5 +++++ 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/docker/models/images.py b/docker/models/images.py index 32068e6927..6f8f4fe273 100644 --- a/docker/models/images.py +++ b/docker/models/images.py @@ -30,10 +30,10 @@ def tags(self): """ The image's tags. """ - return [ - tag for tag in self.attrs.get('RepoTags', []) - if tag != ':' - ] + tags = self.attrs.get('RepoTags') + if tags is None: + tags = [] + return [tag for tag in tags if tag != ':'] def history(self): """ diff --git a/tests/unit/models_images_test.py b/tests/unit/models_images_test.py index 392c58d79f..efb2116660 100644 --- a/tests/unit/models_images_test.py +++ b/tests/unit/models_images_test.py @@ -83,6 +83,11 @@ def test_tags(self): }) assert image.tags == [] + image = Image(attrs={ + 'RepoTags': None + }) + assert image.tags == [] + def test_history(self): client = make_fake_client() image = client.images.get(FAKE_IMAGE_ID) From 66d57333981f1f8a0f46c6c90a71c1ad020b0ac1 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 18 Jan 2017 15:38:53 -0800 Subject: [PATCH 0237/1301] Fix a number of docs formatting issues Signed-off-by: Joffrey F --- docker/api/container.py | 48 ++++++++++++++++++------------------- docker/api/network.py | 21 ++++++++-------- docker/api/service.py | 32 ++++++++++++------------- docker/api/swarm.py | 4 ++-- docker/models/containers.py | 32 ++++++++++++------------- docker/models/networks.py | 19 ++++++++------- docker/models/services.py | 20 +++++++--------- docker/types/networks.py | 2 +- docker/types/services.py | 14 +++++------ docs/api.rst | 2 -- docs/index.rst | 3 +-- 11 files changed, 96 insertions(+), 101 deletions(-) diff --git a/docker/api/container.py b/docker/api/container.py index afe696ca19..efcae9b0c6 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -388,13 +388,13 @@ def create_container(self, image, command=None, hostname=None, user=None, environment (dict or list): A dictionary or a list of strings in the following format ``["PASSWORD=xxx"]`` or ``{"PASSWORD": "xxx"}``. - dns (list): DNS name servers. Deprecated since API version 1.10. - Use ``host_config`` instead. - dns_opt (list): Additional options to be added to the container's - ``resolv.conf`` file + dns (:py:class:`list`): DNS name servers. Deprecated since API + version 1.10. Use ``host_config`` instead. + dns_opt (:py:class:`list`): Additional options to be added to the + container's ``resolv.conf`` file volumes (str or list): - volumes_from (list): List of container names or Ids to get - volumes from. + volumes_from (:py:class:`list`): List of container names or Ids to + get volumes from. network_disabled (bool): Disable networking name (str): A name for the container entrypoint (str or list): An entrypoint @@ -478,19 +478,19 @@ def create_host_config(self, *args, **kwargs): device_write_bps: Limit write rate (bytes per second) from a device. device_write_iops: Limit write rate (IO per second) from a device. - devices (list): Expose host devices to the container, as a list - of strings in the form + devices (:py:class:`list`): Expose host devices to the container, + as a list of strings in the form ``::``. For example, ``/dev/sda:/dev/xvda:rwm`` allows the container to have read-write access to the host's ``/dev/sda`` via a node named ``/dev/xvda`` inside the container. - dns (list): Set custom DNS servers. - dns_search (list): DNS search domains. + dns (:py:class:`list`): Set custom DNS servers. + dns_search (:py:class:`list`): DNS search domains. extra_hosts (dict): Addtional hostnames to resolve inside the container, as a mapping of hostname to IP address. - group_add (list): List of additional group names and/or IDs that - the container process will run as. + group_add (:py:class:`list`): List of additional group names and/or + IDs that the container process will run as. ipc_mode (str): Set the IPC mode for the container. isolation (str): Isolation technology to use. Default: `None`. links (dict or list of tuples): Either a dictionary mapping name @@ -539,8 +539,8 @@ def create_host_config(self, *args, **kwargs): - ``Name`` One of ``on-failure``, or ``always``. - ``MaximumRetryCount`` Number of times to restart the container on failure. - security_opt (list): A list of string values to customize labels - for MLS systems, such as SELinux. + security_opt (:py:class:`list`): A list of string values to + customize labels for MLS systems, such as SELinux. shm_size (str or int): Size of /dev/shm (e.g. ``1G``). sysctls (dict): Kernel parameters to set in the container. tmpfs (dict): Temporary filesystems to mount, as a dictionary @@ -555,13 +555,13 @@ def create_host_config(self, *args, **kwargs): '/mnt/vol1': 'size=3G,uid=1000' } - ulimits (list): Ulimits to set inside the container, as a list of - dicts. + ulimits (:py:class:`list`): Ulimits to set inside the container, + as a list of dicts. userns_mode (str): Sets the user namespace mode for the container when user namespace remapping option is enabled. Supported values are: ``host`` - volumes_from (list): List of container names or IDs to get - volumes from. + volumes_from (:py:class:`list`): List of container names or IDs to + get volumes from. Returns: @@ -618,17 +618,17 @@ def create_endpoint_config(self, *args, **kwargs): :py:meth:`create_networking_config`. Args: - aliases (list): A list of aliases for this endpoint. Names in - that list can be used within the network to reach the + aliases (:py:class:`list`): A list of aliases for this endpoint. + Names in that list can be used within the network to reach the + container. Defaults to ``None``. + links (:py:class:`list`): A list of links for this endpoint. + Containers declared in this list will be linked to this container. Defaults to ``None``. - links (list): A list of links for this endpoint. Containers - declared in this list will be linked to this container. - Defaults to ``None``. ipv4_address (str): The IP address of this container on the network, using the IPv4 protocol. Defaults to ``None``. ipv6_address (str): The IP address of this container on the network, using the IPv6 protocol. Defaults to ``None``. - link_local_ips (list): A list of link-local (IPv4/IPv6) + link_local_ips (:py:class:`list`): A list of link-local (IPv4/IPv6) addresses. Returns: diff --git a/docker/api/network.py b/docker/api/network.py index 7ccda559d6..9f6d98fea3 100644 --- a/docker/api/network.py +++ b/docker/api/network.py @@ -11,13 +11,13 @@ def networks(self, names=None, ids=None, filters=None): List networks. Similar to the ``docker networks ls`` command. Args: - names (list): List of names to filter by - ids (list): List of ids to filter by + names (:py:class:`list`): List of names to filter by + ids (:py:class:`list`): List of ids to filter by filters (dict): Filters to be processed on the network list. Available filters: - ``driver=[]`` Matches a network's driver. - ``label=[]`` or ``label=[=]``. - - ``type=["custom"|"builtin"] `` Filters networks by type. + - ``type=["custom"|"builtin"]`` Filters networks by type. Returns: (dict): List of network objects. @@ -169,17 +169,18 @@ def connect_container_to_network(self, container, net_id, Args: container (str): container-id/name to be connected to the network net_id (str): network id - aliases (list): A list of aliases for this endpoint. Names in that - list can be used within the network to reach the container. - Defaults to ``None``. - links (list): A list of links for this endpoint. Containers - declared in this list will be linkedto this container. - Defaults to ``None``. + aliases (:py:class:`list`): A list of aliases for this endpoint. + Names in that list can be used within the network to reach the + container. Defaults to ``None``. + links (:py:class:`list`): A list of links for this endpoint. + Containers declared in this list will be linked to this + container. Defaults to ``None``. ipv4_address (str): The IP address of this container on the network, using the IPv4 protocol. Defaults to ``None``. ipv6_address (str): The IP address of this container on the network, using the IPv6 protocol. Defaults to ``None``. - link_local_ips (list): A list of link-local (IPv4/IPv6) addresses. + link_local_ips (:py:class:`list`): A list of link-local + (IPv4/IPv6) addresses. """ data = { "Container": container, diff --git a/docker/api/service.py b/docker/api/service.py index 7708b75274..0d8421ecb2 100644 --- a/docker/api/service.py +++ b/docker/api/service.py @@ -13,18 +13,18 @@ def create_service( Create a service. Args: - task_template (dict): Specification of the task to start as part - of the new service. + task_template (TaskTemplate): Specification of the task to start as + part of the new service. name (string): User-defined name for the service. Optional. labels (dict): A map of labels to associate with the service. Optional. mode (string): Scheduling mode for the service (``replicated`` or ``global``). Defaults to ``replicated``. - update_config (dict): Specification for the update strategy of the - service. Default: ``None`` - networks (list): List of network names or IDs to attach the - service to. Default: ``None``. - endpoint_config (dict): Properties that can be configured to + update_config (UpdateConfig): Specification for the update strategy + of the service. Default: ``None`` + networks (:py:class:`list`): List of network names or IDs to attach + the service to. Default: ``None``. + endpoint_spec (EndpointSpec): Properties that can be configured to access and load balance a service. Default: ``None``. Returns: @@ -159,7 +159,7 @@ def tasks(self, filters=None): ``label`` and ``desired-state``. Returns: - (list): List of task dictionaries. + (:py:class:`list`): List of task dictionaries. Raises: :py:class:`docker.errors.APIError` @@ -186,20 +186,18 @@ def update_service(self, service, version, task_template=None, name=None, ID). version (int): The version number of the service object being updated. This is required to avoid conflicting writes. - task_template (dict): Specification of the updated task to start - as part of the service. See the [TaskTemplate - class](#TaskTemplate) for details. + task_template (TaskTemplate): Specification of the updated task to + start as part of the service. name (string): New name for the service. Optional. labels (dict): A map of labels to associate with the service. Optional. mode (string): Scheduling mode for the service (``replicated`` or ``global``). Defaults to ``replicated``. - update_config (dict): Specification for the update strategy of the - service. See the [UpdateConfig class](#UpdateConfig) for - details. Default: ``None``. - networks (list): List of network names or IDs to attach the - service to. Default: ``None``. - endpoint_config (dict): Properties that can be configured to + update_config (UpdateConfig): Specification for the update strategy + of the service. Default: ``None``. + networks (:py:class:`list`): List of network names or IDs to attach + the service to. Default: ``None``. + endpoint_spec (EndpointSpec): Properties that can be configured to access and load balance a service. Default: ``None``. Returns: diff --git a/docker/api/swarm.py b/docker/api/swarm.py index 3ada538389..88770562f2 100644 --- a/docker/api/swarm.py +++ b/docker/api/swarm.py @@ -143,8 +143,8 @@ def join_swarm(self, remote_addrs, join_token, listen_addr=None, Make this Engine join a swarm that has already been created. Args: - remote_addrs (list): Addresses of one or more manager nodes already - participating in the Swarm to join. + remote_addrs (:py:class:`list`): Addresses of one or more manager + nodes already participating in the Swarm to join. join_token (string): Secret token for joining this Swarm. listen_addr (string): Listen address used for inter-manager communication if the node gets promoted to manager, as well as diff --git a/docker/models/containers.py b/docker/models/containers.py index ad1cb6139c..b1cdd8f870 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -468,17 +468,17 @@ def run(self, image, command=None, stdout=True, stderr=False, device_write_bps: Limit write rate (bytes per second) from a device. device_write_iops: Limit write rate (IO per second) from a device. - devices (list): Expose host devices to the container, as a list - of strings in the form + devices (:py:class:`list`): Expose host devices to the container, + as a list of strings in the form ``::``. For example, ``/dev/sda:/dev/xvda:rwm`` allows the container to have read-write access to the host's ``/dev/sda`` via a node named ``/dev/xvda`` inside the container. - dns (list): Set custom DNS servers. - dns_opt (list): Additional options to be added to the container's - ``resolv.conf`` file. - dns_search (list): DNS search domains. + dns (:py:class:`list`): Set custom DNS servers. + dns_opt (:py:class:`list`): Additional options to be added to the + container's ``resolv.conf`` file. + dns_search (:py:class:`list`): DNS search domains. domainname (str or list): Set custom DNS search domains. entrypoint (str or list): The entrypoint for the container. environment (dict or list): Environment variables to set inside @@ -486,8 +486,8 @@ def run(self, image, command=None, stdout=True, stderr=False, format ``["SOMEVARIABLE=xxx"]``. extra_hosts (dict): Addtional hostnames to resolve inside the container, as a mapping of hostname to IP address. - group_add (list): List of additional group names and/or IDs that - the container process will run as. + group_add (:py:class:`list`): List of additional group names and/or + IDs that the container process will run as. hostname (str): Optional hostname for the container. ipc_mode (str): Set the IPC mode for the container. isolation (str): Isolation technology to use. Default: `None`. @@ -517,8 +517,8 @@ def run(self, image, command=None, stdout=True, stderr=False, behavior. Accepts number between 0 and 100. memswap_limit (str or int): Maximum amount of memory + swap a container is allowed to consume. - networks (list): A list of network names to connect this - container to. + networks (:py:class:`list`): A list of network names to connect + this container to. name (str): The name for this container. network_disabled (bool): Disable networking. network_mode (str): One of: @@ -574,8 +574,8 @@ def run(self, image, command=None, stdout=True, stderr=False, For example: ``{"Name": "on-failure", "MaximumRetryCount": 5}`` - security_opt (list): A list of string values to customize labels - for MLS systems, such as SELinux. + security_opt (:py:class:`list`): A list of string values to + customize labels for MLS systems, such as SELinux. shm_size (str or int): Size of /dev/shm (e.g. ``1G``). stdin_open (bool): Keep ``STDIN`` open even if not attached. stdout (bool): Return logs from ``STDOUT`` when ``detach=False``. @@ -598,8 +598,8 @@ def run(self, image, command=None, stdout=True, stderr=False, } tty (bool): Allocate a pseudo-TTY. - ulimits (list): Ulimits to set inside the container, as a list of - dicts. + ulimits (:py:class:`list`): Ulimits to set inside the container, as + a list of dicts. user (str or int): Username or UID to run commands as inside the container. userns_mode (str): Sets the user namespace mode for the container @@ -621,8 +621,8 @@ def run(self, image, command=None, stdout=True, stderr=False, {'/home/user1/': {'bind': '/mnt/vol2', 'mode': 'rw'}, '/var/www': {'bind': '/mnt/vol1', 'mode': 'ro'}} - volumes_from (list): List of container names or IDs to get - volumes from. + volumes_from (:py:class:`list`): List of container names or IDs to + get volumes from. working_dir (str): Path to the working directory. Returns: diff --git a/docker/models/networks.py b/docker/models/networks.py index d5e2097295..a80c9f5f8d 100644 --- a/docker/models/networks.py +++ b/docker/models/networks.py @@ -32,17 +32,18 @@ def connect(self, container): container (str): Container to connect to this network, as either an ID, name, or :py:class:`~docker.models.containers.Container` object. - aliases (list): A list of aliases for this endpoint. Names in that - list can be used within the network to reach the container. - Defaults to ``None``. - links (list): A list of links for this endpoint. Containers - declared in this list will be linkedto this container. - Defaults to ``None``. + aliases (:py:class:`list`): A list of aliases for this endpoint. + Names in that list can be used within the network to reach the + container. Defaults to ``None``. + links (:py:class:`list`): A list of links for this endpoint. + Containers declared in this list will be linkedto this + container. Defaults to ``None``. ipv4_address (str): The IP address of this container on the network, using the IPv4 protocol. Defaults to ``None``. ipv6_address (str): The IP address of this container on the network, using the IPv6 protocol. Defaults to ``None``. - link_local_ips (list): A list of link-local (IPv4/IPv6) addresses. + link_local_ips (:py:class:`list`): A list of link-local (IPv4/IPv6) + addresses. Raises: :py:class:`docker.errors.APIError` @@ -167,8 +168,8 @@ def list(self, *args, **kwargs): List networks. Similar to the ``docker networks ls`` command. Args: - names (list): List of names to filter by. - ids (list): List of ids to filter by. + names (:py:class:`list`): List of names to filter by. + ids (:py:class:`list`): List of ids to filter by. Returns: (list of :py:class:`Network`) The networks on the server. diff --git a/docker/models/services.py b/docker/models/services.py index d70c9e7a08..ef6c3e3a91 100644 --- a/docker/models/services.py +++ b/docker/models/services.py @@ -42,7 +42,7 @@ def tasks(self, filters=None): ``label``, and ``desired-state``. Returns: - (list): List of task dictionaries. + (:py:class:`list`): List of task dictionaries. Raises: :py:class:`docker.errors.APIError` @@ -92,29 +92,27 @@ def create(self, image, command=None, **kwargs): args (list of str): Arguments to the command. constraints (list of str): Placement constraints. container_labels (dict): Labels to apply to the container. - endpoint_spec (dict): Properties that can be configured to + endpoint_spec (EndpointSpec): Properties that can be configured to access and load balance a service. Default: ``None``. env (list of str): Environment variables, in the form ``KEY=val``. labels (dict): Labels to apply to the service. log_driver (str): Log driver to use for containers. log_driver_options (dict): Log driver options. - mode (string): Scheduling mode for the service (``replicated`` or + mode (str): Scheduling mode for the service (``replicated`` or ``global``). Defaults to ``replicated``. mounts (list of str): Mounts for the containers, in the form ``source:target:options``, where options is either ``ro`` or ``rw``. name (str): Name to give to the service. - networks (list): List of network names or IDs to attach the - service to. Default: ``None``. - resources (dict): Resource limits and reservations. For the - format, see the Remote API documentation. - restart_policy (dict): Restart policy for containers. For the - format, see the Remote API documentation. + networks (list of str): List of network names or IDs to attach + the service to. Default: ``None``. + resources (Resources): Resource limits and reservations. + restart_policy (RestartPolicy): Restart policy for containers. stop_grace_period (int): Amount of time to wait for containers to terminate before forcefully killing them. - update_config (dict): Specification for the update strategy of the - service. Default: ``None`` + update_config (UpdateConfig): Specification for the update strategy + of the service. Default: ``None`` user (str): User to run commands as. workdir (str): Working directory for commands to run. diff --git a/docker/types/networks.py b/docker/types/networks.py index 628ea65ad2..1c7b2c9e69 100644 --- a/docker/types/networks.py +++ b/docker/types/networks.py @@ -48,7 +48,7 @@ class IPAMConfig(dict): Args: driver (str): The IPAM driver to use. Defaults to ``default``. - pool_configs (list): A list of pool configurations + pool_configs (:py:class:`list`): A list of pool configurations (:py:class:`~docker.types.IPAMPool`). Defaults to empty list. options (dict): Driver options as a key-value dictionary. Defaults to `None`. diff --git a/docker/types/services.py b/docker/types/services.py index 93503dc054..6e1ad321b9 100644 --- a/docker/types/services.py +++ b/docker/types/services.py @@ -20,7 +20,7 @@ class TaskTemplate(dict): individual container created as part of the service. restart_policy (RestartPolicy): Specification for the restart policy which applies to containers created as part of this service. - placement (list): A list of constraints. + placement (:py:class:`list`): A list of constraints. """ def __init__(self, container_spec, resources=None, restart_policy=None, placement=None, log_driver=None): @@ -62,16 +62,16 @@ class ContainerSpec(dict): image (string): The image name to use for the container. command (string or list): The command to be run in the image. - args (list): Arguments to the command. + args (:py:class:`list`): Arguments to the command. env (dict): Environment variables. dir (string): The working directory for commands to run in. user (string): The user inside the container. labels (dict): A map of labels to associate with the service. - mounts (list): A list of specifications for mounts to be added to - containers created as part of the service. See the - :py:class:`~docker.types.Mount` class for details. + mounts (:py:class:`list`): A list of specifications for mounts to be + added to containers created as part of the service. See the + :py:class:`~docker.types.Mount` class for details. stop_grace_period (int): Amount of time to wait for the container to - terminate before forcefully killing it. + terminate before forcefully killing it. """ def __init__(self, image, command=None, args=None, env=None, workdir=None, user=None, labels=None, mounts=None, stop_grace_period=None): @@ -106,7 +106,7 @@ def __init__(self, image, command=None, args=None, env=None, workdir=None, class Mount(dict): """ Describes a mounted folder's configuration inside a container. A list of - ``Mount``s would be used as part of a + :py:class:`Mount`s would be used as part of a :py:class:`~docker.types.ContainerSpec`. Args: diff --git a/docs/api.rst b/docs/api.rst index 5e59aa7ad1..110b0a7f1d 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -19,8 +19,6 @@ Containers :members: :undoc-members: -.. py:module:: docker.api.image - Images ------ diff --git a/docs/index.rst b/docs/index.rst index 7eadf4c7e1..9f484cdbaa 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -73,13 +73,12 @@ You can manage images: >>> client.images.list() [, , ...] -That's just a taster of what you can do with the Docker SDK for Python. For more, :doc:`take a look at the reference `. +That's just a taste of what you can do with the Docker SDK for Python. For more, :doc:`take a look at the reference `. .. toctree:: :hidden: :maxdepth: 2 - Home client containers images From e9691db91bc1c25999185b055afa62c4a9ee3265 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 18 Jan 2017 16:26:22 -0800 Subject: [PATCH 0238/1301] Prevent issues when installing docker and docker-py in the same environment Signed-off-by: Joffrey F --- setup.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/setup.py b/setup.py index b82a74f7d1..9fc4ad66e9 100644 --- a/setup.py +++ b/setup.py @@ -1,10 +1,20 @@ #!/usr/bin/env python +from __future__ import print_function + import codecs import os import sys +import pip + from setuptools import setup, find_packages +if 'docker-py' in [x.project_name for x in pip.get_installed_distributions()]: + print( + 'ERROR: "docker-py" needs to be uninstalled before installing this' + ' package:\npip uninstall docker-py', file=sys.stderr + ) + sys.exit(1) ROOT_DIR = os.path.dirname(__file__) SOURCE_DIR = os.path.join(ROOT_DIR) From 06e808179991612b604ef1cecc776843df8aceac Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 11 Jan 2017 18:07:25 -0800 Subject: [PATCH 0239/1301] Convert mode argument to valid structure in create_service Signed-off-by: Joffrey F --- docker/api/service.py | 14 ++++++++--- docker/types/__init__.py | 2 +- docker/types/services.py | 35 +++++++++++++++++++++++++++ docs/api.rst | 1 + tests/integration/api_service_test.py | 28 +++++++++++++++++++++ tests/unit/dockertypes_test.py | 33 +++++++++++++++++++++++-- 6 files changed, 106 insertions(+), 7 deletions(-) diff --git a/docker/api/service.py b/docker/api/service.py index 0d8421ecb2..d2621e685c 100644 --- a/docker/api/service.py +++ b/docker/api/service.py @@ -1,5 +1,6 @@ import warnings from .. import auth, errors, utils +from ..types import ServiceMode class ServiceApiMixin(object): @@ -18,8 +19,8 @@ def create_service( name (string): User-defined name for the service. Optional. labels (dict): A map of labels to associate with the service. Optional. - mode (string): Scheduling mode for the service (``replicated`` or - ``global``). Defaults to ``replicated``. + mode (ServiceMode): Scheduling mode for the service (replicated + or global). Defaults to replicated. update_config (UpdateConfig): Specification for the update strategy of the service. Default: ``None`` networks (:py:class:`list`): List of network names or IDs to attach @@ -49,6 +50,9 @@ def create_service( raise errors.DockerException( 'Missing mandatory Image key in ContainerSpec' ) + if mode and not isinstance(mode, dict): + mode = ServiceMode(mode) + registry, repo_name = auth.resolve_repository_name(image) auth_header = auth.get_config_header(self, registry) if auth_header: @@ -191,8 +195,8 @@ def update_service(self, service, version, task_template=None, name=None, name (string): New name for the service. Optional. labels (dict): A map of labels to associate with the service. Optional. - mode (string): Scheduling mode for the service (``replicated`` or - ``global``). Defaults to ``replicated``. + mode (ServiceMode): Scheduling mode for the service (replicated + or global). Defaults to replicated. update_config (UpdateConfig): Specification for the update strategy of the service. Default: ``None``. networks (:py:class:`list`): List of network names or IDs to attach @@ -222,6 +226,8 @@ def update_service(self, service, version, task_template=None, name=None, if labels is not None: data['Labels'] = labels if mode is not None: + if not isinstance(mode, dict): + mode = ServiceMode(mode) data['Mode'] = mode if task_template is not None: image = task_template.get('ContainerSpec', {}).get('Image', None) diff --git a/docker/types/__init__.py b/docker/types/__init__.py index 7230723ee4..8e2fc17472 100644 --- a/docker/types/__init__.py +++ b/docker/types/__init__.py @@ -4,6 +4,6 @@ from .networks import EndpointConfig, IPAMConfig, IPAMPool, NetworkingConfig from .services import ( ContainerSpec, DriverConfig, EndpointSpec, Mount, Resources, RestartPolicy, - TaskTemplate, UpdateConfig + ServiceMode, TaskTemplate, UpdateConfig ) from .swarm import SwarmSpec, SwarmExternalCA diff --git a/docker/types/services.py b/docker/types/services.py index 6e1ad321b9..ec0fcb15f0 100644 --- a/docker/types/services.py +++ b/docker/types/services.py @@ -348,3 +348,38 @@ def convert_service_ports(ports): result.append(port_spec) return result + + +class ServiceMode(dict): + """ + Indicate whether a service should be deployed as a replicated or global + service, and associated parameters + + Args: + mode (string): Can be either ``replicated`` or ``global`` + replicas (int): Number of replicas. For replicated services only. + """ + def __init__(self, mode, replicas=None): + if mode not in ('replicated', 'global'): + raise errors.InvalidArgument( + 'mode must be either "replicated" or "global"' + ) + if mode != 'replicated' and replicas is not None: + raise errors.InvalidArgument( + 'replicas can only be used for replicated mode' + ) + self[mode] = {} + if replicas: + self[mode]['Replicas'] = replicas + + @property + def mode(self): + if 'global' in self: + return 'global' + return 'replicated' + + @property + def replicas(self): + if self.mode != 'replicated': + return None + return self['replicated'].get('Replicas') diff --git a/docs/api.rst b/docs/api.rst index 110b0a7f1d..b5c1e92998 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -110,5 +110,6 @@ Configuration types .. autoclass:: Mount .. autoclass:: Resources .. autoclass:: RestartPolicy +.. autoclass:: ServiceMode .. autoclass:: TaskTemplate .. autoclass:: UpdateConfig diff --git a/tests/integration/api_service_test.py b/tests/integration/api_service_test.py index fc7940023f..77d7d28f7e 100644 --- a/tests/integration/api_service_test.py +++ b/tests/integration/api_service_test.py @@ -251,3 +251,31 @@ def test_create_service_with_env(self): con_spec = svc_info['Spec']['TaskTemplate']['ContainerSpec'] assert 'Env' in con_spec assert con_spec['Env'] == ['DOCKER_PY_TEST=1'] + + def test_create_service_global_mode(self): + container_spec = docker.types.ContainerSpec( + 'busybox', ['echo', 'hello'] + ) + task_tmpl = docker.types.TaskTemplate(container_spec) + name = self.get_service_name() + svc_id = self.client.create_service( + task_tmpl, name=name, mode='global' + ) + svc_info = self.client.inspect_service(svc_id) + assert 'Mode' in svc_info['Spec'] + assert 'Global' in svc_info['Spec']['Mode'] + + def test_create_service_replicated_mode(self): + container_spec = docker.types.ContainerSpec( + 'busybox', ['echo', 'hello'] + ) + task_tmpl = docker.types.TaskTemplate(container_spec) + name = self.get_service_name() + svc_id = self.client.create_service( + task_tmpl, name=name, + mode=docker.types.ServiceMode('replicated', 5) + ) + svc_info = self.client.inspect_service(svc_id) + assert 'Mode' in svc_info['Spec'] + assert 'Replicated' in svc_info['Spec']['Mode'] + assert svc_info['Spec']['Mode']['Replicated'] == {'Replicas': 5} diff --git a/tests/unit/dockertypes_test.py b/tests/unit/dockertypes_test.py index d11e4f03f0..5c470ffa2f 100644 --- a/tests/unit/dockertypes_test.py +++ b/tests/unit/dockertypes_test.py @@ -7,7 +7,8 @@ from docker.constants import DEFAULT_DOCKER_API_VERSION from docker.errors import InvalidArgument, InvalidVersion from docker.types import ( - EndpointConfig, HostConfig, IPAMConfig, IPAMPool, LogConfig, Mount, Ulimit, + EndpointConfig, HostConfig, IPAMConfig, IPAMPool, LogConfig, Mount, + ServiceMode, Ulimit, ) try: @@ -260,7 +261,35 @@ def test_create_ipam_config(self): }) -class TestMounts(unittest.TestCase): +class ServiceModeTest(unittest.TestCase): + def test_replicated_simple(self): + mode = ServiceMode('replicated') + assert mode == {'replicated': {}} + assert mode.mode == 'replicated' + assert mode.replicas is None + + def test_global_simple(self): + mode = ServiceMode('global') + assert mode == {'global': {}} + assert mode.mode == 'global' + assert mode.replicas is None + + def test_global_replicas_error(self): + with pytest.raises(InvalidArgument): + ServiceMode('global', 21) + + def test_replicated_replicas(self): + mode = ServiceMode('replicated', 21) + assert mode == {'replicated': {'Replicas': 21}} + assert mode.mode == 'replicated' + assert mode.replicas == 21 + + def test_invalid_mode(self): + with pytest.raises(InvalidArgument): + ServiceMode('foobar') + + +class MountTest(unittest.TestCase): def test_parse_mount_string_ro(self): mount = Mount.parse_mount_string("/foo/bar:/baz:ro") assert mount['Source'] == "/foo/bar" From fbea15861e3730039855db0c65c3c426a4113e13 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 19 Jan 2017 18:05:11 -0800 Subject: [PATCH 0240/1301] Update dockerVersions Signed-off-by: Joffrey F --- Jenkinsfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index b73a78eb95..91bb2382c1 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -4,7 +4,10 @@ def imageNameBase = "dockerbuildbot/docker-py" def imageNamePy2 def imageNamePy3 def images = [:] -def dockerVersions = ["1.12.0", "1.13.0-rc3"] + +// Note: Swarm in dind seem notoriously flimsy with 1.12.1+, which is why we're +// sticking with 1.12.0 for the 1.12 series +def dockerVersions = ["1.12.0", "1.13.0"] def buildImage = { name, buildargs, pyTag -> img = docker.image(name) From 886e6ff9ea61a5c2177be1b64af397fedf420750 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 19 Jan 2017 17:53:18 -0800 Subject: [PATCH 0241/1301] Bump 2.0.2 Signed-off-by: Joffrey F --- docs/change-log.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/docs/change-log.md b/docs/change-log.md index 91eafcc4dc..1dda4415d2 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -1,6 +1,26 @@ Change log ========== +2.0.2 +----- + +[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/28?closed=1) + +### Bugfixes + +* Installation of the package now fails if the `docker-py` package is + installed in order to prevent obscure naming conflicts when both + packages co-exist. +* Added missing `filters` parameter to `APIClient.networks`. +* Resource objects generated by the `DockerClient` are now hashable. +* Fixed a bug where retrieving untagged images using `DockerClient` + would raise a `TypeError` exception. +* `mode` parameter in `create_service` is now properly converted to + a valid data type for the Engine API. Use `ServiceMode` for advanced + configurations. +* Fixed a bug where the decoded `APIClient.events` stream would sometimes raise + an exception when a container is stopped or restarted. + 2.0.1 ----- From 8ca5b2b392ad5625dc726b8ae43f866196ba40df Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 20 Jan 2017 14:07:54 -0800 Subject: [PATCH 0242/1301] Fix milestone link Signed-off-by: Joffrey F --- docs/change-log.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/change-log.md b/docs/change-log.md index 1dda4415d2..8b6d859ea8 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -4,7 +4,7 @@ Change log 2.0.2 ----- -[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/28?closed=1) +[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/29?closed=1) ### Bugfixes From e87ed38f694866416e471e43a136584c463fd42a Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 24 Jan 2017 17:45:16 -0800 Subject: [PATCH 0243/1301] Ignore socket files in utils.tar Signed-off-by: Joffrey F --- docker/utils/utils.py | 9 +++++++-- tests/unit/utils_test.py | 15 +++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/docker/utils/utils.py b/docker/utils/utils.py index e12fcf00dc..8026c4dfde 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -4,7 +4,6 @@ import os.path import json import shlex -import sys import tarfile import tempfile import warnings @@ -15,6 +14,7 @@ import requests import six +from .. import constants from .. import errors from .. import tls @@ -90,7 +90,12 @@ def tar(path, exclude=None, dockerfile=None, fileobj=None, gzip=False): for path in sorted(exclude_paths(root, exclude, dockerfile=dockerfile)): i = t.gettarinfo(os.path.join(root, path), arcname=path) - if sys.platform == 'win32': + if i is None: + # This happens when we encounter a socket file. We can safely + # ignore it and proceed. + continue + + if constants.IS_WINDOWS_PLATFORM: # Windows doesn't keep track of the execute bit, so we make files # and directories executable by default. i.mode = i.mode & 0o755 | 0o111 diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index cf00616d3d..71a8cc7089 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -5,6 +5,7 @@ import os import os.path import shutil +import socket import sys import tarfile import tempfile @@ -894,6 +895,20 @@ def test_tar_with_directory_symlinks(self): sorted(tar_data.getnames()), ['bar', 'bar/foo', 'foo'] ) + def test_tar_socket_file(self): + base = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, base) + for d in ['foo', 'bar']: + os.makedirs(os.path.join(base, d)) + sock = socket.socket(socket.AF_UNIX) + self.addCleanup(sock.close) + sock.bind(os.path.join(base, 'test.sock')) + with tar(base) as archive: + tar_data = tarfile.open(fileobj=archive) + self.assertEqual( + sorted(tar_data.getnames()), ['bar', 'foo'] + ) + class ShouldCheckDirectoryTest(unittest.TestCase): exclude_patterns = [ From df7e709fb661cb5f8a1a430550a9e9b7395c1190 Mon Sep 17 00:00:00 2001 From: Mehdi Bayazee Date: Wed, 25 Jan 2017 13:03:12 +0100 Subject: [PATCH 0244/1301] Remove duplicate line in exec_run documentation Signed-off-by: Mehdi Bayazee --- docker/models/containers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/docker/models/containers.py b/docker/models/containers.py index b1cdd8f870..259828a933 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -121,7 +121,6 @@ def exec_run(self, cmd, stdout=True, stderr=True, stdin=False, tty=False, user (str): User to execute command as. Default: root detach (bool): If true, detach from the exec command. Default: False - tty (bool): Allocate a pseudo-TTY. Default: False stream (bool): Stream response data. Default: False Returns: From 93877241f9c4d25967e41b050a1fe2966355ede5 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 25 Jan 2017 12:02:53 -0800 Subject: [PATCH 0245/1301] Fix ImageNotFound detection Signed-off-by: Joffrey F --- Makefile | 4 ++-- docker/errors.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 8727ada4dc..6788cf6f4e 100644 --- a/Makefile +++ b/Makefile @@ -44,7 +44,7 @@ integration-test-py3: build-py3 .PHONY: integration-dind integration-dind: build build-py3 docker rm -vf dpy-dind || : - docker run -d --name dpy-dind --privileged dockerswarm/dind:1.13.0-rc3 docker daemon\ + docker run -d --name dpy-dind --privileged dockerswarm/dind:1.13.0 docker daemon\ -H tcp://0.0.0.0:2375 docker run --rm --env="DOCKER_HOST=tcp://docker:2375" --link=dpy-dind:docker docker-sdk-python\ py.test tests/integration @@ -57,7 +57,7 @@ integration-dind-ssl: build-dind-certs build build-py3 docker run -d --name dpy-dind-certs dpy-dind-certs docker run -d --env="DOCKER_HOST=tcp://localhost:2375" --env="DOCKER_TLS_VERIFY=1"\ --env="DOCKER_CERT_PATH=/certs" --volumes-from dpy-dind-certs --name dpy-dind-ssl\ - -v /tmp --privileged dockerswarm/dind:1.13.0-rc3 docker daemon --tlsverify\ + -v /tmp --privileged dockerswarm/dind:1.13.0 docker daemon --tlsverify\ --tlscacert=/certs/ca.pem --tlscert=/certs/server-cert.pem\ --tlskey=/certs/server-key.pem -H tcp://0.0.0.0:2375 docker run --rm --volumes-from dpy-dind-ssl --env="DOCKER_HOST=tcp://docker:2375"\ diff --git a/docker/errors.py b/docker/errors.py index 95c462b9d2..d9b197d1a3 100644 --- a/docker/errors.py +++ b/docker/errors.py @@ -22,7 +22,7 @@ def create_api_error_from_http_exception(e): cls = APIError if response.status_code == 404: if explanation and ('No such image' in str(explanation) or - 'not found: does not exist or no read access' + 'not found: does not exist or no pull access' in str(explanation)): cls = ImageNotFound else: From 3e5bb7b0e6ae6286451f8af8cc717bd995709c13 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 25 Jan 2017 15:28:43 -0800 Subject: [PATCH 0246/1301] Fix Swarm model init to correctly pass arguments through to init_swarm Signed-off-by: Joffrey F --- docker/models/swarm.py | 12 ++++++------ tests/integration/models_nodes_test.py | 2 +- tests/integration/models_services_test.py | 2 +- tests/integration/models_swarm_test.py | 3 ++- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/docker/models/swarm.py b/docker/models/swarm.py index adfc51d920..d3d07ee711 100644 --- a/docker/models/swarm.py +++ b/docker/models/swarm.py @@ -29,7 +29,7 @@ def version(self): return self.attrs.get('Version').get('Index') def init(self, advertise_addr=None, listen_addr='0.0.0.0:2377', - force_new_cluster=False, swarm_spec=None, **kwargs): + force_new_cluster=False, **kwargs): """ Initialize a new swarm on this Engine. @@ -87,11 +87,11 @@ def init(self, advertise_addr=None, listen_addr='0.0.0.0:2377', ) """ - init_kwargs = {} - for arg in ['advertise_addr', 'listen_addr', 'force_new_cluster']: - if arg in kwargs: - init_kwargs[arg] = kwargs[arg] - del kwargs[arg] + init_kwargs = { + 'advertise_addr': advertise_addr, + 'listen_addr': listen_addr, + 'force_new_cluster': force_new_cluster + } init_kwargs['swarm_spec'] = SwarmSpec(**kwargs) self.client.api.init_swarm(**init_kwargs) self.reload() diff --git a/tests/integration/models_nodes_test.py b/tests/integration/models_nodes_test.py index 9fd16593ac..b3aba805ac 100644 --- a/tests/integration/models_nodes_test.py +++ b/tests/integration/models_nodes_test.py @@ -14,7 +14,7 @@ def tearDown(self): def test_list_get_update(self): client = docker.from_env() - client.swarm.init(listen_addr=helpers.swarm_listen_addr()) + client.swarm.init('eth0', listen_addr=helpers.swarm_listen_addr()) nodes = client.nodes.list() assert len(nodes) == 1 assert nodes[0].attrs['Spec']['Role'] == 'manager' diff --git a/tests/integration/models_services_test.py b/tests/integration/models_services_test.py index a795df9841..27979ddb76 100644 --- a/tests/integration/models_services_test.py +++ b/tests/integration/models_services_test.py @@ -11,7 +11,7 @@ class ServiceTest(unittest.TestCase): def setUpClass(cls): client = docker.from_env() helpers.force_leave_swarm(client) - client.swarm.init(listen_addr=helpers.swarm_listen_addr()) + client.swarm.init('eth0', listen_addr=helpers.swarm_listen_addr()) @classmethod def tearDownClass(cls): diff --git a/tests/integration/models_swarm_test.py b/tests/integration/models_swarm_test.py index 4f177f1005..2808b45f40 100644 --- a/tests/integration/models_swarm_test.py +++ b/tests/integration/models_swarm_test.py @@ -15,7 +15,8 @@ def tearDown(self): def test_init_update_leave(self): client = docker.from_env() client.swarm.init( - snapshot_interval=5000, listen_addr=helpers.swarm_listen_addr() + advertise_addr='eth0', snapshot_interval=5000, + listen_addr=helpers.swarm_listen_addr() ) assert client.swarm.attrs['Spec']['Raft']['SnapshotInterval'] == 5000 client.swarm.update(snapshot_interval=10000) From 848b7aa6a4721a58831d3d9c611cc261217a1b4b Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 25 Jan 2017 16:45:59 -0800 Subject: [PATCH 0247/1301] Add support for auto_remove in HostConfig Signed-off-by: Joffrey F --- docker/api/container.py | 2 ++ docker/models/containers.py | 2 ++ docker/types/containers.py | 7 ++++++- tests/integration/api_container_test.py | 12 ++++++++++++ 4 files changed, 22 insertions(+), 1 deletion(-) diff --git a/docker/api/container.py b/docker/api/container.py index efcae9b0c6..482b7b64cb 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -457,6 +457,8 @@ def create_host_config(self, *args, **kwargs): :py:meth:`create_container`. Args: + auto_remove (bool): enable auto-removal of the container on daemon + side when the container's process exits. binds (dict): Volumes to bind. See :py:meth:`create_container` for more information. blkio_weight_device: Block IO weight (relative device weight) in diff --git a/docker/models/containers.py b/docker/models/containers.py index 259828a933..6acc4bb85a 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -446,6 +446,8 @@ def run(self, image, command=None, stdout=True, stderr=False, Args: image (str): The image to run. command (str or list): The command to run in the container. + auto_remove (bool): enable auto-removal of the container on daemon + side when the container's process exits. blkio_weight_device: Block IO weight (relative device weight) in the form of: ``[{"Path": "device_path", "Weight": weight}]``. blkio_weight: Block IO weight (relative weight), accepts a weight diff --git a/docker/types/containers.py b/docker/types/containers.py index 8fdecb3e3d..7e7d9eaa3b 100644 --- a/docker/types/containers.py +++ b/docker/types/containers.py @@ -117,7 +117,7 @@ def __init__(self, version, binds=None, port_bindings=None, oom_kill_disable=False, shm_size=None, sysctls=None, tmpfs=None, oom_score_adj=None, dns_opt=None, cpu_shares=None, cpuset_cpus=None, userns_mode=None, pids_limit=None, - isolation=None): + isolation=None, auto_remove=False): if mem_limit is not None: self['Memory'] = parse_bytes(mem_limit) @@ -407,6 +407,11 @@ def __init__(self, version, binds=None, port_bindings=None, raise host_config_version_error('isolation', '1.24') self['Isolation'] = isolation + if auto_remove: + if version_lt(version, '1.25'): + raise host_config_version_error('auto_remove', '1.25') + self['AutoRemove'] = auto_remove + def host_config_type_error(param, param_value, expected): error_msg = 'Invalid type for {0} param: expected {1} but found {2}' diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index bebadb71b5..fc748f1c8b 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -401,6 +401,18 @@ def test_create_with_isolation(self): config = self.client.inspect_container(container) assert config['HostConfig']['Isolation'] == 'default' + @requires_api_version('1.25') + def test_create_with_auto_remove(self): + host_config = self.client.create_host_config( + auto_remove=True + ) + container = self.client.create_container( + BUSYBOX, ['echo', 'test'], host_config=host_config + ) + self.tmp_containers.append(container['Id']) + config = self.client.inspect_container(container) + assert config['HostConfig']['AutoRemove'] is True + class VolumeBindTest(BaseAPIIntegrationTest): def setUp(self): From a1d550a14c188dd9ce30d6344e35c5e48ad8f75d Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 25 Jan 2017 16:31:43 -0800 Subject: [PATCH 0248/1301] Allow configuring API version for integration test with env var Signed-off-by: Joffrey F --- Jenkinsfile | 13 +++++- Makefile | 12 +++--- tests/integration/api_client_test.py | 10 +++-- tests/integration/base.py | 8 +++- tests/integration/client_test.py | 8 ++-- tests/integration/conftest.py | 2 +- tests/integration/models_containers_test.py | 44 ++++++++++----------- tests/integration/models_images_test.py | 14 +++---- tests/integration/models_networks_test.py | 10 ++--- tests/integration/models_nodes_test.py | 7 ++-- tests/integration/models_resources_test.py | 4 +- tests/integration/models_services_test.py | 15 +++---- tests/integration/models_swarm_test.py | 7 ++-- tests/integration/models_volumes_test.py | 6 +-- 14 files changed, 90 insertions(+), 70 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 91bb2382c1..bc4cc06dc0 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -34,6 +34,13 @@ def buildImages = { -> } } +def getAPIVersion = { engineVersion -> + def versionMap = ['1.12': '1.24', '1.13': '1.25'] + + engineVersion = engineVersion.substring(0, 4) + return versionMap[engineVersion] +} + def runTests = { Map settings -> def dockerVersion = settings.get("dockerVersion", null) def pythonVersion = settings.get("pythonVersion", null) @@ -53,8 +60,9 @@ def runTests = { Map settings -> wrappedNode(label: "ubuntu && !zfs && amd64", cleanWorkspace: true) { stage("test python=${pythonVersion} / docker=${dockerVersion}") { checkout(scm) - def dindContainerName = "dpy-dind-\$BUILD_NUMBER-\$EXECUTOR_NUMBER" - def testContainerName = "dpy-tests-\$BUILD_NUMBER-\$EXECUTOR_NUMBER" + def dindContainerName = "dpy-dind-\$BUILD_NUMBER-\$EXECUTOR_NUMBER-${pythonVersion}-${dockerVersion}" + def testContainerName = "dpy-tests-\$BUILD_NUMBER-\$EXECUTOR_NUMBER-${pythonVersion}-${dockerVersion}" + def apiVersion = getAPIVersion(dockerVersion) try { sh """docker run -d --name ${dindContainerName} -v /tmp --privileged \\ dockerswarm/dind:${dockerVersion} docker daemon -H tcp://0.0.0.0:2375 @@ -62,6 +70,7 @@ def runTests = { Map settings -> sh """docker run \\ --name ${testContainerName} --volumes-from ${dindContainerName} \\ -e 'DOCKER_HOST=tcp://docker:2375' \\ + -e 'DOCKER_TEST_API_VERSION=${apiVersion}' \\ --link=${dindContainerName}:docker \\ ${testImage} \\ py.test -v -rxs tests/integration diff --git a/Makefile b/Makefile index 6788cf6f4e..148c50a4c0 100644 --- a/Makefile +++ b/Makefile @@ -46,10 +46,10 @@ integration-dind: build build-py3 docker rm -vf dpy-dind || : docker run -d --name dpy-dind --privileged dockerswarm/dind:1.13.0 docker daemon\ -H tcp://0.0.0.0:2375 - docker run --rm --env="DOCKER_HOST=tcp://docker:2375" --link=dpy-dind:docker docker-sdk-python\ - py.test tests/integration - docker run --rm --env="DOCKER_HOST=tcp://docker:2375" --link=dpy-dind:docker docker-sdk-python3\ - py.test tests/integration + docker run --rm --env="DOCKER_HOST=tcp://docker:2375" --env="DOCKER_TEST_API_VERSION=1.25"\ + --link=dpy-dind:docker docker-sdk-python py.test tests/integration + docker run --rm --env="DOCKER_HOST=tcp://docker:2375" --env="DOCKER_TEST_API_VERSION=1.25"\ + --link=dpy-dind:docker docker-sdk-python3 py.test tests/integration docker rm -vf dpy-dind .PHONY: integration-dind-ssl @@ -61,10 +61,10 @@ integration-dind-ssl: build-dind-certs build build-py3 --tlscacert=/certs/ca.pem --tlscert=/certs/server-cert.pem\ --tlskey=/certs/server-key.pem -H tcp://0.0.0.0:2375 docker run --rm --volumes-from dpy-dind-ssl --env="DOCKER_HOST=tcp://docker:2375"\ - --env="DOCKER_TLS_VERIFY=1" --env="DOCKER_CERT_PATH=/certs"\ + --env="DOCKER_TLS_VERIFY=1" --env="DOCKER_CERT_PATH=/certs" --env="DOCKER_TEST_API_VERSION=1.25"\ --link=dpy-dind-ssl:docker docker-sdk-python py.test tests/integration docker run --rm --volumes-from dpy-dind-ssl --env="DOCKER_HOST=tcp://docker:2375"\ - --env="DOCKER_TLS_VERIFY=1" --env="DOCKER_CERT_PATH=/certs"\ + --env="DOCKER_TLS_VERIFY=1" --env="DOCKER_CERT_PATH=/certs" --env="DOCKER_TEST_API_VERSION=1.25"\ --link=dpy-dind-ssl:docker docker-sdk-python3 py.test tests/integration docker rm -vf dpy-dind-ssl dpy-dind-certs diff --git a/tests/integration/api_client_test.py b/tests/integration/api_client_test.py index dab8ddf382..8f6a375790 100644 --- a/tests/integration/api_client_test.py +++ b/tests/integration/api_client_test.py @@ -25,8 +25,7 @@ def test_info(self): self.assertIn('Debug', res) def test_search(self): - client = docker.APIClient(timeout=10, **kwargs_from_env()) - res = client.search('busybox') + res = self.client.search('busybox') self.assertTrue(len(res) >= 1) base_img = [x for x in res if x['name'] == 'busybox'] self.assertEqual(len(base_img), 1) @@ -126,8 +125,11 @@ def test_client_init(self): class ConnectionTimeoutTest(unittest.TestCase): def setUp(self): self.timeout = 0.5 - self.client = docker.api.APIClient(base_url='http://192.168.10.2:4243', - timeout=self.timeout) + self.client = docker.api.APIClient( + version=docker.constants.MINIMUM_DOCKER_API_VERSION, + base_url='http://192.168.10.2:4243', + timeout=self.timeout + ) def test_timeout(self): start = time.time() diff --git a/tests/integration/base.py b/tests/integration/base.py index 4a41e6b81a..f0f5a910fe 100644 --- a/tests/integration/base.py +++ b/tests/integration/base.py @@ -1,3 +1,4 @@ +import os import shutil import unittest @@ -8,6 +9,7 @@ from .. import helpers BUSYBOX = 'busybox:buildroot-2014.02' +TEST_API_VERSION = os.environ.get('DOCKER_TEST_API_VERSION') class BaseIntegrationTest(unittest.TestCase): @@ -27,7 +29,7 @@ def setUp(self): self.tmp_networks = [] def tearDown(self): - client = docker.from_env() + client = docker.from_env(version=TEST_API_VERSION) for img in self.tmp_imgs: try: client.api.remove_image(img) @@ -61,7 +63,9 @@ class BaseAPIIntegrationTest(BaseIntegrationTest): def setUp(self): super(BaseAPIIntegrationTest, self).setUp() - self.client = docker.APIClient(timeout=60, **kwargs_from_env()) + self.client = docker.APIClient( + version=TEST_API_VERSION, timeout=60, **kwargs_from_env() + ) def run_container(self, *args, **kwargs): container = self.client.create_container(*args, **kwargs) diff --git a/tests/integration/client_test.py b/tests/integration/client_test.py index dfced9b66f..20e8cd55e7 100644 --- a/tests/integration/client_test.py +++ b/tests/integration/client_test.py @@ -2,19 +2,21 @@ import docker +from .base import TEST_API_VERSION + class ClientTest(unittest.TestCase): def test_info(self): - client = docker.from_env() + client = docker.from_env(version=TEST_API_VERSION) info = client.info() assert 'ID' in info assert 'Name' in info def test_ping(self): - client = docker.from_env() + client = docker.from_env(version=TEST_API_VERSION) assert client.ping() is True def test_version(self): - client = docker.from_env() + client = docker.from_env(version=TEST_API_VERSION) assert 'Version' in client.version() diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 7217fe07a3..4e8d26831d 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -13,7 +13,7 @@ @pytest.fixture(autouse=True, scope='session') def setup_test_session(): warnings.simplefilter('error') - c = docker.APIClient(**kwargs_from_env()) + c = docker.APIClient(version='auto', **kwargs_from_env()) try: c.inspect_image(BUSYBOX) except docker.errors.NotFound: diff --git a/tests/integration/models_containers_test.py b/tests/integration/models_containers_test.py index d8b4c62c35..d0f87d6023 100644 --- a/tests/integration/models_containers_test.py +++ b/tests/integration/models_containers_test.py @@ -1,25 +1,25 @@ import docker -from .base import BaseIntegrationTest +from .base import BaseIntegrationTest, TEST_API_VERSION class ContainerCollectionTest(BaseIntegrationTest): def test_run(self): - client = docker.from_env() + client = docker.from_env(version=TEST_API_VERSION) self.assertEqual( client.containers.run("alpine", "echo hello world", remove=True), b'hello world\n' ) def test_run_detach(self): - client = docker.from_env() + client = docker.from_env(version=TEST_API_VERSION) container = client.containers.run("alpine", "sleep 300", detach=True) self.tmp_containers.append(container.id) assert container.attrs['Config']['Image'] == "alpine" assert container.attrs['Config']['Cmd'] == ['sleep', '300'] def test_run_with_error(self): - client = docker.from_env() + client = docker.from_env(version=TEST_API_VERSION) with self.assertRaises(docker.errors.ContainerError) as cm: client.containers.run("alpine", "cat /test", remove=True) assert cm.exception.exit_status == 1 @@ -28,19 +28,19 @@ def test_run_with_error(self): assert "No such file or directory" in str(cm.exception) def test_run_with_image_that_does_not_exist(self): - client = docker.from_env() + client = docker.from_env(version=TEST_API_VERSION) with self.assertRaises(docker.errors.ImageNotFound): client.containers.run("dockerpytest_does_not_exist") def test_get(self): - client = docker.from_env() + client = docker.from_env(version=TEST_API_VERSION) container = client.containers.run("alpine", "sleep 300", detach=True) self.tmp_containers.append(container.id) assert client.containers.get(container.id).attrs[ 'Config']['Image'] == "alpine" def test_list(self): - client = docker.from_env() + client = docker.from_env(version=TEST_API_VERSION) container_id = client.containers.run( "alpine", "sleep 300", detach=True).id self.tmp_containers.append(container_id) @@ -59,7 +59,7 @@ def test_list(self): class ContainerTest(BaseIntegrationTest): def test_commit(self): - client = docker.from_env() + client = docker.from_env(version=TEST_API_VERSION) container = client.containers.run( "alpine", "sh -c 'echo \"hello\" > /test'", detach=True @@ -73,14 +73,14 @@ def test_commit(self): ) def test_diff(self): - client = docker.from_env() + client = docker.from_env(version=TEST_API_VERSION) container = client.containers.run("alpine", "touch /test", detach=True) self.tmp_containers.append(container.id) container.wait() assert container.diff() == [{'Path': '/test', 'Kind': 1}] def test_exec_run(self): - client = docker.from_env() + client = docker.from_env(version=TEST_API_VERSION) container = client.containers.run( "alpine", "sh -c 'echo \"hello\" > /test; sleep 60'", detach=True ) @@ -88,7 +88,7 @@ def test_exec_run(self): assert container.exec_run("cat /test") == b"hello\n" def test_kill(self): - client = docker.from_env() + client = docker.from_env(version=TEST_API_VERSION) container = client.containers.run("alpine", "sleep 300", detach=True) self.tmp_containers.append(container.id) while container.status != 'running': @@ -99,7 +99,7 @@ def test_kill(self): assert container.status == 'exited' def test_logs(self): - client = docker.from_env() + client = docker.from_env(version=TEST_API_VERSION) container = client.containers.run("alpine", "echo hello world", detach=True) self.tmp_containers.append(container.id) @@ -107,7 +107,7 @@ def test_logs(self): assert container.logs() == b"hello world\n" def test_pause(self): - client = docker.from_env() + client = docker.from_env(version=TEST_API_VERSION) container = client.containers.run("alpine", "sleep 300", detach=True) self.tmp_containers.append(container.id) container.pause() @@ -118,7 +118,7 @@ def test_pause(self): assert container.status == "running" def test_remove(self): - client = docker.from_env() + client = docker.from_env(version=TEST_API_VERSION) container = client.containers.run("alpine", "echo hello", detach=True) self.tmp_containers.append(container.id) assert container.id in [c.id for c in client.containers.list(all=True)] @@ -128,7 +128,7 @@ def test_remove(self): assert container.id not in [c.id for c in containers] def test_rename(self): - client = docker.from_env() + client = docker.from_env(version=TEST_API_VERSION) container = client.containers.run("alpine", "echo hello", name="test1", detach=True) self.tmp_containers.append(container.id) @@ -138,7 +138,7 @@ def test_rename(self): assert container.name == "test2" def test_restart(self): - client = docker.from_env() + client = docker.from_env(version=TEST_API_VERSION) container = client.containers.run("alpine", "sleep 100", detach=True) self.tmp_containers.append(container.id) first_started_at = container.attrs['State']['StartedAt'] @@ -148,7 +148,7 @@ def test_restart(self): assert first_started_at != second_started_at def test_start(self): - client = docker.from_env() + client = docker.from_env(version=TEST_API_VERSION) container = client.containers.create("alpine", "sleep 50", detach=True) self.tmp_containers.append(container.id) assert container.status == "created" @@ -157,7 +157,7 @@ def test_start(self): assert container.status == "running" def test_stats(self): - client = docker.from_env() + client = docker.from_env(version=TEST_API_VERSION) container = client.containers.run("alpine", "sleep 100", detach=True) self.tmp_containers.append(container.id) stats = container.stats(stream=False) @@ -166,7 +166,7 @@ def test_stats(self): assert key in stats def test_stop(self): - client = docker.from_env() + client = docker.from_env(version=TEST_API_VERSION) container = client.containers.run("alpine", "top", detach=True) self.tmp_containers.append(container.id) assert container.status in ("running", "created") @@ -175,7 +175,7 @@ def test_stop(self): assert container.status == "exited" def test_top(self): - client = docker.from_env() + client = docker.from_env(version=TEST_API_VERSION) container = client.containers.run("alpine", "sleep 60", detach=True) self.tmp_containers.append(container.id) top = container.top() @@ -183,7 +183,7 @@ def test_top(self): assert 'sleep 60' in top['Processes'][0] def test_update(self): - client = docker.from_env() + client = docker.from_env(version=TEST_API_VERSION) container = client.containers.run("alpine", "sleep 60", detach=True, cpu_shares=2) self.tmp_containers.append(container.id) @@ -193,7 +193,7 @@ def test_update(self): assert container.attrs['HostConfig']['CpuShares'] == 3 def test_wait(self): - client = docker.from_env() + client = docker.from_env(version=TEST_API_VERSION) container = client.containers.run("alpine", "sh -c 'exit 0'", detach=True) self.tmp_containers.append(container.id) diff --git a/tests/integration/models_images_test.py b/tests/integration/models_images_test.py index 876ec292b6..4f8bb26cd5 100644 --- a/tests/integration/models_images_test.py +++ b/tests/integration/models_images_test.py @@ -3,13 +3,13 @@ import docker import pytest -from .base import BaseIntegrationTest +from .base import BaseIntegrationTest, TEST_API_VERSION class ImageCollectionTest(BaseIntegrationTest): def test_build(self): - client = docker.from_env() + client = docker.from_env(version=TEST_API_VERSION) image = client.images.build(fileobj=io.BytesIO( "FROM alpine\n" "CMD echo hello world".encode('ascii') @@ -19,7 +19,7 @@ def test_build(self): @pytest.mark.xfail(reason='Engine 1.13 responds with status 500') def test_build_with_error(self): - client = docker.from_env() + client = docker.from_env(version=TEST_API_VERSION) with self.assertRaises(docker.errors.BuildError) as cm: client.images.build(fileobj=io.BytesIO( "FROM alpine\n" @@ -29,18 +29,18 @@ def test_build_with_error(self): "NOTADOCKERFILECOMMAND") def test_list(self): - client = docker.from_env() + client = docker.from_env(version=TEST_API_VERSION) image = client.images.pull('alpine:latest') assert image.id in get_ids(client.images.list()) def test_list_with_repository(self): - client = docker.from_env() + client = docker.from_env(version=TEST_API_VERSION) image = client.images.pull('alpine:latest') assert image.id in get_ids(client.images.list('alpine')) assert image.id in get_ids(client.images.list('alpine:latest')) def test_pull(self): - client = docker.from_env() + client = docker.from_env(version=TEST_API_VERSION) image = client.images.pull('alpine:latest') assert 'alpine:latest' in image.attrs['RepoTags'] @@ -52,7 +52,7 @@ def test_tag_and_remove(self): tag = 'some-tag' identifier = '{}:{}'.format(repo, tag) - client = docker.from_env() + client = docker.from_env(version=TEST_API_VERSION) image = client.images.pull('alpine:latest') image.tag(repo, tag) diff --git a/tests/integration/models_networks_test.py b/tests/integration/models_networks_test.py index 771ee7d346..105dcc594a 100644 --- a/tests/integration/models_networks_test.py +++ b/tests/integration/models_networks_test.py @@ -1,12 +1,12 @@ import docker from .. import helpers -from .base import BaseIntegrationTest +from .base import BaseIntegrationTest, TEST_API_VERSION class ImageCollectionTest(BaseIntegrationTest): def test_create(self): - client = docker.from_env() + client = docker.from_env(version=TEST_API_VERSION) name = helpers.random_name() network = client.networks.create(name, labels={'foo': 'bar'}) self.tmp_networks.append(network.id) @@ -14,7 +14,7 @@ def test_create(self): assert network.attrs['Labels']['foo'] == "bar" def test_get(self): - client = docker.from_env() + client = docker.from_env(version=TEST_API_VERSION) name = helpers.random_name() network_id = client.networks.create(name).id self.tmp_networks.append(network_id) @@ -22,7 +22,7 @@ def test_get(self): assert network.name == name def test_list_remove(self): - client = docker.from_env() + client = docker.from_env(version=TEST_API_VERSION) name = helpers.random_name() network = client.networks.create(name) self.tmp_networks.append(network.id) @@ -50,7 +50,7 @@ def test_list_remove(self): class ImageTest(BaseIntegrationTest): def test_connect_disconnect(self): - client = docker.from_env() + client = docker.from_env(version=TEST_API_VERSION) network = client.networks.create(helpers.random_name()) self.tmp_networks.append(network.id) container = client.containers.create("alpine", "sleep 300") diff --git a/tests/integration/models_nodes_test.py b/tests/integration/models_nodes_test.py index b3aba805ac..5823e6b1a3 100644 --- a/tests/integration/models_nodes_test.py +++ b/tests/integration/models_nodes_test.py @@ -3,17 +3,18 @@ import docker from .. import helpers +from .base import TEST_API_VERSION class NodesTest(unittest.TestCase): def setUp(self): - helpers.force_leave_swarm(docker.from_env()) + helpers.force_leave_swarm(docker.from_env(version=TEST_API_VERSION)) def tearDown(self): - helpers.force_leave_swarm(docker.from_env()) + helpers.force_leave_swarm(docker.from_env(version=TEST_API_VERSION)) def test_list_get_update(self): - client = docker.from_env() + client = docker.from_env(version=TEST_API_VERSION) client.swarm.init('eth0', listen_addr=helpers.swarm_listen_addr()) nodes = client.nodes.list() assert len(nodes) == 1 diff --git a/tests/integration/models_resources_test.py b/tests/integration/models_resources_test.py index b8eba81c6e..4aafe0cc74 100644 --- a/tests/integration/models_resources_test.py +++ b/tests/integration/models_resources_test.py @@ -1,11 +1,11 @@ import docker -from .base import BaseIntegrationTest +from .base import BaseIntegrationTest, TEST_API_VERSION class ModelTest(BaseIntegrationTest): def test_reload(self): - client = docker.from_env() + client = docker.from_env(version=TEST_API_VERSION) container = client.containers.run("alpine", "sleep 300", detach=True) self.tmp_containers.append(container.id) first_started_at = container.attrs['State']['StartedAt'] diff --git a/tests/integration/models_services_test.py b/tests/integration/models_services_test.py index 27979ddb76..9b5676d694 100644 --- a/tests/integration/models_services_test.py +++ b/tests/integration/models_services_test.py @@ -4,21 +4,22 @@ import pytest from .. import helpers +from .base import TEST_API_VERSION class ServiceTest(unittest.TestCase): @classmethod def setUpClass(cls): - client = docker.from_env() + client = docker.from_env(version=TEST_API_VERSION) helpers.force_leave_swarm(client) client.swarm.init('eth0', listen_addr=helpers.swarm_listen_addr()) @classmethod def tearDownClass(cls): - helpers.force_leave_swarm(docker.from_env()) + helpers.force_leave_swarm(docker.from_env(version=TEST_API_VERSION)) def test_create(self): - client = docker.from_env() + client = docker.from_env(version=TEST_API_VERSION) name = helpers.random_name() service = client.services.create( # create arguments @@ -36,7 +37,7 @@ def test_create(self): assert container_spec['Labels'] == {'container': 'label'} def test_get(self): - client = docker.from_env() + client = docker.from_env(version=TEST_API_VERSION) name = helpers.random_name() service = client.services.create( name=name, @@ -47,7 +48,7 @@ def test_get(self): assert service.name == name def test_list_remove(self): - client = docker.from_env() + client = docker.from_env(version=TEST_API_VERSION) service = client.services.create( name=helpers.random_name(), image="alpine", @@ -58,7 +59,7 @@ def test_list_remove(self): assert service not in client.services.list() def test_tasks(self): - client = docker.from_env() + client = docker.from_env(version=TEST_API_VERSION) service1 = client.services.create( name=helpers.random_name(), image="alpine", @@ -83,7 +84,7 @@ def test_tasks(self): @pytest.mark.skip(reason="Makes Swarm unstable?") def test_update(self): - client = docker.from_env() + client = docker.from_env(version=TEST_API_VERSION) service = client.services.create( # create arguments name=helpers.random_name(), diff --git a/tests/integration/models_swarm_test.py b/tests/integration/models_swarm_test.py index 2808b45f40..e45ff3cb72 100644 --- a/tests/integration/models_swarm_test.py +++ b/tests/integration/models_swarm_test.py @@ -3,17 +3,18 @@ import docker from .. import helpers +from .base import TEST_API_VERSION class SwarmTest(unittest.TestCase): def setUp(self): - helpers.force_leave_swarm(docker.from_env()) + helpers.force_leave_swarm(docker.from_env(version=TEST_API_VERSION)) def tearDown(self): - helpers.force_leave_swarm(docker.from_env()) + helpers.force_leave_swarm(docker.from_env(version=TEST_API_VERSION)) def test_init_update_leave(self): - client = docker.from_env() + client = docker.from_env(version=TEST_API_VERSION) client.swarm.init( advertise_addr='eth0', snapshot_interval=5000, listen_addr=helpers.swarm_listen_addr() diff --git a/tests/integration/models_volumes_test.py b/tests/integration/models_volumes_test.py index 094e68fadb..47b4a4550f 100644 --- a/tests/integration/models_volumes_test.py +++ b/tests/integration/models_volumes_test.py @@ -1,10 +1,10 @@ import docker -from .base import BaseIntegrationTest +from .base import BaseIntegrationTest, TEST_API_VERSION class VolumesTest(BaseIntegrationTest): def test_create_get(self): - client = docker.from_env() + client = docker.from_env(version=TEST_API_VERSION) volume = client.volumes.create( 'dockerpytest_1', driver='local', @@ -19,7 +19,7 @@ def test_create_get(self): assert volume.name == 'dockerpytest_1' def test_list_remove(self): - client = docker.from_env() + client = docker.from_env(version=TEST_API_VERSION) volume = client.volumes.create('dockerpytest_1') self.tmp_volumes.append(volume.id) assert volume in client.volumes.list() From 686d8e9536d89ac9c0259d9598fd03b65a9409e2 Mon Sep 17 00:00:00 2001 From: Thomas Schaaf Date: Mon, 9 Jan 2017 22:41:14 +0100 Subject: [PATCH 0249/1301] Implement cachefrom Signed-off-by: Thomas Schaaf --- docker/api/build.py | 11 ++++++++++- tests/integration/api_build_test.py | 18 ++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/docker/api/build.py b/docker/api/build.py index eb01bce389..c009f1a273 100644 --- a/docker/api/build.py +++ b/docker/api/build.py @@ -18,7 +18,7 @@ def build(self, path=None, tag=None, quiet=False, fileobj=None, custom_context=False, encoding=None, pull=False, forcerm=False, dockerfile=None, container_limits=None, decode=False, buildargs=None, gzip=False, shmsize=None, - labels=None): + labels=None, cachefrom=None): """ Similar to the ``docker build`` command. Either ``path`` or ``fileobj`` needs to be set. ``path`` can be a local path (to a directory @@ -92,6 +92,7 @@ def build(self, path=None, tag=None, quiet=False, fileobj=None, shmsize (int): Size of `/dev/shm` in bytes. The size must be greater than 0. If omitted the system uses 64MB. labels (dict): A dictionary of labels to set on the image. + cachefrom (list): A list of images used for build cache resolution. Returns: A generator for the build output. @@ -188,6 +189,14 @@ def build(self, path=None, tag=None, quiet=False, fileobj=None, 'labels was only introduced in API version 1.23' ) + if cachefrom: + if utils.version_gte(self._version, '1.25'): + params.update({'cachefrom': json.dumps(cachefrom)}) + else: + raise errors.InvalidVersion( + 'cachefrom was only introduced in API version 1.25' + ) + if context is not None: headers = {'Content-Type': 'application/tar'} if encoding: diff --git a/tests/integration/api_build_test.py b/tests/integration/api_build_test.py index 3dac0e932d..e7479bfb29 100644 --- a/tests/integration/api_build_test.py +++ b/tests/integration/api_build_test.py @@ -153,6 +153,24 @@ def test_build_labels(self): info = self.client.inspect_image('labels') self.assertEqual(info['Config']['Labels'], labels) + @requires_api_version('1.25') + def test_build_cachefrom(self): + script = io.BytesIO('\n'.join([ + 'FROM scratch', + ]).encode('ascii')) + + cachefrom = ['build1'] + + stream = self.client.build( + fileobj=script, tag='cachefrom', cachefrom=cachefrom + ) + self.tmp_imgs.append('cachefrom') + for chunk in stream: + pass + + info = self.client.inspect_image('cachefrom') + self.assertEqual(info['Config']['CacheFrom'], cachefrom) + def test_build_stderr_data(self): control_chars = ['\x1b[91m', '\x1b[0m'] snippet = 'Ancient Temple (Mystic Oriental Dream ~ Ancient Temple)' From 0d9adcdfdca94daa589faf10af1f123cb7d7e5b8 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Thu, 26 Jan 2017 11:22:03 +0000 Subject: [PATCH 0250/1301] Add cachefrom to build docstring Signed-off-by: Thomas Grainger --- docker/models/images.py | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/models/images.py b/docker/models/images.py index 6f8f4fe273..1265374605 100644 --- a/docker/models/images.py +++ b/docker/models/images.py @@ -140,6 +140,7 @@ def build(self, **kwargs): ``"0-3"``, ``"0,1"`` decode (bool): If set to ``True``, the returned stream will be decoded into dicts on the fly. Default ``False``. + cachefrom (list): A list of images used for build cache resolution. Returns: (:py:class:`Image`): The built image. From 62c94f9634f8f8cfb5f353aac786cae8278f96fa Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 26 Jan 2017 14:07:41 -0800 Subject: [PATCH 0251/1301] Remove integration test for APIClient.search method Signed-off-by: Joffrey F --- tests/integration/api_client_test.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/tests/integration/api_client_test.py b/tests/integration/api_client_test.py index 8f6a375790..02bb435ada 100644 --- a/tests/integration/api_client_test.py +++ b/tests/integration/api_client_test.py @@ -24,13 +24,6 @@ def test_info(self): self.assertIn('Images', res) self.assertIn('Debug', res) - def test_search(self): - res = self.client.search('busybox') - self.assertTrue(len(res) >= 1) - base_img = [x for x in res if x['name'] == 'busybox'] - self.assertEqual(len(base_img), 1) - self.assertIn('description', base_img[0]) - class LinkTest(BaseAPIIntegrationTest): def test_remove_link(self): From a24b114af3aec0430fd27f8bbf460cfa396ae274 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 25 Jan 2017 15:46:58 -0800 Subject: [PATCH 0252/1301] Add 'force' parameter in remove_volume Signed-off-by: Joffrey F --- docker/api/volume.py | 18 ++++++++++++++---- docker/models/volumes.py | 15 ++++++++++++--- tests/integration/api_volume_test.py | 7 +++++++ 3 files changed, 33 insertions(+), 7 deletions(-) diff --git a/docker/api/volume.py b/docker/api/volume.py index 9c6d5f8351..5de089e39e 100644 --- a/docker/api/volume.py +++ b/docker/api/volume.py @@ -115,17 +115,27 @@ def inspect_volume(self, name): return self._result(self._get(url), True) @utils.minimum_version('1.21') - def remove_volume(self, name): + def remove_volume(self, name, force=False): """ Remove a volume. Similar to the ``docker volume rm`` command. Args: name (str): The volume's name + force (bool): Force removal of volumes that were already removed + out of band by the volume driver plugin. Raises: - - ``docker.errors.APIError``: If volume failed to remove. + :py:class:`docker.errors.APIError` + If volume failed to remove. """ - url = self._url('/volumes/{0}', name) + params = {} + if force: + if utils.version_lt(self._version, '1.25'): + raise errors.InvalidVersion( + 'force removal was introduced in API 1.25' + ) + params = {'force': force} + + url = self._url('/volumes/{0}', name, params=params) resp = self._delete(url) self._raise_for_status(resp) diff --git a/docker/models/volumes.py b/docker/models/volumes.py index 5a31541260..5fb0d1c5b8 100644 --- a/docker/models/volumes.py +++ b/docker/models/volumes.py @@ -10,9 +10,18 @@ def name(self): """The name of the volume.""" return self.attrs['Name'] - def remove(self): - """Remove this volume.""" - return self.client.api.remove_volume(self.id) + def remove(self, force=False): + """ + Remove this volume. + + Args: + force (bool): Force removal of volumes that were already removed + out of band by the volume driver plugin. + Raises: + :py:class:`docker.errors.APIError` + If volume failed to remove. + """ + return self.client.api.remove_volume(self.id, force=force) class VolumeCollection(Collection): diff --git a/tests/integration/api_volume_test.py b/tests/integration/api_volume_test.py index bc97f462e5..4bfc672b57 100644 --- a/tests/integration/api_volume_test.py +++ b/tests/integration/api_volume_test.py @@ -49,6 +49,13 @@ def test_remove_volume(self): self.client.create_volume(name) self.client.remove_volume(name) + @requires_api_version('1.25') + def test_force_remove_volume(self): + name = 'shootthebullet' + self.tmp_volumes.append(name) + self.client.create_volume(name) + self.client.remove_volume(name, force=True) + def test_remove_nonexistent_volume(self): name = 'shootthebullet' with pytest.raises(docker.errors.NotFound): From 847f209865cb7c9234cd19a9ae8cc9a158d9a140 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 26 Jan 2017 16:14:57 -0800 Subject: [PATCH 0253/1301] Add stop_timeout to create_container Fix requires_api_version test decorator Signed-off-by: Joffrey F --- Jenkinsfile | 6 ++---- docker/api/container.py | 5 ++++- docker/types/containers.py | 7 +++++++ tests/helpers.py | 8 +++++--- tests/integration/api_build_test.py | 4 ++++ tests/integration/api_container_test.py | 9 +++++++++ 6 files changed, 31 insertions(+), 8 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index bc4cc06dc0..b8b932aed1 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -36,15 +36,14 @@ def buildImages = { -> def getAPIVersion = { engineVersion -> def versionMap = ['1.12': '1.24', '1.13': '1.25'] - - engineVersion = engineVersion.substring(0, 4) - return versionMap[engineVersion] + return versionMap[engineVersion.substring(0, 4)] } def runTests = { Map settings -> def dockerVersion = settings.get("dockerVersion", null) def pythonVersion = settings.get("pythonVersion", null) def testImage = settings.get("testImage", null) + def apiVersion = getAPIVersion(dockerVersion) if (!testImage) { throw new Exception("Need test image object, e.g.: `runTests(testImage: img)`") @@ -62,7 +61,6 @@ def runTests = { Map settings -> checkout(scm) def dindContainerName = "dpy-dind-\$BUILD_NUMBER-\$EXECUTOR_NUMBER-${pythonVersion}-${dockerVersion}" def testContainerName = "dpy-tests-\$BUILD_NUMBER-\$EXECUTOR_NUMBER-${pythonVersion}-${dockerVersion}" - def apiVersion = getAPIVersion(dockerVersion) try { sh """docker run -d --name ${dindContainerName} -v /tmp --privileged \\ dockerswarm/dind:${dockerVersion} docker daemon -H tcp://0.0.0.0:2375 diff --git a/docker/api/container.py b/docker/api/container.py index 482b7b64cb..acb0ffa8bf 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -238,7 +238,7 @@ def create_container(self, image, command=None, hostname=None, user=None, memswap_limit=None, cpuset=None, host_config=None, mac_address=None, labels=None, volume_driver=None, stop_signal=None, networking_config=None, - healthcheck=None): + healthcheck=None, stop_timeout=None): """ Creates a container. Parameters are similar to those for the ``docker run`` command except it doesn't support the attach options (``-a``). @@ -411,6 +411,8 @@ def create_container(self, image, command=None, hostname=None, user=None, volume_driver (str): The name of a volume driver/plugin. stop_signal (str): The stop signal to use to stop the container (e.g. ``SIGINT``). + stop_timeout (int): Timeout to stop the container, in seconds. + Default: 10 networking_config (dict): A networking configuration generated by :py:meth:`create_networking_config`. @@ -437,6 +439,7 @@ def create_container(self, image, command=None, hostname=None, user=None, network_disabled, entrypoint, cpu_shares, working_dir, domainname, memswap_limit, cpuset, host_config, mac_address, labels, volume_driver, stop_signal, networking_config, healthcheck, + stop_timeout ) return self.create_container_from_config(config, name) diff --git a/docker/types/containers.py b/docker/types/containers.py index 7e7d9eaa3b..3c0e41e0d7 100644 --- a/docker/types/containers.py +++ b/docker/types/containers.py @@ -438,6 +438,7 @@ def __init__( working_dir=None, domainname=None, memswap_limit=None, cpuset=None, host_config=None, mac_address=None, labels=None, volume_driver=None, stop_signal=None, networking_config=None, healthcheck=None, + stop_timeout=None ): if isinstance(command, six.string_types): command = split_command(command) @@ -466,6 +467,11 @@ def __init__( 'stop_signal was only introduced in API version 1.21' ) + if stop_timeout is not None and version_lt(version, '1.25'): + raise errors.InvalidVersion( + 'stop_timeout was only introduced in API version 1.25' + ) + if healthcheck is not None and version_lt(version, '1.24'): raise errors.InvalidVersion( 'Health options were only introduced in API version 1.24' @@ -584,4 +590,5 @@ def __init__( 'VolumeDriver': volume_driver, 'StopSignal': stop_signal, 'Healthcheck': healthcheck, + 'StopTimeout': stop_timeout }) diff --git a/tests/helpers.py b/tests/helpers.py index 1e42363144..b742c960cb 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -43,10 +43,12 @@ def untar_file(tardata, filename): def requires_api_version(version): + test_version = os.environ.get( + 'DOCKER_TEST_API_VERSION', docker.constants.DEFAULT_DOCKER_API_VERSION + ) + return pytest.mark.skipif( - docker.utils.version_lt( - docker.constants.DEFAULT_DOCKER_API_VERSION, version - ), + docker.utils.version_lt(test_version, version), reason="API version is too low (< {0})".format(version) ) diff --git a/tests/integration/api_build_test.py b/tests/integration/api_build_test.py index e7479bfb29..c2fd26c1f3 100644 --- a/tests/integration/api_build_test.py +++ b/tests/integration/api_build_test.py @@ -3,6 +3,7 @@ import shutil import tempfile +import pytest import six from docker import errors @@ -154,9 +155,11 @@ def test_build_labels(self): self.assertEqual(info['Config']['Labels'], labels) @requires_api_version('1.25') + @pytest.mark.xfail(reason='Bad test') def test_build_cachefrom(self): script = io.BytesIO('\n'.join([ 'FROM scratch', + 'CMD sh -c "echo \'Hello, World!\'"', ]).encode('ascii')) cachefrom = ['build1'] @@ -169,6 +172,7 @@ def test_build_cachefrom(self): pass info = self.client.inspect_image('cachefrom') + # FIXME: Config.CacheFrom is not a real thing self.assertEqual(info['Config']['CacheFrom'], cachefrom) def test_build_stderr_data(self): diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index fc748f1c8b..3cede45de2 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -413,6 +413,15 @@ def test_create_with_auto_remove(self): config = self.client.inspect_container(container) assert config['HostConfig']['AutoRemove'] is True + @requires_api_version('1.25') + def test_create_with_stop_timeout(self): + container = self.client.create_container( + BUSYBOX, ['echo', 'test'], stop_timeout=25 + ) + self.tmp_containers.append(container['Id']) + config = self.client.inspect_container(container) + assert config['Config']['StopTimeout'] == 25 + class VolumeBindTest(BaseAPIIntegrationTest): def setUp(self): From fc5cd1a914868716549e42f1d0ac57b012b9e271 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 27 Jan 2017 16:19:26 -0800 Subject: [PATCH 0254/1301] Add support for max_failure_ratio and monitor in UpdateConfig Signed-off-by: Joffrey F --- docker/types/services.py | 22 +++++++++++++++++++++- tests/integration/api_service_test.py | 17 +++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/docker/types/services.py b/docker/types/services.py index ec0fcb15f0..51b6e0d06d 100644 --- a/docker/types/services.py +++ b/docker/types/services.py @@ -233,8 +233,14 @@ class UpdateConfig(dict): failure_action (string): Action to take if an updated task fails to run, or stops running during the update. Acceptable values are ``continue`` and ``pause``. Default: ``continue`` + monitor (int): Amount of time to monitor each updated task for + failures, in nanoseconds. + max_failure_ratio (float): The fraction of tasks that may fail during + an update before the failure action is invoked, specified as a + floating point number between 0 and 1. Default: 0 """ - def __init__(self, parallelism=0, delay=None, failure_action='continue'): + def __init__(self, parallelism=0, delay=None, failure_action='continue', + monitor=None, max_failure_ratio=None): self['Parallelism'] = parallelism if delay is not None: self['Delay'] = delay @@ -244,6 +250,20 @@ def __init__(self, parallelism=0, delay=None, failure_action='continue'): ) self['FailureAction'] = failure_action + if monitor is not None: + if not isinstance(monitor, int): + raise TypeError('monitor must be an integer') + self['Monitor'] = monitor + + if max_failure_ratio is not None: + if not isinstance(max_failure_ratio, (float, int)): + raise TypeError('max_failure_ratio must be a float') + if max_failure_ratio > 1 or max_failure_ratio < 0: + raise errors.InvalidArgument( + 'max_failure_ratio must be a number between 0 and 1' + ) + self['MaxFailureRatio'] = max_failure_ratio + class RestartConditionTypesEnum(object): _values = ( diff --git a/tests/integration/api_service_test.py b/tests/integration/api_service_test.py index 77d7d28f7e..f4656d431c 100644 --- a/tests/integration/api_service_test.py +++ b/tests/integration/api_service_test.py @@ -155,6 +155,23 @@ def test_create_service_with_update_config(self): assert update_config['Delay'] == uc['Delay'] assert update_config['FailureAction'] == uc['FailureAction'] + @requires_api_version('1.25') + def test_create_service_with_update_config_monitor(self): + container_spec = docker.types.ContainerSpec('busybox', ['true']) + task_tmpl = docker.types.TaskTemplate(container_spec) + update_config = docker.types.UpdateConfig( + monitor=300000000, max_failure_ratio=0.4 + ) + name = self.get_service_name() + svc_id = self.client.create_service( + task_tmpl, update_config=update_config, name=name + ) + svc_info = self.client.inspect_service(svc_id) + assert 'UpdateConfig' in svc_info['Spec'] + uc = svc_info['Spec']['UpdateConfig'] + assert update_config['Monitor'] == uc['Monitor'] + assert update_config['MaxFailureRatio'] == uc['MaxFailureRatio'] + def test_create_service_with_restart_policy(self): container_spec = docker.types.ContainerSpec('busybox', ['true']) policy = docker.types.RestartPolicy( From d22e2fec6ff7156fcd533467c7aff95d2e3e1d13 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 27 Jan 2017 17:19:18 -0800 Subject: [PATCH 0255/1301] Add support for force_update in TaskTemplate Add min version checks in create_service and update_service Signed-off-by: Joffrey F --- docker/api/service.py | 34 ++++++++++++++++++++++++++- docker/types/services.py | 9 ++++++- tests/helpers.py | 4 +++- tests/integration/api_service_test.py | 27 ++++++++++++++++++--- tests/integration/api_swarm_test.py | 6 ++--- 5 files changed, 71 insertions(+), 9 deletions(-) diff --git a/docker/api/service.py b/docker/api/service.py index d2621e685c..0b2abdc9af 100644 --- a/docker/api/service.py +++ b/docker/api/service.py @@ -62,10 +62,24 @@ def create_service( 'Labels': labels, 'TaskTemplate': task_template, 'Mode': mode, - 'UpdateConfig': update_config, 'Networks': utils.convert_service_networks(networks), 'EndpointSpec': endpoint_spec } + + if update_config is not None: + if utils.version_lt(self._version, '1.25'): + if 'MaxFailureRatio' in update_config: + raise errors.InvalidVersion( + 'UpdateConfig.max_failure_ratio is not supported in' + ' API version < 1.25' + ) + if 'Monitor' in update_config: + raise errors.InvalidVersion( + 'UpdateConfig.monitor is not supported in' + ' API version < 1.25' + ) + data['UpdateConfig'] = update_config + return self._result( self._post_json(url, data=data, headers=headers), True ) @@ -230,6 +244,12 @@ def update_service(self, service, version, task_template=None, name=None, mode = ServiceMode(mode) data['Mode'] = mode if task_template is not None: + if 'ForceUpdate' in task_template and utils.version_lt( + self._version, '1.25'): + raise errors.InvalidVersion( + 'force_update is not supported in API version < 1.25' + ) + image = task_template.get('ContainerSpec', {}).get('Image', None) if image is not None: registry, repo_name = auth.resolve_repository_name(image) @@ -238,7 +258,19 @@ def update_service(self, service, version, task_template=None, name=None, headers['X-Registry-Auth'] = auth_header data['TaskTemplate'] = task_template if update_config is not None: + if utils.version_lt(self._version, '1.25'): + if 'MaxFailureRatio' in update_config: + raise errors.InvalidVersion( + 'UpdateConfig.max_failure_ratio is not supported in' + ' API version < 1.25' + ) + if 'Monitor' in update_config: + raise errors.InvalidVersion( + 'UpdateConfig.monitor is not supported in' + ' API version < 1.25' + ) data['UpdateConfig'] = update_config + if networks is not None: data['Networks'] = utils.convert_service_networks(networks) if endpoint_spec is not None: diff --git a/docker/types/services.py b/docker/types/services.py index 51b6e0d06d..5f7b2fb0d0 100644 --- a/docker/types/services.py +++ b/docker/types/services.py @@ -21,9 +21,11 @@ class TaskTemplate(dict): restart_policy (RestartPolicy): Specification for the restart policy which applies to containers created as part of this service. placement (:py:class:`list`): A list of constraints. + force_update (int): A counter that triggers an update even if no + relevant parameters have been changed. """ def __init__(self, container_spec, resources=None, restart_policy=None, - placement=None, log_driver=None): + placement=None, log_driver=None, force_update=None): self['ContainerSpec'] = container_spec if resources: self['Resources'] = resources @@ -36,6 +38,11 @@ def __init__(self, container_spec, resources=None, restart_policy=None, if log_driver: self['LogDriver'] = log_driver + if force_update is not None: + if not isinstance(force_update, int): + raise TypeError('force_update must be an integer') + self['ForceUpdate'] = force_update + @property def container_spec(self): return self.get('ContainerSpec') diff --git a/tests/helpers.py b/tests/helpers.py index b742c960cb..e8ba4d6bf9 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -70,7 +70,9 @@ def force_leave_swarm(client): occasionally throws "context deadline exceeded" errors when leaving.""" while True: try: - return client.swarm.leave(force=True) + if isinstance(client, docker.DockerClient): + return client.swarm.leave(force=True) + return client.leave_swarm(force=True) # elif APIClient except docker.errors.APIError as e: if e.explanation == "context deadline exceeded": continue diff --git a/tests/integration/api_service_test.py b/tests/integration/api_service_test.py index f4656d431c..46b0a79e6e 100644 --- a/tests/integration/api_service_test.py +++ b/tests/integration/api_service_test.py @@ -2,14 +2,14 @@ import docker -from ..helpers import requires_api_version +from ..helpers import force_leave_swarm, requires_api_version from .base import BaseAPIIntegrationTest class ServiceTest(BaseAPIIntegrationTest): def setUp(self): super(ServiceTest, self).setUp() - self.client.leave_swarm(force=True) + force_leave_swarm(self.client) self.init_swarm() def tearDown(self): @@ -19,7 +19,7 @@ def tearDown(self): self.client.remove_service(service['ID']) except docker.errors.APIError: pass - self.client.leave_swarm(force=True) + force_leave_swarm(self.client) def get_service_name(self): return 'dockerpytest_{0:x}'.format(random.getrandbits(64)) @@ -296,3 +296,24 @@ def test_create_service_replicated_mode(self): assert 'Mode' in svc_info['Spec'] assert 'Replicated' in svc_info['Spec']['Mode'] assert svc_info['Spec']['Mode']['Replicated'] == {'Replicas': 5} + + @requires_api_version('1.25') + def test_update_service_force_update(self): + container_spec = docker.types.ContainerSpec( + 'busybox', ['echo', 'hello'] + ) + task_tmpl = docker.types.TaskTemplate(container_spec) + name = self.get_service_name() + svc_id = self.client.create_service(task_tmpl, name=name) + svc_info = self.client.inspect_service(svc_id) + assert 'TaskTemplate' in svc_info['Spec'] + assert 'ForceUpdate' in svc_info['Spec']['TaskTemplate'] + assert svc_info['Spec']['TaskTemplate']['ForceUpdate'] == 0 + version_index = svc_info['Version']['Index'] + + task_tmpl = docker.types.TaskTemplate(container_spec, force_update=10) + self.client.update_service(name, version_index, task_tmpl, name=name) + svc_info = self.client.inspect_service(svc_id) + new_index = svc_info['Version']['Index'] + assert new_index > version_index + assert svc_info['Spec']['TaskTemplate']['ForceUpdate'] == 10 diff --git a/tests/integration/api_swarm_test.py b/tests/integration/api_swarm_test.py index a8f439c8b5..d06cac21bd 100644 --- a/tests/integration/api_swarm_test.py +++ b/tests/integration/api_swarm_test.py @@ -2,18 +2,18 @@ import docker import pytest -from ..helpers import requires_api_version +from ..helpers import force_leave_swarm, requires_api_version from .base import BaseAPIIntegrationTest class SwarmTest(BaseAPIIntegrationTest): def setUp(self): super(SwarmTest, self).setUp() - self.client.leave_swarm(force=True) + force_leave_swarm(self.client) def tearDown(self): super(SwarmTest, self).tearDown() - self.client.leave_swarm(force=True) + force_leave_swarm(self.client) @requires_api_version('1.24') def test_init_swarm_simple(self): From 00de2055f93af28c3911b349786c0faa41908e1c Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Wed, 14 Dec 2016 11:57:19 +0000 Subject: [PATCH 0256/1301] Change "Remote API" to "Engine API" This is currently inconsistent, but mostly called "Engine API". For the release of Docker 1.13, this will be "Engine API" all over the Engine documentation, too. Signed-off-by: Ben Firshman --- docker/api/client.py | 2 +- docker/api/container.py | 2 +- docker/models/containers.py | 2 +- docs/conf.py | 2 +- docs/index.rst | 4 ++-- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docker/api/client.py b/docker/api/client.py index 22c32b44d9..032f57758d 100644 --- a/docker/api/client.py +++ b/docker/api/client.py @@ -50,7 +50,7 @@ class APIClient( SwarmApiMixin, VolumeApiMixin): """ - A low-level client for the Docker Remote API. + A low-level client for the Docker Engine API. Example: diff --git a/docker/api/container.py b/docker/api/container.py index acb0ffa8bf..6a764fbf58 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -108,7 +108,7 @@ def commit(self, container, repository=None, tag=None, message=None, author (str): The name of the author changes (str): Dockerfile instructions to apply while committing conf (dict): The configuration for the container. See the - `Remote API documentation + `Engine API documentation `_ for full details. diff --git a/docker/models/containers.py b/docker/models/containers.py index 6acc4bb85a..c4a4add440 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -78,7 +78,7 @@ def commit(self, repository=None, tag=None, **kwargs): author (str): The name of the author changes (str): Dockerfile instructions to apply while committing conf (dict): The configuration for the container. See the - `Remote API documentation + `Engine API documentation `_ for full details. diff --git a/docs/conf.py b/docs/conf.py index 4901279619..3e17678a83 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -139,7 +139,7 @@ # documentation. # html_theme_options = { - 'description': 'A Python library for the Docker Remote API', + 'description': 'A Python library for the Docker Engine API', 'fixed_sidebar': True, } diff --git a/docs/index.rst b/docs/index.rst index 8a86cc60b6..b297fc08b4 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,9 +1,9 @@ Docker SDK for Python ===================== -A Python library for the Docker Remote API. It lets you do anything the ``docker`` command does, but from within Python apps – run containers, manage containers, manage Swarms, etc. +A Python library for the Docker Engine API. It lets you do anything the ``docker`` command does, but from within Python apps – run containers, manage containers, manage Swarms, etc. -For more information about the Remote API, `see its documentation `_. +For more information about the Engine API, `see its documentation `_. Installation ------------ From bf49438a2188b60d5a9b462238dba76d373ab084 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 30 Jan 2017 11:55:59 -0800 Subject: [PATCH 0257/1301] Optional name on VolumeCollection.create Signed-off-by: Joffrey F --- docker/models/volumes.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docker/models/volumes.py b/docker/models/volumes.py index 5fb0d1c5b8..3111f67479 100644 --- a/docker/models/volumes.py +++ b/docker/models/volumes.py @@ -28,12 +28,13 @@ class VolumeCollection(Collection): """Volumes on the Docker server.""" model = Volume - def create(self, name, **kwargs): + def create(self, name=None, **kwargs): """ Create a volume. Args: - name (str): Name of the volume + name (str): Name of the volume. If not specified, the engine + generates a name. driver (str): Name of the driver used to create the volume driver_opts (dict): Driver options as a key-value dictionary labels (dict): Labels to set on the volume From bf41c7fa67b8f02cde37ab2869eb33e7b49f39ae Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 30 Jan 2017 18:52:11 -0800 Subject: [PATCH 0258/1301] Improve robustness of remove_network integration test Signed-off-by: Joffrey F --- tests/integration/api_network_test.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/integration/api_network_test.py b/tests/integration/api_network_test.py index 2c297a00a6..982a5182f6 100644 --- a/tests/integration/api_network_test.py +++ b/tests/integration/api_network_test.py @@ -88,13 +88,11 @@ def test_create_network_with_host_driver_fails(self): @requires_api_version('1.21') def test_remove_network(self): - initial_size = len(self.client.networks()) - net_name, net_id = self.create_network() - self.assertEqual(len(self.client.networks()), initial_size + 1) + assert net_name in [n['Name'] for n in self.client.networks()] self.client.remove_network(net_id) - self.assertEqual(len(self.client.networks()), initial_size) + assert net_name not in [n['Name'] for n in self.client.networks()] @requires_api_version('1.21') def test_connect_and_disconnect_container(self): From f2a867f04b30489968a2dcffce9faff4db4af6ae Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 27 Jan 2017 17:59:09 -0800 Subject: [PATCH 0259/1301] Add prune_containers method Signed-off-by: Joffrey F --- docker/api/container.py | 25 ++++++++++++++++++++++--- docker/models/containers.py | 17 +++++++++++++++++ tests/integration/api_container_test.py | 14 ++++++++++++++ 3 files changed, 53 insertions(+), 3 deletions(-) diff --git a/docker/api/container.py b/docker/api/container.py index 6a764fbf58..9fa6d7636d 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -911,9 +911,6 @@ def put_archive(self, container, path, data): Raises: :py:class:`docker.errors.APIError` If the server returns an error. - - Raises: - :py:class:`~docker.errors.APIError` If an error occurs. """ params = {'path': path} url = self._url('/containers/{0}/archive', container) @@ -921,6 +918,28 @@ def put_archive(self, container, path, data): self._raise_for_status(res) return res.status_code == 200 + @utils.minimum_version('1.25') + def prune_containers(self, filters=None): + """ + Delete stopped containers + + Args: + filters (dict): Filters to process on the prune list. + + Returns: + (dict): A dict containing a list of deleted container IDs and + the amount of disk space reclaimed in bytes. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + params = {} + if filters: + params['filters'] = utils.convert_filters(filters) + url = self._url('/containers/prune') + return self._result(self._post(url, params=params), True) + @utils.check_resource def remove_container(self, container, v=False, link=False, force=False): """ diff --git a/docker/models/containers.py b/docker/models/containers.py index c4a4add440..134db4ec0b 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -763,6 +763,23 @@ def list(self, all=False, before=None, filters=None, limit=-1, since=None): since=since) return [self.get(r['Id']) for r in resp] + def prune(self, filters=None): + """ + Delete stopped containers + + Args: + filters (dict): Filters to process on the prune list. + + Returns: + (dict): A dict containing a list of deleted container IDs and + the amount of disk space reclaimed in bytes. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + return self.client.api.prune_containers(filters=filters) + # kwargs to copy straight from run to create RUN_CREATE_KWARGS = [ diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index 3cede45de2..c0e5b9327c 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -1094,6 +1094,20 @@ def test_pause_unpause(self): self.assertEqual(state['Paused'], False) +class PruneTest(BaseAPIIntegrationTest): + @requires_api_version('1.25') + def test_prune_containers(self): + container1 = self.client.create_container(BUSYBOX, ['echo', 'hello']) + container2 = self.client.create_container(BUSYBOX, ['sleep', '9999']) + self.client.start(container1) + self.client.start(container2) + self.client.wait(container1) + result = self.client.prune_containers() + assert container1['Id'] in result['ContainersDeleted'] + assert result['SpaceReclaimed'] > 0 + assert container2['Id'] not in result['ContainersDeleted'] + + class GetContainerStatsTest(BaseAPIIntegrationTest): @requires_api_version('1.19') def test_get_container_stats_no_stream(self): From 7f0c2e7531d6cdef846176e175a148773da13ddb Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 27 Jan 2017 18:29:37 -0800 Subject: [PATCH 0260/1301] Add prune_images method Signed-off-by: Joffrey F --- docker/api/image.py | 25 +++++++++++++++++++++++++ docker/models/containers.py | 16 ++-------------- docker/models/images.py | 4 ++++ tests/integration/api_image_test.py | 16 ++++++++++++++++ 4 files changed, 47 insertions(+), 14 deletions(-) diff --git a/docker/api/image.py b/docker/api/image.py index c1ebc69ca6..09eb086d78 100644 --- a/docker/api/image.py +++ b/docker/api/image.py @@ -274,6 +274,31 @@ def load_image(self, data): res = self._post(self._url("/images/load"), data=data) self._raise_for_status(res) + @utils.minimum_version('1.25') + def prune_images(self, filters=None): + """ + Delete unused images + + Args: + filters (dict): Filters to process on the prune list. + Available filters: + - dangling (bool): When set to true (or 1), prune only + unused and untagged images. + + Returns: + (dict): A dict containing a list of deleted image IDs and + the amount of disk space reclaimed in bytes. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + url = self._url("/images/prune") + params = {} + if filters is not None: + params['filters'] = utils.convert_filters(filters) + return self._result(self._post(url, params=params), True) + def pull(self, repository, tag=None, stream=False, insecure_registry=False, auth_config=None, decode=False): """ diff --git a/docker/models/containers.py b/docker/models/containers.py index 134db4ec0b..78463fd8bf 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -1,5 +1,6 @@ import copy +from ..api import APIClient from ..errors import (ContainerError, ImageNotFound, create_unexpected_kwargs_error) from ..types import HostConfig @@ -764,21 +765,8 @@ def list(self, all=False, before=None, filters=None, limit=-1, since=None): return [self.get(r['Id']) for r in resp] def prune(self, filters=None): - """ - Delete stopped containers - - Args: - filters (dict): Filters to process on the prune list. - - Returns: - (dict): A dict containing a list of deleted container IDs and - the amount of disk space reclaimed in bytes. - - Raises: - :py:class:`docker.errors.APIError` - If the server returns an error. - """ return self.client.api.prune_containers(filters=filters) + prune.__doc__ = APIClient.prune_containers.__doc__ # kwargs to copy straight from run to create diff --git a/docker/models/images.py b/docker/models/images.py index 968e4e329f..a749f63b35 100644 --- a/docker/models/images.py +++ b/docker/models/images.py @@ -269,3 +269,7 @@ def remove(self, *args, **kwargs): def search(self, *args, **kwargs): return self.client.api.search(*args, **kwargs) search.__doc__ = APIClient.search.__doc__ + + def prune(self, filters=None): + return self.client.api.prune_images(filters=filters) + prune.__doc__ = APIClient.prune_images.__doc__ diff --git a/tests/integration/api_image_test.py b/tests/integration/api_image_test.py index 135f115b1c..0f6753fcaa 100644 --- a/tests/integration/api_image_test.py +++ b/tests/integration/api_image_test.py @@ -14,6 +14,7 @@ import docker +from ..helpers import requires_api_version from .base import BaseAPIIntegrationTest, BUSYBOX @@ -285,3 +286,18 @@ def test_import_from_url(self): self.assertIn('status', result) img_id = result['status'] self.tmp_imgs.append(img_id) + + +@requires_api_version('1.25') +class PruneImagesTest(BaseAPIIntegrationTest): + def test_prune_images(self): + try: + self.client.remove_image('hello-world') + except docker.errors.APIError: + pass + self.client.pull('hello-world') + self.tmp_imgs.append('hello-world') + img_id = self.client.inspect_image('hello-world')['Id'] + result = self.client.prune_images() + assert img_id in result['ImagesDeleted'] + assert result['SpaceReclaimed'] > 0 From 89d3803344585bf12496ecfb2a956249991be3d1 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 30 Jan 2017 17:59:03 -0800 Subject: [PATCH 0261/1301] Add prune_volumes method Signed-off-by: Joffrey F --- docker/api/volume.py | 22 ++++++++++++++++++++++ docker/models/volumes.py | 5 +++++ tests/integration/api_image_test.py | 2 +- tests/integration/api_volume_test.py | 8 ++++++++ 4 files changed, 36 insertions(+), 1 deletion(-) diff --git a/docker/api/volume.py b/docker/api/volume.py index 5bffaa0715..cfd9a21ac9 100644 --- a/docker/api/volume.py +++ b/docker/api/volume.py @@ -115,6 +115,28 @@ def inspect_volume(self, name): url = self._url('/volumes/{0}', name) return self._result(self._get(url), True) + @utils.minimum_version('1.25') + def prune_volumes(self, filters=None): + """ + Delete unused volumes + + Args: + filters (dict): Filters to process on the prune list. + + Returns: + (dict): A dict containing a list of deleted volume IDs and + the amount of disk space reclaimed in bytes. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + params = {} + if filters: + params['filters'] = utils.convert_filters(filters) + url = self._url('/volumes/prune') + return self._result(self._post(url, params=params), True) + @utils.minimum_version('1.21') def remove_volume(self, name, force=False): """ diff --git a/docker/models/volumes.py b/docker/models/volumes.py index 5fb0d1c5b8..426e0020b2 100644 --- a/docker/models/volumes.py +++ b/docker/models/volumes.py @@ -1,3 +1,4 @@ +from ..api import APIClient from .resource import Model, Collection @@ -91,3 +92,7 @@ def list(self, **kwargs): if not resp.get('Volumes'): return [] return [self.prepare_model(obj) for obj in resp['Volumes']] + + def prune(self, filters=None): + return self.client.api.prune_volumes(filters=filters) + prune.__doc__ = APIClient.prune_volumes.__doc__ diff --git a/tests/integration/api_image_test.py b/tests/integration/api_image_test.py index 0f6753fcaa..10e95fee33 100644 --- a/tests/integration/api_image_test.py +++ b/tests/integration/api_image_test.py @@ -299,5 +299,5 @@ def test_prune_images(self): self.tmp_imgs.append('hello-world') img_id = self.client.inspect_image('hello-world')['Id'] result = self.client.prune_images() - assert img_id in result['ImagesDeleted'] + assert img_id in [img['Deleted'] for img in result['ImagesDeleted']] assert result['SpaceReclaimed'] > 0 diff --git a/tests/integration/api_volume_test.py b/tests/integration/api_volume_test.py index 4bfc672b57..5a4bb1e0bc 100644 --- a/tests/integration/api_volume_test.py +++ b/tests/integration/api_volume_test.py @@ -56,6 +56,14 @@ def test_force_remove_volume(self): self.client.create_volume(name) self.client.remove_volume(name, force=True) + @requires_api_version('1.25') + def test_prune_volumes(self): + name = 'hopelessmasquerade' + self.client.create_volume(name) + self.tmp_volumes.append(name) + result = self.client.prune_volumes() + assert name in result['VolumesDeleted'] + def test_remove_nonexistent_volume(self): name = 'shootthebullet' with pytest.raises(docker.errors.NotFound): From 83b45b7d54e7011887b05fc9709ac4338f17339e Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 30 Jan 2017 18:46:44 -0800 Subject: [PATCH 0262/1301] Add prune_networks method Ensure all integration tests use the same version of the busybox image Signed-off-by: Joffrey F --- docker/api/network.py | 22 ++++++++++++++++++++++ docker/models/networks.py | 5 +++++ tests/integration/api_image_test.py | 16 +++++++++++++++- tests/integration/api_network_test.py | 24 +++++++++++++++--------- tests/integration/api_service_test.py | 26 +++++++++++++------------- tests/integration/base.py | 2 +- 6 files changed, 71 insertions(+), 24 deletions(-) diff --git a/docker/api/network.py b/docker/api/network.py index 9f6d98fea3..9652228de1 100644 --- a/docker/api/network.py +++ b/docker/api/network.py @@ -133,6 +133,28 @@ def create_network(self, name, driver=None, options=None, ipam=None, res = self._post_json(url, data=data) return self._result(res, json=True) + @minimum_version('1.25') + def prune_networks(self, filters=None): + """ + Delete unused networks + + Args: + filters (dict): Filters to process on the prune list. + + Returns: + (dict): A dict containing a list of deleted network names and + the amount of disk space reclaimed in bytes. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + params = {} + if filters: + params['filters'] = utils.convert_filters(filters) + url = self._url('/networks/prune') + return self._result(self._post(url, params=params), True) + @minimum_version('1.21') def remove_network(self, net_id): """ diff --git a/docker/models/networks.py b/docker/models/networks.py index a80c9f5f8d..a712e9bc43 100644 --- a/docker/models/networks.py +++ b/docker/models/networks.py @@ -1,3 +1,4 @@ +from ..api import APIClient from .containers import Container from .resource import Model, Collection @@ -180,3 +181,7 @@ def list(self, *args, **kwargs): """ resp = self.client.api.networks(*args, **kwargs) return [self.prepare_model(item) for item in resp] + + def prune(self, filters=None): + self.client.api.prune_networks(filters=filters) + prune.__doc__ = APIClient.prune_networks.__doc__ diff --git a/tests/integration/api_image_test.py b/tests/integration/api_image_test.py index 10e95fee33..11146a8a00 100644 --- a/tests/integration/api_image_test.py +++ b/tests/integration/api_image_test.py @@ -295,9 +295,23 @@ def test_prune_images(self): self.client.remove_image('hello-world') except docker.errors.APIError: pass + + # Ensure busybox does not get pruned + ctnr = self.client.create_container(BUSYBOX, ['sleep', '9999']) + self.tmp_containers.append(ctnr) + self.client.pull('hello-world') self.tmp_imgs.append('hello-world') img_id = self.client.inspect_image('hello-world')['Id'] result = self.client.prune_images() - assert img_id in [img['Deleted'] for img in result['ImagesDeleted']] + assert img_id not in [ + img.get('Deleted') for img in result['ImagesDeleted'] + ] + result = self.client.prune_images({'dangling': False}) assert result['SpaceReclaimed'] > 0 + assert 'hello-world:latest' in [ + img.get('Untagged') for img in result['ImagesDeleted'] + ] + assert img_id in [ + img.get('Deleted') for img in result['ImagesDeleted'] + ] diff --git a/tests/integration/api_network_test.py b/tests/integration/api_network_test.py index 2c297a00a6..ea6b02ea47 100644 --- a/tests/integration/api_network_test.py +++ b/tests/integration/api_network_test.py @@ -3,7 +3,7 @@ import pytest from ..helpers import random_name, requires_api_version -from .base import BaseAPIIntegrationTest +from .base import BaseAPIIntegrationTest, BUSYBOX class TestNetworks(BaseAPIIntegrationTest): @@ -100,7 +100,7 @@ def test_remove_network(self): def test_connect_and_disconnect_container(self): net_name, net_id = self.create_network() - container = self.client.create_container('busybox', 'top') + container = self.client.create_container(BUSYBOX, 'top') self.tmp_containers.append(container) self.client.start(container) @@ -128,7 +128,7 @@ def test_connect_and_disconnect_container(self): def test_connect_and_force_disconnect_container(self): net_name, net_id = self.create_network() - container = self.client.create_container('busybox', 'top') + container = self.client.create_container(BUSYBOX, 'top') self.tmp_containers.append(container) self.client.start(container) @@ -155,7 +155,7 @@ def test_connect_and_force_disconnect_container(self): def test_connect_with_aliases(self): net_name, net_id = self.create_network() - container = self.client.create_container('busybox', 'top') + container = self.client.create_container(BUSYBOX, 'top') self.tmp_containers.append(container) self.client.start(container) @@ -173,7 +173,7 @@ def test_connect_on_container_create(self): net_name, net_id = self.create_network() container = self.client.create_container( - image='busybox', + image=BUSYBOX, command='top', host_config=self.client.create_host_config(network_mode=net_name), ) @@ -194,7 +194,7 @@ def test_create_with_aliases(self): net_name, net_id = self.create_network() container = self.client.create_container( - image='busybox', + image=BUSYBOX, command='top', host_config=self.client.create_host_config( network_mode=net_name, @@ -224,7 +224,7 @@ def test_create_with_ipv4_address(self): ), ) container = self.client.create_container( - image='busybox', command='top', + image=BUSYBOX, command='top', host_config=self.client.create_host_config(network_mode=net_name), networking_config=self.client.create_networking_config({ net_name: self.client.create_endpoint_config( @@ -253,7 +253,7 @@ def test_create_with_ipv6_address(self): ), ) container = self.client.create_container( - image='busybox', command='top', + image=BUSYBOX, command='top', host_config=self.client.create_host_config(network_mode=net_name), networking_config=self.client.create_networking_config({ net_name: self.client.create_endpoint_config( @@ -276,7 +276,7 @@ def test_create_with_ipv6_address(self): @requires_api_version('1.24') def test_create_with_linklocal_ips(self): container = self.client.create_container( - 'busybox', 'top', + BUSYBOX, 'top', networking_config=self.client.create_networking_config( { 'bridge': self.client.create_endpoint_config( @@ -453,3 +453,9 @@ def test_create_network_attachable(self): _, net_id = self.create_network(driver='overlay', attachable=True) net = self.client.inspect_network(net_id) assert net['Attachable'] is True + + @requires_api_version('1.25') + def test_prune_networks(self): + net_name, _ = self.create_network() + result = self.client.prune_networks() + assert net_name in result['NetworksDeleted'] diff --git a/tests/integration/api_service_test.py b/tests/integration/api_service_test.py index 46b0a79e6e..fe964596d8 100644 --- a/tests/integration/api_service_test.py +++ b/tests/integration/api_service_test.py @@ -3,7 +3,7 @@ import docker from ..helpers import force_leave_swarm, requires_api_version -from .base import BaseAPIIntegrationTest +from .base import BaseAPIIntegrationTest, BUSYBOX class ServiceTest(BaseAPIIntegrationTest): @@ -31,7 +31,7 @@ def create_simple_service(self, name=None): name = self.get_service_name() container_spec = docker.types.ContainerSpec( - 'busybox', ['echo', 'hello'] + BUSYBOX, ['echo', 'hello'] ) task_tmpl = docker.types.TaskTemplate(container_spec) return name, self.client.create_service(task_tmpl, name=name) @@ -81,7 +81,7 @@ def test_create_service_simple(self): def test_create_service_custom_log_driver(self): container_spec = docker.types.ContainerSpec( - 'busybox', ['echo', 'hello'] + BUSYBOX, ['echo', 'hello'] ) log_cfg = docker.types.DriverConfig('none') task_tmpl = docker.types.TaskTemplate( @@ -99,7 +99,7 @@ def test_create_service_custom_log_driver(self): def test_create_service_with_volume_mount(self): vol_name = self.get_service_name() container_spec = docker.types.ContainerSpec( - 'busybox', ['ls'], + BUSYBOX, ['ls'], mounts=[ docker.types.Mount(target='/test', source=vol_name) ] @@ -119,7 +119,7 @@ def test_create_service_with_volume_mount(self): assert mount['Type'] == 'volume' def test_create_service_with_resources_constraints(self): - container_spec = docker.types.ContainerSpec('busybox', ['true']) + container_spec = docker.types.ContainerSpec(BUSYBOX, ['true']) resources = docker.types.Resources( cpu_limit=4000000, mem_limit=3 * 1024 * 1024 * 1024, cpu_reservation=3500000, mem_reservation=2 * 1024 * 1024 * 1024 @@ -139,7 +139,7 @@ def test_create_service_with_resources_constraints(self): ] def test_create_service_with_update_config(self): - container_spec = docker.types.ContainerSpec('busybox', ['true']) + container_spec = docker.types.ContainerSpec(BUSYBOX, ['true']) task_tmpl = docker.types.TaskTemplate(container_spec) update_config = docker.types.UpdateConfig( parallelism=10, delay=5, failure_action='pause' @@ -173,7 +173,7 @@ def test_create_service_with_update_config_monitor(self): assert update_config['MaxFailureRatio'] == uc['MaxFailureRatio'] def test_create_service_with_restart_policy(self): - container_spec = docker.types.ContainerSpec('busybox', ['true']) + container_spec = docker.types.ContainerSpec(BUSYBOX, ['true']) policy = docker.types.RestartPolicy( docker.types.RestartPolicy.condition_types.ANY, delay=5, max_attempts=5 @@ -196,7 +196,7 @@ def test_create_service_with_custom_networks(self): 'dockerpytest_2', driver='overlay', ipam={'Driver': 'default'} ) self.tmp_networks.append(net2['Id']) - container_spec = docker.types.ContainerSpec('busybox', ['true']) + container_spec = docker.types.ContainerSpec(BUSYBOX, ['true']) task_tmpl = docker.types.TaskTemplate(container_spec) name = self.get_service_name() svc_id = self.client.create_service( @@ -212,7 +212,7 @@ def test_create_service_with_custom_networks(self): def test_create_service_with_placement(self): node_id = self.client.nodes()[0]['ID'] - container_spec = docker.types.ContainerSpec('busybox', ['true']) + container_spec = docker.types.ContainerSpec(BUSYBOX, ['true']) task_tmpl = docker.types.TaskTemplate( container_spec, placement=['node.id=={}'.format(node_id)] ) @@ -224,7 +224,7 @@ def test_create_service_with_placement(self): {'Constraints': ['node.id=={}'.format(node_id)]}) def test_create_service_with_endpoint_spec(self): - container_spec = docker.types.ContainerSpec('busybox', ['true']) + container_spec = docker.types.ContainerSpec(BUSYBOX, ['true']) task_tmpl = docker.types.TaskTemplate(container_spec) name = self.get_service_name() endpoint_spec = docker.types.EndpointSpec(ports={ @@ -255,7 +255,7 @@ def test_create_service_with_endpoint_spec(self): def test_create_service_with_env(self): container_spec = docker.types.ContainerSpec( - 'busybox', ['true'], env={'DOCKER_PY_TEST': 1} + BUSYBOX, ['true'], env={'DOCKER_PY_TEST': 1} ) task_tmpl = docker.types.TaskTemplate( container_spec, @@ -271,7 +271,7 @@ def test_create_service_with_env(self): def test_create_service_global_mode(self): container_spec = docker.types.ContainerSpec( - 'busybox', ['echo', 'hello'] + BUSYBOX, ['echo', 'hello'] ) task_tmpl = docker.types.TaskTemplate(container_spec) name = self.get_service_name() @@ -284,7 +284,7 @@ def test_create_service_global_mode(self): def test_create_service_replicated_mode(self): container_spec = docker.types.ContainerSpec( - 'busybox', ['echo', 'hello'] + BUSYBOX, ['echo', 'hello'] ) task_tmpl = docker.types.TaskTemplate(container_spec) name = self.get_service_name() diff --git a/tests/integration/base.py b/tests/integration/base.py index f0f5a910fe..7da3aa75d5 100644 --- a/tests/integration/base.py +++ b/tests/integration/base.py @@ -81,7 +81,7 @@ def run_container(self, *args, **kwargs): return container - def create_and_start(self, image='busybox', command='top', **kwargs): + def create_and_start(self, image=BUSYBOX, command='top', **kwargs): container = self.client.create_container( image=image, command=command, **kwargs) self.tmp_containers.append(container) From 62e223208fa85e55fbcdd36d2f65dd4185ce0452 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 30 Jan 2017 18:47:24 -0800 Subject: [PATCH 0263/1301] Reference new methods in docs Signed-off-by: Joffrey F --- docker/api/volume.py | 2 +- docs/containers.rst | 1 + docs/images.rst | 1 + docs/networks.rst | 1 + docs/volumes.rst | 1 + 5 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docker/api/volume.py b/docker/api/volume.py index cfd9a21ac9..69abe92038 100644 --- a/docker/api/volume.py +++ b/docker/api/volume.py @@ -124,7 +124,7 @@ def prune_volumes(self, filters=None): filters (dict): Filters to process on the prune list. Returns: - (dict): A dict containing a list of deleted volume IDs and + (dict): A dict containing a list of deleted volume names and the amount of disk space reclaimed in bytes. Raises: diff --git a/docs/containers.rst b/docs/containers.rst index eb51ae4c97..9b27a306b8 100644 --- a/docs/containers.rst +++ b/docs/containers.rst @@ -14,6 +14,7 @@ Methods available on ``client.containers``: .. automethod:: create(image, command=None, **kwargs) .. automethod:: get(id_or_name) .. automethod:: list(**kwargs) + .. automethod:: prune Container objects ----------------- diff --git a/docs/images.rst b/docs/images.rst index 7572c2d6a5..866786ded4 100644 --- a/docs/images.rst +++ b/docs/images.rst @@ -18,6 +18,7 @@ Methods available on ``client.images``: .. automethod:: push .. automethod:: remove .. automethod:: search + .. automethod:: prune Image objects diff --git a/docs/networks.rst b/docs/networks.rst index f6de38bd71..b585f0bdaa 100644 --- a/docs/networks.rst +++ b/docs/networks.rst @@ -13,6 +13,7 @@ Methods available on ``client.networks``: .. automethod:: create .. automethod:: get .. automethod:: list + .. automethod:: prune Network objects ----------------- diff --git a/docs/volumes.rst b/docs/volumes.rst index 8c0574b562..fcd022a574 100644 --- a/docs/volumes.rst +++ b/docs/volumes.rst @@ -13,6 +13,7 @@ Methods available on ``client.volumes``: .. automethod:: create .. automethod:: get .. automethod:: list + .. automethod:: prune Volume objects -------------- From 9296971e4cbba272e4f6ee2bf74f6dddd4f8a3a7 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 31 Jan 2017 18:39:33 -0800 Subject: [PATCH 0264/1301] APIClient implementation of plugin methods Signed-off-by: Joffrey F --- docker/api/client.py | 6 +- docker/api/plugin.py | 207 +++++++++++++++++++++++++++ docs/api.rst | 11 ++ tests/integration/api_plugin_test.py | 114 +++++++++++++++ 4 files changed, 337 insertions(+), 1 deletion(-) create mode 100644 docker/api/plugin.py create mode 100644 tests/integration/api_plugin_test.py diff --git a/docker/api/client.py b/docker/api/client.py index 032f57758d..3f701de2ae 100644 --- a/docker/api/client.py +++ b/docker/api/client.py @@ -14,6 +14,7 @@ from .exec_api import ExecApiMixin from .image import ImageApiMixin from .network import NetworkApiMixin +from .plugin import PluginApiMixin from .service import ServiceApiMixin from .swarm import SwarmApiMixin from .volume import VolumeApiMixin @@ -46,6 +47,7 @@ class APIClient( ExecApiMixin, ImageApiMixin, NetworkApiMixin, + PluginApiMixin, ServiceApiMixin, SwarmApiMixin, VolumeApiMixin): @@ -225,10 +227,12 @@ def _post_json(self, url, data, **kwargs): # Go <1.1 can't unserialize null to a string # so we do this disgusting thing here. data2 = {} - if data is not None: + if data is not None and isinstance(data, dict): for k, v in six.iteritems(data): if v is not None: data2[k] = v + else: + data2 = data if 'headers' not in kwargs: kwargs['headers'] = {} diff --git a/docker/api/plugin.py b/docker/api/plugin.py new file mode 100644 index 0000000000..0a80034947 --- /dev/null +++ b/docker/api/plugin.py @@ -0,0 +1,207 @@ +import six + +from .. import auth, utils + + +class PluginApiMixin(object): + @utils.minimum_version('1.25') + @utils.check_resource + def configure_plugin(self, name, options): + """ + Configure a plugin. + + Args: + name (string): The name of the plugin. The ``:latest`` tag is + optional, and is the default if omitted. + options (dict): A key-value mapping of options + + Returns: + ``True`` if successful + """ + url = self._url('/plugins/{0}/set', name) + data = options + if isinstance(data, dict): + data = ['{0}={1}'.format(k, v) for k, v in six.iteritems(data)] + res = self._post_json(url, data=data) + self._raise_for_status(res) + return True + + def create_plugin(self, name, rootfs, manifest): + """ + Create a new plugin. + + Args: + name (string): The name of the plugin. The ``:latest`` tag is + optional, and is the default if omitted. + rootfs (string): Path to the plugin's ``rootfs`` + manifest (string): Path to the plugin's manifest file + + Returns: + ``True`` if successful + """ + # FIXME: Needs implementation + raise NotImplementedError() + + @utils.minimum_version('1.25') + def disable_plugin(self, name): + """ + Disable an installed plugin. + + Args: + name (string): The name of the plugin. The ``:latest`` tag is + optional, and is the default if omitted. + + Returns: + ``True`` if successful + """ + url = self._url('/plugins/{0}/disable', name) + res = self._post(url) + self._raise_for_status(res) + return True + + @utils.minimum_version('1.25') + def enable_plugin(self, name, timeout=0): + """ + Enable an installed plugin. + + Args: + name (string): The name of the plugin. The ``:latest`` tag is + optional, and is the default if omitted. + timeout (int): Operation timeout (in seconds). Default: 0 + + Returns: + ``True`` if successful + """ + url = self._url('/plugins/{0}/enable', name) + params = {'timeout': timeout} + res = self._post(url, params=params) + self._raise_for_status(res) + return True + + @utils.minimum_version('1.25') + def inspect_plugin(self, name): + """ + Retrieve plugin metadata. + + Args: + name (string): The name of the plugin. The ``:latest`` tag is + optional, and is the default if omitted. + + Returns: + A dict containing plugin info + """ + url = self._url('/plugins/{0}/json', name) + return self._result(self._get(url), True) + + @utils.minimum_version('1.25') + def pull_plugin(self, remote, privileges, name=None): + """ + Pull and install a plugin. After the plugin is installed, it can be + enabled using :py:meth:`~enable_plugin`. + + Args: + remote (string): Remote reference for the plugin to install. + The ``:latest`` tag is optional, and is the default if + omitted. + privileges (list): A list of privileges the user consents to + grant to the plugin. Can be retrieved using + :py:meth:`~plugin_privileges`. + name (string): Local name for the pulled plugin. The + ``:latest`` tag is optional, and is the default if omitted. + + Returns: + An iterable object streaming the decoded API logs + """ + url = self._url('/plugins/pull') + params = { + 'remote': remote, + } + if name: + params['name'] = name + + headers = {} + registry, repo_name = auth.resolve_repository_name(remote) + header = auth.get_config_header(self, registry) + if header: + headers['X-Registry-Auth'] = header + response = self._post_json( + url, params=params, headers=headers, data=privileges, + stream=True + ) + self._raise_for_status(response) + return self._stream_helper(response, decode=True) + + @utils.minimum_version('1.25') + def plugins(self): + """ + Retrieve a list of installed plugins. + + Returns: + A list of dicts, one per plugin + """ + url = self._url('/plugins') + return self._result(self._get(url), True) + + @utils.minimum_version('1.25') + def plugin_privileges(self, name): + """ + Retrieve list of privileges to be granted to a plugin. + + Args: + name (string): Name of the remote plugin to examine. The + ``:latest`` tag is optional, and is the default if omitted. + + Returns: + A list of dictionaries representing the plugin's + permissions + + """ + params = { + 'remote': name, + } + + url = self._url('/plugins/privileges') + return self._result(self._get(url, params=params), True) + + @utils.minimum_version('1.25') + @utils.check_resource + def push_plugin(self, name): + """ + Push a plugin to the registry. + + Args: + name (string): Name of the plugin to upload. The ``:latest`` + tag is optional, and is the default if omitted. + + Returns: + ``True`` if successful + """ + url = self._url('/plugins/{0}/pull', name) + + headers = {} + registry, repo_name = auth.resolve_repository_name(name) + header = auth.get_config_header(self, registry) + if header: + headers['X-Registry-Auth'] = header + res = self._post(url, headers=headers) + self._raise_for_status(res) + return self._stream_helper(res, decode=True) + + @utils.minimum_version('1.25') + def remove_plugin(self, name, force=False): + """ + Remove an installed plugin. + + Args: + name (string): Name of the plugin to remove. The ``:latest`` + tag is optional, and is the default if omitted. + force (bool): Disable the plugin before removing. This may + result in issues if the plugin is in use by a container. + + Returns: + ``True`` if successful + """ + url = self._url('/plugins/{0}', name) + res = self._delete(url, params={'force': force}) + self._raise_for_status(res) + return True diff --git a/docs/api.rst b/docs/api.rst index b5c1e92998..52d12aedde 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -87,6 +87,17 @@ Services :members: :undoc-members: +Plugins +------- + +.. py:module:: docker.api.plugin + +.. rst-class:: hide-signature +.. autoclass:: PluginApiMixin + :members: + :undoc-members: + + The Docker daemon ----------------- diff --git a/tests/integration/api_plugin_test.py b/tests/integration/api_plugin_test.py new file mode 100644 index 0000000000..29e1fa1833 --- /dev/null +++ b/tests/integration/api_plugin_test.py @@ -0,0 +1,114 @@ +import docker +import pytest + +from .base import BaseAPIIntegrationTest, TEST_API_VERSION + +SSHFS = 'vieux/sshfs:latest' + + +class PluginTest(BaseAPIIntegrationTest): + @classmethod + def teardown_class(cls): + c = docker.APIClient( + version=TEST_API_VERSION, timeout=60, + **docker.utils.kwargs_from_env() + ) + try: + c.remove_plugin(SSHFS, force=True) + except docker.errors.APIError: + pass + + def teardown_method(self, method): + try: + self.client.disable_plugin(SSHFS) + except docker.errors.APIError: + pass + + def ensure_plugin_installed(self, plugin_name): + try: + return self.client.inspect_plugin(plugin_name) + except docker.errors.NotFound: + prv = self.client.plugin_privileges(plugin_name) + for d in self.client.pull_plugin(plugin_name, prv): + pass + return self.client.inspect_plugin(plugin_name) + + def test_enable_plugin(self): + pl_data = self.ensure_plugin_installed(SSHFS) + assert pl_data['Enabled'] is False + assert self.client.enable_plugin(SSHFS) + pl_data = self.client.inspect_plugin(SSHFS) + assert pl_data['Enabled'] is True + with pytest.raises(docker.errors.APIError): + self.client.enable_plugin(SSHFS) + + def test_disable_plugin(self): + pl_data = self.ensure_plugin_installed(SSHFS) + assert pl_data['Enabled'] is False + assert self.client.enable_plugin(SSHFS) + pl_data = self.client.inspect_plugin(SSHFS) + assert pl_data['Enabled'] is True + self.client.disable_plugin(SSHFS) + pl_data = self.client.inspect_plugin(SSHFS) + assert pl_data['Enabled'] is False + with pytest.raises(docker.errors.APIError): + self.client.disable_plugin(SSHFS) + + def test_inspect_plugin(self): + self.ensure_plugin_installed(SSHFS) + data = self.client.inspect_plugin(SSHFS) + assert 'Config' in data + assert 'Name' in data + assert data['Name'] == SSHFS + + def test_plugin_privileges(self): + prv = self.client.plugin_privileges(SSHFS) + assert isinstance(prv, list) + for item in prv: + assert 'Name' in item + assert 'Value' in item + assert 'Description' in item + + def test_list_plugins(self): + self.ensure_plugin_installed(SSHFS) + data = self.client.plugins() + assert len(data) > 0 + plugin = [p for p in data if p['Name'] == SSHFS][0] + assert 'Config' in plugin + + def test_configure_plugin(self): + pl_data = self.ensure_plugin_installed(SSHFS) + assert pl_data['Enabled'] is False + self.client.configure_plugin(SSHFS, { + 'DEBUG': '1' + }) + pl_data = self.client.inspect_plugin(SSHFS) + assert 'Env' in pl_data['Settings'] + assert 'DEBUG=1' in pl_data['Settings']['Env'] + + self.client.configure_plugin(SSHFS, ['DEBUG=0']) + pl_data = self.client.inspect_plugin(SSHFS) + assert 'DEBUG=0' in pl_data['Settings']['Env'] + + def test_remove_plugin(self): + pl_data = self.ensure_plugin_installed(SSHFS) + assert pl_data['Enabled'] is False + assert self.client.remove_plugin(SSHFS) is True + + def test_force_remove_plugin(self): + self.ensure_plugin_installed(SSHFS) + self.client.enable_plugin(SSHFS) + assert self.client.inspect_plugin(SSHFS)['Enabled'] is True + assert self.client.remove_plugin(SSHFS, force=True) is True + + def test_install_plugin(self): + try: + self.client.remove_plugin(SSHFS, force=True) + except docker.errors.APIError: + pass + + prv = self.client.plugin_privileges(SSHFS) + logs = [d for d in self.client.pull_plugin(SSHFS, prv)] + assert filter(lambda x: x['status'] == 'Download complete', logs) + assert self.client.inspect_plugin(SSHFS) + assert self.client.enable_plugin(SSHFS) From 39f6a89b697dc1c55e1c16595e4ac1deccf3e59c Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 3 Feb 2017 19:46:58 -0800 Subject: [PATCH 0265/1301] Add plugin API implementation to DockerClient Signed-off-by: Joffrey F --- docker/client.py | 9 ++ docker/models/plugins.py | 173 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 182 insertions(+) create mode 100644 docker/models/plugins.py diff --git a/docker/client.py b/docker/client.py index 171175d328..127f8dd0aa 100644 --- a/docker/client.py +++ b/docker/client.py @@ -3,6 +3,7 @@ from .models.images import ImageCollection from .models.networks import NetworkCollection from .models.nodes import NodeCollection +from .models.plugins import PluginCollection from .models.services import ServiceCollection from .models.swarm import Swarm from .models.volumes import VolumeCollection @@ -109,6 +110,14 @@ def nodes(self): """ return NodeCollection(client=self) + @property + def plugins(self): + """ + An object for managing plugins on the server. See the + :doc:`plugins documentation ` for full details. + """ + return PluginCollection(client=self) + @property def services(self): """ diff --git a/docker/models/plugins.py b/docker/models/plugins.py new file mode 100644 index 0000000000..04c8bde54f --- /dev/null +++ b/docker/models/plugins.py @@ -0,0 +1,173 @@ +from .resource import Collection, Model + + +class Plugin(Model): + """ + A plugin on the server. + """ + def __repr__(self): + return "<%s: '%s'>" % (self.__class__.__name__, self.name) + + @property + def name(self): + """ + The plugin's name. + """ + return self.attrs.get('Name') + + @property + def enabled(self): + """ + Whether the plugin is enabled. + """ + return self.attrs.get('Enabled') + + @property + def settings(self): + """ + A dictionary representing the plugin's configuration. + """ + return self.attrs.get('Settings') + + def configure(self, options): + """ + Update the plugin's settings. + + Args: + options (dict): A key-value mapping of options. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + self.client.api.configure_plugin(self.name, options) + self.reload() + + def disable(self): + """ + Disable the plugin. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + + self.client.api.disable_plugin(self.name) + self.reload() + + def enable(self, timeout=0): + """ + Enable the plugin. + + Args: + timeout (int): Timeout in seconds. Default: 0 + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + self.client.api.enable_plugin(self.name, timeout) + self.reload() + + def push(self): + """ + Push the plugin to a remote registry. + + Returns: + A dict iterator streaming the status of the upload. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + return self.client.api.push_plugin(self.name) + + def remove(self, force=False): + """ + Remove the plugin from the server. + + Args: + force (bool): Remove even if the plugin is enabled. + Default: False + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + return self.client.api.remove_plugin(self.name, force=force) + + +class PluginCollection(Collection): + model = Plugin + + def create(self, name, rootfs, manifest): + """ + Create a new plugin. + + Args: + name (string): The name of the plugin. The ``:latest`` tag is + optional, and is the default if omitted. + rootfs (string): Path to the plugin's ``rootfs`` + manifest (string): Path to the plugin's manifest file + + Returns: + (:py:class:`Plugin`): The newly created plugin. + """ + self.client.api.create_plugin(name, rootfs, manifest) + return self.get(name) + + def get(self, name): + """ + Gets a plugin. + + Args: + name (str): The name of the plugin. + + Returns: + (:py:class:`Plugin`): The plugin. + + Raises: + :py:class:`docker.errors.NotFound` If the plugin does not + exist. + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + return self.prepare_model(self.client.api.inspect_plugin(name)) + + def install(self, remote_name, local_name=None): + """ + Pull and install a plugin. + + Args: + remote_name (string): Remote reference for the plugin to + install. The ``:latest`` tag is optional, and is the + default if omitted. + local_name (string): Local name for the pulled plugin. + The ``:latest`` tag is optional, and is the default if + omitted. Optional. + + Returns: + (:py:class:`Plugin`): The installed plugin + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + privileges = self.client.api.plugin_privileges(remote_name) + it = self.client.api.pull_plugin(remote_name, privileges, local_name) + for data in it: + pass + return self.get(local_name or remote_name) + + def list(self): + """ + List plugins installed on the server. + + Returns: + (list of :py:class:`Plugin`): The plugins. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + resp = self.client.api.plugins() + return [self.prepare_model(r) for r in resp] From 14d81d96c1455d0f3ca45a7e9398c3bc2e9ea4da Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 3 Feb 2017 19:47:34 -0800 Subject: [PATCH 0266/1301] Plugins API documentation Signed-off-by: Joffrey F --- docs/client.rst | 1 + docs/index.rst | 1 + docs/plugins.rst | 37 +++++++++++++++++++++++++++++++++++++ 3 files changed, 39 insertions(+) create mode 100644 docs/plugins.rst diff --git a/docs/client.rst b/docs/client.rst index 63bce2c875..5096bcc435 100644 --- a/docs/client.rst +++ b/docs/client.rst @@ -19,6 +19,7 @@ Client reference .. autoattribute:: images .. autoattribute:: networks .. autoattribute:: nodes + .. autoattribute:: plugins .. autoattribute:: services .. autoattribute:: swarm .. autoattribute:: volumes diff --git a/docs/index.rst b/docs/index.rst index b297fc08b4..70f570ea2e 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -84,6 +84,7 @@ That's just a taste of what you can do with the Docker SDK for Python. For more, images networks nodes + plugins services swarm volumes diff --git a/docs/plugins.rst b/docs/plugins.rst new file mode 100644 index 0000000000..a171b2bdad --- /dev/null +++ b/docs/plugins.rst @@ -0,0 +1,37 @@ +Plugins +======= + +.. py:module:: docker.models.plugins + +Manage plugins on the server. + +Methods available on ``client.plugins``: + +.. rst-class:: hide-signature +.. py:class:: PluginCollection + + .. automethod:: get + .. automethod:: install + .. automethod:: list + + +Plugin objects +-------------- + +.. autoclass:: Plugin() + + .. autoattribute:: id + .. autoattribute:: short_id + .. autoattribute:: name + .. autoattribute:: enabled + .. autoattribute:: settings + .. py:attribute:: attrs + + The raw representation of this object from the server. + + .. automethod:: configure + .. automethod:: disable + .. automethod:: enable + .. automethod:: reload + .. automethod:: push + .. automethod:: remove From cd05d8d53dd2215b8b778e6f3ae193c36ec183c2 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 3 Feb 2017 19:53:58 -0800 Subject: [PATCH 0267/1301] Fix _post_json behavior Signed-off-by: Joffrey F --- docker/api/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/api/client.py b/docker/api/client.py index 3f701de2ae..107297391f 100644 --- a/docker/api/client.py +++ b/docker/api/client.py @@ -231,7 +231,7 @@ def _post_json(self, url, data, **kwargs): for k, v in six.iteritems(data): if v is not None: data2[k] = v - else: + elif data is not None: data2 = data if 'headers' not in kwargs: From 956fe1cac17c7b902694889f3044503046801185 Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Tue, 7 Feb 2017 16:31:25 +0100 Subject: [PATCH 0268/1301] Fix volume path passed by run to create_container Seems like this is pretty much ignored by Docker, so it wasn't causing any visible issues, except when a volume name was used instead of a path. Also, added integration tests. Ref #1380 Signed-off-by: Ben Firshman --- docker/api/container.py | 10 +++--- docker/models/containers.py | 2 +- tests/integration/models_containers_test.py | 37 +++++++++++++++++++++ tests/unit/models_containers_test.py | 4 ++- 4 files changed, 47 insertions(+), 6 deletions(-) diff --git a/docker/api/container.py b/docker/api/container.py index 9fa6d7636d..551e72aad6 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -313,9 +313,10 @@ def create_container(self, image, command=None, hostname=None, user=None, **Using volumes** - Volume declaration is done in two parts. Provide a list of mountpoints - to the with the ``volumes`` parameter, and declare mappings in the - ``host_config`` section. + Volume declaration is done in two parts. Provide a list of + paths to use as mountpoints inside the container with the + ``volumes`` parameter, and declare mappings from paths on the host + in the ``host_config`` section. .. code-block:: python @@ -392,7 +393,8 @@ def create_container(self, image, command=None, hostname=None, user=None, version 1.10. Use ``host_config`` instead. dns_opt (:py:class:`list`): Additional options to be added to the container's ``resolv.conf`` file - volumes (str or list): + volumes (str or list): List of paths inside the container to use + as volumes. volumes_from (:py:class:`list`): List of container names or Ids to get volumes from. network_disabled (bool): Disable networking diff --git a/docker/models/containers.py b/docker/models/containers.py index 78463fd8bf..02773922d4 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -885,5 +885,5 @@ def _create_container_args(kwargs): for p in sorted(port_bindings.keys())] binds = create_kwargs['host_config'].get('Binds') if binds: - create_kwargs['volumes'] = [v.split(':')[0] for v in binds] + create_kwargs['volumes'] = [v.split(':')[1] for v in binds] return create_kwargs diff --git a/tests/integration/models_containers_test.py b/tests/integration/models_containers_test.py index d0f87d6023..4f1e6a1fe9 100644 --- a/tests/integration/models_containers_test.py +++ b/tests/integration/models_containers_test.py @@ -1,4 +1,5 @@ import docker +import tempfile from .base import BaseIntegrationTest, TEST_API_VERSION @@ -32,6 +33,42 @@ def test_run_with_image_that_does_not_exist(self): with self.assertRaises(docker.errors.ImageNotFound): client.containers.run("dockerpytest_does_not_exist") + def test_run_with_volume(self): + client = docker.from_env(version=TEST_API_VERSION) + path = tempfile.mkdtemp() + + container = client.containers.run( + "alpine", "sh -c 'echo \"hello\" > /insidecontainer/test'", + volumes=["%s:/insidecontainer" % path], + detach=True + ) + self.tmp_containers.append(container.id) + container.wait() + + out = client.containers.run( + "alpine", "cat /insidecontainer/test", + volumes=["%s:/insidecontainer" % path] + ) + self.assertEqual(out, b'hello\n') + + def test_run_with_named_volume(self): + client = docker.from_env(version=TEST_API_VERSION) + client.volumes.create(name="somevolume") + + container = client.containers.run( + "alpine", "sh -c 'echo \"hello\" > /insidecontainer/test'", + volumes=["somevolume:/insidecontainer"], + detach=True + ) + self.tmp_containers.append(container.id) + container.wait() + + out = client.containers.run( + "alpine", "cat /insidecontainer/test", + volumes=["somevolume:/insidecontainer"] + ) + self.assertEqual(out, b'hello\n') + def test_get(self): client = docker.from_env(version=TEST_API_VERSION) container = client.containers.run("alpine", "sleep 300", detach=True) diff --git a/tests/unit/models_containers_test.py b/tests/unit/models_containers_test.py index c3086c629a..de727b0e5e 100644 --- a/tests/unit/models_containers_test.py +++ b/tests/unit/models_containers_test.py @@ -100,6 +100,7 @@ def test_create_container_args(self): volumes=[ '/home/user1/:/mnt/vol2', '/var/www:/mnt/vol1:ro', + 'volumename:/mnt/vol3', ], volumes_from=['container'], working_dir='/code' @@ -116,6 +117,7 @@ def test_create_container_args(self): 'Binds': [ '/home/user1/:/mnt/vol2', '/var/www:/mnt/vol1:ro', + 'volumename:/mnt/vol3', ], 'BlkioDeviceReadBps': [{'Path': 'foo', 'Rate': 3}], 'BlkioDeviceReadIOps': [{'Path': 'foo', 'Rate': 3}], @@ -181,7 +183,7 @@ def test_create_container_args(self): tty=True, user='bob', volume_driver='some_driver', - volumes=['/home/user1/', '/var/www'], + volumes=['/mnt/vol2', '/mnt/vol1', '/mnt/vol3'], working_dir='/code' ) From f83993de0a7bbde043240beb85fd2eec65a21cf6 Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Tue, 7 Feb 2017 17:03:02 +0100 Subject: [PATCH 0269/1301] Fix passing volumes to run with no host path Technically we shouldn't be passing them as binds, but the daemon doesn't seem to mind. Fixes #1380 Signed-off-by: Ben Firshman --- docker/models/containers.py | 12 +++++++++++- tests/unit/models_containers_test.py | 12 +++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/docker/models/containers.py b/docker/models/containers.py index 02773922d4..330ac92c56 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -885,5 +885,15 @@ def _create_container_args(kwargs): for p in sorted(port_bindings.keys())] binds = create_kwargs['host_config'].get('Binds') if binds: - create_kwargs['volumes'] = [v.split(':')[1] for v in binds] + create_kwargs['volumes'] = [_host_volume_from_bind(v) for v in binds] return create_kwargs + + +def _host_volume_from_bind(bind): + bits = bind.split(':') + if len(bits) == 1: + return bits[0] + elif len(bits) == 2 and bits[1] in ('ro', 'rw'): + return bits[0] + else: + return bits[1] diff --git a/tests/unit/models_containers_test.py b/tests/unit/models_containers_test.py index de727b0e5e..ae1bd12aae 100644 --- a/tests/unit/models_containers_test.py +++ b/tests/unit/models_containers_test.py @@ -101,6 +101,8 @@ def test_create_container_args(self): '/home/user1/:/mnt/vol2', '/var/www:/mnt/vol1:ro', 'volumename:/mnt/vol3', + '/volumewithnohostpath', + '/anothervolumewithnohostpath:ro', ], volumes_from=['container'], working_dir='/code' @@ -118,6 +120,8 @@ def test_create_container_args(self): '/home/user1/:/mnt/vol2', '/var/www:/mnt/vol1:ro', 'volumename:/mnt/vol3', + '/volumewithnohostpath', + '/anothervolumewithnohostpath:ro' ], 'BlkioDeviceReadBps': [{'Path': 'foo', 'Rate': 3}], 'BlkioDeviceReadIOps': [{'Path': 'foo', 'Rate': 3}], @@ -183,7 +187,13 @@ def test_create_container_args(self): tty=True, user='bob', volume_driver='some_driver', - volumes=['/mnt/vol2', '/mnt/vol1', '/mnt/vol3'], + volumes=[ + '/mnt/vol2', + '/mnt/vol1', + '/mnt/vol3', + '/volumewithnohostpath', + '/anothervolumewithnohostpath' + ], working_dir='/code' ) From 9ac3666c27452e602e5e34188b3e2d441162df0d Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 8 Feb 2017 18:08:12 -0800 Subject: [PATCH 0270/1301] Bump test engine version Signed-off-by: Joffrey F --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index b8b932aed1..566d5494c1 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -7,7 +7,7 @@ def images = [:] // Note: Swarm in dind seem notoriously flimsy with 1.12.1+, which is why we're // sticking with 1.12.0 for the 1.12 series -def dockerVersions = ["1.12.0", "1.13.0"] +def dockerVersions = ["1.12.0", "1.13.1"] def buildImage = { name, buildargs, pyTag -> img = docker.image(name) From e1ad3186ef86c0a91f1051da96049facae7a0325 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 8 Feb 2017 18:05:24 -0800 Subject: [PATCH 0271/1301] Add create_plugin implementation Signed-off-by: Joffrey F --- MANIFEST.in | 1 + docker/api/plugin.py | 17 +++++--- docker/models/plugins.py | 10 +++-- docker/utils/__init__.py | 2 +- docker/utils/utils.py | 39 +++++++++++++------ tests/integration/api_plugin_test.py | 21 ++++++++++ tests/integration/base.py | 1 + .../testdata/dummy-plugin/config.json | 19 +++++++++ .../dummy-plugin/rootfs/dummy/file.txt | 0 9 files changed, 89 insertions(+), 21 deletions(-) create mode 100644 tests/integration/testdata/dummy-plugin/config.json create mode 100644 tests/integration/testdata/dummy-plugin/rootfs/dummy/file.txt diff --git a/MANIFEST.in b/MANIFEST.in index ee6cdbbd6f..41b3fa9f8b 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -5,3 +5,4 @@ include README.rst include LICENSE recursive-include tests *.py recursive-include tests/unit/testdata * +recursive-include tests/integration/testdata * diff --git a/docker/api/plugin.py b/docker/api/plugin.py index 0a80034947..772d263387 100644 --- a/docker/api/plugin.py +++ b/docker/api/plugin.py @@ -26,21 +26,28 @@ def configure_plugin(self, name, options): self._raise_for_status(res) return True - def create_plugin(self, name, rootfs, manifest): + @utils.minimum_version('1.25') + def create_plugin(self, name, plugin_data_dir, gzip=False): """ Create a new plugin. Args: name (string): The name of the plugin. The ``:latest`` tag is optional, and is the default if omitted. - rootfs (string): Path to the plugin's ``rootfs`` - manifest (string): Path to the plugin's manifest file + plugin_data_dir (string): Path to the plugin data directory. + Plugin data directory must contain the ``config.json`` + manifest file and the ``rootfs`` directory. + gzip (bool): Compress the context using gzip. Default: False Returns: ``True`` if successful """ - # FIXME: Needs implementation - raise NotImplementedError() + url = self._url('/plugins/create') + + with utils.create_archive(root=plugin_data_dir, gzip=gzip) as archv: + res = self._post(url, params={'name': name}, data=archv) + self._raise_for_status(res) + return True @utils.minimum_version('1.25') def disable_plugin(self, name): diff --git a/docker/models/plugins.py b/docker/models/plugins.py index 04c8bde54f..8b6ede95bf 100644 --- a/docker/models/plugins.py +++ b/docker/models/plugins.py @@ -100,20 +100,22 @@ def remove(self, force=False): class PluginCollection(Collection): model = Plugin - def create(self, name, rootfs, manifest): + def create(self, name, plugin_data_dir, gzip=False): """ Create a new plugin. Args: name (string): The name of the plugin. The ``:latest`` tag is optional, and is the default if omitted. - rootfs (string): Path to the plugin's ``rootfs`` - manifest (string): Path to the plugin's manifest file + plugin_data_dir (string): Path to the plugin data directory. + Plugin data directory must contain the ``config.json`` + manifest file and the ``rootfs`` directory. + gzip (bool): Compress the context using gzip. Default: False Returns: (:py:class:`Plugin`): The newly created plugin. """ - self.client.api.create_plugin(name, rootfs, manifest) + self.client.api.create_plugin(name, plugin_data_dir, gzip) return self.get(name) def get(self, name): diff --git a/docker/utils/__init__.py b/docker/utils/__init__.py index 747743cc6a..8f8eb2706a 100644 --- a/docker/utils/__init__.py +++ b/docker/utils/__init__.py @@ -6,7 +6,7 @@ create_host_config, parse_bytes, ping_registry, parse_env_file, version_lt, version_gte, decode_json_header, split_command, create_ipam_config, create_ipam_pool, parse_devices, normalize_links, convert_service_networks, - format_environment + format_environment, create_archive ) from .decorators import check_resource, minimum_version, update_headers diff --git a/docker/utils/utils.py b/docker/utils/utils.py index 8026c4dfde..01eb16c32e 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -80,16 +80,35 @@ def decode_json_header(header): def tar(path, exclude=None, dockerfile=None, fileobj=None, gzip=False): - if not fileobj: - fileobj = tempfile.NamedTemporaryFile() - t = tarfile.open(mode='w:gz' if gzip else 'w', fileobj=fileobj) - root = os.path.abspath(path) exclude = exclude or [] - for path in sorted(exclude_paths(root, exclude, dockerfile=dockerfile)): - i = t.gettarinfo(os.path.join(root, path), arcname=path) + return create_archive( + files=sorted(exclude_paths(root, exclude, dockerfile=dockerfile)), + root=root, fileobj=fileobj, gzip=gzip + ) + + +def build_file_list(root): + files = [] + for dirname, dirnames, fnames in os.walk(root): + for filename in fnames + dirnames: + longpath = os.path.join(dirname, filename) + files.append( + longpath.replace(root, '', 1).lstrip('/') + ) + return files + + +def create_archive(root, files=None, fileobj=None, gzip=False): + if not fileobj: + fileobj = tempfile.NamedTemporaryFile() + t = tarfile.open(mode='w:gz' if gzip else 'w', fileobj=fileobj) + if files is None: + files = build_file_list(root) + for path in files: + i = t.gettarinfo(os.path.join(root, path), arcname=path) if i is None: # This happens when we encounter a socket file. We can safely # ignore it and proceed. @@ -102,13 +121,11 @@ def tar(path, exclude=None, dockerfile=None, fileobj=None, gzip=False): try: # We open the file object in binary mode for Windows support. - f = open(os.path.join(root, path), 'rb') + with open(os.path.join(root, path), 'rb') as f: + t.addfile(i, f) except IOError: # When we encounter a directory the file object is set to None. - f = None - - t.addfile(i, f) - + t.addfile(i, None) t.close() fileobj.seek(0) return fileobj diff --git a/tests/integration/api_plugin_test.py b/tests/integration/api_plugin_test.py index 29e1fa1833..e90a1088fc 100644 --- a/tests/integration/api_plugin_test.py +++ b/tests/integration/api_plugin_test.py @@ -1,11 +1,15 @@ +import os + import docker import pytest from .base import BaseAPIIntegrationTest, TEST_API_VERSION +from ..helpers import requires_api_version SSHFS = 'vieux/sshfs:latest' +@requires_api_version('1.25') class PluginTest(BaseAPIIntegrationTest): @classmethod def teardown_class(cls): @@ -24,6 +28,12 @@ def teardown_method(self, method): except docker.errors.APIError: pass + for p in self.tmp_plugins: + try: + self.client.remove_plugin(p, force=True) + except docker.errors.APIError: + pass + def ensure_plugin_installed(self, plugin_name): try: return self.client.inspect_plugin(plugin_name) @@ -112,3 +122,14 @@ def test_install_plugin(self): assert filter(lambda x: x['status'] == 'Download complete', logs) assert self.client.inspect_plugin(SSHFS) assert self.client.enable_plugin(SSHFS) + + def test_create_plugin(self): + plugin_data_dir = os.path.join( + os.path.dirname(__file__), 'testdata/dummy-plugin' + ) + assert self.client.create_plugin( + 'docker-sdk-py/dummy', plugin_data_dir + ) + self.tmp_plugins.append('docker-sdk-py/dummy') + data = self.client.inspect_plugin('docker-sdk-py/dummy') + assert data['Config']['Entrypoint'] == ['/dummy'] diff --git a/tests/integration/base.py b/tests/integration/base.py index f0f5a910fe..8b75acc9c6 100644 --- a/tests/integration/base.py +++ b/tests/integration/base.py @@ -27,6 +27,7 @@ def setUp(self): self.tmp_folders = [] self.tmp_volumes = [] self.tmp_networks = [] + self.tmp_plugins = [] def tearDown(self): client = docker.from_env(version=TEST_API_VERSION) diff --git a/tests/integration/testdata/dummy-plugin/config.json b/tests/integration/testdata/dummy-plugin/config.json new file mode 100644 index 0000000000..53b4e7aa98 --- /dev/null +++ b/tests/integration/testdata/dummy-plugin/config.json @@ -0,0 +1,19 @@ +{ + "description": "Dummy test plugin for docker python SDK", + "documentation": "https://github.com/docker/docker-py", + "entrypoint": ["/dummy"], + "network": { + "type": "host" + }, + "interface" : { + "types": ["docker.volumedriver/1.0"], + "socket": "dummy.sock" + }, + "env": [ + { + "name":"DEBUG", + "settable":["value"], + "value":"0" + } + ] +} diff --git a/tests/integration/testdata/dummy-plugin/rootfs/dummy/file.txt b/tests/integration/testdata/dummy-plugin/rootfs/dummy/file.txt new file mode 100644 index 0000000000..e69de29bb2 From 52bae3ca2cf9547d760d0dff8643a2f83133f5ad Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 9 Feb 2017 19:08:24 -0800 Subject: [PATCH 0272/1301] Implement secrets API Signed-off-by: Joffrey F --- docker/api/client.py | 2 + docker/api/secret.py | 87 ++++++++++++++++++++++++++++++++++++++++ docker/client.py | 8 ++++ docker/models/secrets.py | 69 +++++++++++++++++++++++++++++++ 4 files changed, 166 insertions(+) create mode 100644 docker/api/secret.py create mode 100644 docker/models/secrets.py diff --git a/docker/api/client.py b/docker/api/client.py index 0098d44a32..99d7879cb8 100644 --- a/docker/api/client.py +++ b/docker/api/client.py @@ -15,6 +15,7 @@ from .image import ImageApiMixin from .network import NetworkApiMixin from .plugin import PluginApiMixin +from .secret import SecretApiMixin from .service import ServiceApiMixin from .swarm import SwarmApiMixin from .volume import VolumeApiMixin @@ -48,6 +49,7 @@ class APIClient( ImageApiMixin, NetworkApiMixin, PluginApiMixin, + SecretApiMixin, ServiceApiMixin, SwarmApiMixin, VolumeApiMixin): diff --git a/docker/api/secret.py b/docker/api/secret.py new file mode 100644 index 0000000000..4802a4afe3 --- /dev/null +++ b/docker/api/secret.py @@ -0,0 +1,87 @@ +import base64 + +from .. import utils + + +class SecretApiMixin(object): + @utils.minimum_version('1.25') + def create_secret(self, name, data, labels=None): + """ + Create a secret + + Args: + name (string): Name of the secret + data (bytes): Secret data to be stored + labels (dict): A mapping of labels to assign to the secret + + Returns (dict): ID of the newly created secret + """ + if not isinstance(data, bytes): + data = data.encode('utf-8') + + data = base64.b64encode(data) + body = { + 'Data': data, + 'Name': name, + 'Labels': labels + } + + url = self._url('/secrets/create') + return self._result( + self._post_json(url, data=body), True + ) + + @utils.minimum_version('1.25') + @utils.check_resource + def inspect_secret(self, id): + """ + Retrieve secret metadata + + Args: + id (string): Full ID of the secret to remove + + Returns (dict): A dictionary of metadata + + Raises: + :py:class:`docker.errors.NotFound` + if no secret with that ID exists + """ + url = self._url('/secrets/{0}', id) + return self._result(self._get(url), True) + + @utils.minimum_version('1.25') + @utils.check_resource + def remove_secret(self, id): + """ + Remove a secret + + Args: + id (string): Full ID of the secret to remove + + Returns (boolean): True if successful + + Raises: + :py:class:`docker.errors.NotFound` + if no secret with that ID exists + """ + url = self._url('/secrets/{0}', id) + res = self._delete(url) + self._raise_for_status(res) + return True + + @utils.minimum_version('1.25') + def secrets(self, filters=None): + """ + List secrets + + Args: + filters (dict): A map of filters to process on the secrets + list. Available filters: ``names`` + + Returns (list): A list of secrets + """ + url = self._url('/secrets') + params = {} + if filters: + params['filters'] = utils.convert_filters(filters) + return self._result(self._get(url, params=params), True) diff --git a/docker/client.py b/docker/client.py index 127f8dd0aa..09bda67f82 100644 --- a/docker/client.py +++ b/docker/client.py @@ -4,6 +4,7 @@ from .models.networks import NetworkCollection from .models.nodes import NodeCollection from .models.plugins import PluginCollection +from .models.secrets import SecretCollection from .models.services import ServiceCollection from .models.swarm import Swarm from .models.volumes import VolumeCollection @@ -118,6 +119,13 @@ def plugins(self): """ return PluginCollection(client=self) + def secrets(self): + """ + An object for managing secrets on the server. See the + :doc:`secrets documentation ` for full details. + """ + return SecretCollection(client=self) + @property def services(self): """ diff --git a/docker/models/secrets.py b/docker/models/secrets.py new file mode 100644 index 0000000000..ca11edeb08 --- /dev/null +++ b/docker/models/secrets.py @@ -0,0 +1,69 @@ +from ..api import APIClient +from .resource import Model, Collection + + +class Secret(Model): + """A secret.""" + id_attribute = 'ID' + + def __repr__(self): + return "<%s: '%s'>" % (self.__class__.__name__, self.name) + + @property + def name(self): + return self.attrs['Spec']['Name'] + + def remove(self): + """ + Remove this secret. + + Raises: + :py:class:`docker.errors.APIError` + If secret failed to remove. + """ + return self.client.api.remove_secret(self.id) + + +class SecretCollection(Collection): + """Secrets on the Docker server.""" + model = Secret + + def create(self, **kwargs): + obj = self.client.api.create_secret(**kwargs) + return self.prepare_model(obj) + create.__doc__ = APIClient.create_secret.__doc__ + + def get(self, secret_id): + """ + Get a secret. + + Args: + secret_id (str): Secret ID. + + Returns: + (:py:class:`Secret`): The secret. + + Raises: + :py:class:`docker.errors.NotFound` + If the secret does not exist. + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + return self.prepare_model(self.client.api.inspect_secret(secret_id)) + + def list(self, **kwargs): + """ + List secrets. Similar to the ``docker secret ls`` command. + + Args: + filters (dict): Server-side list filtering options. + + Returns: + (list of :py:class:`Secret`): The secrets. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + resp = self.client.api.secrets(**kwargs) + return [self.prepare_model(obj) for obj in resp] From d1038c422b2a069494476dd743cc06e11e8939e7 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 10 Feb 2017 14:51:15 -0800 Subject: [PATCH 0273/1301] Add support for secrets in ContainerSpec Signed-off-by: Joffrey F --- docker/models/services.py | 3 +++ docker/types/__init__.py | 2 +- docker/types/services.py | 40 ++++++++++++++++++++++++++++++++++++-- docker/utils/decorators.py | 2 +- 4 files changed, 43 insertions(+), 4 deletions(-) diff --git a/docker/models/services.py b/docker/models/services.py index ef6c3e3a91..bd95b5f965 100644 --- a/docker/models/services.py +++ b/docker/models/services.py @@ -109,6 +109,8 @@ def create(self, image, command=None, **kwargs): the service to. Default: ``None``. resources (Resources): Resource limits and reservations. restart_policy (RestartPolicy): Restart policy for containers. + secrets (list of :py:class:`docker.types.SecretReference`): List + of secrets accessible to containers for this service. stop_grace_period (int): Amount of time to wait for containers to terminate before forcefully killing them. update_config (UpdateConfig): Specification for the update strategy @@ -179,6 +181,7 @@ def list(self, **kwargs): 'labels', 'mounts', 'stop_grace_period', + 'secrets', ] # kwargs to copy straight over to TaskTemplate diff --git a/docker/types/__init__.py b/docker/types/__init__.py index 8e2fc17472..0e88776013 100644 --- a/docker/types/__init__.py +++ b/docker/types/__init__.py @@ -4,6 +4,6 @@ from .networks import EndpointConfig, IPAMConfig, IPAMPool, NetworkingConfig from .services import ( ContainerSpec, DriverConfig, EndpointSpec, Mount, Resources, RestartPolicy, - ServiceMode, TaskTemplate, UpdateConfig + SecretReference, ServiceMode, TaskTemplate, UpdateConfig ) from .swarm import SwarmSpec, SwarmExternalCA diff --git a/docker/types/services.py b/docker/types/services.py index 5f7b2fb0d0..b903fa434b 100644 --- a/docker/types/services.py +++ b/docker/types/services.py @@ -2,7 +2,7 @@ from .. import errors from ..constants import IS_WINDOWS_PLATFORM -from ..utils import format_environment, split_command +from ..utils import check_resource, format_environment, split_command class TaskTemplate(dict): @@ -79,9 +79,12 @@ class ContainerSpec(dict): :py:class:`~docker.types.Mount` class for details. stop_grace_period (int): Amount of time to wait for the container to terminate before forcefully killing it. + secrets (list of py:class:`SecretReference`): List of secrets to be + made available inside the containers. """ def __init__(self, image, command=None, args=None, env=None, workdir=None, - user=None, labels=None, mounts=None, stop_grace_period=None): + user=None, labels=None, mounts=None, stop_grace_period=None, + secrets=None): self['Image'] = image if isinstance(command, six.string_types): @@ -109,6 +112,11 @@ def __init__(self, image, command=None, args=None, env=None, workdir=None, if stop_grace_period is not None: self['StopGracePeriod'] = stop_grace_period + if secrets is not None: + if not isinstance(secrets, list): + raise TypeError('secrets must be a list') + self['Secrets'] = secrets + class Mount(dict): """ @@ -410,3 +418,31 @@ def replicas(self): if self.mode != 'replicated': return None return self['replicated'].get('Replicas') + + +class SecretReference(dict): + """ + Secret reference to be used as part of a :py:class:`ContainerSpec`. + Describes how a secret is made accessible inside the service's + containers. + + Args: + secret_id (string): Secret's ID + secret_name (string): Secret's name as defined at its creation. + filename (string): Name of the file containing the secret. Defaults + to the secret's name if not specified. + uid (string): UID of the secret file's owner. Default: 0 + gid (string): GID of the secret file's group. Default: 0 + mode (int): File access mode inside the container. Default: 0o444 + """ + @check_resource + def __init__(self, secret_id, secret_name, filename=None, uid=None, + gid=None, mode=0o444): + self['SecretName'] = secret_name + self['SecretID'] = secret_id + self['File'] = { + 'Name': filename or secret_name, + 'UID': uid or '0', + 'GID': gid or '0', + 'Mode': mode + } diff --git a/docker/utils/decorators.py b/docker/utils/decorators.py index 2fe880c4a5..18cde412ff 100644 --- a/docker/utils/decorators.py +++ b/docker/utils/decorators.py @@ -16,7 +16,7 @@ def wrapped(self, resource_id=None, *args, **kwargs): resource_id = resource_id.get('Id', resource_id.get('ID')) if not resource_id: raise errors.NullResource( - 'image or container param is undefined' + 'Resource ID was not provided' ) return f(self, resource_id, *args, **kwargs) return wrapped From e8a86e40cc387e98fa40e7e3638b921e1b9e18d1 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 10 Feb 2017 15:40:55 -0800 Subject: [PATCH 0274/1301] Add tests for secret API implementation Signed-off-by: Joffrey F --- docker/api/secret.py | 4 ++ tests/integration/api_secret_test.py | 69 ++++++++++++++++++++++++++ tests/integration/api_service_test.py | 70 +++++++++++++++++++++++++++ tests/integration/base.py | 7 +++ tests/unit/api_container_test.py | 6 +-- tests/unit/api_image_test.py | 2 +- 6 files changed, 154 insertions(+), 4 deletions(-) create mode 100644 tests/integration/api_secret_test.py diff --git a/docker/api/secret.py b/docker/api/secret.py index 4802a4afe3..03534a6236 100644 --- a/docker/api/secret.py +++ b/docker/api/secret.py @@ -1,5 +1,7 @@ import base64 +import six + from .. import utils @@ -20,6 +22,8 @@ def create_secret(self, name, data, labels=None): data = data.encode('utf-8') data = base64.b64encode(data) + if six.PY3: + data = data.decode('ascii') body = { 'Data': data, 'Name': name, diff --git a/tests/integration/api_secret_test.py b/tests/integration/api_secret_test.py new file mode 100644 index 0000000000..dcd880f49c --- /dev/null +++ b/tests/integration/api_secret_test.py @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- + +import docker +import pytest + +from ..helpers import force_leave_swarm, requires_api_version +from .base import BaseAPIIntegrationTest + + +@requires_api_version('1.25') +class SecretAPITest(BaseAPIIntegrationTest): + def setUp(self): + super(SecretAPITest, self).setUp() + self.init_swarm() + + def tearDown(self): + super(SecretAPITest, self).tearDown() + force_leave_swarm(self.client) + + def test_create_secret(self): + secret_id = self.client.create_secret( + 'favorite_character', 'sakuya izayoi' + ) + self.tmp_secrets.append(secret_id) + assert 'ID' in secret_id + data = self.client.inspect_secret(secret_id) + assert data['Spec']['Name'] == 'favorite_character' + + def test_create_secret_unicode_data(self): + secret_id = self.client.create_secret( + 'favorite_character', u'いざよいさくや' + ) + self.tmp_secrets.append(secret_id) + assert 'ID' in secret_id + data = self.client.inspect_secret(secret_id) + assert data['Spec']['Name'] == 'favorite_character' + + def test_inspect_secret(self): + secret_name = 'favorite_character' + secret_id = self.client.create_secret( + secret_name, 'sakuya izayoi' + ) + self.tmp_secrets.append(secret_id) + data = self.client.inspect_secret(secret_id) + assert data['Spec']['Name'] == secret_name + assert 'ID' in data + assert 'Version' in data + + def test_remove_secret(self): + secret_name = 'favorite_character' + secret_id = self.client.create_secret( + secret_name, 'sakuya izayoi' + ) + self.tmp_secrets.append(secret_id) + + assert self.client.remove_secret(secret_id) + with pytest.raises(docker.errors.NotFound): + self.client.inspect_secret(secret_id) + + def test_list_secrets(self): + secret_name = 'favorite_character' + secret_id = self.client.create_secret( + secret_name, 'sakuya izayoi' + ) + self.tmp_secrets.append(secret_id) + + data = self.client.secrets(filters={'names': ['favorite_character']}) + assert len(data) == 1 + assert data[0]['ID'] == secret_id['ID'] diff --git a/tests/integration/api_service_test.py b/tests/integration/api_service_test.py index fe964596d8..1dd295dfb5 100644 --- a/tests/integration/api_service_test.py +++ b/tests/integration/api_service_test.py @@ -1,4 +1,7 @@ +# -*- coding: utf-8 -*- + import random +import time import docker @@ -24,6 +27,21 @@ def tearDown(self): def get_service_name(self): return 'dockerpytest_{0:x}'.format(random.getrandbits(64)) + def get_service_container(self, service_name, attempts=20, interval=0.5): + # There is some delay between the service's creation and the creation + # of the service's containers. This method deals with the uncertainty + # when trying to retrieve the container associated with a service. + while True: + containers = self.client.containers( + filters={'name': [service_name]}, quiet=True + ) + if len(containers) > 0: + return containers[0] + attempts -= 1 + if attempts <= 0: + return None + time.sleep(interval) + def create_simple_service(self, name=None): if name: name = 'dockerpytest_{0}'.format(name) @@ -317,3 +335,55 @@ def test_update_service_force_update(self): new_index = svc_info['Version']['Index'] assert new_index > version_index assert svc_info['Spec']['TaskTemplate']['ForceUpdate'] == 10 + + @requires_api_version('1.25') + def test_create_service_with_secret(self): + secret_name = 'favorite_touhou' + secret_data = b'phantasmagoria of flower view' + secret_id = self.client.create_secret(secret_name, secret_data) + self.tmp_secrets.append(secret_id) + secret_ref = docker.types.SecretReference(secret_id, secret_name) + container_spec = docker.types.ContainerSpec( + 'busybox', ['top'], secrets=[secret_ref] + ) + task_tmpl = docker.types.TaskTemplate(container_spec) + name = self.get_service_name() + svc_id = self.client.create_service(task_tmpl, name=name) + svc_info = self.client.inspect_service(svc_id) + assert 'Secrets' in svc_info['Spec']['TaskTemplate']['ContainerSpec'] + secrets = svc_info['Spec']['TaskTemplate']['ContainerSpec']['Secrets'] + assert secrets[0] == secret_ref + + container = self.get_service_container(name) + assert container is not None + exec_id = self.client.exec_create( + container, 'cat /run/secrets/{0}'.format(secret_name) + ) + assert self.client.exec_start(exec_id) == secret_data + + @requires_api_version('1.25') + def test_create_service_with_unicode_secret(self): + secret_name = 'favorite_touhou' + secret_data = u'東方花映塚' + secret_id = self.client.create_secret(secret_name, secret_data) + self.tmp_secrets.append(secret_id) + secret_ref = docker.types.SecretReference(secret_id, secret_name) + container_spec = docker.types.ContainerSpec( + 'busybox', ['top'], secrets=[secret_ref] + ) + task_tmpl = docker.types.TaskTemplate(container_spec) + name = self.get_service_name() + svc_id = self.client.create_service(task_tmpl, name=name) + svc_info = self.client.inspect_service(svc_id) + assert 'Secrets' in svc_info['Spec']['TaskTemplate']['ContainerSpec'] + secrets = svc_info['Spec']['TaskTemplate']['ContainerSpec']['Secrets'] + assert secrets[0] == secret_ref + + container = self.get_service_container(name) + assert container is not None + exec_id = self.client.exec_create( + container, 'cat /run/secrets/{0}'.format(secret_name) + ) + container_secret = self.client.exec_start(exec_id) + container_secret = container_secret.decode('utf-8') + assert container_secret == secret_data diff --git a/tests/integration/base.py b/tests/integration/base.py index 6f00a46ad5..aa7c6afd1c 100644 --- a/tests/integration/base.py +++ b/tests/integration/base.py @@ -28,6 +28,7 @@ def setUp(self): self.tmp_volumes = [] self.tmp_networks = [] self.tmp_plugins = [] + self.tmp_secrets = [] def tearDown(self): client = docker.from_env(version=TEST_API_VERSION) @@ -52,6 +53,12 @@ def tearDown(self): except docker.errors.APIError: pass + for secret in self.tmp_secrets: + try: + client.api.remove_secret(secret) + except docker.errors.APIError: + pass + for folder in self.tmp_folders: shutil.rmtree(folder) diff --git a/tests/unit/api_container_test.py b/tests/unit/api_container_test.py index abf3613885..51d6678151 100644 --- a/tests/unit/api_container_test.py +++ b/tests/unit/api_container_test.py @@ -45,7 +45,7 @@ def test_start_container_none(self): self.assertEqual( str(excinfo.value), - 'image or container param is undefined', + 'Resource ID was not provided', ) with pytest.raises(ValueError) as excinfo: @@ -53,7 +53,7 @@ def test_start_container_none(self): self.assertEqual( str(excinfo.value), - 'image or container param is undefined', + 'Resource ID was not provided', ) def test_start_container_regression_573(self): @@ -1559,7 +1559,7 @@ def test_inspect_container_undefined_id(self): self.client.inspect_container(arg) self.assertEqual( - excinfo.value.args[0], 'image or container param is undefined' + excinfo.value.args[0], 'Resource ID was not provided' ) def test_container_stats(self): diff --git a/tests/unit/api_image_test.py b/tests/unit/api_image_test.py index fbfb146bb7..36b2a46833 100644 --- a/tests/unit/api_image_test.py +++ b/tests/unit/api_image_test.py @@ -204,7 +204,7 @@ def test_inspect_image_undefined_id(self): self.client.inspect_image(arg) self.assertEqual( - excinfo.value.args[0], 'image or container param is undefined' + excinfo.value.args[0], 'Resource ID was not provided' ) def test_insert_image(self): From 20c6fe31e067f647af52b05ef44ba36aad887882 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 14 Feb 2017 18:24:23 -0800 Subject: [PATCH 0275/1301] Add support for recursive wildcard pattern in .dockerignore Signed-off-by: Joffrey F --- docker/utils/__init__.py | 5 +- docker/utils/build.py | 138 +++++++++++++++++++++++++++++++++++++++ docker/utils/fnmatch.py | 106 ++++++++++++++++++++++++++++++ docker/utils/utils.py | 132 ------------------------------------- tests/unit/utils_test.py | 16 ++++- 5 files changed, 260 insertions(+), 137 deletions(-) create mode 100644 docker/utils/build.py create mode 100644 docker/utils/fnmatch.py diff --git a/docker/utils/__init__.py b/docker/utils/__init__.py index 8f8eb2706a..b758cbd4ec 100644 --- a/docker/utils/__init__.py +++ b/docker/utils/__init__.py @@ -1,7 +1,9 @@ # flake8: noqa +from .build import tar, exclude_paths +from .decorators import check_resource, minimum_version, update_headers from .utils import ( compare_version, convert_port_bindings, convert_volume_binds, - mkbuildcontext, tar, exclude_paths, parse_repository_tag, parse_host, + mkbuildcontext, parse_repository_tag, parse_host, kwargs_from_env, convert_filters, datetime_to_timestamp, create_host_config, parse_bytes, ping_registry, parse_env_file, version_lt, version_gte, decode_json_header, split_command, create_ipam_config, @@ -9,4 +11,3 @@ format_environment, create_archive ) -from .decorators import check_resource, minimum_version, update_headers diff --git a/docker/utils/build.py b/docker/utils/build.py new file mode 100644 index 0000000000..6ba47b39fb --- /dev/null +++ b/docker/utils/build.py @@ -0,0 +1,138 @@ +import os + +from .fnmatch import fnmatch +from .utils import create_archive + + +def tar(path, exclude=None, dockerfile=None, fileobj=None, gzip=False): + root = os.path.abspath(path) + exclude = exclude or [] + + return create_archive( + files=sorted(exclude_paths(root, exclude, dockerfile=dockerfile)), + root=root, fileobj=fileobj, gzip=gzip + ) + + +def exclude_paths(root, patterns, dockerfile=None): + """ + Given a root directory path and a list of .dockerignore patterns, return + an iterator of all paths (both regular files and directories) in the root + directory that do *not* match any of the patterns. + + All paths returned are relative to the root. + """ + if dockerfile is None: + dockerfile = 'Dockerfile' + + exceptions = [p for p in patterns if p.startswith('!')] + + include_patterns = [p[1:] for p in exceptions] + include_patterns += [dockerfile, '.dockerignore'] + + exclude_patterns = list(set(patterns) - set(exceptions)) + + paths = get_paths(root, exclude_patterns, include_patterns, + has_exceptions=len(exceptions) > 0) + + return set(paths).union( + # If the Dockerfile is in a subdirectory that is excluded, get_paths + # will not descend into it and the file will be skipped. This ensures + # it doesn't happen. + set([dockerfile]) + if os.path.exists(os.path.join(root, dockerfile)) else set() + ) + + +def should_include(path, exclude_patterns, include_patterns): + """ + Given a path, a list of exclude patterns, and a list of inclusion patterns: + + 1. Returns True if the path doesn't match any exclusion pattern + 2. Returns False if the path matches an exclusion pattern and doesn't match + an inclusion pattern + 3. Returns true if the path matches an exclusion pattern and matches an + inclusion pattern + """ + for pattern in exclude_patterns: + if match_path(path, pattern): + for pattern in include_patterns: + if match_path(path, pattern): + return True + return False + return True + + +def should_check_directory(directory_path, exclude_patterns, include_patterns): + """ + Given a directory path, a list of exclude patterns, and a list of inclusion + patterns: + + 1. Returns True if the directory path should be included according to + should_include. + 2. Returns True if the directory path is the prefix for an inclusion + pattern + 3. Returns False otherwise + """ + + # To account for exception rules, check directories if their path is a + # a prefix to an inclusion pattern. This logic conforms with the current + # docker logic (2016-10-27): + # https://github.com/docker/docker/blob/bc52939b0455116ab8e0da67869ec81c1a1c3e2c/pkg/archive/archive.go#L640-L671 + + def normalize_path(path): + return path.replace(os.path.sep, '/') + + path_with_slash = normalize_path(directory_path) + '/' + possible_child_patterns = [ + pattern for pattern in map(normalize_path, include_patterns) + if (pattern + '/').startswith(path_with_slash) + ] + directory_included = should_include( + directory_path, exclude_patterns, include_patterns + ) + return directory_included or len(possible_child_patterns) > 0 + + +def get_paths(root, exclude_patterns, include_patterns, has_exceptions=False): + paths = [] + + for parent, dirs, files in os.walk(root, topdown=True, followlinks=False): + parent = os.path.relpath(parent, root) + if parent == '.': + parent = '' + + # Remove excluded patterns from the list of directories to traverse + # by mutating the dirs we're iterating over. + # This looks strange, but is considered the correct way to skip + # traversal. See https://docs.python.org/2/library/os.html#os.walk + dirs[:] = [ + d for d in dirs if should_check_directory( + os.path.join(parent, d), exclude_patterns, include_patterns + ) + ] + + for path in dirs: + if should_include(os.path.join(parent, path), + exclude_patterns, include_patterns): + paths.append(os.path.join(parent, path)) + + for path in files: + if should_include(os.path.join(parent, path), + exclude_patterns, include_patterns): + paths.append(os.path.join(parent, path)) + + return paths + + +def match_path(path, pattern): + pattern = pattern.rstrip('/' + os.path.sep) + if pattern: + pattern = os.path.relpath(pattern) + + if '**' not in pattern: + pattern_components = pattern.split(os.path.sep) + path_components = path.split(os.path.sep)[:len(pattern_components)] + else: + path_components = path.split(os.path.sep) + return fnmatch('/'.join(path_components), pattern) diff --git a/docker/utils/fnmatch.py b/docker/utils/fnmatch.py new file mode 100644 index 0000000000..80bdf77329 --- /dev/null +++ b/docker/utils/fnmatch.py @@ -0,0 +1,106 @@ +"""Filename matching with shell patterns. + +fnmatch(FILENAME, PATTERN) matches according to the local convention. +fnmatchcase(FILENAME, PATTERN) always takes case in account. + +The functions operate by translating the pattern into a regular +expression. They cache the compiled regular expressions for speed. + +The function translate(PATTERN) returns a regular expression +corresponding to PATTERN. (It does not compile it.) +""" + +import re + +__all__ = ["fnmatch", "fnmatchcase", "translate"] + +_cache = {} +_MAXCACHE = 100 + + +def _purge(): + """Clear the pattern cache""" + _cache.clear() + + +def fnmatch(name, pat): + """Test whether FILENAME matches PATTERN. + + Patterns are Unix shell style: + + * matches everything + ? matches any single character + [seq] matches any character in seq + [!seq] matches any char not in seq + + An initial period in FILENAME is not special. + Both FILENAME and PATTERN are first case-normalized + if the operating system requires it. + If you don't want this, use fnmatchcase(FILENAME, PATTERN). + """ + + import os + name = os.path.normcase(name) + pat = os.path.normcase(pat) + return fnmatchcase(name, pat) + + +def fnmatchcase(name, pat): + """Test whether FILENAME matches PATTERN, including case. + + This is a version of fnmatch() which doesn't case-normalize + its arguments. + """ + + try: + re_pat = _cache[pat] + except KeyError: + res = translate(pat) + if len(_cache) >= _MAXCACHE: + _cache.clear() + _cache[pat] = re_pat = re.compile(res) + return re_pat.match(name) is not None + + +def translate(pat): + """Translate a shell PATTERN to a regular expression. + + There is no way to quote meta-characters. + """ + + recursive_mode = False + i, n = 0, len(pat) + res = '' + while i < n: + c = pat[i] + i = i + 1 + if c == '*': + if i < n and pat[i] == '*': + recursive_mode = True + i = i + 1 + res = res + '.*' + elif c == '?': + res = res + '.' + elif c == '[': + j = i + if j < n and pat[j] == '!': + j = j + 1 + if j < n and pat[j] == ']': + j = j + 1 + while j < n and pat[j] != ']': + j = j + 1 + if j >= n: + res = res + '\\[' + else: + stuff = pat[i:j].replace('\\', '\\\\') + i = j + 1 + if stuff[0] == '!': + stuff = '^' + stuff[1:] + elif stuff[0] == '^': + stuff = '\\' + stuff + res = '%s[%s]' % (res, stuff) + elif recursive_mode and c == '/': + res = res + '/?' + else: + res = res + re.escape(c) + return res + '\Z(?ms)' diff --git a/docker/utils/utils.py b/docker/utils/utils.py index 01eb16c32e..d9a6d7c1ba 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -9,7 +9,6 @@ import warnings from distutils.version import StrictVersion from datetime import datetime -from fnmatch import fnmatch import requests import six @@ -79,16 +78,6 @@ def decode_json_header(header): return json.loads(data) -def tar(path, exclude=None, dockerfile=None, fileobj=None, gzip=False): - root = os.path.abspath(path) - exclude = exclude or [] - - return create_archive( - files=sorted(exclude_paths(root, exclude, dockerfile=dockerfile)), - root=root, fileobj=fileobj, gzip=gzip - ) - - def build_file_list(root): files = [] for dirname, dirnames, fnames in os.walk(root): @@ -131,127 +120,6 @@ def create_archive(root, files=None, fileobj=None, gzip=False): return fileobj -def exclude_paths(root, patterns, dockerfile=None): - """ - Given a root directory path and a list of .dockerignore patterns, return - an iterator of all paths (both regular files and directories) in the root - directory that do *not* match any of the patterns. - - All paths returned are relative to the root. - """ - if dockerfile is None: - dockerfile = 'Dockerfile' - - exceptions = [p for p in patterns if p.startswith('!')] - - include_patterns = [p[1:] for p in exceptions] - include_patterns += [dockerfile, '.dockerignore'] - - exclude_patterns = list(set(patterns) - set(exceptions)) - - paths = get_paths(root, exclude_patterns, include_patterns, - has_exceptions=len(exceptions) > 0) - - return set(paths).union( - # If the Dockerfile is in a subdirectory that is excluded, get_paths - # will not descend into it and the file will be skipped. This ensures - # it doesn't happen. - set([dockerfile]) - if os.path.exists(os.path.join(root, dockerfile)) else set() - ) - - -def should_include(path, exclude_patterns, include_patterns): - """ - Given a path, a list of exclude patterns, and a list of inclusion patterns: - - 1. Returns True if the path doesn't match any exclusion pattern - 2. Returns False if the path matches an exclusion pattern and doesn't match - an inclusion pattern - 3. Returns true if the path matches an exclusion pattern and matches an - inclusion pattern - """ - for pattern in exclude_patterns: - if match_path(path, pattern): - for pattern in include_patterns: - if match_path(path, pattern): - return True - return False - return True - - -def should_check_directory(directory_path, exclude_patterns, include_patterns): - """ - Given a directory path, a list of exclude patterns, and a list of inclusion - patterns: - - 1. Returns True if the directory path should be included according to - should_include. - 2. Returns True if the directory path is the prefix for an inclusion - pattern - 3. Returns False otherwise - """ - - # To account for exception rules, check directories if their path is a - # a prefix to an inclusion pattern. This logic conforms with the current - # docker logic (2016-10-27): - # https://github.com/docker/docker/blob/bc52939b0455116ab8e0da67869ec81c1a1c3e2c/pkg/archive/archive.go#L640-L671 - - def normalize_path(path): - return path.replace(os.path.sep, '/') - - path_with_slash = normalize_path(directory_path) + '/' - possible_child_patterns = [ - pattern for pattern in map(normalize_path, include_patterns) - if (pattern + '/').startswith(path_with_slash) - ] - directory_included = should_include( - directory_path, exclude_patterns, include_patterns - ) - return directory_included or len(possible_child_patterns) > 0 - - -def get_paths(root, exclude_patterns, include_patterns, has_exceptions=False): - paths = [] - - for parent, dirs, files in os.walk(root, topdown=True, followlinks=False): - parent = os.path.relpath(parent, root) - if parent == '.': - parent = '' - - # Remove excluded patterns from the list of directories to traverse - # by mutating the dirs we're iterating over. - # This looks strange, but is considered the correct way to skip - # traversal. See https://docs.python.org/2/library/os.html#os.walk - dirs[:] = [ - d for d in dirs if should_check_directory( - os.path.join(parent, d), exclude_patterns, include_patterns - ) - ] - - for path in dirs: - if should_include(os.path.join(parent, path), - exclude_patterns, include_patterns): - paths.append(os.path.join(parent, path)) - - for path in files: - if should_include(os.path.join(parent, path), - exclude_patterns, include_patterns): - paths.append(os.path.join(parent, path)) - - return paths - - -def match_path(path, pattern): - pattern = pattern.rstrip('/' + os.path.sep) - if pattern: - pattern = os.path.relpath(pattern) - - pattern_components = pattern.split(os.path.sep) - path_components = path.split(os.path.sep)[:len(pattern_components)] - return fnmatch('/'.join(path_components), pattern) - - def compare_version(v1, v2): """Compare docker versions diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index 71a8cc7089..854d0ef2cd 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -23,10 +23,9 @@ decode_json_header, tar, split_command, parse_devices, update_headers, ) +from docker.utils.build import should_check_directory from docker.utils.ports import build_port_bindings, split_port -from docker.utils.utils import ( - format_environment, should_check_directory -) +from docker.utils.utils import format_environment from ..helpers import make_tree @@ -811,6 +810,17 @@ def test_subdirectory_win32_pathsep(self): self.all_paths - set(['foo/bar', 'foo/bar/a.py']) ) + def test_double_wildcard(self): + assert self.exclude(['**/a.py']) == convert_paths( + self.all_paths - set( + ['a.py', 'foo/a.py', 'foo/bar/a.py', 'bar/a.py'] + ) + ) + + assert self.exclude(['foo/**/bar']) == convert_paths( + self.all_paths - set(['foo/bar', 'foo/bar/a.py']) + ) + class TarTest(unittest.TestCase): def test_tar_with_excludes(self): From ece3b1978255ae8ef7874b84fe5e9a9c044e2f23 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 15 Feb 2017 17:43:24 -0800 Subject: [PATCH 0276/1301] Add support for storage_opt in host_config Signed-off-by: Joffrey F --- docker/api/container.py | 2 ++ docker/models/containers.py | 3 +++ docker/types/containers.py | 7 ++++++- tests/integration/api_container_test.py | 16 ++++++++++++++++ tests/integration/base.py | 4 ++++ 5 files changed, 31 insertions(+), 1 deletion(-) diff --git a/docker/api/container.py b/docker/api/container.py index 6a764fbf58..d8a2319370 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -547,6 +547,8 @@ def create_host_config(self, *args, **kwargs): security_opt (:py:class:`list`): A list of string values to customize labels for MLS systems, such as SELinux. shm_size (str or int): Size of /dev/shm (e.g. ``1G``). + storage_opt (dict): Storage driver options per container as a + key-value mapping. sysctls (dict): Kernel parameters to set in the container. tmpfs (dict): Temporary filesystems to mount, as a dictionary mapping a path inside the container to options for that path. diff --git a/docker/models/containers.py b/docker/models/containers.py index c4a4add440..8602531e8a 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -585,6 +585,8 @@ def run(self, image, command=None, stdout=True, stderr=False, Default: ``False``. stop_signal (str): The stop signal to use to stop the container (e.g. ``SIGINT``). + storage_opt (dict): Storage driver options per container as a + key-value mapping. sysctls (dict): Kernel parameters to set in the container. tmpfs (dict): Temporary filesystems to mount, as a dictionary mapping a path inside the container to options for that path. @@ -828,6 +830,7 @@ def list(self, all=False, before=None, filters=None, limit=-1, since=None): 'restart_policy', 'security_opt', 'shm_size', + 'storage_opt', 'sysctls', 'tmpfs', 'ulimits', diff --git a/docker/types/containers.py b/docker/types/containers.py index 3c0e41e0d7..9a8d1574e8 100644 --- a/docker/types/containers.py +++ b/docker/types/containers.py @@ -117,7 +117,7 @@ def __init__(self, version, binds=None, port_bindings=None, oom_kill_disable=False, shm_size=None, sysctls=None, tmpfs=None, oom_score_adj=None, dns_opt=None, cpu_shares=None, cpuset_cpus=None, userns_mode=None, pids_limit=None, - isolation=None, auto_remove=False): + isolation=None, auto_remove=False, storage_opt=None): if mem_limit is not None: self['Memory'] = parse_bytes(mem_limit) @@ -412,6 +412,11 @@ def __init__(self, version, binds=None, port_bindings=None, raise host_config_version_error('auto_remove', '1.25') self['AutoRemove'] = auto_remove + if storage_opt is not None: + if version_lt(version, '1.24'): + raise host_config_version_error('storage_opt', '1.24') + self['StorageOpt'] = storage_opt + def host_config_type_error(param, param_value, expected): error_msg = 'Invalid type for {0} param: expected {1} but found {2}' diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index 3cede45de2..42a5b4ad36 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -422,6 +422,22 @@ def test_create_with_stop_timeout(self): config = self.client.inspect_container(container) assert config['Config']['StopTimeout'] == 25 + @requires_api_version('1.24') + def test_create_with_storage_opt(self): + if self.client.info()['Driver'] == 'aufs': + return pytest.skip('Not supported on AUFS') + host_config = self.client.create_host_config( + storage_opt={'size': '120G'} + ) + container = self.client.create_container( + BUSYBOX, ['echo', 'test'], host_config=host_config + ) + self.tmp_containers.append(container) + config = self.client.inspect_container(container) + assert config['HostConfig']['StorageOpt'] == { + 'size': '120G' + } + class VolumeBindTest(BaseAPIIntegrationTest): def setUp(self): diff --git a/tests/integration/base.py b/tests/integration/base.py index 8b75acc9c6..503efc5b85 100644 --- a/tests/integration/base.py +++ b/tests/integration/base.py @@ -68,6 +68,10 @@ def setUp(self): version=TEST_API_VERSION, timeout=60, **kwargs_from_env() ) + def tearDown(self): + super(BaseAPIIntegrationTest, self).tearDown() + self.client.close() + def run_container(self, *args, **kwargs): container = self.client.create_container(*args, **kwargs) self.tmp_containers.append(container) From 3a9c83509cc21dd319f03d15b5a3af87dcf29cd4 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 15 Feb 2017 19:02:31 -0800 Subject: [PATCH 0277/1301] Add xfail mark to storageopt test Signed-off-by: Joffrey F --- tests/integration/api_container_test.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index 0e69cdaf1f..07097ed863 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -6,12 +6,14 @@ from docker.constants import IS_WINDOWS_PLATFORM from docker.utils.socket import next_frame_size from docker.utils.socket import read_exactly + import pytest + import six -from ..helpers import requires_api_version +from .base import BUSYBOX, BaseAPIIntegrationTest from .. import helpers -from .base import BaseAPIIntegrationTest, BUSYBOX +from ..helpers import requires_api_version class ListContainersTest(BaseAPIIntegrationTest): @@ -423,9 +425,8 @@ def test_create_with_stop_timeout(self): assert config['Config']['StopTimeout'] == 25 @requires_api_version('1.24') + @pytest.mark.xfail(True, reason='Not supported on most drivers') def test_create_with_storage_opt(self): - if self.client.info()['Driver'] == 'aufs': - return pytest.skip('Not supported on AUFS') host_config = self.client.create_host_config( storage_opt={'size': '120G'} ) From 0a97df1abc7236793b82041626088f54ad8d204f Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 15 Feb 2017 18:42:49 -0800 Subject: [PATCH 0278/1301] Rename cachefrom -> cache_from Fix cache_from integration test Fix image ID detection in ImageCollection.build Signed-off-by: Joffrey F --- docker/api/build.py | 15 +++++----- docker/models/images.py | 7 +++-- tests/integration/api_build_test.py | 44 +++++++++++++++++++---------- 3 files changed, 41 insertions(+), 25 deletions(-) diff --git a/docker/api/build.py b/docker/api/build.py index c009f1a273..5c34c47b38 100644 --- a/docker/api/build.py +++ b/docker/api/build.py @@ -1,11 +1,11 @@ +import json import logging import os import re -import json +from .. import auth from .. import constants from .. import errors -from .. import auth from .. import utils @@ -18,7 +18,7 @@ def build(self, path=None, tag=None, quiet=False, fileobj=None, custom_context=False, encoding=None, pull=False, forcerm=False, dockerfile=None, container_limits=None, decode=False, buildargs=None, gzip=False, shmsize=None, - labels=None, cachefrom=None): + labels=None, cache_from=None): """ Similar to the ``docker build`` command. Either ``path`` or ``fileobj`` needs to be set. ``path`` can be a local path (to a directory @@ -92,7 +92,8 @@ def build(self, path=None, tag=None, quiet=False, fileobj=None, shmsize (int): Size of `/dev/shm` in bytes. The size must be greater than 0. If omitted the system uses 64MB. labels (dict): A dictionary of labels to set on the image. - cachefrom (list): A list of images used for build cache resolution. + cache_from (list): A list of images used for build cache + resolution. Returns: A generator for the build output. @@ -189,12 +190,12 @@ def build(self, path=None, tag=None, quiet=False, fileobj=None, 'labels was only introduced in API version 1.23' ) - if cachefrom: + if cache_from: if utils.version_gte(self._version, '1.25'): - params.update({'cachefrom': json.dumps(cachefrom)}) + params.update({'cachefrom': json.dumps(cache_from)}) else: raise errors.InvalidVersion( - 'cachefrom was only introduced in API version 1.25' + 'cache_from was only introduced in API version 1.25' ) if context is not None: diff --git a/docker/models/images.py b/docker/models/images.py index a749f63b35..51ee6f4ab9 100644 --- a/docker/models/images.py +++ b/docker/models/images.py @@ -141,7 +141,8 @@ def build(self, **kwargs): ``"0-3"``, ``"0,1"`` decode (bool): If set to ``True``, the returned stream will be decoded into dicts on the fly. Default ``False``. - cachefrom (list): A list of images used for build cache resolution. + cache_from (list): A list of images used for build cache + resolution. Returns: (:py:class:`Image`): The built image. @@ -162,10 +163,10 @@ def build(self, **kwargs): return BuildError('Unknown') event = events[-1] if 'stream' in event: - match = re.search(r'Successfully built ([0-9a-f]+)', + match = re.search(r'(Successfully built |sha256:)([0-9a-f]+)', event.get('stream', '')) if match: - image_id = match.group(1) + image_id = match.group(2) return self.get(image_id) raise BuildError(event.get('error') or event) diff --git a/tests/integration/api_build_test.py b/tests/integration/api_build_test.py index c2fd26c1f3..fe5d994dd6 100644 --- a/tests/integration/api_build_test.py +++ b/tests/integration/api_build_test.py @@ -3,13 +3,12 @@ import shutil import tempfile -import pytest -import six - from docker import errors -from ..helpers import requires_api_version +import six + from .base import BaseAPIIntegrationTest +from ..helpers import requires_api_version class BuildTest(BaseAPIIntegrationTest): @@ -155,25 +154,40 @@ def test_build_labels(self): self.assertEqual(info['Config']['Labels'], labels) @requires_api_version('1.25') - @pytest.mark.xfail(reason='Bad test') - def test_build_cachefrom(self): + def test_build_with_cache_from(self): script = io.BytesIO('\n'.join([ - 'FROM scratch', - 'CMD sh -c "echo \'Hello, World!\'"', + 'FROM busybox', + 'ENV FOO=bar', + 'RUN touch baz', + 'RUN touch bax', ]).encode('ascii')) - cachefrom = ['build1'] + stream = self.client.build(fileobj=script, tag='build1') + self.tmp_imgs.append('build1') + for chunk in stream: + pass stream = self.client.build( - fileobj=script, tag='cachefrom', cachefrom=cachefrom + fileobj=script, tag='build2', cache_from=['build1'], + decode=True ) - self.tmp_imgs.append('cachefrom') + self.tmp_imgs.append('build2') + counter = 0 for chunk in stream: - pass + if 'Using cache' in chunk.get('stream', ''): + counter += 1 + assert counter == 3 + self.client.remove_image('build2') - info = self.client.inspect_image('cachefrom') - # FIXME: Config.CacheFrom is not a real thing - self.assertEqual(info['Config']['CacheFrom'], cachefrom) + counter = 0 + stream = self.client.build( + fileobj=script, tag='build2', cache_from=['nosuchtag'], + decode=True + ) + for chunk in stream: + if 'Using cache' in chunk.get('stream', ''): + counter += 1 + assert counter == 0 def test_build_stderr_data(self): control_chars = ['\x1b[91m', '\x1b[0m'] From 1042b06e50cc7db858ba11fc623a6ffc2a7887d8 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 16 Feb 2017 16:00:34 -0800 Subject: [PATCH 0279/1301] Add missing secrets doc Signed-off-by: Joffrey F --- docs/api.rst | 10 ++++++++++ docs/client.rst | 1 + docs/images.rst | 2 +- docs/index.rst | 1 + docs/secrets.rst | 29 +++++++++++++++++++++++++++++ 5 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 docs/secrets.rst diff --git a/docs/api.rst b/docs/api.rst index 52d12aedde..52cd26b2ca 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -97,6 +97,15 @@ Plugins :members: :undoc-members: +Secrets +------- + +.. py:module:: docker.api.secret + +.. rst-class:: hide-signature +.. autoclass:: SecretApiMixin + :members: + :undoc-members: The Docker daemon ----------------- @@ -121,6 +130,7 @@ Configuration types .. autoclass:: Mount .. autoclass:: Resources .. autoclass:: RestartPolicy +.. autoclass:: SecretReference .. autoclass:: ServiceMode .. autoclass:: TaskTemplate .. autoclass:: UpdateConfig diff --git a/docs/client.rst b/docs/client.rst index 5096bcc435..9d9edeb1b2 100644 --- a/docs/client.rst +++ b/docs/client.rst @@ -20,6 +20,7 @@ Client reference .. autoattribute:: networks .. autoattribute:: nodes .. autoattribute:: plugins + .. autoattribute:: secrets .. autoattribute:: services .. autoattribute:: swarm .. autoattribute:: volumes diff --git a/docs/images.rst b/docs/images.rst index 866786ded4..25fcffc83d 100644 --- a/docs/images.rst +++ b/docs/images.rst @@ -14,11 +14,11 @@ Methods available on ``client.images``: .. automethod:: get .. automethod:: list(**kwargs) .. automethod:: load + .. automethod:: prune .. automethod:: pull .. automethod:: push .. automethod:: remove .. automethod:: search - .. automethod:: prune Image objects diff --git a/docs/index.rst b/docs/index.rst index 70f570ea2e..9113bffcc8 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -85,6 +85,7 @@ That's just a taste of what you can do with the Docker SDK for Python. For more, networks nodes plugins + secrets services swarm volumes diff --git a/docs/secrets.rst b/docs/secrets.rst new file mode 100644 index 0000000000..49e149847d --- /dev/null +++ b/docs/secrets.rst @@ -0,0 +1,29 @@ +Secrets +======= + +.. py:module:: docker.models.secrets + +Manage secrets on the server. + +Methods available on ``client.secrets``: + +.. rst-class:: hide-signature +.. py:class:: SecretCollection + + .. automethod:: create + .. automethod:: get + .. automethod:: list + + +Secret objects +-------------- + +.. autoclass:: Secret() + + .. autoattribute:: id + .. autoattribute:: name + .. py:attribute:: attrs + + The raw representation of this object from the server. + + .. automethod:: remove From a90e3e442c8b0f08434fe9c88c8f50d1c38910bb Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 16 Feb 2017 16:12:07 -0800 Subject: [PATCH 0280/1301] Bump version 2.1.0 Signed-off-by: Joffrey F --- docker/version.py | 2 +- docs/change-log.md | 51 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index 957097006b..491566cd9a 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "2.1.0-dev" +version = "2.1.0" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) diff --git a/docs/change-log.md b/docs/change-log.md index 8b6d859ea8..68b27b8bbb 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -1,6 +1,57 @@ Change log ========== +2.1.0 +----- + +[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/27?closed=1) + +### Features + +* Added the following pruning methods: + * In `APIClient`: `prune_containers`, `prune_images`, `prune_networks`, + `prune_volumes` + * In `DockerClient`: `containers.prune`, `images.prune`, `networks.prune`, + `volumes.prune` +* Added support for the plugins API: + * In `APIClient`: `configure_plugin`, `create_plugin`, `disable_plugin`, + `enable_plugin`, `inspect_plugin`, `pull_plugin`, `plugins`, + `plugin_privileges`, `push_plugin`, `remove_plugin` + * In `DockerClient`: `plugins.create`, `plugins.get`, `plugins.install`, + `plugins.list`, and the `Plugin` model. +* Added support for the secrets API: + * In `APIClient`: `create_secret`, `inspect_secret`, `remove_secret`, + `secrets` + * In `DockerClient`: `secret.create`, `secret.get`, `secret.list` and + the `Secret` model. + * Added `secrets` parameter to `ContainerSpec`. Each item in the `secrets` + list must be a `docker.types.SecretReference` instance. +* Added support for `cache_from` in `APIClient.build` and + `DockerClient.images.build`. +* Added support for `auto_remove` and `storage_opt` in + `APIClient.create_host_config` and `DockerClient.containers.run` +* Added support for `stop_timeout` in `APIClient.create_container` and + `DockerClient.containers.run` +* Added support for the `force` parameter in `APIClient.remove_volume` and + `Volume.remove` +* Added support for `max_failure_ratio` and `monitor` in `UpdateConfig` +* Added support for `force_update` in `TaskTemplate` +* Made `name` parameter optional in `APIClient.create_volume` and + `DockerClient.volumes.create` + +### Bugfixes + +* Fixed a bug where building from a directory containing socket-type files + would raise an unexpected `AttributeError`. +* Fixed an issue that was preventing the `DockerClient.swarm.init` method to + take into account arguments passed to it. +* `Image.tag` now correctly returns a boolean value upon completion. +* Fixed several issues related to passing `volumes` in + `DockerClient.containers.run` +* Fixed an issue where `DockerClient.image.build` wouldn't return an `Image` + object even when the build was successful + + 2.0.2 ----- From b791aa10fafd8d64f2d98141aa270e45702bb4b7 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 16 Feb 2017 16:54:45 -0800 Subject: [PATCH 0281/1301] Bump dev version Signed-off-by: Joffrey F --- docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index 491566cd9a..8a12c002ff 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "2.1.0" +version = "2.2.0-dev" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) From 44c31e47e0222d9c43039d2ef55d29c91630ae83 Mon Sep 17 00:00:00 2001 From: Nils Krabshuis Date: Sun, 19 Feb 2017 06:18:26 +0100 Subject: [PATCH 0282/1301] Add ability to set 'Hostname' on a Service. Signed-off-by: Nils Krabshuis --- docker/models/services.py | 2 ++ docker/types/services.py | 9 ++++++--- tests/unit/models_services_test.py | 5 +++-- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/docker/models/services.py b/docker/models/services.py index bd95b5f965..590729eb19 100644 --- a/docker/models/services.py +++ b/docker/models/services.py @@ -96,6 +96,7 @@ def create(self, image, command=None, **kwargs): access and load balance a service. Default: ``None``. env (list of str): Environment variables, in the form ``KEY=val``. + hostname (string): Hostname to set on the container. labels (dict): Labels to apply to the service. log_driver (str): Log driver to use for containers. log_driver_options (dict): Log driver options. @@ -176,6 +177,7 @@ def list(self, **kwargs): 'command', 'args', 'env', + 'hostname', 'workdir', 'user', 'labels', diff --git a/docker/types/services.py b/docker/types/services.py index b903fa434b..ccf920d015 100644 --- a/docker/types/services.py +++ b/docker/types/services.py @@ -70,6 +70,7 @@ class ContainerSpec(dict): image (string): The image name to use for the container. command (string or list): The command to be run in the image. args (:py:class:`list`): Arguments to the command. + hostname (string): The hostname to set on the container. env (dict): Environment variables. dir (string): The working directory for commands to run in. user (string): The user inside the container. @@ -82,9 +83,9 @@ class ContainerSpec(dict): secrets (list of py:class:`SecretReference`): List of secrets to be made available inside the containers. """ - def __init__(self, image, command=None, args=None, env=None, workdir=None, - user=None, labels=None, mounts=None, stop_grace_period=None, - secrets=None): + def __init__(self, image, command=None, args=None, hostname=None, env=None, + workdir=None, user=None, labels=None, mounts=None, + stop_grace_period=None, secrets=None): self['Image'] = image if isinstance(command, six.string_types): @@ -92,6 +93,8 @@ def __init__(self, image, command=None, args=None, env=None, workdir=None, self['Command'] = command self['Args'] = args + if hostname is not None: + self['Hostname'] = hostname if env is not None: if isinstance(env, dict): self['Env'] = format_environment(env) diff --git a/tests/unit/models_services_test.py b/tests/unit/models_services_test.py index c3b63ae087..e7e317d52d 100644 --- a/tests/unit/models_services_test.py +++ b/tests/unit/models_services_test.py @@ -9,6 +9,7 @@ def test_get_create_service_kwargs(self): 'command': 'true', 'name': 'somename', 'labels': {'key': 'value'}, + 'hostname': 'test_host', 'mode': 'global', 'update_config': {'update': 'config'}, 'networks': ['somenet'], @@ -47,6 +48,6 @@ def test_get_create_service_kwargs(self): 'Options': {'foo': 'bar'} } assert set(task_template['ContainerSpec'].keys()) == set([ - 'Image', 'Command', 'Args', 'Env', 'Dir', 'User', 'Labels', - 'Mounts', 'StopGracePeriod' + 'Image', 'Command', 'Args', 'Hostname', 'Env', 'Dir', 'User', + 'Labels', 'Mounts', 'StopGracePeriod' ]) From 9ec31ffa2763c5c7a99b933de945137a8b9162ee Mon Sep 17 00:00:00 2001 From: Nils Krabshuis Date: Sun, 19 Feb 2017 06:26:16 +0100 Subject: [PATCH 0283/1301] Flake8: Fix indent. Signed-off-by: Nils Krabshuis --- docker/types/services.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/types/services.py b/docker/types/services.py index ccf920d015..9291c9bd42 100644 --- a/docker/types/services.py +++ b/docker/types/services.py @@ -84,8 +84,8 @@ class ContainerSpec(dict): made available inside the containers. """ def __init__(self, image, command=None, args=None, hostname=None, env=None, - workdir=None, user=None, labels=None, mounts=None, - stop_grace_period=None, secrets=None): + workdir=None, user=None, labels=None, mounts=None, + stop_grace_period=None, secrets=None): self['Image'] = image if isinstance(command, six.string_types): From cfc11515bcf7d9c76605149ae3a0e0ae7c1e7857 Mon Sep 17 00:00:00 2001 From: crierr Date: Mon, 20 Feb 2017 00:06:27 +0900 Subject: [PATCH 0284/1301] Allow port range like 8000-8010:80 Signed-off-by: SeungJin Oh --- docker/utils/ports.py | 9 ++++++--- tests/unit/utils_test.py | 5 +++++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/docker/utils/ports.py b/docker/utils/ports.py index 326ef94f47..1813f83fe8 100644 --- a/docker/utils/ports.py +++ b/docker/utils/ports.py @@ -24,7 +24,7 @@ def build_port_bindings(ports): return port_bindings -def to_port_range(port): +def to_port_range(port, randomly_available_port = False): if not port: return None @@ -37,6 +37,9 @@ def to_port_range(port): port, protocol = parts protocol = "/" + protocol + if randomly_available_port: + return ["%s%s" % (port, protocol)] + parts = str(port).split('-') if len(parts) == 1: @@ -69,7 +72,7 @@ def split_port(port): external_port, internal_port = parts internal_range = to_port_range(internal_port) - external_range = to_port_range(external_port) + external_range = to_port_range(external_port, len(internal_range) == 1) if internal_range is None or external_range is None: _raise_invalid_port(port) @@ -81,7 +84,7 @@ def split_port(port): external_ip, external_port, internal_port = parts internal_range = to_port_range(internal_port) - external_range = to_port_range(external_port) + external_range = to_port_range(external_port, len(internal_range) == 1) if not external_range: external_range = [None] * len(internal_range) diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index 854d0ef2cd..ad6d9e09b4 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -530,6 +530,11 @@ def test_split_port_range_with_host_port(self): self.assertEqual(internal_port, ["2000", "2001"]) self.assertEqual(external_port, ["1000", "1001"]) + def test_split_port_random_port_range_with_host_port(self): + internal_port, external_port = split_port("1000-1001:2000") + self.assertEqual(internal_port, ["2000"]) + self.assertEqual(external_port, ["1000-1001"]) + def test_split_port_no_host_port(self): internal_port, external_port = split_port("2000") self.assertEqual(internal_port, ["2000"]) From 8e47f297200b0bcf64d89dcb11242aace2e21a6a Mon Sep 17 00:00:00 2001 From: SeungJin Oh Date: Mon, 20 Feb 2017 01:36:09 +0900 Subject: [PATCH 0285/1301] check NoneType before using Signed-off-by: SeungJin Oh --- docker/utils/ports.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docker/utils/ports.py b/docker/utils/ports.py index 1813f83fe8..39b158ac9b 100644 --- a/docker/utils/ports.py +++ b/docker/utils/ports.py @@ -72,9 +72,11 @@ def split_port(port): external_port, internal_port = parts internal_range = to_port_range(internal_port) - external_range = to_port_range(external_port, len(internal_range) == 1) + if internal_range is None: + _raise_invalid_port(port) - if internal_range is None or external_range is None: + external_range = to_port_range(external_port, len(internal_range) == 1) + if external_range is None: _raise_invalid_port(port) if len(internal_range) != len(external_range): From 60e7fd93a81bccf7b3992d5dbe23d958ac229a8f Mon Sep 17 00:00:00 2001 From: SeungJin Oh Date: Mon, 20 Feb 2017 02:05:57 +0900 Subject: [PATCH 0286/1301] passing flake8 test Signed-off-by: SeungJin Oh --- docker/utils/ports.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/utils/ports.py b/docker/utils/ports.py index 39b158ac9b..5bb7079aef 100644 --- a/docker/utils/ports.py +++ b/docker/utils/ports.py @@ -24,7 +24,7 @@ def build_port_bindings(ports): return port_bindings -def to_port_range(port, randomly_available_port = False): +def to_port_range(port, randomly_available_port=False): if not port: return None From b587139d3cb70f7a784870923af6b59f65996653 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damien=20Nad=C3=A9?= Date: Tue, 21 Feb 2017 17:53:02 +0100 Subject: [PATCH 0287/1301] Allow events daemon command to read config.json MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Damien Nadé --- docker/api/daemon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/api/daemon.py b/docker/api/daemon.py index d40631f59c..0334584915 100644 --- a/docker/api/daemon.py +++ b/docker/api/daemon.py @@ -54,7 +54,7 @@ def events(self, since=None, until=None, filters=None, decode=None): } return self._stream_helper( - self.get(self._url('/events'), params=params, stream=True), + self._get(self._url('/events'), params=params, stream=True), decode=decode ) From f36ef399ad127f7f5409c2c4e67c45d45b5b013b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damien=20Nad=C3=A9?= Date: Tue, 21 Feb 2017 18:17:34 +0100 Subject: [PATCH 0288/1301] Fixed events command related unit tests by passing a timeout value MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Damien Nadé --- tests/unit/api_test.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/unit/api_test.py b/tests/unit/api_test.py index 15e4d7cc69..b632d209be 100644 --- a/tests/unit/api_test.py +++ b/tests/unit/api_test.py @@ -228,7 +228,8 @@ def test_events(self): 'GET', url_prefix + 'events', params={'since': None, 'until': None, 'filters': None}, - stream=True + stream=True, + timeout=DEFAULT_TIMEOUT_SECONDS ) def test_events_with_since_until(self): @@ -247,7 +248,8 @@ def test_events_with_since_until(self): 'until': ts + 10, 'filters': None }, - stream=True + stream=True, + timeout=DEFAULT_TIMEOUT_SECONDS ) def test_events_with_filters(self): @@ -265,7 +267,8 @@ def test_events_with_filters(self): 'until': None, 'filters': expected_filters }, - stream=True + stream=True, + timeout=DEFAULT_TIMEOUT_SECONDS ) def _socket_path_for_client_session(self, client): From 48ac7729941f078227060d177d643c2272c5baef Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 23 Feb 2017 17:27:04 -0800 Subject: [PATCH 0289/1301] Add upgrade_plugin method Signed-off-by: Joffrey F --- Jenkinsfile | 2 +- Makefile | 12 ++++----- docker/api/plugin.py | 37 ++++++++++++++++++++++++++++ docker/models/plugins.py | 25 +++++++++++++++++++ tests/integration/api_plugin_test.py | 10 ++++++++ 5 files changed, 79 insertions(+), 7 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 566d5494c1..b5af1bada2 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -35,7 +35,7 @@ def buildImages = { -> } def getAPIVersion = { engineVersion -> - def versionMap = ['1.12': '1.24', '1.13': '1.25'] + def versionMap = ['1.12': '1.24', '1.13': '1.26'] return versionMap[engineVersion.substring(0, 4)] } diff --git a/Makefile b/Makefile index 148c50a4c0..e8fa711aba 100644 --- a/Makefile +++ b/Makefile @@ -44,11 +44,11 @@ integration-test-py3: build-py3 .PHONY: integration-dind integration-dind: build build-py3 docker rm -vf dpy-dind || : - docker run -d --name dpy-dind --privileged dockerswarm/dind:1.13.0 docker daemon\ + docker run -d --name dpy-dind --privileged dockerswarm/dind:1.13.1 docker daemon\ -H tcp://0.0.0.0:2375 - docker run --rm --env="DOCKER_HOST=tcp://docker:2375" --env="DOCKER_TEST_API_VERSION=1.25"\ + docker run --rm --env="DOCKER_HOST=tcp://docker:2375" --env="DOCKER_TEST_API_VERSION=1.26"\ --link=dpy-dind:docker docker-sdk-python py.test tests/integration - docker run --rm --env="DOCKER_HOST=tcp://docker:2375" --env="DOCKER_TEST_API_VERSION=1.25"\ + docker run --rm --env="DOCKER_HOST=tcp://docker:2375" --env="DOCKER_TEST_API_VERSION=1.26"\ --link=dpy-dind:docker docker-sdk-python3 py.test tests/integration docker rm -vf dpy-dind @@ -57,14 +57,14 @@ integration-dind-ssl: build-dind-certs build build-py3 docker run -d --name dpy-dind-certs dpy-dind-certs docker run -d --env="DOCKER_HOST=tcp://localhost:2375" --env="DOCKER_TLS_VERIFY=1"\ --env="DOCKER_CERT_PATH=/certs" --volumes-from dpy-dind-certs --name dpy-dind-ssl\ - -v /tmp --privileged dockerswarm/dind:1.13.0 docker daemon --tlsverify\ + -v /tmp --privileged dockerswarm/dind:1.13.1 docker daemon --tlsverify\ --tlscacert=/certs/ca.pem --tlscert=/certs/server-cert.pem\ --tlskey=/certs/server-key.pem -H tcp://0.0.0.0:2375 docker run --rm --volumes-from dpy-dind-ssl --env="DOCKER_HOST=tcp://docker:2375"\ - --env="DOCKER_TLS_VERIFY=1" --env="DOCKER_CERT_PATH=/certs" --env="DOCKER_TEST_API_VERSION=1.25"\ + --env="DOCKER_TLS_VERIFY=1" --env="DOCKER_CERT_PATH=/certs" --env="DOCKER_TEST_API_VERSION=1.26"\ --link=dpy-dind-ssl:docker docker-sdk-python py.test tests/integration docker run --rm --volumes-from dpy-dind-ssl --env="DOCKER_HOST=tcp://docker:2375"\ - --env="DOCKER_TLS_VERIFY=1" --env="DOCKER_CERT_PATH=/certs" --env="DOCKER_TEST_API_VERSION=1.25"\ + --env="DOCKER_TLS_VERIFY=1" --env="DOCKER_CERT_PATH=/certs" --env="DOCKER_TEST_API_VERSION=1.26"\ --link=dpy-dind-ssl:docker docker-sdk-python3 py.test tests/integration docker rm -vf dpy-dind-ssl dpy-dind-certs diff --git a/docker/api/plugin.py b/docker/api/plugin.py index 772d263387..ba40c88297 100644 --- a/docker/api/plugin.py +++ b/docker/api/plugin.py @@ -195,6 +195,7 @@ def push_plugin(self, name): return self._stream_helper(res, decode=True) @utils.minimum_version('1.25') + @utils.check_resource def remove_plugin(self, name, force=False): """ Remove an installed plugin. @@ -212,3 +213,39 @@ def remove_plugin(self, name, force=False): res = self._delete(url, params={'force': force}) self._raise_for_status(res) return True + + @utils.minimum_version('1.26') + @utils.check_resource + def upgrade_plugin(self, name, remote, privileges): + """ + Upgrade an installed plugin. + + Args: + name (string): Name of the plugin to upgrade. The ``:latest`` + tag is optional and is the default if omitted. + remote (string): Remote reference to upgrade to. The + ``:latest`` tag is optional and is the default if omitted. + privileges (list): A list of privileges the user consents to + grant to the plugin. Can be retrieved using + :py:meth:`~plugin_privileges`. + + Returns: + An iterable object streaming the decoded API logs + """ + + url = self._url('/plugins/{0}/upgrade', name) + params = { + 'remote': remote, + } + + headers = {} + registry, repo_name = auth.resolve_repository_name(remote) + header = auth.get_config_header(self, registry) + if header: + headers['X-Registry-Auth'] = header + response = self._post_json( + url, params=params, headers=headers, data=privileges, + stream=True + ) + self._raise_for_status(response) + return self._stream_helper(response, decode=True) diff --git a/docker/models/plugins.py b/docker/models/plugins.py index 8b6ede95bf..6cdf01ca00 100644 --- a/docker/models/plugins.py +++ b/docker/models/plugins.py @@ -1,3 +1,4 @@ +from .. import errors from .resource import Collection, Model @@ -96,6 +97,30 @@ def remove(self, force=False): """ return self.client.api.remove_plugin(self.name, force=force) + def upgrade(self, remote=None): + """ + Upgrade the plugin. + + Args: + remote (string): Remote reference to upgrade to. The + ``:latest`` tag is optional and is the default if omitted. + Default: this plugin's name. + + Returns: + A generator streaming the decoded API logs + """ + if self.enabled: + raise errors.DockerError( + 'Plugin must be disabled before upgrading.' + ) + + if remote is None: + remote = self.name + privileges = self.client.api.plugin_privileges(remote) + for d in self.client.api.upgrade_plugin(self.name, remote, privileges): + yield d + self._reload() + class PluginCollection(Collection): model = Plugin diff --git a/tests/integration/api_plugin_test.py b/tests/integration/api_plugin_test.py index e90a1088fc..433d44d101 100644 --- a/tests/integration/api_plugin_test.py +++ b/tests/integration/api_plugin_test.py @@ -123,6 +123,16 @@ def test_install_plugin(self): assert self.client.inspect_plugin(SSHFS) assert self.client.enable_plugin(SSHFS) + @requires_api_version('1.26') + def test_upgrade_plugin(self): + pl_data = self.ensure_plugin_installed(SSHFS) + assert pl_data['Enabled'] is False + prv = self.client.plugin_privileges(SSHFS) + logs = [d for d in self.client.upgrade_plugin(SSHFS, SSHFS, prv)] + assert filter(lambda x: x['status'] == 'Download complete', logs) + assert self.client.inspect_plugin(SSHFS) + assert self.client.enable_plugin(SSHFS) + def test_create_plugin(self): plugin_data_dir = os.path.join( os.path.dirname(__file__), 'testdata/dummy-plugin' From 5268a3111621b14c8d713f0a9fb2c8e29ca3b990 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 23 Feb 2017 18:15:40 -0800 Subject: [PATCH 0290/1301] Add Plugin.upgrade to documentation Signed-off-by: Joffrey F --- docs/plugins.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/plugins.rst b/docs/plugins.rst index a171b2bdad..560bc38262 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -35,3 +35,4 @@ Plugin objects .. automethod:: reload .. automethod:: push .. automethod:: remove + .. automethod:: upgrade From 820af34940825da2bdbe74baaf010a6bb7664ab3 Mon Sep 17 00:00:00 2001 From: Ben Doan Date: Sun, 26 Feb 2017 18:33:22 -0600 Subject: [PATCH 0291/1301] removed unused/typoed param from models.containers.list docstring Signed-off-by: Ben Doan --- docker/models/containers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/models/containers.py b/docker/models/containers.py index b7a77875ff..2b8481c0df 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -726,7 +726,7 @@ def list(self, all=False, before=None, filters=None, limit=-1, since=None): Args: all (bool): Show all containers. Only running containers are shown - by default trunc (bool): Truncate output + by default since (str): Show only containers created since Id or Name, include non-running ones before (str): Show only container created before Id or Name, From 5dced6579a48adb4803a053d203b316af79ed74f Mon Sep 17 00:00:00 2001 From: Boaz Shuster Date: Mon, 27 Feb 2017 21:39:38 +0200 Subject: [PATCH 0292/1301] Update assert in test_create_with_restart_policy In https://github.com/docker/docker/pull/30870 a new error message is displayed if the container is restarting. To make "test_create_with_restart_policy" pass against the above change, the test checks that the error message contains "You cannot remove " instead of "You cannot remove a running container" Signed-off-by: Boaz Shuster --- tests/integration/api_container_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index 07097ed863..e261ef3dcc 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -122,7 +122,7 @@ def test_create_with_restart_policy(self): self.client.remove_container(id) err = exc.exception.explanation self.assertIn( - 'You cannot remove a running container', err + 'You cannot remove ', err ) self.client.remove_container(id, force=True) From cfb14fa78f6650c956e679d1c926c0a9a1e7081b Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 23 Feb 2017 17:51:02 -0800 Subject: [PATCH 0293/1301] Add df method Signed-off-by: Joffrey F --- docker/api/daemon.py | 16 ++++++++++++++++ docker/client.py | 4 ++++ docs/client.rst | 1 + tests/integration/client_test.py | 19 +++++++++++++------ 4 files changed, 34 insertions(+), 6 deletions(-) diff --git a/docker/api/daemon.py b/docker/api/daemon.py index 0334584915..00367bc309 100644 --- a/docker/api/daemon.py +++ b/docker/api/daemon.py @@ -7,6 +7,22 @@ class DaemonApiMixin(object): + @utils.minimum_version('1.25') + def df(self): + """ + Get data usage information. + + Returns: + (dict): A dictionary representing different resource categories + and their respective data usage. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + url = self._url('/system/df') + return self._result(self._get(url), True) + def events(self, since=None, until=None, filters=None, decode=None): """ Get real-time events from the server. Similar to the ``docker events`` diff --git a/docker/client.py b/docker/client.py index 09bda67f82..151e1944ef 100644 --- a/docker/client.py +++ b/docker/client.py @@ -155,6 +155,10 @@ def events(self, *args, **kwargs): return self.api.events(*args, **kwargs) events.__doc__ = APIClient.events.__doc__ + def df(self): + return self.api.df() + df.__doc__ = APIClient.df.__doc__ + def info(self, *args, **kwargs): return self.api.info(*args, **kwargs) info.__doc__ = APIClient.info.__doc__ diff --git a/docs/client.rst b/docs/client.rst index 9d9edeb1b2..ac7a256a05 100644 --- a/docs/client.rst +++ b/docs/client.rst @@ -25,6 +25,7 @@ Client reference .. autoattribute:: swarm .. autoattribute:: volumes + .. automethod:: df() .. automethod:: events() .. automethod:: info() .. automethod:: login() diff --git a/tests/integration/client_test.py b/tests/integration/client_test.py index 20e8cd55e7..8f6bd86b84 100644 --- a/tests/integration/client_test.py +++ b/tests/integration/client_test.py @@ -2,21 +2,28 @@ import docker +from ..helpers import requires_api_version from .base import TEST_API_VERSION class ClientTest(unittest.TestCase): + client = docker.from_env(version=TEST_API_VERSION) def test_info(self): - client = docker.from_env(version=TEST_API_VERSION) - info = client.info() + info = self.client.info() assert 'ID' in info assert 'Name' in info def test_ping(self): - client = docker.from_env(version=TEST_API_VERSION) - assert client.ping() is True + assert self.client.ping() is True def test_version(self): - client = docker.from_env(version=TEST_API_VERSION) - assert 'Version' in client.version() + assert 'Version' in self.client.version() + + @requires_api_version('1.25') + def test_df(self): + data = self.client.df() + assert 'LayersSize' in data + assert 'Containers' in data + assert 'Volumes' in data + assert 'Images' in data From 4273d3f63d84523c983927742960c71cc2812806 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 23 Feb 2017 17:31:45 -0800 Subject: [PATCH 0294/1301] Bump default API version to 1.26 Signed-off-by: Joffrey F --- docker/constants.py | 2 +- tests/integration/api_client_test.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/constants.py b/docker/constants.py index 4337fedced..91a65282a1 100644 --- a/docker/constants.py +++ b/docker/constants.py @@ -1,7 +1,7 @@ import sys from .version import version -DEFAULT_DOCKER_API_VERSION = '1.24' +DEFAULT_DOCKER_API_VERSION = '1.26' MINIMUM_DOCKER_API_VERSION = '1.21' DEFAULT_TIMEOUT_SECONDS = 60 STREAM_HEADER_SIZE_BYTES = 8 diff --git a/tests/integration/api_client_test.py b/tests/integration/api_client_test.py index 02bb435ada..1fef783b4d 100644 --- a/tests/integration/api_client_test.py +++ b/tests/integration/api_client_test.py @@ -150,7 +150,7 @@ def test_resource_warnings(self): with warnings.catch_warnings(record=True) as w: warnings.simplefilter('always') - client = docker.APIClient(**kwargs_from_env()) + client = docker.APIClient(version='auto', **kwargs_from_env()) client.images() client.close() del client From 13b5f785a7ab459960aae82fae00e4245e391387 Mon Sep 17 00:00:00 2001 From: Tomasz Madycki Date: Thu, 16 Feb 2017 01:26:03 +0100 Subject: [PATCH 0295/1301] Add init parameter to container HostConfig Signed-off-by: Tomasz Madycki --- docker/models/containers.py | 3 +++ docker/types/containers.py | 8 +++++++- tests/integration/api_container_test.py | 12 ++++++++++++ 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/docker/models/containers.py b/docker/models/containers.py index 2b8481c0df..f297a5d2eb 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -491,6 +491,8 @@ def run(self, image, command=None, stdout=True, stderr=False, group_add (:py:class:`list`): List of additional group names and/or IDs that the container process will run as. hostname (str): Optional hostname for the container. + init (bool): Run an init inside the container that forwards + signals and reaps processes ipc_mode (str): Set the IPC mode for the container. isolation (str): Isolation technology to use. Default: `None`. labels (dict or list): A dictionary of name-value labels (e.g. @@ -814,6 +816,7 @@ def prune(self, filters=None): 'dns', 'extra_hosts', 'group_add', + 'init', 'ipc_mode', 'isolation', 'kernel_memory', diff --git a/docker/types/containers.py b/docker/types/containers.py index 9a8d1574e8..8e8a87d6d3 100644 --- a/docker/types/containers.py +++ b/docker/types/containers.py @@ -117,7 +117,8 @@ def __init__(self, version, binds=None, port_bindings=None, oom_kill_disable=False, shm_size=None, sysctls=None, tmpfs=None, oom_score_adj=None, dns_opt=None, cpu_shares=None, cpuset_cpus=None, userns_mode=None, pids_limit=None, - isolation=None, auto_remove=False, storage_opt=None): + isolation=None, auto_remove=False, storage_opt=None, + init=None): if mem_limit is not None: self['Memory'] = parse_bytes(mem_limit) @@ -417,6 +418,11 @@ def __init__(self, version, binds=None, port_bindings=None, raise host_config_version_error('storage_opt', '1.24') self['StorageOpt'] = storage_opt + if init is not None: + if version_lt(version, '1.25'): + raise host_config_version_error('init', '1.25') + self['Init'] = init + def host_config_type_error(param, param_value, expected): error_msg = 'Invalid type for {0} param: expected {1} but found {2}' diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index e261ef3dcc..a98e8b1169 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -439,6 +439,18 @@ def test_create_with_storage_opt(self): 'size': '120G' } + @requires_api_version('1.25') + def test_create_with_init(self): + ctnr = self.client.create_container( + BUSYBOX, 'true', + host_config=self.client.create_host_config( + init=True + ) + ) + self.tmp_containers.append(ctnr['Id']) + config = self.client.inspect_container(ctnr) + assert config['HostConfig']['Init'] is True + class VolumeBindTest(BaseAPIIntegrationTest): def setUp(self): From 8c6534d7be9130c7889164cd5ec054cb1e051569 Mon Sep 17 00:00:00 2001 From: Tomasz Madycki Date: Fri, 17 Feb 2017 23:18:45 +0100 Subject: [PATCH 0296/1301] Add init_path parameter to container HostConfig Signed-off-by: Tomasz Madycki --- docker/models/containers.py | 2 ++ docker/types/containers.py | 7 ++++++- tests/integration/api_container_test.py | 12 ++++++++++++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/docker/models/containers.py b/docker/models/containers.py index f297a5d2eb..b31c22f6dd 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -493,6 +493,7 @@ def run(self, image, command=None, stdout=True, stderr=False, hostname (str): Optional hostname for the container. init (bool): Run an init inside the container that forwards signals and reaps processes + init_path (bool): Path to the docker-init binary ipc_mode (str): Set the IPC mode for the container. isolation (str): Isolation technology to use. Default: `None`. labels (dict or list): A dictionary of name-value labels (e.g. @@ -817,6 +818,7 @@ def prune(self, filters=None): 'extra_hosts', 'group_add', 'init', + 'init_path', 'ipc_mode', 'isolation', 'kernel_memory', diff --git a/docker/types/containers.py b/docker/types/containers.py index 8e8a87d6d3..4823d9904d 100644 --- a/docker/types/containers.py +++ b/docker/types/containers.py @@ -118,7 +118,7 @@ def __init__(self, version, binds=None, port_bindings=None, tmpfs=None, oom_score_adj=None, dns_opt=None, cpu_shares=None, cpuset_cpus=None, userns_mode=None, pids_limit=None, isolation=None, auto_remove=False, storage_opt=None, - init=None): + init=None, init_path=None): if mem_limit is not None: self['Memory'] = parse_bytes(mem_limit) @@ -423,6 +423,11 @@ def __init__(self, version, binds=None, port_bindings=None, raise host_config_version_error('init', '1.25') self['Init'] = init + if init_path: + if version_lt(version, '1.25'): + raise host_config_version_error('init', '1.25') + self['InitPath'] = init_path + def host_config_type_error(param, param_value, expected): error_msg = 'Invalid type for {0} param: expected {1} but found {2}' diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index a98e8b1169..f539697e9e 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -451,6 +451,18 @@ def test_create_with_init(self): config = self.client.inspect_container(ctnr) assert config['HostConfig']['Init'] is True + @requires_api_version('1.25') + def test_create_with_init_path(self): + ctnr = self.client.create_container( + BUSYBOX, 'true', + host_config=self.client.create_host_config( + init_path="/usr/libexec/docker-init" + ) + ) + self.tmp_containers.append(ctnr['Id']) + config = self.client.inspect_container(ctnr) + assert config['HostConfig']['InitPath'] == "/usr/libexec/docker-init" + class VolumeBindTest(BaseAPIIntegrationTest): def setUp(self): From c0c3cb44973e4376cbd2c187d0287c6505857cca Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 1 Mar 2017 14:00:05 -0800 Subject: [PATCH 0297/1301] Docstring fixes for init and init_path Signed-off-by: Joffrey F --- docker/api/container.py | 3 +++ docker/models/containers.py | 2 +- docker/types/containers.py | 4 ++-- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/docker/api/container.py b/docker/api/container.py index 453e378516..4e7364b6db 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -498,6 +498,9 @@ def create_host_config(self, *args, **kwargs): container, as a mapping of hostname to IP address. group_add (:py:class:`list`): List of additional group names and/or IDs that the container process will run as. + init (bool): Run an init inside the container that forwards + signals and reaps processes + init_path (str): Path to the docker-init binary ipc_mode (str): Set the IPC mode for the container. isolation (str): Isolation technology to use. Default: `None`. links (dict or list of tuples): Either a dictionary mapping name diff --git a/docker/models/containers.py b/docker/models/containers.py index b31c22f6dd..0d328e72a9 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -493,7 +493,7 @@ def run(self, image, command=None, stdout=True, stderr=False, hostname (str): Optional hostname for the container. init (bool): Run an init inside the container that forwards signals and reaps processes - init_path (bool): Path to the docker-init binary + init_path (str): Path to the docker-init binary ipc_mode (str): Set the IPC mode for the container. isolation (str): Isolation technology to use. Default: `None`. labels (dict or list): A dictionary of name-value labels (e.g. diff --git a/docker/types/containers.py b/docker/types/containers.py index 4823d9904d..5a5079a81a 100644 --- a/docker/types/containers.py +++ b/docker/types/containers.py @@ -423,9 +423,9 @@ def __init__(self, version, binds=None, port_bindings=None, raise host_config_version_error('init', '1.25') self['Init'] = init - if init_path: + if init_path is not None: if version_lt(version, '1.25'): - raise host_config_version_error('init', '1.25') + raise host_config_version_error('init_path', '1.25') self['InitPath'] = init_path From eba20084f68fffda28dffad083371ce9c0a27890 Mon Sep 17 00:00:00 2001 From: Lei Gong Date: Thu, 9 Mar 2017 15:40:22 +0800 Subject: [PATCH 0298/1301] fix: Missing exception handling in split_port when no container port "localhost:host_port:" case will raise TypeError exception directly Catch the "TypeError" and give proper error message * docker/utils/ports.py Signed-off-by: Lei Gong --- docker/utils/ports.py | 5 +++++ tests/unit/utils_test.py | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/docker/utils/ports.py b/docker/utils/ports.py index 5bb7079aef..e2aeb8cc52 100644 --- a/docker/utils/ports.py +++ b/docker/utils/ports.py @@ -85,8 +85,13 @@ def split_port(port): return internal_range, external_range external_ip, external_port, internal_port = parts + + if not internal_port: + _raise_invalid_port(port) + internal_range = to_port_range(internal_port) external_range = to_port_range(external_port, len(internal_range) == 1) + if not external_range: external_range = [None] * len(internal_range) diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index ad6d9e09b4..ed84b3a1fd 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -574,6 +574,10 @@ def test_host_only_with_colon(self): self.assertRaises(ValueError, lambda: split_port("localhost:")) + def test_with_no_container_port(self): + self.assertRaises(ValueError, + lambda: split_port("localhost:80:")) + def test_build_port_bindings_with_one_port(self): port_bindings = build_port_bindings(["127.0.0.1:1000:1000"]) self.assertEqual(port_bindings["1000"], [("127.0.0.1", "1000")]) From eb91d709cb1d25d7533199013fc7a4b28bff9cb8 Mon Sep 17 00:00:00 2001 From: Anda Xu Date: Thu, 9 Mar 2017 13:29:17 -0800 Subject: [PATCH 0299/1301] documentation change Signed-off-by: Anda Xu --- docker/models/services.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/models/services.py b/docker/models/services.py index 590729eb19..d19573a4b6 100644 --- a/docker/models/services.py +++ b/docker/models/services.py @@ -100,8 +100,8 @@ def create(self, image, command=None, **kwargs): labels (dict): Labels to apply to the service. log_driver (str): Log driver to use for containers. log_driver_options (dict): Log driver options. - mode (str): Scheduling mode for the service (``replicated`` or - ``global``). Defaults to ``replicated``. + mode (ServiceMode): Scheduling mode for the service. + Default:``None`` mounts (list of str): Mounts for the containers, in the form ``source:target:options``, where options is either ``ro`` or ``rw``. From c2d114c067d4cbc8a6a5cbe5788503e94e29f33c Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 8 Mar 2017 16:10:47 -0800 Subject: [PATCH 0300/1301] Move LinksTest to appropriate file Signed-off-by: Joffrey F --- tests/integration/api_client_test.py | 46 ++----------------------- tests/integration/api_container_test.py | 42 ++++++++++++++++++++++ 2 files changed, 45 insertions(+), 43 deletions(-) diff --git a/tests/integration/api_client_test.py b/tests/integration/api_client_test.py index 1fef783b4d..d7873a8586 100644 --- a/tests/integration/api_client_test.py +++ b/tests/integration/api_client_test.py @@ -8,7 +8,7 @@ import docker from docker.utils import kwargs_from_env -from .base import BaseAPIIntegrationTest, BUSYBOX +from .base import BaseAPIIntegrationTest class InformationTest(BaseAPIIntegrationTest): @@ -23,48 +23,8 @@ def test_info(self): self.assertIn('Containers', res) self.assertIn('Images', res) self.assertIn('Debug', res) - - -class LinkTest(BaseAPIIntegrationTest): - def test_remove_link(self): - # Create containers - container1 = self.client.create_container( - BUSYBOX, 'cat', detach=True, stdin_open=True - ) - container1_id = container1['Id'] - self.tmp_containers.append(container1_id) - self.client.start(container1_id) - - # Create Link - # we don't want the first / - link_path = self.client.inspect_container(container1_id)['Name'][1:] - link_alias = 'mylink' - - container2 = self.client.create_container( - BUSYBOX, 'cat', host_config=self.client.create_host_config( - links={link_path: link_alias} - ) - ) - container2_id = container2['Id'] - self.tmp_containers.append(container2_id) - self.client.start(container2_id) - - # Remove link - linked_name = self.client.inspect_container(container2_id)['Name'][1:] - link_name = '%s/%s' % (linked_name, link_alias) - self.client.remove_container(link_name, link=True) - - # Link is gone - containers = self.client.containers(all=True) - retrieved = [x for x in containers if link_name in x['Names']] - self.assertEqual(len(retrieved), 0) - - # Containers are still there - retrieved = [ - x for x in containers if x['Id'].startswith(container1_id) or - x['Id'].startswith(container2_id) - ] - self.assertEqual(len(retrieved), 2) + print(res) + self.fail() class LoadConfigTest(BaseAPIIntegrationTest): diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index f539697e9e..9514261512 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -1253,3 +1253,45 @@ def test_container_cpuset(self): self.client.start(container) inspect_data = self.client.inspect_container(container) self.assertEqual(inspect_data['HostConfig']['CpusetCpus'], cpuset_cpus) + + +class LinkTest(BaseAPIIntegrationTest): + def test_remove_link(self): + # Create containers + container1 = self.client.create_container( + BUSYBOX, 'cat', detach=True, stdin_open=True + ) + container1_id = container1['Id'] + self.tmp_containers.append(container1_id) + self.client.start(container1_id) + + # Create Link + # we don't want the first / + link_path = self.client.inspect_container(container1_id)['Name'][1:] + link_alias = 'mylink' + + container2 = self.client.create_container( + BUSYBOX, 'cat', host_config=self.client.create_host_config( + links={link_path: link_alias} + ) + ) + container2_id = container2['Id'] + self.tmp_containers.append(container2_id) + self.client.start(container2_id) + + # Remove link + linked_name = self.client.inspect_container(container2_id)['Name'][1:] + link_name = '%s/%s' % (linked_name, link_alias) + self.client.remove_container(link_name, link=True) + + # Link is gone + containers = self.client.containers(all=True) + retrieved = [x for x in containers if link_name in x['Names']] + self.assertEqual(len(retrieved), 0) + + # Containers are still there + retrieved = [ + x for x in containers if x['Id'].startswith(container1_id) or + x['Id'].startswith(container2_id) + ] + self.assertEqual(len(retrieved), 2) From 54b3c364cbe8091768d342db5d167e174b6a416f Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 8 Mar 2017 16:13:56 -0800 Subject: [PATCH 0301/1301] Raise an error when passing an empty string to split_port Signed-off-by: Joffrey F --- docker/utils/ports.py | 2 ++ tests/integration/api_client_test.py | 2 -- tests/unit/utils_test.py | 3 +++ 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/docker/utils/ports.py b/docker/utils/ports.py index e2aeb8cc52..3708958d4e 100644 --- a/docker/utils/ports.py +++ b/docker/utils/ports.py @@ -67,6 +67,8 @@ def split_port(port): if len(parts) == 1: internal_port, = parts + if not internal_port: + _raise_invalid_port(port) return to_port_range(internal_port), None if len(parts) == 2: external_port, internal_port = parts diff --git a/tests/integration/api_client_test.py b/tests/integration/api_client_test.py index d7873a8586..cc641582c0 100644 --- a/tests/integration/api_client_test.py +++ b/tests/integration/api_client_test.py @@ -23,8 +23,6 @@ def test_info(self): self.assertIn('Containers', res) self.assertIn('Images', res) self.assertIn('Debug', res) - print(res) - self.fail() class LoadConfigTest(BaseAPIIntegrationTest): diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index ed84b3a1fd..4c3c3664a2 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -578,6 +578,9 @@ def test_with_no_container_port(self): self.assertRaises(ValueError, lambda: split_port("localhost:80:")) + def test_split_port_empty_string(self): + self.assertRaises(ValueError, lambda: split_port("")) + def test_build_port_bindings_with_one_port(self): port_bindings = build_port_bindings(["127.0.0.1:1000:1000"]) self.assertEqual(port_bindings["1000"], [("127.0.0.1", "1000")]) From be463bb27e5cce3d767b1948a2b6d6ee8a6af726 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 8 Feb 2017 14:56:48 -0800 Subject: [PATCH 0302/1301] Add service_logs command to APIClient and logs command to models.Service Signed-off-by: Joffrey F --- docker/api/service.py | 49 +++++++++++++++++++++++++++++++++++++++ docker/models/services.py | 28 ++++++++++++++++++++++ 2 files changed, 77 insertions(+) diff --git a/docker/api/service.py b/docker/api/service.py index 0b2abdc9af..b7cdc9a09f 100644 --- a/docker/api/service.py +++ b/docker/api/service.py @@ -166,6 +166,55 @@ def services(self, filters=None): url = self._url('/services') return self._result(self._get(url, params=params), True) + @utils.minimum_version('1.25') + def service_logs(self, service, details=False, follow=False, stdout=False, + stderr=False, since=0, timestamps=False, tail='all', + is_tty=None): + """ + Get log stream for a service. + Note: This endpoint works only for services with the ``json-file`` + or ``journald`` logging drivers. + + Args: + service (str): ID or name of the container + details (bool): Show extra details provided to logs. + Default: ``False`` + follow (bool): Keep connection open to read logs as they are + sent by the Engine. Default: ``False`` + stdout (bool): Return logs from ``stdout``. Default: ``False`` + stderr (bool): Return logs from ``stderr``. Default: ``False`` + since (int): UNIX timestamp for the logs staring point. + Default: 0 + timestamps (bool): Add timestamps to every log line. + tail (string or int): Number of log lines to be returned, + counting from the current end of the logs. Specify an + integer or ``'all'`` to output all log lines. + Default: ``all`` + is_tty (bool): Whether the service's :py:class:`ContainerSpec` + enables the TTY option. If omitted, the method will query + the Engine for the information, causing an additional + roundtrip. + + Returns (generator): Logs for the service. + """ + params = { + 'details': details, + 'follow': follow, + 'stdout': stdout, + 'stderr': stderr, + 'since': since, + 'timestamps': timestamps, + 'tail': tail + } + + url = self._url('/services/{0}/logs', service) + res = self._get(url, params=params, stream=True) + if is_tty is None: + is_tty = self.inspect_service( + service + )['Spec']['TaskTemplate']['ContainerSpec'].get('TTY', False) + return self._get_result_tty(True, res, is_tty) + @utils.minimum_version('1.24') def tasks(self, filters=None): """ diff --git a/docker/models/services.py b/docker/models/services.py index d19573a4b6..c10804dedf 100644 --- a/docker/models/services.py +++ b/docker/models/services.py @@ -77,6 +77,34 @@ def update(self, **kwargs): **create_kwargs ) + def logs(self, **kwargs): + """ + Get log stream for the service. + Note: This method works only for services with the ``json-file`` + or ``journald`` logging drivers. + + Args: + details (bool): Show extra details provided to logs. + Default: ``False`` + follow (bool): Keep connection open to read logs as they are + sent by the Engine. Default: ``False`` + stdout (bool): Return logs from ``stdout``. Default: ``False`` + stderr (bool): Return logs from ``stderr``. Default: ``False`` + since (int): UNIX timestamp for the logs staring point. + Default: 0 + timestamps (bool): Add timestamps to every log line. + tail (string or int): Number of log lines to be returned, + counting from the current end of the logs. Specify an + integer or ``'all'`` to output all log lines. + Default: ``all`` + + Returns (generator): Logs for the service. + """ + is_tty = self.attrs['Spec']['TaskTemplate']['ContainerSpec'].get( + 'TTY', False + ) + return self.client.api.service_logs(self.id, is_tty=is_tty, **kwargs) + class ServiceCollection(Collection): """Services on the Docker server.""" From daac15c1fafdde058163ebe031a2565f9d0a9662 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 13 Mar 2017 17:13:26 -0700 Subject: [PATCH 0303/1301] Add service_logs integration test Signed-off-by: Joffrey F --- docker/api/service.py | 3 ++- tests/helpers.py | 10 ++++++++++ tests/integration/api_service_test.py | 22 +++++++++++++++++++--- 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/docker/api/service.py b/docker/api/service.py index b7cdc9a09f..4972c16d1c 100644 --- a/docker/api/service.py +++ b/docker/api/service.py @@ -167,6 +167,7 @@ def services(self, filters=None): return self._result(self._get(url, params=params), True) @utils.minimum_version('1.25') + @utils.check_resource def service_logs(self, service, details=False, follow=False, stdout=False, stderr=False, since=0, timestamps=False, tail='all', is_tty=None): @@ -176,7 +177,7 @@ def service_logs(self, service, details=False, follow=False, stdout=False, or ``journald`` logging drivers. Args: - service (str): ID or name of the container + service (str): ID or name of the service details (bool): Show extra details provided to logs. Default: ``False`` follow (bool): Keep connection open to read logs as they are diff --git a/tests/helpers.py b/tests/helpers.py index e8ba4d6bf9..1d86619a1c 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -1,3 +1,4 @@ +import functools import os import os.path import random @@ -53,6 +54,15 @@ def requires_api_version(version): ) +def requires_experimental(f): + @functools.wraps(f) + def wrapped(self, *args, **kwargs): + if not self.client.info()['ExperimentalBuild']: + pytest.skip('Feature requires Docker Engine experimental mode') + return f(self, *args, **kwargs) + return wrapped + + def wait_on_condition(condition, delay=0.1, timeout=40): start_time = time.time() while not condition(): diff --git a/tests/integration/api_service_test.py b/tests/integration/api_service_test.py index 1dd295dfb5..b366573146 100644 --- a/tests/integration/api_service_test.py +++ b/tests/integration/api_service_test.py @@ -5,7 +5,9 @@ import docker -from ..helpers import force_leave_swarm, requires_api_version +from ..helpers import ( + force_leave_swarm, requires_api_version, requires_experimental +) from .base import BaseAPIIntegrationTest, BUSYBOX @@ -27,13 +29,15 @@ def tearDown(self): def get_service_name(self): return 'dockerpytest_{0:x}'.format(random.getrandbits(64)) - def get_service_container(self, service_name, attempts=20, interval=0.5): + def get_service_container(self, service_name, attempts=20, interval=0.5, + include_stopped=False): # There is some delay between the service's creation and the creation # of the service's containers. This method deals with the uncertainty # when trying to retrieve the container associated with a service. while True: containers = self.client.containers( - filters={'name': [service_name]}, quiet=True + filters={'name': [service_name]}, quiet=True, + all=include_stopped ) if len(containers) > 0: return containers[0] @@ -97,6 +101,18 @@ def test_create_service_simple(self): assert len(services) == 1 assert services[0]['ID'] == svc_id['ID'] + @requires_api_version('1.25') + @requires_experimental + def test_service_logs(self): + name, svc_id = self.create_simple_service() + assert self.get_service_container(name, include_stopped=True) + logs = self.client.service_logs(svc_id, stdout=True, is_tty=False) + log_line = next(logs) + assert 'hello\n' in log_line + assert 'com.docker.swarm.service.id={}'.format( + svc_id['ID'] + ) in log_line + def test_create_service_custom_log_driver(self): container_spec = docker.types.ContainerSpec( BUSYBOX, ['echo', 'hello'] From 279b058752770d44dbd3875b3fb04a549e63708e Mon Sep 17 00:00:00 2001 From: Scott Miller Date: Fri, 17 Mar 2017 13:34:38 -0400 Subject: [PATCH 0304/1301] Add License to PyPi metadata Signed-off-by: Scott Miller --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index 00b8f37d4a..907746f013 100644 --- a/setup.cfg +++ b/setup.cfg @@ -3,3 +3,4 @@ universal = 1 [metadata] description_file = README.rst +license = Apache License 2.0 From aeb5479fd5ea99eabc91cec7ecce2472c7e3abae Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 13 Mar 2017 17:29:50 -0700 Subject: [PATCH 0305/1301] Use experimental engine for testing in Jenkins/Makefile Signed-off-by: Joffrey F --- Jenkinsfile | 2 +- Makefile | 4 ++-- tests/integration/api_service_test.py | 3 +++ 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index b5af1bada2..148568d3d8 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -63,7 +63,7 @@ def runTests = { Map settings -> def testContainerName = "dpy-tests-\$BUILD_NUMBER-\$EXECUTOR_NUMBER-${pythonVersion}-${dockerVersion}" try { sh """docker run -d --name ${dindContainerName} -v /tmp --privileged \\ - dockerswarm/dind:${dockerVersion} docker daemon -H tcp://0.0.0.0:2375 + dockerswarm/dind:${dockerVersion} docker daemon -H tcp://0.0.0.0:2375 --experimental """ sh """docker run \\ --name ${testContainerName} --volumes-from ${dindContainerName} \\ diff --git a/Makefile b/Makefile index e8fa711aba..411795848e 100644 --- a/Makefile +++ b/Makefile @@ -45,7 +45,7 @@ integration-test-py3: build-py3 integration-dind: build build-py3 docker rm -vf dpy-dind || : docker run -d --name dpy-dind --privileged dockerswarm/dind:1.13.1 docker daemon\ - -H tcp://0.0.0.0:2375 + -H tcp://0.0.0.0:2375 --experimental docker run --rm --env="DOCKER_HOST=tcp://docker:2375" --env="DOCKER_TEST_API_VERSION=1.26"\ --link=dpy-dind:docker docker-sdk-python py.test tests/integration docker run --rm --env="DOCKER_HOST=tcp://docker:2375" --env="DOCKER_TEST_API_VERSION=1.26"\ @@ -59,7 +59,7 @@ integration-dind-ssl: build-dind-certs build build-py3 --env="DOCKER_CERT_PATH=/certs" --volumes-from dpy-dind-certs --name dpy-dind-ssl\ -v /tmp --privileged dockerswarm/dind:1.13.1 docker daemon --tlsverify\ --tlscacert=/certs/ca.pem --tlscert=/certs/server-cert.pem\ - --tlskey=/certs/server-key.pem -H tcp://0.0.0.0:2375 + --tlskey=/certs/server-key.pem -H tcp://0.0.0.0:2375 --experimental docker run --rm --volumes-from dpy-dind-ssl --env="DOCKER_HOST=tcp://docker:2375"\ --env="DOCKER_TLS_VERIFY=1" --env="DOCKER_CERT_PATH=/certs" --env="DOCKER_TEST_API_VERSION=1.26"\ --link=dpy-dind-ssl:docker docker-sdk-python py.test tests/integration diff --git a/tests/integration/api_service_test.py b/tests/integration/api_service_test.py index b366573146..3bfabe91dd 100644 --- a/tests/integration/api_service_test.py +++ b/tests/integration/api_service_test.py @@ -4,6 +4,7 @@ import time import docker +import six from ..helpers import ( force_leave_swarm, requires_api_version, requires_experimental @@ -108,6 +109,8 @@ def test_service_logs(self): assert self.get_service_container(name, include_stopped=True) logs = self.client.service_logs(svc_id, stdout=True, is_tty=False) log_line = next(logs) + if six.PY3: + log_line = log_line.decode('utf-8') assert 'hello\n' in log_line assert 'com.docker.swarm.service.id={}'.format( svc_id['ID'] From b7fbbb56c7555138874d81b61044317cf13b3989 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Sat, 18 Mar 2017 15:19:15 -0700 Subject: [PATCH 0306/1301] Remove unsupported --experimental flag from Jenkinsfile Signed-off-by: Joffrey F --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 148568d3d8..b5af1bada2 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -63,7 +63,7 @@ def runTests = { Map settings -> def testContainerName = "dpy-tests-\$BUILD_NUMBER-\$EXECUTOR_NUMBER-${pythonVersion}-${dockerVersion}" try { sh """docker run -d --name ${dindContainerName} -v /tmp --privileged \\ - dockerswarm/dind:${dockerVersion} docker daemon -H tcp://0.0.0.0:2375 --experimental + dockerswarm/dind:${dockerVersion} docker daemon -H tcp://0.0.0.0:2375 """ sh """docker run \\ --name ${testContainerName} --volumes-from ${dindContainerName} \\ From 04c662bd23b66168441b761540cd365cdeca74e8 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 17 Mar 2017 19:04:36 -0700 Subject: [PATCH 0307/1301] Bump 2.2.0, update changelog Signed-off-by: Joffrey F --- docker/version.py | 2 +- docs/change-log.md | 30 ++++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index 8a12c002ff..b2474bd739 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "2.2.0-dev" +version = "2.2.0" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) diff --git a/docs/change-log.md b/docs/change-log.md index 68b27b8bbb..c04cd4be8f 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -1,6 +1,36 @@ Change log ========== +2.2.0 +----- + +[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/30?closed=1) + +### Features + +* Default API version has been bumped to `1.26` (Engine 1.13.1+) +* Upgrade plugin: + * Added the `upgrade_plugin` method to the `APIClient` class + * Added the `upgrade` method to the `Plugin` class +* Service logs: + * Added the `service_logs` method to the `APIClient` class + * Added the `logs` method to the `Service` class +* Added the `df` method to `APIClient` and `DockerClient` +* Added support for `init` and `init_path` parameters in `HostConfig` + and `DockerClient.containers.run` +* Added support for `hostname` parameter in `ContainerSpec` and + `DockerClient.service.create` +* Added support for port range to single port in port mappings + (e.g. `8000-8010:80`) + +### Bugfixes + +* Fixed a bug where a missing container port in a port mapping would raise + an unexpected `TypeError` +* Fixed a bug where the `events` method in `APIClient` and `DockerClient` + would not respect custom headers set in `config.json` + + 2.1.0 ----- From e0e740438087971bc976a57cdb8d3044fb3fecd4 Mon Sep 17 00:00:00 2001 From: alex-dr Date: Wed, 22 Mar 2017 00:41:32 -0400 Subject: [PATCH 0308/1301] Fix APIError status_code property for client/server errors requests.Response objects evaluate as falsy when the status_code attribute is in the 400-500 range. Therefore we are assured that prior to this change, APIError would show `is_server_error() == False` when generated with a 500-level response and `is_client_error() == False` when generated with a 400-level response. This is not desirable. Added some seemingly dry (not DRY) unit tests to ensure nothing silly slips back in here. Signed-off-by: alex-dr --- docker/errors.py | 2 +- tests/unit/errors_test.py | 65 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 1 deletion(-) diff --git a/docker/errors.py b/docker/errors.py index d9b197d1a3..03b89c11e3 100644 --- a/docker/errors.py +++ b/docker/errors.py @@ -59,7 +59,7 @@ def __str__(self): @property def status_code(self): - if self.response: + if self.response is not None: return self.response.status_code def is_client_error(self): diff --git a/tests/unit/errors_test.py b/tests/unit/errors_test.py index 876ede3b5f..b78af4e109 100644 --- a/tests/unit/errors_test.py +++ b/tests/unit/errors_test.py @@ -1,5 +1,7 @@ import unittest +import requests + from docker.errors import (APIError, DockerException, create_unexpected_kwargs_error) @@ -11,6 +13,69 @@ def test_api_error_is_caught_by_dockerexception(self): except DockerException: pass + def test_status_code_200(self): + """The status_code property is present with 200 response.""" + resp = requests.Response() + resp.status_code = 200 + err = APIError('', response=resp) + assert err.status_code == 200 + + def test_status_code_400(self): + """The status_code property is present with 400 response.""" + resp = requests.Response() + resp.status_code = 400 + err = APIError('', response=resp) + assert err.status_code == 400 + + def test_status_code_500(self): + """The status_code property is present with 500 response.""" + resp = requests.Response() + resp.status_code = 500 + err = APIError('', response=resp) + assert err.status_code == 500 + + def test_is_server_error_200(self): + """Report not server error on 200 response.""" + resp = requests.Response() + resp.status_code = 200 + err = APIError('', response=resp) + assert err.is_server_error() is False + + def test_is_server_error_300(self): + """Report not server error on 300 response.""" + resp = requests.Response() + resp.status_code = 300 + err = APIError('', response=resp) + assert err.is_server_error() is False + + def test_is_server_error_400(self): + """Report not server error on 400 response.""" + resp = requests.Response() + resp.status_code = 400 + err = APIError('', response=resp) + assert err.is_server_error() is False + + def test_is_server_error_500(self): + """Report server error on 500 response.""" + resp = requests.Response() + resp.status_code = 500 + err = APIError('', response=resp) + assert err.is_server_error() is True + + def test_is_client_error_500(self): + """Report not client error on 500 response.""" + resp = requests.Response() + resp.status_code = 500 + err = APIError('', response=resp) + assert err.is_client_error() is False + + def test_is_client_error_400(self): + """Report client error on 400 response.""" + resp = requests.Response() + resp.status_code = 400 + err = APIError('', response=resp) + assert err.is_client_error() is True + class CreateUnexpectedKwargsErrorTest(unittest.TestCase): def test_create_unexpected_kwargs_error_single(self): From c5d35026ce1d889782f639d47285656aa3a6cc6a Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 22 Mar 2017 16:13:33 -0700 Subject: [PATCH 0309/1301] Set infinite timeout for the `events` method Signed-off-by: Joffrey F --- docker/api/daemon.py | 3 ++- tests/integration/api_service_test.py | 4 ++-- tests/unit/api_test.py | 6 +++--- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/docker/api/daemon.py b/docker/api/daemon.py index 00367bc309..91c777f092 100644 --- a/docker/api/daemon.py +++ b/docker/api/daemon.py @@ -68,9 +68,10 @@ def events(self, since=None, until=None, filters=None, decode=None): 'until': until, 'filters': filters } + url = self._url('/events') return self._stream_helper( - self._get(self._url('/events'), params=params, stream=True), + self._get(url, params=params, stream=True, timeout=None), decode=decode ) diff --git a/tests/integration/api_service_test.py b/tests/integration/api_service_test.py index 3bfabe91dd..6858ad0e54 100644 --- a/tests/integration/api_service_test.py +++ b/tests/integration/api_service_test.py @@ -363,7 +363,7 @@ def test_create_service_with_secret(self): self.tmp_secrets.append(secret_id) secret_ref = docker.types.SecretReference(secret_id, secret_name) container_spec = docker.types.ContainerSpec( - 'busybox', ['top'], secrets=[secret_ref] + 'busybox', ['sleep', '999'], secrets=[secret_ref] ) task_tmpl = docker.types.TaskTemplate(container_spec) name = self.get_service_name() @@ -388,7 +388,7 @@ def test_create_service_with_unicode_secret(self): self.tmp_secrets.append(secret_id) secret_ref = docker.types.SecretReference(secret_id, secret_name) container_spec = docker.types.ContainerSpec( - 'busybox', ['top'], secrets=[secret_ref] + 'busybox', ['sleep', '999'], secrets=[secret_ref] ) task_tmpl = docker.types.TaskTemplate(container_spec) name = self.get_service_name() diff --git a/tests/unit/api_test.py b/tests/unit/api_test.py index b632d209be..83848c524a 100644 --- a/tests/unit/api_test.py +++ b/tests/unit/api_test.py @@ -229,7 +229,7 @@ def test_events(self): url_prefix + 'events', params={'since': None, 'until': None, 'filters': None}, stream=True, - timeout=DEFAULT_TIMEOUT_SECONDS + timeout=None ) def test_events_with_since_until(self): @@ -249,7 +249,7 @@ def test_events_with_since_until(self): 'filters': None }, stream=True, - timeout=DEFAULT_TIMEOUT_SECONDS + timeout=None ) def test_events_with_filters(self): @@ -268,7 +268,7 @@ def test_events_with_filters(self): 'filters': expected_filters }, stream=True, - timeout=DEFAULT_TIMEOUT_SECONDS + timeout=None ) def _socket_path_for_client_session(self, client): From 277a6e13c21cc55732883da6654c7bafc3c64347 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 28 Mar 2017 15:04:05 -0700 Subject: [PATCH 0310/1301] Add reload() in docs for Container and Secret classes Signed-off-by: Joffrey F --- docs/change-log.md | 13 +++++++++++++ docs/containers.rst | 1 + docs/secrets.rst | 1 + 3 files changed, 15 insertions(+) diff --git a/docs/change-log.md b/docs/change-log.md index c04cd4be8f..5d9b05b37d 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -1,6 +1,19 @@ Change log ========== +2.2.1 +----- + +[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/32?closed=1) + +### Bugfixes + +* Fixed a bug where the `status_code` attribute of `APIError` exceptions would + not reflect the expected value. +* Fixed an issue where the `events` method would time out unexpectedly if no + data was sent by the engine for a given amount of time. + + 2.2.0 ----- diff --git a/docs/containers.rst b/docs/containers.rst index 9b27a306b8..20529b0eff 100644 --- a/docs/containers.rst +++ b/docs/containers.rst @@ -40,6 +40,7 @@ Container objects .. automethod:: logs .. automethod:: pause .. automethod:: put_archive + .. automethod:: reload .. automethod:: remove .. automethod:: rename .. automethod:: resize diff --git a/docs/secrets.rst b/docs/secrets.rst index 49e149847d..d1c39f1a16 100644 --- a/docs/secrets.rst +++ b/docs/secrets.rst @@ -26,4 +26,5 @@ Secret objects The raw representation of this object from the server. + .. automethod:: reload .. automethod:: remove From 09813334c1a9d54df45f1d11832b5f5421f90e6c Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 28 Mar 2017 16:22:35 -0700 Subject: [PATCH 0311/1301] Add 17.04 (CE) RC1 to list of engine versions to be tested Signed-off-by: Joffrey F --- Jenkinsfile | 6 +++--- Makefile | 15 +++++++++------ 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index b5af1bada2..a61e6d5165 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -7,7 +7,7 @@ def images = [:] // Note: Swarm in dind seem notoriously flimsy with 1.12.1+, which is why we're // sticking with 1.12.0 for the 1.12 series -def dockerVersions = ["1.12.0", "1.13.1"] +def dockerVersions = ["1.12.0", "1.13.1", "17.04.0-ce-rc1"] def buildImage = { name, buildargs, pyTag -> img = docker.image(name) @@ -35,8 +35,8 @@ def buildImages = { -> } def getAPIVersion = { engineVersion -> - def versionMap = ['1.12': '1.24', '1.13': '1.26'] - return versionMap[engineVersion.substring(0, 4)] + def versionMap = ['1.12.': '1.24', '1.13.': '1.26', '17.04': '1.27'] + return versionMap[engineVersion.substring(0, 5)] } def runTests = { Map settings -> diff --git a/Makefile b/Makefile index 411795848e..e4c64e71e7 100644 --- a/Makefile +++ b/Makefile @@ -41,14 +41,17 @@ integration-test: build integration-test-py3: build-py3 docker run --rm -v /var/run/docker.sock:/var/run/docker.sock docker-sdk-python3 py.test tests/integration/${file} +TEST_API_VERSION ?= 1.27 +TEST_ENGINE_VERSION ?= 17.04.0-ce-rc1 + .PHONY: integration-dind integration-dind: build build-py3 docker rm -vf dpy-dind || : - docker run -d --name dpy-dind --privileged dockerswarm/dind:1.13.1 docker daemon\ + docker run -d --name dpy-dind --privileged dockerswarm/dind:${TEST_ENGINE_VERSION} docker daemon\ -H tcp://0.0.0.0:2375 --experimental - docker run --rm --env="DOCKER_HOST=tcp://docker:2375" --env="DOCKER_TEST_API_VERSION=1.26"\ + docker run --rm --env="DOCKER_HOST=tcp://docker:2375" --env="DOCKER_TEST_API_VERSION=${TEST_API_VERSION}"\ --link=dpy-dind:docker docker-sdk-python py.test tests/integration - docker run --rm --env="DOCKER_HOST=tcp://docker:2375" --env="DOCKER_TEST_API_VERSION=1.26"\ + docker run --rm --env="DOCKER_HOST=tcp://docker:2375" --env="DOCKER_TEST_API_VERSION=${TEST_API_VERSION}"\ --link=dpy-dind:docker docker-sdk-python3 py.test tests/integration docker rm -vf dpy-dind @@ -57,14 +60,14 @@ integration-dind-ssl: build-dind-certs build build-py3 docker run -d --name dpy-dind-certs dpy-dind-certs docker run -d --env="DOCKER_HOST=tcp://localhost:2375" --env="DOCKER_TLS_VERIFY=1"\ --env="DOCKER_CERT_PATH=/certs" --volumes-from dpy-dind-certs --name dpy-dind-ssl\ - -v /tmp --privileged dockerswarm/dind:1.13.1 docker daemon --tlsverify\ + -v /tmp --privileged dockerswarm/dind:${TEST_ENGINE_VERSION} docker daemon --tlsverify\ --tlscacert=/certs/ca.pem --tlscert=/certs/server-cert.pem\ --tlskey=/certs/server-key.pem -H tcp://0.0.0.0:2375 --experimental docker run --rm --volumes-from dpy-dind-ssl --env="DOCKER_HOST=tcp://docker:2375"\ - --env="DOCKER_TLS_VERIFY=1" --env="DOCKER_CERT_PATH=/certs" --env="DOCKER_TEST_API_VERSION=1.26"\ + --env="DOCKER_TLS_VERIFY=1" --env="DOCKER_CERT_PATH=/certs" --env="DOCKER_TEST_API_VERSION=${TEST_API_VERSION}"\ --link=dpy-dind-ssl:docker docker-sdk-python py.test tests/integration docker run --rm --volumes-from dpy-dind-ssl --env="DOCKER_HOST=tcp://docker:2375"\ - --env="DOCKER_TLS_VERIFY=1" --env="DOCKER_CERT_PATH=/certs" --env="DOCKER_TEST_API_VERSION=1.26"\ + --env="DOCKER_TLS_VERIFY=1" --env="DOCKER_CERT_PATH=/certs" --env="DOCKER_TEST_API_VERSION=${TEST_API_VERSION}"\ --link=dpy-dind-ssl:docker docker-sdk-python3 py.test tests/integration docker rm -vf dpy-dind-ssl dpy-dind-certs From 7fa30a713e328e007c982e0e1bf113a2f81ef1ba Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 28 Mar 2017 16:29:41 -0700 Subject: [PATCH 0312/1301] Add appveyor.yml config Signed-off-by: Joffrey F --- appveyor.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 appveyor.yml diff --git a/appveyor.yml b/appveyor.yml new file mode 100644 index 0000000000..10e32a540b --- /dev/null +++ b/appveyor.yml @@ -0,0 +1,13 @@ + +version: '{branch}-{build}' + +install: + - "SET PATH=C:\\Python27-x64;C:\\Python27-x64\\Scripts;%PATH%" + - "python --version" + - "pip install tox==2.1.1 virtualenv==13.1.2" + +# Build the binary after tests +build: false + +test_script: + - "tox" From 73d8097b3d63dcc0aba41cef3f7bdcf0459ae8ee Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 28 Mar 2017 16:58:51 -0700 Subject: [PATCH 0313/1301] Fix test issues Signed-off-by: Joffrey F --- appveyor.yml | 1 - docker/utils/build.py | 10 +++-- docker/utils/fnmatch.py | 9 ++--- tests/unit/utils_test.py | 80 +++++++++++++++++++--------------------- tox.ini | 2 +- 5 files changed, 48 insertions(+), 54 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 10e32a540b..1fc67cc024 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,4 +1,3 @@ - version: '{branch}-{build}' install: diff --git a/docker/utils/build.py b/docker/utils/build.py index 6ba47b39fb..79b72495d9 100644 --- a/docker/utils/build.py +++ b/docker/utils/build.py @@ -1,5 +1,6 @@ import os +from ..constants import IS_WINDOWS_PLATFORM from .fnmatch import fnmatch from .utils import create_archive @@ -39,7 +40,7 @@ def exclude_paths(root, patterns, dockerfile=None): # If the Dockerfile is in a subdirectory that is excluded, get_paths # will not descend into it and the file will be skipped. This ensures # it doesn't happen. - set([dockerfile]) + set([dockerfile.replace('/', os.path.sep)]) if os.path.exists(os.path.join(root, dockerfile)) else set() ) @@ -130,9 +131,12 @@ def match_path(path, pattern): if pattern: pattern = os.path.relpath(pattern) + pattern_components = pattern.split(os.path.sep) + if len(pattern_components) == 1 and IS_WINDOWS_PLATFORM: + pattern_components = pattern.split('/') + if '**' not in pattern: - pattern_components = pattern.split(os.path.sep) path_components = path.split(os.path.sep)[:len(pattern_components)] else: path_components = path.split(os.path.sep) - return fnmatch('/'.join(path_components), pattern) + return fnmatch('/'.join(path_components), '/'.join(pattern_components)) diff --git a/docker/utils/fnmatch.py b/docker/utils/fnmatch.py index 80bdf77329..e95b63ceb1 100644 --- a/docker/utils/fnmatch.py +++ b/docker/utils/fnmatch.py @@ -39,15 +39,13 @@ def fnmatch(name, pat): If you don't want this, use fnmatchcase(FILENAME, PATTERN). """ - import os - name = os.path.normcase(name) - pat = os.path.normcase(pat) + name = name.lower() + pat = pat.lower() return fnmatchcase(name, pat) def fnmatchcase(name, pat): """Test whether FILENAME matches PATTERN, including case. - This is a version of fnmatch() which doesn't case-normalize its arguments. """ @@ -67,7 +65,6 @@ def translate(pat): There is no way to quote meta-characters. """ - recursive_mode = False i, n = 0, len(pat) res = '' @@ -100,7 +97,7 @@ def translate(pat): stuff = '\\' + stuff res = '%s[%s]' % (res, stuff) elif recursive_mode and c == '/': - res = res + '/?' + res = res + re.escape(c) + '?' else: res = res + re.escape(c) return res + '\Z(?ms)' diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index 4c3c3664a2..25ed0f9b7f 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -618,9 +618,11 @@ def test_build_port_bindings_with_nonmatching_internal_port_ranges(self): def convert_paths(collection): - if not IS_WINDOWS_PLATFORM: - return collection - return set(map(lambda x: x.replace('/', '\\'), collection)) + return set(map(convert_path, collection)) + + +def convert_path(path): + return path.replace('/', os.path.sep) class ExcludePathsTest(unittest.TestCase): @@ -685,12 +687,12 @@ def test_exclude_custom_dockerfile(self): set(['Dockerfile.alt', '.dockerignore']) assert self.exclude(['*'], dockerfile='foo/Dockerfile3') == \ - set(['foo/Dockerfile3', '.dockerignore']) + convert_paths(set(['foo/Dockerfile3', '.dockerignore'])) def test_exclude_dockerfile_child(self): includes = self.exclude(['foo/'], dockerfile='foo/Dockerfile3') - assert 'foo/Dockerfile3' in includes - assert 'foo/a.py' not in includes + assert convert_path('foo/Dockerfile3') in includes + assert convert_path('foo/a.py') not in includes def test_single_filename(self): assert self.exclude(['a.py']) == convert_paths( @@ -917,6 +919,7 @@ def test_tar_with_directory_symlinks(self): sorted(tar_data.getnames()), ['bar', 'bar/foo', 'foo'] ) + @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='No UNIX sockets on Win32') def test_tar_socket_file(self): base = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, base) @@ -945,62 +948,53 @@ class ShouldCheckDirectoryTest(unittest.TestCase): ] def test_should_check_directory_not_excluded(self): - self.assertTrue( - should_check_directory('not_excluded', self.exclude_patterns, - self.include_patterns) + assert should_check_directory( + 'not_excluded', self.exclude_patterns, self.include_patterns ) - - self.assertTrue( - should_check_directory('dir/with', self.exclude_patterns, - self.include_patterns) + assert should_check_directory( + convert_path('dir/with'), self.exclude_patterns, + self.include_patterns ) def test_shoud_check_parent_directories_of_excluded(self): - self.assertTrue( - should_check_directory('dir', self.exclude_patterns, - self.include_patterns) + assert should_check_directory( + 'dir', self.exclude_patterns, self.include_patterns ) - self.assertTrue( - should_check_directory('dir/with', self.exclude_patterns, - self.include_patterns) + assert should_check_directory( + convert_path('dir/with'), self.exclude_patterns, + self.include_patterns ) def test_should_not_check_excluded_directories_with_no_exceptions(self): - self.assertFalse( - should_check_directory('exclude_rather_large_directory', - self.exclude_patterns, self.include_patterns - ) + assert not should_check_directory( + 'exclude_rather_large_directory', self.exclude_patterns, + self.include_patterns ) - self.assertFalse( - should_check_directory('dir/with/subdir_excluded', - self.exclude_patterns, self.include_patterns - ) + assert not should_check_directory( + convert_path('dir/with/subdir_excluded'), self.exclude_patterns, + self.include_patterns ) def test_should_check_excluded_directory_with_exceptions(self): - self.assertTrue( - should_check_directory('dir/with/exceptions', - self.exclude_patterns, self.include_patterns - ) + assert should_check_directory( + convert_path('dir/with/exceptions'), self.exclude_patterns, + self.include_patterns ) - self.assertTrue( - should_check_directory('dir/with/exceptions/in', - self.exclude_patterns, self.include_patterns - ) + assert should_check_directory( + convert_path('dir/with/exceptions/in'), self.exclude_patterns, + self.include_patterns ) def test_should_not_check_siblings_of_exceptions(self): - self.assertFalse( - should_check_directory('dir/with/exceptions/but_not_here', - self.exclude_patterns, self.include_patterns - ) + assert not should_check_directory( + convert_path('dir/with/exceptions/but_not_here'), + self.exclude_patterns, self.include_patterns ) def test_should_check_subdirectories_of_exceptions(self): - self.assertTrue( - should_check_directory('dir/with/exceptions/like_this_one/subdir', - self.exclude_patterns, self.include_patterns - ) + assert should_check_directory( + convert_path('dir/with/exceptions/like_this_one/subdir'), + self.exclude_patterns, self.include_patterns ) diff --git a/tox.ini b/tox.ini index 1a41c6edac..5a5e5415ad 100644 --- a/tox.ini +++ b/tox.ini @@ -5,7 +5,7 @@ skipsdist=True [testenv] usedevelop=True commands = - py.test --cov=docker {posargs:tests/unit} + py.test -v --cov=docker {posargs:tests/unit} deps = -r{toxinidir}/test-requirements.txt -r{toxinidir}/requirements.txt From eea1271cd0d407644bcc2a4a110dc408604fa917 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 3 Apr 2017 16:45:09 -0700 Subject: [PATCH 0314/1301] Update mentions of the default API version in docs Signed-off-by: Joffrey F --- docker/api/client.py | 2 +- docker/client.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/api/client.py b/docker/api/client.py index 99d7879cb8..749b061dce 100644 --- a/docker/api/client.py +++ b/docker/api/client.py @@ -75,7 +75,7 @@ class APIClient( base_url (str): URL to the Docker server. For example, ``unix:///var/run/docker.sock`` or ``tcp://127.0.0.1:1234``. version (str): The version of the API to use. Set to ``auto`` to - automatically detect the server's version. Default: ``1.24`` + automatically detect the server's version. Default: ``1.26`` timeout (int): Default timeout for API calls, in seconds. tls (bool or :py:class:`~docker.tls.TLSConfig`): Enable TLS. Pass ``True`` to enable it with default options, or pass a diff --git a/docker/client.py b/docker/client.py index 151e1944ef..09abd63322 100644 --- a/docker/client.py +++ b/docker/client.py @@ -24,7 +24,7 @@ class DockerClient(object): base_url (str): URL to the Docker server. For example, ``unix:///var/run/docker.sock`` or ``tcp://127.0.0.1:1234``. version (str): The version of the API to use. Set to ``auto`` to - automatically detect the server's version. Default: ``1.24`` + automatically detect the server's version. Default: ``1.26`` timeout (int): Default timeout for API calls, in seconds. tls (bool or :py:class:`~docker.tls.TLSConfig`): Enable TLS. Pass ``True`` to enable it with default options, or pass a @@ -58,7 +58,7 @@ def from_env(cls, **kwargs): Args: version (str): The version of the API to use. Set to ``auto`` to - automatically detect the server's version. Default: ``1.24`` + automatically detect the server's version. Default: ``1.26`` timeout (int): Default timeout for API calls, in seconds. ssl_version (int): A valid `SSL version`_. assert_hostname (bool): Verify the hostname of the server. From 79edcc28f7b87785578f13c99145c33db81697e5 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 5 Apr 2017 17:05:08 -0700 Subject: [PATCH 0315/1301] Add support for volume_driver in HostConfig Some cleanup in ContainerConfig + warning if volume_driver is provided (API>1.20) Signed-off-by: Joffrey F --- docker/types/containers.py | 91 ++++++++++++++++++-------------- tests/unit/api_container_test.py | 11 ++-- tests/unit/dockertypes_test.py | 25 ++++++++- 3 files changed, 79 insertions(+), 48 deletions(-) diff --git a/docker/types/containers.py b/docker/types/containers.py index 5a5079a81a..0af24cb845 100644 --- a/docker/types/containers.py +++ b/docker/types/containers.py @@ -118,7 +118,7 @@ def __init__(self, version, binds=None, port_bindings=None, tmpfs=None, oom_score_adj=None, dns_opt=None, cpu_shares=None, cpuset_cpus=None, userns_mode=None, pids_limit=None, isolation=None, auto_remove=False, storage_opt=None, - init=None, init_path=None): + init=None, init_path=None, volume_driver=None): if mem_limit is not None: self['Memory'] = parse_bytes(mem_limit) @@ -428,6 +428,11 @@ def __init__(self, version, binds=None, port_bindings=None, raise host_config_version_error('init_path', '1.25') self['InitPath'] = init_path + if volume_driver is not None: + if version_lt(version, '1.21'): + raise host_config_version_error('volume_driver', '1.21') + self['VolumeDriver'] = volume_driver + def host_config_type_error(param, param_value, expected): error_msg = 'Invalid type for {0} param: expected {1} but found {2}' @@ -456,43 +461,27 @@ def __init__( stop_signal=None, networking_config=None, healthcheck=None, stop_timeout=None ): - if isinstance(command, six.string_types): - command = split_command(command) - - if isinstance(entrypoint, six.string_types): - entrypoint = split_command(entrypoint) - - if isinstance(environment, dict): - environment = format_environment(environment) - - if labels is not None and version_lt(version, '1.18'): - raise errors.InvalidVersion( - 'labels were only introduced in API version 1.18' - ) + if version_gte(version, '1.10'): + message = ('{0!r} parameter has no effect on create_container().' + ' It has been moved to host_config') + if dns is not None: + raise errors.InvalidVersion(message.format('dns')) + if volumes_from is not None: + raise errors.InvalidVersion(message.format('volumes_from')) - if cpuset is not None or cpu_shares is not None: - if version_gte(version, '1.18'): + if version_lt(version, '1.18'): + if labels is not None: + raise errors.InvalidVersion( + 'labels were only introduced in API version 1.18' + ) + else: + if cpuset is not None or cpu_shares is not None: warnings.warn( 'The cpuset_cpus and cpu_shares options have been moved to' ' host_config in API version 1.18, and will be removed', DeprecationWarning ) - if stop_signal is not None and version_lt(version, '1.21'): - raise errors.InvalidVersion( - 'stop_signal was only introduced in API version 1.21' - ) - - if stop_timeout is not None and version_lt(version, '1.25'): - raise errors.InvalidVersion( - 'stop_timeout was only introduced in API version 1.25' - ) - - if healthcheck is not None and version_lt(version, '1.24'): - raise errors.InvalidVersion( - 'Health options were only introduced in API version 1.24' - ) - if version_lt(version, '1.19'): if volume_driver is not None: raise errors.InvalidVersion( @@ -513,6 +502,38 @@ def __init__( 'version 1.19' ) + if version_lt(version, '1.21'): + if stop_signal is not None: + raise errors.InvalidVersion( + 'stop_signal was only introduced in API version 1.21' + ) + else: + if volume_driver is not None: + warnings.warn( + 'The volume_driver option has been moved to' + ' host_config in API version 1.21, and will be removed', + DeprecationWarning + ) + + if stop_timeout is not None and version_lt(version, '1.25'): + raise errors.InvalidVersion( + 'stop_timeout was only introduced in API version 1.25' + ) + + if healthcheck is not None and version_lt(version, '1.24'): + raise errors.InvalidVersion( + 'Health options were only introduced in API version 1.24' + ) + + if isinstance(command, six.string_types): + command = split_command(command) + + if isinstance(entrypoint, six.string_types): + entrypoint = split_command(entrypoint) + + if isinstance(environment, dict): + environment = format_environment(environment) + if isinstance(labels, list): labels = dict((lbl, six.text_type('')) for lbl in labels) @@ -566,14 +587,6 @@ def __init__( attach_stdin = True stdin_once = True - if version_gte(version, '1.10'): - message = ('{0!r} parameter has no effect on create_container().' - ' It has been moved to host_config') - if dns is not None: - raise errors.InvalidVersion(message.format('dns')) - if volumes_from is not None: - raise errors.InvalidVersion(message.format('volumes_from')) - self.update({ 'Hostname': hostname, 'Domainname': domainname, diff --git a/tests/unit/api_container_test.py b/tests/unit/api_container_test.py index 51d6678151..ad79c5c6fe 100644 --- a/tests/unit/api_container_test.py +++ b/tests/unit/api_container_test.py @@ -407,11 +407,8 @@ def test_create_container_with_volumes_from(self): {'Content-Type': 'application/json'}) def test_create_container_empty_volumes_from(self): - self.client.create_container('busybox', 'true', volumes_from=[]) - - args = fake_request.call_args - data = json.loads(args[1]['data']) - self.assertTrue('VolumesFrom' not in data) + with pytest.raises(docker.errors.InvalidVersion): + self.client.create_container('busybox', 'true', volumes_from=[]) def test_create_named_container(self): self.client.create_container('busybox', 'true', @@ -978,11 +975,11 @@ def test_create_container_with_named_volume(self): self.client.create_container( 'busybox', 'true', host_config=self.client.create_host_config( + volume_driver='foodriver', binds={volume_name: { "bind": mount_dest, "ro": False }}), - volume_driver='foodriver', ) args = fake_request.call_args @@ -990,8 +987,8 @@ def test_create_container_with_named_volume(self): args[0][1], url_prefix + 'containers/create' ) expected_payload = self.base_create_payload() - expected_payload['VolumeDriver'] = 'foodriver' expected_payload['HostConfig'] = self.client.create_host_config() + expected_payload['HostConfig']['VolumeDriver'] = 'foodriver' expected_payload['HostConfig']['Binds'] = ["name:/mnt:rw"] self.assertEqual(json.loads(args[1]['data']), expected_payload) self.assertEqual(args[1]['headers'], diff --git a/tests/unit/dockertypes_test.py b/tests/unit/dockertypes_test.py index 5c470ffa2f..cb1d90ca2d 100644 --- a/tests/unit/dockertypes_test.py +++ b/tests/unit/dockertypes_test.py @@ -1,14 +1,15 @@ # -*- coding: utf-8 -*- import unittest +import warnings import pytest from docker.constants import DEFAULT_DOCKER_API_VERSION from docker.errors import InvalidArgument, InvalidVersion from docker.types import ( - EndpointConfig, HostConfig, IPAMConfig, IPAMPool, LogConfig, Mount, - ServiceMode, Ulimit, + ContainerConfig, EndpointConfig, HostConfig, IPAMConfig, IPAMPool, + LogConfig, Mount, ServiceMode, Ulimit, ) try: @@ -165,6 +166,26 @@ def test_create_host_config_invalid_mem_swappiness(self): with pytest.raises(TypeError): create_host_config(version='1.24', mem_swappiness='40') + def test_create_host_config_with_volume_driver(self): + with pytest.raises(InvalidVersion): + create_host_config(version='1.20', volume_driver='local') + + config = create_host_config(version='1.21', volume_driver='local') + assert config.get('VolumeDriver') == 'local' + + +class ContainerConfigTest(unittest.TestCase): + def test_create_container_config_volume_driver_warning(self): + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter('always') + ContainerConfig( + version='1.21', image='scratch', command=None, + volume_driver='local' + ) + + assert len(w) == 1 + assert 'The volume_driver option has been moved' in str(w[0].message) + class UlimitTest(unittest.TestCase): def test_create_host_config_dict_ulimit(self): From 3076a9ac40b91458f7e95e3c6167e1bbb92682b1 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 5 Apr 2017 17:07:43 -0700 Subject: [PATCH 0316/1301] 2.3.0-dev Signed-off-by: Joffrey F --- docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index b2474bd739..320dae622d 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "2.2.0" +version = "2.3.0-dev" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) From 1f609208fd43292f276d94d1c7bc7b0e86bb6381 Mon Sep 17 00:00:00 2001 From: Peter Slovak Date: Fri, 7 Apr 2017 00:18:58 +0200 Subject: [PATCH 0317/1301] docs renames: cpu_group->cpu_period, cpu_period->cpu_quota Signed-off-by: Peter Slovak --- docker/api/container.py | 4 ++-- docker/models/containers.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docker/api/container.py b/docker/api/container.py index 4e7364b6db..85e5e90ae7 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -473,8 +473,8 @@ def create_host_config(self, *args, **kwargs): cap_add (list of str): Add kernel capabilities. For example, ``["SYS_ADMIN", "MKNOD"]``. cap_drop (list of str): Drop kernel capabilities. - cpu_group (int): The length of a CPU period in microseconds. - cpu_period (int): Microseconds of CPU time that the container can + cpu_period (int): The length of a CPU period in microseconds. + cpu_quota (int): Microseconds of CPU time that the container can get in a CPU period. cpu_shares (int): CPU shares (relative weight). cpuset_cpus (str): CPUs in which to allow execution (``0-3``, diff --git a/docker/models/containers.py b/docker/models/containers.py index 0d328e72a9..15a5b73789 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -456,8 +456,8 @@ def run(self, image, command=None, stdout=True, stderr=False, cap_add (list of str): Add kernel capabilities. For example, ``["SYS_ADMIN", "MKNOD"]``. cap_drop (list of str): Drop kernel capabilities. - cpu_group (int): The length of a CPU period in microseconds. - cpu_period (int): Microseconds of CPU time that the container can + cpu_period (int): The length of a CPU period in microseconds. + cpu_quota (int): Microseconds of CPU time that the container can get in a CPU period. cpu_shares (int): CPU shares (relative weight). cpuset_cpus (str): CPUs in which to allow execution (``0-3``, From c80762d3766ee0b1879dc5fa155b76b426b7dfe8 Mon Sep 17 00:00:00 2001 From: Peter Slovak Date: Fri, 7 Apr 2017 00:22:22 +0200 Subject: [PATCH 0318/1301] removed duplicate mem_limit arg desc; type now consistent accross models (float->int) Signed-off-by: Peter Slovak --- docker/models/containers.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docker/models/containers.py b/docker/models/containers.py index 15a5b73789..1f882cd54c 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -510,14 +510,12 @@ def run(self, image, command=None, stdout=True, stderr=False, driver. mac_address (str): MAC address to assign to the container. - mem_limit (float or str): Memory limit. Accepts float values + mem_limit (int or str): Memory limit. Accepts float values (which represent the memory limit of the created container in bytes) or a string with a units identification char (``100000b``, ``1000k``, ``128m``, ``1g``). If a string is specified without a units character, bytes are assumed as an intended unit. - mem_limit (str or int): Maximum amount of memory container is - allowed to consume. (e.g. ``1G``). mem_swappiness (int): Tune a container's memory swappiness behavior. Accepts number between 0 and 100. memswap_limit (str or int): Maximum amount of memory + swap a From 112dc12bc04bfc00b62de85017c9cd6e8a45e9c7 Mon Sep 17 00:00:00 2001 From: ewanbarr Date: Wed, 5 Apr 2017 19:06:59 +0200 Subject: [PATCH 0319/1301] Minor typo correction The stdout argument name was repeated in the run method docstring. The second should be replaced by stderr. Signed-off-by: Ewan Barr --- docker/models/containers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/models/containers.py b/docker/models/containers.py index 0d328e72a9..a86e503718 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -585,7 +585,7 @@ def run(self, image, command=None, stdout=True, stderr=False, stdin_open (bool): Keep ``STDIN`` open even if not attached. stdout (bool): Return logs from ``STDOUT`` when ``detach=False``. Default: ``True``. - stdout (bool): Return logs from ``STDERR`` when ``detach=False``. + stderr (bool): Return logs from ``STDERR`` when ``detach=False``. Default: ``False``. stop_signal (str): The stop signal to use to stop the container (e.g. ``SIGINT``). From abd5370d8750dd4472dfef3b51751ec7ffc1ed2c Mon Sep 17 00:00:00 2001 From: Santhosh Manohar Date: Fri, 7 Apr 2017 11:19:37 -0700 Subject: [PATCH 0320/1301] Add 'verbose' option for network inspect api Signed-off-by: Santhosh Manohar --- docker/api/network.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/docker/api/network.py b/docker/api/network.py index 9652228de1..46cd68c5ed 100644 --- a/docker/api/network.py +++ b/docker/api/network.py @@ -168,14 +168,23 @@ def remove_network(self, net_id): self._raise_for_status(res) @minimum_version('1.21') - def inspect_network(self, net_id): + def inspect_network(self, net_id, verbose=False): """ Get detailed information about a network. Args: net_id (str): ID of network + verbose (bool): If set shows the service details across the cluster + in swarm mode """ - url = self._url("/networks/{0}", net_id) + if verbose is True: + if version_lt(self._version, '1.28'): + raise InvalidVersion( + 'Verbose option was introduced in API 1.28' + ) + url = self._url("/networks/{0}?verbose=true", net_id) + else: + url = self._url("/networks/{0}", net_id) res = self._get(url) return self._result(res, json=True) From e506a2b8eaef4cea0453af0f40870e488a66d568 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 10 Apr 2017 16:02:43 -0700 Subject: [PATCH 0321/1301] Standardize handling of verbose param in inspect_network Signed-off-by: Joffrey F --- docker/api/network.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/docker/api/network.py b/docker/api/network.py index 46cd68c5ed..74f4cd2b30 100644 --- a/docker/api/network.py +++ b/docker/api/network.py @@ -168,24 +168,23 @@ def remove_network(self, net_id): self._raise_for_status(res) @minimum_version('1.21') - def inspect_network(self, net_id, verbose=False): + def inspect_network(self, net_id, verbose=None): """ Get detailed information about a network. Args: net_id (str): ID of network - verbose (bool): If set shows the service details across the cluster - in swarm mode + verbose (bool): Show the service details across the cluster in + swarm mode. """ - if verbose is True: + params = {} + if verbose is not None: if version_lt(self._version, '1.28'): - raise InvalidVersion( - 'Verbose option was introduced in API 1.28' - ) - url = self._url("/networks/{0}?verbose=true", net_id) - else: - url = self._url("/networks/{0}", net_id) - res = self._get(url) + raise InvalidVersion('verbose was introduced in API 1.28') + params['verbose'] = verbose + + url = self._url("/networks/{0}", net_id) + res = self._get(url, params=params) return self._result(res, json=True) @check_resource From 6529fa599c9ee2a4576fe8fab92f461f0fba798d Mon Sep 17 00:00:00 2001 From: Frank Sachsenheim Date: Sat, 4 Mar 2017 00:18:56 +0100 Subject: [PATCH 0322/1301] Makes docs builds faster and ensures proper ownership Signed-off-by: Frank Sachsenheim --- .dockerignore | 3 +-- Dockerfile-docs | 15 ++++++++------- Makefile | 4 ++-- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.dockerignore b/.dockerignore index 050b8bce1a..198b23ecbb 100644 --- a/.dockerignore +++ b/.dockerignore @@ -13,5 +13,4 @@ html/* __pycache__ # Compiled Documentation -site/ -Makefile +docs/_build diff --git a/Dockerfile-docs b/Dockerfile-docs index 6f4194009a..105083e8cb 100644 --- a/Dockerfile-docs +++ b/Dockerfile-docs @@ -1,12 +1,13 @@ FROM python:3.5 -RUN mkdir /src -WORKDIR /src +ARG uid=1000 +ARG gid=1000 -COPY requirements.txt /src/requirements.txt -RUN pip install -r requirements.txt +RUN addgroup --gid $gid sphinx \ + && useradd --uid $uid --gid $gid -M sphinx -COPY docs-requirements.txt /src/docs-requirements.txt -RUN pip install -r docs-requirements.txt +WORKDIR /src +COPY requirements.txt docs-requirements.txt ./ +RUN pip install -r requirements.txt -r docs-requirements.txt -COPY . /src +USER sphinx diff --git a/Makefile b/Makefile index e4c64e71e7..cd1174675b 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ build-py3: .PHONY: build-docs build-docs: - docker build -t docker-sdk-python-docs -f Dockerfile-docs . + docker build -t docker-sdk-python-docs -f Dockerfile-docs --build-arg uid=$(shell id -u) --build-arg gid=$(shell id -g) . .PHONY: build-dind-certs build-dind-certs: @@ -77,7 +77,7 @@ flake8: build .PHONY: docs docs: build-docs - docker run --rm -it -v `pwd`:/code docker-sdk-python-docs sphinx-build docs ./_build + docker run --rm -it -v `pwd`:/src docker-sdk-python-docs sphinx-build docs docs/_build .PHONY: shell shell: build From 1cd56b9f0c85af580c59597643a307b3177ab7c9 Mon Sep 17 00:00:00 2001 From: Frank Sachsenheim Date: Sat, 4 Mar 2017 00:22:19 +0100 Subject: [PATCH 0323/1301] Adds a 'labels' property to the container model The Docker API seems to respond with a 'null' value for the 'Labels' attribute from containers that were created with older Docker versions. An empty dictionary is returned in this case. Signed-off-by: Frank Sachsenheim --- docker/models/containers.py | 8 ++++++++ docs/containers.rst | 1 + tests/unit/fake_api.py | 2 +- tests/unit/models_containers_test.py | 5 +++++ 4 files changed, 15 insertions(+), 1 deletion(-) diff --git a/docker/models/containers.py b/docker/models/containers.py index fb10ba90b3..493f180438 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -18,6 +18,14 @@ def name(self): if self.attrs.get('Name') is not None: return self.attrs['Name'].lstrip('/') + @property + def labels(self): + """ + The labels of a container as dictionary. + """ + result = self.attrs['Config'].get('Labels') + return result or {} + @property def status(self): """ diff --git a/docs/containers.rst b/docs/containers.rst index 20529b0eff..b67a066d0c 100644 --- a/docs/containers.rst +++ b/docs/containers.rst @@ -25,6 +25,7 @@ Container objects .. autoattribute:: short_id .. autoattribute:: name .. autoattribute:: status + .. autoattribute:: labels .. py:attribute:: attrs The raw representation of this object from the server. diff --git a/tests/unit/fake_api.py b/tests/unit/fake_api.py index 2d0a0b4541..2914b63ae6 100644 --- a/tests/unit/fake_api.py +++ b/tests/unit/fake_api.py @@ -134,7 +134,7 @@ def get_fake_inspect_container(tty=False): status_code = 200 response = { 'Id': FAKE_CONTAINER_ID, - 'Config': {'Privileged': True, 'Tty': tty}, + 'Config': {'Labels': {'foo': 'bar'}, 'Privileged': True, 'Tty': tty}, 'ID': FAKE_CONTAINER_ID, 'Image': 'busybox:latest', 'Name': 'foobar', diff --git a/tests/unit/models_containers_test.py b/tests/unit/models_containers_test.py index ae1bd12aae..a5ef4a1145 100644 --- a/tests/unit/models_containers_test.py +++ b/tests/unit/models_containers_test.py @@ -390,6 +390,11 @@ def test_kill(self): container.kill(signal=5) client.api.kill.assert_called_with(FAKE_CONTAINER_ID, signal=5) + def test_labels(self): + client = make_fake_client() + container = client.containers.get(FAKE_CONTAINER_ID) + assert container.labels == {'foo': 'bar'} + def test_logs(self): client = make_fake_client() container = client.containers.get(FAKE_CONTAINER_ID) From b585ec59a8a3e7f6f7252d106df0a027eb0b5ab6 Mon Sep 17 00:00:00 2001 From: Frank Sachsenheim Date: Sat, 4 Mar 2017 00:36:11 +0100 Subject: [PATCH 0324/1301] Adds a 'labels' property to the image model Signed-off-by: Frank Sachsenheim --- docker/models/images.py | 8 ++++++++ docs/images.rst | 1 + tests/unit/fake_api.py | 1 + tests/unit/models_images_test.py | 5 +++++ 4 files changed, 15 insertions(+) diff --git a/docker/models/images.py b/docker/models/images.py index 51ee6f4ab9..3fd3dc1956 100644 --- a/docker/models/images.py +++ b/docker/models/images.py @@ -15,6 +15,14 @@ class Image(Model): def __repr__(self): return "<%s: '%s'>" % (self.__class__.__name__, "', '".join(self.tags)) + @property + def labels(self): + """ + The labels of an image as dictionary. + """ + result = self.attrs['Config'].get('Labels') + return result or {} + @property def short_id(self): """ diff --git a/docs/images.rst b/docs/images.rst index 25fcffc83d..44f006c08e 100644 --- a/docs/images.rst +++ b/docs/images.rst @@ -29,6 +29,7 @@ Image objects .. autoattribute:: id .. autoattribute:: short_id .. autoattribute:: tags + .. autoattribute:: labels .. py:attribute:: attrs The raw representation of this object from the server. diff --git a/tests/unit/fake_api.py b/tests/unit/fake_api.py index 2914b63ae6..ff0f1b65cc 100644 --- a/tests/unit/fake_api.py +++ b/tests/unit/fake_api.py @@ -158,6 +158,7 @@ def get_fake_inspect_image(): 'Parent': "27cf784147099545", 'Created': "2013-03-23T22:24:18.818426-07:00", 'Container': FAKE_CONTAINER_ID, + 'Config': {'Labels': {'bar': 'foo'}}, 'ContainerConfig': { "Hostname": "", diff --git a/tests/unit/models_images_test.py b/tests/unit/models_images_test.py index efb2116660..784717be8e 100644 --- a/tests/unit/models_images_test.py +++ b/tests/unit/models_images_test.py @@ -21,6 +21,11 @@ def test_get(self): assert isinstance(image, Image) assert image.id == FAKE_IMAGE_ID + def test_labels(self): + client = make_fake_client() + image = client.images.get(FAKE_IMAGE_ID) + assert image.labels == {'bar': 'foo'} + def test_list(self): client = make_fake_client() images = client.images.list(all=True) From 659090fc99b87bc9bb4fb5533b09d7ba781ac28d Mon Sep 17 00:00:00 2001 From: Frank Sachsenheim Date: Sat, 4 Mar 2017 01:04:20 +0100 Subject: [PATCH 0325/1301] Adds an 'image' property to the container model Signed-off-by: Frank Sachsenheim --- docker/models/containers.py | 10 ++++++++++ docs/containers.rst | 1 + tests/unit/models_containers_test.py | 5 +++++ 3 files changed, 16 insertions(+) diff --git a/docker/models/containers.py b/docker/models/containers.py index 493f180438..93f637252f 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -18,6 +18,16 @@ def name(self): if self.attrs.get('Name') is not None: return self.attrs['Name'].lstrip('/') + @property + def image(self): + """ + The image of the container. + """ + image_id = self.attrs['Image'] + if image_id is None: + return None + return self.client.images.get(image_id.split(':')[1]) + @property def labels(self): """ diff --git a/docs/containers.rst b/docs/containers.rst index b67a066d0c..9ea64ad549 100644 --- a/docs/containers.rst +++ b/docs/containers.rst @@ -25,6 +25,7 @@ Container objects .. autoattribute:: short_id .. autoattribute:: name .. autoattribute:: status + .. autoattribute:: image .. autoattribute:: labels .. py:attribute:: attrs diff --git a/tests/unit/models_containers_test.py b/tests/unit/models_containers_test.py index a5ef4a1145..c594606170 100644 --- a/tests/unit/models_containers_test.py +++ b/tests/unit/models_containers_test.py @@ -384,6 +384,11 @@ def test_get_archive(self): container.get_archive('foo') client.api.get_archive.assert_called_with(FAKE_CONTAINER_ID, 'foo') + def test_image(self): + client = make_fake_client() + container = client.containers.get(FAKE_CONTAINER_ID) + assert container.image.id == FAKE_IMAGE_ID + def test_kill(self): client = make_fake_client() container = client.containers.get(FAKE_CONTAINER_ID) From 9536c8653dc02624fea01ce9bd6ee01df4369269 Mon Sep 17 00:00:00 2001 From: Frank Sachsenheim Date: Sat, 4 Mar 2017 01:10:40 +0100 Subject: [PATCH 0326/1301] Sorts model attributes in api docs alphabetically Signed-off-by: Frank Sachsenheim --- docs/containers.rst | 8 ++++---- docs/images.rst | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/containers.rst b/docs/containers.rst index 9ea64ad549..6c895c6b2d 100644 --- a/docs/containers.rst +++ b/docs/containers.rst @@ -21,13 +21,13 @@ Container objects .. autoclass:: Container() + .. py:attribute:: attrs .. autoattribute:: id - .. autoattribute:: short_id - .. autoattribute:: name - .. autoattribute:: status .. autoattribute:: image .. autoattribute:: labels - .. py:attribute:: attrs + .. autoattribute:: name + .. autoattribute:: short_id + .. autoattribute:: status The raw representation of this object from the server. diff --git a/docs/images.rst b/docs/images.rst index 44f006c08e..3ba06010a6 100644 --- a/docs/images.rst +++ b/docs/images.rst @@ -26,11 +26,11 @@ Image objects .. autoclass:: Image() - .. autoattribute:: id - .. autoattribute:: short_id - .. autoattribute:: tags - .. autoattribute:: labels - .. py:attribute:: attrs +.. py:attribute:: attrs +.. autoattribute:: id +.. autoattribute:: labels +.. autoattribute:: short_id +.. autoattribute:: tags The raw representation of this object from the server. From 3f7d622143cdb6c793ab6beb6dda6a50d17504de Mon Sep 17 00:00:00 2001 From: Alexey Rokhin Date: Wed, 19 Apr 2017 15:06:48 +0300 Subject: [PATCH 0327/1301] Add cpu_count, cpu_percent, cpus parameters to container HostConfig. Signed-off-by: Alexey Rokhin --- docker/models/containers.py | 6 ++++++ docker/types/containers.py | 27 +++++++++++++++++++++++++- tests/unit/api_container_test.py | 30 +++++++++++++++++++++++++++++ tests/unit/dockertypes_test.py | 33 ++++++++++++++++++++++++++++++++ 4 files changed, 95 insertions(+), 1 deletion(-) diff --git a/docker/models/containers.py b/docker/models/containers.py index fb10ba90b3..f623f6b818 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -456,10 +456,13 @@ def run(self, image, command=None, stdout=True, stderr=False, cap_add (list of str): Add kernel capabilities. For example, ``["SYS_ADMIN", "MKNOD"]``. cap_drop (list of str): Drop kernel capabilities. + cpu_count (int): CPU count (Windows only). + cpu_percent (int): CPU percent (Windows only). cpu_period (int): The length of a CPU period in microseconds. cpu_quota (int): Microseconds of CPU time that the container can get in a CPU period. cpu_shares (int): CPU shares (relative weight). + cpus (float): Number of CPUs. cpuset_cpus (str): CPUs in which to allow execution (``0-3``, ``0,1``). detach (bool): Run container in the background and return a @@ -801,9 +804,12 @@ def prune(self, filters=None): 'cap_add', 'cap_drop', 'cgroup_parent', + 'cpu_count', + 'cpu_percent', 'cpu_period', 'cpu_quota', 'cpu_shares', + 'cpus', 'cpuset_cpus', 'device_read_bps', 'device_read_iops', diff --git a/docker/types/containers.py b/docker/types/containers.py index 0af24cb845..8c5e5ef7b5 100644 --- a/docker/types/containers.py +++ b/docker/types/containers.py @@ -118,7 +118,8 @@ def __init__(self, version, binds=None, port_bindings=None, tmpfs=None, oom_score_adj=None, dns_opt=None, cpu_shares=None, cpuset_cpus=None, userns_mode=None, pids_limit=None, isolation=None, auto_remove=False, storage_opt=None, - init=None, init_path=None, volume_driver=None): + init=None, init_path=None, volume_driver=None, + cpu_count=None, cpu_percent=None, cpus=None): if mem_limit is not None: self['Memory'] = parse_bytes(mem_limit) @@ -433,6 +434,30 @@ def __init__(self, version, binds=None, port_bindings=None, raise host_config_version_error('volume_driver', '1.21') self['VolumeDriver'] = volume_driver + if cpu_count: + if not isinstance(cpu_count, int): + raise host_config_type_error('cpu_count', cpu_count, 'int') + if version_lt(version, '1.25'): + raise host_config_version_error('cpu_count', '1.25') + + self['CpuCount'] = cpu_count + + if cpu_percent: + if not isinstance(cpu_percent, int): + raise host_config_type_error('cpu_percent', cpu_percent, 'int') + if version_lt(version, '1.25'): + raise host_config_version_error('cpu_percent', '1.25') + + self['CpuPercent'] = cpu_percent + + if cpus: + if not isinstance(cpus, (float, int)): + raise host_config_type_error('cpus', cpus, 'float') + if version_lt(version, '1.25'): + raise host_config_version_error('cpus', '1.25') + + self['NanoCpus'] = int(1000000000 * cpus) + def host_config_type_error(param, param_value, expected): error_msg = 'Invalid type for {0} param: expected {1} but found {2}' diff --git a/tests/unit/api_container_test.py b/tests/unit/api_container_test.py index ad79c5c6fe..30287110d4 100644 --- a/tests/unit/api_container_test.py +++ b/tests/unit/api_container_test.py @@ -1152,6 +1152,36 @@ def test_create_container_with_unicode_envvars(self): self.assertEqual(args[0][1], url_prefix + 'containers/create') self.assertEqual(json.loads(args[1]['data'])['Env'], expected) + @requires_api_version('1.25') + def test_create_container_with_host_config_cpus(self): + self.client.create_container( + 'busybox', 'ls', host_config=self.client.create_host_config( + cpu_count=1, + cpu_percent=20, + cpus=10 + ) + ) + + args = fake_request.call_args + self.assertEqual(args[0][1], + url_prefix + 'containers/create') + + self.assertEqual(json.loads(args[1]['data']), + json.loads(''' + {"Tty": false, "Image": "busybox", + "Cmd": ["ls"], "AttachStdin": false, + "AttachStderr": true, + "AttachStdout": true, "OpenStdin": false, + "StdinOnce": false, + "NetworkDisabled": false, + "HostConfig": { + "CpuCount": 1, + "CpuPercent": 20, + "NanoCpus": 10000000000, + "NetworkMode": "default" + }}''')) + self.assertEqual(args[1]['headers'], {'Content-Type': 'application/json'}) + class ContainerTest(BaseAPIClientTest): def test_list_containers(self): diff --git a/tests/unit/dockertypes_test.py b/tests/unit/dockertypes_test.py index cb1d90ca2d..c3ee2e263e 100644 --- a/tests/unit/dockertypes_test.py +++ b/tests/unit/dockertypes_test.py @@ -172,6 +172,39 @@ def test_create_host_config_with_volume_driver(self): config = create_host_config(version='1.21', volume_driver='local') assert config.get('VolumeDriver') == 'local' + + def test_create_host_config_invalid_cpu_count_types(self): + with pytest.raises(TypeError): + create_host_config(version='1.25', cpu_count='1') + + def test_create_host_config_with_cpu_count(self): + config = create_host_config(version='1.25', cpu_count=2) + self.assertEqual(config.get('CpuCount'), 2) + self.assertRaises( + InvalidVersion, lambda: create_host_config( + version='1.24', cpu_count=1)) + + def test_create_host_config_invalid_cpu_percent_types(self): + with pytest.raises(TypeError): + create_host_config(version='1.25', cpu_percent='1') + + def test_create_host_config_with_cpu_percent(self): + config = create_host_config(version='1.25', cpu_percent=15) + self.assertEqual(config.get('CpuPercent'), 15) + self.assertRaises( + InvalidVersion, lambda: create_host_config( + version='1.24', cpu_percent=10)) + + def test_create_host_config_invalid_cpus_types(self): + with pytest.raises(TypeError): + create_host_config(version='1.25', cpus='0') + + def test_create_host_config_with_cpus(self): + config = create_host_config(version='1.25', cpus=100) + self.assertEqual(config.get('NanoCpus'), 100000000000) + self.assertRaises( + InvalidVersion, lambda: create_host_config( + version='1.24', cpus=1)) class ContainerConfigTest(unittest.TestCase): From 2d026fd1e56e39a45f1335ec0e0102828e87d9f1 Mon Sep 17 00:00:00 2001 From: hhHypo Date: Wed, 26 Apr 2017 01:41:07 +0800 Subject: [PATCH 0328/1301] fix can't get a dict when Containers is None Signed-off-by: hhHypo --- docker/models/networks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/models/networks.py b/docker/models/networks.py index a712e9bc43..e620e15745 100644 --- a/docker/models/networks.py +++ b/docker/models/networks.py @@ -22,7 +22,7 @@ def containers(self): """ return [ self.client.containers.get(cid) for cid in - self.attrs.get('Containers', {}).keys() + (self.attrs.get('Containers') or {}).keys() ] def connect(self, container): From 5f9a599b0ffa4da5679fe98e5f9ae933e8bb924d Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Wed, 26 Apr 2017 22:50:17 -0500 Subject: [PATCH 0329/1301] Fix if replicas is set to 0, Fixes #1572 Signed-off-by: Rob Kooper --- docker/types/services.py | 2 +- tests/unit/dockertypes_test.py | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/docker/types/services.py b/docker/types/services.py index 9291c9bd42..e7787ec81b 100644 --- a/docker/types/services.py +++ b/docker/types/services.py @@ -407,7 +407,7 @@ def __init__(self, mode, replicas=None): 'replicas can only be used for replicated mode' ) self[mode] = {} - if replicas: + if replicas is not None: self[mode]['Replicas'] = replicas @property diff --git a/tests/unit/dockertypes_test.py b/tests/unit/dockertypes_test.py index cb1d90ca2d..160fabdd7d 100644 --- a/tests/unit/dockertypes_test.py +++ b/tests/unit/dockertypes_test.py @@ -305,6 +305,12 @@ def test_replicated_replicas(self): assert mode.mode == 'replicated' assert mode.replicas == 21 + def test_replicated_replicas_0(self): + mode = ServiceMode('replicated', 0) + assert mode == {'replicated': {'Replicas': 0}} + assert mode.mode == 'replicated' + assert mode.replicas == 0 + def test_invalid_mode(self): with pytest.raises(InvalidArgument): ServiceMode('foobar') From c5cc23884a53bb2d074b290ae7e3beb8b26d1703 Mon Sep 17 00:00:00 2001 From: Aashutosh Rathi Date: Tue, 18 Apr 2017 03:56:13 +0530 Subject: [PATCH 0330/1301] Update services.py Signed-off-by: Aashutosh Rathi --- docker/types/services.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/types/services.py b/docker/types/services.py index 9291c9bd42..512b80fd31 100644 --- a/docker/types/services.py +++ b/docker/types/services.py @@ -302,8 +302,8 @@ class RestartPolicy(dict): condition (string): Condition for restart (``none``, ``on-failure``, or ``any``). Default: `none`. delay (int): Delay between restart attempts. Default: 0 - attempts (int): Maximum attempts to restart a given container before - giving up. Default value is 0, which is ignored. + max_attempts (int): Maximum attempts to restart a given container + before giving up. Default value is 0, which is ignored. window (int): Time window used to evaluate the restart policy. Default value is 0, which is unbounded. """ From 9412e21f1ab4a9ff849885f7fcfd86f17ab1f59c Mon Sep 17 00:00:00 2001 From: "Jesper L. Nielsen" Date: Mon, 24 Apr 2017 17:57:38 +0000 Subject: [PATCH 0331/1301] Network model functions 'connect' and 'disconnect' did not accept or passthrough keyword arguments. Signed-off-by: Jesper L. Nielsen --- docker/models/networks.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/docker/models/networks.py b/docker/models/networks.py index e620e15745..50601ad4ed 100644 --- a/docker/models/networks.py +++ b/docker/models/networks.py @@ -25,7 +25,7 @@ def containers(self): (self.attrs.get('Containers') or {}).keys() ] - def connect(self, container): + def connect(self, container, *args, **kwargs): """ Connect a container to this network. @@ -52,9 +52,12 @@ def connect(self, container): """ if isinstance(container, Container): container = container.id - return self.client.api.connect_container_to_network(container, self.id) + return self.client.api.connect_container_to_network(container, + self.id, + *args, + **kwargs) - def disconnect(self, container): + def disconnect(self, container, *args, **kwargs): """ Disconnect a container from this network. @@ -72,7 +75,9 @@ def disconnect(self, container): if isinstance(container, Container): container = container.id return self.client.api.disconnect_container_from_network(container, - self.id) + self.id, + *args, + **kwargs) def remove(self): """ From 16d32b40e64c134e690bc4ad72e547958b6263e7 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 27 Apr 2017 15:46:38 -0700 Subject: [PATCH 0332/1301] Formatting Signed-off-by: Joffrey F --- docker/models/networks.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/docker/models/networks.py b/docker/models/networks.py index 50601ad4ed..586809753b 100644 --- a/docker/models/networks.py +++ b/docker/models/networks.py @@ -52,10 +52,9 @@ def connect(self, container, *args, **kwargs): """ if isinstance(container, Container): container = container.id - return self.client.api.connect_container_to_network(container, - self.id, - *args, - **kwargs) + return self.client.api.connect_container_to_network( + container, self.id, *args, **kwargs + ) def disconnect(self, container, *args, **kwargs): """ @@ -74,10 +73,9 @@ def disconnect(self, container, *args, **kwargs): """ if isinstance(container, Container): container = container.id - return self.client.api.disconnect_container_from_network(container, - self.id, - *args, - **kwargs) + return self.client.api.disconnect_container_from_network( + container, self.id, *args, **kwargs + ) def remove(self): """ From c2f83d558e74c4e4d2faca8bcecf35eb9f9f8d96 Mon Sep 17 00:00:00 2001 From: Alexey Rokhin Date: Fri, 28 Apr 2017 14:49:40 +0300 Subject: [PATCH 0333/1301] cpus renamed to nano_cpus. Type and scale of nano_cpus are changed. Comments for new parameters are changed. Signed-off-by: Alexey Rokhin --- docker/models/containers.py | 9 +++++---- docker/types/containers.py | 12 ++++++------ tests/unit/api_container_test.py | 8 +++++--- tests/unit/dockertypes_test.py | 14 +++++++------- 4 files changed, 23 insertions(+), 20 deletions(-) diff --git a/docker/models/containers.py b/docker/models/containers.py index f623f6b818..c2c0c404bb 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -456,13 +456,13 @@ def run(self, image, command=None, stdout=True, stderr=False, cap_add (list of str): Add kernel capabilities. For example, ``["SYS_ADMIN", "MKNOD"]``. cap_drop (list of str): Drop kernel capabilities. - cpu_count (int): CPU count (Windows only). - cpu_percent (int): CPU percent (Windows only). + cpu_count (int): Number of usable CPUs (Windows only). + cpu_percent (int): Usable percentage of the available CPUs + (Windows only). cpu_period (int): The length of a CPU period in microseconds. cpu_quota (int): Microseconds of CPU time that the container can get in a CPU period. cpu_shares (int): CPU shares (relative weight). - cpus (float): Number of CPUs. cpuset_cpus (str): CPUs in which to allow execution (``0-3``, ``0,1``). detach (bool): Run container in the background and return a @@ -526,6 +526,7 @@ def run(self, image, command=None, stdout=True, stderr=False, networks (:py:class:`list`): A list of network names to connect this container to. name (str): The name for this container. + nano_cpus (int): CPU quota in units of 10-9 CPUs. network_disabled (bool): Disable networking. network_mode (str): One of: @@ -809,7 +810,6 @@ def prune(self, filters=None): 'cpu_period', 'cpu_quota', 'cpu_shares', - 'cpus', 'cpuset_cpus', 'device_read_bps', 'device_read_iops', @@ -833,6 +833,7 @@ def prune(self, filters=None): 'mem_reservation', 'mem_swappiness', 'memswap_limit', + 'nano_cpus', 'network_mode', 'oom_kill_disable', 'oom_score_adj', diff --git a/docker/types/containers.py b/docker/types/containers.py index 8c5e5ef7b5..9f1a04d104 100644 --- a/docker/types/containers.py +++ b/docker/types/containers.py @@ -119,7 +119,7 @@ def __init__(self, version, binds=None, port_bindings=None, cpuset_cpus=None, userns_mode=None, pids_limit=None, isolation=None, auto_remove=False, storage_opt=None, init=None, init_path=None, volume_driver=None, - cpu_count=None, cpu_percent=None, cpus=None): + cpu_count=None, cpu_percent=None, nano_cpus=None): if mem_limit is not None: self['Memory'] = parse_bytes(mem_limit) @@ -450,13 +450,13 @@ def __init__(self, version, binds=None, port_bindings=None, self['CpuPercent'] = cpu_percent - if cpus: - if not isinstance(cpus, (float, int)): - raise host_config_type_error('cpus', cpus, 'float') + if nano_cpus: + if not isinstance(nano_cpus, int): + raise host_config_type_error('nano_cpus', nano_cpus, 'int') if version_lt(version, '1.25'): - raise host_config_version_error('cpus', '1.25') + raise host_config_version_error('nano_cpus', '1.25') - self['NanoCpus'] = int(1000000000 * cpus) + self['NanoCpus'] = nano_cpus def host_config_type_error(param, param_value, expected): diff --git a/tests/unit/api_container_test.py b/tests/unit/api_container_test.py index 30287110d4..901934ee7a 100644 --- a/tests/unit/api_container_test.py +++ b/tests/unit/api_container_test.py @@ -1158,7 +1158,7 @@ def test_create_container_with_host_config_cpus(self): 'busybox', 'ls', host_config=self.client.create_host_config( cpu_count=1, cpu_percent=20, - cpus=10 + nano_cpus=1000 ) ) @@ -1177,10 +1177,12 @@ def test_create_container_with_host_config_cpus(self): "HostConfig": { "CpuCount": 1, "CpuPercent": 20, - "NanoCpus": 10000000000, + "NanoCpus": 1000, "NetworkMode": "default" }}''')) - self.assertEqual(args[1]['headers'], {'Content-Type': 'application/json'}) + self.assertEqual( + args[1]['headers'], {'Content-Type': 'application/json'} + ) class ContainerTest(BaseAPIClientTest): diff --git a/tests/unit/dockertypes_test.py b/tests/unit/dockertypes_test.py index c3ee2e263e..63383677d7 100644 --- a/tests/unit/dockertypes_test.py +++ b/tests/unit/dockertypes_test.py @@ -172,7 +172,7 @@ def test_create_host_config_with_volume_driver(self): config = create_host_config(version='1.21', volume_driver='local') assert config.get('VolumeDriver') == 'local' - + def test_create_host_config_invalid_cpu_count_types(self): with pytest.raises(TypeError): create_host_config(version='1.25', cpu_count='1') @@ -195,16 +195,16 @@ def test_create_host_config_with_cpu_percent(self): InvalidVersion, lambda: create_host_config( version='1.24', cpu_percent=10)) - def test_create_host_config_invalid_cpus_types(self): + def test_create_host_config_invalid_nano_cpus_types(self): with pytest.raises(TypeError): - create_host_config(version='1.25', cpus='0') + create_host_config(version='1.25', nano_cpus='0') - def test_create_host_config_with_cpus(self): - config = create_host_config(version='1.25', cpus=100) - self.assertEqual(config.get('NanoCpus'), 100000000000) + def test_create_host_config_with_nano_cpus(self): + config = create_host_config(version='1.25', nano_cpus=1000) + self.assertEqual(config.get('NanoCpus'), 1000) self.assertRaises( InvalidVersion, lambda: create_host_config( - version='1.24', cpus=1)) + version='1.24', nano_cpus=1)) class ContainerConfigTest(unittest.TestCase): From 59ba27068b524bdbfe60dd74762e28acd709caea Mon Sep 17 00:00:00 2001 From: Aaron Cowdin Date: Fri, 28 Apr 2017 14:36:43 -0700 Subject: [PATCH 0334/1301] Handle multiple success messages during image building. Signed-off-by: Aaron Cowdin --- docker/models/images.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/docker/models/images.py b/docker/models/images.py index 3fd3dc1956..cb9c80d218 100644 --- a/docker/models/images.py +++ b/docker/models/images.py @@ -169,14 +169,15 @@ def build(self, **kwargs): events = list(json_stream(resp)) if not events: return BuildError('Unknown') - event = events[-1] - if 'stream' in event: - match = re.search(r'(Successfully built |sha256:)([0-9a-f]+)', - event.get('stream', '')) - if match: - image_id = match.group(2) - return self.get(image_id) + for event in events: + if 'stream' in event: + match = re.search(r'(Successfully built |sha256:)([0-9a-f]+)', + event.get('stream', '')) + if match: + image_id = match.group(2) + return self.get(image_id) + event = events[-1] raise BuildError(event.get('error') or event) def get(self, name): From 4633dac5808f5a02a0fa61a2738e1ae1862887d2 Mon Sep 17 00:00:00 2001 From: Tomas Tomecek Date: Fri, 28 Apr 2017 11:23:44 +0200 Subject: [PATCH 0335/1301] exec: add support for `Env` Signed-off-by: Tomas Tomecek --- docker/api/exec_api.py | 16 ++++++++++++++-- docker/models/containers.py | 7 +++++-- tests/integration/api_exec_test.py | 15 +++++++++++++++ tests/unit/models_containers_test.py | 2 +- 4 files changed, 35 insertions(+), 5 deletions(-) diff --git a/docker/api/exec_api.py b/docker/api/exec_api.py index 6c3e638338..0382a64797 100644 --- a/docker/api/exec_api.py +++ b/docker/api/exec_api.py @@ -8,7 +8,8 @@ class ExecApiMixin(object): @utils.minimum_version('1.15') @utils.check_resource def exec_create(self, container, cmd, stdout=True, stderr=True, - stdin=False, tty=False, privileged=False, user=''): + stdin=False, tty=False, privileged=False, user='', + environment=None): """ Sets up an exec instance in a running container. @@ -22,6 +23,9 @@ def exec_create(self, container, cmd, stdout=True, stderr=True, tty (bool): Allocate a pseudo-TTY. Default: False privileged (bool): Run as privileged. user (str): User to execute command as. Default: root + environment (dict or list): A dictionary or a list of strings in + the following format ``["PASSWORD=xxx"]`` or + ``{"PASSWORD": "xxx"}``. Returns: (dict): A dictionary with an exec ``Id`` key. @@ -39,9 +43,16 @@ def exec_create(self, container, cmd, stdout=True, stderr=True, raise errors.InvalidVersion( 'User-specific exec is not supported in API < 1.19' ) + if environment and utils.compare_version('1.25', self._version) < 0: + raise errors.InvalidVersion( + 'Setting environment for exec is not supported in API < 1.25' + ) if isinstance(cmd, six.string_types): cmd = utils.split_command(cmd) + if isinstance(environment, dict): + environment = utils.utils.format_environment(environment) + data = { 'Container': container, 'User': user, @@ -50,7 +61,8 @@ def exec_create(self, container, cmd, stdout=True, stderr=True, 'AttachStdin': stdin, 'AttachStdout': stdout, 'AttachStderr': stderr, - 'Cmd': cmd + 'Cmd': cmd, + 'Env': environment, } url = self._url('/containers/{0}/exec', container) diff --git a/docker/models/containers.py b/docker/models/containers.py index 93f637252f..6c8e163056 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -125,7 +125,7 @@ def diff(self): def exec_run(self, cmd, stdout=True, stderr=True, stdin=False, tty=False, privileged=False, user='', detach=False, stream=False, - socket=False): + socket=False, environment=None): """ Run a command inside this container. Similar to ``docker exec``. @@ -141,6 +141,9 @@ def exec_run(self, cmd, stdout=True, stderr=True, stdin=False, tty=False, detach (bool): If true, detach from the exec command. Default: False stream (bool): Stream response data. Default: False + environment (dict or list): A dictionary or a list of strings in + the following format ``["PASSWORD=xxx"]`` or + ``{"PASSWORD": "xxx"}``. Returns: (generator or str): If ``stream=True``, a generator yielding @@ -152,7 +155,7 @@ def exec_run(self, cmd, stdout=True, stderr=True, stdin=False, tty=False, """ resp = self.client.api.exec_create( self.id, cmd, stdout=stdout, stderr=stderr, stdin=stdin, tty=tty, - privileged=privileged, user=user + privileged=privileged, user=user, environment=environment ) return self.client.api.exec_start( resp['Id'], detach=detach, tty=tty, stream=stream, socket=socket diff --git a/tests/integration/api_exec_test.py b/tests/integration/api_exec_test.py index 55286e374e..fb3c6f9320 100644 --- a/tests/integration/api_exec_test.py +++ b/tests/integration/api_exec_test.py @@ -2,6 +2,7 @@ from docker.utils.socket import read_exactly from .base import BaseAPIIntegrationTest, BUSYBOX +from ..helpers import requires_api_version class ExecTest(BaseAPIIntegrationTest): @@ -121,3 +122,17 @@ def test_exec_inspect(self): exec_info = self.client.exec_inspect(exec_id) self.assertIn('ExitCode', exec_info) self.assertNotEqual(exec_info['ExitCode'], 0) + + @requires_api_version('1.25') + def test_exec_command_with_env(self): + container = self.client.create_container(BUSYBOX, 'cat', + detach=True, stdin_open=True) + id = container['Id'] + self.client.start(id) + self.tmp_containers.append(id) + + res = self.client.exec_create(id, 'env', environment=["X=Y"]) + self.assertIn('Id', res) + + exec_log = self.client.exec_start(res) + self.assertIn(b'X=Y\n', exec_log) diff --git a/tests/unit/models_containers_test.py b/tests/unit/models_containers_test.py index c594606170..f8ec708548 100644 --- a/tests/unit/models_containers_test.py +++ b/tests/unit/models_containers_test.py @@ -366,7 +366,7 @@ def test_exec_run(self): container.exec_run("echo hello world", privileged=True, stream=True) client.api.exec_create.assert_called_with( FAKE_CONTAINER_ID, "echo hello world", stdout=True, stderr=True, - stdin=False, tty=False, privileged=True, user='' + stdin=False, tty=False, privileged=True, user='', environment=None ) client.api.exec_start.assert_called_with( FAKE_EXEC_ID, detach=False, tty=False, stream=True, socket=False From b9ca8755bb04fb366364f7cf579010060977808b Mon Sep 17 00:00:00 2001 From: Dan Liew Date: Mon, 1 May 2017 14:39:07 +0100 Subject: [PATCH 0336/1301] Add missing support for `CpusetMems` parameter to HostConfig. Signed-off-by: Dan Liew --- docker/api/container.py | 2 ++ docker/models/containers.py | 3 +++ docker/types/containers.py | 13 ++++++++++++- tests/unit/api_container_test.py | 27 +++++++++++++++++++++++++++ 4 files changed, 44 insertions(+), 1 deletion(-) diff --git a/docker/api/container.py b/docker/api/container.py index 85e5e90ae7..97b5405935 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -479,6 +479,8 @@ def create_host_config(self, *args, **kwargs): cpu_shares (int): CPU shares (relative weight). cpuset_cpus (str): CPUs in which to allow execution (``0-3``, ``0,1``). + cpuset_mems (str): Memory nodes (MEMs) in which to allow execution + (``0-3``, ``0,1``). Only effective on NUMA systems. device_read_bps: Limit read rate (bytes per second) from a device in the form of: `[{"Path": "device_path", "Rate": rate}]` device_read_iops: Limit read rate (IO per second) from a device. diff --git a/docker/models/containers.py b/docker/models/containers.py index f1c0fd262d..2213583a71 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -483,6 +483,8 @@ def run(self, image, command=None, stdout=True, stderr=False, cpu_shares (int): CPU shares (relative weight). cpuset_cpus (str): CPUs in which to allow execution (``0-3``, ``0,1``). + cpuset_mems (str): Memory nodes (MEMs) in which to allow execution + (``0-3``, ``0,1``). Only effective on NUMA systems. detach (bool): Run container in the background and return a :py:class:`Container` object. device_read_bps: Limit read rate (bytes per second) from a device @@ -829,6 +831,7 @@ def prune(self, filters=None): 'cpu_quota', 'cpu_shares', 'cpuset_cpus', + 'cpuset_mems', 'device_read_bps', 'device_read_iops', 'device_write_bps', diff --git a/docker/types/containers.py b/docker/types/containers.py index 9f1a04d104..ad162555a8 100644 --- a/docker/types/containers.py +++ b/docker/types/containers.py @@ -119,7 +119,8 @@ def __init__(self, version, binds=None, port_bindings=None, cpuset_cpus=None, userns_mode=None, pids_limit=None, isolation=None, auto_remove=False, storage_opt=None, init=None, init_path=None, volume_driver=None, - cpu_count=None, cpu_percent=None, nano_cpus=None): + cpu_count=None, cpu_percent=None, nano_cpus=None, + cpuset_mems=None): if mem_limit is not None: self['Memory'] = parse_bytes(mem_limit) @@ -328,6 +329,16 @@ def __init__(self, version, binds=None, port_bindings=None, self['CpuSetCpus'] = cpuset_cpus + if cpuset_mems: + if version_lt(version, '1.19'): + raise host_config_version_error('cpuset_mems', '1.19') + + if not isinstance(cpuset_mems, str): + raise host_config_type_error( + 'cpuset_mems', cpuset_mems, 'str' + ) + self['CpusetMems'] = cpuset_mems + if blkio_weight: if not isinstance(blkio_weight, int): raise host_config_type_error( diff --git a/tests/unit/api_container_test.py b/tests/unit/api_container_test.py index 901934ee7a..39df293923 100644 --- a/tests/unit/api_container_test.py +++ b/tests/unit/api_container_test.py @@ -338,6 +338,33 @@ def test_create_container_with_host_config_cpuset(self): self.assertEqual(args[1]['headers'], {'Content-Type': 'application/json'}) + @requires_api_version('1.19') + def test_create_container_with_host_config_cpuset_mems(self): + self.client.create_container( + 'busybox', 'ls', host_config=self.client.create_host_config( + cpuset_mems='0' + ) + ) + + args = fake_request.call_args + self.assertEqual(args[0][1], + url_prefix + 'containers/create') + + self.assertEqual(json.loads(args[1]['data']), + json.loads(''' + {"Tty": false, "Image": "busybox", + "Cmd": ["ls"], "AttachStdin": false, + "AttachStderr": true, + "AttachStdout": true, "OpenStdin": false, + "StdinOnce": false, + "NetworkDisabled": false, + "HostConfig": { + "CpusetMems": "0", + "NetworkMode": "default" + }}''')) + self.assertEqual(args[1]['headers'], + {'Content-Type': 'application/json'}) + def test_create_container_with_cgroup_parent(self): self.client.create_container( 'busybox', 'ls', host_config=self.client.create_host_config( From 2a0a7adeced6120cf53ed10936769748cdb3a34a Mon Sep 17 00:00:00 2001 From: Dan Liew Date: Mon, 1 May 2017 14:42:46 +0100 Subject: [PATCH 0337/1301] Fix typo s/CpuSetCpus/CpusetCpus/ According to Docker's API documentation [1]. The parameter name is `CpusetCpus` not `CpuSetCpus`. [1] https://docs.docker.com/engine/api/v1.25/#operation/ContainerCreate Signed-off-by: Dan Liew --- docker/types/containers.py | 2 +- tests/unit/api_container_test.py | 2 +- tests/unit/models_containers_test.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/types/containers.py b/docker/types/containers.py index ad162555a8..06d0ee43ac 100644 --- a/docker/types/containers.py +++ b/docker/types/containers.py @@ -327,7 +327,7 @@ def __init__(self, version, binds=None, port_bindings=None, if version_lt(version, '1.18'): raise host_config_version_error('cpuset_cpus', '1.18') - self['CpuSetCpus'] = cpuset_cpus + self['CpusetCpus'] = cpuset_cpus if cpuset_mems: if version_lt(version, '1.19'): diff --git a/tests/unit/api_container_test.py b/tests/unit/api_container_test.py index 39df293923..662d3f5908 100644 --- a/tests/unit/api_container_test.py +++ b/tests/unit/api_container_test.py @@ -332,7 +332,7 @@ def test_create_container_with_host_config_cpuset(self): "StdinOnce": false, "NetworkDisabled": false, "HostConfig": { - "CpuSetCpus": "0,1", + "CpusetCpus": "0,1", "NetworkMode": "default" }}''')) self.assertEqual(args[1]['headers'], diff --git a/tests/unit/models_containers_test.py b/tests/unit/models_containers_test.py index c594606170..4f4dc0f398 100644 --- a/tests/unit/models_containers_test.py +++ b/tests/unit/models_containers_test.py @@ -135,7 +135,7 @@ def test_create_container_args(self): 'CpuPeriod': 1, 'CpuQuota': 2, 'CpuShares': 5, - 'CpuSetCpus': '0-3', + 'CpusetCpus': '0-3', 'Devices': [{'PathOnHost': '/dev/sda', 'CgroupPermissions': 'rwm', 'PathInContainer': '/dev/xvda'}], From 3a4fa79e1c707b0f9354677410cc84b1f5db1cb8 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Tue, 2 May 2017 11:23:36 -0500 Subject: [PATCH 0338/1301] Documentation fixes for login func This makes a small tweak to the grammar of the documentation for the reauth argument, and also updates the dockercfg_path docs for accuracy. Signed-off-by: Erik Johnson --- docker/api/daemon.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docker/api/daemon.py b/docker/api/daemon.py index 91c777f092..285b7429ad 100644 --- a/docker/api/daemon.py +++ b/docker/api/daemon.py @@ -100,10 +100,11 @@ def login(self, username, password=None, email=None, registry=None, email (str): The email for the registry account registry (str): URL to the registry. E.g. ``https://index.docker.io/v1/`` - reauth (bool): Whether refresh existing authentication on the - Docker server. - dockercfg_path (str): Use a custom path for the ``.dockercfg`` file - (default ``$HOME/.dockercfg``) + reauth (bool): Whether or not to refresh existing authentication on + the Docker server. + dockercfg_path (str): Use a custom path for the Docker config file + (default ``$HOME/.docker/config.json`` if present, + otherwise``$HOME/.dockercfg``) Returns: (dict): The response from the login request From 0f843414e5b4ba4c2344abb8bf3489eedba055d3 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Tue, 2 May 2017 14:38:07 -0500 Subject: [PATCH 0339/1301] Add a reload_config function to the DaemonApiMixin This allows the client to reload the config.json for an existing APIClient instance. Signed-off-by: Erik Johnson --- docker/api/daemon.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/docker/api/daemon.py b/docker/api/daemon.py index 91c777f092..aa93643679 100644 --- a/docker/api/daemon.py +++ b/docker/api/daemon.py @@ -123,9 +123,9 @@ def login(self, username, password=None, email=None, registry=None, # If dockercfg_path is passed check to see if the config file exists, # if so load that config. if dockercfg_path and os.path.exists(dockercfg_path): - self._auth_configs = auth.load_config(dockercfg_path) + self._auth_configs = self.reload_config(dockercfg_path) elif not self._auth_configs: - self._auth_configs = auth.load_config() + self._auth_configs = self.reload_config() authcfg = auth.resolve_authconfig(self._auth_configs, registry) # If we found an existing auth config for this registry and username @@ -174,3 +174,17 @@ def version(self, api_version=True): """ url = self._url("/version", versioned_api=api_version) return self._result(self._get(url), json=True) + + def reload_config(self, dockercfg_path=None): + """ + Forces a reload of the auth configuration + + Args: + dockercfg_path (str): Use a custom path for the Docker config file + (default ``$HOME/.docker/config.json`` if present, + otherwise``$HOME/.dockercfg``) + + Returns: + None + """ + self._auth_configs = auth.load_config(dockercfg_path) From 550c31e2b701ddaa275f5b260e702987638467dc Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Tue, 2 May 2017 16:23:41 -0500 Subject: [PATCH 0340/1301] Move reload_config func into the APIClient Also revert an incorrect change in the DaemonApiMixin's login func Signed-off-by: Erik Johnson --- docker/api/client.py | 14 ++++++++++++++ docker/api/daemon.py | 18 ++---------------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/docker/api/client.py b/docker/api/client.py index 749b061dce..54ec6abb45 100644 --- a/docker/api/client.py +++ b/docker/api/client.py @@ -433,3 +433,17 @@ def get_adapter(self, url): @property def api_version(self): return self._version + + def reload_config(self, dockercfg_path=None): + """ + Force a reload of the auth configuration + + Args: + dockercfg_path (str): Use a custom path for the Docker config file + (default ``$HOME/.docker/config.json`` if present, + otherwise``$HOME/.dockercfg``) + + Returns: + None + """ + self._auth_configs = auth.load_config(dockercfg_path) diff --git a/docker/api/daemon.py b/docker/api/daemon.py index aa93643679..91c777f092 100644 --- a/docker/api/daemon.py +++ b/docker/api/daemon.py @@ -123,9 +123,9 @@ def login(self, username, password=None, email=None, registry=None, # If dockercfg_path is passed check to see if the config file exists, # if so load that config. if dockercfg_path and os.path.exists(dockercfg_path): - self._auth_configs = self.reload_config(dockercfg_path) + self._auth_configs = auth.load_config(dockercfg_path) elif not self._auth_configs: - self._auth_configs = self.reload_config() + self._auth_configs = auth.load_config() authcfg = auth.resolve_authconfig(self._auth_configs, registry) # If we found an existing auth config for this registry and username @@ -174,17 +174,3 @@ def version(self, api_version=True): """ url = self._url("/version", versioned_api=api_version) return self._result(self._get(url), json=True) - - def reload_config(self, dockercfg_path=None): - """ - Forces a reload of the auth configuration - - Args: - dockercfg_path (str): Use a custom path for the Docker config file - (default ``$HOME/.docker/config.json`` if present, - otherwise``$HOME/.dockercfg``) - - Returns: - None - """ - self._auth_configs = auth.load_config(dockercfg_path) From 7dffc4623485d705ff088817f4439f1edb734547 Mon Sep 17 00:00:00 2001 From: Aaron Cowdin Date: Tue, 2 May 2017 17:01:34 -0700 Subject: [PATCH 0341/1301] Add integration tests Signed-off-by: Aaron Cowdin --- tests/integration/models_images_test.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/integration/models_images_test.py b/tests/integration/models_images_test.py index 4f8bb26cd5..49e06f6974 100644 --- a/tests/integration/models_images_test.py +++ b/tests/integration/models_images_test.py @@ -28,6 +28,15 @@ def test_build_with_error(self): assert str(cm.exception) == ("Unknown instruction: " "NOTADOCKERFILECOMMAND") + def test_build_with_multiple_success(self): + client = docker.from_env(version=TEST_API_VERSION) + image = client.images.build(tag='some-tag', fileobj=io.BytesIO( + "FROM alpine\n" + "CMD echo hello world".encode('ascii') + )) + self.tmp_imgs.append(image.id) + assert client.containers.run(image) == b"hello world\n" + def test_list(self): client = docker.from_env(version=TEST_API_VERSION) image = client.images.pull('alpine:latest') From c6030027f530c30c68e7240a294d2ae901df6492 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 2 May 2017 17:09:09 -0700 Subject: [PATCH 0342/1301] Modernize exec_api.py Signed-off-by: Joffrey F --- docker/api/exec_api.py | 10 +++++----- tests/integration/api_exec_test.py | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docker/api/exec_api.py b/docker/api/exec_api.py index 0382a64797..3ff65256ef 100644 --- a/docker/api/exec_api.py +++ b/docker/api/exec_api.py @@ -35,18 +35,19 @@ def exec_create(self, container, cmd, stdout=True, stderr=True, If the server returns an error. """ - if privileged and utils.compare_version('1.19', self._version) < 0: + if privileged and utils.version_lt(self._version, '1.19'): raise errors.InvalidVersion( 'Privileged exec is not supported in API < 1.19' ) - if user and utils.compare_version('1.19', self._version) < 0: + if user and utils.version_lt(self._version, '1.19'): raise errors.InvalidVersion( 'User-specific exec is not supported in API < 1.19' ) - if environment and utils.compare_version('1.25', self._version) < 0: + if environment is not None and utils.version_lt(self._version, '1.25'): raise errors.InvalidVersion( 'Setting environment for exec is not supported in API < 1.25' ) + if isinstance(cmd, six.string_types): cmd = utils.split_command(cmd) @@ -109,6 +110,7 @@ def exec_resize(self, exec_id, height=None, width=None): self._raise_for_status(res) @utils.minimum_version('1.15') + @utils.check_resource def exec_start(self, exec_id, detach=False, tty=False, stream=False, socket=False): """ @@ -130,8 +132,6 @@ def exec_start(self, exec_id, detach=False, tty=False, stream=False, If the server returns an error. """ # we want opened socket if socket == True - if isinstance(exec_id, dict): - exec_id = exec_id.get('Id') data = { 'Tty': tty, diff --git a/tests/integration/api_exec_test.py b/tests/integration/api_exec_test.py index fb3c6f9320..7a65041963 100644 --- a/tests/integration/api_exec_test.py +++ b/tests/integration/api_exec_test.py @@ -132,7 +132,7 @@ def test_exec_command_with_env(self): self.tmp_containers.append(id) res = self.client.exec_create(id, 'env', environment=["X=Y"]) - self.assertIn('Id', res) + assert 'Id' in res exec_log = self.client.exec_start(res) - self.assertIn(b'X=Y\n', exec_log) + assert b'X=Y\n' in exec_log From a164f4661bb92eb962e6954836b33f6d10b173d0 Mon Sep 17 00:00:00 2001 From: Aaron Cowdin Date: Tue, 2 May 2017 17:14:05 -0700 Subject: [PATCH 0343/1301] Better error handling, itterate on json stream directly. Signed-off-by: Aaron Cowdin --- docker/models/images.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/docker/models/images.py b/docker/models/images.py index cb9c80d218..55e7ced8f4 100644 --- a/docker/models/images.py +++ b/docker/models/images.py @@ -166,19 +166,18 @@ def build(self, **kwargs): resp = self.client.api.build(**kwargs) if isinstance(resp, six.string_types): return self.get(resp) - events = list(json_stream(resp)) - if not events: - return BuildError('Unknown') - for event in events: - if 'stream' in event: + for chunk in json_stream(resp): + if 'error' in chunk: + raise BuildError(chunk['error']) + break + if 'stream' in chunk: match = re.search(r'(Successfully built |sha256:)([0-9a-f]+)', - event.get('stream', '')) + chunk['stream']) if match: image_id = match.group(2) return self.get(image_id) - event = events[-1] - raise BuildError(event.get('error') or event) + return BuildError('Unknown') def get(self, name): """ From 02c5914d29a8706cadc22317f8b5abd1630ce373 Mon Sep 17 00:00:00 2001 From: Alfred Landrum Date: Thu, 4 May 2017 16:22:15 -0700 Subject: [PATCH 0344/1301] Update image create error parsing Support new error message returned for image creation in: https://github.com/moby/moby/pull/33005 Signed-off-by: Alfred Landrum --- docker/errors.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/errors.py b/docker/errors.py index 03b89c11e3..0da97f4e3f 100644 --- a/docker/errors.py +++ b/docker/errors.py @@ -23,7 +23,8 @@ def create_api_error_from_http_exception(e): if response.status_code == 404: if explanation and ('No such image' in str(explanation) or 'not found: does not exist or no pull access' - in str(explanation)): + in str(explanation) or + 'repository does not exist' in str(explanation)): cls = ImageNotFound else: cls = NotFound From 431f7c6432031b0d405b81e567ffb6cb57c8a6db Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 5 May 2017 15:08:38 -0700 Subject: [PATCH 0345/1301] Improve build result detection Signed-off-by: Joffrey F --- docker/models/images.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/docker/models/images.py b/docker/models/images.py index 55e7ced8f4..48f35909e0 100644 --- a/docker/models/images.py +++ b/docker/models/images.py @@ -166,18 +166,21 @@ def build(self, **kwargs): resp = self.client.api.build(**kwargs) if isinstance(resp, six.string_types): return self.get(resp) + last_event = None for chunk in json_stream(resp): if 'error' in chunk: raise BuildError(chunk['error']) - break if 'stream' in chunk: - match = re.search(r'(Successfully built |sha256:)([0-9a-f]+)', - chunk['stream']) + match = re.search( + r'(Successfully built |sha256:)([0-9a-f]+)', + chunk['stream'] + ) if match: image_id = match.group(2) return self.get(image_id) + last_event = chunk - return BuildError('Unknown') + raise BuildError(last_event or 'Unknown') def get(self, name): """ From 933a303ede8b032ef56dee3c68f35b67b1ec4f98 Mon Sep 17 00:00:00 2001 From: Antoine Verlant Date: Mon, 8 May 2017 10:09:36 +0200 Subject: [PATCH 0346/1301] Fix the way the list of mounts is made for service. Signed-off-by: Antoine Verlant --- docker/types/services.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/docker/types/services.py b/docker/types/services.py index 9291c9bd42..a95e3e5478 100644 --- a/docker/types/services.py +++ b/docker/types/services.py @@ -107,11 +107,14 @@ def __init__(self, image, command=None, args=None, hostname=None, env=None, if labels is not None: self['Labels'] = labels if mounts is not None: + parsed_mounts = [] for mount in mounts: if isinstance(mount, six.string_types): - mounts.append(Mount.parse_mount_string(mount)) - mounts.remove(mount) - self['Mounts'] = mounts + parsed_mounts.append(Mount.parse_mount_string(mount)) + else: + # If mount already parsed + parsed_mounts.append(mount) + self['Mounts'] = parsed_mounts if stop_grace_period is not None: self['StopGracePeriod'] = stop_grace_period From f27ecf3f8822e01636cc4298d8f4fbccec313830 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 8 May 2017 14:13:59 -0700 Subject: [PATCH 0347/1301] Add ContainerSpec mounts test Signed-off-by: Joffrey F --- tests/unit/dockertypes_test.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/tests/unit/dockertypes_test.py b/tests/unit/dockertypes_test.py index 9b530b7f5b..8dbb35ecca 100644 --- a/tests/unit/dockertypes_test.py +++ b/tests/unit/dockertypes_test.py @@ -8,8 +8,8 @@ from docker.constants import DEFAULT_DOCKER_API_VERSION from docker.errors import InvalidArgument, InvalidVersion from docker.types import ( - ContainerConfig, EndpointConfig, HostConfig, IPAMConfig, IPAMPool, - LogConfig, Mount, ServiceMode, Ulimit, + ContainerConfig, ContainerSpec, EndpointConfig, HostConfig, IPAMConfig, + IPAMPool, LogConfig, Mount, ServiceMode, Ulimit, ) try: @@ -220,6 +220,22 @@ def test_create_container_config_volume_driver_warning(self): assert 'The volume_driver option has been moved' in str(w[0].message) +class ContainerSpecTest(unittest.TestCase): + def test_parse_mounts(self): + spec = ContainerSpec( + image='scratch', mounts=[ + '/local:/container', + '/local2:/container2:ro', + Mount(target='/target', source='/source') + ] + ) + + assert 'Mounts' in spec + assert len(spec['Mounts']) == 3 + for mount in spec['Mounts']: + assert isinstance(mount, Mount) + + class UlimitTest(unittest.TestCase): def test_create_host_config_dict_ulimit(self): ulimit_dct = {'name': 'nofile', 'soft': 8096} From c6ddea469f276608940f6cc6d3196b8d5d12e421 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 9 May 2017 12:19:35 -0700 Subject: [PATCH 0348/1301] Include tag in images.get after pulling if provided separately Signed-off-by: Joffrey F --- docker/models/images.py | 6 +++--- tests/integration/models_images_test.py | 15 +++++++++++---- tests/unit/models_containers_test.py | 2 +- tests/unit/models_images_test.py | 2 +- 4 files changed, 16 insertions(+), 9 deletions(-) diff --git a/docker/models/images.py b/docker/models/images.py index 48f35909e0..a12ede3067 100644 --- a/docker/models/images.py +++ b/docker/models/images.py @@ -238,7 +238,7 @@ def load(self, data): """ return self.client.api.load_image(data) - def pull(self, name, **kwargs): + def pull(self, name, tag=None, **kwargs): """ Pull an image of the given name and return it. Similar to the ``docker pull`` command. @@ -267,8 +267,8 @@ def pull(self, name, **kwargs): >>> image = client.images.pull('busybox') """ - self.client.api.pull(name, **kwargs) - return self.get(name) + self.client.api.pull(name, tag=tag, **kwargs) + return self.get('{0}:{1}'.format(name, tag) if tag else name) def push(self, repository, tag=None, **kwargs): return self.client.api.push(repository, tag=tag, **kwargs) diff --git a/tests/integration/models_images_test.py b/tests/integration/models_images_test.py index 49e06f6974..6d61e4977c 100644 --- a/tests/integration/models_images_test.py +++ b/tests/integration/models_images_test.py @@ -30,10 +30,12 @@ def test_build_with_error(self): def test_build_with_multiple_success(self): client = docker.from_env(version=TEST_API_VERSION) - image = client.images.build(tag='some-tag', fileobj=io.BytesIO( - "FROM alpine\n" - "CMD echo hello world".encode('ascii') - )) + image = client.images.build( + tag='some-tag', fileobj=io.BytesIO( + "FROM alpine\n" + "CMD echo hello world".encode('ascii') + ) + ) self.tmp_imgs.append(image.id) assert client.containers.run(image) == b"hello world\n" @@ -53,6 +55,11 @@ def test_pull(self): image = client.images.pull('alpine:latest') assert 'alpine:latest' in image.attrs['RepoTags'] + def test_pull_with_tag(self): + client = docker.from_env(version=TEST_API_VERSION) + image = client.images.pull('alpine', tag='3.3') + assert 'alpine:3.3' in image.attrs['RepoTags'] + class ImageTest(BaseIntegrationTest): diff --git a/tests/unit/models_containers_test.py b/tests/unit/models_containers_test.py index e74bb7cd35..0fb69f3e34 100644 --- a/tests/unit/models_containers_test.py +++ b/tests/unit/models_containers_test.py @@ -227,7 +227,7 @@ def test_run_pull(self): container = client.containers.run('alpine', 'sleep 300', detach=True) assert container.id == FAKE_CONTAINER_ID - client.api.pull.assert_called_with('alpine') + client.api.pull.assert_called_with('alpine', tag=None) def test_run_with_error(self): client = make_fake_client() diff --git a/tests/unit/models_images_test.py b/tests/unit/models_images_test.py index 784717be8e..9ecb7e490d 100644 --- a/tests/unit/models_images_test.py +++ b/tests/unit/models_images_test.py @@ -42,7 +42,7 @@ def test_load(self): def test_pull(self): client = make_fake_client() image = client.images.pull('test_image') - client.api.pull.assert_called_with('test_image') + client.api.pull.assert_called_with('test_image', tag=None) client.api.inspect_image.assert_called_with('test_image') assert isinstance(image, Image) assert image.id == FAKE_IMAGE_ID From bf60e2a330f964c43cc379ead6c209adaa4c74f4 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 9 May 2017 16:16:10 -0700 Subject: [PATCH 0349/1301] init_path removed in Engine 17.05 Signed-off-by: Joffrey F --- docker/types/containers.py | 4 ++++ tests/integration/api_container_test.py | 1 + 2 files changed, 5 insertions(+) diff --git a/docker/types/containers.py b/docker/types/containers.py index 06d0ee43ac..18d18381da 100644 --- a/docker/types/containers.py +++ b/docker/types/containers.py @@ -438,6 +438,10 @@ def __init__(self, version, binds=None, port_bindings=None, if init_path is not None: if version_lt(version, '1.25'): raise host_config_version_error('init_path', '1.25') + + if version_gte(version, '1.29'): + # https://github.com/moby/moby/pull/32470 + raise host_config_version_error('init_path', '1.29', False) self['InitPath'] = init_path if volume_driver is not None: diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index 9514261512..fb4c4e4adc 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -451,6 +451,7 @@ def test_create_with_init(self): config = self.client.inspect_container(ctnr) assert config['HostConfig']['Init'] is True + @pytest.mark.xfail(True, reason='init-path removed in 17.05.0') @requires_api_version('1.25') def test_create_with_init_path(self): ctnr = self.client.create_container( From 6ed0c010189dc4d0ffb0511ae145346bfdc49117 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 9 May 2017 17:18:46 -0700 Subject: [PATCH 0350/1301] Adjust tests and add newest engine version to Jenkinsfile Signed-off-by: Joffrey F --- Jenkinsfile | 4 ++-- Makefile | 4 ++-- tests/helpers.py | 24 +++++++++++++++++------- tests/integration/api_service_test.py | 22 ++++++++++++++++------ 4 files changed, 37 insertions(+), 17 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index a61e6d5165..987df7aff7 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -7,7 +7,7 @@ def images = [:] // Note: Swarm in dind seem notoriously flimsy with 1.12.1+, which is why we're // sticking with 1.12.0 for the 1.12 series -def dockerVersions = ["1.12.0", "1.13.1", "17.04.0-ce-rc1"] +def dockerVersions = ["1.12.0", "1.13.1", "17.04.0-ce", "17.05.0-ce"] def buildImage = { name, buildargs, pyTag -> img = docker.image(name) @@ -35,7 +35,7 @@ def buildImages = { -> } def getAPIVersion = { engineVersion -> - def versionMap = ['1.12.': '1.24', '1.13.': '1.26', '17.04': '1.27'] + def versionMap = ['1.12.': '1.24', '1.13.': '1.26', '17.04': '1.27', '17.05': '1.29'] return versionMap[engineVersion.substring(0, 5)] } diff --git a/Makefile b/Makefile index cd1174675b..e4cd3f7b88 100644 --- a/Makefile +++ b/Makefile @@ -41,8 +41,8 @@ integration-test: build integration-test-py3: build-py3 docker run --rm -v /var/run/docker.sock:/var/run/docker.sock docker-sdk-python3 py.test tests/integration/${file} -TEST_API_VERSION ?= 1.27 -TEST_ENGINE_VERSION ?= 17.04.0-ce-rc1 +TEST_API_VERSION ?= 1.29 +TEST_ENGINE_VERSION ?= 17.05.0-ce .PHONY: integration-dind integration-dind: build build-py3 diff --git a/tests/helpers.py b/tests/helpers.py index 1d86619a1c..124ae2da51 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -54,13 +54,23 @@ def requires_api_version(version): ) -def requires_experimental(f): - @functools.wraps(f) - def wrapped(self, *args, **kwargs): - if not self.client.info()['ExperimentalBuild']: - pytest.skip('Feature requires Docker Engine experimental mode') - return f(self, *args, **kwargs) - return wrapped +def requires_experimental(until=None): + test_version = os.environ.get( + 'DOCKER_TEST_API_VERSION', docker.constants.DEFAULT_DOCKER_API_VERSION + ) + + def req_exp(f): + @functools.wraps(f) + def wrapped(self, *args, **kwargs): + if not self.client.info()['ExperimentalBuild']: + pytest.skip('Feature requires Docker Engine experimental mode') + return f(self, *args, **kwargs) + + if until and docker.utils.version_gte(test_version, until): + return f + return wrapped + + return req_exp def wait_on_condition(condition, delay=0.1, timeout=40): diff --git a/tests/integration/api_service_test.py b/tests/integration/api_service_test.py index 6858ad0e54..914e516bc4 100644 --- a/tests/integration/api_service_test.py +++ b/tests/integration/api_service_test.py @@ -103,18 +103,28 @@ def test_create_service_simple(self): assert services[0]['ID'] == svc_id['ID'] @requires_api_version('1.25') - @requires_experimental + @requires_experimental(until='1.29') def test_service_logs(self): name, svc_id = self.create_simple_service() assert self.get_service_container(name, include_stopped=True) - logs = self.client.service_logs(svc_id, stdout=True, is_tty=False) - log_line = next(logs) + attempts = 20 + while True: + if attempts == 0: + self.fail('No service logs produced by endpoint') + return + logs = self.client.service_logs(svc_id, stdout=True, is_tty=False) + try: + log_line = next(logs) + except StopIteration: + attempts -= 1 + time.sleep(0.1) + continue + else: + break + if six.PY3: log_line = log_line.decode('utf-8') assert 'hello\n' in log_line - assert 'com.docker.swarm.service.id={}'.format( - svc_id['ID'] - ) in log_line def test_create_service_custom_log_driver(self): container_spec = docker.types.ContainerSpec( From 95297dc2e760b992d7d31d485a66d02728944f36 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 11 May 2017 17:44:23 -0700 Subject: [PATCH 0351/1301] Replace erroneous networks argument in containers.run with singular network equivalent. Small docfixes Signed-off-by: Joffrey F --- docker/models/containers.py | 23 ++++++++++++++------- docker/models/plugins.py | 4 ++-- docker/types/services.py | 2 +- tests/integration/models_containers_test.py | 19 +++++++++++++++++ tests/unit/models_containers_test.py | 5 ++--- 5 files changed, 40 insertions(+), 13 deletions(-) diff --git a/docker/models/containers.py b/docker/models/containers.py index 7a1cd7167f..4bb2cf8638 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -147,7 +147,7 @@ def exec_run(self, cmd, stdout=True, stderr=True, stdin=False, tty=False, Returns: (generator or str): If ``stream=True``, a generator yielding - response chunks. A string containing response data otherwise. + response chunks. A string containing response data otherwise. Raises: :py:class:`docker.errors.APIError` @@ -546,10 +546,12 @@ def run(self, image, command=None, stdout=True, stderr=False, behavior. Accepts number between 0 and 100. memswap_limit (str or int): Maximum amount of memory + swap a container is allowed to consume. - networks (:py:class:`list`): A list of network names to connect - this container to. name (str): The name for this container. nano_cpus (int): CPU quota in units of 10-9 CPUs. + network (str): Name of the network this container will be connected + to at creation time. You can connect to additional networks + using :py:meth:`Network.connect`. Incompatible with + ``network_mode``. network_disabled (bool): Disable networking. network_mode (str): One of: @@ -559,6 +561,7 @@ def run(self, image, command=None, stdout=True, stderr=False, - ``container:`` Reuse another container's network stack. - ``host`` Use the host network stack. + Incompatible with ``network``. oom_kill_disable (bool): Whether to disable OOM killer. oom_score_adj (int): An integer value containing the score given to the container in order to tune OOM killer preferences. @@ -680,6 +683,12 @@ def run(self, image, command=None, stdout=True, stderr=False, raise RuntimeError("The options 'detach' and 'remove' cannot be " "used together.") + if kwargs.get('network') and kwargs.get('network_mode'): + raise RuntimeError( + 'The options "network" and "network_mode" can not be used ' + 'together.' + ) + try: container = self.create(image=image, command=command, detach=detach, **kwargs) @@ -902,10 +911,10 @@ def _create_container_args(kwargs): if volumes: host_config_kwargs['binds'] = volumes - networks = kwargs.pop('networks', []) - if networks: - create_kwargs['networking_config'] = {network: None - for network in networks} + network = kwargs.pop('network', None) + if network: + create_kwargs['networking_config'] = {network: None} + host_config_kwargs['network_mode'] = network # All kwargs should have been consumed by this point, so raise # error if any are left diff --git a/docker/models/plugins.py b/docker/models/plugins.py index 6cdf01ca00..06880181f5 100644 --- a/docker/models/plugins.py +++ b/docker/models/plugins.py @@ -103,8 +103,8 @@ def upgrade(self, remote=None): Args: remote (string): Remote reference to upgrade to. The - ``:latest`` tag is optional and is the default if omitted. - Default: this plugin's name. + ``:latest`` tag is optional and is the default if omitted. + Default: this plugin's name. Returns: A generator streaming the decoded API logs diff --git a/docker/types/services.py b/docker/types/services.py index 07409cd816..012f7b0199 100644 --- a/docker/types/services.py +++ b/docker/types/services.py @@ -127,7 +127,7 @@ def __init__(self, image, command=None, args=None, hostname=None, env=None, class Mount(dict): """ Describes a mounted folder's configuration inside a container. A list of - :py:class:`Mount`s would be used as part of a + :py:class:`Mount` would be used as part of a :py:class:`~docker.types.ContainerSpec`. Args: diff --git a/tests/integration/models_containers_test.py b/tests/integration/models_containers_test.py index 4f1e6a1fe9..b76a88ffcf 100644 --- a/tests/integration/models_containers_test.py +++ b/tests/integration/models_containers_test.py @@ -1,6 +1,7 @@ import docker import tempfile from .base import BaseIntegrationTest, TEST_API_VERSION +from ..helpers import random_name class ContainerCollectionTest(BaseIntegrationTest): @@ -69,6 +70,24 @@ def test_run_with_named_volume(self): ) self.assertEqual(out, b'hello\n') + def test_run_with_network(self): + net_name = random_name() + client = docker.from_env(version=TEST_API_VERSION) + client.networks.create(net_name) + self.tmp_networks.append(net_name) + + container = client.containers.run( + 'alpine', 'echo hello world', network=net_name, + detach=True + ) + self.tmp_containers.append(container.id) + + attrs = container.attrs + + assert 'NetworkSettings' in attrs + assert 'Networks' in attrs['NetworkSettings'] + assert list(attrs['NetworkSettings']['Networks'].keys()) == [net_name] + def test_get(self): client = docker.from_env(version=TEST_API_VERSION) container = client.containers.run("alpine", "sleep 300", detach=True) diff --git a/tests/unit/models_containers_test.py b/tests/unit/models_containers_test.py index 0fb69f3e34..70c86480c8 100644 --- a/tests/unit/models_containers_test.py +++ b/tests/unit/models_containers_test.py @@ -71,8 +71,7 @@ def test_create_container_args(self): memswap_limit=456, name='somename', network_disabled=False, - network_mode='blah', - networks=['foo'], + network='foo', oom_kill_disable=True, oom_score_adj=5, pid_mode='host', @@ -153,7 +152,7 @@ def test_create_container_args(self): 'MemoryReservation': 123, 'MemorySwap': 456, 'MemorySwappiness': 2, - 'NetworkMode': 'blah', + 'NetworkMode': 'foo', 'OomKillDisable': True, 'OomScoreAdj': 5, 'PidMode': 'host', From 717459db0ed693f56009d24ae0ae1607705bc8b5 Mon Sep 17 00:00:00 2001 From: allencloud Date: Fri, 12 May 2017 10:25:40 +0800 Subject: [PATCH 0352/1301] update docker-py test status code from 500 to 400 Signed-off-by: allencloud --- tests/integration/api_swarm_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/api_swarm_test.py b/tests/integration/api_swarm_test.py index d06cac21bd..0a2f69f3c9 100644 --- a/tests/integration/api_swarm_test.py +++ b/tests/integration/api_swarm_test.py @@ -168,7 +168,7 @@ def test_remove_main_node(self): with pytest.raises(docker.errors.APIError) as e: self.client.remove_node(node_id) - assert e.value.response.status_code == 500 + assert e.value.response.status_code >= 400 with pytest.raises(docker.errors.APIError) as e: self.client.remove_node(node_id, True) From 6ea1ea8a51c54c0c7f63ed52c6a98fb567a0d4dd Mon Sep 17 00:00:00 2001 From: Yusuke Miyazaki Date: Sun, 14 May 2017 05:31:21 +0900 Subject: [PATCH 0353/1301] Fix docstring of ImageCollection.get Signed-off-by: Yusuke Miyazaki --- docker/models/images.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/models/images.py b/docker/models/images.py index a12ede3067..52a44b27bf 100644 --- a/docker/models/images.py +++ b/docker/models/images.py @@ -193,8 +193,8 @@ def get(self, name): (:py:class:`Image`): The image. Raises: - :py:class:`docker.errors.ImageNotFound` If the image does not - exist. + :py:class:`docker.errors.ImageNotFound` + If the image does not exist. :py:class:`docker.errors.APIError` If the server returns an error. """ From e4093ab258df6e1978e1090ddadede1c869abc1e Mon Sep 17 00:00:00 2001 From: Yong Wen Chua Date: Tue, 16 May 2017 15:36:09 +0800 Subject: [PATCH 0354/1301] Add `target` argument to image building This is related to the multi-stage image building that was introduced in 17.05 (API 1.29). This allows a user to specify the stage of a multi-stage Dockerfile to build for, rather than the final stage. Signed-off-by: Yong Wen Chua --- docker/api/build.py | 12 +++++++++++- docker/models/images.py | 2 ++ tests/integration/api_build_test.py | 22 ++++++++++++++++++++++ 3 files changed, 35 insertions(+), 1 deletion(-) diff --git a/docker/api/build.py b/docker/api/build.py index 5c34c47b38..f30be4168a 100644 --- a/docker/api/build.py +++ b/docker/api/build.py @@ -18,7 +18,7 @@ def build(self, path=None, tag=None, quiet=False, fileobj=None, custom_context=False, encoding=None, pull=False, forcerm=False, dockerfile=None, container_limits=None, decode=False, buildargs=None, gzip=False, shmsize=None, - labels=None, cache_from=None): + labels=None, cache_from=None, target=None): """ Similar to the ``docker build`` command. Either ``path`` or ``fileobj`` needs to be set. ``path`` can be a local path (to a directory @@ -94,6 +94,8 @@ def build(self, path=None, tag=None, quiet=False, fileobj=None, labels (dict): A dictionary of labels to set on the image. cache_from (list): A list of images used for build cache resolution. + target (str): Name of the build-stage to build in a multi-stage + Dockerfile. Returns: A generator for the build output. @@ -198,6 +200,14 @@ def build(self, path=None, tag=None, quiet=False, fileobj=None, 'cache_from was only introduced in API version 1.25' ) + if target: + if utils.version_gte(self._version, '1.29'): + params.update({'target': target}) + else: + raise errors.InvalidVersion( + 'target was only introduced in API version 1.29' + ) + if context is not None: headers = {'Content-Type': 'application/tar'} if encoding: diff --git a/docker/models/images.py b/docker/models/images.py index 52a44b27bf..a9ed65ee35 100644 --- a/docker/models/images.py +++ b/docker/models/images.py @@ -151,6 +151,8 @@ def build(self, **kwargs): decoded into dicts on the fly. Default ``False``. cache_from (list): A list of images used for build cache resolution. + target (str): Name of the build-stage to build in a multi-stage + Dockerfile. Returns: (:py:class:`Image`): The built image. diff --git a/tests/integration/api_build_test.py b/tests/integration/api_build_test.py index fe5d994dd6..623b660931 100644 --- a/tests/integration/api_build_test.py +++ b/tests/integration/api_build_test.py @@ -189,6 +189,28 @@ def test_build_with_cache_from(self): counter += 1 assert counter == 0 + @requires_api_version('1.29') + def test_build_container_with_target(self): + script = io.BytesIO('\n'.join([ + 'FROM busybox as first', + 'RUN mkdir -p /tmp/test', + 'RUN touch /tmp/silence.tar.gz', + 'FROM alpine:latest', + 'WORKDIR /root/' + 'COPY --from=first /tmp/silence.tar.gz .', + 'ONBUILD RUN echo "This should not be in the final image"' + ]).encode('ascii')) + + stream = self.client.build( + fileobj=script, target='first', tag='build1' + ) + self.tmp_imgs.append('build1') + for chunk in stream: + pass + + info = self.client.inspect_image('build1') + self.assertEqual(info['Config']['OnBuild'], []) + def test_build_stderr_data(self): control_chars = ['\x1b[91m', '\x1b[0m'] snippet = 'Ancient Temple (Mystic Oriental Dream ~ Ancient Temple)' From ba334f8bd5ddf517b8411998b657070ba0166f53 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 16 May 2017 14:53:41 -0700 Subject: [PATCH 0355/1301] Bump 2.3.0 Signed-off-by: Joffrey F --- docker/version.py | 2 +- docs/change-log.md | 41 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index 320dae622d..c734a16114 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "2.3.0-dev" +version = "2.3.0" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) diff --git a/docs/change-log.md b/docs/change-log.md index 5d9b05b37d..3d58f931ff 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -1,6 +1,47 @@ Change log ========== +2.3.0 +----- + +[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/31?closed=1) + +### Features + +* Added support for the following `HostConfig` parameters: `volume_driver`, + `cpu_count`, `cpu_percent`, `nano_cpus`, `cpuset_mems`. +* Added support for `verbose` parameter in `APIClient.inspect_network` and + `DockerClient.networks.get`. +* Added support for the `environment` parameter in `APIClient.exec_create` + and `Container.exec_run` +* Added `reload_config` method to `APIClient`, that lets the user reload + the `config.json` data from disk. +* Added `labels` property to the `Image` and `Container` classes. +* Added `image` property to the `Container` class. + +### Bugfixes + +* Fixed a bug where setting `replicas` to zero in `ServiceMode` would not + register as a valid entry. +* Fixed a bug where `DockerClient.images.build` would report a failure after + a successful build if a `tag` was set. +* Fixed an issue where `DockerClient.images.pull` would fail to return the + corresponding image object if a `tag` was set. +* Fixed a bug where a list of `mounts` provided to `APIClient.create_service` + would sometimes be parsed incorrectly. +* Fixed a bug where calling `Network.containers` would crash when no containers + were associated with the network. +* Fixed an issue where `Network.connect` and `Network.disconnect` would not + accept some of the documented parameters. +* Fixed a bug where the `cpuset_cpus` parameter would not be properly set in + `APIClient.create_host_config`. + +### Miscellaneous + +* The invalid `networks` argument in `DockerClient.containers.run` has been + replaced with a (working) singular `network` argument. + + 2.2.1 ----- From 7880c5af1de66ed4555a30eeb19dc0093536f2f0 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 16 May 2017 17:19:37 -0700 Subject: [PATCH 0356/1301] dev version Signed-off-by: Joffrey F --- docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index c734a16114..6979e1bef8 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "2.3.0" +version = "2.4.0-dev" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) From 9cc021dfa684ab1a614d473e78f9c4c0fc960585 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 16 May 2017 19:05:32 -0700 Subject: [PATCH 0357/1301] Add support for placement preferences and platforms in TaskTemplate Signed-off-by: Joffrey F --- docker/api/service.py | 69 ++++++++++++++++----------- docker/types/__init__.py | 4 +- docker/types/services.py | 31 +++++++++++- tests/integration/api_service_test.py | 43 +++++++++++++++++ 4 files changed, 115 insertions(+), 32 deletions(-) diff --git a/docker/api/service.py b/docker/api/service.py index 4972c16d1c..aea93cbfc9 100644 --- a/docker/api/service.py +++ b/docker/api/service.py @@ -3,6 +3,43 @@ from ..types import ServiceMode +def _check_api_features(version, task_template, update_config): + if update_config is not None: + if utils.version_lt(version, '1.25'): + if 'MaxFailureRatio' in update_config: + raise errors.InvalidVersion( + 'UpdateConfig.max_failure_ratio is not supported in' + ' API version < 1.25' + ) + if 'Monitor' in update_config: + raise errors.InvalidVersion( + 'UpdateConfig.monitor is not supported in' + ' API version < 1.25' + ) + + if task_template is not None: + if 'ForceUpdate' in task_template and utils.version_lt( + version, '1.25'): + raise errors.InvalidVersion( + 'force_update is not supported in API version < 1.25' + ) + + if task_template.get('Placement'): + if utils.version_lt(version, '1.30'): + if task_template['Placement'].get('Platforms'): + raise errors.InvalidVersion( + 'Placement.platforms is not supported in' + ' API version < 1.30' + ) + + if utils.version_lt(version, '1.27'): + if task_template['Placement'].get('Preferences'): + raise errors.InvalidVersion( + 'Placement.preferences is not supported in' + ' API version < 1.27' + ) + + class ServiceApiMixin(object): @utils.minimum_version('1.24') def create_service( @@ -43,6 +80,8 @@ def create_service( ) endpoint_spec = endpoint_config + _check_api_features(self._version, task_template, update_config) + url = self._url('/services/create') headers = {} image = task_template.get('ContainerSpec', {}).get('Image', None) @@ -67,17 +106,6 @@ def create_service( } if update_config is not None: - if utils.version_lt(self._version, '1.25'): - if 'MaxFailureRatio' in update_config: - raise errors.InvalidVersion( - 'UpdateConfig.max_failure_ratio is not supported in' - ' API version < 1.25' - ) - if 'Monitor' in update_config: - raise errors.InvalidVersion( - 'UpdateConfig.monitor is not supported in' - ' API version < 1.25' - ) data['UpdateConfig'] = update_config return self._result( @@ -282,6 +310,8 @@ def update_service(self, service, version, task_template=None, name=None, ) endpoint_spec = endpoint_config + _check_api_features(self._version, task_template, update_config) + url = self._url('/services/{0}/update', service) data = {} headers = {} @@ -294,12 +324,6 @@ def update_service(self, service, version, task_template=None, name=None, mode = ServiceMode(mode) data['Mode'] = mode if task_template is not None: - if 'ForceUpdate' in task_template and utils.version_lt( - self._version, '1.25'): - raise errors.InvalidVersion( - 'force_update is not supported in API version < 1.25' - ) - image = task_template.get('ContainerSpec', {}).get('Image', None) if image is not None: registry, repo_name = auth.resolve_repository_name(image) @@ -308,17 +332,6 @@ def update_service(self, service, version, task_template=None, name=None, headers['X-Registry-Auth'] = auth_header data['TaskTemplate'] = task_template if update_config is not None: - if utils.version_lt(self._version, '1.25'): - if 'MaxFailureRatio' in update_config: - raise errors.InvalidVersion( - 'UpdateConfig.max_failure_ratio is not supported in' - ' API version < 1.25' - ) - if 'Monitor' in update_config: - raise errors.InvalidVersion( - 'UpdateConfig.monitor is not supported in' - ' API version < 1.25' - ) data['UpdateConfig'] = update_config if networks is not None: diff --git a/docker/types/__init__.py b/docker/types/__init__.py index 0e88776013..edc919dfcf 100644 --- a/docker/types/__init__.py +++ b/docker/types/__init__.py @@ -3,7 +3,7 @@ from .healthcheck import Healthcheck from .networks import EndpointConfig, IPAMConfig, IPAMPool, NetworkingConfig from .services import ( - ContainerSpec, DriverConfig, EndpointSpec, Mount, Resources, RestartPolicy, - SecretReference, ServiceMode, TaskTemplate, UpdateConfig + ContainerSpec, DriverConfig, EndpointSpec, Mount, Placement, Resources, + RestartPolicy, SecretReference, ServiceMode, TaskTemplate, UpdateConfig ) from .swarm import SwarmSpec, SwarmExternalCA diff --git a/docker/types/services.py b/docker/types/services.py index 012f7b0199..7456a42ba5 100644 --- a/docker/types/services.py +++ b/docker/types/services.py @@ -20,7 +20,9 @@ class TaskTemplate(dict): individual container created as part of the service. restart_policy (RestartPolicy): Specification for the restart policy which applies to containers created as part of this service. - placement (:py:class:`list`): A list of constraints. + placement (Placement): Placement instructions for the scheduler. + If a list is passed instead, it is assumed to be a list of + constraints as part of a :py:class:`Placement` object. force_update (int): A counter that triggers an update even if no relevant parameters have been changed. """ @@ -33,7 +35,7 @@ def __init__(self, container_spec, resources=None, restart_policy=None, self['RestartPolicy'] = restart_policy if placement: if isinstance(placement, list): - placement = {'Constraints': placement} + placement = Placement(constraints=placement) self['Placement'] = placement if log_driver: self['LogDriver'] = log_driver @@ -452,3 +454,28 @@ def __init__(self, secret_id, secret_name, filename=None, uid=None, 'GID': gid or '0', 'Mode': mode } + + +class Placement(dict): + """ + Placement constraints to be used as part of a :py:class:`TaskTemplate` + + Args: + constraints (list): A list of constraints + preferences (list): Preferences provide a way to make the + scheduler aware of factors such as topology. They are provided + in order from highest to lowest precedence. + platforms (list): A list of platforms expressed as ``(arch, os)`` + tuples + """ + def __init__(self, constraints=None, preferences=None, platforms=None): + if constraints is not None: + self['Constraints'] = constraints + if preferences is not None: + self['Preferences'] = preferences + if platforms: + self['Platforms'] = [] + for plat in platforms: + self['Platforms'].append({ + 'Architecture': plat[0], 'OS': plat[1] + }) diff --git a/tests/integration/api_service_test.py b/tests/integration/api_service_test.py index 914e516bc4..8ac852d960 100644 --- a/tests/integration/api_service_test.py +++ b/tests/integration/api_service_test.py @@ -270,6 +270,49 @@ def test_create_service_with_placement(self): assert (svc_info['Spec']['TaskTemplate']['Placement'] == {'Constraints': ['node.id=={}'.format(node_id)]}) + def test_create_service_with_placement_object(self): + node_id = self.client.nodes()[0]['ID'] + container_spec = docker.types.ContainerSpec(BUSYBOX, ['true']) + placemt = docker.types.Placement( + constraints=['node.id=={}'.format(node_id)] + ) + task_tmpl = docker.types.TaskTemplate( + container_spec, placement=placemt + ) + name = self.get_service_name() + svc_id = self.client.create_service(task_tmpl, name=name) + svc_info = self.client.inspect_service(svc_id) + assert 'Placement' in svc_info['Spec']['TaskTemplate'] + assert svc_info['Spec']['TaskTemplate']['Placement'] == placemt + + @requires_api_version('1.30') + def test_create_service_with_placement_platform(self): + container_spec = docker.types.ContainerSpec(BUSYBOX, ['true']) + placemt = docker.types.Placement(platforms=[('x86_64', 'linux')]) + task_tmpl = docker.types.TaskTemplate( + container_spec, placement=placemt + ) + name = self.get_service_name() + svc_id = self.client.create_service(task_tmpl, name=name) + svc_info = self.client.inspect_service(svc_id) + assert 'Placement' in svc_info['Spec']['TaskTemplate'] + assert svc_info['Spec']['TaskTemplate']['Placement'] == placemt + + @requires_api_version('1.27') + def test_create_service_with_placement_preferences(self): + container_spec = docker.types.ContainerSpec(BUSYBOX, ['true']) + placemt = docker.types.Placement(preferences=[ + {'Spread': {'SpreadDescriptor': 'com.dockerpy.test'}} + ]) + task_tmpl = docker.types.TaskTemplate( + container_spec, placement=placemt + ) + name = self.get_service_name() + svc_id = self.client.create_service(task_tmpl, name=name) + svc_info = self.client.inspect_service(svc_id) + assert 'Placement' in svc_info['Spec']['TaskTemplate'] + assert svc_info['Spec']['TaskTemplate']['Placement'] == placemt + def test_create_service_with_endpoint_spec(self): container_spec = docker.types.ContainerSpec(BUSYBOX, ['true']) task_tmpl = docker.types.TaskTemplate(container_spec) From ff718f5dac2ba00ffbb52c0f3b1af5b687f07930 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 17 May 2017 15:23:36 -0700 Subject: [PATCH 0358/1301] Add support for ingress in create_network Signed-off-by: Joffrey F --- docker/api/network.py | 13 ++++++++++++- docker/models/networks.py | 2 ++ tests/integration/api_network_test.py | 8 ++++++++ 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/docker/api/network.py b/docker/api/network.py index 74f4cd2b30..3a454546c7 100644 --- a/docker/api/network.py +++ b/docker/api/network.py @@ -41,7 +41,8 @@ def networks(self, names=None, ids=None, filters=None): @minimum_version('1.21') def create_network(self, name, driver=None, options=None, ipam=None, check_duplicate=None, internal=False, labels=None, - enable_ipv6=False, attachable=None, scope=None): + enable_ipv6=False, attachable=None, scope=None, + ingress=None): """ Create a network. Similar to the ``docker network create``. @@ -60,6 +61,8 @@ def create_network(self, name, driver=None, options=None, ipam=None, attachable (bool): If enabled, and the network is in the global scope, non-service containers on worker nodes will be able to connect to the network. + ingress (bool): If set, create an ingress network which provides + the routing-mesh in swarm mode. Returns: (dict): The created network reference object @@ -129,6 +132,14 @@ def create_network(self, name, driver=None, options=None, ipam=None, ) data['Attachable'] = attachable + if ingress is not None: + if version_lt(self._version, '1.29'): + raise InvalidVersion( + 'ingress is not supported in API version < 1.29' + ) + + data['Ingress'] = ingress + url = self._url("/networks/create") res = self._post_json(url, data=data) return self._result(res, json=True) diff --git a/docker/models/networks.py b/docker/models/networks.py index 586809753b..afb0ebe8b2 100644 --- a/docker/models/networks.py +++ b/docker/models/networks.py @@ -111,6 +111,8 @@ def create(self, name, *args, **kwargs): labels (dict): Map of labels to set on the network. Default ``None``. enable_ipv6 (bool): Enable IPv6 on the network. Default ``False``. + ingress (bool): If set, create an ingress network which provides + the routing-mesh in swarm mode. Returns: (:py:class:`Network`): The network that was created. diff --git a/tests/integration/api_network_test.py b/tests/integration/api_network_test.py index b3ae512080..5439dd7b2e 100644 --- a/tests/integration/api_network_test.py +++ b/tests/integration/api_network_test.py @@ -452,6 +452,14 @@ def test_create_network_attachable(self): net = self.client.inspect_network(net_id) assert net['Attachable'] is True + @requires_api_version('1.29') + def test_create_network_ingress(self): + assert self.client.init_swarm('eth0') + self.client.remove_network('ingress') + _, net_id = self.create_network(driver='overlay', ingress=True) + net = self.client.inspect_network(net_id) + assert net['Ingress'] is True + @requires_api_version('1.25') def test_prune_networks(self): net_name, _ = self.create_network() From f6f5652eb265fb8311b1e6ad4b16979b9808f908 Mon Sep 17 00:00:00 2001 From: Alexey Rokhin Date: Wed, 17 May 2017 23:18:18 +0300 Subject: [PATCH 0359/1301] fix type checking for nano_cpus Signed-off-by: Alexey Rokhin --- docker/types/containers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/types/containers.py b/docker/types/containers.py index 18d18381da..f33c5e836f 100644 --- a/docker/types/containers.py +++ b/docker/types/containers.py @@ -466,7 +466,7 @@ def __init__(self, version, binds=None, port_bindings=None, self['CpuPercent'] = cpu_percent if nano_cpus: - if not isinstance(nano_cpus, int): + if not isinstance(nano_cpus, six.integer_types): raise host_config_type_error('nano_cpus', nano_cpus, 'int') if version_lt(version, '1.25'): raise host_config_version_error('nano_cpus', '1.25') From 41aae65ab2714168e56118c494d16128ef0929b2 Mon Sep 17 00:00:00 2001 From: allencloud Date: Thu, 18 May 2017 10:06:58 +0800 Subject: [PATCH 0360/1301] update swarm remove test status code from 500 to >= 400 Signed-off-by: allencloud --- tests/integration/api_swarm_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/api_swarm_test.py b/tests/integration/api_swarm_test.py index 0a2f69f3c9..666c689f55 100644 --- a/tests/integration/api_swarm_test.py +++ b/tests/integration/api_swarm_test.py @@ -173,4 +173,4 @@ def test_remove_main_node(self): with pytest.raises(docker.errors.APIError) as e: self.client.remove_node(node_id, True) - assert e.value.response.status_code == 500 + assert e.value.response.status_code >= 400 From 45aec93089b8b5133ed4eae125ffc29878b788ea Mon Sep 17 00:00:00 2001 From: Chris Ottinger Date: Sat, 27 May 2017 00:21:19 +1000 Subject: [PATCH 0361/1301] fix #1625 where ImageCollection.build() could return early with incorrect image_id depending on docer build output Signed-off-by: Chris Ottinger --- docker/models/images.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docker/models/images.py b/docker/models/images.py index a9ed65ee35..9af040cfcf 100644 --- a/docker/models/images.py +++ b/docker/models/images.py @@ -169,19 +169,20 @@ def build(self, **kwargs): if isinstance(resp, six.string_types): return self.get(resp) last_event = None + image_id = None for chunk in json_stream(resp): if 'error' in chunk: raise BuildError(chunk['error']) if 'stream' in chunk: match = re.search( - r'(Successfully built |sha256:)([0-9a-f]+)', + r'(^Successfully built |sha256:)([0-9a-f]+)$', chunk['stream'] ) if match: image_id = match.group(2) - return self.get(image_id) last_event = chunk - + if image_id: + return self.get(image_id) raise BuildError(last_event or 'Unknown') def get(self, name): From 6ef9d426eb259650c8a4ff5c25c878f462c2bb9d Mon Sep 17 00:00:00 2001 From: Chris Ottinger Date: Sat, 27 May 2017 10:29:36 +1000 Subject: [PATCH 0362/1301] added integration test for #1625 for ImageCollection.build() that verfies that the build method uses the last success message for extracting the image id Signed-off-by: Chris Ottinger --- tests/integration/models_images_test.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/integration/models_images_test.py b/tests/integration/models_images_test.py index 6d61e4977c..2b45429d8a 100644 --- a/tests/integration/models_images_test.py +++ b/tests/integration/models_images_test.py @@ -39,6 +39,17 @@ def test_build_with_multiple_success(self): self.tmp_imgs.append(image.id) assert client.containers.run(image) == b"hello world\n" + def test_build_with_success_build_output(self): + client = docker.from_env(version=TEST_API_VERSION) + image = client.images.build( + tag='dup-txt-tag', fileobj=io.BytesIO( + "FROM alpine\n" + "CMD echo Successfully built 33c838732b70".encode('ascii') + ) + ) + self.tmp_imgs.append(image.id) + assert client.containers.run(image) == b"Successfully built 33c838732b70\n" + def test_list(self): client = docker.from_env(version=TEST_API_VERSION) image = client.images.pull('alpine:latest') From 1223fc144fc9b529959ba554f1f5e45e63c50514 Mon Sep 17 00:00:00 2001 From: Chris Ottinger Date: Sat, 27 May 2017 11:24:58 +1000 Subject: [PATCH 0363/1301] new integration task linting for #1625 Signed-off-by: Chris Ottinger --- tests/integration/models_images_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/models_images_test.py b/tests/integration/models_images_test.py index 2b45429d8a..721a19db61 100644 --- a/tests/integration/models_images_test.py +++ b/tests/integration/models_images_test.py @@ -44,11 +44,11 @@ def test_build_with_success_build_output(self): image = client.images.build( tag='dup-txt-tag', fileobj=io.BytesIO( "FROM alpine\n" - "CMD echo Successfully built 33c838732b70".encode('ascii') + "CMD echo Successfully built abcd1234".encode('ascii') ) ) self.tmp_imgs.append(image.id) - assert client.containers.run(image) == b"Successfully built 33c838732b70\n" + assert client.containers.run(image) == b"Successfully built abcd1234\n" def test_list(self): client = docker.from_env(version=TEST_API_VERSION) From 9eecfb0d2fa6973c08dd78e1cc81cbd2098ec3b8 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 30 May 2017 11:45:00 -0700 Subject: [PATCH 0364/1301] Fix misleading build method docs Signed-off-by: Joffrey F --- docker/models/images.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/docker/models/images.py b/docker/models/images.py index a9ed65ee35..81a21d66ba 100644 --- a/docker/models/images.py +++ b/docker/models/images.py @@ -126,9 +126,6 @@ def build(self, **kwargs): rm (bool): Remove intermediate containers. The ``docker build`` command now defaults to ``--rm=true``, but we have kept the old default of `False` to preserve backward compatibility - stream (bool): *Deprecated for API version > 1.8 (always True)*. - Return a blocking generator you can iterate over to retrieve - build output as it happens timeout (int): HTTP timeout custom_context (bool): Optional if using ``fileobj`` encoding (str): The encoding for a stream. Set to ``gzip`` for From 6ae24b9e60cd8f080a6c657ccbdffd22056169dd Mon Sep 17 00:00:00 2001 From: Madhuri Kumari Date: Thu, 1 Jun 2017 15:09:46 +0000 Subject: [PATCH 0365/1301] Add support for ``runtime`` in container create and run API --- docker/api/container.py | 6 ++++-- docker/models/containers.py | 2 ++ docker/types/containers.py | 10 +++++++--- docs/change-log.md | 2 ++ tests/integration/api_container_test.py | 9 +++++++++ 5 files changed, 24 insertions(+), 5 deletions(-) diff --git a/docker/api/container.py b/docker/api/container.py index 97b5405935..0abfca4f7c 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -238,7 +238,7 @@ def create_container(self, image, command=None, hostname=None, user=None, memswap_limit=None, cpuset=None, host_config=None, mac_address=None, labels=None, volume_driver=None, stop_signal=None, networking_config=None, - healthcheck=None, stop_timeout=None): + healthcheck=None, stop_timeout=None, runtime=None): """ Creates a container. Parameters are similar to those for the ``docker run`` command except it doesn't support the attach options (``-a``). @@ -417,6 +417,7 @@ def create_container(self, image, command=None, hostname=None, user=None, Default: 10 networking_config (dict): A networking configuration generated by :py:meth:`create_networking_config`. + runtime (str): The name of the runtime tool to create container. Returns: A dictionary with an image 'Id' key and a 'Warnings' key. @@ -441,7 +442,7 @@ def create_container(self, image, command=None, hostname=None, user=None, network_disabled, entrypoint, cpu_shares, working_dir, domainname, memswap_limit, cpuset, host_config, mac_address, labels, volume_driver, stop_signal, networking_config, healthcheck, - stop_timeout + stop_timeout, runtime ) return self.create_container_from_config(config, name) @@ -576,6 +577,7 @@ def create_host_config(self, *args, **kwargs): values are: ``host`` volumes_from (:py:class:`list`): List of container names or IDs to get volumes from. + runtime (str): The name of the runtime tool to manage container. Returns: diff --git a/docker/models/containers.py b/docker/models/containers.py index 4bb2cf8638..46f900e06e 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -659,6 +659,7 @@ def run(self, image, command=None, stdout=True, stderr=False, volumes_from (:py:class:`list`): List of container names or IDs to get volumes from. working_dir (str): Path to the working directory. + runtime (str): The name of the runtime tool to create container. Returns: The container logs, either ``STDOUT``, ``STDERR``, or both, @@ -885,6 +886,7 @@ def prune(self, filters=None): 'userns_mode', 'version', 'volumes_from', + 'runtime' ] diff --git a/docker/types/containers.py b/docker/types/containers.py index f33c5e836f..f834c78515 100644 --- a/docker/types/containers.py +++ b/docker/types/containers.py @@ -120,7 +120,7 @@ def __init__(self, version, binds=None, port_bindings=None, isolation=None, auto_remove=False, storage_opt=None, init=None, init_path=None, volume_driver=None, cpu_count=None, cpu_percent=None, nano_cpus=None, - cpuset_mems=None): + cpuset_mems=None, runtime=None): if mem_limit is not None: self['Memory'] = parse_bytes(mem_limit) @@ -473,6 +473,9 @@ def __init__(self, version, binds=None, port_bindings=None, self['NanoCpus'] = nano_cpus + if runtime: + self['Runtime'] = runtime + def host_config_type_error(param, param_value, expected): error_msg = 'Invalid type for {0} param: expected {1} but found {2}' @@ -499,7 +502,7 @@ def __init__( working_dir=None, domainname=None, memswap_limit=None, cpuset=None, host_config=None, mac_address=None, labels=None, volume_driver=None, stop_signal=None, networking_config=None, healthcheck=None, - stop_timeout=None + stop_timeout=None, runtime=None ): if version_gte(version, '1.10'): message = ('{0!r} parameter has no effect on create_container().' @@ -659,5 +662,6 @@ def __init__( 'VolumeDriver': volume_driver, 'StopSignal': stop_signal, 'Healthcheck': healthcheck, - 'StopTimeout': stop_timeout + 'StopTimeout': stop_timeout, + 'Runtime': runtime }) diff --git a/docs/change-log.md b/docs/change-log.md index 3d58f931ff..20bf9e092e 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -122,6 +122,8 @@ Change log * Added support for `force_update` in `TaskTemplate` * Made `name` parameter optional in `APIClient.create_volume` and `DockerClient.volumes.create` +* Added support for `runtime` in `APIClient.create_container` and + `DockerClient.containers.run` ### Bugfixes diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index fb4c4e4adc..c499e35e42 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -1256,6 +1256,15 @@ def test_container_cpuset(self): self.assertEqual(inspect_data['HostConfig']['CpusetCpus'], cpuset_cpus) + def test_create_with_runtime(self): + container = self.client.create_container( + BUSYBOX, ['echo', 'test'], runtime='runc' + ) + self.tmp_containers.append(container['Id']) + config = self.client.inspect_container(container) + assert config['Config']['Runtime'] == 'runc' + + class LinkTest(BaseAPIIntegrationTest): def test_remove_link(self): # Create containers From 5dd91cd4aaa2e7cd8dde1dd316d53cab25ef9b78 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Mon, 5 Jun 2017 17:51:37 +0200 Subject: [PATCH 0366/1301] Rewrite the split_port function using re In the case of a defined format with specific parts, a regular expression with named capturing bits make reasoning about the parts simpler than imlementing a parser from scratch. Signed-off-by: kaiyou --- docker/utils/ports.py | 107 ++++++++++++++++-------------------------- 1 file changed, 40 insertions(+), 67 deletions(-) diff --git a/docker/utils/ports.py b/docker/utils/ports.py index 3708958d4e..57332deee4 100644 --- a/docker/utils/ports.py +++ b/docker/utils/ports.py @@ -1,3 +1,16 @@ +import re + +PORT_SPEC = re.compile( + "^" # Match full string + "(" # External part + "((?P[a-fA-F\d.:]+):)?" # Address + "(?P[\d]*)(-(?P[\d]+))?:" # External range + ")?" + "(?P[\d]+)(-(?P[\d]+))?" # Internal range + "(?P/(udp|tcp))?" # Protocol + "$" # Match full string +) + def add_port_mapping(port_bindings, internal_port, external): if internal_port in port_bindings: @@ -24,81 +37,41 @@ def build_port_bindings(ports): return port_bindings -def to_port_range(port, randomly_available_port=False): - if not port: - return None - - protocol = "" - if "/" in port: - parts = port.split("/") - if len(parts) != 2: - _raise_invalid_port(port) - - port, protocol = parts - protocol = "/" + protocol - - if randomly_available_port: - return ["%s%s" % (port, protocol)] - - parts = str(port).split('-') - - if len(parts) == 1: - return ["%s%s" % (port, protocol)] - - if len(parts) == 2: - full_port_range = range(int(parts[0]), int(parts[1]) + 1) - return ["%s%s" % (p, protocol) for p in full_port_range] - - raise ValueError('Invalid port range "%s", should be ' - 'port or startport-endport' % port) - - def _raise_invalid_port(port): raise ValueError('Invalid port "%s", should be ' '[[remote_ip:]remote_port[-remote_port]:]' 'port[/protocol]' % port) -def split_port(port): - parts = str(port).split(':') - - if not 1 <= len(parts) <= 3: - _raise_invalid_port(port) - - if len(parts) == 1: - internal_port, = parts - if not internal_port: - _raise_invalid_port(port) - return to_port_range(internal_port), None - if len(parts) == 2: - external_port, internal_port = parts - - internal_range = to_port_range(internal_port) - if internal_range is None: - _raise_invalid_port(port) - - external_range = to_port_range(external_port, len(internal_range) == 1) - if external_range is None: - _raise_invalid_port(port) - - if len(internal_range) != len(external_range): - raise ValueError('Port ranges don\'t match in length') - - return internal_range, external_range +def port_range(start, end, proto, randomly_available_port=False): + if not start: + return start + if not end: + return [start + proto] + if randomly_available_port: + return ['{}-{}'.format(start, end) + proto] + return [str(port) + proto for port in range(int(start), int(end) + 1)] - external_ip, external_port, internal_port = parts - if not internal_port: +def split_port(port): + match = PORT_SPEC.match(port) + if match is None: _raise_invalid_port(port) + parts = match.groupdict() - internal_range = to_port_range(internal_port) - external_range = to_port_range(external_port, len(internal_range) == 1) - - if not external_range: - external_range = [None] * len(internal_range) - - if len(internal_range) != len(external_range): - raise ValueError('Port ranges don\'t match in length') + host = parts['host'] + proto = parts['proto'] or '' + internal = port_range(parts['int'], parts['int_end'], proto) + external = port_range( + parts['ext'], parts['ext_end'], '', len(internal) == 1) - return internal_range, [(external_ip, ex_port or None) - for ex_port in external_range] + if host is None: + if external is not None and len(internal) != len(external): + raise ValueError('Port ranges don\'t match in length') + return internal, external + else: + if not external: + external = [None] * len(internal) + elif len(internal) != len(external): + raise ValueError('Port ranges don\'t match in length') + return internal, [(host, ext_port) for ext_port in external] From 0c1271350db33cb21265309a31da2d1c399b8243 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Mon, 5 Jun 2017 17:57:46 +0200 Subject: [PATCH 0367/1301] Add a specific unit test for splitting port with IPv6 The test was copied from https://github.com/greybyte/docker-py/commit/ccec87ca2c2aacfcfe3b38c5bc7d59dd73551c51 Signed-off-by: kaiyou --- tests/unit/utils_test.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index 25ed0f9b7f..c25881d142 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -552,6 +552,12 @@ def test_split_port_range_with_protocol(self): self.assertEqual(external_port, [("127.0.0.1", "1000"), ("127.0.0.1", "1001")]) + def test_split_port_with_ipv6_address(self): + internal_port, external_port = split_port( + "2001:abcd:ef00::2:1000:2000") + self.assertEqual(internal_port, ["2000"]) + self.assertEqual(external_port, [("2001:abcd:ef00::2", "1000")]) + def test_split_port_invalid(self): self.assertRaises(ValueError, lambda: split_port("0.0.0.0:1000:2000:tcp")) From 612c0f3d0de298884b80766d285f8ad47ad742a0 Mon Sep 17 00:00:00 2001 From: Madhuri Kumari Date: Thu, 1 Jun 2017 16:53:58 +0000 Subject: [PATCH 0368/1301] Fix test cases for ``runtime`` config Signed-off-by: Madhuri Kumari --- docker/api/container.py | 4 ++-- docker/models/containers.py | 2 +- docker/types/containers.py | 2 ++ docs/change-log.md | 2 -- tests/integration/api_container_test.py | 6 +++--- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docker/api/container.py b/docker/api/container.py index 0abfca4f7c..2352df9b4b 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -417,7 +417,7 @@ def create_container(self, image, command=None, hostname=None, user=None, Default: 10 networking_config (dict): A networking configuration generated by :py:meth:`create_networking_config`. - runtime (str): The name of the runtime tool to create container. + runtime (str): Runtime to use with this container. Returns: A dictionary with an image 'Id' key and a 'Warnings' key. @@ -577,7 +577,7 @@ def create_host_config(self, *args, **kwargs): values are: ``host`` volumes_from (:py:class:`list`): List of container names or IDs to get volumes from. - runtime (str): The name of the runtime tool to manage container. + runtime (str): Runtime to use with this container. Returns: diff --git a/docker/models/containers.py b/docker/models/containers.py index 46f900e06e..300c5a9d3b 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -659,7 +659,7 @@ def run(self, image, command=None, stdout=True, stderr=False, volumes_from (:py:class:`list`): List of container names or IDs to get volumes from. working_dir (str): Path to the working directory. - runtime (str): The name of the runtime tool to create container. + runtime (str): Runtime to use with this container. Returns: The container logs, either ``STDOUT``, ``STDERR``, or both, diff --git a/docker/types/containers.py b/docker/types/containers.py index f834c78515..6bbb57ae79 100644 --- a/docker/types/containers.py +++ b/docker/types/containers.py @@ -474,6 +474,8 @@ def __init__(self, version, binds=None, port_bindings=None, self['NanoCpus'] = nano_cpus if runtime: + if version_lt(version, '1.25'): + raise host_config_version_error('runtime', '1.25') self['Runtime'] = runtime diff --git a/docs/change-log.md b/docs/change-log.md index 20bf9e092e..3d58f931ff 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -122,8 +122,6 @@ Change log * Added support for `force_update` in `TaskTemplate` * Made `name` parameter optional in `APIClient.create_volume` and `DockerClient.volumes.create` -* Added support for `runtime` in `APIClient.create_container` and - `DockerClient.containers.run` ### Bugfixes diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index c499e35e42..de3fe7183c 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -1255,14 +1255,14 @@ def test_container_cpuset(self): inspect_data = self.client.inspect_container(container) self.assertEqual(inspect_data['HostConfig']['CpusetCpus'], cpuset_cpus) - - def test_create_with_runtime(self): + @requires_api_version('1.25') + def test_create_with_runtime(self): container = self.client.create_container( BUSYBOX, ['echo', 'test'], runtime='runc' ) self.tmp_containers.append(container['Id']) config = self.client.inspect_container(container) - assert config['Config']['Runtime'] == 'runc' + assert config['HostConfig']['Runtime'] == 'runc' class LinkTest(BaseAPIIntegrationTest): From ee75a1c2e349fccab4a1bcb49142756c9a8495db Mon Sep 17 00:00:00 2001 From: grahamlyons Date: Thu, 8 Jun 2017 14:31:25 +0100 Subject: [PATCH 0369/1301] Ensure default timeout is used by API Client The `from_env` method on the `docker` module passed `None` as the value for the `timeout` keyword argument which overrode the default value in the initialiser, taken from `constants` module. This sets the default in the initialiser to `None` and adds logic to set that, in the same way that `version` is handled. Signed-off-by: grahamlyons --- docker/api/client.py | 9 ++++++--- tests/unit/client_test.py | 13 +++++++++++++ 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/docker/api/client.py b/docker/api/client.py index 54ec6abb45..6822f7c7bc 100644 --- a/docker/api/client.py +++ b/docker/api/client.py @@ -83,8 +83,7 @@ class APIClient( configuration. user_agent (str): Set a custom user agent for requests to the server. """ - def __init__(self, base_url=None, version=None, - timeout=DEFAULT_TIMEOUT_SECONDS, tls=False, + def __init__(self, base_url=None, version=None, timeout=None, tls=False, user_agent=DEFAULT_USER_AGENT, num_pools=DEFAULT_NUM_POOLS): super(APIClient, self).__init__() @@ -94,7 +93,11 @@ def __init__(self, base_url=None, version=None, ) self.base_url = base_url - self.timeout = timeout + if timeout is not None: + self.timeout = timeout + else: + self.timeout = DEFAULT_TIMEOUT_SECONDS + self.headers['User-Agent'] = user_agent self._auth_configs = auth.load_config() diff --git a/tests/unit/client_test.py b/tests/unit/client_test.py index b79c68e155..c4996f1330 100644 --- a/tests/unit/client_test.py +++ b/tests/unit/client_test.py @@ -1,6 +1,9 @@ import datetime import docker from docker.utils import kwargs_from_env +from docker.constants import ( + DEFAULT_DOCKER_API_VERSION, DEFAULT_TIMEOUT_SECONDS +) import os import unittest @@ -96,3 +99,13 @@ def test_from_env_with_version(self): client = docker.from_env(version='2.32') self.assertEqual(client.api.base_url, "https://192.168.59.103:2376") self.assertEqual(client.api._version, '2.32') + + def test_from_env_without_version_uses_default(self): + client = docker.from_env() + + self.assertEqual(client.api._version, DEFAULT_DOCKER_API_VERSION) + + def test_from_env_without_timeout_uses_default(self): + client = docker.from_env() + + self.assertEqual(client.api.timeout, DEFAULT_TIMEOUT_SECONDS) From ff993dd858ffb3c6367013ed2c468903f0cf4fe9 Mon Sep 17 00:00:00 2001 From: grahamlyons Date: Fri, 9 Jun 2017 09:47:00 +0100 Subject: [PATCH 0370/1301] Move default `timeout` into `from_env` We'd like to be able to pass `None` as a value for `timeout` because it has meaning to the `requests` library (http://docs.python-requests.org/en/master/user/advanced/#timeouts) Signed-off-by: grahamlyons --- docker/api/client.py | 9 +++------ docker/client.py | 3 ++- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/docker/api/client.py b/docker/api/client.py index 6822f7c7bc..54ec6abb45 100644 --- a/docker/api/client.py +++ b/docker/api/client.py @@ -83,7 +83,8 @@ class APIClient( configuration. user_agent (str): Set a custom user agent for requests to the server. """ - def __init__(self, base_url=None, version=None, timeout=None, tls=False, + def __init__(self, base_url=None, version=None, + timeout=DEFAULT_TIMEOUT_SECONDS, tls=False, user_agent=DEFAULT_USER_AGENT, num_pools=DEFAULT_NUM_POOLS): super(APIClient, self).__init__() @@ -93,11 +94,7 @@ def __init__(self, base_url=None, version=None, timeout=None, tls=False, ) self.base_url = base_url - if timeout is not None: - self.timeout = timeout - else: - self.timeout = DEFAULT_TIMEOUT_SECONDS - + self.timeout = timeout self.headers['User-Agent'] = user_agent self._auth_configs = auth.load_config() diff --git a/docker/client.py b/docker/client.py index 09abd63322..fcfb01d8b1 100644 --- a/docker/client.py +++ b/docker/client.py @@ -1,4 +1,5 @@ from .api.client import APIClient +from .constants import DEFAULT_TIMEOUT_SECONDS from .models.containers import ContainerCollection from .models.images import ImageCollection from .models.networks import NetworkCollection @@ -73,7 +74,7 @@ def from_env(cls, **kwargs): .. _`SSL version`: https://docs.python.org/3.5/library/ssl.html#ssl.PROTOCOL_TLSv1 """ - timeout = kwargs.pop('timeout', None) + timeout = kwargs.pop('timeout', DEFAULT_TIMEOUT_SECONDS) version = kwargs.pop('version', None) return cls(timeout=timeout, version=version, **kwargs_from_env(**kwargs)) From 234296171f2389b89fa3d4c359b6f6042419b7c7 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Wed, 14 Jun 2017 13:46:21 +0000 Subject: [PATCH 0371/1301] Only pull the 'latest' tag when testing images Signed-off-by: Bryan Boreham --- tests/integration/api_image_test.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/integration/api_image_test.py b/tests/integration/api_image_test.py index 11146a8a00..917bc50555 100644 --- a/tests/integration/api_image_test.py +++ b/tests/integration/api_image_test.py @@ -42,7 +42,7 @@ def test_pull(self): self.client.remove_image('hello-world') except docker.errors.APIError: pass - res = self.client.pull('hello-world') + res = self.client.pull('hello-world', tag='latest') self.tmp_imgs.append('hello-world') self.assertEqual(type(res), six.text_type) self.assertGreaterEqual( @@ -56,7 +56,8 @@ def test_pull_streaming(self): self.client.remove_image('hello-world') except docker.errors.APIError: pass - stream = self.client.pull('hello-world', stream=True, decode=True) + stream = self.client.pull( + 'hello-world', tag='latest', stream=True, decode=True) self.tmp_imgs.append('hello-world') for chunk in stream: assert isinstance(chunk, dict) @@ -300,7 +301,7 @@ def test_prune_images(self): ctnr = self.client.create_container(BUSYBOX, ['sleep', '9999']) self.tmp_containers.append(ctnr) - self.client.pull('hello-world') + self.client.pull('hello-world', tag='latest') self.tmp_imgs.append('hello-world') img_id = self.client.inspect_image('hello-world')['Id'] result = self.client.prune_images() From ae1f596d3703c35e7f33abe3c14155382d0bad42 Mon Sep 17 00:00:00 2001 From: Matt Oberle Date: Wed, 14 Jun 2017 14:41:04 -0400 Subject: [PATCH 0372/1301] excludes requests 2.18.0 from compatible versions The 2.18.0 version of requests breaks compatibility with docker-py: https://github.com/requests/requests/issues/4160 [This block](https://github.com/shazow/urllib3/blob/master/urllib3/connectionpool.py#L292) of code from urllib3 fails: ```python def _get_timeout(self, timeout): """ Helper that always returns a :class:`urllib3.util.Timeout` """ if timeout is _Default: return self.timeout.clone() if isinstance(timeout, Timeout): return timeout.clone() else: # User passed us an int/float. This is for backwards compatibility, # can be removed later return Timeout.from_float(timeout) ``` In the case of requests version 2.18.0: `timeout` was an instance of `urllib3.util.timeout.Timeout` `Timeout` was an instance of `requests.packages.urllib3.util.timeout.Timeout` When the `isinstance(timeout, Timeout)` check fails the `urllib3.util.timeout.Timeout` object is passed as the `connection` argument to `requests.packages.urllib3.util.timeout.Timeout.from_float`. Signed-off-by: Matt Oberle --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 9fc4ad66e9..31180d2397 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ SOURCE_DIR = os.path.join(ROOT_DIR) requirements = [ - 'requests >= 2.5.2, != 2.11.0, != 2.12.2', + 'requests >= 2.5.2, != 2.11.0, != 2.12.2, != 2.18.0', 'six >= 1.4.0', 'websocket-client >= 0.32.0', 'docker-pycreds >= 0.2.1' From b0c30c8ac4ddf86ee77dcc69214cc93de4b03543 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 14 Jun 2017 12:20:47 -0700 Subject: [PATCH 0373/1301] DockerClient.secrets should be a @property Signed-off-by: Joffrey F --- docker/client.py | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/client.py b/docker/client.py index 09abd63322..66ef60f3f1 100644 --- a/docker/client.py +++ b/docker/client.py @@ -119,6 +119,7 @@ def plugins(self): """ return PluginCollection(client=self) + @property def secrets(self): """ An object for managing secrets on the server. See the From d33e9ad030effb126dea39181ef401c80c442f15 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 15 Jun 2017 18:34:00 -0700 Subject: [PATCH 0374/1301] Update check_resource decorator to account for new resource names Signed-off-by: Joffrey F --- docker/api/client.py | 2 +- docker/api/container.py | 48 +++++++++++++++++++------------------- docker/api/exec_api.py | 4 ++-- docker/api/image.py | 12 +++++----- docker/api/network.py | 6 +++-- docker/api/plugin.py | 8 +++---- docker/api/secret.py | 4 ++-- docker/api/service.py | 10 ++++---- docker/api/swarm.py | 4 ++-- docker/types/services.py | 2 +- docker/utils/decorators.py | 31 ++++++++++++------------ 11 files changed, 66 insertions(+), 65 deletions(-) diff --git a/docker/api/client.py b/docker/api/client.py index 54ec6abb45..6e567b161d 100644 --- a/docker/api/client.py +++ b/docker/api/client.py @@ -248,7 +248,7 @@ def _attach_params(self, override=None): 'stream': 1 } - @check_resource + @check_resource('container') def _attach_websocket(self, container, params=None): url = self._url("/containers/{0}/attach/ws", container) req = requests.Request("POST", url, params=self._attach_params(params)) diff --git a/docker/api/container.py b/docker/api/container.py index 97b5405935..f7ff971b71 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -10,7 +10,7 @@ class ContainerApiMixin(object): - @utils.check_resource + @utils.check_resource('container') def attach(self, container, stdout=True, stderr=True, stream=False, logs=False): """ @@ -54,7 +54,7 @@ def attach(self, container, stdout=True, stderr=True, return self._read_from_socket(response, stream) - @utils.check_resource + @utils.check_resource('container') def attach_socket(self, container, params=None, ws=False): """ Like ``attach``, but returns the underlying socket-like object for the @@ -93,7 +93,7 @@ def attach_socket(self, container, params=None, ws=False): ) ) - @utils.check_resource + @utils.check_resource('container') def commit(self, container, repository=None, tag=None, message=None, author=None, changes=None, conf=None): """ @@ -195,7 +195,7 @@ def containers(self, quiet=False, all=False, trunc=False, latest=False, x['Id'] = x['Id'][:12] return res - @utils.check_resource + @utils.check_resource('container') def copy(self, container, resource): """ Identical to the ``docker cp`` command. Get files/folders from the @@ -659,7 +659,7 @@ def create_endpoint_config(self, *args, **kwargs): """ return EndpointConfig(self._version, *args, **kwargs) - @utils.check_resource + @utils.check_resource('container') def diff(self, container): """ Inspect changes on a container's filesystem. @@ -678,7 +678,7 @@ def diff(self, container): self._get(self._url("/containers/{0}/changes", container)), True ) - @utils.check_resource + @utils.check_resource('container') def export(self, container): """ Export the contents of a filesystem as a tar archive. @@ -699,7 +699,7 @@ def export(self, container): self._raise_for_status(res) return res.raw - @utils.check_resource + @utils.check_resource('container') @utils.minimum_version('1.20') def get_archive(self, container, path): """ @@ -730,7 +730,7 @@ def get_archive(self, container, path): utils.decode_json_header(encoded_stat) if encoded_stat else None ) - @utils.check_resource + @utils.check_resource('container') def inspect_container(self, container): """ Identical to the `docker inspect` command, but only for containers. @@ -750,7 +750,7 @@ def inspect_container(self, container): self._get(self._url("/containers/{0}/json", container)), True ) - @utils.check_resource + @utils.check_resource('container') def kill(self, container, signal=None): """ Kill a container or send a signal to a container. @@ -773,7 +773,7 @@ def kill(self, container, signal=None): self._raise_for_status(res) - @utils.check_resource + @utils.check_resource('container') def logs(self, container, stdout=True, stderr=True, stream=False, timestamps=False, tail='all', since=None, follow=None): """ @@ -836,7 +836,7 @@ def logs(self, container, stdout=True, stderr=True, stream=False, logs=True ) - @utils.check_resource + @utils.check_resource('container') def pause(self, container): """ Pauses all processes within a container. @@ -852,7 +852,7 @@ def pause(self, container): res = self._post(url) self._raise_for_status(res) - @utils.check_resource + @utils.check_resource('container') def port(self, container, private_port): """ Lookup the public-facing port that is NAT-ed to ``private_port``. @@ -901,7 +901,7 @@ def port(self, container, private_port): return h_ports - @utils.check_resource + @utils.check_resource('container') @utils.minimum_version('1.20') def put_archive(self, container, path, data): """ @@ -949,7 +949,7 @@ def prune_containers(self, filters=None): url = self._url('/containers/prune') return self._result(self._post(url, params=params), True) - @utils.check_resource + @utils.check_resource('container') def remove_container(self, container, v=False, link=False, force=False): """ Remove a container. Similar to the ``docker rm`` command. @@ -973,7 +973,7 @@ def remove_container(self, container, v=False, link=False, force=False): self._raise_for_status(res) @utils.minimum_version('1.17') - @utils.check_resource + @utils.check_resource('container') def rename(self, container, name): """ Rename a container. Similar to the ``docker rename`` command. @@ -991,7 +991,7 @@ def rename(self, container, name): res = self._post(url, params=params) self._raise_for_status(res) - @utils.check_resource + @utils.check_resource('container') def resize(self, container, height, width): """ Resize the tty session. @@ -1010,7 +1010,7 @@ def resize(self, container, height, width): res = self._post(url, params=params) self._raise_for_status(res) - @utils.check_resource + @utils.check_resource('container') def restart(self, container, timeout=10): """ Restart a container. Similar to the ``docker restart`` command. @@ -1031,7 +1031,7 @@ def restart(self, container, timeout=10): res = self._post(url, params=params) self._raise_for_status(res) - @utils.check_resource + @utils.check_resource('container') def start(self, container, *args, **kwargs): """ Start a container. Similar to the ``docker start`` command, but @@ -1070,7 +1070,7 @@ def start(self, container, *args, **kwargs): self._raise_for_status(res) @utils.minimum_version('1.17') - @utils.check_resource + @utils.check_resource('container') def stats(self, container, decode=None, stream=True): """ Stream statistics for a specific container. Similar to the @@ -1096,7 +1096,7 @@ def stats(self, container, decode=None, stream=True): return self._result(self._get(url, params={'stream': False}), json=True) - @utils.check_resource + @utils.check_resource('container') def stop(self, container, timeout=10): """ Stops a container. Similar to the ``docker stop`` command. @@ -1117,7 +1117,7 @@ def stop(self, container, timeout=10): timeout=(timeout + (self.timeout or 0))) self._raise_for_status(res) - @utils.check_resource + @utils.check_resource('container') def top(self, container, ps_args=None): """ Display the running processes of a container. @@ -1139,7 +1139,7 @@ def top(self, container, ps_args=None): params['ps_args'] = ps_args return self._result(self._get(u, params=params), True) - @utils.check_resource + @utils.check_resource('container') def unpause(self, container): """ Unpause all processes within a container. @@ -1152,7 +1152,7 @@ def unpause(self, container): self._raise_for_status(res) @utils.minimum_version('1.22') - @utils.check_resource + @utils.check_resource('container') def update_container( self, container, blkio_weight=None, cpu_period=None, cpu_quota=None, cpu_shares=None, cpuset_cpus=None, cpuset_mems=None, mem_limit=None, @@ -1217,7 +1217,7 @@ def update_container( res = self._post_json(url, data=data) return self._result(res, True) - @utils.check_resource + @utils.check_resource('container') def wait(self, container, timeout=None): """ Block until a container stops, then return its exit code. Similar to diff --git a/docker/api/exec_api.py b/docker/api/exec_api.py index 3ff65256ef..2b407cef40 100644 --- a/docker/api/exec_api.py +++ b/docker/api/exec_api.py @@ -6,7 +6,7 @@ class ExecApiMixin(object): @utils.minimum_version('1.15') - @utils.check_resource + @utils.check_resource('container') def exec_create(self, container, cmd, stdout=True, stderr=True, stdin=False, tty=False, privileged=False, user='', environment=None): @@ -110,7 +110,7 @@ def exec_resize(self, exec_id, height=None, width=None): self._raise_for_status(res) @utils.minimum_version('1.15') - @utils.check_resource + @utils.check_resource('exec_id') def exec_start(self, exec_id, detach=False, tty=False, stream=False, socket=False): """ diff --git a/docker/api/image.py b/docker/api/image.py index 09eb086d78..181c4a1e4a 100644 --- a/docker/api/image.py +++ b/docker/api/image.py @@ -12,7 +12,7 @@ class ImageApiMixin(object): - @utils.check_resource + @utils.check_resource('image') def get_image(self, image): """ Get a tarball of an image. Similar to the ``docker save`` command. @@ -39,7 +39,7 @@ def get_image(self, image): self._raise_for_status(res) return res.raw - @utils.check_resource + @utils.check_resource('image') def history(self, image): """ Show the history of an image. @@ -228,7 +228,7 @@ def import_image_from_image(self, image, repository=None, tag=None, image=image, repository=repository, tag=tag, changes=changes ) - @utils.check_resource + @utils.check_resource('image') def insert(self, image, url, path): if utils.compare_version('1.12', self._version) >= 0: raise errors.DeprecatedMethod( @@ -241,7 +241,7 @@ def insert(self, image, url, path): } return self._result(self._post(api_url, params=params)) - @utils.check_resource + @utils.check_resource('image') def inspect_image(self, image): """ Get detailed information about an image. Similar to the ``docker @@ -443,7 +443,7 @@ def push(self, repository, tag=None, stream=False, return self._result(response) - @utils.check_resource + @utils.check_resource('image') def remove_image(self, image, force=False, noprune=False): """ Remove an image. Similar to the ``docker rmi`` command. @@ -477,7 +477,7 @@ def search(self, term): True ) - @utils.check_resource + @utils.check_resource('image') def tag(self, image, repository, tag=None, force=False): """ Tag an image into a repository. Similar to the ``docker tag`` command. diff --git a/docker/api/network.py b/docker/api/network.py index 74f4cd2b30..bd2959fdc5 100644 --- a/docker/api/network.py +++ b/docker/api/network.py @@ -156,6 +156,7 @@ def prune_networks(self, filters=None): return self._result(self._post(url, params=params), True) @minimum_version('1.21') + @check_resource('net_id') def remove_network(self, net_id): """ Remove a network. Similar to the ``docker network rm`` command. @@ -168,6 +169,7 @@ def remove_network(self, net_id): self._raise_for_status(res) @minimum_version('1.21') + @check_resource('net_id') def inspect_network(self, net_id, verbose=None): """ Get detailed information about a network. @@ -187,7 +189,7 @@ def inspect_network(self, net_id, verbose=None): res = self._get(url, params=params) return self._result(res, json=True) - @check_resource + @check_resource('image') @minimum_version('1.21') def connect_container_to_network(self, container, net_id, ipv4_address=None, ipv6_address=None, @@ -224,7 +226,7 @@ def connect_container_to_network(self, container, net_id, res = self._post_json(url, data=data) self._raise_for_status(res) - @check_resource + @check_resource('image') @minimum_version('1.21') def disconnect_container_from_network(self, container, net_id, force=False): diff --git a/docker/api/plugin.py b/docker/api/plugin.py index ba40c88297..87520ccee3 100644 --- a/docker/api/plugin.py +++ b/docker/api/plugin.py @@ -5,7 +5,7 @@ class PluginApiMixin(object): @utils.minimum_version('1.25') - @utils.check_resource + @utils.check_resource('name') def configure_plugin(self, name, options): """ Configure a plugin. @@ -171,7 +171,7 @@ def plugin_privileges(self, name): return self._result(self._get(url, params=params), True) @utils.minimum_version('1.25') - @utils.check_resource + @utils.check_resource('name') def push_plugin(self, name): """ Push a plugin to the registry. @@ -195,7 +195,7 @@ def push_plugin(self, name): return self._stream_helper(res, decode=True) @utils.minimum_version('1.25') - @utils.check_resource + @utils.check_resource('name') def remove_plugin(self, name, force=False): """ Remove an installed plugin. @@ -215,7 +215,7 @@ def remove_plugin(self, name, force=False): return True @utils.minimum_version('1.26') - @utils.check_resource + @utils.check_resource('name') def upgrade_plugin(self, name, remote, privileges): """ Upgrade an installed plugin. diff --git a/docker/api/secret.py b/docker/api/secret.py index 03534a6236..1760a39469 100644 --- a/docker/api/secret.py +++ b/docker/api/secret.py @@ -36,7 +36,7 @@ def create_secret(self, name, data, labels=None): ) @utils.minimum_version('1.25') - @utils.check_resource + @utils.check_resource('id') def inspect_secret(self, id): """ Retrieve secret metadata @@ -54,7 +54,7 @@ def inspect_secret(self, id): return self._result(self._get(url), True) @utils.minimum_version('1.25') - @utils.check_resource + @utils.check_resource('id') def remove_secret(self, id): """ Remove a secret diff --git a/docker/api/service.py b/docker/api/service.py index aea93cbfc9..0f14776dbb 100644 --- a/docker/api/service.py +++ b/docker/api/service.py @@ -113,7 +113,7 @@ def create_service( ) @utils.minimum_version('1.24') - @utils.check_resource + @utils.check_resource('service') def inspect_service(self, service): """ Return information about a service. @@ -132,7 +132,7 @@ def inspect_service(self, service): return self._result(self._get(url), True) @utils.minimum_version('1.24') - @utils.check_resource + @utils.check_resource('task') def inspect_task(self, task): """ Retrieve information about a task. @@ -151,7 +151,7 @@ def inspect_task(self, task): return self._result(self._get(url), True) @utils.minimum_version('1.24') - @utils.check_resource + @utils.check_resource('service') def remove_service(self, service): """ Stop and remove a service. @@ -195,7 +195,7 @@ def services(self, filters=None): return self._result(self._get(url, params=params), True) @utils.minimum_version('1.25') - @utils.check_resource + @utils.check_resource('service') def service_logs(self, service, details=False, follow=False, stdout=False, stderr=False, since=0, timestamps=False, tail='all', is_tty=None): @@ -269,7 +269,7 @@ def tasks(self, filters=None): return self._result(self._get(url, params=params), True) @utils.minimum_version('1.24') - @utils.check_resource + @utils.check_resource('service') def update_service(self, service, version, task_template=None, name=None, labels=None, mode=None, update_config=None, networks=None, endpoint_config=None, diff --git a/docker/api/swarm.py b/docker/api/swarm.py index 88770562f2..4fa0c4a120 100644 --- a/docker/api/swarm.py +++ b/docker/api/swarm.py @@ -117,7 +117,7 @@ def inspect_swarm(self): url = self._url('/swarm') return self._result(self._get(url), True) - @utils.check_resource + @utils.check_resource('node_id') @utils.minimum_version('1.24') def inspect_node(self, node_id): """ @@ -228,7 +228,7 @@ def nodes(self, filters=None): return self._result(self._get(url, params=params), True) - @utils.check_resource + @utils.check_resource('node_id') @utils.minimum_version('1.24') def remove_node(self, node_id, force=False): """ diff --git a/docker/types/services.py b/docker/types/services.py index 7456a42ba5..9cec34ef8d 100644 --- a/docker/types/services.py +++ b/docker/types/services.py @@ -443,7 +443,7 @@ class SecretReference(dict): gid (string): GID of the secret file's group. Default: 0 mode (int): File access mode inside the container. Default: 0o444 """ - @check_resource + @check_resource('secret_id') def __init__(self, secret_id, secret_name, filename=None, uid=None, gid=None, mode=0o444): self['SecretName'] = secret_name diff --git a/docker/utils/decorators.py b/docker/utils/decorators.py index 18cde412ff..5e195c0ea6 100644 --- a/docker/utils/decorators.py +++ b/docker/utils/decorators.py @@ -4,22 +4,21 @@ from . import utils -def check_resource(f): - @functools.wraps(f) - def wrapped(self, resource_id=None, *args, **kwargs): - if resource_id is None: - if kwargs.get('container'): - resource_id = kwargs.pop('container') - elif kwargs.get('image'): - resource_id = kwargs.pop('image') - if isinstance(resource_id, dict): - resource_id = resource_id.get('Id', resource_id.get('ID')) - if not resource_id: - raise errors.NullResource( - 'Resource ID was not provided' - ) - return f(self, resource_id, *args, **kwargs) - return wrapped +def check_resource(resource_name): + def decorator(f): + @functools.wraps(f) + def wrapped(self, resource_id=None, *args, **kwargs): + if resource_id is None and kwargs.get(resource_name): + resource_id = kwargs.pop(resource_name) + if isinstance(resource_id, dict): + resource_id = resource_id.get('Id', resource_id.get('ID')) + if not resource_id: + raise errors.NullResource( + 'Resource ID was not provided' + ) + return f(self, resource_id, *args, **kwargs) + return wrapped + return decorator def minimum_version(version): From c03a009e2dc2548de2ef280752109a9dc0660acb Mon Sep 17 00:00:00 2001 From: Chris Mark Date: Fri, 16 Jun 2017 18:30:24 +0300 Subject: [PATCH 0375/1301] Raising error in case of invalid value of since kwarg on Container.logs Signed-off-by: Chris Mark --- docker/api/container.py | 5 +++++ tests/unit/api_container_test.py | 7 +++++++ 2 files changed, 12 insertions(+) diff --git a/docker/api/container.py b/docker/api/container.py index 97b5405935..7aeea2028a 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -825,6 +825,11 @@ def logs(self, container, stdout=True, stderr=True, stream=False, params['since'] = utils.datetime_to_timestamp(since) elif (isinstance(since, int) and since > 0): params['since'] = since + else: + raise errors.InvalidArgument( + 'since value should be datetime or int, not {}'. + format(type(since)) + ) url = self._url("/containers/{0}/logs", container) res = self._get(url, params=params, stream=stream) return self._get_result(container, stream, res) diff --git a/tests/unit/api_container_test.py b/tests/unit/api_container_test.py index 662d3f5908..3b135a8135 100644 --- a/tests/unit/api_container_test.py +++ b/tests/unit/api_container_test.py @@ -1421,6 +1421,13 @@ def test_log_since_with_datetime(self): stream=False ) + def test_log_since_with_invalid_value_raises_error(self): + with mock.patch('docker.api.client.APIClient.inspect_container', + fake_inspect_container): + with self.assertRaises(docker.errors.InvalidArgument): + self.client.logs(fake_api.FAKE_CONTAINER_ID, stream=False, + follow=False, since=42.42) + def test_log_tty(self): m = mock.Mock() with mock.patch('docker.api.client.APIClient.inspect_container', From d638829f736aa7a942a780ff7796facb2713a959 Mon Sep 17 00:00:00 2001 From: Olivier Sallou Date: Fri, 16 Jun 2017 17:49:43 +0200 Subject: [PATCH 0376/1301] Closes #1588, image.tag does not return anything This patch returns the check made against api when tagging an image as stated in documentation Signed-off-by: Olivier Sallou --- docker/models/images.py | 2 +- tests/integration/models_images_test.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docker/models/images.py b/docker/models/images.py index a9ed65ee35..e8af101d79 100644 --- a/docker/models/images.py +++ b/docker/models/images.py @@ -96,7 +96,7 @@ def tag(self, repository, tag=None, **kwargs): Returns: (bool): ``True`` if successful """ - self.client.api.tag(self.id, repository, tag=tag, **kwargs) + return self.client.api.tag(self.id, repository, tag=tag, **kwargs) class ImageCollection(Collection): diff --git a/tests/integration/models_images_test.py b/tests/integration/models_images_test.py index 6d61e4977c..881df0a1e5 100644 --- a/tests/integration/models_images_test.py +++ b/tests/integration/models_images_test.py @@ -71,7 +71,8 @@ def test_tag_and_remove(self): client = docker.from_env(version=TEST_API_VERSION) image = client.images.pull('alpine:latest') - image.tag(repo, tag) + result = image.tag(repo, tag) + assert result is True self.tmp_imgs.append(identifier) assert image.id in get_ids(client.images.list(repo)) assert image.id in get_ids(client.images.list(identifier)) From 1ea6618b09e3a74531a0fed1d44a5d698afe2339 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 17 May 2017 18:12:26 -0700 Subject: [PATCH 0377/1301] Add support for start_period in Healthcheck spec Signed-off-by: Joffrey F --- docker/api/container.py | 2 ++ docker/models/containers.py | 2 ++ docker/types/containers.py | 15 ++++++++++---- docker/types/healthcheck.py | 12 ++++++++++- tests/integration/api_healthcheck_test.py | 25 +++++++++++++++++++---- 5 files changed, 47 insertions(+), 9 deletions(-) diff --git a/docker/api/container.py b/docker/api/container.py index 97a39b65d7..5668b43cea 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -418,6 +418,8 @@ def create_container(self, image, command=None, hostname=None, user=None, networking_config (dict): A networking configuration generated by :py:meth:`create_networking_config`. runtime (str): Runtime to use with this container. + healthcheck (dict): Specify a test to perform to check that the + container is healthy. Returns: A dictionary with an image 'Id' key and a 'Warnings' key. diff --git a/docker/models/containers.py b/docker/models/containers.py index 300c5a9d3b..cf01b2750a 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -516,6 +516,8 @@ def run(self, image, command=None, stdout=True, stderr=False, container, as a mapping of hostname to IP address. group_add (:py:class:`list`): List of additional group names and/or IDs that the container process will run as. + healthcheck (dict): Specify a test to perform to check that the + container is healthy. hostname (str): Optional hostname for the container. init (bool): Run an init inside the container that forwards signals and reaps processes diff --git a/docker/types/containers.py b/docker/types/containers.py index 6bbb57ae79..030e292bc6 100644 --- a/docker/types/containers.py +++ b/docker/types/containers.py @@ -565,10 +565,17 @@ def __init__( 'stop_timeout was only introduced in API version 1.25' ) - if healthcheck is not None and version_lt(version, '1.24'): - raise errors.InvalidVersion( - 'Health options were only introduced in API version 1.24' - ) + if healthcheck is not None: + if version_lt(version, '1.24'): + raise errors.InvalidVersion( + 'Health options were only introduced in API version 1.24' + ) + + if version_lt(version, '1.29') and 'StartPeriod' in healthcheck: + raise errors.InvalidVersion( + 'healthcheck start period was introduced in API ' + 'version 1.29' + ) if isinstance(command, six.string_types): command = split_command(command) diff --git a/docker/types/healthcheck.py b/docker/types/healthcheck.py index ba63d21ed6..8ea9a35f5b 100644 --- a/docker/types/healthcheck.py +++ b/docker/types/healthcheck.py @@ -12,12 +12,14 @@ def __init__(self, **kwargs): interval = kwargs.get('interval', kwargs.get('Interval')) timeout = kwargs.get('timeout', kwargs.get('Timeout')) retries = kwargs.get('retries', kwargs.get('Retries')) + start_period = kwargs.get('start_period', kwargs.get('StartPeriod')) super(Healthcheck, self).__init__({ 'Test': test, 'Interval': interval, 'Timeout': timeout, - 'Retries': retries + 'Retries': retries, + 'StartPeriod': start_period }) @property @@ -51,3 +53,11 @@ def retries(self): @retries.setter def retries(self, value): self['Retries'] = value + + @property + def start_period(self): + return self['StartPeriod'] + + @start_period.setter + def start_period(self, value): + self['StartPeriod'] = value diff --git a/tests/integration/api_healthcheck_test.py b/tests/integration/api_healthcheck_test.py index afe1dea21c..211042d486 100644 --- a/tests/integration/api_healthcheck_test.py +++ b/tests/integration/api_healthcheck_test.py @@ -28,8 +28,8 @@ def test_healthcheck_passes(self): container = self.client.create_container( BUSYBOX, 'top', healthcheck=dict( test="true", - interval=1*SECOND, - timeout=1*SECOND, + interval=1 * SECOND, + timeout=1 * SECOND, retries=1, )) self.tmp_containers.append(container) @@ -41,10 +41,27 @@ def test_healthcheck_fails(self): container = self.client.create_container( BUSYBOX, 'top', healthcheck=dict( test="false", - interval=1*SECOND, - timeout=1*SECOND, + interval=1 * SECOND, + timeout=1 * SECOND, retries=1, )) self.tmp_containers.append(container) self.client.start(container) wait_on_health_status(self.client, container, "unhealthy") + + @helpers.requires_api_version('1.29') + def test_healthcheck_start_period(self): + container = self.client.create_container( + BUSYBOX, 'top', healthcheck=dict( + test="echo 'x' >> /counter.txt && " + "test `cat /counter.txt | wc -l` -ge 3", + interval=1 * SECOND, + timeout=1 * SECOND, + retries=1, + start_period=3 * SECOND + ) + ) + + self.tmp_containers.append(container) + self.client.start(container) + wait_on_health_status(self.client, container, "healthy") From 39bb78ac694d8d6e53882d3dbc9ebc3c92f5519d Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 19 Jun 2017 15:50:28 -0700 Subject: [PATCH 0378/1301] Add network_mode support to Client.build Signed-off-by: Joffrey F --- docker/api/build.py | 22 +++++++++++++------ docker/models/images.py | 11 ++++++---- tests/integration/api_build_test.py | 33 +++++++++++++++++++++++++++++ 3 files changed, 56 insertions(+), 10 deletions(-) diff --git a/docker/api/build.py b/docker/api/build.py index f30be4168a..cbef4a8b17 100644 --- a/docker/api/build.py +++ b/docker/api/build.py @@ -18,7 +18,7 @@ def build(self, path=None, tag=None, quiet=False, fileobj=None, custom_context=False, encoding=None, pull=False, forcerm=False, dockerfile=None, container_limits=None, decode=False, buildargs=None, gzip=False, shmsize=None, - labels=None, cache_from=None, target=None): + labels=None, cache_from=None, target=None, network_mode=None): """ Similar to the ``docker build`` command. Either ``path`` or ``fileobj`` needs to be set. ``path`` can be a local path (to a directory @@ -88,14 +88,16 @@ def build(self, path=None, tag=None, quiet=False, fileobj=None, - cpusetcpus (str): CPUs in which to allow execution, e.g., ``"0-3"``, ``"0,1"`` decode (bool): If set to ``True``, the returned stream will be - decoded into dicts on the fly. Default ``False``. + decoded into dicts on the fly. Default ``False`` shmsize (int): Size of `/dev/shm` in bytes. The size must be - greater than 0. If omitted the system uses 64MB. - labels (dict): A dictionary of labels to set on the image. + greater than 0. If omitted the system uses 64MB + labels (dict): A dictionary of labels to set on the image cache_from (list): A list of images used for build cache - resolution. + resolution target (str): Name of the build-stage to build in a multi-stage - Dockerfile. + Dockerfile + network_mode (str): networking mode for the run commands during + build Returns: A generator for the build output. @@ -208,6 +210,14 @@ def build(self, path=None, tag=None, quiet=False, fileobj=None, 'target was only introduced in API version 1.29' ) + if network_mode: + if utils.version_gte(self._version, '1.25'): + params.update({'networkmode': network_mode}) + else: + raise errors.InvalidVersion( + 'network_mode was only introduced in API version 1.25' + ) + if context is not None: headers = {'Content-Type': 'application/tar'} if encoding: diff --git a/docker/models/images.py b/docker/models/images.py index 7e999b0c48..d4e24c6060 100644 --- a/docker/models/images.py +++ b/docker/models/images.py @@ -144,12 +144,15 @@ def build(self, **kwargs): - cpushares (int): CPU shares (relative weight) - cpusetcpus (str): CPUs in which to allow execution, e.g., ``"0-3"``, ``"0,1"`` - decode (bool): If set to ``True``, the returned stream will be - decoded into dicts on the fly. Default ``False``. + shmsize (int): Size of `/dev/shm` in bytes. The size must be + greater than 0. If omitted the system uses 64MB + labels (dict): A dictionary of labels to set on the image cache_from (list): A list of images used for build cache - resolution. + resolution target (str): Name of the build-stage to build in a multi-stage - Dockerfile. + Dockerfile + network_mode (str): networking mode for the run commands during + build Returns: (:py:class:`Image`): The built image. diff --git a/tests/integration/api_build_test.py b/tests/integration/api_build_test.py index 623b660931..609964f0b0 100644 --- a/tests/integration/api_build_test.py +++ b/tests/integration/api_build_test.py @@ -5,6 +5,7 @@ from docker import errors +import pytest import six from .base import BaseAPIIntegrationTest @@ -211,6 +212,38 @@ def test_build_container_with_target(self): info = self.client.inspect_image('build1') self.assertEqual(info['Config']['OnBuild'], []) + @requires_api_version('1.25') + def test_build_with_network_mode(self): + script = io.BytesIO('\n'.join([ + 'FROM busybox', + 'RUN wget http://google.com' + ]).encode('ascii')) + + stream = self.client.build( + fileobj=script, network_mode='bridge', + tag='dockerpytest_bridgebuild' + ) + + self.tmp_imgs.append('dockerpytest_bridgebuild') + for chunk in stream: + pass + + assert self.client.inspect_image('dockerpytest_bridgebuild') + + script.seek(0) + stream = self.client.build( + fileobj=script, network_mode='none', + tag='dockerpytest_nonebuild', nocache=True, decode=True + ) + + self.tmp_imgs.append('dockerpytest_nonebuild') + logs = [chunk for chunk in stream] + assert 'errorDetail' in logs[-1] + assert logs[-1]['errorDetail']['code'] == 1 + + with pytest.raises(errors.NotFound): + self.client.inspect_image('dockerpytest_nonebuild') + def test_build_stderr_data(self): control_chars = ['\x1b[91m', '\x1b[0m'] snippet = 'Ancient Temple (Mystic Oriental Dream ~ Ancient Temple)' From 0165a343d51c8051f1793f62202e2f053ab1b594 Mon Sep 17 00:00:00 2001 From: An Ha Date: Mon, 12 Jun 2017 16:33:56 -0400 Subject: [PATCH 0379/1301] Add attributes for pickling When using the multiprocessing module, it throws an AttributeError, complaining that the object does not have the attribute used. This adds the missing attributes and allows them to be pickled. Signed-off-by: An Ha --- docker/api/client.py | 6 ++++++ docker/transport/npipeconn.py | 5 +++++ docker/transport/ssladapter.py | 4 ++++ docker/transport/unixconn.py | 5 +++++ 4 files changed, 20 insertions(+) diff --git a/docker/api/client.py b/docker/api/client.py index 6e567b161d..65b5baa967 100644 --- a/docker/api/client.py +++ b/docker/api/client.py @@ -83,6 +83,12 @@ class APIClient( configuration. user_agent (str): Set a custom user agent for requests to the server. """ + + __attrs__ = requests.Session.__attrs__ + ['_auth_configs', + '_version', + 'base_url', + 'timeout'] + def __init__(self, base_url=None, version=None, timeout=DEFAULT_TIMEOUT_SECONDS, tls=False, user_agent=DEFAULT_USER_AGENT, num_pools=DEFAULT_NUM_POOLS): diff --git a/docker/transport/npipeconn.py b/docker/transport/npipeconn.py index db059b445a..ab9b90480a 100644 --- a/docker/transport/npipeconn.py +++ b/docker/transport/npipeconn.py @@ -69,6 +69,11 @@ def _get_conn(self, timeout): class NpipeAdapter(requests.adapters.HTTPAdapter): + + __attrs__ = requests.adapters.HTTPAdapter.__attrs__ + ['npipe_path', + 'pools', + 'timeout'] + def __init__(self, base_url, timeout=60, pool_connections=constants.DEFAULT_NUM_POOLS): self.npipe_path = base_url.replace('npipe://', '') diff --git a/docker/transport/ssladapter.py b/docker/transport/ssladapter.py index 31f45fc459..8fafec3550 100644 --- a/docker/transport/ssladapter.py +++ b/docker/transport/ssladapter.py @@ -25,6 +25,10 @@ class SSLAdapter(HTTPAdapter): '''An HTTPS Transport Adapter that uses an arbitrary SSL version.''' + __attrs__ = HTTPAdapter.__attrs__ + ['assert_fingerprint', + 'assert_hostname', + 'ssl_version'] + def __init__(self, ssl_version=None, assert_hostname=None, assert_fingerprint=None, **kwargs): self.ssl_version = ssl_version diff --git a/docker/transport/unixconn.py b/docker/transport/unixconn.py index 978c87a1bf..3565cfb629 100644 --- a/docker/transport/unixconn.py +++ b/docker/transport/unixconn.py @@ -50,6 +50,11 @@ def _new_conn(self): class UnixAdapter(requests.adapters.HTTPAdapter): + + __attrs__ = requests.adapters.HTTPAdapter.__attrs__ + ['pools', + 'socket_path', + 'timeout'] + def __init__(self, socket_url, timeout=60, pool_connections=constants.DEFAULT_NUM_POOLS): socket_path = socket_url.replace('http+unix://', '') From 9b9fb0aa0140cae1fc2c6eb549f2615501453eb9 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 20 Jun 2017 16:07:15 -0700 Subject: [PATCH 0380/1301] Make sure data is written in prune test so space can be reclaimed Signed-off-by: Joffrey F --- tests/integration/api_container_test.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index de3fe7183c..f8b474a113 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -1139,7 +1139,9 @@ def test_pause_unpause(self): class PruneTest(BaseAPIIntegrationTest): @requires_api_version('1.25') def test_prune_containers(self): - container1 = self.client.create_container(BUSYBOX, ['echo', 'hello']) + container1 = self.client.create_container( + BUSYBOX, ['sh', '-c', 'echo hello > /data.txt'] + ) container2 = self.client.create_container(BUSYBOX, ['sleep', '9999']) self.client.start(container1) self.client.start(container2) From 06d2553b9c3a4c91d1ef2110ea120da15605de2f Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 21 Jun 2017 16:29:25 -0700 Subject: [PATCH 0381/1301] Add support for ContainerSpec.TTY Signed-off-by: Joffrey F --- docker/api/service.py | 5 +++++ docker/models/services.py | 2 ++ docker/types/services.py | 6 +++++- tests/integration/api_service_test.py | 17 +++++++++++++++++ 4 files changed, 29 insertions(+), 1 deletion(-) diff --git a/docker/api/service.py b/docker/api/service.py index 0f14776dbb..cc16cc37dd 100644 --- a/docker/api/service.py +++ b/docker/api/service.py @@ -38,6 +38,11 @@ def _check_api_features(version, task_template, update_config): 'Placement.preferences is not supported in' ' API version < 1.27' ) + if task_template.container_spec.get('TTY'): + if utils.version_lt(version, '1.25'): + raise errors.InvalidVersion( + 'ContainerSpec.TTY is not supported in API version < 1.25' + ) class ServiceApiMixin(object): diff --git a/docker/models/services.py b/docker/models/services.py index c10804dedf..e1e2ea6a44 100644 --- a/docker/models/services.py +++ b/docker/models/services.py @@ -146,6 +146,7 @@ def create(self, image, command=None, **kwargs): of the service. Default: ``None`` user (str): User to run commands as. workdir (str): Working directory for commands to run. + tty (boolean): Whether a pseudo-TTY should be allocated. Returns: (:py:class:`Service`) The created service. @@ -212,6 +213,7 @@ def list(self, **kwargs): 'mounts', 'stop_grace_period', 'secrets', + 'tty' ] # kwargs to copy straight over to TaskTemplate diff --git a/docker/types/services.py b/docker/types/services.py index 9cec34ef8d..8411b70a40 100644 --- a/docker/types/services.py +++ b/docker/types/services.py @@ -84,10 +84,11 @@ class ContainerSpec(dict): terminate before forcefully killing it. secrets (list of py:class:`SecretReference`): List of secrets to be made available inside the containers. + tty (boolean): Whether a pseudo-TTY should be allocated. """ def __init__(self, image, command=None, args=None, hostname=None, env=None, workdir=None, user=None, labels=None, mounts=None, - stop_grace_period=None, secrets=None): + stop_grace_period=None, secrets=None, tty=None): self['Image'] = image if isinstance(command, six.string_types): @@ -125,6 +126,9 @@ def __init__(self, image, command=None, args=None, hostname=None, env=None, raise TypeError('secrets must be a list') self['Secrets'] = secrets + if tty is not None: + self['TTY'] = tty + class Mount(dict): """ diff --git a/tests/integration/api_service_test.py b/tests/integration/api_service_test.py index 8ac852d960..54111a7bb1 100644 --- a/tests/integration/api_service_test.py +++ b/tests/integration/api_service_test.py @@ -359,6 +359,23 @@ def test_create_service_with_env(self): assert 'Env' in con_spec assert con_spec['Env'] == ['DOCKER_PY_TEST=1'] + @requires_api_version('1.25') + def test_create_service_with_tty(self): + container_spec = docker.types.ContainerSpec( + BUSYBOX, ['true'], tty=True + ) + task_tmpl = docker.types.TaskTemplate( + container_spec, + ) + name = self.get_service_name() + svc_id = self.client.create_service(task_tmpl, name=name) + svc_info = self.client.inspect_service(svc_id) + assert 'TaskTemplate' in svc_info['Spec'] + assert 'ContainerSpec' in svc_info['Spec']['TaskTemplate'] + con_spec = svc_info['Spec']['TaskTemplate']['ContainerSpec'] + assert 'TTY' in con_spec + assert con_spec['TTY'] is True + def test_create_service_global_mode(self): container_spec = docker.types.ContainerSpec( BUSYBOX, ['echo', 'hello'] From 320c81047107a4350bb430f24825d116a91d1d8f Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 22 Jun 2017 11:51:31 -0700 Subject: [PATCH 0382/1301] Support credHelpers section in config.json Signed-off-by: Joffrey F --- docker/auth.py | 30 ++++++++++++++++++------ tests/unit/auth_test.py | 51 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 7 deletions(-) diff --git a/docker/auth.py b/docker/auth.py index 7c1ce7618e..ec9c45b97d 100644 --- a/docker/auth.py +++ b/docker/auth.py @@ -70,6 +70,15 @@ def split_repo_name(repo_name): return tuple(parts) +def get_credential_store(authconfig, registry): + if not registry or registry == INDEX_NAME: + registry = 'https://index.docker.io/v1/' + + return authconfig.get('credHelpers', {}).get(registry) or authconfig.get( + 'credsStore' + ) + + def resolve_authconfig(authconfig, registry=None): """ Returns the authentication data from the given auth configuration for a @@ -77,13 +86,17 @@ def resolve_authconfig(authconfig, registry=None): with full URLs are stripped down to hostnames before checking for a match. Returns None if no match was found. """ - if 'credsStore' in authconfig: - log.debug( - 'Using credentials store "{0}"'.format(authconfig['credsStore']) - ) - return _resolve_authconfig_credstore( - authconfig, registry, authconfig['credsStore'] - ) + + if 'credHelpers' in authconfig or 'credsStore' in authconfig: + store_name = get_credential_store(authconfig, registry) + if store_name is not None: + log.debug( + 'Using credentials store "{0}"'.format(store_name) + ) + return _resolve_authconfig_credstore( + authconfig, registry, store_name + ) + # Default to the public index server registry = resolve_index_name(registry) if registry else INDEX_NAME log.debug("Looking for auth entry for {0}".format(repr(registry))) @@ -274,6 +287,9 @@ def load_config(config_path=None): if data.get('credsStore'): log.debug("Found 'credsStore' section") res.update({'credsStore': data['credsStore']}) + if data.get('credHelpers'): + log.debug("Found 'credHelpers' section") + res.update({'credHelpers': data['credHelpers']}) if res: return res else: diff --git a/tests/unit/auth_test.py b/tests/unit/auth_test.py index f9f6fc1462..56fd50c250 100644 --- a/tests/unit/auth_test.py +++ b/tests/unit/auth_test.py @@ -272,6 +272,57 @@ def test_resolve_registry_and_auth_unauthenticated_registry(self): ) +class CredStoreTest(unittest.TestCase): + def test_get_credential_store(self): + auth_config = { + 'credHelpers': { + 'registry1.io': 'truesecret', + 'registry2.io': 'powerlock' + }, + 'credsStore': 'blackbox', + } + + assert auth.get_credential_store( + auth_config, 'registry1.io' + ) == 'truesecret' + assert auth.get_credential_store( + auth_config, 'registry2.io' + ) == 'powerlock' + assert auth.get_credential_store( + auth_config, 'registry3.io' + ) == 'blackbox' + + def test_get_credential_store_no_default(self): + auth_config = { + 'credHelpers': { + 'registry1.io': 'truesecret', + 'registry2.io': 'powerlock' + }, + } + assert auth.get_credential_store( + auth_config, 'registry2.io' + ) == 'powerlock' + assert auth.get_credential_store( + auth_config, 'registry3.io' + ) is None + + def test_get_credential_store_default_index(self): + auth_config = { + 'credHelpers': { + 'https://index.docker.io/v1/': 'powerlock' + }, + 'credsStore': 'truesecret' + } + + assert auth.get_credential_store(auth_config, None) == 'powerlock' + assert auth.get_credential_store( + auth_config, 'docker.io' + ) == 'powerlock' + assert auth.get_credential_store( + auth_config, 'images.io' + ) == 'truesecret' + + class FindConfigFileTest(unittest.TestCase): def tmpdir(self, name): tmpdir = ensuretemp(name) From 015fe1cf5eacf93f965bd68b6e618adf2d9c115a Mon Sep 17 00:00:00 2001 From: Boik Date: Tue, 20 Dec 2016 10:44:00 +0800 Subject: [PATCH 0383/1301] Correct the description of dns_opt option of create_container Signed-off-by: Boik --- docker/api/container.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/api/container.py b/docker/api/container.py index 91084219a0..532a9c6d8b 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -391,8 +391,6 @@ def create_container(self, image, command=None, hostname=None, user=None, ``{"PASSWORD": "xxx"}``. dns (:py:class:`list`): DNS name servers. Deprecated since API version 1.10. Use ``host_config`` instead. - dns_opt (:py:class:`list`): Additional options to be added to the - container's ``resolv.conf`` file volumes (str or list): List of paths inside the container to use as volumes. volumes_from (:py:class:`list`): List of container names or Ids to @@ -498,6 +496,8 @@ def create_host_config(self, *args, **kwargs): to have read-write access to the host's ``/dev/sda`` via a node named ``/dev/xvda`` inside the container. dns (:py:class:`list`): Set custom DNS servers. + dns_opt (:py:class:`list`): Additional options to be added to the + container's ``resolv.conf`` file dns_search (:py:class:`list`): DNS search domains. extra_hosts (dict): Addtional hostnames to resolve inside the container, as a mapping of hostname to IP address. From 1ad6859600258eca17f91e4f71c1de5788321777 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 22 Jun 2017 17:17:13 -0700 Subject: [PATCH 0384/1301] Bump 2.4.0 Signed-off-by: Joffrey F --- docker/version.py | 2 +- docs/api.rst | 1 + docs/change-log.md | 38 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 40 insertions(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index 6979e1bef8..8f40f467a4 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "2.4.0-dev" +version = "2.4.0" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) diff --git a/docs/api.rst b/docs/api.rst index 52cd26b2ca..0b10f387db 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -128,6 +128,7 @@ Configuration types .. autoclass:: DriverConfig .. autoclass:: EndpointSpec .. autoclass:: Mount +.. autoclass:: Placement .. autoclass:: Resources .. autoclass:: RestartPolicy .. autoclass:: SecretReference diff --git a/docs/change-log.md b/docs/change-log.md index 3d58f931ff..194f734795 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -1,6 +1,44 @@ Change log ========== +2.4.0 +----- + +[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/33?closed=1) + +### Features + +* Added support for the `target` and `network_mode` parameters in + `APIClient.build` and `DockerClient.images.build`. +* Added support for the `runtime` parameter in `APIClient.create_container` + and `DockerClient.containers.run`. +* Added support for the `ingress` parameter in `APIClient.create_network` and + `DockerClient.networks.create`. +* Added support for `placement` configuration in `docker.types.TaskTemplate`. +* Added support for `tty` configuration in `docker.types.ContainerSpec`. +* Added support for `start_period` configuration in `docker.types.Healthcheck`. +* The `credHelpers` section in Docker's configuration file is now recognized. +* Port specifications including IPv6 endpoints are now supported. + +### Bugfixes + +* Fixed a bug where instantiating a `DockerClient` using `docker.from_env` + wouldn't correctly set the default timeout value. +* Fixed a bug where `DockerClient.secrets` was not accessible as a property. +* Fixed a bug where `DockerClient.build` would sometimes return the wrong + image. +* Fixed a bug where values for `HostConfig.nano_cpus` exceeding 2^32 would + raise a type error. +* `Image.tag` now properly returns `True` when the operation is successful. +* `APIClient.logs` and `Container.logs` now raise an exception if the `since` + argument uses an unsupported type instead of ignoring the value. +* Fixed a bug where some methods would raise a `NullResource` exception when + the resource ID was provided using a keyword argument. + +### Miscellaneous + +* `APIClient` instances can now be pickled. + 2.3.0 ----- From f4a4982ae05b6824372ff681b8ff5c30ea4ac80c Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 22 Jun 2017 16:42:15 -0700 Subject: [PATCH 0385/1301] Shift test matrix forward Signed-off-by: Joffrey F --- Jenkinsfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 987df7aff7..357927888c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -7,7 +7,7 @@ def images = [:] // Note: Swarm in dind seem notoriously flimsy with 1.12.1+, which is why we're // sticking with 1.12.0 for the 1.12 series -def dockerVersions = ["1.12.0", "1.13.1", "17.04.0-ce", "17.05.0-ce"] +def dockerVersions = ["1.13.1", "17.04.0-ce", "17.05.0-ce", "17.06.0-ce-rc5"] def buildImage = { name, buildargs, pyTag -> img = docker.image(name) @@ -35,7 +35,7 @@ def buildImages = { -> } def getAPIVersion = { engineVersion -> - def versionMap = ['1.12.': '1.24', '1.13.': '1.26', '17.04': '1.27', '17.05': '1.29'] + def versionMap = ['1.13.': '1.26', '17.04': '1.27', '17.05': '1.29', '17.06': '1.30'] return versionMap[engineVersion.substring(0, 5)] } @@ -63,7 +63,7 @@ def runTests = { Map settings -> def testContainerName = "dpy-tests-\$BUILD_NUMBER-\$EXECUTOR_NUMBER-${pythonVersion}-${dockerVersion}" try { sh """docker run -d --name ${dindContainerName} -v /tmp --privileged \\ - dockerswarm/dind:${dockerVersion} docker daemon -H tcp://0.0.0.0:2375 + dockerswarm/dind:${dockerVersion} dockerd -H tcp://0.0.0.0:2375 """ sh """docker run \\ --name ${testContainerName} --volumes-from ${dindContainerName} \\ From f611eaf9e392da691c5623400bdb3152d6bab44c Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 27 Jun 2017 18:06:12 -0700 Subject: [PATCH 0386/1301] dev version Signed-off-by: Joffrey F --- docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index 8f40f467a4..a7452d4f13 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "2.4.0" +version = "2.5.0-dev" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) From a8722fb0c23e3b7198a4d8fb2ad96cd64af11632 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 28 Jun 2017 12:06:03 -0700 Subject: [PATCH 0387/1301] split_port should not break when passed a non-string argument Signed-off-by: Joffrey F --- docker/utils/ports.py | 1 + tests/unit/utils_test.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/docker/utils/ports.py b/docker/utils/ports.py index 57332deee4..8f713c720a 100644 --- a/docker/utils/ports.py +++ b/docker/utils/ports.py @@ -54,6 +54,7 @@ def port_range(start, end, proto, randomly_available_port=False): def split_port(port): + port = str(port) match = PORT_SPEC.match(port) if match is None: _raise_invalid_port(port) diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index c25881d142..a2d463d715 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -587,6 +587,9 @@ def test_with_no_container_port(self): def test_split_port_empty_string(self): self.assertRaises(ValueError, lambda: split_port("")) + def test_split_port_non_string(self): + assert split_port(1243) == (['1243'], None) + def test_build_port_bindings_with_one_port(self): port_bindings = build_port_bindings(["127.0.0.1:1000:1000"]) self.assertEqual(port_bindings["1000"], [("127.0.0.1", "1000")]) From 50a60717f064f77974807cba7f9defd8f4e1cf4e Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 28 Jun 2017 12:06:03 -0700 Subject: [PATCH 0388/1301] split_port should not break when passed a non-string argument Signed-off-by: Joffrey F --- docker/utils/ports.py | 1 + tests/unit/utils_test.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/docker/utils/ports.py b/docker/utils/ports.py index 57332deee4..8f713c720a 100644 --- a/docker/utils/ports.py +++ b/docker/utils/ports.py @@ -54,6 +54,7 @@ def port_range(start, end, proto, randomly_available_port=False): def split_port(port): + port = str(port) match = PORT_SPEC.match(port) if match is None: _raise_invalid_port(port) diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index c25881d142..a2d463d715 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -587,6 +587,9 @@ def test_with_no_container_port(self): def test_split_port_empty_string(self): self.assertRaises(ValueError, lambda: split_port("")) + def test_split_port_non_string(self): + assert split_port(1243) == (['1243'], None) + def test_build_port_bindings_with_one_port(self): port_bindings = build_port_bindings(["127.0.0.1:1000:1000"]) self.assertEqual(port_bindings["1000"], [("127.0.0.1", "1000")]) From 3d84dbee59ba289d3f55455dbe38b929c677e687 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 28 Jun 2017 12:25:10 -0700 Subject: [PATCH 0389/1301] Bump 2.4.1 Signed-off-by: Joffrey F --- docker/version.py | 2 +- docs/change-log.md | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index 8f40f467a4..7953c904c9 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "2.4.0" +version = "2.4.1" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) diff --git a/docs/change-log.md b/docs/change-log.md index 194f734795..4ccb617da4 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -1,6 +1,16 @@ Change log ========== +2.4.1 +----- + +[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/35?closed=1) + +### Bugfixes + +* Fixed a bug where the `split_port` utility would raise an exception when + passed a non-string argument. + 2.4.0 ----- From 48957e726bf598b7aa467fee116189e930c57290 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 28 Jun 2017 13:26:04 -0700 Subject: [PATCH 0390/1301] Compose 1.14.0 hack Signed-off-by: Joffrey F --- docker/utils/ports.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docker/utils/ports.py b/docker/utils/ports.py index 8f713c720a..bf7d697271 100644 --- a/docker/utils/ports.py +++ b/docker/utils/ports.py @@ -54,6 +54,11 @@ def port_range(start, end, proto, randomly_available_port=False): def split_port(port): + if hasattr(port, 'legacy_repr'): + # This is the worst hack, but it prevents a bug in Compose 1.14.0 + # https://github.com/docker/docker-py/issues/1668 + # TODO: remove once fixed in Compose stable + port = port.legacy_repr() port = str(port) match = PORT_SPEC.match(port) if match is None: From 14e61848146c45c4892417833c8a7e98f48e864e Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 28 Jun 2017 13:26:04 -0700 Subject: [PATCH 0391/1301] Compose 1.14.0 hack Signed-off-by: Joffrey F --- docker/utils/ports.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docker/utils/ports.py b/docker/utils/ports.py index 8f713c720a..bf7d697271 100644 --- a/docker/utils/ports.py +++ b/docker/utils/ports.py @@ -54,6 +54,11 @@ def port_range(start, end, proto, randomly_available_port=False): def split_port(port): + if hasattr(port, 'legacy_repr'): + # This is the worst hack, but it prevents a bug in Compose 1.14.0 + # https://github.com/docker/docker-py/issues/1668 + # TODO: remove once fixed in Compose stable + port = port.legacy_repr() port = str(port) match = PORT_SPEC.match(port) if match is None: From 43f87e9f63a99c56f05afbf28e3a973151cacee5 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 28 Jun 2017 13:32:18 -0700 Subject: [PATCH 0392/1301] Bump 2.4.2 Signed-off-by: Joffrey F --- docker/version.py | 2 +- docs/change-log.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/version.py b/docker/version.py index 7953c904c9..af1bd5b580 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "2.4.1" +version = "2.4.2" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) diff --git a/docs/change-log.md b/docs/change-log.md index 4ccb617da4..7099d7942e 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -1,10 +1,10 @@ Change log ========== -2.4.1 +2.4.2 ----- -[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/35?closed=1) +[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/36?closed=1) ### Bugfixes From c9960c78b09254436f6856413c18b48801a14970 Mon Sep 17 00:00:00 2001 From: Matthew Berry Date: Thu, 13 Jul 2017 23:20:24 -0500 Subject: [PATCH 0393/1301] Fix #1673 check resource error in container network API Container network functions checked 'image' as resource ID and not 'container'. This caused a traceback when using container as named argument. Signed-off-by: Matthew Berry --- docker/api/network.py | 4 ++-- tests/unit/api_network_test.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docker/api/network.py b/docker/api/network.py index 5ebb41af34..5549bf0cf9 100644 --- a/docker/api/network.py +++ b/docker/api/network.py @@ -200,7 +200,7 @@ def inspect_network(self, net_id, verbose=None): res = self._get(url, params=params) return self._result(res, json=True) - @check_resource('image') + @check_resource('container') @minimum_version('1.21') def connect_container_to_network(self, container, net_id, ipv4_address=None, ipv6_address=None, @@ -237,7 +237,7 @@ def connect_container_to_network(self, container, net_id, res = self._post_json(url, data=data) self._raise_for_status(res) - @check_resource('image') + @check_resource('container') @minimum_version('1.21') def disconnect_container_from_network(self, container, net_id, force=False): diff --git a/tests/unit/api_network_test.py b/tests/unit/api_network_test.py index f997a1b829..96cdc4b194 100644 --- a/tests/unit/api_network_test.py +++ b/tests/unit/api_network_test.py @@ -147,8 +147,8 @@ def test_connect_container_to_network(self): with mock.patch('docker.api.client.APIClient.post', post): self.client.connect_container_to_network( - {'Id': container_id}, - network_id, + container={'Id': container_id}, + net_id=network_id, aliases=['foo', 'bar'], links=[('baz', 'quux')] ) @@ -176,7 +176,7 @@ def test_disconnect_container_from_network(self): with mock.patch('docker.api.client.APIClient.post', post): self.client.disconnect_container_from_network( - {'Id': container_id}, network_id) + container={'Id': container_id}, net_id=network_id) self.assertEqual( post.call_args[0][0], From d68996f953f2fcdf611868e0ec4f7b0513eb1b00 Mon Sep 17 00:00:00 2001 From: Brandon Jackson Date: Tue, 18 Jul 2017 09:35:31 -0500 Subject: [PATCH 0394/1301] By not specifying a specific tag, the example would download every Ubuntu tag that exists. This oversight caused my machine to run out of disk space holding all the image diffs. Signed-off-by: Brandon Jackson --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 38963b325c..747b98b250 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ client = docker.from_env() You can run containers: ```python ->>> client.containers.run("ubuntu", "echo hello world") +>>> client.containers.run("ubuntu:latest", "echo hello world") 'hello world\n' ``` From c2925a384bc76abba9260a4d75f71742ebb615c3 Mon Sep 17 00:00:00 2001 From: Dima Spivak Date: Mon, 31 Jul 2017 15:04:15 -0700 Subject: [PATCH 0395/1301] client.networks.create check_duplicates docs not reflective of behavior Fixes #1693 Signed-off-by: Dima Spivak --- docker/api/network.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/api/network.py b/docker/api/network.py index 5549bf0cf9..befbb583ce 100644 --- a/docker/api/network.py +++ b/docker/api/network.py @@ -52,7 +52,7 @@ def create_network(self, name, driver=None, options=None, ipam=None, options (dict): Driver options as a key-value dictionary ipam (IPAMConfig): Optional custom IP scheme for the network. check_duplicate (bool): Request daemon to check for networks with - same name. Default: ``True``. + same name. Default: ``None``. internal (bool): Restrict external access to the network. Default ``False``. labels (dict): Map of labels to set on the network. Default From d798afca7e866641a357275ae768a0a686d68882 Mon Sep 17 00:00:00 2001 From: Jakub Goszczurny Date: Mon, 3 Jul 2017 22:22:37 +0200 Subject: [PATCH 0396/1301] Generating regexp from .dockerignore file in a similar way as docker-ce. Signed-off-by: Jakub Goszczurny --- docker/utils/fnmatch.py | 27 +++++++++++++++++++-------- tests/unit/utils_test.py | 25 +++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 8 deletions(-) diff --git a/docker/utils/fnmatch.py b/docker/utils/fnmatch.py index e95b63ceb1..e51bd81552 100644 --- a/docker/utils/fnmatch.py +++ b/docker/utils/fnmatch.py @@ -65,19 +65,32 @@ def translate(pat): There is no way to quote meta-characters. """ - recursive_mode = False i, n = 0, len(pat) - res = '' + res = '^' while i < n: c = pat[i] i = i + 1 if c == '*': if i < n and pat[i] == '*': - recursive_mode = True + # is some flavor of "**" i = i + 1 - res = res + '.*' + # Treat **/ as ** so eat the "/" + if pat[i] == '/': + i = i + 1 + if i >= n: + # is "**EOF" - to align with .gitignore just accept all + res = res + '.*' + else: + # is "**" + # Note that this allows for any # of /'s (even 0) because + # the .* will eat everything, even /'s + res = res + '(.*/)?' + else: + # is "*" so map it to anything but "/" + res = res + '[^/]*' elif c == '?': - res = res + '.' + # "?" is any char except "/" + res = res + '[^/]' elif c == '[': j = i if j < n and pat[j] == '!': @@ -96,8 +109,6 @@ def translate(pat): elif stuff[0] == '^': stuff = '\\' + stuff res = '%s[%s]' % (res, stuff) - elif recursive_mode and c == '/': - res = res + re.escape(c) + '?' else: res = res + re.escape(c) - return res + '\Z(?ms)' + return res + '$' diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index a2d463d715..7045d23c27 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -639,6 +639,14 @@ class ExcludePathsTest(unittest.TestCase): 'foo', 'foo/bar', 'bar', + 'target', + 'target/subdir', + 'subdir', + 'subdir/target', + 'subdir/target/subdir', + 'subdir/subdir2', + 'subdir/subdir2/target', + 'subdir/subdir2/target/subdir' ] files = [ @@ -654,6 +662,14 @@ class ExcludePathsTest(unittest.TestCase): 'foo/bar/a.py', 'bar/a.py', 'foo/Dockerfile3', + 'target/file.txt', + 'target/subdir/file.txt', + 'subdir/file.txt', + 'subdir/target/file.txt', + 'subdir/target/subdir/file.txt', + 'subdir/subdir2/file.txt', + 'subdir/subdir2/target/file.txt', + 'subdir/subdir2/target/subdir/file.txt', ] all_paths = set(dirs + files) @@ -844,6 +860,15 @@ def test_double_wildcard(self): self.all_paths - set(['foo/bar', 'foo/bar/a.py']) ) + def test_single_and_double_wildcard(self): + assert self.exclude(['**/target/*/*']) == convert_paths( + self.all_paths - set( + ['target/subdir/file.txt', + 'subdir/target/subdir/file.txt', + 'subdir/subdir2/target/subdir/file.txt'] + ) + ) + class TarTest(unittest.TestCase): def test_tar_with_excludes(self): From bcfd6dadd1a2631b5a390220d4bd201faa359774 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 1 Aug 2017 13:49:48 -0700 Subject: [PATCH 0397/1301] Temporarily - do not run py33 tests on travis Signed-off-by: Joffrey F --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 6b48142f72..cd64b4456e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,8 @@ python: - "3.5" env: - TOX_ENV=py27 - - TOX_ENV=py33 +# FIXME: default travis worker does not carry py33 anymore. Can this be configured? +# - TOX_ENV=py33 - TOX_ENV=py34 - TOX_ENV=py35 - TOX_ENV=flake8 From 8a6b168467998e47a5364974a4876aaa2a3e9741 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Mon, 7 Aug 2017 09:14:21 -0500 Subject: [PATCH 0398/1301] Fix domainname documentation in create_container function It looks like this was probably originally copypasta'ed from dns_search and not edited afterward. Signed-off-by: Erik Johnson --- docker/api/container.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/api/container.py b/docker/api/container.py index 532a9c6d8b..06c575d5c0 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -399,7 +399,7 @@ def create_container(self, image, command=None, hostname=None, user=None, name (str): A name for the container entrypoint (str or list): An entrypoint working_dir (str): Path to the working directory - domainname (str or list): Set custom DNS search domains + domainname (str): The domain name to use for the container memswap_limit (int): host_config (dict): A dictionary created with :py:meth:`create_host_config`. From 1a923c561ff0d31347e7b48b7c728e6cbe427317 Mon Sep 17 00:00:00 2001 From: Andreas Backx Date: Sun, 6 Aug 2017 17:15:09 +0200 Subject: [PATCH 0399/1301] Added wait to the Container class documentation. The container class documentation did not automatically document the `Container.wait` method. Signed-off-by: Andreas Backx --- docs/containers.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/containers.rst b/docs/containers.rst index 6c895c6b2d..7c41bfdd92 100644 --- a/docs/containers.rst +++ b/docs/containers.rst @@ -53,3 +53,4 @@ Container objects .. automethod:: top .. automethod:: unpause .. automethod:: update + .. automethod:: wait From b8fd8213364ae21c2981f3e51cee924738cb9b93 Mon Sep 17 00:00:00 2001 From: Artem Bolshakov Date: Tue, 25 Jul 2017 12:38:23 +0300 Subject: [PATCH 0400/1301] client.containers.run returns None if none of json-file or journald logging drivers used Signed-off-by: Artem Bolshakov --- docker/errors.py | 10 ++++-- docker/models/containers.py | 16 +++++++++- tests/integration/models_containers_test.py | 18 +++++++++++ tests/unit/errors_test.py | 34 ++++++++++++++++++++- tests/unit/fake_api.py | 6 ++++ 5 files changed, 80 insertions(+), 4 deletions(-) diff --git a/docker/errors.py b/docker/errors.py index 0da97f4e3f..1f8ac23c40 100644 --- a/docker/errors.py +++ b/docker/errors.py @@ -127,8 +127,14 @@ def __init__(self, container, exit_status, command, image, stderr): self.command = command self.image = image self.stderr = stderr - msg = ("Command '{}' in image '{}' returned non-zero exit status {}: " - "{}").format(command, image, exit_status, stderr) + + if stderr is None: + msg = ("Command '{}' in image '{}' returned non-zero exit " + "status {}").format(command, image, exit_status, stderr) + else: + msg = ("Command '{}' in image '{}' returned non-zero exit " + "status {}: {}").format(command, image, exit_status, stderr) + super(ContainerError, self).__init__(msg) diff --git a/docker/models/containers.py b/docker/models/containers.py index cf01b2750a..a3598f28f9 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -667,6 +667,13 @@ def run(self, image, command=None, stdout=True, stderr=False, The container logs, either ``STDOUT``, ``STDERR``, or both, depending on the value of the ``stdout`` and ``stderr`` arguments. + ``STDOUT`` and ``STDERR`` may be read only if either ``json-file`` + or ``journald`` logging driver used. Thus, if you are using none of + these drivers, a ``None`` object is returned instead. See the + `Engine API documentation + `_ + for full details. + If ``detach`` is ``True``, a :py:class:`Container` object is returned instead. @@ -709,7 +716,14 @@ def run(self, image, command=None, stdout=True, stderr=False, if exit_status != 0: stdout = False stderr = True - out = container.logs(stdout=stdout, stderr=stderr) + + logging_driver = container.attrs['HostConfig']['LogConfig']['Type'] + + if logging_driver == 'json-file' or logging_driver == 'journald': + out = container.logs(stdout=stdout, stderr=stderr) + else: + out = None + if remove: container.remove() if exit_status != 0: diff --git a/tests/integration/models_containers_test.py b/tests/integration/models_containers_test.py index b76a88ffcf..ce3349baa7 100644 --- a/tests/integration/models_containers_test.py +++ b/tests/integration/models_containers_test.py @@ -88,6 +88,24 @@ def test_run_with_network(self): assert 'Networks' in attrs['NetworkSettings'] assert list(attrs['NetworkSettings']['Networks'].keys()) == [net_name] + def test_run_with_none_driver(self): + client = docker.from_env(version=TEST_API_VERSION) + + out = client.containers.run( + "alpine", "echo hello", + log_config=dict(type='none') + ) + self.assertEqual(out, None) + + def test_run_with_json_file_driver(self): + client = docker.from_env(version=TEST_API_VERSION) + + out = client.containers.run( + "alpine", "echo hello", + log_config=dict(type='json-file') + ) + self.assertEqual(out, b'hello\n') + def test_get(self): client = docker.from_env(version=TEST_API_VERSION) container = client.containers.run("alpine", "sleep 300", detach=True) diff --git a/tests/unit/errors_test.py b/tests/unit/errors_test.py index b78af4e109..9678669c3f 100644 --- a/tests/unit/errors_test.py +++ b/tests/unit/errors_test.py @@ -2,8 +2,10 @@ import requests -from docker.errors import (APIError, DockerException, +from docker.errors import (APIError, ContainerError, DockerException, create_unexpected_kwargs_error) +from .fake_api import FAKE_CONTAINER_ID, FAKE_IMAGE_ID +from .fake_api_client import make_fake_client class APIErrorTest(unittest.TestCase): @@ -77,6 +79,36 @@ def test_is_client_error_400(self): assert err.is_client_error() is True +class ContainerErrorTest(unittest.TestCase): + def test_container_without_stderr(self): + """The massage does not contain stderr""" + client = make_fake_client() + container = client.containers.get(FAKE_CONTAINER_ID) + command = "echo Hello World" + exit_status = 42 + image = FAKE_IMAGE_ID + stderr = None + + err = ContainerError(container, exit_status, command, image, stderr) + msg = ("Command '{}' in image '{}' returned non-zero exit status {}" + ).format(command, image, exit_status, stderr) + assert str(err) == msg + + def test_container_with_stderr(self): + """The massage contains stderr""" + client = make_fake_client() + container = client.containers.get(FAKE_CONTAINER_ID) + command = "echo Hello World" + exit_status = 42 + image = FAKE_IMAGE_ID + stderr = "Something went wrong" + + err = ContainerError(container, exit_status, command, image, stderr) + msg = ("Command '{}' in image '{}' returned non-zero exit status {}: " + "{}").format(command, image, exit_status, stderr) + assert str(err) == msg + + class CreateUnexpectedKwargsErrorTest(unittest.TestCase): def test_create_unexpected_kwargs_error_single(self): e = create_unexpected_kwargs_error('f', {'foo': 'bar'}) diff --git a/tests/unit/fake_api.py b/tests/unit/fake_api.py index ff0f1b65cc..2ba85bbf53 100644 --- a/tests/unit/fake_api.py +++ b/tests/unit/fake_api.py @@ -146,6 +146,12 @@ def get_fake_inspect_container(tty=False): "StartedAt": "2013-09-25T14:01:18.869545111+02:00", "Ghost": False }, + "HostConfig": { + "LogConfig": { + "Type": "json-file", + "Config": {} + }, + }, "MacAddress": "02:42:ac:11:00:0a" } return status_code, response From 8bcf2f27fb7f11edc454bc26c98801e1e5423020 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 7 Aug 2017 12:05:40 -0700 Subject: [PATCH 0401/1301] Improve ContainerError message compute Signed-off-by: Joffrey F --- docker/errors.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/docker/errors.py b/docker/errors.py index 1f8ac23c40..2a2f871e5d 100644 --- a/docker/errors.py +++ b/docker/errors.py @@ -128,12 +128,9 @@ def __init__(self, container, exit_status, command, image, stderr): self.image = image self.stderr = stderr - if stderr is None: - msg = ("Command '{}' in image '{}' returned non-zero exit " - "status {}").format(command, image, exit_status, stderr) - else: - msg = ("Command '{}' in image '{}' returned non-zero exit " - "status {}: {}").format(command, image, exit_status, stderr) + err = ": {}".format(stderr) if stderr is not None else "" + msg = ("Command '{}' in image '{}' returned non-zero exit " + "status {}{}").format(command, image, exit_status, err) super(ContainerError, self).__init__(msg) From d74f1bc3805176af229ae3d2e9e5dbe1f0b40a15 Mon Sep 17 00:00:00 2001 From: Tzu-Chiao Yeh Date: Sun, 13 Aug 2017 09:01:50 +0000 Subject: [PATCH 0402/1301] Fix #1575 - Add cpu_rt_period and cpu_rt_runtime args Add cpu_rt_period and cpu_rt_runtime in hostconfig with version(1.25), types(int) checks. Also add version and type checks in dockertype unit test. Signed-off-by: Tzu-Chiao Yeh --- docker/models/containers.py | 2 ++ docker/types/containers.py | 23 ++++++++++++++++++++++- tests/unit/dockertypes_test.py | 22 ++++++++++++++++++++++ 3 files changed, 46 insertions(+), 1 deletion(-) diff --git a/docker/models/containers.py b/docker/models/containers.py index a3598f28f9..204369643a 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -861,6 +861,8 @@ def prune(self, filters=None): 'cpu_shares', 'cpuset_cpus', 'cpuset_mems', + 'cpu_rt_period', + 'cpu_rt_runtime', 'device_read_bps', 'device_read_iops', 'device_write_bps', diff --git a/docker/types/containers.py b/docker/types/containers.py index 030e292bc6..fee93c0756 100644 --- a/docker/types/containers.py +++ b/docker/types/containers.py @@ -120,7 +120,8 @@ def __init__(self, version, binds=None, port_bindings=None, isolation=None, auto_remove=False, storage_opt=None, init=None, init_path=None, volume_driver=None, cpu_count=None, cpu_percent=None, nano_cpus=None, - cpuset_mems=None, runtime=None): + cpuset_mems=None, runtime=None, cpu_rt_period=None, + cpu_rt_runtime=None): if mem_limit is not None: self['Memory'] = parse_bytes(mem_limit) @@ -339,6 +340,26 @@ def __init__(self, version, binds=None, port_bindings=None, ) self['CpusetMems'] = cpuset_mems + if cpu_rt_period: + if version_lt(version, '1.25'): + raise host_config_version_error('cpu_rt_period', '1.25') + + if not isinstance(cpu_rt_period, int): + raise host_config_type_error( + 'cpu_rt_period', cpu_rt_period, 'int' + ) + self['CPURealtimePeriod'] = cpu_rt_period + + if cpu_rt_runtime: + if version_lt(version, '1.25'): + raise host_config_version_error('cpu_rt_runtime', '1.25') + + if not isinstance(cpu_rt_runtime, int): + raise host_config_type_error( + 'cpu_rt_runtime', cpu_rt_runtime, 'int' + ) + self['CPURealtimeRuntime'] = cpu_rt_runtime + if blkio_weight: if not isinstance(blkio_weight, int): raise host_config_type_error( diff --git a/tests/unit/dockertypes_test.py b/tests/unit/dockertypes_test.py index 8dbb35ecca..40adbb782f 100644 --- a/tests/unit/dockertypes_test.py +++ b/tests/unit/dockertypes_test.py @@ -206,6 +206,28 @@ def test_create_host_config_with_nano_cpus(self): InvalidVersion, lambda: create_host_config( version='1.24', nano_cpus=1)) + def test_create_host_config_with_cpu_rt_period_types(self): + with pytest.raises(TypeError): + create_host_config(version='1.25', cpu_rt_period='1000') + + def test_create_host_config_with_cpu_rt_period(self): + config = create_host_config(version='1.25', cpu_rt_period=1000) + self.assertEqual(config.get('CPURealtimePeriod'), 1000) + self.assertRaises( + InvalidVersion, lambda: create_host_config( + version='1.24', cpu_rt_period=1000)) + + def test_ctrate_host_config_with_cpu_rt_runtime_types(self): + with pytest.raises(TypeError): + create_host_config(version='1.25', cpu_rt_runtime='1000') + + def test_create_host_config_with_cpu_rt_runtime(self): + config = create_host_config(version='1.25', cpu_rt_runtime=1000) + self.assertEqual(config.get('CPURealtimeRuntime'), 1000) + self.assertRaises( + InvalidVersion, lambda: create_host_config( + version='1.24', cpu_rt_runtime=1000)) + class ContainerConfigTest(unittest.TestCase): def test_create_container_config_volume_driver_warning(self): From 56dc7db069f8b0bdfd739d1ff47dafbe6f39513e Mon Sep 17 00:00:00 2001 From: Cecile Tonglet Date: Tue, 1 Aug 2017 12:16:56 +0200 Subject: [PATCH 0403/1301] Return the result of the API when using remove_image and load_image Those calls return result that can be used by the developers. Signed-off-by: Cecile Tonglet --- docker/api/image.py | 4 ++-- tests/integration/api_image_test.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docker/api/image.py b/docker/api/image.py index 181c4a1e4a..85ff435d44 100644 --- a/docker/api/image.py +++ b/docker/api/image.py @@ -272,7 +272,7 @@ def load_image(self, data): data (binary): Image data to be loaded. """ res = self._post(self._url("/images/load"), data=data) - self._raise_for_status(res) + return self._result(res, True) @utils.minimum_version('1.25') def prune_images(self, filters=None): @@ -455,7 +455,7 @@ def remove_image(self, image, force=False, noprune=False): """ params = {'force': force, 'noprune': noprune} res = self._delete(self._url("/images/{0}", image), params=params) - self._raise_for_status(res) + return self._result(res, True) def search(self, term): """ diff --git a/tests/integration/api_image_test.py b/tests/integration/api_image_test.py index 917bc50555..192e6f8d1b 100644 --- a/tests/integration/api_image_test.py +++ b/tests/integration/api_image_test.py @@ -113,7 +113,8 @@ def test_remove(self): self.assertIn('Id', res) img_id = res['Id'] self.tmp_imgs.append(img_id) - self.client.remove_image(img_id, force=True) + logs = self.client.remove_image(img_id, force=True) + self.assertIn({"Deleted": img_id}, logs) images = self.client.images(all=True) res = [x for x in images if x['Id'].startswith(img_id)] self.assertEqual(len(res), 0) From 5e4a69bbdafef6f1036b733ce356c6692c65e775 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 16 Jun 2017 14:05:54 -0700 Subject: [PATCH 0404/1301] Return generator for output of load_image endpoint Signed-off-by: Joffrey F --- Jenkinsfile | 5 ++--- docker/api/image.py | 29 ++++++++++++++++++++++++++--- docker/models/images.py | 3 +++ tests/integration/api_image_test.py | 13 +++++++++++++ tests/unit/api_image_test.py | 14 ++++++++++++++ 5 files changed, 58 insertions(+), 6 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 357927888c..9e1b4912a4 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -5,9 +5,8 @@ def imageNamePy2 def imageNamePy3 def images = [:] -// Note: Swarm in dind seem notoriously flimsy with 1.12.1+, which is why we're -// sticking with 1.12.0 for the 1.12 series -def dockerVersions = ["1.13.1", "17.04.0-ce", "17.05.0-ce", "17.06.0-ce-rc5"] + +def dockerVersions = ["1.13.1", "17.04.0-ce", "17.05.0-ce", "17.06.0-ce"] def buildImage = { name, buildargs, pyTag -> img = docker.image(name) diff --git a/docker/api/image.py b/docker/api/image.py index 85ff435d44..41cc267e9d 100644 --- a/docker/api/image.py +++ b/docker/api/image.py @@ -262,7 +262,7 @@ def inspect_image(self, image): self._get(self._url("/images/{0}/json", image)), True ) - def load_image(self, data): + def load_image(self, data, quiet=None): """ Load an image that was previously saved using :py:meth:`~docker.api.image.ImageApiMixin.get_image` (or ``docker @@ -270,9 +270,32 @@ def load_image(self, data): Args: data (binary): Image data to be loaded. + quiet (boolean): Suppress progress details in response. + + Returns: + (generator): Progress output as JSON objects. Only available for + API version >= 1.23 + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. """ - res = self._post(self._url("/images/load"), data=data) - return self._result(res, True) + params = {} + + if quiet is not None: + if utils.version_lt(self._version, '1.23'): + raise errors.InvalidVersion( + 'quiet is not supported in API version < 1.23' + ) + params['quiet'] = quiet + + res = self._post( + self._url("/images/load"), data=data, params=params, stream=True + ) + if utils.version_gte(self._version, '1.23'): + return self._stream_helper(res, decode=True) + + self._raise_for_status(res) @utils.minimum_version('1.25') def prune_images(self, filters=None): diff --git a/docker/models/images.py b/docker/models/images.py index d4e24c6060..3837929dfa 100644 --- a/docker/models/images.py +++ b/docker/models/images.py @@ -235,6 +235,9 @@ def load(self, data): Args: data (binary): Image data to be loaded. + Returns: + (generator): Progress output as JSON objects + Raises: :py:class:`docker.errors.APIError` If the server returns an error. diff --git a/tests/integration/api_image_test.py b/tests/integration/api_image_test.py index 192e6f8d1b..14fb77aa46 100644 --- a/tests/integration/api_image_test.py +++ b/tests/integration/api_image_test.py @@ -249,6 +249,19 @@ def test_import_image_with_changes(self): assert img_data['Config']['Cmd'] == ['echo'] assert img_data['Config']['User'] == 'foobar' + # Docs say output is available in 1.23, but this test fails on 1.12.0 + @requires_api_version('1.24') + def test_get_load_image(self): + test_img = 'hello-world:latest' + self.client.pull(test_img) + data = self.client.get_image(test_img) + assert data + output = self.client.load_image(data) + assert any([ + line for line in output + if 'Loaded image: {}'.format(test_img) in line.get('stream', '') + ]) + @contextlib.contextmanager def temporary_http_file_server(self, stream): '''Serve data from an IO stream over HTTP.''' diff --git a/tests/unit/api_image_test.py b/tests/unit/api_image_test.py index 36b2a46833..f1e42cc147 100644 --- a/tests/unit/api_image_test.py +++ b/tests/unit/api_image_test.py @@ -369,5 +369,19 @@ def test_load_image(self): 'POST', url_prefix + 'images/load', data='Byte Stream....', + stream=True, + params={}, + timeout=DEFAULT_TIMEOUT_SECONDS + ) + + def test_load_image_quiet(self): + self.client.load_image('Byte Stream....', quiet=True) + + fake_request.assert_called_with( + 'POST', + url_prefix + 'images/load', + data='Byte Stream....', + stream=True, + params={'quiet': True}, timeout=DEFAULT_TIMEOUT_SECONDS ) From 92a2e48e1740f9ecb055db6c960990483b4b349c Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 15 Aug 2017 14:54:53 -0700 Subject: [PATCH 0405/1301] Leading slash in .dockerignore should be ignored Signed-off-by: Joffrey F --- docker/utils/build.py | 1 + tests/unit/utils_test.py | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/docker/utils/build.py b/docker/utils/build.py index 79b72495d9..d4223e749f 100644 --- a/docker/utils/build.py +++ b/docker/utils/build.py @@ -26,6 +26,7 @@ def exclude_paths(root, patterns, dockerfile=None): if dockerfile is None: dockerfile = 'Dockerfile' + patterns = [p.lstrip('/') for p in patterns] exceptions = [p for p in patterns if p.startswith('!')] include_patterns = [p[1:] for p in exceptions] diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index 7045d23c27..4a391facb7 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -768,6 +768,11 @@ def test_single_subdir_single_filename(self): self.all_paths - set(['foo/a.py']) ) + def test_single_subdir_single_filename_leading_slash(self): + assert self.exclude(['/foo/a.py']) == convert_paths( + self.all_paths - set(['foo/a.py']) + ) + def test_single_subdir_with_path_traversal(self): assert self.exclude(['foo/whoops/../a.py']) == convert_paths( self.all_paths - set(['foo/a.py']) From 3b95da3ea48c3f60f133cdefe1382fce5d44a770 Mon Sep 17 00:00:00 2001 From: cyli Date: Thu, 13 Apr 2017 17:25:55 -0700 Subject: [PATCH 0406/1301] Require "requests[security]" if the `[tls]` option is selected, which also installs: pyOpenSSL, cryptography, idna and installs cryptography's version of openssl in Mac OS (which by default has an ancient version of openssl that doesn't support TLS 1.2). Signed-off-by: cyli --- README.md | 4 ++++ requirements.txt | 2 +- setup.py | 10 ++++++++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 747b98b250..3ff124d7a5 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,10 @@ The latest stable version [is available on PyPI](https://pypi.python.org/pypi/do pip install docker +If you are intending to connect to a docker host via TLS, add `docker[tls]` to your requirements instead, or install with pip: + + pip install docker[tls] + ## Usage Connect to Docker using the default socket or the configuration in your environment: diff --git a/requirements.txt b/requirements.txt index 375413122b..423ffb7004 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -requests==2.11.1 +requests[security]==2.11.1 six>=1.4.0 websocket-client==0.32.0 backports.ssl_match_hostname>=3.5 ; python_version < '3.5' diff --git a/setup.py b/setup.py index 31180d2397..534c9495da 100644 --- a/setup.py +++ b/setup.py @@ -35,6 +35,16 @@ # ssl_match_hostname to verify hosts match with certificates via # ServerAltname: https://pypi.python.org/pypi/backports.ssl_match_hostname ':python_version < "3.3"': 'ipaddress >= 1.0.16', + + # If using docker-py over TLS, highly recommend this option is pip-installed + # or pinned. + + # TODO: if pip installign both "requests" and "requests[security]", the + # extra package from the "security" option are not installed (see + # https://github.com/pypa/pip/issues/4391). Once that's fixed, instead of + # installing the extra dependencies, install the following instead: + # 'requests[security] >= 2.5.2, != 2.11.0, != 2.12.2' + 'tls': ['pyOpenSSL>=0.14', 'cryptography>=1.3.4', 'idna>=2.0.0'], } version = None From c163375591f075c641804effbfa65be5cd08adce Mon Sep 17 00:00:00 2001 From: cyli Date: Mon, 22 May 2017 15:45:06 -0700 Subject: [PATCH 0407/1301] If we're pinning exact versions of things for requirements.txt, pin all dependencies of dependencies as well so we can get a consistent build. Signed-off-by: cyli --- requirements.txt | 20 +++++++++++++++----- setup.py | 6 +++--- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/requirements.txt b/requirements.txt index 423ffb7004..f3c61e790b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,16 @@ -requests[security]==2.11.1 -six>=1.4.0 -websocket-client==0.32.0 -backports.ssl_match_hostname>=3.5 ; python_version < '3.5' -ipaddress==1.0.16 ; python_version < '3.3' +appdirs==1.4.3 +asn1crypto==0.22.0 +backports.ssl-match-hostname==3.5.0.1 +cffi==1.10.0 +cryptography==1.9 docker-pycreds==0.2.1 +enum34==1.1.6 +idna==2.5 +ipaddress==1.0.18 +packaging==16.8 +pycparser==2.17 +pyOpenSSL==17.0.0 +pyparsing==2.2.0 +requests==2.14.2 +six==1.10.0 +websocket-client==0.40.0 diff --git a/setup.py b/setup.py index 534c9495da..4a33c8df02 100644 --- a/setup.py +++ b/setup.py @@ -36,10 +36,10 @@ # ServerAltname: https://pypi.python.org/pypi/backports.ssl_match_hostname ':python_version < "3.3"': 'ipaddress >= 1.0.16', - # If using docker-py over TLS, highly recommend this option is pip-installed - # or pinned. + # If using docker-py over TLS, highly recommend this option is + # pip-installed or pinned. - # TODO: if pip installign both "requests" and "requests[security]", the + # TODO: if pip installing both "requests" and "requests[security]", the # extra package from the "security" option are not installed (see # https://github.com/pypa/pip/issues/4391). Once that's fixed, instead of # installing the extra dependencies, install the following instead: From 35f29d08d748fbf78b08c18a1cd3cad92944d022 Mon Sep 17 00:00:00 2001 From: Ying Date: Fri, 16 Jun 2017 18:46:09 -0700 Subject: [PATCH 0408/1301] Upgrade tox and virtualenv in appveyor to make sure we have the latest pip. Signed-off-by: Ying --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 1fc67cc024..41cde6252b 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -3,7 +3,7 @@ version: '{branch}-{build}' install: - "SET PATH=C:\\Python27-x64;C:\\Python27-x64\\Scripts;%PATH%" - "python --version" - - "pip install tox==2.1.1 virtualenv==13.1.2" + - "pip install tox==2.7.0 virtualenv==15.1.0" # Build the binary after tests build: false From e9fab1432b974ceaa888b371e382dfcf2f6556e4 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 15 Aug 2017 15:38:09 -0700 Subject: [PATCH 0409/1301] Daemon expects full URL of hub in auth config dict in build payload Signed-off-by: Joffrey F --- docker/api/build.py | 5 ++++- docker/auth.py | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/docker/api/build.py b/docker/api/build.py index cbef4a8b17..5d4e7720a1 100644 --- a/docker/api/build.py +++ b/docker/api/build.py @@ -274,7 +274,10 @@ def _set_auth_headers(self, headers): self._auth_configs, registry ) else: - auth_data = self._auth_configs + auth_data = self._auth_configs.copy() + # See https://github.com/docker/docker-py/issues/1683 + if auth.INDEX_NAME in auth_data: + auth_data[auth.INDEX_URL] = auth_data[auth.INDEX_NAME] log.debug( 'Sending auth config ({0})'.format( diff --git a/docker/auth.py b/docker/auth.py index ec9c45b97d..c3fb062e9b 100644 --- a/docker/auth.py +++ b/docker/auth.py @@ -10,7 +10,7 @@ from .constants import IS_WINDOWS_PLATFORM INDEX_NAME = 'docker.io' -INDEX_URL = 'https://{0}/v1/'.format(INDEX_NAME) +INDEX_URL = 'https://index.{0}/v1/'.format(INDEX_NAME) DOCKER_CONFIG_FILENAME = os.path.join('.docker', 'config.json') LEGACY_DOCKER_CONFIG_FILENAME = '.dockercfg' TOKEN_USERNAME = '' @@ -118,7 +118,7 @@ def _resolve_authconfig_credstore(authconfig, registry, credstore_name): if not registry or registry == INDEX_NAME: # The ecosystem is a little schizophrenic with index.docker.io VS # docker.io - in that case, it seems the full URL is necessary. - registry = 'https://index.docker.io/v1/' + registry = INDEX_URL log.debug("Looking for auth entry for {0}".format(repr(registry))) store = dockerpycreds.Store(credstore_name) try: From 18acd569a1c7aadd81f0eb877093a4bd28de86f5 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 15 Aug 2017 16:07:47 -0700 Subject: [PATCH 0410/1301] Handle untyped ContainerSpec dict in _check_api_features Signed-off-by: Joffrey F --- docker/api/service.py | 2 +- tests/integration/api_service_test.py | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/docker/api/service.py b/docker/api/service.py index cc16cc37dd..4b555a5f59 100644 --- a/docker/api/service.py +++ b/docker/api/service.py @@ -38,7 +38,7 @@ def _check_api_features(version, task_template, update_config): 'Placement.preferences is not supported in' ' API version < 1.27' ) - if task_template.container_spec.get('TTY'): + if task_template.get('ContainerSpec', {}).get('TTY'): if utils.version_lt(version, '1.25'): raise errors.InvalidVersion( 'ContainerSpec.TTY is not supported in API version < 1.25' diff --git a/tests/integration/api_service_test.py b/tests/integration/api_service_test.py index 54111a7bb1..c966916ebb 100644 --- a/tests/integration/api_service_test.py +++ b/tests/integration/api_service_test.py @@ -376,6 +376,23 @@ def test_create_service_with_tty(self): assert 'TTY' in con_spec assert con_spec['TTY'] is True + @requires_api_version('1.25') + def test_create_service_with_tty_dict(self): + container_spec = { + 'Image': BUSYBOX, + 'Command': ['true'], + 'TTY': True + } + task_tmpl = docker.types.TaskTemplate(container_spec) + name = self.get_service_name() + svc_id = self.client.create_service(task_tmpl, name=name) + svc_info = self.client.inspect_service(svc_id) + assert 'TaskTemplate' in svc_info['Spec'] + assert 'ContainerSpec' in svc_info['Spec']['TaskTemplate'] + con_spec = svc_info['Spec']['TaskTemplate']['ContainerSpec'] + assert 'TTY' in con_spec + assert con_spec['TTY'] is True + def test_create_service_global_mode(self): container_spec = docker.types.ContainerSpec( BUSYBOX, ['echo', 'hello'] From 05e5db5853f3798be950c6d20949c9bf7a3c0fa0 Mon Sep 17 00:00:00 2001 From: adrianliaw Date: Sat, 6 May 2017 19:29:39 +0800 Subject: [PATCH 0411/1301] Use collection's get method inside ImageCollection's list method Signed-off-by: Adrian Liaw --- docker/models/images.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/models/images.py b/docker/models/images.py index 3837929dfa..d1b29ad8a6 100644 --- a/docker/models/images.py +++ b/docker/models/images.py @@ -224,7 +224,7 @@ def list(self, name=None, all=False, filters=None): If the server returns an error. """ resp = self.client.api.images(name=name, all=all, filters=filters) - return [self.prepare_model(r) for r in resp] + return [self.get(r["Id"]) for r in resp] def load(self, data): """ From e17a545aa5b17a2aa8de486a2e2363a6274ea6a7 Mon Sep 17 00:00:00 2001 From: David Steines Date: Mon, 3 Apr 2017 21:58:59 -0400 Subject: [PATCH 0412/1301] Allow detach and remove for api version >= 1.25 and use auto_remove when both are set. Continue raising an exception for api versions <1.25. Signed-off-by: David Steines --- docker/models/containers.py | 10 ++++++++-- tests/unit/models_containers_test.py | 30 ++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/docker/models/containers.py b/docker/models/containers.py index a3598f28f9..d9db79dfba 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -4,6 +4,7 @@ from ..errors import (ContainerError, ImageNotFound, create_unexpected_kwargs_error) from ..types import HostConfig +from ..utils import compare_version from .images import Image from .resource import Collection, Model @@ -690,8 +691,12 @@ def run(self, image, command=None, stdout=True, stderr=False, image = image.id detach = kwargs.pop("detach", False) if detach and remove: - raise RuntimeError("The options 'detach' and 'remove' cannot be " - "used together.") + if compare_version("1.24", + self.client.api._version) > 0: + kwargs["auto_remove"] = True + else: + raise RuntimeError("The options 'detach' and 'remove' cannot " + "be used together in api versions < 1.25.") if kwargs.get('network') and kwargs.get('network_mode'): raise RuntimeError( @@ -849,6 +854,7 @@ def prune(self, filters=None): # kwargs to copy straight from run to host_config RUN_HOST_CONFIG_KWARGS = [ + 'auto_remove', 'blkio_weight_device', 'blkio_weight', 'cap_add', diff --git a/tests/unit/models_containers_test.py b/tests/unit/models_containers_test.py index 70c86480c8..5eaa45ac66 100644 --- a/tests/unit/models_containers_test.py +++ b/tests/unit/models_containers_test.py @@ -273,9 +273,39 @@ def test_run_remove(self): client.api.remove_container.assert_called_with(FAKE_CONTAINER_ID) client = make_fake_client() + client.api._version = '1.24' with self.assertRaises(RuntimeError): client.containers.run("alpine", detach=True, remove=True) + client = make_fake_client() + client.api._version = '1.23' + with self.assertRaises(RuntimeError): + client.containers.run("alpine", detach=True, remove=True) + + client = make_fake_client() + client.api._version = '1.25' + client.containers.run("alpine", detach=True, remove=True) + client.api.remove_container.assert_not_called() + client.api.create_container.assert_called_with( + command=None, + image='alpine', + detach=True, + host_config={'AutoRemove': True, + 'NetworkMode': 'default'} + ) + + client = make_fake_client() + client.api._version = '1.26' + client.containers.run("alpine", detach=True, remove=True) + client.api.remove_container.assert_not_called() + client.api.create_container.assert_called_with( + command=None, + image='alpine', + detach=True, + host_config={'AutoRemove': True, + 'NetworkMode': 'default'} + ) + def test_create(self): client = make_fake_client() container = client.containers.create( From b78fa6405aa5621cc9857dd8eda98a6d1b4e8ad0 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 15 Aug 2017 17:40:07 -0700 Subject: [PATCH 0413/1301] Use better version comparison function Signed-off-by: Joffrey F --- docker/models/containers.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docker/models/containers.py b/docker/models/containers.py index d9db79dfba..688deccadc 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -4,7 +4,7 @@ from ..errors import (ContainerError, ImageNotFound, create_unexpected_kwargs_error) from ..types import HostConfig -from ..utils import compare_version +from ..utils import version_gte from .images import Image from .resource import Collection, Model @@ -691,8 +691,7 @@ def run(self, image, command=None, stdout=True, stderr=False, image = image.id detach = kwargs.pop("detach", False) if detach and remove: - if compare_version("1.24", - self.client.api._version) > 0: + if version_gte(self.client.api._version, '1.25'): kwargs["auto_remove"] = True else: raise RuntimeError("The options 'detach' and 'remove' cannot " From 76eb0298c69c74545e9389e0458f27b2ea12fd88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gabriel=20F=C3=A9ron?= Date: Mon, 30 Jan 2017 19:06:20 +0100 Subject: [PATCH 0414/1301] Add support for the `squash` flag when building MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Also added a test that compares the number of layers in the default mode, and with the new flag Signed-off-by: Gabriel Féron --- docker/api/build.py | 13 ++++++++++++- tests/integration/api_build_test.py | 28 +++++++++++++++++++++++++++- 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/docker/api/build.py b/docker/api/build.py index 5d4e7720a1..f9678a390a 100644 --- a/docker/api/build.py +++ b/docker/api/build.py @@ -18,7 +18,8 @@ def build(self, path=None, tag=None, quiet=False, fileobj=None, custom_context=False, encoding=None, pull=False, forcerm=False, dockerfile=None, container_limits=None, decode=False, buildargs=None, gzip=False, shmsize=None, - labels=None, cache_from=None, target=None, network_mode=None): + labels=None, cache_from=None, target=None, network_mode=None, + squash=None): """ Similar to the ``docker build`` command. Either ``path`` or ``fileobj`` needs to be set. ``path`` can be a local path (to a directory @@ -98,6 +99,8 @@ def build(self, path=None, tag=None, quiet=False, fileobj=None, Dockerfile network_mode (str): networking mode for the run commands during build + squash (bool): Squash the resulting images layers into a + single layer. Returns: A generator for the build output. @@ -218,6 +221,14 @@ def build(self, path=None, tag=None, quiet=False, fileobj=None, 'network_mode was only introduced in API version 1.25' ) + if squash: + if utils.version_gte(self._version, '1.25'): + params.update({'squash': squash}) + else: + raise errors.InvalidVersion( + 'squash was only introduced in API version 1.25' + ) + if context is not None: headers = {'Content-Type': 'application/tar'} if encoding: diff --git a/tests/integration/api_build_test.py b/tests/integration/api_build_test.py index 609964f0b0..209c1f28b2 100644 --- a/tests/integration/api_build_test.py +++ b/tests/integration/api_build_test.py @@ -9,7 +9,7 @@ import six from .base import BaseAPIIntegrationTest -from ..helpers import requires_api_version +from ..helpers import requires_api_version, requires_experimental class BuildTest(BaseAPIIntegrationTest): @@ -244,6 +244,32 @@ def test_build_with_network_mode(self): with pytest.raises(errors.NotFound): self.client.inspect_image('dockerpytest_nonebuild') + @requires_api_version('1.25') + @requires_experimental + def test_build_squash(self): + script = io.BytesIO('\n'.join([ + 'FROM busybox', + 'RUN echo blah > /file_1', + 'RUN echo blahblah > /file_2', + 'RUN echo blahblahblah > /file_3' + ]).encode('ascii')) + + def build_squashed(squash): + tag = 'squash' if squash else 'nosquash' + stream = self.client.build( + fileobj=script, tag=tag, squash=squash + ) + self.tmp_imgs.append(tag) + for chunk in stream: + pass + + return self.client.inspect_image(tag) + + non_squashed = build_squashed(False) + squashed = build_squashed(True) + self.assertEqual(len(non_squashed['RootFS']['Layers']), 4) + self.assertEqual(len(squashed['RootFS']['Layers']), 2) + def test_build_stderr_data(self): control_chars = ['\x1b[91m', '\x1b[0m'] snippet = 'Ancient Temple (Mystic Oriental Dream ~ Ancient Temple)' From 13b9349216162e24c4bfedca1f401363ab732615 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 16 Aug 2017 17:31:36 -0700 Subject: [PATCH 0415/1301] Fix handling of non-multiplexed (TTY) streams over upgraded sockets Signed-off-by: Joffrey F --- docker/api/client.py | 22 ++++++++++++++++------ docker/api/container.py | 4 +++- docker/api/exec_api.py | 2 +- docker/utils/socket.py | 21 ++++++++++++++++++++- tests/integration/api_build_test.py | 2 +- tests/unit/api_test.py | 2 +- 6 files changed, 42 insertions(+), 11 deletions(-) diff --git a/docker/api/client.py b/docker/api/client.py index 65b5baa967..1de10c77c0 100644 --- a/docker/api/client.py +++ b/docker/api/client.py @@ -32,7 +32,7 @@ from ..tls import TLSConfig from ..transport import SSLAdapter, UnixAdapter from ..utils import utils, check_resource, update_headers -from ..utils.socket import frames_iter +from ..utils.socket import frames_iter, socket_raw_iter from ..utils.json_stream import json_stream try: from ..transport import NpipeAdapter @@ -362,13 +362,19 @@ def _stream_raw_result(self, response): for out in response.iter_content(chunk_size=1, decode_unicode=True): yield out - def _read_from_socket(self, response, stream): + def _read_from_socket(self, response, stream, tty=False): socket = self._get_raw_response_socket(response) + gen = None + if tty is False: + gen = frames_iter(socket) + else: + gen = socket_raw_iter(socket) + if stream: - return frames_iter(socket) + return gen else: - return six.binary_type().join(frames_iter(socket)) + return six.binary_type().join(gen) def _disable_socket_timeout(self, socket): """ Depending on the combination of python version and whether we're @@ -398,9 +404,13 @@ def _disable_socket_timeout(self, socket): s.settimeout(None) - def _get_result(self, container, stream, res): + @check_resource('container') + def _check_is_tty(self, container): cont = self.inspect_container(container) - return self._get_result_tty(stream, res, cont['Config']['Tty']) + return cont['Config']['Tty'] + + def _get_result(self, container, stream, res): + return self._get_result_tty(stream, res, self._check_is_tty(container)) def _get_result_tty(self, stream, res, is_tty): # Stream multi-plexing was only introduced in API v1.6. Anything diff --git a/docker/api/container.py b/docker/api/container.py index 06c575d5c0..dde1325422 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -52,7 +52,9 @@ def attach(self, container, stdout=True, stderr=True, u = self._url("/containers/{0}/attach", container) response = self._post(u, headers=headers, params=params, stream=stream) - return self._read_from_socket(response, stream) + return self._read_from_socket( + response, stream, self._check_is_tty(container) + ) @utils.check_resource('container') def attach_socket(self, container, params=None, ws=False): diff --git a/docker/api/exec_api.py b/docker/api/exec_api.py index 2b407cef40..6f42524e65 100644 --- a/docker/api/exec_api.py +++ b/docker/api/exec_api.py @@ -153,4 +153,4 @@ def exec_start(self, exec_id, detach=False, tty=False, stream=False, return self._result(res) if socket: return self._get_raw_response_socket(res) - return self._read_from_socket(res, stream) + return self._read_from_socket(res, stream, tty) diff --git a/docker/utils/socket.py b/docker/utils/socket.py index 4080f253f5..54392d2b74 100644 --- a/docker/utils/socket.py +++ b/docker/utils/socket.py @@ -75,5 +75,24 @@ def frames_iter(socket): break while n > 0: result = read(socket, n) - n -= len(result) + if result is None: + continue + data_length = len(result) + if data_length == 0: + # We have reached EOF + return + n -= data_length yield result + + +def socket_raw_iter(socket): + """ + Returns a generator of data read from the socket. + This is used for non-multiplexed streams. + """ + while True: + result = read(socket) + if len(result) == 0: + # We have reached EOF + return + yield result diff --git a/tests/integration/api_build_test.py b/tests/integration/api_build_test.py index 209c1f28b2..d0aa5c213c 100644 --- a/tests/integration/api_build_test.py +++ b/tests/integration/api_build_test.py @@ -244,8 +244,8 @@ def test_build_with_network_mode(self): with pytest.raises(errors.NotFound): self.client.inspect_image('dockerpytest_nonebuild') + @requires_experimental(until=None) @requires_api_version('1.25') - @requires_experimental def test_build_squash(self): script = io.BytesIO('\n'.join([ 'FROM busybox', diff --git a/tests/unit/api_test.py b/tests/unit/api_test.py index 83848c524a..6ac92c4076 100644 --- a/tests/unit/api_test.py +++ b/tests/unit/api_test.py @@ -83,7 +83,7 @@ def fake_delete(self, url, *args, **kwargs): return fake_request('DELETE', url, *args, **kwargs) -def fake_read_from_socket(self, response, stream): +def fake_read_from_socket(self, response, stream, tty=False): return six.binary_type() From 2b128077c1fffc0d9f308dc31543bb602b599282 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 22 Jun 2017 16:42:15 -0700 Subject: [PATCH 0416/1301] Shift test matrix forward Signed-off-by: Joffrey F --- Jenkinsfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 987df7aff7..357927888c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -7,7 +7,7 @@ def images = [:] // Note: Swarm in dind seem notoriously flimsy with 1.12.1+, which is why we're // sticking with 1.12.0 for the 1.12 series -def dockerVersions = ["1.12.0", "1.13.1", "17.04.0-ce", "17.05.0-ce"] +def dockerVersions = ["1.13.1", "17.04.0-ce", "17.05.0-ce", "17.06.0-ce-rc5"] def buildImage = { name, buildargs, pyTag -> img = docker.image(name) @@ -35,7 +35,7 @@ def buildImages = { -> } def getAPIVersion = { engineVersion -> - def versionMap = ['1.12.': '1.24', '1.13.': '1.26', '17.04': '1.27', '17.05': '1.29'] + def versionMap = ['1.13.': '1.26', '17.04': '1.27', '17.05': '1.29', '17.06': '1.30'] return versionMap[engineVersion.substring(0, 5)] } @@ -63,7 +63,7 @@ def runTests = { Map settings -> def testContainerName = "dpy-tests-\$BUILD_NUMBER-\$EXECUTOR_NUMBER-${pythonVersion}-${dockerVersion}" try { sh """docker run -d --name ${dindContainerName} -v /tmp --privileged \\ - dockerswarm/dind:${dockerVersion} docker daemon -H tcp://0.0.0.0:2375 + dockerswarm/dind:${dockerVersion} dockerd -H tcp://0.0.0.0:2375 """ sh """docker run \\ --name ${testContainerName} --volumes-from ${dindContainerName} \\ From e0c7e4d60e1e76cdfceeb782bbcc91d87d2a5d0d Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 27 Jun 2017 18:06:12 -0700 Subject: [PATCH 0417/1301] dev version Signed-off-by: Joffrey F --- docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index af1bd5b580..a7452d4f13 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "2.4.2" +version = "2.5.0-dev" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) From a23cd3d8e874635718e93b121dec58a08c8f766c Mon Sep 17 00:00:00 2001 From: Matthew Berry Date: Thu, 13 Jul 2017 23:20:24 -0500 Subject: [PATCH 0418/1301] Fix #1673 check resource error in container network API Container network functions checked 'image' as resource ID and not 'container'. This caused a traceback when using container as named argument. Signed-off-by: Matthew Berry --- docker/api/network.py | 4 ++-- tests/unit/api_network_test.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docker/api/network.py b/docker/api/network.py index 5ebb41af34..5549bf0cf9 100644 --- a/docker/api/network.py +++ b/docker/api/network.py @@ -200,7 +200,7 @@ def inspect_network(self, net_id, verbose=None): res = self._get(url, params=params) return self._result(res, json=True) - @check_resource('image') + @check_resource('container') @minimum_version('1.21') def connect_container_to_network(self, container, net_id, ipv4_address=None, ipv6_address=None, @@ -237,7 +237,7 @@ def connect_container_to_network(self, container, net_id, res = self._post_json(url, data=data) self._raise_for_status(res) - @check_resource('image') + @check_resource('container') @minimum_version('1.21') def disconnect_container_from_network(self, container, net_id, force=False): diff --git a/tests/unit/api_network_test.py b/tests/unit/api_network_test.py index f997a1b829..96cdc4b194 100644 --- a/tests/unit/api_network_test.py +++ b/tests/unit/api_network_test.py @@ -147,8 +147,8 @@ def test_connect_container_to_network(self): with mock.patch('docker.api.client.APIClient.post', post): self.client.connect_container_to_network( - {'Id': container_id}, - network_id, + container={'Id': container_id}, + net_id=network_id, aliases=['foo', 'bar'], links=[('baz', 'quux')] ) @@ -176,7 +176,7 @@ def test_disconnect_container_from_network(self): with mock.patch('docker.api.client.APIClient.post', post): self.client.disconnect_container_from_network( - {'Id': container_id}, network_id) + container={'Id': container_id}, net_id=network_id) self.assertEqual( post.call_args[0][0], From 9abcaccb89fe5767e541061befc1d24c64380243 Mon Sep 17 00:00:00 2001 From: Brandon Jackson Date: Tue, 18 Jul 2017 09:35:31 -0500 Subject: [PATCH 0419/1301] By not specifying a specific tag, the example would download every Ubuntu tag that exists. This oversight caused my machine to run out of disk space holding all the image diffs. Signed-off-by: Brandon Jackson --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 38963b325c..747b98b250 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ client = docker.from_env() You can run containers: ```python ->>> client.containers.run("ubuntu", "echo hello world") +>>> client.containers.run("ubuntu:latest", "echo hello world") 'hello world\n' ``` From 48b5c07c3a7c008befe5349095eee0117ba8e3de Mon Sep 17 00:00:00 2001 From: Dima Spivak Date: Mon, 31 Jul 2017 15:04:15 -0700 Subject: [PATCH 0420/1301] client.networks.create check_duplicates docs not reflective of behavior Fixes #1693 Signed-off-by: Dima Spivak --- docker/api/network.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/api/network.py b/docker/api/network.py index 5549bf0cf9..befbb583ce 100644 --- a/docker/api/network.py +++ b/docker/api/network.py @@ -52,7 +52,7 @@ def create_network(self, name, driver=None, options=None, ipam=None, options (dict): Driver options as a key-value dictionary ipam (IPAMConfig): Optional custom IP scheme for the network. check_duplicate (bool): Request daemon to check for networks with - same name. Default: ``True``. + same name. Default: ``None``. internal (bool): Restrict external access to the network. Default ``False``. labels (dict): Map of labels to set on the network. Default From bf9d06db251fc3befbdbe560bc03d2009ce927e7 Mon Sep 17 00:00:00 2001 From: Jakub Goszczurny Date: Mon, 3 Jul 2017 22:22:37 +0200 Subject: [PATCH 0421/1301] Generating regexp from .dockerignore file in a similar way as docker-ce. Signed-off-by: Jakub Goszczurny --- docker/utils/fnmatch.py | 27 +++++++++++++++++++-------- tests/unit/utils_test.py | 25 +++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 8 deletions(-) diff --git a/docker/utils/fnmatch.py b/docker/utils/fnmatch.py index e95b63ceb1..e51bd81552 100644 --- a/docker/utils/fnmatch.py +++ b/docker/utils/fnmatch.py @@ -65,19 +65,32 @@ def translate(pat): There is no way to quote meta-characters. """ - recursive_mode = False i, n = 0, len(pat) - res = '' + res = '^' while i < n: c = pat[i] i = i + 1 if c == '*': if i < n and pat[i] == '*': - recursive_mode = True + # is some flavor of "**" i = i + 1 - res = res + '.*' + # Treat **/ as ** so eat the "/" + if pat[i] == '/': + i = i + 1 + if i >= n: + # is "**EOF" - to align with .gitignore just accept all + res = res + '.*' + else: + # is "**" + # Note that this allows for any # of /'s (even 0) because + # the .* will eat everything, even /'s + res = res + '(.*/)?' + else: + # is "*" so map it to anything but "/" + res = res + '[^/]*' elif c == '?': - res = res + '.' + # "?" is any char except "/" + res = res + '[^/]' elif c == '[': j = i if j < n and pat[j] == '!': @@ -96,8 +109,6 @@ def translate(pat): elif stuff[0] == '^': stuff = '\\' + stuff res = '%s[%s]' % (res, stuff) - elif recursive_mode and c == '/': - res = res + re.escape(c) + '?' else: res = res + re.escape(c) - return res + '\Z(?ms)' + return res + '$' diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index a2d463d715..7045d23c27 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -639,6 +639,14 @@ class ExcludePathsTest(unittest.TestCase): 'foo', 'foo/bar', 'bar', + 'target', + 'target/subdir', + 'subdir', + 'subdir/target', + 'subdir/target/subdir', + 'subdir/subdir2', + 'subdir/subdir2/target', + 'subdir/subdir2/target/subdir' ] files = [ @@ -654,6 +662,14 @@ class ExcludePathsTest(unittest.TestCase): 'foo/bar/a.py', 'bar/a.py', 'foo/Dockerfile3', + 'target/file.txt', + 'target/subdir/file.txt', + 'subdir/file.txt', + 'subdir/target/file.txt', + 'subdir/target/subdir/file.txt', + 'subdir/subdir2/file.txt', + 'subdir/subdir2/target/file.txt', + 'subdir/subdir2/target/subdir/file.txt', ] all_paths = set(dirs + files) @@ -844,6 +860,15 @@ def test_double_wildcard(self): self.all_paths - set(['foo/bar', 'foo/bar/a.py']) ) + def test_single_and_double_wildcard(self): + assert self.exclude(['**/target/*/*']) == convert_paths( + self.all_paths - set( + ['target/subdir/file.txt', + 'subdir/target/subdir/file.txt', + 'subdir/subdir2/target/subdir/file.txt'] + ) + ) + class TarTest(unittest.TestCase): def test_tar_with_excludes(self): From bf15e27d6dcb0fe66c03dd0822fa1c88ee09e914 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 1 Aug 2017 13:49:48 -0700 Subject: [PATCH 0422/1301] Temporarily - do not run py33 tests on travis Signed-off-by: Joffrey F --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 6b48142f72..cd64b4456e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,8 @@ python: - "3.5" env: - TOX_ENV=py27 - - TOX_ENV=py33 +# FIXME: default travis worker does not carry py33 anymore. Can this be configured? +# - TOX_ENV=py33 - TOX_ENV=py34 - TOX_ENV=py35 - TOX_ENV=flake8 From f7e7a8564e6d40fe6b4dfc739db460c56245c63b Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Mon, 7 Aug 2017 09:14:21 -0500 Subject: [PATCH 0423/1301] Fix domainname documentation in create_container function It looks like this was probably originally copypasta'ed from dns_search and not edited afterward. Signed-off-by: Erik Johnson --- docker/api/container.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/api/container.py b/docker/api/container.py index 532a9c6d8b..06c575d5c0 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -399,7 +399,7 @@ def create_container(self, image, command=None, hostname=None, user=None, name (str): A name for the container entrypoint (str or list): An entrypoint working_dir (str): Path to the working directory - domainname (str or list): Set custom DNS search domains + domainname (str): The domain name to use for the container memswap_limit (int): host_config (dict): A dictionary created with :py:meth:`create_host_config`. From 48377d52e9fe1f76add3c91fe3d4ede1898be37a Mon Sep 17 00:00:00 2001 From: Andreas Backx Date: Sun, 6 Aug 2017 17:15:09 +0200 Subject: [PATCH 0424/1301] Added wait to the Container class documentation. The container class documentation did not automatically document the `Container.wait` method. Signed-off-by: Andreas Backx --- docs/containers.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/containers.rst b/docs/containers.rst index 6c895c6b2d..7c41bfdd92 100644 --- a/docs/containers.rst +++ b/docs/containers.rst @@ -53,3 +53,4 @@ Container objects .. automethod:: top .. automethod:: unpause .. automethod:: update + .. automethod:: wait From 62fda980e4159e54ea7920f174a2395101f964bf Mon Sep 17 00:00:00 2001 From: Artem Bolshakov Date: Tue, 25 Jul 2017 12:38:23 +0300 Subject: [PATCH 0425/1301] client.containers.run returns None if none of json-file or journald logging drivers used Signed-off-by: Artem Bolshakov --- docker/errors.py | 10 ++++-- docker/models/containers.py | 16 +++++++++- tests/integration/models_containers_test.py | 18 +++++++++++ tests/unit/errors_test.py | 34 ++++++++++++++++++++- tests/unit/fake_api.py | 6 ++++ 5 files changed, 80 insertions(+), 4 deletions(-) diff --git a/docker/errors.py b/docker/errors.py index 0da97f4e3f..1f8ac23c40 100644 --- a/docker/errors.py +++ b/docker/errors.py @@ -127,8 +127,14 @@ def __init__(self, container, exit_status, command, image, stderr): self.command = command self.image = image self.stderr = stderr - msg = ("Command '{}' in image '{}' returned non-zero exit status {}: " - "{}").format(command, image, exit_status, stderr) + + if stderr is None: + msg = ("Command '{}' in image '{}' returned non-zero exit " + "status {}").format(command, image, exit_status, stderr) + else: + msg = ("Command '{}' in image '{}' returned non-zero exit " + "status {}: {}").format(command, image, exit_status, stderr) + super(ContainerError, self).__init__(msg) diff --git a/docker/models/containers.py b/docker/models/containers.py index cf01b2750a..a3598f28f9 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -667,6 +667,13 @@ def run(self, image, command=None, stdout=True, stderr=False, The container logs, either ``STDOUT``, ``STDERR``, or both, depending on the value of the ``stdout`` and ``stderr`` arguments. + ``STDOUT`` and ``STDERR`` may be read only if either ``json-file`` + or ``journald`` logging driver used. Thus, if you are using none of + these drivers, a ``None`` object is returned instead. See the + `Engine API documentation + `_ + for full details. + If ``detach`` is ``True``, a :py:class:`Container` object is returned instead. @@ -709,7 +716,14 @@ def run(self, image, command=None, stdout=True, stderr=False, if exit_status != 0: stdout = False stderr = True - out = container.logs(stdout=stdout, stderr=stderr) + + logging_driver = container.attrs['HostConfig']['LogConfig']['Type'] + + if logging_driver == 'json-file' or logging_driver == 'journald': + out = container.logs(stdout=stdout, stderr=stderr) + else: + out = None + if remove: container.remove() if exit_status != 0: diff --git a/tests/integration/models_containers_test.py b/tests/integration/models_containers_test.py index b76a88ffcf..ce3349baa7 100644 --- a/tests/integration/models_containers_test.py +++ b/tests/integration/models_containers_test.py @@ -88,6 +88,24 @@ def test_run_with_network(self): assert 'Networks' in attrs['NetworkSettings'] assert list(attrs['NetworkSettings']['Networks'].keys()) == [net_name] + def test_run_with_none_driver(self): + client = docker.from_env(version=TEST_API_VERSION) + + out = client.containers.run( + "alpine", "echo hello", + log_config=dict(type='none') + ) + self.assertEqual(out, None) + + def test_run_with_json_file_driver(self): + client = docker.from_env(version=TEST_API_VERSION) + + out = client.containers.run( + "alpine", "echo hello", + log_config=dict(type='json-file') + ) + self.assertEqual(out, b'hello\n') + def test_get(self): client = docker.from_env(version=TEST_API_VERSION) container = client.containers.run("alpine", "sleep 300", detach=True) diff --git a/tests/unit/errors_test.py b/tests/unit/errors_test.py index b78af4e109..9678669c3f 100644 --- a/tests/unit/errors_test.py +++ b/tests/unit/errors_test.py @@ -2,8 +2,10 @@ import requests -from docker.errors import (APIError, DockerException, +from docker.errors import (APIError, ContainerError, DockerException, create_unexpected_kwargs_error) +from .fake_api import FAKE_CONTAINER_ID, FAKE_IMAGE_ID +from .fake_api_client import make_fake_client class APIErrorTest(unittest.TestCase): @@ -77,6 +79,36 @@ def test_is_client_error_400(self): assert err.is_client_error() is True +class ContainerErrorTest(unittest.TestCase): + def test_container_without_stderr(self): + """The massage does not contain stderr""" + client = make_fake_client() + container = client.containers.get(FAKE_CONTAINER_ID) + command = "echo Hello World" + exit_status = 42 + image = FAKE_IMAGE_ID + stderr = None + + err = ContainerError(container, exit_status, command, image, stderr) + msg = ("Command '{}' in image '{}' returned non-zero exit status {}" + ).format(command, image, exit_status, stderr) + assert str(err) == msg + + def test_container_with_stderr(self): + """The massage contains stderr""" + client = make_fake_client() + container = client.containers.get(FAKE_CONTAINER_ID) + command = "echo Hello World" + exit_status = 42 + image = FAKE_IMAGE_ID + stderr = "Something went wrong" + + err = ContainerError(container, exit_status, command, image, stderr) + msg = ("Command '{}' in image '{}' returned non-zero exit status {}: " + "{}").format(command, image, exit_status, stderr) + assert str(err) == msg + + class CreateUnexpectedKwargsErrorTest(unittest.TestCase): def test_create_unexpected_kwargs_error_single(self): e = create_unexpected_kwargs_error('f', {'foo': 'bar'}) diff --git a/tests/unit/fake_api.py b/tests/unit/fake_api.py index ff0f1b65cc..2ba85bbf53 100644 --- a/tests/unit/fake_api.py +++ b/tests/unit/fake_api.py @@ -146,6 +146,12 @@ def get_fake_inspect_container(tty=False): "StartedAt": "2013-09-25T14:01:18.869545111+02:00", "Ghost": False }, + "HostConfig": { + "LogConfig": { + "Type": "json-file", + "Config": {} + }, + }, "MacAddress": "02:42:ac:11:00:0a" } return status_code, response From f3374959b7145d0c25042d51065b5d387832d19c Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 7 Aug 2017 12:05:40 -0700 Subject: [PATCH 0426/1301] Improve ContainerError message compute Signed-off-by: Joffrey F --- docker/errors.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/docker/errors.py b/docker/errors.py index 1f8ac23c40..2a2f871e5d 100644 --- a/docker/errors.py +++ b/docker/errors.py @@ -128,12 +128,9 @@ def __init__(self, container, exit_status, command, image, stderr): self.image = image self.stderr = stderr - if stderr is None: - msg = ("Command '{}' in image '{}' returned non-zero exit " - "status {}").format(command, image, exit_status, stderr) - else: - msg = ("Command '{}' in image '{}' returned non-zero exit " - "status {}: {}").format(command, image, exit_status, stderr) + err = ": {}".format(stderr) if stderr is not None else "" + msg = ("Command '{}' in image '{}' returned non-zero exit " + "status {}{}").format(command, image, exit_status, err) super(ContainerError, self).__init__(msg) From 9e793806ff79559c3bc591d8c52a3bbe3cdb7350 Mon Sep 17 00:00:00 2001 From: Cecile Tonglet Date: Tue, 1 Aug 2017 12:16:56 +0200 Subject: [PATCH 0427/1301] Return the result of the API when using remove_image and load_image Those calls return result that can be used by the developers. Signed-off-by: Cecile Tonglet --- docker/api/image.py | 4 ++-- tests/integration/api_image_test.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docker/api/image.py b/docker/api/image.py index 181c4a1e4a..85ff435d44 100644 --- a/docker/api/image.py +++ b/docker/api/image.py @@ -272,7 +272,7 @@ def load_image(self, data): data (binary): Image data to be loaded. """ res = self._post(self._url("/images/load"), data=data) - self._raise_for_status(res) + return self._result(res, True) @utils.minimum_version('1.25') def prune_images(self, filters=None): @@ -455,7 +455,7 @@ def remove_image(self, image, force=False, noprune=False): """ params = {'force': force, 'noprune': noprune} res = self._delete(self._url("/images/{0}", image), params=params) - self._raise_for_status(res) + return self._result(res, True) def search(self, term): """ diff --git a/tests/integration/api_image_test.py b/tests/integration/api_image_test.py index 917bc50555..192e6f8d1b 100644 --- a/tests/integration/api_image_test.py +++ b/tests/integration/api_image_test.py @@ -113,7 +113,8 @@ def test_remove(self): self.assertIn('Id', res) img_id = res['Id'] self.tmp_imgs.append(img_id) - self.client.remove_image(img_id, force=True) + logs = self.client.remove_image(img_id, force=True) + self.assertIn({"Deleted": img_id}, logs) images = self.client.images(all=True) res = [x for x in images if x['Id'].startswith(img_id)] self.assertEqual(len(res), 0) From 7139e2d8f1ea82340417add02090bfaf7794f159 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 16 Jun 2017 14:05:54 -0700 Subject: [PATCH 0428/1301] Return generator for output of load_image endpoint Signed-off-by: Joffrey F --- Jenkinsfile | 5 ++--- docker/api/image.py | 29 ++++++++++++++++++++++++++--- docker/models/images.py | 3 +++ tests/integration/api_image_test.py | 13 +++++++++++++ tests/unit/api_image_test.py | 14 ++++++++++++++ 5 files changed, 58 insertions(+), 6 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 357927888c..9e1b4912a4 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -5,9 +5,8 @@ def imageNamePy2 def imageNamePy3 def images = [:] -// Note: Swarm in dind seem notoriously flimsy with 1.12.1+, which is why we're -// sticking with 1.12.0 for the 1.12 series -def dockerVersions = ["1.13.1", "17.04.0-ce", "17.05.0-ce", "17.06.0-ce-rc5"] + +def dockerVersions = ["1.13.1", "17.04.0-ce", "17.05.0-ce", "17.06.0-ce"] def buildImage = { name, buildargs, pyTag -> img = docker.image(name) diff --git a/docker/api/image.py b/docker/api/image.py index 85ff435d44..41cc267e9d 100644 --- a/docker/api/image.py +++ b/docker/api/image.py @@ -262,7 +262,7 @@ def inspect_image(self, image): self._get(self._url("/images/{0}/json", image)), True ) - def load_image(self, data): + def load_image(self, data, quiet=None): """ Load an image that was previously saved using :py:meth:`~docker.api.image.ImageApiMixin.get_image` (or ``docker @@ -270,9 +270,32 @@ def load_image(self, data): Args: data (binary): Image data to be loaded. + quiet (boolean): Suppress progress details in response. + + Returns: + (generator): Progress output as JSON objects. Only available for + API version >= 1.23 + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. """ - res = self._post(self._url("/images/load"), data=data) - return self._result(res, True) + params = {} + + if quiet is not None: + if utils.version_lt(self._version, '1.23'): + raise errors.InvalidVersion( + 'quiet is not supported in API version < 1.23' + ) + params['quiet'] = quiet + + res = self._post( + self._url("/images/load"), data=data, params=params, stream=True + ) + if utils.version_gte(self._version, '1.23'): + return self._stream_helper(res, decode=True) + + self._raise_for_status(res) @utils.minimum_version('1.25') def prune_images(self, filters=None): diff --git a/docker/models/images.py b/docker/models/images.py index d4e24c6060..3837929dfa 100644 --- a/docker/models/images.py +++ b/docker/models/images.py @@ -235,6 +235,9 @@ def load(self, data): Args: data (binary): Image data to be loaded. + Returns: + (generator): Progress output as JSON objects + Raises: :py:class:`docker.errors.APIError` If the server returns an error. diff --git a/tests/integration/api_image_test.py b/tests/integration/api_image_test.py index 192e6f8d1b..14fb77aa46 100644 --- a/tests/integration/api_image_test.py +++ b/tests/integration/api_image_test.py @@ -249,6 +249,19 @@ def test_import_image_with_changes(self): assert img_data['Config']['Cmd'] == ['echo'] assert img_data['Config']['User'] == 'foobar' + # Docs say output is available in 1.23, but this test fails on 1.12.0 + @requires_api_version('1.24') + def test_get_load_image(self): + test_img = 'hello-world:latest' + self.client.pull(test_img) + data = self.client.get_image(test_img) + assert data + output = self.client.load_image(data) + assert any([ + line for line in output + if 'Loaded image: {}'.format(test_img) in line.get('stream', '') + ]) + @contextlib.contextmanager def temporary_http_file_server(self, stream): '''Serve data from an IO stream over HTTP.''' diff --git a/tests/unit/api_image_test.py b/tests/unit/api_image_test.py index 36b2a46833..f1e42cc147 100644 --- a/tests/unit/api_image_test.py +++ b/tests/unit/api_image_test.py @@ -369,5 +369,19 @@ def test_load_image(self): 'POST', url_prefix + 'images/load', data='Byte Stream....', + stream=True, + params={}, + timeout=DEFAULT_TIMEOUT_SECONDS + ) + + def test_load_image_quiet(self): + self.client.load_image('Byte Stream....', quiet=True) + + fake_request.assert_called_with( + 'POST', + url_prefix + 'images/load', + data='Byte Stream....', + stream=True, + params={'quiet': True}, timeout=DEFAULT_TIMEOUT_SECONDS ) From 7f5739dc025af101e939b1403cf46a68fbc2dc97 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 15 Aug 2017 14:54:53 -0700 Subject: [PATCH 0429/1301] Leading slash in .dockerignore should be ignored Signed-off-by: Joffrey F --- docker/utils/build.py | 1 + tests/unit/utils_test.py | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/docker/utils/build.py b/docker/utils/build.py index 79b72495d9..d4223e749f 100644 --- a/docker/utils/build.py +++ b/docker/utils/build.py @@ -26,6 +26,7 @@ def exclude_paths(root, patterns, dockerfile=None): if dockerfile is None: dockerfile = 'Dockerfile' + patterns = [p.lstrip('/') for p in patterns] exceptions = [p for p in patterns if p.startswith('!')] include_patterns = [p[1:] for p in exceptions] diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index 7045d23c27..4a391facb7 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -768,6 +768,11 @@ def test_single_subdir_single_filename(self): self.all_paths - set(['foo/a.py']) ) + def test_single_subdir_single_filename_leading_slash(self): + assert self.exclude(['/foo/a.py']) == convert_paths( + self.all_paths - set(['foo/a.py']) + ) + def test_single_subdir_with_path_traversal(self): assert self.exclude(['foo/whoops/../a.py']) == convert_paths( self.all_paths - set(['foo/a.py']) From 0494c4f262cc489469e4c8d6356e825d849f9125 Mon Sep 17 00:00:00 2001 From: cyli Date: Thu, 13 Apr 2017 17:25:55 -0700 Subject: [PATCH 0430/1301] Require "requests[security]" if the `[tls]` option is selected, which also installs: pyOpenSSL, cryptography, idna and installs cryptography's version of openssl in Mac OS (which by default has an ancient version of openssl that doesn't support TLS 1.2). Signed-off-by: cyli --- README.md | 4 ++++ requirements.txt | 2 +- setup.py | 10 ++++++++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 747b98b250..3ff124d7a5 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,10 @@ The latest stable version [is available on PyPI](https://pypi.python.org/pypi/do pip install docker +If you are intending to connect to a docker host via TLS, add `docker[tls]` to your requirements instead, or install with pip: + + pip install docker[tls] + ## Usage Connect to Docker using the default socket or the configuration in your environment: diff --git a/requirements.txt b/requirements.txt index 375413122b..423ffb7004 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -requests==2.11.1 +requests[security]==2.11.1 six>=1.4.0 websocket-client==0.32.0 backports.ssl_match_hostname>=3.5 ; python_version < '3.5' diff --git a/setup.py b/setup.py index 31180d2397..534c9495da 100644 --- a/setup.py +++ b/setup.py @@ -35,6 +35,16 @@ # ssl_match_hostname to verify hosts match with certificates via # ServerAltname: https://pypi.python.org/pypi/backports.ssl_match_hostname ':python_version < "3.3"': 'ipaddress >= 1.0.16', + + # If using docker-py over TLS, highly recommend this option is pip-installed + # or pinned. + + # TODO: if pip installign both "requests" and "requests[security]", the + # extra package from the "security" option are not installed (see + # https://github.com/pypa/pip/issues/4391). Once that's fixed, instead of + # installing the extra dependencies, install the following instead: + # 'requests[security] >= 2.5.2, != 2.11.0, != 2.12.2' + 'tls': ['pyOpenSSL>=0.14', 'cryptography>=1.3.4', 'idna>=2.0.0'], } version = None From 380914aaaa2cd3ebb0f2367f214d5de6c8eada89 Mon Sep 17 00:00:00 2001 From: cyli Date: Mon, 22 May 2017 15:45:06 -0700 Subject: [PATCH 0431/1301] If we're pinning exact versions of things for requirements.txt, pin all dependencies of dependencies as well so we can get a consistent build. Signed-off-by: cyli --- requirements.txt | 20 +++++++++++++++----- setup.py | 6 +++--- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/requirements.txt b/requirements.txt index 423ffb7004..f3c61e790b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,16 @@ -requests[security]==2.11.1 -six>=1.4.0 -websocket-client==0.32.0 -backports.ssl_match_hostname>=3.5 ; python_version < '3.5' -ipaddress==1.0.16 ; python_version < '3.3' +appdirs==1.4.3 +asn1crypto==0.22.0 +backports.ssl-match-hostname==3.5.0.1 +cffi==1.10.0 +cryptography==1.9 docker-pycreds==0.2.1 +enum34==1.1.6 +idna==2.5 +ipaddress==1.0.18 +packaging==16.8 +pycparser==2.17 +pyOpenSSL==17.0.0 +pyparsing==2.2.0 +requests==2.14.2 +six==1.10.0 +websocket-client==0.40.0 diff --git a/setup.py b/setup.py index 534c9495da..4a33c8df02 100644 --- a/setup.py +++ b/setup.py @@ -36,10 +36,10 @@ # ServerAltname: https://pypi.python.org/pypi/backports.ssl_match_hostname ':python_version < "3.3"': 'ipaddress >= 1.0.16', - # If using docker-py over TLS, highly recommend this option is pip-installed - # or pinned. + # If using docker-py over TLS, highly recommend this option is + # pip-installed or pinned. - # TODO: if pip installign both "requests" and "requests[security]", the + # TODO: if pip installing both "requests" and "requests[security]", the # extra package from the "security" option are not installed (see # https://github.com/pypa/pip/issues/4391). Once that's fixed, instead of # installing the extra dependencies, install the following instead: From b54c76c3c1e89390d27f816eed97afbd2c3b9bf8 Mon Sep 17 00:00:00 2001 From: Ying Date: Fri, 16 Jun 2017 18:46:09 -0700 Subject: [PATCH 0432/1301] Upgrade tox and virtualenv in appveyor to make sure we have the latest pip. Signed-off-by: Ying --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 1fc67cc024..41cde6252b 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -3,7 +3,7 @@ version: '{branch}-{build}' install: - "SET PATH=C:\\Python27-x64;C:\\Python27-x64\\Scripts;%PATH%" - "python --version" - - "pip install tox==2.1.1 virtualenv==13.1.2" + - "pip install tox==2.7.0 virtualenv==15.1.0" # Build the binary after tests build: false From d49c136d042db105a20053ed933776534ff8f5b3 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 15 Aug 2017 15:38:09 -0700 Subject: [PATCH 0433/1301] Daemon expects full URL of hub in auth config dict in build payload Signed-off-by: Joffrey F --- docker/api/build.py | 5 ++++- docker/auth.py | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/docker/api/build.py b/docker/api/build.py index cbef4a8b17..5d4e7720a1 100644 --- a/docker/api/build.py +++ b/docker/api/build.py @@ -274,7 +274,10 @@ def _set_auth_headers(self, headers): self._auth_configs, registry ) else: - auth_data = self._auth_configs + auth_data = self._auth_configs.copy() + # See https://github.com/docker/docker-py/issues/1683 + if auth.INDEX_NAME in auth_data: + auth_data[auth.INDEX_URL] = auth_data[auth.INDEX_NAME] log.debug( 'Sending auth config ({0})'.format( diff --git a/docker/auth.py b/docker/auth.py index ec9c45b97d..c3fb062e9b 100644 --- a/docker/auth.py +++ b/docker/auth.py @@ -10,7 +10,7 @@ from .constants import IS_WINDOWS_PLATFORM INDEX_NAME = 'docker.io' -INDEX_URL = 'https://{0}/v1/'.format(INDEX_NAME) +INDEX_URL = 'https://index.{0}/v1/'.format(INDEX_NAME) DOCKER_CONFIG_FILENAME = os.path.join('.docker', 'config.json') LEGACY_DOCKER_CONFIG_FILENAME = '.dockercfg' TOKEN_USERNAME = '' @@ -118,7 +118,7 @@ def _resolve_authconfig_credstore(authconfig, registry, credstore_name): if not registry or registry == INDEX_NAME: # The ecosystem is a little schizophrenic with index.docker.io VS # docker.io - in that case, it seems the full URL is necessary. - registry = 'https://index.docker.io/v1/' + registry = INDEX_URL log.debug("Looking for auth entry for {0}".format(repr(registry))) store = dockerpycreds.Store(credstore_name) try: From b4802ea12626bb9d987c34f679ac04d97a402f9f Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 15 Aug 2017 16:07:47 -0700 Subject: [PATCH 0434/1301] Handle untyped ContainerSpec dict in _check_api_features Signed-off-by: Joffrey F --- docker/api/service.py | 2 +- tests/integration/api_service_test.py | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/docker/api/service.py b/docker/api/service.py index cc16cc37dd..4b555a5f59 100644 --- a/docker/api/service.py +++ b/docker/api/service.py @@ -38,7 +38,7 @@ def _check_api_features(version, task_template, update_config): 'Placement.preferences is not supported in' ' API version < 1.27' ) - if task_template.container_spec.get('TTY'): + if task_template.get('ContainerSpec', {}).get('TTY'): if utils.version_lt(version, '1.25'): raise errors.InvalidVersion( 'ContainerSpec.TTY is not supported in API version < 1.25' diff --git a/tests/integration/api_service_test.py b/tests/integration/api_service_test.py index 54111a7bb1..c966916ebb 100644 --- a/tests/integration/api_service_test.py +++ b/tests/integration/api_service_test.py @@ -376,6 +376,23 @@ def test_create_service_with_tty(self): assert 'TTY' in con_spec assert con_spec['TTY'] is True + @requires_api_version('1.25') + def test_create_service_with_tty_dict(self): + container_spec = { + 'Image': BUSYBOX, + 'Command': ['true'], + 'TTY': True + } + task_tmpl = docker.types.TaskTemplate(container_spec) + name = self.get_service_name() + svc_id = self.client.create_service(task_tmpl, name=name) + svc_info = self.client.inspect_service(svc_id) + assert 'TaskTemplate' in svc_info['Spec'] + assert 'ContainerSpec' in svc_info['Spec']['TaskTemplate'] + con_spec = svc_info['Spec']['TaskTemplate']['ContainerSpec'] + assert 'TTY' in con_spec + assert con_spec['TTY'] is True + def test_create_service_global_mode(self): container_spec = docker.types.ContainerSpec( BUSYBOX, ['echo', 'hello'] From 2a6926b5aba00f83c8c1702f5dde3d5eaa855d29 Mon Sep 17 00:00:00 2001 From: adrianliaw Date: Sat, 6 May 2017 19:29:39 +0800 Subject: [PATCH 0435/1301] Use collection's get method inside ImageCollection's list method Signed-off-by: Adrian Liaw --- docker/models/images.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/models/images.py b/docker/models/images.py index 3837929dfa..d1b29ad8a6 100644 --- a/docker/models/images.py +++ b/docker/models/images.py @@ -224,7 +224,7 @@ def list(self, name=None, all=False, filters=None): If the server returns an error. """ resp = self.client.api.images(name=name, all=all, filters=filters) - return [self.prepare_model(r) for r in resp] + return [self.get(r["Id"]) for r in resp] def load(self, data): """ From 6b59dc62715a1e387543de02ade80de84aa2171c Mon Sep 17 00:00:00 2001 From: David Steines Date: Mon, 3 Apr 2017 21:58:59 -0400 Subject: [PATCH 0436/1301] Allow detach and remove for api version >= 1.25 and use auto_remove when both are set. Continue raising an exception for api versions <1.25. Signed-off-by: David Steines --- docker/models/containers.py | 10 ++++++++-- tests/unit/models_containers_test.py | 30 ++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/docker/models/containers.py b/docker/models/containers.py index a3598f28f9..d9db79dfba 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -4,6 +4,7 @@ from ..errors import (ContainerError, ImageNotFound, create_unexpected_kwargs_error) from ..types import HostConfig +from ..utils import compare_version from .images import Image from .resource import Collection, Model @@ -690,8 +691,12 @@ def run(self, image, command=None, stdout=True, stderr=False, image = image.id detach = kwargs.pop("detach", False) if detach and remove: - raise RuntimeError("The options 'detach' and 'remove' cannot be " - "used together.") + if compare_version("1.24", + self.client.api._version) > 0: + kwargs["auto_remove"] = True + else: + raise RuntimeError("The options 'detach' and 'remove' cannot " + "be used together in api versions < 1.25.") if kwargs.get('network') and kwargs.get('network_mode'): raise RuntimeError( @@ -849,6 +854,7 @@ def prune(self, filters=None): # kwargs to copy straight from run to host_config RUN_HOST_CONFIG_KWARGS = [ + 'auto_remove', 'blkio_weight_device', 'blkio_weight', 'cap_add', diff --git a/tests/unit/models_containers_test.py b/tests/unit/models_containers_test.py index 70c86480c8..5eaa45ac66 100644 --- a/tests/unit/models_containers_test.py +++ b/tests/unit/models_containers_test.py @@ -273,9 +273,39 @@ def test_run_remove(self): client.api.remove_container.assert_called_with(FAKE_CONTAINER_ID) client = make_fake_client() + client.api._version = '1.24' with self.assertRaises(RuntimeError): client.containers.run("alpine", detach=True, remove=True) + client = make_fake_client() + client.api._version = '1.23' + with self.assertRaises(RuntimeError): + client.containers.run("alpine", detach=True, remove=True) + + client = make_fake_client() + client.api._version = '1.25' + client.containers.run("alpine", detach=True, remove=True) + client.api.remove_container.assert_not_called() + client.api.create_container.assert_called_with( + command=None, + image='alpine', + detach=True, + host_config={'AutoRemove': True, + 'NetworkMode': 'default'} + ) + + client = make_fake_client() + client.api._version = '1.26' + client.containers.run("alpine", detach=True, remove=True) + client.api.remove_container.assert_not_called() + client.api.create_container.assert_called_with( + command=None, + image='alpine', + detach=True, + host_config={'AutoRemove': True, + 'NetworkMode': 'default'} + ) + def test_create(self): client = make_fake_client() container = client.containers.create( From d5c4ce203aa5839966a221079d1be44e572e92af Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 15 Aug 2017 17:40:07 -0700 Subject: [PATCH 0437/1301] Use better version comparison function Signed-off-by: Joffrey F --- docker/models/containers.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docker/models/containers.py b/docker/models/containers.py index d9db79dfba..688deccadc 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -4,7 +4,7 @@ from ..errors import (ContainerError, ImageNotFound, create_unexpected_kwargs_error) from ..types import HostConfig -from ..utils import compare_version +from ..utils import version_gte from .images import Image from .resource import Collection, Model @@ -691,8 +691,7 @@ def run(self, image, command=None, stdout=True, stderr=False, image = image.id detach = kwargs.pop("detach", False) if detach and remove: - if compare_version("1.24", - self.client.api._version) > 0: + if version_gte(self.client.api._version, '1.25'): kwargs["auto_remove"] = True else: raise RuntimeError("The options 'detach' and 'remove' cannot " From a6065df64d848a0fc1ce9f2638b7b2a33f407145 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gabriel=20F=C3=A9ron?= Date: Mon, 30 Jan 2017 19:06:20 +0100 Subject: [PATCH 0438/1301] Add support for the `squash` flag when building MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Also added a test that compares the number of layers in the default mode, and with the new flag Signed-off-by: Gabriel Féron --- docker/api/build.py | 13 ++++++++++++- tests/integration/api_build_test.py | 28 +++++++++++++++++++++++++++- 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/docker/api/build.py b/docker/api/build.py index 5d4e7720a1..f9678a390a 100644 --- a/docker/api/build.py +++ b/docker/api/build.py @@ -18,7 +18,8 @@ def build(self, path=None, tag=None, quiet=False, fileobj=None, custom_context=False, encoding=None, pull=False, forcerm=False, dockerfile=None, container_limits=None, decode=False, buildargs=None, gzip=False, shmsize=None, - labels=None, cache_from=None, target=None, network_mode=None): + labels=None, cache_from=None, target=None, network_mode=None, + squash=None): """ Similar to the ``docker build`` command. Either ``path`` or ``fileobj`` needs to be set. ``path`` can be a local path (to a directory @@ -98,6 +99,8 @@ def build(self, path=None, tag=None, quiet=False, fileobj=None, Dockerfile network_mode (str): networking mode for the run commands during build + squash (bool): Squash the resulting images layers into a + single layer. Returns: A generator for the build output. @@ -218,6 +221,14 @@ def build(self, path=None, tag=None, quiet=False, fileobj=None, 'network_mode was only introduced in API version 1.25' ) + if squash: + if utils.version_gte(self._version, '1.25'): + params.update({'squash': squash}) + else: + raise errors.InvalidVersion( + 'squash was only introduced in API version 1.25' + ) + if context is not None: headers = {'Content-Type': 'application/tar'} if encoding: diff --git a/tests/integration/api_build_test.py b/tests/integration/api_build_test.py index 609964f0b0..209c1f28b2 100644 --- a/tests/integration/api_build_test.py +++ b/tests/integration/api_build_test.py @@ -9,7 +9,7 @@ import six from .base import BaseAPIIntegrationTest -from ..helpers import requires_api_version +from ..helpers import requires_api_version, requires_experimental class BuildTest(BaseAPIIntegrationTest): @@ -244,6 +244,32 @@ def test_build_with_network_mode(self): with pytest.raises(errors.NotFound): self.client.inspect_image('dockerpytest_nonebuild') + @requires_api_version('1.25') + @requires_experimental + def test_build_squash(self): + script = io.BytesIO('\n'.join([ + 'FROM busybox', + 'RUN echo blah > /file_1', + 'RUN echo blahblah > /file_2', + 'RUN echo blahblahblah > /file_3' + ]).encode('ascii')) + + def build_squashed(squash): + tag = 'squash' if squash else 'nosquash' + stream = self.client.build( + fileobj=script, tag=tag, squash=squash + ) + self.tmp_imgs.append(tag) + for chunk in stream: + pass + + return self.client.inspect_image(tag) + + non_squashed = build_squashed(False) + squashed = build_squashed(True) + self.assertEqual(len(non_squashed['RootFS']['Layers']), 4) + self.assertEqual(len(squashed['RootFS']['Layers']), 2) + def test_build_stderr_data(self): control_chars = ['\x1b[91m', '\x1b[0m'] snippet = 'Ancient Temple (Mystic Oriental Dream ~ Ancient Temple)' From d9df2a8b75d8a36691a15cdb27213b1db5fa4a61 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 16 Aug 2017 17:31:36 -0700 Subject: [PATCH 0439/1301] Fix handling of non-multiplexed (TTY) streams over upgraded sockets Signed-off-by: Joffrey F --- docker/api/client.py | 22 ++++++++++++++++------ docker/api/container.py | 4 +++- docker/api/exec_api.py | 2 +- docker/utils/socket.py | 21 ++++++++++++++++++++- tests/integration/api_build_test.py | 2 +- tests/unit/api_test.py | 2 +- 6 files changed, 42 insertions(+), 11 deletions(-) diff --git a/docker/api/client.py b/docker/api/client.py index 65b5baa967..1de10c77c0 100644 --- a/docker/api/client.py +++ b/docker/api/client.py @@ -32,7 +32,7 @@ from ..tls import TLSConfig from ..transport import SSLAdapter, UnixAdapter from ..utils import utils, check_resource, update_headers -from ..utils.socket import frames_iter +from ..utils.socket import frames_iter, socket_raw_iter from ..utils.json_stream import json_stream try: from ..transport import NpipeAdapter @@ -362,13 +362,19 @@ def _stream_raw_result(self, response): for out in response.iter_content(chunk_size=1, decode_unicode=True): yield out - def _read_from_socket(self, response, stream): + def _read_from_socket(self, response, stream, tty=False): socket = self._get_raw_response_socket(response) + gen = None + if tty is False: + gen = frames_iter(socket) + else: + gen = socket_raw_iter(socket) + if stream: - return frames_iter(socket) + return gen else: - return six.binary_type().join(frames_iter(socket)) + return six.binary_type().join(gen) def _disable_socket_timeout(self, socket): """ Depending on the combination of python version and whether we're @@ -398,9 +404,13 @@ def _disable_socket_timeout(self, socket): s.settimeout(None) - def _get_result(self, container, stream, res): + @check_resource('container') + def _check_is_tty(self, container): cont = self.inspect_container(container) - return self._get_result_tty(stream, res, cont['Config']['Tty']) + return cont['Config']['Tty'] + + def _get_result(self, container, stream, res): + return self._get_result_tty(stream, res, self._check_is_tty(container)) def _get_result_tty(self, stream, res, is_tty): # Stream multi-plexing was only introduced in API v1.6. Anything diff --git a/docker/api/container.py b/docker/api/container.py index 06c575d5c0..dde1325422 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -52,7 +52,9 @@ def attach(self, container, stdout=True, stderr=True, u = self._url("/containers/{0}/attach", container) response = self._post(u, headers=headers, params=params, stream=stream) - return self._read_from_socket(response, stream) + return self._read_from_socket( + response, stream, self._check_is_tty(container) + ) @utils.check_resource('container') def attach_socket(self, container, params=None, ws=False): diff --git a/docker/api/exec_api.py b/docker/api/exec_api.py index 2b407cef40..6f42524e65 100644 --- a/docker/api/exec_api.py +++ b/docker/api/exec_api.py @@ -153,4 +153,4 @@ def exec_start(self, exec_id, detach=False, tty=False, stream=False, return self._result(res) if socket: return self._get_raw_response_socket(res) - return self._read_from_socket(res, stream) + return self._read_from_socket(res, stream, tty) diff --git a/docker/utils/socket.py b/docker/utils/socket.py index 4080f253f5..54392d2b74 100644 --- a/docker/utils/socket.py +++ b/docker/utils/socket.py @@ -75,5 +75,24 @@ def frames_iter(socket): break while n > 0: result = read(socket, n) - n -= len(result) + if result is None: + continue + data_length = len(result) + if data_length == 0: + # We have reached EOF + return + n -= data_length yield result + + +def socket_raw_iter(socket): + """ + Returns a generator of data read from the socket. + This is used for non-multiplexed streams. + """ + while True: + result = read(socket) + if len(result) == 0: + # We have reached EOF + return + yield result diff --git a/tests/integration/api_build_test.py b/tests/integration/api_build_test.py index 209c1f28b2..d0aa5c213c 100644 --- a/tests/integration/api_build_test.py +++ b/tests/integration/api_build_test.py @@ -244,8 +244,8 @@ def test_build_with_network_mode(self): with pytest.raises(errors.NotFound): self.client.inspect_image('dockerpytest_nonebuild') + @requires_experimental(until=None) @requires_api_version('1.25') - @requires_experimental def test_build_squash(self): script = io.BytesIO('\n'.join([ 'FROM busybox', diff --git a/tests/unit/api_test.py b/tests/unit/api_test.py index 83848c524a..6ac92c4076 100644 --- a/tests/unit/api_test.py +++ b/tests/unit/api_test.py @@ -83,7 +83,7 @@ def fake_delete(self, url, *args, **kwargs): return fake_request('DELETE', url, *args, **kwargs) -def fake_read_from_socket(self, response, stream): +def fake_read_from_socket(self, response, stream, tty=False): return six.binary_type() From 921aba107b75b4571d8e7391d199011e07048b8a Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 17 Aug 2017 12:38:24 -0700 Subject: [PATCH 0440/1301] Update test versions Signed-off-by: Joffrey F --- Jenkinsfile | 4 ++-- Makefile | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 9e1b4912a4..6fa2728946 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -6,7 +6,7 @@ def imageNamePy3 def images = [:] -def dockerVersions = ["1.13.1", "17.04.0-ce", "17.05.0-ce", "17.06.0-ce"] +def dockerVersions = ["1.13.1", "17.04.0-ce", "17.05.0-ce", "17.06.1-ce", "17.07.0-ce-rc3"] def buildImage = { name, buildargs, pyTag -> img = docker.image(name) @@ -34,7 +34,7 @@ def buildImages = { -> } def getAPIVersion = { engineVersion -> - def versionMap = ['1.13.': '1.26', '17.04': '1.27', '17.05': '1.29', '17.06': '1.30'] + def versionMap = ['1.13.': '1.26', '17.04': '1.27', '17.05': '1.29', '17.06': '1.30', '17.07': '1.31'] return versionMap[engineVersion.substring(0, 5)] } diff --git a/Makefile b/Makefile index e4cd3f7b88..c6c6d56cda 100644 --- a/Makefile +++ b/Makefile @@ -41,8 +41,8 @@ integration-test: build integration-test-py3: build-py3 docker run --rm -v /var/run/docker.sock:/var/run/docker.sock docker-sdk-python3 py.test tests/integration/${file} -TEST_API_VERSION ?= 1.29 -TEST_ENGINE_VERSION ?= 17.05.0-ce +TEST_API_VERSION ?= 1.30 +TEST_ENGINE_VERSION ?= 17.06.0-ce .PHONY: integration-dind integration-dind: build build-py3 From 7d559a957c5908a3cc2b7bee3336869b33d87107 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 17 Aug 2017 12:38:36 -0700 Subject: [PATCH 0441/1301] Update default API version Signed-off-by: Joffrey F --- docker/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/constants.py b/docker/constants.py index 91a65282a1..6de8fad632 100644 --- a/docker/constants.py +++ b/docker/constants.py @@ -1,7 +1,7 @@ import sys from .version import version -DEFAULT_DOCKER_API_VERSION = '1.26' +DEFAULT_DOCKER_API_VERSION = '1.30' MINIMUM_DOCKER_API_VERSION = '1.21' DEFAULT_TIMEOUT_SECONDS = 60 STREAM_HEADER_SIZE_BYTES = 8 From 9b6ff333ac0e1fbebb8fe4881d29b36c07f15a51 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 17 Aug 2017 12:38:52 -0700 Subject: [PATCH 0442/1301] Bump 2.5.0 Signed-off-by: Joffrey F --- Jenkinsfile | 2 +- docker/version.py | 2 +- docs/change-log.md | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 36 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 6fa2728946..a83d7bf1a7 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -6,7 +6,7 @@ def imageNamePy3 def images = [:] -def dockerVersions = ["1.13.1", "17.04.0-ce", "17.05.0-ce", "17.06.1-ce", "17.07.0-ce-rc3"] +def dockerVersions = ["1.13.1", "17.04.0-ce", "17.05.0-ce", "17.06.0-ce", "17.07.0-ce-rc3"] def buildImage = { name, buildargs, pyTag -> img = docker.image(name) diff --git a/docker/version.py b/docker/version.py index a7452d4f13..066b62e746 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "2.5.0-dev" +version = "2.5.0" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) diff --git a/docs/change-log.md b/docs/change-log.md index 7099d7942e..894cd1e313 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -1,6 +1,40 @@ Change log ========== +2.5.0 +----- + +[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/33?closed=1) + +### Features + +* Added support for the `squash` parameter in `APIClient.build` and + `DockerClient.images.build`. +* When using API version 1.23 or above, `load_image` will now return a + generator of progress as JSON `dict`s. +* `remove_image` now returns the content of the API's response. + + +### Bugfixes + +* Fixed an issue where the `auto_remove` parameter in + `DockerClient.containers.run` was not taken into account. +* Fixed a bug where `.dockerignore` patterns starting with a slash + were ignored. +* Fixed an issue with the handling of `**` patterns in `.dockerignore` +* Fixed a bug where building `FROM` a private Docker Hub image when not + using a cred store would fail. +* Fixed a bug where calling `create_service` or `update_service` with + `task_template` as a `dict` would raise an exception. +* Fixed the handling of TTY-enabled containers in `attach` and `exec_run`. +* `DockerClient.containers.run` will no longer attempt to stream logs if the + log driver doesn't support the operation. + +### Miscellaneous + +* Added extra requirements for better TLS support on some platforms. + These can be installed or required through the `docker[tls]` notation. + 2.4.2 ----- From 8d14709c1804b3803351c1a6509820eaab52b6ef Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 17 Aug 2017 13:50:20 -0700 Subject: [PATCH 0443/1301] Changelog typo Signed-off-by: Joffrey F --- docs/change-log.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/change-log.md b/docs/change-log.md index 894cd1e313..199e5ce816 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -4,7 +4,7 @@ Change log 2.5.0 ----- -[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/33?closed=1) +[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/34?closed=1) ### Features From 477f236c71b4892e876337c79f1b88be1ad9f570 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 17 Aug 2017 13:52:13 -0700 Subject: [PATCH 0444/1301] Bump 2.6.0-dev Signed-off-by: Joffrey F --- docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index 066b62e746..3b899fb539 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "2.5.0" +version = "2.6.0-dev" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) From 3df0653493d5f2ff5e17de8211a32aed8f4c1965 Mon Sep 17 00:00:00 2001 From: Veli-Matti Lintu Date: Fri, 18 Aug 2017 11:48:09 +0300 Subject: [PATCH 0445/1301] Commit d798afca made changes for the handling of '**' patterns in .dockerignore. This causes an IndexError with patterns ending with '**', e.g. 'subdir/**'. This adds a missing boundary check before checking for trailing '/'. Signed-off-by: Veli-Matti Lintu --- docker/utils/fnmatch.py | 2 +- tests/unit/utils_test.py | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/docker/utils/fnmatch.py b/docker/utils/fnmatch.py index e51bd81552..42461dd7d3 100644 --- a/docker/utils/fnmatch.py +++ b/docker/utils/fnmatch.py @@ -75,7 +75,7 @@ def translate(pat): # is some flavor of "**" i = i + 1 # Treat **/ as ** so eat the "/" - if pat[i] == '/': + if i < n and pat[i] == '/': i = i + 1 if i >= n: # is "**EOF" - to align with .gitignore just accept all diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index 4a391facb7..2fa1d051f2 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -874,6 +874,23 @@ def test_single_and_double_wildcard(self): ) ) + def test_trailing_double_wildcard(self): + assert self.exclude(['subdir/**']) == convert_paths( + self.all_paths - set( + ['subdir/file.txt', + 'subdir/target/file.txt', + 'subdir/target/subdir/file.txt', + 'subdir/subdir2/file.txt', + 'subdir/subdir2/target/file.txt', + 'subdir/subdir2/target/subdir/file.txt', + 'subdir/target', + 'subdir/target/subdir', + 'subdir/subdir2', + 'subdir/subdir2/target', + 'subdir/subdir2/target/subdir'] + ) + ) + class TarTest(unittest.TestCase): def test_tar_with_excludes(self): From 89195146adde47181156b0c204c72b790c69cf85 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 21 Aug 2017 14:41:10 -0700 Subject: [PATCH 0446/1301] Always send attach request as streaming Signed-off-by: Joffrey F --- docker/api/container.py | 2 +- tests/integration/api_container_test.py | 16 ++++++++++++---- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/docker/api/container.py b/docker/api/container.py index dde1325422..918f8a3a90 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -50,7 +50,7 @@ def attach(self, container, stdout=True, stderr=True, } u = self._url("/containers/{0}/attach", container) - response = self._post(u, headers=headers, params=params, stream=stream) + response = self._post(u, headers=headers, params=params, stream=True) return self._read_from_socket( response, stream, self._check_is_tty(container) diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index f8b474a113..a972c1cdca 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -1092,20 +1092,28 @@ def test_run_container_reading_socket(self): command = "printf '{0}'".format(line) container = self.client.create_container(BUSYBOX, command, detach=True, tty=False) - ident = container['Id'] - self.tmp_containers.append(ident) + self.tmp_containers.append(container) opts = {"stdout": 1, "stream": 1, "logs": 1} - pty_stdout = self.client.attach_socket(ident, opts) + pty_stdout = self.client.attach_socket(container, opts) self.addCleanup(pty_stdout.close) - self.client.start(ident) + self.client.start(container) next_size = next_frame_size(pty_stdout) self.assertEqual(next_size, len(line)) data = read_exactly(pty_stdout, next_size) self.assertEqual(data.decode('utf-8'), line) + def test_attach_no_stream(self): + container = self.client.create_container( + BUSYBOX, 'echo hello' + ) + self.tmp_containers.append(container) + self.client.start(container) + output = self.client.attach(container, stream=False, logs=True) + assert output == 'hello\n'.encode(encoding='ascii') + class PauseTest(BaseAPIIntegrationTest): def test_pause_unpause(self): From ba7580d6b94324b7fe97e90e3f40693a12f5ce2c Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 17 Aug 2017 13:52:13 -0700 Subject: [PATCH 0447/1301] Bump 2.6.0-dev Signed-off-by: Joffrey F --- docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index 066b62e746..3b899fb539 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "2.5.0" +version = "2.6.0-dev" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) From fc6773d6732a433bda71dde24712584b7885deb8 Mon Sep 17 00:00:00 2001 From: Veli-Matti Lintu Date: Fri, 18 Aug 2017 11:48:09 +0300 Subject: [PATCH 0448/1301] Commit d798afca made changes for the handling of '**' patterns in .dockerignore. This causes an IndexError with patterns ending with '**', e.g. 'subdir/**'. This adds a missing boundary check before checking for trailing '/'. Signed-off-by: Veli-Matti Lintu --- docker/utils/fnmatch.py | 2 +- tests/unit/utils_test.py | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/docker/utils/fnmatch.py b/docker/utils/fnmatch.py index e51bd81552..42461dd7d3 100644 --- a/docker/utils/fnmatch.py +++ b/docker/utils/fnmatch.py @@ -75,7 +75,7 @@ def translate(pat): # is some flavor of "**" i = i + 1 # Treat **/ as ** so eat the "/" - if pat[i] == '/': + if i < n and pat[i] == '/': i = i + 1 if i >= n: # is "**EOF" - to align with .gitignore just accept all diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index 4a391facb7..2fa1d051f2 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -874,6 +874,23 @@ def test_single_and_double_wildcard(self): ) ) + def test_trailing_double_wildcard(self): + assert self.exclude(['subdir/**']) == convert_paths( + self.all_paths - set( + ['subdir/file.txt', + 'subdir/target/file.txt', + 'subdir/target/subdir/file.txt', + 'subdir/subdir2/file.txt', + 'subdir/subdir2/target/file.txt', + 'subdir/subdir2/target/subdir/file.txt', + 'subdir/target', + 'subdir/target/subdir', + 'subdir/subdir2', + 'subdir/subdir2/target', + 'subdir/subdir2/target/subdir'] + ) + ) + class TarTest(unittest.TestCase): def test_tar_with_excludes(self): From 0c2b4e4d3aa15efbf2cfac68571e7506ea86b8c9 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 21 Aug 2017 14:41:10 -0700 Subject: [PATCH 0449/1301] Always send attach request as streaming Signed-off-by: Joffrey F --- docker/api/container.py | 2 +- tests/integration/api_container_test.py | 16 ++++++++++++---- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/docker/api/container.py b/docker/api/container.py index dde1325422..918f8a3a90 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -50,7 +50,7 @@ def attach(self, container, stdout=True, stderr=True, } u = self._url("/containers/{0}/attach", container) - response = self._post(u, headers=headers, params=params, stream=stream) + response = self._post(u, headers=headers, params=params, stream=True) return self._read_from_socket( response, stream, self._check_is_tty(container) diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index f8b474a113..a972c1cdca 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -1092,20 +1092,28 @@ def test_run_container_reading_socket(self): command = "printf '{0}'".format(line) container = self.client.create_container(BUSYBOX, command, detach=True, tty=False) - ident = container['Id'] - self.tmp_containers.append(ident) + self.tmp_containers.append(container) opts = {"stdout": 1, "stream": 1, "logs": 1} - pty_stdout = self.client.attach_socket(ident, opts) + pty_stdout = self.client.attach_socket(container, opts) self.addCleanup(pty_stdout.close) - self.client.start(ident) + self.client.start(container) next_size = next_frame_size(pty_stdout) self.assertEqual(next_size, len(line)) data = read_exactly(pty_stdout, next_size) self.assertEqual(data.decode('utf-8'), line) + def test_attach_no_stream(self): + container = self.client.create_container( + BUSYBOX, 'echo hello' + ) + self.tmp_containers.append(container) + self.client.start(container) + output = self.client.attach(container, stream=False, logs=True) + assert output == 'hello\n'.encode(encoding='ascii') + class PauseTest(BaseAPIIntegrationTest): def test_pause_unpause(self): From e9fe07768145d848fd67c8d84b7766bd39ec5e38 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 22 Aug 2017 16:38:15 -0700 Subject: [PATCH 0450/1301] Bump 2.5.1 Signed-off-by: Joffrey F --- docker/version.py | 2 +- docs/change-log.md | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index 3b899fb539..273270d599 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "2.6.0-dev" +version = "2.5.1" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) diff --git a/docs/change-log.md b/docs/change-log.md index 199e5ce816..9fe15e1924 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -1,6 +1,18 @@ Change log ========== +2.5.1 +----- + +[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/37?closed=1) + +### Bugfixes + +* Fixed a bug where patterns ending with `**` in `.dockerignore` would + raise an exception +* Fixed a bug where using `attach` with the `stream` argument set to `False` + would raise an exception + 2.5.0 ----- From 7fa2cb7be339e575ade092442f94fcf40a19e6a0 Mon Sep 17 00:00:00 2001 From: Maxime Belanger Date: Thu, 24 Aug 2017 16:15:30 -0400 Subject: [PATCH 0451/1301] Add join_swarm default listen address Since the docker CLI adds a default listen address (0.0.0.0:2377) when joining a node to the swarm, the docker-py api will support the same behavior to easy configuration. Signed-off-by: Maxime Belanger --- docker/api/swarm.py | 2 +- tests/unit/fake_api.py | 6 ++++++ tests/unit/swarm_test.py | 43 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 50 insertions(+), 1 deletion(-) diff --git a/docker/api/swarm.py b/docker/api/swarm.py index 4fa0c4a120..cab7a45e5c 100644 --- a/docker/api/swarm.py +++ b/docker/api/swarm.py @@ -137,7 +137,7 @@ def inspect_node(self, node_id): return self._result(self._get(url), True) @utils.minimum_version('1.24') - def join_swarm(self, remote_addrs, join_token, listen_addr=None, + def join_swarm(self, remote_addrs, join_token, listen_addr='0.0.0.0:2377', advertise_addr=None): """ Make this Engine join a swarm that has already been created. diff --git a/tests/unit/fake_api.py b/tests/unit/fake_api.py index 2ba85bbf53..b3051886bc 100644 --- a/tests/unit/fake_api.py +++ b/tests/unit/fake_api.py @@ -435,6 +435,10 @@ def post_fake_update_node(): return 200, None +def post_fake_join_swarm(): + return 200, None + + def get_fake_network_list(): return 200, [{ "Name": "bridge", @@ -599,6 +603,8 @@ def post_fake_network_disconnect(): CURRENT_VERSION, prefix, FAKE_NODE_ID ), 'POST'): post_fake_update_node, + ('{1}/{0}/swarm/join'.format(CURRENT_VERSION, prefix), 'POST'): + post_fake_join_swarm, ('{1}/{0}/networks'.format(CURRENT_VERSION, prefix), 'GET'): get_fake_network_list, ('{1}/{0}/networks/create'.format(CURRENT_VERSION, prefix), 'POST'): diff --git a/tests/unit/swarm_test.py b/tests/unit/swarm_test.py index 374f8b2473..9a66c0c049 100644 --- a/tests/unit/swarm_test.py +++ b/tests/unit/swarm_test.py @@ -30,3 +30,46 @@ def test_node_update(self): self.assertEqual( args[1]['headers']['Content-Type'], 'application/json' ) + + @requires_api_version('1.24') + def test_join_swarm(self): + remote_addr = ['1.2.3.4:2377'] + listen_addr = '2.3.4.5:2377' + join_token = 'A_BEAUTIFUL_JOIN_TOKEN' + + data = { + 'RemoteAddrs': remote_addr, + 'ListenAddr': listen_addr, + 'JoinToken': join_token + } + + self.client.join_swarm( + remote_addrs=remote_addr, + listen_addr=listen_addr, + join_token=join_token + ) + + args = fake_request.call_args + + assert (args[0][1] == url_prefix + 'swarm/join') + assert (json.loads(args[1]['data']) == data) + assert (args[1]['headers']['Content-Type'] == 'application/json') + + @requires_api_version('1.24') + def test_join_swarm_no_listen_address_takes_default(self): + remote_addr = ['1.2.3.4:2377'] + join_token = 'A_BEAUTIFUL_JOIN_TOKEN' + + data = { + 'RemoteAddrs': remote_addr, + 'ListenAddr': '0.0.0.0:2377', + 'JoinToken': join_token + } + + self.client.join_swarm(remote_addrs=remote_addr, join_token=join_token) + + args = fake_request.call_args + + assert (args[0][1] == url_prefix + 'swarm/join') + assert (json.loads(args[1]['data']) == data) + assert (args[1]['headers']['Content-Type'] == 'application/json') From 3c9c8b181c1029e9a1e819d5da712614f70c510d Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 31 Aug 2017 12:22:59 -0700 Subject: [PATCH 0452/1301] Use unambiguous advertise-addr when initializing a swarm Signed-off-by: Joffrey F --- Makefile | 4 ++-- tests/integration/api_network_test.py | 4 ++-- tests/integration/base.py | 2 +- tests/integration/models_nodes_test.py | 2 +- tests/integration/models_services_test.py | 2 +- tests/integration/models_swarm_test.py | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index c6c6d56cda..991b93a1db 100644 --- a/Makefile +++ b/Makefile @@ -47,7 +47,7 @@ TEST_ENGINE_VERSION ?= 17.06.0-ce .PHONY: integration-dind integration-dind: build build-py3 docker rm -vf dpy-dind || : - docker run -d --name dpy-dind --privileged dockerswarm/dind:${TEST_ENGINE_VERSION} docker daemon\ + docker run -d --name dpy-dind --privileged dockerswarm/dind:${TEST_ENGINE_VERSION} dockerd\ -H tcp://0.0.0.0:2375 --experimental docker run --rm --env="DOCKER_HOST=tcp://docker:2375" --env="DOCKER_TEST_API_VERSION=${TEST_API_VERSION}"\ --link=dpy-dind:docker docker-sdk-python py.test tests/integration @@ -60,7 +60,7 @@ integration-dind-ssl: build-dind-certs build build-py3 docker run -d --name dpy-dind-certs dpy-dind-certs docker run -d --env="DOCKER_HOST=tcp://localhost:2375" --env="DOCKER_TLS_VERIFY=1"\ --env="DOCKER_CERT_PATH=/certs" --volumes-from dpy-dind-certs --name dpy-dind-ssl\ - -v /tmp --privileged dockerswarm/dind:${TEST_ENGINE_VERSION} docker daemon --tlsverify\ + -v /tmp --privileged dockerswarm/dind:${TEST_ENGINE_VERSION} dockerd --tlsverify\ --tlscacert=/certs/ca.pem --tlscert=/certs/server-cert.pem\ --tlskey=/certs/server-key.pem -H tcp://0.0.0.0:2375 --experimental docker run --rm --volumes-from dpy-dind-ssl --env="DOCKER_HOST=tcp://docker:2375"\ diff --git a/tests/integration/api_network_test.py b/tests/integration/api_network_test.py index 5439dd7b2e..1cc632fac7 100644 --- a/tests/integration/api_network_test.py +++ b/tests/integration/api_network_test.py @@ -447,14 +447,14 @@ def test_create_network_ipv6_enabled(self): @requires_api_version('1.25') def test_create_network_attachable(self): - assert self.client.init_swarm('eth0') + assert self.init_swarm() _, net_id = self.create_network(driver='overlay', attachable=True) net = self.client.inspect_network(net_id) assert net['Attachable'] is True @requires_api_version('1.29') def test_create_network_ingress(self): - assert self.client.init_swarm('eth0') + assert self.init_swarm() self.client.remove_network('ingress') _, net_id = self.create_network(driver='overlay', ingress=True) net = self.client.inspect_network(net_id) diff --git a/tests/integration/base.py b/tests/integration/base.py index 3c01689ab3..0c0cd06564 100644 --- a/tests/integration/base.py +++ b/tests/integration/base.py @@ -110,5 +110,5 @@ def execute(self, container, cmd, exit_code=0, **kwargs): def init_swarm(self, **kwargs): return self.client.init_swarm( - 'eth0', listen_addr=helpers.swarm_listen_addr(), **kwargs + '127.0.0.1', listen_addr=helpers.swarm_listen_addr(), **kwargs ) diff --git a/tests/integration/models_nodes_test.py b/tests/integration/models_nodes_test.py index 5823e6b1a3..3c8d48adb5 100644 --- a/tests/integration/models_nodes_test.py +++ b/tests/integration/models_nodes_test.py @@ -15,7 +15,7 @@ def tearDown(self): def test_list_get_update(self): client = docker.from_env(version=TEST_API_VERSION) - client.swarm.init('eth0', listen_addr=helpers.swarm_listen_addr()) + client.swarm.init('127.0.0.1', listen_addr=helpers.swarm_listen_addr()) nodes = client.nodes.list() assert len(nodes) == 1 assert nodes[0].attrs['Spec']['Role'] == 'manager' diff --git a/tests/integration/models_services_test.py b/tests/integration/models_services_test.py index 9b5676d694..6b5dab5312 100644 --- a/tests/integration/models_services_test.py +++ b/tests/integration/models_services_test.py @@ -12,7 +12,7 @@ class ServiceTest(unittest.TestCase): def setUpClass(cls): client = docker.from_env(version=TEST_API_VERSION) helpers.force_leave_swarm(client) - client.swarm.init('eth0', listen_addr=helpers.swarm_listen_addr()) + client.swarm.init('127.0.0.1', listen_addr=helpers.swarm_listen_addr()) @classmethod def tearDownClass(cls): diff --git a/tests/integration/models_swarm_test.py b/tests/integration/models_swarm_test.py index e45ff3cb72..ac18030504 100644 --- a/tests/integration/models_swarm_test.py +++ b/tests/integration/models_swarm_test.py @@ -16,7 +16,7 @@ def tearDown(self): def test_init_update_leave(self): client = docker.from_env(version=TEST_API_VERSION) client.swarm.init( - advertise_addr='eth0', snapshot_interval=5000, + advertise_addr='127.0.0.1', snapshot_interval=5000, listen_addr=helpers.swarm_listen_addr() ) assert client.swarm.attrs['Spec']['Raft']['SnapshotInterval'] == 5000 From 2671d87843e9b8a60ef51c2f60c4f70f13d083f1 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 31 Aug 2017 16:09:52 -0700 Subject: [PATCH 0453/1301] Fix prune_images docstring Signed-off-by: Joffrey F --- docker/api/image.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/api/image.py b/docker/api/image.py index 41cc267e9d..44e60e209e 100644 --- a/docker/api/image.py +++ b/docker/api/image.py @@ -305,8 +305,8 @@ def prune_images(self, filters=None): Args: filters (dict): Filters to process on the prune list. Available filters: - - dangling (bool): When set to true (or 1), prune only - unused and untagged images. + - dangling (bool): When set to true (or 1), prune only + unused and untagged images. Returns: (dict): A dict containing a list of deleted image IDs and From 37fbc8b4fda62f104d32a87946eb9bdd9e3698ba Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 31 Aug 2017 15:58:28 -0700 Subject: [PATCH 0454/1301] Do not interrupt streaming when encountering 0-length frames Signed-off-by: Joffrey F --- docker/utils/socket.py | 4 ++-- tests/unit/fake_api.py | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docker/utils/socket.py b/docker/utils/socket.py index 54392d2b74..c3a5f90fc3 100644 --- a/docker/utils/socket.py +++ b/docker/utils/socket.py @@ -59,7 +59,7 @@ def next_frame_size(socket): try: data = read_exactly(socket, 8) except SocketError: - return 0 + return -1 _, actual = struct.unpack('>BxxxL', data) return actual @@ -71,7 +71,7 @@ def frames_iter(socket): """ while True: n = next_frame_size(socket) - if n == 0: + if n < 0: break while n > 0: result = read(socket, n) diff --git a/tests/unit/fake_api.py b/tests/unit/fake_api.py index 2ba85bbf53..045c342566 100644 --- a/tests/unit/fake_api.py +++ b/tests/unit/fake_api.py @@ -205,7 +205,9 @@ def get_fake_wait(): def get_fake_logs(): status_code = 200 - response = (b'\x01\x00\x00\x00\x00\x00\x00\x11Flowering Nights\n' + response = (b'\x01\x00\x00\x00\x00\x00\x00\x00' + b'\x02\x00\x00\x00\x00\x00\x00\x00' + b'\x01\x00\x00\x00\x00\x00\x00\x11Flowering Nights\n' b'\x01\x00\x00\x00\x00\x00\x00\x10(Sakuya Iyazoi)\n') return status_code, response From 35ceefe1f121c8d829173cbc8f18dd5965308073 Mon Sep 17 00:00:00 2001 From: Hongbin Lu Date: Sun, 10 Sep 2017 14:34:51 -0400 Subject: [PATCH 0455/1301] Return Image objects on image.load In before, image.load returns what Docker API returns, which is a text stream. This commits propose an improvement for returning more useful information, which is a list of Image objects being loaded. Signed-off-by: Hongbin Lu --- docker/models/images.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/docker/models/images.py b/docker/models/images.py index d1b29ad8a6..89e0e80319 100644 --- a/docker/models/images.py +++ b/docker/models/images.py @@ -236,13 +236,24 @@ def load(self, data): data (binary): Image data to be loaded. Returns: - (generator): Progress output as JSON objects + (list of :py:class:`Image`): The images. Raises: :py:class:`docker.errors.APIError` If the server returns an error. """ - return self.client.api.load_image(data) + resp = self.client.api.load_image(data) + images = [] + for chunk in resp: + if 'stream' in chunk: + match = re.search( + r'(^Loaded image ID: |^Loaded image: )(.+)$', + chunk['stream'] + ) + if match: + image_id = match.group(2) + images.append(image_id) + return [self.get(i) for i in images] def pull(self, name, tag=None, **kwargs): """ From be3900b806cf48933643757077f3b77d9c3f4593 Mon Sep 17 00:00:00 2001 From: brett55 Date: Wed, 13 Sep 2017 15:40:37 -0600 Subject: [PATCH 0456/1301] Fix docs, incorrect param name Signed-off-by: brett55 --- docker/models/images.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/models/images.py b/docker/models/images.py index d1b29ad8a6..2aae46d8a0 100644 --- a/docker/models/images.py +++ b/docker/models/images.py @@ -254,7 +254,7 @@ def pull(self, name, tag=None, **kwargs): low-level API. Args: - repository (str): The repository to pull + name (str): The repository to pull tag (str): The tag to pull insecure_registry (bool): Use an insecure registry auth_config (dict): Override the credentials that From ca435af52e06b2c189708a749372708ff69161ff Mon Sep 17 00:00:00 2001 From: Steve Clark Date: Mon, 18 Sep 2017 11:35:53 +0100 Subject: [PATCH 0457/1301] Adding swarm id_attribute to match docker output Swarm id is returned in a attribute with the key ID. The swarm model was using the default behaviour and looking for Id. Signed-off-by: Steve Clark --- docker/models/swarm.py | 2 ++ tests/integration/models_swarm_test.py | 1 + 2 files changed, 3 insertions(+) diff --git a/docker/models/swarm.py b/docker/models/swarm.py index d3d07ee711..df3afd36b7 100644 --- a/docker/models/swarm.py +++ b/docker/models/swarm.py @@ -9,6 +9,8 @@ class Swarm(Model): The server's Swarm state. This a singleton that must be reloaded to get the current state of the Swarm. """ + id_attribute = 'ID' + def __init__(self, *args, **kwargs): super(Swarm, self).__init__(*args, **kwargs) if self.client: diff --git a/tests/integration/models_swarm_test.py b/tests/integration/models_swarm_test.py index ac18030504..dadd77d981 100644 --- a/tests/integration/models_swarm_test.py +++ b/tests/integration/models_swarm_test.py @@ -22,6 +22,7 @@ def test_init_update_leave(self): assert client.swarm.attrs['Spec']['Raft']['SnapshotInterval'] == 5000 client.swarm.update(snapshot_interval=10000) assert client.swarm.attrs['Spec']['Raft']['SnapshotInterval'] == 10000 + assert client.swarm.id assert client.swarm.leave(force=True) with self.assertRaises(docker.errors.APIError) as cm: client.swarm.reload() From ec9356d3a07df473f9dc3ba410a6e542d15d6619 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 2 Oct 2017 12:24:17 -0700 Subject: [PATCH 0458/1301] Remove superfluous version validation Signed-off-by: Joffrey F --- tests/integration/api_client_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/api_client_test.py b/tests/integration/api_client_test.py index cc641582c0..cfb45a3e31 100644 --- a/tests/integration/api_client_test.py +++ b/tests/integration/api_client_test.py @@ -16,7 +16,6 @@ def test_version(self): res = self.client.version() self.assertIn('GoVersion', res) self.assertIn('Version', res) - self.assertEqual(len(res['Version'].split('.')), 3) def test_info(self): res = self.client.info() From 8cb5b52c3f00911f57803e34afae9640dcf2d3ab Mon Sep 17 00:00:00 2001 From: Jan Losinski Date: Tue, 17 Oct 2017 02:46:35 +0200 Subject: [PATCH 0459/1301] Fix simple documentation copy/paste error. Signed-off-by: Jan Losinski --- docker/api/image.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/api/image.py b/docker/api/image.py index 44e60e209e..77553122d6 100644 --- a/docker/api/image.py +++ b/docker/api/image.py @@ -245,10 +245,10 @@ def insert(self, image, url, path): def inspect_image(self, image): """ Get detailed information about an image. Similar to the ``docker - inspect`` command, but only for containers. + inspect`` command, but only for images. Args: - container (str): The container to inspect + image (str): The image to inspect Returns: (dict): Similar to the output of ``docker inspect``, but as a From 877fc817d74a3a44501d07d32ef67dcf95787898 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 24 Oct 2017 16:16:39 -0700 Subject: [PATCH 0460/1301] Add support for new types and options to docker.types.Mount Signed-off-by: Joffrey F --- docker/types/services.py | 55 ++++++++++++++++++++++++++++++---------- 1 file changed, 41 insertions(+), 14 deletions(-) diff --git a/docker/types/services.py b/docker/types/services.py index 8411b70a40..c2767404a6 100644 --- a/docker/types/services.py +++ b/docker/types/services.py @@ -2,7 +2,9 @@ from .. import errors from ..constants import IS_WINDOWS_PLATFORM -from ..utils import check_resource, format_environment, split_command +from ..utils import ( + check_resource, format_environment, parse_bytes, split_command +) class TaskTemplate(dict): @@ -140,9 +142,11 @@ class Mount(dict): target (string): Container path. source (string): Mount source (e.g. a volume name or a host path). - type (string): The mount type (``bind`` or ``volume``). - Default: ``volume``. + type (string): The mount type (``bind`` / ``volume`` / ``tmpfs`` / + ``npipe``). Default: ``volume``. read_only (bool): Whether the mount should be read-only. + consistency (string): The consistency requirement for the mount. One of + ``default```, ``consistent``, ``cached``, ``delegated``. propagation (string): A propagation mode with the value ``[r]private``, ``[r]shared``, or ``[r]slave``. Only valid for the ``bind`` type. no_copy (bool): False if the volume should be populated with the data @@ -152,30 +156,36 @@ class Mount(dict): for the ``volume`` type. driver_config (DriverConfig): Volume driver configuration. Only valid for the ``volume`` type. + tmpfs_size (int or string): The size for the tmpfs mount in bytes. + tmpfs_mode (int): The permission mode for the tmpfs mount. """ def __init__(self, target, source, type='volume', read_only=False, - propagation=None, no_copy=False, labels=None, - driver_config=None): + consistency=None, propagation=None, no_copy=False, + labels=None, driver_config=None, tmpfs_size=None, + tmpfs_mode=None): self['Target'] = target self['Source'] = source - if type not in ('bind', 'volume'): + if type not in ('bind', 'volume', 'tmpfs', 'npipe'): raise errors.InvalidArgument( - 'Only acceptable mount types are `bind` and `volume`.' + 'Unsupported mount type: "{}"'.format(type) ) self['Type'] = type self['ReadOnly'] = read_only + if consistency: + self['Consistency'] = consistency + if type == 'bind': if propagation is not None: self['BindOptions'] = { 'Propagation': propagation } - if any([labels, driver_config, no_copy]): + if any([labels, driver_config, no_copy, tmpfs_size, tmpfs_mode]): raise errors.InvalidArgument( - 'Mount type is binding but volume options have been ' - 'provided.' + 'Incompatible options have been provided for the bind ' + 'type mount.' ) - else: + elif type == 'volume': volume_opts = {} if no_copy: volume_opts['NoCopy'] = True @@ -185,10 +195,27 @@ def __init__(self, target, source, type='volume', read_only=False, volume_opts['DriverConfig'] = driver_config if volume_opts: self['VolumeOptions'] = volume_opts - if propagation: + if any([propagation, tmpfs_size, tmpfs_mode]): + raise errors.InvalidArgument( + 'Incompatible options have been provided for the volume ' + 'type mount.' + ) + elif type == 'tmpfs': + tmpfs_opts = {} + if tmpfs_mode: + if not isinstance(tmpfs_mode, six.integer_types): + raise errors.InvalidArgument( + 'tmpfs_mode must be an integer' + ) + tmpfs_opts['Mode'] = tmpfs_mode + if tmpfs_size: + tmpfs_opts['SizeBytes'] = parse_bytes(tmpfs_size) + if tmpfs_opts: + self['TmpfsOptions'] = tmpfs_opts + if any([propagation, labels, driver_config, no_copy]): raise errors.InvalidArgument( - 'Mount type is volume but `propagation` argument has been ' - 'provided.' + 'Incompatible options have been provided for the tmpfs ' + 'type mount.' ) @classmethod From 5552deed8620fa5652fbc899f2e94358ef82f844 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 24 Oct 2017 16:17:25 -0700 Subject: [PATCH 0461/1301] Add support for mounts in HostConfig Signed-off-by: Joffrey F --- docker/api/container.py | 4 ++ docker/models/containers.py | 5 ++ docker/types/containers.py | 7 ++- tests/integration/api_container_test.py | 65 +++++++++++++++++++++++++ 4 files changed, 80 insertions(+), 1 deletion(-) diff --git a/docker/api/container.py b/docker/api/container.py index 918f8a3a90..f3c33c9786 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -529,6 +529,10 @@ def create_host_config(self, *args, **kwargs): behavior. Accepts number between 0 and 100. memswap_limit (str or int): Maximum amount of memory + swap a container is allowed to consume. + mounts (:py:class:`list`): Specification for mounts to be added to + the container. More powerful alternative to ``binds``. Each + item in the list is expected to be a + :py:class:`docker.types.Mount` object. network_mode (str): One of: - ``bridge`` Create a new network stack for the container on diff --git a/docker/models/containers.py b/docker/models/containers.py index 688deccadc..ea8c10b5be 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -549,6 +549,10 @@ def run(self, image, command=None, stdout=True, stderr=False, behavior. Accepts number between 0 and 100. memswap_limit (str or int): Maximum amount of memory + swap a container is allowed to consume. + mounts (:py:class:`list`): Specification for mounts to be added to + the container. More powerful alternative to ``volumes``. Each + item in the list is expected to be a + :py:class:`docker.types.Mount` object. name (str): The name for this container. nano_cpus (int): CPU quota in units of 10-9 CPUs. network (str): Name of the network this container will be connected @@ -888,6 +892,7 @@ def prune(self, filters=None): 'mem_reservation', 'mem_swappiness', 'memswap_limit', + 'mounts', 'nano_cpus', 'network_mode', 'oom_kill_disable', diff --git a/docker/types/containers.py b/docker/types/containers.py index 030e292bc6..3fc13d9d7f 100644 --- a/docker/types/containers.py +++ b/docker/types/containers.py @@ -120,7 +120,7 @@ def __init__(self, version, binds=None, port_bindings=None, isolation=None, auto_remove=False, storage_opt=None, init=None, init_path=None, volume_driver=None, cpu_count=None, cpu_percent=None, nano_cpus=None, - cpuset_mems=None, runtime=None): + cpuset_mems=None, runtime=None, mounts=None): if mem_limit is not None: self['Memory'] = parse_bytes(mem_limit) @@ -478,6 +478,11 @@ def __init__(self, version, binds=None, port_bindings=None, raise host_config_version_error('runtime', '1.25') self['Runtime'] = runtime + if mounts is not None: + if version_lt(version, '1.30'): + raise host_config_version_error('mounts', '1.30') + self['Mounts'] = mounts + def host_config_type_error(param, param_value, expected): error_msg = 'Invalid type for {0} param: expected {1} but found {2}' diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index a972c1cdca..f03ccdb436 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -522,6 +522,71 @@ def test_create_with_binds_ro(self): inspect_data = self.client.inspect_container(container) self.check_container_data(inspect_data, False) + @pytest.mark.xfail( + IS_WINDOWS_PLATFORM, reason='Test not designed for Windows platform' + ) + @requires_api_version('1.30') + def test_create_with_mounts(self): + mount = docker.types.Mount( + type="bind", source=self.mount_origin, target=self.mount_dest + ) + host_config = self.client.create_host_config(mounts=[mount]) + container = self.run_container( + BUSYBOX, ['ls', self.mount_dest], + host_config=host_config + ) + assert container + logs = self.client.logs(container) + if six.PY3: + logs = logs.decode('utf-8') + assert self.filename in logs + inspect_data = self.client.inspect_container(container) + self.check_container_data(inspect_data, True) + + @pytest.mark.xfail( + IS_WINDOWS_PLATFORM, reason='Test not designed for Windows platform' + ) + @requires_api_version('1.30') + def test_create_with_mounts_ro(self): + mount = docker.types.Mount( + type="bind", source=self.mount_origin, target=self.mount_dest, + read_only=True + ) + host_config = self.client.create_host_config(mounts=[mount]) + container = self.run_container( + BUSYBOX, ['ls', self.mount_dest], + host_config=host_config + ) + assert container + logs = self.client.logs(container) + if six.PY3: + logs = logs.decode('utf-8') + assert self.filename in logs + inspect_data = self.client.inspect_container(container) + self.check_container_data(inspect_data, False) + + @requires_api_version('1.30') + def test_create_with_volume_mount(self): + mount = docker.types.Mount( + type="volume", source=helpers.random_name(), + target=self.mount_dest, labels={'com.dockerpy.test': 'true'} + ) + host_config = self.client.create_host_config(mounts=[mount]) + container = self.client.create_container( + BUSYBOX, ['true'], host_config=host_config, + ) + assert container + inspect_data = self.client.inspect_container(container) + assert 'Mounts' in inspect_data + filtered = list(filter( + lambda x: x['Destination'] == self.mount_dest, + inspect_data['Mounts'] + )) + assert len(filtered) == 1 + mount_data = filtered[0] + assert mount['Source'] == mount_data['Name'] + assert mount_data['RW'] is True + def check_container_data(self, inspect_data, rw): if docker.utils.compare_version('1.20', self.client._version) < 0: self.assertIn('Volumes', inspect_data) From 0d21b5b2549f013889491e02226b5facd758e514 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 24 Oct 2017 16:26:16 -0700 Subject: [PATCH 0462/1301] Pin flake8 version Signed-off-by: Joffrey F --- test-requirements.txt | 2 +- tox.ini | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index 460db10734..f79e815907 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -2,4 +2,4 @@ mock==1.0.1 pytest==2.9.1 coverage==3.7.1 pytest-cov==2.1.0 -flake8==2.4.1 +flake8==3.4.1 diff --git a/tox.ini b/tox.ini index 5a5e5415ad..3bf2b7164d 100644 --- a/tox.ini +++ b/tox.ini @@ -12,4 +12,5 @@ deps = [testenv:flake8] commands = flake8 docker tests setup.py -deps = flake8 +deps = + -r{toxinidir}/test-requirements.txt From 93f2ab1530b86afb5020ba06d9786bc351b0996d Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 25 Oct 2017 14:30:18 -0700 Subject: [PATCH 0463/1301] Add support for extra_hosts option in build Signed-off-by: Joffrey F --- docker/api/build.py | 15 +++++++++++++- docker/models/images.py | 4 ++++ tests/integration/api_build_test.py | 32 +++++++++++++++++++++++++++++ 3 files changed, 50 insertions(+), 1 deletion(-) diff --git a/docker/api/build.py b/docker/api/build.py index f9678a390a..42a1a2965e 100644 --- a/docker/api/build.py +++ b/docker/api/build.py @@ -19,7 +19,7 @@ def build(self, path=None, tag=None, quiet=False, fileobj=None, forcerm=False, dockerfile=None, container_limits=None, decode=False, buildargs=None, gzip=False, shmsize=None, labels=None, cache_from=None, target=None, network_mode=None, - squash=None): + squash=None, extra_hosts=None): """ Similar to the ``docker build`` command. Either ``path`` or ``fileobj`` needs to be set. ``path`` can be a local path (to a directory @@ -101,6 +101,8 @@ def build(self, path=None, tag=None, quiet=False, fileobj=None, build squash (bool): Squash the resulting images layers into a single layer. + extra_hosts (dict): Extra hosts to add to /etc/hosts in building + containers, as a mapping of hostname to IP address. Returns: A generator for the build output. @@ -229,6 +231,17 @@ def build(self, path=None, tag=None, quiet=False, fileobj=None, 'squash was only introduced in API version 1.25' ) + if extra_hosts is not None: + if utils.version_lt(self._version, '1.27'): + raise errors.InvalidVersion( + 'extra_hosts was only introduced in API version 1.27' + ) + + encoded_extra_hosts = [ + '{}:{}'.format(k, v) for k, v in extra_hosts.items() + ] + params.update({'extrahosts': encoded_extra_hosts}) + if context is not None: headers = {'Content-Type': 'application/tar'} if encoding: diff --git a/docker/models/images.py b/docker/models/images.py index 2aae46d8a0..82ca54135e 100644 --- a/docker/models/images.py +++ b/docker/models/images.py @@ -153,6 +153,10 @@ def build(self, **kwargs): Dockerfile network_mode (str): networking mode for the run commands during build + squash (bool): Squash the resulting images layers into a + single layer. + extra_hosts (dict): Extra hosts to add to /etc/hosts in building + containers, as a mapping of hostname to IP address. Returns: (:py:class:`Image`): The built image. diff --git a/tests/integration/api_build_test.py b/tests/integration/api_build_test.py index d0aa5c213c..21464ff64b 100644 --- a/tests/integration/api_build_test.py +++ b/tests/integration/api_build_test.py @@ -244,6 +244,38 @@ def test_build_with_network_mode(self): with pytest.raises(errors.NotFound): self.client.inspect_image('dockerpytest_nonebuild') + @requires_api_version('1.27') + def test_build_with_extra_hosts(self): + img_name = 'dockerpytest_extrahost_build' + self.tmp_imgs.append(img_name) + + script = io.BytesIO('\n'.join([ + 'FROM busybox', + 'RUN ping -c1 hello.world.test', + 'RUN ping -c1 extrahost.local.test', + 'RUN cp /etc/hosts /hosts-file' + ]).encode('ascii')) + + stream = self.client.build( + fileobj=script, tag=img_name, + extra_hosts={ + 'extrahost.local.test': '127.0.0.1', + 'hello.world.test': '8.8.8.8', + }, decode=True + ) + for chunk in stream: + if 'errorDetail' in chunk: + pytest.fail(chunk) + + assert self.client.inspect_image(img_name) + ctnr = self.run_container(img_name, 'cat /hosts-file') + self.tmp_containers.append(ctnr) + logs = self.client.logs(ctnr) + if six.PY3: + logs = logs.decode('utf-8') + assert '127.0.0.1\textrahost.local.test' in logs + assert '8.8.8.8\thello.world.test' in logs + @requires_experimental(until=None) @requires_api_version('1.25') def test_build_squash(self): From 378bd763770d77ea86ad9a608a683bbf1cf53cec Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 25 Oct 2017 15:14:19 -0700 Subject: [PATCH 0464/1301] Update test engine versions in Jenkinsfile Signed-off-by: Joffrey F --- Jenkinsfile | 4 ++-- Makefile | 4 ++-- tests/integration/api_build_test.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index a83d7bf1a7..e2b76ad73f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -6,7 +6,7 @@ def imageNamePy3 def images = [:] -def dockerVersions = ["1.13.1", "17.04.0-ce", "17.05.0-ce", "17.06.0-ce", "17.07.0-ce-rc3"] +def dockerVersions = ["17.06.2-ce", "17.09.0-ce", "17.10.0-ce"] def buildImage = { name, buildargs, pyTag -> img = docker.image(name) @@ -34,7 +34,7 @@ def buildImages = { -> } def getAPIVersion = { engineVersion -> - def versionMap = ['1.13.': '1.26', '17.04': '1.27', '17.05': '1.29', '17.06': '1.30', '17.07': '1.31'] + def versionMap = ['17.06': '1.30', '17.09': '1.32', '17.10': '1.33'] return versionMap[engineVersion.substring(0, 5)] } diff --git a/Makefile b/Makefile index 991b93a1db..98926933e9 100644 --- a/Makefile +++ b/Makefile @@ -41,8 +41,8 @@ integration-test: build integration-test-py3: build-py3 docker run --rm -v /var/run/docker.sock:/var/run/docker.sock docker-sdk-python3 py.test tests/integration/${file} -TEST_API_VERSION ?= 1.30 -TEST_ENGINE_VERSION ?= 17.06.0-ce +TEST_API_VERSION ?= 1.33 +TEST_ENGINE_VERSION ?= 17.10.0-ce .PHONY: integration-dind integration-dind: build build-py3 diff --git a/tests/integration/api_build_test.py b/tests/integration/api_build_test.py index 21464ff64b..a808981f2f 100644 --- a/tests/integration/api_build_test.py +++ b/tests/integration/api_build_test.py @@ -210,7 +210,7 @@ def test_build_container_with_target(self): pass info = self.client.inspect_image('build1') - self.assertEqual(info['Config']['OnBuild'], []) + assert not info['Config']['OnBuild'] @requires_api_version('1.25') def test_build_with_network_mode(self): From 10ea65f5ab34cc9924e09ee9ed54d7005d997c40 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Thu, 26 Oct 2017 10:29:21 -0500 Subject: [PATCH 0465/1301] Fix indentation in docstring The incorrect indentation causes improper formatting when the docs are published. Signed-off-by: Erik Johnson --- docker/api/network.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/api/network.py b/docker/api/network.py index befbb583ce..071a12a63f 100644 --- a/docker/api/network.py +++ b/docker/api/network.py @@ -223,7 +223,7 @@ def connect_container_to_network(self, container, net_id, ipv6_address (str): The IP address of this container on the network, using the IPv6 protocol. Defaults to ``None``. link_local_ips (:py:class:`list`): A list of link-local - (IPv4/IPv6) addresses. + (IPv4/IPv6) addresses. """ data = { "Container": container, From 601d6be5266f4c71145be4a50ea8ef7e44ed575c Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 25 Oct 2017 19:03:12 -0700 Subject: [PATCH 0466/1301] Add support for new ContainerSpec parameters Signed-off-by: Joffrey F --- docker/api/build.py | 7 +- docker/api/service.py | 65 +++++++++------ docker/models/services.py | 37 +++++++-- docker/types/__init__.py | 5 +- docker/types/containers.py | 9 +-- docker/types/healthcheck.py | 24 ++++++ docker/types/services.py | 152 ++++++++++++++++++++++++++++++++++-- docker/utils/__init__.py | 2 +- docker/utils/utils.py | 6 ++ docs/api.rst | 8 +- 10 files changed, 265 insertions(+), 50 deletions(-) diff --git a/docker/api/build.py b/docker/api/build.py index 42a1a2965e..25f271a4e3 100644 --- a/docker/api/build.py +++ b/docker/api/build.py @@ -237,10 +237,9 @@ def build(self, path=None, tag=None, quiet=False, fileobj=None, 'extra_hosts was only introduced in API version 1.27' ) - encoded_extra_hosts = [ - '{}:{}'.format(k, v) for k, v in extra_hosts.items() - ] - params.update({'extrahosts': encoded_extra_hosts}) + if isinstance(extra_hosts, dict): + extra_hosts = utils.format_extra_hosts(extra_hosts) + params.update({'extrahosts': extra_hosts}) if context is not None: headers = {'Content-Type': 'application/tar'} diff --git a/docker/api/service.py b/docker/api/service.py index 4b555a5f59..9ce830ca48 100644 --- a/docker/api/service.py +++ b/docker/api/service.py @@ -4,45 +4,62 @@ def _check_api_features(version, task_template, update_config): + + def raise_version_error(param, min_version): + raise errors.InvalidVersion( + '{} is not supported in API version < {}'.format( + param, min_version + ) + ) + if update_config is not None: if utils.version_lt(version, '1.25'): if 'MaxFailureRatio' in update_config: - raise errors.InvalidVersion( - 'UpdateConfig.max_failure_ratio is not supported in' - ' API version < 1.25' - ) + raise_version_error('UpdateConfig.max_failure_ratio', '1.25') if 'Monitor' in update_config: - raise errors.InvalidVersion( - 'UpdateConfig.monitor is not supported in' - ' API version < 1.25' - ) + raise_version_error('UpdateConfig.monitor', '1.25') if task_template is not None: if 'ForceUpdate' in task_template and utils.version_lt( version, '1.25'): - raise errors.InvalidVersion( - 'force_update is not supported in API version < 1.25' - ) + raise_version_error('force_update', '1.25') if task_template.get('Placement'): if utils.version_lt(version, '1.30'): if task_template['Placement'].get('Platforms'): - raise errors.InvalidVersion( - 'Placement.platforms is not supported in' - ' API version < 1.30' - ) - + raise_version_error('Placement.platforms', '1.30') if utils.version_lt(version, '1.27'): if task_template['Placement'].get('Preferences'): - raise errors.InvalidVersion( - 'Placement.preferences is not supported in' - ' API version < 1.27' - ) - if task_template.get('ContainerSpec', {}).get('TTY'): + raise_version_error('Placement.preferences', '1.27') + + if task_template.get('ContainerSpec'): + container_spec = task_template.get('ContainerSpec') + if utils.version_lt(version, '1.25'): - raise errors.InvalidVersion( - 'ContainerSpec.TTY is not supported in API version < 1.25' - ) + if container_spec.get('TTY'): + raise_version_error('ContainerSpec.tty', '1.25') + if container_spec.get('Hostname') is not None: + raise_version_error('ContainerSpec.hostname', '1.25') + if container_spec.get('Hosts') is not None: + raise_version_error('ContainerSpec.hosts', '1.25') + if container_spec.get('Groups') is not None: + raise_version_error('ContainerSpec.groups', '1.25') + if container_spec.get('DNSConfig') is not None: + raise_version_error('ContainerSpec.dns_config', '1.25') + if container_spec.get('Healthcheck') is not None: + raise_version_error('ContainerSpec.healthcheck', '1.25') + + if utils.version_lt(version, '1.28'): + if container_spec.get('ReadOnly') is not None: + raise_version_error('ContainerSpec.dns_config', '1.28') + if container_spec.get('StopSignal') is not None: + raise_version_error('ContainerSpec.stop_signal', '1.28') + + if utils.version_lt(version, '1.30'): + if container_spec.get('Configs') is not None: + raise_version_error('ContainerSpec.configs', '1.30') + if container_spec.get('Privileges') is not None: + raise_version_error('ContainerSpec.privileges', '1.30') class ServiceApiMixin(object): diff --git a/docker/models/services.py b/docker/models/services.py index e1e2ea6a44..d45621bb4d 100644 --- a/docker/models/services.py +++ b/docker/models/services.py @@ -147,6 +147,22 @@ def create(self, image, command=None, **kwargs): user (str): User to run commands as. workdir (str): Working directory for commands to run. tty (boolean): Whether a pseudo-TTY should be allocated. + groups (:py:class:`list`): A list of additional groups that the + container process will run as. + open_stdin (boolean): Open ``stdin`` + read_only (boolean): Mount the container's root filesystem as read + only. + stop_signal (string): Set signal to stop the service's containers + healthcheck (Healthcheck): Healthcheck + configuration for this service. + hosts (:py:class:`dict`): A set of host to IP mappings to add to + the container's `hosts` file. + dns_config (DNSConfig): Specification for DNS + related configurations in resolver configuration file. + configs (:py:class:`list`): List of :py:class:`ConfigReference` + that will be exposed to the service. + privileges (Privileges): Security options for the service's + containers. Returns: (:py:class:`Service`) The created service. @@ -202,18 +218,27 @@ def list(self, **kwargs): # kwargs to copy straight over to ContainerSpec CONTAINER_SPEC_KWARGS = [ - 'image', - 'command', 'args', + 'command', + 'configs', + 'dns_config', 'env', + 'groups', + 'healthcheck', 'hostname', - 'workdir', - 'user', + 'hosts', + 'image', 'labels', 'mounts', - 'stop_grace_period', + 'open_stdin', + 'privileges' + 'read_only', 'secrets', - 'tty' + 'stop_grace_period', + 'stop_signal', + 'tty', + 'user', + 'workdir', ] # kwargs to copy straight over to TaskTemplate diff --git a/docker/types/__init__.py b/docker/types/__init__.py index edc919dfcf..39c93e344d 100644 --- a/docker/types/__init__.py +++ b/docker/types/__init__.py @@ -3,7 +3,8 @@ from .healthcheck import Healthcheck from .networks import EndpointConfig, IPAMConfig, IPAMPool, NetworkingConfig from .services import ( - ContainerSpec, DriverConfig, EndpointSpec, Mount, Placement, Resources, - RestartPolicy, SecretReference, ServiceMode, TaskTemplate, UpdateConfig + ConfigReference, ContainerSpec, DNSConfig, DriverConfig, EndpointSpec, + Mount, Placement, Privileges, Resources, RestartPolicy, SecretReference, + ServiceMode, TaskTemplate, UpdateConfig ) from .swarm import SwarmSpec, SwarmExternalCA diff --git a/docker/types/containers.py b/docker/types/containers.py index 3fc13d9d7f..13bea713ed 100644 --- a/docker/types/containers.py +++ b/docker/types/containers.py @@ -4,8 +4,8 @@ from .. import errors from ..utils.utils import ( convert_port_bindings, convert_tmpfs_mounts, convert_volume_binds, - format_environment, normalize_links, parse_bytes, parse_devices, - split_command, version_gte, version_lt, + format_environment, format_extra_hosts, normalize_links, parse_bytes, + parse_devices, split_command, version_gte, version_lt, ) from .base import DictType from .healthcheck import Healthcheck @@ -257,10 +257,7 @@ def __init__(self, version, binds=None, port_bindings=None, if extra_hosts is not None: if isinstance(extra_hosts, dict): - extra_hosts = [ - '{0}:{1}'.format(k, v) - for k, v in sorted(six.iteritems(extra_hosts)) - ] + extra_hosts = format_extra_hosts(extra_hosts) self['ExtraHosts'] = extra_hosts diff --git a/docker/types/healthcheck.py b/docker/types/healthcheck.py index 8ea9a35f5b..5a6a931576 100644 --- a/docker/types/healthcheck.py +++ b/docker/types/healthcheck.py @@ -4,6 +4,30 @@ class Healthcheck(DictType): + """ + Defines a healthcheck configuration for a container or service. + + Args: + + test (:py:class:`list` or str): Test to perform to determine + container health. Possible values: + - Empty list: Inherit healthcheck from parent image + - ``["NONE"]``: Disable healthcheck + - ``["CMD", args...]``: exec arguments directly. + - ``["CMD-SHELL", command]``: RUn command in the system's + default shell. + If a string is provided, it will be used as a ``CMD-SHELL`` + command. + interval (int): The time to wait between checks in nanoseconds. It + should be 0 or at least 1000000 (1 ms). + timeout (int): The time to wait before considering the check to + have hung. It should be 0 or at least 1000000 (1 ms). + retries (integer): The number of consecutive failures needed to + consider a container as unhealthy. + start_period (integer): Start period for the container to + initialize before starting health-retries countdown in + nanoseconds. It should be 0 or at least 1000000 (1 ms). + """ def __init__(self, **kwargs): test = kwargs.get('test', kwargs.get('Test')) if isinstance(test, six.string_types): diff --git a/docker/types/services.py b/docker/types/services.py index c2767404a6..c77db166fb 100644 --- a/docker/types/services.py +++ b/docker/types/services.py @@ -3,7 +3,8 @@ from .. import errors from ..constants import IS_WINDOWS_PLATFORM from ..utils import ( - check_resource, format_environment, parse_bytes, split_command + check_resource, format_environment, format_extra_hosts, parse_bytes, + split_command, ) @@ -84,13 +85,31 @@ class ContainerSpec(dict): :py:class:`~docker.types.Mount` class for details. stop_grace_period (int): Amount of time to wait for the container to terminate before forcefully killing it. - secrets (list of py:class:`SecretReference`): List of secrets to be + secrets (:py:class:`list`): List of :py:class:`SecretReference` to be made available inside the containers. tty (boolean): Whether a pseudo-TTY should be allocated. + groups (:py:class:`list`): A list of additional groups that the + container process will run as. + open_stdin (boolean): Open ``stdin`` + read_only (boolean): Mount the container's root filesystem as read + only. + stop_signal (string): Set signal to stop the service's containers + healthcheck (Healthcheck): Healthcheck + configuration for this service. + hosts (:py:class:`dict`): A set of host to IP mappings to add to + the container's `hosts` file. + dns_config (DNSConfig): Specification for DNS + related configurations in resolver configuration file. + configs (:py:class:`list`): List of :py:class:`ConfigReference` that + will be exposed to the service. + privileges (Privileges): Security options for the service's containers. """ def __init__(self, image, command=None, args=None, hostname=None, env=None, workdir=None, user=None, labels=None, mounts=None, - stop_grace_period=None, secrets=None, tty=None): + stop_grace_period=None, secrets=None, tty=None, groups=None, + open_stdin=None, read_only=None, stop_signal=None, + healthcheck=None, hosts=None, dns_config=None, configs=None, + privileges=None): self['Image'] = image if isinstance(command, six.string_types): @@ -109,8 +128,17 @@ def __init__(self, image, command=None, args=None, hostname=None, env=None, self['Dir'] = workdir if user is not None: self['User'] = user + if groups is not None: + self['Groups'] = groups + if stop_signal is not None: + self['StopSignal'] = stop_signal + if stop_grace_period is not None: + self['StopGracePeriod'] = stop_grace_period if labels is not None: self['Labels'] = labels + if hosts is not None: + self['Hosts'] = format_extra_hosts(hosts) + if mounts is not None: parsed_mounts = [] for mount in mounts: @@ -120,16 +148,30 @@ def __init__(self, image, command=None, args=None, hostname=None, env=None, # If mount already parsed parsed_mounts.append(mount) self['Mounts'] = parsed_mounts - if stop_grace_period is not None: - self['StopGracePeriod'] = stop_grace_period if secrets is not None: if not isinstance(secrets, list): raise TypeError('secrets must be a list') self['Secrets'] = secrets + if configs is not None: + if not isinstance(configs, list): + raise TypeError('configs must be a list') + self['Configs'] = configs + + if dns_config is not None: + self['DNSConfig'] = dns_config + if privileges is not None: + self['Privileges'] = privileges + if healthcheck is not None: + self['Healthcheck'] = healthcheck + if tty is not None: self['TTY'] = tty + if open_stdin is not None: + self['OpenStdin'] = open_stdin + if read_only is not None: + self['ReadOnly'] = read_only class Mount(dict): @@ -487,6 +529,34 @@ def __init__(self, secret_id, secret_name, filename=None, uid=None, } +class ConfigReference(dict): + """ + Config reference to be used as part of a :py:class:`ContainerSpec`. + Describes how a config is made accessible inside the service's + containers. + + Args: + config_id (string): Config's ID + config_name (string): Config's name as defined at its creation. + filename (string): Name of the file containing the config. Defaults + to the config's name if not specified. + uid (string): UID of the config file's owner. Default: 0 + gid (string): GID of the config file's group. Default: 0 + mode (int): File access mode inside the container. Default: 0o444 + """ + @check_resource('config_id') + def __init__(self, config_id, config_name, filename=None, uid=None, + gid=None, mode=0o444): + self['ConfigName'] = config_name + self['ConfigID'] = config_id + self['File'] = { + 'Name': filename or config_name, + 'UID': uid or '0', + 'GID': gid or '0', + 'Mode': mode + } + + class Placement(dict): """ Placement constraints to be used as part of a :py:class:`TaskTemplate` @@ -510,3 +580,75 @@ def __init__(self, constraints=None, preferences=None, platforms=None): self['Platforms'].append({ 'Architecture': plat[0], 'OS': plat[1] }) + + +class DNSConfig(dict): + """ + Specification for DNS related configurations in resolver configuration + file (``resolv.conf``). Part of a :py:class:`ContainerSpec` definition. + + Args: + nameservers (:py:class:`list`): The IP addresses of the name + servers. + search (:py:class:`list`): A search list for host-name lookup. + options (:py:class:`list`): A list of internal resolver variables + to be modified (e.g., ``debug``, ``ndots:3``, etc.). + """ + def __init__(self, nameservers=None, search=None, options=None): + self['Nameservers'] = nameservers + self['Search'] = search + self['Options'] = options + + +class Privileges(dict): + """ + Security options for a service's containers. + Part of a :py:class:`ContainerSpec` definition. + + Args: + credentialspec_file (str): Load credential spec from this file. + The file is read by the daemon, and must be present in the + CredentialSpecs subdirectory in the docker data directory, + which defaults to ``C:\ProgramData\Docker\`` on Windows. + Can not be combined with credentialspec_registry. + + credentialspec_registry (str): Load credential spec from this value + in the Windows registry. The specified registry value must be + located in: ``HKLM\SOFTWARE\Microsoft\Windows NT\CurrentVersion + \Virtualization\Containers\CredentialSpecs``. + Can not be combined with credentialspec_file. + + selinux_disable (boolean): Disable SELinux + selinux_user (string): SELinux user label + selinux_role (string): SELinux role label + selinux_type (string): SELinux type label + selinux_level (string): SELinux level label + """ + def __init__(self, credentialspec_file=None, credentialspec_registry=None, + selinux_disable=None, selinux_user=None, selinux_role=None, + selinux_type=None, selinux_level=None): + credential_spec = {} + if credentialspec_registry is not None: + credential_spec['Registry'] = credentialspec_registry + if credentialspec_file is not None: + credential_spec['File'] = credentialspec_file + + if len(credential_spec) > 1: + raise errors.InvalidArgument( + 'credentialspec_file and credentialspec_registry are mutually' + ' exclusive' + ) + + selinux_context = { + 'Disable': selinux_disable, + 'User': selinux_user, + 'Role': selinux_role, + 'Type': selinux_type, + 'Level': selinux_level, + } + + if len(credential_spec) > 0: + self['CredentialSpec'] = credential_spec + + if len(selinux_context) > 0: + self['SELinuxContext'] = selinux_context diff --git a/docker/utils/__init__.py b/docker/utils/__init__.py index b758cbd4ec..c162e3bd6a 100644 --- a/docker/utils/__init__.py +++ b/docker/utils/__init__.py @@ -8,6 +8,6 @@ create_host_config, parse_bytes, ping_registry, parse_env_file, version_lt, version_gte, decode_json_header, split_command, create_ipam_config, create_ipam_pool, parse_devices, normalize_links, convert_service_networks, - format_environment, create_archive + format_environment, create_archive, format_extra_hosts ) diff --git a/docker/utils/utils.py b/docker/utils/utils.py index d9a6d7c1ba..a123fd8f83 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -564,6 +564,12 @@ def format_env(key, value): return [format_env(*var) for var in six.iteritems(environment)] +def format_extra_hosts(extra_hosts): + return [ + '{}:{}'.format(k, v) for k, v in sorted(six.iteritems(extra_hosts)) + ] + + def create_host_config(self, *args, **kwargs): raise errors.DeprecatedMethod( 'utils.create_host_config has been removed. Please use a ' diff --git a/docs/api.rst b/docs/api.rst index 0b10f387db..2fce0a77a2 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -122,13 +122,17 @@ Configuration types .. py:module:: docker.types -.. autoclass:: IPAMConfig -.. autoclass:: IPAMPool +.. autoclass:: ConfigReference .. autoclass:: ContainerSpec +.. autoclass:: DNSConfig .. autoclass:: DriverConfig .. autoclass:: EndpointSpec +.. autoclass:: Healthcheck +.. autoclass:: IPAMConfig +.. autoclass:: IPAMPool .. autoclass:: Mount .. autoclass:: Placement +.. autoclass:: Privileges .. autoclass:: Resources .. autoclass:: RestartPolicy .. autoclass:: SecretReference From 856414bf85fc1cde5c186547807c9633ca14cef4 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 26 Oct 2017 16:13:05 -0700 Subject: [PATCH 0467/1301] Add support for configs management Signed-off-by: Joffrey F --- docker/api/client.py | 2 + docker/api/config.py | 91 +++++++++++++ docker/client.py | 9 ++ docker/models/configs.py | 69 ++++++++++ docs/api.rst | 10 ++ docs/client.rst | 1 + docs/configs.rst | 30 +++++ docs/index.rst | 1 + tests/integration/api_config_test.py | 69 ++++++++++ tests/integration/api_service_test.py | 178 +++++++++++++++++++++++++- tests/integration/base.py | 7 + 11 files changed, 464 insertions(+), 3 deletions(-) create mode 100644 docker/api/config.py create mode 100644 docker/models/configs.py create mode 100644 docs/configs.rst create mode 100644 tests/integration/api_config_test.py diff --git a/docker/api/client.py b/docker/api/client.py index 1de10c77c0..cbe74b916f 100644 --- a/docker/api/client.py +++ b/docker/api/client.py @@ -9,6 +9,7 @@ import websocket from .build import BuildApiMixin +from .config import ConfigApiMixin from .container import ContainerApiMixin from .daemon import DaemonApiMixin from .exec_api import ExecApiMixin @@ -43,6 +44,7 @@ class APIClient( requests.Session, BuildApiMixin, + ConfigApiMixin, ContainerApiMixin, DaemonApiMixin, ExecApiMixin, diff --git a/docker/api/config.py b/docker/api/config.py new file mode 100644 index 0000000000..b46b09c7c1 --- /dev/null +++ b/docker/api/config.py @@ -0,0 +1,91 @@ +import base64 + +import six + +from .. import utils + + +class ConfigApiMixin(object): + @utils.minimum_version('1.25') + def create_config(self, name, data, labels=None): + """ + Create a config + + Args: + name (string): Name of the config + data (bytes): Config data to be stored + labels (dict): A mapping of labels to assign to the config + + Returns (dict): ID of the newly created config + """ + if not isinstance(data, bytes): + data = data.encode('utf-8') + + data = base64.b64encode(data) + if six.PY3: + data = data.decode('ascii') + body = { + 'Data': data, + 'Name': name, + 'Labels': labels + } + + url = self._url('/configs/create') + return self._result( + self._post_json(url, data=body), True + ) + + @utils.minimum_version('1.25') + @utils.check_resource('id') + def inspect_config(self, id): + """ + Retrieve config metadata + + Args: + id (string): Full ID of the config to remove + + Returns (dict): A dictionary of metadata + + Raises: + :py:class:`docker.errors.NotFound` + if no config with that ID exists + """ + url = self._url('/configs/{0}', id) + return self._result(self._get(url), True) + + @utils.minimum_version('1.25') + @utils.check_resource('id') + def remove_config(self, id): + """ + Remove a config + + Args: + id (string): Full ID of the config to remove + + Returns (boolean): True if successful + + Raises: + :py:class:`docker.errors.NotFound` + if no config with that ID exists + """ + url = self._url('/configs/{0}', id) + res = self._delete(url) + self._raise_for_status(res) + return True + + @utils.minimum_version('1.25') + def configs(self, filters=None): + """ + List configs + + Args: + filters (dict): A map of filters to process on the configs + list. Available filters: ``names`` + + Returns (list): A list of configs + """ + url = self._url('/configs') + params = {} + if filters: + params['filters'] = utils.convert_filters(filters) + return self._result(self._get(url, params=params), True) diff --git a/docker/client.py b/docker/client.py index ee361bb961..29968c1f0d 100644 --- a/docker/client.py +++ b/docker/client.py @@ -1,5 +1,6 @@ from .api.client import APIClient from .constants import DEFAULT_TIMEOUT_SECONDS +from .models.configs import ConfigCollection from .models.containers import ContainerCollection from .models.images import ImageCollection from .models.networks import NetworkCollection @@ -80,6 +81,14 @@ def from_env(cls, **kwargs): **kwargs_from_env(**kwargs)) # Resources + @property + def configs(self): + """ + An object for managing configs on the server. See the + :doc:`configs documentation ` for full details. + """ + return ConfigCollection(client=self) + @property def containers(self): """ diff --git a/docker/models/configs.py b/docker/models/configs.py new file mode 100644 index 0000000000..7f23f65007 --- /dev/null +++ b/docker/models/configs.py @@ -0,0 +1,69 @@ +from ..api import APIClient +from .resource import Model, Collection + + +class Config(Model): + """A config.""" + id_attribute = 'ID' + + def __repr__(self): + return "<%s: '%s'>" % (self.__class__.__name__, self.name) + + @property + def name(self): + return self.attrs['Spec']['Name'] + + def remove(self): + """ + Remove this config. + + Raises: + :py:class:`docker.errors.APIError` + If config failed to remove. + """ + return self.client.api.remove_config(self.id) + + +class ConfigCollection(Collection): + """Configs on the Docker server.""" + model = Config + + def create(self, **kwargs): + obj = self.client.api.create_config(**kwargs) + return self.prepare_model(obj) + create.__doc__ = APIClient.create_config.__doc__ + + def get(self, config_id): + """ + Get a config. + + Args: + config_id (str): Config ID. + + Returns: + (:py:class:`Config`): The config. + + Raises: + :py:class:`docker.errors.NotFound` + If the config does not exist. + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + return self.prepare_model(self.client.api.inspect_config(config_id)) + + def list(self, **kwargs): + """ + List configs. Similar to the ``docker config ls`` command. + + Args: + filters (dict): Server-side list filtering options. + + Returns: + (list of :py:class:`Config`): The configs. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + resp = self.client.api.configs(**kwargs) + return [self.prepare_model(obj) for obj in resp] diff --git a/docs/api.rst b/docs/api.rst index 2fce0a77a2..18993ad343 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -9,6 +9,16 @@ It's possible to use :py:class:`APIClient` directly. Some basic things (e.g. run .. autoclass:: docker.api.client.APIClient +Configs +------- + +.. py:module:: docker.api.config + +.. rst-class:: hide-signature +.. autoclass:: ConfigApiMixin + :members: + :undoc-members: + Containers ---------- diff --git a/docs/client.rst b/docs/client.rst index ac7a256a05..43d7c63be7 100644 --- a/docs/client.rst +++ b/docs/client.rst @@ -15,6 +15,7 @@ Client reference .. autoclass:: DockerClient() + .. autoattribute:: configs .. autoattribute:: containers .. autoattribute:: images .. autoattribute:: networks diff --git a/docs/configs.rst b/docs/configs.rst new file mode 100644 index 0000000000..d907ad4216 --- /dev/null +++ b/docs/configs.rst @@ -0,0 +1,30 @@ +Configs +======= + +.. py:module:: docker.models.configs + +Manage configs on the server. + +Methods available on ``client.configs``: + +.. rst-class:: hide-signature +.. py:class:: ConfigCollection + + .. automethod:: create + .. automethod:: get + .. automethod:: list + + +Config objects +-------------- + +.. autoclass:: Config() + + .. autoattribute:: id + .. autoattribute:: name + .. py:attribute:: attrs + + The raw representation of this object from the server. + + .. automethod:: reload + .. automethod:: remove diff --git a/docs/index.rst b/docs/index.rst index 9113bffcc8..39426b6819 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -80,6 +80,7 @@ That's just a taste of what you can do with the Docker SDK for Python. For more, :maxdepth: 2 client + configs containers images networks diff --git a/tests/integration/api_config_test.py b/tests/integration/api_config_test.py new file mode 100644 index 0000000000..fb6002a760 --- /dev/null +++ b/tests/integration/api_config_test.py @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- + +import docker +import pytest + +from ..helpers import force_leave_swarm, requires_api_version +from .base import BaseAPIIntegrationTest + + +@requires_api_version('1.30') +class ConfigAPITest(BaseAPIIntegrationTest): + def setUp(self): + super(ConfigAPITest, self).setUp() + self.init_swarm() + + def tearDown(self): + super(ConfigAPITest, self).tearDown() + force_leave_swarm(self.client) + + def test_create_config(self): + config_id = self.client.create_config( + 'favorite_character', 'sakuya izayoi' + ) + self.tmp_configs.append(config_id) + assert 'ID' in config_id + data = self.client.inspect_config(config_id) + assert data['Spec']['Name'] == 'favorite_character' + + def test_create_config_unicode_data(self): + config_id = self.client.create_config( + 'favorite_character', u'いざよいさくや' + ) + self.tmp_configs.append(config_id) + assert 'ID' in config_id + data = self.client.inspect_config(config_id) + assert data['Spec']['Name'] == 'favorite_character' + + def test_inspect_config(self): + config_name = 'favorite_character' + config_id = self.client.create_config( + config_name, 'sakuya izayoi' + ) + self.tmp_configs.append(config_id) + data = self.client.inspect_config(config_id) + assert data['Spec']['Name'] == config_name + assert 'ID' in data + assert 'Version' in data + + def test_remove_config(self): + config_name = 'favorite_character' + config_id = self.client.create_config( + config_name, 'sakuya izayoi' + ) + self.tmp_configs.append(config_id) + + assert self.client.remove_config(config_id) + with pytest.raises(docker.errors.NotFound): + self.client.inspect_config(config_id) + + def test_list_configs(self): + config_name = 'favorite_character' + config_id = self.client.create_config( + config_name, 'sakuya izayoi' + ) + self.tmp_configs.append(config_id) + + data = self.client.configs(filters={'name': ['favorite_character']}) + assert len(data) == 1 + assert data[0]['ID'] == config_id['ID'] diff --git a/tests/integration/api_service_test.py b/tests/integration/api_service_test.py index c966916ebb..56c3e683cf 100644 --- a/tests/integration/api_service_test.py +++ b/tests/integration/api_service_test.py @@ -473,7 +473,7 @@ def test_create_service_with_unicode_secret(self): secret_data = u'東方花映塚' secret_id = self.client.create_secret(secret_name, secret_data) self.tmp_secrets.append(secret_id) - secret_ref = docker.types.SecretReference(secret_id, secret_name) + secret_ref = docker.types.ConfigReference(secret_id, secret_name) container_spec = docker.types.ContainerSpec( 'busybox', ['sleep', '999'], secrets=[secret_ref] ) @@ -481,8 +481,8 @@ def test_create_service_with_unicode_secret(self): name = self.get_service_name() svc_id = self.client.create_service(task_tmpl, name=name) svc_info = self.client.inspect_service(svc_id) - assert 'Secrets' in svc_info['Spec']['TaskTemplate']['ContainerSpec'] - secrets = svc_info['Spec']['TaskTemplate']['ContainerSpec']['Secrets'] + assert 'Configs' in svc_info['Spec']['TaskTemplate']['ContainerSpec'] + secrets = svc_info['Spec']['TaskTemplate']['ContainerSpec']['Configs'] assert secrets[0] == secret_ref container = self.get_service_container(name) @@ -493,3 +493,175 @@ def test_create_service_with_unicode_secret(self): container_secret = self.client.exec_start(exec_id) container_secret = container_secret.decode('utf-8') assert container_secret == secret_data + + @requires_api_version('1.25') + def test_create_service_with_config(self): + config_name = 'favorite_touhou' + config_data = b'phantasmagoria of flower view' + config_id = self.client.create_config(config_name, config_data) + self.tmp_configs.append(config_id) + config_ref = docker.types.ConfigReference(config_id, config_name) + container_spec = docker.types.ContainerSpec( + 'busybox', ['sleep', '999'], configs=[config_ref] + ) + task_tmpl = docker.types.TaskTemplate(container_spec) + name = self.get_service_name() + svc_id = self.client.create_service(task_tmpl, name=name) + svc_info = self.client.inspect_service(svc_id) + assert 'Configs' in svc_info['Spec']['TaskTemplate']['ContainerSpec'] + configs = svc_info['Spec']['TaskTemplate']['ContainerSpec']['Configs'] + assert configs[0] == config_ref + + container = self.get_service_container(name) + assert container is not None + exec_id = self.client.exec_create( + container, 'cat /run/configs/{0}'.format(config_name) + ) + assert self.client.exec_start(exec_id) == config_data + + @requires_api_version('1.25') + def test_create_service_with_unicode_config(self): + config_name = 'favorite_touhou' + config_data = u'東方花映塚' + config_id = self.client.create_config(config_name, config_data) + self.tmp_configs.append(config_id) + config_ref = docker.types.ConfigReference(config_id, config_name) + container_spec = docker.types.ContainerSpec( + 'busybox', ['sleep', '999'], configs=[config_ref] + ) + task_tmpl = docker.types.TaskTemplate(container_spec) + name = self.get_service_name() + svc_id = self.client.create_service(task_tmpl, name=name) + svc_info = self.client.inspect_service(svc_id) + assert 'Configs' in svc_info['Spec']['TaskTemplate']['ContainerSpec'] + configs = svc_info['Spec']['TaskTemplate']['ContainerSpec']['Configs'] + assert configs[0] == config_ref + + container = self.get_service_container(name) + assert container is not None + exec_id = self.client.exec_create( + container, 'cat /run/configs/{0}'.format(config_name) + ) + container_config = self.client.exec_start(exec_id) + container_config = container_config.decode('utf-8') + assert container_config == config_data + + @requires_api_version('1.25') + def test_create_service_with_hosts(self): + container_spec = docker.types.ContainerSpec( + 'busybox', ['sleep', '999'], hosts={ + 'foobar': '127.0.0.1', + 'baz': '8.8.8.8', + } + ) + task_tmpl = docker.types.TaskTemplate(container_spec) + name = self.get_service_name() + svc_id = self.client.create_service(task_tmpl, name=name) + svc_info = self.client.inspect_service(svc_id) + assert 'Hosts' in svc_info['Spec']['TaskTemplate']['ContainerSpec'] + hosts = svc_info['Spec']['TaskTemplate']['ContainerSpec']['Hosts'] + assert len(hosts) == 2 + assert 'foobar:127.0.0.1' in hosts + assert 'baz:8.8.8.8' in hosts + + @requires_api_version('1.25') + def test_create_service_with_hostname(self): + container_spec = docker.types.ContainerSpec( + 'busybox', ['sleep', '999'], hostname='foobar.baz.com' + ) + task_tmpl = docker.types.TaskTemplate(container_spec) + name = self.get_service_name() + svc_id = self.client.create_service(task_tmpl, name=name) + svc_info = self.client.inspect_service(svc_id) + assert 'Hostname' in svc_info['Spec']['TaskTemplate']['ContainerSpec'] + assert ( + svc_info['Spec']['TaskTemplate']['ContainerSpec']['Hostname'] == + 'foobar.baz.com' + ) + + @requires_api_version('1.25') + def test_create_service_with_groups(self): + container_spec = docker.types.ContainerSpec( + 'busybox', ['sleep', '999'], groups=['shrinemaidens', 'youkais'] + ) + task_tmpl = docker.types.TaskTemplate(container_spec) + name = self.get_service_name() + svc_id = self.client.create_service(task_tmpl, name=name) + svc_info = self.client.inspect_service(svc_id) + assert 'Groups' in svc_info['Spec']['TaskTemplate']['ContainerSpec'] + groups = svc_info['Spec']['TaskTemplate']['ContainerSpec']['Groups'] + assert len(groups) == 2 + assert 'shrinemaidens' in groups + assert 'youkais' in groups + + @requires_api_version('1.25') + def test_create_service_with_dns_config(self): + dns_config = docker.types.DNSConfig( + nameservers=['8.8.8.8', '8.8.4.4'], + search=['local'], options=['debug'] + ) + container_spec = docker.types.ContainerSpec( + BUSYBOX, ['sleep', '999'], dns_config=dns_config + ) + task_tmpl = docker.types.TaskTemplate(container_spec) + name = self.get_service_name() + svc_id = self.client.create_service(task_tmpl, name=name) + svc_info = self.client.inspect_service(svc_id) + assert 'DNSConfig' in svc_info['Spec']['TaskTemplate']['ContainerSpec'] + assert ( + dns_config == + svc_info['Spec']['TaskTemplate']['ContainerSpec']['DNSConfig'] + ) + + @requires_api_version('1.25') + def test_create_service_with_healthcheck(self): + second = 1000000000 + hc = docker.types.Healthcheck( + test='true', retries=3, timeout=1 * second, + start_period=3 * second, interval=second / 2, + ) + container_spec = docker.types.ContainerSpec( + BUSYBOX, ['sleep', '999'], healthcheck=hc + ) + task_tmpl = docker.types.TaskTemplate(container_spec) + name = self.get_service_name() + svc_id = self.client.create_service(task_tmpl, name=name) + svc_info = self.client.inspect_service(svc_id) + assert ( + 'Healthcheck' in svc_info['Spec']['TaskTemplate']['ContainerSpec'] + ) + assert ( + hc == + svc_info['Spec']['TaskTemplate']['ContainerSpec']['Healthcheck'] + ) + + @requires_api_version('1.28') + def test_create_service_with_readonly(self): + container_spec = docker.types.ContainerSpec( + BUSYBOX, ['sleep', '999'], read_only=True + ) + task_tmpl = docker.types.TaskTemplate(container_spec) + name = self.get_service_name() + svc_id = self.client.create_service(task_tmpl, name=name) + svc_info = self.client.inspect_service(svc_id) + assert ( + 'ReadOnly' in svc_info['Spec']['TaskTemplate']['ContainerSpec'] + ) + assert svc_info['Spec']['TaskTemplate']['ContainerSpec']['ReadOnly'] + + @requires_api_version('1.28') + def test_create_service_with_stop_signal(self): + container_spec = docker.types.ContainerSpec( + BUSYBOX, ['sleep', '999'], stop_signal='SIGINT' + ) + task_tmpl = docker.types.TaskTemplate(container_spec) + name = self.get_service_name() + svc_id = self.client.create_service(task_tmpl, name=name) + svc_info = self.client.inspect_service(svc_id) + assert ( + 'StopSignal' in svc_info['Spec']['TaskTemplate']['ContainerSpec'] + ) + assert ( + svc_info['Spec']['TaskTemplate']['ContainerSpec']['StopSignal'] == + 'SIGINT' + ) diff --git a/tests/integration/base.py b/tests/integration/base.py index 0c0cd06564..701e7fc293 100644 --- a/tests/integration/base.py +++ b/tests/integration/base.py @@ -29,6 +29,7 @@ def setUp(self): self.tmp_networks = [] self.tmp_plugins = [] self.tmp_secrets = [] + self.tmp_configs = [] def tearDown(self): client = docker.from_env(version=TEST_API_VERSION) @@ -59,6 +60,12 @@ def tearDown(self): except docker.errors.APIError: pass + for config in self.tmp_configs: + try: + client.api.remove_config(config) + except docker.errors.APIError: + pass + for folder in self.tmp_folders: shutil.rmtree(folder) From bb148380e1ea92e26c8e4dc783dd18b8a39506c0 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 26 Oct 2017 16:50:53 -0700 Subject: [PATCH 0468/1301] More ContainerSpec tests Signed-off-by: Joffrey F --- tests/integration/api_config_test.py | 15 ++++---- tests/integration/api_secret_test.py | 15 ++++---- tests/integration/api_service_test.py | 51 +++++++++++++++++++-------- tests/integration/base.py | 20 +++++++---- 4 files changed, 69 insertions(+), 32 deletions(-) diff --git a/tests/integration/api_config_test.py b/tests/integration/api_config_test.py index fb6002a760..0ffd7675c8 100644 --- a/tests/integration/api_config_test.py +++ b/tests/integration/api_config_test.py @@ -9,13 +9,16 @@ @requires_api_version('1.30') class ConfigAPITest(BaseAPIIntegrationTest): - def setUp(self): - super(ConfigAPITest, self).setUp() - self.init_swarm() + @classmethod + def setup_class(cls): + client = cls.get_client_instance() + force_leave_swarm(client) + cls._init_swarm(client) - def tearDown(self): - super(ConfigAPITest, self).tearDown() - force_leave_swarm(self.client) + @classmethod + def teardown_class(cls): + client = cls.get_client_instance() + force_leave_swarm(client) def test_create_config(self): config_id = self.client.create_config( diff --git a/tests/integration/api_secret_test.py b/tests/integration/api_secret_test.py index dcd880f49c..b3d93b8fc1 100644 --- a/tests/integration/api_secret_test.py +++ b/tests/integration/api_secret_test.py @@ -9,13 +9,16 @@ @requires_api_version('1.25') class SecretAPITest(BaseAPIIntegrationTest): - def setUp(self): - super(SecretAPITest, self).setUp() - self.init_swarm() + @classmethod + def setup_class(cls): + client = cls.get_client_instance() + force_leave_swarm(client) + cls._init_swarm(client) - def tearDown(self): - super(SecretAPITest, self).tearDown() - force_leave_swarm(self.client) + @classmethod + def teardown_class(cls): + client = cls.get_client_instance() + force_leave_swarm(client) def test_create_secret(self): secret_id = self.client.create_secret( diff --git a/tests/integration/api_service_test.py b/tests/integration/api_service_test.py index 56c3e683cf..baa6afa783 100644 --- a/tests/integration/api_service_test.py +++ b/tests/integration/api_service_test.py @@ -13,19 +13,24 @@ class ServiceTest(BaseAPIIntegrationTest): - def setUp(self): - super(ServiceTest, self).setUp() - force_leave_swarm(self.client) - self.init_swarm() + @classmethod + def setup_class(cls): + client = cls.get_client_instance() + force_leave_swarm(client) + cls._init_swarm(client) + + @classmethod + def teardown_class(cls): + client = cls.get_client_instance() + force_leave_swarm(client) def tearDown(self): - super(ServiceTest, self).tearDown() for service in self.client.services(filters={'name': 'dockerpytest_'}): try: self.client.remove_service(service['ID']) except docker.errors.APIError: pass - force_leave_swarm(self.client) + super(ServiceTest, self).tearDown() def get_service_name(self): return 'dockerpytest_{0:x}'.format(random.getrandbits(64)) @@ -473,7 +478,7 @@ def test_create_service_with_unicode_secret(self): secret_data = u'東方花映塚' secret_id = self.client.create_secret(secret_name, secret_data) self.tmp_secrets.append(secret_id) - secret_ref = docker.types.ConfigReference(secret_id, secret_name) + secret_ref = docker.types.SecretReference(secret_id, secret_name) container_spec = docker.types.ContainerSpec( 'busybox', ['sleep', '999'], secrets=[secret_ref] ) @@ -481,8 +486,8 @@ def test_create_service_with_unicode_secret(self): name = self.get_service_name() svc_id = self.client.create_service(task_tmpl, name=name) svc_info = self.client.inspect_service(svc_id) - assert 'Configs' in svc_info['Spec']['TaskTemplate']['ContainerSpec'] - secrets = svc_info['Spec']['TaskTemplate']['ContainerSpec']['Configs'] + assert 'Secrets' in svc_info['Spec']['TaskTemplate']['ContainerSpec'] + secrets = svc_info['Spec']['TaskTemplate']['ContainerSpec']['Secrets'] assert secrets[0] == secret_ref container = self.get_service_container(name) @@ -494,7 +499,7 @@ def test_create_service_with_unicode_secret(self): container_secret = container_secret.decode('utf-8') assert container_secret == secret_data - @requires_api_version('1.25') + @requires_api_version('1.30') def test_create_service_with_config(self): config_name = 'favorite_touhou' config_data = b'phantasmagoria of flower view' @@ -515,11 +520,11 @@ def test_create_service_with_config(self): container = self.get_service_container(name) assert container is not None exec_id = self.client.exec_create( - container, 'cat /run/configs/{0}'.format(config_name) + container, 'cat /{0}'.format(config_name) ) assert self.client.exec_start(exec_id) == config_data - @requires_api_version('1.25') + @requires_api_version('1.30') def test_create_service_with_unicode_config(self): config_name = 'favorite_touhou' config_data = u'東方花映塚' @@ -540,7 +545,7 @@ def test_create_service_with_unicode_config(self): container = self.get_service_container(name) assert container is not None exec_id = self.client.exec_create( - container, 'cat /run/configs/{0}'.format(config_name) + container, 'cat /{0}'.format(config_name) ) container_config = self.client.exec_start(exec_id) container_config = container_config.decode('utf-8') @@ -618,7 +623,7 @@ def test_create_service_with_healthcheck(self): second = 1000000000 hc = docker.types.Healthcheck( test='true', retries=3, timeout=1 * second, - start_period=3 * second, interval=second / 2, + start_period=3 * second, interval=int(second / 2), ) container_spec = docker.types.ContainerSpec( BUSYBOX, ['sleep', '999'], healthcheck=hc @@ -665,3 +670,21 @@ def test_create_service_with_stop_signal(self): svc_info['Spec']['TaskTemplate']['ContainerSpec']['StopSignal'] == 'SIGINT' ) + + @requires_api_version('1.30') + def test_create_service_with_privileges(self): + priv = docker.types.Privileges(selinux_disable=True) + container_spec = docker.types.ContainerSpec( + BUSYBOX, ['sleep', '999'], privileges=priv + ) + task_tmpl = docker.types.TaskTemplate(container_spec) + name = self.get_service_name() + svc_id = self.client.create_service(task_tmpl, name=name) + svc_info = self.client.inspect_service(svc_id) + assert ( + 'Privileges' in svc_info['Spec']['TaskTemplate']['ContainerSpec'] + ) + privileges = ( + svc_info['Spec']['TaskTemplate']['ContainerSpec']['Privileges'] + ) + assert privileges['SELinuxContext']['Disable'] is True diff --git a/tests/integration/base.py b/tests/integration/base.py index 701e7fc293..4f929014bd 100644 --- a/tests/integration/base.py +++ b/tests/integration/base.py @@ -78,14 +78,24 @@ class BaseAPIIntegrationTest(BaseIntegrationTest): def setUp(self): super(BaseAPIIntegrationTest, self).setUp() - self.client = docker.APIClient( - version=TEST_API_VERSION, timeout=60, **kwargs_from_env() - ) + self.client = self.get_client_instance() def tearDown(self): super(BaseAPIIntegrationTest, self).tearDown() self.client.close() + @staticmethod + def get_client_instance(): + return docker.APIClient( + version=TEST_API_VERSION, timeout=60, **kwargs_from_env() + ) + + @staticmethod + def _init_swarm(client, **kwargs): + return client.init_swarm( + '127.0.0.1', listen_addr=helpers.swarm_listen_addr(), **kwargs + ) + def run_container(self, *args, **kwargs): container = self.client.create_container(*args, **kwargs) self.tmp_containers.append(container) @@ -116,6 +126,4 @@ def execute(self, container, cmd, exit_code=0, **kwargs): assert actual_exit_code == exit_code, msg def init_swarm(self, **kwargs): - return self.client.init_swarm( - '127.0.0.1', listen_addr=helpers.swarm_listen_addr(), **kwargs - ) + return self._init_swarm(self.client, **kwargs) From 76b138a0a1c4258517cf8f6876028bbc810166d7 Mon Sep 17 00:00:00 2001 From: Alessandro Baldo Date: Wed, 1 Nov 2017 01:44:21 +0100 Subject: [PATCH 0469/1301] Improve docs for service list filters - add "label" and "mode" to the list of available filter keys in the high-level service API - add "label" and "mode" to the list of available filter keys in the low-level service API - add integration tests Signed-off-by: Alessandro Baldo --- docker/api/service.py | 3 ++- docker/models/services.py | 3 ++- tests/integration/api_service_test.py | 15 +++++++++++++-- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/docker/api/service.py b/docker/api/service.py index 9ce830ca48..e6b48768b4 100644 --- a/docker/api/service.py +++ b/docker/api/service.py @@ -201,7 +201,8 @@ def services(self, filters=None): Args: filters (dict): Filters to process on the nodes list. Valid - filters: ``id`` and ``name``. Default: ``None``. + filters: ``id``, ``name`` , ``label`` and ``mode``. + Default: ``None``. Returns: A list of dictionaries containing data about each service. diff --git a/docker/models/services.py b/docker/models/services.py index d45621bb4d..f2a5d355a0 100644 --- a/docker/models/services.py +++ b/docker/models/services.py @@ -201,7 +201,8 @@ def list(self, **kwargs): Args: filters (dict): Filters to process on the nodes list. Valid - filters: ``id`` and ``name``. Default: ``None``. + filters: ``id``, ``name`` , ``label`` and ``mode``. + Default: ``None``. Returns: (list of :py:class:`Service`): The services. diff --git a/tests/integration/api_service_test.py b/tests/integration/api_service_test.py index baa6afa783..8c6d4af54c 100644 --- a/tests/integration/api_service_test.py +++ b/tests/integration/api_service_test.py @@ -52,7 +52,7 @@ def get_service_container(self, service_name, attempts=20, interval=0.5, return None time.sleep(interval) - def create_simple_service(self, name=None): + def create_simple_service(self, name=None, labels=None): if name: name = 'dockerpytest_{0}'.format(name) else: @@ -62,7 +62,9 @@ def create_simple_service(self, name=None): BUSYBOX, ['echo', 'hello'] ) task_tmpl = docker.types.TaskTemplate(container_spec) - return name, self.client.create_service(task_tmpl, name=name) + return name, self.client.create_service( + task_tmpl, name=name, labels=labels + ) @requires_api_version('1.24') def test_list_services(self): @@ -76,6 +78,15 @@ def test_list_services(self): assert len(test_services) == 1 assert 'dockerpytest_' in test_services[0]['Spec']['Name'] + @requires_api_version('1.24') + def test_list_services_filter_by_label(self): + test_services = self.client.services(filters={'label': 'test_label'}) + assert len(test_services) == 0 + self.create_simple_service(labels={'test_label': 'testing'}) + test_services = self.client.services(filters={'label': 'test_label'}) + assert len(test_services) == 1 + assert test_services[0]['Spec']['Labels']['test_label'] == 'testing' + def test_inspect_service_by_id(self): svc_name, svc_id = self.create_simple_service() svc_info = self.client.inspect_service(svc_id) From c0a075810e3d92295ade789c24d141c1dbba60c4 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 1 Nov 2017 17:13:09 -0700 Subject: [PATCH 0470/1301] Add support for secret driver in create_secret Signed-off-by: Joffrey F --- docker/api/secret.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/docker/api/secret.py b/docker/api/secret.py index 1760a39469..fa4c2ab81d 100644 --- a/docker/api/secret.py +++ b/docker/api/secret.py @@ -2,12 +2,13 @@ import six +from .. import errors from .. import utils class SecretApiMixin(object): @utils.minimum_version('1.25') - def create_secret(self, name, data, labels=None): + def create_secret(self, name, data, labels=None, driver=None): """ Create a secret @@ -15,6 +16,8 @@ def create_secret(self, name, data, labels=None): name (string): Name of the secret data (bytes): Secret data to be stored labels (dict): A mapping of labels to assign to the secret + driver (DriverConfig): A custom driver configuration. If + unspecified, the default ``internal`` driver will be used Returns (dict): ID of the newly created secret """ @@ -30,6 +33,14 @@ def create_secret(self, name, data, labels=None): 'Labels': labels } + if driver is not None: + if utils.version_lt(self._version, '1.31'): + raise errors.InvalidVersion( + 'Secret driver is only available for API version > 1.31' + ) + + body['Driver'] = driver + url = self._url('/secrets/create') return self._result( self._post_json(url, data=body), True From b99f4f2c6952f2e0e98e9a75d9087da6e7f2ef3c Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 1 Nov 2017 17:13:28 -0700 Subject: [PATCH 0471/1301] Doc fixes Signed-off-by: Joffrey F --- docker/api/build.py | 4 ++-- docker/api/plugin.py | 8 ++++---- docker/types/healthcheck.py | 13 +++++++------ docker/types/services.py | 17 +++++++++-------- 4 files changed, 22 insertions(+), 20 deletions(-) diff --git a/docker/api/build.py b/docker/api/build.py index 25f271a4e3..9ff2dfb3c9 100644 --- a/docker/api/build.py +++ b/docker/api/build.py @@ -93,8 +93,8 @@ def build(self, path=None, tag=None, quiet=False, fileobj=None, shmsize (int): Size of `/dev/shm` in bytes. The size must be greater than 0. If omitted the system uses 64MB labels (dict): A dictionary of labels to set on the image - cache_from (list): A list of images used for build cache - resolution + cache_from (:py:class:`list`): A list of images used for build + cache resolution target (str): Name of the build-stage to build in a multi-stage Dockerfile network_mode (str): networking mode for the run commands during diff --git a/docker/api/plugin.py b/docker/api/plugin.py index 87520ccee3..73f185251e 100644 --- a/docker/api/plugin.py +++ b/docker/api/plugin.py @@ -110,8 +110,8 @@ def pull_plugin(self, remote, privileges, name=None): remote (string): Remote reference for the plugin to install. The ``:latest`` tag is optional, and is the default if omitted. - privileges (list): A list of privileges the user consents to - grant to the plugin. Can be retrieved using + privileges (:py:class:`list`): A list of privileges the user + consents to grant to the plugin. Can be retrieved using :py:meth:`~plugin_privileges`. name (string): Local name for the pulled plugin. The ``:latest`` tag is optional, and is the default if omitted. @@ -225,8 +225,8 @@ def upgrade_plugin(self, name, remote, privileges): tag is optional and is the default if omitted. remote (string): Remote reference to upgrade to. The ``:latest`` tag is optional and is the default if omitted. - privileges (list): A list of privileges the user consents to - grant to the plugin. Can be retrieved using + privileges (:py:class:`list`): A list of privileges the user + consents to grant to the plugin. Can be retrieved using :py:meth:`~plugin_privileges`. Returns: diff --git a/docker/types/healthcheck.py b/docker/types/healthcheck.py index 5a6a931576..61857c21ce 100644 --- a/docker/types/healthcheck.py +++ b/docker/types/healthcheck.py @@ -8,14 +8,15 @@ class Healthcheck(DictType): Defines a healthcheck configuration for a container or service. Args: - test (:py:class:`list` or str): Test to perform to determine container health. Possible values: - - Empty list: Inherit healthcheck from parent image - - ``["NONE"]``: Disable healthcheck - - ``["CMD", args...]``: exec arguments directly. - - ``["CMD-SHELL", command]``: RUn command in the system's - default shell. + + - Empty list: Inherit healthcheck from parent image + - ``["NONE"]``: Disable healthcheck + - ``["CMD", args...]``: exec arguments directly. + - ``["CMD-SHELL", command]``: RUn command in the system's + default shell. + If a string is provided, it will be used as a ``CMD-SHELL`` command. interval (int): The time to wait between checks in nanoseconds. It diff --git a/docker/types/services.py b/docker/types/services.py index c77db166fb..9031e609ab 100644 --- a/docker/types/services.py +++ b/docker/types/services.py @@ -405,8 +405,9 @@ class DriverConfig(dict): """ Indicates which driver to use, as well as its configuration. Can be used as ``log_driver`` in a :py:class:`~docker.types.ContainerSpec`, - and for the `driver_config` in a volume - :py:class:`~docker.types.Mount`. + for the `driver_config` in a volume :py:class:`~docker.types.Mount`, or + as the driver object in + :py:meth:`create_secret`. Args: @@ -562,12 +563,12 @@ class Placement(dict): Placement constraints to be used as part of a :py:class:`TaskTemplate` Args: - constraints (list): A list of constraints - preferences (list): Preferences provide a way to make the - scheduler aware of factors such as topology. They are provided - in order from highest to lowest precedence. - platforms (list): A list of platforms expressed as ``(arch, os)`` - tuples + constraints (:py:class:`list`): A list of constraints + preferences (:py:class:`list`): Preferences provide a way to make + the scheduler aware of factors such as topology. They are + provided in order from highest to lowest precedence. + platforms (:py:class:`list`): A list of platforms expressed as + ``(arch, os)`` tuples """ def __init__(self, constraints=None, preferences=None, platforms=None): if constraints is not None: From ff86324c4f51495b0f0395e4cb28c83a37d4d3e0 Mon Sep 17 00:00:00 2001 From: timvisee Date: Thu, 2 Nov 2017 14:30:18 +0100 Subject: [PATCH 0472/1301] Require at least requests v2.14.2 to fix chardet Signed-off-by: timvisee --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4a33c8df02..d59d8124ba 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ SOURCE_DIR = os.path.join(ROOT_DIR) requirements = [ - 'requests >= 2.5.2, != 2.11.0, != 2.12.2, != 2.18.0', + 'requests >= 2.14.2, != 2.18.0', 'six >= 1.4.0', 'websocket-client >= 0.32.0', 'docker-pycreds >= 0.2.1' From f8b5bc62df3057b6dc7108b2a7cf7920033b4e07 Mon Sep 17 00:00:00 2001 From: Chris Harris Date: Thu, 5 Oct 2017 12:14:17 -0400 Subject: [PATCH 0473/1301] Prevent data loss when attaching to container The use of buffering within httplib.HTTPResponse can cause data to be lost. socket.makefile() is called without a bufsize, which causes a buffer to be used when recieving data. The attach methods do a HTTP upgrade to tcp before the raw socket is using to stream data from the container. The problem is that if the container starts stream data while httplib/http.client is reading the response to the attach request part of the data ends will end up in the buffer of fileobject created within the HTTPResponse object. This data is lost as after the attach request data is read directly from the raw socket. Signed-off-by: Chris Harris --- docker/transport/unixconn.py | 33 ++++++++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/docker/transport/unixconn.py b/docker/transport/unixconn.py index 3565cfb629..16e22a8ecb 100644 --- a/docker/transport/unixconn.py +++ b/docker/transport/unixconn.py @@ -34,6 +34,25 @@ def connect(self): self.sock = sock +class AttachHTTPResponse(httplib.HTTPResponse): + ''' + A HTTPResponse object that doesn't use a buffered fileobject. + ''' + def __init__(self, sock, *args, **kwargs): + # Delegate to super class + httplib.HTTPResponse.__init__(self, sock, *args, **kwargs) + + # Override fp with a fileobject that doesn't buffer + self.fp = sock.makefile('rb', 0) + + +class AttachUnixHTTPConnection(UnixHTTPConnection): + ''' + A HTTPConnection that returns responses that don't used buffering. + ''' + response_class = AttachHTTPResponse + + class UnixHTTPConnectionPool(urllib3.connectionpool.HTTPConnectionPool): def __init__(self, base_url, socket_path, timeout=60, maxsize=10): super(UnixHTTPConnectionPool, self).__init__( @@ -44,9 +63,17 @@ def __init__(self, base_url, socket_path, timeout=60, maxsize=10): self.timeout = timeout def _new_conn(self): - return UnixHTTPConnection( - self.base_url, self.socket_path, self.timeout - ) + # Special case for attach url, as we do a http upgrade to tcp and + # a buffered connection can cause data loss. + path = urllib3.util.parse_url(self.base_url).path + if path.endswith('attach'): + return AttachUnixHTTPConnection( + self.base_url, self.socket_path, self.timeout + ) + else: + return UnixHTTPConnection( + self.base_url, self.socket_path, self.timeout + ) class UnixAdapter(requests.adapters.HTTPAdapter): From 1359eb1100d602c5115c07175724002d30a042ee Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 1 Nov 2017 18:03:29 -0700 Subject: [PATCH 0474/1301] Disable buffering based on presence of Connection Upgrade headers Signed-off-by: Joffrey F --- Makefile | 20 +++++++------- docker/transport/unixconn.py | 52 +++++++++++++++++------------------- 2 files changed, 35 insertions(+), 37 deletions(-) diff --git a/Makefile b/Makefile index 98926933e9..32ef510675 100644 --- a/Makefile +++ b/Makefile @@ -27,19 +27,19 @@ test: flake8 unit-test unit-test-py3 integration-dind integration-dind-ssl .PHONY: unit-test unit-test: build - docker run --rm docker-sdk-python py.test tests/unit + docker run -t --rm docker-sdk-python py.test tests/unit .PHONY: unit-test-py3 unit-test-py3: build-py3 - docker run --rm docker-sdk-python3 py.test tests/unit + docker run -t --rm docker-sdk-python3 py.test tests/unit .PHONY: integration-test integration-test: build - docker run --rm -v /var/run/docker.sock:/var/run/docker.sock docker-sdk-python py.test tests/integration/${file} + docker run -t --rm -v /var/run/docker.sock:/var/run/docker.sock docker-sdk-python py.test -v tests/integration/${file} .PHONY: integration-test-py3 integration-test-py3: build-py3 - docker run --rm -v /var/run/docker.sock:/var/run/docker.sock docker-sdk-python3 py.test tests/integration/${file} + docker run -t --rm -v /var/run/docker.sock:/var/run/docker.sock docker-sdk-python3 py.test tests/integration/${file} TEST_API_VERSION ?= 1.33 TEST_ENGINE_VERSION ?= 17.10.0-ce @@ -49,9 +49,9 @@ integration-dind: build build-py3 docker rm -vf dpy-dind || : docker run -d --name dpy-dind --privileged dockerswarm/dind:${TEST_ENGINE_VERSION} dockerd\ -H tcp://0.0.0.0:2375 --experimental - docker run --rm --env="DOCKER_HOST=tcp://docker:2375" --env="DOCKER_TEST_API_VERSION=${TEST_API_VERSION}"\ + docker run -t --rm --env="DOCKER_HOST=tcp://docker:2375" --env="DOCKER_TEST_API_VERSION=${TEST_API_VERSION}"\ --link=dpy-dind:docker docker-sdk-python py.test tests/integration - docker run --rm --env="DOCKER_HOST=tcp://docker:2375" --env="DOCKER_TEST_API_VERSION=${TEST_API_VERSION}"\ + docker run -t --rm --env="DOCKER_HOST=tcp://docker:2375" --env="DOCKER_TEST_API_VERSION=${TEST_API_VERSION}"\ --link=dpy-dind:docker docker-sdk-python3 py.test tests/integration docker rm -vf dpy-dind @@ -63,21 +63,21 @@ integration-dind-ssl: build-dind-certs build build-py3 -v /tmp --privileged dockerswarm/dind:${TEST_ENGINE_VERSION} dockerd --tlsverify\ --tlscacert=/certs/ca.pem --tlscert=/certs/server-cert.pem\ --tlskey=/certs/server-key.pem -H tcp://0.0.0.0:2375 --experimental - docker run --rm --volumes-from dpy-dind-ssl --env="DOCKER_HOST=tcp://docker:2375"\ + docker run -t --rm --volumes-from dpy-dind-ssl --env="DOCKER_HOST=tcp://docker:2375"\ --env="DOCKER_TLS_VERIFY=1" --env="DOCKER_CERT_PATH=/certs" --env="DOCKER_TEST_API_VERSION=${TEST_API_VERSION}"\ --link=dpy-dind-ssl:docker docker-sdk-python py.test tests/integration - docker run --rm --volumes-from dpy-dind-ssl --env="DOCKER_HOST=tcp://docker:2375"\ + docker run -t --rm --volumes-from dpy-dind-ssl --env="DOCKER_HOST=tcp://docker:2375"\ --env="DOCKER_TLS_VERIFY=1" --env="DOCKER_CERT_PATH=/certs" --env="DOCKER_TEST_API_VERSION=${TEST_API_VERSION}"\ --link=dpy-dind-ssl:docker docker-sdk-python3 py.test tests/integration docker rm -vf dpy-dind-ssl dpy-dind-certs .PHONY: flake8 flake8: build - docker run --rm docker-sdk-python flake8 docker tests + docker run -t --rm docker-sdk-python flake8 docker tests .PHONY: docs docs: build-docs - docker run --rm -it -v `pwd`:/src docker-sdk-python-docs sphinx-build docs docs/_build + docker run --rm -t -v `pwd`:/src docker-sdk-python-docs sphinx-build docs docs/_build .PHONY: shell shell: build diff --git a/docker/transport/unixconn.py b/docker/transport/unixconn.py index 16e22a8ecb..7cb877141e 100644 --- a/docker/transport/unixconn.py +++ b/docker/transport/unixconn.py @@ -18,7 +18,20 @@ RecentlyUsedContainer = urllib3._collections.RecentlyUsedContainer +class UnixHTTPResponse(httplib.HTTPResponse, object): + def __init__(self, sock, *args, **kwargs): + disable_buffering = kwargs.pop('disable_buffering', False) + super(UnixHTTPResponse, self).__init__(sock, *args, **kwargs) + if disable_buffering is True: + # We must first create a new pointer then close the old one + # to avoid closing the underlying socket. + new_fp = sock.makefile('rb', 0) + self.fp.close() + self.fp = new_fp + + class UnixHTTPConnection(httplib.HTTPConnection, object): + def __init__(self, base_url, unix_socket, timeout=60): super(UnixHTTPConnection, self).__init__( 'localhost', timeout=timeout @@ -26,6 +39,7 @@ def __init__(self, base_url, unix_socket, timeout=60): self.base_url = base_url self.unix_socket = unix_socket self.timeout = timeout + self.disable_buffering = False def connect(self): sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) @@ -33,24 +47,16 @@ def connect(self): sock.connect(self.unix_socket) self.sock = sock + def putheader(self, header, *values): + super(UnixHTTPConnection, self).putheader(header, *values) + if header == 'Connection' and 'Upgrade' in values: + self.disable_buffering = True -class AttachHTTPResponse(httplib.HTTPResponse): - ''' - A HTTPResponse object that doesn't use a buffered fileobject. - ''' - def __init__(self, sock, *args, **kwargs): - # Delegate to super class - httplib.HTTPResponse.__init__(self, sock, *args, **kwargs) - - # Override fp with a fileobject that doesn't buffer - self.fp = sock.makefile('rb', 0) - + def response_class(self, sock, *args, **kwargs): + if self.disable_buffering: + kwargs['disable_buffering'] = True -class AttachUnixHTTPConnection(UnixHTTPConnection): - ''' - A HTTPConnection that returns responses that don't used buffering. - ''' - response_class = AttachHTTPResponse + return UnixHTTPResponse(sock, *args, **kwargs) class UnixHTTPConnectionPool(urllib3.connectionpool.HTTPConnectionPool): @@ -63,17 +69,9 @@ def __init__(self, base_url, socket_path, timeout=60, maxsize=10): self.timeout = timeout def _new_conn(self): - # Special case for attach url, as we do a http upgrade to tcp and - # a buffered connection can cause data loss. - path = urllib3.util.parse_url(self.base_url).path - if path.endswith('attach'): - return AttachUnixHTTPConnection( - self.base_url, self.socket_path, self.timeout - ) - else: - return UnixHTTPConnection( - self.base_url, self.socket_path, self.timeout - ) + return UnixHTTPConnection( + self.base_url, self.socket_path, self.timeout + ) class UnixAdapter(requests.adapters.HTTPAdapter): From d5094a81267d94d1c05a7250e5a93d2d464607cd Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 2 Nov 2017 14:06:05 -0700 Subject: [PATCH 0475/1301] Fix build tests to not rely on internet connectivity Signed-off-by: Joffrey F --- tests/integration/api_build_test.py | 30 +++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/tests/integration/api_build_test.py b/tests/integration/api_build_test.py index a808981f2f..715a21c426 100644 --- a/tests/integration/api_build_test.py +++ b/tests/integration/api_build_test.py @@ -8,8 +8,8 @@ import pytest import six -from .base import BaseAPIIntegrationTest -from ..helpers import requires_api_version, requires_experimental +from .base import BaseAPIIntegrationTest, BUSYBOX +from ..helpers import random_name, requires_api_version, requires_experimental class BuildTest(BaseAPIIntegrationTest): @@ -214,21 +214,31 @@ def test_build_container_with_target(self): @requires_api_version('1.25') def test_build_with_network_mode(self): + # Set up pingable endpoint on custom network + network = self.client.create_network(random_name())['Id'] + self.tmp_networks.append(network) + container = self.client.create_container(BUSYBOX, 'top') + self.tmp_containers.append(container) + self.client.start(container) + self.client.connect_container_to_network( + container, network, aliases=['pingtarget.docker'] + ) + script = io.BytesIO('\n'.join([ 'FROM busybox', - 'RUN wget http://google.com' + 'RUN ping -c1 pingtarget.docker' ]).encode('ascii')) stream = self.client.build( - fileobj=script, network_mode='bridge', - tag='dockerpytest_bridgebuild' + fileobj=script, network_mode=network, + tag='dockerpytest_customnetbuild' ) - self.tmp_imgs.append('dockerpytest_bridgebuild') + self.tmp_imgs.append('dockerpytest_customnetbuild') for chunk in stream: - pass + print chunk - assert self.client.inspect_image('dockerpytest_bridgebuild') + assert self.client.inspect_image('dockerpytest_customnetbuild') script.seek(0) stream = self.client.build( @@ -260,7 +270,7 @@ def test_build_with_extra_hosts(self): fileobj=script, tag=img_name, extra_hosts={ 'extrahost.local.test': '127.0.0.1', - 'hello.world.test': '8.8.8.8', + 'hello.world.test': '127.0.0.1', }, decode=True ) for chunk in stream: @@ -274,7 +284,7 @@ def test_build_with_extra_hosts(self): if six.PY3: logs = logs.decode('utf-8') assert '127.0.0.1\textrahost.local.test' in logs - assert '8.8.8.8\thello.world.test' in logs + assert '127.0.0.1\thello.world.test' in logs @requires_experimental(until=None) @requires_api_version('1.25') From ca7a6132a418c32df6bb11ba9b2a8b9b2727227a Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 2 Nov 2017 14:10:13 -0700 Subject: [PATCH 0476/1301] Oops Signed-off-by: Joffrey F --- tests/integration/api_build_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/api_build_test.py b/tests/integration/api_build_test.py index 715a21c426..8e98cc9fa5 100644 --- a/tests/integration/api_build_test.py +++ b/tests/integration/api_build_test.py @@ -236,7 +236,7 @@ def test_build_with_network_mode(self): self.tmp_imgs.append('dockerpytest_customnetbuild') for chunk in stream: - print chunk + pass assert self.client.inspect_image('dockerpytest_customnetbuild') From 1ce93ac6e75f0aaf83aededcc165e1e9fe49c52e Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 2 Nov 2017 15:35:43 -0700 Subject: [PATCH 0477/1301] Add support for scope filter in inspect_network Fix missing scope implementation in create_network Signed-off-by: Joffrey F --- docker/api/network.py | 17 ++++++++++++++++- docker/models/networks.py | 14 +++++++++++--- tests/integration/api_network_test.py | 19 +++++++++++++++++++ 3 files changed, 46 insertions(+), 4 deletions(-) diff --git a/docker/api/network.py b/docker/api/network.py index 071a12a63f..797780858a 100644 --- a/docker/api/network.py +++ b/docker/api/network.py @@ -61,6 +61,8 @@ def create_network(self, name, driver=None, options=None, ipam=None, attachable (bool): If enabled, and the network is in the global scope, non-service containers on worker nodes will be able to connect to the network. + scope (str): Specify the network's scope (``local``, ``global`` or + ``swarm``) ingress (bool): If set, create an ingress network which provides the routing-mesh in swarm mode. @@ -140,6 +142,13 @@ def create_network(self, name, driver=None, options=None, ipam=None, data['Ingress'] = ingress + if scope is not None: + if version_lt(self._version, '1.30'): + raise InvalidVersion( + 'scope is not supported in API version < 1.30' + ) + data['Scope'] = scope + url = self._url("/networks/create") res = self._post_json(url, data=data) return self._result(res, json=True) @@ -181,7 +190,7 @@ def remove_network(self, net_id): @minimum_version('1.21') @check_resource('net_id') - def inspect_network(self, net_id, verbose=None): + def inspect_network(self, net_id, verbose=None, scope=None): """ Get detailed information about a network. @@ -189,12 +198,18 @@ def inspect_network(self, net_id, verbose=None): net_id (str): ID of network verbose (bool): Show the service details across the cluster in swarm mode. + scope (str): Filter the network by scope (``swarm``, ``global`` + or ``local``). """ params = {} if verbose is not None: if version_lt(self._version, '1.28'): raise InvalidVersion('verbose was introduced in API 1.28') params['verbose'] = verbose + if scope is not None: + if version_lt(self._version, '1.31'): + raise InvalidVersion('scope was introduced in API 1.31') + params['scope'] = scope url = self._url("/networks/{0}", net_id) res = self._get(url, params=params) diff --git a/docker/models/networks.py b/docker/models/networks.py index afb0ebe8b2..158af99b8d 100644 --- a/docker/models/networks.py +++ b/docker/models/networks.py @@ -102,15 +102,19 @@ def create(self, name, *args, **kwargs): name (str): Name of the network driver (str): Name of the driver used to create the network options (dict): Driver options as a key-value dictionary - ipam (dict): Optional custom IP scheme for the network. - Created with :py:class:`~docker.types.IPAMConfig`. + ipam (IPAMConfig): Optional custom IP scheme for the network. check_duplicate (bool): Request daemon to check for networks with - same name. Default: ``True``. + same name. Default: ``None``. internal (bool): Restrict external access to the network. Default ``False``. labels (dict): Map of labels to set on the network. Default ``None``. enable_ipv6 (bool): Enable IPv6 on the network. Default ``False``. + attachable (bool): If enabled, and the network is in the global + scope, non-service containers on worker nodes will be able to + connect to the network. + scope (str): Specify the network's scope (``local``, ``global`` or + ``swarm``) ingress (bool): If set, create an ingress network which provides the routing-mesh in swarm mode. @@ -155,6 +159,10 @@ def get(self, network_id): Args: network_id (str): The ID of the network. + verbose (bool): Retrieve the service details across the cluster in + swarm mode. + scope (str): Filter the network by scope (``swarm``, ``global`` + or ``local``). Returns: (:py:class:`Network`) The network. diff --git a/tests/integration/api_network_test.py b/tests/integration/api_network_test.py index 1cc632fac7..f4fefde5b9 100644 --- a/tests/integration/api_network_test.py +++ b/tests/integration/api_network_test.py @@ -465,3 +465,22 @@ def test_prune_networks(self): net_name, _ = self.create_network() result = self.client.prune_networks() assert net_name in result['NetworksDeleted'] + + @requires_api_version('1.31') + def test_create_inspect_network_with_scope(self): + assert self.init_swarm() + net_name_loc, net_id_loc = self.create_network(scope='local') + + assert self.client.inspect_network(net_name_loc) + assert self.client.inspect_network(net_name_loc, scope='local') + with pytest.raises(docker.errors.NotFound): + self.client.inspect_network(net_name_loc, scope='global') + + net_name_swarm, net_id_swarm = self.create_network( + driver='overlay', scope='swarm' + ) + + assert self.client.inspect_network(net_name_swarm) + assert self.client.inspect_network(net_name_swarm, scope='swarm') + with pytest.raises(docker.errors.NotFound): + self.client.inspect_network(net_name_swarm, scope='local') From 11a260225c5875584cc2c9af60e891dbed51bbba Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 2 Nov 2017 18:46:25 -0700 Subject: [PATCH 0478/1301] Update SwarmSpec to include new parameters Signed-off-by: Joffrey F --- docker/api/swarm.py | 29 ++++++--- docker/models/swarm.py | 17 +++++- docker/types/swarm.py | 92 ++++++++++++++++++++++++++--- docs/api.rst | 2 + tests/integration/api_swarm_test.py | 38 ++++++++++++ 5 files changed, 160 insertions(+), 18 deletions(-) diff --git a/docker/api/swarm.py b/docker/api/swarm.py index 4fa0c4a120..576fd79bf8 100644 --- a/docker/api/swarm.py +++ b/docker/api/swarm.py @@ -9,8 +9,8 @@ class SwarmApiMixin(object): def create_swarm_spec(self, *args, **kwargs): """ - Create a ``docker.types.SwarmSpec`` instance that can be used as the - ``swarm_spec`` argument in + Create a :py:class:`docker.types.SwarmSpec` instance that can be used + as the ``swarm_spec`` argument in :py:meth:`~docker.api.swarm.SwarmApiMixin.init_swarm`. Args: @@ -29,13 +29,25 @@ def create_swarm_spec(self, *args, **kwargs): dispatcher_heartbeat_period (int): The delay for an agent to send a heartbeat to the dispatcher. node_cert_expiry (int): Automatic expiry for nodes certificates. - external_ca (dict): Configuration for forwarding signing requests - to an external certificate authority. Use - ``docker.types.SwarmExternalCA``. + external_cas (:py:class:`list`): Configuration for forwarding + signing requests to an external certificate authority. Use + a list of :py:class:`docker.types.SwarmExternalCA`. name (string): Swarm's name + labels (dict): User-defined key/value metadata. + signing_ca_cert (str): The desired signing CA certificate for all + swarm node TLS leaf certificates, in PEM format. + signing_ca_key (str): The desired signing CA key for all swarm + node TLS leaf certificates, in PEM format. + ca_force_rotate (int): An integer whose purpose is to force swarm + to generate a new signing CA certificate and key, if none have + been specified. + autolock_managers (boolean): If set, generate a key and use it to + lock data stored on the managers. + log_driver (DriverConfig): The default log driver to use for tasks + created in the orchestrator. Returns: - ``docker.types.SwarmSpec`` instance. + :py:class:`docker.types.SwarmSpec` Raises: :py:class:`docker.errors.APIError` @@ -51,7 +63,10 @@ def create_swarm_spec(self, *args, **kwargs): force_new_cluster=False, swarm_spec=spec ) """ - return types.SwarmSpec(*args, **kwargs) + ext_ca = kwargs.pop('external_ca', None) + if ext_ca: + kwargs['external_cas'] = [ext_ca] + return types.SwarmSpec(self._version, *args, **kwargs) @utils.minimum_version('1.24') def init_swarm(self, advertise_addr=None, listen_addr='0.0.0.0:2377', diff --git a/docker/models/swarm.py b/docker/models/swarm.py index df3afd36b7..5a253c57b5 100644 --- a/docker/models/swarm.py +++ b/docker/models/swarm.py @@ -1,6 +1,5 @@ from docker.api import APIClient from docker.errors import APIError -from docker.types import SwarmSpec from .resource import Model @@ -72,6 +71,18 @@ def init(self, advertise_addr=None, listen_addr='0.0.0.0:2377', to an external certificate authority. Use ``docker.types.SwarmExternalCA``. name (string): Swarm's name + labels (dict): User-defined key/value metadata. + signing_ca_cert (str): The desired signing CA certificate for all + swarm node TLS leaf certificates, in PEM format. + signing_ca_key (str): The desired signing CA key for all swarm + node TLS leaf certificates, in PEM format. + ca_force_rotate (int): An integer whose purpose is to force swarm + to generate a new signing CA certificate and key, if none have + been specified. + autolock_managers (boolean): If set, generate a key and use it to + lock data stored on the managers. + log_driver (DriverConfig): The default log driver to use for tasks + created in the orchestrator. Returns: ``True`` if the request went through. @@ -94,7 +105,7 @@ def init(self, advertise_addr=None, listen_addr='0.0.0.0:2377', 'listen_addr': listen_addr, 'force_new_cluster': force_new_cluster } - init_kwargs['swarm_spec'] = SwarmSpec(**kwargs) + init_kwargs['swarm_spec'] = self.client.api.create_swarm_spec(**kwargs) self.client.api.init_swarm(**init_kwargs) self.reload() @@ -143,7 +154,7 @@ def update(self, rotate_worker_token=False, rotate_manager_token=False, return self.client.api.update_swarm( version=self.version, - swarm_spec=SwarmSpec(**kwargs), + swarm_spec=self.client.api.create_swarm_spec(**kwargs), rotate_worker_token=rotate_worker_token, rotate_manager_token=rotate_manager_token ) diff --git a/docker/types/swarm.py b/docker/types/swarm.py index 49beaa11f7..9687a82d82 100644 --- a/docker/types/swarm.py +++ b/docker/types/swarm.py @@ -1,9 +1,21 @@ +from ..errors import InvalidVersion +from ..utils import version_lt + + class SwarmSpec(dict): - def __init__(self, task_history_retention_limit=None, + """ + Describe a Swarm's configuration and options. Use + :py:meth:`~docker.api.swarm.SwarmApiMixin.create_swarm_spec` + to instantiate. + """ + def __init__(self, version, task_history_retention_limit=None, snapshot_interval=None, keep_old_snapshots=None, log_entries_for_slow_followers=None, heartbeat_tick=None, election_tick=None, dispatcher_heartbeat_period=None, - node_cert_expiry=None, external_ca=None, name=None): + node_cert_expiry=None, external_cas=None, name=None, + labels=None, signing_ca_cert=None, signing_ca_key=None, + ca_force_rotate=None, autolock_managers=None, + log_driver=None): if task_history_retention_limit is not None: self['Orchestration'] = { 'TaskHistoryRetentionLimit': task_history_retention_limit @@ -26,18 +38,82 @@ def __init__(self, task_history_retention_limit=None, 'HeartbeatPeriod': dispatcher_heartbeat_period } - if node_cert_expiry or external_ca: - self['CAConfig'] = { - 'NodeCertExpiry': node_cert_expiry, - 'ExternalCA': external_ca - } + ca_config = {} + if node_cert_expiry is not None: + ca_config['NodeCertExpiry'] = node_cert_expiry + if external_cas: + if version_lt(version, '1.25'): + if len(external_cas) > 1: + raise InvalidVersion( + 'Support for multiple external CAs is not available ' + 'for API version < 1.25' + ) + ca_config['ExternalCA'] = external_cas[0] + else: + ca_config['ExternalCAs'] = external_cas + if signing_ca_key: + if version_lt(version, '1.30'): + raise InvalidVersion( + 'signing_ca_key is not supported in API version < 1.30' + ) + ca_config['SigningCAKey'] = signing_ca_key + if signing_ca_cert: + if version_lt(version, '1.30'): + raise InvalidVersion( + 'signing_ca_cert is not supported in API version < 1.30' + ) + ca_config['SigningCACert'] = signing_ca_cert + if ca_force_rotate is not None: + if version_lt(version, '1.30'): + raise InvalidVersion( + 'force_rotate is not supported in API version < 1.30' + ) + ca_config['ForceRotate'] = ca_force_rotate + if ca_config: + self['CAConfig'] = ca_config + + if autolock_managers is not None: + if version_lt(version, '1.25'): + raise InvalidVersion( + 'autolock_managers is not supported in API version < 1.25' + ) + + self['EncryptionConfig'] = {'AutoLockManagers': autolock_managers} + + if log_driver is not None: + if version_lt(version, '1.25'): + raise InvalidVersion( + 'log_driver is not supported in API version < 1.25' + ) + + self['TaskDefaults'] = {'LogDriver': log_driver} if name is not None: self['Name'] = name + if labels is not None: + self['Labels'] = labels class SwarmExternalCA(dict): - def __init__(self, url, protocol=None, options=None): + """ + Configuration for forwarding signing requests to an external + certificate authority. + + Args: + url (string): URL where certificate signing requests should be + sent. + protocol (string): Protocol for communication with the external CA. + options (dict): An object with key/value pairs that are interpreted + as protocol-specific options for the external CA driver. + ca_cert (string): The root CA certificate (in PEM format) this + external CA uses to issue TLS certificates (assumed to be to + the current swarm root CA certificate if not provided). + + + + """ + def __init__(self, url, protocol=None, options=None, ca_cert=None): self['URL'] = url self['Protocol'] = protocol self['Options'] = options + self['CACert'] = ca_cert diff --git a/docs/api.rst b/docs/api.rst index 18993ad343..ff466a1763 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -147,5 +147,7 @@ Configuration types .. autoclass:: RestartPolicy .. autoclass:: SecretReference .. autoclass:: ServiceMode +.. autoclass:: SwarmExternalCA +.. autoclass:: SwarmSpec(*args, **kwargs) .. autoclass:: TaskTemplate .. autoclass:: UpdateConfig diff --git a/tests/integration/api_swarm_test.py b/tests/integration/api_swarm_test.py index 666c689f55..34b0879ce4 100644 --- a/tests/integration/api_swarm_test.py +++ b/tests/integration/api_swarm_test.py @@ -45,6 +45,44 @@ def test_init_swarm_custom_raft_spec(self): assert swarm_info['Spec']['Raft']['SnapshotInterval'] == 5000 assert swarm_info['Spec']['Raft']['LogEntriesForSlowFollowers'] == 1200 + @requires_api_version('1.30') + def test_init_swarm_with_ca_config(self): + spec = self.client.create_swarm_spec( + node_cert_expiry=7776000000000000, ca_force_rotate=6000000000000 + ) + + assert self.init_swarm(swarm_spec=spec) + swarm_info = self.client.inspect_swarm() + assert swarm_info['Spec']['CAConfig']['NodeCertExpiry'] == ( + spec['CAConfig']['NodeCertExpiry'] + ) + assert swarm_info['Spec']['CAConfig']['ForceRotate'] == ( + spec['CAConfig']['ForceRotate'] + ) + + @requires_api_version('1.25') + def test_init_swarm_with_autolock_managers(self): + spec = self.client.create_swarm_spec(autolock_managers=True) + assert self.init_swarm(swarm_spec=spec) + swarm_info = self.client.inspect_swarm() + + assert ( + swarm_info['Spec']['EncryptionConfig']['AutoLockManagers'] is True + ) + + @requires_api_version('1.25') + @pytest.mark.xfail( + reason="This doesn't seem to be taken into account by the engine" + ) + def test_init_swarm_with_log_driver(self): + spec = {'TaskDefaults': {'LogDriver': {'Name': 'syslog'}}} + assert self.init_swarm(swarm_spec=spec) + swarm_info = self.client.inspect_swarm() + + assert swarm_info['Spec']['TaskDefaults']['LogDriver']['Name'] == ( + 'syslog' + ) + @requires_api_version('1.24') def test_leave_swarm(self): assert self.init_swarm() From 1a4b1813449a7add39c13cbf00101bbf8443b76f Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 6 Nov 2017 16:04:00 -0800 Subject: [PATCH 0479/1301] Add support for insert_defaults in inspect_service Signed-off-by: Joffrey F --- docker/api/service.py | 16 +++++++++++++--- docker/models/services.py | 11 +++++++++-- tests/integration/api_service_test.py | 11 +++++++++++ 3 files changed, 33 insertions(+), 5 deletions(-) diff --git a/docker/api/service.py b/docker/api/service.py index e6b48768b4..4c10ef8efd 100644 --- a/docker/api/service.py +++ b/docker/api/service.py @@ -136,12 +136,14 @@ def create_service( @utils.minimum_version('1.24') @utils.check_resource('service') - def inspect_service(self, service): + def inspect_service(self, service, insert_defaults=None): """ Return information about a service. Args: - service (str): Service name or ID + service (str): Service name or ID. + insert_defaults (boolean): If true, default values will be merged + into the service inspect output. Returns: ``True`` if successful. @@ -151,7 +153,15 @@ def inspect_service(self, service): If the server returns an error. """ url = self._url('/services/{0}', service) - return self._result(self._get(url), True) + params = {} + if insert_defaults is not None: + if utils.version_lt(self._version, '1.29'): + raise errors.InvalidVersion( + 'insert_defaults is not supported in API version < 1.29' + ) + params['insertDefaults'] = insert_defaults + + return self._result(self._get(url, params=params), True) @utils.minimum_version('1.24') @utils.check_resource('task') diff --git a/docker/models/services.py b/docker/models/services.py index f2a5d355a0..6fc5c2a5c1 100644 --- a/docker/models/services.py +++ b/docker/models/services.py @@ -177,12 +177,14 @@ def create(self, image, command=None, **kwargs): service_id = self.client.api.create_service(**create_kwargs) return self.get(service_id) - def get(self, service_id): + def get(self, service_id, insert_defaults=None): """ Get a service. Args: service_id (str): The ID of the service. + insert_defaults (boolean): If true, default values will be merged + into the output. Returns: (:py:class:`Service`): The service. @@ -192,8 +194,13 @@ def get(self, service_id): If the service does not exist. :py:class:`docker.errors.APIError` If the server returns an error. + :py:class:`docker.errors.InvalidVersion` + If one of the arguments is not supported with the current + API version. """ - return self.prepare_model(self.client.api.inspect_service(service_id)) + return self.prepare_model( + self.client.api.inspect_service(service_id, insert_defaults) + ) def list(self, **kwargs): """ diff --git a/tests/integration/api_service_test.py b/tests/integration/api_service_test.py index 8c6d4af54c..b931154945 100644 --- a/tests/integration/api_service_test.py +++ b/tests/integration/api_service_test.py @@ -99,6 +99,17 @@ def test_inspect_service_by_name(self): assert 'ID' in svc_info assert svc_info['ID'] == svc_id['ID'] + @requires_api_version('1.29') + def test_inspect_service_insert_defaults(self): + svc_name, svc_id = self.create_simple_service() + svc_info = self.client.inspect_service(svc_id) + svc_info_defaults = self.client.inspect_service( + svc_id, insert_defaults=True + ) + assert svc_info != svc_info_defaults + assert 'RollbackConfig' in svc_info_defaults['Spec'] + assert 'RollbackConfig' not in svc_info['Spec'] + def test_remove_service_by_id(self): svc_name, svc_id = self.create_simple_service() assert self.client.remove_service(svc_id) From 05f40f038172af00fd8e5a0a4b284daf55358dae Mon Sep 17 00:00:00 2001 From: Martin Monperrus Date: Mon, 2 Oct 2017 09:40:21 +0200 Subject: [PATCH 0480/1301] explain the socket parameter of exec_run Signed-off-by: Martin Monperrus --- docker/models/containers.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docker/models/containers.py b/docker/models/containers.py index ea8c10b5be..689d8ddc04 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -142,14 +142,16 @@ def exec_run(self, cmd, stdout=True, stderr=True, stdin=False, tty=False, detach (bool): If true, detach from the exec command. Default: False stream (bool): Stream response data. Default: False + socket (bool): Whether to return a socket object or not. Default: False environment (dict or list): A dictionary or a list of strings in the following format ``["PASSWORD=xxx"]`` or ``{"PASSWORD": "xxx"}``. Returns: - (generator or str): If ``stream=True``, a generator yielding - response chunks. A string containing response data otherwise. - + (generator or str): + If ``stream=True``, a generator yielding response chunks. + If ``socket=True``, a socket object of the connection (an SSL wrapped socket for TLS-based docker, on which one must call ``sendall`` and ``recv`` -- and **not** os.read / os.write). + A string containing response data otherwise. Raises: :py:class:`docker.errors.APIError` If the server returns an error. From f238fe5554cca48bb9bb366a07bd219c090e445b Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 6 Nov 2017 17:04:27 -0800 Subject: [PATCH 0481/1301] Style fixes. Copied docs to APIClient as well Signed-off-by: Joffrey F --- docker/api/exec_api.py | 5 ++++- docker/models/containers.py | 7 ++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/docker/api/exec_api.py b/docker/api/exec_api.py index 6f42524e65..cff5cfa7b3 100644 --- a/docker/api/exec_api.py +++ b/docker/api/exec_api.py @@ -122,10 +122,13 @@ def exec_start(self, exec_id, detach=False, tty=False, stream=False, Default: False tty (bool): Allocate a pseudo-TTY. Default: False stream (bool): Stream response data. Default: False + socket (bool): Return the connection socket to allow custom + read/write operations. Returns: (generator or str): If ``stream=True``, a generator yielding - response chunks. A string containing response data otherwise. + response chunks. If ``socket=True``, a socket object for the + connection. A string containing response data otherwise. Raises: :py:class:`docker.errors.APIError` diff --git a/docker/models/containers.py b/docker/models/containers.py index 689d8ddc04..97a08b9d83 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -142,15 +142,16 @@ def exec_run(self, cmd, stdout=True, stderr=True, stdin=False, tty=False, detach (bool): If true, detach from the exec command. Default: False stream (bool): Stream response data. Default: False - socket (bool): Whether to return a socket object or not. Default: False + socket (bool): Return the connection socket to allow custom + read/write operations. Default: False environment (dict or list): A dictionary or a list of strings in the following format ``["PASSWORD=xxx"]`` or ``{"PASSWORD": "xxx"}``. Returns: - (generator or str): + (generator or str): If ``stream=True``, a generator yielding response chunks. - If ``socket=True``, a socket object of the connection (an SSL wrapped socket for TLS-based docker, on which one must call ``sendall`` and ``recv`` -- and **not** os.read / os.write). + If ``socket=True``, a socket object for the connection. A string containing response data otherwise. Raises: :py:class:`docker.errors.APIError` From f470955a7750b9b471ebecd921cf2c4269956aae Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 22 Jun 2017 16:42:15 -0700 Subject: [PATCH 0482/1301] Shift test matrix forward Signed-off-by: Joffrey F --- Jenkinsfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index a83d7bf1a7..178653a87c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -5,7 +5,6 @@ def imageNamePy2 def imageNamePy3 def images = [:] - def dockerVersions = ["1.13.1", "17.04.0-ce", "17.05.0-ce", "17.06.0-ce", "17.07.0-ce-rc3"] def buildImage = { name, buildargs, pyTag -> From 303b303855a4862abd35db5a6afd914ec1a3ec89 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 31 Aug 2017 12:22:59 -0700 Subject: [PATCH 0483/1301] Use unambiguous advertise-addr when initializing a swarm Signed-off-by: Joffrey F --- Makefile | 4 ++-- tests/integration/api_network_test.py | 4 ++-- tests/integration/base.py | 2 +- tests/integration/models_nodes_test.py | 2 +- tests/integration/models_services_test.py | 2 +- tests/integration/models_swarm_test.py | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index c6c6d56cda..991b93a1db 100644 --- a/Makefile +++ b/Makefile @@ -47,7 +47,7 @@ TEST_ENGINE_VERSION ?= 17.06.0-ce .PHONY: integration-dind integration-dind: build build-py3 docker rm -vf dpy-dind || : - docker run -d --name dpy-dind --privileged dockerswarm/dind:${TEST_ENGINE_VERSION} docker daemon\ + docker run -d --name dpy-dind --privileged dockerswarm/dind:${TEST_ENGINE_VERSION} dockerd\ -H tcp://0.0.0.0:2375 --experimental docker run --rm --env="DOCKER_HOST=tcp://docker:2375" --env="DOCKER_TEST_API_VERSION=${TEST_API_VERSION}"\ --link=dpy-dind:docker docker-sdk-python py.test tests/integration @@ -60,7 +60,7 @@ integration-dind-ssl: build-dind-certs build build-py3 docker run -d --name dpy-dind-certs dpy-dind-certs docker run -d --env="DOCKER_HOST=tcp://localhost:2375" --env="DOCKER_TLS_VERIFY=1"\ --env="DOCKER_CERT_PATH=/certs" --volumes-from dpy-dind-certs --name dpy-dind-ssl\ - -v /tmp --privileged dockerswarm/dind:${TEST_ENGINE_VERSION} docker daemon --tlsverify\ + -v /tmp --privileged dockerswarm/dind:${TEST_ENGINE_VERSION} dockerd --tlsverify\ --tlscacert=/certs/ca.pem --tlscert=/certs/server-cert.pem\ --tlskey=/certs/server-key.pem -H tcp://0.0.0.0:2375 --experimental docker run --rm --volumes-from dpy-dind-ssl --env="DOCKER_HOST=tcp://docker:2375"\ diff --git a/tests/integration/api_network_test.py b/tests/integration/api_network_test.py index 5439dd7b2e..1cc632fac7 100644 --- a/tests/integration/api_network_test.py +++ b/tests/integration/api_network_test.py @@ -447,14 +447,14 @@ def test_create_network_ipv6_enabled(self): @requires_api_version('1.25') def test_create_network_attachable(self): - assert self.client.init_swarm('eth0') + assert self.init_swarm() _, net_id = self.create_network(driver='overlay', attachable=True) net = self.client.inspect_network(net_id) assert net['Attachable'] is True @requires_api_version('1.29') def test_create_network_ingress(self): - assert self.client.init_swarm('eth0') + assert self.init_swarm() self.client.remove_network('ingress') _, net_id = self.create_network(driver='overlay', ingress=True) net = self.client.inspect_network(net_id) diff --git a/tests/integration/base.py b/tests/integration/base.py index 3c01689ab3..0c0cd06564 100644 --- a/tests/integration/base.py +++ b/tests/integration/base.py @@ -110,5 +110,5 @@ def execute(self, container, cmd, exit_code=0, **kwargs): def init_swarm(self, **kwargs): return self.client.init_swarm( - 'eth0', listen_addr=helpers.swarm_listen_addr(), **kwargs + '127.0.0.1', listen_addr=helpers.swarm_listen_addr(), **kwargs ) diff --git a/tests/integration/models_nodes_test.py b/tests/integration/models_nodes_test.py index 5823e6b1a3..3c8d48adb5 100644 --- a/tests/integration/models_nodes_test.py +++ b/tests/integration/models_nodes_test.py @@ -15,7 +15,7 @@ def tearDown(self): def test_list_get_update(self): client = docker.from_env(version=TEST_API_VERSION) - client.swarm.init('eth0', listen_addr=helpers.swarm_listen_addr()) + client.swarm.init('127.0.0.1', listen_addr=helpers.swarm_listen_addr()) nodes = client.nodes.list() assert len(nodes) == 1 assert nodes[0].attrs['Spec']['Role'] == 'manager' diff --git a/tests/integration/models_services_test.py b/tests/integration/models_services_test.py index 9b5676d694..6b5dab5312 100644 --- a/tests/integration/models_services_test.py +++ b/tests/integration/models_services_test.py @@ -12,7 +12,7 @@ class ServiceTest(unittest.TestCase): def setUpClass(cls): client = docker.from_env(version=TEST_API_VERSION) helpers.force_leave_swarm(client) - client.swarm.init('eth0', listen_addr=helpers.swarm_listen_addr()) + client.swarm.init('127.0.0.1', listen_addr=helpers.swarm_listen_addr()) @classmethod def tearDownClass(cls): diff --git a/tests/integration/models_swarm_test.py b/tests/integration/models_swarm_test.py index e45ff3cb72..ac18030504 100644 --- a/tests/integration/models_swarm_test.py +++ b/tests/integration/models_swarm_test.py @@ -16,7 +16,7 @@ def tearDown(self): def test_init_update_leave(self): client = docker.from_env(version=TEST_API_VERSION) client.swarm.init( - advertise_addr='eth0', snapshot_interval=5000, + advertise_addr='127.0.0.1', snapshot_interval=5000, listen_addr=helpers.swarm_listen_addr() ) assert client.swarm.attrs['Spec']['Raft']['SnapshotInterval'] == 5000 From a49d73e9df5c1bf1e37f65b8f66ae4a8bad1fc9b Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 31 Aug 2017 16:09:52 -0700 Subject: [PATCH 0484/1301] Fix prune_images docstring Signed-off-by: Joffrey F --- docker/api/image.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/api/image.py b/docker/api/image.py index 41cc267e9d..44e60e209e 100644 --- a/docker/api/image.py +++ b/docker/api/image.py @@ -305,8 +305,8 @@ def prune_images(self, filters=None): Args: filters (dict): Filters to process on the prune list. Available filters: - - dangling (bool): When set to true (or 1), prune only - unused and untagged images. + - dangling (bool): When set to true (or 1), prune only + unused and untagged images. Returns: (dict): A dict containing a list of deleted image IDs and From 7107e265b1b972b05ec1dc34a9bb3006a90a0c3e Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 31 Aug 2017 15:58:28 -0700 Subject: [PATCH 0485/1301] Do not interrupt streaming when encountering 0-length frames Signed-off-by: Joffrey F --- docker/utils/socket.py | 4 ++-- tests/unit/fake_api.py | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docker/utils/socket.py b/docker/utils/socket.py index 54392d2b74..c3a5f90fc3 100644 --- a/docker/utils/socket.py +++ b/docker/utils/socket.py @@ -59,7 +59,7 @@ def next_frame_size(socket): try: data = read_exactly(socket, 8) except SocketError: - return 0 + return -1 _, actual = struct.unpack('>BxxxL', data) return actual @@ -71,7 +71,7 @@ def frames_iter(socket): """ while True: n = next_frame_size(socket) - if n == 0: + if n < 0: break while n > 0: result = read(socket, n) diff --git a/tests/unit/fake_api.py b/tests/unit/fake_api.py index 2ba85bbf53..045c342566 100644 --- a/tests/unit/fake_api.py +++ b/tests/unit/fake_api.py @@ -205,7 +205,9 @@ def get_fake_wait(): def get_fake_logs(): status_code = 200 - response = (b'\x01\x00\x00\x00\x00\x00\x00\x11Flowering Nights\n' + response = (b'\x01\x00\x00\x00\x00\x00\x00\x00' + b'\x02\x00\x00\x00\x00\x00\x00\x00' + b'\x01\x00\x00\x00\x00\x00\x00\x11Flowering Nights\n' b'\x01\x00\x00\x00\x00\x00\x00\x10(Sakuya Iyazoi)\n') return status_code, response From ba66b09e2b48651a9ecd7d783797938336c134a5 Mon Sep 17 00:00:00 2001 From: brett55 Date: Wed, 13 Sep 2017 15:40:37 -0600 Subject: [PATCH 0486/1301] Fix docs, incorrect param name Signed-off-by: brett55 --- docker/models/images.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/models/images.py b/docker/models/images.py index d1b29ad8a6..2aae46d8a0 100644 --- a/docker/models/images.py +++ b/docker/models/images.py @@ -254,7 +254,7 @@ def pull(self, name, tag=None, **kwargs): low-level API. Args: - repository (str): The repository to pull + name (str): The repository to pull tag (str): The tag to pull insecure_registry (bool): Use an insecure registry auth_config (dict): Override the credentials that From 1d77ef9e53dac9b325797fdb7984f42f09434375 Mon Sep 17 00:00:00 2001 From: Steve Clark Date: Mon, 18 Sep 2017 11:35:53 +0100 Subject: [PATCH 0487/1301] Adding swarm id_attribute to match docker output Swarm id is returned in a attribute with the key ID. The swarm model was using the default behaviour and looking for Id. Signed-off-by: Steve Clark --- docker/models/swarm.py | 2 ++ tests/integration/models_swarm_test.py | 1 + 2 files changed, 3 insertions(+) diff --git a/docker/models/swarm.py b/docker/models/swarm.py index d3d07ee711..df3afd36b7 100644 --- a/docker/models/swarm.py +++ b/docker/models/swarm.py @@ -9,6 +9,8 @@ class Swarm(Model): The server's Swarm state. This a singleton that must be reloaded to get the current state of the Swarm. """ + id_attribute = 'ID' + def __init__(self, *args, **kwargs): super(Swarm, self).__init__(*args, **kwargs) if self.client: diff --git a/tests/integration/models_swarm_test.py b/tests/integration/models_swarm_test.py index ac18030504..dadd77d981 100644 --- a/tests/integration/models_swarm_test.py +++ b/tests/integration/models_swarm_test.py @@ -22,6 +22,7 @@ def test_init_update_leave(self): assert client.swarm.attrs['Spec']['Raft']['SnapshotInterval'] == 5000 client.swarm.update(snapshot_interval=10000) assert client.swarm.attrs['Spec']['Raft']['SnapshotInterval'] == 10000 + assert client.swarm.id assert client.swarm.leave(force=True) with self.assertRaises(docker.errors.APIError) as cm: client.swarm.reload() From f94fae3aa88ec968575cfd3aaebee97d3c954356 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 2 Oct 2017 12:24:17 -0700 Subject: [PATCH 0488/1301] Remove superfluous version validation Signed-off-by: Joffrey F --- tests/integration/api_client_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/api_client_test.py b/tests/integration/api_client_test.py index cc641582c0..cfb45a3e31 100644 --- a/tests/integration/api_client_test.py +++ b/tests/integration/api_client_test.py @@ -16,7 +16,6 @@ def test_version(self): res = self.client.version() self.assertIn('GoVersion', res) self.assertIn('Version', res) - self.assertEqual(len(res['Version'].split('.')), 3) def test_info(self): res = self.client.info() From 87426093917470b1937ec7bb5eb54d36418033d1 Mon Sep 17 00:00:00 2001 From: Jan Losinski Date: Tue, 17 Oct 2017 02:46:35 +0200 Subject: [PATCH 0489/1301] Fix simple documentation copy/paste error. Signed-off-by: Jan Losinski --- docker/api/image.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/api/image.py b/docker/api/image.py index 44e60e209e..77553122d6 100644 --- a/docker/api/image.py +++ b/docker/api/image.py @@ -245,10 +245,10 @@ def insert(self, image, url, path): def inspect_image(self, image): """ Get detailed information about an image. Similar to the ``docker - inspect`` command, but only for containers. + inspect`` command, but only for images. Args: - container (str): The container to inspect + image (str): The image to inspect Returns: (dict): Similar to the output of ``docker inspect``, but as a From eee9cbbf0810fed71f68bac3fd4d93cfe16208a5 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 24 Oct 2017 16:16:39 -0700 Subject: [PATCH 0490/1301] Add support for new types and options to docker.types.Mount Signed-off-by: Joffrey F --- docker/types/services.py | 55 ++++++++++++++++++++++++++++++---------- 1 file changed, 41 insertions(+), 14 deletions(-) diff --git a/docker/types/services.py b/docker/types/services.py index 8411b70a40..c2767404a6 100644 --- a/docker/types/services.py +++ b/docker/types/services.py @@ -2,7 +2,9 @@ from .. import errors from ..constants import IS_WINDOWS_PLATFORM -from ..utils import check_resource, format_environment, split_command +from ..utils import ( + check_resource, format_environment, parse_bytes, split_command +) class TaskTemplate(dict): @@ -140,9 +142,11 @@ class Mount(dict): target (string): Container path. source (string): Mount source (e.g. a volume name or a host path). - type (string): The mount type (``bind`` or ``volume``). - Default: ``volume``. + type (string): The mount type (``bind`` / ``volume`` / ``tmpfs`` / + ``npipe``). Default: ``volume``. read_only (bool): Whether the mount should be read-only. + consistency (string): The consistency requirement for the mount. One of + ``default```, ``consistent``, ``cached``, ``delegated``. propagation (string): A propagation mode with the value ``[r]private``, ``[r]shared``, or ``[r]slave``. Only valid for the ``bind`` type. no_copy (bool): False if the volume should be populated with the data @@ -152,30 +156,36 @@ class Mount(dict): for the ``volume`` type. driver_config (DriverConfig): Volume driver configuration. Only valid for the ``volume`` type. + tmpfs_size (int or string): The size for the tmpfs mount in bytes. + tmpfs_mode (int): The permission mode for the tmpfs mount. """ def __init__(self, target, source, type='volume', read_only=False, - propagation=None, no_copy=False, labels=None, - driver_config=None): + consistency=None, propagation=None, no_copy=False, + labels=None, driver_config=None, tmpfs_size=None, + tmpfs_mode=None): self['Target'] = target self['Source'] = source - if type not in ('bind', 'volume'): + if type not in ('bind', 'volume', 'tmpfs', 'npipe'): raise errors.InvalidArgument( - 'Only acceptable mount types are `bind` and `volume`.' + 'Unsupported mount type: "{}"'.format(type) ) self['Type'] = type self['ReadOnly'] = read_only + if consistency: + self['Consistency'] = consistency + if type == 'bind': if propagation is not None: self['BindOptions'] = { 'Propagation': propagation } - if any([labels, driver_config, no_copy]): + if any([labels, driver_config, no_copy, tmpfs_size, tmpfs_mode]): raise errors.InvalidArgument( - 'Mount type is binding but volume options have been ' - 'provided.' + 'Incompatible options have been provided for the bind ' + 'type mount.' ) - else: + elif type == 'volume': volume_opts = {} if no_copy: volume_opts['NoCopy'] = True @@ -185,10 +195,27 @@ def __init__(self, target, source, type='volume', read_only=False, volume_opts['DriverConfig'] = driver_config if volume_opts: self['VolumeOptions'] = volume_opts - if propagation: + if any([propagation, tmpfs_size, tmpfs_mode]): + raise errors.InvalidArgument( + 'Incompatible options have been provided for the volume ' + 'type mount.' + ) + elif type == 'tmpfs': + tmpfs_opts = {} + if tmpfs_mode: + if not isinstance(tmpfs_mode, six.integer_types): + raise errors.InvalidArgument( + 'tmpfs_mode must be an integer' + ) + tmpfs_opts['Mode'] = tmpfs_mode + if tmpfs_size: + tmpfs_opts['SizeBytes'] = parse_bytes(tmpfs_size) + if tmpfs_opts: + self['TmpfsOptions'] = tmpfs_opts + if any([propagation, labels, driver_config, no_copy]): raise errors.InvalidArgument( - 'Mount type is volume but `propagation` argument has been ' - 'provided.' + 'Incompatible options have been provided for the tmpfs ' + 'type mount.' ) @classmethod From 5d1b6522462c6672482555b479630f3a4c6b7dcf Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 24 Oct 2017 16:17:25 -0700 Subject: [PATCH 0491/1301] Add support for mounts in HostConfig Signed-off-by: Joffrey F --- docker/api/container.py | 4 ++ docker/models/containers.py | 5 ++ docker/types/containers.py | 7 ++- tests/integration/api_container_test.py | 65 +++++++++++++++++++++++++ 4 files changed, 80 insertions(+), 1 deletion(-) diff --git a/docker/api/container.py b/docker/api/container.py index 918f8a3a90..f3c33c9786 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -529,6 +529,10 @@ def create_host_config(self, *args, **kwargs): behavior. Accepts number between 0 and 100. memswap_limit (str or int): Maximum amount of memory + swap a container is allowed to consume. + mounts (:py:class:`list`): Specification for mounts to be added to + the container. More powerful alternative to ``binds``. Each + item in the list is expected to be a + :py:class:`docker.types.Mount` object. network_mode (str): One of: - ``bridge`` Create a new network stack for the container on diff --git a/docker/models/containers.py b/docker/models/containers.py index 688deccadc..ea8c10b5be 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -549,6 +549,10 @@ def run(self, image, command=None, stdout=True, stderr=False, behavior. Accepts number between 0 and 100. memswap_limit (str or int): Maximum amount of memory + swap a container is allowed to consume. + mounts (:py:class:`list`): Specification for mounts to be added to + the container. More powerful alternative to ``volumes``. Each + item in the list is expected to be a + :py:class:`docker.types.Mount` object. name (str): The name for this container. nano_cpus (int): CPU quota in units of 10-9 CPUs. network (str): Name of the network this container will be connected @@ -888,6 +892,7 @@ def prune(self, filters=None): 'mem_reservation', 'mem_swappiness', 'memswap_limit', + 'mounts', 'nano_cpus', 'network_mode', 'oom_kill_disable', diff --git a/docker/types/containers.py b/docker/types/containers.py index 030e292bc6..3fc13d9d7f 100644 --- a/docker/types/containers.py +++ b/docker/types/containers.py @@ -120,7 +120,7 @@ def __init__(self, version, binds=None, port_bindings=None, isolation=None, auto_remove=False, storage_opt=None, init=None, init_path=None, volume_driver=None, cpu_count=None, cpu_percent=None, nano_cpus=None, - cpuset_mems=None, runtime=None): + cpuset_mems=None, runtime=None, mounts=None): if mem_limit is not None: self['Memory'] = parse_bytes(mem_limit) @@ -478,6 +478,11 @@ def __init__(self, version, binds=None, port_bindings=None, raise host_config_version_error('runtime', '1.25') self['Runtime'] = runtime + if mounts is not None: + if version_lt(version, '1.30'): + raise host_config_version_error('mounts', '1.30') + self['Mounts'] = mounts + def host_config_type_error(param, param_value, expected): error_msg = 'Invalid type for {0} param: expected {1} but found {2}' diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index a972c1cdca..f03ccdb436 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -522,6 +522,71 @@ def test_create_with_binds_ro(self): inspect_data = self.client.inspect_container(container) self.check_container_data(inspect_data, False) + @pytest.mark.xfail( + IS_WINDOWS_PLATFORM, reason='Test not designed for Windows platform' + ) + @requires_api_version('1.30') + def test_create_with_mounts(self): + mount = docker.types.Mount( + type="bind", source=self.mount_origin, target=self.mount_dest + ) + host_config = self.client.create_host_config(mounts=[mount]) + container = self.run_container( + BUSYBOX, ['ls', self.mount_dest], + host_config=host_config + ) + assert container + logs = self.client.logs(container) + if six.PY3: + logs = logs.decode('utf-8') + assert self.filename in logs + inspect_data = self.client.inspect_container(container) + self.check_container_data(inspect_data, True) + + @pytest.mark.xfail( + IS_WINDOWS_PLATFORM, reason='Test not designed for Windows platform' + ) + @requires_api_version('1.30') + def test_create_with_mounts_ro(self): + mount = docker.types.Mount( + type="bind", source=self.mount_origin, target=self.mount_dest, + read_only=True + ) + host_config = self.client.create_host_config(mounts=[mount]) + container = self.run_container( + BUSYBOX, ['ls', self.mount_dest], + host_config=host_config + ) + assert container + logs = self.client.logs(container) + if six.PY3: + logs = logs.decode('utf-8') + assert self.filename in logs + inspect_data = self.client.inspect_container(container) + self.check_container_data(inspect_data, False) + + @requires_api_version('1.30') + def test_create_with_volume_mount(self): + mount = docker.types.Mount( + type="volume", source=helpers.random_name(), + target=self.mount_dest, labels={'com.dockerpy.test': 'true'} + ) + host_config = self.client.create_host_config(mounts=[mount]) + container = self.client.create_container( + BUSYBOX, ['true'], host_config=host_config, + ) + assert container + inspect_data = self.client.inspect_container(container) + assert 'Mounts' in inspect_data + filtered = list(filter( + lambda x: x['Destination'] == self.mount_dest, + inspect_data['Mounts'] + )) + assert len(filtered) == 1 + mount_data = filtered[0] + assert mount['Source'] == mount_data['Name'] + assert mount_data['RW'] is True + def check_container_data(self, inspect_data, rw): if docker.utils.compare_version('1.20', self.client._version) < 0: self.assertIn('Volumes', inspect_data) From cdf9acb185557c82fde6d4a55f3c9aea45d0cbd2 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 24 Oct 2017 16:26:16 -0700 Subject: [PATCH 0492/1301] Pin flake8 version Signed-off-by: Joffrey F --- test-requirements.txt | 2 +- tox.ini | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index 460db10734..f79e815907 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -2,4 +2,4 @@ mock==1.0.1 pytest==2.9.1 coverage==3.7.1 pytest-cov==2.1.0 -flake8==2.4.1 +flake8==3.4.1 diff --git a/tox.ini b/tox.ini index 5a5e5415ad..3bf2b7164d 100644 --- a/tox.ini +++ b/tox.ini @@ -12,4 +12,5 @@ deps = [testenv:flake8] commands = flake8 docker tests setup.py -deps = flake8 +deps = + -r{toxinidir}/test-requirements.txt From 53582a9cf53b66aa898e23a2ff987a0db8ccdae3 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 25 Oct 2017 14:30:18 -0700 Subject: [PATCH 0493/1301] Add support for extra_hosts option in build Signed-off-by: Joffrey F --- docker/api/build.py | 15 +++++++++++++- docker/models/images.py | 4 ++++ tests/integration/api_build_test.py | 32 +++++++++++++++++++++++++++++ 3 files changed, 50 insertions(+), 1 deletion(-) diff --git a/docker/api/build.py b/docker/api/build.py index f9678a390a..42a1a2965e 100644 --- a/docker/api/build.py +++ b/docker/api/build.py @@ -19,7 +19,7 @@ def build(self, path=None, tag=None, quiet=False, fileobj=None, forcerm=False, dockerfile=None, container_limits=None, decode=False, buildargs=None, gzip=False, shmsize=None, labels=None, cache_from=None, target=None, network_mode=None, - squash=None): + squash=None, extra_hosts=None): """ Similar to the ``docker build`` command. Either ``path`` or ``fileobj`` needs to be set. ``path`` can be a local path (to a directory @@ -101,6 +101,8 @@ def build(self, path=None, tag=None, quiet=False, fileobj=None, build squash (bool): Squash the resulting images layers into a single layer. + extra_hosts (dict): Extra hosts to add to /etc/hosts in building + containers, as a mapping of hostname to IP address. Returns: A generator for the build output. @@ -229,6 +231,17 @@ def build(self, path=None, tag=None, quiet=False, fileobj=None, 'squash was only introduced in API version 1.25' ) + if extra_hosts is not None: + if utils.version_lt(self._version, '1.27'): + raise errors.InvalidVersion( + 'extra_hosts was only introduced in API version 1.27' + ) + + encoded_extra_hosts = [ + '{}:{}'.format(k, v) for k, v in extra_hosts.items() + ] + params.update({'extrahosts': encoded_extra_hosts}) + if context is not None: headers = {'Content-Type': 'application/tar'} if encoding: diff --git a/docker/models/images.py b/docker/models/images.py index 2aae46d8a0..82ca54135e 100644 --- a/docker/models/images.py +++ b/docker/models/images.py @@ -153,6 +153,10 @@ def build(self, **kwargs): Dockerfile network_mode (str): networking mode for the run commands during build + squash (bool): Squash the resulting images layers into a + single layer. + extra_hosts (dict): Extra hosts to add to /etc/hosts in building + containers, as a mapping of hostname to IP address. Returns: (:py:class:`Image`): The built image. diff --git a/tests/integration/api_build_test.py b/tests/integration/api_build_test.py index d0aa5c213c..21464ff64b 100644 --- a/tests/integration/api_build_test.py +++ b/tests/integration/api_build_test.py @@ -244,6 +244,38 @@ def test_build_with_network_mode(self): with pytest.raises(errors.NotFound): self.client.inspect_image('dockerpytest_nonebuild') + @requires_api_version('1.27') + def test_build_with_extra_hosts(self): + img_name = 'dockerpytest_extrahost_build' + self.tmp_imgs.append(img_name) + + script = io.BytesIO('\n'.join([ + 'FROM busybox', + 'RUN ping -c1 hello.world.test', + 'RUN ping -c1 extrahost.local.test', + 'RUN cp /etc/hosts /hosts-file' + ]).encode('ascii')) + + stream = self.client.build( + fileobj=script, tag=img_name, + extra_hosts={ + 'extrahost.local.test': '127.0.0.1', + 'hello.world.test': '8.8.8.8', + }, decode=True + ) + for chunk in stream: + if 'errorDetail' in chunk: + pytest.fail(chunk) + + assert self.client.inspect_image(img_name) + ctnr = self.run_container(img_name, 'cat /hosts-file') + self.tmp_containers.append(ctnr) + logs = self.client.logs(ctnr) + if six.PY3: + logs = logs.decode('utf-8') + assert '127.0.0.1\textrahost.local.test' in logs + assert '8.8.8.8\thello.world.test' in logs + @requires_experimental(until=None) @requires_api_version('1.25') def test_build_squash(self): From bb437e921ee926350df132b7d5cf55bcbda0cbea Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Thu, 26 Oct 2017 10:29:21 -0500 Subject: [PATCH 0494/1301] Fix indentation in docstring The incorrect indentation causes improper formatting when the docs are published. Signed-off-by: Erik Johnson --- docker/api/network.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/api/network.py b/docker/api/network.py index befbb583ce..071a12a63f 100644 --- a/docker/api/network.py +++ b/docker/api/network.py @@ -223,7 +223,7 @@ def connect_container_to_network(self, container, net_id, ipv6_address (str): The IP address of this container on the network, using the IPv6 protocol. Defaults to ``None``. link_local_ips (:py:class:`list`): A list of link-local - (IPv4/IPv6) addresses. + (IPv4/IPv6) addresses. """ data = { "Container": container, From cd47a1f9f5e7273fde56e1058927991b2e609cae Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 25 Oct 2017 19:03:12 -0700 Subject: [PATCH 0495/1301] Add support for new ContainerSpec parameters Signed-off-by: Joffrey F --- docker/api/build.py | 7 +- docker/api/service.py | 65 +++++++++------ docker/models/services.py | 37 +++++++-- docker/types/__init__.py | 5 +- docker/types/containers.py | 9 +-- docker/types/healthcheck.py | 24 ++++++ docker/types/services.py | 152 ++++++++++++++++++++++++++++++++++-- docker/utils/__init__.py | 2 +- docker/utils/utils.py | 6 ++ docs/api.rst | 8 +- 10 files changed, 265 insertions(+), 50 deletions(-) diff --git a/docker/api/build.py b/docker/api/build.py index 42a1a2965e..25f271a4e3 100644 --- a/docker/api/build.py +++ b/docker/api/build.py @@ -237,10 +237,9 @@ def build(self, path=None, tag=None, quiet=False, fileobj=None, 'extra_hosts was only introduced in API version 1.27' ) - encoded_extra_hosts = [ - '{}:{}'.format(k, v) for k, v in extra_hosts.items() - ] - params.update({'extrahosts': encoded_extra_hosts}) + if isinstance(extra_hosts, dict): + extra_hosts = utils.format_extra_hosts(extra_hosts) + params.update({'extrahosts': extra_hosts}) if context is not None: headers = {'Content-Type': 'application/tar'} diff --git a/docker/api/service.py b/docker/api/service.py index 4b555a5f59..9ce830ca48 100644 --- a/docker/api/service.py +++ b/docker/api/service.py @@ -4,45 +4,62 @@ def _check_api_features(version, task_template, update_config): + + def raise_version_error(param, min_version): + raise errors.InvalidVersion( + '{} is not supported in API version < {}'.format( + param, min_version + ) + ) + if update_config is not None: if utils.version_lt(version, '1.25'): if 'MaxFailureRatio' in update_config: - raise errors.InvalidVersion( - 'UpdateConfig.max_failure_ratio is not supported in' - ' API version < 1.25' - ) + raise_version_error('UpdateConfig.max_failure_ratio', '1.25') if 'Monitor' in update_config: - raise errors.InvalidVersion( - 'UpdateConfig.monitor is not supported in' - ' API version < 1.25' - ) + raise_version_error('UpdateConfig.monitor', '1.25') if task_template is not None: if 'ForceUpdate' in task_template and utils.version_lt( version, '1.25'): - raise errors.InvalidVersion( - 'force_update is not supported in API version < 1.25' - ) + raise_version_error('force_update', '1.25') if task_template.get('Placement'): if utils.version_lt(version, '1.30'): if task_template['Placement'].get('Platforms'): - raise errors.InvalidVersion( - 'Placement.platforms is not supported in' - ' API version < 1.30' - ) - + raise_version_error('Placement.platforms', '1.30') if utils.version_lt(version, '1.27'): if task_template['Placement'].get('Preferences'): - raise errors.InvalidVersion( - 'Placement.preferences is not supported in' - ' API version < 1.27' - ) - if task_template.get('ContainerSpec', {}).get('TTY'): + raise_version_error('Placement.preferences', '1.27') + + if task_template.get('ContainerSpec'): + container_spec = task_template.get('ContainerSpec') + if utils.version_lt(version, '1.25'): - raise errors.InvalidVersion( - 'ContainerSpec.TTY is not supported in API version < 1.25' - ) + if container_spec.get('TTY'): + raise_version_error('ContainerSpec.tty', '1.25') + if container_spec.get('Hostname') is not None: + raise_version_error('ContainerSpec.hostname', '1.25') + if container_spec.get('Hosts') is not None: + raise_version_error('ContainerSpec.hosts', '1.25') + if container_spec.get('Groups') is not None: + raise_version_error('ContainerSpec.groups', '1.25') + if container_spec.get('DNSConfig') is not None: + raise_version_error('ContainerSpec.dns_config', '1.25') + if container_spec.get('Healthcheck') is not None: + raise_version_error('ContainerSpec.healthcheck', '1.25') + + if utils.version_lt(version, '1.28'): + if container_spec.get('ReadOnly') is not None: + raise_version_error('ContainerSpec.dns_config', '1.28') + if container_spec.get('StopSignal') is not None: + raise_version_error('ContainerSpec.stop_signal', '1.28') + + if utils.version_lt(version, '1.30'): + if container_spec.get('Configs') is not None: + raise_version_error('ContainerSpec.configs', '1.30') + if container_spec.get('Privileges') is not None: + raise_version_error('ContainerSpec.privileges', '1.30') class ServiceApiMixin(object): diff --git a/docker/models/services.py b/docker/models/services.py index e1e2ea6a44..d45621bb4d 100644 --- a/docker/models/services.py +++ b/docker/models/services.py @@ -147,6 +147,22 @@ def create(self, image, command=None, **kwargs): user (str): User to run commands as. workdir (str): Working directory for commands to run. tty (boolean): Whether a pseudo-TTY should be allocated. + groups (:py:class:`list`): A list of additional groups that the + container process will run as. + open_stdin (boolean): Open ``stdin`` + read_only (boolean): Mount the container's root filesystem as read + only. + stop_signal (string): Set signal to stop the service's containers + healthcheck (Healthcheck): Healthcheck + configuration for this service. + hosts (:py:class:`dict`): A set of host to IP mappings to add to + the container's `hosts` file. + dns_config (DNSConfig): Specification for DNS + related configurations in resolver configuration file. + configs (:py:class:`list`): List of :py:class:`ConfigReference` + that will be exposed to the service. + privileges (Privileges): Security options for the service's + containers. Returns: (:py:class:`Service`) The created service. @@ -202,18 +218,27 @@ def list(self, **kwargs): # kwargs to copy straight over to ContainerSpec CONTAINER_SPEC_KWARGS = [ - 'image', - 'command', 'args', + 'command', + 'configs', + 'dns_config', 'env', + 'groups', + 'healthcheck', 'hostname', - 'workdir', - 'user', + 'hosts', + 'image', 'labels', 'mounts', - 'stop_grace_period', + 'open_stdin', + 'privileges' + 'read_only', 'secrets', - 'tty' + 'stop_grace_period', + 'stop_signal', + 'tty', + 'user', + 'workdir', ] # kwargs to copy straight over to TaskTemplate diff --git a/docker/types/__init__.py b/docker/types/__init__.py index edc919dfcf..39c93e344d 100644 --- a/docker/types/__init__.py +++ b/docker/types/__init__.py @@ -3,7 +3,8 @@ from .healthcheck import Healthcheck from .networks import EndpointConfig, IPAMConfig, IPAMPool, NetworkingConfig from .services import ( - ContainerSpec, DriverConfig, EndpointSpec, Mount, Placement, Resources, - RestartPolicy, SecretReference, ServiceMode, TaskTemplate, UpdateConfig + ConfigReference, ContainerSpec, DNSConfig, DriverConfig, EndpointSpec, + Mount, Placement, Privileges, Resources, RestartPolicy, SecretReference, + ServiceMode, TaskTemplate, UpdateConfig ) from .swarm import SwarmSpec, SwarmExternalCA diff --git a/docker/types/containers.py b/docker/types/containers.py index 3fc13d9d7f..13bea713ed 100644 --- a/docker/types/containers.py +++ b/docker/types/containers.py @@ -4,8 +4,8 @@ from .. import errors from ..utils.utils import ( convert_port_bindings, convert_tmpfs_mounts, convert_volume_binds, - format_environment, normalize_links, parse_bytes, parse_devices, - split_command, version_gte, version_lt, + format_environment, format_extra_hosts, normalize_links, parse_bytes, + parse_devices, split_command, version_gte, version_lt, ) from .base import DictType from .healthcheck import Healthcheck @@ -257,10 +257,7 @@ def __init__(self, version, binds=None, port_bindings=None, if extra_hosts is not None: if isinstance(extra_hosts, dict): - extra_hosts = [ - '{0}:{1}'.format(k, v) - for k, v in sorted(six.iteritems(extra_hosts)) - ] + extra_hosts = format_extra_hosts(extra_hosts) self['ExtraHosts'] = extra_hosts diff --git a/docker/types/healthcheck.py b/docker/types/healthcheck.py index 8ea9a35f5b..5a6a931576 100644 --- a/docker/types/healthcheck.py +++ b/docker/types/healthcheck.py @@ -4,6 +4,30 @@ class Healthcheck(DictType): + """ + Defines a healthcheck configuration for a container or service. + + Args: + + test (:py:class:`list` or str): Test to perform to determine + container health. Possible values: + - Empty list: Inherit healthcheck from parent image + - ``["NONE"]``: Disable healthcheck + - ``["CMD", args...]``: exec arguments directly. + - ``["CMD-SHELL", command]``: RUn command in the system's + default shell. + If a string is provided, it will be used as a ``CMD-SHELL`` + command. + interval (int): The time to wait between checks in nanoseconds. It + should be 0 or at least 1000000 (1 ms). + timeout (int): The time to wait before considering the check to + have hung. It should be 0 or at least 1000000 (1 ms). + retries (integer): The number of consecutive failures needed to + consider a container as unhealthy. + start_period (integer): Start period for the container to + initialize before starting health-retries countdown in + nanoseconds. It should be 0 or at least 1000000 (1 ms). + """ def __init__(self, **kwargs): test = kwargs.get('test', kwargs.get('Test')) if isinstance(test, six.string_types): diff --git a/docker/types/services.py b/docker/types/services.py index c2767404a6..c77db166fb 100644 --- a/docker/types/services.py +++ b/docker/types/services.py @@ -3,7 +3,8 @@ from .. import errors from ..constants import IS_WINDOWS_PLATFORM from ..utils import ( - check_resource, format_environment, parse_bytes, split_command + check_resource, format_environment, format_extra_hosts, parse_bytes, + split_command, ) @@ -84,13 +85,31 @@ class ContainerSpec(dict): :py:class:`~docker.types.Mount` class for details. stop_grace_period (int): Amount of time to wait for the container to terminate before forcefully killing it. - secrets (list of py:class:`SecretReference`): List of secrets to be + secrets (:py:class:`list`): List of :py:class:`SecretReference` to be made available inside the containers. tty (boolean): Whether a pseudo-TTY should be allocated. + groups (:py:class:`list`): A list of additional groups that the + container process will run as. + open_stdin (boolean): Open ``stdin`` + read_only (boolean): Mount the container's root filesystem as read + only. + stop_signal (string): Set signal to stop the service's containers + healthcheck (Healthcheck): Healthcheck + configuration for this service. + hosts (:py:class:`dict`): A set of host to IP mappings to add to + the container's `hosts` file. + dns_config (DNSConfig): Specification for DNS + related configurations in resolver configuration file. + configs (:py:class:`list`): List of :py:class:`ConfigReference` that + will be exposed to the service. + privileges (Privileges): Security options for the service's containers. """ def __init__(self, image, command=None, args=None, hostname=None, env=None, workdir=None, user=None, labels=None, mounts=None, - stop_grace_period=None, secrets=None, tty=None): + stop_grace_period=None, secrets=None, tty=None, groups=None, + open_stdin=None, read_only=None, stop_signal=None, + healthcheck=None, hosts=None, dns_config=None, configs=None, + privileges=None): self['Image'] = image if isinstance(command, six.string_types): @@ -109,8 +128,17 @@ def __init__(self, image, command=None, args=None, hostname=None, env=None, self['Dir'] = workdir if user is not None: self['User'] = user + if groups is not None: + self['Groups'] = groups + if stop_signal is not None: + self['StopSignal'] = stop_signal + if stop_grace_period is not None: + self['StopGracePeriod'] = stop_grace_period if labels is not None: self['Labels'] = labels + if hosts is not None: + self['Hosts'] = format_extra_hosts(hosts) + if mounts is not None: parsed_mounts = [] for mount in mounts: @@ -120,16 +148,30 @@ def __init__(self, image, command=None, args=None, hostname=None, env=None, # If mount already parsed parsed_mounts.append(mount) self['Mounts'] = parsed_mounts - if stop_grace_period is not None: - self['StopGracePeriod'] = stop_grace_period if secrets is not None: if not isinstance(secrets, list): raise TypeError('secrets must be a list') self['Secrets'] = secrets + if configs is not None: + if not isinstance(configs, list): + raise TypeError('configs must be a list') + self['Configs'] = configs + + if dns_config is not None: + self['DNSConfig'] = dns_config + if privileges is not None: + self['Privileges'] = privileges + if healthcheck is not None: + self['Healthcheck'] = healthcheck + if tty is not None: self['TTY'] = tty + if open_stdin is not None: + self['OpenStdin'] = open_stdin + if read_only is not None: + self['ReadOnly'] = read_only class Mount(dict): @@ -487,6 +529,34 @@ def __init__(self, secret_id, secret_name, filename=None, uid=None, } +class ConfigReference(dict): + """ + Config reference to be used as part of a :py:class:`ContainerSpec`. + Describes how a config is made accessible inside the service's + containers. + + Args: + config_id (string): Config's ID + config_name (string): Config's name as defined at its creation. + filename (string): Name of the file containing the config. Defaults + to the config's name if not specified. + uid (string): UID of the config file's owner. Default: 0 + gid (string): GID of the config file's group. Default: 0 + mode (int): File access mode inside the container. Default: 0o444 + """ + @check_resource('config_id') + def __init__(self, config_id, config_name, filename=None, uid=None, + gid=None, mode=0o444): + self['ConfigName'] = config_name + self['ConfigID'] = config_id + self['File'] = { + 'Name': filename or config_name, + 'UID': uid or '0', + 'GID': gid or '0', + 'Mode': mode + } + + class Placement(dict): """ Placement constraints to be used as part of a :py:class:`TaskTemplate` @@ -510,3 +580,75 @@ def __init__(self, constraints=None, preferences=None, platforms=None): self['Platforms'].append({ 'Architecture': plat[0], 'OS': plat[1] }) + + +class DNSConfig(dict): + """ + Specification for DNS related configurations in resolver configuration + file (``resolv.conf``). Part of a :py:class:`ContainerSpec` definition. + + Args: + nameservers (:py:class:`list`): The IP addresses of the name + servers. + search (:py:class:`list`): A search list for host-name lookup. + options (:py:class:`list`): A list of internal resolver variables + to be modified (e.g., ``debug``, ``ndots:3``, etc.). + """ + def __init__(self, nameservers=None, search=None, options=None): + self['Nameservers'] = nameservers + self['Search'] = search + self['Options'] = options + + +class Privileges(dict): + """ + Security options for a service's containers. + Part of a :py:class:`ContainerSpec` definition. + + Args: + credentialspec_file (str): Load credential spec from this file. + The file is read by the daemon, and must be present in the + CredentialSpecs subdirectory in the docker data directory, + which defaults to ``C:\ProgramData\Docker\`` on Windows. + Can not be combined with credentialspec_registry. + + credentialspec_registry (str): Load credential spec from this value + in the Windows registry. The specified registry value must be + located in: ``HKLM\SOFTWARE\Microsoft\Windows NT\CurrentVersion + \Virtualization\Containers\CredentialSpecs``. + Can not be combined with credentialspec_file. + + selinux_disable (boolean): Disable SELinux + selinux_user (string): SELinux user label + selinux_role (string): SELinux role label + selinux_type (string): SELinux type label + selinux_level (string): SELinux level label + """ + def __init__(self, credentialspec_file=None, credentialspec_registry=None, + selinux_disable=None, selinux_user=None, selinux_role=None, + selinux_type=None, selinux_level=None): + credential_spec = {} + if credentialspec_registry is not None: + credential_spec['Registry'] = credentialspec_registry + if credentialspec_file is not None: + credential_spec['File'] = credentialspec_file + + if len(credential_spec) > 1: + raise errors.InvalidArgument( + 'credentialspec_file and credentialspec_registry are mutually' + ' exclusive' + ) + + selinux_context = { + 'Disable': selinux_disable, + 'User': selinux_user, + 'Role': selinux_role, + 'Type': selinux_type, + 'Level': selinux_level, + } + + if len(credential_spec) > 0: + self['CredentialSpec'] = credential_spec + + if len(selinux_context) > 0: + self['SELinuxContext'] = selinux_context diff --git a/docker/utils/__init__.py b/docker/utils/__init__.py index b758cbd4ec..c162e3bd6a 100644 --- a/docker/utils/__init__.py +++ b/docker/utils/__init__.py @@ -8,6 +8,6 @@ create_host_config, parse_bytes, ping_registry, parse_env_file, version_lt, version_gte, decode_json_header, split_command, create_ipam_config, create_ipam_pool, parse_devices, normalize_links, convert_service_networks, - format_environment, create_archive + format_environment, create_archive, format_extra_hosts ) diff --git a/docker/utils/utils.py b/docker/utils/utils.py index d9a6d7c1ba..a123fd8f83 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -564,6 +564,12 @@ def format_env(key, value): return [format_env(*var) for var in six.iteritems(environment)] +def format_extra_hosts(extra_hosts): + return [ + '{}:{}'.format(k, v) for k, v in sorted(six.iteritems(extra_hosts)) + ] + + def create_host_config(self, *args, **kwargs): raise errors.DeprecatedMethod( 'utils.create_host_config has been removed. Please use a ' diff --git a/docs/api.rst b/docs/api.rst index 0b10f387db..2fce0a77a2 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -122,13 +122,17 @@ Configuration types .. py:module:: docker.types -.. autoclass:: IPAMConfig -.. autoclass:: IPAMPool +.. autoclass:: ConfigReference .. autoclass:: ContainerSpec +.. autoclass:: DNSConfig .. autoclass:: DriverConfig .. autoclass:: EndpointSpec +.. autoclass:: Healthcheck +.. autoclass:: IPAMConfig +.. autoclass:: IPAMPool .. autoclass:: Mount .. autoclass:: Placement +.. autoclass:: Privileges .. autoclass:: Resources .. autoclass:: RestartPolicy .. autoclass:: SecretReference From b1301637cf2669205b048df80f7d21c2ac3c4d68 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 26 Oct 2017 16:13:05 -0700 Subject: [PATCH 0496/1301] Add support for configs management Signed-off-by: Joffrey F --- docker/api/client.py | 2 + docker/api/config.py | 91 +++++++++++++ docker/client.py | 9 ++ docker/models/configs.py | 69 ++++++++++ docs/api.rst | 10 ++ docs/client.rst | 1 + docs/configs.rst | 30 +++++ docs/index.rst | 1 + tests/integration/api_config_test.py | 69 ++++++++++ tests/integration/api_service_test.py | 178 +++++++++++++++++++++++++- tests/integration/base.py | 7 + 11 files changed, 464 insertions(+), 3 deletions(-) create mode 100644 docker/api/config.py create mode 100644 docker/models/configs.py create mode 100644 docs/configs.rst create mode 100644 tests/integration/api_config_test.py diff --git a/docker/api/client.py b/docker/api/client.py index 1de10c77c0..cbe74b916f 100644 --- a/docker/api/client.py +++ b/docker/api/client.py @@ -9,6 +9,7 @@ import websocket from .build import BuildApiMixin +from .config import ConfigApiMixin from .container import ContainerApiMixin from .daemon import DaemonApiMixin from .exec_api import ExecApiMixin @@ -43,6 +44,7 @@ class APIClient( requests.Session, BuildApiMixin, + ConfigApiMixin, ContainerApiMixin, DaemonApiMixin, ExecApiMixin, diff --git a/docker/api/config.py b/docker/api/config.py new file mode 100644 index 0000000000..b46b09c7c1 --- /dev/null +++ b/docker/api/config.py @@ -0,0 +1,91 @@ +import base64 + +import six + +from .. import utils + + +class ConfigApiMixin(object): + @utils.minimum_version('1.25') + def create_config(self, name, data, labels=None): + """ + Create a config + + Args: + name (string): Name of the config + data (bytes): Config data to be stored + labels (dict): A mapping of labels to assign to the config + + Returns (dict): ID of the newly created config + """ + if not isinstance(data, bytes): + data = data.encode('utf-8') + + data = base64.b64encode(data) + if six.PY3: + data = data.decode('ascii') + body = { + 'Data': data, + 'Name': name, + 'Labels': labels + } + + url = self._url('/configs/create') + return self._result( + self._post_json(url, data=body), True + ) + + @utils.minimum_version('1.25') + @utils.check_resource('id') + def inspect_config(self, id): + """ + Retrieve config metadata + + Args: + id (string): Full ID of the config to remove + + Returns (dict): A dictionary of metadata + + Raises: + :py:class:`docker.errors.NotFound` + if no config with that ID exists + """ + url = self._url('/configs/{0}', id) + return self._result(self._get(url), True) + + @utils.minimum_version('1.25') + @utils.check_resource('id') + def remove_config(self, id): + """ + Remove a config + + Args: + id (string): Full ID of the config to remove + + Returns (boolean): True if successful + + Raises: + :py:class:`docker.errors.NotFound` + if no config with that ID exists + """ + url = self._url('/configs/{0}', id) + res = self._delete(url) + self._raise_for_status(res) + return True + + @utils.minimum_version('1.25') + def configs(self, filters=None): + """ + List configs + + Args: + filters (dict): A map of filters to process on the configs + list. Available filters: ``names`` + + Returns (list): A list of configs + """ + url = self._url('/configs') + params = {} + if filters: + params['filters'] = utils.convert_filters(filters) + return self._result(self._get(url, params=params), True) diff --git a/docker/client.py b/docker/client.py index ee361bb961..29968c1f0d 100644 --- a/docker/client.py +++ b/docker/client.py @@ -1,5 +1,6 @@ from .api.client import APIClient from .constants import DEFAULT_TIMEOUT_SECONDS +from .models.configs import ConfigCollection from .models.containers import ContainerCollection from .models.images import ImageCollection from .models.networks import NetworkCollection @@ -80,6 +81,14 @@ def from_env(cls, **kwargs): **kwargs_from_env(**kwargs)) # Resources + @property + def configs(self): + """ + An object for managing configs on the server. See the + :doc:`configs documentation ` for full details. + """ + return ConfigCollection(client=self) + @property def containers(self): """ diff --git a/docker/models/configs.py b/docker/models/configs.py new file mode 100644 index 0000000000..7f23f65007 --- /dev/null +++ b/docker/models/configs.py @@ -0,0 +1,69 @@ +from ..api import APIClient +from .resource import Model, Collection + + +class Config(Model): + """A config.""" + id_attribute = 'ID' + + def __repr__(self): + return "<%s: '%s'>" % (self.__class__.__name__, self.name) + + @property + def name(self): + return self.attrs['Spec']['Name'] + + def remove(self): + """ + Remove this config. + + Raises: + :py:class:`docker.errors.APIError` + If config failed to remove. + """ + return self.client.api.remove_config(self.id) + + +class ConfigCollection(Collection): + """Configs on the Docker server.""" + model = Config + + def create(self, **kwargs): + obj = self.client.api.create_config(**kwargs) + return self.prepare_model(obj) + create.__doc__ = APIClient.create_config.__doc__ + + def get(self, config_id): + """ + Get a config. + + Args: + config_id (str): Config ID. + + Returns: + (:py:class:`Config`): The config. + + Raises: + :py:class:`docker.errors.NotFound` + If the config does not exist. + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + return self.prepare_model(self.client.api.inspect_config(config_id)) + + def list(self, **kwargs): + """ + List configs. Similar to the ``docker config ls`` command. + + Args: + filters (dict): Server-side list filtering options. + + Returns: + (list of :py:class:`Config`): The configs. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + resp = self.client.api.configs(**kwargs) + return [self.prepare_model(obj) for obj in resp] diff --git a/docs/api.rst b/docs/api.rst index 2fce0a77a2..18993ad343 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -9,6 +9,16 @@ It's possible to use :py:class:`APIClient` directly. Some basic things (e.g. run .. autoclass:: docker.api.client.APIClient +Configs +------- + +.. py:module:: docker.api.config + +.. rst-class:: hide-signature +.. autoclass:: ConfigApiMixin + :members: + :undoc-members: + Containers ---------- diff --git a/docs/client.rst b/docs/client.rst index ac7a256a05..43d7c63be7 100644 --- a/docs/client.rst +++ b/docs/client.rst @@ -15,6 +15,7 @@ Client reference .. autoclass:: DockerClient() + .. autoattribute:: configs .. autoattribute:: containers .. autoattribute:: images .. autoattribute:: networks diff --git a/docs/configs.rst b/docs/configs.rst new file mode 100644 index 0000000000..d907ad4216 --- /dev/null +++ b/docs/configs.rst @@ -0,0 +1,30 @@ +Configs +======= + +.. py:module:: docker.models.configs + +Manage configs on the server. + +Methods available on ``client.configs``: + +.. rst-class:: hide-signature +.. py:class:: ConfigCollection + + .. automethod:: create + .. automethod:: get + .. automethod:: list + + +Config objects +-------------- + +.. autoclass:: Config() + + .. autoattribute:: id + .. autoattribute:: name + .. py:attribute:: attrs + + The raw representation of this object from the server. + + .. automethod:: reload + .. automethod:: remove diff --git a/docs/index.rst b/docs/index.rst index 9113bffcc8..39426b6819 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -80,6 +80,7 @@ That's just a taste of what you can do with the Docker SDK for Python. For more, :maxdepth: 2 client + configs containers images networks diff --git a/tests/integration/api_config_test.py b/tests/integration/api_config_test.py new file mode 100644 index 0000000000..fb6002a760 --- /dev/null +++ b/tests/integration/api_config_test.py @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- + +import docker +import pytest + +from ..helpers import force_leave_swarm, requires_api_version +from .base import BaseAPIIntegrationTest + + +@requires_api_version('1.30') +class ConfigAPITest(BaseAPIIntegrationTest): + def setUp(self): + super(ConfigAPITest, self).setUp() + self.init_swarm() + + def tearDown(self): + super(ConfigAPITest, self).tearDown() + force_leave_swarm(self.client) + + def test_create_config(self): + config_id = self.client.create_config( + 'favorite_character', 'sakuya izayoi' + ) + self.tmp_configs.append(config_id) + assert 'ID' in config_id + data = self.client.inspect_config(config_id) + assert data['Spec']['Name'] == 'favorite_character' + + def test_create_config_unicode_data(self): + config_id = self.client.create_config( + 'favorite_character', u'いざよいさくや' + ) + self.tmp_configs.append(config_id) + assert 'ID' in config_id + data = self.client.inspect_config(config_id) + assert data['Spec']['Name'] == 'favorite_character' + + def test_inspect_config(self): + config_name = 'favorite_character' + config_id = self.client.create_config( + config_name, 'sakuya izayoi' + ) + self.tmp_configs.append(config_id) + data = self.client.inspect_config(config_id) + assert data['Spec']['Name'] == config_name + assert 'ID' in data + assert 'Version' in data + + def test_remove_config(self): + config_name = 'favorite_character' + config_id = self.client.create_config( + config_name, 'sakuya izayoi' + ) + self.tmp_configs.append(config_id) + + assert self.client.remove_config(config_id) + with pytest.raises(docker.errors.NotFound): + self.client.inspect_config(config_id) + + def test_list_configs(self): + config_name = 'favorite_character' + config_id = self.client.create_config( + config_name, 'sakuya izayoi' + ) + self.tmp_configs.append(config_id) + + data = self.client.configs(filters={'name': ['favorite_character']}) + assert len(data) == 1 + assert data[0]['ID'] == config_id['ID'] diff --git a/tests/integration/api_service_test.py b/tests/integration/api_service_test.py index c966916ebb..56c3e683cf 100644 --- a/tests/integration/api_service_test.py +++ b/tests/integration/api_service_test.py @@ -473,7 +473,7 @@ def test_create_service_with_unicode_secret(self): secret_data = u'東方花映塚' secret_id = self.client.create_secret(secret_name, secret_data) self.tmp_secrets.append(secret_id) - secret_ref = docker.types.SecretReference(secret_id, secret_name) + secret_ref = docker.types.ConfigReference(secret_id, secret_name) container_spec = docker.types.ContainerSpec( 'busybox', ['sleep', '999'], secrets=[secret_ref] ) @@ -481,8 +481,8 @@ def test_create_service_with_unicode_secret(self): name = self.get_service_name() svc_id = self.client.create_service(task_tmpl, name=name) svc_info = self.client.inspect_service(svc_id) - assert 'Secrets' in svc_info['Spec']['TaskTemplate']['ContainerSpec'] - secrets = svc_info['Spec']['TaskTemplate']['ContainerSpec']['Secrets'] + assert 'Configs' in svc_info['Spec']['TaskTemplate']['ContainerSpec'] + secrets = svc_info['Spec']['TaskTemplate']['ContainerSpec']['Configs'] assert secrets[0] == secret_ref container = self.get_service_container(name) @@ -493,3 +493,175 @@ def test_create_service_with_unicode_secret(self): container_secret = self.client.exec_start(exec_id) container_secret = container_secret.decode('utf-8') assert container_secret == secret_data + + @requires_api_version('1.25') + def test_create_service_with_config(self): + config_name = 'favorite_touhou' + config_data = b'phantasmagoria of flower view' + config_id = self.client.create_config(config_name, config_data) + self.tmp_configs.append(config_id) + config_ref = docker.types.ConfigReference(config_id, config_name) + container_spec = docker.types.ContainerSpec( + 'busybox', ['sleep', '999'], configs=[config_ref] + ) + task_tmpl = docker.types.TaskTemplate(container_spec) + name = self.get_service_name() + svc_id = self.client.create_service(task_tmpl, name=name) + svc_info = self.client.inspect_service(svc_id) + assert 'Configs' in svc_info['Spec']['TaskTemplate']['ContainerSpec'] + configs = svc_info['Spec']['TaskTemplate']['ContainerSpec']['Configs'] + assert configs[0] == config_ref + + container = self.get_service_container(name) + assert container is not None + exec_id = self.client.exec_create( + container, 'cat /run/configs/{0}'.format(config_name) + ) + assert self.client.exec_start(exec_id) == config_data + + @requires_api_version('1.25') + def test_create_service_with_unicode_config(self): + config_name = 'favorite_touhou' + config_data = u'東方花映塚' + config_id = self.client.create_config(config_name, config_data) + self.tmp_configs.append(config_id) + config_ref = docker.types.ConfigReference(config_id, config_name) + container_spec = docker.types.ContainerSpec( + 'busybox', ['sleep', '999'], configs=[config_ref] + ) + task_tmpl = docker.types.TaskTemplate(container_spec) + name = self.get_service_name() + svc_id = self.client.create_service(task_tmpl, name=name) + svc_info = self.client.inspect_service(svc_id) + assert 'Configs' in svc_info['Spec']['TaskTemplate']['ContainerSpec'] + configs = svc_info['Spec']['TaskTemplate']['ContainerSpec']['Configs'] + assert configs[0] == config_ref + + container = self.get_service_container(name) + assert container is not None + exec_id = self.client.exec_create( + container, 'cat /run/configs/{0}'.format(config_name) + ) + container_config = self.client.exec_start(exec_id) + container_config = container_config.decode('utf-8') + assert container_config == config_data + + @requires_api_version('1.25') + def test_create_service_with_hosts(self): + container_spec = docker.types.ContainerSpec( + 'busybox', ['sleep', '999'], hosts={ + 'foobar': '127.0.0.1', + 'baz': '8.8.8.8', + } + ) + task_tmpl = docker.types.TaskTemplate(container_spec) + name = self.get_service_name() + svc_id = self.client.create_service(task_tmpl, name=name) + svc_info = self.client.inspect_service(svc_id) + assert 'Hosts' in svc_info['Spec']['TaskTemplate']['ContainerSpec'] + hosts = svc_info['Spec']['TaskTemplate']['ContainerSpec']['Hosts'] + assert len(hosts) == 2 + assert 'foobar:127.0.0.1' in hosts + assert 'baz:8.8.8.8' in hosts + + @requires_api_version('1.25') + def test_create_service_with_hostname(self): + container_spec = docker.types.ContainerSpec( + 'busybox', ['sleep', '999'], hostname='foobar.baz.com' + ) + task_tmpl = docker.types.TaskTemplate(container_spec) + name = self.get_service_name() + svc_id = self.client.create_service(task_tmpl, name=name) + svc_info = self.client.inspect_service(svc_id) + assert 'Hostname' in svc_info['Spec']['TaskTemplate']['ContainerSpec'] + assert ( + svc_info['Spec']['TaskTemplate']['ContainerSpec']['Hostname'] == + 'foobar.baz.com' + ) + + @requires_api_version('1.25') + def test_create_service_with_groups(self): + container_spec = docker.types.ContainerSpec( + 'busybox', ['sleep', '999'], groups=['shrinemaidens', 'youkais'] + ) + task_tmpl = docker.types.TaskTemplate(container_spec) + name = self.get_service_name() + svc_id = self.client.create_service(task_tmpl, name=name) + svc_info = self.client.inspect_service(svc_id) + assert 'Groups' in svc_info['Spec']['TaskTemplate']['ContainerSpec'] + groups = svc_info['Spec']['TaskTemplate']['ContainerSpec']['Groups'] + assert len(groups) == 2 + assert 'shrinemaidens' in groups + assert 'youkais' in groups + + @requires_api_version('1.25') + def test_create_service_with_dns_config(self): + dns_config = docker.types.DNSConfig( + nameservers=['8.8.8.8', '8.8.4.4'], + search=['local'], options=['debug'] + ) + container_spec = docker.types.ContainerSpec( + BUSYBOX, ['sleep', '999'], dns_config=dns_config + ) + task_tmpl = docker.types.TaskTemplate(container_spec) + name = self.get_service_name() + svc_id = self.client.create_service(task_tmpl, name=name) + svc_info = self.client.inspect_service(svc_id) + assert 'DNSConfig' in svc_info['Spec']['TaskTemplate']['ContainerSpec'] + assert ( + dns_config == + svc_info['Spec']['TaskTemplate']['ContainerSpec']['DNSConfig'] + ) + + @requires_api_version('1.25') + def test_create_service_with_healthcheck(self): + second = 1000000000 + hc = docker.types.Healthcheck( + test='true', retries=3, timeout=1 * second, + start_period=3 * second, interval=second / 2, + ) + container_spec = docker.types.ContainerSpec( + BUSYBOX, ['sleep', '999'], healthcheck=hc + ) + task_tmpl = docker.types.TaskTemplate(container_spec) + name = self.get_service_name() + svc_id = self.client.create_service(task_tmpl, name=name) + svc_info = self.client.inspect_service(svc_id) + assert ( + 'Healthcheck' in svc_info['Spec']['TaskTemplate']['ContainerSpec'] + ) + assert ( + hc == + svc_info['Spec']['TaskTemplate']['ContainerSpec']['Healthcheck'] + ) + + @requires_api_version('1.28') + def test_create_service_with_readonly(self): + container_spec = docker.types.ContainerSpec( + BUSYBOX, ['sleep', '999'], read_only=True + ) + task_tmpl = docker.types.TaskTemplate(container_spec) + name = self.get_service_name() + svc_id = self.client.create_service(task_tmpl, name=name) + svc_info = self.client.inspect_service(svc_id) + assert ( + 'ReadOnly' in svc_info['Spec']['TaskTemplate']['ContainerSpec'] + ) + assert svc_info['Spec']['TaskTemplate']['ContainerSpec']['ReadOnly'] + + @requires_api_version('1.28') + def test_create_service_with_stop_signal(self): + container_spec = docker.types.ContainerSpec( + BUSYBOX, ['sleep', '999'], stop_signal='SIGINT' + ) + task_tmpl = docker.types.TaskTemplate(container_spec) + name = self.get_service_name() + svc_id = self.client.create_service(task_tmpl, name=name) + svc_info = self.client.inspect_service(svc_id) + assert ( + 'StopSignal' in svc_info['Spec']['TaskTemplate']['ContainerSpec'] + ) + assert ( + svc_info['Spec']['TaskTemplate']['ContainerSpec']['StopSignal'] == + 'SIGINT' + ) diff --git a/tests/integration/base.py b/tests/integration/base.py index 0c0cd06564..701e7fc293 100644 --- a/tests/integration/base.py +++ b/tests/integration/base.py @@ -29,6 +29,7 @@ def setUp(self): self.tmp_networks = [] self.tmp_plugins = [] self.tmp_secrets = [] + self.tmp_configs = [] def tearDown(self): client = docker.from_env(version=TEST_API_VERSION) @@ -59,6 +60,12 @@ def tearDown(self): except docker.errors.APIError: pass + for config in self.tmp_configs: + try: + client.api.remove_config(config) + except docker.errors.APIError: + pass + for folder in self.tmp_folders: shutil.rmtree(folder) From 2cb78062b14029c2834b30d6a407168bcf200ed3 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 26 Oct 2017 16:50:53 -0700 Subject: [PATCH 0497/1301] More ContainerSpec tests Signed-off-by: Joffrey F --- tests/integration/api_config_test.py | 15 ++++---- tests/integration/api_secret_test.py | 15 ++++---- tests/integration/api_service_test.py | 51 +++++++++++++++++++-------- tests/integration/base.py | 20 +++++++---- 4 files changed, 69 insertions(+), 32 deletions(-) diff --git a/tests/integration/api_config_test.py b/tests/integration/api_config_test.py index fb6002a760..0ffd7675c8 100644 --- a/tests/integration/api_config_test.py +++ b/tests/integration/api_config_test.py @@ -9,13 +9,16 @@ @requires_api_version('1.30') class ConfigAPITest(BaseAPIIntegrationTest): - def setUp(self): - super(ConfigAPITest, self).setUp() - self.init_swarm() + @classmethod + def setup_class(cls): + client = cls.get_client_instance() + force_leave_swarm(client) + cls._init_swarm(client) - def tearDown(self): - super(ConfigAPITest, self).tearDown() - force_leave_swarm(self.client) + @classmethod + def teardown_class(cls): + client = cls.get_client_instance() + force_leave_swarm(client) def test_create_config(self): config_id = self.client.create_config( diff --git a/tests/integration/api_secret_test.py b/tests/integration/api_secret_test.py index dcd880f49c..b3d93b8fc1 100644 --- a/tests/integration/api_secret_test.py +++ b/tests/integration/api_secret_test.py @@ -9,13 +9,16 @@ @requires_api_version('1.25') class SecretAPITest(BaseAPIIntegrationTest): - def setUp(self): - super(SecretAPITest, self).setUp() - self.init_swarm() + @classmethod + def setup_class(cls): + client = cls.get_client_instance() + force_leave_swarm(client) + cls._init_swarm(client) - def tearDown(self): - super(SecretAPITest, self).tearDown() - force_leave_swarm(self.client) + @classmethod + def teardown_class(cls): + client = cls.get_client_instance() + force_leave_swarm(client) def test_create_secret(self): secret_id = self.client.create_secret( diff --git a/tests/integration/api_service_test.py b/tests/integration/api_service_test.py index 56c3e683cf..baa6afa783 100644 --- a/tests/integration/api_service_test.py +++ b/tests/integration/api_service_test.py @@ -13,19 +13,24 @@ class ServiceTest(BaseAPIIntegrationTest): - def setUp(self): - super(ServiceTest, self).setUp() - force_leave_swarm(self.client) - self.init_swarm() + @classmethod + def setup_class(cls): + client = cls.get_client_instance() + force_leave_swarm(client) + cls._init_swarm(client) + + @classmethod + def teardown_class(cls): + client = cls.get_client_instance() + force_leave_swarm(client) def tearDown(self): - super(ServiceTest, self).tearDown() for service in self.client.services(filters={'name': 'dockerpytest_'}): try: self.client.remove_service(service['ID']) except docker.errors.APIError: pass - force_leave_swarm(self.client) + super(ServiceTest, self).tearDown() def get_service_name(self): return 'dockerpytest_{0:x}'.format(random.getrandbits(64)) @@ -473,7 +478,7 @@ def test_create_service_with_unicode_secret(self): secret_data = u'東方花映塚' secret_id = self.client.create_secret(secret_name, secret_data) self.tmp_secrets.append(secret_id) - secret_ref = docker.types.ConfigReference(secret_id, secret_name) + secret_ref = docker.types.SecretReference(secret_id, secret_name) container_spec = docker.types.ContainerSpec( 'busybox', ['sleep', '999'], secrets=[secret_ref] ) @@ -481,8 +486,8 @@ def test_create_service_with_unicode_secret(self): name = self.get_service_name() svc_id = self.client.create_service(task_tmpl, name=name) svc_info = self.client.inspect_service(svc_id) - assert 'Configs' in svc_info['Spec']['TaskTemplate']['ContainerSpec'] - secrets = svc_info['Spec']['TaskTemplate']['ContainerSpec']['Configs'] + assert 'Secrets' in svc_info['Spec']['TaskTemplate']['ContainerSpec'] + secrets = svc_info['Spec']['TaskTemplate']['ContainerSpec']['Secrets'] assert secrets[0] == secret_ref container = self.get_service_container(name) @@ -494,7 +499,7 @@ def test_create_service_with_unicode_secret(self): container_secret = container_secret.decode('utf-8') assert container_secret == secret_data - @requires_api_version('1.25') + @requires_api_version('1.30') def test_create_service_with_config(self): config_name = 'favorite_touhou' config_data = b'phantasmagoria of flower view' @@ -515,11 +520,11 @@ def test_create_service_with_config(self): container = self.get_service_container(name) assert container is not None exec_id = self.client.exec_create( - container, 'cat /run/configs/{0}'.format(config_name) + container, 'cat /{0}'.format(config_name) ) assert self.client.exec_start(exec_id) == config_data - @requires_api_version('1.25') + @requires_api_version('1.30') def test_create_service_with_unicode_config(self): config_name = 'favorite_touhou' config_data = u'東方花映塚' @@ -540,7 +545,7 @@ def test_create_service_with_unicode_config(self): container = self.get_service_container(name) assert container is not None exec_id = self.client.exec_create( - container, 'cat /run/configs/{0}'.format(config_name) + container, 'cat /{0}'.format(config_name) ) container_config = self.client.exec_start(exec_id) container_config = container_config.decode('utf-8') @@ -618,7 +623,7 @@ def test_create_service_with_healthcheck(self): second = 1000000000 hc = docker.types.Healthcheck( test='true', retries=3, timeout=1 * second, - start_period=3 * second, interval=second / 2, + start_period=3 * second, interval=int(second / 2), ) container_spec = docker.types.ContainerSpec( BUSYBOX, ['sleep', '999'], healthcheck=hc @@ -665,3 +670,21 @@ def test_create_service_with_stop_signal(self): svc_info['Spec']['TaskTemplate']['ContainerSpec']['StopSignal'] == 'SIGINT' ) + + @requires_api_version('1.30') + def test_create_service_with_privileges(self): + priv = docker.types.Privileges(selinux_disable=True) + container_spec = docker.types.ContainerSpec( + BUSYBOX, ['sleep', '999'], privileges=priv + ) + task_tmpl = docker.types.TaskTemplate(container_spec) + name = self.get_service_name() + svc_id = self.client.create_service(task_tmpl, name=name) + svc_info = self.client.inspect_service(svc_id) + assert ( + 'Privileges' in svc_info['Spec']['TaskTemplate']['ContainerSpec'] + ) + privileges = ( + svc_info['Spec']['TaskTemplate']['ContainerSpec']['Privileges'] + ) + assert privileges['SELinuxContext']['Disable'] is True diff --git a/tests/integration/base.py b/tests/integration/base.py index 701e7fc293..4f929014bd 100644 --- a/tests/integration/base.py +++ b/tests/integration/base.py @@ -78,14 +78,24 @@ class BaseAPIIntegrationTest(BaseIntegrationTest): def setUp(self): super(BaseAPIIntegrationTest, self).setUp() - self.client = docker.APIClient( - version=TEST_API_VERSION, timeout=60, **kwargs_from_env() - ) + self.client = self.get_client_instance() def tearDown(self): super(BaseAPIIntegrationTest, self).tearDown() self.client.close() + @staticmethod + def get_client_instance(): + return docker.APIClient( + version=TEST_API_VERSION, timeout=60, **kwargs_from_env() + ) + + @staticmethod + def _init_swarm(client, **kwargs): + return client.init_swarm( + '127.0.0.1', listen_addr=helpers.swarm_listen_addr(), **kwargs + ) + def run_container(self, *args, **kwargs): container = self.client.create_container(*args, **kwargs) self.tmp_containers.append(container) @@ -116,6 +126,4 @@ def execute(self, container, cmd, exit_code=0, **kwargs): assert actual_exit_code == exit_code, msg def init_swarm(self, **kwargs): - return self.client.init_swarm( - '127.0.0.1', listen_addr=helpers.swarm_listen_addr(), **kwargs - ) + return self._init_swarm(self.client, **kwargs) From 80985cb5b2262dcd5263ef78635aa66609e132eb Mon Sep 17 00:00:00 2001 From: Alessandro Baldo Date: Wed, 1 Nov 2017 01:44:21 +0100 Subject: [PATCH 0498/1301] Improve docs for service list filters - add "label" and "mode" to the list of available filter keys in the high-level service API - add "label" and "mode" to the list of available filter keys in the low-level service API - add integration tests Signed-off-by: Alessandro Baldo --- docker/api/service.py | 3 ++- docker/models/services.py | 3 ++- tests/integration/api_service_test.py | 15 +++++++++++++-- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/docker/api/service.py b/docker/api/service.py index 9ce830ca48..e6b48768b4 100644 --- a/docker/api/service.py +++ b/docker/api/service.py @@ -201,7 +201,8 @@ def services(self, filters=None): Args: filters (dict): Filters to process on the nodes list. Valid - filters: ``id`` and ``name``. Default: ``None``. + filters: ``id``, ``name`` , ``label`` and ``mode``. + Default: ``None``. Returns: A list of dictionaries containing data about each service. diff --git a/docker/models/services.py b/docker/models/services.py index d45621bb4d..f2a5d355a0 100644 --- a/docker/models/services.py +++ b/docker/models/services.py @@ -201,7 +201,8 @@ def list(self, **kwargs): Args: filters (dict): Filters to process on the nodes list. Valid - filters: ``id`` and ``name``. Default: ``None``. + filters: ``id``, ``name`` , ``label`` and ``mode``. + Default: ``None``. Returns: (list of :py:class:`Service`): The services. diff --git a/tests/integration/api_service_test.py b/tests/integration/api_service_test.py index baa6afa783..8c6d4af54c 100644 --- a/tests/integration/api_service_test.py +++ b/tests/integration/api_service_test.py @@ -52,7 +52,7 @@ def get_service_container(self, service_name, attempts=20, interval=0.5, return None time.sleep(interval) - def create_simple_service(self, name=None): + def create_simple_service(self, name=None, labels=None): if name: name = 'dockerpytest_{0}'.format(name) else: @@ -62,7 +62,9 @@ def create_simple_service(self, name=None): BUSYBOX, ['echo', 'hello'] ) task_tmpl = docker.types.TaskTemplate(container_spec) - return name, self.client.create_service(task_tmpl, name=name) + return name, self.client.create_service( + task_tmpl, name=name, labels=labels + ) @requires_api_version('1.24') def test_list_services(self): @@ -76,6 +78,15 @@ def test_list_services(self): assert len(test_services) == 1 assert 'dockerpytest_' in test_services[0]['Spec']['Name'] + @requires_api_version('1.24') + def test_list_services_filter_by_label(self): + test_services = self.client.services(filters={'label': 'test_label'}) + assert len(test_services) == 0 + self.create_simple_service(labels={'test_label': 'testing'}) + test_services = self.client.services(filters={'label': 'test_label'}) + assert len(test_services) == 1 + assert test_services[0]['Spec']['Labels']['test_label'] == 'testing' + def test_inspect_service_by_id(self): svc_name, svc_id = self.create_simple_service() svc_info = self.client.inspect_service(svc_id) From a0853622f98abebe4cb0a837d4fc287000e9805a Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 1 Nov 2017 17:13:09 -0700 Subject: [PATCH 0499/1301] Add support for secret driver in create_secret Signed-off-by: Joffrey F --- docker/api/secret.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/docker/api/secret.py b/docker/api/secret.py index 1760a39469..fa4c2ab81d 100644 --- a/docker/api/secret.py +++ b/docker/api/secret.py @@ -2,12 +2,13 @@ import six +from .. import errors from .. import utils class SecretApiMixin(object): @utils.minimum_version('1.25') - def create_secret(self, name, data, labels=None): + def create_secret(self, name, data, labels=None, driver=None): """ Create a secret @@ -15,6 +16,8 @@ def create_secret(self, name, data, labels=None): name (string): Name of the secret data (bytes): Secret data to be stored labels (dict): A mapping of labels to assign to the secret + driver (DriverConfig): A custom driver configuration. If + unspecified, the default ``internal`` driver will be used Returns (dict): ID of the newly created secret """ @@ -30,6 +33,14 @@ def create_secret(self, name, data, labels=None): 'Labels': labels } + if driver is not None: + if utils.version_lt(self._version, '1.31'): + raise errors.InvalidVersion( + 'Secret driver is only available for API version > 1.31' + ) + + body['Driver'] = driver + url = self._url('/secrets/create') return self._result( self._post_json(url, data=body), True From 114512a9bf5aaccaf4c1fc58f86c3677c80436f1 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 1 Nov 2017 17:13:28 -0700 Subject: [PATCH 0500/1301] Doc fixes Signed-off-by: Joffrey F --- docker/api/build.py | 4 ++-- docker/api/plugin.py | 8 ++++---- docker/types/healthcheck.py | 13 +++++++------ docker/types/services.py | 17 +++++++++-------- 4 files changed, 22 insertions(+), 20 deletions(-) diff --git a/docker/api/build.py b/docker/api/build.py index 25f271a4e3..9ff2dfb3c9 100644 --- a/docker/api/build.py +++ b/docker/api/build.py @@ -93,8 +93,8 @@ def build(self, path=None, tag=None, quiet=False, fileobj=None, shmsize (int): Size of `/dev/shm` in bytes. The size must be greater than 0. If omitted the system uses 64MB labels (dict): A dictionary of labels to set on the image - cache_from (list): A list of images used for build cache - resolution + cache_from (:py:class:`list`): A list of images used for build + cache resolution target (str): Name of the build-stage to build in a multi-stage Dockerfile network_mode (str): networking mode for the run commands during diff --git a/docker/api/plugin.py b/docker/api/plugin.py index 87520ccee3..73f185251e 100644 --- a/docker/api/plugin.py +++ b/docker/api/plugin.py @@ -110,8 +110,8 @@ def pull_plugin(self, remote, privileges, name=None): remote (string): Remote reference for the plugin to install. The ``:latest`` tag is optional, and is the default if omitted. - privileges (list): A list of privileges the user consents to - grant to the plugin. Can be retrieved using + privileges (:py:class:`list`): A list of privileges the user + consents to grant to the plugin. Can be retrieved using :py:meth:`~plugin_privileges`. name (string): Local name for the pulled plugin. The ``:latest`` tag is optional, and is the default if omitted. @@ -225,8 +225,8 @@ def upgrade_plugin(self, name, remote, privileges): tag is optional and is the default if omitted. remote (string): Remote reference to upgrade to. The ``:latest`` tag is optional and is the default if omitted. - privileges (list): A list of privileges the user consents to - grant to the plugin. Can be retrieved using + privileges (:py:class:`list`): A list of privileges the user + consents to grant to the plugin. Can be retrieved using :py:meth:`~plugin_privileges`. Returns: diff --git a/docker/types/healthcheck.py b/docker/types/healthcheck.py index 5a6a931576..61857c21ce 100644 --- a/docker/types/healthcheck.py +++ b/docker/types/healthcheck.py @@ -8,14 +8,15 @@ class Healthcheck(DictType): Defines a healthcheck configuration for a container or service. Args: - test (:py:class:`list` or str): Test to perform to determine container health. Possible values: - - Empty list: Inherit healthcheck from parent image - - ``["NONE"]``: Disable healthcheck - - ``["CMD", args...]``: exec arguments directly. - - ``["CMD-SHELL", command]``: RUn command in the system's - default shell. + + - Empty list: Inherit healthcheck from parent image + - ``["NONE"]``: Disable healthcheck + - ``["CMD", args...]``: exec arguments directly. + - ``["CMD-SHELL", command]``: RUn command in the system's + default shell. + If a string is provided, it will be used as a ``CMD-SHELL`` command. interval (int): The time to wait between checks in nanoseconds. It diff --git a/docker/types/services.py b/docker/types/services.py index c77db166fb..9031e609ab 100644 --- a/docker/types/services.py +++ b/docker/types/services.py @@ -405,8 +405,9 @@ class DriverConfig(dict): """ Indicates which driver to use, as well as its configuration. Can be used as ``log_driver`` in a :py:class:`~docker.types.ContainerSpec`, - and for the `driver_config` in a volume - :py:class:`~docker.types.Mount`. + for the `driver_config` in a volume :py:class:`~docker.types.Mount`, or + as the driver object in + :py:meth:`create_secret`. Args: @@ -562,12 +563,12 @@ class Placement(dict): Placement constraints to be used as part of a :py:class:`TaskTemplate` Args: - constraints (list): A list of constraints - preferences (list): Preferences provide a way to make the - scheduler aware of factors such as topology. They are provided - in order from highest to lowest precedence. - platforms (list): A list of platforms expressed as ``(arch, os)`` - tuples + constraints (:py:class:`list`): A list of constraints + preferences (:py:class:`list`): Preferences provide a way to make + the scheduler aware of factors such as topology. They are + provided in order from highest to lowest precedence. + platforms (:py:class:`list`): A list of platforms expressed as + ``(arch, os)`` tuples """ def __init__(self, constraints=None, preferences=None, platforms=None): if constraints is not None: From 047c67b31e2087d5e900072166921d55649f8b6f Mon Sep 17 00:00:00 2001 From: Chris Harris Date: Thu, 5 Oct 2017 12:14:17 -0400 Subject: [PATCH 0501/1301] Prevent data loss when attaching to container The use of buffering within httplib.HTTPResponse can cause data to be lost. socket.makefile() is called without a bufsize, which causes a buffer to be used when recieving data. The attach methods do a HTTP upgrade to tcp before the raw socket is using to stream data from the container. The problem is that if the container starts stream data while httplib/http.client is reading the response to the attach request part of the data ends will end up in the buffer of fileobject created within the HTTPResponse object. This data is lost as after the attach request data is read directly from the raw socket. Signed-off-by: Chris Harris --- docker/transport/unixconn.py | 33 ++++++++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/docker/transport/unixconn.py b/docker/transport/unixconn.py index 3565cfb629..16e22a8ecb 100644 --- a/docker/transport/unixconn.py +++ b/docker/transport/unixconn.py @@ -34,6 +34,25 @@ def connect(self): self.sock = sock +class AttachHTTPResponse(httplib.HTTPResponse): + ''' + A HTTPResponse object that doesn't use a buffered fileobject. + ''' + def __init__(self, sock, *args, **kwargs): + # Delegate to super class + httplib.HTTPResponse.__init__(self, sock, *args, **kwargs) + + # Override fp with a fileobject that doesn't buffer + self.fp = sock.makefile('rb', 0) + + +class AttachUnixHTTPConnection(UnixHTTPConnection): + ''' + A HTTPConnection that returns responses that don't used buffering. + ''' + response_class = AttachHTTPResponse + + class UnixHTTPConnectionPool(urllib3.connectionpool.HTTPConnectionPool): def __init__(self, base_url, socket_path, timeout=60, maxsize=10): super(UnixHTTPConnectionPool, self).__init__( @@ -44,9 +63,17 @@ def __init__(self, base_url, socket_path, timeout=60, maxsize=10): self.timeout = timeout def _new_conn(self): - return UnixHTTPConnection( - self.base_url, self.socket_path, self.timeout - ) + # Special case for attach url, as we do a http upgrade to tcp and + # a buffered connection can cause data loss. + path = urllib3.util.parse_url(self.base_url).path + if path.endswith('attach'): + return AttachUnixHTTPConnection( + self.base_url, self.socket_path, self.timeout + ) + else: + return UnixHTTPConnection( + self.base_url, self.socket_path, self.timeout + ) class UnixAdapter(requests.adapters.HTTPAdapter): From e055729104dbc60bb9bbebce09686ac8a94c5809 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 1 Nov 2017 18:03:29 -0700 Subject: [PATCH 0502/1301] Disable buffering based on presence of Connection Upgrade headers Signed-off-by: Joffrey F --- Makefile | 20 +++++++------- docker/transport/unixconn.py | 52 +++++++++++++++++------------------- 2 files changed, 35 insertions(+), 37 deletions(-) diff --git a/Makefile b/Makefile index 991b93a1db..efa4232e9b 100644 --- a/Makefile +++ b/Makefile @@ -27,19 +27,19 @@ test: flake8 unit-test unit-test-py3 integration-dind integration-dind-ssl .PHONY: unit-test unit-test: build - docker run --rm docker-sdk-python py.test tests/unit + docker run -t --rm docker-sdk-python py.test tests/unit .PHONY: unit-test-py3 unit-test-py3: build-py3 - docker run --rm docker-sdk-python3 py.test tests/unit + docker run -t --rm docker-sdk-python3 py.test tests/unit .PHONY: integration-test integration-test: build - docker run --rm -v /var/run/docker.sock:/var/run/docker.sock docker-sdk-python py.test tests/integration/${file} + docker run -t --rm -v /var/run/docker.sock:/var/run/docker.sock docker-sdk-python py.test -v tests/integration/${file} .PHONY: integration-test-py3 integration-test-py3: build-py3 - docker run --rm -v /var/run/docker.sock:/var/run/docker.sock docker-sdk-python3 py.test tests/integration/${file} + docker run -t --rm -v /var/run/docker.sock:/var/run/docker.sock docker-sdk-python3 py.test tests/integration/${file} TEST_API_VERSION ?= 1.30 TEST_ENGINE_VERSION ?= 17.06.0-ce @@ -49,9 +49,9 @@ integration-dind: build build-py3 docker rm -vf dpy-dind || : docker run -d --name dpy-dind --privileged dockerswarm/dind:${TEST_ENGINE_VERSION} dockerd\ -H tcp://0.0.0.0:2375 --experimental - docker run --rm --env="DOCKER_HOST=tcp://docker:2375" --env="DOCKER_TEST_API_VERSION=${TEST_API_VERSION}"\ + docker run -t --rm --env="DOCKER_HOST=tcp://docker:2375" --env="DOCKER_TEST_API_VERSION=${TEST_API_VERSION}"\ --link=dpy-dind:docker docker-sdk-python py.test tests/integration - docker run --rm --env="DOCKER_HOST=tcp://docker:2375" --env="DOCKER_TEST_API_VERSION=${TEST_API_VERSION}"\ + docker run -t --rm --env="DOCKER_HOST=tcp://docker:2375" --env="DOCKER_TEST_API_VERSION=${TEST_API_VERSION}"\ --link=dpy-dind:docker docker-sdk-python3 py.test tests/integration docker rm -vf dpy-dind @@ -63,21 +63,21 @@ integration-dind-ssl: build-dind-certs build build-py3 -v /tmp --privileged dockerswarm/dind:${TEST_ENGINE_VERSION} dockerd --tlsverify\ --tlscacert=/certs/ca.pem --tlscert=/certs/server-cert.pem\ --tlskey=/certs/server-key.pem -H tcp://0.0.0.0:2375 --experimental - docker run --rm --volumes-from dpy-dind-ssl --env="DOCKER_HOST=tcp://docker:2375"\ + docker run -t --rm --volumes-from dpy-dind-ssl --env="DOCKER_HOST=tcp://docker:2375"\ --env="DOCKER_TLS_VERIFY=1" --env="DOCKER_CERT_PATH=/certs" --env="DOCKER_TEST_API_VERSION=${TEST_API_VERSION}"\ --link=dpy-dind-ssl:docker docker-sdk-python py.test tests/integration - docker run --rm --volumes-from dpy-dind-ssl --env="DOCKER_HOST=tcp://docker:2375"\ + docker run -t --rm --volumes-from dpy-dind-ssl --env="DOCKER_HOST=tcp://docker:2375"\ --env="DOCKER_TLS_VERIFY=1" --env="DOCKER_CERT_PATH=/certs" --env="DOCKER_TEST_API_VERSION=${TEST_API_VERSION}"\ --link=dpy-dind-ssl:docker docker-sdk-python3 py.test tests/integration docker rm -vf dpy-dind-ssl dpy-dind-certs .PHONY: flake8 flake8: build - docker run --rm docker-sdk-python flake8 docker tests + docker run -t --rm docker-sdk-python flake8 docker tests .PHONY: docs docs: build-docs - docker run --rm -it -v `pwd`:/src docker-sdk-python-docs sphinx-build docs docs/_build + docker run --rm -t -v `pwd`:/src docker-sdk-python-docs sphinx-build docs docs/_build .PHONY: shell shell: build diff --git a/docker/transport/unixconn.py b/docker/transport/unixconn.py index 16e22a8ecb..7cb877141e 100644 --- a/docker/transport/unixconn.py +++ b/docker/transport/unixconn.py @@ -18,7 +18,20 @@ RecentlyUsedContainer = urllib3._collections.RecentlyUsedContainer +class UnixHTTPResponse(httplib.HTTPResponse, object): + def __init__(self, sock, *args, **kwargs): + disable_buffering = kwargs.pop('disable_buffering', False) + super(UnixHTTPResponse, self).__init__(sock, *args, **kwargs) + if disable_buffering is True: + # We must first create a new pointer then close the old one + # to avoid closing the underlying socket. + new_fp = sock.makefile('rb', 0) + self.fp.close() + self.fp = new_fp + + class UnixHTTPConnection(httplib.HTTPConnection, object): + def __init__(self, base_url, unix_socket, timeout=60): super(UnixHTTPConnection, self).__init__( 'localhost', timeout=timeout @@ -26,6 +39,7 @@ def __init__(self, base_url, unix_socket, timeout=60): self.base_url = base_url self.unix_socket = unix_socket self.timeout = timeout + self.disable_buffering = False def connect(self): sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) @@ -33,24 +47,16 @@ def connect(self): sock.connect(self.unix_socket) self.sock = sock + def putheader(self, header, *values): + super(UnixHTTPConnection, self).putheader(header, *values) + if header == 'Connection' and 'Upgrade' in values: + self.disable_buffering = True -class AttachHTTPResponse(httplib.HTTPResponse): - ''' - A HTTPResponse object that doesn't use a buffered fileobject. - ''' - def __init__(self, sock, *args, **kwargs): - # Delegate to super class - httplib.HTTPResponse.__init__(self, sock, *args, **kwargs) - - # Override fp with a fileobject that doesn't buffer - self.fp = sock.makefile('rb', 0) - + def response_class(self, sock, *args, **kwargs): + if self.disable_buffering: + kwargs['disable_buffering'] = True -class AttachUnixHTTPConnection(UnixHTTPConnection): - ''' - A HTTPConnection that returns responses that don't used buffering. - ''' - response_class = AttachHTTPResponse + return UnixHTTPResponse(sock, *args, **kwargs) class UnixHTTPConnectionPool(urllib3.connectionpool.HTTPConnectionPool): @@ -63,17 +69,9 @@ def __init__(self, base_url, socket_path, timeout=60, maxsize=10): self.timeout = timeout def _new_conn(self): - # Special case for attach url, as we do a http upgrade to tcp and - # a buffered connection can cause data loss. - path = urllib3.util.parse_url(self.base_url).path - if path.endswith('attach'): - return AttachUnixHTTPConnection( - self.base_url, self.socket_path, self.timeout - ) - else: - return UnixHTTPConnection( - self.base_url, self.socket_path, self.timeout - ) + return UnixHTTPConnection( + self.base_url, self.socket_path, self.timeout + ) class UnixAdapter(requests.adapters.HTTPAdapter): From 9756a4ec4c7235ca6aea1c63a97e82313613f0fe Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 2 Nov 2017 14:06:05 -0700 Subject: [PATCH 0503/1301] Fix build tests to not rely on internet connectivity Signed-off-by: Joffrey F --- tests/integration/api_build_test.py | 30 +++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/tests/integration/api_build_test.py b/tests/integration/api_build_test.py index 21464ff64b..f72c7e6cf2 100644 --- a/tests/integration/api_build_test.py +++ b/tests/integration/api_build_test.py @@ -8,8 +8,8 @@ import pytest import six -from .base import BaseAPIIntegrationTest -from ..helpers import requires_api_version, requires_experimental +from .base import BaseAPIIntegrationTest, BUSYBOX +from ..helpers import random_name, requires_api_version, requires_experimental class BuildTest(BaseAPIIntegrationTest): @@ -214,21 +214,31 @@ def test_build_container_with_target(self): @requires_api_version('1.25') def test_build_with_network_mode(self): + # Set up pingable endpoint on custom network + network = self.client.create_network(random_name())['Id'] + self.tmp_networks.append(network) + container = self.client.create_container(BUSYBOX, 'top') + self.tmp_containers.append(container) + self.client.start(container) + self.client.connect_container_to_network( + container, network, aliases=['pingtarget.docker'] + ) + script = io.BytesIO('\n'.join([ 'FROM busybox', - 'RUN wget http://google.com' + 'RUN ping -c1 pingtarget.docker' ]).encode('ascii')) stream = self.client.build( - fileobj=script, network_mode='bridge', - tag='dockerpytest_bridgebuild' + fileobj=script, network_mode=network, + tag='dockerpytest_customnetbuild' ) - self.tmp_imgs.append('dockerpytest_bridgebuild') + self.tmp_imgs.append('dockerpytest_customnetbuild') for chunk in stream: - pass + print chunk - assert self.client.inspect_image('dockerpytest_bridgebuild') + assert self.client.inspect_image('dockerpytest_customnetbuild') script.seek(0) stream = self.client.build( @@ -260,7 +270,7 @@ def test_build_with_extra_hosts(self): fileobj=script, tag=img_name, extra_hosts={ 'extrahost.local.test': '127.0.0.1', - 'hello.world.test': '8.8.8.8', + 'hello.world.test': '127.0.0.1', }, decode=True ) for chunk in stream: @@ -274,7 +284,7 @@ def test_build_with_extra_hosts(self): if six.PY3: logs = logs.decode('utf-8') assert '127.0.0.1\textrahost.local.test' in logs - assert '8.8.8.8\thello.world.test' in logs + assert '127.0.0.1\thello.world.test' in logs @requires_experimental(until=None) @requires_api_version('1.25') From 6e1f9333d35cc3c50e1d225a3c690c31cbf57843 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 2 Nov 2017 14:10:13 -0700 Subject: [PATCH 0504/1301] Oops Signed-off-by: Joffrey F --- tests/integration/api_build_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/api_build_test.py b/tests/integration/api_build_test.py index f72c7e6cf2..7a0e6b16be 100644 --- a/tests/integration/api_build_test.py +++ b/tests/integration/api_build_test.py @@ -236,7 +236,7 @@ def test_build_with_network_mode(self): self.tmp_imgs.append('dockerpytest_customnetbuild') for chunk in stream: - print chunk + pass assert self.client.inspect_image('dockerpytest_customnetbuild') From a0f6758c76d2524c54ef212aff5744ca19c6a975 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 2 Nov 2017 15:35:43 -0700 Subject: [PATCH 0505/1301] Add support for scope filter in inspect_network Fix missing scope implementation in create_network Signed-off-by: Joffrey F --- docker/api/network.py | 17 ++++++++++++++++- docker/models/networks.py | 14 +++++++++++--- tests/integration/api_network_test.py | 19 +++++++++++++++++++ 3 files changed, 46 insertions(+), 4 deletions(-) diff --git a/docker/api/network.py b/docker/api/network.py index 071a12a63f..797780858a 100644 --- a/docker/api/network.py +++ b/docker/api/network.py @@ -61,6 +61,8 @@ def create_network(self, name, driver=None, options=None, ipam=None, attachable (bool): If enabled, and the network is in the global scope, non-service containers on worker nodes will be able to connect to the network. + scope (str): Specify the network's scope (``local``, ``global`` or + ``swarm``) ingress (bool): If set, create an ingress network which provides the routing-mesh in swarm mode. @@ -140,6 +142,13 @@ def create_network(self, name, driver=None, options=None, ipam=None, data['Ingress'] = ingress + if scope is not None: + if version_lt(self._version, '1.30'): + raise InvalidVersion( + 'scope is not supported in API version < 1.30' + ) + data['Scope'] = scope + url = self._url("/networks/create") res = self._post_json(url, data=data) return self._result(res, json=True) @@ -181,7 +190,7 @@ def remove_network(self, net_id): @minimum_version('1.21') @check_resource('net_id') - def inspect_network(self, net_id, verbose=None): + def inspect_network(self, net_id, verbose=None, scope=None): """ Get detailed information about a network. @@ -189,12 +198,18 @@ def inspect_network(self, net_id, verbose=None): net_id (str): ID of network verbose (bool): Show the service details across the cluster in swarm mode. + scope (str): Filter the network by scope (``swarm``, ``global`` + or ``local``). """ params = {} if verbose is not None: if version_lt(self._version, '1.28'): raise InvalidVersion('verbose was introduced in API 1.28') params['verbose'] = verbose + if scope is not None: + if version_lt(self._version, '1.31'): + raise InvalidVersion('scope was introduced in API 1.31') + params['scope'] = scope url = self._url("/networks/{0}", net_id) res = self._get(url, params=params) diff --git a/docker/models/networks.py b/docker/models/networks.py index afb0ebe8b2..158af99b8d 100644 --- a/docker/models/networks.py +++ b/docker/models/networks.py @@ -102,15 +102,19 @@ def create(self, name, *args, **kwargs): name (str): Name of the network driver (str): Name of the driver used to create the network options (dict): Driver options as a key-value dictionary - ipam (dict): Optional custom IP scheme for the network. - Created with :py:class:`~docker.types.IPAMConfig`. + ipam (IPAMConfig): Optional custom IP scheme for the network. check_duplicate (bool): Request daemon to check for networks with - same name. Default: ``True``. + same name. Default: ``None``. internal (bool): Restrict external access to the network. Default ``False``. labels (dict): Map of labels to set on the network. Default ``None``. enable_ipv6 (bool): Enable IPv6 on the network. Default ``False``. + attachable (bool): If enabled, and the network is in the global + scope, non-service containers on worker nodes will be able to + connect to the network. + scope (str): Specify the network's scope (``local``, ``global`` or + ``swarm``) ingress (bool): If set, create an ingress network which provides the routing-mesh in swarm mode. @@ -155,6 +159,10 @@ def get(self, network_id): Args: network_id (str): The ID of the network. + verbose (bool): Retrieve the service details across the cluster in + swarm mode. + scope (str): Filter the network by scope (``swarm``, ``global`` + or ``local``). Returns: (:py:class:`Network`) The network. diff --git a/tests/integration/api_network_test.py b/tests/integration/api_network_test.py index 1cc632fac7..f4fefde5b9 100644 --- a/tests/integration/api_network_test.py +++ b/tests/integration/api_network_test.py @@ -465,3 +465,22 @@ def test_prune_networks(self): net_name, _ = self.create_network() result = self.client.prune_networks() assert net_name in result['NetworksDeleted'] + + @requires_api_version('1.31') + def test_create_inspect_network_with_scope(self): + assert self.init_swarm() + net_name_loc, net_id_loc = self.create_network(scope='local') + + assert self.client.inspect_network(net_name_loc) + assert self.client.inspect_network(net_name_loc, scope='local') + with pytest.raises(docker.errors.NotFound): + self.client.inspect_network(net_name_loc, scope='global') + + net_name_swarm, net_id_swarm = self.create_network( + driver='overlay', scope='swarm' + ) + + assert self.client.inspect_network(net_name_swarm) + assert self.client.inspect_network(net_name_swarm, scope='swarm') + with pytest.raises(docker.errors.NotFound): + self.client.inspect_network(net_name_swarm, scope='local') From ecca6e0740a24521808c193ae7d4b9499c1f5637 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 2 Nov 2017 18:46:25 -0700 Subject: [PATCH 0506/1301] Update SwarmSpec to include new parameters Signed-off-by: Joffrey F --- docker/api/swarm.py | 29 ++++++--- docker/models/swarm.py | 17 +++++- docker/types/swarm.py | 92 ++++++++++++++++++++++++++--- docs/api.rst | 2 + tests/integration/api_swarm_test.py | 38 ++++++++++++ 5 files changed, 160 insertions(+), 18 deletions(-) diff --git a/docker/api/swarm.py b/docker/api/swarm.py index 4fa0c4a120..576fd79bf8 100644 --- a/docker/api/swarm.py +++ b/docker/api/swarm.py @@ -9,8 +9,8 @@ class SwarmApiMixin(object): def create_swarm_spec(self, *args, **kwargs): """ - Create a ``docker.types.SwarmSpec`` instance that can be used as the - ``swarm_spec`` argument in + Create a :py:class:`docker.types.SwarmSpec` instance that can be used + as the ``swarm_spec`` argument in :py:meth:`~docker.api.swarm.SwarmApiMixin.init_swarm`. Args: @@ -29,13 +29,25 @@ def create_swarm_spec(self, *args, **kwargs): dispatcher_heartbeat_period (int): The delay for an agent to send a heartbeat to the dispatcher. node_cert_expiry (int): Automatic expiry for nodes certificates. - external_ca (dict): Configuration for forwarding signing requests - to an external certificate authority. Use - ``docker.types.SwarmExternalCA``. + external_cas (:py:class:`list`): Configuration for forwarding + signing requests to an external certificate authority. Use + a list of :py:class:`docker.types.SwarmExternalCA`. name (string): Swarm's name + labels (dict): User-defined key/value metadata. + signing_ca_cert (str): The desired signing CA certificate for all + swarm node TLS leaf certificates, in PEM format. + signing_ca_key (str): The desired signing CA key for all swarm + node TLS leaf certificates, in PEM format. + ca_force_rotate (int): An integer whose purpose is to force swarm + to generate a new signing CA certificate and key, if none have + been specified. + autolock_managers (boolean): If set, generate a key and use it to + lock data stored on the managers. + log_driver (DriverConfig): The default log driver to use for tasks + created in the orchestrator. Returns: - ``docker.types.SwarmSpec`` instance. + :py:class:`docker.types.SwarmSpec` Raises: :py:class:`docker.errors.APIError` @@ -51,7 +63,10 @@ def create_swarm_spec(self, *args, **kwargs): force_new_cluster=False, swarm_spec=spec ) """ - return types.SwarmSpec(*args, **kwargs) + ext_ca = kwargs.pop('external_ca', None) + if ext_ca: + kwargs['external_cas'] = [ext_ca] + return types.SwarmSpec(self._version, *args, **kwargs) @utils.minimum_version('1.24') def init_swarm(self, advertise_addr=None, listen_addr='0.0.0.0:2377', diff --git a/docker/models/swarm.py b/docker/models/swarm.py index df3afd36b7..5a253c57b5 100644 --- a/docker/models/swarm.py +++ b/docker/models/swarm.py @@ -1,6 +1,5 @@ from docker.api import APIClient from docker.errors import APIError -from docker.types import SwarmSpec from .resource import Model @@ -72,6 +71,18 @@ def init(self, advertise_addr=None, listen_addr='0.0.0.0:2377', to an external certificate authority. Use ``docker.types.SwarmExternalCA``. name (string): Swarm's name + labels (dict): User-defined key/value metadata. + signing_ca_cert (str): The desired signing CA certificate for all + swarm node TLS leaf certificates, in PEM format. + signing_ca_key (str): The desired signing CA key for all swarm + node TLS leaf certificates, in PEM format. + ca_force_rotate (int): An integer whose purpose is to force swarm + to generate a new signing CA certificate and key, if none have + been specified. + autolock_managers (boolean): If set, generate a key and use it to + lock data stored on the managers. + log_driver (DriverConfig): The default log driver to use for tasks + created in the orchestrator. Returns: ``True`` if the request went through. @@ -94,7 +105,7 @@ def init(self, advertise_addr=None, listen_addr='0.0.0.0:2377', 'listen_addr': listen_addr, 'force_new_cluster': force_new_cluster } - init_kwargs['swarm_spec'] = SwarmSpec(**kwargs) + init_kwargs['swarm_spec'] = self.client.api.create_swarm_spec(**kwargs) self.client.api.init_swarm(**init_kwargs) self.reload() @@ -143,7 +154,7 @@ def update(self, rotate_worker_token=False, rotate_manager_token=False, return self.client.api.update_swarm( version=self.version, - swarm_spec=SwarmSpec(**kwargs), + swarm_spec=self.client.api.create_swarm_spec(**kwargs), rotate_worker_token=rotate_worker_token, rotate_manager_token=rotate_manager_token ) diff --git a/docker/types/swarm.py b/docker/types/swarm.py index 49beaa11f7..9687a82d82 100644 --- a/docker/types/swarm.py +++ b/docker/types/swarm.py @@ -1,9 +1,21 @@ +from ..errors import InvalidVersion +from ..utils import version_lt + + class SwarmSpec(dict): - def __init__(self, task_history_retention_limit=None, + """ + Describe a Swarm's configuration and options. Use + :py:meth:`~docker.api.swarm.SwarmApiMixin.create_swarm_spec` + to instantiate. + """ + def __init__(self, version, task_history_retention_limit=None, snapshot_interval=None, keep_old_snapshots=None, log_entries_for_slow_followers=None, heartbeat_tick=None, election_tick=None, dispatcher_heartbeat_period=None, - node_cert_expiry=None, external_ca=None, name=None): + node_cert_expiry=None, external_cas=None, name=None, + labels=None, signing_ca_cert=None, signing_ca_key=None, + ca_force_rotate=None, autolock_managers=None, + log_driver=None): if task_history_retention_limit is not None: self['Orchestration'] = { 'TaskHistoryRetentionLimit': task_history_retention_limit @@ -26,18 +38,82 @@ def __init__(self, task_history_retention_limit=None, 'HeartbeatPeriod': dispatcher_heartbeat_period } - if node_cert_expiry or external_ca: - self['CAConfig'] = { - 'NodeCertExpiry': node_cert_expiry, - 'ExternalCA': external_ca - } + ca_config = {} + if node_cert_expiry is not None: + ca_config['NodeCertExpiry'] = node_cert_expiry + if external_cas: + if version_lt(version, '1.25'): + if len(external_cas) > 1: + raise InvalidVersion( + 'Support for multiple external CAs is not available ' + 'for API version < 1.25' + ) + ca_config['ExternalCA'] = external_cas[0] + else: + ca_config['ExternalCAs'] = external_cas + if signing_ca_key: + if version_lt(version, '1.30'): + raise InvalidVersion( + 'signing_ca_key is not supported in API version < 1.30' + ) + ca_config['SigningCAKey'] = signing_ca_key + if signing_ca_cert: + if version_lt(version, '1.30'): + raise InvalidVersion( + 'signing_ca_cert is not supported in API version < 1.30' + ) + ca_config['SigningCACert'] = signing_ca_cert + if ca_force_rotate is not None: + if version_lt(version, '1.30'): + raise InvalidVersion( + 'force_rotate is not supported in API version < 1.30' + ) + ca_config['ForceRotate'] = ca_force_rotate + if ca_config: + self['CAConfig'] = ca_config + + if autolock_managers is not None: + if version_lt(version, '1.25'): + raise InvalidVersion( + 'autolock_managers is not supported in API version < 1.25' + ) + + self['EncryptionConfig'] = {'AutoLockManagers': autolock_managers} + + if log_driver is not None: + if version_lt(version, '1.25'): + raise InvalidVersion( + 'log_driver is not supported in API version < 1.25' + ) + + self['TaskDefaults'] = {'LogDriver': log_driver} if name is not None: self['Name'] = name + if labels is not None: + self['Labels'] = labels class SwarmExternalCA(dict): - def __init__(self, url, protocol=None, options=None): + """ + Configuration for forwarding signing requests to an external + certificate authority. + + Args: + url (string): URL where certificate signing requests should be + sent. + protocol (string): Protocol for communication with the external CA. + options (dict): An object with key/value pairs that are interpreted + as protocol-specific options for the external CA driver. + ca_cert (string): The root CA certificate (in PEM format) this + external CA uses to issue TLS certificates (assumed to be to + the current swarm root CA certificate if not provided). + + + + """ + def __init__(self, url, protocol=None, options=None, ca_cert=None): self['URL'] = url self['Protocol'] = protocol self['Options'] = options + self['CACert'] = ca_cert diff --git a/docs/api.rst b/docs/api.rst index 18993ad343..ff466a1763 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -147,5 +147,7 @@ Configuration types .. autoclass:: RestartPolicy .. autoclass:: SecretReference .. autoclass:: ServiceMode +.. autoclass:: SwarmExternalCA +.. autoclass:: SwarmSpec(*args, **kwargs) .. autoclass:: TaskTemplate .. autoclass:: UpdateConfig diff --git a/tests/integration/api_swarm_test.py b/tests/integration/api_swarm_test.py index 666c689f55..34b0879ce4 100644 --- a/tests/integration/api_swarm_test.py +++ b/tests/integration/api_swarm_test.py @@ -45,6 +45,44 @@ def test_init_swarm_custom_raft_spec(self): assert swarm_info['Spec']['Raft']['SnapshotInterval'] == 5000 assert swarm_info['Spec']['Raft']['LogEntriesForSlowFollowers'] == 1200 + @requires_api_version('1.30') + def test_init_swarm_with_ca_config(self): + spec = self.client.create_swarm_spec( + node_cert_expiry=7776000000000000, ca_force_rotate=6000000000000 + ) + + assert self.init_swarm(swarm_spec=spec) + swarm_info = self.client.inspect_swarm() + assert swarm_info['Spec']['CAConfig']['NodeCertExpiry'] == ( + spec['CAConfig']['NodeCertExpiry'] + ) + assert swarm_info['Spec']['CAConfig']['ForceRotate'] == ( + spec['CAConfig']['ForceRotate'] + ) + + @requires_api_version('1.25') + def test_init_swarm_with_autolock_managers(self): + spec = self.client.create_swarm_spec(autolock_managers=True) + assert self.init_swarm(swarm_spec=spec) + swarm_info = self.client.inspect_swarm() + + assert ( + swarm_info['Spec']['EncryptionConfig']['AutoLockManagers'] is True + ) + + @requires_api_version('1.25') + @pytest.mark.xfail( + reason="This doesn't seem to be taken into account by the engine" + ) + def test_init_swarm_with_log_driver(self): + spec = {'TaskDefaults': {'LogDriver': {'Name': 'syslog'}}} + assert self.init_swarm(swarm_spec=spec) + swarm_info = self.client.inspect_swarm() + + assert swarm_info['Spec']['TaskDefaults']['LogDriver']['Name'] == ( + 'syslog' + ) + @requires_api_version('1.24') def test_leave_swarm(self): assert self.init_swarm() From af0071403cb348c3dd5c253457078806303efec4 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 6 Nov 2017 16:04:00 -0800 Subject: [PATCH 0507/1301] Add support for insert_defaults in inspect_service Signed-off-by: Joffrey F --- docker/api/service.py | 16 +++++++++++++--- docker/models/services.py | 11 +++++++++-- tests/integration/api_service_test.py | 11 +++++++++++ 3 files changed, 33 insertions(+), 5 deletions(-) diff --git a/docker/api/service.py b/docker/api/service.py index e6b48768b4..4c10ef8efd 100644 --- a/docker/api/service.py +++ b/docker/api/service.py @@ -136,12 +136,14 @@ def create_service( @utils.minimum_version('1.24') @utils.check_resource('service') - def inspect_service(self, service): + def inspect_service(self, service, insert_defaults=None): """ Return information about a service. Args: - service (str): Service name or ID + service (str): Service name or ID. + insert_defaults (boolean): If true, default values will be merged + into the service inspect output. Returns: ``True`` if successful. @@ -151,7 +153,15 @@ def inspect_service(self, service): If the server returns an error. """ url = self._url('/services/{0}', service) - return self._result(self._get(url), True) + params = {} + if insert_defaults is not None: + if utils.version_lt(self._version, '1.29'): + raise errors.InvalidVersion( + 'insert_defaults is not supported in API version < 1.29' + ) + params['insertDefaults'] = insert_defaults + + return self._result(self._get(url, params=params), True) @utils.minimum_version('1.24') @utils.check_resource('task') diff --git a/docker/models/services.py b/docker/models/services.py index f2a5d355a0..6fc5c2a5c1 100644 --- a/docker/models/services.py +++ b/docker/models/services.py @@ -177,12 +177,14 @@ def create(self, image, command=None, **kwargs): service_id = self.client.api.create_service(**create_kwargs) return self.get(service_id) - def get(self, service_id): + def get(self, service_id, insert_defaults=None): """ Get a service. Args: service_id (str): The ID of the service. + insert_defaults (boolean): If true, default values will be merged + into the output. Returns: (:py:class:`Service`): The service. @@ -192,8 +194,13 @@ def get(self, service_id): If the service does not exist. :py:class:`docker.errors.APIError` If the server returns an error. + :py:class:`docker.errors.InvalidVersion` + If one of the arguments is not supported with the current + API version. """ - return self.prepare_model(self.client.api.inspect_service(service_id)) + return self.prepare_model( + self.client.api.inspect_service(service_id, insert_defaults) + ) def list(self, **kwargs): """ diff --git a/tests/integration/api_service_test.py b/tests/integration/api_service_test.py index 8c6d4af54c..b931154945 100644 --- a/tests/integration/api_service_test.py +++ b/tests/integration/api_service_test.py @@ -99,6 +99,17 @@ def test_inspect_service_by_name(self): assert 'ID' in svc_info assert svc_info['ID'] == svc_id['ID'] + @requires_api_version('1.29') + def test_inspect_service_insert_defaults(self): + svc_name, svc_id = self.create_simple_service() + svc_info = self.client.inspect_service(svc_id) + svc_info_defaults = self.client.inspect_service( + svc_id, insert_defaults=True + ) + assert svc_info != svc_info_defaults + assert 'RollbackConfig' in svc_info_defaults['Spec'] + assert 'RollbackConfig' not in svc_info['Spec'] + def test_remove_service_by_id(self): svc_name, svc_id = self.create_simple_service() assert self.client.remove_service(svc_id) From 34709689372a090f6135d9d179eeff0e86d528e9 Mon Sep 17 00:00:00 2001 From: Martin Monperrus Date: Mon, 2 Oct 2017 09:40:21 +0200 Subject: [PATCH 0508/1301] explain the socket parameter of exec_run Signed-off-by: Martin Monperrus --- docker/models/containers.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docker/models/containers.py b/docker/models/containers.py index ea8c10b5be..689d8ddc04 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -142,14 +142,16 @@ def exec_run(self, cmd, stdout=True, stderr=True, stdin=False, tty=False, detach (bool): If true, detach from the exec command. Default: False stream (bool): Stream response data. Default: False + socket (bool): Whether to return a socket object or not. Default: False environment (dict or list): A dictionary or a list of strings in the following format ``["PASSWORD=xxx"]`` or ``{"PASSWORD": "xxx"}``. Returns: - (generator or str): If ``stream=True``, a generator yielding - response chunks. A string containing response data otherwise. - + (generator or str): + If ``stream=True``, a generator yielding response chunks. + If ``socket=True``, a socket object of the connection (an SSL wrapped socket for TLS-based docker, on which one must call ``sendall`` and ``recv`` -- and **not** os.read / os.write). + A string containing response data otherwise. Raises: :py:class:`docker.errors.APIError` If the server returns an error. From fe6c9a64b04f6ea4d440998debcf3d0739832be4 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 6 Nov 2017 17:04:27 -0800 Subject: [PATCH 0509/1301] Style fixes. Copied docs to APIClient as well Signed-off-by: Joffrey F --- docker/api/exec_api.py | 5 ++++- docker/models/containers.py | 7 ++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/docker/api/exec_api.py b/docker/api/exec_api.py index 6f42524e65..cff5cfa7b3 100644 --- a/docker/api/exec_api.py +++ b/docker/api/exec_api.py @@ -122,10 +122,13 @@ def exec_start(self, exec_id, detach=False, tty=False, stream=False, Default: False tty (bool): Allocate a pseudo-TTY. Default: False stream (bool): Stream response data. Default: False + socket (bool): Return the connection socket to allow custom + read/write operations. Returns: (generator or str): If ``stream=True``, a generator yielding - response chunks. A string containing response data otherwise. + response chunks. If ``socket=True``, a socket object for the + connection. A string containing response data otherwise. Raises: :py:class:`docker.errors.APIError` diff --git a/docker/models/containers.py b/docker/models/containers.py index 689d8ddc04..97a08b9d83 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -142,15 +142,16 @@ def exec_run(self, cmd, stdout=True, stderr=True, stdin=False, tty=False, detach (bool): If true, detach from the exec command. Default: False stream (bool): Stream response data. Default: False - socket (bool): Whether to return a socket object or not. Default: False + socket (bool): Return the connection socket to allow custom + read/write operations. Default: False environment (dict or list): A dictionary or a list of strings in the following format ``["PASSWORD=xxx"]`` or ``{"PASSWORD": "xxx"}``. Returns: - (generator or str): + (generator or str): If ``stream=True``, a generator yielding response chunks. - If ``socket=True``, a socket object of the connection (an SSL wrapped socket for TLS-based docker, on which one must call ``sendall`` and ``recv`` -- and **not** os.read / os.write). + If ``socket=True``, a socket object for the connection. A string containing response data otherwise. Raises: :py:class:`docker.errors.APIError` From 3bad05136a6366c4e4a80bc13a79250fd7ca2657 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 6 Nov 2017 19:13:19 -0800 Subject: [PATCH 0510/1301] Bump 2.6.0 Signed-off-by: Joffrey F --- docker/version.py | 2 +- docs/change-log.md | 44 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index 273270d599..bdf1346465 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "2.5.1" +version = "2.6.0" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) diff --git a/docs/change-log.md b/docs/change-log.md index 9fe15e1924..ca19981f84 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -1,6 +1,50 @@ Change log ========== +2.6.0 +----- + +[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/34?closed=1) + +### Features + +* Added support for `mounts` in `APIClient.create_host_config` and + `DockerClient.containers.run` +* Added support for `consistency`, `tmpfs_size` and `tmpfs_mode` when + creating mount objects. +* `Mount` objects now support the `tmpfs` and `npipe` types. +* Added support for `extra_hosts` in the `build` methods. +* Added support for the configs API: + * In `APIClient`: `create_config`, `inspect_config`, `remove_config`, + `configs` + * In `DockerClient`: `configs.create`, `configs.get`, `configs.list` and + the `Config` model. + * Added `configs` parameter to `ContainerSpec`. Each item in the `configs` + list must be a `docker.types.ConfigReference` instance. +* Added support for the following parameters when creating a `ContainerSpec` + object: `groups`, `open_stdin`, `read_only`, `stop_signal`, `helathcheck`, + `hosts`, `ns_config`, `configs`, `privileges`. +* Added the following configuration classes to `docker.types`: + `ConfigReference`, `DNSConfig`, `Privileges`, `SwarmExternalCA`. +* Added support for `driver` in `APIClient.create_secret` and + `DockerClient.secrets.create`. +* Added support for `scope` in `APIClient.inspect_network` and + `APIClient.create_network`, and their `DockerClient` equivalent. +* Added support for the following parameters to `create_swarm_spec`: + `external_cas`, `labels`, `signing_ca_cert`, `signing_ca_key`, + `ca_force_rotate`, `autolock_managers`, `log_driver`. These additions + also apply to `DockerClient.swarm.init`. +* Added support for `insert_defaults` in `APIClient.inspect_service` and + `DockerClient.services.get`. + +### Bugfixes + +* Fixed a bug where reading a 0-length frame in log streams would incorrectly + interrupt streaming. +* Fixed a bug where the `id` member on `Swarm` objects wasn't being populated. +* Fixed a bug that would cause some data at the beginning of an upgraded + connection stream (`attach`, `exec_run`) to disappear. + 2.5.1 ----- From 65ba043d158792ea7a596f306293a6503cc12e9a Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 25 Oct 2017 15:14:19 -0700 Subject: [PATCH 0511/1301] Update test engine versions in Jenkinsfile Signed-off-by: Joffrey F Conflicts: Jenkinsfile --- Jenkinsfile | 4 ++-- Makefile | 4 ++-- tests/integration/api_build_test.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 178653a87c..e3168cd703 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -5,7 +5,7 @@ def imageNamePy2 def imageNamePy3 def images = [:] -def dockerVersions = ["1.13.1", "17.04.0-ce", "17.05.0-ce", "17.06.0-ce", "17.07.0-ce-rc3"] +def dockerVersions = ["17.06.2-ce", "17.09.0-ce", "17.10.0-ce"] def buildImage = { name, buildargs, pyTag -> img = docker.image(name) @@ -33,7 +33,7 @@ def buildImages = { -> } def getAPIVersion = { engineVersion -> - def versionMap = ['1.13.': '1.26', '17.04': '1.27', '17.05': '1.29', '17.06': '1.30', '17.07': '1.31'] + def versionMap = ['17.06': '1.30', '17.09': '1.32', '17.10': '1.33'] return versionMap[engineVersion.substring(0, 5)] } diff --git a/Makefile b/Makefile index efa4232e9b..32ef510675 100644 --- a/Makefile +++ b/Makefile @@ -41,8 +41,8 @@ integration-test: build integration-test-py3: build-py3 docker run -t --rm -v /var/run/docker.sock:/var/run/docker.sock docker-sdk-python3 py.test tests/integration/${file} -TEST_API_VERSION ?= 1.30 -TEST_ENGINE_VERSION ?= 17.06.0-ce +TEST_API_VERSION ?= 1.33 +TEST_ENGINE_VERSION ?= 17.10.0-ce .PHONY: integration-dind integration-dind: build build-py3 diff --git a/tests/integration/api_build_test.py b/tests/integration/api_build_test.py index 7a0e6b16be..8e98cc9fa5 100644 --- a/tests/integration/api_build_test.py +++ b/tests/integration/api_build_test.py @@ -210,7 +210,7 @@ def test_build_container_with_target(self): pass info = self.client.inspect_image('build1') - self.assertEqual(info['Config']['OnBuild'], []) + assert not info['Config']['OnBuild'] @requires_api_version('1.25') def test_build_with_network_mode(self): From aa3c4f026d435af98391568c30998414fe2baedf Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 7 Nov 2017 15:19:07 -0800 Subject: [PATCH 0512/1301] Add unlock_swarm and get_unlock_key to APIClient Signed-off-by: Joffrey F --- docker/api/swarm.py | 50 ++++++++++++++++++++++++++++- tests/integration/api_swarm_test.py | 10 ++++++ 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/docker/api/swarm.py b/docker/api/swarm.py index 576fd79bf8..26ec75a913 100644 --- a/docker/api/swarm.py +++ b/docker/api/swarm.py @@ -1,7 +1,9 @@ import logging from six.moves import http_client +from .. import errors from .. import types from .. import utils + log = logging.getLogger(__name__) @@ -68,6 +70,16 @@ def create_swarm_spec(self, *args, **kwargs): kwargs['external_cas'] = [ext_ca] return types.SwarmSpec(self._version, *args, **kwargs) + @utils.minimum_version('1.24') + def get_unlock_key(self): + """ + Get the unlock key for this Swarm manager. + + Returns: + A ``dict`` containing an ``UnlockKey`` member + """ + return self._result(self._get(self._url('/swarm/unlockkey')), True) + @utils.minimum_version('1.24') def init_swarm(self, advertise_addr=None, listen_addr='0.0.0.0:2377', force_new_cluster=False, swarm_spec=None): @@ -270,10 +282,46 @@ def remove_node(self, node_id, force=False): self._raise_for_status(res) return True + @utils.minimum_version('1.24') + def unlock_swarm(self, key): + """ + Unlock a locked swarm. + + Args: + key (string): The unlock key as provided by + :py:meth:`get_unlock_key` + + Raises: + :py:class:`docker.errors.InvalidArgument` + If the key argument is in an incompatible format + + :py:class:`docker.errors.APIError` + If the server returns an error. + + Returns: + `True` if the request was successful. + + Example: + + >>> key = client.get_unlock_key() + >>> client.unlock_node(key) + + """ + if isinstance(key, dict): + if 'UnlockKey' not in key: + raise errors.InvalidArgument('Invalid unlock key format') + else: + key = {'UnlockKey': key} + + url = self._url('/swarm/unlock') + res = self._post_json(url, data=key) + self._raise_for_status(res) + return True + @utils.minimum_version('1.24') def update_node(self, node_id, version, node_spec=None): """ - Update the Node's configuration + Update the node's configuration Args: diff --git a/tests/integration/api_swarm_test.py b/tests/integration/api_swarm_test.py index 34b0879ce4..1a945c5f25 100644 --- a/tests/integration/api_swarm_test.py +++ b/tests/integration/api_swarm_test.py @@ -13,6 +13,13 @@ def setUp(self): def tearDown(self): super(SwarmTest, self).tearDown() + try: + unlock_key = self.client.get_unlock_key() + if unlock_key.get('UnlockKey'): + self.unlock_swarm(unlock_key) + except docker.errors.APIError: + pass + force_leave_swarm(self.client) @requires_api_version('1.24') @@ -70,6 +77,9 @@ def test_init_swarm_with_autolock_managers(self): swarm_info['Spec']['EncryptionConfig']['AutoLockManagers'] is True ) + unlock_key = self.get_unlock_key() + assert unlock_key.get('UnlockKey') + @requires_api_version('1.25') @pytest.mark.xfail( reason="This doesn't seem to be taken into account by the engine" From 3bd053a4b703156e5e1f66e3e1b4c72beada2b33 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 7 Nov 2017 15:29:53 -0800 Subject: [PATCH 0513/1301] Add unlock methods to Swarm model Signed-off-by: Joffrey F --- docker/models/swarm.py | 8 ++++++++ docs/swarm.rst | 2 ++ tests/integration/api_swarm_test.py | 11 ++++++----- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/docker/models/swarm.py b/docker/models/swarm.py index 5a253c57b5..7396e730d7 100644 --- a/docker/models/swarm.py +++ b/docker/models/swarm.py @@ -29,6 +29,10 @@ def version(self): """ return self.attrs.get('Version').get('Index') + def get_unlock_key(self): + return self.client.api.get_unlock_key() + get_unlock_key.__doc__ = APIClient.get_unlock_key.__doc__ + def init(self, advertise_addr=None, listen_addr='0.0.0.0:2377', force_new_cluster=False, **kwargs): """ @@ -128,6 +132,10 @@ def reload(self): """ self.attrs = self.client.api.inspect_swarm() + def unlock(self, key): + return self.client.api.unlock_swarm(key) + unlock.__doc__ = APIClient.unlock_swarm.__doc__ + def update(self, rotate_worker_token=False, rotate_manager_token=False, **kwargs): """ diff --git a/docs/swarm.rst b/docs/swarm.rst index 0c21bae1ab..cab9def70a 100644 --- a/docs/swarm.rst +++ b/docs/swarm.rst @@ -12,9 +12,11 @@ These methods are available on ``client.swarm``: .. rst-class:: hide-signature .. py:class:: Swarm + .. automethod:: get_unlock_key() .. automethod:: init() .. automethod:: join() .. automethod:: leave() + .. automethod:: unlock() .. automethod:: update() .. automethod:: reload() diff --git a/tests/integration/api_swarm_test.py b/tests/integration/api_swarm_test.py index 1a945c5f25..56b0129679 100644 --- a/tests/integration/api_swarm_test.py +++ b/tests/integration/api_swarm_test.py @@ -10,13 +10,13 @@ class SwarmTest(BaseAPIIntegrationTest): def setUp(self): super(SwarmTest, self).setUp() force_leave_swarm(self.client) + self._unlock_key = None def tearDown(self): super(SwarmTest, self).tearDown() try: - unlock_key = self.client.get_unlock_key() - if unlock_key.get('UnlockKey'): - self.unlock_swarm(unlock_key) + if self._unlock_key: + self.client.unlock_swarm(self._unlock_key) except docker.errors.APIError: pass @@ -71,14 +71,15 @@ def test_init_swarm_with_ca_config(self): def test_init_swarm_with_autolock_managers(self): spec = self.client.create_swarm_spec(autolock_managers=True) assert self.init_swarm(swarm_spec=spec) + # save unlock key for tearDown + self._unlock_key = self.client.get_unlock_key() swarm_info = self.client.inspect_swarm() assert ( swarm_info['Spec']['EncryptionConfig']['AutoLockManagers'] is True ) - unlock_key = self.get_unlock_key() - assert unlock_key.get('UnlockKey') + assert self._unlock_key.get('UnlockKey') @requires_api_version('1.25') @pytest.mark.xfail( From 700fbef42b0a269204bfbe03c9404c059ec95d98 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 8 Nov 2017 16:01:19 -0800 Subject: [PATCH 0514/1301] Fix broken unbuffered streaming with Py3 Signed-off-by: Joffrey F --- docker/transport/unixconn.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/docker/transport/unixconn.py b/docker/transport/unixconn.py index 7cb877141e..cc35d00466 100644 --- a/docker/transport/unixconn.py +++ b/docker/transport/unixconn.py @@ -21,13 +21,12 @@ class UnixHTTPResponse(httplib.HTTPResponse, object): def __init__(self, sock, *args, **kwargs): disable_buffering = kwargs.pop('disable_buffering', False) + if six.PY2: + # FIXME: We may need to disable buffering on Py3 as well, + # but there's no clear way to do it at the moment. See: + # https://github.com/docker/docker-py/issues/1799 + kwargs['buffering'] = not disable_buffering super(UnixHTTPResponse, self).__init__(sock, *args, **kwargs) - if disable_buffering is True: - # We must first create a new pointer then close the old one - # to avoid closing the underlying socket. - new_fp = sock.makefile('rb', 0) - self.fp.close() - self.fp = new_fp class UnixHTTPConnection(httplib.HTTPConnection, object): From aafae3dd86d9c1052b41935a6d8b0f590b0abf26 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 8 Nov 2017 16:41:03 -0800 Subject: [PATCH 0515/1301] dev version Signed-off-by: Joffrey F --- docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index bdf1346465..fd82246174 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "2.6.0" +version = "2.7.0-dev" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) From 2008f526607f9ef4d7e2b6084d51969876fea0a8 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 8 Nov 2017 16:43:56 -0800 Subject: [PATCH 0516/1301] 2.6.1 release Signed-off-by: Joffrey F --- docker/version.py | 2 +- docs/change-log.md | 13 ++++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/docker/version.py b/docker/version.py index fd82246174..87c864313b 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "2.7.0-dev" +version = "2.6.1" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) diff --git a/docs/change-log.md b/docs/change-log.md index ca19981f84..57293f3e14 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -1,10 +1,21 @@ Change log ========== +2.6.1 +----- + +[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/40?closed=1) + +### Bugfixes + +* Fixed a bug on Python 3 installations preventing the use of the `attach` and + `exec_run` methods. + + 2.6.0 ----- -[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/34?closed=1) +[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/38?closed=1) ### Features From c7f1b5f84f9b574de370ef0bb00e84ad3b8a556f Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 8 Nov 2017 17:04:53 -0800 Subject: [PATCH 0517/1301] dev version Signed-off-by: Joffrey F --- docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index 87c864313b..fd82246174 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "2.6.1" +version = "2.7.0-dev" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) From 436306f09d0c3460ba5586d015ec151405f42b12 Mon Sep 17 00:00:00 2001 From: HuyNQ Date: Tue, 7 Nov 2017 16:05:35 +0700 Subject: [PATCH 0518/1301] Add exit code to exec_run Signed-off-by: HuyNQ --- docker/models/containers.py | 21 ++++++++++++++++----- tests/integration/models_containers_test.py | 15 +++++++++++++-- tests/unit/models_containers_test.py | 11 +++++++++++ 3 files changed, 40 insertions(+), 7 deletions(-) diff --git a/docker/models/containers.py b/docker/models/containers.py index 97a08b9d83..d70aa5069f 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -149,10 +149,14 @@ def exec_run(self, cmd, stdout=True, stderr=True, stdin=False, tty=False, ``{"PASSWORD": "xxx"}``. Returns: - (generator or str): - If ``stream=True``, a generator yielding response chunks. - If ``socket=True``, a socket object for the connection. - A string containing response data otherwise. + dict: + output: (generator or str): + If ``stream=True``, a generator yielding response chunks. + If ``socket=True``, a socket object for the connection. + A string containing response data otherwise. + exit_code: (int): + Exited code of execution + Raises: :py:class:`docker.errors.APIError` If the server returns an error. @@ -161,9 +165,16 @@ def exec_run(self, cmd, stdout=True, stderr=True, stdin=False, tty=False, self.id, cmd, stdout=stdout, stderr=stderr, stdin=stdin, tty=tty, privileged=privileged, user=user, environment=environment ) - return self.client.api.exec_start( + exec_output = self.client.api.exec_start( resp['Id'], detach=detach, tty=tty, stream=stream, socket=socket ) + exit_code = 0 + if stream is False: + exit_code = self.client.api.exec_inspect(resp['Id'])['ExitCode'] + return { + 'exit_code': exit_code, + 'output': exec_output + } def export(self): """ diff --git a/tests/integration/models_containers_test.py b/tests/integration/models_containers_test.py index ce3349baa7..1055a26fd8 100644 --- a/tests/integration/models_containers_test.py +++ b/tests/integration/models_containers_test.py @@ -153,13 +153,24 @@ def test_diff(self): container.wait() assert container.diff() == [{'Path': '/test', 'Kind': 1}] - def test_exec_run(self): + def test_exec_run_success(self): client = docker.from_env(version=TEST_API_VERSION) container = client.containers.run( "alpine", "sh -c 'echo \"hello\" > /test; sleep 60'", detach=True ) self.tmp_containers.append(container.id) - assert container.exec_run("cat /test") == b"hello\n" + exec_output = container.exec_run("cat /test") + assert exec_output["output"] == b"hello\n" + assert exec_output["exit_code"] == 0 + + def test_exec_run_failed(self): + client = docker.from_env(version=TEST_API_VERSION) + container = client.containers.run( + "alpine", "sh -c 'sleep 60'", detach=True + ) + self.tmp_containers.append(container.id) + exec_output = container.exec_run("docker ps") + assert exec_output["exit_code"] == 126 def test_kill(self): client = docker.from_env(version=TEST_API_VERSION) diff --git a/tests/unit/models_containers_test.py b/tests/unit/models_containers_test.py index 5eaa45ac66..75d128a149 100644 --- a/tests/unit/models_containers_test.py +++ b/tests/unit/models_containers_test.py @@ -400,6 +400,17 @@ def test_exec_run(self): client.api.exec_start.assert_called_with( FAKE_EXEC_ID, detach=False, tty=False, stream=True, socket=False ) + container.exec_run("docker ps", privileged=True, stream=False) + client.api.exec_create.assert_called_with( + FAKE_CONTAINER_ID, "docker ps", stdout=True, stderr=True, + stdin=False, tty=False, privileged=True, user='', environment=None + ) + client.api.exec_start.assert_called_with( + FAKE_EXEC_ID, detach=False, tty=False, stream=False, socket=False + ) + client.api.exec_start.assert_called_with( + FAKE_EXEC_ID, detach=False, tty=False, stream=False, socket=False + ) def test_export(self): client = make_fake_client() From e6cc3c15400a386efff2c7672758d424637b7c14 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Sat, 11 Nov 2017 03:07:27 +0100 Subject: [PATCH 0519/1301] Remove test_update_swarm_name Docker currently only supports the "default" cluster in Swarm-mode, and an upcoming SwarmKit release will produce an error if the name of the cluster is updated, causing the test to fail. Given that renaming the cluster is not supported, this patch removes the test Signed-off-by: Sebastiaan van Stijn --- tests/integration/api_swarm_test.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/tests/integration/api_swarm_test.py b/tests/integration/api_swarm_test.py index 34b0879ce4..4b00dd73e2 100644 --- a/tests/integration/api_swarm_test.py +++ b/tests/integration/api_swarm_test.py @@ -126,24 +126,6 @@ def test_update_swarm(self): swarm_info_2['JoinTokens']['Worker'] ) - @requires_api_version('1.24') - def test_update_swarm_name(self): - assert self.init_swarm() - swarm_info_1 = self.client.inspect_swarm() - spec = self.client.create_swarm_spec( - node_cert_expiry=7776000000000000, name='reimuhakurei' - ) - assert self.client.update_swarm( - version=swarm_info_1['Version']['Index'], swarm_spec=spec - ) - swarm_info_2 = self.client.inspect_swarm() - - assert ( - swarm_info_1['Version']['Index'] != - swarm_info_2['Version']['Index'] - ) - assert swarm_info_2['Spec']['Name'] == 'reimuhakurei' - @requires_api_version('1.24') def test_list_nodes(self): assert self.init_swarm() From 6e5eb2eba707f95d03168dfa1384127134260aae Mon Sep 17 00:00:00 2001 From: Viktor Adam Date: Tue, 14 Nov 2017 21:10:23 +0000 Subject: [PATCH 0520/1301] Update service using previous spec Signed-off-by: Viktor Adam --- docker/api/service.py | 52 ++++- tests/integration/api_service_test.py | 313 ++++++++++++++++++++++++++ 2 files changed, 357 insertions(+), 8 deletions(-) diff --git a/docker/api/service.py b/docker/api/service.py index 4c10ef8efd..8e39ff5109 100644 --- a/docker/api/service.py +++ b/docker/api/service.py @@ -306,7 +306,7 @@ def tasks(self, filters=None): def update_service(self, service, version, task_template=None, name=None, labels=None, mode=None, update_config=None, networks=None, endpoint_config=None, - endpoint_spec=None): + endpoint_spec=None, use_current_spec=False): """ Update a service. @@ -328,6 +328,8 @@ def update_service(self, service, version, task_template=None, name=None, the service to. Default: ``None``. endpoint_spec (EndpointSpec): Properties that can be configured to access and load balance a service. Default: ``None``. + use_current_spec (boolean): Use the undefined settings from the + previous specification of the service. Default: ``False`` Returns: ``True`` if successful. @@ -345,32 +347,66 @@ def update_service(self, service, version, task_template=None, name=None, _check_api_features(self._version, task_template, update_config) + if use_current_spec: + if utils.version_lt(self._version, '1.29'): + inspect_defaults = None + else: + inspect_defaults = True + current = self.inspect_service( + service, insert_defaults=inspect_defaults + )['Spec'] + + else: + current = {} + url = self._url('/services/{0}/update', service) data = {} headers = {} - if name is not None: - data['Name'] = name - if labels is not None: - data['Labels'] = labels + + data['Name'] = name if name is not None else current.get('Name') + data['Labels'] = labels if labels is not None else current.get('Labels') + if mode is not None: if not isinstance(mode, dict): mode = ServiceMode(mode) data['Mode'] = mode + else: + data['Mode'] = current.get('Mode') + + merged_task_template = current.get('TaskTemplate', {}) if task_template is not None: - image = task_template.get('ContainerSpec', {}).get('Image', None) + for task_template_key, task_template_value in task_template.items(): + if task_template_key == 'ContainerSpec': + if 'ContainerSpec' not in merged_task_template: + merged_task_template['ContainerSpec'] = {} + for container_spec_key, container_spec_value in task_template['ContainerSpec'].items(): + merged_task_template['ContainerSpec'][container_spec_key] = container_spec_value + else: + merged_task_template[task_template_key] = task_template_value + image = merged_task_template.get('ContainerSpec', {}).get('Image', None) if image is not None: registry, repo_name = auth.resolve_repository_name(image) auth_header = auth.get_config_header(self, registry) if auth_header: headers['X-Registry-Auth'] = auth_header - data['TaskTemplate'] = task_template + data['TaskTemplate'] = merged_task_template + if update_config is not None: data['UpdateConfig'] = update_config + else: + data['UpdateConfig'] = current.get('UpdateConfig') if networks is not None: - data['Networks'] = utils.convert_service_networks(networks) + data['TaskTemplate']['Networks'] = utils.convert_service_networks(networks) + else: + existing_networks = current.get('TaskTemplate', {}).get('Networks') or current.get('Networks') + if existing_networks is not None: + data['TaskTemplate']['Networks'] = existing_networks + if endpoint_spec is not None: data['EndpointSpec'] = endpoint_spec + else: + data['EndpointSpec'] = current.get('EndpointSpec') resp = self._post_json( url, data=data, params={'version': version}, headers=headers diff --git a/tests/integration/api_service_test.py b/tests/integration/api_service_test.py index b931154945..00ad84cce1 100644 --- a/tests/integration/api_service_test.py +++ b/tests/integration/api_service_test.py @@ -710,3 +710,316 @@ def test_create_service_with_privileges(self): svc_info['Spec']['TaskTemplate']['ContainerSpec']['Privileges'] ) assert privileges['SELinuxContext']['Disable'] is True + + @requires_api_version('1.25') + def test_update_service_with_defaults_name(self): + container_spec = docker.types.ContainerSpec( + 'busybox', ['echo', 'hello'] + ) + task_tmpl = docker.types.TaskTemplate(container_spec) + name = self.get_service_name() + svc_id = self.client.create_service(task_tmpl, name=name) + svc_info = self.client.inspect_service(svc_id) + assert 'Name' in svc_info['Spec'] + assert svc_info['Spec']['Name'] == name + version_index = svc_info['Version']['Index'] + + task_tmpl = docker.types.TaskTemplate(container_spec, force_update=10) + self.client.update_service(name, version_index, task_tmpl, use_current_spec=True) + svc_info = self.client.inspect_service(svc_id) + new_index = svc_info['Version']['Index'] + assert new_index > version_index + assert 'Name' in svc_info['Spec'] + assert svc_info['Spec']['Name'] == name + + @requires_api_version('1.25') + def test_update_service_with_defaults_labels(self): + container_spec = docker.types.ContainerSpec( + 'busybox', ['echo', 'hello'] + ) + task_tmpl = docker.types.TaskTemplate(container_spec) + name = self.get_service_name() + svc_id = self.client.create_service(task_tmpl, name=name, labels={'service.label': 'SampleLabel'}) + svc_info = self.client.inspect_service(svc_id) + assert 'Labels' in svc_info['Spec'] + assert 'service.label' in svc_info['Spec']['Labels'] + assert svc_info['Spec']['Labels']['service.label'] == 'SampleLabel' + version_index = svc_info['Version']['Index'] + + task_tmpl = docker.types.TaskTemplate(container_spec, force_update=10) + self.client.update_service(name, version_index, task_tmpl, name=name, use_current_spec=True) + svc_info = self.client.inspect_service(svc_id) + new_index = svc_info['Version']['Index'] + assert new_index > version_index + assert 'Labels' in svc_info['Spec'] + assert 'service.label' in svc_info['Spec']['Labels'] + assert svc_info['Spec']['Labels']['service.label'] == 'SampleLabel' + + def test_update_service_with_defaults_mode(self): + container_spec = docker.types.ContainerSpec( + 'busybox', ['echo', 'hello'] + ) + task_tmpl = docker.types.TaskTemplate(container_spec) + name = self.get_service_name() + svc_id = self.client.create_service(task_tmpl, name=name, + mode=docker.types.ServiceMode(mode='replicated', replicas=2)) + svc_info = self.client.inspect_service(svc_id) + assert 'Mode' in svc_info['Spec'] + assert 'Replicated' in svc_info['Spec']['Mode'] + assert 'Replicas' in svc_info['Spec']['Mode']['Replicated'] + assert svc_info['Spec']['Mode']['Replicated']['Replicas'] == 2 + version_index = svc_info['Version']['Index'] + + self.client.update_service(name, version_index, labels={'force': 'update'}, use_current_spec=True) + svc_info = self.client.inspect_service(svc_id) + new_index = svc_info['Version']['Index'] + assert new_index > version_index + assert 'Mode' in svc_info['Spec'] + assert 'Replicated' in svc_info['Spec']['Mode'] + assert 'Replicas' in svc_info['Spec']['Mode']['Replicated'] + assert svc_info['Spec']['Mode']['Replicated']['Replicas'] == 2 + + def test_update_service_with_defaults_container_labels(self): + container_spec = docker.types.ContainerSpec( + 'busybox', ['echo', 'hello'], + labels={'container.label': 'SampleLabel'} + ) + task_tmpl = docker.types.TaskTemplate(container_spec) + name = self.get_service_name() + svc_id = self.client.create_service(task_tmpl, name=name, labels={'service.label': 'SampleLabel'}) + svc_info = self.client.inspect_service(svc_id) + assert 'TaskTemplate' in svc_info['Spec'] + assert 'ContainerSpec' in svc_info['Spec']['TaskTemplate'] + assert 'Labels' in svc_info['Spec']['TaskTemplate']['ContainerSpec'] + assert svc_info['Spec']['TaskTemplate']['ContainerSpec']['Labels']['container.label'] == 'SampleLabel' + version_index = svc_info['Version']['Index'] + + self.client.update_service(name, version_index, labels={'force': 'update'}, use_current_spec=True) + svc_info = self.client.inspect_service(svc_id) + new_index = svc_info['Version']['Index'] + assert new_index > version_index + assert 'TaskTemplate' in svc_info['Spec'] + assert 'ContainerSpec' in svc_info['Spec']['TaskTemplate'] + assert 'Labels' in svc_info['Spec']['TaskTemplate']['ContainerSpec'] + assert svc_info['Spec']['TaskTemplate']['ContainerSpec']['Labels']['container.label'] == 'SampleLabel' + + container_spec = docker.types.ContainerSpec( + 'busybox', ['echo', 'hello'] + ) + task_tmpl = docker.types.TaskTemplate(container_spec) + self.client.update_service(name, new_index, task_tmpl, use_current_spec=True) + svc_info = self.client.inspect_service(svc_id) + newer_index = svc_info['Version']['Index'] + assert newer_index > new_index + assert 'TaskTemplate' in svc_info['Spec'] + assert 'ContainerSpec' in svc_info['Spec']['TaskTemplate'] + assert 'Labels' in svc_info['Spec']['TaskTemplate']['ContainerSpec'] + assert svc_info['Spec']['TaskTemplate']['ContainerSpec']['Labels']['container.label'] == 'SampleLabel' + + def test_update_service_with_defaults_update_config(self): + container_spec = docker.types.ContainerSpec(BUSYBOX, ['true']) + task_tmpl = docker.types.TaskTemplate(container_spec) + update_config = docker.types.UpdateConfig( + parallelism=10, delay=5, failure_action='pause' + ) + name = self.get_service_name() + svc_id = self.client.create_service( + task_tmpl, update_config=update_config, name=name + ) + svc_info = self.client.inspect_service(svc_id) + assert 'UpdateConfig' in svc_info['Spec'] + uc = svc_info['Spec']['UpdateConfig'] + assert update_config['Parallelism'] == uc['Parallelism'] + assert update_config['Delay'] == uc['Delay'] + assert update_config['FailureAction'] == uc['FailureAction'] + version_index = svc_info['Version']['Index'] + + self.client.update_service(name, version_index, labels={'force': 'update'}, use_current_spec=True) + svc_info = self.client.inspect_service(svc_id) + new_index = svc_info['Version']['Index'] + assert new_index > version_index + assert 'UpdateConfig' in svc_info['Spec'] + uc = svc_info['Spec']['UpdateConfig'] + assert update_config['Parallelism'] == uc['Parallelism'] + assert update_config['Delay'] == uc['Delay'] + assert update_config['FailureAction'] == uc['FailureAction'] + + def test_update_service_with_defaults_networks(self): + net1 = self.client.create_network( + 'dockerpytest_1', driver='overlay', ipam={'Driver': 'default'} + ) + self.tmp_networks.append(net1['Id']) + net2 = self.client.create_network( + 'dockerpytest_2', driver='overlay', ipam={'Driver': 'default'} + ) + self.tmp_networks.append(net2['Id']) + container_spec = docker.types.ContainerSpec(BUSYBOX, ['true']) + task_tmpl = docker.types.TaskTemplate(container_spec) + name = self.get_service_name() + svc_id = self.client.create_service( + task_tmpl, name=name, networks=[ + 'dockerpytest_1', {'Target': 'dockerpytest_2'} + ] + ) + svc_info = self.client.inspect_service(svc_id) + assert 'Networks' in svc_info['Spec'] + assert svc_info['Spec']['Networks'] == [ + {'Target': net1['Id']}, {'Target': net2['Id']} + ] + + version_index = svc_info['Version']['Index'] + + self.client.update_service(name, version_index, labels={'force': 'update'}, use_current_spec=True) + svc_info = self.client.inspect_service(svc_id) + new_index = svc_info['Version']['Index'] + assert new_index > version_index + assert 'Networks' in svc_info['Spec']['TaskTemplate'] + assert svc_info['Spec']['TaskTemplate']['Networks'] == [ + {'Target': net1['Id']}, {'Target': net2['Id']} + ] + + self.client.update_service(name, new_index, networks=[net1['Id']], use_current_spec=True) + svc_info = self.client.inspect_service(svc_id) + assert 'Networks' in svc_info['Spec']['TaskTemplate'] + assert svc_info['Spec']['TaskTemplate']['Networks'] == [ + {'Target': net1['Id']} + ] + + def test_update_service_with_defaults_endpoint_spec(self): + container_spec = docker.types.ContainerSpec(BUSYBOX, ['true']) + task_tmpl = docker.types.TaskTemplate(container_spec) + name = self.get_service_name() + endpoint_spec = docker.types.EndpointSpec(ports={ + 12357: (1990, 'udp'), + 12562: (678,), + 53243: 8080, + }) + svc_id = self.client.create_service( + task_tmpl, name=name, endpoint_spec=endpoint_spec + ) + svc_info = self.client.inspect_service(svc_id) + print(svc_info) + ports = svc_info['Spec']['EndpointSpec']['Ports'] + for port in ports: + if port['PublishedPort'] == 12562: + assert port['TargetPort'] == 678 + assert port['Protocol'] == 'tcp' + elif port['PublishedPort'] == 53243: + assert port['TargetPort'] == 8080 + assert port['Protocol'] == 'tcp' + elif port['PublishedPort'] == 12357: + assert port['TargetPort'] == 1990 + assert port['Protocol'] == 'udp' + else: + self.fail('Invalid port specification: {0}'.format(port)) + + assert len(ports) == 3 + + svc_info = self.client.inspect_service(svc_id) + version_index = svc_info['Version']['Index'] + + self.client.update_service(name, version_index, labels={'force': 'update'}, use_current_spec=True) + svc_info = self.client.inspect_service(svc_id) + new_index = svc_info['Version']['Index'] + assert new_index > version_index + + ports = svc_info['Spec']['EndpointSpec']['Ports'] + for port in ports: + if port['PublishedPort'] == 12562: + assert port['TargetPort'] == 678 + assert port['Protocol'] == 'tcp' + elif port['PublishedPort'] == 53243: + assert port['TargetPort'] == 8080 + assert port['Protocol'] == 'tcp' + elif port['PublishedPort'] == 12357: + assert port['TargetPort'] == 1990 + assert port['Protocol'] == 'udp' + else: + self.fail('Invalid port specification: {0}'.format(port)) + + assert len(ports) == 3 + + @requires_api_version('1.25') + def test_update_service_remove_healthcheck(self): + second = 1000000000 + hc = docker.types.Healthcheck( + test='true', retries=3, timeout=1 * second, + start_period=3 * second, interval=int(second / 2), + ) + container_spec = docker.types.ContainerSpec( + BUSYBOX, ['sleep', '999'], healthcheck=hc + ) + task_tmpl = docker.types.TaskTemplate(container_spec) + name = self.get_service_name() + svc_id = self.client.create_service(task_tmpl, name=name) + svc_info = self.client.inspect_service(svc_id) + assert ( + 'Healthcheck' in svc_info['Spec']['TaskTemplate']['ContainerSpec'] + ) + assert ( + hc == + svc_info['Spec']['TaskTemplate']['ContainerSpec']['Healthcheck'] + ) + + container_spec = docker.types.ContainerSpec( + BUSYBOX, ['sleep', '999'], healthcheck={} + ) + task_tmpl = docker.types.TaskTemplate(container_spec) + + version_index = svc_info['Version']['Index'] + + self.client.update_service(name, version_index, task_tmpl, use_current_spec=True) + svc_info = self.client.inspect_service(svc_id) + new_index = svc_info['Version']['Index'] + assert new_index > version_index + assert ( + 'Healthcheck' not in svc_info['Spec']['TaskTemplate']['ContainerSpec'] or + not svc_info['Spec']['TaskTemplate']['ContainerSpec']['Healthcheck'] + ) + + def test_update_service_remove_labels(self): + container_spec = docker.types.ContainerSpec( + 'busybox', ['echo', 'hello'] + ) + task_tmpl = docker.types.TaskTemplate(container_spec) + name = self.get_service_name() + svc_id = self.client.create_service(task_tmpl, name=name, labels={'service.label': 'SampleLabel'}) + svc_info = self.client.inspect_service(svc_id) + assert 'Labels' in svc_info['Spec'] + assert 'service.label' in svc_info['Spec']['Labels'] + assert svc_info['Spec']['Labels']['service.label'] == 'SampleLabel' + version_index = svc_info['Version']['Index'] + + self.client.update_service(name, version_index, labels={}, use_current_spec=True) + svc_info = self.client.inspect_service(svc_id) + new_index = svc_info['Version']['Index'] + assert new_index > version_index + assert not svc_info['Spec'].get('Labels') + + def test_update_service_remove_container_labels(self): + container_spec = docker.types.ContainerSpec( + 'busybox', ['echo', 'hello'], + labels={'container.label': 'SampleLabel'} + ) + task_tmpl = docker.types.TaskTemplate(container_spec) + name = self.get_service_name() + svc_id = self.client.create_service(task_tmpl, name=name, labels={'service.label': 'SampleLabel'}) + svc_info = self.client.inspect_service(svc_id) + assert 'TaskTemplate' in svc_info['Spec'] + assert 'ContainerSpec' in svc_info['Spec']['TaskTemplate'] + assert 'Labels' in svc_info['Spec']['TaskTemplate']['ContainerSpec'] + assert svc_info['Spec']['TaskTemplate']['ContainerSpec']['Labels']['container.label'] == 'SampleLabel' + version_index = svc_info['Version']['Index'] + + container_spec = docker.types.ContainerSpec( + 'busybox', ['echo', 'hello'], + labels={} + ) + task_tmpl = docker.types.TaskTemplate(container_spec) + self.client.update_service(name, version_index, task_tmpl, use_current_spec=True) + svc_info = self.client.inspect_service(svc_id) + new_index = svc_info['Version']['Index'] + assert new_index > version_index + assert 'TaskTemplate' in svc_info['Spec'] + assert 'ContainerSpec' in svc_info['Spec']['TaskTemplate'] + assert not svc_info['Spec']['TaskTemplate']['ContainerSpec'].get('Labels') From b2d08e64bceb81b75df8de6b0ad1948488bb4b28 Mon Sep 17 00:00:00 2001 From: Viktor Adam Date: Tue, 14 Nov 2017 23:32:19 +0000 Subject: [PATCH 0521/1301] Service model update changes Signed-off-by: Viktor Adam --- docker/api/service.py | 67 +++++--- docker/models/services.py | 11 +- docker/types/services.py | 9 +- tests/integration/api_service_test.py | 187 +++++++++++++++++++--- tests/integration/models_services_test.py | 123 +++++++++++++- tests/unit/models_services_test.py | 4 +- 6 files changed, 345 insertions(+), 56 deletions(-) diff --git a/docker/api/service.py b/docker/api/service.py index 8e39ff5109..67eb02c39f 100644 --- a/docker/api/service.py +++ b/docker/api/service.py @@ -363,8 +363,15 @@ def update_service(self, service, version, task_template=None, name=None, data = {} headers = {} - data['Name'] = name if name is not None else current.get('Name') - data['Labels'] = labels if labels is not None else current.get('Labels') + if name is not None: + data['Name'] = name + else: + data['Name'] = current.get('Name') + + if labels is not None: + data['Labels'] = labels + else: + data['Labels'] = current.get('Labels') if mode is not None: if not isinstance(mode, dict): @@ -373,23 +380,17 @@ def update_service(self, service, version, task_template=None, name=None, else: data['Mode'] = current.get('Mode') - merged_task_template = current.get('TaskTemplate', {}) - if task_template is not None: - for task_template_key, task_template_value in task_template.items(): - if task_template_key == 'ContainerSpec': - if 'ContainerSpec' not in merged_task_template: - merged_task_template['ContainerSpec'] = {} - for container_spec_key, container_spec_value in task_template['ContainerSpec'].items(): - merged_task_template['ContainerSpec'][container_spec_key] = container_spec_value - else: - merged_task_template[task_template_key] = task_template_value - image = merged_task_template.get('ContainerSpec', {}).get('Image', None) - if image is not None: - registry, repo_name = auth.resolve_repository_name(image) - auth_header = auth.get_config_header(self, registry) - if auth_header: - headers['X-Registry-Auth'] = auth_header - data['TaskTemplate'] = merged_task_template + data['TaskTemplate'] = self._merge_task_template( + current.get('TaskTemplate', {}), task_template + ) + + container_spec = data['TaskTemplate'].get('ContainerSpec', {}) + image = container_spec.get('Image', None) + if image is not None: + registry, repo_name = auth.resolve_repository_name(image) + auth_header = auth.get_config_header(self, registry) + if auth_header: + headers['X-Registry-Auth'] = auth_header if update_config is not None: data['UpdateConfig'] = update_config @@ -397,11 +398,15 @@ def update_service(self, service, version, task_template=None, name=None, data['UpdateConfig'] = current.get('UpdateConfig') if networks is not None: - data['TaskTemplate']['Networks'] = utils.convert_service_networks(networks) - else: - existing_networks = current.get('TaskTemplate', {}).get('Networks') or current.get('Networks') - if existing_networks is not None: - data['TaskTemplate']['Networks'] = existing_networks + converted_networks = utils.convert_service_networks(networks) + data['TaskTemplate']['Networks'] = converted_networks + elif data['TaskTemplate'].get('Networks') is None: + current_task_template = current.get('TaskTemplate', {}) + current_networks = current_task_template.get('Networks') + if current_networks is None: + current_networks = current.get('Networks') + if current_networks is not None: + data['TaskTemplate']['Networks'] = current_networks if endpoint_spec is not None: data['EndpointSpec'] = endpoint_spec @@ -413,3 +418,17 @@ def update_service(self, service, version, task_template=None, name=None, ) self._raise_for_status(resp) return True + + @staticmethod + def _merge_task_template(current, override): + merged = current.copy() + if override is not None: + for ts_key, ts_value in override.items(): + if ts_key == 'ContainerSpec': + if 'ContainerSpec' not in merged: + merged['ContainerSpec'] = {} + for cs_key, cs_value in override['ContainerSpec'].items(): + merged['ContainerSpec'][cs_key] = cs_value + else: + merged[ts_key] = ts_value + return merged diff --git a/docker/models/services.py b/docker/models/services.py index 6fc5c2a5c1..39c86efb27 100644 --- a/docker/models/services.py +++ b/docker/models/services.py @@ -251,6 +251,7 @@ def list(self, **kwargs): # kwargs to copy straight over to TaskTemplate TASK_TEMPLATE_KWARGS = [ + 'networks', 'resources', 'restart_policy', ] @@ -261,7 +262,6 @@ def list(self, **kwargs): 'labels', 'mode', 'update_config', - 'networks', 'endpoint_spec', ] @@ -295,6 +295,15 @@ def _get_create_service_kwargs(func_name, kwargs): 'Options': kwargs.pop('log_driver_options', {}) } + if func_name == 'update': + if 'force_update' in kwargs: + task_template_kwargs['force_update'] = kwargs.pop('force_update') + + # use the current spec by default if updating the service + # through the model + use_current_spec = kwargs.pop('use_current_spec', True) + create_kwargs['use_current_spec'] = use_current_spec + # All kwargs should have been consumed by this point, so raise # error if any are left if kwargs: diff --git a/docker/types/services.py b/docker/types/services.py index 9031e609ab..109b22e502 100644 --- a/docker/types/services.py +++ b/docker/types/services.py @@ -4,7 +4,7 @@ from ..constants import IS_WINDOWS_PLATFORM from ..utils import ( check_resource, format_environment, format_extra_hosts, parse_bytes, - split_command, + split_command, convert_service_networks, ) @@ -26,11 +26,14 @@ class TaskTemplate(dict): placement (Placement): Placement instructions for the scheduler. If a list is passed instead, it is assumed to be a list of constraints as part of a :py:class:`Placement` object. + networks (:py:class:`list`): List of network names or IDs to attach + the containers to. force_update (int): A counter that triggers an update even if no relevant parameters have been changed. """ def __init__(self, container_spec, resources=None, restart_policy=None, - placement=None, log_driver=None, force_update=None): + placement=None, log_driver=None, networks=None, + force_update=None): self['ContainerSpec'] = container_spec if resources: self['Resources'] = resources @@ -42,6 +45,8 @@ def __init__(self, container_spec, resources=None, restart_policy=None, self['Placement'] = placement if log_driver: self['LogDriver'] = log_driver + if networks: + self['Networks'] = convert_service_networks(networks) if force_update is not None: if not isinstance(force_update, int): diff --git a/tests/integration/api_service_test.py b/tests/integration/api_service_test.py index 00ad84cce1..60fdcf8808 100644 --- a/tests/integration/api_service_test.py +++ b/tests/integration/api_service_test.py @@ -725,7 +725,9 @@ def test_update_service_with_defaults_name(self): version_index = svc_info['Version']['Index'] task_tmpl = docker.types.TaskTemplate(container_spec, force_update=10) - self.client.update_service(name, version_index, task_tmpl, use_current_spec=True) + self._update_service( + svc_id, name, version_index, task_tmpl, use_current_spec=True + ) svc_info = self.client.inspect_service(svc_id) new_index = svc_info['Version']['Index'] assert new_index > version_index @@ -739,7 +741,9 @@ def test_update_service_with_defaults_labels(self): ) task_tmpl = docker.types.TaskTemplate(container_spec) name = self.get_service_name() - svc_id = self.client.create_service(task_tmpl, name=name, labels={'service.label': 'SampleLabel'}) + svc_id = self.client.create_service( + task_tmpl, name=name, labels={'service.label': 'SampleLabel'} + ) svc_info = self.client.inspect_service(svc_id) assert 'Labels' in svc_info['Spec'] assert 'service.label' in svc_info['Spec']['Labels'] @@ -747,7 +751,10 @@ def test_update_service_with_defaults_labels(self): version_index = svc_info['Version']['Index'] task_tmpl = docker.types.TaskTemplate(container_spec, force_update=10) - self.client.update_service(name, version_index, task_tmpl, name=name, use_current_spec=True) + self._update_service( + svc_id, name, version_index, task_tmpl, name=name, + use_current_spec=True + ) svc_info = self.client.inspect_service(svc_id) new_index = svc_info['Version']['Index'] assert new_index > version_index @@ -761,8 +768,10 @@ def test_update_service_with_defaults_mode(self): ) task_tmpl = docker.types.TaskTemplate(container_spec) name = self.get_service_name() - svc_id = self.client.create_service(task_tmpl, name=name, - mode=docker.types.ServiceMode(mode='replicated', replicas=2)) + svc_id = self.client.create_service( + task_tmpl, name=name, + mode=docker.types.ServiceMode(mode='replicated', replicas=2) + ) svc_info = self.client.inspect_service(svc_id) assert 'Mode' in svc_info['Spec'] assert 'Replicated' in svc_info['Spec']['Mode'] @@ -770,7 +779,10 @@ def test_update_service_with_defaults_mode(self): assert svc_info['Spec']['Mode']['Replicated']['Replicas'] == 2 version_index = svc_info['Version']['Index'] - self.client.update_service(name, version_index, labels={'force': 'update'}, use_current_spec=True) + self._update_service( + svc_id, name, version_index, labels={'force': 'update'}, + use_current_spec=True + ) svc_info = self.client.inspect_service(svc_id) new_index = svc_info['Version']['Index'] assert new_index > version_index @@ -786,35 +798,45 @@ def test_update_service_with_defaults_container_labels(self): ) task_tmpl = docker.types.TaskTemplate(container_spec) name = self.get_service_name() - svc_id = self.client.create_service(task_tmpl, name=name, labels={'service.label': 'SampleLabel'}) + svc_id = self.client.create_service( + task_tmpl, name=name, labels={'service.label': 'SampleLabel'} + ) svc_info = self.client.inspect_service(svc_id) assert 'TaskTemplate' in svc_info['Spec'] assert 'ContainerSpec' in svc_info['Spec']['TaskTemplate'] assert 'Labels' in svc_info['Spec']['TaskTemplate']['ContainerSpec'] - assert svc_info['Spec']['TaskTemplate']['ContainerSpec']['Labels']['container.label'] == 'SampleLabel' + labels = svc_info['Spec']['TaskTemplate']['ContainerSpec']['Labels'] + assert labels['container.label'] == 'SampleLabel' version_index = svc_info['Version']['Index'] - self.client.update_service(name, version_index, labels={'force': 'update'}, use_current_spec=True) + self._update_service( + svc_id, name, version_index, labels={'force': 'update'}, + use_current_spec=True + ) svc_info = self.client.inspect_service(svc_id) new_index = svc_info['Version']['Index'] assert new_index > version_index assert 'TaskTemplate' in svc_info['Spec'] assert 'ContainerSpec' in svc_info['Spec']['TaskTemplate'] assert 'Labels' in svc_info['Spec']['TaskTemplate']['ContainerSpec'] - assert svc_info['Spec']['TaskTemplate']['ContainerSpec']['Labels']['container.label'] == 'SampleLabel' + labels = svc_info['Spec']['TaskTemplate']['ContainerSpec']['Labels'] + assert labels['container.label'] == 'SampleLabel' container_spec = docker.types.ContainerSpec( 'busybox', ['echo', 'hello'] ) task_tmpl = docker.types.TaskTemplate(container_spec) - self.client.update_service(name, new_index, task_tmpl, use_current_spec=True) + self._update_service( + svc_id, name, new_index, task_tmpl, use_current_spec=True + ) svc_info = self.client.inspect_service(svc_id) newer_index = svc_info['Version']['Index'] assert newer_index > new_index assert 'TaskTemplate' in svc_info['Spec'] assert 'ContainerSpec' in svc_info['Spec']['TaskTemplate'] assert 'Labels' in svc_info['Spec']['TaskTemplate']['ContainerSpec'] - assert svc_info['Spec']['TaskTemplate']['ContainerSpec']['Labels']['container.label'] == 'SampleLabel' + labels = svc_info['Spec']['TaskTemplate']['ContainerSpec']['Labels'] + assert labels['container.label'] == 'SampleLabel' def test_update_service_with_defaults_update_config(self): container_spec = docker.types.ContainerSpec(BUSYBOX, ['true']) @@ -834,7 +856,10 @@ def test_update_service_with_defaults_update_config(self): assert update_config['FailureAction'] == uc['FailureAction'] version_index = svc_info['Version']['Index'] - self.client.update_service(name, version_index, labels={'force': 'update'}, use_current_spec=True) + self._update_service( + svc_id, name, version_index, labels={'force': 'update'}, + use_current_spec=True + ) svc_info = self.client.inspect_service(svc_id) new_index = svc_info['Version']['Index'] assert new_index > version_index @@ -869,7 +894,10 @@ def test_update_service_with_defaults_networks(self): version_index = svc_info['Version']['Index'] - self.client.update_service(name, version_index, labels={'force': 'update'}, use_current_spec=True) + self._update_service( + svc_id, name, version_index, labels={'force': 'update'}, + use_current_spec=True + ) svc_info = self.client.inspect_service(svc_id) new_index = svc_info['Version']['Index'] assert new_index > version_index @@ -878,7 +906,10 @@ def test_update_service_with_defaults_networks(self): {'Target': net1['Id']}, {'Target': net2['Id']} ] - self.client.update_service(name, new_index, networks=[net1['Id']], use_current_spec=True) + self._update_service( + svc_id, name, new_index, networks=[net1['Id']], + use_current_spec=True + ) svc_info = self.client.inspect_service(svc_id) assert 'Networks' in svc_info['Spec']['TaskTemplate'] assert svc_info['Spec']['TaskTemplate']['Networks'] == [ @@ -918,7 +949,10 @@ def test_update_service_with_defaults_endpoint_spec(self): svc_info = self.client.inspect_service(svc_id) version_index = svc_info['Version']['Index'] - self.client.update_service(name, version_index, labels={'force': 'update'}, use_current_spec=True) + self._update_service( + svc_id, name, version_index, labels={'force': 'update'}, + use_current_spec=True + ) svc_info = self.client.inspect_service(svc_id) new_index = svc_info['Version']['Index'] assert new_index > version_index @@ -968,13 +1002,16 @@ def test_update_service_remove_healthcheck(self): version_index = svc_info['Version']['Index'] - self.client.update_service(name, version_index, task_tmpl, use_current_spec=True) + self._update_service( + svc_id, name, version_index, task_tmpl, use_current_spec=True + ) svc_info = self.client.inspect_service(svc_id) new_index = svc_info['Version']['Index'] assert new_index > version_index + container_spec = svc_info['Spec']['TaskTemplate']['ContainerSpec'] assert ( - 'Healthcheck' not in svc_info['Spec']['TaskTemplate']['ContainerSpec'] or - not svc_info['Spec']['TaskTemplate']['ContainerSpec']['Healthcheck'] + 'Healthcheck' not in container_spec or + not container_spec['Healthcheck'] ) def test_update_service_remove_labels(self): @@ -983,14 +1020,18 @@ def test_update_service_remove_labels(self): ) task_tmpl = docker.types.TaskTemplate(container_spec) name = self.get_service_name() - svc_id = self.client.create_service(task_tmpl, name=name, labels={'service.label': 'SampleLabel'}) + svc_id = self.client.create_service( + task_tmpl, name=name, labels={'service.label': 'SampleLabel'} + ) svc_info = self.client.inspect_service(svc_id) assert 'Labels' in svc_info['Spec'] assert 'service.label' in svc_info['Spec']['Labels'] assert svc_info['Spec']['Labels']['service.label'] == 'SampleLabel' version_index = svc_info['Version']['Index'] - self.client.update_service(name, version_index, labels={}, use_current_spec=True) + self._update_service( + svc_id, name, version_index, labels={}, use_current_spec=True + ) svc_info = self.client.inspect_service(svc_id) new_index = svc_info['Version']['Index'] assert new_index > version_index @@ -1003,12 +1044,15 @@ def test_update_service_remove_container_labels(self): ) task_tmpl = docker.types.TaskTemplate(container_spec) name = self.get_service_name() - svc_id = self.client.create_service(task_tmpl, name=name, labels={'service.label': 'SampleLabel'}) + svc_id = self.client.create_service( + task_tmpl, name=name, labels={'service.label': 'SampleLabel'} + ) svc_info = self.client.inspect_service(svc_id) assert 'TaskTemplate' in svc_info['Spec'] assert 'ContainerSpec' in svc_info['Spec']['TaskTemplate'] assert 'Labels' in svc_info['Spec']['TaskTemplate']['ContainerSpec'] - assert svc_info['Spec']['TaskTemplate']['ContainerSpec']['Labels']['container.label'] == 'SampleLabel' + labels = svc_info['Spec']['TaskTemplate']['ContainerSpec']['Labels'] + assert labels['container.label'] == 'SampleLabel' version_index = svc_info['Version']['Index'] container_spec = docker.types.ContainerSpec( @@ -1016,10 +1060,103 @@ def test_update_service_remove_container_labels(self): labels={} ) task_tmpl = docker.types.TaskTemplate(container_spec) - self.client.update_service(name, version_index, task_tmpl, use_current_spec=True) + self._update_service( + svc_id, name, version_index, task_tmpl, use_current_spec=True + ) svc_info = self.client.inspect_service(svc_id) new_index = svc_info['Version']['Index'] assert new_index > version_index assert 'TaskTemplate' in svc_info['Spec'] assert 'ContainerSpec' in svc_info['Spec']['TaskTemplate'] - assert not svc_info['Spec']['TaskTemplate']['ContainerSpec'].get('Labels') + container_spec = svc_info['Spec']['TaskTemplate']['ContainerSpec'] + assert not container_spec.get('Labels') + + @requires_api_version('1.29') + def test_update_service_with_network_change(self): + container_spec = docker.types.ContainerSpec( + 'busybox', ['echo', 'hello'] + ) + task_tmpl = docker.types.TaskTemplate(container_spec) + net1 = self.client.create_network( + 'dockerpytest_1', driver='overlay', ipam={'Driver': 'default'} + ) + self.tmp_networks.append(net1['Id']) + net2 = self.client.create_network( + 'dockerpytest_2', driver='overlay', ipam={'Driver': 'default'} + ) + self.tmp_networks.append(net2['Id']) + name = self.get_service_name() + svc_id = self.client.create_service( + task_tmpl, name=name, networks=[net1['Id']] + ) + svc_info = self.client.inspect_service(svc_id) + assert 'Networks' in svc_info['Spec'] + assert len(svc_info['Spec']['Networks']) > 0 + assert svc_info['Spec']['Networks'][0]['Target'] == net1['Id'] + + svc_info = self.client.inspect_service(svc_id) + version_index = svc_info['Version']['Index'] + + task_tmpl = docker.types.TaskTemplate(container_spec) + self._update_service( + svc_id, name, version_index, task_tmpl, name=name, + networks=[net2['Id']], use_current_spec=True + ) + svc_info = self.client.inspect_service(svc_id) + task_template = svc_info['Spec']['TaskTemplate'] + assert 'Networks' in task_template + assert len(task_template['Networks']) > 0 + assert task_template['Networks'][0]['Target'] == net2['Id'] + + svc_info = self.client.inspect_service(svc_id) + new_index = svc_info['Version']['Index'] + assert new_index > version_index + + self._update_service( + svc_id, name, new_index, name=name, networks=[net1['Id']], + use_current_spec=True + ) + svc_info = self.client.inspect_service(svc_id) + task_template = svc_info['Spec']['TaskTemplate'] + assert 'ContainerSpec' in task_template + new_spec = task_template['ContainerSpec'] + assert 'Image' in new_spec + assert new_spec['Image'].split(':')[0] == 'busybox' + assert 'Command' in new_spec + assert new_spec['Command'] == ['echo', 'hello'] + assert 'Networks' in task_template + assert len(task_template['Networks']) > 0 + assert task_template['Networks'][0]['Target'] == net1['Id'] + + svc_info = self.client.inspect_service(svc_id) + new_index = svc_info['Version']['Index'] + + task_tmpl = docker.types.TaskTemplate( + container_spec, networks=[net2['Id']] + ) + self._update_service( + svc_id, name, new_index, task_tmpl, name=name, + use_current_spec=True + ) + svc_info = self.client.inspect_service(svc_id) + task_template = svc_info['Spec']['TaskTemplate'] + assert 'Networks' in task_template + assert len(task_template['Networks']) > 0 + assert task_template['Networks'][0]['Target'] == net2['Id'] + + def _update_service(self, svc_id, *args, **kwargs): + # service update tests seem to be a bit flaky + # give them a chance to retry the update with a new version index + try: + self.client.update_service(*args, **kwargs) + except docker.errors.APIError as e: + if e.explanation == "update out of sequence": + svc_info = self.client.inspect_service(svc_id) + version_index = svc_info['Version']['Index'] + + if len(args) > 1: + args = (args[0], version_index) + args[2:] + else: + kwargs['version'] = version_index + + self.client.update_service(*args, **kwargs) diff --git a/tests/integration/models_services_test.py b/tests/integration/models_services_test.py index 6b5dab5312..ee8a3487fd 100644 --- a/tests/integration/models_services_test.py +++ b/tests/integration/models_services_test.py @@ -1,7 +1,6 @@ import unittest import docker -import pytest from .. import helpers from .base import TEST_API_VERSION @@ -36,6 +35,25 @@ def test_create(self): assert "alpine" in container_spec['Image'] assert container_spec['Labels'] == {'container': 'label'} + def test_create_with_network(self): + client = docker.from_env(version=TEST_API_VERSION) + name = helpers.random_name() + network = client.networks.create( + helpers.random_name(), driver='overlay' + ) + service = client.services.create( + # create arguments + name=name, + # ContainerSpec arguments + image="alpine", + command="sleep 300", + networks=[network.id] + ) + assert 'Networks' in service.attrs['Spec']['TaskTemplate'] + networks = service.attrs['Spec']['TaskTemplate']['Networks'] + assert len(networks) == 1 + assert networks[0]['Target'] == network.id + def test_get(self): client = docker.from_env(version=TEST_API_VERSION) name = helpers.random_name() @@ -82,7 +100,6 @@ def test_tasks(self): assert len(tasks) == 1 assert tasks[0]['ServiceID'] == service2.id - @pytest.mark.skip(reason="Makes Swarm unstable?") def test_update(self): client = docker.from_env(version=TEST_API_VERSION) service = client.services.create( @@ -101,3 +118,105 @@ def test_update(self): service.reload() container_spec = service.attrs['Spec']['TaskTemplate']['ContainerSpec'] assert container_spec['Command'] == ["sleep", "600"] + + def test_update_retains_service_labels(self): + client = docker.from_env(version=TEST_API_VERSION) + service = client.services.create( + # create arguments + name=helpers.random_name(), + labels={'service.label': 'SampleLabel'}, + # ContainerSpec arguments + image="alpine", + command="sleep 300" + ) + service.update( + # create argument + name=service.name, + # ContainerSpec argument + command="sleep 600" + ) + service.reload() + labels = service.attrs['Spec']['Labels'] + assert labels == {'service.label': 'SampleLabel'} + + def test_update_retains_container_labels(self): + client = docker.from_env(version=TEST_API_VERSION) + service = client.services.create( + # create arguments + name=helpers.random_name(), + # ContainerSpec arguments + image="alpine", + command="sleep 300", + container_labels={'container.label': 'SampleLabel'} + ) + service.update( + # create argument + name=service.name, + # ContainerSpec argument + command="sleep 600" + ) + service.reload() + container_spec = service.attrs['Spec']['TaskTemplate']['ContainerSpec'] + assert container_spec['Labels'] == {'container.label': 'SampleLabel'} + + def test_update_remove_service_labels(self): + client = docker.from_env(version=TEST_API_VERSION) + service = client.services.create( + # create arguments + name=helpers.random_name(), + labels={'service.label': 'SampleLabel'}, + # ContainerSpec arguments + image="alpine", + command="sleep 300" + ) + service.update( + # create argument + name=service.name, + labels={}, + # ContainerSpec argument + command="sleep 600" + ) + service.reload() + assert not service.attrs['Spec'].get('Labels') + + def test_scale_service(self): + client = docker.from_env(version=TEST_API_VERSION) + service = client.services.create( + # create arguments + name=helpers.random_name(), + # ContainerSpec arguments + image="alpine", + command="sleep 300" + ) + assert len(service.tasks()) == 1 + service.update( + # create argument + name=service.name, + mode=docker.types.ServiceMode('replicated', replicas=2), + # ContainerSpec argument + command="sleep 600" + ) + service.reload() + assert len(service.tasks()) >= 2 + + @helpers.requires_api_version('1.25') + def test_restart_service(self): + client = docker.from_env(version=TEST_API_VERSION) + service = client.services.create( + # create arguments + name=helpers.random_name(), + # ContainerSpec arguments + image="alpine", + command="sleep 300" + ) + initial_version = service.version + service.update( + # create argument + name=service.name, + # task template argument + force_update=10, + # ContainerSpec argument + command="sleep 600" + ) + service.reload() + assert service.version > initial_version diff --git a/tests/unit/models_services_test.py b/tests/unit/models_services_test.py index e7e317d52d..247bb4a4aa 100644 --- a/tests/unit/models_services_test.py +++ b/tests/unit/models_services_test.py @@ -35,18 +35,18 @@ def test_get_create_service_kwargs(self): 'labels': {'key': 'value'}, 'mode': 'global', 'update_config': {'update': 'config'}, - 'networks': ['somenet'], 'endpoint_spec': {'blah': 'blah'}, } assert set(task_template.keys()) == set([ 'ContainerSpec', 'Resources', 'RestartPolicy', 'Placement', - 'LogDriver' + 'LogDriver', 'Networks' ]) assert task_template['Placement'] == {'Constraints': ['foo=bar']} assert task_template['LogDriver'] == { 'Name': 'logdriver', 'Options': {'foo': 'bar'} } + assert task_template['Networks'] == [{'Target': 'somenet'}] assert set(task_template['ContainerSpec'].keys()) == set([ 'Image', 'Command', 'Args', 'Hostname', 'Env', 'Dir', 'User', 'Labels', 'Mounts', 'StopGracePeriod' From c78e73bf7ac87feb8bef35921492a98e5727d9a5 Mon Sep 17 00:00:00 2001 From: Viktor Adam Date: Wed, 15 Nov 2017 08:17:16 +0000 Subject: [PATCH 0522/1301] Attempting to make service update tests less flaky Signed-off-by: Viktor Adam --- tests/integration/api_service_test.py | 4 +++- tests/integration/models_services_test.py | 10 +++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/tests/integration/api_service_test.py b/tests/integration/api_service_test.py index 60fdcf8808..10ae1804b2 100644 --- a/tests/integration/api_service_test.py +++ b/tests/integration/api_service_test.py @@ -1150,7 +1150,7 @@ def _update_service(self, svc_id, *args, **kwargs): try: self.client.update_service(*args, **kwargs) except docker.errors.APIError as e: - if e.explanation == "update out of sequence": + if e.explanation.endswith("update out of sequence"): svc_info = self.client.inspect_service(svc_id) version_index = svc_info['Version']['Index'] @@ -1160,3 +1160,5 @@ def _update_service(self, svc_id, *args, **kwargs): kwargs['version'] = version_index self.client.update_service(*args, **kwargs) + else: + raise diff --git a/tests/integration/models_services_test.py b/tests/integration/models_services_test.py index ee8a3487fd..cb96ff1880 100644 --- a/tests/integration/models_services_test.py +++ b/tests/integration/models_services_test.py @@ -188,7 +188,10 @@ def test_scale_service(self): image="alpine", command="sleep 300" ) - assert len(service.tasks()) == 1 + tasks = [] + while len(tasks) == 0: + tasks = service.tasks() + assert len(tasks) == 1 service.update( # create argument name=service.name, @@ -196,8 +199,9 @@ def test_scale_service(self): # ContainerSpec argument command="sleep 600" ) - service.reload() - assert len(service.tasks()) >= 2 + while len(tasks) == 1: + tasks = service.tasks() + assert len(tasks) >= 2 @helpers.requires_api_version('1.25') def test_restart_service(self): From 828b865bd7629c08ae5dcc67ca0c606253d8d4ec Mon Sep 17 00:00:00 2001 From: Viktor Adam Date: Wed, 15 Nov 2017 18:30:05 +0000 Subject: [PATCH 0523/1301] Fix resetting ContainerSpec properties to None Signed-off-by: Viktor Adam --- docker/api/service.py | 5 +++-- tests/integration/models_services_test.py | 8 ++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/docker/api/service.py b/docker/api/service.py index 67eb02c39f..768d513120 100644 --- a/docker/api/service.py +++ b/docker/api/service.py @@ -428,7 +428,8 @@ def _merge_task_template(current, override): if 'ContainerSpec' not in merged: merged['ContainerSpec'] = {} for cs_key, cs_value in override['ContainerSpec'].items(): - merged['ContainerSpec'][cs_key] = cs_value - else: + if cs_value is not None: + merged['ContainerSpec'][cs_key] = cs_value + elif ts_value is not None: merged[ts_key] = ts_value return merged diff --git a/tests/integration/models_services_test.py b/tests/integration/models_services_test.py index cb96ff1880..ca8be48de4 100644 --- a/tests/integration/models_services_test.py +++ b/tests/integration/models_services_test.py @@ -193,15 +193,15 @@ def test_scale_service(self): tasks = service.tasks() assert len(tasks) == 1 service.update( - # create argument - name=service.name, mode=docker.types.ServiceMode('replicated', replicas=2), - # ContainerSpec argument - command="sleep 600" ) while len(tasks) == 1: tasks = service.tasks() assert len(tasks) >= 2 + # check that the container spec is not overridden with None + service.reload() + spec = service.attrs['Spec']['TaskTemplate']['ContainerSpec'] + assert spec.get('Command') == ['sleep', '300'] @helpers.requires_api_version('1.25') def test_restart_service(self): From 7829b728a40b59b6efd1d0a3d8b92f48d351e5aa Mon Sep 17 00:00:00 2001 From: Viktor Adam Date: Thu, 16 Nov 2017 23:15:31 +0000 Subject: [PATCH 0524/1301] Fetch network details with network lists greedily Signed-off-by: Viktor Adam --- docker/api/network.py | 10 ++++-- docker/models/networks.py | 13 ++++++-- tests/integration/api_network_test.py | 37 +++++++++++++++++++++++ tests/integration/models_networks_test.py | 10 ++++-- tests/unit/models_networks_test.py | 8 +++-- 5 files changed, 70 insertions(+), 8 deletions(-) diff --git a/docker/api/network.py b/docker/api/network.py index 797780858a..09f5a8bd5d 100644 --- a/docker/api/network.py +++ b/docker/api/network.py @@ -6,7 +6,7 @@ class NetworkApiMixin(object): @minimum_version('1.21') - def networks(self, names=None, ids=None, filters=None): + def networks(self, names=None, ids=None, filters=None, greedy=False): """ List networks. Similar to the ``docker networks ls`` command. @@ -18,6 +18,8 @@ def networks(self, names=None, ids=None, filters=None): - ``driver=[]`` Matches a network's driver. - ``label=[]`` or ``label=[=]``. - ``type=["custom"|"builtin"]`` Filters networks by type. + greedy (bool): Fetch more details for each network individually. + You might want this to get the containers attached to them. Returns: (dict): List of network objects. @@ -36,7 +38,11 @@ def networks(self, names=None, ids=None, filters=None): params = {'filters': utils.convert_filters(filters)} url = self._url("/networks") res = self._get(url, params=params) - return self._result(res, json=True) + result = self._result(res, json=True) + if greedy: + return [self.inspect_network(net['Id']) for net in result] + else: + return result @minimum_version('1.21') def create_network(self, name, driver=None, options=None, ipam=None, diff --git a/docker/models/networks.py b/docker/models/networks.py index 158af99b8d..06ff22b165 100644 --- a/docker/models/networks.py +++ b/docker/models/networks.py @@ -153,7 +153,7 @@ def create(self, name, *args, **kwargs): resp = self.client.api.create_network(name, *args, **kwargs) return self.get(resp['Id']) - def get(self, network_id): + def get(self, network_id, *args, **kwargs): """ Get a network by its ID. @@ -175,7 +175,9 @@ def get(self, network_id): If the server returns an error. """ - return self.prepare_model(self.client.api.inspect_network(network_id)) + return self.prepare_model( + self.client.api.inspect_network(network_id, *args, **kwargs) + ) def list(self, *args, **kwargs): """ @@ -184,6 +186,13 @@ def list(self, *args, **kwargs): Args: names (:py:class:`list`): List of names to filter by. ids (:py:class:`list`): List of ids to filter by. + filters (dict): Filters to be processed on the network list. + Available filters: + - ``driver=[]`` Matches a network's driver. + - ``label=[]`` or ``label=[=]``. + - ``type=["custom"|"builtin"]`` Filters networks by type. + greedy (bool): Fetch more details for each network individually. + You might want this to get the containers attached to them. Returns: (list of :py:class:`Network`) The networks on the server. diff --git a/tests/integration/api_network_test.py b/tests/integration/api_network_test.py index f4fefde5b9..57d2630454 100644 --- a/tests/integration/api_network_test.py +++ b/tests/integration/api_network_test.py @@ -112,6 +112,16 @@ def test_connect_and_disconnect_container(self): [container['Id']] ) + network_list = self.client.networks(ids=[net_id], greedy=True) + self.assertEqual( + list( + key + for net in network_list + for key in net['Containers'].keys() + ), + [container['Id']] + ) + with pytest.raises(docker.errors.APIError): self.client.connect_container_to_network(container, net_id) @@ -140,10 +150,27 @@ def test_connect_and_force_disconnect_container(self): [container['Id']] ) + network_list = self.client.networks(ids=[net_id], greedy=True) + self.assertEqual( + list( + key + for net in network_list + for key in net['Containers'].keys() + ), + [container['Id']] + ) + self.client.disconnect_container_from_network(container, net_id, True) network_data = self.client.inspect_network(net_id) self.assertFalse(network_data.get('Containers')) + network_list = self.client.networks(ids=[net_id], greedy=True) + self.assertFalse(list( + key + for net in network_list + for key in net['Containers'].keys() + )) + with pytest.raises(docker.errors.APIError): self.client.disconnect_container_from_network( container, net_id, force=True @@ -183,6 +210,16 @@ def test_connect_on_container_create(self): list(network_data['Containers'].keys()), [container['Id']]) + network_list = self.client.networks(ids=[net_id], greedy=True) + self.assertEqual( + list( + key + for net in network_list + for key in net['Containers'].keys() + ), + [container['Id']] + ) + self.client.disconnect_container_from_network(container, net_id) network_data = self.client.inspect_network(net_id) self.assertFalse(network_data.get('Containers')) diff --git a/tests/integration/models_networks_test.py b/tests/integration/models_networks_test.py index 105dcc594a..25fea33eda 100644 --- a/tests/integration/models_networks_test.py +++ b/tests/integration/models_networks_test.py @@ -3,7 +3,7 @@ from .base import BaseIntegrationTest, TEST_API_VERSION -class ImageCollectionTest(BaseIntegrationTest): +class NetworkCollectionTest(BaseIntegrationTest): def test_create(self): client = docker.from_env(version=TEST_API_VERSION) @@ -47,7 +47,7 @@ def test_list_remove(self): assert network.id not in [n.id for n in client.networks.list()] -class ImageTest(BaseIntegrationTest): +class NetworkTest(BaseIntegrationTest): def test_connect_disconnect(self): client = docker.from_env(version=TEST_API_VERSION) @@ -59,6 +59,12 @@ def test_connect_disconnect(self): network.connect(container) container.start() assert client.networks.get(network.id).containers == [container] + network_containers = list( + c + for net in client.networks.list(greedy=True) + for c in net.containers + ) + assert network_containers == [container] network.disconnect(container) assert network.containers == [] assert client.networks.get(network.id).containers == [] diff --git a/tests/unit/models_networks_test.py b/tests/unit/models_networks_test.py index 943b904568..df0650a298 100644 --- a/tests/unit/models_networks_test.py +++ b/tests/unit/models_networks_test.py @@ -4,7 +4,7 @@ from .fake_api_client import make_fake_client -class ImageCollectionTest(unittest.TestCase): +class NetworkCollectionTest(unittest.TestCase): def test_create(self): client = make_fake_client() @@ -36,8 +36,12 @@ def test_list(self): client.networks.list(names=["foobar"]) assert client.api.networks.called_once_with(names=["foobar"]) + client = make_fake_client() + client.networks.list(greedy=True) + assert client.api.networks.called_once_with(greedy=True) + -class ImageTest(unittest.TestCase): +class NetworkTest(unittest.TestCase): def test_connect(self): client = make_fake_client() From 2878900a71a026803fd89cc14c47ac31adbf1485 Mon Sep 17 00:00:00 2001 From: Viktor Adam Date: Sun, 19 Nov 2017 21:03:07 +0000 Subject: [PATCH 0525/1301] Fixing integration tests Signed-off-by: Viktor Adam --- tests/integration/api_network_test.py | 4 ++++ tests/integration/models_networks_test.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/integration/api_network_test.py b/tests/integration/api_network_test.py index 57d2630454..bb900d4566 100644 --- a/tests/integration/api_network_test.py +++ b/tests/integration/api_network_test.py @@ -118,6 +118,7 @@ def test_connect_and_disconnect_container(self): key for net in network_list for key in net['Containers'].keys() + if net['Id'] == net_id ), [container['Id']] ) @@ -156,6 +157,7 @@ def test_connect_and_force_disconnect_container(self): key for net in network_list for key in net['Containers'].keys() + if net['Id'] == net_id ), [container['Id']] ) @@ -169,6 +171,7 @@ def test_connect_and_force_disconnect_container(self): key for net in network_list for key in net['Containers'].keys() + if net['Id'] == net_id )) with pytest.raises(docker.errors.APIError): @@ -216,6 +219,7 @@ def test_connect_on_container_create(self): key for net in network_list for key in net['Containers'].keys() + if net['Id'] == net_id ), [container['Id']] ) diff --git a/tests/integration/models_networks_test.py b/tests/integration/models_networks_test.py index 25fea33eda..08d7ad2955 100644 --- a/tests/integration/models_networks_test.py +++ b/tests/integration/models_networks_test.py @@ -61,7 +61,7 @@ def test_connect_disconnect(self): assert client.networks.get(network.id).containers == [container] network_containers = list( c - for net in client.networks.list(greedy=True) + for net in client.networks.list(ids=[network.id], greedy=True) for c in net.containers ) assert network_containers == [container] From f3dbd017f8c41774f7176319ba1d80a8a1101593 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damien=20Nad=C3=A9?= Date: Mon, 20 Nov 2017 16:47:36 +0100 Subject: [PATCH 0526/1301] Fix for #1815: make APIClient.stop honor container StopTimeout value MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Damien Nadé --- docker/api/container.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/docker/api/container.py b/docker/api/container.py index f3c33c9786..fea64cde7c 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -1112,20 +1112,26 @@ def stats(self, container, decode=None, stream=True): json=True) @utils.check_resource('container') - def stop(self, container, timeout=10): + def stop(self, container, timeout=None): """ Stops a container. Similar to the ``docker stop`` command. Args: container (str): The container to stop timeout (int): Timeout in seconds to wait for the container to - stop before sending a ``SIGKILL``. Default: 10 + stop before sending a ``SIGKILL``. If None, then the + StopTimeout value of the container will be used. + Default: None Raises: :py:class:`docker.errors.APIError` If the server returns an error. """ - params = {'t': timeout} + if timeout is None: + params = {} + timeout = 10 + else: + params = {'t': timeout} url = self._url("/containers/{0}/stop", container) res = self._post(url, params=params, From 6cce101f012d1875029e0a10f45d2d3ad01aedfe Mon Sep 17 00:00:00 2001 From: Alex Villarreal Date: Tue, 21 Nov 2017 10:07:02 -0600 Subject: [PATCH 0527/1301] Add missing call to string format in log message Signed-off-by: Alejandro Villarreal --- docker/auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/auth.py b/docker/auth.py index c3fb062e9b..c0cae5d97a 100644 --- a/docker/auth.py +++ b/docker/auth.py @@ -203,7 +203,7 @@ def parse_auth(entries, raise_on_error=False): # https://github.com/docker/compose/issues/3265 log.debug( 'Auth data for {0} is absent. Client might be using a ' - 'credentials store instead.' + 'credentials store instead.'.format(registry) ) conf[registry] = {} continue From 36ed843e2bbfd50698c16bfbd898d915019ca94d Mon Sep 17 00:00:00 2001 From: Viktor Adam Date: Tue, 21 Nov 2017 21:59:11 +0000 Subject: [PATCH 0528/1301] Only allow greedy queries on the model Signed-off-by: Viktor Adam --- docker/api/network.py | 10 ++----- docker/models/networks.py | 8 +++++- tests/integration/api_network_test.py | 41 --------------------------- tests/unit/models_networks_test.py | 4 --- 4 files changed, 9 insertions(+), 54 deletions(-) diff --git a/docker/api/network.py b/docker/api/network.py index 09f5a8bd5d..797780858a 100644 --- a/docker/api/network.py +++ b/docker/api/network.py @@ -6,7 +6,7 @@ class NetworkApiMixin(object): @minimum_version('1.21') - def networks(self, names=None, ids=None, filters=None, greedy=False): + def networks(self, names=None, ids=None, filters=None): """ List networks. Similar to the ``docker networks ls`` command. @@ -18,8 +18,6 @@ def networks(self, names=None, ids=None, filters=None, greedy=False): - ``driver=[]`` Matches a network's driver. - ``label=[]`` or ``label=[=]``. - ``type=["custom"|"builtin"]`` Filters networks by type. - greedy (bool): Fetch more details for each network individually. - You might want this to get the containers attached to them. Returns: (dict): List of network objects. @@ -38,11 +36,7 @@ def networks(self, names=None, ids=None, filters=None, greedy=False): params = {'filters': utils.convert_filters(filters)} url = self._url("/networks") res = self._get(url, params=params) - result = self._result(res, json=True) - if greedy: - return [self.inspect_network(net['Id']) for net in result] - else: - return result + return self._result(res, json=True) @minimum_version('1.21') def create_network(self, name, driver=None, options=None, ipam=None, diff --git a/docker/models/networks.py b/docker/models/networks.py index 06ff22b165..1c2fbf2465 100644 --- a/docker/models/networks.py +++ b/docker/models/networks.py @@ -1,4 +1,5 @@ from ..api import APIClient +from ..utils import version_gte from .containers import Container from .resource import Model, Collection @@ -201,8 +202,13 @@ def list(self, *args, **kwargs): :py:class:`docker.errors.APIError` If the server returns an error. """ + greedy = kwargs.pop('greedy', False) resp = self.client.api.networks(*args, **kwargs) - return [self.prepare_model(item) for item in resp] + networks = [self.prepare_model(item) for item in resp] + if greedy and version_gte(self.client.api._version, '1.28'): + for net in networks: + net.reload() + return networks def prune(self, filters=None): self.client.api.prune_networks(filters=filters) diff --git a/tests/integration/api_network_test.py b/tests/integration/api_network_test.py index bb900d4566..f4fefde5b9 100644 --- a/tests/integration/api_network_test.py +++ b/tests/integration/api_network_test.py @@ -112,17 +112,6 @@ def test_connect_and_disconnect_container(self): [container['Id']] ) - network_list = self.client.networks(ids=[net_id], greedy=True) - self.assertEqual( - list( - key - for net in network_list - for key in net['Containers'].keys() - if net['Id'] == net_id - ), - [container['Id']] - ) - with pytest.raises(docker.errors.APIError): self.client.connect_container_to_network(container, net_id) @@ -151,29 +140,10 @@ def test_connect_and_force_disconnect_container(self): [container['Id']] ) - network_list = self.client.networks(ids=[net_id], greedy=True) - self.assertEqual( - list( - key - for net in network_list - for key in net['Containers'].keys() - if net['Id'] == net_id - ), - [container['Id']] - ) - self.client.disconnect_container_from_network(container, net_id, True) network_data = self.client.inspect_network(net_id) self.assertFalse(network_data.get('Containers')) - network_list = self.client.networks(ids=[net_id], greedy=True) - self.assertFalse(list( - key - for net in network_list - for key in net['Containers'].keys() - if net['Id'] == net_id - )) - with pytest.raises(docker.errors.APIError): self.client.disconnect_container_from_network( container, net_id, force=True @@ -213,17 +183,6 @@ def test_connect_on_container_create(self): list(network_data['Containers'].keys()), [container['Id']]) - network_list = self.client.networks(ids=[net_id], greedy=True) - self.assertEqual( - list( - key - for net in network_list - for key in net['Containers'].keys() - if net['Id'] == net_id - ), - [container['Id']] - ) - self.client.disconnect_container_from_network(container, net_id) network_data = self.client.inspect_network(net_id) self.assertFalse(network_data.get('Containers')) diff --git a/tests/unit/models_networks_test.py b/tests/unit/models_networks_test.py index df0650a298..58c9fce669 100644 --- a/tests/unit/models_networks_test.py +++ b/tests/unit/models_networks_test.py @@ -36,10 +36,6 @@ def test_list(self): client.networks.list(names=["foobar"]) assert client.api.networks.called_once_with(names=["foobar"]) - client = make_fake_client() - client.networks.list(greedy=True) - assert client.api.networks.called_once_with(greedy=True) - class NetworkTest(unittest.TestCase): From 5c5705045be72530091a51372ae920f958192bfb Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 29 Nov 2017 16:42:28 -0800 Subject: [PATCH 0529/1301] Fix common issues with build context creation: inaccessible files and fifos Signed-off-by: Joffrey F --- docker/utils/utils.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/docker/utils/utils.py b/docker/utils/utils.py index a123fd8f83..6cac4bce6e 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -97,7 +97,12 @@ def create_archive(root, files=None, fileobj=None, gzip=False): if files is None: files = build_file_list(root) for path in files: - i = t.gettarinfo(os.path.join(root, path), arcname=path) + full_path = os.path.join(root, path) + if not os.access(full_path, os.R_OK): + raise IOError( + 'Can not access file in context: {}'.format(full_path) + ) + i = t.gettarinfo(full_path, arcname=path) if i is None: # This happens when we encounter a socket file. We can safely # ignore it and proceed. @@ -108,12 +113,14 @@ def create_archive(root, files=None, fileobj=None, gzip=False): # and directories executable by default. i.mode = i.mode & 0o755 | 0o111 - try: - # We open the file object in binary mode for Windows support. - with open(os.path.join(root, path), 'rb') as f: - t.addfile(i, f) - except IOError: - # When we encounter a directory the file object is set to None. + if i.isfile(): + try: + with open(full_path, 'rb') as f: + t.addfile(i, f) + except IOError: + t.addfile(i, None) + else: + # Directories, FIFOs, symlinks... don't need to be read. t.addfile(i, None) t.close() fileobj.seek(0) From 8d770b012d6786ffb468e0fc929bde309a9500b1 Mon Sep 17 00:00:00 2001 From: Michael Hankin Date: Sun, 3 Dec 2017 14:54:28 -0600 Subject: [PATCH 0530/1301] Change format of extra hosts Signed-off-by: Michael Hankin --- docker/utils/utils.py | 2 +- tests/integration/api_service_test.py | 4 ++-- tests/unit/models_containers_test.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docker/utils/utils.py b/docker/utils/utils.py index a123fd8f83..3e2a710d07 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -566,7 +566,7 @@ def format_env(key, value): def format_extra_hosts(extra_hosts): return [ - '{}:{}'.format(k, v) for k, v in sorted(six.iteritems(extra_hosts)) + '{} {}'.format(v, k) for k, v in sorted(six.iteritems(extra_hosts)) ] diff --git a/tests/integration/api_service_test.py b/tests/integration/api_service_test.py index b931154945..05b5ba752e 100644 --- a/tests/integration/api_service_test.py +++ b/tests/integration/api_service_test.py @@ -588,8 +588,8 @@ def test_create_service_with_hosts(self): assert 'Hosts' in svc_info['Spec']['TaskTemplate']['ContainerSpec'] hosts = svc_info['Spec']['TaskTemplate']['ContainerSpec']['Hosts'] assert len(hosts) == 2 - assert 'foobar:127.0.0.1' in hosts - assert 'baz:8.8.8.8' in hosts + assert '127.0.0.1 foobar' in hosts + assert '8.8.8.8 baz' in hosts @requires_api_version('1.25') def test_create_service_with_hostname(self): diff --git a/tests/unit/models_containers_test.py b/tests/unit/models_containers_test.py index 5eaa45ac66..29a5caad2d 100644 --- a/tests/unit/models_containers_test.py +++ b/tests/unit/models_containers_test.py @@ -141,7 +141,7 @@ def test_create_container_args(self): 'Dns': ['8.8.8.8'], 'DnsOptions': ['foo'], 'DnsSearch': ['example.com'], - 'ExtraHosts': ['foo:1.2.3.4'], + 'ExtraHosts': ['1.2.3.4 foo'], 'GroupAdd': ['blah'], 'IpcMode': 'foo', 'KernelMemory': 123, From 0134939c2c5cc6920339a65a69305227849a452d Mon Sep 17 00:00:00 2001 From: Michael Hankin Date: Tue, 5 Dec 2017 21:19:37 -0600 Subject: [PATCH 0531/1301] Change format in which hosts are being stored for Swarm services Signed-off-by: Michael Hankin --- docker/types/services.py | 2 +- docker/utils/utils.py | 10 ++++++++-- tests/unit/models_containers_test.py | 2 +- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/docker/types/services.py b/docker/types/services.py index 9031e609ab..5b6af8f5a7 100644 --- a/docker/types/services.py +++ b/docker/types/services.py @@ -137,7 +137,7 @@ def __init__(self, image, command=None, args=None, hostname=None, env=None, if labels is not None: self['Labels'] = labels if hosts is not None: - self['Hosts'] = format_extra_hosts(hosts) + self['Hosts'] = format_extra_hosts(hosts, task=True) if mounts is not None: parsed_mounts = [] diff --git a/docker/utils/utils.py b/docker/utils/utils.py index 3e2a710d07..845af6538f 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -564,9 +564,15 @@ def format_env(key, value): return [format_env(*var) for var in six.iteritems(environment)] -def format_extra_hosts(extra_hosts): +def format_extra_hosts(extra_hosts, task=False): + # Use format dictated by Swarm API if container is part of a task + if task: + return [ + '{} {}'.format(v, k) for k, v in sorted(six.iteritems(extra_hosts)) + ] + return [ - '{} {}'.format(v, k) for k, v in sorted(six.iteritems(extra_hosts)) + '{}:{}'.format(k, v) for k, v in sorted(six.iteritems(extra_hosts)) ] diff --git a/tests/unit/models_containers_test.py b/tests/unit/models_containers_test.py index 29a5caad2d..5eaa45ac66 100644 --- a/tests/unit/models_containers_test.py +++ b/tests/unit/models_containers_test.py @@ -141,7 +141,7 @@ def test_create_container_args(self): 'Dns': ['8.8.8.8'], 'DnsOptions': ['foo'], 'DnsSearch': ['example.com'], - 'ExtraHosts': ['1.2.3.4 foo'], + 'ExtraHosts': ['foo:1.2.3.4'], 'GroupAdd': ['blah'], 'IpcMode': 'foo', 'KernelMemory': 123, From 9d23278643cb6b4a097e833915a73ab9a2eba10d Mon Sep 17 00:00:00 2001 From: Anthony Sottile Date: Thu, 7 Dec 2017 13:52:27 -0800 Subject: [PATCH 0532/1301] container: fix docstring for containers() Signed-off-by: Anthony Sottile --- docker/api/container.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/api/container.py b/docker/api/container.py index f3c33c9786..8dd89ccf95 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -139,7 +139,8 @@ def containers(self, quiet=False, all=False, trunc=False, latest=False, Args: quiet (bool): Only display numeric Ids all (bool): Show all containers. Only running containers are shown - by default trunc (bool): Truncate output + by default + trunc (bool): Truncate output latest (bool): Show only the latest created container, include non-running ones. since (str): Show only containers created since Id or Name, include From 61bc8bea7f4f4cc9e57da2a9ae36ce5002e129f9 Mon Sep 17 00:00:00 2001 From: Michael Hankin Date: Tue, 12 Dec 2017 15:49:07 -0600 Subject: [PATCH 0533/1301] Add support for order property when updating a service Signed-off-by: Michael Hankin --- docker/api/service.py | 4 ++++ docker/types/services.py | 11 ++++++++++- tests/integration/api_service_test.py | 18 ++++++++++++++++++ 3 files changed, 32 insertions(+), 1 deletion(-) diff --git a/docker/api/service.py b/docker/api/service.py index 4c10ef8efd..df899f514b 100644 --- a/docker/api/service.py +++ b/docker/api/service.py @@ -19,6 +19,10 @@ def raise_version_error(param, min_version): if 'Monitor' in update_config: raise_version_error('UpdateConfig.monitor', '1.25') + if utils.version_lt(version, '1.29'): + if 'Order' in update_config: + raise_version_error('UpdateConfig.order', '1.29') + if task_template is not None: if 'ForceUpdate' in task_template and utils.version_lt( version, '1.25'): diff --git a/docker/types/services.py b/docker/types/services.py index 9031e609ab..14e2cc32bb 100644 --- a/docker/types/services.py +++ b/docker/types/services.py @@ -334,9 +334,12 @@ class UpdateConfig(dict): max_failure_ratio (float): The fraction of tasks that may fail during an update before the failure action is invoked, specified as a floating point number between 0 and 1. Default: 0 + order (string): Specifies the order of operations when rolling out an + updated task. Either ``start_first`` or ``stop_first`` are accepted. + Default: ``stop_first`` """ def __init__(self, parallelism=0, delay=None, failure_action='continue', - monitor=None, max_failure_ratio=None): + monitor=None, max_failure_ratio=None, order='stop-first'): self['Parallelism'] = parallelism if delay is not None: self['Delay'] = delay @@ -360,6 +363,12 @@ def __init__(self, parallelism=0, delay=None, failure_action='continue', ) self['MaxFailureRatio'] = max_failure_ratio + if order not in ('start-first', 'stop-first'): + raise errors.InvalidArgument( + 'order must be either `start-first` or `stop-first`' + ) + self['Order'] = order + class RestartConditionTypesEnum(object): _values = ( diff --git a/tests/integration/api_service_test.py b/tests/integration/api_service_test.py index b931154945..fb57f63950 100644 --- a/tests/integration/api_service_test.py +++ b/tests/integration/api_service_test.py @@ -386,6 +386,24 @@ def test_create_service_with_env(self): assert 'Env' in con_spec assert con_spec['Env'] == ['DOCKER_PY_TEST=1'] + @requires_api_version('1.29') + def test_create_service_with_update_order(self): + container_spec = docker.types.ContainerSpec(BUSYBOX, ['true']) + task_tmpl = docker.types.TaskTemplate(container_spec) + update_config = docker.types.UpdateConfig( + parallelism=10, delay=5, order='start-first' + ) + name = self.get_service_name() + svc_id = self.client.create_service( + task_tmpl, update_config=update_config, name=name + ) + svc_info = self.client.inspect_service(svc_id) + assert 'UpdateConfig' in svc_info['Spec'] + uc = svc_info['Spec']['UpdateConfig'] + assert update_config['Parallelism'] == uc['Parallelism'] + assert update_config['Delay'] == uc['Delay'] + assert update_config['Order'] == uc['Order'] + @requires_api_version('1.25') def test_create_service_with_tty(self): container_spec = docker.types.ContainerSpec( From 7db76737ca661f242f9345a2be9a317c79da7575 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 13 Dec 2017 18:32:59 -0800 Subject: [PATCH 0534/1301] Fix URL-quoting for resource names containing spaces Signed-off-by: Joffrey F --- docker/api/client.py | 2 +- tests/integration/api_network_test.py | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/docker/api/client.py b/docker/api/client.py index cbe74b916f..01a83ea497 100644 --- a/docker/api/client.py +++ b/docker/api/client.py @@ -206,7 +206,7 @@ def _url(self, pathfmt, *args, **kwargs): 'instead'.format(arg, type(arg)) ) - quote_f = partial(six.moves.urllib.parse.quote_plus, safe="/:") + quote_f = partial(six.moves.urllib.parse.quote, safe="/:") args = map(quote_f, args) if kwargs.get('versioned_api', True): diff --git a/tests/integration/api_network_test.py b/tests/integration/api_network_test.py index f4fefde5b9..10e09dd70d 100644 --- a/tests/integration/api_network_test.py +++ b/tests/integration/api_network_test.py @@ -484,3 +484,10 @@ def test_create_inspect_network_with_scope(self): assert self.client.inspect_network(net_name_swarm, scope='swarm') with pytest.raises(docker.errors.NotFound): self.client.inspect_network(net_name_swarm, scope='local') + + @requires_api_version('1.21') + def test_create_remove_network_with_space_in_name(self): + net_id = self.client.create_network('test 01') + self.tmp_networks.append(net_id) + assert self.client.inspect_network('test 01') + assert self.client.remove_network('test 01') is None # does not raise From 445cb18723fe4e9aa3f98020b001842cc9ee8273 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 13 Dec 2017 19:06:29 -0800 Subject: [PATCH 0535/1301] Add integration test for CPU realtime options Signed-off-by: Joffrey F --- tests/integration/api_container_test.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index f03ccdb436..5e30eee27d 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -464,6 +464,20 @@ def test_create_with_init_path(self): config = self.client.inspect_container(ctnr) assert config['HostConfig']['InitPath'] == "/usr/libexec/docker-init" + @requires_api_version('1.24') + @pytest.mark.xfail(not os.path.exists('/sys/fs/cgroup/cpu.rt_runtime_us'), + reason='CONFIG_RT_GROUP_SCHED isn\'t enabled') + def test_create_with_cpu_rt_options(self): + ctnr = self.client.create_container( + BUSYBOX, 'true', host_config=self.client.create_host_config( + cpu_rt_period=1000, cpu_rt_runtime=500 + ) + ) + self.tmp_containers.append(ctnr) + config = self.client.inspect_container(ctnr) + assert config['HostConfig']['CpuRealtimeRuntime'] == 500 + assert config['HostConfig']['CpuRealtimePeriod'] == 1000 + class VolumeBindTest(BaseAPIIntegrationTest): def setUp(self): From a66c89247a1f896090e23ea6820d77a40cca978b Mon Sep 17 00:00:00 2001 From: Viktor Adam Date: Thu, 14 Dec 2017 09:55:36 +0000 Subject: [PATCH 0536/1301] Renaming new argument Signed-off-by: Viktor Adam --- docker/api/service.py | 53 ++++++++++++--------------- docker/models/services.py | 6 +-- tests/integration/api_service_test.py | 30 +++++++-------- 3 files changed, 41 insertions(+), 48 deletions(-) diff --git a/docker/api/service.py b/docker/api/service.py index 768d513120..66d98c7845 100644 --- a/docker/api/service.py +++ b/docker/api/service.py @@ -62,6 +62,21 @@ def raise_version_error(param, min_version): raise_version_error('ContainerSpec.privileges', '1.30') +def _merge_task_template(current, override): + merged = current.copy() + if override is not None: + for ts_key, ts_value in override.items(): + if ts_key == 'ContainerSpec': + if 'ContainerSpec' not in merged: + merged['ContainerSpec'] = {} + for cs_key, cs_value in override['ContainerSpec'].items(): + if cs_value is not None: + merged['ContainerSpec'][cs_key] = cs_value + elif ts_value is not None: + merged[ts_key] = ts_value + return merged + + class ServiceApiMixin(object): @utils.minimum_version('1.24') def create_service( @@ -306,7 +321,7 @@ def tasks(self, filters=None): def update_service(self, service, version, task_template=None, name=None, labels=None, mode=None, update_config=None, networks=None, endpoint_config=None, - endpoint_spec=None, use_current_spec=False): + endpoint_spec=None, fetch_current_spec=False): """ Update a service. @@ -328,8 +343,8 @@ def update_service(self, service, version, task_template=None, name=None, the service to. Default: ``None``. endpoint_spec (EndpointSpec): Properties that can be configured to access and load balance a service. Default: ``None``. - use_current_spec (boolean): Use the undefined settings from the - previous specification of the service. Default: ``False`` + fetch_current_spec (boolean): Use the undefined settings from the + current specification of the service. Default: ``False`` Returns: ``True`` if successful. @@ -347,11 +362,10 @@ def update_service(self, service, version, task_template=None, name=None, _check_api_features(self._version, task_template, update_config) - if use_current_spec: + if fetch_current_spec: + inspect_defaults = True if utils.version_lt(self._version, '1.29'): inspect_defaults = None - else: - inspect_defaults = True current = self.inspect_service( service, insert_defaults=inspect_defaults )['Spec'] @@ -363,15 +377,9 @@ def update_service(self, service, version, task_template=None, name=None, data = {} headers = {} - if name is not None: - data['Name'] = name - else: - data['Name'] = current.get('Name') + data['Name'] = current.get('Name') if name is None else name - if labels is not None: - data['Labels'] = labels - else: - data['Labels'] = current.get('Labels') + data['Labels'] = current.get('Labels') if labels is None else labels if mode is not None: if not isinstance(mode, dict): @@ -380,7 +388,7 @@ def update_service(self, service, version, task_template=None, name=None, else: data['Mode'] = current.get('Mode') - data['TaskTemplate'] = self._merge_task_template( + data['TaskTemplate'] = _merge_task_template( current.get('TaskTemplate', {}), task_template ) @@ -418,18 +426,3 @@ def update_service(self, service, version, task_template=None, name=None, ) self._raise_for_status(resp) return True - - @staticmethod - def _merge_task_template(current, override): - merged = current.copy() - if override is not None: - for ts_key, ts_value in override.items(): - if ts_key == 'ContainerSpec': - if 'ContainerSpec' not in merged: - merged['ContainerSpec'] = {} - for cs_key, cs_value in override['ContainerSpec'].items(): - if cs_value is not None: - merged['ContainerSpec'][cs_key] = cs_value - elif ts_value is not None: - merged[ts_key] = ts_value - return merged diff --git a/docker/models/services.py b/docker/models/services.py index 39c86efb27..009e4551ac 100644 --- a/docker/models/services.py +++ b/docker/models/services.py @@ -299,10 +299,10 @@ def _get_create_service_kwargs(func_name, kwargs): if 'force_update' in kwargs: task_template_kwargs['force_update'] = kwargs.pop('force_update') - # use the current spec by default if updating the service + # fetch the current spec by default if updating the service # through the model - use_current_spec = kwargs.pop('use_current_spec', True) - create_kwargs['use_current_spec'] = use_current_spec + fetch_current_spec = kwargs.pop('fetch_current_spec', True) + create_kwargs['fetch_current_spec'] = fetch_current_spec # All kwargs should have been consumed by this point, so raise # error if any are left diff --git a/tests/integration/api_service_test.py b/tests/integration/api_service_test.py index 10ae1804b2..a35d3a5764 100644 --- a/tests/integration/api_service_test.py +++ b/tests/integration/api_service_test.py @@ -726,7 +726,7 @@ def test_update_service_with_defaults_name(self): task_tmpl = docker.types.TaskTemplate(container_spec, force_update=10) self._update_service( - svc_id, name, version_index, task_tmpl, use_current_spec=True + svc_id, name, version_index, task_tmpl, fetch_current_spec=True ) svc_info = self.client.inspect_service(svc_id) new_index = svc_info['Version']['Index'] @@ -753,7 +753,7 @@ def test_update_service_with_defaults_labels(self): task_tmpl = docker.types.TaskTemplate(container_spec, force_update=10) self._update_service( svc_id, name, version_index, task_tmpl, name=name, - use_current_spec=True + fetch_current_spec=True ) svc_info = self.client.inspect_service(svc_id) new_index = svc_info['Version']['Index'] @@ -781,7 +781,7 @@ def test_update_service_with_defaults_mode(self): self._update_service( svc_id, name, version_index, labels={'force': 'update'}, - use_current_spec=True + fetch_current_spec=True ) svc_info = self.client.inspect_service(svc_id) new_index = svc_info['Version']['Index'] @@ -811,7 +811,7 @@ def test_update_service_with_defaults_container_labels(self): self._update_service( svc_id, name, version_index, labels={'force': 'update'}, - use_current_spec=True + fetch_current_spec=True ) svc_info = self.client.inspect_service(svc_id) new_index = svc_info['Version']['Index'] @@ -827,7 +827,7 @@ def test_update_service_with_defaults_container_labels(self): ) task_tmpl = docker.types.TaskTemplate(container_spec) self._update_service( - svc_id, name, new_index, task_tmpl, use_current_spec=True + svc_id, name, new_index, task_tmpl, fetch_current_spec=True ) svc_info = self.client.inspect_service(svc_id) newer_index = svc_info['Version']['Index'] @@ -858,7 +858,7 @@ def test_update_service_with_defaults_update_config(self): self._update_service( svc_id, name, version_index, labels={'force': 'update'}, - use_current_spec=True + fetch_current_spec=True ) svc_info = self.client.inspect_service(svc_id) new_index = svc_info['Version']['Index'] @@ -896,7 +896,7 @@ def test_update_service_with_defaults_networks(self): self._update_service( svc_id, name, version_index, labels={'force': 'update'}, - use_current_spec=True + fetch_current_spec=True ) svc_info = self.client.inspect_service(svc_id) new_index = svc_info['Version']['Index'] @@ -908,7 +908,7 @@ def test_update_service_with_defaults_networks(self): self._update_service( svc_id, name, new_index, networks=[net1['Id']], - use_current_spec=True + fetch_current_spec=True ) svc_info = self.client.inspect_service(svc_id) assert 'Networks' in svc_info['Spec']['TaskTemplate'] @@ -951,7 +951,7 @@ def test_update_service_with_defaults_endpoint_spec(self): self._update_service( svc_id, name, version_index, labels={'force': 'update'}, - use_current_spec=True + fetch_current_spec=True ) svc_info = self.client.inspect_service(svc_id) new_index = svc_info['Version']['Index'] @@ -1003,7 +1003,7 @@ def test_update_service_remove_healthcheck(self): version_index = svc_info['Version']['Index'] self._update_service( - svc_id, name, version_index, task_tmpl, use_current_spec=True + svc_id, name, version_index, task_tmpl, fetch_current_spec=True ) svc_info = self.client.inspect_service(svc_id) new_index = svc_info['Version']['Index'] @@ -1030,7 +1030,7 @@ def test_update_service_remove_labels(self): version_index = svc_info['Version']['Index'] self._update_service( - svc_id, name, version_index, labels={}, use_current_spec=True + svc_id, name, version_index, labels={}, fetch_current_spec=True ) svc_info = self.client.inspect_service(svc_id) new_index = svc_info['Version']['Index'] @@ -1061,7 +1061,7 @@ def test_update_service_remove_container_labels(self): ) task_tmpl = docker.types.TaskTemplate(container_spec) self._update_service( - svc_id, name, version_index, task_tmpl, use_current_spec=True + svc_id, name, version_index, task_tmpl, fetch_current_spec=True ) svc_info = self.client.inspect_service(svc_id) new_index = svc_info['Version']['Index'] @@ -1100,7 +1100,7 @@ def test_update_service_with_network_change(self): task_tmpl = docker.types.TaskTemplate(container_spec) self._update_service( svc_id, name, version_index, task_tmpl, name=name, - networks=[net2['Id']], use_current_spec=True + networks=[net2['Id']], fetch_current_spec=True ) svc_info = self.client.inspect_service(svc_id) task_template = svc_info['Spec']['TaskTemplate'] @@ -1114,7 +1114,7 @@ def test_update_service_with_network_change(self): self._update_service( svc_id, name, new_index, name=name, networks=[net1['Id']], - use_current_spec=True + fetch_current_spec=True ) svc_info = self.client.inspect_service(svc_id) task_template = svc_info['Spec']['TaskTemplate'] @@ -1136,7 +1136,7 @@ def test_update_service_with_network_change(self): ) self._update_service( svc_id, name, new_index, task_tmpl, name=name, - use_current_spec=True + fetch_current_spec=True ) svc_info = self.client.inspect_service(svc_id) task_template = svc_info['Spec']['TaskTemplate'] From 49d09583aa7a82859cfaa2415c26833cd2473519 Mon Sep 17 00:00:00 2001 From: Michael Hankin Date: Thu, 14 Dec 2017 08:58:26 -0600 Subject: [PATCH 0537/1301] Correct default value of order parameter Signed-off-by: Michael Hankin --- docker/types/services.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docker/types/services.py b/docker/types/services.py index 14e2cc32bb..bc77b0abca 100644 --- a/docker/types/services.py +++ b/docker/types/services.py @@ -336,10 +336,9 @@ class UpdateConfig(dict): floating point number between 0 and 1. Default: 0 order (string): Specifies the order of operations when rolling out an updated task. Either ``start_first`` or ``stop_first`` are accepted. - Default: ``stop_first`` """ def __init__(self, parallelism=0, delay=None, failure_action='continue', - monitor=None, max_failure_ratio=None, order='stop-first'): + monitor=None, max_failure_ratio=None, order=None): self['Parallelism'] = parallelism if delay is not None: self['Delay'] = delay @@ -363,11 +362,12 @@ def __init__(self, parallelism=0, delay=None, failure_action='continue', ) self['MaxFailureRatio'] = max_failure_ratio - if order not in ('start-first', 'stop-first'): - raise errors.InvalidArgument( - 'order must be either `start-first` or `stop-first`' - ) - self['Order'] = order + if order is not None: + if order not in ('start-first', 'stop-first'): + raise errors.InvalidArgument( + 'order must be either `start-first` or `stop-first`' + ) + self['Order'] = order class RestartConditionTypesEnum(object): From b6d0dc1e5a67e39a497052e713df9e478ea15d29 Mon Sep 17 00:00:00 2001 From: Felipe Ruhland Date: Thu, 14 Dec 2017 22:33:11 -0200 Subject: [PATCH 0538/1301] Fixed DEFAULT API VERSION in docstrings. Signed-off-by: Felipe Ruhland --- docker/api/client.py | 16 ++++++++-------- docker/client.py | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docker/api/client.py b/docker/api/client.py index 01a83ea497..f0a86d4596 100644 --- a/docker/api/client.py +++ b/docker/api/client.py @@ -63,21 +63,21 @@ class APIClient( >>> import docker >>> client = docker.APIClient(base_url='unix://var/run/docker.sock') >>> client.version() - {u'ApiVersion': u'1.24', + {u'ApiVersion': u'1.33', u'Arch': u'amd64', - u'BuildTime': u'2016-09-27T23:38:15.810178467+00:00', - u'Experimental': True, - u'GitCommit': u'45bed2c', - u'GoVersion': u'go1.6.3', - u'KernelVersion': u'4.4.22-moby', + u'BuildTime': u'2017-11-19T18:46:37.000000000+00:00', + u'GitCommit': u'f4ffd2511c', + u'GoVersion': u'go1.9.2', + u'KernelVersion': u'4.14.3-1-ARCH', + u'MinAPIVersion': u'1.12', u'Os': u'linux', - u'Version': u'1.12.2-rc1'} + u'Version': u'17.10.0-ce'} Args: base_url (str): URL to the Docker server. For example, ``unix:///var/run/docker.sock`` or ``tcp://127.0.0.1:1234``. version (str): The version of the API to use. Set to ``auto`` to - automatically detect the server's version. Default: ``1.26`` + automatically detect the server's version. Default: ``1.30`` timeout (int): Default timeout for API calls, in seconds. tls (bool or :py:class:`~docker.tls.TLSConfig`): Enable TLS. Pass ``True`` to enable it with default options, or pass a diff --git a/docker/client.py b/docker/client.py index 29968c1f0d..467583e639 100644 --- a/docker/client.py +++ b/docker/client.py @@ -26,7 +26,7 @@ class DockerClient(object): base_url (str): URL to the Docker server. For example, ``unix:///var/run/docker.sock`` or ``tcp://127.0.0.1:1234``. version (str): The version of the API to use. Set to ``auto`` to - automatically detect the server's version. Default: ``1.26`` + automatically detect the server's version. Default: ``1.30`` timeout (int): Default timeout for API calls, in seconds. tls (bool or :py:class:`~docker.tls.TLSConfig`): Enable TLS. Pass ``True`` to enable it with default options, or pass a @@ -60,7 +60,7 @@ def from_env(cls, **kwargs): Args: version (str): The version of the API to use. Set to ``auto`` to - automatically detect the server's version. Default: ``1.26`` + automatically detect the server's version. Default: ``1.30`` timeout (int): Default timeout for API calls, in seconds. ssl_version (int): A valid `SSL version`_. assert_hostname (bool): Verify the hostname of the server. From 6b8dfe42499345aaa1701d835ea0a9b86a00f1a6 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 14 Dec 2017 15:12:08 -0800 Subject: [PATCH 0539/1301] Retrieve container logs before container exits / is removed Signed-off-by: Joffrey F --- docker/models/containers.py | 29 ++++++++++++++------- tests/integration/models_containers_test.py | 23 +++++++++++++--- tests/unit/fake_api_client.py | 2 +- tests/unit/models_containers_test.py | 7 +++-- 4 files changed, 43 insertions(+), 18 deletions(-) diff --git a/docker/models/containers.py b/docker/models/containers.py index 4e3d218e53..f16b7cd60d 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -629,6 +629,9 @@ def run(self, image, command=None, stdout=True, stderr=False, (e.g. ``SIGINT``). storage_opt (dict): Storage driver options per container as a key-value mapping. + stream (bool): If true and ``detach`` is false, return a log + generator instead of a string. Ignored if ``detach`` is true. + Default: ``False``. sysctls (dict): Kernel parameters to set in the container. tmpfs (dict): Temporary filesystems to mount, as a dictionary mapping a path inside the container to options for that path. @@ -696,6 +699,7 @@ def run(self, image, command=None, stdout=True, stderr=False, """ if isinstance(image, Image): image = image.id + stream = kwargs.pop('stream', False) detach = kwargs.pop("detach", False) if detach and remove: if version_gte(self.client.api._version, '1.25'): @@ -723,23 +727,28 @@ def run(self, image, command=None, stdout=True, stderr=False, if detach: return container - exit_status = container.wait() - if exit_status != 0: - stdout = False - stderr = True - logging_driver = container.attrs['HostConfig']['LogConfig']['Type'] + out = None if logging_driver == 'json-file' or logging_driver == 'journald': - out = container.logs(stdout=stdout, stderr=stderr) - else: - out = None + out = container.logs( + stdout=stdout, stderr=stderr, stream=True, follow=True + ) + + exit_status = container.wait() + if exit_status != 0: + out = container.logs(stdout=False, stderr=True) if remove: container.remove() if exit_status != 0: - raise ContainerError(container, exit_status, command, image, out) - return out + raise ContainerError( + container, exit_status, command, image, out + ) + + return out if stream or out is None else b''.join( + [line for line in out] + ) def create(self, image, command=None, **kwargs): """ diff --git a/tests/integration/models_containers_test.py b/tests/integration/models_containers_test.py index ce3349baa7..7707ae2654 100644 --- a/tests/integration/models_containers_test.py +++ b/tests/integration/models_containers_test.py @@ -1,7 +1,7 @@ import docker import tempfile from .base import BaseIntegrationTest, TEST_API_VERSION -from ..helpers import random_name +from ..helpers import random_name, requires_api_version class ContainerCollectionTest(BaseIntegrationTest): @@ -95,7 +95,7 @@ def test_run_with_none_driver(self): "alpine", "echo hello", log_config=dict(type='none') ) - self.assertEqual(out, None) + assert out is None def test_run_with_json_file_driver(self): client = docker.from_env(version=TEST_API_VERSION) @@ -104,7 +104,24 @@ def test_run_with_json_file_driver(self): "alpine", "echo hello", log_config=dict(type='json-file') ) - self.assertEqual(out, b'hello\n') + assert out == b'hello\n' + + @requires_api_version('1.25') + def test_run_with_auto_remove(self): + client = docker.from_env(version=TEST_API_VERSION) + out = client.containers.run( + 'alpine', 'echo hello', auto_remove=True + ) + assert out == b'hello\n' + + def test_run_with_streamed_logs(self): + client = docker.from_env(version=TEST_API_VERSION) + out = client.containers.run( + 'alpine', 'sh -c "echo hello && echo world"', stream=True + ) + logs = [line for line in out] + assert logs[0] == b'hello\n' + assert logs[1] == b'world\n' def test_get(self): client = docker.from_env(version=TEST_API_VERSION) diff --git a/tests/unit/fake_api_client.py b/tests/unit/fake_api_client.py index 47890ace91..f908355101 100644 --- a/tests/unit/fake_api_client.py +++ b/tests/unit/fake_api_client.py @@ -43,7 +43,7 @@ def make_fake_api_client(): fake_api.get_fake_inspect_container()[1], 'inspect_image.return_value': fake_api.get_fake_inspect_image()[1], 'inspect_network.return_value': fake_api.get_fake_network()[1], - 'logs.return_value': 'hello world\n', + 'logs.return_value': [b'hello world\n'], 'networks.return_value': fake_api.get_fake_network_list()[1], 'start.return_value': None, 'wait.return_value': 0, diff --git a/tests/unit/models_containers_test.py b/tests/unit/models_containers_test.py index 5eaa45ac66..a479e836e6 100644 --- a/tests/unit/models_containers_test.py +++ b/tests/unit/models_containers_test.py @@ -12,7 +12,7 @@ def test_run(self): client = make_fake_client() out = client.containers.run("alpine", "echo hello world") - assert out == 'hello world\n' + assert out == b'hello world\n' client.api.create_container.assert_called_with( image="alpine", @@ -24,9 +24,8 @@ def test_run(self): client.api.start.assert_called_with(FAKE_CONTAINER_ID) client.api.wait.assert_called_with(FAKE_CONTAINER_ID) client.api.logs.assert_called_with( - FAKE_CONTAINER_ID, - stderr=False, - stdout=True + FAKE_CONTAINER_ID, stderr=False, stdout=True, stream=True, + follow=True ) def test_create_container_args(self): From adbb3079ddb99eb64c3e8ec4ee1548306ebc2844 Mon Sep 17 00:00:00 2001 From: Boik Date: Sun, 10 Dec 2017 14:41:52 +0800 Subject: [PATCH 0540/1301] make the error message clearer Signed-off-by: Boik --- docker/api/container.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/api/container.py b/docker/api/container.py index 8dd89ccf95..af054fef8c 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -838,7 +838,8 @@ def logs(self, container, stdout=True, stderr=True, stream=False, params['since'] = since else: raise errors.InvalidArgument( - 'since value should be datetime or int, not {}'. + 'since value should be datetime or positive int\ + , not {}'. format(type(since)) ) url = self._url("/containers/{0}/logs", container) From b20f800db6f6521995268a5d7a4746c017fc6d9e Mon Sep 17 00:00:00 2001 From: Constantine Peresypkin Date: Sat, 2 Dec 2017 17:18:09 -0500 Subject: [PATCH 0541/1301] fixes create_api_error_from_http_exception() `create_api_error_from_http_exception()` is never tested in the original code and will fail miserably when fed with empty `HTTPError` object see fixes in requests for this behaviour: https://github.com/requests/requests/pull/3179 Signed-off-by: Constantine Peresypkin --- docker/errors.py | 2 +- tests/unit/errors_test.py | 16 +++++++++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/docker/errors.py b/docker/errors.py index 2a2f871e5d..50423a268d 100644 --- a/docker/errors.py +++ b/docker/errors.py @@ -18,7 +18,7 @@ def create_api_error_from_http_exception(e): try: explanation = response.json()['message'] except ValueError: - explanation = response.content.strip() + explanation = (response.content or '').strip() cls = APIError if response.status_code == 404: if explanation and ('No such image' in str(explanation) or diff --git a/tests/unit/errors_test.py b/tests/unit/errors_test.py index 9678669c3f..e27a9b1975 100644 --- a/tests/unit/errors_test.py +++ b/tests/unit/errors_test.py @@ -3,7 +3,8 @@ import requests from docker.errors import (APIError, ContainerError, DockerException, - create_unexpected_kwargs_error) + create_unexpected_kwargs_error, + create_api_error_from_http_exception) from .fake_api import FAKE_CONTAINER_ID, FAKE_IMAGE_ID from .fake_api_client import make_fake_client @@ -78,6 +79,19 @@ def test_is_client_error_400(self): err = APIError('', response=resp) assert err.is_client_error() is True + def test_create_error_from_exception(self): + resp = requests.Response() + resp.status_code = 500 + err = APIError('') + try: + resp.raise_for_status() + except requests.exceptions.HTTPError as e: + try: + create_api_error_from_http_exception(e) + except APIError as e: + err = e + assert err.is_server_error() is True + class ContainerErrorTest(unittest.TestCase): def test_container_without_stderr(self): From f10c008aa57c4c48cce1a718a0160a54a2b1a371 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 18 Dec 2017 18:21:51 -0800 Subject: [PATCH 0542/1301] Bump 2.7.0 + changelog Signed-off-by: Joffrey F --- docker/version.py | 2 +- docs/change-log.md | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index fd82246174..2502183331 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "2.7.0-dev" +version = "2.7.0" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) diff --git a/docs/change-log.md b/docs/change-log.md index 57293f3e14..b8298a7981 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -1,6 +1,41 @@ Change log ========== +2.7.0 +----- + +[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/41?closed=1) + +### Features + +* Added `unlock_swarm` and `get_unlock_key` methods to the `APIClient`. + * Added `unlock` and `get_unlock_key` to `DockerClient.swarm`. +* Added a `greedy` parameter to `DockerClient.networks.list`, yielding + additional details about the listed networks. +* Added `cpu_rt_runtime` and `cpu_rt_period` as parameters to + `APIClient.create_host_config` and `DockerClient.containers.run`. +* Added the `order` argument to `UpdateConfig`. +* Added `fetch_current_spec` to `APIClient.update_service` and `Service.update` + that will retrieve the current configuration of the service and merge it with + the provided parameters to determine the new configuration. + +### Bugfixes + +* Fixed a bug where the `build` method tried to include inaccessible files + in the context, leading to obscure errors during the build phase + (inaccessible files inside the context now raise an `IOError` instead). +* Fixed a bug where the `build` method would try to read from FIFOs present + inside the build context, causing it to hang. +* `APIClient.stop` will no longer override the `stop_timeout` value present + in the container's configuration. +* Fixed a bug preventing removal of networks with names containing a space. +* Fixed a bug where `DockerClient.containers.run` would crash if the + `auto_remove` parameter was set to `True`. +* Changed the default value of `listen_addr` in `join_swarm` to match the + one in `init_swarm`. +* Fixed a bug where handling HTTP errors with no body would cause an unexpected + exception to be thrown while generating an `APIError` object. + 2.6.1 ----- From 2250fa2ebd03f5f01db06fa85998212d97882680 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 19 Dec 2017 13:21:56 -0800 Subject: [PATCH 0543/1301] Don't attempt to retrieve container's stderr if `auto_remove` was set Signed-off-by: Joffrey F --- docker/models/containers.py | 4 +++- tests/integration/models_containers_test.py | 14 +++++++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/docker/models/containers.py b/docker/models/containers.py index f16b7cd60d..6ba308e492 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -737,7 +737,9 @@ def run(self, image, command=None, stdout=True, stderr=False, exit_status = container.wait() if exit_status != 0: - out = container.logs(stdout=False, stderr=True) + out = None + if not kwargs.get('auto_remove'): + out = container.logs(stdout=False, stderr=True) if remove: container.remove() diff --git a/tests/integration/models_containers_test.py b/tests/integration/models_containers_test.py index 7707ae2654..d246189d09 100644 --- a/tests/integration/models_containers_test.py +++ b/tests/integration/models_containers_test.py @@ -1,5 +1,7 @@ -import docker import tempfile + +import docker +import pytest from .base import BaseIntegrationTest, TEST_API_VERSION from ..helpers import random_name, requires_api_version @@ -114,6 +116,16 @@ def test_run_with_auto_remove(self): ) assert out == b'hello\n' + @requires_api_version('1.25') + def test_run_with_auto_remove_error(self): + client = docker.from_env(version=TEST_API_VERSION) + with pytest.raises(docker.errors.ContainerError) as e: + client.containers.run( + 'alpine', 'sh -c ">&2 echo error && exit 1"', auto_remove=True + ) + assert e.value.exit_status == 1 + assert e.value.stderr is None + def test_run_with_streamed_logs(self): client = docker.from_env(version=TEST_API_VERSION) out = client.containers.run( From 598f16771ca4673886c3ea9b86fa280d77829beb Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 19 Dec 2017 13:21:56 -0800 Subject: [PATCH 0544/1301] Don't attempt to retrieve container's stderr if `auto_remove` was set Signed-off-by: Joffrey F --- docker/models/containers.py | 4 +++- tests/integration/models_containers_test.py | 14 +++++++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/docker/models/containers.py b/docker/models/containers.py index f16b7cd60d..6ba308e492 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -737,7 +737,9 @@ def run(self, image, command=None, stdout=True, stderr=False, exit_status = container.wait() if exit_status != 0: - out = container.logs(stdout=False, stderr=True) + out = None + if not kwargs.get('auto_remove'): + out = container.logs(stdout=False, stderr=True) if remove: container.remove() diff --git a/tests/integration/models_containers_test.py b/tests/integration/models_containers_test.py index 7707ae2654..d246189d09 100644 --- a/tests/integration/models_containers_test.py +++ b/tests/integration/models_containers_test.py @@ -1,5 +1,7 @@ -import docker import tempfile + +import docker +import pytest from .base import BaseIntegrationTest, TEST_API_VERSION from ..helpers import random_name, requires_api_version @@ -114,6 +116,16 @@ def test_run_with_auto_remove(self): ) assert out == b'hello\n' + @requires_api_version('1.25') + def test_run_with_auto_remove_error(self): + client = docker.from_env(version=TEST_API_VERSION) + with pytest.raises(docker.errors.ContainerError) as e: + client.containers.run( + 'alpine', 'sh -c ">&2 echo error && exit 1"', auto_remove=True + ) + assert e.value.exit_status == 1 + assert e.value.stderr is None + def test_run_with_streamed_logs(self): client = docker.from_env(version=TEST_API_VERSION) out = client.containers.run( From 9fc45f108af80decf26e1c95aa327f60ac794974 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 19 Dec 2017 14:37:17 -0800 Subject: [PATCH 0545/1301] 2.8.0 dev Signed-off-by: Joffrey F --- docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index 2502183331..5b76748510 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "2.7.0" +version = "2.8.0-dev" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) From 758344fdf4b6447fc46270c61c8a4f914f0445a5 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 19 Dec 2017 14:38:38 -0800 Subject: [PATCH 0546/1301] Formatting Signed-off-by: Joffrey F --- docker/api/container.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/api/container.py b/docker/api/container.py index 63c0018587..494f7b4662 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -838,8 +838,8 @@ def logs(self, container, stdout=True, stderr=True, stream=False, params['since'] = since else: raise errors.InvalidArgument( - 'since value should be datetime or positive int\ - , not {}'. + 'since value should be datetime or positive int, ' + 'not {}'. format(type(since)) ) url = self._url("/containers/{0}/logs", container) From edb9e3c2ae85f382549df8365a5b2f18986fc5e2 Mon Sep 17 00:00:00 2001 From: Felipe Ruhland Date: Thu, 21 Dec 2017 23:13:18 -0200 Subject: [PATCH 0547/1301] Added scale method to the Service model. Signed-off-by: Felipe Ruhland --- docker/models/services.py | 18 +++++++++++++++++- tests/integration/models_services_test.py | 22 ++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/docker/models/services.py b/docker/models/services.py index 009e4551ac..e87e2d43ad 100644 --- a/docker/models/services.py +++ b/docker/models/services.py @@ -1,6 +1,6 @@ import copy from docker.errors import create_unexpected_kwargs_error -from docker.types import TaskTemplate, ContainerSpec +from docker.types import TaskTemplate, ContainerSpec, ServiceMode from .resource import Model, Collection @@ -105,6 +105,22 @@ def logs(self, **kwargs): ) return self.client.api.service_logs(self.id, is_tty=is_tty, **kwargs) + def scale(self, replicas): + """ + Scale service container. + + Args: + replicas (int): The number of containers that should be running. + + Returns: + ``True``if successful. + """ + + service_mode = ServiceMode('replicated', replicas) + return self.client.api.update_service(self.id, self.version, + service_mode, + fetch_current_spec=True) + class ServiceCollection(Collection): """Services on the Docker server.""" diff --git a/tests/integration/models_services_test.py b/tests/integration/models_services_test.py index ca8be48de4..e7f16cdff2 100644 --- a/tests/integration/models_services_test.py +++ b/tests/integration/models_services_test.py @@ -203,6 +203,28 @@ def test_scale_service(self): spec = service.attrs['Spec']['TaskTemplate']['ContainerSpec'] assert spec.get('Command') == ['sleep', '300'] + def test_scale_method_service(self): + client = docker.from_env(version=TEST_API_VERSION) + service = client.services.create( + # create arguments + name=helpers.random_name(), + # ContainerSpec arguments + image="alpine", + command="sleep 300" + ) + tasks = [] + while len(tasks) == 0: + tasks = service.tasks() + assert len(tasks) == 1 + service.scale(2) + while len(tasks) == 1: + tasks = service.tasks() + assert len(tasks) >= 2 + # check that the container spec is not overridden with None + service.reload() + spec = service.attrs['Spec']['TaskTemplate']['ContainerSpec'] + assert spec.get('Command') == ['sleep', '300'] + @helpers.requires_api_version('1.25') def test_restart_service(self): client = docker.from_env(version=TEST_API_VERSION) From 0e0a8526801d207bfdf2e9011ab1348525437a79 Mon Sep 17 00:00:00 2001 From: Felipe Ruhland Date: Tue, 26 Dec 2017 18:25:20 -0200 Subject: [PATCH 0548/1301] Ensure that global containers are not scaled Signed-off-by: Felipe Ruhland --- docker/models/services.py | 5 ++++- tests/integration/models_services_test.py | 24 +++++++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/docker/models/services.py b/docker/models/services.py index e87e2d43ad..4c8accefdb 100644 --- a/docker/models/services.py +++ b/docker/models/services.py @@ -1,5 +1,5 @@ import copy -from docker.errors import create_unexpected_kwargs_error +from docker.errors import create_unexpected_kwargs_error, InvalidArgument from docker.types import TaskTemplate, ContainerSpec, ServiceMode from .resource import Model, Collection @@ -116,6 +116,9 @@ def scale(self, replicas): ``True``if successful. """ + if not self.attrs['Spec']['Mode'].get('Global'): + raise InvalidArgument('Cannot scale a global container') + service_mode = ServiceMode('replicated', replicas) return self.client.api.update_service(self.id, self.version, service_mode, diff --git a/tests/integration/models_services_test.py b/tests/integration/models_services_test.py index e7f16cdff2..64aba66e95 100644 --- a/tests/integration/models_services_test.py +++ b/tests/integration/models_services_test.py @@ -4,6 +4,8 @@ from .. import helpers from .base import TEST_API_VERSION +from docker.errors import InvalidArgument +from docker.models.services import ServiceMode class ServiceTest(unittest.TestCase): @@ -225,6 +227,28 @@ def test_scale_method_service(self): spec = service.attrs['Spec']['TaskTemplate']['ContainerSpec'] assert spec.get('Command') == ['sleep', '300'] + def test_scale_method_global_service(self): + client = docker.from_env(version=TEST_API_VERSION) + mode = ServiceMode('global') + service = client.services.create( + name=helpers.random_name(), + image="alpine", + command="sleep 300", + mode=mode + ) + tasks = [] + while len(tasks) == 0: + tasks = service.tasks() + assert len(tasks) == 1 + with self.assertRaises(InvalidArgument, + msg='Cannot scale a global container'): + service.scale(2) + + assert len(tasks) == 1 + service.reload() + spec = service.attrs['Spec']['TaskTemplate']['ContainerSpec'] + assert spec.get('Command') == ['sleep', '300'] + @helpers.requires_api_version('1.25') def test_restart_service(self): client = docker.from_env(version=TEST_API_VERSION) From 1bb4155dcbb2048b5e4728b5711f7974dd07055a Mon Sep 17 00:00:00 2001 From: Johannes Postler Date: Wed, 27 Dec 2017 17:01:49 +0100 Subject: [PATCH 0549/1301] Switch ports in documentation for EndpointSpec Signed-off-by: Johannes Postler --- docker/types/services.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/types/services.py b/docker/types/services.py index 18d4d2adf2..a1f34e02ee 100644 --- a/docker/types/services.py +++ b/docker/types/services.py @@ -444,8 +444,8 @@ class EndpointSpec(dict): balancing between tasks (``'vip'`` or ``'dnsrr'``). Defaults to ``'vip'`` if not provided. ports (dict): Exposed ports that this service is accessible on from the - outside, in the form of ``{ target_port: published_port }`` or - ``{ target_port: (published_port, protocol) }``. Ports can only be + outside, in the form of ``{ published_port: target_port }`` or + ``{ published_port: (target_port, protocol) }``. Ports can only be provided if the ``vip`` resolution mode is used. """ def __init__(self, mode=None, ports=None): From 663c6089e962f024eafad7fb5bcf88847dd8b28a Mon Sep 17 00:00:00 2001 From: Felipe Ruhland Date: Tue, 2 Jan 2018 22:46:17 -0200 Subject: [PATCH 0550/1301] Fix test to make sure the initial mode is replicated Signed-off-by: Felipe Ruhland --- docker/models/services.py | 2 +- tests/integration/models_services_test.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/models/services.py b/docker/models/services.py index 4c8accefdb..337ed44460 100644 --- a/docker/models/services.py +++ b/docker/models/services.py @@ -116,7 +116,7 @@ def scale(self, replicas): ``True``if successful. """ - if not self.attrs['Spec']['Mode'].get('Global'): + if 'Global' in self.attrs['Spec']['Mode'].keys(): raise InvalidArgument('Cannot scale a global container') service_mode = ServiceMode('replicated', replicas) diff --git a/tests/integration/models_services_test.py b/tests/integration/models_services_test.py index 64aba66e95..15b87da494 100644 --- a/tests/integration/models_services_test.py +++ b/tests/integration/models_services_test.py @@ -5,7 +5,7 @@ from .. import helpers from .base import TEST_API_VERSION from docker.errors import InvalidArgument -from docker.models.services import ServiceMode +from docker.types.services import ServiceMode class ServiceTest(unittest.TestCase): @@ -212,7 +212,7 @@ def test_scale_method_service(self): name=helpers.random_name(), # ContainerSpec arguments image="alpine", - command="sleep 300" + command="sleep 300", ) tasks = [] while len(tasks) == 0: From bf06a361e225c8562e872025fbc836c454cefbae Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 3 Jan 2018 12:16:21 -0800 Subject: [PATCH 0551/1301] Ignore dockerignore lines that contain only whitespace Signed-off-by: Joffrey F --- docker/api/build.py | 4 +++- tests/integration/api_build_test.py | 22 ++++++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/docker/api/build.py b/docker/api/build.py index 9ff2dfb3c9..34456ab373 100644 --- a/docker/api/build.py +++ b/docker/api/build.py @@ -145,7 +145,9 @@ def build(self, path=None, tag=None, quiet=False, fileobj=None, exclude = None if os.path.exists(dockerignore): with open(dockerignore, 'r') as f: - exclude = list(filter(bool, f.read().splitlines())) + exclude = list(filter( + bool, [l.strip() for l in f.read().splitlines()] + )) context = utils.tar( path, exclude=exclude, dockerfile=dockerfile, gzip=gzip ) diff --git a/tests/integration/api_build_test.py b/tests/integration/api_build_test.py index 8e98cc9fa5..7cc32346ba 100644 --- a/tests/integration/api_build_test.py +++ b/tests/integration/api_build_test.py @@ -352,6 +352,28 @@ def test_build_gzip_encoding(self): assert 'Successfully built' in lines[-1]['stream'] + def test_build_with_dockerfile_empty_lines(self): + base_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, base_dir) + with open(os.path.join(base_dir, 'Dockerfile'), 'w') as f: + f.write('FROM busybox\n') + with open(os.path.join(base_dir, '.dockerignore'), 'w') as f: + f.write('\n'.join([ + ' ', + '', + '\t\t', + '\t ', + ])) + + stream = self.client.build( + path=base_dir, stream=True, decode=True, nocache=True + ) + + lines = [] + for chunk in stream: + lines.append(chunk) + assert 'Successfully built' in lines[-1]['stream'] + def test_build_gzip_custom_encoding(self): with self.assertRaises(errors.DockerException): self.client.build(path='.', gzip=True, encoding='text/html') From e75a03fd6d744a96d84179eadfcc47301ca968c7 Mon Sep 17 00:00:00 2001 From: Viktor Adam Date: Fri, 12 Jan 2018 22:18:47 +0000 Subject: [PATCH 0552/1301] Fix regression on API < 1.25 Signed-off-by: Viktor Adam --- docker/api/service.py | 9 ++++++++- tests/integration/models_services_test.py | 24 +++++++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/docker/api/service.py b/docker/api/service.py index 86f4b07361..1a8b8b5bf8 100644 --- a/docker/api/service.py +++ b/docker/api/service.py @@ -137,6 +137,8 @@ def create_service( auth_header = auth.get_config_header(self, registry) if auth_header: headers['X-Registry-Auth'] = auth_header + if utils.version_lt(self._version, '1.25'): + networks = networks or task_template.pop('Networks', None) data = { 'Name': name, 'Labels': labels, @@ -411,7 +413,12 @@ def update_service(self, service, version, task_template=None, name=None, if networks is not None: converted_networks = utils.convert_service_networks(networks) - data['TaskTemplate']['Networks'] = converted_networks + if utils.version_lt(self._version, '1.25'): + data['Networks'] = converted_networks + else: + data['TaskTemplate']['Networks'] = converted_networks + elif utils.version_lt(self._version, '1.25'): + data['Networks'] = current.get('Networks') elif data['TaskTemplate'].get('Networks') is None: current_task_template = current.get('TaskTemplate', {}) current_networks = current_task_template.get('Networks') diff --git a/tests/integration/models_services_test.py b/tests/integration/models_services_test.py index ca8be48de4..b4df745e0c 100644 --- a/tests/integration/models_services_test.py +++ b/tests/integration/models_services_test.py @@ -179,6 +179,30 @@ def test_update_remove_service_labels(self): service.reload() assert not service.attrs['Spec'].get('Labels') + def test_update_retains_networks(self): + client = docker.from_env(version=TEST_API_VERSION) + network_name = helpers.random_name() + network = client.networks.create( + network_name, driver='overlay' + ) + service = client.services.create( + # create arguments + name=helpers.random_name(), + networks=[network.id], + # ContainerSpec arguments + image="alpine", + command="sleep 300" + ) + service.update( + # create argument + name=service.name, + # ContainerSpec argument + command="sleep 600" + ) + service.reload() + networks = service.attrs['Spec']['TaskTemplate']['Networks'] + assert networks == [{'Target': network.id}] + def test_scale_service(self): client = docker.from_env(version=TEST_API_VERSION) service = client.services.create( From 0acef5f6d299d0f82d62034a356efe212ba186d5 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 18 Jan 2018 18:19:25 -0800 Subject: [PATCH 0553/1301] Add Python 3.6 testing Signed-off-by: Joffrey F --- .travis.yml | 23 +++++++++++++---------- Dockerfile-py3 | 2 +- Jenkinsfile | 2 +- Makefile | 24 +++++++++++++++++------- appveyor.yml | 2 +- setup.py | 1 + tox.ini | 2 +- win32-requirements.txt | 2 +- 8 files changed, 36 insertions(+), 22 deletions(-) diff --git a/.travis.yml b/.travis.yml index cd64b4456e..842e352836 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,15 +1,18 @@ sudo: false language: python -python: - - "3.5" -env: - - TOX_ENV=py27 -# FIXME: default travis worker does not carry py33 anymore. Can this be configured? -# - TOX_ENV=py33 - - TOX_ENV=py34 - - TOX_ENV=py35 - - TOX_ENV=flake8 +matrix: + include: + - python: 2.7 + env: TOXENV=py27 + - python: 3.4 + env: TOXENV=py34 + - python: 3.5 + env: TOXENV=py35 + - python: 3.6 + env: TOXENV=py36 + - env: TOXENV=flake8 + install: - pip install tox script: - - tox -e $TOX_ENV + - tox diff --git a/Dockerfile-py3 b/Dockerfile-py3 index 543cf4d63f..d558ba3e4f 100644 --- a/Dockerfile-py3 +++ b/Dockerfile-py3 @@ -1,4 +1,4 @@ -FROM python:3.5 +FROM python:3.6 RUN mkdir /src WORKDIR /src diff --git a/Jenkinsfile b/Jenkinsfile index e3168cd703..6dc9a32ca2 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -27,7 +27,7 @@ def buildImages = { -> imageNamePy3 = "${imageNameBase}:py3-${gitCommit()}" buildImage(imageNamePy2, ".", "py2.7") - buildImage(imageNamePy3, "-f Dockerfile-py3 .", "py3.5") + buildImage(imageNamePy3, "-f Dockerfile-py3 .", "py3.6") } } } diff --git a/Makefile b/Makefile index 32ef510675..a61fe82836 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ all: test .PHONY: clean clean: - -docker rm -vf dpy-dind + -docker rm -f dpy-dind-py2 dpy-dind-py3 find -name "__pycache__" | xargs rm -rf .PHONY: build @@ -45,15 +45,25 @@ TEST_API_VERSION ?= 1.33 TEST_ENGINE_VERSION ?= 17.10.0-ce .PHONY: integration-dind -integration-dind: build build-py3 - docker rm -vf dpy-dind || : - docker run -d --name dpy-dind --privileged dockerswarm/dind:${TEST_ENGINE_VERSION} dockerd\ +integration-dind: integration-dind-py2 integration-dind-py3 + +.PHONY: integration-dind-py2 +integration-dind-py2: build + docker rm -vf dpy-dind-py2 || : + docker run -d --name dpy-dind-py2 --privileged dockerswarm/dind:${TEST_ENGINE_VERSION} dockerd\ -H tcp://0.0.0.0:2375 --experimental docker run -t --rm --env="DOCKER_HOST=tcp://docker:2375" --env="DOCKER_TEST_API_VERSION=${TEST_API_VERSION}"\ - --link=dpy-dind:docker docker-sdk-python py.test tests/integration + --link=dpy-dind-py2:docker docker-sdk-python py.test tests/integration + docker rm -vf dpy-dind-py3 + +.PHONY: integration-dind-py3 +integration-dind-py3: build-py3 + docker rm -vf dpy-dind-py3 || : + docker run -d --name dpy-dind-py3 --privileged dockerswarm/dind:${TEST_ENGINE_VERSION} dockerd\ + -H tcp://0.0.0.0:2375 --experimental docker run -t --rm --env="DOCKER_HOST=tcp://docker:2375" --env="DOCKER_TEST_API_VERSION=${TEST_API_VERSION}"\ - --link=dpy-dind:docker docker-sdk-python3 py.test tests/integration - docker rm -vf dpy-dind + --link=dpy-dind-py3:docker docker-sdk-python3 py.test tests/integration + docker rm -vf dpy-dind-py3 .PHONY: integration-dind-ssl integration-dind-ssl: build-dind-certs build build-py3 diff --git a/appveyor.yml b/appveyor.yml index 41cde6252b..d659b586ee 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -3,7 +3,7 @@ version: '{branch}-{build}' install: - "SET PATH=C:\\Python27-x64;C:\\Python27-x64\\Scripts;%PATH%" - "python --version" - - "pip install tox==2.7.0 virtualenv==15.1.0" + - "pip install tox==2.9.1" # Build the binary after tests build: false diff --git a/setup.py b/setup.py index d59d8124ba..468245101d 100644 --- a/setup.py +++ b/setup.py @@ -87,6 +87,7 @@ 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', 'Topic :: Utilities', 'License :: OSI Approved :: Apache Software License', ], diff --git a/tox.ini b/tox.ini index 3bf2b7164d..41d88605d3 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27, py33, py34, py35, flake8 +envlist = py27, py33, py34, py35, py36, flake8 skipsdist=True [testenv] diff --git a/win32-requirements.txt b/win32-requirements.txt index e77c3d90f8..6db52a50b4 100644 --- a/win32-requirements.txt +++ b/win32-requirements.txt @@ -1,2 +1,2 @@ -r requirements.txt -pypiwin32==219 \ No newline at end of file +pypiwin32>=219 From f95b958429b38dab50929e013db3c636a12e1536 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 8 Jan 2018 18:11:29 -0800 Subject: [PATCH 0554/1301] Add support for experimental platform flag in build and pull Signed-off-by: Joffrey F --- docker/api/build.py | 10 +++++++++- docker/api/image.py | 13 +++++++++++-- docker/models/containers.py | 8 ++++++-- docker/models/images.py | 2 ++ tests/integration/api_build_test.py | 15 +++++++++++++++ tests/integration/api_image_test.py | 11 ++++++++++- tests/unit/models_containers_test.py | 2 +- 7 files changed, 54 insertions(+), 7 deletions(-) diff --git a/docker/api/build.py b/docker/api/build.py index 34456ab373..32238efed9 100644 --- a/docker/api/build.py +++ b/docker/api/build.py @@ -19,7 +19,7 @@ def build(self, path=None, tag=None, quiet=False, fileobj=None, forcerm=False, dockerfile=None, container_limits=None, decode=False, buildargs=None, gzip=False, shmsize=None, labels=None, cache_from=None, target=None, network_mode=None, - squash=None, extra_hosts=None): + squash=None, extra_hosts=None, platform=None): """ Similar to the ``docker build`` command. Either ``path`` or ``fileobj`` needs to be set. ``path`` can be a local path (to a directory @@ -103,6 +103,7 @@ def build(self, path=None, tag=None, quiet=False, fileobj=None, single layer. extra_hosts (dict): Extra hosts to add to /etc/hosts in building containers, as a mapping of hostname to IP address. + platform (str): Platform in the format ``os[/arch[/variant]]`` Returns: A generator for the build output. @@ -243,6 +244,13 @@ def build(self, path=None, tag=None, quiet=False, fileobj=None, extra_hosts = utils.format_extra_hosts(extra_hosts) params.update({'extrahosts': extra_hosts}) + if platform is not None: + if utils.version_lt(self._version, '1.32'): + raise errors.InvalidVersion( + 'platform was only introduced in API version 1.32' + ) + params['platform'] = platform + if context is not None: headers = {'Content-Type': 'application/tar'} if encoding: diff --git a/docker/api/image.py b/docker/api/image.py index 77553122d6..065fae3959 100644 --- a/docker/api/image.py +++ b/docker/api/image.py @@ -323,7 +323,8 @@ def prune_images(self, filters=None): return self._result(self._post(url, params=params), True) def pull(self, repository, tag=None, stream=False, - insecure_registry=False, auth_config=None, decode=False): + insecure_registry=False, auth_config=None, decode=False, + platform=None): """ Pulls an image. Similar to the ``docker pull`` command. @@ -336,6 +337,7 @@ def pull(self, repository, tag=None, stream=False, :py:meth:`~docker.api.daemon.DaemonApiMixin.login` has set for this request. ``auth_config`` should contain the ``username`` and ``password`` keys to be valid. + platform (str): Platform in the format ``os[/arch[/variant]]`` Returns: (generator or str): The output @@ -376,7 +378,7 @@ def pull(self, repository, tag=None, stream=False, } headers = {} - if utils.compare_version('1.5', self._version) >= 0: + if utils.version_gte(self._version, '1.5'): if auth_config is None: header = auth.get_config_header(self, registry) if header: @@ -385,6 +387,13 @@ def pull(self, repository, tag=None, stream=False, log.debug('Sending supplied auth config') headers['X-Registry-Auth'] = auth.encode_header(auth_config) + if platform is not None: + if utils.version_lt(self._version, '1.32'): + raise errors.InvalidVersion( + 'platform was only introduced in API version 1.32' + ) + params['platform'] = platform + response = self._post( self._url('/images/create'), params=params, headers=headers, stream=stream, timeout=None diff --git a/docker/models/containers.py b/docker/models/containers.py index 6ba308e492..5e2aa88a3d 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -579,6 +579,8 @@ def run(self, image, command=None, stdout=True, stderr=False, inside the container. pids_limit (int): Tune a container's pids limit. Set ``-1`` for unlimited. + platform (str): Platform in the format ``os[/arch[/variant]]``. + Only used if the method needs to pull the requested image. ports (dict): Ports to bind inside the container. The keys of the dictionary are the ports to bind inside the @@ -700,7 +702,9 @@ def run(self, image, command=None, stdout=True, stderr=False, if isinstance(image, Image): image = image.id stream = kwargs.pop('stream', False) - detach = kwargs.pop("detach", False) + detach = kwargs.pop('detach', False) + platform = kwargs.pop('platform', None) + if detach and remove: if version_gte(self.client.api._version, '1.25'): kwargs["auto_remove"] = True @@ -718,7 +722,7 @@ def run(self, image, command=None, stdout=True, stderr=False, container = self.create(image=image, command=command, detach=detach, **kwargs) except ImageNotFound: - self.client.images.pull(image) + self.client.images.pull(image, platform=platform) container = self.create(image=image, command=command, detach=detach, **kwargs) diff --git a/docker/models/images.py b/docker/models/images.py index 82ca54135e..891c565f66 100644 --- a/docker/models/images.py +++ b/docker/models/images.py @@ -157,6 +157,7 @@ def build(self, **kwargs): single layer. extra_hosts (dict): Extra hosts to add to /etc/hosts in building containers, as a mapping of hostname to IP address. + platform (str): Platform in the format ``os[/arch[/variant]]``. Returns: (:py:class:`Image`): The built image. @@ -265,6 +266,7 @@ def pull(self, name, tag=None, **kwargs): :py:meth:`~docker.client.DockerClient.login` has set for this request. ``auth_config`` should contain the ``username`` and ``password`` keys to be valid. + platform (str): Platform in the format ``os[/arch[/variant]]`` Returns: (:py:class:`Image`): The image that has been pulled. diff --git a/tests/integration/api_build_test.py b/tests/integration/api_build_test.py index 7cc32346ba..245214e1a2 100644 --- a/tests/integration/api_build_test.py +++ b/tests/integration/api_build_test.py @@ -377,3 +377,18 @@ def test_build_with_dockerfile_empty_lines(self): def test_build_gzip_custom_encoding(self): with self.assertRaises(errors.DockerException): self.client.build(path='.', gzip=True, encoding='text/html') + + @requires_api_version('1.32') + @requires_experimental(until=None) + def test_build_invalid_platform(self): + script = io.BytesIO('FROM busybox\n'.encode('ascii')) + + with pytest.raises(errors.APIError) as excinfo: + stream = self.client.build( + fileobj=script, stream=True, platform='foobar' + ) + for _ in stream: + pass + + assert excinfo.value.status_code == 400 + assert 'invalid platform' in excinfo.exconly() diff --git a/tests/integration/api_image_test.py b/tests/integration/api_image_test.py index 14fb77aa46..178c34e995 100644 --- a/tests/integration/api_image_test.py +++ b/tests/integration/api_image_test.py @@ -14,7 +14,7 @@ import docker -from ..helpers import requires_api_version +from ..helpers import requires_api_version, requires_experimental from .base import BaseAPIIntegrationTest, BUSYBOX @@ -67,6 +67,15 @@ def test_pull_streaming(self): img_info = self.client.inspect_image('hello-world') self.assertIn('Id', img_info) + @requires_api_version('1.32') + @requires_experimental(until=None) + def test_pull_invalid_platform(self): + with pytest.raises(docker.errors.APIError) as excinfo: + self.client.pull('hello-world', platform='foobar') + + assert excinfo.value.status_code == 500 + assert 'invalid platform' in excinfo.exconly() + class CommitTest(BaseAPIIntegrationTest): def test_commit(self): diff --git a/tests/unit/models_containers_test.py b/tests/unit/models_containers_test.py index a479e836e6..95295a91b7 100644 --- a/tests/unit/models_containers_test.py +++ b/tests/unit/models_containers_test.py @@ -225,7 +225,7 @@ def test_run_pull(self): container = client.containers.run('alpine', 'sleep 300', detach=True) assert container.id == FAKE_CONTAINER_ID - client.api.pull.assert_called_with('alpine', tag=None) + client.api.pull.assert_called_with('alpine', platform=None, tag=None) def test_run_with_error(self): client = make_fake_client() From ccd79323dae2af09d58323c9fb0486b045e6730d Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 23 Jan 2018 17:32:27 -0800 Subject: [PATCH 0555/1301] Shift test matrix forward Signed-off-by: Joffrey F --- Jenkinsfile | 4 ++-- Makefile | 4 ++-- tests/integration/api_service_test.py | 6 ++++-- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 6dc9a32ca2..6d9d3436f2 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -5,7 +5,7 @@ def imageNamePy2 def imageNamePy3 def images = [:] -def dockerVersions = ["17.06.2-ce", "17.09.0-ce", "17.10.0-ce"] +def dockerVersions = ["17.06.2-ce", "17.12.0-ce", "18.01.0-ce"] def buildImage = { name, buildargs, pyTag -> img = docker.image(name) @@ -33,7 +33,7 @@ def buildImages = { -> } def getAPIVersion = { engineVersion -> - def versionMap = ['17.06': '1.30', '17.09': '1.32', '17.10': '1.33'] + def versionMap = ['17.06': '1.30', '17.12': '1.35', '18.01': '1.35'] return versionMap[engineVersion.substring(0, 5)] } diff --git a/Makefile b/Makefile index a61fe82836..d07b8c5968 100644 --- a/Makefile +++ b/Makefile @@ -41,8 +41,8 @@ integration-test: build integration-test-py3: build-py3 docker run -t --rm -v /var/run/docker.sock:/var/run/docker.sock docker-sdk-python3 py.test tests/integration/${file} -TEST_API_VERSION ?= 1.33 -TEST_ENGINE_VERSION ?= 17.10.0-ce +TEST_API_VERSION ?= 1.35 +TEST_ENGINE_VERSION ?= 17.12.0-ce .PHONY: integration-dind integration-dind: integration-dind-py2 integration-dind-py3 diff --git a/tests/integration/api_service_test.py b/tests/integration/api_service_test.py index 4a2093dae6..7620cb47d7 100644 --- a/tests/integration/api_service_test.py +++ b/tests/integration/api_service_test.py @@ -1096,11 +1096,13 @@ def test_update_service_with_network_change(self): ) task_tmpl = docker.types.TaskTemplate(container_spec) net1 = self.client.create_network( - 'dockerpytest_1', driver='overlay', ipam={'Driver': 'default'} + self.get_service_name(), driver='overlay', + ipam={'Driver': 'default'} ) self.tmp_networks.append(net1['Id']) net2 = self.client.create_network( - 'dockerpytest_2', driver='overlay', ipam={'Driver': 'default'} + self.get_service_name(), driver='overlay', + ipam={'Driver': 'default'} ) self.tmp_networks.append(net2['Id']) name = self.get_service_name() From 500286d51e63510e9765868cbc1f8cc01ff36bbb Mon Sep 17 00:00:00 2001 From: Drew Erny Date: Thu, 18 Jan 2018 13:27:33 -0800 Subject: [PATCH 0556/1301] Change default TLS version Detects if python has an up-to-date version of OpenSSL that supports TLSv1.2. If it does, choose that as the default TLS version, instead of TLSv1. The Docker Engine and the majority of other Docker API servers should suppot TLSv1.2, and if they do not, the user can manually set a different (lower) version. Signed-off-by: Drew Erny --- docker/tls.py | 41 ++++++++++++++++++++++++++++++++++------- 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/docker/tls.py b/docker/tls.py index 6488bbccc1..8fdf3596b1 100644 --- a/docker/tls.py +++ b/docker/tls.py @@ -37,13 +37,40 @@ def __init__(self, client_cert=None, ca_cert=None, verify=None, self.assert_hostname = assert_hostname self.assert_fingerprint = assert_fingerprint - # TLS v1.0 seems to be the safest default; SSLv23 fails in mysterious - # ways: https://github.com/docker/docker-py/issues/963 - - self.ssl_version = ssl_version or ssl.PROTOCOL_TLSv1 - - # "tls" and "tls_verify" must have both or neither cert/key files - # In either case, Alert the user when both are expected, but any are + # TODO(dperny): according to the python docs, PROTOCOL_TLSvWhatever is + # depcreated, and it's recommended to use OPT_NO_TLSvWhatever instead + # to exclude versions. But I think that might require a bigger + # architectural change, so I've opted not to pursue it at this time + + # If the user provides an SSL version, we should use their preference + if ssl_version: + self.ssl_version = ssl_version + else: + # If the user provides no ssl version, we should default to + # TLSv1_2. This option is the most secure, and will work for the + # majority of users with reasonably up-to-date software. However, + # before doing so, detect openssl version to ensure we can support + # it. + + # ssl.OPENSSL_VERSION_INFO returns a tuple of 5 integers + # representing version info. We want any OpenSSL version greater + # than 1.0.1. Python compares tuples lexigraphically, which means + # this comparison will work. + if ssl.OPENSSL_VERSION_INFO > (1, 0, 1, 0, 0): + # If this version is high enough to support TLSv1_2, then we + # should use it. + self.ssl_version = ssl.PROTOCOL_TLSv1_2 + else: + # If we can't, use a differnent default. Before the commit + # introducing this version detection, the comment read: + # >>> TLS v1.0 seems to be the safest default; SSLv23 fails in + # >>> mysterious ways: + # >>> https://github.com/docker/docker-py/issues/963 + # Which is why we choose PROTOCOL_TLSv1 + self.ssl_version = ssl.PROTOCOL_TLSv1 + + # "tls" and "tls_verify" must have both or neither cert/key files In + # either case, Alert the user when both are expected, but any are # missing. if client_cert: From bab7ca3cde63295a4cd775c7e4da9516340af7f4 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 23 Jan 2018 17:22:34 -0800 Subject: [PATCH 0557/1301] Don't use PROTOCOL_TLSv1_2 directly to avoid ImportErrors Signed-off-by: Joffrey F --- docker/tls.py | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/docker/tls.py b/docker/tls.py index 8fdf3596b1..4900e9fdf7 100644 --- a/docker/tls.py +++ b/docker/tls.py @@ -51,22 +51,15 @@ def __init__(self, client_cert=None, ca_cert=None, verify=None, # majority of users with reasonably up-to-date software. However, # before doing so, detect openssl version to ensure we can support # it. - - # ssl.OPENSSL_VERSION_INFO returns a tuple of 5 integers - # representing version info. We want any OpenSSL version greater - # than 1.0.1. Python compares tuples lexigraphically, which means - # this comparison will work. - if ssl.OPENSSL_VERSION_INFO > (1, 0, 1, 0, 0): - # If this version is high enough to support TLSv1_2, then we - # should use it. - self.ssl_version = ssl.PROTOCOL_TLSv1_2 + if ssl.OPENSSL_VERSION_INFO[:3] >= (1, 0, 1) and hasattr( + ssl, 'PROTOCOL_TLSv1_2'): + # If the OpenSSL version is high enough to support TLSv1_2, + # then we should use it. + self.ssl_version = getattr(ssl, 'PROTOCOL_TLSv1_2') else: - # If we can't, use a differnent default. Before the commit - # introducing this version detection, the comment read: - # >>> TLS v1.0 seems to be the safest default; SSLv23 fails in - # >>> mysterious ways: - # >>> https://github.com/docker/docker-py/issues/963 - # Which is why we choose PROTOCOL_TLSv1 + # Otherwise, TLS v1.0 seems to be the safest default; + # SSLv23 fails in mysterious ways: + # https://github.com/docker/docker-py/issues/963 self.ssl_version = ssl.PROTOCOL_TLSv1 # "tls" and "tls_verify" must have both or neither cert/key files In From 24bd5d8e53414098b4d4c70f93f3a69ce094510f Mon Sep 17 00:00:00 2001 From: mccalluc Date: Fri, 26 Jan 2018 10:44:15 -0500 Subject: [PATCH 0558/1301] Replace missing "^" with "e" Signed-off-by: Chuck McCallum --- docker/models/containers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/models/containers.py b/docker/models/containers.py index 5e2aa88a3d..08f63edeca 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -557,7 +557,7 @@ def run(self, image, command=None, stdout=True, stderr=False, item in the list is expected to be a :py:class:`docker.types.Mount` object. name (str): The name for this container. - nano_cpus (int): CPU quota in units of 10-9 CPUs. + nano_cpus (int): CPU quota in units of 1e-9 CPUs. network (str): Name of the network this container will be connected to at creation time. You can connect to additional networks using :py:meth:`Network.connect`. Incompatible with From a5490ad0be6840961758d090f573ceb1ef0a5d96 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 26 Jan 2018 12:52:03 -0800 Subject: [PATCH 0559/1301] Fix appveyor tests Signed-off-by: Joffrey F --- requirements.txt | 2 ++ setup.py | 9 ++++++--- win32-requirements.txt | 1 - 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index f3c61e790b..1602750fde 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,6 +11,8 @@ packaging==16.8 pycparser==2.17 pyOpenSSL==17.0.0 pyparsing==2.2.0 +pypiwin32==219; sys_platform == 'win32' and python_version < '3.6' +pypiwin32==220; sys_platform == 'win32' and python_version >= '3.6' requests==2.14.2 six==1.10.0 websocket-client==0.40.0 diff --git a/setup.py b/setup.py index 468245101d..b628f4a878 100644 --- a/setup.py +++ b/setup.py @@ -26,9 +26,6 @@ 'docker-pycreds >= 0.2.1' ] -if sys.platform == 'win32': - requirements.append('pypiwin32 >= 219') - extras_require = { ':python_version < "3.5"': 'backports.ssl_match_hostname >= 3.5', # While not imported explicitly, the ipaddress module is required for @@ -36,6 +33,12 @@ # ServerAltname: https://pypi.python.org/pypi/backports.ssl_match_hostname ':python_version < "3.3"': 'ipaddress >= 1.0.16', + # win32 APIs if on Windows (required for npipe support) + # Python 3.6 is only compatible with v220 ; Python < 3.5 is not supported + # on v220 ; ALL versions are broken for v222 (as of 2018-01-26) + ':sys_platform == "win32" and python_version < "3.6"': 'pypiwin32==219', + ':sys_platform == "win32" and python_version >= "3.6"': 'pypiwin32==220', + # If using docker-py over TLS, highly recommend this option is # pip-installed or pinned. diff --git a/win32-requirements.txt b/win32-requirements.txt index 6db52a50b4..bc04b4960a 100644 --- a/win32-requirements.txt +++ b/win32-requirements.txt @@ -1,2 +1 @@ -r requirements.txt -pypiwin32>=219 From abd60aedc7e3df813006919222d86717eb8c6fc2 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 25 Jan 2018 14:17:26 -0800 Subject: [PATCH 0560/1301] Bump default API version to 1.35 Add ContainerSpec.isolation support Add until support in logs Add condition support in wait Add workdir support in exec_create Signed-off-by: Joffrey F --- docker/api/container.py | 64 ++++++++++++++++------- docker/api/exec_api.py | 10 +++- docker/api/service.py | 4 ++ docker/constants.py | 2 +- docker/models/containers.py | 9 +++- docker/models/services.py | 3 ++ docker/types/services.py | 9 +++- tests/integration/api_container_test.py | 33 ++++++++++++ tests/integration/api_exec_test.py | 12 +++++ tests/integration/models_services_test.py | 1 + tests/unit/api_container_test.py | 6 ++- tests/unit/models_containers_test.py | 3 +- 12 files changed, 129 insertions(+), 27 deletions(-) diff --git a/docker/api/container.py b/docker/api/container.py index 494f7b4662..b08032c403 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -786,7 +786,8 @@ def kill(self, container, signal=None): @utils.check_resource('container') def logs(self, container, stdout=True, stderr=True, stream=False, - timestamps=False, tail='all', since=None, follow=None): + timestamps=False, tail='all', since=None, follow=None, + until=None): """ Get logs from a container. Similar to the ``docker logs`` command. @@ -805,6 +806,8 @@ def logs(self, container, stdout=True, stderr=True, stream=False, since (datetime or int): Show logs since a given datetime or integer epoch (in seconds) follow (bool): Follow log output + until (datetime or int): Show logs that occurred before the given + datetime or integer epoch (in seconds) Returns: (generator or str) @@ -827,21 +830,35 @@ def logs(self, container, stdout=True, stderr=True, stream=False, params['tail'] = tail if since is not None: - if utils.compare_version('1.19', self._version) < 0: + if utils.version_lt(self._version, '1.19'): raise errors.InvalidVersion( - 'since is not supported in API < 1.19' + 'since is not supported for API version < 1.19' ) + if isinstance(since, datetime): + params['since'] = utils.datetime_to_timestamp(since) + elif (isinstance(since, int) and since > 0): + params['since'] = since else: - if isinstance(since, datetime): - params['since'] = utils.datetime_to_timestamp(since) - elif (isinstance(since, int) and since > 0): - params['since'] = since - else: - raise errors.InvalidArgument( - 'since value should be datetime or positive int, ' - 'not {}'. - format(type(since)) - ) + raise errors.InvalidArgument( + 'since value should be datetime or positive int, ' + 'not {}'.format(type(since)) + ) + + if until is not None: + if utils.version_lt(self._version, '1.35'): + raise errors.InvalidVersion( + 'until is not supported for API version < 1.35' + ) + if isinstance(until, datetime): + params['until'] = utils.datetime_to_timestamp(until) + elif (isinstance(until, int) and until > 0): + params['until'] = until + else: + raise errors.InvalidArgument( + 'until value should be datetime or positive int, ' + 'not {}'.format(type(until)) + ) + url = self._url("/containers/{0}/logs", container) res = self._get(url, params=params, stream=stream) return self._get_result(container, stream, res) @@ -1241,7 +1258,7 @@ def update_container( return self._result(res, True) @utils.check_resource('container') - def wait(self, container, timeout=None): + def wait(self, container, timeout=None, condition=None): """ Block until a container stops, then return its exit code. Similar to the ``docker wait`` command. @@ -1250,10 +1267,13 @@ def wait(self, container, timeout=None): container (str or dict): The container to wait on. If a dict, the ``Id`` key is used. timeout (int): Request timeout + condition (str): Wait until a container state reaches the given + condition, either ``not-running`` (default), ``next-exit``, + or ``removed`` Returns: - (int): The exit code of the container. Returns ``-1`` if the API - responds without a ``StatusCode`` attribute. + (int or dict): The exit code of the container. Returns the full API + response if no ``StatusCode`` field is included. Raises: :py:class:`requests.exceptions.ReadTimeout` @@ -1262,9 +1282,17 @@ def wait(self, container, timeout=None): If the server returns an error. """ url = self._url("/containers/{0}/wait", container) - res = self._post(url, timeout=timeout) + params = {} + if condition is not None: + if utils.version_lt(self._version, '1.30'): + raise errors.InvalidVersion( + 'wait condition is not supported for API version < 1.30' + ) + params['condition'] = condition + + res = self._post(url, timeout=timeout, params=params) self._raise_for_status(res) json_ = res.json() if 'StatusCode' in json_: return json_['StatusCode'] - return -1 + return json_ diff --git a/docker/api/exec_api.py b/docker/api/exec_api.py index cff5cfa7b3..029c984a6f 100644 --- a/docker/api/exec_api.py +++ b/docker/api/exec_api.py @@ -9,7 +9,7 @@ class ExecApiMixin(object): @utils.check_resource('container') def exec_create(self, container, cmd, stdout=True, stderr=True, stdin=False, tty=False, privileged=False, user='', - environment=None): + environment=None, workdir=None): """ Sets up an exec instance in a running container. @@ -26,6 +26,7 @@ def exec_create(self, container, cmd, stdout=True, stderr=True, environment (dict or list): A dictionary or a list of strings in the following format ``["PASSWORD=xxx"]`` or ``{"PASSWORD": "xxx"}``. + workdir (str): Path to working directory for this exec session Returns: (dict): A dictionary with an exec ``Id`` key. @@ -66,6 +67,13 @@ def exec_create(self, container, cmd, stdout=True, stderr=True, 'Env': environment, } + if workdir is not None: + if utils.version_lt(self._version, '1.35'): + raise errors.InvalidVersion( + 'workdir is not supported for API version < 1.35' + ) + data['WorkingDir'] = workdir + url = self._url('/containers/{0}/exec', container) res = self._post_json(url, data=data) return self._result(res, True) diff --git a/docker/api/service.py b/docker/api/service.py index 1a8b8b5bf8..4f7123e5a0 100644 --- a/docker/api/service.py +++ b/docker/api/service.py @@ -65,6 +65,10 @@ def raise_version_error(param, min_version): if container_spec.get('Privileges') is not None: raise_version_error('ContainerSpec.privileges', '1.30') + if utils.version_lt(version, '1.35'): + if container_spec.get('Isolation') is not None: + raise_version_error('ContainerSpec.isolation', '1.35') + def _merge_task_template(current, override): merged = current.copy() diff --git a/docker/constants.py b/docker/constants.py index 6de8fad632..9ab3673255 100644 --- a/docker/constants.py +++ b/docker/constants.py @@ -1,7 +1,7 @@ import sys from .version import version -DEFAULT_DOCKER_API_VERSION = '1.30' +DEFAULT_DOCKER_API_VERSION = '1.35' MINIMUM_DOCKER_API_VERSION = '1.21' DEFAULT_TIMEOUT_SECONDS = 60 STREAM_HEADER_SIZE_BYTES = 8 diff --git a/docker/models/containers.py b/docker/models/containers.py index 08f63edeca..bdc05cdb72 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -126,7 +126,7 @@ def diff(self): def exec_run(self, cmd, stdout=True, stderr=True, stdin=False, tty=False, privileged=False, user='', detach=False, stream=False, - socket=False, environment=None): + socket=False, environment=None, workdir=None): """ Run a command inside this container. Similar to ``docker exec``. @@ -147,6 +147,7 @@ def exec_run(self, cmd, stdout=True, stderr=True, stdin=False, tty=False, environment (dict or list): A dictionary or a list of strings in the following format ``["PASSWORD=xxx"]`` or ``{"PASSWORD": "xxx"}``. + workdir (str): Path to working directory for this exec session Returns: (generator or str): @@ -159,7 +160,8 @@ def exec_run(self, cmd, stdout=True, stderr=True, stdin=False, tty=False, """ resp = self.client.api.exec_create( self.id, cmd, stdout=stdout, stderr=stderr, stdin=stdin, tty=tty, - privileged=privileged, user=user, environment=environment + privileged=privileged, user=user, environment=environment, + workdir=workdir ) return self.client.api.exec_start( resp['Id'], detach=detach, tty=tty, stream=stream, socket=socket @@ -427,6 +429,9 @@ def wait(self, **kwargs): Args: timeout (int): Request timeout + condition (str): Wait until a container state reaches the given + condition, either ``not-running`` (default), ``next-exit``, + or ``removed`` Returns: (int): The exit code of the container. Returns ``-1`` if the API diff --git a/docker/models/services.py b/docker/models/services.py index 337ed44460..8a633dfa01 100644 --- a/docker/models/services.py +++ b/docker/models/services.py @@ -144,6 +144,8 @@ def create(self, image, command=None, **kwargs): env (list of str): Environment variables, in the form ``KEY=val``. hostname (string): Hostname to set on the container. + isolation (string): Isolation technology used by the service's + containers. Only used for Windows containers. labels (dict): Labels to apply to the service. log_driver (str): Log driver to use for containers. log_driver_options (dict): Log driver options. @@ -255,6 +257,7 @@ def list(self, **kwargs): 'hostname', 'hosts', 'image', + 'isolation', 'labels', 'mounts', 'open_stdin', diff --git a/docker/types/services.py b/docker/types/services.py index a1f34e02ee..ef1ca690d3 100644 --- a/docker/types/services.py +++ b/docker/types/services.py @@ -102,19 +102,21 @@ class ContainerSpec(dict): healthcheck (Healthcheck): Healthcheck configuration for this service. hosts (:py:class:`dict`): A set of host to IP mappings to add to - the container's `hosts` file. + the container's ``hosts`` file. dns_config (DNSConfig): Specification for DNS related configurations in resolver configuration file. configs (:py:class:`list`): List of :py:class:`ConfigReference` that will be exposed to the service. privileges (Privileges): Security options for the service's containers. + isolation (string): Isolation technology used by the service's + containers. Only used for Windows containers. """ def __init__(self, image, command=None, args=None, hostname=None, env=None, workdir=None, user=None, labels=None, mounts=None, stop_grace_period=None, secrets=None, tty=None, groups=None, open_stdin=None, read_only=None, stop_signal=None, healthcheck=None, hosts=None, dns_config=None, configs=None, - privileges=None): + privileges=None, isolation=None): self['Image'] = image if isinstance(command, six.string_types): @@ -178,6 +180,9 @@ def __init__(self, image, command=None, args=None, hostname=None, env=None, if read_only is not None: self['ReadOnly'] = read_only + if isolation is not None: + self['Isolation'] = isolation + class Mount(dict): """ diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index 5e30eee27d..5d06bc46a3 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -1,6 +1,7 @@ import os import signal import tempfile +from datetime import datetime import docker from docker.constants import IS_WINDOWS_PLATFORM @@ -9,6 +10,7 @@ import pytest +import requests import six from .base import BUSYBOX, BaseAPIIntegrationTest @@ -816,6 +818,21 @@ def test_wait_with_dict_instead_of_id(self): self.assertIn('ExitCode', inspect['State']) self.assertEqual(inspect['State']['ExitCode'], exitcode) + @requires_api_version('1.30') + def test_wait_with_condition(self): + ctnr = self.client.create_container(BUSYBOX, 'true') + self.tmp_containers.append(ctnr) + with pytest.raises(requests.exceptions.ConnectionError): + self.client.wait(ctnr, condition='removed', timeout=1) + + ctnr = self.client.create_container( + BUSYBOX, ['sleep', '3'], + host_config=self.client.create_host_config(auto_remove=True) + ) + self.tmp_containers.append(ctnr) + self.client.start(ctnr) + assert self.client.wait(ctnr, condition='removed', timeout=5) == 0 + class LogsTest(BaseAPIIntegrationTest): def test_logs(self): @@ -888,6 +905,22 @@ def test_logs_with_tail_0(self): logs = self.client.logs(id, tail=0) self.assertEqual(logs, ''.encode(encoding='ascii')) + @requires_api_version('1.35') + def test_logs_with_until(self): + snippet = 'Shanghai Teahouse (Hong Meiling)' + container = self.client.create_container( + BUSYBOX, 'echo "{0}"'.format(snippet) + ) + + self.tmp_containers.append(container) + self.client.start(container) + exitcode = self.client.wait(container) + assert exitcode == 0 + logs_until_1 = self.client.logs(container, until=1) + assert logs_until_1 == b'' + logs_until_now = self.client.logs(container, datetime.now()) + assert logs_until_now == (snippet + '\n').encode(encoding='ascii') + class DiffTest(BaseAPIIntegrationTest): def test_diff(self): diff --git a/tests/integration/api_exec_test.py b/tests/integration/api_exec_test.py index 7a65041963..0d42e19ac6 100644 --- a/tests/integration/api_exec_test.py +++ b/tests/integration/api_exec_test.py @@ -136,3 +136,15 @@ def test_exec_command_with_env(self): exec_log = self.client.exec_start(res) assert b'X=Y\n' in exec_log + + @requires_api_version('1.35') + def test_exec_command_with_workdir(self): + container = self.client.create_container( + BUSYBOX, 'cat', detach=True, stdin_open=True + ) + self.tmp_containers.append(container) + self.client.start(container) + + res = self.client.exec_create(container, 'pwd', workdir='/var/www') + exec_log = self.client.exec_start(res) + assert exec_log == b'/var/www\n' diff --git a/tests/integration/models_services_test.py b/tests/integration/models_services_test.py index ce83428488..827242a01b 100644 --- a/tests/integration/models_services_test.py +++ b/tests/integration/models_services_test.py @@ -195,6 +195,7 @@ def test_update_retains_networks(self): image="alpine", command="sleep 300" ) + service.reload() service.update( # create argument name=service.name, diff --git a/tests/unit/api_container_test.py b/tests/unit/api_container_test.py index 3b135a8135..8a897ccafc 100644 --- a/tests/unit/api_container_test.py +++ b/tests/unit/api_container_test.py @@ -1263,7 +1263,8 @@ def test_wait(self): fake_request.assert_called_with( 'POST', url_prefix + 'containers/3cc2351ab11b/wait', - timeout=None + timeout=None, + params={} ) def test_wait_with_dict_instead_of_id(self): @@ -1272,7 +1273,8 @@ def test_wait_with_dict_instead_of_id(self): fake_request.assert_called_with( 'POST', url_prefix + 'containers/3cc2351ab11b/wait', - timeout=None + timeout=None, + params={} ) def test_logs(self): diff --git a/tests/unit/models_containers_test.py b/tests/unit/models_containers_test.py index 95295a91b7..62a29b37db 100644 --- a/tests/unit/models_containers_test.py +++ b/tests/unit/models_containers_test.py @@ -394,7 +394,8 @@ def test_exec_run(self): container.exec_run("echo hello world", privileged=True, stream=True) client.api.exec_create.assert_called_with( FAKE_CONTAINER_ID, "echo hello world", stdout=True, stderr=True, - stdin=False, tty=False, privileged=True, user='', environment=None + stdin=False, tty=False, privileged=True, user='', environment=None, + workdir=None ) client.api.exec_start.assert_called_with( FAKE_EXEC_ID, detach=False, tty=False, stream=True, socket=False From 6e6eaece81ebeb43f644a14e498189a9d27d647e Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 26 Jan 2018 14:21:23 -0800 Subject: [PATCH 0561/1301] Return tuple instead of dict in exec_run Signed-off-by: Joffrey F --- docker/models/containers.py | 11 ++++------- tests/integration/models_containers_test.py | 6 +++--- tests/unit/models_containers_test.py | 10 ++++++---- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/docker/models/containers.py b/docker/models/containers.py index 343535bab5..9644b00236 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -150,13 +150,13 @@ def exec_run(self, cmd, stdout=True, stderr=True, stdin=False, tty=False, workdir (str): Path to working directory for this exec session Returns: - dict: + (tuple): A tuple of (exit_code, output) + exit_code: (int): + Exit code for the executed command output: (generator or str): If ``stream=True``, a generator yielding response chunks. If ``socket=True``, a socket object for the connection. A string containing response data otherwise. - exit_code: (int): - Exited code of execution Raises: :py:class:`docker.errors.APIError` @@ -173,10 +173,7 @@ def exec_run(self, cmd, stdout=True, stderr=True, stdin=False, tty=False, exit_code = 0 if stream is False: exit_code = self.client.api.exec_inspect(resp['Id'])['ExitCode'] - return { - 'exit_code': exit_code, - 'output': exec_output - } + return (exit_code, exec_output) def export(self): """ diff --git a/tests/integration/models_containers_test.py b/tests/integration/models_containers_test.py index fd052bef19..3c33cb0721 100644 --- a/tests/integration/models_containers_test.py +++ b/tests/integration/models_containers_test.py @@ -189,8 +189,8 @@ def test_exec_run_success(self): ) self.tmp_containers.append(container.id) exec_output = container.exec_run("cat /test") - assert exec_output["output"] == b"hello\n" - assert exec_output["exit_code"] == 0 + assert exec_output[0] == 0 + assert exec_output[1] == b"hello\n" def test_exec_run_failed(self): client = docker.from_env(version=TEST_API_VERSION) @@ -199,7 +199,7 @@ def test_exec_run_failed(self): ) self.tmp_containers.append(container.id) exec_output = container.exec_run("docker ps") - assert exec_output["exit_code"] == 126 + assert exec_output[0] == 126 def test_kill(self): client = docker.from_env(version=TEST_API_VERSION) diff --git a/tests/unit/models_containers_test.py b/tests/unit/models_containers_test.py index f94c5cfa35..d7457ba4a1 100644 --- a/tests/unit/models_containers_test.py +++ b/tests/unit/models_containers_test.py @@ -400,13 +400,15 @@ def test_exec_run(self): client.api.exec_start.assert_called_with( FAKE_EXEC_ID, detach=False, tty=False, stream=True, socket=False ) + + def test_exec_run_failure(self): + client = make_fake_client() + container = client.containers.get(FAKE_CONTAINER_ID) container.exec_run("docker ps", privileged=True, stream=False) client.api.exec_create.assert_called_with( FAKE_CONTAINER_ID, "docker ps", stdout=True, stderr=True, - stdin=False, tty=False, privileged=True, user='', environment=None - ) - client.api.exec_start.assert_called_with( - FAKE_EXEC_ID, detach=False, tty=False, stream=False, socket=False + stdin=False, tty=False, privileged=True, user='', environment=None, + workdir=None ) client.api.exec_start.assert_called_with( FAKE_EXEC_ID, detach=False, tty=False, stream=False, socket=False From 8b5a52ae0ced380fbd761f891830107d8817f240 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 26 Jan 2018 14:34:20 -0800 Subject: [PATCH 0562/1301] Error handling in ImageCollection.load Signed-off-by: Joffrey F --- docker/errors.py | 4 ++++ docker/models/images.py | 5 ++++- tests/integration/models_images_test.py | 5 +++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/docker/errors.py b/docker/errors.py index 50423a268d..eeeac5791f 100644 --- a/docker/errors.py +++ b/docker/errors.py @@ -144,6 +144,10 @@ class BuildError(Exception): pass +class ImageLoadError(DockerException): + pass + + def create_unexpected_kwargs_error(name, kwargs): quoted_kwargs = ["'{}'".format(k) for k in sorted(kwargs)] text = ["{}() ".format(name)] diff --git a/docker/models/images.py b/docker/models/images.py index 437273013b..dcdeac98aa 100644 --- a/docker/models/images.py +++ b/docker/models/images.py @@ -3,7 +3,7 @@ import six from ..api import APIClient -from ..errors import BuildError +from ..errors import BuildError, ImageLoadError from ..utils.json_stream import json_stream from .resource import Collection, Model @@ -258,6 +258,9 @@ def load(self, data): if match: image_id = match.group(2) images.append(image_id) + if 'error' in chunk: + raise ImageLoadError(chunk['error']) + return [self.get(i) for i in images] def pull(self, name, tag=None, **kwargs): diff --git a/tests/integration/models_images_test.py b/tests/integration/models_images_test.py index 8f812d9390..8840e15de0 100644 --- a/tests/integration/models_images_test.py +++ b/tests/integration/models_images_test.py @@ -71,6 +71,11 @@ def test_pull_with_tag(self): image = client.images.pull('alpine', tag='3.3') assert 'alpine:3.3' in image.attrs['RepoTags'] + def test_load_error(self): + client = docker.from_env(version=TEST_API_VERSION) + with pytest.raises(docker.errors.ImageLoadError): + client.images.load('abc') + class ImageTest(BaseIntegrationTest): From 631cc3c1215441edb075a999a77061c1275c5e5a Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 26 Jan 2018 15:59:46 -0800 Subject: [PATCH 0563/1301] ImageCollection.build now also returns build logs along with the built image reference BuildError.build_logs has a copy of the logs generator Signed-off-by: Joffrey F --- docker/errors.py | 7 +++++-- docker/models/images.py | 14 +++++++++----- tests/integration/models_images_test.py | 17 ++++++++++------- 3 files changed, 24 insertions(+), 14 deletions(-) diff --git a/docker/errors.py b/docker/errors.py index eeeac5791f..0253695a5f 100644 --- a/docker/errors.py +++ b/docker/errors.py @@ -140,8 +140,11 @@ def __init__(self, reason): self.msg = reason -class BuildError(Exception): - pass +class BuildError(DockerException): + def __init__(self, reason, build_log): + super(BuildError, self).__init__(reason) + self.msg = reason + self.build_log = build_log class ImageLoadError(DockerException): diff --git a/docker/models/images.py b/docker/models/images.py index dcdeac98aa..c4e727b598 100644 --- a/docker/models/images.py +++ b/docker/models/images.py @@ -1,3 +1,4 @@ +import itertools import re import six @@ -160,7 +161,9 @@ def build(self, **kwargs): platform (str): Platform in the format ``os[/arch[/variant]]``. Returns: - (:py:class:`Image`): The built image. + (tuple): The first item is the :py:class:`Image` object for the + image that was build. The second item is a generator of the + build logs as JSON-decoded objects. Raises: :py:class:`docker.errors.BuildError` @@ -175,9 +178,10 @@ def build(self, **kwargs): return self.get(resp) last_event = None image_id = None - for chunk in json_stream(resp): + result_stream, internal_stream = itertools.tee(json_stream(resp)) + for chunk in internal_stream: if 'error' in chunk: - raise BuildError(chunk['error']) + raise BuildError(chunk['error'], result_stream) if 'stream' in chunk: match = re.search( r'(^Successfully built |sha256:)([0-9a-f]+)$', @@ -187,8 +191,8 @@ def build(self, **kwargs): image_id = match.group(2) last_event = chunk if image_id: - return self.get(image_id) - raise BuildError(last_event or 'Unknown') + return (self.get(image_id), result_stream) + raise BuildError(last_event or 'Unknown', result_stream) def get(self, name): """ diff --git a/tests/integration/models_images_test.py b/tests/integration/models_images_test.py index 8840e15de0..900555d7f2 100644 --- a/tests/integration/models_images_test.py +++ b/tests/integration/models_images_test.py @@ -10,27 +10,30 @@ class ImageCollectionTest(BaseIntegrationTest): def test_build(self): client = docker.from_env(version=TEST_API_VERSION) - image = client.images.build(fileobj=io.BytesIO( + image, _ = client.images.build(fileobj=io.BytesIO( "FROM alpine\n" "CMD echo hello world".encode('ascii') )) self.tmp_imgs.append(image.id) assert client.containers.run(image) == b"hello world\n" - @pytest.mark.xfail(reason='Engine 1.13 responds with status 500') + # @pytest.mark.xfail(reason='Engine 1.13 responds with status 500') def test_build_with_error(self): client = docker.from_env(version=TEST_API_VERSION) with self.assertRaises(docker.errors.BuildError) as cm: client.images.build(fileobj=io.BytesIO( "FROM alpine\n" - "NOTADOCKERFILECOMMAND".encode('ascii') + "RUN exit 1".encode('ascii') )) - assert str(cm.exception) == ("Unknown instruction: " - "NOTADOCKERFILECOMMAND") + print(cm.exception) + assert str(cm.exception) == ( + "The command '/bin/sh -c exit 1' returned a non-zero code: 1" + ) + assert cm.exception.build_log def test_build_with_multiple_success(self): client = docker.from_env(version=TEST_API_VERSION) - image = client.images.build( + image, _ = client.images.build( tag='some-tag', fileobj=io.BytesIO( "FROM alpine\n" "CMD echo hello world".encode('ascii') @@ -41,7 +44,7 @@ def test_build_with_multiple_success(self): def test_build_with_success_build_output(self): client = docker.from_env(version=TEST_API_VERSION) - image = client.images.build( + image, _ = client.images.build( tag='dup-txt-tag', fileobj=io.BytesIO( "FROM alpine\n" "CMD echo Successfully built abcd1234".encode('ascii') From 388f291b13fca76f4974a1ee89225ff7f3afb85b Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 26 Jan 2018 15:32:04 -0800 Subject: [PATCH 0564/1301] Update save / export methods to return data generators Signed-off-by: Joffrey F --- docker/api/container.py | 7 +++---- docker/api/image.py | 13 ++++++------- docker/models/images.py | 12 +++++------- tests/integration/api_image_test.py | 24 +++++++++++++++++++++++- tests/integration/models_images_test.py | 17 ++++++++++++++++- 5 files changed, 53 insertions(+), 20 deletions(-) diff --git a/docker/api/container.py b/docker/api/container.py index b08032c403..49230c7b66 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -698,7 +698,7 @@ def export(self, container): container (str): The container to export Returns: - (str): The filesystem tar archive + (generator): The archived filesystem data stream Raises: :py:class:`docker.errors.APIError` @@ -707,8 +707,7 @@ def export(self, container): res = self._get( self._url("/containers/{0}/export", container), stream=True ) - self._raise_for_status(res) - return res.raw + return self._stream_raw_result(res) @utils.check_resource('container') @utils.minimum_version('1.20') @@ -737,7 +736,7 @@ def get_archive(self, container, path): self._raise_for_status(res) encoded_stat = res.headers.get('x-docker-container-path-stat') return ( - res.raw, + self._stream_raw_result(res), utils.decode_json_header(encoded_stat) if encoded_stat else None ) diff --git a/docker/api/image.py b/docker/api/image.py index 065fae3959..b3dcd3ab5a 100644 --- a/docker/api/image.py +++ b/docker/api/image.py @@ -21,8 +21,7 @@ def get_image(self, image): image (str): Image name to get Returns: - (urllib3.response.HTTPResponse object): The response from the - daemon. + (generator): A stream of raw archive data. Raises: :py:class:`docker.errors.APIError` @@ -30,14 +29,14 @@ def get_image(self, image): Example: - >>> image = cli.get_image("fedora:latest") - >>> f = open('/tmp/fedora-latest.tar', 'w') - >>> f.write(image.data) + >>> image = cli.get_image("busybox:latest") + >>> f = open('/tmp/busybox-latest.tar', 'w') + >>> for chunk in image: + >>> f.write(chunk) >>> f.close() """ res = self._get(self._url("/images/{0}/get", image), stream=True) - self._raise_for_status(res) - return res.raw + return self._stream_raw_result(res) @utils.check_resource('image') def history(self, image): diff --git a/docker/models/images.py b/docker/models/images.py index dcdeac98aa..8229cfc6a6 100644 --- a/docker/models/images.py +++ b/docker/models/images.py @@ -61,8 +61,7 @@ def save(self): Get a tarball of an image. Similar to the ``docker save`` command. Returns: - (urllib3.response.HTTPResponse object): The response from the - daemon. + (generator): A stream of raw archive data. Raises: :py:class:`docker.errors.APIError` @@ -70,11 +69,10 @@ def save(self): Example: - >>> image = cli.images.get("fedora:latest") - >>> resp = image.save() - >>> f = open('/tmp/fedora-latest.tar', 'w') - >>> for chunk in resp.stream(): - >>> f.write(chunk) + >>> image = cli.get_image("busybox:latest") + >>> f = open('/tmp/busybox-latest.tar', 'w') + >>> for chunk in image: + >>> f.write(chunk) >>> f.close() """ return self.client.api.get_image(self.id) diff --git a/tests/integration/api_image_test.py b/tests/integration/api_image_test.py index 178c34e995..ae93190e52 100644 --- a/tests/integration/api_image_test.py +++ b/tests/integration/api_image_test.py @@ -329,7 +329,7 @@ def test_prune_images(self): img_id = self.client.inspect_image('hello-world')['Id'] result = self.client.prune_images() assert img_id not in [ - img.get('Deleted') for img in result['ImagesDeleted'] + img.get('Deleted') for img in result.get('ImagesDeleted') or [] ] result = self.client.prune_images({'dangling': False}) assert result['SpaceReclaimed'] > 0 @@ -339,3 +339,25 @@ def test_prune_images(self): assert img_id in [ img.get('Deleted') for img in result['ImagesDeleted'] ] + + +class SaveLoadImagesTest(BaseAPIIntegrationTest): + @requires_api_version('1.23') + def test_get_image_load_image(self): + with tempfile.TemporaryFile() as f: + stream = self.client.get_image(BUSYBOX) + for chunk in stream: + f.write(chunk) + + f.seek(0) + result = self.client.load_image(f.read()) + + success = False + result_line = 'Loaded image: {}\n'.format(BUSYBOX) + for data in result: + print(data) + if 'stream' in data: + if data['stream'] == result_line: + success = True + break + assert success is True diff --git a/tests/integration/models_images_test.py b/tests/integration/models_images_test.py index 8840e15de0..2a28e1265e 100644 --- a/tests/integration/models_images_test.py +++ b/tests/integration/models_images_test.py @@ -1,9 +1,10 @@ import io +import tempfile import docker import pytest -from .base import BaseIntegrationTest, TEST_API_VERSION +from .base import BaseIntegrationTest, BUSYBOX, TEST_API_VERSION class ImageCollectionTest(BaseIntegrationTest): @@ -76,6 +77,20 @@ def test_load_error(self): with pytest.raises(docker.errors.ImageLoadError): client.images.load('abc') + def test_save_and_load(self): + client = docker.from_env(version=TEST_API_VERSION) + image = client.images.get(BUSYBOX) + with tempfile.TemporaryFile() as f: + stream = image.save() + for chunk in stream: + f.write(chunk) + + f.seek(0) + result = client.images.load(f.read()) + + assert len(result) == 1 + assert result[0].id == image.id + class ImageTest(BaseIntegrationTest): From ad208dfd29e6b5b8426763b835c29948f9dd8604 Mon Sep 17 00:00:00 2001 From: Frank Sachsenheim Date: Sun, 28 Jan 2018 01:26:58 +0100 Subject: [PATCH 0565/1301] Container.exec_run returns None as exit_code if stream or socket Signed-off-by: Frank Sachsenheim --- docker/models/containers.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/docker/models/containers.py b/docker/models/containers.py index 9644b00236..42c9d5564e 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -152,7 +152,8 @@ def exec_run(self, cmd, stdout=True, stderr=True, stdin=False, tty=False, Returns: (tuple): A tuple of (exit_code, output) exit_code: (int): - Exit code for the executed command + Exit code for the executed command or ``None`` if + either ``stream```or ``socket`` is ``True``. output: (generator or str): If ``stream=True``, a generator yielding response chunks. If ``socket=True``, a socket object for the connection. @@ -170,10 +171,11 @@ def exec_run(self, cmd, stdout=True, stderr=True, stdin=False, tty=False, exec_output = self.client.api.exec_start( resp['Id'], detach=detach, tty=tty, stream=stream, socket=socket ) - exit_code = 0 - if stream is False: - exit_code = self.client.api.exec_inspect(resp['Id'])['ExitCode'] - return (exit_code, exec_output) + if socket or stream: + return None, exec_output + else: + return (self.client.api.exec_inspect(resp['Id'])['ExitCode'], + exec_output) def export(self): """ From a63b726d40096e45ebb0598511573efea13b320f Mon Sep 17 00:00:00 2001 From: Frank Sachsenheim Date: Sun, 28 Jan 2018 01:38:31 +0100 Subject: [PATCH 0566/1301] Container.exec_run returns a namedtuple w/ attrs exit_code & output Signed-off-by: Frank Sachsenheim --- docker/models/containers.py | 11 ++++++----- docker/types/__init__.py | 3 ++- docker/types/containers.py | 6 ++++++ 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/docker/models/containers.py b/docker/models/containers.py index 42c9d5564e..ed3f7ace4a 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -3,7 +3,7 @@ from ..api import APIClient from ..errors import (ContainerError, ImageNotFound, create_unexpected_kwargs_error) -from ..types import HostConfig +from ..types import ExecResult, HostConfig from ..utils import version_gte from .images import Image from .resource import Collection, Model @@ -150,7 +150,7 @@ def exec_run(self, cmd, stdout=True, stderr=True, stdin=False, tty=False, workdir (str): Path to working directory for this exec session Returns: - (tuple): A tuple of (exit_code, output) + (ExecResult): A tuple of (exit_code, output) exit_code: (int): Exit code for the executed command or ``None`` if either ``stream```or ``socket`` is ``True``. @@ -172,10 +172,11 @@ def exec_run(self, cmd, stdout=True, stderr=True, stdin=False, tty=False, resp['Id'], detach=detach, tty=tty, stream=stream, socket=socket ) if socket or stream: - return None, exec_output + return ExecResult(None, exec_output) else: - return (self.client.api.exec_inspect(resp['Id'])['ExitCode'], - exec_output) + return ExecResult( + self.client.api.exec_inspect(resp['Id'])['ExitCode'], + exec_output) def export(self): """ diff --git a/docker/types/__init__.py b/docker/types/__init__.py index 39c93e344d..bd19638c8f 100644 --- a/docker/types/__init__.py +++ b/docker/types/__init__.py @@ -1,5 +1,6 @@ # flake8: noqa -from .containers import ContainerConfig, HostConfig, LogConfig, Ulimit +from .containers import (ContainerConfig, ExecResult, HostConfig, LogConfig, + Ulimit) from .healthcheck import Healthcheck from .networks import EndpointConfig, IPAMConfig, IPAMPool, NetworkingConfig from .services import ( diff --git a/docker/types/containers.py b/docker/types/containers.py index 15dd86c991..8d1f271f29 100644 --- a/docker/types/containers.py +++ b/docker/types/containers.py @@ -1,3 +1,4 @@ +from collections import namedtuple import six import warnings @@ -11,6 +12,11 @@ from .healthcheck import Healthcheck +ExecResult = namedtuple('ExecResult', 'exit_code,output') +""" A result of Container.exec_run with the properties ``exit_code`` and + ``output``. """ + + class LogConfigTypesEnum(object): _values = ( 'json-file', From 947c47f609bf7fc97b0117aef65b1d035a19f24f Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 29 Jan 2018 16:34:03 -0800 Subject: [PATCH 0567/1301] Move ExecResult definition to models.containers Signed-off-by: Joffrey F --- docker/models/containers.py | 17 ++++++++++++----- docker/types/__init__.py | 3 +-- docker/types/containers.py | 6 ------ 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/docker/models/containers.py b/docker/models/containers.py index ed3f7ace4a..79fd71df1e 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -1,9 +1,10 @@ import copy +from collections import namedtuple from ..api import APIClient from ..errors import (ContainerError, ImageNotFound, create_unexpected_kwargs_error) -from ..types import ExecResult, HostConfig +from ..types import HostConfig from ..utils import version_gte from .images import Image from .resource import Collection, Model @@ -173,10 +174,11 @@ def exec_run(self, cmd, stdout=True, stderr=True, stdin=False, tty=False, ) if socket or stream: return ExecResult(None, exec_output) - else: - return ExecResult( - self.client.api.exec_inspect(resp['Id'])['ExitCode'], - exec_output) + + return ExecResult( + self.client.api.exec_inspect(resp['Id'])['ExitCode'], + exec_output + ) def export(self): """ @@ -1007,3 +1009,8 @@ def _host_volume_from_bind(bind): return bits[0] else: return bits[1] + + +ExecResult = namedtuple('ExecResult', 'exit_code,output') +""" A result of Container.exec_run with the properties ``exit_code`` and + ``output``. """ diff --git a/docker/types/__init__.py b/docker/types/__init__.py index bd19638c8f..39c93e344d 100644 --- a/docker/types/__init__.py +++ b/docker/types/__init__.py @@ -1,6 +1,5 @@ # flake8: noqa -from .containers import (ContainerConfig, ExecResult, HostConfig, LogConfig, - Ulimit) +from .containers import ContainerConfig, HostConfig, LogConfig, Ulimit from .healthcheck import Healthcheck from .networks import EndpointConfig, IPAMConfig, IPAMPool, NetworkingConfig from .services import ( diff --git a/docker/types/containers.py b/docker/types/containers.py index 8d1f271f29..15dd86c991 100644 --- a/docker/types/containers.py +++ b/docker/types/containers.py @@ -1,4 +1,3 @@ -from collections import namedtuple import six import warnings @@ -12,11 +11,6 @@ from .healthcheck import Healthcheck -ExecResult = namedtuple('ExecResult', 'exit_code,output') -""" A result of Container.exec_run with the properties ``exit_code`` and - ``output``. """ - - class LogConfigTypesEnum(object): _values = ( 'json-file', From 4e34300379a9d85d99192e890cef2f1f656c6761 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 29 Jan 2018 18:13:46 -0800 Subject: [PATCH 0568/1301] Do not break when archiving broken symlinks Signed-off-by: Joffrey F --- docker/utils/utils.py | 3 ++- tests/unit/utils_test.py | 26 ++++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/docker/utils/utils.py b/docker/utils/utils.py index 2de995c402..c4db1750b3 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -98,7 +98,8 @@ def create_archive(root, files=None, fileobj=None, gzip=False): files = build_file_list(root) for path in files: full_path = os.path.join(root, path) - if not os.access(full_path, os.R_OK): + + if os.lstat(full_path).st_mode & os.R_OK == 0: raise IOError( 'Can not access file in context: {}'.format(full_path) ) diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index 2fa1d051f2..8674e853f9 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -948,6 +948,20 @@ def test_tar_with_empty_directory(self): tar_data = tarfile.open(fileobj=archive) self.assertEqual(sorted(tar_data.getnames()), ['bar', 'foo']) + @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='No chmod on Windows') + def test_tar_with_inaccessible_file(self): + base = tempfile.mkdtemp() + full_path = os.path.join(base, 'foo') + self.addCleanup(shutil.rmtree, base) + with open(full_path, 'w') as f: + f.write('content') + os.chmod(full_path, 0o222) + with pytest.raises(IOError) as ei: + tar(base) + + assert 'Can not access file in context: {}'.format(full_path) in \ + ei.exconly() + @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='No symlinks on Windows') def test_tar_with_file_symlinks(self): base = tempfile.mkdtemp() @@ -975,6 +989,18 @@ def test_tar_with_directory_symlinks(self): sorted(tar_data.getnames()), ['bar', 'bar/foo', 'foo'] ) + @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='No symlinks on Windows') + def test_tar_with_broken_symlinks(self): + base = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, base) + for d in ['foo', 'bar']: + os.makedirs(os.path.join(base, d)) + + os.symlink('../baz', os.path.join(base, 'bar/foo')) + with tar(base) as archive: + tar_data = tarfile.open(fileobj=archive) + assert sorted(tar_data.getnames()) == ['bar', 'bar/foo', 'foo'] + @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='No UNIX sockets on Win32') def test_tar_socket_file(self): base = tempfile.mkdtemp() From 342221130918f4525f01e31d3697cfc077df090e Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 29 Jan 2018 19:10:12 -0800 Subject: [PATCH 0569/1301] Use pytest asserts Signed-off-by: Joffrey F --- tests/integration/api_build_test.py | 23 +- tests/integration/api_client_test.py | 43 +- tests/integration/api_container_test.py | 439 +++++----- tests/integration/api_exec_test.py | 36 +- tests/integration/api_healthcheck_test.py | 5 +- tests/integration/api_image_test.py | 60 +- tests/integration/api_network_test.py | 98 +-- tests/integration/api_volume_test.py | 14 +- tests/integration/errors_test.py | 5 +- tests/integration/models_containers_test.py | 30 +- tests/integration/models_images_test.py | 9 +- tests/integration/models_services_test.py | 4 +- tests/integration/models_swarm_test.py | 9 +- tests/integration/regression_test.py | 30 +- tests/unit/api_build_test.py | 13 +- tests/unit/api_container_test.py | 846 ++++++++------------ tests/unit/api_exec_test.py | 90 +-- tests/unit/api_image_test.py | 38 +- tests/unit/api_network_test.py | 97 +-- tests/unit/api_test.py | 79 +- tests/unit/api_volume_test.py | 68 +- tests/unit/auth_test.py | 284 +++---- tests/unit/client_test.py | 23 +- tests/unit/dockertypes_test.py | 194 +++-- tests/unit/models_containers_test.py | 15 +- tests/unit/ssladapter_test.py | 11 +- tests/unit/swarm_test.py | 12 +- tests/unit/utils_test.py | 303 ++++--- 28 files changed, 1270 insertions(+), 1608 deletions(-) diff --git a/tests/integration/api_build_test.py b/tests/integration/api_build_test.py index 245214e1a2..ee9b68a619 100644 --- a/tests/integration/api_build_test.py +++ b/tests/integration/api_build_test.py @@ -43,7 +43,7 @@ def test_build_from_stringio(self): if six.PY3: chunk = chunk.decode('utf-8') logs += chunk - self.assertNotEqual(logs, '') + assert logs != '' @requires_api_version('1.8') def test_build_with_dockerignore(self): @@ -92,11 +92,10 @@ def test_build_with_dockerignore(self): if six.PY3: logs = logs.decode('utf-8') - self.assertEqual( - sorted(list(filter(None, logs.split('\n')))), - sorted(['/test/ignored/subdir/excepted-file', - '/test/not-ignored']), - ) + assert sorted(list(filter(None, logs.split('\n')))) == sorted([ + '/test/ignored/subdir/excepted-file', + '/test/not-ignored' + ]) @requires_api_version('1.21') def test_build_with_buildargs(self): @@ -114,7 +113,7 @@ def test_build_with_buildargs(self): pass info = self.client.inspect_image('buildargs') - self.assertEqual(info['Config']['User'], 'OK') + assert info['Config']['User'] == 'OK' @requires_api_version('1.22') def test_build_shmsize(self): @@ -152,7 +151,7 @@ def test_build_labels(self): pass info = self.client.inspect_image('labels') - self.assertEqual(info['Config']['Labels'], labels) + assert info['Config']['Labels'] == labels @requires_api_version('1.25') def test_build_with_cache_from(self): @@ -309,8 +308,8 @@ def build_squashed(squash): non_squashed = build_squashed(False) squashed = build_squashed(True) - self.assertEqual(len(non_squashed['RootFS']['Layers']), 4) - self.assertEqual(len(squashed['RootFS']['Layers']), 2) + assert len(non_squashed['RootFS']['Layers']) == 4 + assert len(squashed['RootFS']['Layers']) == 2 def test_build_stderr_data(self): control_chars = ['\x1b[91m', '\x1b[0m'] @@ -329,7 +328,7 @@ def test_build_stderr_data(self): expected = '{0}{2}\n{1}'.format( control_chars[0], control_chars[1], snippet ) - self.assertTrue(any([line == expected for line in lines])) + assert any([line == expected for line in lines]) def test_build_gzip_encoding(self): base_dir = tempfile.mkdtemp() @@ -375,7 +374,7 @@ def test_build_with_dockerfile_empty_lines(self): assert 'Successfully built' in lines[-1]['stream'] def test_build_gzip_custom_encoding(self): - with self.assertRaises(errors.DockerException): + with pytest.raises(errors.DockerException): self.client.build(path='.', gzip=True, encoding='text/html') @requires_api_version('1.32') diff --git a/tests/integration/api_client_test.py b/tests/integration/api_client_test.py index cfb45a3e31..05281f8849 100644 --- a/tests/integration/api_client_test.py +++ b/tests/integration/api_client_test.py @@ -14,14 +14,14 @@ class InformationTest(BaseAPIIntegrationTest): def test_version(self): res = self.client.version() - self.assertIn('GoVersion', res) - self.assertIn('Version', res) + assert 'GoVersion' in res + assert 'Version' in res def test_info(self): res = self.client.info() - self.assertIn('Containers', res) - self.assertIn('Images', res) - self.assertIn('Debug', res) + assert 'Containers' in res + assert 'Images' in res + assert 'Debug' in res class LoadConfigTest(BaseAPIIntegrationTest): @@ -35,12 +35,12 @@ def test_load_legacy_config(self): f.write('email = sakuya@scarlet.net') f.close() cfg = docker.auth.load_config(cfg_path) - self.assertNotEqual(cfg[docker.auth.INDEX_NAME], None) + assert cfg[docker.auth.INDEX_NAME] is not None cfg = cfg[docker.auth.INDEX_NAME] - self.assertEqual(cfg['username'], 'sakuya') - self.assertEqual(cfg['password'], 'izayoi') - self.assertEqual(cfg['email'], 'sakuya@scarlet.net') - self.assertEqual(cfg.get('Auth'), None) + assert cfg['username'] == 'sakuya' + assert cfg['password'] == 'izayoi' + assert cfg['email'] == 'sakuya@scarlet.net' + assert cfg.get('Auth') is None def test_load_json_config(self): folder = tempfile.mkdtemp() @@ -53,12 +53,12 @@ def test_load_json_config(self): docker.auth.INDEX_URL, auth_, email_)) f.close() cfg = docker.auth.load_config(cfg_path) - self.assertNotEqual(cfg[docker.auth.INDEX_URL], None) + assert cfg[docker.auth.INDEX_URL] is not None cfg = cfg[docker.auth.INDEX_URL] - self.assertEqual(cfg['username'], 'sakuya') - self.assertEqual(cfg['password'], 'izayoi') - self.assertEqual(cfg['email'], 'sakuya@scarlet.net') - self.assertEqual(cfg.get('Auth'), None) + assert cfg['username'] == 'sakuya' + assert cfg['password'] == 'izayoi' + assert cfg['email'] == 'sakuya@scarlet.net' + assert cfg.get('Auth') is None class AutoDetectVersionTest(unittest.TestCase): @@ -66,9 +66,9 @@ def test_client_init(self): client = docker.APIClient(version='auto', **kwargs_from_env()) client_version = client._version api_version = client.version(api_version=False)['ApiVersion'] - self.assertEqual(client_version, api_version) + assert client_version == api_version api_version_2 = client.version()['ApiVersion'] - self.assertEqual(client_version, api_version_2) + assert client_version == api_version_2 client.close() @@ -90,8 +90,8 @@ def test_timeout(self): except: pass end = time.time() - self.assertTrue(res is None) - self.assertTrue(end - start < 2 * self.timeout) + assert res is None + assert end - start < 2 * self.timeout class UnixconnTest(unittest.TestCase): @@ -112,5 +112,6 @@ def test_resource_warnings(self): client.close() del client - assert len(w) == 0, \ - "No warnings produced: {0}".format(w[0].message) + assert len(w) == 0, "No warnings produced: {0}".format( + w[0].message + ) diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index 5d06bc46a3..4585c442d9 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -16,6 +16,7 @@ from .base import BUSYBOX, BaseAPIIntegrationTest from .. import helpers from ..helpers import requires_api_version +import re class ListContainersTest(BaseAPIIntegrationTest): @@ -23,26 +24,26 @@ def test_list_containers(self): res0 = self.client.containers(all=True) size = len(res0) res1 = self.client.create_container(BUSYBOX, 'true') - self.assertIn('Id', res1) + assert 'Id' in res1 self.client.start(res1['Id']) self.tmp_containers.append(res1['Id']) res2 = self.client.containers(all=True) - self.assertEqual(size + 1, len(res2)) + assert size + 1 == len(res2) retrieved = [x for x in res2 if x['Id'].startswith(res1['Id'])] - self.assertEqual(len(retrieved), 1) + assert len(retrieved) == 1 retrieved = retrieved[0] - self.assertIn('Command', retrieved) - self.assertEqual(retrieved['Command'], six.text_type('true')) - self.assertIn('Image', retrieved) - self.assertRegex(retrieved['Image'], r'busybox:.*') - self.assertIn('Status', retrieved) + assert 'Command' in retrieved + assert retrieved['Command'] == six.text_type('true') + assert 'Image' in retrieved + assert re.search(r'busybox:.*', retrieved['Image']) + assert 'Status' in retrieved class CreateContainerTest(BaseAPIIntegrationTest): def test_create(self): res = self.client.create_container(BUSYBOX, 'true') - self.assertIn('Id', res) + assert 'Id' in res self.tmp_containers.append(res['Id']) def test_create_with_host_pid_mode(self): @@ -51,14 +52,14 @@ def test_create_with_host_pid_mode(self): pid_mode='host', network_mode='none' ) ) - self.assertIn('Id', ctnr) + assert 'Id' in ctnr self.tmp_containers.append(ctnr['Id']) self.client.start(ctnr) inspect = self.client.inspect_container(ctnr) - self.assertIn('HostConfig', inspect) + assert 'HostConfig' in inspect host_config = inspect['HostConfig'] - self.assertIn('PidMode', host_config) - self.assertEqual(host_config['PidMode'], 'host') + assert 'PidMode' in host_config + assert host_config['PidMode'] == 'host' def test_create_with_links(self): res0 = self.client.create_container( @@ -99,15 +100,15 @@ def test_create_with_links(self): container3_id = res2['Id'] self.tmp_containers.append(container3_id) self.client.start(container3_id) - self.assertEqual(self.client.wait(container3_id), 0) + assert self.client.wait(container3_id) == 0 logs = self.client.logs(container3_id) if six.PY3: logs = logs.decode('utf-8') - self.assertIn('{0}_NAME='.format(link_env_prefix1), logs) - self.assertIn('{0}_ENV_FOO=1'.format(link_env_prefix1), logs) - self.assertIn('{0}_NAME='.format(link_env_prefix2), logs) - self.assertIn('{0}_ENV_FOO=1'.format(link_env_prefix2), logs) + assert '{0}_NAME='.format(link_env_prefix1) in logs + assert '{0}_ENV_FOO=1'.format(link_env_prefix1) in logs + assert '{0}_NAME='.format(link_env_prefix2) in logs + assert '{0}_ENV_FOO=1'.format(link_env_prefix2) in logs def test_create_with_restart_policy(self): container = self.client.create_container( @@ -120,12 +121,10 @@ def test_create_with_restart_policy(self): id = container['Id'] self.client.start(id) self.client.wait(id) - with self.assertRaises(docker.errors.APIError) as exc: + with pytest.raises(docker.errors.APIError) as exc: self.client.remove_container(id) - err = exc.exception.explanation - self.assertIn( - 'You cannot remove ', err - ) + err = exc.value.explanation + assert 'You cannot remove ' in err self.client.remove_container(id, force=True) def test_create_container_with_volumes_from(self): @@ -144,7 +143,7 @@ def test_create_container_with_volumes_from(self): container2_id = res1['Id'] self.tmp_containers.append(container2_id) self.client.start(container2_id) - with self.assertRaises(docker.errors.DockerException): + with pytest.raises(docker.errors.DockerException): self.client.create_container( BUSYBOX, 'cat', detach=True, stdin_open=True, volumes_from=vol_names @@ -169,19 +168,19 @@ def create_container_readonly_fs(self): read_only=True, network_mode='none' ) ) - self.assertIn('Id', ctnr) + assert 'Id' in ctnr self.tmp_containers.append(ctnr['Id']) self.client.start(ctnr) res = self.client.wait(ctnr) - self.assertNotEqual(res, 0) + assert res != 0 def create_container_with_name(self): res = self.client.create_container(BUSYBOX, 'true', name='foobar') - self.assertIn('Id', res) + assert 'Id' in res self.tmp_containers.append(res['Id']) inspect = self.client.inspect_container(res['Id']) - self.assertIn('Name', inspect) - self.assertEqual('/foobar', inspect['Name']) + assert 'Name' in inspect + assert '/foobar' == inspect['Name'] def create_container_privileged(self): res = self.client.create_container( @@ -189,24 +188,24 @@ def create_container_privileged(self): privileged=True, network_mode='none' ) ) - self.assertIn('Id', res) + assert 'Id' in res self.tmp_containers.append(res['Id']) self.client.start(res['Id']) inspect = self.client.inspect_container(res['Id']) - self.assertIn('Config', inspect) - self.assertIn('Id', inspect) - self.assertTrue(inspect['Id'].startswith(res['Id'])) - self.assertIn('Image', inspect) - self.assertIn('State', inspect) - self.assertIn('Running', inspect['State']) + assert 'Config' in inspect + assert 'Id' in inspect + assert inspect['Id'].startswith(res['Id']) + assert 'Image' in inspect + assert 'State' in inspect + assert 'Running' in inspect['State'] if not inspect['State']['Running']: - self.assertIn('ExitCode', inspect['State']) - self.assertEqual(inspect['State']['ExitCode'], 0) + assert 'ExitCode' in inspect['State'] + assert inspect['State']['ExitCode'] == 0 # Since Nov 2013, the Privileged flag is no longer part of the # container's config exposed via the API (safety concerns?). # if 'Privileged' in inspect['Config']: - self.assertEqual(inspect['Config']['Privileged'], True) + assert inspect['Config']['Privileged'] is True def test_create_with_mac_address(self): mac_address_expected = "02:42:ac:11:00:0a" @@ -217,8 +216,7 @@ def test_create_with_mac_address(self): self.client.start(container) res = self.client.inspect_container(container['Id']) - self.assertEqual(mac_address_expected, - res['NetworkSettings']['MacAddress']) + assert mac_address_expected == res['NetworkSettings']['MacAddress'] self.client.kill(id) @@ -236,8 +234,8 @@ def test_group_id_ints(self): if six.PY3: logs = logs.decode('utf-8') groups = logs.strip().split(' ') - self.assertIn('1000', groups) - self.assertIn('1001', groups) + assert '1000' in groups + assert '1001' in groups @requires_api_version('1.20') def test_group_id_strings(self): @@ -255,8 +253,8 @@ def test_group_id_strings(self): logs = logs.decode('utf-8') groups = logs.strip().split(' ') - self.assertIn('1000', groups) - self.assertIn('1001', groups) + assert '1000' in groups + assert '1001' in groups def test_valid_log_driver_and_log_opt(self): log_config = docker.types.LogConfig( @@ -274,8 +272,8 @@ def test_valid_log_driver_and_log_opt(self): info = self.client.inspect_container(container) container_log_config = info['HostConfig']['LogConfig'] - self.assertEqual(container_log_config['Type'], log_config.type) - self.assertEqual(container_log_config['Config'], log_config.config) + assert container_log_config['Type'] == log_config.type + assert container_log_config['Config'] == log_config.config def test_invalid_log_driver_raises_exception(self): log_config = docker.types.LogConfig( @@ -311,8 +309,8 @@ def test_valid_no_log_driver_specified(self): info = self.client.inspect_container(container) container_log_config = info['HostConfig']['LogConfig'] - self.assertEqual(container_log_config['Type'], "json-file") - self.assertEqual(container_log_config['Config'], log_config.config) + assert container_log_config['Type'] == "json-file" + assert container_log_config['Config'] == log_config.config def test_valid_no_config_specified(self): log_config = docker.types.LogConfig( @@ -330,8 +328,8 @@ def test_valid_no_config_specified(self): info = self.client.inspect_container(container) container_log_config = info['HostConfig']['LogConfig'] - self.assertEqual(container_log_config['Type'], "json-file") - self.assertEqual(container_log_config['Config'], {}) + assert container_log_config['Type'] == "json-file" + assert container_log_config['Config'] == {} def test_create_with_memory_constraints_with_str(self): ctnr = self.client.create_container( @@ -341,29 +339,29 @@ def test_create_with_memory_constraints_with_str(self): mem_limit='700M' ) ) - self.assertIn('Id', ctnr) + assert 'Id' in ctnr self.tmp_containers.append(ctnr['Id']) self.client.start(ctnr) inspect = self.client.inspect_container(ctnr) - self.assertIn('HostConfig', inspect) + assert 'HostConfig' in inspect host_config = inspect['HostConfig'] for limit in ['Memory', 'MemorySwap']: - self.assertIn(limit, host_config) + assert limit in host_config def test_create_with_memory_constraints_with_int(self): ctnr = self.client.create_container( BUSYBOX, 'true', host_config=self.client.create_host_config(mem_swappiness=40) ) - self.assertIn('Id', ctnr) + assert 'Id' in ctnr self.tmp_containers.append(ctnr['Id']) self.client.start(ctnr) inspect = self.client.inspect_container(ctnr) - self.assertIn('HostConfig', inspect) + assert 'HostConfig' in inspect host_config = inspect['HostConfig'] - self.assertIn('MemorySwappiness', host_config) + assert 'MemorySwappiness' in host_config def test_create_with_environment_variable_no_value(self): container = self.client.create_container( @@ -511,7 +509,7 @@ def test_create_with_binds_rw(self): if six.PY3: logs = logs.decode('utf-8') - self.assertIn(self.filename, logs) + assert self.filename in logs inspect_data = self.client.inspect_container(container) self.check_container_data(inspect_data, True) @@ -533,7 +531,7 @@ def test_create_with_binds_ro(self): if six.PY3: logs = logs.decode('utf-8') - self.assertIn(self.filename, logs) + assert self.filename in logs inspect_data = self.client.inspect_container(container) self.check_container_data(inspect_data, False) @@ -605,23 +603,23 @@ def test_create_with_volume_mount(self): def check_container_data(self, inspect_data, rw): if docker.utils.compare_version('1.20', self.client._version) < 0: - self.assertIn('Volumes', inspect_data) - self.assertIn(self.mount_dest, inspect_data['Volumes']) - self.assertEqual( - self.mount_origin, inspect_data['Volumes'][self.mount_dest] + assert 'Volumes' in inspect_data + assert self.mount_dest in inspect_data['Volumes'] + assert ( + self.mount_origin == inspect_data['Volumes'][self.mount_dest] ) - self.assertIn(self.mount_dest, inspect_data['VolumesRW']) - self.assertFalse(inspect_data['VolumesRW'][self.mount_dest]) + assert self.mount_dest in inspect_data['VolumesRW'] + assert not inspect_data['VolumesRW'][self.mount_dest] else: - self.assertIn('Mounts', inspect_data) + assert 'Mounts' in inspect_data filtered = list(filter( lambda x: x['Destination'] == self.mount_dest, inspect_data['Mounts'] )) - self.assertEqual(len(filtered), 1) + assert len(filtered) == 1 mount_data = filtered[0] - self.assertEqual(mount_data['Source'], self.mount_origin) - self.assertEqual(mount_data['RW'], rw) + assert mount_data['Source'] == self.mount_origin + assert mount_data['RW'] == rw def run_with_volume(self, ro, *args, **kwargs): return self.run_container( @@ -659,7 +657,7 @@ def test_get_file_archive_from_container(self): retrieved_data = helpers.untar_file(destination, 'data.txt') if six.PY3: retrieved_data = retrieved_data.decode('utf-8') - self.assertEqual(data, retrieved_data.strip()) + assert data == retrieved_data.strip() def test_get_file_stat_from_container(self): data = 'The Maid and the Pocket Watch of Blood' @@ -671,10 +669,10 @@ def test_get_file_stat_from_container(self): self.client.start(ctnr) self.client.wait(ctnr) strm, stat = self.client.get_archive(ctnr, '/vol1/data.txt') - self.assertIn('name', stat) - self.assertEqual(stat['name'], 'data.txt') - self.assertIn('size', stat) - self.assertEqual(stat['size'], len(data)) + assert 'name' in stat + assert stat['name'] == 'data.txt' + assert 'size' in stat + assert stat['size'] == len(data) def test_copy_file_to_container(self): data = b'Deaf To All But The Song' @@ -697,7 +695,7 @@ def test_copy_file_to_container(self): if six.PY3: logs = logs.decode('utf-8') data = data.decode('utf-8') - self.assertEqual(logs.strip(), data) + assert logs.strip() == data def test_copy_directory_to_container(self): files = ['a.py', 'b.py', 'foo/b.py'] @@ -715,10 +713,10 @@ def test_copy_directory_to_container(self): if six.PY3: logs = logs.decode('utf-8') results = logs.strip().split() - self.assertIn('a.py', results) - self.assertIn('b.py', results) - self.assertIn('foo/', results) - self.assertIn('bar/', results) + assert 'a.py' in results + assert 'b.py' in results + assert 'foo/' in results + assert 'bar/' in results class RenameContainerTest(BaseAPIIntegrationTest): @@ -726,49 +724,49 @@ def test_rename_container(self): version = self.client.version()['Version'] name = 'hong_meiling' res = self.client.create_container(BUSYBOX, 'true') - self.assertIn('Id', res) + assert 'Id' in res self.tmp_containers.append(res['Id']) self.client.rename(res, name) inspect = self.client.inspect_container(res['Id']) - self.assertIn('Name', inspect) + assert 'Name' in inspect if version == '1.5.0': - self.assertEqual(name, inspect['Name']) + assert name == inspect['Name'] else: - self.assertEqual('/{0}'.format(name), inspect['Name']) + assert '/{0}'.format(name) == inspect['Name'] class StartContainerTest(BaseAPIIntegrationTest): def test_start_container(self): res = self.client.create_container(BUSYBOX, 'true') - self.assertIn('Id', res) + assert 'Id' in res self.tmp_containers.append(res['Id']) self.client.start(res['Id']) inspect = self.client.inspect_container(res['Id']) - self.assertIn('Config', inspect) - self.assertIn('Id', inspect) - self.assertTrue(inspect['Id'].startswith(res['Id'])) - self.assertIn('Image', inspect) - self.assertIn('State', inspect) - self.assertIn('Running', inspect['State']) + assert 'Config' in inspect + assert 'Id' in inspect + assert inspect['Id'].startswith(res['Id']) + assert 'Image' in inspect + assert 'State' in inspect + assert 'Running' in inspect['State'] if not inspect['State']['Running']: - self.assertIn('ExitCode', inspect['State']) - self.assertEqual(inspect['State']['ExitCode'], 0) + assert 'ExitCode' in inspect['State'] + assert inspect['State']['ExitCode'] == 0 def test_start_container_with_dict_instead_of_id(self): res = self.client.create_container(BUSYBOX, 'true') - self.assertIn('Id', res) + assert 'Id' in res self.tmp_containers.append(res['Id']) self.client.start(res) inspect = self.client.inspect_container(res['Id']) - self.assertIn('Config', inspect) - self.assertIn('Id', inspect) - self.assertTrue(inspect['Id'].startswith(res['Id'])) - self.assertIn('Image', inspect) - self.assertIn('State', inspect) - self.assertIn('Running', inspect['State']) + assert 'Config' in inspect + assert 'Id' in inspect + assert inspect['Id'].startswith(res['Id']) + assert 'Image' in inspect + assert 'State' in inspect + assert 'Running' in inspect['State'] if not inspect['State']['Running']: - self.assertIn('ExitCode', inspect['State']) - self.assertEqual(inspect['State']['ExitCode'], 0) + assert 'ExitCode' in inspect['State'] + assert inspect['State']['ExitCode'] == 0 def test_run_shlex_commands(self): commands = [ @@ -788,7 +786,7 @@ def test_run_shlex_commands(self): self.client.start(id) self.tmp_containers.append(id) exitcode = self.client.wait(id) - self.assertEqual(exitcode, 0, msg=cmd) + assert exitcode == 0, cmd class WaitTest(BaseAPIIntegrationTest): @@ -798,12 +796,12 @@ def test_wait(self): self.tmp_containers.append(id) self.client.start(id) exitcode = self.client.wait(id) - self.assertEqual(exitcode, 0) + assert exitcode == 0 inspect = self.client.inspect_container(id) - self.assertIn('Running', inspect['State']) - self.assertEqual(inspect['State']['Running'], False) - self.assertIn('ExitCode', inspect['State']) - self.assertEqual(inspect['State']['ExitCode'], exitcode) + assert 'Running' in inspect['State'] + assert inspect['State']['Running'] is False + assert 'ExitCode' in inspect['State'] + assert inspect['State']['ExitCode'] == exitcode def test_wait_with_dict_instead_of_id(self): res = self.client.create_container(BUSYBOX, ['sleep', '3']) @@ -811,12 +809,12 @@ def test_wait_with_dict_instead_of_id(self): self.tmp_containers.append(id) self.client.start(res) exitcode = self.client.wait(res) - self.assertEqual(exitcode, 0) + assert exitcode == 0 inspect = self.client.inspect_container(res) - self.assertIn('Running', inspect['State']) - self.assertEqual(inspect['State']['Running'], False) - self.assertIn('ExitCode', inspect['State']) - self.assertEqual(inspect['State']['ExitCode'], exitcode) + assert 'Running' in inspect['State'] + assert inspect['State']['Running'] is False + assert 'ExitCode' in inspect['State'] + assert inspect['State']['ExitCode'] == exitcode @requires_api_version('1.30') def test_wait_with_condition(self): @@ -844,9 +842,9 @@ def test_logs(self): self.tmp_containers.append(id) self.client.start(id) exitcode = self.client.wait(id) - self.assertEqual(exitcode, 0) + assert exitcode == 0 logs = self.client.logs(id) - self.assertEqual(logs, (snippet + '\n').encode(encoding='ascii')) + assert logs == (snippet + '\n').encode(encoding='ascii') def test_logs_tail_option(self): snippet = '''Line1 @@ -858,9 +856,9 @@ def test_logs_tail_option(self): self.tmp_containers.append(id) self.client.start(id) exitcode = self.client.wait(id) - self.assertEqual(exitcode, 0) + assert exitcode == 0 logs = self.client.logs(id, tail=1) - self.assertEqual(logs, 'Line2\n'.encode(encoding='ascii')) + assert logs == 'Line2\n'.encode(encoding='ascii') def test_logs_streaming_and_follow(self): snippet = 'Flowering Nights (Sakuya Iyazoi)' @@ -875,9 +873,9 @@ def test_logs_streaming_and_follow(self): logs += chunk exitcode = self.client.wait(id) - self.assertEqual(exitcode, 0) + assert exitcode == 0 - self.assertEqual(logs, (snippet + '\n').encode(encoding='ascii')) + assert logs == (snippet + '\n').encode(encoding='ascii') def test_logs_with_dict_instead_of_id(self): snippet = 'Flowering Nights (Sakuya Iyazoi)' @@ -888,9 +886,9 @@ def test_logs_with_dict_instead_of_id(self): self.tmp_containers.append(id) self.client.start(id) exitcode = self.client.wait(id) - self.assertEqual(exitcode, 0) + assert exitcode == 0 logs = self.client.logs(container) - self.assertEqual(logs, (snippet + '\n').encode(encoding='ascii')) + assert logs == (snippet + '\n').encode(encoding='ascii') def test_logs_with_tail_0(self): snippet = 'Flowering Nights (Sakuya Iyazoi)' @@ -901,9 +899,9 @@ def test_logs_with_tail_0(self): self.tmp_containers.append(id) self.client.start(id) exitcode = self.client.wait(id) - self.assertEqual(exitcode, 0) + assert exitcode == 0 logs = self.client.logs(id, tail=0) - self.assertEqual(logs, ''.encode(encoding='ascii')) + assert logs == ''.encode(encoding='ascii') @requires_api_version('1.35') def test_logs_with_until(self): @@ -929,12 +927,12 @@ def test_diff(self): self.client.start(id) self.tmp_containers.append(id) exitcode = self.client.wait(id) - self.assertEqual(exitcode, 0) + assert exitcode == 0 diff = self.client.diff(id) test_diff = [x for x in diff if x.get('Path', None) == '/test'] - self.assertEqual(len(test_diff), 1) - self.assertIn('Kind', test_diff[0]) - self.assertEqual(test_diff[0]['Kind'], 1) + assert len(test_diff) == 1 + assert 'Kind' in test_diff[0] + assert test_diff[0]['Kind'] == 1 def test_diff_with_dict_instead_of_id(self): container = self.client.create_container(BUSYBOX, ['touch', '/test']) @@ -942,12 +940,12 @@ def test_diff_with_dict_instead_of_id(self): self.client.start(id) self.tmp_containers.append(id) exitcode = self.client.wait(id) - self.assertEqual(exitcode, 0) + assert exitcode == 0 diff = self.client.diff(container) test_diff = [x for x in diff if x.get('Path', None) == '/test'] - self.assertEqual(len(test_diff), 1) - self.assertIn('Kind', test_diff[0]) - self.assertEqual(test_diff[0]['Kind'], 1) + assert len(test_diff) == 1 + assert 'Kind' in test_diff[0] + assert test_diff[0]['Kind'] == 1 class StopTest(BaseAPIIntegrationTest): @@ -958,23 +956,23 @@ def test_stop(self): self.tmp_containers.append(id) self.client.stop(id, timeout=2) container_info = self.client.inspect_container(id) - self.assertIn('State', container_info) + assert 'State' in container_info state = container_info['State'] - self.assertIn('Running', state) - self.assertEqual(state['Running'], False) + assert 'Running' in state + assert state['Running'] is False def test_stop_with_dict_instead_of_id(self): container = self.client.create_container(BUSYBOX, ['sleep', '9999']) - self.assertIn('Id', container) + assert 'Id' in container id = container['Id'] self.client.start(container) self.tmp_containers.append(id) self.client.stop(container, timeout=2) container_info = self.client.inspect_container(id) - self.assertIn('State', container_info) + assert 'State' in container_info state = container_info['State'] - self.assertIn('Running', state) - self.assertEqual(state['Running'], False) + assert 'Running' in state + assert state['Running'] is False class KillTest(BaseAPIIntegrationTest): @@ -985,12 +983,12 @@ def test_kill(self): self.tmp_containers.append(id) self.client.kill(id) container_info = self.client.inspect_container(id) - self.assertIn('State', container_info) + assert 'State' in container_info state = container_info['State'] - self.assertIn('ExitCode', state) - self.assertNotEqual(state['ExitCode'], 0) - self.assertIn('Running', state) - self.assertEqual(state['Running'], False) + assert 'ExitCode' in state + assert state['ExitCode'] != 0 + assert 'Running' in state + assert state['Running'] is False def test_kill_with_dict_instead_of_id(self): container = self.client.create_container(BUSYBOX, ['sleep', '9999']) @@ -999,12 +997,12 @@ def test_kill_with_dict_instead_of_id(self): self.tmp_containers.append(id) self.client.kill(container) container_info = self.client.inspect_container(id) - self.assertIn('State', container_info) + assert 'State' in container_info state = container_info['State'] - self.assertIn('ExitCode', state) - self.assertNotEqual(state['ExitCode'], 0) - self.assertIn('Running', state) - self.assertEqual(state['Running'], False) + assert 'ExitCode' in state + assert state['ExitCode'] != 0 + assert 'Running' in state + assert state['Running'] is False def test_kill_with_signal(self): id = self.client.create_container(BUSYBOX, ['sleep', '60']) @@ -1014,14 +1012,14 @@ def test_kill_with_signal(self): id, signal=signal.SIGKILL if not IS_WINDOWS_PLATFORM else 9 ) exitcode = self.client.wait(id) - self.assertNotEqual(exitcode, 0) + assert exitcode != 0 container_info = self.client.inspect_container(id) - self.assertIn('State', container_info) + assert 'State' in container_info state = container_info['State'] - self.assertIn('ExitCode', state) - self.assertNotEqual(state['ExitCode'], 0) - self.assertIn('Running', state) - self.assertEqual(state['Running'], False, state) + assert 'ExitCode' in state + assert state['ExitCode'] != 0 + assert 'Running' in state + assert state['Running'] is False, state def test_kill_with_signal_name(self): id = self.client.create_container(BUSYBOX, ['sleep', '60']) @@ -1029,14 +1027,14 @@ def test_kill_with_signal_name(self): self.tmp_containers.append(id) self.client.kill(id, signal='SIGKILL') exitcode = self.client.wait(id) - self.assertNotEqual(exitcode, 0) + assert exitcode != 0 container_info = self.client.inspect_container(id) - self.assertIn('State', container_info) + assert 'State' in container_info state = container_info['State'] - self.assertIn('ExitCode', state) - self.assertNotEqual(state['ExitCode'], 0) - self.assertIn('Running', state) - self.assertEqual(state['Running'], False, state) + assert 'ExitCode' in state + assert state['ExitCode'] != 0 + assert 'Running' in state + assert state['Running'] is False, state def test_kill_with_signal_integer(self): id = self.client.create_container(BUSYBOX, ['sleep', '60']) @@ -1044,14 +1042,14 @@ def test_kill_with_signal_integer(self): self.tmp_containers.append(id) self.client.kill(id, signal=9) exitcode = self.client.wait(id) - self.assertNotEqual(exitcode, 0) + assert exitcode != 0 container_info = self.client.inspect_container(id) - self.assertIn('State', container_info) + assert 'State' in container_info state = container_info['State'] - self.assertIn('ExitCode', state) - self.assertNotEqual(state['ExitCode'], 0) - self.assertIn('Running', state) - self.assertEqual(state['Running'], False, state) + assert 'ExitCode' in state + assert state['ExitCode'] != 0 + assert 'Running' in state + assert state['Running'] is False, state class PortTest(BaseAPIIntegrationTest): @@ -1079,8 +1077,8 @@ def test_port(self): ip, host_port = port_binding['HostIp'], port_binding['HostPort'] - self.assertEqual(ip, port_bindings[port][0]) - self.assertEqual(host_port, port_bindings[port][1]) + assert ip == port_bindings[port][0] + assert host_port == port_bindings[port][1] self.client.kill(id) @@ -1116,13 +1114,12 @@ def test_top_with_psargs(self): self.client.start(container) res = self.client.top(container, 'waux') - self.assertEqual( - res['Titles'], - ['USER', 'PID', '%CPU', '%MEM', 'VSZ', 'RSS', - 'TTY', 'STAT', 'START', 'TIME', 'COMMAND'], - ) - self.assertEqual(len(res['Processes']), 1) - self.assertEqual(res['Processes'][0][10], 'sleep 60') + assert res['Titles'] == [ + 'USER', 'PID', '%CPU', '%MEM', 'VSZ', 'RSS', + 'TTY', 'STAT', 'START', 'TIME', 'COMMAND' + ] + assert len(res['Processes']) == 1 + assert res['Processes'][0][10] == 'sleep 60' class RestartContainerTest(BaseAPIIntegrationTest): @@ -1132,37 +1129,37 @@ def test_restart(self): self.client.start(id) self.tmp_containers.append(id) info = self.client.inspect_container(id) - self.assertIn('State', info) - self.assertIn('StartedAt', info['State']) + assert 'State' in info + assert 'StartedAt' in info['State'] start_time1 = info['State']['StartedAt'] self.client.restart(id, timeout=2) info2 = self.client.inspect_container(id) - self.assertIn('State', info2) - self.assertIn('StartedAt', info2['State']) + assert 'State' in info2 + assert 'StartedAt' in info2['State'] start_time2 = info2['State']['StartedAt'] - self.assertNotEqual(start_time1, start_time2) - self.assertIn('Running', info2['State']) - self.assertEqual(info2['State']['Running'], True) + assert start_time1 != start_time2 + assert 'Running' in info2['State'] + assert info2['State']['Running'] is True self.client.kill(id) def test_restart_with_dict_instead_of_id(self): container = self.client.create_container(BUSYBOX, ['sleep', '9999']) - self.assertIn('Id', container) + assert 'Id' in container id = container['Id'] self.client.start(container) self.tmp_containers.append(id) info = self.client.inspect_container(id) - self.assertIn('State', info) - self.assertIn('StartedAt', info['State']) + assert 'State' in info + assert 'StartedAt' in info['State'] start_time1 = info['State']['StartedAt'] self.client.restart(container, timeout=2) info2 = self.client.inspect_container(id) - self.assertIn('State', info2) - self.assertIn('StartedAt', info2['State']) + assert 'State' in info2 + assert 'StartedAt' in info2['State'] start_time2 = info2['State']['StartedAt'] - self.assertNotEqual(start_time1, start_time2) - self.assertIn('Running', info2['State']) - self.assertEqual(info2['State']['Running'], True) + assert start_time1 != start_time2 + assert 'Running' in info2['State'] + assert info2['State']['Running'] is True self.client.kill(id) @@ -1175,7 +1172,7 @@ def test_remove(self): self.client.remove_container(id) containers = self.client.containers(all=True) res = [x for x in containers if 'Id' in x and x['Id'].startswith(id)] - self.assertEqual(len(res), 0) + assert len(res) == 0 def test_remove_with_dict_instead_of_id(self): container = self.client.create_container(BUSYBOX, ['true']) @@ -1185,7 +1182,7 @@ def test_remove_with_dict_instead_of_id(self): self.client.remove_container(container) containers = self.client.containers(all=True) res = [x for x in containers if 'Id' in x and x['Id'].startswith(id)] - self.assertEqual(len(res), 0) + assert len(res) == 0 class AttachContainerTest(BaseAPIIntegrationTest): @@ -1196,7 +1193,7 @@ def test_run_container_streaming(self): self.tmp_containers.append(id) self.client.start(id) sock = self.client.attach_socket(container, ws=False) - self.assertTrue(sock.fileno() > -1) + assert sock.fileno() > -1 def test_run_container_reading_socket(self): line = 'hi there and stuff and things, words!' @@ -1213,9 +1210,9 @@ def test_run_container_reading_socket(self): self.client.start(container) next_size = next_frame_size(pty_stdout) - self.assertEqual(next_size, len(line)) + assert next_size == len(line) data = read_exactly(pty_stdout, next_size) - self.assertEqual(data.decode('utf-8'), line) + assert data.decode('utf-8') == line def test_attach_no_stream(self): container = self.client.create_container( @@ -1235,25 +1232,25 @@ def test_pause_unpause(self): self.client.start(container) self.client.pause(id) container_info = self.client.inspect_container(id) - self.assertIn('State', container_info) + assert 'State' in container_info state = container_info['State'] - self.assertIn('ExitCode', state) - self.assertEqual(state['ExitCode'], 0) - self.assertIn('Running', state) - self.assertEqual(state['Running'], True) - self.assertIn('Paused', state) - self.assertEqual(state['Paused'], True) + assert 'ExitCode' in state + assert state['ExitCode'] == 0 + assert 'Running' in state + assert state['Running'] is True + assert 'Paused' in state + assert state['Paused'] is True self.client.unpause(id) container_info = self.client.inspect_container(id) - self.assertIn('State', container_info) + assert 'State' in container_info state = container_info['State'] - self.assertIn('ExitCode', state) - self.assertEqual(state['ExitCode'], 0) - self.assertIn('Running', state) - self.assertEqual(state['Running'], True) - self.assertIn('Paused', state) - self.assertEqual(state['Paused'], False) + assert 'ExitCode' in state + assert state['ExitCode'] == 0 + assert 'Running' in state + assert state['Running'] is True + assert 'Paused' in state + assert state['Paused'] is False class PruneTest(BaseAPIIntegrationTest): @@ -1283,10 +1280,10 @@ def test_get_container_stats_no_stream(self): response = self.client.stats(container, stream=0) self.client.kill(container) - self.assertEqual(type(response), dict) + assert type(response) == dict for key in ['read', 'networks', 'precpu_stats', 'cpu_stats', 'memory_stats', 'blkio_stats']: - self.assertIn(key, response) + assert key in response @requires_api_version('1.17') def test_get_container_stats_stream(self): @@ -1297,10 +1294,10 @@ def test_get_container_stats_stream(self): self.client.start(container) stream = self.client.stats(container) for chunk in stream: - self.assertEqual(type(chunk), dict) + assert type(chunk) == dict for key in ['read', 'network', 'precpu_stats', 'cpu_stats', 'memory_stats', 'blkio_stats']: - self.assertIn(key, chunk) + assert key in chunk class ContainerUpdateTest(BaseAPIIntegrationTest): @@ -1317,7 +1314,7 @@ def test_update_container(self): self.client.start(container) self.client.update_container(container, mem_limit=new_mem_limit) inspect_data = self.client.inspect_container(container) - self.assertEqual(inspect_data['HostConfig']['Memory'], new_mem_limit) + assert inspect_data['HostConfig']['Memory'] == new_mem_limit @requires_api_version('1.23') def test_restart_policy_update(self): @@ -1340,12 +1337,12 @@ def test_restart_policy_update(self): self.client.update_container(container, restart_policy=new_restart_policy) inspect_data = self.client.inspect_container(container) - self.assertEqual( - inspect_data['HostConfig']['RestartPolicy']['MaximumRetryCount'], + assert ( + inspect_data['HostConfig']['RestartPolicy']['MaximumRetryCount'] == new_restart_policy['MaximumRetryCount'] ) - self.assertEqual( - inspect_data['HostConfig']['RestartPolicy']['Name'], + assert ( + inspect_data['HostConfig']['RestartPolicy']['Name'] == new_restart_policy['Name'] ) @@ -1362,7 +1359,7 @@ def test_container_cpu_shares(self): self.tmp_containers.append(container) self.client.start(container) inspect_data = self.client.inspect_container(container) - self.assertEqual(inspect_data['HostConfig']['CpuShares'], 512) + assert inspect_data['HostConfig']['CpuShares'] == 512 @requires_api_version('1.18') def test_container_cpuset(self): @@ -1375,7 +1372,7 @@ def test_container_cpuset(self): self.tmp_containers.append(container) self.client.start(container) inspect_data = self.client.inspect_container(container) - self.assertEqual(inspect_data['HostConfig']['CpusetCpus'], cpuset_cpus) + assert inspect_data['HostConfig']['CpusetCpus'] == cpuset_cpus @requires_api_version('1.25') def test_create_with_runtime(self): @@ -1419,11 +1416,11 @@ def test_remove_link(self): # Link is gone containers = self.client.containers(all=True) retrieved = [x for x in containers if link_name in x['Names']] - self.assertEqual(len(retrieved), 0) + assert len(retrieved) == 0 # Containers are still there retrieved = [ x for x in containers if x['Id'].startswith(container1_id) or x['Id'].startswith(container2_id) ] - self.assertEqual(len(retrieved), 2) + assert len(retrieved) == 2 diff --git a/tests/integration/api_exec_test.py b/tests/integration/api_exec_test.py index 0d42e19ac6..cd97c68b5b 100644 --- a/tests/integration/api_exec_test.py +++ b/tests/integration/api_exec_test.py @@ -14,10 +14,10 @@ def test_execute_command(self): self.tmp_containers.append(id) res = self.client.exec_create(id, ['echo', 'hello']) - self.assertIn('Id', res) + assert 'Id' in res exec_log = self.client.exec_start(res) - self.assertEqual(exec_log, b'hello\n') + assert exec_log == b'hello\n' def test_exec_command_string(self): container = self.client.create_container(BUSYBOX, 'cat', @@ -27,10 +27,10 @@ def test_exec_command_string(self): self.tmp_containers.append(id) res = self.client.exec_create(id, 'echo hello world') - self.assertIn('Id', res) + assert 'Id' in res exec_log = self.client.exec_start(res) - self.assertEqual(exec_log, b'hello world\n') + assert exec_log == b'hello world\n' def test_exec_command_as_user(self): container = self.client.create_container(BUSYBOX, 'cat', @@ -40,10 +40,10 @@ def test_exec_command_as_user(self): self.tmp_containers.append(id) res = self.client.exec_create(id, 'whoami', user='default') - self.assertIn('Id', res) + assert 'Id' in res exec_log = self.client.exec_start(res) - self.assertEqual(exec_log, b'default\n') + assert exec_log == b'default\n' def test_exec_command_as_root(self): container = self.client.create_container(BUSYBOX, 'cat', @@ -53,10 +53,10 @@ def test_exec_command_as_root(self): self.tmp_containers.append(id) res = self.client.exec_create(id, 'whoami') - self.assertIn('Id', res) + assert 'Id' in res exec_log = self.client.exec_start(res) - self.assertEqual(exec_log, b'root\n') + assert exec_log == b'root\n' def test_exec_command_streaming(self): container = self.client.create_container(BUSYBOX, 'cat', @@ -66,12 +66,12 @@ def test_exec_command_streaming(self): self.client.start(id) exec_id = self.client.exec_create(id, ['echo', 'hello\nworld']) - self.assertIn('Id', exec_id) + assert 'Id' in exec_id res = b'' for chunk in self.client.exec_start(exec_id, stream=True): res += chunk - self.assertEqual(res, b'hello\nworld\n') + assert res == b'hello\nworld\n' def test_exec_start_socket(self): container = self.client.create_container(BUSYBOX, 'cat', @@ -84,15 +84,15 @@ def test_exec_start_socket(self): # `echo` appends CRLF, `printf` doesn't exec_id = self.client.exec_create( container_id, ['printf', line], tty=True) - self.assertIn('Id', exec_id) + assert 'Id' in exec_id socket = self.client.exec_start(exec_id, socket=True) self.addCleanup(socket.close) next_size = next_frame_size(socket) - self.assertEqual(next_size, len(line)) + assert next_size == len(line) data = read_exactly(socket, next_size) - self.assertEqual(data.decode('utf-8'), line) + assert data.decode('utf-8') == line def test_exec_start_detached(self): container = self.client.create_container(BUSYBOX, 'cat', @@ -103,11 +103,11 @@ def test_exec_start_detached(self): exec_id = self.client.exec_create( container_id, ['printf', "asdqwe"]) - self.assertIn('Id', exec_id) + assert 'Id' in exec_id response = self.client.exec_start(exec_id, detach=True) - self.assertEqual(response, "") + assert response == "" def test_exec_inspect(self): container = self.client.create_container(BUSYBOX, 'cat', @@ -117,11 +117,11 @@ def test_exec_inspect(self): self.tmp_containers.append(id) exec_id = self.client.exec_create(id, ['mkdir', '/does/not/exist']) - self.assertIn('Id', exec_id) + assert 'Id' in exec_id self.client.exec_start(exec_id) exec_info = self.client.exec_inspect(exec_id) - self.assertIn('ExitCode', exec_info) - self.assertNotEqual(exec_info['ExitCode'], 0) + assert 'ExitCode' in exec_info + assert exec_info['ExitCode'] != 0 @requires_api_version('1.25') def test_exec_command_with_env(self): diff --git a/tests/integration/api_healthcheck_test.py b/tests/integration/api_healthcheck_test.py index 211042d486..5dbac3769f 100644 --- a/tests/integration/api_healthcheck_test.py +++ b/tests/integration/api_healthcheck_test.py @@ -20,8 +20,9 @@ def test_healthcheck_shell_command(self): self.tmp_containers.append(container) res = self.client.inspect_container(container) - assert res['Config']['Healthcheck']['Test'] == \ - ['CMD-SHELL', 'echo "hello world"'] + assert res['Config']['Healthcheck']['Test'] == [ + 'CMD-SHELL', 'echo "hello world"' + ] @helpers.requires_api_version('1.24') def test_healthcheck_passes(self): diff --git a/tests/integration/api_image_test.py b/tests/integration/api_image_test.py index ae93190e52..ab638c9e46 100644 --- a/tests/integration/api_image_test.py +++ b/tests/integration/api_image_test.py @@ -21,19 +21,19 @@ class ListImagesTest(BaseAPIIntegrationTest): def test_images(self): res1 = self.client.images(all=True) - self.assertIn('Id', res1[0]) + assert 'Id' in res1[0] res10 = res1[0] - self.assertIn('Created', res10) - self.assertIn('RepoTags', res10) + assert 'Created' in res10 + assert 'RepoTags' in res10 distinct = [] for img in res1: if img['Id'] not in distinct: distinct.append(img['Id']) - self.assertEqual(len(distinct), self.client.info()['Images']) + assert len(distinct) == self.client.info()['Images'] def test_images_quiet(self): res1 = self.client.images(quiet=True) - self.assertEqual(type(res1[0]), six.text_type) + assert type(res1[0]) == six.text_type class PullImageTest(BaseAPIIntegrationTest): @@ -44,12 +44,10 @@ def test_pull(self): pass res = self.client.pull('hello-world', tag='latest') self.tmp_imgs.append('hello-world') - self.assertEqual(type(res), six.text_type) - self.assertGreaterEqual( - len(self.client.images('hello-world')), 1 - ) + assert type(res) == six.text_type + assert len(self.client.images('hello-world')) >= 1 img_info = self.client.inspect_image('hello-world') - self.assertIn('Id', img_info) + assert 'Id' in img_info def test_pull_streaming(self): try: @@ -61,11 +59,9 @@ def test_pull_streaming(self): self.tmp_imgs.append('hello-world') for chunk in stream: assert isinstance(chunk, dict) - self.assertGreaterEqual( - len(self.client.images('hello-world')), 1 - ) + assert len(self.client.images('hello-world')) >= 1 img_info = self.client.inspect_image('hello-world') - self.assertIn('Id', img_info) + assert 'Id' in img_info @requires_api_version('1.32') @requires_experimental(until=None) @@ -84,18 +80,18 @@ def test_commit(self): self.client.start(id) self.tmp_containers.append(id) res = self.client.commit(id) - self.assertIn('Id', res) + assert 'Id' in res img_id = res['Id'] self.tmp_imgs.append(img_id) img = self.client.inspect_image(img_id) - self.assertIn('Container', img) - self.assertTrue(img['Container'].startswith(id)) - self.assertIn('ContainerConfig', img) - self.assertIn('Image', img['ContainerConfig']) - self.assertEqual(BUSYBOX, img['ContainerConfig']['Image']) + assert 'Container' in img + assert img['Container'].startswith(id) + assert 'ContainerConfig' in img + assert 'Image' in img['ContainerConfig'] + assert BUSYBOX == img['ContainerConfig']['Image'] busybox_id = self.client.inspect_image(BUSYBOX)['Id'] - self.assertIn('Parent', img) - self.assertEqual(img['Parent'], busybox_id) + assert 'Parent' in img + assert img['Parent'] == busybox_id def test_commit_with_changes(self): cid = self.client.create_container(BUSYBOX, ['touch', '/test']) @@ -119,14 +115,14 @@ def test_remove(self): self.client.start(id) self.tmp_containers.append(id) res = self.client.commit(id) - self.assertIn('Id', res) + assert 'Id' in res img_id = res['Id'] self.tmp_imgs.append(img_id) logs = self.client.remove_image(img_id, force=True) - self.assertIn({"Deleted": img_id}, logs) + assert {"Deleted": img_id} in logs images = self.client.images(all=True) res = [x for x in images if x['Id'].startswith(img_id)] - self.assertEqual(len(res), 0) + assert len(res) == 0 class ImportImageTest(BaseAPIIntegrationTest): @@ -180,7 +176,7 @@ def test_import_from_bytes(self): result_text = statuses.splitlines()[-1] result = json.loads(result_text) - self.assertNotIn('error', result) + assert 'error' not in result img_id = result['status'] self.tmp_imgs.append(img_id) @@ -195,9 +191,9 @@ def test_import_from_file(self): result_text = statuses.splitlines()[-1] result = json.loads(result_text) - self.assertNotIn('error', result) + assert 'error' not in result - self.assertIn('status', result) + assert 'status' in result img_id = result['status'] self.tmp_imgs.append(img_id) @@ -210,9 +206,9 @@ def test_import_from_stream(self): result_text = statuses.splitlines()[-1] result = json.loads(result_text) - self.assertNotIn('error', result) + assert 'error' not in result - self.assertIn('status', result) + assert 'status' in result img_id = result['status'] self.tmp_imgs.append(img_id) @@ -305,9 +301,9 @@ def test_import_from_url(self): result_text = statuses.splitlines()[-1] result = json.loads(result_text) - self.assertNotIn('error', result) + assert 'error' not in result - self.assertIn('status', result) + assert 'status' in result img_id = result['status'] self.tmp_imgs.append(img_id) diff --git a/tests/integration/api_network_test.py b/tests/integration/api_network_test.py index 10e09dd70d..ec92bd7956 100644 --- a/tests/integration/api_network_test.py +++ b/tests/integration/api_network_test.py @@ -24,24 +24,24 @@ def test_list_networks(self): net_name, net_id = self.create_network() networks = self.client.networks() - self.assertTrue(net_id in [n['Id'] for n in networks]) + assert net_id in [n['Id'] for n in networks] networks_by_name = self.client.networks(names=[net_name]) - self.assertEqual([n['Id'] for n in networks_by_name], [net_id]) + assert [n['Id'] for n in networks_by_name] == [net_id] networks_by_partial_id = self.client.networks(ids=[net_id[:8]]) - self.assertEqual([n['Id'] for n in networks_by_partial_id], [net_id]) + assert [n['Id'] for n in networks_by_partial_id] == [net_id] @requires_api_version('1.21') def test_inspect_network(self): net_name, net_id = self.create_network() net = self.client.inspect_network(net_id) - self.assertEqual(net['Id'], net_id) - self.assertEqual(net['Name'], net_name) - self.assertEqual(net['Driver'], 'bridge') - self.assertEqual(net['Scope'], 'local') - self.assertEqual(net['IPAM']['Driver'], 'default') + assert net['Id'] == net_id + assert net['Name'] == net_name + assert net['Driver'] == 'bridge' + assert net['Scope'] == 'local' + assert net['IPAM']['Driver'] == 'default' @requires_api_version('1.21') def test_create_network_with_ipam_config(self): @@ -103,21 +103,20 @@ def test_connect_and_disconnect_container(self): self.client.start(container) network_data = self.client.inspect_network(net_id) - self.assertFalse(network_data.get('Containers')) + assert not network_data.get('Containers') self.client.connect_container_to_network(container, net_id) network_data = self.client.inspect_network(net_id) - self.assertEqual( - list(network_data['Containers'].keys()), - [container['Id']] - ) + assert list(network_data['Containers'].keys()) == [ + container['Id'] + ] with pytest.raises(docker.errors.APIError): self.client.connect_container_to_network(container, net_id) self.client.disconnect_container_from_network(container, net_id) network_data = self.client.inspect_network(net_id) - self.assertFalse(network_data.get('Containers')) + assert not network_data.get('Containers') with pytest.raises(docker.errors.APIError): self.client.disconnect_container_from_network(container, net_id) @@ -131,18 +130,16 @@ def test_connect_and_force_disconnect_container(self): self.client.start(container) network_data = self.client.inspect_network(net_id) - self.assertFalse(network_data.get('Containers')) + assert not network_data.get('Containers') self.client.connect_container_to_network(container, net_id) network_data = self.client.inspect_network(net_id) - self.assertEqual( - list(network_data['Containers'].keys()), + assert list(network_data['Containers'].keys()) == \ [container['Id']] - ) self.client.disconnect_container_from_network(container, net_id, True) network_data = self.client.inspect_network(net_id) - self.assertFalse(network_data.get('Containers')) + assert not network_data.get('Containers') with pytest.raises(docker.errors.APIError): self.client.disconnect_container_from_network( @@ -179,13 +176,12 @@ def test_connect_on_container_create(self): self.client.start(container) network_data = self.client.inspect_network(net_id) - self.assertEqual( - list(network_data['Containers'].keys()), - [container['Id']]) + assert list(network_data['Containers'].keys()) == \ + [container['Id']] self.client.disconnect_container_from_network(container, net_id) network_data = self.client.inspect_network(net_id) - self.assertFalse(network_data.get('Containers')) + assert not network_data.get('Containers') @requires_api_version('1.22') def test_create_with_aliases(self): @@ -233,14 +229,11 @@ def test_create_with_ipv4_address(self): self.tmp_containers.append(container) self.client.start(container) - container_data = self.client.inspect_container(container) - self.assertEqual( - container_data[ - 'NetworkSettings']['Networks'][net_name]['IPAMConfig'][ - 'IPv4Address' - ], - '132.124.0.23' - ) + net_settings = self.client.inspect_container(container)[ + 'NetworkSettings' + ] + assert net_settings['Networks'][net_name]['IPAMConfig']['IPv4Address']\ + == '132.124.0.23' @requires_api_version('1.22') def test_create_with_ipv6_address(self): @@ -262,14 +255,11 @@ def test_create_with_ipv6_address(self): self.tmp_containers.append(container) self.client.start(container) - container_data = self.client.inspect_container(container) - self.assertEqual( - container_data[ - 'NetworkSettings']['Networks'][net_name]['IPAMConfig'][ - 'IPv6Address' - ], - '2001:389::f00d' - ) + net_settings = self.client.inspect_container(container)[ + 'NetworkSettings' + ] + assert net_settings['Networks'][net_name]['IPAMConfig']['IPv6Address']\ + == '2001:389::f00d' @requires_api_version('1.24') def test_create_with_linklocal_ips(self): @@ -305,10 +295,12 @@ def test_create_with_links(self): }), ) - container_data = self.client.inspect_container(container) - self.assertEqual( - container_data['NetworkSettings']['Networks'][net_name]['Links'], - ['docker-py-test-upstream:bar']) + net_settings = self.client.inspect_container(container)[ + 'NetworkSettings' + ] + assert net_settings['Networks'][net_name]['Links'] == [ + 'docker-py-test-upstream:bar' + ] self.create_and_start( name='docker-py-test-upstream', @@ -320,7 +312,7 @@ def test_create_with_links(self): @requires_api_version('1.21') def test_create_check_duplicate(self): net_name, net_id = self.create_network() - with self.assertRaises(docker.errors.APIError): + with pytest.raises(docker.errors.APIError): self.client.create_network(net_name, check_duplicate=True) net_id = self.client.create_network(net_name, check_duplicate=False) self.tmp_networks.append(net_id['Id']) @@ -337,10 +329,12 @@ def test_connect_with_links(self): container, net_name, links=[('docker-py-test-upstream', 'bar')]) - container_data = self.client.inspect_container(container) - self.assertEqual( - container_data['NetworkSettings']['Networks'][net_name]['Links'], - ['docker-py-test-upstream:bar']) + net_settings = self.client.inspect_container(container)[ + 'NetworkSettings' + ] + assert net_settings['Networks'][net_name]['Links'] == [ + 'docker-py-test-upstream:bar' + ] self.create_and_start( name='docker-py-test-upstream', @@ -373,9 +367,7 @@ def test_connect_with_ipv4_address(self): container_data = self.client.inspect_container(container) net_data = container_data['NetworkSettings']['Networks'][net_name] - self.assertEqual( - net_data['IPAMConfig']['IPv4Address'], '172.28.5.24' - ) + assert net_data['IPAMConfig']['IPv4Address'] == '172.28.5.24' @requires_api_version('1.22') def test_connect_with_ipv6_address(self): @@ -401,9 +393,7 @@ def test_connect_with_ipv6_address(self): container_data = self.client.inspect_container(container) net_data = container_data['NetworkSettings']['Networks'][net_name] - self.assertEqual( - net_data['IPAMConfig']['IPv6Address'], '2001:389::f00d' - ) + assert net_data['IPAMConfig']['IPv6Address'] == '2001:389::f00d' @requires_api_version('1.23') def test_create_internal_networks(self): diff --git a/tests/integration/api_volume_test.py b/tests/integration/api_volume_test.py index 5a4bb1e0bc..e09f12a7fe 100644 --- a/tests/integration/api_volume_test.py +++ b/tests/integration/api_volume_test.py @@ -11,10 +11,10 @@ def test_create_volume(self): name = 'perfectcherryblossom' self.tmp_volumes.append(name) result = self.client.create_volume(name) - self.assertIn('Name', result) - self.assertEqual(result['Name'], name) - self.assertIn('Driver', result) - self.assertEqual(result['Driver'], 'local') + assert 'Name' in result + assert result['Name'] == name + assert 'Driver' in result + assert result['Driver'] == 'local' def test_create_volume_invalid_driver(self): driver_name = 'invalid.driver' @@ -27,16 +27,16 @@ def test_list_volumes(self): self.tmp_volumes.append(name) volume_info = self.client.create_volume(name) result = self.client.volumes() - self.assertIn('Volumes', result) + assert 'Volumes' in result volumes = result['Volumes'] - self.assertIn(volume_info, volumes) + assert volume_info in volumes def test_inspect_volume(self): name = 'embodimentofscarletdevil' self.tmp_volumes.append(name) volume_info = self.client.create_volume(name) result = self.client.inspect_volume(name) - self.assertEqual(volume_info, result) + assert volume_info == result def test_inspect_nonexistent_volume(self): name = 'embodimentofscarletdevil' diff --git a/tests/integration/errors_test.py b/tests/integration/errors_test.py index dc5cef4945..ac74d72100 100644 --- a/tests/integration/errors_test.py +++ b/tests/integration/errors_test.py @@ -1,14 +1,15 @@ from docker.errors import APIError from .base import BaseAPIIntegrationTest, BUSYBOX +import pytest class ErrorsTest(BaseAPIIntegrationTest): def test_api_error_parses_json(self): container = self.client.create_container(BUSYBOX, ['sleep', '10']) self.client.start(container['Id']) - with self.assertRaises(APIError) as cm: + with pytest.raises(APIError) as cm: self.client.remove_container(container['Id']) - explanation = cm.exception.explanation + explanation = cm.value.explanation assert 'You cannot remove a running container' in explanation assert '{"message":' not in explanation self.client.remove_container(container['Id'], force=True) diff --git a/tests/integration/models_containers_test.py b/tests/integration/models_containers_test.py index 3c33cb0721..2d01f222aa 100644 --- a/tests/integration/models_containers_test.py +++ b/tests/integration/models_containers_test.py @@ -10,10 +10,9 @@ class ContainerCollectionTest(BaseIntegrationTest): def test_run(self): client = docker.from_env(version=TEST_API_VERSION) - self.assertEqual( - client.containers.run("alpine", "echo hello world", remove=True), - b'hello world\n' - ) + assert client.containers.run( + "alpine", "echo hello world", remove=True + ) == b'hello world\n' def test_run_detach(self): client = docker.from_env(version=TEST_API_VERSION) @@ -24,16 +23,16 @@ def test_run_detach(self): def test_run_with_error(self): client = docker.from_env(version=TEST_API_VERSION) - with self.assertRaises(docker.errors.ContainerError) as cm: + with pytest.raises(docker.errors.ContainerError) as cm: client.containers.run("alpine", "cat /test", remove=True) - assert cm.exception.exit_status == 1 - assert "cat /test" in str(cm.exception) - assert "alpine" in str(cm.exception) - assert "No such file or directory" in str(cm.exception) + assert cm.value.exit_status == 1 + assert "cat /test" in cm.exconly() + assert "alpine" in cm.exconly() + assert "No such file or directory" in cm.exconly() def test_run_with_image_that_does_not_exist(self): client = docker.from_env(version=TEST_API_VERSION) - with self.assertRaises(docker.errors.ImageNotFound): + with pytest.raises(docker.errors.ImageNotFound): client.containers.run("dockerpytest_does_not_exist") def test_run_with_volume(self): @@ -52,7 +51,7 @@ def test_run_with_volume(self): "alpine", "cat /insidecontainer/test", volumes=["%s:/insidecontainer" % path] ) - self.assertEqual(out, b'hello\n') + assert out == b'hello\n' def test_run_with_named_volume(self): client = docker.from_env(version=TEST_API_VERSION) @@ -70,7 +69,7 @@ def test_run_with_named_volume(self): "alpine", "cat /insidecontainer/test", volumes=["somevolume:/insidecontainer"] ) - self.assertEqual(out, b'hello\n') + assert out == b'hello\n' def test_run_with_network(self): net_name = random_name() @@ -170,10 +169,9 @@ def test_commit(self): self.tmp_containers.append(container.id) container.wait() image = container.commit() - self.assertEqual( - client.containers.run(image.id, "cat /test", remove=True), - b"hello\n" - ) + assert client.containers.run( + image.id, "cat /test", remove=True + ) == b"hello\n" def test_diff(self): client = docker.from_env(version=TEST_API_VERSION) diff --git a/tests/integration/models_images_test.py b/tests/integration/models_images_test.py index 94164ce1e4..931391691e 100644 --- a/tests/integration/models_images_test.py +++ b/tests/integration/models_images_test.py @@ -21,16 +21,15 @@ def test_build(self): # @pytest.mark.xfail(reason='Engine 1.13 responds with status 500') def test_build_with_error(self): client = docker.from_env(version=TEST_API_VERSION) - with self.assertRaises(docker.errors.BuildError) as cm: + with pytest.raises(docker.errors.BuildError) as cm: client.images.build(fileobj=io.BytesIO( "FROM alpine\n" "RUN exit 1".encode('ascii') )) - print(cm.exception) - assert str(cm.exception) == ( + assert ( "The command '/bin/sh -c exit 1' returned a non-zero code: 1" - ) - assert cm.exception.build_log + ) in cm.exconly() + assert cm.value.build_log def test_build_with_multiple_success(self): client = docker.from_env(version=TEST_API_VERSION) diff --git a/tests/integration/models_services_test.py b/tests/integration/models_services_test.py index 827242a01b..39cccf8ee6 100644 --- a/tests/integration/models_services_test.py +++ b/tests/integration/models_services_test.py @@ -6,6 +6,7 @@ from .base import TEST_API_VERSION from docker.errors import InvalidArgument from docker.types.services import ServiceMode +import pytest class ServiceTest(unittest.TestCase): @@ -265,8 +266,7 @@ def test_scale_method_global_service(self): while len(tasks) == 0: tasks = service.tasks() assert len(tasks) == 1 - with self.assertRaises(InvalidArgument, - msg='Cannot scale a global container'): + with pytest.raises(InvalidArgument): service.scale(2) assert len(tasks) == 1 diff --git a/tests/integration/models_swarm_test.py b/tests/integration/models_swarm_test.py index dadd77d981..f39f0d34cf 100644 --- a/tests/integration/models_swarm_test.py +++ b/tests/integration/models_swarm_test.py @@ -4,6 +4,7 @@ from .. import helpers from .base import TEST_API_VERSION +import pytest class SwarmTest(unittest.TestCase): @@ -24,11 +25,9 @@ def test_init_update_leave(self): assert client.swarm.attrs['Spec']['Raft']['SnapshotInterval'] == 10000 assert client.swarm.id assert client.swarm.leave(force=True) - with self.assertRaises(docker.errors.APIError) as cm: + with pytest.raises(docker.errors.APIError) as cm: client.swarm.reload() assert ( - # FIXME: test for both until - # https://github.com/docker/docker/issues/29192 is resolved - cm.exception.response.status_code == 406 or - cm.exception.response.status_code == 503 + cm.value.response.status_code == 406 or + cm.value.response.status_code == 503 ) diff --git a/tests/integration/regression_test.py b/tests/integration/regression_test.py index e3e6d9b7e3..0fd4e43104 100644 --- a/tests/integration/regression_test.py +++ b/tests/integration/regression_test.py @@ -5,15 +5,16 @@ import six from .base import BaseAPIIntegrationTest, BUSYBOX +import pytest class TestRegressions(BaseAPIIntegrationTest): def test_443_handle_nonchunked_response_in_stream(self): dfile = io.BytesIO() - with self.assertRaises(docker.errors.APIError) as exc: + with pytest.raises(docker.errors.APIError) as exc: for line in self.client.build(fileobj=dfile, tag="a/b/c"): pass - self.assertEqual(exc.exception.response.status_code, 500) + assert exc.value.response.status_code == 500 dfile.close() def test_542_truncate_ids_client_side(self): @@ -21,10 +22,10 @@ def test_542_truncate_ids_client_side(self): self.client.create_container(BUSYBOX, ['true']) ) result = self.client.containers(all=True, trunc=True) - self.assertEqual(len(result[0]['Id']), 12) + assert len(result[0]['Id']) == 12 def test_647_support_doubleslash_in_image_names(self): - with self.assertRaises(docker.errors.APIError): + with pytest.raises(docker.errors.APIError): self.client.inspect_image('gensokyo.jp//kirisame') def test_649_handle_timeout_value_none(self): @@ -53,15 +54,12 @@ def test_792_explicit_port_protocol(self): ) self.tmp_containers.append(ctnr) self.client.start(ctnr) - self.assertEqual( - self.client.port(ctnr, 2000)[0]['HostPort'], - six.text_type(tcp_port) - ) - self.assertEqual( - self.client.port(ctnr, '2000/tcp')[0]['HostPort'], - six.text_type(tcp_port) - ) - self.assertEqual( - self.client.port(ctnr, '2000/udp')[0]['HostPort'], - six.text_type(udp_port) - ) + assert self.client.port( + ctnr, 2000 + )[0]['HostPort'] == six.text_type(tcp_port) + assert self.client.port( + ctnr, '2000/tcp' + )[0]['HostPort'] == six.text_type(tcp_port) + assert self.client.port( + ctnr, '2000/udp' + )[0]['HostPort'] == six.text_type(udp_port) diff --git a/tests/unit/api_build_test.py b/tests/unit/api_build_test.py index 927aa9749e..e366bced69 100644 --- a/tests/unit/api_build_test.py +++ b/tests/unit/api_build_test.py @@ -5,6 +5,7 @@ from docker import auth from .api_test import BaseAPIClientTest, fake_request, url_prefix +import pytest class BuildTest(BaseAPIClientTest): @@ -110,12 +111,10 @@ def test_build_container_with_container_limits(self): }) def test_build_container_invalid_container_limits(self): - self.assertRaises( - docker.errors.DockerException, - lambda: self.client.build('.', container_limits={ + with pytest.raises(docker.errors.DockerException): + self.client.build('.', container_limits={ 'foo': 'bar' }) - ) def test_set_auth_headers_with_empty_dict_and_auth_configs(self): self.client._auth_configs = { @@ -130,7 +129,7 @@ def test_set_auth_headers_with_empty_dict_and_auth_configs(self): expected_headers = { 'X-Registry-Config': auth.encode_header(self.client._auth_configs)} self.client._set_auth_headers(headers) - self.assertEqual(headers, expected_headers) + assert headers == expected_headers def test_set_auth_headers_with_dict_and_auth_configs(self): self.client._auth_configs = { @@ -147,7 +146,7 @@ def test_set_auth_headers_with_dict_and_auth_configs(self): 'X-Registry-Config': auth.encode_header(self.client._auth_configs)} self.client._set_auth_headers(headers) - self.assertEqual(headers, expected_headers) + assert headers == expected_headers def test_set_auth_headers_with_dict_and_no_auth_configs(self): headers = {'foo': 'bar'} @@ -156,4 +155,4 @@ def test_set_auth_headers_with_dict_and_no_auth_configs(self): } self.client._set_auth_headers(headers) - self.assertEqual(headers, expected_headers) + assert headers == expected_headers diff --git a/tests/unit/api_container_test.py b/tests/unit/api_container_test.py index 8a897ccafc..3cb718a2f0 100644 --- a/tests/unit/api_container_test.py +++ b/tests/unit/api_container_test.py @@ -30,31 +30,20 @@ def test_start_container(self): self.client.start(fake_api.FAKE_CONTAINER_ID) args = fake_request.call_args - self.assertEqual( - args[0][1], - url_prefix + 'containers/3cc2351ab11b/start' - ) + assert args[0][1] == url_prefix + 'containers/3cc2351ab11b/start' assert 'data' not in args[1] - self.assertEqual( - args[1]['timeout'], DEFAULT_TIMEOUT_SECONDS - ) + assert args[1]['timeout'] == DEFAULT_TIMEOUT_SECONDS def test_start_container_none(self): with pytest.raises(ValueError) as excinfo: self.client.start(container=None) - self.assertEqual( - str(excinfo.value), - 'Resource ID was not provided', - ) + assert str(excinfo.value) == 'Resource ID was not provided' with pytest.raises(ValueError) as excinfo: self.client.start(None) - self.assertEqual( - str(excinfo.value), - 'Resource ID was not provided', - ) + assert str(excinfo.value) == 'Resource ID was not provided' def test_start_container_regression_573(self): self.client.start(**{'container': fake_api.FAKE_CONTAINER_ID}) @@ -134,14 +123,9 @@ def test_start_container_with_dict_instead_of_id(self): self.client.start({'Id': fake_api.FAKE_CONTAINER_ID}) args = fake_request.call_args - self.assertEqual( - args[0][1], - url_prefix + 'containers/3cc2351ab11b/start' - ) + assert args[0][1] == url_prefix + 'containers/3cc2351ab11b/start' assert 'data' not in args[1] - self.assertEqual( - args[1]['timeout'], DEFAULT_TIMEOUT_SECONDS - ) + assert args[1]['timeout'] == DEFAULT_TIMEOUT_SECONDS class CreateContainerTest(BaseAPIClientTest): @@ -149,17 +133,15 @@ def test_create_container(self): self.client.create_container('busybox', 'true') args = fake_request.call_args - self.assertEqual(args[0][1], - url_prefix + 'containers/create') - self.assertEqual(json.loads(args[1]['data']), - json.loads(''' - {"Tty": false, "Image": "busybox", "Cmd": ["true"], - "AttachStdin": false, - "AttachStderr": true, "AttachStdout": true, - "StdinOnce": false, - "OpenStdin": false, "NetworkDisabled": false}''')) - self.assertEqual(args[1]['headers'], - {'Content-Type': 'application/json'}) + assert args[0][1] == url_prefix + 'containers/create' + assert json.loads(args[1]['data']) == json.loads(''' + {"Tty": false, "Image": "busybox", "Cmd": ["true"], + "AttachStdin": false, + "AttachStderr": true, "AttachStdout": true, + "StdinOnce": false, + "OpenStdin": false, "NetworkDisabled": false} + ''') + assert args[1]['headers'] == {'Content-Type': 'application/json'} def test_create_container_with_binds(self): mount_dest = '/mnt' @@ -168,19 +150,17 @@ def test_create_container_with_binds(self): volumes=[mount_dest]) args = fake_request.call_args - self.assertEqual(args[0][1], - url_prefix + 'containers/create') - self.assertEqual(json.loads(args[1]['data']), - json.loads(''' - {"Tty": false, "Image": "busybox", - "Cmd": ["ls", "/mnt"], "AttachStdin": false, - "Volumes": {"/mnt": {}}, - "AttachStderr": true, - "AttachStdout": true, "OpenStdin": false, - "StdinOnce": false, - "NetworkDisabled": false}''')) - self.assertEqual(args[1]['headers'], - {'Content-Type': 'application/json'}) + assert args[0][1] == url_prefix + 'containers/create' + assert json.loads(args[1]['data']) == json.loads(''' + {"Tty": false, "Image": "busybox", + "Cmd": ["ls", "/mnt"], "AttachStdin": false, + "Volumes": {"/mnt": {}}, + "AttachStderr": true, + "AttachStdout": true, "OpenStdin": false, + "StdinOnce": false, + "NetworkDisabled": false} + ''') + assert args[1]['headers'] == {'Content-Type': 'application/json'} def test_create_container_with_volume_string(self): mount_dest = '/mnt' @@ -189,80 +169,72 @@ def test_create_container_with_volume_string(self): volumes=mount_dest) args = fake_request.call_args - self.assertEqual(args[0][1], - url_prefix + 'containers/create') - self.assertEqual(json.loads(args[1]['data']), - json.loads(''' - {"Tty": false, "Image": "busybox", - "Cmd": ["ls", "/mnt"], "AttachStdin": false, - "Volumes": {"/mnt": {}}, - "AttachStderr": true, - "AttachStdout": true, "OpenStdin": false, - "StdinOnce": false, - "NetworkDisabled": false}''')) - self.assertEqual(args[1]['headers'], - {'Content-Type': 'application/json'}) + assert args[0][1] == url_prefix + 'containers/create' + assert json.loads(args[1]['data']) == json.loads(''' + {"Tty": false, "Image": "busybox", + "Cmd": ["ls", "/mnt"], "AttachStdin": false, + "Volumes": {"/mnt": {}}, + "AttachStderr": true, + "AttachStdout": true, "OpenStdin": false, + "StdinOnce": false, + "NetworkDisabled": false} + ''') + assert args[1]['headers'] == {'Content-Type': 'application/json'} def test_create_container_with_ports(self): self.client.create_container('busybox', 'ls', ports=[1111, (2222, 'udp'), (3333,)]) args = fake_request.call_args - self.assertEqual(args[0][1], - url_prefix + 'containers/create') - self.assertEqual(json.loads(args[1]['data']), - json.loads(''' - {"Tty": false, "Image": "busybox", - "Cmd": ["ls"], "AttachStdin": false, - "ExposedPorts": { - "1111/tcp": {}, - "2222/udp": {}, - "3333/tcp": {} - }, - "AttachStderr": true, - "AttachStdout": true, "OpenStdin": false, - "StdinOnce": false, - "NetworkDisabled": false}''')) - self.assertEqual(args[1]['headers'], - {'Content-Type': 'application/json'}) + assert args[0][1] == url_prefix + 'containers/create' + assert json.loads(args[1]['data']) == json.loads(''' + {"Tty": false, "Image": "busybox", + "Cmd": ["ls"], "AttachStdin": false, + "ExposedPorts": { + "1111/tcp": {}, + "2222/udp": {}, + "3333/tcp": {} + }, + "AttachStderr": true, + "AttachStdout": true, "OpenStdin": false, + "StdinOnce": false, + "NetworkDisabled": false} + ''') + assert args[1]['headers'] == {'Content-Type': 'application/json'} def test_create_container_with_entrypoint(self): self.client.create_container('busybox', 'hello', entrypoint='cowsay entry') args = fake_request.call_args - self.assertEqual(args[0][1], - url_prefix + 'containers/create') - self.assertEqual(json.loads(args[1]['data']), - json.loads(''' - {"Tty": false, "Image": "busybox", - "Cmd": ["hello"], "AttachStdin": false, - "AttachStderr": true, - "AttachStdout": true, "OpenStdin": false, - "StdinOnce": false, - "NetworkDisabled": false, - "Entrypoint": ["cowsay", "entry"]}''')) - self.assertEqual(args[1]['headers'], - {'Content-Type': 'application/json'}) + assert args[0][1] == url_prefix + 'containers/create' + assert json.loads(args[1]['data']) == json.loads(''' + {"Tty": false, "Image": "busybox", + "Cmd": ["hello"], "AttachStdin": false, + "AttachStderr": true, + "AttachStdout": true, "OpenStdin": false, + "StdinOnce": false, + "NetworkDisabled": false, + "Entrypoint": ["cowsay", "entry"]} + ''') + assert args[1]['headers'] == {'Content-Type': 'application/json'} def test_create_container_with_cpu_shares(self): with pytest.deprecated_call(): self.client.create_container('busybox', 'ls', cpu_shares=5) args = fake_request.call_args - self.assertEqual(args[0][1], - url_prefix + 'containers/create') - self.assertEqual(json.loads(args[1]['data']), - json.loads(''' - {"Tty": false, "Image": "busybox", - "Cmd": ["ls"], "AttachStdin": false, - "AttachStderr": true, - "AttachStdout": true, "OpenStdin": false, - "StdinOnce": false, - "NetworkDisabled": false, - "CpuShares": 5}''')) - self.assertEqual(args[1]['headers'], - {'Content-Type': 'application/json'}) + assert args[0][1] == url_prefix + 'containers/create' + assert json.loads(args[1]['data']) == json.loads(''' + {"Tty": false, "Image": "busybox", + "Cmd": ["ls"], "AttachStdin": false, + "AttachStderr": true, + "AttachStdout": true, "OpenStdin": false, + "StdinOnce": false, + "NetworkDisabled": false, + "CpuShares": 5} + ''') + assert args[1]['headers'] == {'Content-Type': 'application/json'} @requires_api_version('1.18') def test_create_container_with_host_config_cpu_shares(self): @@ -273,43 +245,39 @@ def test_create_container_with_host_config_cpu_shares(self): ) args = fake_request.call_args - self.assertEqual(args[0][1], - url_prefix + 'containers/create') - - self.assertEqual(json.loads(args[1]['data']), - json.loads(''' - {"Tty": false, "Image": "busybox", - "Cmd": ["ls"], "AttachStdin": false, - "AttachStderr": true, - "AttachStdout": true, "OpenStdin": false, - "StdinOnce": false, - "NetworkDisabled": false, - "HostConfig": { - "CpuShares": 512, - "NetworkMode": "default" - }}''')) - self.assertEqual(args[1]['headers'], - {'Content-Type': 'application/json'}) + assert args[0][1] == url_prefix + 'containers/create' + + assert json.loads(args[1]['data']) == json.loads(''' + {"Tty": false, "Image": "busybox", + "Cmd": ["ls"], "AttachStdin": false, + "AttachStderr": true, + "AttachStdout": true, "OpenStdin": false, + "StdinOnce": false, + "NetworkDisabled": false, + "HostConfig": { + "CpuShares": 512, + "NetworkMode": "default" + }} + ''') + assert args[1]['headers'] == {'Content-Type': 'application/json'} def test_create_container_with_cpuset(self): with pytest.deprecated_call(): self.client.create_container('busybox', 'ls', cpuset='0,1') args = fake_request.call_args - self.assertEqual(args[0][1], - url_prefix + 'containers/create') - self.assertEqual(json.loads(args[1]['data']), - json.loads(''' - {"Tty": false, "Image": "busybox", - "Cmd": ["ls"], "AttachStdin": false, - "AttachStderr": true, - "AttachStdout": true, "OpenStdin": false, - "StdinOnce": false, - "NetworkDisabled": false, - "Cpuset": "0,1", - "CpusetCpus": "0,1"}''')) - self.assertEqual(args[1]['headers'], - {'Content-Type': 'application/json'}) + assert args[0][1] == url_prefix + 'containers/create' + assert json.loads(args[1]['data']) == json.loads(''' + {"Tty": false, "Image": "busybox", + "Cmd": ["ls"], "AttachStdin": false, + "AttachStderr": true, + "AttachStdout": true, "OpenStdin": false, + "StdinOnce": false, + "NetworkDisabled": false, + "Cpuset": "0,1", + "CpusetCpus": "0,1"} + ''') + assert args[1]['headers'] == {'Content-Type': 'application/json'} @requires_api_version('1.18') def test_create_container_with_host_config_cpuset(self): @@ -320,23 +288,21 @@ def test_create_container_with_host_config_cpuset(self): ) args = fake_request.call_args - self.assertEqual(args[0][1], - url_prefix + 'containers/create') - - self.assertEqual(json.loads(args[1]['data']), - json.loads(''' - {"Tty": false, "Image": "busybox", - "Cmd": ["ls"], "AttachStdin": false, - "AttachStderr": true, - "AttachStdout": true, "OpenStdin": false, - "StdinOnce": false, - "NetworkDisabled": false, - "HostConfig": { - "CpusetCpus": "0,1", - "NetworkMode": "default" - }}''')) - self.assertEqual(args[1]['headers'], - {'Content-Type': 'application/json'}) + assert args[0][1] == url_prefix + 'containers/create' + + assert json.loads(args[1]['data']) == json.loads(''' + {"Tty": false, "Image": "busybox", + "Cmd": ["ls"], "AttachStdin": false, + "AttachStderr": true, + "AttachStdout": true, "OpenStdin": false, + "StdinOnce": false, + "NetworkDisabled": false, + "HostConfig": { + "CpusetCpus": "0,1", + "NetworkMode": "default" + }} + ''') + assert args[1]['headers'] == {'Content-Type': 'application/json'} @requires_api_version('1.19') def test_create_container_with_host_config_cpuset_mems(self): @@ -347,23 +313,21 @@ def test_create_container_with_host_config_cpuset_mems(self): ) args = fake_request.call_args - self.assertEqual(args[0][1], - url_prefix + 'containers/create') - - self.assertEqual(json.loads(args[1]['data']), - json.loads(''' - {"Tty": false, "Image": "busybox", - "Cmd": ["ls"], "AttachStdin": false, - "AttachStderr": true, - "AttachStdout": true, "OpenStdin": false, - "StdinOnce": false, - "NetworkDisabled": false, - "HostConfig": { - "CpusetMems": "0", - "NetworkMode": "default" - }}''')) - self.assertEqual(args[1]['headers'], - {'Content-Type': 'application/json'}) + assert args[0][1] == url_prefix + 'containers/create' + + assert json.loads(args[1]['data']) == json.loads(''' + {"Tty": false, "Image": "busybox", + "Cmd": ["ls"], "AttachStdin": false, + "AttachStderr": true, + "AttachStdout": true, "OpenStdin": false, + "StdinOnce": false, + "NetworkDisabled": false, + "HostConfig": { + "CpusetMems": "0", + "NetworkMode": "default" + }} + ''') + assert args[1]['headers'] == {'Content-Type': 'application/json'} def test_create_container_with_cgroup_parent(self): self.client.create_container( @@ -373,47 +337,42 @@ def test_create_container_with_cgroup_parent(self): ) args = fake_request.call_args - self.assertEqual(args[0][1], - url_prefix + 'containers/create') + assert args[0][1] == url_prefix + 'containers/create' data = json.loads(args[1]['data']) - self.assertIn('HostConfig', data) - self.assertIn('CgroupParent', data['HostConfig']) - self.assertEqual(data['HostConfig']['CgroupParent'], 'test') + assert 'HostConfig' in data + assert 'CgroupParent' in data['HostConfig'] + assert data['HostConfig']['CgroupParent'] == 'test' def test_create_container_with_working_dir(self): self.client.create_container('busybox', 'ls', working_dir='/root') args = fake_request.call_args - self.assertEqual(args[0][1], - url_prefix + 'containers/create') - self.assertEqual(json.loads(args[1]['data']), - json.loads(''' - {"Tty": false, "Image": "busybox", - "Cmd": ["ls"], "AttachStdin": false, - "AttachStderr": true, - "AttachStdout": true, "OpenStdin": false, - "StdinOnce": false, - "NetworkDisabled": false, - "WorkingDir": "/root"}''')) - self.assertEqual(args[1]['headers'], - {'Content-Type': 'application/json'}) + assert args[0][1] == url_prefix + 'containers/create' + assert json.loads(args[1]['data']) == json.loads(''' + {"Tty": false, "Image": "busybox", + "Cmd": ["ls"], "AttachStdin": false, + "AttachStderr": true, + "AttachStdout": true, "OpenStdin": false, + "StdinOnce": false, + "NetworkDisabled": false, + "WorkingDir": "/root"} + ''') + assert args[1]['headers'] == {'Content-Type': 'application/json'} def test_create_container_with_stdin_open(self): self.client.create_container('busybox', 'true', stdin_open=True) args = fake_request.call_args - self.assertEqual(args[0][1], - url_prefix + 'containers/create') - self.assertEqual(json.loads(args[1]['data']), - json.loads(''' - {"Tty": false, "Image": "busybox", "Cmd": ["true"], - "AttachStdin": true, - "AttachStderr": true, "AttachStdout": true, - "StdinOnce": true, - "OpenStdin": true, "NetworkDisabled": false}''')) - self.assertEqual(args[1]['headers'], - {'Content-Type': 'application/json'}) + assert args[0][1] == url_prefix + 'containers/create' + assert json.loads(args[1]['data']) == json.loads(''' + {"Tty": false, "Image": "busybox", "Cmd": ["true"], + "AttachStdin": true, + "AttachStderr": true, "AttachStdout": true, + "StdinOnce": true, + "OpenStdin": true, "NetworkDisabled": false} + ''') + assert args[1]['headers'] == {'Content-Type': 'application/json'} def test_create_container_with_volumes_from(self): vol_names = ['foo', 'bar'] @@ -421,17 +380,17 @@ def test_create_container_with_volumes_from(self): self.client.create_container('busybox', 'true', volumes_from=vol_names) except docker.errors.DockerException: - self.assertTrue( - docker.utils.compare_version('1.10', self.client._version) >= 0 - ) + assert docker.utils.compare_version( + '1.10', self.client._version + ) >= 0 return args = fake_request.call_args - self.assertEqual(args[0][1], url_prefix + 'containers/create') - self.assertEqual(json.loads(args[1]['data'])['VolumesFrom'], - ','.join(vol_names)) - self.assertEqual(args[1]['headers'], - {'Content-Type': 'application/json'}) + assert args[0][1] == url_prefix + 'containers/create' + assert json.loads(args[1]['data'])['VolumesFrom'] == ','.join( + vol_names + ) + assert args[1]['headers'] == {'Content-Type': 'application/json'} def test_create_container_empty_volumes_from(self): with pytest.raises(docker.errors.InvalidVersion): @@ -442,18 +401,16 @@ def test_create_named_container(self): name='marisa-kirisame') args = fake_request.call_args - self.assertEqual(args[0][1], - url_prefix + 'containers/create') - self.assertEqual(json.loads(args[1]['data']), - json.loads(''' - {"Tty": false, "Image": "busybox", "Cmd": ["true"], - "AttachStdin": false, - "AttachStderr": true, "AttachStdout": true, - "StdinOnce": false, - "OpenStdin": false, "NetworkDisabled": false}''')) - self.assertEqual(args[1]['headers'], - {'Content-Type': 'application/json'}) - self.assertEqual(args[1]['params'], {'name': 'marisa-kirisame'}) + assert args[0][1] == url_prefix + 'containers/create' + assert json.loads(args[1]['data']) == json.loads(''' + {"Tty": false, "Image": "busybox", "Cmd": ["true"], + "AttachStdin": false, + "AttachStderr": true, "AttachStdout": true, + "StdinOnce": false, + "OpenStdin": false, "NetworkDisabled": false} + ''') + assert args[1]['headers'] == {'Content-Type': 'application/json'} + assert args[1]['params'] == {'name': 'marisa-kirisame'} def test_create_container_with_mem_limit_as_int(self): self.client.create_container( @@ -464,7 +421,7 @@ def test_create_container_with_mem_limit_as_int(self): args = fake_request.call_args data = json.loads(args[1]['data']) - self.assertEqual(data['HostConfig']['Memory'], 128.0) + assert data['HostConfig']['Memory'] == 128.0 def test_create_container_with_mem_limit_as_string(self): self.client.create_container( @@ -475,7 +432,7 @@ def test_create_container_with_mem_limit_as_string(self): args = fake_request.call_args data = json.loads(args[1]['data']) - self.assertEqual(data['HostConfig']['Memory'], 128.0) + assert data['HostConfig']['Memory'] == 128.0 def test_create_container_with_mem_limit_as_string_with_k_unit(self): self.client.create_container( @@ -486,7 +443,7 @@ def test_create_container_with_mem_limit_as_string_with_k_unit(self): args = fake_request.call_args data = json.loads(args[1]['data']) - self.assertEqual(data['HostConfig']['Memory'], 128.0 * 1024) + assert data['HostConfig']['Memory'] == 128.0 * 1024 def test_create_container_with_mem_limit_as_string_with_m_unit(self): self.client.create_container( @@ -497,7 +454,7 @@ def test_create_container_with_mem_limit_as_string_with_m_unit(self): args = fake_request.call_args data = json.loads(args[1]['data']) - self.assertEqual(data['HostConfig']['Memory'], 128.0 * 1024 * 1024) + assert data['HostConfig']['Memory'] == 128.0 * 1024 * 1024 def test_create_container_with_mem_limit_as_string_with_g_unit(self): self.client.create_container( @@ -508,20 +465,14 @@ def test_create_container_with_mem_limit_as_string_with_g_unit(self): args = fake_request.call_args data = json.loads(args[1]['data']) - self.assertEqual( - data['HostConfig']['Memory'], 128.0 * 1024 * 1024 * 1024 - ) + assert data['HostConfig']['Memory'] == 128.0 * 1024 * 1024 * 1024 def test_create_container_with_mem_limit_as_string_with_wrong_value(self): - self.assertRaises( - docker.errors.DockerException, - self.client.create_host_config, mem_limit='128p' - ) + with pytest.raises(docker.errors.DockerException): + self.client.create_host_config(mem_limit='128p') - self.assertRaises( - docker.errors.DockerException, - self.client.create_host_config, mem_limit='1f28' - ) + with pytest.raises(docker.errors.DockerException): + self.client.create_host_config(mem_limit='1f28') def test_create_container_with_lxc_conf(self): self.client.create_container( @@ -531,25 +482,16 @@ def test_create_container_with_lxc_conf(self): ) args = fake_request.call_args - self.assertEqual( - args[0][1], - url_prefix + 'containers/create' - ) + assert args[0][1] == url_prefix + 'containers/create' expected_payload = self.base_create_payload() expected_payload['HostConfig'] = self.client.create_host_config() expected_payload['HostConfig']['LxcConf'] = [ {"Value": "lxc.conf.value", "Key": "lxc.conf.k"} ] - self.assertEqual(json.loads(args[1]['data']), expected_payload) - self.assertEqual( - args[1]['headers'], - {'Content-Type': 'application/json'} - ) - self.assertEqual( - args[1]['timeout'], - DEFAULT_TIMEOUT_SECONDS - ) + assert json.loads(args[1]['data']) == expected_payload + assert args[1]['headers'] == {'Content-Type': 'application/json'} + assert args[1]['timeout'] == DEFAULT_TIMEOUT_SECONDS def test_create_container_with_lxc_conf_compat(self): self.client.create_container( @@ -559,20 +501,15 @@ def test_create_container_with_lxc_conf_compat(self): ) args = fake_request.call_args - self.assertEqual(args[0][1], url_prefix + 'containers/create') + assert args[0][1] == url_prefix + 'containers/create' expected_payload = self.base_create_payload() expected_payload['HostConfig'] = self.client.create_host_config() expected_payload['HostConfig']['LxcConf'] = [ {"Value": "lxc.conf.value", "Key": "lxc.conf.k"} ] - self.assertEqual( - json.loads(args[1]['data']), expected_payload) - self.assertEqual(args[1]['headers'], - {'Content-Type': 'application/json'}) - self.assertEqual( - args[1]['timeout'], - DEFAULT_TIMEOUT_SECONDS - ) + assert json.loads(args[1]['data']) == expected_payload + assert args[1]['headers'] == {'Content-Type': 'application/json'} + assert args[1]['timeout'] == DEFAULT_TIMEOUT_SECONDS def test_create_container_with_binds_ro(self): mount_dest = '/mnt' @@ -588,18 +525,13 @@ def test_create_container_with_binds_ro(self): ) args = fake_request.call_args - self.assertEqual(args[0][1], url_prefix + - 'containers/create') + assert args[0][1] == url_prefix + 'containers/create' expected_payload = self.base_create_payload() expected_payload['HostConfig'] = self.client.create_host_config() expected_payload['HostConfig']['Binds'] = ["/tmp:/mnt:ro"] - self.assertEqual(json.loads(args[1]['data']), expected_payload) - self.assertEqual(args[1]['headers'], - {'Content-Type': 'application/json'}) - self.assertEqual( - args[1]['timeout'], - DEFAULT_TIMEOUT_SECONDS - ) + assert json.loads(args[1]['data']) == expected_payload + assert args[1]['headers'] == {'Content-Type': 'application/json'} + assert args[1]['timeout'] == DEFAULT_TIMEOUT_SECONDS def test_create_container_with_binds_rw(self): mount_dest = '/mnt' @@ -615,18 +547,13 @@ def test_create_container_with_binds_rw(self): ) args = fake_request.call_args - self.assertEqual(args[0][1], url_prefix + - 'containers/create') + assert args[0][1] == url_prefix + 'containers/create' expected_payload = self.base_create_payload() expected_payload['HostConfig'] = self.client.create_host_config() expected_payload['HostConfig']['Binds'] = ["/tmp:/mnt:rw"] - self.assertEqual(json.loads(args[1]['data']), expected_payload) - self.assertEqual(args[1]['headers'], - {'Content-Type': 'application/json'}) - self.assertEqual( - args[1]['timeout'], - DEFAULT_TIMEOUT_SECONDS - ) + assert json.loads(args[1]['data']) == expected_payload + assert args[1]['headers'] == {'Content-Type': 'application/json'} + assert args[1]['timeout'] == DEFAULT_TIMEOUT_SECONDS def test_create_container_with_binds_mode(self): mount_dest = '/mnt' @@ -642,18 +569,13 @@ def test_create_container_with_binds_mode(self): ) args = fake_request.call_args - self.assertEqual(args[0][1], url_prefix + - 'containers/create') + assert args[0][1] == url_prefix + 'containers/create' expected_payload = self.base_create_payload() expected_payload['HostConfig'] = self.client.create_host_config() expected_payload['HostConfig']['Binds'] = ["/tmp:/mnt:z"] - self.assertEqual(json.loads(args[1]['data']), expected_payload) - self.assertEqual(args[1]['headers'], - {'Content-Type': 'application/json'}) - self.assertEqual( - args[1]['timeout'], - DEFAULT_TIMEOUT_SECONDS - ) + assert json.loads(args[1]['data']) == expected_payload + assert args[1]['headers'] == {'Content-Type': 'application/json'} + assert args[1]['timeout'] == DEFAULT_TIMEOUT_SECONDS def test_create_container_with_binds_mode_and_ro_error(self): with pytest.raises(ValueError): @@ -680,21 +602,16 @@ def test_create_container_with_binds_list(self): ) args = fake_request.call_args - self.assertEqual(args[0][1], url_prefix + - 'containers/create') + assert args[0][1] == url_prefix + 'containers/create' expected_payload = self.base_create_payload() expected_payload['HostConfig'] = self.client.create_host_config() expected_payload['HostConfig']['Binds'] = [ "/tmp:/mnt/1:ro", "/tmp:/mnt/2", ] - self.assertEqual(json.loads(args[1]['data']), expected_payload) - self.assertEqual(args[1]['headers'], - {'Content-Type': 'application/json'}) - self.assertEqual( - args[1]['timeout'], - DEFAULT_TIMEOUT_SECONDS - ) + assert json.loads(args[1]['data']) == expected_payload + assert args[1]['headers'] == {'Content-Type': 'application/json'} + assert args[1]['timeout'] == DEFAULT_TIMEOUT_SECONDS def test_create_container_with_port_binds(self): self.maxDiff = None @@ -713,42 +630,31 @@ def test_create_container_with_port_binds(self): ) args = fake_request.call_args - self.assertEqual(args[0][1], url_prefix + 'containers/create') + assert args[0][1] == url_prefix + 'containers/create' data = json.loads(args[1]['data']) port_bindings = data['HostConfig']['PortBindings'] - self.assertTrue('1111/tcp' in port_bindings) - self.assertTrue('2222/tcp' in port_bindings) - self.assertTrue('3333/udp' in port_bindings) - self.assertTrue('4444/tcp' in port_bindings) - self.assertTrue('5555/tcp' in port_bindings) - self.assertTrue('6666/tcp' in port_bindings) - self.assertEqual( - [{"HostPort": "", "HostIp": ""}], - port_bindings['1111/tcp'] - ) - self.assertEqual( - [{"HostPort": "2222", "HostIp": ""}], - port_bindings['2222/tcp'] - ) - self.assertEqual( - [{"HostPort": "3333", "HostIp": ""}], - port_bindings['3333/udp'] - ) - self.assertEqual( - [{"HostPort": "", "HostIp": "127.0.0.1"}], - port_bindings['4444/tcp'] - ) - self.assertEqual( - [{"HostPort": "5555", "HostIp": "127.0.0.1"}], - port_bindings['5555/tcp'] - ) - self.assertEqual(len(port_bindings['6666/tcp']), 2) - self.assertEqual(args[1]['headers'], - {'Content-Type': 'application/json'}) - self.assertEqual( - args[1]['timeout'], - DEFAULT_TIMEOUT_SECONDS - ) + assert '1111/tcp' in port_bindings + assert '2222/tcp' in port_bindings + assert '3333/udp' in port_bindings + assert '4444/tcp' in port_bindings + assert '5555/tcp' in port_bindings + assert '6666/tcp' in port_bindings + assert [{"HostPort": "", "HostIp": ""}] == port_bindings['1111/tcp'] + assert [ + {"HostPort": "2222", "HostIp": ""} + ] == port_bindings['2222/tcp'] + assert [ + {"HostPort": "3333", "HostIp": ""} + ] == port_bindings['3333/udp'] + assert [ + {"HostPort": "", "HostIp": "127.0.0.1"} + ] == port_bindings['4444/tcp'] + assert [ + {"HostPort": "5555", "HostIp": "127.0.0.1"} + ] == port_bindings['5555/tcp'] + assert len(port_bindings['6666/tcp']) == 2 + assert args[1]['headers'] == {'Content-Type': 'application/json'} + assert args[1]['timeout'] == DEFAULT_TIMEOUT_SECONDS def test_create_container_with_mac_address(self): expected = "02:42:ac:11:00:0a" @@ -760,7 +666,7 @@ def test_create_container_with_mac_address(self): ) args = fake_request.call_args - self.assertEqual(args[0][1], url_prefix + 'containers/create') + assert args[0][1] == url_prefix + 'containers/create' data = json.loads(args[1]['data']) assert data['MacAddress'] == expected @@ -775,17 +681,13 @@ def test_create_container_with_links(self): ) args = fake_request.call_args - self.assertEqual( - args[0][1], url_prefix + 'containers/create' - ) + assert args[0][1] == url_prefix + 'containers/create' expected_payload = self.base_create_payload() expected_payload['HostConfig'] = self.client.create_host_config() expected_payload['HostConfig']['Links'] = ['path:alias'] - self.assertEqual(json.loads(args[1]['data']), expected_payload) - self.assertEqual( - args[1]['headers'], {'Content-Type': 'application/json'} - ) + assert json.loads(args[1]['data']) == expected_payload + assert args[1]['headers'] == {'Content-Type': 'application/json'} def test_create_container_with_multiple_links(self): link_path = 'path' @@ -801,16 +703,14 @@ def test_create_container_with_multiple_links(self): ) args = fake_request.call_args - self.assertEqual(args[0][1], url_prefix + 'containers/create') + assert args[0][1] == url_prefix + 'containers/create' expected_payload = self.base_create_payload() expected_payload['HostConfig'] = self.client.create_host_config() expected_payload['HostConfig']['Links'] = [ 'path1:alias1', 'path2:alias2' ] - self.assertEqual(json.loads(args[1]['data']), expected_payload) - self.assertEqual( - args[1]['headers'], {'Content-Type': 'application/json'} - ) + assert json.loads(args[1]['data']) == expected_payload + assert args[1]['headers'] == {'Content-Type': 'application/json'} def test_create_container_with_links_as_list_of_tuples(self): link_path = 'path' @@ -823,15 +723,13 @@ def test_create_container_with_links_as_list_of_tuples(self): ) args = fake_request.call_args - self.assertEqual(args[0][1], url_prefix + 'containers/create') + assert args[0][1] == url_prefix + 'containers/create' expected_payload = self.base_create_payload() expected_payload['HostConfig'] = self.client.create_host_config() expected_payload['HostConfig']['Links'] = ['path:alias'] - self.assertEqual(json.loads(args[1]['data']), expected_payload) - self.assertEqual( - args[1]['headers'], {'Content-Type': 'application/json'} - ) + assert json.loads(args[1]['data']) == expected_payload + assert args[1]['headers'] == {'Content-Type': 'application/json'} def test_create_container_privileged(self): self.client.create_container( @@ -843,14 +741,10 @@ def test_create_container_privileged(self): expected_payload['HostConfig'] = self.client.create_host_config() expected_payload['HostConfig']['Privileged'] = True args = fake_request.call_args - self.assertEqual(args[0][1], url_prefix + 'containers/create') - self.assertEqual(json.loads(args[1]['data']), expected_payload) - self.assertEqual(args[1]['headers'], - {'Content-Type': 'application/json'}) - self.assertEqual( - args[1]['timeout'], - DEFAULT_TIMEOUT_SECONDS - ) + assert args[0][1] == url_prefix + 'containers/create' + assert json.loads(args[1]['data']) == expected_payload + assert args[1]['headers'] == {'Content-Type': 'application/json'} + assert args[1]['timeout'] == DEFAULT_TIMEOUT_SECONDS def test_create_container_with_restart_policy(self): self.client.create_container( @@ -863,21 +757,17 @@ def test_create_container_with_restart_policy(self): ) args = fake_request.call_args - self.assertEqual(args[0][1], url_prefix + 'containers/create') + assert args[0][1] == url_prefix + 'containers/create' expected_payload = self.base_create_payload() expected_payload['HostConfig'] = self.client.create_host_config() expected_payload['HostConfig']['RestartPolicy'] = { "MaximumRetryCount": 0, "Name": "always" } - self.assertEqual(json.loads(args[1]['data']), expected_payload) + assert json.loads(args[1]['data']) == expected_payload - self.assertEqual( - args[1]['headers'], {'Content-Type': 'application/json'} - ) - self.assertEqual( - args[1]['timeout'], DEFAULT_TIMEOUT_SECONDS - ) + assert args[1]['headers'] == {'Content-Type': 'application/json'} + assert args[1]['timeout'] == DEFAULT_TIMEOUT_SECONDS def test_create_container_with_added_capabilities(self): self.client.create_container( @@ -886,17 +776,13 @@ def test_create_container_with_added_capabilities(self): ) args = fake_request.call_args - self.assertEqual(args[0][1], url_prefix + 'containers/create') + assert args[0][1] == url_prefix + 'containers/create' expected_payload = self.base_create_payload() expected_payload['HostConfig'] = self.client.create_host_config() expected_payload['HostConfig']['CapAdd'] = ['MKNOD'] - self.assertEqual(json.loads(args[1]['data']), expected_payload) - self.assertEqual( - args[1]['headers'], {'Content-Type': 'application/json'} - ) - self.assertEqual( - args[1]['timeout'], DEFAULT_TIMEOUT_SECONDS - ) + assert json.loads(args[1]['data']) == expected_payload + assert args[1]['headers'] == {'Content-Type': 'application/json'} + assert args[1]['timeout'] == DEFAULT_TIMEOUT_SECONDS def test_create_container_with_dropped_capabilities(self): self.client.create_container( @@ -905,17 +791,13 @@ def test_create_container_with_dropped_capabilities(self): ) args = fake_request.call_args - self.assertEqual(args[0][1], url_prefix + 'containers/create') + assert args[0][1] == url_prefix + 'containers/create' expected_payload = self.base_create_payload() expected_payload['HostConfig'] = self.client.create_host_config() expected_payload['HostConfig']['CapDrop'] = ['MKNOD'] - self.assertEqual(json.loads(args[1]['data']), expected_payload) - self.assertEqual( - args[1]['headers'], {'Content-Type': 'application/json'} - ) - self.assertEqual( - args[1]['timeout'], DEFAULT_TIMEOUT_SECONDS - ) + assert json.loads(args[1]['data']) == expected_payload + assert args[1]['headers'] == {'Content-Type': 'application/json'} + assert args[1]['timeout'] == DEFAULT_TIMEOUT_SECONDS def test_create_container_with_devices(self): self.client.create_container( @@ -927,7 +809,7 @@ def test_create_container_with_devices(self): ) args = fake_request.call_args - self.assertEqual(args[0][1], url_prefix + 'containers/create') + assert args[0][1] == url_prefix + 'containers/create' expected_payload = self.base_create_payload() expected_payload['HostConfig'] = self.client.create_host_config() expected_payload['HostConfig']['Devices'] = [ @@ -941,13 +823,9 @@ def test_create_container_with_devices(self): 'PathInContainer': '/dev/sdc', 'PathOnHost': '/dev/sdc'} ] - self.assertEqual(json.loads(args[1]['data']), expected_payload) - self.assertEqual( - args[1]['headers'], {'Content-Type': 'application/json'} - ) - self.assertEqual( - args[1]['timeout'], DEFAULT_TIMEOUT_SECONDS - ) + assert json.loads(args[1]['data']) == expected_payload + assert args[1]['headers'] == {'Content-Type': 'application/json'} + assert args[1]['timeout'] == DEFAULT_TIMEOUT_SECONDS def test_create_container_with_labels_dict(self): labels_dict = { @@ -961,14 +839,10 @@ def test_create_container_with_labels_dict(self): ) args = fake_request.call_args - self.assertEqual(args[0][1], url_prefix + 'containers/create') - self.assertEqual(json.loads(args[1]['data'])['Labels'], labels_dict) - self.assertEqual( - args[1]['headers'], {'Content-Type': 'application/json'} - ) - self.assertEqual( - args[1]['timeout'], DEFAULT_TIMEOUT_SECONDS - ) + assert args[0][1] == url_prefix + 'containers/create' + assert json.loads(args[1]['data'])['Labels'] == labels_dict + assert args[1]['headers'] == {'Content-Type': 'application/json'} + assert args[1]['timeout'] == DEFAULT_TIMEOUT_SECONDS def test_create_container_with_labels_list(self): labels_list = [ @@ -986,14 +860,10 @@ def test_create_container_with_labels_list(self): ) args = fake_request.call_args - self.assertEqual(args[0][1], url_prefix + 'containers/create') - self.assertEqual(json.loads(args[1]['data'])['Labels'], labels_dict) - self.assertEqual( - args[1]['headers'], {'Content-Type': 'application/json'} - ) - self.assertEqual( - args[1]['timeout'], DEFAULT_TIMEOUT_SECONDS - ) + assert args[0][1] == url_prefix + 'containers/create' + assert json.loads(args[1]['data'])['Labels'] == labels_dict + assert args[1]['headers'] == {'Content-Type': 'application/json'} + assert args[1]['timeout'] == DEFAULT_TIMEOUT_SECONDS def test_create_container_with_named_volume(self): mount_dest = '/mnt' @@ -1010,39 +880,31 @@ def test_create_container_with_named_volume(self): ) args = fake_request.call_args - self.assertEqual( - args[0][1], url_prefix + 'containers/create' - ) + assert args[0][1] == url_prefix + 'containers/create' expected_payload = self.base_create_payload() expected_payload['HostConfig'] = self.client.create_host_config() expected_payload['HostConfig']['VolumeDriver'] = 'foodriver' expected_payload['HostConfig']['Binds'] = ["name:/mnt:rw"] - self.assertEqual(json.loads(args[1]['data']), expected_payload) - self.assertEqual(args[1]['headers'], - {'Content-Type': 'application/json'}) - self.assertEqual( - args[1]['timeout'], - DEFAULT_TIMEOUT_SECONDS - ) + assert json.loads(args[1]['data']) == expected_payload + assert args[1]['headers'] == {'Content-Type': 'application/json'} + assert args[1]['timeout'] == DEFAULT_TIMEOUT_SECONDS def test_create_container_with_stop_signal(self): self.client.create_container('busybox', 'ls', stop_signal='SIGINT') args = fake_request.call_args - self.assertEqual(args[0][1], - url_prefix + 'containers/create') - self.assertEqual(json.loads(args[1]['data']), - json.loads(''' - {"Tty": false, "Image": "busybox", - "Cmd": ["ls"], "AttachStdin": false, - "AttachStderr": true, - "AttachStdout": true, "OpenStdin": false, - "StdinOnce": false, - "NetworkDisabled": false, - "StopSignal": "SIGINT"}''')) - self.assertEqual(args[1]['headers'], - {'Content-Type': 'application/json'}) + assert args[0][1] == url_prefix + 'containers/create' + assert json.loads(args[1]['data']) == json.loads(''' + {"Tty": false, "Image": "busybox", + "Cmd": ["ls"], "AttachStdin": false, + "AttachStderr": true, + "AttachStdout": true, "OpenStdin": false, + "StdinOnce": false, + "NetworkDisabled": false, + "StopSignal": "SIGINT"} + ''') + assert args[1]['headers'] == {'Content-Type': 'application/json'} @requires_api_version('1.22') def test_create_container_with_aliases(self): @@ -1059,22 +921,22 @@ def test_create_container_with_aliases(self): ) args = fake_request.call_args - self.assertEqual(json.loads(args[1]['data']), - json.loads(''' - {"Tty": false, "Image": "busybox", - "Cmd": ["ls"], "AttachStdin": false, - "AttachStderr": true, - "AttachStdout": true, "OpenStdin": false, - "StdinOnce": false, - "NetworkDisabled": false, - "HostConfig": { - "NetworkMode": "some-network" - }, - "NetworkingConfig": { - "EndpointsConfig": { - "some-network": {"Aliases": ["foo", "bar"]} - } - }}''')) + assert json.loads(args[1]['data']) == json.loads(''' + {"Tty": false, "Image": "busybox", + "Cmd": ["ls"], "AttachStdin": false, + "AttachStderr": true, + "AttachStdout": true, "OpenStdin": false, + "StdinOnce": false, + "NetworkDisabled": false, + "HostConfig": { + "NetworkMode": "some-network" + }, + "NetworkingConfig": { + "EndpointsConfig": { + "some-network": {"Aliases": ["foo", "bar"]} + } + }} + ''') @requires_api_version('1.22') def test_create_container_with_tmpfs_list(self): @@ -1089,21 +951,16 @@ def test_create_container_with_tmpfs_list(self): ) args = fake_request.call_args - self.assertEqual(args[0][1], url_prefix + - 'containers/create') + assert args[0][1] == url_prefix + 'containers/create' expected_payload = self.base_create_payload() expected_payload['HostConfig'] = self.client.create_host_config() expected_payload['HostConfig']['Tmpfs'] = { "/tmp": "", "/mnt": "size=3G,uid=100" } - self.assertEqual(json.loads(args[1]['data']), expected_payload) - self.assertEqual(args[1]['headers'], - {'Content-Type': 'application/json'}) - self.assertEqual( - args[1]['timeout'], - DEFAULT_TIMEOUT_SECONDS - ) + assert json.loads(args[1]['data']) == expected_payload + assert args[1]['headers'] == {'Content-Type': 'application/json'} + assert args[1]['timeout'] == DEFAULT_TIMEOUT_SECONDS @requires_api_version('1.22') def test_create_container_with_tmpfs_dict(self): @@ -1118,21 +975,16 @@ def test_create_container_with_tmpfs_dict(self): ) args = fake_request.call_args - self.assertEqual(args[0][1], url_prefix + - 'containers/create') + assert args[0][1] == url_prefix + 'containers/create' expected_payload = self.base_create_payload() expected_payload['HostConfig'] = self.client.create_host_config() expected_payload['HostConfig']['Tmpfs'] = { "/tmp": "", "/mnt": "size=3G,uid=100" } - self.assertEqual(json.loads(args[1]['data']), expected_payload) - self.assertEqual(args[1]['headers'], - {'Content-Type': 'application/json'}) - self.assertEqual( - args[1]['timeout'], - DEFAULT_TIMEOUT_SECONDS - ) + assert json.loads(args[1]['data']) == expected_payload + assert args[1]['headers'] == {'Content-Type': 'application/json'} + assert args[1]['timeout'] == DEFAULT_TIMEOUT_SECONDS @requires_api_version('1.24') def test_create_container_with_sysctl(self): @@ -1147,19 +999,15 @@ def test_create_container_with_sysctl(self): ) args = fake_request.call_args - self.assertEqual(args[0][1], url_prefix + 'containers/create') + assert args[0][1] == url_prefix + 'containers/create' expected_payload = self.base_create_payload() expected_payload['HostConfig'] = self.client.create_host_config() expected_payload['HostConfig']['Sysctls'] = { 'net.core.somaxconn': '1024', 'net.ipv4.tcp_syncookies': '0', } - self.assertEqual(json.loads(args[1]['data']), expected_payload) - self.assertEqual( - args[1]['headers'], {'Content-Type': 'application/json'} - ) - self.assertEqual( - args[1]['timeout'], DEFAULT_TIMEOUT_SECONDS - ) + assert json.loads(args[1]['data']) == expected_payload + assert args[1]['headers'] == {'Content-Type': 'application/json'} + assert args[1]['timeout'] == DEFAULT_TIMEOUT_SECONDS def test_create_container_with_unicode_envvars(self): envvars_dict = { @@ -1176,8 +1024,8 @@ def test_create_container_with_unicode_envvars(self): ) args = fake_request.call_args - self.assertEqual(args[0][1], url_prefix + 'containers/create') - self.assertEqual(json.loads(args[1]['data'])['Env'], expected) + assert args[0][1] == url_prefix + 'containers/create' + assert json.loads(args[1]['data'])['Env'] == expected @requires_api_version('1.25') def test_create_container_with_host_config_cpus(self): @@ -1190,26 +1038,23 @@ def test_create_container_with_host_config_cpus(self): ) args = fake_request.call_args - self.assertEqual(args[0][1], - url_prefix + 'containers/create') - - self.assertEqual(json.loads(args[1]['data']), - json.loads(''' - {"Tty": false, "Image": "busybox", - "Cmd": ["ls"], "AttachStdin": false, - "AttachStderr": true, - "AttachStdout": true, "OpenStdin": false, - "StdinOnce": false, - "NetworkDisabled": false, - "HostConfig": { - "CpuCount": 1, - "CpuPercent": 20, - "NanoCpus": 1000, - "NetworkMode": "default" - }}''')) - self.assertEqual( - args[1]['headers'], {'Content-Type': 'application/json'} - ) + assert args[0][1] == url_prefix + 'containers/create' + + assert json.loads(args[1]['data']) == json.loads(''' + {"Tty": false, "Image": "busybox", + "Cmd": ["ls"], "AttachStdin": false, + "AttachStderr": true, + "AttachStdout": true, "OpenStdin": false, + "StdinOnce": false, + "NetworkDisabled": false, + "HostConfig": { + "CpuCount": 1, + "CpuPercent": 20, + "NanoCpus": 1000, + "NetworkMode": "default" + }} + ''') + assert args[1]['headers'] == {'Content-Type': 'application/json'} class ContainerTest(BaseAPIClientTest): @@ -1291,10 +1136,7 @@ def test_logs(self): stream=False ) - self.assertEqual( - logs, - 'Flowering Nights\n(Sakuya Iyazoi)\n'.encode('ascii') - ) + assert logs == 'Flowering Nights\n(Sakuya Iyazoi)\n'.encode('ascii') def test_logs_with_dict_instead_of_id(self): with mock.patch('docker.api.client.APIClient.inspect_container', @@ -1310,10 +1152,7 @@ def test_logs_with_dict_instead_of_id(self): stream=False ) - self.assertEqual( - logs, - 'Flowering Nights\n(Sakuya Iyazoi)\n'.encode('ascii') - ) + assert logs == 'Flowering Nights\n(Sakuya Iyazoi)\n'.encode('ascii') def test_log_streaming(self): with mock.patch('docker.api.client.APIClient.inspect_container', @@ -1426,7 +1265,7 @@ def test_log_since_with_datetime(self): def test_log_since_with_invalid_value_raises_error(self): with mock.patch('docker.api.client.APIClient.inspect_container', fake_inspect_container): - with self.assertRaises(docker.errors.InvalidArgument): + with pytest.raises(docker.errors.InvalidArgument): self.client.logs(fake_api.FAKE_CONTAINER_ID, stream=False, follow=False, since=42.42) @@ -1439,7 +1278,7 @@ def test_log_tty(self): self.client.logs(fake_api.FAKE_CONTAINER_ID, follow=True, stream=True) - self.assertTrue(m.called) + assert m.called fake_request.assert_called_with( 'GET', url_prefix + 'containers/3cc2351ab11b/logs', @@ -1623,9 +1462,7 @@ def test_inspect_container_undefined_id(self): with pytest.raises(docker.errors.NullResource) as excinfo: self.client.inspect_container(arg) - self.assertEqual( - excinfo.value.args[0], 'Resource ID was not provided' - ) + assert excinfo.value.args[0] == 'Resource ID was not provided' def test_container_stats(self): self.client.stats(fake_api.FAKE_CONTAINER_ID) @@ -1664,13 +1501,8 @@ def test_container_update(self): blkio_weight=345 ) args = fake_request.call_args - self.assertEqual( - args[0][1], url_prefix + 'containers/3cc2351ab11b/update' - ) - self.assertEqual( - json.loads(args[1]['data']), - {'Memory': 2 * 1024, 'CpuShares': 124, 'BlkioWeight': 345} - ) - self.assertEqual( - args[1]['headers']['Content-Type'], 'application/json' - ) + assert args[0][1] == url_prefix + 'containers/3cc2351ab11b/update' + assert json.loads(args[1]['data']) == { + 'Memory': 2 * 1024, 'CpuShares': 124, 'BlkioWeight': 345 + } + assert args[1]['headers']['Content-Type'] == 'application/json' diff --git a/tests/unit/api_exec_test.py b/tests/unit/api_exec_test.py index 41ee940a8a..a9d2dd5b65 100644 --- a/tests/unit/api_exec_test.py +++ b/tests/unit/api_exec_test.py @@ -11,85 +11,65 @@ def test_exec_create(self): self.client.exec_create(fake_api.FAKE_CONTAINER_ID, ['ls', '-1']) args = fake_request.call_args - self.assertEqual( - 'POST', - args[0][0], url_prefix + 'containers/{0}/exec'.format( - fake_api.FAKE_CONTAINER_ID - ) + assert 'POST' == args[0][0], url_prefix + 'containers/{0}/exec'.format( + fake_api.FAKE_CONTAINER_ID ) - self.assertEqual( - json.loads(args[1]['data']), { - 'Tty': False, - 'AttachStdout': True, - 'Container': fake_api.FAKE_CONTAINER_ID, - 'Cmd': ['ls', '-1'], - 'Privileged': False, - 'AttachStdin': False, - 'AttachStderr': True, - 'User': '' - } - ) + assert json.loads(args[1]['data']) == { + 'Tty': False, + 'AttachStdout': True, + 'Container': fake_api.FAKE_CONTAINER_ID, + 'Cmd': ['ls', '-1'], + 'Privileged': False, + 'AttachStdin': False, + 'AttachStderr': True, + 'User': '' + } - self.assertEqual(args[1]['headers'], - {'Content-Type': 'application/json'}) + assert args[1]['headers'] == {'Content-Type': 'application/json'} def test_exec_start(self): self.client.exec_start(fake_api.FAKE_EXEC_ID) args = fake_request.call_args - self.assertEqual( - args[0][1], url_prefix + 'exec/{0}/start'.format( - fake_api.FAKE_EXEC_ID - ) + assert args[0][1] == url_prefix + 'exec/{0}/start'.format( + fake_api.FAKE_EXEC_ID ) - self.assertEqual( - json.loads(args[1]['data']), { - 'Tty': False, - 'Detach': False, - } - ) + assert json.loads(args[1]['data']) == { + 'Tty': False, + 'Detach': False, + } - self.assertEqual( - args[1]['headers'], { - 'Content-Type': 'application/json', - 'Connection': 'Upgrade', - 'Upgrade': 'tcp' - } - ) + assert args[1]['headers'] == { + 'Content-Type': 'application/json', + 'Connection': 'Upgrade', + 'Upgrade': 'tcp' + } def test_exec_start_detached(self): self.client.exec_start(fake_api.FAKE_EXEC_ID, detach=True) args = fake_request.call_args - self.assertEqual( - args[0][1], url_prefix + 'exec/{0}/start'.format( - fake_api.FAKE_EXEC_ID - ) + assert args[0][1] == url_prefix + 'exec/{0}/start'.format( + fake_api.FAKE_EXEC_ID ) - self.assertEqual( - json.loads(args[1]['data']), { - 'Tty': False, - 'Detach': True - } - ) + assert json.loads(args[1]['data']) == { + 'Tty': False, + 'Detach': True + } - self.assertEqual( - args[1]['headers'], { - 'Content-Type': 'application/json' - } - ) + assert args[1]['headers'] == { + 'Content-Type': 'application/json' + } def test_exec_inspect(self): self.client.exec_inspect(fake_api.FAKE_EXEC_ID) args = fake_request.call_args - self.assertEqual( - args[0][1], url_prefix + 'exec/{0}/json'.format( - fake_api.FAKE_EXEC_ID - ) + assert args[0][1] == url_prefix + 'exec/{0}/json'.format( + fake_api.FAKE_EXEC_ID ) def test_exec_resize(self): diff --git a/tests/unit/api_image_test.py b/tests/unit/api_image_test.py index f1e42cc147..785f887240 100644 --- a/tests/unit/api_image_test.py +++ b/tests/unit/api_image_test.py @@ -65,29 +65,21 @@ def test_pull(self): self.client.pull('joffrey/test001') args = fake_request.call_args - self.assertEqual( - args[0][1], - url_prefix + 'images/create' - ) - self.assertEqual( - args[1]['params'], - {'tag': None, 'fromImage': 'joffrey/test001'} - ) - self.assertFalse(args[1]['stream']) + assert args[0][1] == url_prefix + 'images/create' + assert args[1]['params'] == { + 'tag': None, 'fromImage': 'joffrey/test001' + } + assert not args[1]['stream'] def test_pull_stream(self): self.client.pull('joffrey/test001', stream=True) args = fake_request.call_args - self.assertEqual( - args[0][1], - url_prefix + 'images/create' - ) - self.assertEqual( - args[1]['params'], - {'tag': None, 'fromImage': 'joffrey/test001'} - ) - self.assertTrue(args[1]['stream']) + assert args[0][1] == url_prefix + 'images/create' + assert args[1]['params'] == { + 'tag': None, 'fromImage': 'joffrey/test001' + } + assert args[1]['stream'] def test_commit(self): self.client.commit(fake_api.FAKE_CONTAINER_ID) @@ -203,18 +195,16 @@ def test_inspect_image_undefined_id(self): with pytest.raises(docker.errors.NullResource) as excinfo: self.client.inspect_image(arg) - self.assertEqual( - excinfo.value.args[0], 'Resource ID was not provided' - ) + assert excinfo.value.args[0] == 'Resource ID was not provided' def test_insert_image(self): try: self.client.insert(fake_api.FAKE_IMAGE_NAME, fake_api.FAKE_URL, fake_api.FAKE_PATH) except docker.errors.DeprecatedMethod: - self.assertTrue( - docker.utils.compare_version('1.12', self.client._version) >= 0 - ) + assert docker.utils.compare_version( + '1.12', self.client._version + ) >= 0 return fake_request.assert_called_with( diff --git a/tests/unit/api_network_test.py b/tests/unit/api_network_test.py index 96cdc4b194..fbbc97b0d4 100644 --- a/tests/unit/api_network_test.py +++ b/tests/unit/api_network_test.py @@ -34,20 +34,20 @@ def test_list_networks(self): status_code=200, content=json.dumps(networks).encode('utf-8'))) with mock.patch('docker.api.client.APIClient.get', get): - self.assertEqual(self.client.networks(), networks) + assert self.client.networks() == networks - self.assertEqual(get.call_args[0][0], url_prefix + 'networks') + assert get.call_args[0][0] == url_prefix + 'networks' filters = json.loads(get.call_args[1]['params']['filters']) - self.assertFalse(filters) + assert not filters self.client.networks(names=['foo']) filters = json.loads(get.call_args[1]['params']['filters']) - self.assertEqual(filters, {'name': ['foo']}) + assert filters == {'name': ['foo']} self.client.networks(ids=['123']) filters = json.loads(get.call_args[1]['params']['filters']) - self.assertEqual(filters, {'id': ['123']}) + assert filters == {'id': ['123']} @requires_api_version('1.21') def test_create_network(self): @@ -61,15 +61,11 @@ def test_create_network(self): with mock.patch('docker.api.client.APIClient.post', post): result = self.client.create_network('foo') - self.assertEqual(result, network_data) + assert result == network_data - self.assertEqual( - post.call_args[0][0], - url_prefix + 'networks/create') + assert post.call_args[0][0] == url_prefix + 'networks/create' - self.assertEqual( - json.loads(post.call_args[1]['data']), - {"Name": "foo"}) + assert json.loads(post.call_args[1]['data']) == {"Name": "foo"} opts = { 'com.docker.network.bridge.enable_icc': False, @@ -77,9 +73,9 @@ def test_create_network(self): } self.client.create_network('foo', 'bridge', opts) - self.assertEqual( - json.loads(post.call_args[1]['data']), - {"Name": "foo", "Driver": "bridge", "Options": opts}) + assert json.loads(post.call_args[1]['data']) == { + "Name": "foo", "Driver": "bridge", "Options": opts + } ipam_pool_config = IPAMPool(subnet="192.168.52.0/24", gateway="192.168.52.254") @@ -88,21 +84,19 @@ def test_create_network(self): self.client.create_network("bar", driver="bridge", ipam=ipam_config) - self.assertEqual( - json.loads(post.call_args[1]['data']), - { - "Name": "bar", - "Driver": "bridge", - "IPAM": { - "Driver": "default", - "Config": [{ - "IPRange": None, - "Gateway": "192.168.52.254", - "Subnet": "192.168.52.0/24", - "AuxiliaryAddresses": None, - }], - } - }) + assert json.loads(post.call_args[1]['data']) == { + "Name": "bar", + "Driver": "bridge", + "IPAM": { + "Driver": "default", + "Config": [{ + "IPRange": None, + "Gateway": "192.168.52.254", + "Subnet": "192.168.52.0/24", + "AuxiliaryAddresses": None, + }], + } + } @requires_api_version('1.21') def test_remove_network(self): @@ -113,8 +107,7 @@ def test_remove_network(self): self.client.remove_network(network_id) args = delete.call_args - self.assertEqual(args[0][0], - url_prefix + 'networks/{0}'.format(network_id)) + assert args[0][0] == url_prefix + 'networks/{0}'.format(network_id) @requires_api_version('1.21') def test_inspect_network(self): @@ -132,11 +125,10 @@ def test_inspect_network(self): with mock.patch('docker.api.client.APIClient.get', get): result = self.client.inspect_network(network_id) - self.assertEqual(result, network_data) + assert result == network_data args = get.call_args - self.assertEqual(args[0][0], - url_prefix + 'networks/{0}'.format(network_id)) + assert args[0][0] == url_prefix + 'networks/{0}'.format(network_id) @requires_api_version('1.21') def test_connect_container_to_network(self): @@ -153,19 +145,17 @@ def test_connect_container_to_network(self): links=[('baz', 'quux')] ) - self.assertEqual( - post.call_args[0][0], - url_prefix + 'networks/{0}/connect'.format(network_id)) + assert post.call_args[0][0] == ( + url_prefix + 'networks/{0}/connect'.format(network_id) + ) - self.assertEqual( - json.loads(post.call_args[1]['data']), - { - 'Container': container_id, - 'EndpointConfig': { - 'Aliases': ['foo', 'bar'], - 'Links': ['baz:quux'], - }, - }) + assert json.loads(post.call_args[1]['data']) == { + 'Container': container_id, + 'EndpointConfig': { + 'Aliases': ['foo', 'bar'], + 'Links': ['baz:quux'], + }, + } @requires_api_version('1.21') def test_disconnect_container_from_network(self): @@ -178,10 +168,9 @@ def test_disconnect_container_from_network(self): self.client.disconnect_container_from_network( container={'Id': container_id}, net_id=network_id) - self.assertEqual( - post.call_args[0][0], - url_prefix + 'networks/{0}/disconnect'.format(network_id)) - - self.assertEqual( - json.loads(post.call_args[1]['data']), - {'Container': container_id}) + assert post.call_args[0][0] == ( + url_prefix + 'networks/{0}/disconnect'.format(network_id) + ) + assert json.loads(post.call_args[1]['data']) == { + 'Container': container_id + } diff --git a/tests/unit/api_test.py b/tests/unit/api_test.py index 6ac92c4076..b9e0d5243f 100644 --- a/tests/unit/api_test.py +++ b/tests/unit/api_test.py @@ -128,34 +128,27 @@ def test_ctor(self): with pytest.raises(docker.errors.DockerException) as excinfo: APIClient(version=1.12) - self.assertEqual( - str(excinfo.value), - 'Version parameter must be a string or None. Found float' - ) + assert str( + excinfo.value + ) == 'Version parameter must be a string or None. Found float' def test_url_valid_resource(self): url = self.client._url('/hello/{0}/world', 'somename') - self.assertEqual( - url, '{0}{1}'.format(url_prefix, 'hello/somename/world') - ) + assert url == '{0}{1}'.format(url_prefix, 'hello/somename/world') url = self.client._url( '/hello/{0}/world/{1}', 'somename', 'someothername' ) - self.assertEqual( - url, - '{0}{1}'.format(url_prefix, 'hello/somename/world/someothername') + assert url == '{0}{1}'.format( + url_prefix, 'hello/somename/world/someothername' ) url = self.client._url('/hello/{0}/world', 'some?name') - self.assertEqual( - url, '{0}{1}'.format(url_prefix, 'hello/some%3Fname/world') - ) + assert url == '{0}{1}'.format(url_prefix, 'hello/some%3Fname/world') url = self.client._url("/images/{0}/push", "localhost:5000/image") - self.assertEqual( - url, - '{0}{1}'.format(url_prefix, 'images/localhost:5000/image/push') + assert url == '{0}{1}'.format( + url_prefix, 'images/localhost:5000/image/push' ) def test_url_invalid_resource(self): @@ -164,15 +157,13 @@ def test_url_invalid_resource(self): def test_url_no_resource(self): url = self.client._url('/simple') - self.assertEqual(url, '{0}{1}'.format(url_prefix, 'simple')) + assert url == '{0}{1}'.format(url_prefix, 'simple') def test_url_unversioned_api(self): url = self.client._url( '/hello/{0}/world', 'somename', versioned_api=False ) - self.assertEqual( - url, '{0}{1}'.format(url_base, 'hello/somename/world') - ) + assert url == '{0}{1}'.format(url_base, 'hello/somename/world') def test_version(self): self.client.version() @@ -194,13 +185,13 @@ def test_version_no_api_version(self): def test_retrieve_server_version(self): client = APIClient(version="auto") - self.assertTrue(isinstance(client._version, six.string_types)) - self.assertFalse(client._version == "auto") + assert isinstance(client._version, six.string_types) + assert not (client._version == "auto") client.close() def test_auto_retrieve_server_version(self): version = self.client._retrieve_server_version() - self.assertTrue(isinstance(version, six.string_types)) + assert isinstance(version, six.string_types) def test_info(self): self.client.info() @@ -313,11 +304,10 @@ def test_remove_link(self): def test_create_host_config_secopt(self): security_opt = ['apparmor:test_profile'] result = self.client.create_host_config(security_opt=security_opt) - self.assertIn('SecurityOpt', result) - self.assertEqual(result['SecurityOpt'], security_opt) - self.assertRaises( - TypeError, self.client.create_host_config, security_opt='wrong' - ) + assert 'SecurityOpt' in result + assert result['SecurityOpt'] == security_opt + with pytest.raises(TypeError): + self.client.create_host_config(security_opt='wrong') def test_stream_helper_decoding(self): status_code, content = fake_api.fake_responses[url_prefix + 'events']() @@ -335,26 +325,26 @@ def test_stream_helper_decoding(self): raw_resp._fp.seek(0) resp = response(status_code=status_code, content=content, raw=raw_resp) result = next(self.client._stream_helper(resp)) - self.assertEqual(result, content_str) + assert result == content_str # pass `decode=True` to the helper raw_resp._fp.seek(0) resp = response(status_code=status_code, content=content, raw=raw_resp) result = next(self.client._stream_helper(resp, decode=True)) - self.assertEqual(result, content) + assert result == content # non-chunked response, pass `decode=False` to the helper setattr(raw_resp._fp, 'chunked', False) raw_resp._fp.seek(0) resp = response(status_code=status_code, content=content, raw=raw_resp) result = next(self.client._stream_helper(resp)) - self.assertEqual(result, content_str.decode('utf-8')) + assert result == content_str.decode('utf-8') # non-chunked response, pass `decode=True` to the helper raw_resp._fp.seek(0) resp = response(status_code=status_code, content=content, raw=raw_resp) result = next(self.client._stream_helper(resp, decode=True)) - self.assertEqual(result, content) + assert result == content class StreamTest(unittest.TestCase): @@ -442,8 +432,7 @@ def test_early_stream_response(self): b'\r\n' ) + b'\r\n'.join(lines) - with APIClient(base_url="http+unix://" + self.socket_file) \ - as client: + with APIClient(base_url="http+unix://" + self.socket_file) as client: for i in range(5): try: stream = client.build( @@ -455,8 +444,8 @@ def test_early_stream_response(self): if i == 4: raise e - self.assertEqual(list(stream), [ - str(i).encode() for i in range(50)]) + assert list(stream) == [ + str(i).encode() for i in range(50)] class UserAgentTest(unittest.TestCase): @@ -475,18 +464,18 @@ def test_default_user_agent(self): client = APIClient() client.version() - self.assertEqual(self.mock_send.call_count, 1) + assert self.mock_send.call_count == 1 headers = self.mock_send.call_args[0][0].headers expected = 'docker-sdk-python/%s' % docker.__version__ - self.assertEqual(headers['User-Agent'], expected) + assert headers['User-Agent'] == expected def test_custom_user_agent(self): client = APIClient(user_agent='foo/bar') client.version() - self.assertEqual(self.mock_send.call_count, 1) + assert self.mock_send.call_count == 1 headers = self.mock_send.call_args[0][0].headers - self.assertEqual(headers['User-Agent'], 'foo/bar') + assert headers['User-Agent'] == 'foo/bar' class DisableSocketTest(unittest.TestCase): @@ -509,7 +498,7 @@ def test_disable_socket_timeout(self): self.client._disable_socket_timeout(socket) - self.assertEqual(socket.timeout, None) + assert socket.timeout is None def test_disable_socket_timeout2(self): """Test that the timeouts are disabled on a generic socket object @@ -519,8 +508,8 @@ def test_disable_socket_timeout2(self): self.client._disable_socket_timeout(socket) - self.assertEqual(socket.timeout, None) - self.assertEqual(socket._sock.timeout, None) + assert socket.timeout is None + assert socket._sock.timeout is None def test_disable_socket_timout_non_blocking(self): """Test that a non-blocking socket does not get set to blocking.""" @@ -529,5 +518,5 @@ def test_disable_socket_timout_non_blocking(self): self.client._disable_socket_timeout(socket) - self.assertEqual(socket.timeout, None) - self.assertEqual(socket._sock.timeout, 0.0) + assert socket.timeout is None + assert socket._sock.timeout == 0.0 diff --git a/tests/unit/api_volume_test.py b/tests/unit/api_volume_test.py index fc2a556d29..f5e5001123 100644 --- a/tests/unit/api_volume_test.py +++ b/tests/unit/api_volume_test.py @@ -10,12 +10,12 @@ class VolumeTest(BaseAPIClientTest): @requires_api_version('1.21') def test_list_volumes(self): volumes = self.client.volumes() - self.assertIn('Volumes', volumes) - self.assertEqual(len(volumes['Volumes']), 2) + assert 'Volumes' in volumes + assert len(volumes['Volumes']) == 2 args = fake_request.call_args - self.assertEqual(args[0][0], 'GET') - self.assertEqual(args[0][1], url_prefix + 'volumes') + assert args[0][0] == 'GET' + assert args[0][1] == url_prefix + 'volumes' @requires_api_version('1.21') def test_list_volumes_and_filters(self): @@ -33,25 +33,25 @@ def test_list_volumes_and_filters(self): def test_create_volume(self): name = 'perfectcherryblossom' result = self.client.create_volume(name) - self.assertIn('Name', result) - self.assertEqual(result['Name'], name) - self.assertIn('Driver', result) - self.assertEqual(result['Driver'], 'local') + assert 'Name' in result + assert result['Name'] == name + assert 'Driver' in result + assert result['Driver'] == 'local' args = fake_request.call_args - self.assertEqual(args[0][0], 'POST') - self.assertEqual(args[0][1], url_prefix + 'volumes/create') - self.assertEqual(json.loads(args[1]['data']), {'Name': name}) + assert args[0][0] == 'POST' + assert args[0][1] == url_prefix + 'volumes/create' + assert json.loads(args[1]['data']) == {'Name': name} @requires_api_version('1.23') def test_create_volume_with_labels(self): name = 'perfectcherryblossom' result = self.client.create_volume(name, labels={ - 'com.example.some-label': 'some-value'}) - self.assertEqual( - result["Labels"], - {'com.example.some-label': 'some-value'} - ) + 'com.example.some-label': 'some-value' + }) + assert result["Labels"] == { + 'com.example.some-label': 'some-value' + } @requires_api_version('1.23') def test_create_volume_with_invalid_labels(self): @@ -66,11 +66,11 @@ def test_create_volume_with_driver(self): self.client.create_volume(name, driver=driver_name) args = fake_request.call_args - self.assertEqual(args[0][0], 'POST') - self.assertEqual(args[0][1], url_prefix + 'volumes/create') + assert args[0][0] == 'POST' + assert args[0][1] == url_prefix + 'volumes/create' data = json.loads(args[1]['data']) - self.assertIn('Driver', data) - self.assertEqual(data['Driver'], driver_name) + assert 'Driver' in data + assert data['Driver'] == driver_name @requires_api_version('1.21') def test_create_volume_invalid_opts_type(self): @@ -92,25 +92,25 @@ def test_create_volume_invalid_opts_type(self): @requires_api_version('1.24') def test_create_volume_with_no_specified_name(self): result = self.client.create_volume(name=None) - self.assertIn('Name', result) - self.assertNotEqual(result['Name'], None) - self.assertIn('Driver', result) - self.assertEqual(result['Driver'], 'local') - self.assertIn('Scope', result) - self.assertEqual(result['Scope'], 'local') + assert 'Name' in result + assert result['Name'] is not None + assert 'Driver' in result + assert result['Driver'] == 'local' + assert 'Scope' in result + assert result['Scope'] == 'local' @requires_api_version('1.21') def test_inspect_volume(self): name = 'perfectcherryblossom' result = self.client.inspect_volume(name) - self.assertIn('Name', result) - self.assertEqual(result['Name'], name) - self.assertIn('Driver', result) - self.assertEqual(result['Driver'], 'local') + assert 'Name' in result + assert result['Name'] == name + assert 'Driver' in result + assert result['Driver'] == 'local' args = fake_request.call_args - self.assertEqual(args[0][0], 'GET') - self.assertEqual(args[0][1], '{0}volumes/{1}'.format(url_prefix, name)) + assert args[0][0] == 'GET' + assert args[0][1] == '{0}volumes/{1}'.format(url_prefix, name) @requires_api_version('1.21') def test_remove_volume(self): @@ -118,5 +118,5 @@ def test_remove_volume(self): self.client.remove_volume(name) args = fake_request.call_args - self.assertEqual(args[0][0], 'DELETE') - self.assertEqual(args[0][1], '{0}volumes/{1}'.format(url_prefix, name)) + assert args[0][0] == 'DELETE' + assert args[0][1] == '{0}volumes/{1}'.format(url_prefix, name) diff --git a/tests/unit/auth_test.py b/tests/unit/auth_test.py index 56fd50c250..1506ccbd74 100644 --- a/tests/unit/auth_test.py +++ b/tests/unit/auth_test.py @@ -13,6 +13,7 @@ from pytest import mark from docker import auth, errors +import pytest try: from unittest import mock @@ -33,82 +34,68 @@ def test_803_urlsafe_encode(self): class ResolveRepositoryNameTest(unittest.TestCase): def test_resolve_repository_name_hub_library_image(self): - self.assertEqual( - auth.resolve_repository_name('image'), - ('docker.io', 'image'), + assert auth.resolve_repository_name('image') == ( + 'docker.io', 'image' ) def test_resolve_repository_name_dotted_hub_library_image(self): - self.assertEqual( - auth.resolve_repository_name('image.valid'), - ('docker.io', 'image.valid') + assert auth.resolve_repository_name('image.valid') == ( + 'docker.io', 'image.valid' ) def test_resolve_repository_name_hub_image(self): - self.assertEqual( - auth.resolve_repository_name('username/image'), - ('docker.io', 'username/image'), + assert auth.resolve_repository_name('username/image') == ( + 'docker.io', 'username/image' ) def test_explicit_hub_index_library_image(self): - self.assertEqual( - auth.resolve_repository_name('docker.io/image'), - ('docker.io', 'image') + assert auth.resolve_repository_name('docker.io/image') == ( + 'docker.io', 'image' ) def test_explicit_legacy_hub_index_library_image(self): - self.assertEqual( - auth.resolve_repository_name('index.docker.io/image'), - ('docker.io', 'image') + assert auth.resolve_repository_name('index.docker.io/image') == ( + 'docker.io', 'image' ) def test_resolve_repository_name_private_registry(self): - self.assertEqual( - auth.resolve_repository_name('my.registry.net/image'), - ('my.registry.net', 'image'), + assert auth.resolve_repository_name('my.registry.net/image') == ( + 'my.registry.net', 'image' ) def test_resolve_repository_name_private_registry_with_port(self): - self.assertEqual( - auth.resolve_repository_name('my.registry.net:5000/image'), - ('my.registry.net:5000', 'image'), + assert auth.resolve_repository_name('my.registry.net:5000/image') == ( + 'my.registry.net:5000', 'image' ) def test_resolve_repository_name_private_registry_with_username(self): - self.assertEqual( - auth.resolve_repository_name('my.registry.net/username/image'), - ('my.registry.net', 'username/image'), - ) + assert auth.resolve_repository_name( + 'my.registry.net/username/image' + ) == ('my.registry.net', 'username/image') def test_resolve_repository_name_no_dots_but_port(self): - self.assertEqual( - auth.resolve_repository_name('hostname:5000/image'), - ('hostname:5000', 'image'), + assert auth.resolve_repository_name('hostname:5000/image') == ( + 'hostname:5000', 'image' ) def test_resolve_repository_name_no_dots_but_port_and_username(self): - self.assertEqual( - auth.resolve_repository_name('hostname:5000/username/image'), - ('hostname:5000', 'username/image'), - ) + assert auth.resolve_repository_name( + 'hostname:5000/username/image' + ) == ('hostname:5000', 'username/image') def test_resolve_repository_name_localhost(self): - self.assertEqual( - auth.resolve_repository_name('localhost/image'), - ('localhost', 'image'), + assert auth.resolve_repository_name('localhost/image') == ( + 'localhost', 'image' ) def test_resolve_repository_name_localhost_with_username(self): - self.assertEqual( - auth.resolve_repository_name('localhost/username/image'), - ('localhost', 'username/image'), + assert auth.resolve_repository_name('localhost/username/image') == ( + 'localhost', 'username/image' ) def test_invalid_index_name(self): - self.assertRaises( - errors.InvalidRepository, - lambda: auth.resolve_repository_name('-gecko.com/image') - ) + with pytest.raises(errors.InvalidRepository): + auth.resolve_repository_name('-gecko.com/image') def encode_auth(auth_info): @@ -129,147 +116,100 @@ class ResolveAuthTest(unittest.TestCase): }) def test_resolve_authconfig_hostname_only(self): - self.assertEqual( - auth.resolve_authconfig( - self.auth_config, 'my.registry.net' - )['username'], - 'privateuser' - ) + assert auth.resolve_authconfig( + self.auth_config, 'my.registry.net' + )['username'] == 'privateuser' def test_resolve_authconfig_no_protocol(self): - self.assertEqual( - auth.resolve_authconfig( - self.auth_config, 'my.registry.net/v1/' - )['username'], - 'privateuser' - ) + assert auth.resolve_authconfig( + self.auth_config, 'my.registry.net/v1/' + )['username'] == 'privateuser' def test_resolve_authconfig_no_path(self): - self.assertEqual( - auth.resolve_authconfig( - self.auth_config, 'http://my.registry.net' - )['username'], - 'privateuser' - ) + assert auth.resolve_authconfig( + self.auth_config, 'http://my.registry.net' + )['username'] == 'privateuser' def test_resolve_authconfig_no_path_trailing_slash(self): - self.assertEqual( - auth.resolve_authconfig( - self.auth_config, 'http://my.registry.net/' - )['username'], - 'privateuser' - ) + assert auth.resolve_authconfig( + self.auth_config, 'http://my.registry.net/' + )['username'] == 'privateuser' def test_resolve_authconfig_no_path_wrong_secure_proto(self): - self.assertEqual( - auth.resolve_authconfig( - self.auth_config, 'https://my.registry.net' - )['username'], - 'privateuser' - ) + assert auth.resolve_authconfig( + self.auth_config, 'https://my.registry.net' + )['username'] == 'privateuser' def test_resolve_authconfig_no_path_wrong_insecure_proto(self): - self.assertEqual( - auth.resolve_authconfig( - self.auth_config, 'http://index.docker.io' - )['username'], - 'indexuser' - ) + assert auth.resolve_authconfig( + self.auth_config, 'http://index.docker.io' + )['username'] == 'indexuser' def test_resolve_authconfig_path_wrong_proto(self): - self.assertEqual( - auth.resolve_authconfig( - self.auth_config, 'https://my.registry.net/v1/' - )['username'], - 'privateuser' - ) + assert auth.resolve_authconfig( + self.auth_config, 'https://my.registry.net/v1/' + )['username'] == 'privateuser' def test_resolve_authconfig_default_registry(self): - self.assertEqual( - auth.resolve_authconfig(self.auth_config)['username'], - 'indexuser' - ) + assert auth.resolve_authconfig( + self.auth_config + )['username'] == 'indexuser' def test_resolve_authconfig_default_explicit_none(self): - self.assertEqual( - auth.resolve_authconfig(self.auth_config, None)['username'], - 'indexuser' - ) + assert auth.resolve_authconfig( + self.auth_config, None + )['username'] == 'indexuser' def test_resolve_authconfig_fully_explicit(self): - self.assertEqual( - auth.resolve_authconfig( - self.auth_config, 'http://my.registry.net/v1/' - )['username'], - 'privateuser' - ) + assert auth.resolve_authconfig( + self.auth_config, 'http://my.registry.net/v1/' + )['username'] == 'privateuser' def test_resolve_authconfig_legacy_config(self): - self.assertEqual( - auth.resolve_authconfig( - self.auth_config, 'legacy.registry.url' - )['username'], - 'legacyauth' - ) + assert auth.resolve_authconfig( + self.auth_config, 'legacy.registry.url' + )['username'] == 'legacyauth' def test_resolve_authconfig_no_match(self): - self.assertTrue( - auth.resolve_authconfig(self.auth_config, 'does.not.exist') is None - ) + assert auth.resolve_authconfig( + self.auth_config, 'does.not.exist' + ) is None def test_resolve_registry_and_auth_library_image(self): image = 'image' - self.assertEqual( - auth.resolve_authconfig( - self.auth_config, auth.resolve_repository_name(image)[0] - )['username'], - 'indexuser', - ) + assert auth.resolve_authconfig( + self.auth_config, auth.resolve_repository_name(image)[0] + )['username'] == 'indexuser' def test_resolve_registry_and_auth_hub_image(self): image = 'username/image' - self.assertEqual( - auth.resolve_authconfig( - self.auth_config, auth.resolve_repository_name(image)[0] - )['username'], - 'indexuser', - ) + assert auth.resolve_authconfig( + self.auth_config, auth.resolve_repository_name(image)[0] + )['username'] == 'indexuser' def test_resolve_registry_and_auth_explicit_hub(self): image = 'docker.io/username/image' - self.assertEqual( - auth.resolve_authconfig( - self.auth_config, auth.resolve_repository_name(image)[0] - )['username'], - 'indexuser', - ) + assert auth.resolve_authconfig( + self.auth_config, auth.resolve_repository_name(image)[0] + )['username'] == 'indexuser' def test_resolve_registry_and_auth_explicit_legacy_hub(self): image = 'index.docker.io/username/image' - self.assertEqual( - auth.resolve_authconfig( - self.auth_config, auth.resolve_repository_name(image)[0] - )['username'], - 'indexuser', - ) + assert auth.resolve_authconfig( + self.auth_config, auth.resolve_repository_name(image)[0] + )['username'] == 'indexuser' def test_resolve_registry_and_auth_private_registry(self): image = 'my.registry.net/image' - self.assertEqual( - auth.resolve_authconfig( - self.auth_config, auth.resolve_repository_name(image)[0] - )['username'], - 'privateuser', - ) + assert auth.resolve_authconfig( + self.auth_config, auth.resolve_repository_name(image)[0] + )['username'] == 'privateuser' def test_resolve_registry_and_auth_unauthenticated_registry(self): image = 'other.registry.net/image' - self.assertEqual( - auth.resolve_authconfig( - self.auth_config, auth.resolve_repository_name(image)[0] - ), - None, - ) + assert auth.resolve_authconfig( + self.auth_config, auth.resolve_repository_name(image)[0] + ) is None class CredStoreTest(unittest.TestCase): @@ -378,7 +318,7 @@ def test_load_config_no_file(self): folder = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, folder) cfg = auth.load_config(folder) - self.assertTrue(cfg is not None) + assert cfg is not None def test_load_config(self): folder = tempfile.mkdtemp() @@ -390,12 +330,12 @@ def test_load_config(self): f.write('email = sakuya@scarlet.net') cfg = auth.load_config(dockercfg_path) assert auth.INDEX_NAME in cfg - self.assertNotEqual(cfg[auth.INDEX_NAME], None) + assert cfg[auth.INDEX_NAME] is not None cfg = cfg[auth.INDEX_NAME] - self.assertEqual(cfg['username'], 'sakuya') - self.assertEqual(cfg['password'], 'izayoi') - self.assertEqual(cfg['email'], 'sakuya@scarlet.net') - self.assertEqual(cfg.get('auth'), None) + assert cfg['username'] == 'sakuya' + assert cfg['password'] == 'izayoi' + assert cfg['email'] == 'sakuya@scarlet.net' + assert cfg.get('auth') is None def test_load_config_with_random_name(self): folder = tempfile.mkdtemp() @@ -418,12 +358,12 @@ def test_load_config_with_random_name(self): cfg = auth.load_config(dockercfg_path) assert registry in cfg - self.assertNotEqual(cfg[registry], None) + assert cfg[registry] is not None cfg = cfg[registry] - self.assertEqual(cfg['username'], 'sakuya') - self.assertEqual(cfg['password'], 'izayoi') - self.assertEqual(cfg['email'], 'sakuya@scarlet.net') - self.assertEqual(cfg.get('auth'), None) + assert cfg['username'] == 'sakuya' + assert cfg['password'] == 'izayoi' + assert cfg['email'] == 'sakuya@scarlet.net' + assert cfg.get('auth') is None def test_load_config_custom_config_env(self): folder = tempfile.mkdtemp() @@ -445,12 +385,12 @@ def test_load_config_custom_config_env(self): with mock.patch.dict(os.environ, {'DOCKER_CONFIG': folder}): cfg = auth.load_config(None) assert registry in cfg - self.assertNotEqual(cfg[registry], None) + assert cfg[registry] is not None cfg = cfg[registry] - self.assertEqual(cfg['username'], 'sakuya') - self.assertEqual(cfg['password'], 'izayoi') - self.assertEqual(cfg['email'], 'sakuya@scarlet.net') - self.assertEqual(cfg.get('auth'), None) + assert cfg['username'] == 'sakuya' + assert cfg['password'] == 'izayoi' + assert cfg['email'] == 'sakuya@scarlet.net' + assert cfg.get('auth') is None def test_load_config_custom_config_env_with_auths(self): folder = tempfile.mkdtemp() @@ -474,12 +414,12 @@ def test_load_config_custom_config_env_with_auths(self): with mock.patch.dict(os.environ, {'DOCKER_CONFIG': folder}): cfg = auth.load_config(None) assert registry in cfg - self.assertNotEqual(cfg[registry], None) + assert cfg[registry] is not None cfg = cfg[registry] - self.assertEqual(cfg['username'], 'sakuya') - self.assertEqual(cfg['password'], 'izayoi') - self.assertEqual(cfg['email'], 'sakuya@scarlet.net') - self.assertEqual(cfg.get('auth'), None) + assert cfg['username'] == 'sakuya' + assert cfg['password'] == 'izayoi' + assert cfg['email'] == 'sakuya@scarlet.net' + assert cfg.get('auth') is None def test_load_config_custom_config_env_utf8(self): folder = tempfile.mkdtemp() @@ -504,12 +444,12 @@ def test_load_config_custom_config_env_utf8(self): with mock.patch.dict(os.environ, {'DOCKER_CONFIG': folder}): cfg = auth.load_config(None) assert registry in cfg - self.assertNotEqual(cfg[registry], None) + assert cfg[registry] is not None cfg = cfg[registry] - self.assertEqual(cfg['username'], b'sakuya\xc3\xa6'.decode('utf8')) - self.assertEqual(cfg['password'], b'izayoi\xc3\xa6'.decode('utf8')) - self.assertEqual(cfg['email'], 'sakuya@scarlet.net') - self.assertEqual(cfg.get('auth'), None) + assert cfg['username'] == b'sakuya\xc3\xa6'.decode('utf8') + assert cfg['password'] == b'izayoi\xc3\xa6'.decode('utf8') + assert cfg['email'] == 'sakuya@scarlet.net' + assert cfg.get('auth') is None def test_load_config_custom_config_env_with_headers(self): folder = tempfile.mkdtemp() @@ -529,11 +469,11 @@ def test_load_config_custom_config_env_with_headers(self): with mock.patch.dict(os.environ, {'DOCKER_CONFIG': folder}): cfg = auth.load_config(None) assert 'HttpHeaders' in cfg - self.assertNotEqual(cfg['HttpHeaders'], None) + assert cfg['HttpHeaders'] is not None cfg = cfg['HttpHeaders'] - self.assertEqual(cfg['Name'], 'Spike') - self.assertEqual(cfg['Surname'], 'Spiegel') + assert cfg['Name'] == 'Spike' + assert cfg['Surname'] == 'Spiegel' def test_load_config_unknown_keys(self): folder = tempfile.mkdtemp() diff --git a/tests/unit/client_test.py b/tests/unit/client_test.py index c4996f1330..cce99c53ad 100644 --- a/tests/unit/client_test.py +++ b/tests/unit/client_test.py @@ -8,6 +8,7 @@ import unittest from . import fake_api +import pytest try: from unittest import mock @@ -51,25 +52,25 @@ def test_version(self, mock_func): def test_call_api_client_method(self): client = docker.from_env() - with self.assertRaises(AttributeError) as cm: + with pytest.raises(AttributeError) as cm: client.create_container() - s = str(cm.exception) + s = cm.exconly() assert "'DockerClient' object has no attribute 'create_container'" in s assert "this method is now on the object APIClient" in s - with self.assertRaises(AttributeError) as cm: + with pytest.raises(AttributeError) as cm: client.abcdef() - s = str(cm.exception) + s = cm.exconly() assert "'DockerClient' object has no attribute 'abcdef'" in s assert "this method is now on the object APIClient" not in s def test_call_containers(self): client = docker.DockerClient(**kwargs_from_env()) - with self.assertRaises(TypeError) as cm: + with pytest.raises(TypeError) as cm: client.containers() - s = str(cm.exception) + s = cm.exconly() assert "'ContainerCollection' object is not callable" in s assert "docker.APIClient" in s @@ -90,22 +91,22 @@ def test_from_env(self): DOCKER_CERT_PATH=TEST_CERT_DIR, DOCKER_TLS_VERIFY='1') client = docker.from_env() - self.assertEqual(client.api.base_url, "https://192.168.59.103:2376") + assert client.api.base_url == "https://192.168.59.103:2376" def test_from_env_with_version(self): os.environ.update(DOCKER_HOST='tcp://192.168.59.103:2376', DOCKER_CERT_PATH=TEST_CERT_DIR, DOCKER_TLS_VERIFY='1') client = docker.from_env(version='2.32') - self.assertEqual(client.api.base_url, "https://192.168.59.103:2376") - self.assertEqual(client.api._version, '2.32') + assert client.api.base_url == "https://192.168.59.103:2376" + assert client.api._version == '2.32' def test_from_env_without_version_uses_default(self): client = docker.from_env() - self.assertEqual(client.api._version, DEFAULT_DOCKER_API_VERSION) + assert client.api._version == DEFAULT_DOCKER_API_VERSION def test_from_env_without_timeout_uses_default(self): client = docker.from_env() - self.assertEqual(client.api.timeout, DEFAULT_TIMEOUT_SECONDS) + assert client.api.timeout == DEFAULT_TIMEOUT_SECONDS diff --git a/tests/unit/dockertypes_test.py b/tests/unit/dockertypes_test.py index 40adbb782f..93c1397209 100644 --- a/tests/unit/dockertypes_test.py +++ b/tests/unit/dockertypes_test.py @@ -25,11 +25,11 @@ def create_host_config(*args, **kwargs): class HostConfigTest(unittest.TestCase): def test_create_host_config_no_options(self): config = create_host_config(version='1.19') - self.assertFalse('NetworkMode' in config) + assert not ('NetworkMode' in config) def test_create_host_config_no_options_newer_api_version(self): config = create_host_config(version='1.20') - self.assertEqual(config['NetworkMode'], 'default') + assert config['NetworkMode'] == 'default' def test_create_host_config_invalid_cpu_cfs_types(self): with pytest.raises(TypeError): @@ -46,65 +46,58 @@ def test_create_host_config_invalid_cpu_cfs_types(self): def test_create_host_config_with_cpu_quota(self): config = create_host_config(version='1.20', cpu_quota=1999) - self.assertEqual(config.get('CpuQuota'), 1999) + assert config.get('CpuQuota') == 1999 def test_create_host_config_with_cpu_period(self): config = create_host_config(version='1.20', cpu_period=1999) - self.assertEqual(config.get('CpuPeriod'), 1999) + assert config.get('CpuPeriod') == 1999 def test_create_host_config_with_blkio_constraints(self): blkio_rate = [{"Path": "/dev/sda", "Rate": 1000}] - config = create_host_config(version='1.22', - blkio_weight=1999, - blkio_weight_device=blkio_rate, - device_read_bps=blkio_rate, - device_write_bps=blkio_rate, - device_read_iops=blkio_rate, - device_write_iops=blkio_rate) - - self.assertEqual(config.get('BlkioWeight'), 1999) - self.assertTrue(config.get('BlkioWeightDevice') is blkio_rate) - self.assertTrue(config.get('BlkioDeviceReadBps') is blkio_rate) - self.assertTrue(config.get('BlkioDeviceWriteBps') is blkio_rate) - self.assertTrue(config.get('BlkioDeviceReadIOps') is blkio_rate) - self.assertTrue(config.get('BlkioDeviceWriteIOps') is blkio_rate) - self.assertEqual(blkio_rate[0]['Path'], "/dev/sda") - self.assertEqual(blkio_rate[0]['Rate'], 1000) + config = create_host_config( + version='1.22', blkio_weight=1999, blkio_weight_device=blkio_rate, + device_read_bps=blkio_rate, device_write_bps=blkio_rate, + device_read_iops=blkio_rate, device_write_iops=blkio_rate + ) + + assert config.get('BlkioWeight') == 1999 + assert config.get('BlkioWeightDevice') is blkio_rate + assert config.get('BlkioDeviceReadBps') is blkio_rate + assert config.get('BlkioDeviceWriteBps') is blkio_rate + assert config.get('BlkioDeviceReadIOps') is blkio_rate + assert config.get('BlkioDeviceWriteIOps') is blkio_rate + assert blkio_rate[0]['Path'] == "/dev/sda" + assert blkio_rate[0]['Rate'] == 1000 def test_create_host_config_with_shm_size(self): config = create_host_config(version='1.22', shm_size=67108864) - self.assertEqual(config.get('ShmSize'), 67108864) + assert config.get('ShmSize') == 67108864 def test_create_host_config_with_shm_size_in_mb(self): config = create_host_config(version='1.22', shm_size='64M') - self.assertEqual(config.get('ShmSize'), 67108864) + assert config.get('ShmSize') == 67108864 def test_create_host_config_with_oom_kill_disable(self): config = create_host_config(version='1.20', oom_kill_disable=True) - self.assertEqual(config.get('OomKillDisable'), True) - self.assertRaises( - InvalidVersion, lambda: create_host_config(version='1.18.3', - oom_kill_disable=True)) + assert config.get('OomKillDisable') is True + with pytest.raises(InvalidVersion): + create_host_config(version='1.18.3', oom_kill_disable=True) def test_create_host_config_with_userns_mode(self): config = create_host_config(version='1.23', userns_mode='host') - self.assertEqual(config.get('UsernsMode'), 'host') - self.assertRaises( - InvalidVersion, lambda: create_host_config(version='1.22', - userns_mode='host')) - self.assertRaises( - ValueError, lambda: create_host_config(version='1.23', - userns_mode='host12')) + assert config.get('UsernsMode') == 'host' + with pytest.raises(InvalidVersion): + create_host_config(version='1.22', userns_mode='host') + with pytest.raises(ValueError): + create_host_config(version='1.23', userns_mode='host12') def test_create_host_config_with_oom_score_adj(self): config = create_host_config(version='1.22', oom_score_adj=100) - self.assertEqual(config.get('OomScoreAdj'), 100) - self.assertRaises( - InvalidVersion, lambda: create_host_config(version='1.21', - oom_score_adj=100)) - self.assertRaises( - TypeError, lambda: create_host_config(version='1.22', - oom_score_adj='100')) + assert config.get('OomScoreAdj') == 100 + with pytest.raises(InvalidVersion): + create_host_config(version='1.21', oom_score_adj=100) + with pytest.raises(TypeError): + create_host_config(version='1.22', oom_score_adj='100') def test_create_host_config_with_dns_opt(self): @@ -112,30 +105,27 @@ def test_create_host_config_with_dns_opt(self): config = create_host_config(version='1.21', dns_opt=tested_opts) dns_opts = config.get('DnsOptions') - self.assertTrue('use-vc' in dns_opts) - self.assertTrue('no-tld-query' in dns_opts) + assert 'use-vc' in dns_opts + assert 'no-tld-query' in dns_opts - self.assertRaises( - InvalidVersion, lambda: create_host_config(version='1.20', - dns_opt=tested_opts)) + with pytest.raises(InvalidVersion): + create_host_config(version='1.20', dns_opt=tested_opts) def test_create_host_config_with_mem_reservation(self): config = create_host_config(version='1.21', mem_reservation=67108864) - self.assertEqual(config.get('MemoryReservation'), 67108864) - self.assertRaises( - InvalidVersion, lambda: create_host_config( - version='1.20', mem_reservation=67108864)) + assert config.get('MemoryReservation') == 67108864 + with pytest.raises(InvalidVersion): + create_host_config(version='1.20', mem_reservation=67108864) def test_create_host_config_with_kernel_memory(self): config = create_host_config(version='1.21', kernel_memory=67108864) - self.assertEqual(config.get('KernelMemory'), 67108864) - self.assertRaises( - InvalidVersion, lambda: create_host_config( - version='1.20', kernel_memory=67108864)) + assert config.get('KernelMemory') == 67108864 + with pytest.raises(InvalidVersion): + create_host_config(version='1.20', kernel_memory=67108864) def test_create_host_config_with_pids_limit(self): config = create_host_config(version='1.23', pids_limit=1024) - self.assertEqual(config.get('PidsLimit'), 1024) + assert config.get('PidsLimit') == 1024 with pytest.raises(InvalidVersion): create_host_config(version='1.22', pids_limit=1024) @@ -144,7 +134,7 @@ def test_create_host_config_with_pids_limit(self): def test_create_host_config_with_isolation(self): config = create_host_config(version='1.24', isolation='hyperv') - self.assertEqual(config.get('Isolation'), 'hyperv') + assert config.get('Isolation') == 'hyperv' with pytest.raises(InvalidVersion): create_host_config(version='1.23', isolation='hyperv') @@ -179,10 +169,9 @@ def test_create_host_config_invalid_cpu_count_types(self): def test_create_host_config_with_cpu_count(self): config = create_host_config(version='1.25', cpu_count=2) - self.assertEqual(config.get('CpuCount'), 2) - self.assertRaises( - InvalidVersion, lambda: create_host_config( - version='1.24', cpu_count=1)) + assert config.get('CpuCount') == 2 + with pytest.raises(InvalidVersion): + create_host_config(version='1.24', cpu_count=1) def test_create_host_config_invalid_cpu_percent_types(self): with pytest.raises(TypeError): @@ -190,10 +179,9 @@ def test_create_host_config_invalid_cpu_percent_types(self): def test_create_host_config_with_cpu_percent(self): config = create_host_config(version='1.25', cpu_percent=15) - self.assertEqual(config.get('CpuPercent'), 15) - self.assertRaises( - InvalidVersion, lambda: create_host_config( - version='1.24', cpu_percent=10)) + assert config.get('CpuPercent') == 15 + with pytest.raises(InvalidVersion): + create_host_config(version='1.24', cpu_percent=10) def test_create_host_config_invalid_nano_cpus_types(self): with pytest.raises(TypeError): @@ -201,10 +189,9 @@ def test_create_host_config_invalid_nano_cpus_types(self): def test_create_host_config_with_nano_cpus(self): config = create_host_config(version='1.25', nano_cpus=1000) - self.assertEqual(config.get('NanoCpus'), 1000) - self.assertRaises( - InvalidVersion, lambda: create_host_config( - version='1.24', nano_cpus=1)) + assert config.get('NanoCpus') == 1000 + with pytest.raises(InvalidVersion): + create_host_config(version='1.24', nano_cpus=1) def test_create_host_config_with_cpu_rt_period_types(self): with pytest.raises(TypeError): @@ -212,10 +199,9 @@ def test_create_host_config_with_cpu_rt_period_types(self): def test_create_host_config_with_cpu_rt_period(self): config = create_host_config(version='1.25', cpu_rt_period=1000) - self.assertEqual(config.get('CPURealtimePeriod'), 1000) - self.assertRaises( - InvalidVersion, lambda: create_host_config( - version='1.24', cpu_rt_period=1000)) + assert config.get('CPURealtimePeriod') == 1000 + with pytest.raises(InvalidVersion): + create_host_config(version='1.24', cpu_rt_period=1000) def test_ctrate_host_config_with_cpu_rt_runtime_types(self): with pytest.raises(TypeError): @@ -223,10 +209,9 @@ def test_ctrate_host_config_with_cpu_rt_runtime_types(self): def test_create_host_config_with_cpu_rt_runtime(self): config = create_host_config(version='1.25', cpu_rt_runtime=1000) - self.assertEqual(config.get('CPURealtimeRuntime'), 1000) - self.assertRaises( - InvalidVersion, lambda: create_host_config( - version='1.24', cpu_rt_runtime=1000)) + assert config.get('CPURealtimeRuntime') == 1000 + with pytest.raises(InvalidVersion): + create_host_config(version='1.24', cpu_rt_runtime=1000) class ContainerConfigTest(unittest.TestCase): @@ -264,43 +249,46 @@ def test_create_host_config_dict_ulimit(self): config = create_host_config( ulimits=[ulimit_dct], version=DEFAULT_DOCKER_API_VERSION ) - self.assertIn('Ulimits', config) - self.assertEqual(len(config['Ulimits']), 1) + assert 'Ulimits' in config + assert len(config['Ulimits']) == 1 ulimit_obj = config['Ulimits'][0] - self.assertTrue(isinstance(ulimit_obj, Ulimit)) - self.assertEqual(ulimit_obj.name, ulimit_dct['name']) - self.assertEqual(ulimit_obj.soft, ulimit_dct['soft']) - self.assertEqual(ulimit_obj['Soft'], ulimit_obj.soft) + assert isinstance(ulimit_obj, Ulimit) + assert ulimit_obj.name == ulimit_dct['name'] + assert ulimit_obj.soft == ulimit_dct['soft'] + assert ulimit_obj['Soft'] == ulimit_obj.soft def test_create_host_config_dict_ulimit_capitals(self): ulimit_dct = {'Name': 'nofile', 'Soft': 8096, 'Hard': 8096 * 4} config = create_host_config( ulimits=[ulimit_dct], version=DEFAULT_DOCKER_API_VERSION ) - self.assertIn('Ulimits', config) - self.assertEqual(len(config['Ulimits']), 1) + assert 'Ulimits' in config + assert len(config['Ulimits']) == 1 ulimit_obj = config['Ulimits'][0] - self.assertTrue(isinstance(ulimit_obj, Ulimit)) - self.assertEqual(ulimit_obj.name, ulimit_dct['Name']) - self.assertEqual(ulimit_obj.soft, ulimit_dct['Soft']) - self.assertEqual(ulimit_obj.hard, ulimit_dct['Hard']) - self.assertEqual(ulimit_obj['Soft'], ulimit_obj.soft) + assert isinstance(ulimit_obj, Ulimit) + assert ulimit_obj.name == ulimit_dct['Name'] + assert ulimit_obj.soft == ulimit_dct['Soft'] + assert ulimit_obj.hard == ulimit_dct['Hard'] + assert ulimit_obj['Soft'] == ulimit_obj.soft def test_create_host_config_obj_ulimit(self): ulimit_dct = Ulimit(name='nofile', soft=8096) config = create_host_config( ulimits=[ulimit_dct], version=DEFAULT_DOCKER_API_VERSION ) - self.assertIn('Ulimits', config) - self.assertEqual(len(config['Ulimits']), 1) + assert 'Ulimits' in config + assert len(config['Ulimits']) == 1 ulimit_obj = config['Ulimits'][0] - self.assertTrue(isinstance(ulimit_obj, Ulimit)) - self.assertEqual(ulimit_obj, ulimit_dct) + assert isinstance(ulimit_obj, Ulimit) + assert ulimit_obj == ulimit_dct def test_ulimit_invalid_type(self): - self.assertRaises(ValueError, lambda: Ulimit(name=None)) - self.assertRaises(ValueError, lambda: Ulimit(name='hello', soft='123')) - self.assertRaises(ValueError, lambda: Ulimit(name='hello', hard='456')) + with pytest.raises(ValueError): + Ulimit(name=None) + with pytest.raises(ValueError): + Ulimit(name='hello', soft='123') + with pytest.raises(ValueError): + Ulimit(name='hello', hard='456') class LogConfigTest(unittest.TestCase): @@ -309,18 +297,18 @@ def test_create_host_config_dict_logconfig(self): config = create_host_config( version=DEFAULT_DOCKER_API_VERSION, log_config=dct ) - self.assertIn('LogConfig', config) - self.assertTrue(isinstance(config['LogConfig'], LogConfig)) - self.assertEqual(dct['type'], config['LogConfig'].type) + assert 'LogConfig' in config + assert isinstance(config['LogConfig'], LogConfig) + assert dct['type'] == config['LogConfig'].type def test_create_host_config_obj_logconfig(self): obj = LogConfig(type=LogConfig.types.SYSLOG, config={'key1': 'val1'}) config = create_host_config( version=DEFAULT_DOCKER_API_VERSION, log_config=obj ) - self.assertIn('LogConfig', config) - self.assertTrue(isinstance(config['LogConfig'], LogConfig)) - self.assertEqual(obj, config['LogConfig']) + assert 'LogConfig' in config + assert isinstance(config['LogConfig'], LogConfig) + assert obj == config['LogConfig'] def test_logconfig_invalid_config_type(self): with pytest.raises(ValueError): @@ -342,7 +330,7 @@ def test_create_ipam_config(self): gateway='192.168.52.254') ipam_config = IPAMConfig(pool_configs=[ipam_pool]) - self.assertEqual(ipam_config, { + assert ipam_config == { 'Driver': 'default', 'Config': [{ 'Subnet': '192.168.52.0/24', @@ -350,7 +338,7 @@ def test_create_ipam_config(self): 'AuxiliaryAddresses': None, 'IPRange': None, }] - }) + } class ServiceModeTest(unittest.TestCase): diff --git a/tests/unit/models_containers_test.py b/tests/unit/models_containers_test.py index d7457ba4a1..1fdd7a5c30 100644 --- a/tests/unit/models_containers_test.py +++ b/tests/unit/models_containers_test.py @@ -5,6 +5,7 @@ from .fake_api import FAKE_CONTAINER_ID, FAKE_IMAGE_ID, FAKE_EXEC_ID from .fake_api_client import make_fake_client +import pytest class ContainerCollectionTest(unittest.TestCase): @@ -232,10 +233,10 @@ def test_run_with_error(self): client.api.logs.return_value = "some error" client.api.wait.return_value = 1 - with self.assertRaises(docker.errors.ContainerError) as cm: + with pytest.raises(docker.errors.ContainerError) as cm: client.containers.run('alpine', 'echo hello world') - assert cm.exception.exit_status == 1 - assert "some error" in str(cm.exception) + assert cm.value.exit_status == 1 + assert "some error" in cm.exconly() def test_run_with_image_object(self): client = make_fake_client() @@ -257,7 +258,7 @@ def test_run_remove(self): client = make_fake_client() client.api.wait.return_value = 1 - with self.assertRaises(docker.errors.ContainerError): + with pytest.raises(docker.errors.ContainerError): client.containers.run("alpine") client.api.remove_container.assert_not_called() @@ -267,18 +268,18 @@ def test_run_remove(self): client = make_fake_client() client.api.wait.return_value = 1 - with self.assertRaises(docker.errors.ContainerError): + with pytest.raises(docker.errors.ContainerError): client.containers.run("alpine", remove=True) client.api.remove_container.assert_called_with(FAKE_CONTAINER_ID) client = make_fake_client() client.api._version = '1.24' - with self.assertRaises(RuntimeError): + with pytest.raises(RuntimeError): client.containers.run("alpine", detach=True, remove=True) client = make_fake_client() client.api._version = '1.23' - with self.assertRaises(RuntimeError): + with pytest.raises(RuntimeError): client.containers.run("alpine", detach=True, remove=True) client = make_fake_client() diff --git a/tests/unit/ssladapter_test.py b/tests/unit/ssladapter_test.py index 2b7ce52cb5..73b73360c0 100644 --- a/tests/unit/ssladapter_test.py +++ b/tests/unit/ssladapter_test.py @@ -1,5 +1,6 @@ import unittest from docker.transport import ssladapter +import pytest try: from backports.ssl_match_hostname import ( @@ -69,11 +70,9 @@ def test_match_dns_success(self): assert match_hostname(self.cert, 'touhou.gensokyo.jp') is None def test_match_ip_address_failure(self): - self.assertRaises( - CertificateError, match_hostname, self.cert, '192.168.0.25' - ) + with pytest.raises(CertificateError): + match_hostname(self.cert, '192.168.0.25') def test_match_dns_failure(self): - self.assertRaises( - CertificateError, match_hostname, self.cert, 'foobar.co.uk' - ) + with pytest.raises(CertificateError): + match_hostname(self.cert, 'foobar.co.uk') diff --git a/tests/unit/swarm_test.py b/tests/unit/swarm_test.py index 9a66c0c049..4385380281 100644 --- a/tests/unit/swarm_test.py +++ b/tests/unit/swarm_test.py @@ -21,15 +21,11 @@ def test_node_update(self): node_id=fake_api.FAKE_NODE_ID, version=1, node_spec=node_spec ) args = fake_request.call_args - self.assertEqual( - args[0][1], url_prefix + 'nodes/24ifsmvkjbyhk/update?version=1' - ) - self.assertEqual( - json.loads(args[1]['data']), node_spec - ) - self.assertEqual( - args[1]['headers']['Content-Type'], 'application/json' + assert args[0][1] == ( + url_prefix + 'nodes/24ifsmvkjbyhk/update?version=1' ) + assert json.loads(args[1]['data']) == node_spec + assert args[1]['headers']['Content-Type'] == 'application/json' @requires_api_version('1.24') def test_join_swarm(self): diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index 2fa1d051f2..230b1aa2a3 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -80,25 +80,25 @@ def test_kwargs_from_env_empty(self): os.environ.pop('DOCKER_TLS_VERIFY', None) kwargs = kwargs_from_env() - self.assertEqual(None, kwargs.get('base_url')) - self.assertEqual(None, kwargs.get('tls')) + assert kwargs.get('base_url') is None + assert kwargs.get('tls') is None def test_kwargs_from_env_tls(self): os.environ.update(DOCKER_HOST='tcp://192.168.59.103:2376', DOCKER_CERT_PATH=TEST_CERT_DIR, DOCKER_TLS_VERIFY='1') kwargs = kwargs_from_env(assert_hostname=False) - self.assertEqual('https://192.168.59.103:2376', kwargs['base_url']) - self.assertTrue('ca.pem' in kwargs['tls'].ca_cert) - self.assertTrue('cert.pem' in kwargs['tls'].cert[0]) - self.assertTrue('key.pem' in kwargs['tls'].cert[1]) - self.assertEqual(False, kwargs['tls'].assert_hostname) - self.assertTrue(kwargs['tls'].verify) + assert 'https://192.168.59.103:2376' == kwargs['base_url'] + assert 'ca.pem' in kwargs['tls'].ca_cert + assert 'cert.pem' in kwargs['tls'].cert[0] + assert 'key.pem' in kwargs['tls'].cert[1] + assert kwargs['tls'].assert_hostname is False + assert kwargs['tls'].verify try: client = APIClient(**kwargs) - self.assertEqual(kwargs['base_url'], client.base_url) - self.assertEqual(kwargs['tls'].ca_cert, client.verify) - self.assertEqual(kwargs['tls'].cert, client.cert) + assert kwargs['base_url'] == client.base_url + assert kwargs['tls'].ca_cert == client.verify + assert kwargs['tls'].cert == client.cert except TypeError as e: self.fail(e) @@ -107,17 +107,17 @@ def test_kwargs_from_env_tls_verify_false(self): DOCKER_CERT_PATH=TEST_CERT_DIR, DOCKER_TLS_VERIFY='') kwargs = kwargs_from_env(assert_hostname=True) - self.assertEqual('https://192.168.59.103:2376', kwargs['base_url']) - self.assertTrue('ca.pem' in kwargs['tls'].ca_cert) - self.assertTrue('cert.pem' in kwargs['tls'].cert[0]) - self.assertTrue('key.pem' in kwargs['tls'].cert[1]) - self.assertEqual(True, kwargs['tls'].assert_hostname) - self.assertEqual(False, kwargs['tls'].verify) + assert 'https://192.168.59.103:2376' == kwargs['base_url'] + assert 'ca.pem' in kwargs['tls'].ca_cert + assert 'cert.pem' in kwargs['tls'].cert[0] + assert 'key.pem' in kwargs['tls'].cert[1] + assert kwargs['tls'].assert_hostname is True + assert kwargs['tls'].verify is False try: client = APIClient(**kwargs) - self.assertEqual(kwargs['base_url'], client.base_url) - self.assertEqual(kwargs['tls'].cert, client.cert) - self.assertFalse(kwargs['tls'].verify) + assert kwargs['base_url'] == client.base_url + assert kwargs['tls'].cert == client.cert + assert not kwargs['tls'].verify except TypeError as e: self.fail(e) @@ -131,7 +131,7 @@ def test_kwargs_from_env_tls_verify_false_no_cert(self): DOCKER_TLS_VERIFY='') os.environ.pop('DOCKER_CERT_PATH', None) kwargs = kwargs_from_env(assert_hostname=True) - self.assertEqual('tcp://192.168.59.103:2376', kwargs['base_url']) + assert 'tcp://192.168.59.103:2376' == kwargs['base_url'] def test_kwargs_from_env_no_cert_path(self): try: @@ -144,10 +144,10 @@ def test_kwargs_from_env_no_cert_path(self): DOCKER_TLS_VERIFY='1') kwargs = kwargs_from_env() - self.assertTrue(kwargs['tls'].verify) - self.assertIn(cert_dir, kwargs['tls'].ca_cert) - self.assertIn(cert_dir, kwargs['tls'].cert[0]) - self.assertIn(cert_dir, kwargs['tls'].cert[1]) + assert kwargs['tls'].verify + assert cert_dir in kwargs['tls'].ca_cert + assert cert_dir in kwargs['tls'].cert[0] + assert cert_dir in kwargs['tls'].cert[1] finally: if temp_dir: shutil.rmtree(temp_dir) @@ -169,12 +169,12 @@ def test_kwargs_from_env_alternate_env(self): class ConverVolumeBindsTest(unittest.TestCase): def test_convert_volume_binds_empty(self): - self.assertEqual(convert_volume_binds({}), []) - self.assertEqual(convert_volume_binds([]), []) + assert convert_volume_binds({}) == [] + assert convert_volume_binds([]) == [] def test_convert_volume_binds_list(self): data = ['/a:/a:ro', '/b:/c:z'] - self.assertEqual(convert_volume_binds(data), data) + assert convert_volume_binds(data) == data def test_convert_volume_binds_complete(self): data = { @@ -183,13 +183,13 @@ def test_convert_volume_binds_complete(self): 'mode': 'ro' } } - self.assertEqual(convert_volume_binds(data), ['/mnt/vol1:/data:ro']) + assert convert_volume_binds(data) == ['/mnt/vol1:/data:ro'] def test_convert_volume_binds_compact(self): data = { '/mnt/vol1': '/data' } - self.assertEqual(convert_volume_binds(data), ['/mnt/vol1:/data:rw']) + assert convert_volume_binds(data) == ['/mnt/vol1:/data:rw'] def test_convert_volume_binds_no_mode(self): data = { @@ -197,7 +197,7 @@ def test_convert_volume_binds_no_mode(self): 'bind': '/data' } } - self.assertEqual(convert_volume_binds(data), ['/mnt/vol1:/data:rw']) + assert convert_volume_binds(data) == ['/mnt/vol1:/data:rw'] def test_convert_volume_binds_unicode_bytes_input(self): expected = [u'/mnt/지연:/unicode/박:rw'] @@ -208,9 +208,7 @@ def test_convert_volume_binds_unicode_bytes_input(self): 'mode': 'rw' } } - self.assertEqual( - convert_volume_binds(data), expected - ) + assert convert_volume_binds(data) == expected def test_convert_volume_binds_unicode_unicode_input(self): expected = [u'/mnt/지연:/unicode/박:rw'] @@ -221,9 +219,7 @@ def test_convert_volume_binds_unicode_unicode_input(self): 'mode': 'rw' } } - self.assertEqual( - convert_volume_binds(data), expected - ) + assert convert_volume_binds(data) == expected class ParseEnvFileTest(unittest.TestCase): @@ -242,38 +238,35 @@ def test_parse_env_file_proper(self): env_file = self.generate_tempfile( file_content='USER=jdoe\nPASS=secret') get_parse_env_file = parse_env_file(env_file) - self.assertEqual(get_parse_env_file, - {'USER': 'jdoe', 'PASS': 'secret'}) + assert get_parse_env_file == {'USER': 'jdoe', 'PASS': 'secret'} os.unlink(env_file) def test_parse_env_file_with_equals_character(self): env_file = self.generate_tempfile( file_content='USER=jdoe\nPASS=sec==ret') get_parse_env_file = parse_env_file(env_file) - self.assertEqual(get_parse_env_file, - {'USER': 'jdoe', 'PASS': 'sec==ret'}) + assert get_parse_env_file == {'USER': 'jdoe', 'PASS': 'sec==ret'} os.unlink(env_file) def test_parse_env_file_commented_line(self): env_file = self.generate_tempfile( file_content='USER=jdoe\n#PASS=secret') get_parse_env_file = parse_env_file(env_file) - self.assertEqual(get_parse_env_file, {'USER': 'jdoe'}) + assert get_parse_env_file == {'USER': 'jdoe'} os.unlink(env_file) def test_parse_env_file_newline(self): env_file = self.generate_tempfile( file_content='\nUSER=jdoe\n\n\nPASS=secret') get_parse_env_file = parse_env_file(env_file) - self.assertEqual(get_parse_env_file, - {'USER': 'jdoe', 'PASS': 'secret'}) + assert get_parse_env_file == {'USER': 'jdoe', 'PASS': 'secret'} os.unlink(env_file) def test_parse_env_file_invalid_line(self): env_file = self.generate_tempfile( file_content='USER jdoe') - self.assertRaises( - DockerException, parse_env_file, env_file) + with pytest.raises(DockerException): + parse_env_file(env_file) os.unlink(env_file) @@ -343,46 +336,34 @@ class ParseRepositoryTagTest(unittest.TestCase): sha = 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855' def test_index_image_no_tag(self): - self.assertEqual( - parse_repository_tag("root"), ("root", None) - ) + assert parse_repository_tag("root") == ("root", None) def test_index_image_tag(self): - self.assertEqual( - parse_repository_tag("root:tag"), ("root", "tag") - ) + assert parse_repository_tag("root:tag") == ("root", "tag") def test_index_user_image_no_tag(self): - self.assertEqual( - parse_repository_tag("user/repo"), ("user/repo", None) - ) + assert parse_repository_tag("user/repo") == ("user/repo", None) def test_index_user_image_tag(self): - self.assertEqual( - parse_repository_tag("user/repo:tag"), ("user/repo", "tag") - ) + assert parse_repository_tag("user/repo:tag") == ("user/repo", "tag") def test_private_reg_image_no_tag(self): - self.assertEqual( - parse_repository_tag("url:5000/repo"), ("url:5000/repo", None) - ) + assert parse_repository_tag("url:5000/repo") == ("url:5000/repo", None) def test_private_reg_image_tag(self): - self.assertEqual( - parse_repository_tag("url:5000/repo:tag"), ("url:5000/repo", "tag") + assert parse_repository_tag("url:5000/repo:tag") == ( + "url:5000/repo", "tag" ) def test_index_image_sha(self): - self.assertEqual( - parse_repository_tag("root@sha256:{0}".format(self.sha)), - ("root", "sha256:{0}".format(self.sha)) + assert parse_repository_tag("root@sha256:{0}".format(self.sha)) == ( + "root", "sha256:{0}".format(self.sha) ) def test_private_reg_image_sha(self): - self.assertEqual( - parse_repository_tag("url:5000/repo@sha256:{0}".format(self.sha)), - ("url:5000/repo", "sha256:{0}".format(self.sha)) - ) + assert parse_repository_tag( + "url:5000/repo@sha256:{0}".format(self.sha) + ) == ("url:5000/repo", "sha256:{0}".format(self.sha)) class ParseDeviceTest(unittest.TestCase): @@ -392,35 +373,35 @@ def test_dict(self): 'PathInContainer': '/dev/mnt1', 'CgroupPermissions': 'r' }]) - self.assertEqual(devices[0], { + assert devices[0] == { 'PathOnHost': '/dev/sda1', 'PathInContainer': '/dev/mnt1', 'CgroupPermissions': 'r' - }) + } def test_partial_string_definition(self): devices = parse_devices(['/dev/sda1']) - self.assertEqual(devices[0], { + assert devices[0] == { 'PathOnHost': '/dev/sda1', 'PathInContainer': '/dev/sda1', 'CgroupPermissions': 'rwm' - }) + } def test_permissionless_string_definition(self): devices = parse_devices(['/dev/sda1:/dev/mnt1']) - self.assertEqual(devices[0], { + assert devices[0] == { 'PathOnHost': '/dev/sda1', 'PathInContainer': '/dev/mnt1', 'CgroupPermissions': 'rwm' - }) + } def test_full_string_definition(self): devices = parse_devices(['/dev/sda1:/dev/mnt1:r']) - self.assertEqual(devices[0], { + assert devices[0] == { 'PathOnHost': '/dev/sda1', 'PathInContainer': '/dev/mnt1', 'CgroupPermissions': 'r' - }) + } def test_hybrid_list(self): devices = parse_devices([ @@ -432,36 +413,38 @@ def test_hybrid_list(self): } ]) - self.assertEqual(devices[0], { + assert devices[0] == { 'PathOnHost': '/dev/sda1', 'PathInContainer': '/dev/mnt1', 'CgroupPermissions': 'rw' - }) - self.assertEqual(devices[1], { + } + assert devices[1] == { 'PathOnHost': '/dev/sda2', 'PathInContainer': '/dev/mnt2', 'CgroupPermissions': 'r' - }) + } class ParseBytesTest(unittest.TestCase): def test_parse_bytes_valid(self): - self.assertEqual(parse_bytes("512MB"), 536870912) - self.assertEqual(parse_bytes("512M"), 536870912) - self.assertEqual(parse_bytes("512m"), 536870912) + assert parse_bytes("512MB") == 536870912 + assert parse_bytes("512M") == 536870912 + assert parse_bytes("512m") == 536870912 def test_parse_bytes_invalid(self): - self.assertRaises(DockerException, parse_bytes, "512MK") - self.assertRaises(DockerException, parse_bytes, "512L") - self.assertRaises(DockerException, parse_bytes, "127.0.0.1K") + with pytest.raises(DockerException): + parse_bytes("512MK") + with pytest.raises(DockerException): + parse_bytes("512L") + with pytest.raises(DockerException): + parse_bytes("127.0.0.1K") def test_parse_bytes_float(self): - self.assertRaises(DockerException, parse_bytes, "1.5k") + with pytest.raises(DockerException): + parse_bytes("1.5k") def test_parse_bytes_maxint(self): - self.assertEqual( - parse_bytes("{0}k".format(sys.maxsize)), sys.maxsize * 1024 - ) + assert parse_bytes("{0}k".format(sys.maxsize)) == sys.maxsize * 1024 class UtilsTest(unittest.TestCase): @@ -476,7 +459,7 @@ def test_convert_filters(self): ] for filters, expected in tests: - self.assertEqual(convert_filters(filters), expected) + assert convert_filters(filters) == expected def test_decode_json_header(self): obj = {'a': 'b', 'c': 1} @@ -486,144 +469,144 @@ def test_decode_json_header(self): else: data = base64.urlsafe_b64encode(json.dumps(obj)) decoded_data = decode_json_header(data) - self.assertEqual(obj, decoded_data) + assert obj == decoded_data class SplitCommandTest(unittest.TestCase): def test_split_command_with_unicode(self): - self.assertEqual(split_command(u'echo μμ'), ['echo', 'μμ']) + assert split_command(u'echo μμ') == ['echo', 'μμ'] @pytest.mark.skipif(six.PY3, reason="shlex doesn't support bytes in py3") def test_split_command_with_bytes(self): - self.assertEqual(split_command('echo μμ'), ['echo', 'μμ']) + assert split_command('echo μμ') == ['echo', 'μμ'] class PortsTest(unittest.TestCase): def test_split_port_with_host_ip(self): internal_port, external_port = split_port("127.0.0.1:1000:2000") - self.assertEqual(internal_port, ["2000"]) - self.assertEqual(external_port, [("127.0.0.1", "1000")]) + assert internal_port == ["2000"] + assert external_port == [("127.0.0.1", "1000")] def test_split_port_with_protocol(self): internal_port, external_port = split_port("127.0.0.1:1000:2000/udp") - self.assertEqual(internal_port, ["2000/udp"]) - self.assertEqual(external_port, [("127.0.0.1", "1000")]) + assert internal_port == ["2000/udp"] + assert external_port == [("127.0.0.1", "1000")] def test_split_port_with_host_ip_no_port(self): internal_port, external_port = split_port("127.0.0.1::2000") - self.assertEqual(internal_port, ["2000"]) - self.assertEqual(external_port, [("127.0.0.1", None)]) + assert internal_port == ["2000"] + assert external_port == [("127.0.0.1", None)] def test_split_port_range_with_host_ip_no_port(self): internal_port, external_port = split_port("127.0.0.1::2000-2001") - self.assertEqual(internal_port, ["2000", "2001"]) - self.assertEqual(external_port, - [("127.0.0.1", None), ("127.0.0.1", None)]) + assert internal_port == ["2000", "2001"] + assert external_port == [("127.0.0.1", None), ("127.0.0.1", None)] def test_split_port_with_host_port(self): internal_port, external_port = split_port("1000:2000") - self.assertEqual(internal_port, ["2000"]) - self.assertEqual(external_port, ["1000"]) + assert internal_port == ["2000"] + assert external_port == ["1000"] def test_split_port_range_with_host_port(self): internal_port, external_port = split_port("1000-1001:2000-2001") - self.assertEqual(internal_port, ["2000", "2001"]) - self.assertEqual(external_port, ["1000", "1001"]) + assert internal_port == ["2000", "2001"] + assert external_port == ["1000", "1001"] def test_split_port_random_port_range_with_host_port(self): internal_port, external_port = split_port("1000-1001:2000") - self.assertEqual(internal_port, ["2000"]) - self.assertEqual(external_port, ["1000-1001"]) + assert internal_port == ["2000"] + assert external_port == ["1000-1001"] def test_split_port_no_host_port(self): internal_port, external_port = split_port("2000") - self.assertEqual(internal_port, ["2000"]) - self.assertEqual(external_port, None) + assert internal_port == ["2000"] + assert external_port is None def test_split_port_range_no_host_port(self): internal_port, external_port = split_port("2000-2001") - self.assertEqual(internal_port, ["2000", "2001"]) - self.assertEqual(external_port, None) + assert internal_port == ["2000", "2001"] + assert external_port is None def test_split_port_range_with_protocol(self): internal_port, external_port = split_port( "127.0.0.1:1000-1001:2000-2001/udp") - self.assertEqual(internal_port, ["2000/udp", "2001/udp"]) - self.assertEqual(external_port, - [("127.0.0.1", "1000"), ("127.0.0.1", "1001")]) + assert internal_port == ["2000/udp", "2001/udp"] + assert external_port == [("127.0.0.1", "1000"), ("127.0.0.1", "1001")] def test_split_port_with_ipv6_address(self): internal_port, external_port = split_port( "2001:abcd:ef00::2:1000:2000") - self.assertEqual(internal_port, ["2000"]) - self.assertEqual(external_port, [("2001:abcd:ef00::2", "1000")]) + assert internal_port == ["2000"] + assert external_port == [("2001:abcd:ef00::2", "1000")] def test_split_port_invalid(self): - self.assertRaises(ValueError, - lambda: split_port("0.0.0.0:1000:2000:tcp")) + with pytest.raises(ValueError): + split_port("0.0.0.0:1000:2000:tcp") def test_non_matching_length_port_ranges(self): - self.assertRaises( - ValueError, - lambda: split_port("0.0.0.0:1000-1010:2000-2002/tcp") - ) + with pytest.raises(ValueError): + split_port("0.0.0.0:1000-1010:2000-2002/tcp") def test_port_and_range_invalid(self): - self.assertRaises(ValueError, - lambda: split_port("0.0.0.0:1000:2000-2002/tcp")) + with pytest.raises(ValueError): + split_port("0.0.0.0:1000:2000-2002/tcp") def test_port_only_with_colon(self): - self.assertRaises(ValueError, - lambda: split_port(":80")) + with pytest.raises(ValueError): + split_port(":80") def test_host_only_with_colon(self): - self.assertRaises(ValueError, - lambda: split_port("localhost:")) + with pytest.raises(ValueError): + split_port("localhost:") def test_with_no_container_port(self): - self.assertRaises(ValueError, - lambda: split_port("localhost:80:")) + with pytest.raises(ValueError): + split_port("localhost:80:") def test_split_port_empty_string(self): - self.assertRaises(ValueError, lambda: split_port("")) + with pytest.raises(ValueError): + split_port("") def test_split_port_non_string(self): assert split_port(1243) == (['1243'], None) def test_build_port_bindings_with_one_port(self): port_bindings = build_port_bindings(["127.0.0.1:1000:1000"]) - self.assertEqual(port_bindings["1000"], [("127.0.0.1", "1000")]) + assert port_bindings["1000"] == [("127.0.0.1", "1000")] def test_build_port_bindings_with_matching_internal_ports(self): port_bindings = build_port_bindings( ["127.0.0.1:1000:1000", "127.0.0.1:2000:1000"]) - self.assertEqual(port_bindings["1000"], - [("127.0.0.1", "1000"), ("127.0.0.1", "2000")]) + assert port_bindings["1000"] == [ + ("127.0.0.1", "1000"), ("127.0.0.1", "2000") + ] def test_build_port_bindings_with_nonmatching_internal_ports(self): port_bindings = build_port_bindings( ["127.0.0.1:1000:1000", "127.0.0.1:2000:2000"]) - self.assertEqual(port_bindings["1000"], [("127.0.0.1", "1000")]) - self.assertEqual(port_bindings["2000"], [("127.0.0.1", "2000")]) + assert port_bindings["1000"] == [("127.0.0.1", "1000")] + assert port_bindings["2000"] == [("127.0.0.1", "2000")] def test_build_port_bindings_with_port_range(self): port_bindings = build_port_bindings(["127.0.0.1:1000-1001:1000-1001"]) - self.assertEqual(port_bindings["1000"], [("127.0.0.1", "1000")]) - self.assertEqual(port_bindings["1001"], [("127.0.0.1", "1001")]) + assert port_bindings["1000"] == [("127.0.0.1", "1000")] + assert port_bindings["1001"] == [("127.0.0.1", "1001")] def test_build_port_bindings_with_matching_internal_port_ranges(self): port_bindings = build_port_bindings( ["127.0.0.1:1000-1001:1000-1001", "127.0.0.1:2000-2001:1000-1001"]) - self.assertEqual(port_bindings["1000"], - [("127.0.0.1", "1000"), ("127.0.0.1", "2000")]) - self.assertEqual(port_bindings["1001"], - [("127.0.0.1", "1001"), ("127.0.0.1", "2001")]) + assert port_bindings["1000"] == [ + ("127.0.0.1", "1000"), ("127.0.0.1", "2000") + ] + assert port_bindings["1001"] == [ + ("127.0.0.1", "1001"), ("127.0.0.1", "2001") + ] def test_build_port_bindings_with_nonmatching_internal_port_ranges(self): port_bindings = build_port_bindings( ["127.0.0.1:1000:1000", "127.0.0.1:2000:2000"]) - self.assertEqual(port_bindings["1000"], [("127.0.0.1", "1000")]) - self.assertEqual(port_bindings["2000"], [("127.0.0.1", "2000")]) + assert port_bindings["1000"] == [("127.0.0.1", "1000")] + assert port_bindings["2000"] == [("127.0.0.1", "2000")] def convert_paths(collection): @@ -708,11 +691,13 @@ def test_exclude_custom_dockerfile(self): If we're using a custom Dockerfile, make sure that's not excluded. """ - assert self.exclude(['*'], dockerfile='Dockerfile.alt') == \ - set(['Dockerfile.alt', '.dockerignore']) + assert self.exclude(['*'], dockerfile='Dockerfile.alt') == set( + ['Dockerfile.alt', '.dockerignore'] + ) - assert self.exclude(['*'], dockerfile='foo/Dockerfile3') == \ - convert_paths(set(['foo/Dockerfile3', '.dockerignore'])) + assert self.exclude( + ['*'], dockerfile='foo/Dockerfile3' + ) == convert_paths(set(['foo/Dockerfile3', '.dockerignore'])) def test_exclude_dockerfile_child(self): includes = self.exclude(['foo/'], dockerfile='foo/Dockerfile3') @@ -946,7 +931,7 @@ def test_tar_with_empty_directory(self): os.makedirs(os.path.join(base, d)) with tar(base) as archive: tar_data = tarfile.open(fileobj=archive) - self.assertEqual(sorted(tar_data.getnames()), ['bar', 'foo']) + assert sorted(tar_data.getnames()) == ['bar', 'foo'] @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='No symlinks on Windows') def test_tar_with_file_symlinks(self): @@ -958,9 +943,7 @@ def test_tar_with_file_symlinks(self): os.symlink('../foo', os.path.join(base, 'bar/foo')) with tar(base) as archive: tar_data = tarfile.open(fileobj=archive) - self.assertEqual( - sorted(tar_data.getnames()), ['bar', 'bar/foo', 'foo'] - ) + assert sorted(tar_data.getnames()) == ['bar', 'bar/foo', 'foo'] @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='No symlinks on Windows') def test_tar_with_directory_symlinks(self): @@ -971,9 +954,7 @@ def test_tar_with_directory_symlinks(self): os.symlink('../foo', os.path.join(base, 'bar/foo')) with tar(base) as archive: tar_data = tarfile.open(fileobj=archive) - self.assertEqual( - sorted(tar_data.getnames()), ['bar', 'bar/foo', 'foo'] - ) + assert sorted(tar_data.getnames()) == ['bar', 'bar/foo', 'foo'] @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='No UNIX sockets on Win32') def test_tar_socket_file(self): @@ -986,9 +967,7 @@ def test_tar_socket_file(self): sock.bind(os.path.join(base, 'test.sock')) with tar(base) as archive: tar_data = tarfile.open(fileobj=archive) - self.assertEqual( - sorted(tar_data.getnames()), ['bar', 'foo'] - ) + assert sorted(tar_data.getnames()) == ['bar', 'foo'] class ShouldCheckDirectoryTest(unittest.TestCase): From 17aa31456d99651c651683535243647508eec628 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 30 Jan 2018 16:13:46 -0800 Subject: [PATCH 0570/1301] Properly support pulling all tags in DockerClient.images.pull Signed-off-by: Joffrey F --- docker/models/images.py | 20 +++++++++++++++++--- tests/integration/models_images_test.py | 6 ++++++ tests/integration/models_services_test.py | 3 ++- tests/unit/models_images_test.py | 17 +++++++++++++++-- 4 files changed, 40 insertions(+), 6 deletions(-) diff --git a/docker/models/images.py b/docker/models/images.py index 282d046b75..97c5503074 100644 --- a/docker/models/images.py +++ b/docker/models/images.py @@ -5,6 +5,7 @@ from ..api import APIClient from ..errors import BuildError, ImageLoadError +from ..utils import parse_repository_tag from ..utils.json_stream import json_stream from .resource import Collection, Model @@ -269,6 +270,8 @@ def pull(self, name, tag=None, **kwargs): """ Pull an image of the given name and return it. Similar to the ``docker pull`` command. + If no tag is specified, all tags from that repository will be + pulled. If you want to get the raw pull output, use the :py:meth:`~docker.api.image.ImageApiMixin.pull` method in the @@ -285,7 +288,9 @@ def pull(self, name, tag=None, **kwargs): platform (str): Platform in the format ``os[/arch[/variant]]`` Returns: - (:py:class:`Image`): The image that has been pulled. + (:py:class:`Image` or list): The image that has been pulled. + If no ``tag`` was specified, the method will return a list + of :py:class:`Image` objects belonging to this repository. Raises: :py:class:`docker.errors.APIError` @@ -293,10 +298,19 @@ def pull(self, name, tag=None, **kwargs): Example: - >>> image = client.images.pull('busybox') + >>> # Pull the image tagged `latest` in the busybox repo + >>> image = client.images.pull('busybox:latest') + + >>> # Pull all tags in the busybox repo + >>> images = client.images.pull('busybox') """ + if not tag: + name, tag = parse_repository_tag(name) + self.client.api.pull(name, tag=tag, **kwargs) - return self.get('{0}:{1}'.format(name, tag) if tag else name) + if tag: + return self.get('{0}:{1}'.format(name, tag)) + return self.list(name) def push(self, repository, tag=None, **kwargs): return self.client.api.push(repository, tag=tag, **kwargs) diff --git a/tests/integration/models_images_test.py b/tests/integration/models_images_test.py index 931391691e..2fa71a7947 100644 --- a/tests/integration/models_images_test.py +++ b/tests/integration/models_images_test.py @@ -74,6 +74,12 @@ def test_pull_with_tag(self): image = client.images.pull('alpine', tag='3.3') assert 'alpine:3.3' in image.attrs['RepoTags'] + def test_pull_multiple(self): + client = docker.from_env(version=TEST_API_VERSION) + images = client.images.pull('hello-world') + assert len(images) == 1 + assert 'hello-world:latest' in images[0].attrs['RepoTags'] + def test_load_error(self): client = docker.from_env(version=TEST_API_VERSION) with pytest.raises(docker.errors.ImageLoadError): diff --git a/tests/integration/models_services_test.py b/tests/integration/models_services_test.py index 39cccf8ee6..cb8eca29be 100644 --- a/tests/integration/models_services_test.py +++ b/tests/integration/models_services_test.py @@ -1,12 +1,12 @@ import unittest import docker +import pytest from .. import helpers from .base import TEST_API_VERSION from docker.errors import InvalidArgument from docker.types.services import ServiceMode -import pytest class ServiceTest(unittest.TestCase): @@ -182,6 +182,7 @@ def test_update_remove_service_labels(self): service.reload() assert not service.attrs['Spec'].get('Labels') + @pytest.mark.xfail(reason='Flaky test') def test_update_retains_networks(self): client = docker.from_env(version=TEST_API_VERSION) network_name = helpers.random_name() diff --git a/tests/unit/models_images_test.py b/tests/unit/models_images_test.py index 9ecb7e490d..dacd72be06 100644 --- a/tests/unit/models_images_test.py +++ b/tests/unit/models_images_test.py @@ -41,9 +41,22 @@ def test_load(self): def test_pull(self): client = make_fake_client() - image = client.images.pull('test_image') + image = client.images.pull('test_image:latest') + client.api.pull.assert_called_with('test_image', tag='latest') + client.api.inspect_image.assert_called_with('test_image:latest') + assert isinstance(image, Image) + assert image.id == FAKE_IMAGE_ID + + def test_pull_multiple(self): + client = make_fake_client() + images = client.images.pull('test_image') client.api.pull.assert_called_with('test_image', tag=None) - client.api.inspect_image.assert_called_with('test_image') + client.api.images.assert_called_with( + all=False, name='test_image', filters=None + ) + client.api.inspect_image.assert_called_with(FAKE_IMAGE_ID) + assert len(images) == 1 + image = images[0] assert isinstance(image, Image) assert image.id == FAKE_IMAGE_ID From dd858648a0942177995a74e1eda3468a720a3c58 Mon Sep 17 00:00:00 2001 From: Fumiaki MATSUSHIMA Date: Fri, 1 Dec 2017 02:40:13 +0900 Subject: [PATCH 0571/1301] Use config.json for detachKeys Signed-off-by: Fumiaki Matsushima --- docker/api/client.py | 3 +- docker/api/container.py | 6 ++ docker/api/exec_api.py | 12 +++- docker/auth.py | 49 ++------------- docker/utils/config.py | 65 +++++++++++++++++++ tests/helpers.py | 22 +++++++ tests/integration/api_container_test.py | 58 ++++++++++++++++- tests/integration/api_exec_test.py | 45 ++++++++++++- tests/unit/auth_test.py | 53 ---------------- tests/unit/utils_config_test.py | 84 +++++++++++++++++++++++++ 10 files changed, 294 insertions(+), 103 deletions(-) create mode 100644 docker/utils/config.py create mode 100644 tests/unit/utils_config_test.py diff --git a/docker/api/client.py b/docker/api/client.py index f0a86d4596..10640e1a6d 100644 --- a/docker/api/client.py +++ b/docker/api/client.py @@ -32,7 +32,7 @@ ) from ..tls import TLSConfig from ..transport import SSLAdapter, UnixAdapter -from ..utils import utils, check_resource, update_headers +from ..utils import utils, check_resource, update_headers, config from ..utils.socket import frames_iter, socket_raw_iter from ..utils.json_stream import json_stream try: @@ -106,6 +106,7 @@ def __init__(self, base_url=None, version=None, self.headers['User-Agent'] = user_agent self._auth_configs = auth.load_config() + self._general_configs = config.load_general_config() base_url = utils.parse_host( base_url, IS_WINDOWS_PLATFORM, tls=bool(tls) diff --git a/docker/api/container.py b/docker/api/container.py index 49230c7b66..260fbe91b5 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -66,6 +66,7 @@ def attach_socket(self, container, params=None, ws=False): container (str): The container to attach to. params (dict): Dictionary of request parameters (e.g. ``stdout``, ``stderr``, ``stream``). + For ``detachKeys``, ~/.docker/config.json is used by default. ws (bool): Use websockets instead of raw HTTP. Raises: @@ -79,6 +80,11 @@ def attach_socket(self, container, params=None, ws=False): 'stream': 1 } + if 'detachKeys' not in params \ + and 'detachKeys' in self._general_configs: + + params['detachKeys'] = self._general_configs['detachKeys'] + if ws: return self._attach_websocket(container, params) diff --git a/docker/api/exec_api.py b/docker/api/exec_api.py index 029c984a6f..d607461f5d 100644 --- a/docker/api/exec_api.py +++ b/docker/api/exec_api.py @@ -9,7 +9,7 @@ class ExecApiMixin(object): @utils.check_resource('container') def exec_create(self, container, cmd, stdout=True, stderr=True, stdin=False, tty=False, privileged=False, user='', - environment=None, workdir=None): + environment=None, workdir=None, detach_keys=None): """ Sets up an exec instance in a running container. @@ -27,6 +27,11 @@ def exec_create(self, container, cmd, stdout=True, stderr=True, the following format ``["PASSWORD=xxx"]`` or ``{"PASSWORD": "xxx"}``. workdir (str): Path to working directory for this exec session + detach_keys (str): Override the key sequence for detaching + a container. Format is a single character `[a-Z]` + or `ctrl-` where `` is one of: + `a-z`, `@`, `^`, `[`, `,` or `_`. + ~/.docker/config.json is used by default. Returns: (dict): A dictionary with an exec ``Id`` key. @@ -74,6 +79,11 @@ def exec_create(self, container, cmd, stdout=True, stderr=True, ) data['WorkingDir'] = workdir + if detach_keys: + data['detachKeys'] = detach_keys + elif 'detachKeys' in self._general_configs: + data['detachKeys'] = self._general_configs['detachKeys'] + url = self._url('/containers/{0}/exec', container) res = self._post_json(url, data=data) return self._result(res, True) diff --git a/docker/auth.py b/docker/auth.py index c0cae5d97a..79f63ccbed 100644 --- a/docker/auth.py +++ b/docker/auth.py @@ -1,18 +1,15 @@ import base64 import json import logging -import os import dockerpycreds import six from . import errors -from .constants import IS_WINDOWS_PLATFORM +from .utils import config INDEX_NAME = 'docker.io' INDEX_URL = 'https://index.{0}/v1/'.format(INDEX_NAME) -DOCKER_CONFIG_FILENAME = os.path.join('.docker', 'config.json') -LEGACY_DOCKER_CONFIG_FILENAME = '.dockercfg' TOKEN_USERNAME = '' log = logging.getLogger(__name__) @@ -105,10 +102,10 @@ def resolve_authconfig(authconfig, registry=None): log.debug("Found {0}".format(repr(registry))) return authconfig[registry] - for key, config in six.iteritems(authconfig): + for key, conf in six.iteritems(authconfig): if resolve_index_name(key) == registry: log.debug("Found {0}".format(repr(key))) - return config + return conf log.debug("No entry found") return None @@ -223,44 +220,6 @@ def parse_auth(entries, raise_on_error=False): return conf -def find_config_file(config_path=None): - paths = list(filter(None, [ - config_path, # 1 - config_path_from_environment(), # 2 - os.path.join(home_dir(), DOCKER_CONFIG_FILENAME), # 3 - os.path.join(home_dir(), LEGACY_DOCKER_CONFIG_FILENAME), # 4 - ])) - - log.debug("Trying paths: {0}".format(repr(paths))) - - for path in paths: - if os.path.exists(path): - log.debug("Found file at path: {0}".format(path)) - return path - - log.debug("No config file found") - - return None - - -def config_path_from_environment(): - config_dir = os.environ.get('DOCKER_CONFIG') - if not config_dir: - return None - return os.path.join(config_dir, os.path.basename(DOCKER_CONFIG_FILENAME)) - - -def home_dir(): - """ - Get the user's home directory, using the same logic as the Docker Engine - client - use %USERPROFILE% on Windows, $HOME/getuid on POSIX. - """ - if IS_WINDOWS_PLATFORM: - return os.environ.get('USERPROFILE', '') - else: - return os.path.expanduser('~') - - def load_config(config_path=None): """ Loads authentication data from a Docker configuration file in the given @@ -269,7 +228,7 @@ def load_config(config_path=None): explicit config_path parameter > DOCKER_CONFIG environment variable > ~/.docker/config.json > ~/.dockercfg """ - config_file = find_config_file(config_path) + config_file = config.find_config_file(config_path) if not config_file: return {} diff --git a/docker/utils/config.py b/docker/utils/config.py new file mode 100644 index 0000000000..8417261564 --- /dev/null +++ b/docker/utils/config.py @@ -0,0 +1,65 @@ +import json +import logging +import os + +from ..constants import IS_WINDOWS_PLATFORM + +DOCKER_CONFIG_FILENAME = os.path.join('.docker', 'config.json') +LEGACY_DOCKER_CONFIG_FILENAME = '.dockercfg' + +log = logging.getLogger(__name__) + + +def find_config_file(config_path=None): + paths = list(filter(None, [ + config_path, # 1 + config_path_from_environment(), # 2 + os.path.join(home_dir(), DOCKER_CONFIG_FILENAME), # 3 + os.path.join(home_dir(), LEGACY_DOCKER_CONFIG_FILENAME), # 4 + ])) + + log.debug("Trying paths: {0}".format(repr(paths))) + + for path in paths: + if os.path.exists(path): + log.debug("Found file at path: {0}".format(path)) + return path + + log.debug("No config file found") + + return None + + +def config_path_from_environment(): + config_dir = os.environ.get('DOCKER_CONFIG') + if not config_dir: + return None + return os.path.join(config_dir, os.path.basename(DOCKER_CONFIG_FILENAME)) + + +def home_dir(): + """ + Get the user's home directory, using the same logic as the Docker Engine + client - use %USERPROFILE% on Windows, $HOME/getuid on POSIX. + """ + if IS_WINDOWS_PLATFORM: + return os.environ.get('USERPROFILE', '') + else: + return os.path.expanduser('~') + + +def load_general_config(config_path=None): + config_file = find_config_file(config_path) + + if not config_file: + return {} + + try: + with open(config_file) as f: + return json.load(f) + except Exception as e: + log.debug(e) + pass + + log.debug("All parsing attempts failed - returning empty config") + return {} diff --git a/tests/helpers.py b/tests/helpers.py index 124ae2da51..7c68f6d7c1 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -5,6 +5,9 @@ import tarfile import tempfile import time +import re +import socket +import six import docker import pytest @@ -102,3 +105,22 @@ def force_leave_swarm(client): def swarm_listen_addr(): return '0.0.0.0:{0}'.format(random.randrange(10000, 25000)) + + +def assert_socket_closed_with_keys(sock, inputs): + if six.PY3: + sock = sock._sock + + for i in inputs: + sock.send(i) + time.sleep(1) + + with pytest.raises(socket.error): + sock.send(b"make sure the socket is closed\n") + + +def ctrl_with(char): + if re.match('[a-z]', char): + return chr(ord(char) - ord('a') + 1).encode('ascii') + else: + raise(Exception('char must be [a-z]')) diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index 4585c442d9..ebb5e91af7 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -1,4 +1,5 @@ import os +import re import signal import tempfile from datetime import datetime @@ -15,8 +16,9 @@ from .base import BUSYBOX, BaseAPIIntegrationTest from .. import helpers -from ..helpers import requires_api_version -import re +from ..helpers import ( + requires_api_version, ctrl_with, assert_socket_closed_with_keys +) class ListContainersTest(BaseAPIIntegrationTest): @@ -1223,6 +1225,58 @@ def test_attach_no_stream(self): output = self.client.attach(container, stream=False, logs=True) assert output == 'hello\n'.encode(encoding='ascii') + def test_detach_with_default(self): + container = self.client.create_container( + BUSYBOX, '/bin/sh', + detach=True, stdin_open=True, tty=True + ) + id = container['Id'] + self.tmp_containers.append(id) + self.client.start(id) + + sock = self.client.attach_socket( + container, + {'stdin': True, 'stream': True} + ) + + assert_socket_closed_with_keys(sock, [ctrl_with('p'), ctrl_with('q')]) + + def test_detach_with_config_file(self): + self.client._general_configs['detachKeys'] = 'ctrl-p' + + container = self.client.create_container( + BUSYBOX, '/bin/sh', + detach=True, stdin_open=True, tty=True + ) + id = container['Id'] + self.tmp_containers.append(id) + self.client.start(id) + + sock = self.client.attach_socket( + container, + {'stdin': True, 'stream': True} + ) + + assert_socket_closed_with_keys(sock, [ctrl_with('p')]) + + def test_detach_with_arg(self): + self.client._general_configs['detachKeys'] = 'ctrl-p' + + container = self.client.create_container( + BUSYBOX, '/bin/sh', + detach=True, stdin_open=True, tty=True + ) + id = container['Id'] + self.tmp_containers.append(id) + self.client.start(id) + + sock = self.client.attach_socket( + container, + {'stdin': True, 'stream': True, 'detachKeys': 'ctrl-x'} + ) + + assert_socket_closed_with_keys(sock, [ctrl_with('x')]) + class PauseTest(BaseAPIIntegrationTest): def test_pause_unpause(self): diff --git a/tests/integration/api_exec_test.py b/tests/integration/api_exec_test.py index cd97c68b5b..1a9542e8c4 100644 --- a/tests/integration/api_exec_test.py +++ b/tests/integration/api_exec_test.py @@ -2,7 +2,9 @@ from docker.utils.socket import read_exactly from .base import BaseAPIIntegrationTest, BUSYBOX -from ..helpers import requires_api_version +from ..helpers import ( + requires_api_version, ctrl_with, assert_socket_closed_with_keys +) class ExecTest(BaseAPIIntegrationTest): @@ -148,3 +150,44 @@ def test_exec_command_with_workdir(self): res = self.client.exec_create(container, 'pwd', workdir='/var/www') exec_log = self.client.exec_start(res) assert exec_log == b'/var/www\n' + + def test_detach_with_default(self): + container = self.client.create_container(BUSYBOX, 'cat', + detach=True, stdin_open=True) + id = container['Id'] + self.client.start(id) + self.tmp_containers.append(id) + + exec_id = self.client.exec_create(id, '/bin/sh', stdin=True, tty=True) + sock = self.client.exec_start(exec_id, tty=True, socket=True) + + assert_socket_closed_with_keys(sock, [ctrl_with('p'), ctrl_with('q')]) + + def test_detach_with_config_file(self): + self.client._general_configs['detachKeys'] = 'ctrl-p' + container = self.client.create_container(BUSYBOX, 'cat', + detach=True, stdin_open=True) + id = container['Id'] + self.client.start(id) + self.tmp_containers.append(id) + + exec_id = self.client.exec_create(id, '/bin/sh', stdin=True, tty=True) + sock = self.client.exec_start(exec_id, tty=True, socket=True) + + assert_socket_closed_with_keys(sock, [ctrl_with('p')]) + + def test_detach_with_arg(self): + self.client._general_configs['detachKeys'] = 'ctrl-p' + container = self.client.create_container(BUSYBOX, 'cat', + detach=True, stdin_open=True) + id = container['Id'] + self.client.start(id) + self.tmp_containers.append(id) + + exec_id = self.client.exec_create( + id, '/bin/sh', + stdin=True, tty=True, detach_keys='ctrl-x' + ) + sock = self.client.exec_start(exec_id, tty=True, socket=True) + + assert_socket_closed_with_keys(sock, [ctrl_with('x')]) diff --git a/tests/unit/auth_test.py b/tests/unit/auth_test.py index 1506ccbd74..e3356d3dad 100644 --- a/tests/unit/auth_test.py +++ b/tests/unit/auth_test.py @@ -9,9 +9,6 @@ import tempfile import unittest -from py.test import ensuretemp -from pytest import mark - from docker import auth, errors import pytest @@ -263,56 +260,6 @@ def test_get_credential_store_default_index(self): ) == 'truesecret' -class FindConfigFileTest(unittest.TestCase): - def tmpdir(self, name): - tmpdir = ensuretemp(name) - self.addCleanup(tmpdir.remove) - return tmpdir - - def test_find_config_fallback(self): - tmpdir = self.tmpdir('test_find_config_fallback') - - with mock.patch.dict(os.environ, {'HOME': str(tmpdir)}): - assert auth.find_config_file() is None - - def test_find_config_from_explicit_path(self): - tmpdir = self.tmpdir('test_find_config_from_explicit_path') - config_path = tmpdir.ensure('my-config-file.json') - - assert auth.find_config_file(str(config_path)) == str(config_path) - - def test_find_config_from_environment(self): - tmpdir = self.tmpdir('test_find_config_from_environment') - config_path = tmpdir.ensure('config.json') - - with mock.patch.dict(os.environ, {'DOCKER_CONFIG': str(tmpdir)}): - assert auth.find_config_file() == str(config_path) - - @mark.skipif("sys.platform == 'win32'") - def test_find_config_from_home_posix(self): - tmpdir = self.tmpdir('test_find_config_from_home_posix') - config_path = tmpdir.ensure('.docker', 'config.json') - - with mock.patch.dict(os.environ, {'HOME': str(tmpdir)}): - assert auth.find_config_file() == str(config_path) - - @mark.skipif("sys.platform == 'win32'") - def test_find_config_from_home_legacy_name(self): - tmpdir = self.tmpdir('test_find_config_from_home_legacy_name') - config_path = tmpdir.ensure('.dockercfg') - - with mock.patch.dict(os.environ, {'HOME': str(tmpdir)}): - assert auth.find_config_file() == str(config_path) - - @mark.skipif("sys.platform != 'win32'") - def test_find_config_from_home_windows(self): - tmpdir = self.tmpdir('test_find_config_from_home_windows') - config_path = tmpdir.ensure('.docker', 'config.json') - - with mock.patch.dict(os.environ, {'USERPROFILE': str(tmpdir)}): - assert auth.find_config_file() == str(config_path) - - class LoadConfigTest(unittest.TestCase): def test_load_config_no_file(self): folder = tempfile.mkdtemp() diff --git a/tests/unit/utils_config_test.py b/tests/unit/utils_config_test.py new file mode 100644 index 0000000000..45f75ff487 --- /dev/null +++ b/tests/unit/utils_config_test.py @@ -0,0 +1,84 @@ +import os +import unittest +import shutil +import tempfile +import json + +from py.test import ensuretemp +from pytest import mark +from docker.utils import config + +try: + from unittest import mock +except ImportError: + import mock + + +class FindConfigFileTest(unittest.TestCase): + def tmpdir(self, name): + tmpdir = ensuretemp(name) + self.addCleanup(tmpdir.remove) + return tmpdir + + def test_find_config_fallback(self): + tmpdir = self.tmpdir('test_find_config_fallback') + + with mock.patch.dict(os.environ, {'HOME': str(tmpdir)}): + assert config.find_config_file() is None + + def test_find_config_from_explicit_path(self): + tmpdir = self.tmpdir('test_find_config_from_explicit_path') + config_path = tmpdir.ensure('my-config-file.json') + + assert config.find_config_file(str(config_path)) == str(config_path) + + def test_find_config_from_environment(self): + tmpdir = self.tmpdir('test_find_config_from_environment') + config_path = tmpdir.ensure('config.json') + + with mock.patch.dict(os.environ, {'DOCKER_CONFIG': str(tmpdir)}): + assert config.find_config_file() == str(config_path) + + @mark.skipif("sys.platform == 'win32'") + def test_find_config_from_home_posix(self): + tmpdir = self.tmpdir('test_find_config_from_home_posix') + config_path = tmpdir.ensure('.docker', 'config.json') + + with mock.patch.dict(os.environ, {'HOME': str(tmpdir)}): + assert config.find_config_file() == str(config_path) + + @mark.skipif("sys.platform == 'win32'") + def test_find_config_from_home_legacy_name(self): + tmpdir = self.tmpdir('test_find_config_from_home_legacy_name') + config_path = tmpdir.ensure('.dockercfg') + + with mock.patch.dict(os.environ, {'HOME': str(tmpdir)}): + assert config.find_config_file() == str(config_path) + + @mark.skipif("sys.platform != 'win32'") + def test_find_config_from_home_windows(self): + tmpdir = self.tmpdir('test_find_config_from_home_windows') + config_path = tmpdir.ensure('.docker', 'config.json') + + with mock.patch.dict(os.environ, {'USERPROFILE': str(tmpdir)}): + assert config.find_config_file() == str(config_path) + + +class LoadConfigTest(unittest.TestCase): + def test_load_config_no_file(self): + folder = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, folder) + cfg = config.load_general_config(folder) + self.assertTrue(cfg is not None) + + def test_load_config(self): + folder = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, folder) + dockercfg_path = os.path.join(folder, '.dockercfg') + cfg = { + 'detachKeys': 'ctrl-q, ctrl-u, ctrl-i' + } + with open(dockercfg_path, 'w') as f: + json.dump(cfg, f) + + self.assertEqual(config.load_general_config(dockercfg_path), cfg) From e304f91b4636b59a056ff795d91895c725c6255f Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 29 Jan 2018 17:07:15 -0800 Subject: [PATCH 0572/1301] Update detach tests to work with AF_INET as well Signed-off-by: Joffrey F --- Makefile | 2 +- tests/helpers.py | 17 +++++++---- tests/integration/api_container_test.py | 31 ++++++++++--------- tests/integration/api_exec_test.py | 40 ++++++++++++++++--------- 4 files changed, 54 insertions(+), 36 deletions(-) diff --git a/Makefile b/Makefile index d07b8c5968..f491993926 100644 --- a/Makefile +++ b/Makefile @@ -54,7 +54,7 @@ integration-dind-py2: build -H tcp://0.0.0.0:2375 --experimental docker run -t --rm --env="DOCKER_HOST=tcp://docker:2375" --env="DOCKER_TEST_API_VERSION=${TEST_API_VERSION}"\ --link=dpy-dind-py2:docker docker-sdk-python py.test tests/integration - docker rm -vf dpy-dind-py3 + docker rm -vf dpy-dind-py2 .PHONY: integration-dind-py3 integration-dind-py3: build-py3 diff --git a/tests/helpers.py b/tests/helpers.py index 7c68f6d7c1..c4ea3647dd 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -6,8 +6,8 @@ import tempfile import time import re -import socket import six +import socket import docker import pytest @@ -107,16 +107,23 @@ def swarm_listen_addr(): return '0.0.0.0:{0}'.format(random.randrange(10000, 25000)) -def assert_socket_closed_with_keys(sock, inputs): +def assert_cat_socket_detached_with_keys(sock, inputs): if six.PY3: sock = sock._sock for i in inputs: sock.send(i) - time.sleep(1) - - with pytest.raises(socket.error): + time.sleep(0.5) + + # If we're using a Unix socket, the sock.send call will fail with a + # BrokenPipeError ; INET sockets will just stop receiving / sending data + # but will not raise an error + if sock.family == getattr(socket, 'AF_UNIX', -1): + with pytest.raises(socket.error): + sock.send(b'make sure the socket is closed\n') + else: sock.send(b"make sure the socket is closed\n") + assert sock.recv(32) == b'' def ctrl_with(char): diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index ebb5e91af7..f48e78e90f 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -17,7 +17,7 @@ from .base import BUSYBOX, BaseAPIIntegrationTest from .. import helpers from ..helpers import ( - requires_api_version, ctrl_with, assert_socket_closed_with_keys + requires_api_version, ctrl_with, assert_cat_socket_detached_with_keys ) @@ -1227,55 +1227,54 @@ def test_attach_no_stream(self): def test_detach_with_default(self): container = self.client.create_container( - BUSYBOX, '/bin/sh', + BUSYBOX, 'cat', detach=True, stdin_open=True, tty=True ) - id = container['Id'] - self.tmp_containers.append(id) - self.client.start(id) + self.tmp_containers.append(container) + self.client.start(container) sock = self.client.attach_socket( container, {'stdin': True, 'stream': True} ) - assert_socket_closed_with_keys(sock, [ctrl_with('p'), ctrl_with('q')]) + assert_cat_socket_detached_with_keys( + sock, [ctrl_with('p'), ctrl_with('q')] + ) def test_detach_with_config_file(self): self.client._general_configs['detachKeys'] = 'ctrl-p' container = self.client.create_container( - BUSYBOX, '/bin/sh', + BUSYBOX, 'cat', detach=True, stdin_open=True, tty=True ) - id = container['Id'] - self.tmp_containers.append(id) - self.client.start(id) + self.tmp_containers.append(container) + self.client.start(container) sock = self.client.attach_socket( container, {'stdin': True, 'stream': True} ) - assert_socket_closed_with_keys(sock, [ctrl_with('p')]) + assert_cat_socket_detached_with_keys(sock, [ctrl_with('p')]) def test_detach_with_arg(self): self.client._general_configs['detachKeys'] = 'ctrl-p' container = self.client.create_container( - BUSYBOX, '/bin/sh', + BUSYBOX, 'cat', detach=True, stdin_open=True, tty=True ) - id = container['Id'] - self.tmp_containers.append(id) - self.client.start(id) + self.tmp_containers.append(container) + self.client.start(container) sock = self.client.attach_socket( container, {'stdin': True, 'stream': True, 'detachKeys': 'ctrl-x'} ) - assert_socket_closed_with_keys(sock, [ctrl_with('x')]) + assert_cat_socket_detached_with_keys(sock, [ctrl_with('x')]) class PauseTest(BaseAPIIntegrationTest): diff --git a/tests/integration/api_exec_test.py b/tests/integration/api_exec_test.py index 1a9542e8c4..1a5a4e5472 100644 --- a/tests/integration/api_exec_test.py +++ b/tests/integration/api_exec_test.py @@ -3,7 +3,7 @@ from .base import BaseAPIIntegrationTest, BUSYBOX from ..helpers import ( - requires_api_version, ctrl_with, assert_socket_closed_with_keys + requires_api_version, ctrl_with, assert_cat_socket_detached_with_keys ) @@ -152,42 +152,54 @@ def test_exec_command_with_workdir(self): assert exec_log == b'/var/www\n' def test_detach_with_default(self): - container = self.client.create_container(BUSYBOX, 'cat', - detach=True, stdin_open=True) + container = self.client.create_container( + BUSYBOX, 'cat', detach=True, stdin_open=True + ) id = container['Id'] self.client.start(id) self.tmp_containers.append(id) - exec_id = self.client.exec_create(id, '/bin/sh', stdin=True, tty=True) + exec_id = self.client.exec_create( + id, 'cat', stdin=True, tty=True, stdout=True + ) sock = self.client.exec_start(exec_id, tty=True, socket=True) + self.addCleanup(sock.close) - assert_socket_closed_with_keys(sock, [ctrl_with('p'), ctrl_with('q')]) + assert_cat_socket_detached_with_keys( + sock, [ctrl_with('p'), ctrl_with('q')] + ) def test_detach_with_config_file(self): self.client._general_configs['detachKeys'] = 'ctrl-p' - container = self.client.create_container(BUSYBOX, 'cat', - detach=True, stdin_open=True) + container = self.client.create_container( + BUSYBOX, 'cat', detach=True, stdin_open=True + ) id = container['Id'] self.client.start(id) self.tmp_containers.append(id) - exec_id = self.client.exec_create(id, '/bin/sh', stdin=True, tty=True) + exec_id = self.client.exec_create( + id, 'cat', stdin=True, tty=True, stdout=True + ) sock = self.client.exec_start(exec_id, tty=True, socket=True) + self.addCleanup(sock.close) - assert_socket_closed_with_keys(sock, [ctrl_with('p')]) + assert_cat_socket_detached_with_keys(sock, [ctrl_with('p')]) def test_detach_with_arg(self): self.client._general_configs['detachKeys'] = 'ctrl-p' - container = self.client.create_container(BUSYBOX, 'cat', - detach=True, stdin_open=True) + container = self.client.create_container( + BUSYBOX, 'cat', detach=True, stdin_open=True + ) id = container['Id'] self.client.start(id) self.tmp_containers.append(id) exec_id = self.client.exec_create( - id, '/bin/sh', - stdin=True, tty=True, detach_keys='ctrl-x' + id, 'cat', + stdin=True, tty=True, detach_keys='ctrl-x', stdout=True ) sock = self.client.exec_start(exec_id, tty=True, socket=True) + self.addCleanup(sock.close) - assert_socket_closed_with_keys(sock, [ctrl_with('x')]) + assert_cat_socket_detached_with_keys(sock, [ctrl_with('x')]) From ccbde11c8dbb8773aed84abac92c6361b8e11229 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 30 Jan 2018 18:22:41 -0800 Subject: [PATCH 0573/1301] Improve separation between auth_configs and general_configs Signed-off-by: Joffrey F --- docker/api/build.py | 6 +- docker/api/client.py | 5 +- docker/api/daemon.py | 2 + docker/auth.py | 75 ++++++++++++++----------- docker/utils/config.py | 5 +- docker/utils/decorators.py | 6 +- tests/integration/api_container_test.py | 2 +- tests/integration/base.py | 4 -- tests/unit/api_build_test.py | 48 ++++++++++------ tests/unit/auth_test.py | 52 +++++------------ tests/unit/utils_config_test.py | 51 +++++++++++++++-- tests/unit/utils_test.py | 4 +- 12 files changed, 149 insertions(+), 111 deletions(-) diff --git a/docker/api/build.py b/docker/api/build.py index 32238efed9..220c93f72f 100644 --- a/docker/api/build.py +++ b/docker/api/build.py @@ -300,14 +300,12 @@ def _set_auth_headers(self, headers): # Matches CLI behavior: https://github.com/docker/docker/blob/ # 67b85f9d26f1b0b2b240f2d794748fac0f45243c/cliconfig/ # credentials/native_store.go#L68-L83 - for registry in self._auth_configs.keys(): - if registry == 'credsStore' or registry == 'HttpHeaders': - continue + for registry in self._auth_configs.get('auths', {}).keys(): auth_data[registry] = auth.resolve_authconfig( self._auth_configs, registry ) else: - auth_data = self._auth_configs.copy() + auth_data = self._auth_configs.get('auths', {}).copy() # See https://github.com/docker/docker-py/issues/1683 if auth.INDEX_NAME in auth_data: auth_data[auth.INDEX_URL] = auth_data[auth.INDEX_NAME] diff --git a/docker/api/client.py b/docker/api/client.py index 10640e1a6d..07bcfae4a5 100644 --- a/docker/api/client.py +++ b/docker/api/client.py @@ -87,6 +87,7 @@ class APIClient( """ __attrs__ = requests.Session.__attrs__ + ['_auth_configs', + '_general_configs', '_version', 'base_url', 'timeout'] @@ -105,8 +106,10 @@ def __init__(self, base_url=None, version=None, self.timeout = timeout self.headers['User-Agent'] = user_agent - self._auth_configs = auth.load_config() self._general_configs = config.load_general_config() + self._auth_configs = auth.load_config( + config_dict=self._general_configs + ) base_url = utils.parse_host( base_url, IS_WINDOWS_PLATFORM, tls=bool(tls) diff --git a/docker/api/daemon.py b/docker/api/daemon.py index 285b7429ad..39989679bc 100644 --- a/docker/api/daemon.py +++ b/docker/api/daemon.py @@ -144,6 +144,8 @@ def login(self, username, password=None, email=None, registry=None, response = self._post_json(self._url('/auth'), data=req_data) if response.status_code == 200: + if 'auths' not in self._auth_configs: + self._auth_configs['auths'] = {} self._auth_configs[registry or auth.INDEX_NAME] = req_data return self._result(response, json=True) diff --git a/docker/auth.py b/docker/auth.py index 79f63ccbed..91be2b8502 100644 --- a/docker/auth.py +++ b/docker/auth.py @@ -98,11 +98,12 @@ def resolve_authconfig(authconfig, registry=None): registry = resolve_index_name(registry) if registry else INDEX_NAME log.debug("Looking for auth entry for {0}".format(repr(registry))) - if registry in authconfig: + authdict = authconfig.get('auths', {}) + if registry in authdict: log.debug("Found {0}".format(repr(registry))) - return authconfig[registry] + return authdict[registry] - for key, conf in six.iteritems(authconfig): + for key, conf in six.iteritems(authdict): if resolve_index_name(key) == registry: log.debug("Found {0}".format(repr(key))) return conf @@ -220,7 +221,7 @@ def parse_auth(entries, raise_on_error=False): return conf -def load_config(config_path=None): +def load_config(config_path=None, config_dict=None): """ Loads authentication data from a Docker configuration file in the given root directory or if config_path is passed use given path. @@ -228,39 +229,45 @@ def load_config(config_path=None): explicit config_path parameter > DOCKER_CONFIG environment variable > ~/.docker/config.json > ~/.dockercfg """ - config_file = config.find_config_file(config_path) - if not config_file: - return {} + if not config_dict: + config_file = config.find_config_file(config_path) + + if not config_file: + return {} + try: + with open(config_file) as f: + config_dict = json.load(f) + except (IOError, KeyError, ValueError) as e: + # Likely missing new Docker config file or it's in an + # unknown format, continue to attempt to read old location + # and format. + log.debug(e) + return _load_legacy_config(config_file) + + res = {} + if config_dict.get('auths'): + log.debug("Found 'auths' section") + res.update({ + 'auths': parse_auth(config_dict.pop('auths'), raise_on_error=True) + }) + if config_dict.get('credsStore'): + log.debug("Found 'credsStore' section") + res.update({'credsStore': config_dict.pop('credsStore')}) + if config_dict.get('credHelpers'): + log.debug("Found 'credHelpers' section") + res.update({'credHelpers': config_dict.pop('credHelpers')}) + if res: + return res + + log.debug( + "Couldn't find auth-related section ; attempting to interpret" + "as auth-only file" + ) + return parse_auth(config_dict) - try: - with open(config_file) as f: - data = json.load(f) - res = {} - if data.get('auths'): - log.debug("Found 'auths' section") - res.update(parse_auth(data['auths'], raise_on_error=True)) - if data.get('HttpHeaders'): - log.debug("Found 'HttpHeaders' section") - res.update({'HttpHeaders': data['HttpHeaders']}) - if data.get('credsStore'): - log.debug("Found 'credsStore' section") - res.update({'credsStore': data['credsStore']}) - if data.get('credHelpers'): - log.debug("Found 'credHelpers' section") - res.update({'credHelpers': data['credHelpers']}) - if res: - return res - else: - log.debug("Couldn't find 'auths' or 'HttpHeaders' sections") - f.seek(0) - return parse_auth(json.load(f)) - except (IOError, KeyError, ValueError) as e: - # Likely missing new Docker config file or it's in an - # unknown format, continue to attempt to read old location - # and format. - log.debug(e) +def _load_legacy_config(config_file): log.debug("Attempting to parse legacy auth file format") try: data = [] diff --git a/docker/utils/config.py b/docker/utils/config.py index 8417261564..82a0e2a5ec 100644 --- a/docker/utils/config.py +++ b/docker/utils/config.py @@ -57,9 +57,10 @@ def load_general_config(config_path=None): try: with open(config_file) as f: return json.load(f) - except Exception as e: + except (IOError, ValueError) as e: + # In the case of a legacy `.dockercfg` file, we won't + # be able to load any JSON data. log.debug(e) - pass log.debug("All parsing attempts failed - returning empty config") return {} diff --git a/docker/utils/decorators.py b/docker/utils/decorators.py index 5e195c0ea6..c975d4b401 100644 --- a/docker/utils/decorators.py +++ b/docker/utils/decorators.py @@ -38,10 +38,10 @@ def wrapper(self, *args, **kwargs): def update_headers(f): def inner(self, *args, **kwargs): - if 'HttpHeaders' in self._auth_configs: + if 'HttpHeaders' in self._general_configs: if not kwargs.get('headers'): - kwargs['headers'] = self._auth_configs['HttpHeaders'] + kwargs['headers'] = self._general_configs['HttpHeaders'] else: - kwargs['headers'].update(self._auth_configs['HttpHeaders']) + kwargs['headers'].update(self._general_configs['HttpHeaders']) return f(self, *args, **kwargs) return inner diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index f48e78e90f..e5d79431fb 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -161,7 +161,7 @@ def test_create_container_with_volumes_from(self): self.client.start(container3_id) info = self.client.inspect_container(res2['Id']) - self.assertCountEqual(info['HostConfig']['VolumesFrom'], vol_names) + assert len(info['HostConfig']['VolumesFrom']) == len(vol_names) def create_container_readonly_fs(self): ctnr = self.client.create_container( diff --git a/tests/integration/base.py b/tests/integration/base.py index 4f929014bd..04d9afdb0a 100644 --- a/tests/integration/base.py +++ b/tests/integration/base.py @@ -4,7 +4,6 @@ import docker from docker.utils import kwargs_from_env -import six from .. import helpers @@ -19,9 +18,6 @@ class BaseIntegrationTest(unittest.TestCase): """ def setUp(self): - if six.PY2: - self.assertRegex = self.assertRegexpMatches - self.assertCountEqual = self.assertItemsEqual self.tmp_imgs = [] self.tmp_containers = [] self.tmp_folders = [] diff --git a/tests/unit/api_build_test.py b/tests/unit/api_build_test.py index e366bced69..b20daea87f 100644 --- a/tests/unit/api_build_test.py +++ b/tests/unit/api_build_test.py @@ -73,10 +73,12 @@ def test_build_container_custom_context_gzip(self): def test_build_remote_with_registry_auth(self): self.client._auth_configs = { - 'https://example.com': { - 'user': 'example', - 'password': 'example', - 'email': 'example@example.com' + 'auths': { + 'https://example.com': { + 'user': 'example', + 'password': 'example', + 'email': 'example@example.com' + } } } @@ -85,7 +87,10 @@ def test_build_remote_with_registry_auth(self): 'forcerm': False, 'remote': 'https://github.com/docker-library/mongo'} expected_headers = { - 'X-Registry-Config': auth.encode_header(self.client._auth_configs)} + 'X-Registry-Config': auth.encode_header( + self.client._auth_configs['auths'] + ) + } self.client.build(path='https://github.com/docker-library/mongo') @@ -118,32 +123,43 @@ def test_build_container_invalid_container_limits(self): def test_set_auth_headers_with_empty_dict_and_auth_configs(self): self.client._auth_configs = { - 'https://example.com': { - 'user': 'example', - 'password': 'example', - 'email': 'example@example.com' + 'auths': { + 'https://example.com': { + 'user': 'example', + 'password': 'example', + 'email': 'example@example.com' + } } } headers = {} expected_headers = { - 'X-Registry-Config': auth.encode_header(self.client._auth_configs)} + 'X-Registry-Config': auth.encode_header( + self.client._auth_configs['auths'] + ) + } + self.client._set_auth_headers(headers) assert headers == expected_headers def test_set_auth_headers_with_dict_and_auth_configs(self): self.client._auth_configs = { - 'https://example.com': { - 'user': 'example', - 'password': 'example', - 'email': 'example@example.com' + 'auths': { + 'https://example.com': { + 'user': 'example', + 'password': 'example', + 'email': 'example@example.com' + } } } headers = {'foo': 'bar'} expected_headers = { - 'foo': 'bar', - 'X-Registry-Config': auth.encode_header(self.client._auth_configs)} + 'X-Registry-Config': auth.encode_header( + self.client._auth_configs['auths'] + ), + 'foo': 'bar' + } self.client._set_auth_headers(headers) assert headers == expected_headers diff --git a/tests/unit/auth_test.py b/tests/unit/auth_test.py index e3356d3dad..d6981cd9da 100644 --- a/tests/unit/auth_test.py +++ b/tests/unit/auth_test.py @@ -106,11 +106,13 @@ class ResolveAuthTest(unittest.TestCase): private_config = {'auth': encode_auth({'username': 'privateuser'})} legacy_config = {'auth': encode_auth({'username': 'legacyauth'})} - auth_config = auth.parse_auth({ - 'https://index.docker.io/v1/': index_config, - 'my.registry.net': private_config, - 'http://legacy.registry.url/v1/': legacy_config, - }) + auth_config = { + 'auths': auth.parse_auth({ + 'https://index.docker.io/v1/': index_config, + 'my.registry.net': private_config, + 'http://legacy.registry.url/v1/': legacy_config, + }) + } def test_resolve_authconfig_hostname_only(self): assert auth.resolve_authconfig( @@ -360,9 +362,8 @@ def test_load_config_custom_config_env_with_auths(self): with mock.patch.dict(os.environ, {'DOCKER_CONFIG': folder}): cfg = auth.load_config(None) - assert registry in cfg - assert cfg[registry] is not None - cfg = cfg[registry] + assert registry in cfg['auths'] + cfg = cfg['auths'][registry] assert cfg['username'] == 'sakuya' assert cfg['password'] == 'izayoi' assert cfg['email'] == 'sakuya@scarlet.net' @@ -390,38 +391,13 @@ def test_load_config_custom_config_env_utf8(self): with mock.patch.dict(os.environ, {'DOCKER_CONFIG': folder}): cfg = auth.load_config(None) - assert registry in cfg - assert cfg[registry] is not None - cfg = cfg[registry] + assert registry in cfg['auths'] + cfg = cfg['auths'][registry] assert cfg['username'] == b'sakuya\xc3\xa6'.decode('utf8') assert cfg['password'] == b'izayoi\xc3\xa6'.decode('utf8') assert cfg['email'] == 'sakuya@scarlet.net' assert cfg.get('auth') is None - def test_load_config_custom_config_env_with_headers(self): - folder = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, folder) - - dockercfg_path = os.path.join(folder, 'config.json') - config = { - 'HttpHeaders': { - 'Name': 'Spike', - 'Surname': 'Spiegel' - }, - } - - with open(dockercfg_path, 'w') as f: - json.dump(config, f) - - with mock.patch.dict(os.environ, {'DOCKER_CONFIG': folder}): - cfg = auth.load_config(None) - assert 'HttpHeaders' in cfg - assert cfg['HttpHeaders'] is not None - cfg = cfg['HttpHeaders'] - - assert cfg['Name'] == 'Spike' - assert cfg['Surname'] == 'Spiegel' - def test_load_config_unknown_keys(self): folder = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, folder) @@ -448,7 +424,7 @@ def test_load_config_invalid_auth_dict(self): json.dump(config, f) cfg = auth.load_config(dockercfg_path) - assert cfg == {'scarlet.net': {}} + assert cfg == {'auths': {'scarlet.net': {}}} def test_load_config_identity_token(self): folder = tempfile.mkdtemp() @@ -469,7 +445,7 @@ def test_load_config_identity_token(self): json.dump(config, f) cfg = auth.load_config(dockercfg_path) - assert registry in cfg - cfg = cfg[registry] + assert registry in cfg['auths'] + cfg = cfg['auths'][registry] assert 'IdentityToken' in cfg assert cfg['IdentityToken'] == token diff --git a/tests/unit/utils_config_test.py b/tests/unit/utils_config_test.py index 45f75ff487..50ba3831db 100644 --- a/tests/unit/utils_config_test.py +++ b/tests/unit/utils_config_test.py @@ -69,16 +69,55 @@ def test_load_config_no_file(self): folder = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, folder) cfg = config.load_general_config(folder) - self.assertTrue(cfg is not None) + assert cfg is not None + assert isinstance(cfg, dict) + assert not cfg - def test_load_config(self): + def test_load_config_custom_headers(self): folder = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, folder) - dockercfg_path = os.path.join(folder, '.dockercfg') - cfg = { + + dockercfg_path = os.path.join(folder, 'config.json') + config_data = { + 'HttpHeaders': { + 'Name': 'Spike', + 'Surname': 'Spiegel' + }, + } + + with open(dockercfg_path, 'w') as f: + json.dump(config_data, f) + + cfg = config.load_general_config(dockercfg_path) + assert 'HttpHeaders' in cfg + assert cfg['HttpHeaders'] == { + 'Name': 'Spike', + 'Surname': 'Spiegel' + } + + def test_load_config_detach_keys(self): + folder = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, folder) + dockercfg_path = os.path.join(folder, 'config.json') + config_data = { + 'detachKeys': 'ctrl-q, ctrl-u, ctrl-i' + } + with open(dockercfg_path, 'w') as f: + json.dump(config_data, f) + + cfg = config.load_general_config(dockercfg_path) + assert cfg == config_data + + def test_load_config_from_env(self): + folder = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, folder) + dockercfg_path = os.path.join(folder, 'config.json') + config_data = { 'detachKeys': 'ctrl-q, ctrl-u, ctrl-i' } with open(dockercfg_path, 'w') as f: - json.dump(cfg, f) + json.dump(config_data, f) - self.assertEqual(config.load_general_config(dockercfg_path), cfg) + with mock.patch.dict(os.environ, {'DOCKER_CONFIG': folder}): + cfg = config.load_general_config(None) + assert cfg == config_data diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index 73f95d6500..1f9daf60a2 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -46,7 +46,7 @@ def f(self, headers=None): return headers client = APIClient() - client._auth_configs = {} + client._general_configs = {} g = update_headers(f) assert g(client, headers=None) is None @@ -55,7 +55,7 @@ def f(self, headers=None): 'Content-type': 'application/json', } - client._auth_configs = { + client._general_configs = { 'HttpHeaders': sample_headers } From 5347c168d022836bd19f63c5527c59169ae22f53 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 31 Jan 2018 12:29:26 -0800 Subject: [PATCH 0574/1301] Add support for publish mode for endpointspec ports Signed-off-by: Joffrey F --- docker/api/service.py | 15 ++++-- docker/types/services.py | 14 +++-- tests/integration/api_service_test.py | 21 +++++++- tests/unit/dockertypes_test.py | 75 +++++++++++++++++++++++++++ 4 files changed, 118 insertions(+), 7 deletions(-) diff --git a/docker/api/service.py b/docker/api/service.py index 4f7123e5a0..051d34fd07 100644 --- a/docker/api/service.py +++ b/docker/api/service.py @@ -3,7 +3,7 @@ from ..types import ServiceMode -def _check_api_features(version, task_template, update_config): +def _check_api_features(version, task_template, update_config, endpoint_spec): def raise_version_error(param, min_version): raise errors.InvalidVersion( @@ -23,6 +23,11 @@ def raise_version_error(param, min_version): if 'Order' in update_config: raise_version_error('UpdateConfig.order', '1.29') + if endpoint_spec is not None: + if utils.version_lt(version, '1.32') and 'Ports' in endpoint_spec: + if any(p.get('PublishMode') for p in endpoint_spec['Ports']): + raise_version_error('EndpointSpec.Ports[].mode', '1.32') + if task_template is not None: if 'ForceUpdate' in task_template and utils.version_lt( version, '1.25'): @@ -125,7 +130,9 @@ def create_service( ) endpoint_spec = endpoint_config - _check_api_features(self._version, task_template, update_config) + _check_api_features( + self._version, task_template, update_config, endpoint_spec + ) url = self._url('/services/create') headers = {} @@ -370,7 +377,9 @@ def update_service(self, service, version, task_template=None, name=None, ) endpoint_spec = endpoint_config - _check_api_features(self._version, task_template, update_config) + _check_api_features( + self._version, task_template, update_config, endpoint_spec + ) if fetch_current_spec: inspect_defaults = True diff --git a/docker/types/services.py b/docker/types/services.py index ef1ca690d3..d530e61db6 100644 --- a/docker/types/services.py +++ b/docker/types/services.py @@ -450,8 +450,9 @@ class EndpointSpec(dict): ``'vip'`` if not provided. ports (dict): Exposed ports that this service is accessible on from the outside, in the form of ``{ published_port: target_port }`` or - ``{ published_port: (target_port, protocol) }``. Ports can only be - provided if the ``vip`` resolution mode is used. + ``{ published_port: }``. Port config tuple format + is ``(target_port [, protocol [, publish_mode]])``. + Ports can only be provided if the ``vip`` resolution mode is used. """ def __init__(self, mode=None, ports=None): if ports: @@ -477,8 +478,15 @@ def convert_service_ports(ports): if isinstance(v, tuple): port_spec['TargetPort'] = v[0] - if len(v) == 2: + if len(v) >= 2 and v[1] is not None: port_spec['Protocol'] = v[1] + if len(v) == 3: + port_spec['PublishMode'] = v[2] + if len(v) > 3: + raise ValueError( + 'Service port configuration can have at most 3 elements: ' + '(target_port, protocol, mode)' + ) else: port_spec['TargetPort'] = v diff --git a/tests/integration/api_service_test.py b/tests/integration/api_service_test.py index 7620cb47d7..5cc3fc190f 100644 --- a/tests/integration/api_service_test.py +++ b/tests/integration/api_service_test.py @@ -353,7 +353,6 @@ def test_create_service_with_endpoint_spec(self): task_tmpl, name=name, endpoint_spec=endpoint_spec ) svc_info = self.client.inspect_service(svc_id) - print(svc_info) ports = svc_info['Spec']['EndpointSpec']['Ports'] for port in ports: if port['PublishedPort'] == 12562: @@ -370,6 +369,26 @@ def test_create_service_with_endpoint_spec(self): assert len(ports) == 3 + @requires_api_version('1.32') + def test_create_service_with_endpoint_spec_host_publish_mode(self): + container_spec = docker.types.ContainerSpec(BUSYBOX, ['true']) + task_tmpl = docker.types.TaskTemplate(container_spec) + name = self.get_service_name() + endpoint_spec = docker.types.EndpointSpec(ports={ + 12357: (1990, None, 'host'), + }) + svc_id = self.client.create_service( + task_tmpl, name=name, endpoint_spec=endpoint_spec + ) + svc_info = self.client.inspect_service(svc_id) + ports = svc_info['Spec']['EndpointSpec']['Ports'] + assert len(ports) == 1 + port = ports[0] + assert port['PublishedPort'] == 12357 + assert port['TargetPort'] == 1990 + assert port['Protocol'] == 'tcp' + assert port['PublishMode'] == 'host' + def test_create_service_with_env(self): container_spec = docker.types.ContainerSpec( BUSYBOX, ['true'], env={'DOCKER_PY_TEST': 1} diff --git a/tests/unit/dockertypes_test.py b/tests/unit/dockertypes_test.py index 93c1397209..71dae7eac5 100644 --- a/tests/unit/dockertypes_test.py +++ b/tests/unit/dockertypes_test.py @@ -11,6 +11,7 @@ ContainerConfig, ContainerSpec, EndpointConfig, HostConfig, IPAMConfig, IPAMPool, LogConfig, Mount, ServiceMode, Ulimit, ) +from docker.types.services import convert_service_ports try: from unittest import mock @@ -423,3 +424,77 @@ def test_parse_mount_bind_windows(self): assert mount['Source'] == "C:/foo/bar" assert mount['Target'] == "/baz" assert mount['Type'] == 'bind' + + +class ServicePortsTest(unittest.TestCase): + def test_convert_service_ports_simple(self): + ports = {8080: 80} + assert convert_service_ports(ports) == [{ + 'Protocol': 'tcp', + 'PublishedPort': 8080, + 'TargetPort': 80, + }] + + def test_convert_service_ports_with_protocol(self): + ports = {8080: (80, 'udp')} + + assert convert_service_ports(ports) == [{ + 'Protocol': 'udp', + 'PublishedPort': 8080, + 'TargetPort': 80, + }] + + def test_convert_service_ports_with_protocol_and_mode(self): + ports = {8080: (80, 'udp', 'ingress')} + + assert convert_service_ports(ports) == [{ + 'Protocol': 'udp', + 'PublishedPort': 8080, + 'TargetPort': 80, + 'PublishMode': 'ingress', + }] + + def test_convert_service_ports_invalid(self): + ports = {8080: ('way', 'too', 'many', 'items', 'here')} + + with pytest.raises(ValueError): + convert_service_ports(ports) + + def test_convert_service_ports_no_protocol_and_mode(self): + ports = {8080: (80, None, 'host')} + + assert convert_service_ports(ports) == [{ + 'Protocol': 'tcp', + 'PublishedPort': 8080, + 'TargetPort': 80, + 'PublishMode': 'host', + }] + + def test_convert_service_ports_multiple(self): + ports = { + 8080: (80, None, 'host'), + 9999: 99, + 2375: (2375,) + } + + converted_ports = convert_service_ports(ports) + assert { + 'Protocol': 'tcp', + 'PublishedPort': 8080, + 'TargetPort': 80, + 'PublishMode': 'host', + } in converted_ports + + assert { + 'Protocol': 'tcp', + 'PublishedPort': 9999, + 'TargetPort': 99, + } in converted_ports + + assert { + 'Protocol': 'tcp', + 'PublishedPort': 2375, + 'TargetPort': 2375, + } in converted_ports + + assert len(converted_ports) == 3 From b180b8770a265e33099bd6da76c3e556a1028491 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 31 Jan 2018 13:14:21 -0800 Subject: [PATCH 0575/1301] Remove parameters and methods marked as deprecated Signed-off-by: Joffrey F --- docker/api/container.py | 35 ----------------------------------- docker/api/daemon.py | 9 +-------- docker/api/image.py | 30 ++++++++---------------------- docker/api/service.py | 13 ------------- docker/types/containers.py | 15 --------------- docker/utils/__init__.py | 2 +- docker/utils/utils.py | 25 ------------------------- 7 files changed, 10 insertions(+), 119 deletions(-) diff --git a/docker/api/container.py b/docker/api/container.py index 260fbe91b5..419ae442cc 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -1,5 +1,4 @@ import six -import warnings from datetime import datetime from .. import errors @@ -204,40 +203,6 @@ def containers(self, quiet=False, all=False, trunc=False, latest=False, x['Id'] = x['Id'][:12] return res - @utils.check_resource('container') - def copy(self, container, resource): - """ - Identical to the ``docker cp`` command. Get files/folders from the - container. - - **Deprecated for API version >= 1.20.** Use - :py:meth:`~ContainerApiMixin.get_archive` instead. - - Args: - container (str): The container to copy from - resource (str): The path within the container - - Returns: - The contents of the file as a string - - Raises: - :py:class:`docker.errors.APIError` - If the server returns an error. - """ - if utils.version_gte(self._version, '1.20'): - warnings.warn( - 'APIClient.copy() is deprecated for API version >= 1.20, ' - 'please use get_archive() instead', - DeprecationWarning - ) - res = self._post_json( - self._url("/containers/{0}/copy", container), - data={"Resource": resource}, - stream=True - ) - self._raise_for_status(res) - return res.raw - def create_container(self, image, command=None, hostname=None, user=None, detach=False, stdin_open=False, tty=False, mem_limit=None, ports=None, environment=None, diff --git a/docker/api/daemon.py b/docker/api/daemon.py index 39989679bc..033dbf19e1 100644 --- a/docker/api/daemon.py +++ b/docker/api/daemon.py @@ -1,9 +1,7 @@ import os -import warnings from datetime import datetime from .. import auth, utils -from ..constants import INSECURE_REGISTRY_DEPRECATION_WARNING class DaemonApiMixin(object): @@ -90,7 +88,7 @@ def info(self): return self._result(self._get(self._url("/info")), True) def login(self, username, password=None, email=None, registry=None, - reauth=False, insecure_registry=False, dockercfg_path=None): + reauth=False, dockercfg_path=None): """ Authenticate with a registry. Similar to the ``docker login`` command. @@ -113,11 +111,6 @@ def login(self, username, password=None, email=None, registry=None, :py:class:`docker.errors.APIError` If the server returns an error. """ - if insecure_registry: - warnings.warn( - INSECURE_REGISTRY_DEPRECATION_WARNING.format('login()'), - DeprecationWarning - ) # If we don't have any auth data so far, try reloading the config file # one more time in case anything showed up in there. diff --git a/docker/api/image.py b/docker/api/image.py index b3dcd3ab5a..f37d2dd924 100644 --- a/docker/api/image.py +++ b/docker/api/image.py @@ -1,11 +1,9 @@ import logging import os -import warnings import six from .. import auth, errors, utils -from ..constants import INSECURE_REGISTRY_DEPRECATION_WARNING log = logging.getLogger(__name__) @@ -321,9 +319,8 @@ def prune_images(self, filters=None): params['filters'] = utils.convert_filters(filters) return self._result(self._post(url, params=params), True) - def pull(self, repository, tag=None, stream=False, - insecure_registry=False, auth_config=None, decode=False, - platform=None): + def pull(self, repository, tag=None, stream=False, auth_config=None, + decode=False, platform=None): """ Pulls an image. Similar to the ``docker pull`` command. @@ -331,11 +328,12 @@ def pull(self, repository, tag=None, stream=False, repository (str): The repository to pull tag (str): The tag to pull stream (bool): Stream the output as a generator - insecure_registry (bool): Use an insecure registry auth_config (dict): Override the credentials that :py:meth:`~docker.api.daemon.DaemonApiMixin.login` has set for this request. ``auth_config`` should contain the ``username`` and ``password`` keys to be valid. + decode (bool): Decode the JSON data from the server into dicts. + Only applies with ``stream=True`` platform (str): Platform in the format ``os[/arch[/variant]]`` Returns: @@ -361,12 +359,6 @@ def pull(self, repository, tag=None, stream=False, } """ - if insecure_registry: - warnings.warn( - INSECURE_REGISTRY_DEPRECATION_WARNING.format('pull()'), - DeprecationWarning - ) - if not tag: repository, tag = utils.parse_repository_tag(repository) registry, repo_name = auth.resolve_repository_name(repository) @@ -405,8 +397,8 @@ def pull(self, repository, tag=None, stream=False, return self._result(response) - def push(self, repository, tag=None, stream=False, - insecure_registry=False, auth_config=None, decode=False): + def push(self, repository, tag=None, stream=False, auth_config=None, + decode=False): """ Push an image or a repository to the registry. Similar to the ``docker push`` command. @@ -415,12 +407,12 @@ def push(self, repository, tag=None, stream=False, repository (str): The repository to push to tag (str): An optional tag to push stream (bool): Stream the output as a blocking generator - insecure_registry (bool): Use ``http://`` to connect to the - registry auth_config (dict): Override the credentials that :py:meth:`~docker.api.daemon.DaemonApiMixin.login` has set for this request. ``auth_config`` should contain the ``username`` and ``password`` keys to be valid. + decode (bool): Decode the JSON data from the server into dicts. + Only applies with ``stream=True`` Returns: (generator or str): The output from the server. @@ -439,12 +431,6 @@ def push(self, repository, tag=None, stream=False, ... """ - if insecure_registry: - warnings.warn( - INSECURE_REGISTRY_DEPRECATION_WARNING.format('push()'), - DeprecationWarning - ) - if not tag: repository, tag = utils.parse_repository_tag(repository) registry, repo_name = auth.resolve_repository_name(repository) diff --git a/docker/api/service.py b/docker/api/service.py index 051d34fd07..ceae8fc9a8 100644 --- a/docker/api/service.py +++ b/docker/api/service.py @@ -1,4 +1,3 @@ -import warnings from .. import auth, errors, utils from ..types import ServiceMode @@ -123,12 +122,6 @@ def create_service( :py:class:`docker.errors.APIError` If the server returns an error. """ - if endpoint_config is not None: - warnings.warn( - 'endpoint_config has been renamed to endpoint_spec.', - DeprecationWarning - ) - endpoint_spec = endpoint_config _check_api_features( self._version, task_template, update_config, endpoint_spec @@ -370,12 +363,6 @@ def update_service(self, service, version, task_template=None, name=None, :py:class:`docker.errors.APIError` If the server returns an error. """ - if endpoint_config is not None: - warnings.warn( - 'endpoint_config has been renamed to endpoint_spec.', - DeprecationWarning - ) - endpoint_spec = endpoint_config _check_api_features( self._version, task_template, update_config, endpoint_spec diff --git a/docker/types/containers.py b/docker/types/containers.py index 15dd86c991..3f159058e0 100644 --- a/docker/types/containers.py +++ b/docker/types/containers.py @@ -1,5 +1,4 @@ import six -import warnings from .. import errors from ..utils.utils import ( @@ -542,13 +541,6 @@ def __init__( raise errors.InvalidVersion( 'labels were only introduced in API version 1.18' ) - else: - if cpuset is not None or cpu_shares is not None: - warnings.warn( - 'The cpuset_cpus and cpu_shares options have been moved to' - ' host_config in API version 1.18, and will be removed', - DeprecationWarning - ) if version_lt(version, '1.19'): if volume_driver is not None: @@ -575,13 +567,6 @@ def __init__( raise errors.InvalidVersion( 'stop_signal was only introduced in API version 1.21' ) - else: - if volume_driver is not None: - warnings.warn( - 'The volume_driver option has been moved to' - ' host_config in API version 1.21, and will be removed', - DeprecationWarning - ) if stop_timeout is not None and version_lt(version, '1.25'): raise errors.InvalidVersion( diff --git a/docker/utils/__init__.py b/docker/utils/__init__.py index c162e3bd6a..e70a5e615d 100644 --- a/docker/utils/__init__.py +++ b/docker/utils/__init__.py @@ -5,7 +5,7 @@ compare_version, convert_port_bindings, convert_volume_binds, mkbuildcontext, parse_repository_tag, parse_host, kwargs_from_env, convert_filters, datetime_to_timestamp, - create_host_config, parse_bytes, ping_registry, parse_env_file, version_lt, + create_host_config, parse_bytes, parse_env_file, version_lt, version_gte, decode_json_header, split_command, create_ipam_config, create_ipam_pool, parse_devices, normalize_links, convert_service_networks, format_environment, create_archive, format_extra_hosts diff --git a/docker/utils/utils.py b/docker/utils/utils.py index c4db1750b3..e4e2c0dfc9 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -6,11 +6,9 @@ import shlex import tarfile import tempfile -import warnings from distutils.version import StrictVersion from datetime import datetime -import requests import six from .. import constants @@ -158,29 +156,6 @@ def version_gte(v1, v2): return not version_lt(v1, v2) -def ping_registry(url): - warnings.warn( - 'The `ping_registry` method is deprecated and will be removed.', - DeprecationWarning - ) - - return ping(url + '/v2/', [401]) or ping(url + '/v1/_ping') - - -def ping(url, valid_4xx_statuses=None): - try: - res = requests.get(url, timeout=3) - except Exception: - return False - else: - # We don't send yet auth headers - # and a v2 registry will respond with status 401 - return ( - res.status_code < 400 or - (valid_4xx_statuses and res.status_code in valid_4xx_statuses) - ) - - def _convert_port_binding(binding): result = {'HostIp': '', 'HostPort': ''} if isinstance(binding, tuple): From df8422d0791d7d03cd3e1efe37a9c72f242f1f78 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 31 Jan 2018 14:32:32 -0800 Subject: [PATCH 0576/1301] Refuse API version < 1.21 ; Remove associated code paths Signed-off-by: Joffrey F --- docker/api/build.py | 50 ++------- docker/api/client.py | 27 +---- docker/api/container.py | 137 +++++++++--------------- docker/api/exec_api.py | 12 --- docker/api/image.py | 50 +++------ docker/api/network.py | 6 -- docker/api/volume.py | 4 - docker/types/containers.py | 108 ++----------------- tests/integration/api_build_test.py | 16 ++- tests/integration/api_container_test.py | 46 +++----- tests/integration/api_network_test.py | 9 -- tests/integration/api_volume_test.py | 1 - tests/unit/api_build_test.py | 11 -- tests/unit/api_container_test.py | 60 ----------- tests/unit/api_image_test.py | 20 ---- tests/unit/api_network_test.py | 7 -- tests/unit/api_test.py | 1 - tests/unit/api_volume_test.py | 7 -- tests/unit/dockertypes_test.py | 48 ++------- tests/unit/fake_api.py | 39 ++++--- 20 files changed, 140 insertions(+), 519 deletions(-) diff --git a/docker/api/build.py b/docker/api/build.py index 220c93f72f..56f1fcfc73 100644 --- a/docker/api/build.py +++ b/docker/api/build.py @@ -1,7 +1,6 @@ import json import logging import os -import re from .. import auth from .. import constants @@ -14,7 +13,7 @@ class BuildApiMixin(object): def build(self, path=None, tag=None, quiet=False, fileobj=None, - nocache=False, rm=False, stream=False, timeout=None, + nocache=False, rm=False, timeout=None, custom_context=False, encoding=None, pull=False, forcerm=False, dockerfile=None, container_limits=None, decode=False, buildargs=None, gzip=False, shmsize=None, @@ -67,9 +66,6 @@ def build(self, path=None, tag=None, quiet=False, fileobj=None, rm (bool): Remove intermediate containers. The ``docker build`` command now defaults to ``--rm=true``, but we have kept the old default of `False` to preserve backward compatibility - stream (bool): *Deprecated for API version > 1.8 (always True)*. - Return a blocking generator you can iterate over to retrieve - build output as it happens timeout (int): HTTP timeout custom_context (bool): Optional if using ``fileobj`` encoding (str): The encoding for a stream. Set to ``gzip`` for @@ -154,17 +150,6 @@ def build(self, path=None, tag=None, quiet=False, fileobj=None, ) encoding = 'gzip' if gzip else encoding - if utils.compare_version('1.8', self._version) >= 0: - stream = True - - if dockerfile and utils.compare_version('1.17', self._version) < 0: - raise errors.InvalidVersion( - 'dockerfile was only introduced in API version 1.17' - ) - - if utils.compare_version('1.19', self._version) < 0: - pull = 1 if pull else 0 - u = self._url('/build') params = { 't': tag, @@ -179,12 +164,7 @@ def build(self, path=None, tag=None, quiet=False, fileobj=None, params.update(container_limits) if buildargs: - if utils.version_gte(self._version, '1.21'): - params.update({'buildargs': json.dumps(buildargs)}) - else: - raise errors.InvalidVersion( - 'buildargs was only introduced in API version 1.21' - ) + params.update({'buildargs': json.dumps(buildargs)}) if shmsize: if utils.version_gte(self._version, '1.22'): @@ -256,30 +236,21 @@ def build(self, path=None, tag=None, quiet=False, fileobj=None, if encoding: headers['Content-Encoding'] = encoding - if utils.compare_version('1.9', self._version) >= 0: - self._set_auth_headers(headers) + self._set_auth_headers(headers) response = self._post( u, data=context, params=params, headers=headers, - stream=stream, + stream=True, timeout=timeout, ) if context is not None and not custom_context: context.close() - if stream: - return self._stream_helper(response, decode=decode) - else: - output = self._result(response) - srch = r'Successfully built ([0-9a-f]+)' - match = re.search(srch, output) - if not match: - return None, output - return match.group(1), output + return self._stream_helper(response, decode=decode) def _set_auth_headers(self, headers): log.debug('Looking for auth config') @@ -316,13 +287,8 @@ def _set_auth_headers(self, headers): ) ) - if utils.compare_version('1.19', self._version) >= 0: - headers['X-Registry-Config'] = auth.encode_header( - auth_data - ) - else: - headers['X-Registry-Config'] = auth.encode_header({ - 'configs': auth_data - }) + headers['X-Registry-Config'] = auth.encode_header( + auth_data + ) else: log.debug('No auth config found') diff --git a/docker/api/client.py b/docker/api/client.py index 07bcfae4a5..e69d143b2f 100644 --- a/docker/api/client.py +++ b/docker/api/client.py @@ -1,6 +1,5 @@ import json import struct -import warnings from functools import partial import requests @@ -27,7 +26,7 @@ MINIMUM_DOCKER_API_VERSION ) from ..errors import ( - DockerException, TLSParameterError, + DockerException, InvalidVersion, TLSParameterError, create_api_error_from_http_exception ) from ..tls import TLSConfig @@ -160,11 +159,9 @@ def __init__(self, base_url=None, version=None, ) ) if utils.version_lt(self._version, MINIMUM_DOCKER_API_VERSION): - warnings.warn( - 'The minimum API version supported is {}, but you are using ' - 'version {}. It is recommended you either upgrade Docker ' - 'Engine or use an older version of Docker SDK for ' - 'Python.'.format(MINIMUM_DOCKER_API_VERSION, self._version) + raise InvalidVersion( + 'API versions below {} are no longer supported by this ' + 'library.'.format(MINIMUM_DOCKER_API_VERSION) ) def _retrieve_server_version(self): @@ -353,17 +350,8 @@ def _multiplexed_response_stream_helper(self, response): break yield data - def _stream_raw_result_old(self, response): - ''' Stream raw output for API versions below 1.6 ''' - self._raise_for_status(response) - for line in response.iter_lines(chunk_size=1, - decode_unicode=True): - # filter out keep-alive new lines - if line: - yield line - def _stream_raw_result(self, response): - ''' Stream result for TTY-enabled container above API 1.6 ''' + ''' Stream result for TTY-enabled container ''' self._raise_for_status(response) for out in response.iter_content(chunk_size=1, decode_unicode=True): yield out @@ -419,11 +407,6 @@ def _get_result(self, container, stream, res): return self._get_result_tty(stream, res, self._check_is_tty(container)) def _get_result_tty(self, stream, res, is_tty): - # Stream multi-plexing was only introduced in API v1.6. Anything - # before that needs old-style streaming. - if utils.compare_version('1.6', self._version) < 0: - return self._stream_raw_result_old(res) - # We should also use raw streaming (without keep-alives) # if we're dealing with a tty-enabled container. if is_tty: diff --git a/docker/api/container.py b/docker/api/container.py index 419ae442cc..152a08b325 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -204,15 +204,13 @@ def containers(self, quiet=False, all=False, trunc=False, latest=False, return res def create_container(self, image, command=None, hostname=None, user=None, - detach=False, stdin_open=False, tty=False, - mem_limit=None, ports=None, environment=None, - dns=None, volumes=None, volumes_from=None, + detach=False, stdin_open=False, tty=False, ports=None, + environment=None, volumes=None, network_disabled=False, name=None, entrypoint=None, - cpu_shares=None, working_dir=None, domainname=None, - memswap_limit=None, cpuset=None, host_config=None, - mac_address=None, labels=None, volume_driver=None, - stop_signal=None, networking_config=None, - healthcheck=None, stop_timeout=None, runtime=None): + working_dir=None, domainname=None, host_config=None, + mac_address=None, labels=None, stop_signal=None, + networking_config=None, healthcheck=None, + stop_timeout=None, runtime=None): """ Creates a container. Parameters are similar to those for the ``docker run`` command except it doesn't support the attach options (``-a``). @@ -354,27 +352,17 @@ def create_container(self, image, command=None, hostname=None, user=None, return container ID stdin_open (bool): Keep STDIN open even if not attached tty (bool): Allocate a pseudo-TTY - mem_limit (float or str): Memory limit. Accepts float values (which - represent the memory limit of the created container in bytes) - or a string with a units identification char (``100000b``, - ``1000k``, ``128m``, ``1g``). If a string is specified without - a units character, bytes are assumed as an intended unit. ports (list of ints): A list of port numbers environment (dict or list): A dictionary or a list of strings in the following format ``["PASSWORD=xxx"]`` or ``{"PASSWORD": "xxx"}``. - dns (:py:class:`list`): DNS name servers. Deprecated since API - version 1.10. Use ``host_config`` instead. volumes (str or list): List of paths inside the container to use as volumes. - volumes_from (:py:class:`list`): List of container names or Ids to - get volumes from. network_disabled (bool): Disable networking name (str): A name for the container entrypoint (str or list): An entrypoint working_dir (str): Path to the working directory domainname (str): The domain name to use for the container - memswap_limit (int): host_config (dict): A dictionary created with :py:meth:`create_host_config`. mac_address (str): The Mac Address to assign the container @@ -382,7 +370,6 @@ def create_container(self, image, command=None, hostname=None, user=None, ``{"label1": "value1", "label2": "value2"}``) or a list of names of labels to set with empty values (e.g. ``["label1", "label2"]``) - volume_driver (str): The name of a volume driver/plugin. stop_signal (str): The stop signal to use to stop the container (e.g. ``SIGINT``). stop_timeout (int): Timeout to stop the container, in seconds. @@ -405,17 +392,12 @@ def create_container(self, image, command=None, hostname=None, user=None, if isinstance(volumes, six.string_types): volumes = [volumes, ] - if host_config and utils.compare_version('1.15', self._version) < 0: - raise errors.InvalidVersion( - 'host_config is not supported in API < 1.15' - ) - config = self.create_container_config( - image, command, hostname, user, detach, stdin_open, tty, mem_limit, - ports, dns, environment, volumes, volumes_from, - network_disabled, entrypoint, cpu_shares, working_dir, domainname, - memswap_limit, cpuset, host_config, mac_address, labels, - volume_driver, stop_signal, networking_config, healthcheck, + image, command, hostname, user, detach, stdin_open, tty, + ports, environment, volumes, + network_disabled, entrypoint, working_dir, domainname, + host_config, mac_address, labels, + stop_signal, networking_config, healthcheck, stop_timeout, runtime ) return self.create_container_from_config(config, name) @@ -681,7 +663,6 @@ def export(self, container): return self._stream_raw_result(res) @utils.check_resource('container') - @utils.minimum_version('1.20') def get_archive(self, container, path): """ Retrieve a file or folder from a container in the form of a tar @@ -786,59 +767,46 @@ def logs(self, container, stdout=True, stderr=True, stream=False, :py:class:`docker.errors.APIError` If the server returns an error. """ - if utils.compare_version('1.11', self._version) >= 0: - if follow is None: - follow = stream - params = {'stderr': stderr and 1 or 0, - 'stdout': stdout and 1 or 0, - 'timestamps': timestamps and 1 or 0, - 'follow': follow and 1 or 0, - } - if utils.compare_version('1.13', self._version) >= 0: - if tail != 'all' and (not isinstance(tail, int) or tail < 0): - tail = 'all' - params['tail'] = tail - - if since is not None: - if utils.version_lt(self._version, '1.19'): - raise errors.InvalidVersion( - 'since is not supported for API version < 1.19' - ) - if isinstance(since, datetime): - params['since'] = utils.datetime_to_timestamp(since) - elif (isinstance(since, int) and since > 0): - params['since'] = since - else: - raise errors.InvalidArgument( - 'since value should be datetime or positive int, ' - 'not {}'.format(type(since)) - ) - - if until is not None: - if utils.version_lt(self._version, '1.35'): - raise errors.InvalidVersion( - 'until is not supported for API version < 1.35' - ) - if isinstance(until, datetime): - params['until'] = utils.datetime_to_timestamp(until) - elif (isinstance(until, int) and until > 0): - params['until'] = until - else: - raise errors.InvalidArgument( - 'until value should be datetime or positive int, ' - 'not {}'.format(type(until)) - ) - - url = self._url("/containers/{0}/logs", container) - res = self._get(url, params=params, stream=stream) - return self._get_result(container, stream, res) - return self.attach( - container, - stdout=stdout, - stderr=stderr, - stream=stream, - logs=True - ) + if follow is None: + follow = stream + params = {'stderr': stderr and 1 or 0, + 'stdout': stdout and 1 or 0, + 'timestamps': timestamps and 1 or 0, + 'follow': follow and 1 or 0, + } + if tail != 'all' and (not isinstance(tail, int) or tail < 0): + tail = 'all' + params['tail'] = tail + + if since is not None: + if isinstance(since, datetime): + params['since'] = utils.datetime_to_timestamp(since) + elif (isinstance(since, int) and since > 0): + params['since'] = since + else: + raise errors.InvalidArgument( + 'since value should be datetime or positive int, ' + 'not {}'.format(type(since)) + ) + + if until is not None: + if utils.version_lt(self._version, '1.35'): + raise errors.InvalidVersion( + 'until is not supported for API version < 1.35' + ) + if isinstance(until, datetime): + params['until'] = utils.datetime_to_timestamp(until) + elif (isinstance(until, int) and until > 0): + params['until'] = until + else: + raise errors.InvalidArgument( + 'until value should be datetime or positive int, ' + 'not {}'.format(type(until)) + ) + + url = self._url("/containers/{0}/logs", container) + res = self._get(url, params=params, stream=stream) + return self._get_result(container, stream, res) @utils.check_resource('container') def pause(self, container): @@ -906,7 +874,6 @@ def port(self, container, private_port): return h_ports @utils.check_resource('container') - @utils.minimum_version('1.20') def put_archive(self, container, path, data): """ Insert a file or folder in an existing container using a tar archive as @@ -976,7 +943,6 @@ def remove_container(self, container, v=False, link=False, force=False): ) self._raise_for_status(res) - @utils.minimum_version('1.17') @utils.check_resource('container') def rename(self, container, name): """ @@ -1073,7 +1039,6 @@ def start(self, container, *args, **kwargs): res = self._post(url) self._raise_for_status(res) - @utils.minimum_version('1.17') @utils.check_resource('container') def stats(self, container, decode=None, stream=True): """ diff --git a/docker/api/exec_api.py b/docker/api/exec_api.py index d607461f5d..986d87f21c 100644 --- a/docker/api/exec_api.py +++ b/docker/api/exec_api.py @@ -5,7 +5,6 @@ class ExecApiMixin(object): - @utils.minimum_version('1.15') @utils.check_resource('container') def exec_create(self, container, cmd, stdout=True, stderr=True, stdin=False, tty=False, privileged=False, user='', @@ -41,14 +40,6 @@ def exec_create(self, container, cmd, stdout=True, stderr=True, If the server returns an error. """ - if privileged and utils.version_lt(self._version, '1.19'): - raise errors.InvalidVersion( - 'Privileged exec is not supported in API < 1.19' - ) - if user and utils.version_lt(self._version, '1.19'): - raise errors.InvalidVersion( - 'User-specific exec is not supported in API < 1.19' - ) if environment is not None and utils.version_lt(self._version, '1.25'): raise errors.InvalidVersion( 'Setting environment for exec is not supported in API < 1.25' @@ -88,7 +79,6 @@ def exec_create(self, container, cmd, stdout=True, stderr=True, res = self._post_json(url, data=data) return self._result(res, True) - @utils.minimum_version('1.16') def exec_inspect(self, exec_id): """ Return low-level information about an exec command. @@ -108,7 +98,6 @@ def exec_inspect(self, exec_id): res = self._get(self._url("/exec/{0}/json", exec_id)) return self._result(res, True) - @utils.minimum_version('1.15') def exec_resize(self, exec_id, height=None, width=None): """ Resize the tty session used by the specified exec command. @@ -127,7 +116,6 @@ def exec_resize(self, exec_id, height=None, width=None): res = self._post(url, params=params) self._raise_for_status(res) - @utils.minimum_version('1.15') @utils.check_resource('exec_id') def exec_start(self, exec_id, detach=False, tty=False, stream=False, socket=False): diff --git a/docker/api/image.py b/docker/api/image.py index f37d2dd924..fa832a389a 100644 --- a/docker/api/image.py +++ b/docker/api/image.py @@ -54,8 +54,7 @@ def history(self, image): res = self._get(self._url("/images/{0}/history", image)) return self._result(res, True) - def images(self, name=None, quiet=False, all=False, viz=False, - filters=None): + def images(self, name=None, quiet=False, all=False, filters=None): """ List images. Similar to the ``docker images`` command. @@ -76,10 +75,6 @@ def images(self, name=None, quiet=False, all=False, viz=False, :py:class:`docker.errors.APIError` If the server returns an error. """ - if viz: - if utils.compare_version('1.7', self._version) >= 0: - raise Exception('Viz output is not supported in API >= 1.7!') - return self._result(self._get(self._url("images/viz"))) params = { 'filter': name, 'only_ids': 1 if quiet else 0, @@ -225,19 +220,6 @@ def import_image_from_image(self, image, repository=None, tag=None, image=image, repository=repository, tag=tag, changes=changes ) - @utils.check_resource('image') - def insert(self, image, url, path): - if utils.compare_version('1.12', self._version) >= 0: - raise errors.DeprecatedMethod( - 'insert is not available for API version >=1.12' - ) - api_url = self._url("/images/{0}/insert", image) - params = { - 'url': url, - 'path': path - } - return self._result(self._post(api_url, params=params)) - @utils.check_resource('image') def inspect_image(self, image): """ @@ -369,14 +351,13 @@ def pull(self, repository, tag=None, stream=False, auth_config=None, } headers = {} - if utils.version_gte(self._version, '1.5'): - if auth_config is None: - header = auth.get_config_header(self, registry) - if header: - headers['X-Registry-Auth'] = header - else: - log.debug('Sending supplied auth config') - headers['X-Registry-Auth'] = auth.encode_header(auth_config) + if auth_config is None: + header = auth.get_config_header(self, registry) + if header: + headers['X-Registry-Auth'] = header + else: + log.debug('Sending supplied auth config') + headers['X-Registry-Auth'] = auth.encode_header(auth_config) if platform is not None: if utils.version_lt(self._version, '1.32'): @@ -440,14 +421,13 @@ def push(self, repository, tag=None, stream=False, auth_config=None, } headers = {} - if utils.compare_version('1.5', self._version) >= 0: - if auth_config is None: - header = auth.get_config_header(self, registry) - if header: - headers['X-Registry-Auth'] = header - else: - log.debug('Sending supplied auth config') - headers['X-Registry-Auth'] = auth.encode_header(auth_config) + if auth_config is None: + header = auth.get_config_header(self, registry) + if header: + headers['X-Registry-Auth'] = header + else: + log.debug('Sending supplied auth config') + headers['X-Registry-Auth'] = auth.encode_header(auth_config) response = self._post_json( u, None, headers=headers, stream=stream, params=params diff --git a/docker/api/network.py b/docker/api/network.py index 797780858a..57ed8d3b75 100644 --- a/docker/api/network.py +++ b/docker/api/network.py @@ -5,7 +5,6 @@ class NetworkApiMixin(object): - @minimum_version('1.21') def networks(self, names=None, ids=None, filters=None): """ List networks. Similar to the ``docker networks ls`` command. @@ -38,7 +37,6 @@ def networks(self, names=None, ids=None, filters=None): res = self._get(url, params=params) return self._result(res, json=True) - @minimum_version('1.21') def create_network(self, name, driver=None, options=None, ipam=None, check_duplicate=None, internal=False, labels=None, enable_ipv6=False, attachable=None, scope=None, @@ -175,7 +173,6 @@ def prune_networks(self, filters=None): url = self._url('/networks/prune') return self._result(self._post(url, params=params), True) - @minimum_version('1.21') @check_resource('net_id') def remove_network(self, net_id): """ @@ -188,7 +185,6 @@ def remove_network(self, net_id): res = self._delete(url) self._raise_for_status(res) - @minimum_version('1.21') @check_resource('net_id') def inspect_network(self, net_id, verbose=None, scope=None): """ @@ -216,7 +212,6 @@ def inspect_network(self, net_id, verbose=None, scope=None): return self._result(res, json=True) @check_resource('container') - @minimum_version('1.21') def connect_container_to_network(self, container, net_id, ipv4_address=None, ipv6_address=None, aliases=None, links=None, @@ -253,7 +248,6 @@ def connect_container_to_network(self, container, net_id, self._raise_for_status(res) @check_resource('container') - @minimum_version('1.21') def disconnect_container_from_network(self, container, net_id, force=False): """ diff --git a/docker/api/volume.py b/docker/api/volume.py index ce911c8fcd..900a6086b5 100644 --- a/docker/api/volume.py +++ b/docker/api/volume.py @@ -3,7 +3,6 @@ class VolumeApiMixin(object): - @utils.minimum_version('1.21') def volumes(self, filters=None): """ List volumes currently registered by the docker daemon. Similar to the @@ -37,7 +36,6 @@ def volumes(self, filters=None): url = self._url('/volumes') return self._result(self._get(url, params=params), True) - @utils.minimum_version('1.21') def create_volume(self, name=None, driver=None, driver_opts=None, labels=None): """ @@ -90,7 +88,6 @@ def create_volume(self, name=None, driver=None, driver_opts=None, return self._result(self._post_json(url, data=data), True) - @utils.minimum_version('1.21') def inspect_volume(self, name): """ Retrieve volume info by name. @@ -138,7 +135,6 @@ def prune_volumes(self, filters=None): url = self._url('/volumes/prune') return self._result(self._post(url, params=params), True) - @utils.minimum_version('1.21') def remove_volume(self, name, force=False): """ Remove a volume. Similar to the ``docker volume rm`` command. diff --git a/docker/types/containers.py b/docker/types/containers.py index 3f159058e0..b4a329c220 100644 --- a/docker/types/containers.py +++ b/docker/types/containers.py @@ -129,20 +129,12 @@ def __init__(self, version, binds=None, port_bindings=None, self['MemorySwap'] = parse_bytes(memswap_limit) if mem_reservation: - if version_lt(version, '1.21'): - raise host_config_version_error('mem_reservation', '1.21') - self['MemoryReservation'] = parse_bytes(mem_reservation) if kernel_memory: - if version_lt(version, '1.21'): - raise host_config_version_error('kernel_memory', '1.21') - self['KernelMemory'] = parse_bytes(kernel_memory) if mem_swappiness is not None: - if version_lt(version, '1.20'): - raise host_config_version_error('mem_swappiness', '1.20') if not isinstance(mem_swappiness, int): raise host_config_type_error( 'mem_swappiness', mem_swappiness, 'int' @@ -168,9 +160,6 @@ def __init__(self, version, binds=None, port_bindings=None, self['Privileged'] = privileged if oom_kill_disable: - if version_lt(version, '1.20'): - raise host_config_version_error('oom_kill_disable', '1.19') - self['OomKillDisable'] = oom_kill_disable if oom_score_adj: @@ -193,7 +182,7 @@ def __init__(self, version, binds=None, port_bindings=None, if network_mode: self['NetworkMode'] = network_mode - elif network_mode is None and version_gte(version, '1.20'): + elif network_mode is None: self['NetworkMode'] = 'default' if restart_policy: @@ -214,18 +203,12 @@ def __init__(self, version, binds=None, port_bindings=None, self['Devices'] = parse_devices(devices) if group_add: - if version_lt(version, '1.20'): - raise host_config_version_error('group_add', '1.20') - self['GroupAdd'] = [six.text_type(grp) for grp in group_add] if dns is not None: self['Dns'] = dns if dns_opt is not None: - if version_lt(version, '1.21'): - raise host_config_version_error('dns_opt', '1.21') - self['DnsOptions'] = dns_opt if security_opt is not None: @@ -298,38 +281,23 @@ def __init__(self, version, binds=None, port_bindings=None, if cpu_quota: if not isinstance(cpu_quota, int): raise host_config_type_error('cpu_quota', cpu_quota, 'int') - if version_lt(version, '1.19'): - raise host_config_version_error('cpu_quota', '1.19') - self['CpuQuota'] = cpu_quota if cpu_period: if not isinstance(cpu_period, int): raise host_config_type_error('cpu_period', cpu_period, 'int') - if version_lt(version, '1.19'): - raise host_config_version_error('cpu_period', '1.19') - self['CpuPeriod'] = cpu_period if cpu_shares: - if version_lt(version, '1.18'): - raise host_config_version_error('cpu_shares', '1.18') - if not isinstance(cpu_shares, int): raise host_config_type_error('cpu_shares', cpu_shares, 'int') self['CpuShares'] = cpu_shares if cpuset_cpus: - if version_lt(version, '1.18'): - raise host_config_version_error('cpuset_cpus', '1.18') - self['CpusetCpus'] = cpuset_cpus if cpuset_mems: - if version_lt(version, '1.19'): - raise host_config_version_error('cpuset_mems', '1.19') - if not isinstance(cpuset_mems, str): raise host_config_type_error( 'cpuset_mems', cpuset_mems, 'str' @@ -462,8 +430,6 @@ def __init__(self, version, binds=None, port_bindings=None, self['InitPath'] = init_path if volume_driver is not None: - if version_lt(version, '1.21'): - raise host_config_version_error('volume_driver', '1.21') self['VolumeDriver'] = volume_driver if cpu_count: @@ -520,53 +486,12 @@ def host_config_value_error(param, param_value): class ContainerConfig(dict): def __init__( self, version, image, command, hostname=None, user=None, detach=False, - stdin_open=False, tty=False, mem_limit=None, ports=None, dns=None, - environment=None, volumes=None, volumes_from=None, - network_disabled=False, entrypoint=None, cpu_shares=None, - working_dir=None, domainname=None, memswap_limit=None, cpuset=None, - host_config=None, mac_address=None, labels=None, volume_driver=None, - stop_signal=None, networking_config=None, healthcheck=None, - stop_timeout=None, runtime=None + stdin_open=False, tty=False, ports=None, environment=None, + volumes=None, network_disabled=False, entrypoint=None, + working_dir=None, domainname=None, host_config=None, mac_address=None, + labels=None, stop_signal=None, networking_config=None, + healthcheck=None, stop_timeout=None, runtime=None ): - if version_gte(version, '1.10'): - message = ('{0!r} parameter has no effect on create_container().' - ' It has been moved to host_config') - if dns is not None: - raise errors.InvalidVersion(message.format('dns')) - if volumes_from is not None: - raise errors.InvalidVersion(message.format('volumes_from')) - - if version_lt(version, '1.18'): - if labels is not None: - raise errors.InvalidVersion( - 'labels were only introduced in API version 1.18' - ) - - if version_lt(version, '1.19'): - if volume_driver is not None: - raise errors.InvalidVersion( - 'Volume drivers were only introduced in API version 1.19' - ) - mem_limit = mem_limit if mem_limit is not None else 0 - memswap_limit = memswap_limit if memswap_limit is not None else 0 - else: - if mem_limit is not None: - raise errors.InvalidVersion( - 'mem_limit has been moved to host_config in API version' - ' 1.19' - ) - - if memswap_limit is not None: - raise errors.InvalidVersion( - 'memswap_limit has been moved to host_config in API ' - 'version 1.19' - ) - - if version_lt(version, '1.21'): - if stop_signal is not None: - raise errors.InvalidVersion( - 'stop_signal was only introduced in API version 1.21' - ) if stop_timeout is not None and version_lt(version, '1.25'): raise errors.InvalidVersion( @@ -597,12 +522,6 @@ def __init__( if isinstance(labels, list): labels = dict((lbl, six.text_type('')) for lbl in labels) - if mem_limit is not None: - mem_limit = parse_bytes(mem_limit) - - if memswap_limit is not None: - memswap_limit = parse_bytes(memswap_limit) - if isinstance(ports, list): exposed_ports = {} for port_definition in ports: @@ -624,13 +543,6 @@ def __init__( volumes_dict[vol] = {} volumes = volumes_dict - if volumes_from: - if not isinstance(volumes_from, six.string_types): - volumes_from = ','.join(volumes_from) - else: - # Force None, an empty list or dict causes client.start to fail - volumes_from = None - if healthcheck and isinstance(healthcheck, dict): healthcheck = Healthcheck(**healthcheck) @@ -655,28 +567,20 @@ def __init__( 'Tty': tty, 'OpenStdin': stdin_open, 'StdinOnce': stdin_once, - 'Memory': mem_limit, 'AttachStdin': attach_stdin, 'AttachStdout': attach_stdout, 'AttachStderr': attach_stderr, 'Env': environment, 'Cmd': command, - 'Dns': dns, 'Image': image, 'Volumes': volumes, - 'VolumesFrom': volumes_from, 'NetworkDisabled': network_disabled, 'Entrypoint': entrypoint, - 'CpuShares': cpu_shares, - 'Cpuset': cpuset, - 'CpusetCpus': cpuset, 'WorkingDir': working_dir, - 'MemorySwap': memswap_limit, 'HostConfig': host_config, 'NetworkingConfig': networking_config, 'MacAddress': mac_address, 'Labels': labels, - 'VolumeDriver': volume_driver, 'StopSignal': stop_signal, 'Healthcheck': healthcheck, 'StopTimeout': stop_timeout, diff --git a/tests/integration/api_build_test.py b/tests/integration/api_build_test.py index ee9b68a619..4c2b992033 100644 --- a/tests/integration/api_build_test.py +++ b/tests/integration/api_build_test.py @@ -21,7 +21,7 @@ def test_build_streaming(self): 'ADD https://dl.dropboxusercontent.com/u/20637798/silence.tar.gz' ' /tmp/silence.tar.gz' ]).encode('ascii')) - stream = self.client.build(fileobj=script, stream=True, decode=True) + stream = self.client.build(fileobj=script, decode=True) logs = [] for chunk in stream: logs.append(chunk) @@ -37,7 +37,7 @@ def test_build_from_stringio(self): 'ADD https://dl.dropboxusercontent.com/u/20637798/silence.tar.gz' ' /tmp/silence.tar.gz' ])) - stream = self.client.build(fileobj=script, stream=True) + stream = self.client.build(fileobj=script) logs = '' for chunk in stream: if six.PY3: @@ -45,7 +45,6 @@ def test_build_from_stringio(self): logs += chunk assert logs != '' - @requires_api_version('1.8') def test_build_with_dockerignore(self): base_dir = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, base_dir) @@ -97,7 +96,6 @@ def test_build_with_dockerignore(self): '/test/not-ignored' ]) - @requires_api_version('1.21') def test_build_with_buildargs(self): script = io.BytesIO('\n'.join([ 'FROM scratch', @@ -320,7 +318,7 @@ def test_build_stderr_data(self): ])) stream = self.client.build( - fileobj=script, stream=True, decode=True, nocache=True + fileobj=script, decode=True, nocache=True ) lines = [] for chunk in stream: @@ -341,7 +339,7 @@ def test_build_gzip_encoding(self): ])) stream = self.client.build( - path=base_dir, stream=True, decode=True, nocache=True, + path=base_dir, decode=True, nocache=True, gzip=True ) @@ -365,7 +363,7 @@ def test_build_with_dockerfile_empty_lines(self): ])) stream = self.client.build( - path=base_dir, stream=True, decode=True, nocache=True + path=base_dir, decode=True, nocache=True ) lines = [] @@ -383,9 +381,7 @@ def test_build_invalid_platform(self): script = io.BytesIO('FROM busybox\n'.encode('ascii')) with pytest.raises(errors.APIError) as excinfo: - stream = self.client.build( - fileobj=script, stream=True, platform='foobar' - ) + stream = self.client.build(fileobj=script, platform='foobar') for _ in stream: pass diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index e5d79431fb..09253524a5 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -145,22 +145,18 @@ def test_create_container_with_volumes_from(self): container2_id = res1['Id'] self.tmp_containers.append(container2_id) self.client.start(container2_id) - with pytest.raises(docker.errors.DockerException): - self.client.create_container( - BUSYBOX, 'cat', detach=True, stdin_open=True, - volumes_from=vol_names - ) - res2 = self.client.create_container( + + res = self.client.create_container( BUSYBOX, 'cat', detach=True, stdin_open=True, host_config=self.client.create_host_config( volumes_from=vol_names, network_mode='none' ) ) - container3_id = res2['Id'] + container3_id = res['Id'] self.tmp_containers.append(container3_id) self.client.start(container3_id) - info = self.client.inspect_container(res2['Id']) + info = self.client.inspect_container(res['Id']) assert len(info['HostConfig']['VolumesFrom']) == len(vol_names) def create_container_readonly_fs(self): @@ -222,7 +218,6 @@ def test_create_with_mac_address(self): self.client.kill(id) - @requires_api_version('1.20') def test_group_id_ints(self): container = self.client.create_container( BUSYBOX, 'id -G', @@ -239,7 +234,6 @@ def test_group_id_ints(self): assert '1000' in groups assert '1001' in groups - @requires_api_version('1.20') def test_group_id_strings(self): container = self.client.create_container( BUSYBOX, 'id -G', host_config=self.client.create_host_config( @@ -604,24 +598,15 @@ def test_create_with_volume_mount(self): assert mount_data['RW'] is True def check_container_data(self, inspect_data, rw): - if docker.utils.compare_version('1.20', self.client._version) < 0: - assert 'Volumes' in inspect_data - assert self.mount_dest in inspect_data['Volumes'] - assert ( - self.mount_origin == inspect_data['Volumes'][self.mount_dest] - ) - assert self.mount_dest in inspect_data['VolumesRW'] - assert not inspect_data['VolumesRW'][self.mount_dest] - else: - assert 'Mounts' in inspect_data - filtered = list(filter( - lambda x: x['Destination'] == self.mount_dest, - inspect_data['Mounts'] - )) - assert len(filtered) == 1 - mount_data = filtered[0] - assert mount_data['Source'] == self.mount_origin - assert mount_data['RW'] == rw + assert 'Mounts' in inspect_data + filtered = list(filter( + lambda x: x['Destination'] == self.mount_dest, + inspect_data['Mounts'] + )) + assert len(filtered) == 1 + mount_data = filtered[0] + assert mount_data['Source'] == self.mount_origin + assert mount_data['RW'] == rw def run_with_volume(self, ro, *args, **kwargs): return self.run_container( @@ -640,7 +625,6 @@ def run_with_volume(self, ro, *args, **kwargs): ) -@requires_api_version('1.20') class ArchiveTest(BaseAPIIntegrationTest): def test_get_file_archive_from_container(self): data = 'The Maid and the Pocket Watch of Blood' @@ -1323,7 +1307,6 @@ def test_prune_containers(self): class GetContainerStatsTest(BaseAPIIntegrationTest): - @requires_api_version('1.19') def test_get_container_stats_no_stream(self): container = self.client.create_container( BUSYBOX, ['sleep', '60'], @@ -1338,7 +1321,6 @@ def test_get_container_stats_no_stream(self): 'memory_stats', 'blkio_stats']: assert key in response - @requires_api_version('1.17') def test_get_container_stats_stream(self): container = self.client.create_container( BUSYBOX, ['sleep', '60'], @@ -1401,7 +1383,6 @@ def test_restart_policy_update(self): class ContainerCPUTest(BaseAPIIntegrationTest): - @requires_api_version('1.18') def test_container_cpu_shares(self): cpu_shares = 512 container = self.client.create_container( @@ -1414,7 +1395,6 @@ def test_container_cpu_shares(self): inspect_data = self.client.inspect_container(container) assert inspect_data['HostConfig']['CpuShares'] == 512 - @requires_api_version('1.18') def test_container_cpuset(self): cpuset_cpus = "0,1" container = self.client.create_container( diff --git a/tests/integration/api_network_test.py b/tests/integration/api_network_test.py index ec92bd7956..b6726d0242 100644 --- a/tests/integration/api_network_test.py +++ b/tests/integration/api_network_test.py @@ -17,7 +17,6 @@ def create_network(self, *args, **kwargs): self.tmp_networks.append(net_id) return (net_name, net_id) - @requires_api_version('1.21') def test_list_networks(self): networks = self.client.networks() @@ -32,7 +31,6 @@ def test_list_networks(self): networks_by_partial_id = self.client.networks(ids=[net_id[:8]]) assert [n['Id'] for n in networks_by_partial_id] == [net_id] - @requires_api_version('1.21') def test_inspect_network(self): net_name, net_id = self.create_network() @@ -43,7 +41,6 @@ def test_inspect_network(self): assert net['Scope'] == 'local' assert net['IPAM']['Driver'] == 'default' - @requires_api_version('1.21') def test_create_network_with_ipam_config(self): _, net_id = self.create_network( ipam=IPAMConfig( @@ -81,12 +78,10 @@ def test_create_network_with_ipam_config(self): }, }] - @requires_api_version('1.21') def test_create_network_with_host_driver_fails(self): with pytest.raises(docker.errors.APIError): self.client.create_network(random_name(), driver='host') - @requires_api_version('1.21') def test_remove_network(self): net_name, net_id = self.create_network() assert net_name in [n['Name'] for n in self.client.networks()] @@ -94,7 +89,6 @@ def test_remove_network(self): self.client.remove_network(net_id) assert net_name not in [n['Name'] for n in self.client.networks()] - @requires_api_version('1.21') def test_connect_and_disconnect_container(self): net_name, net_id = self.create_network() @@ -163,7 +157,6 @@ def test_connect_with_aliases(self): assert 'foo' in aliases assert 'bar' in aliases - @requires_api_version('1.21') def test_connect_on_container_create(self): net_name, net_id = self.create_network() @@ -309,7 +302,6 @@ def test_create_with_links(self): self.execute(container, ['nslookup', 'bar']) - @requires_api_version('1.21') def test_create_check_duplicate(self): net_name, net_id = self.create_network() with pytest.raises(docker.errors.APIError): @@ -475,7 +467,6 @@ def test_create_inspect_network_with_scope(self): with pytest.raises(docker.errors.NotFound): self.client.inspect_network(net_name_swarm, scope='local') - @requires_api_version('1.21') def test_create_remove_network_with_space_in_name(self): net_id = self.client.create_network('test 01') self.tmp_networks.append(net_id) diff --git a/tests/integration/api_volume_test.py b/tests/integration/api_volume_test.py index e09f12a7fe..8e7dd3afb1 100644 --- a/tests/integration/api_volume_test.py +++ b/tests/integration/api_volume_test.py @@ -5,7 +5,6 @@ from .base import BaseAPIIntegrationTest -@requires_api_version('1.21') class TestVolumes(BaseAPIIntegrationTest): def test_create_volume(self): name = 'perfectcherryblossom' diff --git a/tests/unit/api_build_test.py b/tests/unit/api_build_test.py index b20daea87f..a7f34fd3f2 100644 --- a/tests/unit/api_build_test.py +++ b/tests/unit/api_build_test.py @@ -31,17 +31,6 @@ def test_build_container_pull(self): self.client.build(fileobj=script, pull=True) - def test_build_container_stream(self): - script = io.BytesIO('\n'.join([ - 'FROM busybox', - 'RUN mkdir -p /tmp/test', - 'EXPOSE 8080', - 'ADD https://dl.dropboxusercontent.com/u/20637798/silence.tar.gz' - ' /tmp/silence.tar.gz' - ]).encode('ascii')) - - self.client.build(fileobj=script, stream=True) - def test_build_container_custom_context(self): script = io.BytesIO('\n'.join([ 'FROM busybox', diff --git a/tests/unit/api_container_test.py b/tests/unit/api_container_test.py index 3cb718a2f0..c33f129eff 100644 --- a/tests/unit/api_container_test.py +++ b/tests/unit/api_container_test.py @@ -219,24 +219,6 @@ def test_create_container_with_entrypoint(self): ''') assert args[1]['headers'] == {'Content-Type': 'application/json'} - def test_create_container_with_cpu_shares(self): - with pytest.deprecated_call(): - self.client.create_container('busybox', 'ls', cpu_shares=5) - - args = fake_request.call_args - assert args[0][1] == url_prefix + 'containers/create' - assert json.loads(args[1]['data']) == json.loads(''' - {"Tty": false, "Image": "busybox", - "Cmd": ["ls"], "AttachStdin": false, - "AttachStderr": true, - "AttachStdout": true, "OpenStdin": false, - "StdinOnce": false, - "NetworkDisabled": false, - "CpuShares": 5} - ''') - assert args[1]['headers'] == {'Content-Type': 'application/json'} - - @requires_api_version('1.18') def test_create_container_with_host_config_cpu_shares(self): self.client.create_container( 'busybox', 'ls', host_config=self.client.create_host_config( @@ -261,25 +243,6 @@ def test_create_container_with_host_config_cpu_shares(self): ''') assert args[1]['headers'] == {'Content-Type': 'application/json'} - def test_create_container_with_cpuset(self): - with pytest.deprecated_call(): - self.client.create_container('busybox', 'ls', cpuset='0,1') - - args = fake_request.call_args - assert args[0][1] == url_prefix + 'containers/create' - assert json.loads(args[1]['data']) == json.loads(''' - {"Tty": false, "Image": "busybox", - "Cmd": ["ls"], "AttachStdin": false, - "AttachStderr": true, - "AttachStdout": true, "OpenStdin": false, - "StdinOnce": false, - "NetworkDisabled": false, - "Cpuset": "0,1", - "CpusetCpus": "0,1"} - ''') - assert args[1]['headers'] == {'Content-Type': 'application/json'} - - @requires_api_version('1.18') def test_create_container_with_host_config_cpuset(self): self.client.create_container( 'busybox', 'ls', host_config=self.client.create_host_config( @@ -304,7 +267,6 @@ def test_create_container_with_host_config_cpuset(self): ''') assert args[1]['headers'] == {'Content-Type': 'application/json'} - @requires_api_version('1.19') def test_create_container_with_host_config_cpuset_mems(self): self.client.create_container( 'busybox', 'ls', host_config=self.client.create_host_config( @@ -374,28 +336,6 @@ def test_create_container_with_stdin_open(self): ''') assert args[1]['headers'] == {'Content-Type': 'application/json'} - def test_create_container_with_volumes_from(self): - vol_names = ['foo', 'bar'] - try: - self.client.create_container('busybox', 'true', - volumes_from=vol_names) - except docker.errors.DockerException: - assert docker.utils.compare_version( - '1.10', self.client._version - ) >= 0 - return - - args = fake_request.call_args - assert args[0][1] == url_prefix + 'containers/create' - assert json.loads(args[1]['data'])['VolumesFrom'] == ','.join( - vol_names - ) - assert args[1]['headers'] == {'Content-Type': 'application/json'} - - def test_create_container_empty_volumes_from(self): - with pytest.raises(docker.errors.InvalidVersion): - self.client.create_container('busybox', 'true', volumes_from=[]) - def test_create_named_container(self): self.client.create_container('busybox', 'true', name='marisa-kirisame') diff --git a/tests/unit/api_image_test.py b/tests/unit/api_image_test.py index 785f887240..1e2315dbb1 100644 --- a/tests/unit/api_image_test.py +++ b/tests/unit/api_image_test.py @@ -197,26 +197,6 @@ def test_inspect_image_undefined_id(self): assert excinfo.value.args[0] == 'Resource ID was not provided' - def test_insert_image(self): - try: - self.client.insert(fake_api.FAKE_IMAGE_NAME, - fake_api.FAKE_URL, fake_api.FAKE_PATH) - except docker.errors.DeprecatedMethod: - assert docker.utils.compare_version( - '1.12', self.client._version - ) >= 0 - return - - fake_request.assert_called_with( - 'POST', - url_prefix + 'images/test_image/insert', - params={ - 'url': fake_api.FAKE_URL, - 'path': fake_api.FAKE_PATH - }, - timeout=DEFAULT_TIMEOUT_SECONDS - ) - def test_push_image(self): with mock.patch('docker.auth.resolve_authconfig', fake_resolve_authconfig): diff --git a/tests/unit/api_network_test.py b/tests/unit/api_network_test.py index fbbc97b0d4..c78554da67 100644 --- a/tests/unit/api_network_test.py +++ b/tests/unit/api_network_test.py @@ -3,7 +3,6 @@ import six from .api_test import BaseAPIClientTest, url_prefix, response -from ..helpers import requires_api_version from docker.types import IPAMConfig, IPAMPool try: @@ -13,7 +12,6 @@ class NetworkTest(BaseAPIClientTest): - @requires_api_version('1.21') def test_list_networks(self): networks = [ { @@ -49,7 +47,6 @@ def test_list_networks(self): filters = json.loads(get.call_args[1]['params']['filters']) assert filters == {'id': ['123']} - @requires_api_version('1.21') def test_create_network(self): network_data = { "id": 'abc12345', @@ -98,7 +95,6 @@ def test_create_network(self): } } - @requires_api_version('1.21') def test_remove_network(self): network_id = 'abc12345' delete = mock.Mock(return_value=response(status_code=200)) @@ -109,7 +105,6 @@ def test_remove_network(self): args = delete.call_args assert args[0][0] == url_prefix + 'networks/{0}'.format(network_id) - @requires_api_version('1.21') def test_inspect_network(self): network_id = 'abc12345' network_name = 'foo' @@ -130,7 +125,6 @@ def test_inspect_network(self): args = get.call_args assert args[0][0] == url_prefix + 'networks/{0}'.format(network_id) - @requires_api_version('1.21') def test_connect_container_to_network(self): network_id = 'abc12345' container_id = 'def45678' @@ -157,7 +151,6 @@ def test_connect_container_to_network(self): }, } - @requires_api_version('1.21') def test_disconnect_container_from_network(self): network_id = 'abc12345' container_id = 'def45678' diff --git a/tests/unit/api_test.py b/tests/unit/api_test.py index b9e0d5243f..c53a4be1f9 100644 --- a/tests/unit/api_test.py +++ b/tests/unit/api_test.py @@ -437,7 +437,6 @@ def test_early_stream_response(self): try: stream = client.build( path=self.build_context, - stream=True ) break except requests.ConnectionError as e: diff --git a/tests/unit/api_volume_test.py b/tests/unit/api_volume_test.py index f5e5001123..7850c224f2 100644 --- a/tests/unit/api_volume_test.py +++ b/tests/unit/api_volume_test.py @@ -7,7 +7,6 @@ class VolumeTest(BaseAPIClientTest): - @requires_api_version('1.21') def test_list_volumes(self): volumes = self.client.volumes() assert 'Volumes' in volumes @@ -17,7 +16,6 @@ def test_list_volumes(self): assert args[0][0] == 'GET' assert args[0][1] == url_prefix + 'volumes' - @requires_api_version('1.21') def test_list_volumes_and_filters(self): volumes = self.client.volumes(filters={'dangling': True}) assert 'Volumes' in volumes @@ -29,7 +27,6 @@ def test_list_volumes_and_filters(self): assert args[1] == {'params': {'filters': '{"dangling": ["true"]}'}, 'timeout': 60} - @requires_api_version('1.21') def test_create_volume(self): name = 'perfectcherryblossom' result = self.client.create_volume(name) @@ -59,7 +56,6 @@ def test_create_volume_with_invalid_labels(self): with pytest.raises(TypeError): self.client.create_volume(name, labels=1) - @requires_api_version('1.21') def test_create_volume_with_driver(self): name = 'perfectcherryblossom' driver_name = 'sshfs' @@ -72,7 +68,6 @@ def test_create_volume_with_driver(self): assert 'Driver' in data assert data['Driver'] == driver_name - @requires_api_version('1.21') def test_create_volume_invalid_opts_type(self): with pytest.raises(TypeError): self.client.create_volume( @@ -99,7 +94,6 @@ def test_create_volume_with_no_specified_name(self): assert 'Scope' in result assert result['Scope'] == 'local' - @requires_api_version('1.21') def test_inspect_volume(self): name = 'perfectcherryblossom' result = self.client.inspect_volume(name) @@ -112,7 +106,6 @@ def test_inspect_volume(self): assert args[0][0] == 'GET' assert args[0][1] == '{0}volumes/{1}'.format(url_prefix, name) - @requires_api_version('1.21') def test_remove_volume(self): name = 'perfectcherryblossom' self.client.remove_volume(name) diff --git a/tests/unit/dockertypes_test.py b/tests/unit/dockertypes_test.py index 71dae7eac5..2be05784bb 100644 --- a/tests/unit/dockertypes_test.py +++ b/tests/unit/dockertypes_test.py @@ -1,14 +1,13 @@ # -*- coding: utf-8 -*- import unittest -import warnings import pytest from docker.constants import DEFAULT_DOCKER_API_VERSION from docker.errors import InvalidArgument, InvalidVersion from docker.types import ( - ContainerConfig, ContainerSpec, EndpointConfig, HostConfig, IPAMConfig, + ContainerSpec, EndpointConfig, HostConfig, IPAMConfig, IPAMPool, LogConfig, Mount, ServiceMode, Ulimit, ) from docker.types.services import convert_service_ports @@ -24,33 +23,29 @@ def create_host_config(*args, **kwargs): class HostConfigTest(unittest.TestCase): - def test_create_host_config_no_options(self): - config = create_host_config(version='1.19') - assert not ('NetworkMode' in config) - def test_create_host_config_no_options_newer_api_version(self): - config = create_host_config(version='1.20') + config = create_host_config(version='1.21') assert config['NetworkMode'] == 'default' def test_create_host_config_invalid_cpu_cfs_types(self): with pytest.raises(TypeError): - create_host_config(version='1.20', cpu_quota='0') + create_host_config(version='1.21', cpu_quota='0') with pytest.raises(TypeError): - create_host_config(version='1.20', cpu_period='0') + create_host_config(version='1.21', cpu_period='0') with pytest.raises(TypeError): - create_host_config(version='1.20', cpu_quota=23.11) + create_host_config(version='1.21', cpu_quota=23.11) with pytest.raises(TypeError): - create_host_config(version='1.20', cpu_period=1999.0) + create_host_config(version='1.21', cpu_period=1999.0) def test_create_host_config_with_cpu_quota(self): - config = create_host_config(version='1.20', cpu_quota=1999) + config = create_host_config(version='1.21', cpu_quota=1999) assert config.get('CpuQuota') == 1999 def test_create_host_config_with_cpu_period(self): - config = create_host_config(version='1.20', cpu_period=1999) + config = create_host_config(version='1.21', cpu_period=1999) assert config.get('CpuPeriod') == 1999 def test_create_host_config_with_blkio_constraints(self): @@ -79,10 +74,8 @@ def test_create_host_config_with_shm_size_in_mb(self): assert config.get('ShmSize') == 67108864 def test_create_host_config_with_oom_kill_disable(self): - config = create_host_config(version='1.20', oom_kill_disable=True) + config = create_host_config(version='1.21', oom_kill_disable=True) assert config.get('OomKillDisable') is True - with pytest.raises(InvalidVersion): - create_host_config(version='1.18.3', oom_kill_disable=True) def test_create_host_config_with_userns_mode(self): config = create_host_config(version='1.23', userns_mode='host') @@ -109,20 +102,13 @@ def test_create_host_config_with_dns_opt(self): assert 'use-vc' in dns_opts assert 'no-tld-query' in dns_opts - with pytest.raises(InvalidVersion): - create_host_config(version='1.20', dns_opt=tested_opts) - def test_create_host_config_with_mem_reservation(self): config = create_host_config(version='1.21', mem_reservation=67108864) assert config.get('MemoryReservation') == 67108864 - with pytest.raises(InvalidVersion): - create_host_config(version='1.20', mem_reservation=67108864) def test_create_host_config_with_kernel_memory(self): config = create_host_config(version='1.21', kernel_memory=67108864) assert config.get('KernelMemory') == 67108864 - with pytest.raises(InvalidVersion): - create_host_config(version='1.20', kernel_memory=67108864) def test_create_host_config_with_pids_limit(self): config = create_host_config(version='1.23', pids_limit=1024) @@ -158,9 +144,6 @@ def test_create_host_config_invalid_mem_swappiness(self): create_host_config(version='1.24', mem_swappiness='40') def test_create_host_config_with_volume_driver(self): - with pytest.raises(InvalidVersion): - create_host_config(version='1.20', volume_driver='local') - config = create_host_config(version='1.21', volume_driver='local') assert config.get('VolumeDriver') == 'local' @@ -215,19 +198,6 @@ def test_create_host_config_with_cpu_rt_runtime(self): create_host_config(version='1.24', cpu_rt_runtime=1000) -class ContainerConfigTest(unittest.TestCase): - def test_create_container_config_volume_driver_warning(self): - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter('always') - ContainerConfig( - version='1.21', image='scratch', command=None, - volume_driver='local' - ) - - assert len(w) == 1 - assert 'The volume_driver option has been moved' in str(w[0].message) - - class ContainerSpecTest(unittest.TestCase): def test_parse_mounts(self): spec = ContainerSpec( diff --git a/tests/unit/fake_api.py b/tests/unit/fake_api.py index 37154a3bd4..63d73317ca 100644 --- a/tests/unit/fake_api.py +++ b/tests/unit/fake_api.py @@ -21,21 +21,36 @@ # for clarity and readability -def get_fake_raw_version(): +def get_fake_version(): status_code = 200 response = { - "ApiVersion": "1.18", - "GitCommit": "fake-commit", - "GoVersion": "go1.3.3", - "Version": "1.5.0" + 'ApiVersion': '1.35', + 'Arch': 'amd64', + 'BuildTime': '2018-01-10T20:09:37.000000000+00:00', + 'Components': [{ + 'Details': { + 'ApiVersion': '1.35', + 'Arch': 'amd64', + 'BuildTime': '2018-01-10T20:09:37.000000000+00:00', + 'Experimental': 'false', + 'GitCommit': '03596f5', + 'GoVersion': 'go1.9.2', + 'KernelVersion': '4.4.0-112-generic', + 'MinAPIVersion': '1.12', + 'Os': 'linux' + }, + 'Name': 'Engine', + 'Version': '18.01.0-ce' + }], + 'GitCommit': '03596f5', + 'GoVersion': 'go1.9.2', + 'KernelVersion': '4.4.0-112-generic', + 'MinAPIVersion': '1.12', + 'Os': 'linux', + 'Platform': {'Name': ''}, + 'Version': '18.01.0-ce' } - return status_code, response - -def get_fake_version(): - status_code = 200 - response = {'GoVersion': '1', 'Version': '1.1.1', - 'GitCommit': 'deadbeef+CHANGES'} return status_code, response @@ -503,7 +518,7 @@ def post_fake_network_disconnect(): fake_responses = { '{0}/version'.format(prefix): - get_fake_raw_version, + get_fake_version, '{1}/{0}/version'.format(CURRENT_VERSION, prefix): get_fake_version, '{1}/{0}/info'.format(CURRENT_VERSION, prefix): From 209ae2423d3fc1f41ed8dc617963d560d9d9e4e1 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 31 Jan 2018 15:12:15 -0800 Subject: [PATCH 0577/1301] Correctly parse volumes with Windows paths Signed-off-by: Joffrey F --- docker/models/containers.py | 24 +++++++++++++++--------- tests/unit/models_containers_test.py | 7 +++++-- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/docker/models/containers.py b/docker/models/containers.py index 79fd71df1e..e05aa626dd 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -1,4 +1,5 @@ import copy +import ntpath from collections import namedtuple from ..api import APIClient @@ -995,20 +996,25 @@ def _create_container_args(kwargs): # sort to make consistent for tests create_kwargs['ports'] = [tuple(p.split('/', 1)) for p in sorted(port_bindings.keys())] - binds = create_kwargs['host_config'].get('Binds') - if binds: - create_kwargs['volumes'] = [_host_volume_from_bind(v) for v in binds] + if volumes: + if isinstance(volumes, dict): + create_kwargs['volumes'] = [ + v.get('bind') for v in volumes.values() + ] + else: + create_kwargs['volumes'] = [ + _host_volume_from_bind(v) for v in volumes + ] return create_kwargs def _host_volume_from_bind(bind): - bits = bind.split(':') - if len(bits) == 1: - return bits[0] - elif len(bits) == 2 and bits[1] in ('ro', 'rw'): - return bits[0] + drive, rest = ntpath.splitdrive(bind) + bits = rest.split(':', 1) + if len(bits) == 1 or bits[1] in ('ro', 'rw'): + return drive + bits[0] else: - return bits[1] + return bits[1].rstrip(':ro').rstrip(':rw') ExecResult = namedtuple('ExecResult', 'exit_code,output') diff --git a/tests/unit/models_containers_test.py b/tests/unit/models_containers_test.py index 1fdd7a5c30..cdc93184df 100644 --- a/tests/unit/models_containers_test.py +++ b/tests/unit/models_containers_test.py @@ -102,6 +102,7 @@ def test_create_container_args(self): 'volumename:/mnt/vol3', '/volumewithnohostpath', '/anothervolumewithnohostpath:ro', + 'C:\\windows\\path:D:\\hello\\world:rw' ], volumes_from=['container'], working_dir='/code' @@ -120,7 +121,8 @@ def test_create_container_args(self): '/var/www:/mnt/vol1:ro', 'volumename:/mnt/vol3', '/volumewithnohostpath', - '/anothervolumewithnohostpath:ro' + '/anothervolumewithnohostpath:ro', + 'C:\\windows\\path:D:\\hello\\world:rw' ], 'BlkioDeviceReadBps': [{'Path': 'foo', 'Rate': 3}], 'BlkioDeviceReadIOps': [{'Path': 'foo', 'Rate': 3}], @@ -191,7 +193,8 @@ def test_create_container_args(self): '/mnt/vol1', '/mnt/vol3', '/volumewithnohostpath', - '/anothervolumewithnohostpath' + '/anothervolumewithnohostpath', + 'D:\\hello\\world' ], working_dir='/code' ) From bf2ea4dcb01ffa3329fef91570fe3bd40fad4d7d Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 31 Jan 2018 15:26:09 -0800 Subject: [PATCH 0578/1301] Rename `name` parameter in `pull` method to `repository` for consistency with APIClient naming Signed-off-by: Joffrey F --- docker/models/images.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/docker/models/images.py b/docker/models/images.py index 97c5503074..0f3c71ab4f 100644 --- a/docker/models/images.py +++ b/docker/models/images.py @@ -266,7 +266,7 @@ def load(self, data): return [self.get(i) for i in images] - def pull(self, name, tag=None, **kwargs): + def pull(self, repository, tag=None, **kwargs): """ Pull an image of the given name and return it. Similar to the ``docker pull`` command. @@ -280,7 +280,6 @@ def pull(self, name, tag=None, **kwargs): Args: name (str): The repository to pull tag (str): The tag to pull - insecure_registry (bool): Use an insecure registry auth_config (dict): Override the credentials that :py:meth:`~docker.client.DockerClient.login` has set for this request. ``auth_config`` should contain the ``username`` @@ -305,12 +304,12 @@ def pull(self, name, tag=None, **kwargs): >>> images = client.images.pull('busybox') """ if not tag: - name, tag = parse_repository_tag(name) + repository, tag = parse_repository_tag(repository) - self.client.api.pull(name, tag=tag, **kwargs) + self.client.api.pull(repository, tag=tag, **kwargs) if tag: - return self.get('{0}:{1}'.format(name, tag)) - return self.list(name) + return self.get('{0}:{1}'.format(repository, tag)) + return self.list(repository) def push(self, repository, tag=None, **kwargs): return self.client.api.push(repository, tag=tag, **kwargs) From 7fabcdaa4cbcf1c7905c2c2e7669a5a4ac594735 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 31 Jan 2018 16:39:39 -0800 Subject: [PATCH 0579/1301] Update wait to always return a dict Signed-off-by: Joffrey F --- docker/api/container.py | 10 ++--- docker/models/containers.py | 6 +-- docker/version.py | 2 +- docs/change-log.md | 41 +++++++++++++++++++++ tests/integration/api_container_test.py | 36 +++++++++--------- tests/integration/base.py | 2 +- tests/integration/models_containers_test.py | 4 +- tests/unit/fake_api_client.py | 2 +- tests/unit/models_containers_test.py | 6 +-- 9 files changed, 74 insertions(+), 35 deletions(-) diff --git a/docker/api/container.py b/docker/api/container.py index 152a08b325..962d8cb91c 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -1207,8 +1207,8 @@ def wait(self, container, timeout=None, condition=None): or ``removed`` Returns: - (int or dict): The exit code of the container. Returns the full API - response if no ``StatusCode`` field is included. + (dict): The API's response as a Python dictionary, including + the container's exit code under the ``StatusCode`` attribute. Raises: :py:class:`requests.exceptions.ReadTimeout` @@ -1226,8 +1226,4 @@ def wait(self, container, timeout=None, condition=None): params['condition'] = condition res = self._post(url, timeout=timeout, params=params) - self._raise_for_status(res) - json_ = res.json() - if 'StatusCode' in json_: - return json_['StatusCode'] - return json_ + return self._result(res, True) diff --git a/docker/models/containers.py b/docker/models/containers.py index e05aa626dd..963fca4ef6 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -448,8 +448,8 @@ def wait(self, **kwargs): or ``removed`` Returns: - (int): The exit code of the container. Returns ``-1`` if the API - responds without a ``StatusCode`` attribute. + (dict): The API's response as a Python dictionary, including + the container's exit code under the ``StatusCode`` attribute. Raises: :py:class:`requests.exceptions.ReadTimeout` @@ -758,7 +758,7 @@ def run(self, image, command=None, stdout=True, stderr=False, stdout=stdout, stderr=stderr, stream=True, follow=True ) - exit_status = container.wait() + exit_status = container.wait()['StatusCode'] if exit_status != 0: out = None if not kwargs.get('auto_remove'): diff --git a/docker/version.py b/docker/version.py index 5b76748510..f141747ae0 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "2.8.0-dev" +version = "3.0.0" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) diff --git a/docs/change-log.md b/docs/change-log.md index b8298a7981..7531ad49b1 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -1,6 +1,47 @@ Change log ========== +3.0.0 +----- + +[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/39?closed=1) + +### Breaking changes + +* Support for API version < 1.21 has been removed. +* The following methods have been removed: + * `APIClient.copy` has been removed. Users should use `APIClient.get_archive` + instead. + * `APIClient.insert` has been removed. Users may use `APIClient.put_archive` + combined with `APIClient.commit` to replicate the method's behavior. + * `utils.ping_registry` and `utils.ping` have been removed. +* The following parameters have been removed: + * `stream` in `APIClient.build` + * `cpu_shares`, `cpuset`, `dns`, `mem_limit`, `memswap_limit`, + `volume_driver`, `volumes_from` in `APIClient.create_container`. These are + all replaced by their equivalent in `create_host_config` + * `insecure_registry` in `APIClient.login`, `APIClient.pull`, + `APIClient.push`, `DockerClient.images.push` and `DockerClient.images.pull` + * `viz` in `APIClient.images` +* The following parameters have been renamed: + * `endpoint_config` in `APIClient.create_service` and + `APIClient.update_service` is now `endpoint_spec` + * `name` in `DockerClient.images.pull` is now `repository` +* The return value for the following methods has changed: + * `APIClient.wait` and `Container.wait` now return a ``dict`` representing + the API's response instead of returning the status code directly. + * `DockerClient.images.load` now returns a list of `Image` objects that have + for the images that were loaded, instead of a log stream. + * `Container.exec_run` now returns a tuple of (exit_code, output) instead of + just the output. + * `DockerClient.images.build` now returns a tuple of (image, build_logs) + instead of just the image object. + * `APIClient.export`, `APIClient.get_archive` and `APIClient.get_image` now + return generators streaming the raw binary data from the server's response. + * When no tag is provided, `DockerClient.images.pull` now returns a list of + `Image`s associated to the pulled repository instead of just the `latest` + image. + 2.7.0 ----- diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index 09253524a5..01780a771b 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -102,7 +102,7 @@ def test_create_with_links(self): container3_id = res2['Id'] self.tmp_containers.append(container3_id) self.client.start(container3_id) - assert self.client.wait(container3_id) == 0 + assert self.client.wait(container3_id)['StatusCode'] == 0 logs = self.client.logs(container3_id) if six.PY3: @@ -169,7 +169,7 @@ def create_container_readonly_fs(self): assert 'Id' in ctnr self.tmp_containers.append(ctnr['Id']) self.client.start(ctnr) - res = self.client.wait(ctnr) + res = self.client.wait(ctnr)['StatusCode'] assert res != 0 def create_container_with_name(self): @@ -771,7 +771,7 @@ def test_run_shlex_commands(self): id = container['Id'] self.client.start(id) self.tmp_containers.append(id) - exitcode = self.client.wait(id) + exitcode = self.client.wait(id)['StatusCode'] assert exitcode == 0, cmd @@ -781,7 +781,7 @@ def test_wait(self): id = res['Id'] self.tmp_containers.append(id) self.client.start(id) - exitcode = self.client.wait(id) + exitcode = self.client.wait(id)['StatusCode'] assert exitcode == 0 inspect = self.client.inspect_container(id) assert 'Running' in inspect['State'] @@ -794,7 +794,7 @@ def test_wait_with_dict_instead_of_id(self): id = res['Id'] self.tmp_containers.append(id) self.client.start(res) - exitcode = self.client.wait(res) + exitcode = self.client.wait(res)['StatusCode'] assert exitcode == 0 inspect = self.client.inspect_container(res) assert 'Running' in inspect['State'] @@ -815,7 +815,9 @@ def test_wait_with_condition(self): ) self.tmp_containers.append(ctnr) self.client.start(ctnr) - assert self.client.wait(ctnr, condition='removed', timeout=5) == 0 + assert self.client.wait( + ctnr, condition='removed', timeout=5 + )['StatusCode'] == 0 class LogsTest(BaseAPIIntegrationTest): @@ -827,7 +829,7 @@ def test_logs(self): id = container['Id'] self.tmp_containers.append(id) self.client.start(id) - exitcode = self.client.wait(id) + exitcode = self.client.wait(id)['StatusCode'] assert exitcode == 0 logs = self.client.logs(id) assert logs == (snippet + '\n').encode(encoding='ascii') @@ -841,7 +843,7 @@ def test_logs_tail_option(self): id = container['Id'] self.tmp_containers.append(id) self.client.start(id) - exitcode = self.client.wait(id) + exitcode = self.client.wait(id)['StatusCode'] assert exitcode == 0 logs = self.client.logs(id, tail=1) assert logs == 'Line2\n'.encode(encoding='ascii') @@ -858,7 +860,7 @@ def test_logs_streaming_and_follow(self): for chunk in self.client.logs(id, stream=True, follow=True): logs += chunk - exitcode = self.client.wait(id) + exitcode = self.client.wait(id)['StatusCode'] assert exitcode == 0 assert logs == (snippet + '\n').encode(encoding='ascii') @@ -871,7 +873,7 @@ def test_logs_with_dict_instead_of_id(self): id = container['Id'] self.tmp_containers.append(id) self.client.start(id) - exitcode = self.client.wait(id) + exitcode = self.client.wait(id)['StatusCode'] assert exitcode == 0 logs = self.client.logs(container) assert logs == (snippet + '\n').encode(encoding='ascii') @@ -884,7 +886,7 @@ def test_logs_with_tail_0(self): id = container['Id'] self.tmp_containers.append(id) self.client.start(id) - exitcode = self.client.wait(id) + exitcode = self.client.wait(id)['StatusCode'] assert exitcode == 0 logs = self.client.logs(id, tail=0) assert logs == ''.encode(encoding='ascii') @@ -898,7 +900,7 @@ def test_logs_with_until(self): self.tmp_containers.append(container) self.client.start(container) - exitcode = self.client.wait(container) + exitcode = self.client.wait(container)['StatusCode'] assert exitcode == 0 logs_until_1 = self.client.logs(container, until=1) assert logs_until_1 == b'' @@ -912,7 +914,7 @@ def test_diff(self): id = container['Id'] self.client.start(id) self.tmp_containers.append(id) - exitcode = self.client.wait(id) + exitcode = self.client.wait(id)['StatusCode'] assert exitcode == 0 diff = self.client.diff(id) test_diff = [x for x in diff if x.get('Path', None) == '/test'] @@ -925,7 +927,7 @@ def test_diff_with_dict_instead_of_id(self): id = container['Id'] self.client.start(id) self.tmp_containers.append(id) - exitcode = self.client.wait(id) + exitcode = self.client.wait(id)['StatusCode'] assert exitcode == 0 diff = self.client.diff(container) test_diff = [x for x in diff if x.get('Path', None) == '/test'] @@ -997,7 +999,7 @@ def test_kill_with_signal(self): self.client.kill( id, signal=signal.SIGKILL if not IS_WINDOWS_PLATFORM else 9 ) - exitcode = self.client.wait(id) + exitcode = self.client.wait(id)['StatusCode'] assert exitcode != 0 container_info = self.client.inspect_container(id) assert 'State' in container_info @@ -1012,7 +1014,7 @@ def test_kill_with_signal_name(self): self.client.start(id) self.tmp_containers.append(id) self.client.kill(id, signal='SIGKILL') - exitcode = self.client.wait(id) + exitcode = self.client.wait(id)['StatusCode'] assert exitcode != 0 container_info = self.client.inspect_container(id) assert 'State' in container_info @@ -1027,7 +1029,7 @@ def test_kill_with_signal_integer(self): self.client.start(id) self.tmp_containers.append(id) self.client.kill(id, signal=9) - exitcode = self.client.wait(id) + exitcode = self.client.wait(id)['StatusCode'] assert exitcode != 0 container_info = self.client.inspect_container(id) assert 'State' in container_info diff --git a/tests/integration/base.py b/tests/integration/base.py index 04d9afdb0a..c22126d5f4 100644 --- a/tests/integration/base.py +++ b/tests/integration/base.py @@ -96,7 +96,7 @@ def run_container(self, *args, **kwargs): container = self.client.create_container(*args, **kwargs) self.tmp_containers.append(container) self.client.start(container) - exitcode = self.client.wait(container) + exitcode = self.client.wait(container)['StatusCode'] if exitcode != 0: output = self.client.logs(container) diff --git a/tests/integration/models_containers_test.py b/tests/integration/models_containers_test.py index 2d01f222aa..a4d9f9c0da 100644 --- a/tests/integration/models_containers_test.py +++ b/tests/integration/models_containers_test.py @@ -309,8 +309,8 @@ def test_wait(self): container = client.containers.run("alpine", "sh -c 'exit 0'", detach=True) self.tmp_containers.append(container.id) - assert container.wait() == 0 + assert container.wait()['StatusCode'] == 0 container = client.containers.run("alpine", "sh -c 'exit 1'", detach=True) self.tmp_containers.append(container.id) - assert container.wait() == 1 + assert container.wait()['StatusCode'] == 1 diff --git a/tests/unit/fake_api_client.py b/tests/unit/fake_api_client.py index f908355101..15b60eaadc 100644 --- a/tests/unit/fake_api_client.py +++ b/tests/unit/fake_api_client.py @@ -46,7 +46,7 @@ def make_fake_api_client(): 'logs.return_value': [b'hello world\n'], 'networks.return_value': fake_api.get_fake_network_list()[1], 'start.return_value': None, - 'wait.return_value': 0, + 'wait.return_value': {'StatusCode': 0}, }) mock_client._version = docker.constants.DEFAULT_DOCKER_API_VERSION return mock_client diff --git a/tests/unit/models_containers_test.py b/tests/unit/models_containers_test.py index cdc93184df..f79f5d5b33 100644 --- a/tests/unit/models_containers_test.py +++ b/tests/unit/models_containers_test.py @@ -234,7 +234,7 @@ def test_run_pull(self): def test_run_with_error(self): client = make_fake_client() client.api.logs.return_value = "some error" - client.api.wait.return_value = 1 + client.api.wait.return_value = {'StatusCode': 1} with pytest.raises(docker.errors.ContainerError) as cm: client.containers.run('alpine', 'echo hello world') @@ -260,7 +260,7 @@ def test_run_remove(self): client.api.remove_container.assert_not_called() client = make_fake_client() - client.api.wait.return_value = 1 + client.api.wait.return_value = {'StatusCode': 1} with pytest.raises(docker.errors.ContainerError): client.containers.run("alpine") client.api.remove_container.assert_not_called() @@ -270,7 +270,7 @@ def test_run_remove(self): client.api.remove_container.assert_called_with(FAKE_CONTAINER_ID) client = make_fake_client() - client.api.wait.return_value = 1 + client.api.wait.return_value = {'StatusCode': 1} with pytest.raises(docker.errors.ContainerError): client.containers.run("alpine", remove=True) client.api.remove_container.assert_called_with(FAKE_CONTAINER_ID) From 3aba34fd0896b4c68186abdb150e5ac42d74d0d1 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 31 Jan 2018 17:26:36 -0800 Subject: [PATCH 0580/1301] Bump 3.0.0 Signed-off-by: Joffrey F --- docker/models/containers.py | 2 ++ docs/change-log.md | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/docker/models/containers.py b/docker/models/containers.py index 963fca4ef6..a67439387f 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -244,6 +244,8 @@ def logs(self, **kwargs): since (datetime or int): Show logs since a given datetime or integer epoch (in seconds) follow (bool): Follow log output + until (datetime or int): Show logs that occurred before the given + datetime or integer epoch (in seconds) Returns: (generator or str): Logs from the container. diff --git a/docs/change-log.md b/docs/change-log.md index 7531ad49b1..08d4e8f7bb 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -42,6 +42,42 @@ Change log `Image`s associated to the pulled repository instead of just the `latest` image. +### Features + +* The Docker Python SDK is now officially supported on Python 3.6 +* Added `scale` method to the `Service` model ; this method is a shorthand + that calls `update_service` with the required number of replicas +* Added support for the `platform` parameter in `APIClient.build`, + `DockerClient.images.build`, `APIClient.pull` and `DockerClient.images.pull` +* Added support for the `until` parameter in `APIClient.logs` and + `Container.logs` +* Added support for the `workdir` argument in `APIClient.exec_create` and + `Container.exec_run` +* Added support for the `condition` argument in `APIClient.wait` and + `Container.wait` +* Users can now specify a publish mode for ports in `EndpointSpec` using + the `{published_port: (target_port, protocol, publish_mode)}` syntax. +* Added support for the `isolation` parameter in `ContainerSpec`, + `DockerClient.services.create` and `Service.update` +* `APIClient.attach_socket`, `APIClient.exec_create` now allow specifying a + `detach_keys` combination. If unspecified, the value from the `config.json` + file will be used +* TLS connections now default to using the TLSv1.2 protocol when available + + +### Bugfixes + +* Fixed a bug where whitespace-only lines in `.dockerignore` would break builds + on Windows +* Fixed a bug where broken symlinks inside a build context would cause the + build to fail +* Fixed a bug where specifying volumes with Windows drives would cause + incorrect parsing in `DockerClient.containers.run` +* Fixed a bug where the `networks` data provided to `create_service` and + `update_service` would be sent incorrectly to the Engine with API < 1.25 +* Pulling all tags from a repository with no `latest` tag using the + `DockerClient` will no longer raise a `NotFound` exception + 2.7.0 ----- From 9a87f80f85557ee3ef808677f982548b33a490c8 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 31 Jan 2018 17:26:51 -0800 Subject: [PATCH 0581/1301] Docs fixes Signed-off-by: Joffrey F --- docker/models/containers.py | 1 + docs/images.rst | 12 +++++++----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/docker/models/containers.py b/docker/models/containers.py index a67439387f..107a0204b1 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -592,6 +592,7 @@ def run(self, image, command=None, stdout=True, stderr=False, - ``container:`` Reuse another container's network stack. - ``host`` Use the host network stack. + Incompatible with ``network``. oom_kill_disable (bool): Whether to disable OOM killer. oom_score_adj (int): An integer value containing the score given diff --git a/docs/images.rst b/docs/images.rst index 3ba06010a6..12b0fd1842 100644 --- a/docs/images.rst +++ b/docs/images.rst @@ -26,14 +26,16 @@ Image objects .. autoclass:: Image() -.. py:attribute:: attrs -.. autoattribute:: id -.. autoattribute:: labels -.. autoattribute:: short_id -.. autoattribute:: tags + .. py:attribute:: attrs The raw representation of this object from the server. + .. autoattribute:: id + .. autoattribute:: labels + .. autoattribute:: short_id + .. autoattribute:: tags + + .. automethod:: history .. automethod:: reload From 05d34ed1fbaa8233a4cf51a0f52b67aef99a9521 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 31 Jan 2018 17:58:23 -0800 Subject: [PATCH 0582/1301] 3.1.0-dev Signed-off-by: Joffrey F --- docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index f141747ae0..fc56840619 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "3.0.0" +version = "3.1.0-dev" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) From 83d185d69533757eac4beef646f1e46737bdd67f Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 1 Feb 2018 16:02:09 -0800 Subject: [PATCH 0583/1301] Add login data to the right subdict in auth_configs Signed-off-by: Joffrey F --- docker/api/daemon.py | 2 +- tests/unit/api_test.py | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/docker/api/daemon.py b/docker/api/daemon.py index 033dbf19e1..0e1c753814 100644 --- a/docker/api/daemon.py +++ b/docker/api/daemon.py @@ -139,7 +139,7 @@ def login(self, username, password=None, email=None, registry=None, if response.status_code == 200: if 'auths' not in self._auth_configs: self._auth_configs['auths'] = {} - self._auth_configs[registry or auth.INDEX_NAME] = req_data + self._auth_configs['auths'][registry or auth.INDEX_NAME] = req_data return self._result(response, json=True) def ping(self): diff --git a/tests/unit/api_test.py b/tests/unit/api_test.py index c53a4be1f9..f65e13ecb3 100644 --- a/tests/unit/api_test.py +++ b/tests/unit/api_test.py @@ -212,6 +212,24 @@ def test_search(self): timeout=DEFAULT_TIMEOUT_SECONDS ) + def test_login(self): + self.client.login('sakuya', 'izayoi') + fake_request.assert_called_with( + 'POST', url_prefix + 'auth', + data=json.dumps({'username': 'sakuya', 'password': 'izayoi'}), + timeout=DEFAULT_TIMEOUT_SECONDS, + headers={'Content-Type': 'application/json'} + ) + + assert self.client._auth_configs['auths'] == { + 'docker.io': { + 'email': None, + 'password': 'izayoi', + 'username': 'sakuya', + 'serveraddress': None, + } + } + def test_events(self): self.client.events() From 04bf470f6e7e06615be453e1adfe7c656bc5f153 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 1 Feb 2018 16:51:36 -0800 Subject: [PATCH 0584/1301] Add workaround for bpo-32713 Signed-off-by: Joffrey F --- docker/utils/utils.py | 4 ++++ tests/unit/utils_test.py | 12 ++++++++++++ 2 files changed, 16 insertions(+) diff --git a/docker/utils/utils.py b/docker/utils/utils.py index e4e2c0dfc9..b145f116ba 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -107,6 +107,10 @@ def create_archive(root, files=None, fileobj=None, gzip=False): # ignore it and proceed. continue + # Workaround https://bugs.python.org/issue32713 + if i.mtime < 0 or i.mtime > 8**11 - 1: + i.mtime = int(i.mtime) + if constants.IS_WINDOWS_PLATFORM: # Windows doesn't keep track of the execute bit, so we make files # and directories executable by default. diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index 1f9daf60a2..1558891815 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -995,6 +995,18 @@ def test_tar_socket_file(self): tar_data = tarfile.open(fileobj=archive) assert sorted(tar_data.getnames()) == ['bar', 'foo'] + def tar_test_negative_mtime_bug(self): + base = tempfile.mkdtemp() + filename = os.path.join(base, 'th.txt') + self.addCleanup(shutil.rmtree, base) + with open(filename, 'w') as f: + f.write('Invisible Full Moon') + os.utime(filename, (12345, -3600.0)) + with tar(base) as archive: + tar_data = tarfile.open(fileobj=archive) + assert tar_data.getnames() == ['th.txt'] + assert tar_data.getmember('th.txt').mtime == -3600 + class ShouldCheckDirectoryTest(unittest.TestCase): exclude_patterns = [ From 58639aecfa50e0bcfbd1415dc8bab2b4448f4d81 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 5 Feb 2018 13:11:19 -0800 Subject: [PATCH 0585/1301] Rewrite access check in create_archive with EAFP Signed-off-by: Joffrey F --- docker/utils/utils.py | 8 +++----- tests/unit/utils_test.py | 8 ++++++-- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/docker/utils/utils.py b/docker/utils/utils.py index e4e2c0dfc9..b86a3f0ae6 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -97,10 +97,6 @@ def create_archive(root, files=None, fileobj=None, gzip=False): for path in files: full_path = os.path.join(root, path) - if os.lstat(full_path).st_mode & os.R_OK == 0: - raise IOError( - 'Can not access file in context: {}'.format(full_path) - ) i = t.gettarinfo(full_path, arcname=path) if i is None: # This happens when we encounter a socket file. We can safely @@ -117,7 +113,9 @@ def create_archive(root, files=None, fileobj=None, gzip=False): with open(full_path, 'rb') as f: t.addfile(i, f) except IOError: - t.addfile(i, None) + raise IOError( + 'Can not read file in context: {}'.format(full_path) + ) else: # Directories, FIFOs, symlinks... don't need to be read. t.addfile(i, None) diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index 1f9daf60a2..3139a97092 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -933,7 +933,10 @@ def test_tar_with_empty_directory(self): tar_data = tarfile.open(fileobj=archive) assert sorted(tar_data.getnames()) == ['bar', 'foo'] - @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='No chmod on Windows') + @pytest.mark.skipif( + IS_WINDOWS_PLATFORM or os.geteuid() == 0, + reason='root user always has access ; no chmod on Windows' + ) def test_tar_with_inaccessible_file(self): base = tempfile.mkdtemp() full_path = os.path.join(base, 'foo') @@ -944,8 +947,9 @@ def test_tar_with_inaccessible_file(self): with pytest.raises(IOError) as ei: tar(base) - assert 'Can not access file in context: {}'.format(full_path) in \ + assert 'Can not read file in context: {}'.format(full_path) in ( ei.exconly() + ) @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='No symlinks on Windows') def test_tar_with_file_symlinks(self): From a60011ca3a7142d28282b70d87708b6c66e85c51 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 1 Feb 2018 16:51:36 -0800 Subject: [PATCH 0586/1301] Add workaround for bpo-32713 Signed-off-by: Joffrey F --- docker/utils/utils.py | 4 ++++ tests/unit/utils_test.py | 12 ++++++++++++ 2 files changed, 16 insertions(+) diff --git a/docker/utils/utils.py b/docker/utils/utils.py index e4e2c0dfc9..b145f116ba 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -107,6 +107,10 @@ def create_archive(root, files=None, fileobj=None, gzip=False): # ignore it and proceed. continue + # Workaround https://bugs.python.org/issue32713 + if i.mtime < 0 or i.mtime > 8**11 - 1: + i.mtime = int(i.mtime) + if constants.IS_WINDOWS_PLATFORM: # Windows doesn't keep track of the execute bit, so we make files # and directories executable by default. diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index 1f9daf60a2..1558891815 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -995,6 +995,18 @@ def test_tar_socket_file(self): tar_data = tarfile.open(fileobj=archive) assert sorted(tar_data.getnames()) == ['bar', 'foo'] + def tar_test_negative_mtime_bug(self): + base = tempfile.mkdtemp() + filename = os.path.join(base, 'th.txt') + self.addCleanup(shutil.rmtree, base) + with open(filename, 'w') as f: + f.write('Invisible Full Moon') + os.utime(filename, (12345, -3600.0)) + with tar(base) as archive: + tar_data = tarfile.open(fileobj=archive) + assert tar_data.getnames() == ['th.txt'] + assert tar_data.getmember('th.txt').mtime == -3600 + class ShouldCheckDirectoryTest(unittest.TestCase): exclude_patterns = [ From 539b321bd1dfc5cbbba4c1573cd46adb0f47a3f1 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 1 Feb 2018 16:02:09 -0800 Subject: [PATCH 0587/1301] Add login data to the right subdict in auth_configs Signed-off-by: Joffrey F --- docker/api/daemon.py | 2 +- tests/unit/api_test.py | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/docker/api/daemon.py b/docker/api/daemon.py index 033dbf19e1..0e1c753814 100644 --- a/docker/api/daemon.py +++ b/docker/api/daemon.py @@ -139,7 +139,7 @@ def login(self, username, password=None, email=None, registry=None, if response.status_code == 200: if 'auths' not in self._auth_configs: self._auth_configs['auths'] = {} - self._auth_configs[registry or auth.INDEX_NAME] = req_data + self._auth_configs['auths'][registry or auth.INDEX_NAME] = req_data return self._result(response, json=True) def ping(self): diff --git a/tests/unit/api_test.py b/tests/unit/api_test.py index c53a4be1f9..f65e13ecb3 100644 --- a/tests/unit/api_test.py +++ b/tests/unit/api_test.py @@ -212,6 +212,24 @@ def test_search(self): timeout=DEFAULT_TIMEOUT_SECONDS ) + def test_login(self): + self.client.login('sakuya', 'izayoi') + fake_request.assert_called_with( + 'POST', url_prefix + 'auth', + data=json.dumps({'username': 'sakuya', 'password': 'izayoi'}), + timeout=DEFAULT_TIMEOUT_SECONDS, + headers={'Content-Type': 'application/json'} + ) + + assert self.client._auth_configs['auths'] == { + 'docker.io': { + 'email': None, + 'password': 'izayoi', + 'username': 'sakuya', + 'serveraddress': None, + } + } + def test_events(self): self.client.events() From 6de7bab22fc339c7e38cc89732a3a05ab3a94eda Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 5 Feb 2018 13:11:19 -0800 Subject: [PATCH 0588/1301] Rewrite access check in create_archive with EAFP Signed-off-by: Joffrey F --- docker/utils/utils.py | 8 +++----- tests/unit/utils_test.py | 8 ++++++-- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/docker/utils/utils.py b/docker/utils/utils.py index b145f116ba..3cd2be8169 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -97,10 +97,6 @@ def create_archive(root, files=None, fileobj=None, gzip=False): for path in files: full_path = os.path.join(root, path) - if os.lstat(full_path).st_mode & os.R_OK == 0: - raise IOError( - 'Can not access file in context: {}'.format(full_path) - ) i = t.gettarinfo(full_path, arcname=path) if i is None: # This happens when we encounter a socket file. We can safely @@ -121,7 +117,9 @@ def create_archive(root, files=None, fileobj=None, gzip=False): with open(full_path, 'rb') as f: t.addfile(i, f) except IOError: - t.addfile(i, None) + raise IOError( + 'Can not read file in context: {}'.format(full_path) + ) else: # Directories, FIFOs, symlinks... don't need to be read. t.addfile(i, None) diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index 1558891815..eedcf71a0d 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -933,7 +933,10 @@ def test_tar_with_empty_directory(self): tar_data = tarfile.open(fileobj=archive) assert sorted(tar_data.getnames()) == ['bar', 'foo'] - @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='No chmod on Windows') + @pytest.mark.skipif( + IS_WINDOWS_PLATFORM or os.geteuid() == 0, + reason='root user always has access ; no chmod on Windows' + ) def test_tar_with_inaccessible_file(self): base = tempfile.mkdtemp() full_path = os.path.join(base, 'foo') @@ -944,8 +947,9 @@ def test_tar_with_inaccessible_file(self): with pytest.raises(IOError) as ei: tar(base) - assert 'Can not access file in context: {}'.format(full_path) in \ + assert 'Can not read file in context: {}'.format(full_path) in ( ei.exconly() + ) @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='No symlinks on Windows') def test_tar_with_file_symlinks(self): From 8649f48a4c2bc89e271b09d17a3e61866a1ead1c Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 5 Feb 2018 13:44:57 -0800 Subject: [PATCH 0589/1301] Bump 3.0.1 Signed-off-by: Joffrey F --- docker/version.py | 2 +- docs/change-log.md | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index f141747ae0..635e84cb5b 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "3.0.0" +version = "3.0.1" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) diff --git a/docs/change-log.md b/docs/change-log.md index 08d4e8f7bb..8ae88ef2aa 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -1,6 +1,20 @@ Change log ========== +3.0.1 +----- + +[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/43?closed=1) + +### Bugfixes + +* Fixed a bug where `APIClient.login` didn't populate the `_auth_configs` + dictionary properly, causing subsequent `pull` and `push` operations to fail +* Fixed a bug where some build context files were incorrectly recognized as + being inaccessible. +* Fixed a bug where files with a negative mtime value would + cause errors when included in a build context + 3.0.0 ----- From 34d50483e20e86cb7ab22700e036a5c4d319268a Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 12 Feb 2018 14:59:41 -0800 Subject: [PATCH 0590/1301] Correctly support absolute paths in .dockerignore Signed-off-by: Joffrey F --- docker/utils/build.py | 20 +++++++++++++------- tests/unit/utils_test.py | 27 +++++++++++++++++---------- 2 files changed, 30 insertions(+), 17 deletions(-) diff --git a/docker/utils/build.py b/docker/utils/build.py index d4223e749f..a21887349c 100644 --- a/docker/utils/build.py +++ b/docker/utils/build.py @@ -46,7 +46,7 @@ def exclude_paths(root, patterns, dockerfile=None): ) -def should_include(path, exclude_patterns, include_patterns): +def should_include(path, exclude_patterns, include_patterns, root): """ Given a path, a list of exclude patterns, and a list of inclusion patterns: @@ -61,11 +61,15 @@ def should_include(path, exclude_patterns, include_patterns): for pattern in include_patterns: if match_path(path, pattern): return True + if os.path.isabs(pattern) and match_path( + os.path.join(root, path), pattern): + return True return False return True -def should_check_directory(directory_path, exclude_patterns, include_patterns): +def should_check_directory(directory_path, exclude_patterns, include_patterns, + root): """ Given a directory path, a list of exclude patterns, and a list of inclusion patterns: @@ -91,7 +95,7 @@ def normalize_path(path): if (pattern + '/').startswith(path_with_slash) ] directory_included = should_include( - directory_path, exclude_patterns, include_patterns + directory_path, exclude_patterns, include_patterns, root ) return directory_included or len(possible_child_patterns) > 0 @@ -110,26 +114,28 @@ def get_paths(root, exclude_patterns, include_patterns, has_exceptions=False): # traversal. See https://docs.python.org/2/library/os.html#os.walk dirs[:] = [ d for d in dirs if should_check_directory( - os.path.join(parent, d), exclude_patterns, include_patterns + os.path.join(parent, d), exclude_patterns, include_patterns, + root ) ] for path in dirs: if should_include(os.path.join(parent, path), - exclude_patterns, include_patterns): + exclude_patterns, include_patterns, root): paths.append(os.path.join(parent, path)) for path in files: if should_include(os.path.join(parent, path), - exclude_patterns, include_patterns): + exclude_patterns, include_patterns, root): paths.append(os.path.join(parent, path)) return paths def match_path(path, pattern): + pattern = pattern.rstrip('/' + os.path.sep) - if pattern: + if pattern and not os.path.isabs(pattern): pattern = os.path.relpath(pattern) pattern_components = pattern.split(os.path.sep) diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index eedcf71a0d..e144b7b15f 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -876,6 +876,13 @@ def test_trailing_double_wildcard(self): ) ) + def test_exclude_include_absolute_path(self): + base = make_tree([], ['a.py', 'b.py']) + assert exclude_paths( + base, + ['/*', '!' + os.path.join(base, '*.py')] + ) == set(['a.py', 'b.py']) + class TarTest(unittest.TestCase): def test_tar_with_excludes(self): @@ -1026,52 +1033,52 @@ class ShouldCheckDirectoryTest(unittest.TestCase): def test_should_check_directory_not_excluded(self): assert should_check_directory( - 'not_excluded', self.exclude_patterns, self.include_patterns + 'not_excluded', self.exclude_patterns, self.include_patterns, '.' ) assert should_check_directory( convert_path('dir/with'), self.exclude_patterns, - self.include_patterns + self.include_patterns, '.' ) def test_shoud_check_parent_directories_of_excluded(self): assert should_check_directory( - 'dir', self.exclude_patterns, self.include_patterns + 'dir', self.exclude_patterns, self.include_patterns, '.' ) assert should_check_directory( convert_path('dir/with'), self.exclude_patterns, - self.include_patterns + self.include_patterns, '.' ) def test_should_not_check_excluded_directories_with_no_exceptions(self): assert not should_check_directory( 'exclude_rather_large_directory', self.exclude_patterns, - self.include_patterns + self.include_patterns, '.' ) assert not should_check_directory( convert_path('dir/with/subdir_excluded'), self.exclude_patterns, - self.include_patterns + self.include_patterns, '.' ) def test_should_check_excluded_directory_with_exceptions(self): assert should_check_directory( convert_path('dir/with/exceptions'), self.exclude_patterns, - self.include_patterns + self.include_patterns, '.' ) assert should_check_directory( convert_path('dir/with/exceptions/in'), self.exclude_patterns, - self.include_patterns + self.include_patterns, '.' ) def test_should_not_check_siblings_of_exceptions(self): assert not should_check_directory( convert_path('dir/with/exceptions/but_not_here'), - self.exclude_patterns, self.include_patterns + self.exclude_patterns, self.include_patterns, '.' ) def test_should_check_subdirectories_of_exceptions(self): assert should_check_directory( convert_path('dir/with/exceptions/like_this_one/subdir'), - self.exclude_patterns, self.include_patterns + self.exclude_patterns, self.include_patterns, '.' ) From 48e45afe88f89a60401e3dfb7af69080204e6077 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 13 Feb 2018 16:28:06 -0800 Subject: [PATCH 0591/1301] Add support for device_cgroup_rules parameter in host config Signed-off-by: Joffrey F --- docker/api/container.py | 2 ++ docker/models/containers.py | 3 +++ docker/types/containers.py | 12 +++++++++++- tests/integration/api_container_test.py | 15 +++++++++++++++ 4 files changed, 31 insertions(+), 1 deletion(-) diff --git a/docker/api/container.py b/docker/api/container.py index 962d8cb91c..8994129a3f 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -438,6 +438,8 @@ def create_host_config(self, *args, **kwargs): ``0,1``). cpuset_mems (str): Memory nodes (MEMs) in which to allow execution (``0-3``, ``0,1``). Only effective on NUMA systems. + device_cgroup_rules (:py:class:`list`): A list of cgroup rules to + apply to the container. device_read_bps: Limit read rate (bytes per second) from a device in the form of: `[{"Path": "device_path", "Rate": rate}]` device_read_iops: Limit read rate (IO per second) from a device. diff --git a/docker/models/containers.py b/docker/models/containers.py index 107a0204b1..84dfa484e0 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -515,6 +515,8 @@ def run(self, image, command=None, stdout=True, stderr=False, (``0-3``, ``0,1``). Only effective on NUMA systems. detach (bool): Run container in the background and return a :py:class:`Container` object. + device_cgroup_rules (:py:class:`list`): A list of cgroup rules to + apply to the container. device_read_bps: Limit read rate (bytes per second) from a device in the form of: `[{"Path": "device_path", "Rate": rate}]` device_read_iops: Limit read rate (IO per second) from a device. @@ -912,6 +914,7 @@ def prune(self, filters=None): 'cpuset_mems', 'cpu_rt_period', 'cpu_rt_runtime', + 'device_cgroup_rules', 'device_read_bps', 'device_read_iops', 'device_write_bps', diff --git a/docker/types/containers.py b/docker/types/containers.py index b4a329c220..252142073f 100644 --- a/docker/types/containers.py +++ b/docker/types/containers.py @@ -120,7 +120,8 @@ def __init__(self, version, binds=None, port_bindings=None, init=None, init_path=None, volume_driver=None, cpu_count=None, cpu_percent=None, nano_cpus=None, cpuset_mems=None, runtime=None, mounts=None, - cpu_rt_period=None, cpu_rt_runtime=None): + cpu_rt_period=None, cpu_rt_runtime=None, + device_cgroup_rules=None): if mem_limit is not None: self['Memory'] = parse_bytes(mem_limit) @@ -466,6 +467,15 @@ def __init__(self, version, binds=None, port_bindings=None, raise host_config_version_error('mounts', '1.30') self['Mounts'] = mounts + if device_cgroup_rules is not None: + if version_lt(version, '1.28'): + raise host_config_version_error('device_cgroup_rules', '1.28') + if not isinstance(device_cgroup_rules, list): + raise host_config_type_error( + 'device_cgroup_rules', device_cgroup_rules, 'list' + ) + self['DeviceCgroupRules'] = device_cgroup_rules + def host_config_type_error(param, param_value, expected): error_msg = 'Invalid type for {0} param: expected {1} but found {2}' diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index 01780a771b..8447aa5f05 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -474,6 +474,21 @@ def test_create_with_cpu_rt_options(self): assert config['HostConfig']['CpuRealtimeRuntime'] == 500 assert config['HostConfig']['CpuRealtimePeriod'] == 1000 + @requires_api_version('1.28') + def test_create_with_device_cgroup_rules(self): + rule = 'c 7:128 rwm' + ctnr = self.client.create_container( + BUSYBOX, 'cat /sys/fs/cgroup/devices/devices.list', + host_config=self.client.create_host_config( + device_cgroup_rules=[rule] + ) + ) + self.tmp_containers.append(ctnr) + config = self.client.inspect_container(ctnr) + assert config['HostConfig']['DeviceCgroupRules'] == [rule] + self.client.start(ctnr) + assert rule in self.client.logs(ctnr).decode('utf-8') + class VolumeBindTest(BaseAPIIntegrationTest): def setUp(self): From 3498b63fb0b9d2fd5a7f1f42e6c6dde772e055ce Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 13 Feb 2018 18:55:56 -0800 Subject: [PATCH 0592/1301] Fix authconfig resolution when credStore is used combined with login() Signed-off-by: Joffrey F --- docker/auth.py | 5 ++++- tests/unit/auth_test.py | 13 +++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/docker/auth.py b/docker/auth.py index 91be2b8502..48fcd8b504 100644 --- a/docker/auth.py +++ b/docker/auth.py @@ -90,9 +90,12 @@ def resolve_authconfig(authconfig, registry=None): log.debug( 'Using credentials store "{0}"'.format(store_name) ) - return _resolve_authconfig_credstore( + cfg = _resolve_authconfig_credstore( authconfig, registry, store_name ) + if cfg is not None: + return cfg + log.debug('No entry in credstore - fetching from auth dict') # Default to the public index server registry = resolve_index_name(registry) if registry else INDEX_NAME diff --git a/tests/unit/auth_test.py b/tests/unit/auth_test.py index d6981cd9da..ee32ca08a9 100644 --- a/tests/unit/auth_test.py +++ b/tests/unit/auth_test.py @@ -210,6 +210,19 @@ def test_resolve_registry_and_auth_unauthenticated_registry(self): self.auth_config, auth.resolve_repository_name(image)[0] ) is None + def test_resolve_auth_with_empty_credstore_and_auth_dict(self): + auth_config = { + 'auths': auth.parse_auth({ + 'https://index.docker.io/v1/': self.index_config, + }), + 'credsStore': 'blackbox' + } + with mock.patch('docker.auth._resolve_authconfig_credstore') as m: + m.return_value = None + assert 'indexuser' == auth.resolve_authconfig( + auth_config, None + )['username'] + class CredStoreTest(unittest.TestCase): def test_get_credential_store(self): From cbbc37ac7b508d1a84ad68fa043f25f99d17b602 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Wed, 14 Feb 2018 13:01:16 +0000 Subject: [PATCH 0593/1301] Clean up created volume from test_run_with_named_volume This fix adds the volume id to the list so that it could be cleaned up on test teardown. The issue was originally from https://github.com/moby/moby/pull/36292 where an additional `somevolume` pre-exists in tests. Signed-off-by: Yong Tang --- tests/integration/models_containers_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration/models_containers_test.py b/tests/integration/models_containers_test.py index a4d9f9c0da..f9f59c43b4 100644 --- a/tests/integration/models_containers_test.py +++ b/tests/integration/models_containers_test.py @@ -55,7 +55,8 @@ def test_run_with_volume(self): def test_run_with_named_volume(self): client = docker.from_env(version=TEST_API_VERSION) - client.volumes.create(name="somevolume") + volume = client.volumes.create(name="somevolume") + self.tmp_volumes.append(volume.id) container = client.containers.run( "alpine", "sh -c 'echo \"hello\" > /insidecontainer/test'", From 15ae1f09f88b943aba35fb45f97e2d86de92a8ce Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 14 Feb 2018 16:02:04 -0800 Subject: [PATCH 0594/1301] Bump docker-pycreds to 0.2.2 (pass support) Signed-off-by: Joffrey F --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 1602750fde..2b281ae82f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ asn1crypto==0.22.0 backports.ssl-match-hostname==3.5.0.1 cffi==1.10.0 cryptography==1.9 -docker-pycreds==0.2.1 +docker-pycreds==0.2.2 enum34==1.1.6 idna==2.5 ipaddress==1.0.18 diff --git a/setup.py b/setup.py index b628f4a878..271d94f268 100644 --- a/setup.py +++ b/setup.py @@ -23,7 +23,7 @@ 'requests >= 2.14.2, != 2.18.0', 'six >= 1.4.0', 'websocket-client >= 0.32.0', - 'docker-pycreds >= 0.2.1' + 'docker-pycreds >= 0.2.2' ] extras_require = { From 581ccc9f7e8e189248054268c98561ca775bd3d7 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 13 Feb 2018 15:17:03 -0800 Subject: [PATCH 0595/1301] Add chunk_size parameter to data downloading methods (export, get_archive, save) Signed-off-by: Joffrey F --- docker/api/client.py | 6 +++--- docker/api/container.py | 15 +++++++++++---- docker/api/image.py | 8 ++++++-- docker/constants.py | 1 + docker/models/containers.py | 17 +++++++++++++---- docker/models/images.py | 10 ++++++++-- tests/unit/models_containers_test.py | 9 +++++++-- tests/unit/models_images_test.py | 5 ++++- 8 files changed, 53 insertions(+), 18 deletions(-) diff --git a/docker/api/client.py b/docker/api/client.py index e69d143b2f..bddab61f31 100644 --- a/docker/api/client.py +++ b/docker/api/client.py @@ -350,10 +350,10 @@ def _multiplexed_response_stream_helper(self, response): break yield data - def _stream_raw_result(self, response): - ''' Stream result for TTY-enabled container ''' + def _stream_raw_result(self, response, chunk_size=1, decode=True): + ''' Stream result for TTY-enabled container and raw binary data''' self._raise_for_status(response) - for out in response.iter_content(chunk_size=1, decode_unicode=True): + for out in response.iter_content(chunk_size, decode): yield out def _read_from_socket(self, response, stream, tty=False): diff --git a/docker/api/container.py b/docker/api/container.py index 962d8cb91c..e986cf22cf 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -3,6 +3,7 @@ from .. import errors from .. import utils +from ..constants import DEFAULT_DATA_CHUNK_SIZE from ..types import ( ContainerConfig, EndpointConfig, HostConfig, NetworkingConfig ) @@ -643,12 +644,15 @@ def diff(self, container): ) @utils.check_resource('container') - def export(self, container): + def export(self, container, chunk_size=DEFAULT_DATA_CHUNK_SIZE): """ Export the contents of a filesystem as a tar archive. Args: container (str): The container to export + chunk_size (int): The number of bytes returned by each iteration + of the generator. If ``None``, data will be streamed as it is + received. Default: 2 MB Returns: (generator): The archived filesystem data stream @@ -660,10 +664,10 @@ def export(self, container): res = self._get( self._url("/containers/{0}/export", container), stream=True ) - return self._stream_raw_result(res) + return self._stream_raw_result(res, chunk_size, False) @utils.check_resource('container') - def get_archive(self, container, path): + def get_archive(self, container, path, chunk_size=DEFAULT_DATA_CHUNK_SIZE): """ Retrieve a file or folder from a container in the form of a tar archive. @@ -671,6 +675,9 @@ def get_archive(self, container, path): Args: container (str): The container where the file is located path (str): Path to the file or folder to retrieve + chunk_size (int): The number of bytes returned by each iteration + of the generator. If ``None``, data will be streamed as it is + received. Default: 2 MB Returns: (tuple): First element is a raw tar data stream. Second element is @@ -688,7 +695,7 @@ def get_archive(self, container, path): self._raise_for_status(res) encoded_stat = res.headers.get('x-docker-container-path-stat') return ( - self._stream_raw_result(res), + self._stream_raw_result(res, chunk_size, False), utils.decode_json_header(encoded_stat) if encoded_stat else None ) diff --git a/docker/api/image.py b/docker/api/image.py index fa832a389a..3ebca32e59 100644 --- a/docker/api/image.py +++ b/docker/api/image.py @@ -4,6 +4,7 @@ import six from .. import auth, errors, utils +from ..constants import DEFAULT_DATA_CHUNK_SIZE log = logging.getLogger(__name__) @@ -11,12 +12,15 @@ class ImageApiMixin(object): @utils.check_resource('image') - def get_image(self, image): + def get_image(self, image, chunk_size=DEFAULT_DATA_CHUNK_SIZE): """ Get a tarball of an image. Similar to the ``docker save`` command. Args: image (str): Image name to get + chunk_size (int): The number of bytes returned by each iteration + of the generator. If ``None``, data will be streamed as it is + received. Default: 2 MB Returns: (generator): A stream of raw archive data. @@ -34,7 +38,7 @@ def get_image(self, image): >>> f.close() """ res = self._get(self._url("/images/{0}/get", image), stream=True) - return self._stream_raw_result(res) + return self._stream_raw_result(res, chunk_size, False) @utils.check_resource('image') def history(self, image): diff --git a/docker/constants.py b/docker/constants.py index 9ab3673255..7565a76889 100644 --- a/docker/constants.py +++ b/docker/constants.py @@ -17,3 +17,4 @@ DEFAULT_USER_AGENT = "docker-sdk-python/{0}".format(version) DEFAULT_NUM_POOLS = 25 +DEFAULT_DATA_CHUNK_SIZE = 1024 * 2048 diff --git a/docker/models/containers.py b/docker/models/containers.py index 107a0204b1..b6e34dd240 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -3,6 +3,7 @@ from collections import namedtuple from ..api import APIClient +from ..constants import DEFAULT_DATA_CHUNK_SIZE from ..errors import (ContainerError, ImageNotFound, create_unexpected_kwargs_error) from ..types import HostConfig @@ -181,10 +182,15 @@ def exec_run(self, cmd, stdout=True, stderr=True, stdin=False, tty=False, exec_output ) - def export(self): + def export(self, chunk_size=DEFAULT_DATA_CHUNK_SIZE): """ Export the contents of the container's filesystem as a tar archive. + Args: + chunk_size (int): The number of bytes returned by each iteration + of the generator. If ``None``, data will be streamed as it is + received. Default: 2 MB + Returns: (str): The filesystem tar archive @@ -192,15 +198,18 @@ def export(self): :py:class:`docker.errors.APIError` If the server returns an error. """ - return self.client.api.export(self.id) + return self.client.api.export(self.id, chunk_size) - def get_archive(self, path): + def get_archive(self, path, chunk_size=DEFAULT_DATA_CHUNK_SIZE): """ Retrieve a file or folder from the container in the form of a tar archive. Args: path (str): Path to the file or folder to retrieve + chunk_size (int): The number of bytes returned by each iteration + of the generator. If ``None``, data will be streamed as it is + received. Default: 2 MB Returns: (tuple): First element is a raw tar data stream. Second element is @@ -210,7 +219,7 @@ def get_archive(self, path): :py:class:`docker.errors.APIError` If the server returns an error. """ - return self.client.api.get_archive(self.id, path) + return self.client.api.get_archive(self.id, path, chunk_size) def kill(self, signal=None): """ diff --git a/docker/models/images.py b/docker/models/images.py index 0f3c71ab4f..d604f7c7f6 100644 --- a/docker/models/images.py +++ b/docker/models/images.py @@ -4,6 +4,7 @@ import six from ..api import APIClient +from ..constants import DEFAULT_DATA_CHUNK_SIZE from ..errors import BuildError, ImageLoadError from ..utils import parse_repository_tag from ..utils.json_stream import json_stream @@ -58,10 +59,15 @@ def history(self): """ return self.client.api.history(self.id) - def save(self): + def save(self, chunk_size=DEFAULT_DATA_CHUNK_SIZE): """ Get a tarball of an image. Similar to the ``docker save`` command. + Args: + chunk_size (int): The number of bytes returned by each iteration + of the generator. If ``None``, data will be streamed as it is + received. Default: 2 MB + Returns: (generator): A stream of raw archive data. @@ -77,7 +83,7 @@ def save(self): >>> f.write(chunk) >>> f.close() """ - return self.client.api.get_image(self.id) + return self.client.api.get_image(self.id, chunk_size) def tag(self, repository, tag=None, **kwargs): """ diff --git a/tests/unit/models_containers_test.py b/tests/unit/models_containers_test.py index f79f5d5b33..2b0b499ef3 100644 --- a/tests/unit/models_containers_test.py +++ b/tests/unit/models_containers_test.py @@ -1,4 +1,5 @@ import docker +from docker.constants import DEFAULT_DATA_CHUNK_SIZE from docker.models.containers import Container, _create_container_args from docker.models.images import Image import unittest @@ -422,13 +423,17 @@ def test_export(self): client = make_fake_client() container = client.containers.get(FAKE_CONTAINER_ID) container.export() - client.api.export.assert_called_with(FAKE_CONTAINER_ID) + client.api.export.assert_called_with( + FAKE_CONTAINER_ID, DEFAULT_DATA_CHUNK_SIZE + ) def test_get_archive(self): client = make_fake_client() container = client.containers.get(FAKE_CONTAINER_ID) container.get_archive('foo') - client.api.get_archive.assert_called_with(FAKE_CONTAINER_ID, 'foo') + client.api.get_archive.assert_called_with( + FAKE_CONTAINER_ID, 'foo', DEFAULT_DATA_CHUNK_SIZE + ) def test_image(self): client = make_fake_client() diff --git a/tests/unit/models_images_test.py b/tests/unit/models_images_test.py index dacd72be06..67832795fe 100644 --- a/tests/unit/models_images_test.py +++ b/tests/unit/models_images_test.py @@ -1,3 +1,4 @@ +from docker.constants import DEFAULT_DATA_CHUNK_SIZE from docker.models.images import Image import unittest @@ -116,7 +117,9 @@ def test_save(self): client = make_fake_client() image = client.images.get(FAKE_IMAGE_ID) image.save() - client.api.get_image.assert_called_with(FAKE_IMAGE_ID) + client.api.get_image.assert_called_with( + FAKE_IMAGE_ID, DEFAULT_DATA_CHUNK_SIZE + ) def test_tag(self): client = make_fake_client() From 4c708f568c55ede2396b01aebf607374f731b3f2 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 14 Feb 2018 16:22:33 -0800 Subject: [PATCH 0596/1301] Fix test_login flakes Signed-off-by: Joffrey F --- tests/unit/api_test.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/unit/api_test.py b/tests/unit/api_test.py index f65e13ecb3..61d24460ef 100644 --- a/tests/unit/api_test.py +++ b/tests/unit/api_test.py @@ -214,12 +214,13 @@ def test_search(self): def test_login(self): self.client.login('sakuya', 'izayoi') - fake_request.assert_called_with( - 'POST', url_prefix + 'auth', - data=json.dumps({'username': 'sakuya', 'password': 'izayoi'}), - timeout=DEFAULT_TIMEOUT_SECONDS, - headers={'Content-Type': 'application/json'} - ) + args = fake_request.call_args + assert args[0][0] == 'POST' + assert args[0][1] == url_prefix + 'auth' + assert json.loads(args[1]['data']) == { + 'username': 'sakuya', 'password': 'izayoi' + } + assert args[1]['headers'] == {'Content-Type': 'application/json'} assert self.client._auth_configs['auths'] == { 'docker.io': { From 181c1c8eb970ae11a707b6b6c3d1e4d546504ccf Mon Sep 17 00:00:00 2001 From: mefyl Date: Fri, 16 Feb 2018 11:03:35 +0100 Subject: [PATCH 0597/1301] Revert "Correctly support absolute paths in .dockerignore" This reverts commit 34d50483e20e86cb7ab22700e036a5c4d319268a. Signed-off-by: mefyl --- docker/utils/build.py | 20 +++++++------------- tests/unit/utils_test.py | 27 ++++++++++----------------- 2 files changed, 17 insertions(+), 30 deletions(-) diff --git a/docker/utils/build.py b/docker/utils/build.py index a21887349c..d4223e749f 100644 --- a/docker/utils/build.py +++ b/docker/utils/build.py @@ -46,7 +46,7 @@ def exclude_paths(root, patterns, dockerfile=None): ) -def should_include(path, exclude_patterns, include_patterns, root): +def should_include(path, exclude_patterns, include_patterns): """ Given a path, a list of exclude patterns, and a list of inclusion patterns: @@ -61,15 +61,11 @@ def should_include(path, exclude_patterns, include_patterns, root): for pattern in include_patterns: if match_path(path, pattern): return True - if os.path.isabs(pattern) and match_path( - os.path.join(root, path), pattern): - return True return False return True -def should_check_directory(directory_path, exclude_patterns, include_patterns, - root): +def should_check_directory(directory_path, exclude_patterns, include_patterns): """ Given a directory path, a list of exclude patterns, and a list of inclusion patterns: @@ -95,7 +91,7 @@ def normalize_path(path): if (pattern + '/').startswith(path_with_slash) ] directory_included = should_include( - directory_path, exclude_patterns, include_patterns, root + directory_path, exclude_patterns, include_patterns ) return directory_included or len(possible_child_patterns) > 0 @@ -114,28 +110,26 @@ def get_paths(root, exclude_patterns, include_patterns, has_exceptions=False): # traversal. See https://docs.python.org/2/library/os.html#os.walk dirs[:] = [ d for d in dirs if should_check_directory( - os.path.join(parent, d), exclude_patterns, include_patterns, - root + os.path.join(parent, d), exclude_patterns, include_patterns ) ] for path in dirs: if should_include(os.path.join(parent, path), - exclude_patterns, include_patterns, root): + exclude_patterns, include_patterns): paths.append(os.path.join(parent, path)) for path in files: if should_include(os.path.join(parent, path), - exclude_patterns, include_patterns, root): + exclude_patterns, include_patterns): paths.append(os.path.join(parent, path)) return paths def match_path(path, pattern): - pattern = pattern.rstrip('/' + os.path.sep) - if pattern and not os.path.isabs(pattern): + if pattern: pattern = os.path.relpath(pattern) pattern_components = pattern.split(os.path.sep) diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index e144b7b15f..eedcf71a0d 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -876,13 +876,6 @@ def test_trailing_double_wildcard(self): ) ) - def test_exclude_include_absolute_path(self): - base = make_tree([], ['a.py', 'b.py']) - assert exclude_paths( - base, - ['/*', '!' + os.path.join(base, '*.py')] - ) == set(['a.py', 'b.py']) - class TarTest(unittest.TestCase): def test_tar_with_excludes(self): @@ -1033,52 +1026,52 @@ class ShouldCheckDirectoryTest(unittest.TestCase): def test_should_check_directory_not_excluded(self): assert should_check_directory( - 'not_excluded', self.exclude_patterns, self.include_patterns, '.' + 'not_excluded', self.exclude_patterns, self.include_patterns ) assert should_check_directory( convert_path('dir/with'), self.exclude_patterns, - self.include_patterns, '.' + self.include_patterns ) def test_shoud_check_parent_directories_of_excluded(self): assert should_check_directory( - 'dir', self.exclude_patterns, self.include_patterns, '.' + 'dir', self.exclude_patterns, self.include_patterns ) assert should_check_directory( convert_path('dir/with'), self.exclude_patterns, - self.include_patterns, '.' + self.include_patterns ) def test_should_not_check_excluded_directories_with_no_exceptions(self): assert not should_check_directory( 'exclude_rather_large_directory', self.exclude_patterns, - self.include_patterns, '.' + self.include_patterns ) assert not should_check_directory( convert_path('dir/with/subdir_excluded'), self.exclude_patterns, - self.include_patterns, '.' + self.include_patterns ) def test_should_check_excluded_directory_with_exceptions(self): assert should_check_directory( convert_path('dir/with/exceptions'), self.exclude_patterns, - self.include_patterns, '.' + self.include_patterns ) assert should_check_directory( convert_path('dir/with/exceptions/in'), self.exclude_patterns, - self.include_patterns, '.' + self.include_patterns ) def test_should_not_check_siblings_of_exceptions(self): assert not should_check_directory( convert_path('dir/with/exceptions/but_not_here'), - self.exclude_patterns, self.include_patterns, '.' + self.exclude_patterns, self.include_patterns ) def test_should_check_subdirectories_of_exceptions(self): assert should_check_directory( convert_path('dir/with/exceptions/like_this_one/subdir'), - self.exclude_patterns, self.include_patterns, '.' + self.exclude_patterns, self.include_patterns ) From cc455d7fd5ac8d192e64965f68ecea74dca011de Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 20 Feb 2018 14:51:49 -0800 Subject: [PATCH 0598/1301] Fix DockerClient pull bug when pulling image by digest Signed-off-by: Joffrey F --- docker/models/images.py | 4 +++- tests/integration/models_images_test.py | 9 +++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/docker/models/images.py b/docker/models/images.py index d604f7c7f6..58d5d93c47 100644 --- a/docker/models/images.py +++ b/docker/models/images.py @@ -314,7 +314,9 @@ def pull(self, repository, tag=None, **kwargs): self.client.api.pull(repository, tag=tag, **kwargs) if tag: - return self.get('{0}:{1}'.format(repository, tag)) + return self.get('{0}{2}{1}'.format( + repository, tag, '@' if tag.startswith('sha256:') else ':' + )) return self.list(repository) def push(self, repository, tag=None, **kwargs): diff --git a/tests/integration/models_images_test.py b/tests/integration/models_images_test.py index 2fa71a7947..ae735baafb 100644 --- a/tests/integration/models_images_test.py +++ b/tests/integration/models_images_test.py @@ -74,6 +74,15 @@ def test_pull_with_tag(self): image = client.images.pull('alpine', tag='3.3') assert 'alpine:3.3' in image.attrs['RepoTags'] + def test_pull_with_sha(self): + image_ref = ( + 'hello-world@sha256:083de497cff944f969d8499ab94f07134c50bcf5e6b95' + '59b27182d3fa80ce3f7' + ) + client = docker.from_env(version=TEST_API_VERSION) + image = client.images.pull(image_ref) + assert image_ref in image.attrs['RepoDigests'] + def test_pull_multiple(self): client = docker.from_env(version=TEST_API_VERSION) images = client.images.pull('hello-world') From 820de848fa73f20cb80215ceb4b8cdafc855867e Mon Sep 17 00:00:00 2001 From: William Myers Date: Fri, 16 Feb 2018 22:29:21 -0700 Subject: [PATCH 0599/1301] Add support for generic resources to docker.types.Resources - Add support for dict and list generic_resources parameter - Add generic_resources integration test Signed-off-by: William Myers --- docker/types/services.py | 34 +++++++++++++++++- tests/integration/api_service_test.py | 51 +++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 1 deletion(-) diff --git a/docker/types/services.py b/docker/types/services.py index d530e61db6..69e0e02401 100644 --- a/docker/types/services.py +++ b/docker/types/services.py @@ -306,9 +306,14 @@ class Resources(dict): mem_limit (int): Memory limit in Bytes. cpu_reservation (int): CPU reservation in units of 10^9 CPU shares. mem_reservation (int): Memory reservation in Bytes. + generic_resources (dict:`list`): List of node level generic + resources, for example a GPU, in the form of + ``{ ResourceSpec: { 'Kind': kind, 'Value': value }}``, where + ResourceSpec is one of 'DiscreteResourceSpec' or 'NamedResourceSpec' + or in the form of ``{ resource_name: resource_value }``. """ def __init__(self, cpu_limit=None, mem_limit=None, cpu_reservation=None, - mem_reservation=None): + mem_reservation=None, generic_resources=None): limits = {} reservation = {} if cpu_limit is not None: @@ -319,6 +324,33 @@ def __init__(self, cpu_limit=None, mem_limit=None, cpu_reservation=None, reservation['NanoCPUs'] = cpu_reservation if mem_reservation is not None: reservation['MemoryBytes'] = mem_reservation + if generic_resources is not None: + # if isinstance(generic_resources, list): + # reservation['GenericResources'] = generic_resources + if isinstance(generic_resources, (list, tuple)): + reservation['GenericResources'] = list(generic_resources) + elif isinstance(generic_resources, dict): + resources = [] + for kind, value in six.iteritems(generic_resources): + resource_type = None + if isinstance(value, int): + resource_type = 'DiscreteResourceSpec' + elif isinstance(value, str): + resource_type = 'NamedResourceSpec' + else: + raise errors.InvalidArgument( + 'Unsupported generic resource reservation ' + 'type: {}'.format({kind: value}) + ) + resources.append({ + resource_type: {'Kind': kind, 'Value': value} + }) + reservation['GenericResources'] = resources + else: + raise errors.InvalidArgument( + 'Unsupported generic resources ' + 'type: {}'.format(generic_resources) + ) if limits: self['Limits'] = limits diff --git a/tests/integration/api_service_test.py b/tests/integration/api_service_test.py index 5cc3fc190f..07a34b95a6 100644 --- a/tests/integration/api_service_test.py +++ b/tests/integration/api_service_test.py @@ -212,6 +212,57 @@ def test_create_service_with_resources_constraints(self): 'Reservations' ] + def _create_service_with_generic_resources(self, generic_resources): + container_spec = docker.types.ContainerSpec(BUSYBOX, ['true']) + + resources = docker.types.Resources( + generic_resources=generic_resources + ) + task_tmpl = docker.types.TaskTemplate( + container_spec, resources=resources + ) + name = self.get_service_name() + svc_id = self.client.create_service(task_tmpl, name=name) + return resources, self.client.inspect_service(svc_id) + + def test_create_service_with_generic_resources(self): + successful = [{ + 'input': [ + {'DiscreteResourceSpec': {'Kind': 'gpu', 'Value': 1}}, + {'NamedResourceSpec': {'Kind': 'gpu', 'Value': 'test'}} + ]}, { + 'input': {'gpu': 2, 'mpi': 'latest'}, + 'expected': [ + {'DiscreteResourceSpec': {'Kind': 'gpu', 'Value': 2}}, + {'NamedResourceSpec': {'Kind': 'mpi', 'Value': 'latest'}} + ]} + ] + + for test in successful: + t = test['input'] + resrcs, svc_info = self._create_service_with_generic_resources(t) + + assert 'TaskTemplate' in svc_info['Spec'] + res_template = svc_info['Spec']['TaskTemplate'] + assert 'Resources' in res_template + res_reservations = res_template['Resources']['Reservations'] + assert res_reservations == resrcs['Reservations'] + assert 'GenericResources' in res_reservations + + def _key(d, specs=('DiscreteResourceSpec', 'NamedResourceSpec')): + return [d.get(s, {}).get('Kind', '') for s in specs] + + actual = res_reservations['GenericResources'] + expected = test.get('expected', test['input']) + assert sorted(actual, key=_key) == sorted(expected, key=_key) + + for test_input in ['1', 1.0, lambda: '1', {1, 2}]: + try: + self._create_service_with_generic_resources(test_input) + self.fail('Should fail: {}'.format(test_input)) + except docker.errors.InvalidArgument: + pass + def test_create_service_with_update_config(self): container_spec = docker.types.ContainerSpec(BUSYBOX, ['true']) task_tmpl = docker.types.TaskTemplate(container_spec) From 9b6b306e173d6b1f8a8fee781332f37735a12573 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 20 Feb 2018 16:25:08 -0800 Subject: [PATCH 0600/1301] Code cleanup and version guards Signed-off-by: Joffrey F --- docker/api/service.py | 5 +++ docker/types/services.py | 65 ++++++++++++++------------- tests/integration/api_service_test.py | 8 ++-- 3 files changed, 42 insertions(+), 36 deletions(-) diff --git a/docker/api/service.py b/docker/api/service.py index ceae8fc9a8..95fb07e476 100644 --- a/docker/api/service.py +++ b/docker/api/service.py @@ -73,6 +73,11 @@ def raise_version_error(param, min_version): if container_spec.get('Isolation') is not None: raise_version_error('ContainerSpec.isolation', '1.35') + if task_template.get('Resources'): + if utils.version_lt(version, '1.35'): + if task_template['Resources'].get('GenericResources'): + raise_version_error('Resources.generic_resources', '1.35') + def _merge_task_template(current, override): merged = current.copy() diff --git a/docker/types/services.py b/docker/types/services.py index 69e0e02401..09eb05edbd 100644 --- a/docker/types/services.py +++ b/docker/types/services.py @@ -306,11 +306,10 @@ class Resources(dict): mem_limit (int): Memory limit in Bytes. cpu_reservation (int): CPU reservation in units of 10^9 CPU shares. mem_reservation (int): Memory reservation in Bytes. - generic_resources (dict:`list`): List of node level generic - resources, for example a GPU, in the form of - ``{ ResourceSpec: { 'Kind': kind, 'Value': value }}``, where - ResourceSpec is one of 'DiscreteResourceSpec' or 'NamedResourceSpec' - or in the form of ``{ resource_name: resource_value }``. + generic_resources (dict or :py:class:`list`): Node level generic + resources, for example a GPU, using the following format: + ``{ resource_name: resource_value }``. Alternatively, a list of + of resource specifications as defined by the Engine API. """ def __init__(self, cpu_limit=None, mem_limit=None, cpu_reservation=None, mem_reservation=None, generic_resources=None): @@ -325,39 +324,41 @@ def __init__(self, cpu_limit=None, mem_limit=None, cpu_reservation=None, if mem_reservation is not None: reservation['MemoryBytes'] = mem_reservation if generic_resources is not None: - # if isinstance(generic_resources, list): - # reservation['GenericResources'] = generic_resources - if isinstance(generic_resources, (list, tuple)): - reservation['GenericResources'] = list(generic_resources) - elif isinstance(generic_resources, dict): - resources = [] - for kind, value in six.iteritems(generic_resources): - resource_type = None - if isinstance(value, int): - resource_type = 'DiscreteResourceSpec' - elif isinstance(value, str): - resource_type = 'NamedResourceSpec' - else: - raise errors.InvalidArgument( - 'Unsupported generic resource reservation ' - 'type: {}'.format({kind: value}) - ) - resources.append({ - resource_type: {'Kind': kind, 'Value': value} - }) - reservation['GenericResources'] = resources - else: - raise errors.InvalidArgument( - 'Unsupported generic resources ' - 'type: {}'.format(generic_resources) - ) - + reservation['GenericResources'] = ( + _convert_generic_resources_dict(generic_resources) + ) if limits: self['Limits'] = limits if reservation: self['Reservations'] = reservation +def _convert_generic_resources_dict(generic_resources): + if isinstance(generic_resources, list): + return generic_resources + if not isinstance(generic_resources, dict): + raise errors.InvalidArgument( + 'generic_resources must be a dict or a list' + ' (found {})'.format(type(generic_resources)) + ) + resources = [] + for kind, value in six.iteritems(generic_resources): + resource_type = None + if isinstance(value, int): + resource_type = 'DiscreteResourceSpec' + elif isinstance(value, str): + resource_type = 'NamedResourceSpec' + else: + raise errors.InvalidArgument( + 'Unsupported generic resource reservation ' + 'type: {}'.format({kind: value}) + ) + resources.append({ + resource_type: {'Kind': kind, 'Value': value} + }) + return resources + + class UpdateConfig(dict): """ diff --git a/tests/integration/api_service_test.py b/tests/integration/api_service_test.py index 07a34b95a6..9d91f9e015 100644 --- a/tests/integration/api_service_test.py +++ b/tests/integration/api_service_test.py @@ -4,6 +4,7 @@ import time import docker +import pytest import six from ..helpers import ( @@ -225,6 +226,7 @@ def _create_service_with_generic_resources(self, generic_resources): svc_id = self.client.create_service(task_tmpl, name=name) return resources, self.client.inspect_service(svc_id) + @requires_api_version('1.35') def test_create_service_with_generic_resources(self): successful = [{ 'input': [ @@ -256,12 +258,10 @@ def _key(d, specs=('DiscreteResourceSpec', 'NamedResourceSpec')): expected = test.get('expected', test['input']) assert sorted(actual, key=_key) == sorted(expected, key=_key) + def test_create_service_with_invalid_generic_resources(self): for test_input in ['1', 1.0, lambda: '1', {1, 2}]: - try: + with pytest.raises(docker.errors.InvalidArgument): self._create_service_with_generic_resources(test_input) - self.fail('Should fail: {}'.format(test_input)) - except docker.errors.InvalidArgument: - pass def test_create_service_with_update_config(self): container_spec = docker.types.ContainerSpec(BUSYBOX, ['true']) From 8fd9d3c99e9314323228af4832054b22d2ac4966 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 20 Feb 2018 17:11:27 -0800 Subject: [PATCH 0601/1301] GenericResources was introduced in 1.32 Signed-off-by: Joffrey F --- docker/api/service.py | 4 ++-- tests/integration/api_service_test.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docker/api/service.py b/docker/api/service.py index 95fb07e476..03b0ca6ea2 100644 --- a/docker/api/service.py +++ b/docker/api/service.py @@ -74,9 +74,9 @@ def raise_version_error(param, min_version): raise_version_error('ContainerSpec.isolation', '1.35') if task_template.get('Resources'): - if utils.version_lt(version, '1.35'): + if utils.version_lt(version, '1.32'): if task_template['Resources'].get('GenericResources'): - raise_version_error('Resources.generic_resources', '1.35') + raise_version_error('Resources.generic_resources', '1.32') def _merge_task_template(current, override): diff --git a/tests/integration/api_service_test.py b/tests/integration/api_service_test.py index 9d91f9e015..85f9dccf26 100644 --- a/tests/integration/api_service_test.py +++ b/tests/integration/api_service_test.py @@ -226,7 +226,7 @@ def _create_service_with_generic_resources(self, generic_resources): svc_id = self.client.create_service(task_tmpl, name=name) return resources, self.client.inspect_service(svc_id) - @requires_api_version('1.35') + @requires_api_version('1.32') def test_create_service_with_generic_resources(self): successful = [{ 'input': [ @@ -258,6 +258,7 @@ def _key(d, specs=('DiscreteResourceSpec', 'NamedResourceSpec')): expected = test.get('expected', test['input']) assert sorted(actual, key=_key) == sorted(expected, key=_key) + @requires_api_version('1.32') def test_create_service_with_invalid_generic_resources(self): for test_input in ['1', 1.0, lambda: '1', {1, 2}]: with pytest.raises(docker.errors.InvalidArgument): From c8f5a5ad4040560ce62d53002ecec12485b531f7 Mon Sep 17 00:00:00 2001 From: mefyl Date: Fri, 16 Feb 2018 11:22:29 +0100 Subject: [PATCH 0602/1301] Fix dockerignore handling of absolute path exceptions. Signed-off-by: mefyl --- docker/utils/build.py | 6 +++--- tests/unit/utils_test.py | 7 +++++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/docker/utils/build.py b/docker/utils/build.py index d4223e749f..e86a04e617 100644 --- a/docker/utils/build.py +++ b/docker/utils/build.py @@ -26,13 +26,13 @@ def exclude_paths(root, patterns, dockerfile=None): if dockerfile is None: dockerfile = 'Dockerfile' - patterns = [p.lstrip('/') for p in patterns] exceptions = [p for p in patterns if p.startswith('!')] - include_patterns = [p[1:] for p in exceptions] + include_patterns = [p[1:].lstrip('/') for p in exceptions] include_patterns += [dockerfile, '.dockerignore'] - exclude_patterns = list(set(patterns) - set(exceptions)) + exclude_patterns = [ + p.lstrip('/') for p in list(set(patterns) - set(exceptions))] paths = get_paths(root, exclude_patterns, include_patterns, has_exceptions=len(exceptions) > 0) diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index eedcf71a0d..0ee041ae94 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -758,6 +758,13 @@ def test_single_subdir_single_filename_leading_slash(self): self.all_paths - set(['foo/a.py']) ) + def test_exclude_include_absolute_path(self): + base = make_tree([], ['a.py', 'b.py']) + assert exclude_paths( + base, + ['/*', '!/*.py'] + ) == set(['a.py', 'b.py']) + def test_single_subdir_with_path_traversal(self): assert self.exclude(['foo/whoops/../a.py']) == convert_paths( self.all_paths - set(['foo/a.py']) From bb3ad64060b1f1136a6a42ed4d2018f71ebd371d Mon Sep 17 00:00:00 2001 From: mefyl Date: Fri, 16 Feb 2018 16:08:11 +0100 Subject: [PATCH 0603/1301] Fix .dockerignore: accept wildcard in inclusion pattern, honor last line precedence. Signed-off-by: mefyl --- docker/utils/build.py | 188 +++++++++++++++------------------------ tests/unit/utils_test.py | 84 +++++------------ 2 files changed, 94 insertions(+), 178 deletions(-) diff --git a/docker/utils/build.py b/docker/utils/build.py index e86a04e617..bfdb87b2f1 100644 --- a/docker/utils/build.py +++ b/docker/utils/build.py @@ -1,20 +1,24 @@ import os +import re from ..constants import IS_WINDOWS_PLATFORM -from .fnmatch import fnmatch +from fnmatch import fnmatch +from itertools import chain from .utils import create_archive def tar(path, exclude=None, dockerfile=None, fileobj=None, gzip=False): root = os.path.abspath(path) exclude = exclude or [] - return create_archive( files=sorted(exclude_paths(root, exclude, dockerfile=dockerfile)), root=root, fileobj=fileobj, gzip=gzip ) +_SEP = re.compile('/|\\\\') if IS_WINDOWS_PLATFORM else re.compile('/') + + def exclude_paths(root, patterns, dockerfile=None): """ Given a root directory path and a list of .dockerignore patterns, return @@ -23,121 +27,77 @@ def exclude_paths(root, patterns, dockerfile=None): All paths returned are relative to the root. """ + if dockerfile is None: dockerfile = 'Dockerfile' - exceptions = [p for p in patterns if p.startswith('!')] - - include_patterns = [p[1:].lstrip('/') for p in exceptions] - include_patterns += [dockerfile, '.dockerignore'] - - exclude_patterns = [ - p.lstrip('/') for p in list(set(patterns) - set(exceptions))] - - paths = get_paths(root, exclude_patterns, include_patterns, - has_exceptions=len(exceptions) > 0) - - return set(paths).union( - # If the Dockerfile is in a subdirectory that is excluded, get_paths - # will not descend into it and the file will be skipped. This ensures - # it doesn't happen. - set([dockerfile.replace('/', os.path.sep)]) - if os.path.exists(os.path.join(root, dockerfile)) else set() - ) - - -def should_include(path, exclude_patterns, include_patterns): - """ - Given a path, a list of exclude patterns, and a list of inclusion patterns: - - 1. Returns True if the path doesn't match any exclusion pattern - 2. Returns False if the path matches an exclusion pattern and doesn't match - an inclusion pattern - 3. Returns true if the path matches an exclusion pattern and matches an - inclusion pattern + def normalize(p): + # Leading and trailing slashes are not relevant. Yes, + # "foo.py/" must exclude the "foo.py" regular file. "." + # components are not relevant either, even if the whole + # pattern is only ".", as the Docker reference states: "For + # historical reasons, the pattern . is ignored." + split = [pt for pt in re.split(_SEP, p) if pt and pt != '.'] + # ".." component must be cleared with the potential previous + # component, regardless of whether it exists: "A preprocessing + # step [...] eliminates . and .. elements using Go's + # filepath.". + i = 0 + while i < len(split): + if split[i] == '..': + del split[i] + if i > 0: + del split[i - 1] + i -= 1 + else: + i += 1 + return split + + patterns = ( + (True, normalize(p[1:])) + if p.startswith('!') else + (False, normalize(p)) + for p in patterns) + patterns = list(reversed(list(chain( + # Exclude empty patterns such as "." or the empty string. + filter(lambda p: p[1], patterns), + # Always include the Dockerfile and .dockerignore + [(True, dockerfile.split('/')), (True, ['.dockerignore'])])))) + return set(walk(root, patterns)) + + +def walk(root, patterns, default=True): """ - for pattern in exclude_patterns: - if match_path(path, pattern): - for pattern in include_patterns: - if match_path(path, pattern): - return True - return False - return True - - -def should_check_directory(directory_path, exclude_patterns, include_patterns): + A collection of file lying below root that should be included according to + patterns. """ - Given a directory path, a list of exclude patterns, and a list of inclusion - patterns: - - 1. Returns True if the directory path should be included according to - should_include. - 2. Returns True if the directory path is the prefix for an inclusion - pattern - 3. Returns False otherwise - """ - - # To account for exception rules, check directories if their path is a - # a prefix to an inclusion pattern. This logic conforms with the current - # docker logic (2016-10-27): - # https://github.com/docker/docker/blob/bc52939b0455116ab8e0da67869ec81c1a1c3e2c/pkg/archive/archive.go#L640-L671 - - def normalize_path(path): - return path.replace(os.path.sep, '/') - - path_with_slash = normalize_path(directory_path) + '/' - possible_child_patterns = [ - pattern for pattern in map(normalize_path, include_patterns) - if (pattern + '/').startswith(path_with_slash) - ] - directory_included = should_include( - directory_path, exclude_patterns, include_patterns - ) - return directory_included or len(possible_child_patterns) > 0 - - -def get_paths(root, exclude_patterns, include_patterns, has_exceptions=False): - paths = [] - - for parent, dirs, files in os.walk(root, topdown=True, followlinks=False): - parent = os.path.relpath(parent, root) - if parent == '.': - parent = '' - - # Remove excluded patterns from the list of directories to traverse - # by mutating the dirs we're iterating over. - # This looks strange, but is considered the correct way to skip - # traversal. See https://docs.python.org/2/library/os.html#os.walk - dirs[:] = [ - d for d in dirs if should_check_directory( - os.path.join(parent, d), exclude_patterns, include_patterns - ) - ] - - for path in dirs: - if should_include(os.path.join(parent, path), - exclude_patterns, include_patterns): - paths.append(os.path.join(parent, path)) - - for path in files: - if should_include(os.path.join(parent, path), - exclude_patterns, include_patterns): - paths.append(os.path.join(parent, path)) - - return paths - - -def match_path(path, pattern): - pattern = pattern.rstrip('/' + os.path.sep) - if pattern: - pattern = os.path.relpath(pattern) - - pattern_components = pattern.split(os.path.sep) - if len(pattern_components) == 1 and IS_WINDOWS_PLATFORM: - pattern_components = pattern.split('/') - if '**' not in pattern: - path_components = path.split(os.path.sep)[:len(pattern_components)] - else: - path_components = path.split(os.path.sep) - return fnmatch('/'.join(path_components), '/'.join(pattern_components)) + def match(p): + if p[1][0] == '**': + rec = (p[0], p[1][1:]) + return [p] + (match(rec) if rec[1] else [rec]) + elif fnmatch(f, p[1][0]): + return [(p[0], p[1][1:])] + else: + return [] + + for f in os.listdir(root): + cur = os.path.join(root, f) + # The patterns if recursing in that directory. + sub = list(chain(*(match(p) for p in patterns))) + # Whether this file is explicitely included / excluded. + hit = next((p[0] for p in sub if not p[1]), None) + # Whether this file is implicitely included / excluded. + matched = default if hit is None else hit + sub = list(filter(lambda p: p[1], sub)) + if os.path.isdir(cur): + children = False + for r in (os.path.join(f, p) for p in walk(cur, sub, matched)): + yield r + children = True + # The current unit tests expect directories only under those + # conditions. It might be simplifiable though. + if (not sub or not children) and hit or hit is None and default: + yield f + elif matched: + yield f diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index 0ee041ae94..8a4b1937e6 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -23,7 +23,6 @@ decode_json_header, tar, split_command, parse_devices, update_headers, ) -from docker.utils.build import should_check_directory from docker.utils.ports import build_port_bindings, split_port from docker.utils.utils import format_environment @@ -883,6 +882,26 @@ def test_trailing_double_wildcard(self): ) ) + def test_include_wildcard(self): + base = make_tree(['a'], ['a/b.py']) + assert exclude_paths( + base, + ['*', '!*/b.py'] + ) == convert_paths(['a/b.py']) + + def test_last_line_precedence(self): + base = make_tree( + [], + ['garbage.md', + 'thrash.md', + 'README.md', + 'README-bis.md', + 'README-secret.md']) + assert exclude_paths( + base, + ['*.md', '!README*.md', 'README-secret.md'] + ) == set(['README.md', 'README-bis.md']) + class TarTest(unittest.TestCase): def test_tar_with_excludes(self): @@ -1019,69 +1038,6 @@ def tar_test_negative_mtime_bug(self): assert tar_data.getmember('th.txt').mtime == -3600 -class ShouldCheckDirectoryTest(unittest.TestCase): - exclude_patterns = [ - 'exclude_rather_large_directory', - 'dir/with/subdir_excluded', - 'dir/with/exceptions' - ] - - include_patterns = [ - 'dir/with/exceptions/like_this_one', - 'dir/with/exceptions/in/descendents' - ] - - def test_should_check_directory_not_excluded(self): - assert should_check_directory( - 'not_excluded', self.exclude_patterns, self.include_patterns - ) - assert should_check_directory( - convert_path('dir/with'), self.exclude_patterns, - self.include_patterns - ) - - def test_shoud_check_parent_directories_of_excluded(self): - assert should_check_directory( - 'dir', self.exclude_patterns, self.include_patterns - ) - assert should_check_directory( - convert_path('dir/with'), self.exclude_patterns, - self.include_patterns - ) - - def test_should_not_check_excluded_directories_with_no_exceptions(self): - assert not should_check_directory( - 'exclude_rather_large_directory', self.exclude_patterns, - self.include_patterns - ) - assert not should_check_directory( - convert_path('dir/with/subdir_excluded'), self.exclude_patterns, - self.include_patterns - ) - - def test_should_check_excluded_directory_with_exceptions(self): - assert should_check_directory( - convert_path('dir/with/exceptions'), self.exclude_patterns, - self.include_patterns - ) - assert should_check_directory( - convert_path('dir/with/exceptions/in'), self.exclude_patterns, - self.include_patterns - ) - - def test_should_not_check_siblings_of_exceptions(self): - assert not should_check_directory( - convert_path('dir/with/exceptions/but_not_here'), - self.exclude_patterns, self.include_patterns - ) - - def test_should_check_subdirectories_of_exceptions(self): - assert should_check_directory( - convert_path('dir/with/exceptions/like_this_one/subdir'), - self.exclude_patterns, self.include_patterns - ) - - class FormatEnvironmentTest(unittest.TestCase): def test_format_env_binary_unicode_value(self): env_dict = { From 3b464f983e77cf85d9bc91e6f725cd7773bc0871 Mon Sep 17 00:00:00 2001 From: mefyl Date: Mon, 19 Feb 2018 12:48:17 +0100 Subject: [PATCH 0604/1301] Skip entirely excluded directories when handling dockerignore. This is pure optimization to not recurse into directories when there are no chances any file will be included. Signed-off-by: mefyl --- docker/utils/build.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docker/utils/build.py b/docker/utils/build.py index bfdb87b2f1..09b2071706 100644 --- a/docker/utils/build.py +++ b/docker/utils/build.py @@ -91,6 +91,10 @@ def match(p): matched = default if hit is None else hit sub = list(filter(lambda p: p[1], sub)) if os.path.isdir(cur): + # Entirely skip directories if there are no chance any subfile will + # be included. + if all(not p[0] for p in sub) and not matched: + continue children = False for r in (os.path.join(f, p) for p in walk(cur, sub, matched)): yield r From 0c948c7df65b0d7378c3c0c8d966c38171f1ef21 Mon Sep 17 00:00:00 2001 From: mefyl Date: Mon, 19 Feb 2018 12:54:04 +0100 Subject: [PATCH 0605/1301] Add note about potential dockerignore optimization. Signed-off-by: mefyl --- docker/utils/build.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docker/utils/build.py b/docker/utils/build.py index 09b2071706..1da56fbcc4 100644 --- a/docker/utils/build.py +++ b/docker/utils/build.py @@ -95,6 +95,15 @@ def match(p): # be included. if all(not p[0] for p in sub) and not matched: continue + # I think this would greatly speed up dockerignore handling by not + # recursing into directories we are sure would be entirely + # included, and only yielding the directory itself, which will be + # recursively archived anyway. However the current unit test expect + # the full list of subfiles and I'm not 100% sure it would make no + # difference yet. + # if all(p[0] for p in sub) and matched: + # yield f + # continue children = False for r in (os.path.join(f, p) for p in walk(cur, sub, matched)): yield r From e54e8f41993e5fd6378b15c5ab9a3d43615b8618 Mon Sep 17 00:00:00 2001 From: Viktor Adam Date: Wed, 21 Feb 2018 19:55:17 +0000 Subject: [PATCH 0606/1301] Shorthand method for service.force_update() Signed-off-by: Viktor Adam --- docker/models/services.py | 15 +++++++++ tests/integration/models_services_test.py | 41 +++++++++++++++++++++-- 2 files changed, 54 insertions(+), 2 deletions(-) diff --git a/docker/models/services.py b/docker/models/services.py index 8a633dfa01..125896bab9 100644 --- a/docker/models/services.py +++ b/docker/models/services.py @@ -69,6 +69,11 @@ def update(self, **kwargs): spec = self.attrs['Spec']['TaskTemplate']['ContainerSpec'] kwargs['image'] = spec['Image'] + if kwargs.get('force_update') is True: + task_template = self.attrs['Spec']['TaskTemplate'] + current_value = int(task_template.get('ForceUpdate', 0)) + kwargs['force_update'] = current_value + 1 + create_kwargs = _get_create_service_kwargs('update', kwargs) return self.client.api.update_service( @@ -124,6 +129,16 @@ def scale(self, replicas): service_mode, fetch_current_spec=True) + def force_update(self): + """ + Force update the service even if no changes require it. + + Returns: + ``True``if successful. + """ + + return self.update(force_update=True, fetch_current_spec=True) + class ServiceCollection(Collection): """Services on the Docker server.""" diff --git a/tests/integration/models_services_test.py b/tests/integration/models_services_test.py index cb8eca29be..36caa8513a 100644 --- a/tests/integration/models_services_test.py +++ b/tests/integration/models_services_test.py @@ -276,7 +276,7 @@ def test_scale_method_global_service(self): assert spec.get('Command') == ['sleep', '300'] @helpers.requires_api_version('1.25') - def test_restart_service(self): + def test_force_update_service(self): client = docker.from_env(version=TEST_API_VERSION) service = client.services.create( # create arguments @@ -286,7 +286,7 @@ def test_restart_service(self): command="sleep 300" ) initial_version = service.version - service.update( + assert service.update( # create argument name=service.name, # task template argument @@ -296,3 +296,40 @@ def test_restart_service(self): ) service.reload() assert service.version > initial_version + + @helpers.requires_api_version('1.25') + def test_force_update_service_using_bool(self): + client = docker.from_env(version=TEST_API_VERSION) + service = client.services.create( + # create arguments + name=helpers.random_name(), + # ContainerSpec arguments + image="alpine", + command="sleep 300" + ) + initial_version = service.version + assert service.update( + # create argument + name=service.name, + # task template argument + force_update=True, + # ContainerSpec argument + command="sleep 600" + ) + service.reload() + assert service.version > initial_version + + @helpers.requires_api_version('1.25') + def test_force_update_service_using_shorthand_method(self): + client = docker.from_env(version=TEST_API_VERSION) + service = client.services.create( + # create arguments + name=helpers.random_name(), + # ContainerSpec arguments + image="alpine", + command="sleep 300" + ) + initial_version = service.version + assert service.force_update() + service.reload() + assert service.version > initial_version From 1d85818f4caeba437d4200a6b8f870beb28ca5a2 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 22 Feb 2018 13:52:44 -0800 Subject: [PATCH 0607/1301] Bump 3.1.0 Signed-off-by: Joffrey F --- docker/version.py | 2 +- docs/change-log.md | 30 ++++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index fc56840619..c79cf93da0 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "3.1.0-dev" +version = "3.1.0" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) diff --git a/docs/change-log.md b/docs/change-log.md index 8ae88ef2aa..ceab083ea0 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -1,6 +1,36 @@ Change log ========== +3.1.0 +----- + +[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/44?closed=1) + +### Features + +* Added support for `device_cgroup_rules` in host config +* Added support for `generic_resources` when creating a `Resources` + object. +* Added support for a configurable `chunk_size` parameter in `export`, + `get_archive` and `get_image` (`Image.save`) +* Added a `force_update` method to the `Service` class. +* In `Service.update`, when the `force_update` parameter is set to `True`, + the current `force_update` counter is incremented by one in the update + request. + +### Bugfixes + +* Fixed a bug where authentication through `login()` was being ignored if the + SDK was configured to use a credential store. +* Fixed a bug where download methods would use an absurdly small chunk size, + leading to slow data retrieval +* Fixed a bug where using `DockerClient.images.pull` to pull an image by digest + would lead to an exception being raised. +* `.dockerignore` rules should now be respected as defined by the spec, + including respect for last-line precedence and proper handling of absolute + paths +* The `pass` credential store is now properly supported. + 3.0.1 ----- From d41e06092dd228c2409c61b0d6ed60d2b83eb5c5 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 22 Feb 2018 14:59:19 -0800 Subject: [PATCH 0608/1301] 3.2.0-dev Signed-off-by: Joffrey F --- docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index c79cf93da0..3429f28411 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "3.1.0" +version = "3.2.0-dev" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) From ab1f90a379bfb781c821ba8210e76bfa5551ea60 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Fri, 23 Feb 2018 02:22:19 +0000 Subject: [PATCH 0609/1301] Cleanup containers during the tests This fix tries to clean up the containers during the tests so that no pre-existing volumes left in docker integration tests. This fix adds `-v` when removing containers, and makes sure containers launched in non-daemon mode are removed. This fix is realted to moby PR 36292 Signed-off-by: Yong Tang --- tests/integration/base.py | 2 +- tests/integration/models_containers_test.py | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/integration/base.py b/tests/integration/base.py index c22126d5f4..56c23ed4af 100644 --- a/tests/integration/base.py +++ b/tests/integration/base.py @@ -36,7 +36,7 @@ def tearDown(self): pass for container in self.tmp_containers: try: - client.api.remove_container(container, force=True) + client.api.remove_container(container, force=True, v=True) except docker.errors.APIError: pass for network in self.tmp_networks: diff --git a/tests/integration/models_containers_test.py b/tests/integration/models_containers_test.py index f9f59c43b4..fac4de2b68 100644 --- a/tests/integration/models_containers_test.py +++ b/tests/integration/models_containers_test.py @@ -47,10 +47,13 @@ def test_run_with_volume(self): self.tmp_containers.append(container.id) container.wait() + name = "container_volume_test" out = client.containers.run( "alpine", "cat /insidecontainer/test", - volumes=["%s:/insidecontainer" % path] + volumes=["%s:/insidecontainer" % path], + name=name ) + self.tmp_containers.append(name) assert out == b'hello\n' def test_run_with_named_volume(self): @@ -66,10 +69,13 @@ def test_run_with_named_volume(self): self.tmp_containers.append(container.id) container.wait() + name = "container_volume_test" out = client.containers.run( "alpine", "cat /insidecontainer/test", - volumes=["somevolume:/insidecontainer"] + volumes=["somevolume:/insidecontainer"], + name=name ) + self.tmp_containers.append(name) assert out == b'hello\n' def test_run_with_network(self): From 429591910359dd7487dc111298eaf1f36121631d Mon Sep 17 00:00:00 2001 From: mefyl Date: Mon, 26 Feb 2018 12:17:34 +0100 Subject: [PATCH 0610/1301] Add test for "/.." patterns in .dockerignore. Signed-off-by: mefyl --- tests/unit/utils_test.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index 8a4b1937e6..c2dd502b7e 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -902,6 +902,22 @@ def test_last_line_precedence(self): ['*.md', '!README*.md', 'README-secret.md'] ) == set(['README.md', 'README-bis.md']) + def test_parent_directory(self): + base = make_tree( + [], + ['a.py', + 'b.py', + 'c.py']) + # Dockerignore reference stipulates that absolute paths are + # equivalent to relative paths, hence /../foo should be + # equivalent to ../foo. It also stipulates that paths are run + # through Go's filepath.Clean, which explicitely "replace + # "/.." by "/" at the beginning of a path". + assert exclude_paths( + base, + ['../a.py', '/../b.py'] + ) == set(['c.py']) + class TarTest(unittest.TestCase): def test_tar_with_excludes(self): From 15c26e7057b6b7a95297c3324ddf5cbe7dad4353 Mon Sep 17 00:00:00 2001 From: Matthieu Nottale Date: Mon, 26 Feb 2018 14:37:27 +0100 Subject: [PATCH 0611/1301] Workaround requests resolving our unix socket URL on macosx. Signed-off-by: Matthieu Nottale --- docker/api/client.py | 4 +++- tests/unit/fake_api.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docker/api/client.py b/docker/api/client.py index bddab61f31..13c292a0a5 100644 --- a/docker/api/client.py +++ b/docker/api/client.py @@ -119,7 +119,9 @@ def __init__(self, base_url=None, version=None, ) self.mount('http+docker://', self._custom_adapter) self._unmount('http://', 'https://') - self.base_url = 'http+docker://localunixsocket' + # host part of URL should be unused, but is resolved by requests + # module in proxy_bypass_macosx_sysconf() + self.base_url = 'http+docker://localhost' elif base_url.startswith('npipe://'): if not IS_WINDOWS_PLATFORM: raise DockerException( diff --git a/tests/unit/fake_api.py b/tests/unit/fake_api.py index 63d73317ca..e609b64edd 100644 --- a/tests/unit/fake_api.py +++ b/tests/unit/fake_api.py @@ -512,7 +512,7 @@ def post_fake_network_disconnect(): # Maps real api url to fake response callback -prefix = 'http+docker://localunixsocket' +prefix = 'http+docker://localhost' if constants.IS_WINDOWS_PLATFORM: prefix = 'http+docker://localnpipe' From ab21746d8f988af474f251792a39fcccd98839cc Mon Sep 17 00:00:00 2001 From: Ronald van Zantvoort Date: Sat, 24 Feb 2018 13:07:29 +0100 Subject: [PATCH 0612/1301] build_prune Signed-off-by: Ronald van Zantvoort --- docker/api/build.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docker/api/build.py b/docker/api/build.py index 56f1fcfc73..62b92c9465 100644 --- a/docker/api/build.py +++ b/docker/api/build.py @@ -292,3 +292,19 @@ def _set_auth_headers(self, headers): ) else: log.debug('No auth config found') + + @utils.minimum_version('1.31') + def prune_build(self): + """ + Delete builder cache + + Returns: + (dict): A dict containing + the amount of disk space reclaimed in bytes. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + url = self._url("/build/prune") + return self._result(self._post(url), True) From 7a28ff351018eae9484451798c22db3c190547d0 Mon Sep 17 00:00:00 2001 From: Wanzhi Du Date: Mon, 5 Mar 2018 18:01:22 +0800 Subject: [PATCH 0613/1301] Ignore comment line from the .dockerignore file This fixed the bug that test comment line in .dockerignore file as ignore rule bug. Add test for "# comment" patterns in .dockerignore. Signed-off-by: Wanzhi Du --- docker/api/build.py | 2 +- tests/integration/api_build_test.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docker/api/build.py b/docker/api/build.py index 56f1fcfc73..6dab14dca4 100644 --- a/docker/api/build.py +++ b/docker/api/build.py @@ -143,7 +143,7 @@ def build(self, path=None, tag=None, quiet=False, fileobj=None, if os.path.exists(dockerignore): with open(dockerignore, 'r') as f: exclude = list(filter( - bool, [l.strip() for l in f.read().splitlines()] + lambda x: x != '' and x[0] != '#', [l.strip() for l in f.read().splitlines()] )) context = utils.tar( path, exclude=exclude, dockerfile=dockerfile, gzip=gzip diff --git a/tests/integration/api_build_test.py b/tests/integration/api_build_test.py index 4c2b992033..a8c0279654 100644 --- a/tests/integration/api_build_test.py +++ b/tests/integration/api_build_test.py @@ -61,7 +61,8 @@ def test_build_with_dockerignore(self): 'Dockerfile', '.dockerignore', '!ignored/subdir/excepted-file', - '', # empty line + '', # empty line, + '#', # comment line ])) with open(os.path.join(base_dir, 'not-ignored'), 'w') as f: From 74586cdd4c661ecc4a3e3bd64fafedde7f121d5e Mon Sep 17 00:00:00 2001 From: Wanzhi Du Date: Mon, 5 Mar 2018 19:26:56 +0800 Subject: [PATCH 0614/1301] Fix flake8 case Signed-off-by: Wanzhi Du --- docker/api/build.py | 3 ++- tests/integration/api_build_test.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docker/api/build.py b/docker/api/build.py index 6dab14dca4..e136a6eedb 100644 --- a/docker/api/build.py +++ b/docker/api/build.py @@ -143,7 +143,8 @@ def build(self, path=None, tag=None, quiet=False, fileobj=None, if os.path.exists(dockerignore): with open(dockerignore, 'r') as f: exclude = list(filter( - lambda x: x != '' and x[0] != '#', [l.strip() for l in f.read().splitlines()] + lambda x: x != '' and x[0] != '#', + [l.strip() for l in f.read().splitlines()] )) context = utils.tar( path, exclude=exclude, dockerfile=dockerfile, gzip=gzip diff --git a/tests/integration/api_build_test.py b/tests/integration/api_build_test.py index a8c0279654..ad5fbaa85f 100644 --- a/tests/integration/api_build_test.py +++ b/tests/integration/api_build_test.py @@ -62,7 +62,7 @@ def test_build_with_dockerignore(self): '.dockerignore', '!ignored/subdir/excepted-file', '', # empty line, - '#', # comment line + '#', # comment line ])) with open(os.path.join(base_dir, 'not-ignored'), 'w') as f: From 13609359acfc33c42bda35ad240c4a8d96df13d4 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 5 Mar 2018 11:49:43 -0800 Subject: [PATCH 0615/1301] Improve dockerignore comment test Signed-off-by: Joffrey F --- tests/integration/api_build_test.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/integration/api_build_test.py b/tests/integration/api_build_test.py index ad5fbaa85f..ce587d54c1 100644 --- a/tests/integration/api_build_test.py +++ b/tests/integration/api_build_test.py @@ -62,12 +62,15 @@ def test_build_with_dockerignore(self): '.dockerignore', '!ignored/subdir/excepted-file', '', # empty line, - '#', # comment line + '#*', # comment line ])) with open(os.path.join(base_dir, 'not-ignored'), 'w') as f: f.write("this file should not be ignored") + with open(os.path.join(base_dir, '#file.txt'), 'w') as f: + f.write('this file should not be ignored') + subdir = os.path.join(base_dir, 'ignored', 'subdir') os.makedirs(subdir) with open(os.path.join(subdir, 'file'), 'w') as f: @@ -93,6 +96,7 @@ def test_build_with_dockerignore(self): logs = logs.decode('utf-8') assert sorted(list(filter(None, logs.split('\n')))) == sorted([ + '/test/#file.txt', '/test/ignored/subdir/excepted-file', '/test/not-ignored' ]) From 52c3d528f64dbc9bd155ae9bc74b454f842761c5 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 5 Mar 2018 15:15:37 -0800 Subject: [PATCH 0616/1301] Bump 3.1.1 Signed-off-by: Joffrey F --- docker/version.py | 2 +- docs/change-log.md | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index 3429f28411..fedd9ed7b0 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "3.2.0-dev" +version = "3.1.1" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) diff --git a/docs/change-log.md b/docs/change-log.md index ceab083ea0..94c325d6f5 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -1,6 +1,18 @@ Change log ========== +3.1.1 +----- + +[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/46?closed=1) + +### Bugfixes + +* Fixed a bug that caused costly DNS lookups on Mac OSX when connecting to the + engine through UNIX socket +* Fixed a bug that caused `.dockerignore` comments to be read as exclusion + patterns + 3.1.0 ----- From 110672d1a8cb5b5f418e4fb7fd7cdff775191122 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 5 Mar 2018 15:26:59 -0800 Subject: [PATCH 0617/1301] Bump test engine versions Signed-off-by: Joffrey F --- Jenkinsfile | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 6d9d3436f2..c548492b40 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -5,7 +5,12 @@ def imageNamePy2 def imageNamePy3 def images = [:] -def dockerVersions = ["17.06.2-ce", "17.12.0-ce", "18.01.0-ce"] +def dockerVersions = [ + "17.06.2-ce", // Latest EE + "17.12.1-ce", // Latest CE stable + "18.02.0-ce", // Latest CE edge + "18.03.0-ce-rc1" // Latest CE RC +] def buildImage = { name, buildargs, pyTag -> img = docker.image(name) @@ -33,7 +38,7 @@ def buildImages = { -> } def getAPIVersion = { engineVersion -> - def versionMap = ['17.06': '1.30', '17.12': '1.35', '18.01': '1.35'] + def versionMap = ['17.06': '1.30', '17.12': '1.35', '18.02': '1.36', '18.03': '1.37'] return versionMap[engineVersion.substring(0, 5)] } From b75799d33a053b6dc99e45721d280e99b21436a6 Mon Sep 17 00:00:00 2001 From: Matthieu Nottale Date: Wed, 14 Mar 2018 14:14:20 +0100 Subject: [PATCH 0618/1301] Add close() method to DockerClient. Signed-off-by: Matthieu Nottale --- docker/client.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docker/client.py b/docker/client.py index 467583e639..b4364c3c07 100644 --- a/docker/client.py +++ b/docker/client.py @@ -186,6 +186,10 @@ def version(self, *args, **kwargs): return self.api.version(*args, **kwargs) version.__doc__ = APIClient.version.__doc__ + def close(self): + return self.api.close() + close.__doc__ = APIClient.close.__doc__ + def __getattr__(self, name): s = ["'DockerClient' object has no attribute '{}'".format(name)] # If a user calls a method on APIClient, they From 1829bd26991a179bfec70d0fe6c28c406fe0c7ee Mon Sep 17 00:00:00 2001 From: Matthieu Nottale Date: Wed, 14 Mar 2018 15:30:30 +0100 Subject: [PATCH 0619/1301] Add sparse argument to DockerClient.containers.list(). Signed-off-by: Matthieu Nottale --- docker/models/containers.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/docker/models/containers.py b/docker/models/containers.py index 895080cab9..d4ed1aa3f4 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -833,7 +833,8 @@ def get(self, container_id): resp = self.client.api.inspect_container(container_id) return self.prepare_model(resp) - def list(self, all=False, before=None, filters=None, limit=-1, since=None): + def list(self, all=False, before=None, filters=None, limit=-1, since=None, + sparse=False): """ List containers. Similar to the ``docker ps`` command. @@ -862,6 +863,9 @@ def list(self, all=False, before=None, filters=None, limit=-1, since=None): container. Give the container name or id. - `since` (str): Only containers created after a particular container. Give container name or id. + sparse (bool): Do not inspect containers. Returns partial + informations, but guaranteed not to block. Use reload() on + each container to get the full list of attributes. A comprehensive list can be found in the documentation for `docker ps @@ -877,7 +881,10 @@ def list(self, all=False, before=None, filters=None, limit=-1, since=None): resp = self.client.api.containers(all=all, before=before, filters=filters, limit=limit, since=since) - return [self.get(r['Id']) for r in resp] + if sparse: + return [self.prepare_model(r) for r in resp] + else: + return [self.get(r['Id']) for r in resp] def prune(self, filters=None): return self.client.api.prune_containers(filters=filters) From 33f1ca9a48dc79661a774fe6ac79b3feba39ed0e Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 14 Mar 2018 14:11:42 -0700 Subject: [PATCH 0620/1301] Use same split rules for Dockerfile as other include/exclude patterns Signed-off-by: Joffrey F --- docker/utils/build.py | 7 +++++-- tests/unit/utils_test.py | 5 +++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/docker/utils/build.py b/docker/utils/build.py index 1da56fbcc4..1622ec356a 100644 --- a/docker/utils/build.py +++ b/docker/utils/build.py @@ -31,18 +31,21 @@ def exclude_paths(root, patterns, dockerfile=None): if dockerfile is None: dockerfile = 'Dockerfile' + def split_path(p): + return [pt for pt in re.split(_SEP, p) if pt and pt != '.'] + def normalize(p): # Leading and trailing slashes are not relevant. Yes, # "foo.py/" must exclude the "foo.py" regular file. "." # components are not relevant either, even if the whole # pattern is only ".", as the Docker reference states: "For # historical reasons, the pattern . is ignored." - split = [pt for pt in re.split(_SEP, p) if pt and pt != '.'] # ".." component must be cleared with the potential previous # component, regardless of whether it exists: "A preprocessing # step [...] eliminates . and .. elements using Go's # filepath.". i = 0 + split = split_path(p) while i < len(split): if split[i] == '..': del split[i] @@ -62,7 +65,7 @@ def normalize(p): # Exclude empty patterns such as "." or the empty string. filter(lambda p: p[1], patterns), # Always include the Dockerfile and .dockerignore - [(True, dockerfile.split('/')), (True, ['.dockerignore'])])))) + [(True, split_path(dockerfile)), (True, ['.dockerignore'])])))) return set(walk(root, patterns)) diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index c2dd502b7e..56800f9e0c 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -698,6 +698,11 @@ def test_exclude_custom_dockerfile(self): ['*'], dockerfile='foo/Dockerfile3' ) == convert_paths(set(['foo/Dockerfile3', '.dockerignore'])) + # https://github.com/docker/docker-py/issues/1956 + assert self.exclude( + ['*'], dockerfile='./foo/Dockerfile3' + ) == convert_paths(set(['foo/Dockerfile3', '.dockerignore'])) + def test_exclude_dockerfile_child(self): includes = self.exclude(['foo/'], dockerfile='foo/Dockerfile3') assert convert_path('foo/Dockerfile3') in includes From 90c0dbe5f8df7ef6d8fd0ccc581fd64cc2ecd1ab Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 14 Mar 2018 16:53:56 -0700 Subject: [PATCH 0621/1301] Add test for container list with sparse=True Signed-off-by: Joffrey F --- docker/models/containers.py | 30 ++++++++++++++------- tests/integration/models_containers_test.py | 22 +++++++++++++++ 2 files changed, 43 insertions(+), 9 deletions(-) diff --git a/docker/models/containers.py b/docker/models/containers.py index d4ed1aa3f4..1e06ed6099 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -4,8 +4,10 @@ from ..api import APIClient from ..constants import DEFAULT_DATA_CHUNK_SIZE -from ..errors import (ContainerError, ImageNotFound, - create_unexpected_kwargs_error) +from ..errors import ( + ContainerError, DockerException, ImageNotFound, + create_unexpected_kwargs_error +) from ..types import HostConfig from ..utils import version_gte from .images import Image @@ -27,7 +29,7 @@ def image(self): """ The image of the container. """ - image_id = self.attrs['Image'] + image_id = self.attrs.get('ImageID', self.attrs['Image']) if image_id is None: return None return self.client.images.get(image_id.split(':')[1]) @@ -37,15 +39,23 @@ def labels(self): """ The labels of a container as dictionary. """ - result = self.attrs['Config'].get('Labels') - return result or {} + try: + result = self.attrs['Config'].get('Labels') + return result or {} + except KeyError: + raise DockerException( + 'Label data is not available for sparse objects. Call reload()' + ' to retrieve all information' + ) @property def status(self): """ The status of the container. For example, ``running``, or ``exited``. """ - return self.attrs['State']['Status'] + if isinstance(self.attrs['State'], dict): + return self.attrs['State']['Status'] + return self.attrs['State'] def attach(self, **kwargs): """ @@ -863,14 +873,16 @@ def list(self, all=False, before=None, filters=None, limit=-1, since=None, container. Give the container name or id. - `since` (str): Only containers created after a particular container. Give container name or id. - sparse (bool): Do not inspect containers. Returns partial - informations, but guaranteed not to block. Use reload() on - each container to get the full list of attributes. A comprehensive list can be found in the documentation for `docker ps `_. + sparse (bool): Do not inspect containers. Returns partial + information, but guaranteed not to block. Use + :py:meth:`Container.reload` on resulting objects to retrieve + all attributes. Default: ``False`` + Returns: (list of :py:class:`Container`) diff --git a/tests/integration/models_containers_test.py b/tests/integration/models_containers_test.py index fac4de2b68..38aae4d2e5 100644 --- a/tests/integration/models_containers_test.py +++ b/tests/integration/models_containers_test.py @@ -159,6 +159,28 @@ def test_list(self): container = containers[0] assert container.attrs['Config']['Image'] == 'alpine' + assert container.status == 'running' + assert container.image == client.images.get('alpine') + + container.kill() + container.remove() + assert container_id not in [c.id for c in client.containers.list()] + + def test_list_sparse(self): + client = docker.from_env(version=TEST_API_VERSION) + container_id = client.containers.run( + "alpine", "sleep 300", detach=True).id + self.tmp_containers.append(container_id) + containers = [c for c in client.containers.list(sparse=True) if c.id == + container_id] + assert len(containers) == 1 + + container = containers[0] + assert container.attrs['Image'] == 'alpine' + assert container.status == 'running' + assert container.image == client.images.get('alpine') + with pytest.raises(docker.errors.DockerException): + container.labels container.kill() container.remove() From 16ccf377a38548ecdbfe8e2317e2ddcc599c6aea Mon Sep 17 00:00:00 2001 From: James Date: Thu, 15 Mar 2018 15:01:13 +0100 Subject: [PATCH 0622/1301] Updates docs for rename of `name` to `repository` Signed-off-by: James Meakin --- docker/models/images.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/models/images.py b/docker/models/images.py index 58d5d93c47..d4c281322f 100644 --- a/docker/models/images.py +++ b/docker/models/images.py @@ -284,7 +284,7 @@ def pull(self, repository, tag=None, **kwargs): low-level API. Args: - name (str): The repository to pull + repository (str): The repository to pull tag (str): The tag to pull auth_config (dict): Override the credentials that :py:meth:`~docker.client.DockerClient.login` has set for From 884261e24103f6732d4f529c19e6f7b56ccf199c Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 15 Mar 2018 14:36:39 -0700 Subject: [PATCH 0623/1301] Fix socket tests for TLS-enabled tests Signed-off-by: Joffrey F --- tests/helpers.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/helpers.py b/tests/helpers.py index c4ea3647dd..b6b493b385 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -108,21 +108,21 @@ def swarm_listen_addr(): def assert_cat_socket_detached_with_keys(sock, inputs): - if six.PY3: + if six.PY3 and hasattr(sock, '_sock'): sock = sock._sock for i in inputs: - sock.send(i) + sock.sendall(i) time.sleep(0.5) # If we're using a Unix socket, the sock.send call will fail with a # BrokenPipeError ; INET sockets will just stop receiving / sending data # but will not raise an error - if sock.family == getattr(socket, 'AF_UNIX', -1): + if getattr(sock, 'family', -9) == getattr(socket, 'AF_UNIX', -1): with pytest.raises(socket.error): - sock.send(b'make sure the socket is closed\n') + sock.sendall(b'make sure the socket is closed\n') else: - sock.send(b"make sure the socket is closed\n") + sock.sendall(b"make sure the socket is closed\n") assert sock.recv(32) == b'' From a4e642b015c50d9c628413341ed00c89599f66be Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 15 Mar 2018 14:37:02 -0700 Subject: [PATCH 0624/1301] Use networks instead of legacy links for test setup Signed-off-by: Joffrey F --- Jenkinsfile | 14 +++++++++----- Makefile | 36 +++++++++++++++++++++--------------- 2 files changed, 30 insertions(+), 20 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index c548492b40..1323f4b827 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -9,7 +9,7 @@ def dockerVersions = [ "17.06.2-ce", // Latest EE "17.12.1-ce", // Latest CE stable "18.02.0-ce", // Latest CE edge - "18.03.0-ce-rc1" // Latest CE RC + "18.03.0-ce-rc4" // Latest CE RC ] def buildImage = { name, buildargs, pyTag -> @@ -64,15 +64,18 @@ def runTests = { Map settings -> checkout(scm) def dindContainerName = "dpy-dind-\$BUILD_NUMBER-\$EXECUTOR_NUMBER-${pythonVersion}-${dockerVersion}" def testContainerName = "dpy-tests-\$BUILD_NUMBER-\$EXECUTOR_NUMBER-${pythonVersion}-${dockerVersion}" + def testNetwork = "dpy-testnet-\$BUILD_NUMBER-\$EXECUTOR_NUMBER-${pythonVersion}-${dockerVersion}" try { - sh """docker run -d --name ${dindContainerName} -v /tmp --privileged \\ + sh """docker network create ${testNetwork}""" + sh """docker run -d --name ${dindContainerName} -v /tmp --privileged --network ${testNetwork} \\ dockerswarm/dind:${dockerVersion} dockerd -H tcp://0.0.0.0:2375 """ sh """docker run \\ - --name ${testContainerName} --volumes-from ${dindContainerName} \\ - -e 'DOCKER_HOST=tcp://docker:2375' \\ + --name ${testContainerName} \\ + -e "DOCKER_HOST=tcp://${dindContainerName}:2375" \\ -e 'DOCKER_TEST_API_VERSION=${apiVersion}' \\ - --link=${dindContainerName}:docker \\ + --network ${testNetwork} \\ + --volumes-from ${dindContainerName} \\ ${testImage} \\ py.test -v -rxs tests/integration """ @@ -80,6 +83,7 @@ def runTests = { Map settings -> sh """ docker stop ${dindContainerName} ${testContainerName} docker rm -vf ${dindContainerName} ${testContainerName} + docker network rm ${testNetwork} """ } } diff --git a/Makefile b/Makefile index f491993926..434d40e1cc 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ all: test .PHONY: clean clean: - -docker rm -f dpy-dind-py2 dpy-dind-py3 + -docker rm -f dpy-dind-py2 dpy-dind-py3 dpy-dind-certs dpy-dind-ssl find -name "__pycache__" | xargs rm -rf .PHONY: build @@ -44,41 +44,47 @@ integration-test-py3: build-py3 TEST_API_VERSION ?= 1.35 TEST_ENGINE_VERSION ?= 17.12.0-ce +.PHONY: setup-network +setup-network: + docker network inspect dpy-tests || docker network create dpy-tests + .PHONY: integration-dind integration-dind: integration-dind-py2 integration-dind-py3 .PHONY: integration-dind-py2 -integration-dind-py2: build +integration-dind-py2: build setup-network docker rm -vf dpy-dind-py2 || : - docker run -d --name dpy-dind-py2 --privileged dockerswarm/dind:${TEST_ENGINE_VERSION} dockerd\ - -H tcp://0.0.0.0:2375 --experimental - docker run -t --rm --env="DOCKER_HOST=tcp://docker:2375" --env="DOCKER_TEST_API_VERSION=${TEST_API_VERSION}"\ - --link=dpy-dind-py2:docker docker-sdk-python py.test tests/integration + docker run -d --network dpy-tests --name dpy-dind-py2 --privileged\ + dockerswarm/dind:${TEST_ENGINE_VERSION} dockerd -H tcp://0.0.0.0:2375 --experimental + docker run -t --rm --env="DOCKER_HOST=tcp://dpy-dind-py2:2375" --env="DOCKER_TEST_API_VERSION=${TEST_API_VERSION}"\ + --network dpy-tests docker-sdk-python py.test tests/integration docker rm -vf dpy-dind-py2 .PHONY: integration-dind-py3 -integration-dind-py3: build-py3 +integration-dind-py3: build-py3 setup-network docker rm -vf dpy-dind-py3 || : - docker run -d --name dpy-dind-py3 --privileged dockerswarm/dind:${TEST_ENGINE_VERSION} dockerd\ - -H tcp://0.0.0.0:2375 --experimental - docker run -t --rm --env="DOCKER_HOST=tcp://docker:2375" --env="DOCKER_TEST_API_VERSION=${TEST_API_VERSION}"\ - --link=dpy-dind-py3:docker docker-sdk-python3 py.test tests/integration + docker run -d --network dpy-tests --name dpy-dind-py3 --privileged\ + dockerswarm/dind:${TEST_ENGINE_VERSION} dockerd -H tcp://0.0.0.0:2375 --experimental + docker run -t --rm --env="DOCKER_HOST=tcp://dpy-dind-py3:2375" --env="DOCKER_TEST_API_VERSION=${TEST_API_VERSION}"\ + --network dpy-tests docker-sdk-python3 py.test tests/integration docker rm -vf dpy-dind-py3 .PHONY: integration-dind-ssl integration-dind-ssl: build-dind-certs build build-py3 + docker rm -vf dpy-dind-certs dpy-dind-ssl || : docker run -d --name dpy-dind-certs dpy-dind-certs docker run -d --env="DOCKER_HOST=tcp://localhost:2375" --env="DOCKER_TLS_VERIFY=1"\ --env="DOCKER_CERT_PATH=/certs" --volumes-from dpy-dind-certs --name dpy-dind-ssl\ - -v /tmp --privileged dockerswarm/dind:${TEST_ENGINE_VERSION} dockerd --tlsverify\ - --tlscacert=/certs/ca.pem --tlscert=/certs/server-cert.pem\ + --network dpy-tests --network-alias docker -v /tmp --privileged\ + dockerswarm/dind:${TEST_ENGINE_VERSION}\ + dockerd --tlsverify --tlscacert=/certs/ca.pem --tlscert=/certs/server-cert.pem\ --tlskey=/certs/server-key.pem -H tcp://0.0.0.0:2375 --experimental docker run -t --rm --volumes-from dpy-dind-ssl --env="DOCKER_HOST=tcp://docker:2375"\ --env="DOCKER_TLS_VERIFY=1" --env="DOCKER_CERT_PATH=/certs" --env="DOCKER_TEST_API_VERSION=${TEST_API_VERSION}"\ - --link=dpy-dind-ssl:docker docker-sdk-python py.test tests/integration + --network dpy-tests docker-sdk-python py.test tests/integration docker run -t --rm --volumes-from dpy-dind-ssl --env="DOCKER_HOST=tcp://docker:2375"\ --env="DOCKER_TLS_VERIFY=1" --env="DOCKER_CERT_PATH=/certs" --env="DOCKER_TEST_API_VERSION=${TEST_API_VERSION}"\ - --link=dpy-dind-ssl:docker docker-sdk-python3 py.test tests/integration + --network dpy-tests docker-sdk-python3 py.test tests/integration docker rm -vf dpy-dind-ssl dpy-dind-certs .PHONY: flake8 From 719d4e9e2091edef8c084857051a751bb8f97ea2 Mon Sep 17 00:00:00 2001 From: Viktor Adam Date: Wed, 21 Feb 2018 22:16:21 +0000 Subject: [PATCH 0625/1301] Allow cancelling the streams from other threads Signed-off-by: Viktor Adam --- docker/api/container.py | 17 +++++- docker/api/daemon.py | 21 ++++--- docker/types/__init__.py | 1 + docker/types/daemon.py | 63 +++++++++++++++++++++ tests/integration/api_container_test.py | 48 ++++++++++++++++ tests/integration/client_test.py | 20 +++++++ tests/integration/models_containers_test.py | 21 +++++++ 7 files changed, 181 insertions(+), 10 deletions(-) create mode 100644 docker/types/daemon.py diff --git a/docker/api/container.py b/docker/api/container.py index f8d52de489..cb97b7940c 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -5,7 +5,8 @@ from .. import utils from ..constants import DEFAULT_DATA_CHUNK_SIZE from ..types import ( - ContainerConfig, EndpointConfig, HostConfig, NetworkingConfig + CancellableStream, ContainerConfig, EndpointConfig, HostConfig, + NetworkingConfig ) @@ -52,10 +53,15 @@ def attach(self, container, stdout=True, stderr=True, u = self._url("/containers/{0}/attach", container) response = self._post(u, headers=headers, params=params, stream=True) - return self._read_from_socket( + output = self._read_from_socket( response, stream, self._check_is_tty(container) ) + if stream: + return CancellableStream(output, response) + else: + return output + @utils.check_resource('container') def attach_socket(self, container, params=None, ws=False): """ @@ -815,7 +821,12 @@ def logs(self, container, stdout=True, stderr=True, stream=False, url = self._url("/containers/{0}/logs", container) res = self._get(url, params=params, stream=stream) - return self._get_result(container, stream, res) + output = self._get_result(container, stream, res) + + if stream: + return CancellableStream(output, res) + else: + return output @utils.check_resource('container') def pause(self, container): diff --git a/docker/api/daemon.py b/docker/api/daemon.py index 0e1c753814..fc3692c248 100644 --- a/docker/api/daemon.py +++ b/docker/api/daemon.py @@ -1,7 +1,7 @@ import os from datetime import datetime -from .. import auth, utils +from .. import auth, types, utils class DaemonApiMixin(object): @@ -34,8 +34,7 @@ def events(self, since=None, until=None, filters=None, decode=None): the fly. False by default. Returns: - (generator): A blocking generator you can iterate over to retrieve - events as they happen. + A :py:class:`docker.types.daemon.CancellableStream` generator Raises: :py:class:`docker.errors.APIError` @@ -50,6 +49,14 @@ def events(self, since=None, until=None, filters=None, decode=None): u'status': u'start', u'time': 1423339459} ... + + or + + >>> events = client.events() + >>> for event in events: + ... print event + >>> # and cancel from another thread + >>> events.close() """ if isinstance(since, datetime): @@ -68,10 +75,10 @@ def events(self, since=None, until=None, filters=None, decode=None): } url = self._url('/events') - return self._stream_helper( - self._get(url, params=params, stream=True, timeout=None), - decode=decode - ) + response = self._get(url, params=params, stream=True, timeout=None) + stream = self._stream_helper(response, decode=decode) + + return types.CancellableStream(stream, response) def info(self): """ diff --git a/docker/types/__init__.py b/docker/types/__init__.py index 39c93e344d..0b0d847fe9 100644 --- a/docker/types/__init__.py +++ b/docker/types/__init__.py @@ -1,5 +1,6 @@ # flake8: noqa from .containers import ContainerConfig, HostConfig, LogConfig, Ulimit +from .daemon import CancellableStream from .healthcheck import Healthcheck from .networks import EndpointConfig, IPAMConfig, IPAMPool, NetworkingConfig from .services import ( diff --git a/docker/types/daemon.py b/docker/types/daemon.py new file mode 100644 index 0000000000..ba0334d0cd --- /dev/null +++ b/docker/types/daemon.py @@ -0,0 +1,63 @@ +import socket + +try: + import requests.packages.urllib3 as urllib3 +except ImportError: + import urllib3 + + +class CancellableStream(object): + """ + Stream wrapper for real-time events, logs, etc. from the server. + + Example: + >>> events = client.events() + >>> for event in events: + ... print event + >>> # and cancel from another thread + >>> events.close() + """ + + def __init__(self, stream, response): + self._stream = stream + self._response = response + + def __iter__(self): + return self + + def __next__(self): + try: + return next(self._stream) + except urllib3.exceptions.ProtocolError: + raise StopIteration + except socket.error: + raise StopIteration + + next = __next__ + + def close(self): + """ + Closes the event streaming. + """ + + if not self._response.raw.closed: + # find the underlying socket object + # based on api.client._get_raw_response_socket + + sock_fp = self._response.raw._fp.fp + + if hasattr(sock_fp, 'raw'): + sock_raw = sock_fp.raw + + if hasattr(sock_raw, 'sock'): + sock = sock_raw.sock + + elif hasattr(sock_raw, '_sock'): + sock = sock_raw._sock + + else: + sock = sock_fp._sock + + sock.shutdown(socket.SHUT_RDWR) + sock.makefile().close() + sock.close() diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index 8447aa5f05..cc2c07198b 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -2,6 +2,7 @@ import re import signal import tempfile +import threading from datetime import datetime import docker @@ -880,6 +881,30 @@ def test_logs_streaming_and_follow(self): assert logs == (snippet + '\n').encode(encoding='ascii') + def test_logs_streaming_and_follow_and_cancel(self): + snippet = 'Flowering Nights (Sakuya Iyazoi)' + container = self.client.create_container( + BUSYBOX, 'sh -c "echo \\"{0}\\" && sleep 3"'.format(snippet) + ) + id = container['Id'] + self.tmp_containers.append(id) + self.client.start(id) + logs = six.binary_type() + + generator = self.client.logs(id, stream=True, follow=True) + + exit_timer = threading.Timer(3, os._exit, args=[1]) + exit_timer.start() + + threading.Timer(1, generator.close).start() + + for chunk in generator: + logs += chunk + + exit_timer.cancel() + + assert logs == (snippet + '\n').encode(encoding='ascii') + def test_logs_with_dict_instead_of_id(self): snippet = 'Flowering Nights (Sakuya Iyazoi)' container = self.client.create_container( @@ -1226,6 +1251,29 @@ def test_attach_no_stream(self): output = self.client.attach(container, stream=False, logs=True) assert output == 'hello\n'.encode(encoding='ascii') + def test_attach_stream_and_cancel(self): + container = self.client.create_container( + BUSYBOX, 'sh -c "echo hello && sleep 60"', + tty=True + ) + self.tmp_containers.append(container) + self.client.start(container) + output = self.client.attach(container, stream=True, logs=True) + + exit_timer = threading.Timer(3, os._exit, args=[1]) + exit_timer.start() + + threading.Timer(1, output.close).start() + + lines = [] + for line in output: + lines.append(line) + + exit_timer.cancel() + + assert len(lines) == 1 + assert lines[0] == 'hello\r\n'.encode(encoding='ascii') + def test_detach_with_default(self): container = self.client.create_container( BUSYBOX, 'cat', diff --git a/tests/integration/client_test.py b/tests/integration/client_test.py index 8f6bd86b84..7df172c885 100644 --- a/tests/integration/client_test.py +++ b/tests/integration/client_test.py @@ -1,7 +1,10 @@ +import threading import unittest import docker +from datetime import datetime, timedelta + from ..helpers import requires_api_version from .base import TEST_API_VERSION @@ -27,3 +30,20 @@ def test_df(self): assert 'Containers' in data assert 'Volumes' in data assert 'Images' in data + + +class CancellableEventsTest(unittest.TestCase): + client = docker.from_env(version=TEST_API_VERSION) + + def test_cancel_events(self): + start = datetime.now() + + events = self.client.events(until=start + timedelta(seconds=5)) + + cancel_thread = threading.Timer(2, events.close) + cancel_thread.start() + + for _ in events: + pass + + self.assertLess(datetime.now() - start, timedelta(seconds=3)) diff --git a/tests/integration/models_containers_test.py b/tests/integration/models_containers_test.py index 38aae4d2e5..41faff3580 100644 --- a/tests/integration/models_containers_test.py +++ b/tests/integration/models_containers_test.py @@ -1,4 +1,6 @@ +import os import tempfile +import threading import docker import pytest @@ -141,6 +143,25 @@ def test_run_with_streamed_logs(self): assert logs[0] == b'hello\n' assert logs[1] == b'world\n' + def test_run_with_streamed_logs_and_cancel(self): + client = docker.from_env(version=TEST_API_VERSION) + out = client.containers.run( + 'alpine', 'sh -c "echo hello && echo world"', stream=True + ) + + exit_timer = threading.Timer(3, os._exit, args=[1]) + exit_timer.start() + + threading.Timer(1, out.close).start() + + logs = [line for line in out] + + exit_timer.cancel() + + assert len(logs) == 2 + assert logs[0] == b'hello\n' + assert logs[1] == b'world\n' + def test_get(self): client = docker.from_env(version=TEST_API_VERSION) container = client.containers.run("alpine", "sleep 300", detach=True) From 284c3d90d6ab1c49410d5622ca8cd3f37dcbe296 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 19 Mar 2018 14:40:49 +0100 Subject: [PATCH 0626/1301] Remove redundant single-socket select call Clean up + use pytest-timeout Signed-off-by: Joffrey F --- docker/types/daemon.py | 1 - docker/utils/socket.py | 3 +-- test-requirements.txt | 5 +++-- tests/integration/api_container_test.py | 13 ++----------- tests/integration/models_containers_test.py | 7 +------ 5 files changed, 7 insertions(+), 22 deletions(-) diff --git a/docker/types/daemon.py b/docker/types/daemon.py index ba0334d0cd..852f3d8292 100644 --- a/docker/types/daemon.py +++ b/docker/types/daemon.py @@ -59,5 +59,4 @@ def close(self): sock = sock_fp._sock sock.shutdown(socket.SHUT_RDWR) - sock.makefile().close() sock.close() diff --git a/docker/utils/socket.py b/docker/utils/socket.py index c3a5f90fc3..0945f0a694 100644 --- a/docker/utils/socket.py +++ b/docker/utils/socket.py @@ -22,8 +22,7 @@ def read(socket, n=4096): recoverable_errors = (errno.EINTR, errno.EDEADLK, errno.EWOULDBLOCK) - # wait for data to become available - if not isinstance(socket, NpipeSocket): + if six.PY3 and not isinstance(socket, NpipeSocket): select.select([socket], [], []) try: diff --git a/test-requirements.txt b/test-requirements.txt index f79e815907..09680b6897 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,5 +1,6 @@ +coverage==3.7.1 +flake8==3.4.1 mock==1.0.1 pytest==2.9.1 -coverage==3.7.1 pytest-cov==2.1.0 -flake8==3.4.1 +pytest-timeout==1.2.1 diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index cc2c07198b..e2125186d1 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -881,6 +881,7 @@ def test_logs_streaming_and_follow(self): assert logs == (snippet + '\n').encode(encoding='ascii') + @pytest.mark.timeout(5) def test_logs_streaming_and_follow_and_cancel(self): snippet = 'Flowering Nights (Sakuya Iyazoi)' container = self.client.create_container( @@ -892,17 +893,11 @@ def test_logs_streaming_and_follow_and_cancel(self): logs = six.binary_type() generator = self.client.logs(id, stream=True, follow=True) - - exit_timer = threading.Timer(3, os._exit, args=[1]) - exit_timer.start() - threading.Timer(1, generator.close).start() for chunk in generator: logs += chunk - exit_timer.cancel() - assert logs == (snippet + '\n').encode(encoding='ascii') def test_logs_with_dict_instead_of_id(self): @@ -1251,6 +1246,7 @@ def test_attach_no_stream(self): output = self.client.attach(container, stream=False, logs=True) assert output == 'hello\n'.encode(encoding='ascii') + @pytest.mark.timeout(5) def test_attach_stream_and_cancel(self): container = self.client.create_container( BUSYBOX, 'sh -c "echo hello && sleep 60"', @@ -1260,17 +1256,12 @@ def test_attach_stream_and_cancel(self): self.client.start(container) output = self.client.attach(container, stream=True, logs=True) - exit_timer = threading.Timer(3, os._exit, args=[1]) - exit_timer.start() - threading.Timer(1, output.close).start() lines = [] for line in output: lines.append(line) - exit_timer.cancel() - assert len(lines) == 1 assert lines[0] == 'hello\r\n'.encode(encoding='ascii') diff --git a/tests/integration/models_containers_test.py b/tests/integration/models_containers_test.py index 41faff3580..6ddb034b41 100644 --- a/tests/integration/models_containers_test.py +++ b/tests/integration/models_containers_test.py @@ -1,4 +1,3 @@ -import os import tempfile import threading @@ -143,21 +142,17 @@ def test_run_with_streamed_logs(self): assert logs[0] == b'hello\n' assert logs[1] == b'world\n' + @pytest.mark.timeout(5) def test_run_with_streamed_logs_and_cancel(self): client = docker.from_env(version=TEST_API_VERSION) out = client.containers.run( 'alpine', 'sh -c "echo hello && echo world"', stream=True ) - exit_timer = threading.Timer(3, os._exit, args=[1]) - exit_timer.start() - threading.Timer(1, out.close).start() logs = [line for line in out] - exit_timer.cancel() - assert len(logs) == 2 assert logs[0] == b'hello\n' assert logs[1] == b'world\n' From 9c2b4e12f87bdec349caf85d97625bd93de1e027 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 14 Mar 2018 14:11:42 -0700 Subject: [PATCH 0627/1301] Use same split rules for Dockerfile as other include/exclude patterns Signed-off-by: Joffrey F --- docker/utils/build.py | 7 +++++-- tests/unit/utils_test.py | 5 +++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/docker/utils/build.py b/docker/utils/build.py index 1da56fbcc4..1622ec356a 100644 --- a/docker/utils/build.py +++ b/docker/utils/build.py @@ -31,18 +31,21 @@ def exclude_paths(root, patterns, dockerfile=None): if dockerfile is None: dockerfile = 'Dockerfile' + def split_path(p): + return [pt for pt in re.split(_SEP, p) if pt and pt != '.'] + def normalize(p): # Leading and trailing slashes are not relevant. Yes, # "foo.py/" must exclude the "foo.py" regular file. "." # components are not relevant either, even if the whole # pattern is only ".", as the Docker reference states: "For # historical reasons, the pattern . is ignored." - split = [pt for pt in re.split(_SEP, p) if pt and pt != '.'] # ".." component must be cleared with the potential previous # component, regardless of whether it exists: "A preprocessing # step [...] eliminates . and .. elements using Go's # filepath.". i = 0 + split = split_path(p) while i < len(split): if split[i] == '..': del split[i] @@ -62,7 +65,7 @@ def normalize(p): # Exclude empty patterns such as "." or the empty string. filter(lambda p: p[1], patterns), # Always include the Dockerfile and .dockerignore - [(True, dockerfile.split('/')), (True, ['.dockerignore'])])))) + [(True, split_path(dockerfile)), (True, ['.dockerignore'])])))) return set(walk(root, patterns)) diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index c2dd502b7e..56800f9e0c 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -698,6 +698,11 @@ def test_exclude_custom_dockerfile(self): ['*'], dockerfile='foo/Dockerfile3' ) == convert_paths(set(['foo/Dockerfile3', '.dockerignore'])) + # https://github.com/docker/docker-py/issues/1956 + assert self.exclude( + ['*'], dockerfile='./foo/Dockerfile3' + ) == convert_paths(set(['foo/Dockerfile3', '.dockerignore'])) + def test_exclude_dockerfile_child(self): includes = self.exclude(['foo/'], dockerfile='foo/Dockerfile3') assert convert_path('foo/Dockerfile3') in includes From cf1d869105641095406db7bf2e5e0e2c1a9bb014 Mon Sep 17 00:00:00 2001 From: James Date: Thu, 15 Mar 2018 15:01:13 +0100 Subject: [PATCH 0628/1301] Updates docs for rename of `name` to `repository` Signed-off-by: James Meakin --- docker/models/images.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/models/images.py b/docker/models/images.py index 58d5d93c47..d4c281322f 100644 --- a/docker/models/images.py +++ b/docker/models/images.py @@ -284,7 +284,7 @@ def pull(self, repository, tag=None, **kwargs): low-level API. Args: - name (str): The repository to pull + repository (str): The repository to pull tag (str): The tag to pull auth_config (dict): Override the credentials that :py:meth:`~docker.client.DockerClient.login` has set for From ffdc0487f5d2f20066c18e29edf2931ca66385fb Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 15 Mar 2018 14:36:39 -0700 Subject: [PATCH 0629/1301] Fix socket tests for TLS-enabled tests Signed-off-by: Joffrey F --- tests/helpers.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/helpers.py b/tests/helpers.py index c4ea3647dd..b6b493b385 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -108,21 +108,21 @@ def swarm_listen_addr(): def assert_cat_socket_detached_with_keys(sock, inputs): - if six.PY3: + if six.PY3 and hasattr(sock, '_sock'): sock = sock._sock for i in inputs: - sock.send(i) + sock.sendall(i) time.sleep(0.5) # If we're using a Unix socket, the sock.send call will fail with a # BrokenPipeError ; INET sockets will just stop receiving / sending data # but will not raise an error - if sock.family == getattr(socket, 'AF_UNIX', -1): + if getattr(sock, 'family', -9) == getattr(socket, 'AF_UNIX', -1): with pytest.raises(socket.error): - sock.send(b'make sure the socket is closed\n') + sock.sendall(b'make sure the socket is closed\n') else: - sock.send(b"make sure the socket is closed\n") + sock.sendall(b"make sure the socket is closed\n") assert sock.recv(32) == b'' From 3f3ca7ed431b18332967cf9d3f0ffce098016011 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 15 Mar 2018 14:37:02 -0700 Subject: [PATCH 0630/1301] Use networks instead of legacy links for test setup Signed-off-by: Joffrey F --- Jenkinsfile | 19 ++++++++++++++----- Makefile | 36 +++++++++++++++++++++--------------- 2 files changed, 35 insertions(+), 20 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 6d9d3436f2..fe684197c7 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -5,7 +5,12 @@ def imageNamePy2 def imageNamePy3 def images = [:] -def dockerVersions = ["17.06.2-ce", "17.12.0-ce", "18.01.0-ce"] +def dockerVersions = [ + "17.06.2-ce", // Latest EE + "17.12.1-ce", // Latest CE stable + "18.02.0-ce", // Latest CE edge + "18.03.0-ce-rc4" // Latest CE RC +] def buildImage = { name, buildargs, pyTag -> img = docker.image(name) @@ -59,15 +64,18 @@ def runTests = { Map settings -> checkout(scm) def dindContainerName = "dpy-dind-\$BUILD_NUMBER-\$EXECUTOR_NUMBER-${pythonVersion}-${dockerVersion}" def testContainerName = "dpy-tests-\$BUILD_NUMBER-\$EXECUTOR_NUMBER-${pythonVersion}-${dockerVersion}" + def testNetwork = "dpy-testnet-\$BUILD_NUMBER-\$EXECUTOR_NUMBER-${pythonVersion}-${dockerVersion}" try { - sh """docker run -d --name ${dindContainerName} -v /tmp --privileged \\ + sh """docker network create ${testNetwork}""" + sh """docker run -d --name ${dindContainerName} -v /tmp --privileged --network ${testNetwork} \\ dockerswarm/dind:${dockerVersion} dockerd -H tcp://0.0.0.0:2375 """ sh """docker run \\ - --name ${testContainerName} --volumes-from ${dindContainerName} \\ - -e 'DOCKER_HOST=tcp://docker:2375' \\ + --name ${testContainerName} \\ + -e "DOCKER_HOST=tcp://${dindContainerName}:2375" \\ -e 'DOCKER_TEST_API_VERSION=${apiVersion}' \\ - --link=${dindContainerName}:docker \\ + --network ${testNetwork} \\ + --volumes-from ${dindContainerName} \\ ${testImage} \\ py.test -v -rxs tests/integration """ @@ -75,6 +83,7 @@ def runTests = { Map settings -> sh """ docker stop ${dindContainerName} ${testContainerName} docker rm -vf ${dindContainerName} ${testContainerName} + docker network rm ${testNetwork} """ } } diff --git a/Makefile b/Makefile index f491993926..434d40e1cc 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ all: test .PHONY: clean clean: - -docker rm -f dpy-dind-py2 dpy-dind-py3 + -docker rm -f dpy-dind-py2 dpy-dind-py3 dpy-dind-certs dpy-dind-ssl find -name "__pycache__" | xargs rm -rf .PHONY: build @@ -44,41 +44,47 @@ integration-test-py3: build-py3 TEST_API_VERSION ?= 1.35 TEST_ENGINE_VERSION ?= 17.12.0-ce +.PHONY: setup-network +setup-network: + docker network inspect dpy-tests || docker network create dpy-tests + .PHONY: integration-dind integration-dind: integration-dind-py2 integration-dind-py3 .PHONY: integration-dind-py2 -integration-dind-py2: build +integration-dind-py2: build setup-network docker rm -vf dpy-dind-py2 || : - docker run -d --name dpy-dind-py2 --privileged dockerswarm/dind:${TEST_ENGINE_VERSION} dockerd\ - -H tcp://0.0.0.0:2375 --experimental - docker run -t --rm --env="DOCKER_HOST=tcp://docker:2375" --env="DOCKER_TEST_API_VERSION=${TEST_API_VERSION}"\ - --link=dpy-dind-py2:docker docker-sdk-python py.test tests/integration + docker run -d --network dpy-tests --name dpy-dind-py2 --privileged\ + dockerswarm/dind:${TEST_ENGINE_VERSION} dockerd -H tcp://0.0.0.0:2375 --experimental + docker run -t --rm --env="DOCKER_HOST=tcp://dpy-dind-py2:2375" --env="DOCKER_TEST_API_VERSION=${TEST_API_VERSION}"\ + --network dpy-tests docker-sdk-python py.test tests/integration docker rm -vf dpy-dind-py2 .PHONY: integration-dind-py3 -integration-dind-py3: build-py3 +integration-dind-py3: build-py3 setup-network docker rm -vf dpy-dind-py3 || : - docker run -d --name dpy-dind-py3 --privileged dockerswarm/dind:${TEST_ENGINE_VERSION} dockerd\ - -H tcp://0.0.0.0:2375 --experimental - docker run -t --rm --env="DOCKER_HOST=tcp://docker:2375" --env="DOCKER_TEST_API_VERSION=${TEST_API_VERSION}"\ - --link=dpy-dind-py3:docker docker-sdk-python3 py.test tests/integration + docker run -d --network dpy-tests --name dpy-dind-py3 --privileged\ + dockerswarm/dind:${TEST_ENGINE_VERSION} dockerd -H tcp://0.0.0.0:2375 --experimental + docker run -t --rm --env="DOCKER_HOST=tcp://dpy-dind-py3:2375" --env="DOCKER_TEST_API_VERSION=${TEST_API_VERSION}"\ + --network dpy-tests docker-sdk-python3 py.test tests/integration docker rm -vf dpy-dind-py3 .PHONY: integration-dind-ssl integration-dind-ssl: build-dind-certs build build-py3 + docker rm -vf dpy-dind-certs dpy-dind-ssl || : docker run -d --name dpy-dind-certs dpy-dind-certs docker run -d --env="DOCKER_HOST=tcp://localhost:2375" --env="DOCKER_TLS_VERIFY=1"\ --env="DOCKER_CERT_PATH=/certs" --volumes-from dpy-dind-certs --name dpy-dind-ssl\ - -v /tmp --privileged dockerswarm/dind:${TEST_ENGINE_VERSION} dockerd --tlsverify\ - --tlscacert=/certs/ca.pem --tlscert=/certs/server-cert.pem\ + --network dpy-tests --network-alias docker -v /tmp --privileged\ + dockerswarm/dind:${TEST_ENGINE_VERSION}\ + dockerd --tlsverify --tlscacert=/certs/ca.pem --tlscert=/certs/server-cert.pem\ --tlskey=/certs/server-key.pem -H tcp://0.0.0.0:2375 --experimental docker run -t --rm --volumes-from dpy-dind-ssl --env="DOCKER_HOST=tcp://docker:2375"\ --env="DOCKER_TLS_VERIFY=1" --env="DOCKER_CERT_PATH=/certs" --env="DOCKER_TEST_API_VERSION=${TEST_API_VERSION}"\ - --link=dpy-dind-ssl:docker docker-sdk-python py.test tests/integration + --network dpy-tests docker-sdk-python py.test tests/integration docker run -t --rm --volumes-from dpy-dind-ssl --env="DOCKER_HOST=tcp://docker:2375"\ --env="DOCKER_TLS_VERIFY=1" --env="DOCKER_CERT_PATH=/certs" --env="DOCKER_TEST_API_VERSION=${TEST_API_VERSION}"\ - --link=dpy-dind-ssl:docker docker-sdk-python3 py.test tests/integration + --network dpy-tests docker-sdk-python3 py.test tests/integration docker rm -vf dpy-dind-ssl dpy-dind-certs .PHONY: flake8 From 791de789ecb9219fa9a8fa9f241213866ee7b7e1 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 20 Mar 2018 09:55:05 +0100 Subject: [PATCH 0631/1301] Bump 3.1.2 Signed-off-by: Joffrey F --- docker/version.py | 2 +- docs/change-log.md | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index fedd9ed7b0..0ce643516a 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "3.1.1" +version = "3.1.2" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) diff --git a/docs/change-log.md b/docs/change-log.md index 94c325d6f5..e92632b55e 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -1,6 +1,16 @@ Change log ========== +3.1.2 +----- + +[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/47?closed=1) + +### Bugfixes + +* Fixed a bug that led to a Dockerfile not being included in the build context + in some situations when the Dockerfile's path was prefixed with `./` + 3.1.1 ----- From 88b0d697aa5386c2ef90a5b480cd400ce5a32646 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 5 Mar 2018 15:26:59 -0800 Subject: [PATCH 0632/1301] Bump test engine versions Signed-off-by: Joffrey F --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index fe684197c7..1323f4b827 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -38,7 +38,7 @@ def buildImages = { -> } def getAPIVersion = { engineVersion -> - def versionMap = ['17.06': '1.30', '17.12': '1.35', '18.01': '1.35'] + def versionMap = ['17.06': '1.30', '17.12': '1.35', '18.02': '1.36', '18.03': '1.37'] return versionMap[engineVersion.substring(0, 5)] } From af674155b78eaf1d014853d9dfcf728b22f1302b Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 20 Mar 2018 11:57:10 +0100 Subject: [PATCH 0633/1301] Bump 3.1.3 Signed-off-by: Joffrey F --- docker/version.py | 2 +- docs/change-log.md | 7 +++++++ scripts/release.sh | 6 ++++++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index 0ce643516a..060797bf1e 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "3.1.2" +version = "3.1.3" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) diff --git a/docs/change-log.md b/docs/change-log.md index e92632b55e..27885b2a36 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -1,6 +1,13 @@ Change log ========== +3.1.3 +----- + +### Bugfixes + +* Regenerated invalid wheel package + 3.1.2 ----- diff --git a/scripts/release.sh b/scripts/release.sh index 814185bd0c..f36efff97a 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -18,6 +18,9 @@ if [ -z $VERSION ]; then exit 1 fi +echo "##> Removing stale build files" +rm -rf ./build || exit 1 + echo "##> Tagging the release as $VERSION" git tag $VERSION || exit 1 if [[ $2 == 'upload' ]]; then @@ -30,4 +33,7 @@ pandoc -f markdown -t rst README.md -o README.rst || exit 1 if [[ $2 == 'upload' ]]; then echo "##> Uploading sdist to pypi" python setup.py sdist bdist_wheel upload +else + echo "##> sdist & wheel" + python setup.py sdist bdist_wheel fi From 7a28cad58ec7c279b91c75a3aa701bb89e0e75cd Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 21 Mar 2018 13:54:43 +0100 Subject: [PATCH 0634/1301] Don't descend into symlinks when building context tar Signed-off-by: Joffrey F --- docker/utils/build.py | 2 +- tests/unit/utils_test.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/docker/utils/build.py b/docker/utils/build.py index 1622ec356a..894b29936b 100644 --- a/docker/utils/build.py +++ b/docker/utils/build.py @@ -93,7 +93,7 @@ def match(p): # Whether this file is implicitely included / excluded. matched = default if hit is None else hit sub = list(filter(lambda p: p[1], sub)) - if os.path.isdir(cur): + if os.path.isdir(cur) and not os.path.islink(cur): # Entirely skip directories if there are no chance any subfile will # be included. if all(not p[0] for p in sub) and not matched: diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index 56800f9e0c..00456e8c11 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -1058,6 +1058,21 @@ def tar_test_negative_mtime_bug(self): assert tar_data.getnames() == ['th.txt'] assert tar_data.getmember('th.txt').mtime == -3600 + @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='No symlinks on Windows') + def test_tar_directory_link(self): + dirs = ['a', 'b', 'a/c'] + files = ['a/hello.py', 'b/utils.py', 'a/c/descend.py'] + base = make_tree(dirs, files) + self.addCleanup(shutil.rmtree, base) + os.symlink(os.path.join(base, 'b'), os.path.join(base, 'a/c/b')) + with tar(base) as archive: + tar_data = tarfile.open(fileobj=archive) + names = tar_data.getnames() + for member in dirs + files: + assert member in names + assert 'a/c/b' in names + assert 'a/c/b/utils.py' not in names + class FormatEnvironmentTest(unittest.TestCase): def test_format_env_binary_unicode_value(self): From a9ecb7234f476989bf28db4f15a5d1d4e47734e6 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 21 Mar 2018 13:54:43 +0100 Subject: [PATCH 0635/1301] Don't descend into symlinks when building context tar Signed-off-by: Joffrey F --- docker/utils/build.py | 2 +- tests/unit/utils_test.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/docker/utils/build.py b/docker/utils/build.py index 1622ec356a..894b29936b 100644 --- a/docker/utils/build.py +++ b/docker/utils/build.py @@ -93,7 +93,7 @@ def match(p): # Whether this file is implicitely included / excluded. matched = default if hit is None else hit sub = list(filter(lambda p: p[1], sub)) - if os.path.isdir(cur): + if os.path.isdir(cur) and not os.path.islink(cur): # Entirely skip directories if there are no chance any subfile will # be included. if all(not p[0] for p in sub) and not matched: diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index 56800f9e0c..00456e8c11 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -1058,6 +1058,21 @@ def tar_test_negative_mtime_bug(self): assert tar_data.getnames() == ['th.txt'] assert tar_data.getmember('th.txt').mtime == -3600 + @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='No symlinks on Windows') + def test_tar_directory_link(self): + dirs = ['a', 'b', 'a/c'] + files = ['a/hello.py', 'b/utils.py', 'a/c/descend.py'] + base = make_tree(dirs, files) + self.addCleanup(shutil.rmtree, base) + os.symlink(os.path.join(base, 'b'), os.path.join(base, 'a/c/b')) + with tar(base) as archive: + tar_data = tarfile.open(fileobj=archive) + names = tar_data.getnames() + for member in dirs + files: + assert member in names + assert 'a/c/b' in names + assert 'a/c/b/utils.py' not in names + class FormatEnvironmentTest(unittest.TestCase): def test_format_env_binary_unicode_value(self): From ea682a69d6c71721f441018fe429e4f1b83ceabf Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 21 Mar 2018 14:23:23 +0100 Subject: [PATCH 0636/1301] Bump 3.1.4 Signed-off-by: Joffrey F --- docker/version.py | 2 +- docs/change-log.md | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index 060797bf1e..0233d23777 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "3.1.3" +version = "3.1.4" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) diff --git a/docs/change-log.md b/docs/change-log.md index 27885b2a36..908519fb22 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -1,6 +1,15 @@ Change log ========== +3.1.4 +----- + +[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/48?closed=1) + + +* Fixed a bug where build contexts containing directory symlinks would produce + invalid tar archives + 3.1.3 ----- From cd9fed108cd06baf318e9a9670fd27298304ef04 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 22 Mar 2018 13:10:35 +0100 Subject: [PATCH 0637/1301] Generate test engines list dynamically Signed-off-by: Joffrey F --- Jenkinsfile | 29 +++++++++++++----- scripts/versions.py | 71 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 92 insertions(+), 8 deletions(-) create mode 100644 scripts/versions.py diff --git a/Jenkinsfile b/Jenkinsfile index 1323f4b827..211159bc28 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -5,13 +5,6 @@ def imageNamePy2 def imageNamePy3 def images = [:] -def dockerVersions = [ - "17.06.2-ce", // Latest EE - "17.12.1-ce", // Latest CE stable - "18.02.0-ce", // Latest CE edge - "18.03.0-ce-rc4" // Latest CE RC -] - def buildImage = { name, buildargs, pyTag -> img = docker.image(name) try { @@ -37,9 +30,27 @@ def buildImages = { -> } } +def getDockerVersions = { -> + def dockerVersions = ["17.06.2-ce"] + wrappedNode(label: "ubuntu && !zfs") { + def result = sh(script: """docker run --rm \\ + --entrypoint=python \\ + ${imageNamePy3} \\ + /src/scripts/versions.py + """, returnStdout: true + ) + dockerVersions = dockerVersions + result.trim().tokenize(' ') + } + return dockerVersions +} + def getAPIVersion = { engineVersion -> def versionMap = ['17.06': '1.30', '17.12': '1.35', '18.02': '1.36', '18.03': '1.37'] - return versionMap[engineVersion.substring(0, 5)] + def result = versionMap[engineVersion.substring(0, 5)] + if (!result) { + return '1.37' + } + return result } def runTests = { Map settings -> @@ -94,6 +105,8 @@ def runTests = { Map settings -> buildImages() +def dockerVersions = getDockerVersions() + def testMatrix = [failFast: false] for (imgKey in new ArrayList(images.keySet())) { diff --git a/scripts/versions.py b/scripts/versions.py new file mode 100644 index 0000000000..77aaf4f182 --- /dev/null +++ b/scripts/versions.py @@ -0,0 +1,71 @@ +import operator +import re +from collections import namedtuple + +import requests + +base_url = 'https://download.docker.com/linux/static/{0}/x86_64/' +categories = [ + 'edge', + 'stable', + 'test' +] + + +class Version(namedtuple('_Version', 'major minor patch rc edition')): + + @classmethod + def parse(cls, version): + edition = None + version = version.lstrip('v') + version, _, rc = version.partition('-') + if rc: + if 'rc' not in rc: + edition = rc + rc = None + elif '-' in rc: + edition, rc = rc.split('-') + + major, minor, patch = version.split('.', 3) + return cls(major, minor, patch, rc, edition) + + @property + def major_minor(self): + return self.major, self.minor + + @property + def order(self): + """Return a representation that allows this object to be sorted + correctly with the default comparator. + """ + # rc releases should appear before official releases + rc = (0, self.rc) if self.rc else (1, ) + return (int(self.major), int(self.minor), int(self.patch)) + rc + + def __str__(self): + rc = '-{}'.format(self.rc) if self.rc else '' + edition = '-{}'.format(self.edition) if self.edition else '' + return '.'.join(map(str, self[:3])) + edition + rc + + +def main(): + results = set() + for url in [base_url.format(cat) for cat in categories]: + res = requests.get(url) + content = res.text + versions = [ + Version.parse( + v.strip('"').lstrip('docker-').rstrip('.tgz').rstrip('-x86_64') + ) for v in re.findall( + r'"docker-[0-9]+\.[0-9]+\.[0-9]+-.*tgz"', content + ) + ] + sorted_versions = sorted( + versions, reverse=True, key=operator.attrgetter('order') + ) + latest = sorted_versions[0] + results.add(str(latest)) + print(' '.join(results)) + +if __name__ == '__main__': + main() From c88db80f01ebef002d3bf9aca49ce273b46c6928 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 22 Mar 2018 09:51:10 +0100 Subject: [PATCH 0638/1301] Add isolation param to build Signed-off-by: Joffrey F --- docker/api/build.py | 11 ++++++++++- docker/models/images.py | 2 ++ tests/integration/api_build_test.py | 15 +++++++++++++++ 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/docker/api/build.py b/docker/api/build.py index e136a6eedb..3067c1051a 100644 --- a/docker/api/build.py +++ b/docker/api/build.py @@ -18,7 +18,7 @@ def build(self, path=None, tag=None, quiet=False, fileobj=None, forcerm=False, dockerfile=None, container_limits=None, decode=False, buildargs=None, gzip=False, shmsize=None, labels=None, cache_from=None, target=None, network_mode=None, - squash=None, extra_hosts=None, platform=None): + squash=None, extra_hosts=None, platform=None, isolation=None): """ Similar to the ``docker build`` command. Either ``path`` or ``fileobj`` needs to be set. ``path`` can be a local path (to a directory @@ -100,6 +100,8 @@ def build(self, path=None, tag=None, quiet=False, fileobj=None, extra_hosts (dict): Extra hosts to add to /etc/hosts in building containers, as a mapping of hostname to IP address. platform (str): Platform in the format ``os[/arch[/variant]]`` + isolation (str): Isolation technology used during build. + Default: `None`. Returns: A generator for the build output. @@ -232,6 +234,13 @@ def build(self, path=None, tag=None, quiet=False, fileobj=None, ) params['platform'] = platform + if isolation is not None: + if utils.version_lt(self._version, '1.24'): + raise errors.InvalidVersion( + 'isolation was only introduced in API version 1.24' + ) + params['isolation'] = isolation + if context is not None: headers = {'Content-Type': 'application/tar'} if encoding: diff --git a/docker/models/images.py b/docker/models/images.py index d4c281322f..bb24eb5c7e 100644 --- a/docker/models/images.py +++ b/docker/models/images.py @@ -164,6 +164,8 @@ def build(self, **kwargs): extra_hosts (dict): Extra hosts to add to /etc/hosts in building containers, as a mapping of hostname to IP address. platform (str): Platform in the format ``os[/arch[/variant]]``. + isolation (str): Isolation technology used during build. + Default: `None`. Returns: (tuple): The first item is the :py:class:`Image` object for the diff --git a/tests/integration/api_build_test.py b/tests/integration/api_build_test.py index ce587d54c1..13bd8ac576 100644 --- a/tests/integration/api_build_test.py +++ b/tests/integration/api_build_test.py @@ -138,6 +138,21 @@ def test_build_shmsize(self): # There is currently no way to get the shmsize # that was used to build the image + @requires_api_version('1.24') + def test_build_isolation(self): + script = io.BytesIO('\n'.join([ + 'FROM scratch', + 'CMD sh -c "echo \'Deaf To All But The Song\'' + ]).encode('ascii')) + + stream = self.client.build( + fileobj=script, tag='isolation', + isolation='default' + ) + + for chunk in stream: + pass + @requires_api_version('1.23') def test_build_labels(self): script = io.BytesIO('\n'.join([ From 12a6833eba4f64be1386d3da0d605156319c5946 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 26 Mar 2018 14:12:42 -0700 Subject: [PATCH 0639/1301] Update MAINTAINERS file Signed-off-by: Joffrey F --- MAINTAINERS | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/MAINTAINERS b/MAINTAINERS index 76aafd8876..b857d13dc0 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -10,13 +10,16 @@ # [Org] [Org."Core maintainers"] + people = [ + "shin-", + ] + [Org.Alumni] people = [ "aanand", "bfirsh", "dnephin", "mnowster", "mpetazzoni", - "shin-", ] [people] From 081b78f15e9a7d3702dd61fa9f01c3babf61a819 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 26 Mar 2018 13:36:45 -0700 Subject: [PATCH 0640/1301] Support building with Dockerfile outside of context Signed-off-by: Joffrey F --- docker/api/build.py | 10 +++++++++ docker/utils/build.py | 22 +++++++++++++------ docker/utils/utils.py | 12 ++++++++++- tests/integration/api_build_test.py | 33 +++++++++++++++++++++++++++++ 4 files changed, 70 insertions(+), 7 deletions(-) diff --git a/docker/api/build.py b/docker/api/build.py index 3067c1051a..2a227591fa 100644 --- a/docker/api/build.py +++ b/docker/api/build.py @@ -1,6 +1,7 @@ import json import logging import os +import random from .. import auth from .. import constants @@ -148,6 +149,15 @@ def build(self, path=None, tag=None, quiet=False, fileobj=None, lambda x: x != '' and x[0] != '#', [l.strip() for l in f.read().splitlines()] )) + if dockerfile and os.path.relpath(dockerfile, path).startswith( + '..'): + with open(dockerfile, 'r') as df: + dockerfile = ( + '.dockerfile.{0:x}'.format(random.getrandbits(160)), + df.read() + ) + else: + dockerfile = (dockerfile, None) context = utils.tar( path, exclude=exclude, dockerfile=dockerfile, gzip=gzip ) diff --git a/docker/utils/build.py b/docker/utils/build.py index 894b29936b..0f173476f2 100644 --- a/docker/utils/build.py +++ b/docker/utils/build.py @@ -2,23 +2,33 @@ import re from ..constants import IS_WINDOWS_PLATFORM +from .utils import create_archive from fnmatch import fnmatch from itertools import chain -from .utils import create_archive + + +_SEP = re.compile('/|\\\\') if IS_WINDOWS_PLATFORM else re.compile('/') def tar(path, exclude=None, dockerfile=None, fileobj=None, gzip=False): root = os.path.abspath(path) exclude = exclude or [] + dockerfile = dockerfile or (None, None) + extra_files = [] + if dockerfile[1] is not None: + dockerignore_contents = '\n'.join( + (exclude or ['.dockerignore']) + [dockerfile[0]] + ) + extra_files = [ + ('.dockerignore', dockerignore_contents), + dockerfile, + ] return create_archive( - files=sorted(exclude_paths(root, exclude, dockerfile=dockerfile)), - root=root, fileobj=fileobj, gzip=gzip + files=sorted(exclude_paths(root, exclude, dockerfile=dockerfile[0])), + root=root, fileobj=fileobj, gzip=gzip, extra_files=extra_files ) -_SEP = re.compile('/|\\\\') if IS_WINDOWS_PLATFORM else re.compile('/') - - def exclude_paths(root, patterns, dockerfile=None): """ Given a root directory path and a list of .dockerignore patterns, return diff --git a/docker/utils/utils.py b/docker/utils/utils.py index 3cd2be8169..5024e471b2 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -88,13 +88,17 @@ def build_file_list(root): return files -def create_archive(root, files=None, fileobj=None, gzip=False): +def create_archive(root, files=None, fileobj=None, gzip=False, + extra_files=None): if not fileobj: fileobj = tempfile.NamedTemporaryFile() t = tarfile.open(mode='w:gz' if gzip else 'w', fileobj=fileobj) if files is None: files = build_file_list(root) for path in files: + if path in [e[0] for e in extra_files]: + # Extra files override context files with the same name + continue full_path = os.path.join(root, path) i = t.gettarinfo(full_path, arcname=path) @@ -123,6 +127,12 @@ def create_archive(root, files=None, fileobj=None, gzip=False): else: # Directories, FIFOs, symlinks... don't need to be read. t.addfile(i, None) + + for name, contents in extra_files: + info = tarfile.TarInfo(name) + info.size = len(contents) + t.addfile(info, io.BytesIO(contents.encode('utf-8'))) + t.close() fileobj.seek(0) return fileobj diff --git a/tests/integration/api_build_test.py b/tests/integration/api_build_test.py index 13bd8ac576..f411efc490 100644 --- a/tests/integration/api_build_test.py +++ b/tests/integration/api_build_test.py @@ -407,3 +407,36 @@ def test_build_invalid_platform(self): assert excinfo.value.status_code == 400 assert 'invalid platform' in excinfo.exconly() + + def test_build_out_of_context_dockerfile(self): + base_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, base_dir) + with open(os.path.join(base_dir, 'file.txt'), 'w') as f: + f.write('hello world') + with open(os.path.join(base_dir, '.dockerignore'), 'w') as f: + f.write('.dockerignore\n') + df = tempfile.NamedTemporaryFile() + self.addCleanup(df.close) + df.write(('\n'.join([ + 'FROM busybox', + 'COPY . /src', + 'WORKDIR /src', + ])).encode('utf-8')) + df.flush() + img_name = random_name() + self.tmp_imgs.append(img_name) + stream = self.client.build( + path=base_dir, dockerfile=df.name, tag=img_name, + decode=True + ) + lines = [] + for chunk in stream: + lines.append(chunk) + assert 'Successfully tagged' in lines[-1]['stream'] + + ctnr = self.client.create_container(img_name, 'ls -a') + self.tmp_containers.append(ctnr) + self.client.start(ctnr) + lsdata = self.client.logs(ctnr).strip().split(b'\n') + assert len(lsdata) == 3 + assert sorted([b'.', b'..', b'file.txt']) == sorted(lsdata) From 3fdc0127c1c42ddde96dbcc1e5611207ba8b8bd7 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 26 Mar 2018 13:38:13 -0700 Subject: [PATCH 0641/1301] Move build utils to appropriate file Signed-off-by: Joffrey F --- docker/utils/__init__.py | 6 +-- docker/utils/build.py | 91 +++++++++++++++++++++++++++++++++++++++- docker/utils/utils.py | 89 --------------------------------------- 3 files changed, 93 insertions(+), 93 deletions(-) diff --git a/docker/utils/__init__.py b/docker/utils/__init__.py index e70a5e615d..81c8186c84 100644 --- a/docker/utils/__init__.py +++ b/docker/utils/__init__.py @@ -1,13 +1,13 @@ # flake8: noqa -from .build import tar, exclude_paths +from .build import create_archive, exclude_paths, mkbuildcontext, tar from .decorators import check_resource, minimum_version, update_headers from .utils import ( compare_version, convert_port_bindings, convert_volume_binds, - mkbuildcontext, parse_repository_tag, parse_host, + parse_repository_tag, parse_host, kwargs_from_env, convert_filters, datetime_to_timestamp, create_host_config, parse_bytes, parse_env_file, version_lt, version_gte, decode_json_header, split_command, create_ipam_config, create_ipam_pool, parse_devices, normalize_links, convert_service_networks, - format_environment, create_archive, format_extra_hosts + format_environment, format_extra_hosts ) diff --git a/docker/utils/build.py b/docker/utils/build.py index 0f173476f2..783273ee8b 100644 --- a/docker/utils/build.py +++ b/docker/utils/build.py @@ -1,8 +1,11 @@ +import io import os import re +import six +import tarfile +import tempfile from ..constants import IS_WINDOWS_PLATFORM -from .utils import create_archive from fnmatch import fnmatch from itertools import chain @@ -127,3 +130,89 @@ def match(p): yield f elif matched: yield f + + +def build_file_list(root): + files = [] + for dirname, dirnames, fnames in os.walk(root): + for filename in fnames + dirnames: + longpath = os.path.join(dirname, filename) + files.append( + longpath.replace(root, '', 1).lstrip('/') + ) + + return files + + +def create_archive(root, files=None, fileobj=None, gzip=False, + extra_files=None): + extra_files = extra_files or [] + if not fileobj: + fileobj = tempfile.NamedTemporaryFile() + t = tarfile.open(mode='w:gz' if gzip else 'w', fileobj=fileobj) + if files is None: + files = build_file_list(root) + for path in files: + if path in [e[0] for e in extra_files]: + # Extra files override context files with the same name + continue + full_path = os.path.join(root, path) + + i = t.gettarinfo(full_path, arcname=path) + if i is None: + # This happens when we encounter a socket file. We can safely + # ignore it and proceed. + continue + + # Workaround https://bugs.python.org/issue32713 + if i.mtime < 0 or i.mtime > 8**11 - 1: + i.mtime = int(i.mtime) + + if IS_WINDOWS_PLATFORM: + # Windows doesn't keep track of the execute bit, so we make files + # and directories executable by default. + i.mode = i.mode & 0o755 | 0o111 + + if i.isfile(): + try: + with open(full_path, 'rb') as f: + t.addfile(i, f) + except IOError: + raise IOError( + 'Can not read file in context: {}'.format(full_path) + ) + else: + # Directories, FIFOs, symlinks... don't need to be read. + t.addfile(i, None) + + for name, contents in extra_files: + info = tarfile.TarInfo(name) + info.size = len(contents) + t.addfile(info, io.BytesIO(contents.encode('utf-8'))) + + t.close() + fileobj.seek(0) + return fileobj + + +def mkbuildcontext(dockerfile): + f = tempfile.NamedTemporaryFile() + t = tarfile.open(mode='w', fileobj=f) + if isinstance(dockerfile, io.StringIO): + dfinfo = tarfile.TarInfo('Dockerfile') + if six.PY3: + raise TypeError('Please use io.BytesIO to create in-memory ' + 'Dockerfiles with Python 3') + else: + dfinfo.size = len(dockerfile.getvalue()) + dockerfile.seek(0) + elif isinstance(dockerfile, io.BytesIO): + dfinfo = tarfile.TarInfo('Dockerfile') + dfinfo.size = len(dockerfile.getvalue()) + dockerfile.seek(0) + else: + dfinfo = t.gettarinfo(fileobj=dockerfile, arcname='Dockerfile') + t.addfile(dfinfo, dockerfile) + t.close() + f.seek(0) + return f diff --git a/docker/utils/utils.py b/docker/utils/utils.py index 5024e471b2..fe3b9a5767 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -1,17 +1,13 @@ import base64 -import io import os import os.path import json import shlex -import tarfile -import tempfile from distutils.version import StrictVersion from datetime import datetime import six -from .. import constants from .. import errors from .. import tls @@ -46,29 +42,6 @@ def create_ipam_config(*args, **kwargs): ) -def mkbuildcontext(dockerfile): - f = tempfile.NamedTemporaryFile() - t = tarfile.open(mode='w', fileobj=f) - if isinstance(dockerfile, io.StringIO): - dfinfo = tarfile.TarInfo('Dockerfile') - if six.PY3: - raise TypeError('Please use io.BytesIO to create in-memory ' - 'Dockerfiles with Python 3') - else: - dfinfo.size = len(dockerfile.getvalue()) - dockerfile.seek(0) - elif isinstance(dockerfile, io.BytesIO): - dfinfo = tarfile.TarInfo('Dockerfile') - dfinfo.size = len(dockerfile.getvalue()) - dockerfile.seek(0) - else: - dfinfo = t.gettarinfo(fileobj=dockerfile, arcname='Dockerfile') - t.addfile(dfinfo, dockerfile) - t.close() - f.seek(0) - return f - - def decode_json_header(header): data = base64.b64decode(header) if six.PY3: @@ -76,68 +49,6 @@ def decode_json_header(header): return json.loads(data) -def build_file_list(root): - files = [] - for dirname, dirnames, fnames in os.walk(root): - for filename in fnames + dirnames: - longpath = os.path.join(dirname, filename) - files.append( - longpath.replace(root, '', 1).lstrip('/') - ) - - return files - - -def create_archive(root, files=None, fileobj=None, gzip=False, - extra_files=None): - if not fileobj: - fileobj = tempfile.NamedTemporaryFile() - t = tarfile.open(mode='w:gz' if gzip else 'w', fileobj=fileobj) - if files is None: - files = build_file_list(root) - for path in files: - if path in [e[0] for e in extra_files]: - # Extra files override context files with the same name - continue - full_path = os.path.join(root, path) - - i = t.gettarinfo(full_path, arcname=path) - if i is None: - # This happens when we encounter a socket file. We can safely - # ignore it and proceed. - continue - - # Workaround https://bugs.python.org/issue32713 - if i.mtime < 0 or i.mtime > 8**11 - 1: - i.mtime = int(i.mtime) - - if constants.IS_WINDOWS_PLATFORM: - # Windows doesn't keep track of the execute bit, so we make files - # and directories executable by default. - i.mode = i.mode & 0o755 | 0o111 - - if i.isfile(): - try: - with open(full_path, 'rb') as f: - t.addfile(i, f) - except IOError: - raise IOError( - 'Can not read file in context: {}'.format(full_path) - ) - else: - # Directories, FIFOs, symlinks... don't need to be read. - t.addfile(i, None) - - for name, contents in extra_files: - info = tarfile.TarInfo(name) - info.size = len(contents) - t.addfile(info, io.BytesIO(contents.encode('utf-8'))) - - t.close() - fileobj.seek(0) - return fileobj - - def compare_version(v1, v2): """Compare docker versions From 899f3cf5a86784dc63eda9545bde73cffc236f0b Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 27 Mar 2018 10:22:17 -0700 Subject: [PATCH 0642/1301] Improve extra_files override check Signed-off-by: Joffrey F --- docker/utils/build.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/utils/build.py b/docker/utils/build.py index 783273ee8b..b644c9fca1 100644 --- a/docker/utils/build.py +++ b/docker/utils/build.py @@ -152,8 +152,9 @@ def create_archive(root, files=None, fileobj=None, gzip=False, t = tarfile.open(mode='w:gz' if gzip else 'w', fileobj=fileobj) if files is None: files = build_file_list(root) + extra_names = set(e[0] for e in extra_files) for path in files: - if path in [e[0] for e in extra_files]: + if path in extra_names: # Extra files override context files with the same name continue full_path = os.path.join(root, path) From 9ff787cb5f6b2ad21669c54a7fa089498538dd2a Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 26 Mar 2018 19:01:50 -0700 Subject: [PATCH 0643/1301] Add methods for /distribution//json endpoint Signed-off-by: Joffrey F --- docker/api/image.py | 21 ++++++ docker/models/images.py | 107 +++++++++++++++++++++++++++- docs/images.rst | 19 +++++ tests/integration/api_image_test.py | 9 +++ 4 files changed, 155 insertions(+), 1 deletion(-) diff --git a/docker/api/image.py b/docker/api/image.py index 3ebca32e59..5f05d8877e 100644 --- a/docker/api/image.py +++ b/docker/api/image.py @@ -245,6 +245,27 @@ def inspect_image(self, image): self._get(self._url("/images/{0}/json", image)), True ) + @utils.minimum_version('1.30') + @utils.check_resource('image') + def inspect_distribution(self, image): + """ + Get image digest and platform information by contacting the registry. + + Args: + image (str): The image name to inspect + + Returns: + (dict): A dict containing distribution data + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + + return self._result( + self._get(self._url("/distribution/{0}/json", image)), True + ) + def load_image(self, data, quiet=None): """ Load an image that was previously saved using diff --git a/docker/models/images.py b/docker/models/images.py index bb24eb5c7e..d4893bb6a1 100644 --- a/docker/models/images.py +++ b/docker/models/images.py @@ -5,7 +5,7 @@ from ..api import APIClient from ..constants import DEFAULT_DATA_CHUNK_SIZE -from ..errors import BuildError, ImageLoadError +from ..errors import BuildError, ImageLoadError, InvalidArgument from ..utils import parse_repository_tag from ..utils.json_stream import json_stream from .resource import Collection, Model @@ -105,6 +105,81 @@ def tag(self, repository, tag=None, **kwargs): return self.client.api.tag(self.id, repository, tag=tag, **kwargs) +class RegistryData(Model): + """ + Image metadata stored on the registry, including available platforms. + """ + def __init__(self, image_name, *args, **kwargs): + super(RegistryData, self).__init__(*args, **kwargs) + self.image_name = image_name + + @property + def id(self): + """ + The ID of the object. + """ + return self.attrs['Descriptor']['digest'] + + @property + def short_id(self): + """ + The ID of the image truncated to 10 characters, plus the ``sha256:`` + prefix. + """ + return self.id[:17] + + def pull(self, platform=None): + """ + Pull the image digest. + + Args: + platform (str): The platform to pull the image for. + Default: ``None`` + + Returns: + (:py:class:`Image`): A reference to the pulled image. + """ + repository, _ = parse_repository_tag(self.image_name) + return self.collection.pull(repository, tag=self.id, platform=platform) + + def has_platform(self, platform): + """ + Check whether the given platform identifier is available for this + digest. + + Args: + platform (str or dict): A string using the ``os[/arch[/variant]]`` + format, or a platform dictionary. + + Returns: + (bool): ``True`` if the platform is recognized as available, + ``False`` otherwise. + + Raises: + :py:class:`docker.errors.InvalidArgument` + If the platform argument is not a valid descriptor. + """ + if platform and not isinstance(platform, dict): + parts = platform.split('/') + if len(parts) > 3 or len(parts) < 1: + raise InvalidArgument( + '"{0}" is not a valid platform descriptor'.format(platform) + ) + platform = {'os': parts[0]} + if len(parts) > 2: + platform['variant'] = parts[2] + if len(parts) > 1: + platform['architecture'] = parts[1] + return normalize_platform( + platform, self.client.version() + ) in self.attrs['Platforms'] + + def reload(self): + self.attrs = self.client.api.inspect_distribution(self.image_name) + + reload.__doc__ = Model.reload.__doc__ + + class ImageCollection(Collection): model = Image @@ -219,6 +294,26 @@ def get(self, name): """ return self.prepare_model(self.client.api.inspect_image(name)) + def get_registry_data(self, name): + """ + Gets the registry data for an image. + + Args: + name (str): The name of the image. + + Returns: + (:py:class:`RegistryData`): The data object. + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + return RegistryData( + image_name=name, + attrs=self.client.api.inspect_distribution(name), + client=self.client, + collection=self, + ) + def list(self, name=None, all=False, filters=None): """ List images on the server. @@ -336,3 +431,13 @@ def search(self, *args, **kwargs): def prune(self, filters=None): return self.client.api.prune_images(filters=filters) prune.__doc__ = APIClient.prune_images.__doc__ + + +def normalize_platform(platform, engine_info): + if platform is None: + platform = {} + if 'os' not in platform: + platform['os'] = engine_info['Os'] + if 'architecture' not in platform: + platform['architecture'] = engine_info['Arch'] + return platform diff --git a/docs/images.rst b/docs/images.rst index 12b0fd1842..4d425e95a1 100644 --- a/docs/images.rst +++ b/docs/images.rst @@ -12,6 +12,7 @@ Methods available on ``client.images``: .. automethod:: build .. automethod:: get + .. automethod:: get_registry_data .. automethod:: list(**kwargs) .. automethod:: load .. automethod:: prune @@ -41,3 +42,21 @@ Image objects .. automethod:: reload .. automethod:: save .. automethod:: tag + +RegistryData objects +-------------------- + +.. autoclass:: RegistryData() + + .. py:attribute:: attrs + + The raw representation of this object from the server. + + .. autoattribute:: id + .. autoattribute:: short_id + + + + .. automethod:: has_platform + .. automethod:: pull + .. automethod:: reload diff --git a/tests/integration/api_image_test.py b/tests/integration/api_image_test.py index ab638c9e46..050e7f339b 100644 --- a/tests/integration/api_image_test.py +++ b/tests/integration/api_image_test.py @@ -357,3 +357,12 @@ def test_get_image_load_image(self): success = True break assert success is True + + +@requires_api_version('1.30') +class InspectDistributionTest(BaseAPIIntegrationTest): + def test_inspect_distribution(self): + data = self.client.inspect_distribution('busybox:latest') + assert data is not None + assert 'Platforms' in data + assert {'os': 'linux', 'architecture': 'amd64'} in data['Platforms'] From 35520ab01fc49052f1ccf145933e89a8c368a364 Mon Sep 17 00:00:00 2001 From: Matthieu Nottale Date: Wed, 14 Mar 2018 14:14:20 +0100 Subject: [PATCH 0644/1301] Add close() method to DockerClient. Signed-off-by: Matthieu Nottale --- docker/client.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docker/client.py b/docker/client.py index 467583e639..b4364c3c07 100644 --- a/docker/client.py +++ b/docker/client.py @@ -186,6 +186,10 @@ def version(self, *args, **kwargs): return self.api.version(*args, **kwargs) version.__doc__ = APIClient.version.__doc__ + def close(self): + return self.api.close() + close.__doc__ = APIClient.close.__doc__ + def __getattr__(self, name): s = ["'DockerClient' object has no attribute '{}'".format(name)] # If a user calls a method on APIClient, they From 726d7f31cabebbd545337733a38b79bf36d2846b Mon Sep 17 00:00:00 2001 From: Matthieu Nottale Date: Wed, 14 Mar 2018 15:30:30 +0100 Subject: [PATCH 0645/1301] Add sparse argument to DockerClient.containers.list(). Signed-off-by: Matthieu Nottale --- docker/models/containers.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/docker/models/containers.py b/docker/models/containers.py index 895080cab9..d4ed1aa3f4 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -833,7 +833,8 @@ def get(self, container_id): resp = self.client.api.inspect_container(container_id) return self.prepare_model(resp) - def list(self, all=False, before=None, filters=None, limit=-1, since=None): + def list(self, all=False, before=None, filters=None, limit=-1, since=None, + sparse=False): """ List containers. Similar to the ``docker ps`` command. @@ -862,6 +863,9 @@ def list(self, all=False, before=None, filters=None, limit=-1, since=None): container. Give the container name or id. - `since` (str): Only containers created after a particular container. Give container name or id. + sparse (bool): Do not inspect containers. Returns partial + informations, but guaranteed not to block. Use reload() on + each container to get the full list of attributes. A comprehensive list can be found in the documentation for `docker ps @@ -877,7 +881,10 @@ def list(self, all=False, before=None, filters=None, limit=-1, since=None): resp = self.client.api.containers(all=all, before=before, filters=filters, limit=limit, since=since) - return [self.get(r['Id']) for r in resp] + if sparse: + return [self.prepare_model(r) for r in resp] + else: + return [self.get(r['Id']) for r in resp] def prune(self, filters=None): return self.client.api.prune_containers(filters=filters) From d310d95fbcdd6ae20d37ca676cb8ece23c3805cc Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 14 Mar 2018 16:53:56 -0700 Subject: [PATCH 0646/1301] Add test for container list with sparse=True Signed-off-by: Joffrey F --- docker/models/containers.py | 30 ++++++++++++++------- tests/integration/models_containers_test.py | 22 +++++++++++++++ 2 files changed, 43 insertions(+), 9 deletions(-) diff --git a/docker/models/containers.py b/docker/models/containers.py index d4ed1aa3f4..1e06ed6099 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -4,8 +4,10 @@ from ..api import APIClient from ..constants import DEFAULT_DATA_CHUNK_SIZE -from ..errors import (ContainerError, ImageNotFound, - create_unexpected_kwargs_error) +from ..errors import ( + ContainerError, DockerException, ImageNotFound, + create_unexpected_kwargs_error +) from ..types import HostConfig from ..utils import version_gte from .images import Image @@ -27,7 +29,7 @@ def image(self): """ The image of the container. """ - image_id = self.attrs['Image'] + image_id = self.attrs.get('ImageID', self.attrs['Image']) if image_id is None: return None return self.client.images.get(image_id.split(':')[1]) @@ -37,15 +39,23 @@ def labels(self): """ The labels of a container as dictionary. """ - result = self.attrs['Config'].get('Labels') - return result or {} + try: + result = self.attrs['Config'].get('Labels') + return result or {} + except KeyError: + raise DockerException( + 'Label data is not available for sparse objects. Call reload()' + ' to retrieve all information' + ) @property def status(self): """ The status of the container. For example, ``running``, or ``exited``. """ - return self.attrs['State']['Status'] + if isinstance(self.attrs['State'], dict): + return self.attrs['State']['Status'] + return self.attrs['State'] def attach(self, **kwargs): """ @@ -863,14 +873,16 @@ def list(self, all=False, before=None, filters=None, limit=-1, since=None, container. Give the container name or id. - `since` (str): Only containers created after a particular container. Give container name or id. - sparse (bool): Do not inspect containers. Returns partial - informations, but guaranteed not to block. Use reload() on - each container to get the full list of attributes. A comprehensive list can be found in the documentation for `docker ps `_. + sparse (bool): Do not inspect containers. Returns partial + information, but guaranteed not to block. Use + :py:meth:`Container.reload` on resulting objects to retrieve + all attributes. Default: ``False`` + Returns: (list of :py:class:`Container`) diff --git a/tests/integration/models_containers_test.py b/tests/integration/models_containers_test.py index fac4de2b68..38aae4d2e5 100644 --- a/tests/integration/models_containers_test.py +++ b/tests/integration/models_containers_test.py @@ -159,6 +159,28 @@ def test_list(self): container = containers[0] assert container.attrs['Config']['Image'] == 'alpine' + assert container.status == 'running' + assert container.image == client.images.get('alpine') + + container.kill() + container.remove() + assert container_id not in [c.id for c in client.containers.list()] + + def test_list_sparse(self): + client = docker.from_env(version=TEST_API_VERSION) + container_id = client.containers.run( + "alpine", "sleep 300", detach=True).id + self.tmp_containers.append(container_id) + containers = [c for c in client.containers.list(sparse=True) if c.id == + container_id] + assert len(containers) == 1 + + container = containers[0] + assert container.attrs['Image'] == 'alpine' + assert container.status == 'running' + assert container.image == client.images.get('alpine') + with pytest.raises(docker.errors.DockerException): + container.labels container.kill() container.remove() From dd743db4b390b71ef86e4d23c618db2e7204e135 Mon Sep 17 00:00:00 2001 From: Viktor Adam Date: Wed, 21 Feb 2018 22:16:21 +0000 Subject: [PATCH 0647/1301] Allow cancelling the streams from other threads Signed-off-by: Viktor Adam --- docker/api/container.py | 17 +++++- docker/api/daemon.py | 21 ++++--- docker/types/__init__.py | 1 + docker/types/daemon.py | 63 +++++++++++++++++++++ tests/integration/api_container_test.py | 48 ++++++++++++++++ tests/integration/client_test.py | 20 +++++++ tests/integration/models_containers_test.py | 21 +++++++ 7 files changed, 181 insertions(+), 10 deletions(-) create mode 100644 docker/types/daemon.py diff --git a/docker/api/container.py b/docker/api/container.py index f8d52de489..cb97b7940c 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -5,7 +5,8 @@ from .. import utils from ..constants import DEFAULT_DATA_CHUNK_SIZE from ..types import ( - ContainerConfig, EndpointConfig, HostConfig, NetworkingConfig + CancellableStream, ContainerConfig, EndpointConfig, HostConfig, + NetworkingConfig ) @@ -52,10 +53,15 @@ def attach(self, container, stdout=True, stderr=True, u = self._url("/containers/{0}/attach", container) response = self._post(u, headers=headers, params=params, stream=True) - return self._read_from_socket( + output = self._read_from_socket( response, stream, self._check_is_tty(container) ) + if stream: + return CancellableStream(output, response) + else: + return output + @utils.check_resource('container') def attach_socket(self, container, params=None, ws=False): """ @@ -815,7 +821,12 @@ def logs(self, container, stdout=True, stderr=True, stream=False, url = self._url("/containers/{0}/logs", container) res = self._get(url, params=params, stream=stream) - return self._get_result(container, stream, res) + output = self._get_result(container, stream, res) + + if stream: + return CancellableStream(output, res) + else: + return output @utils.check_resource('container') def pause(self, container): diff --git a/docker/api/daemon.py b/docker/api/daemon.py index 0e1c753814..fc3692c248 100644 --- a/docker/api/daemon.py +++ b/docker/api/daemon.py @@ -1,7 +1,7 @@ import os from datetime import datetime -from .. import auth, utils +from .. import auth, types, utils class DaemonApiMixin(object): @@ -34,8 +34,7 @@ def events(self, since=None, until=None, filters=None, decode=None): the fly. False by default. Returns: - (generator): A blocking generator you can iterate over to retrieve - events as they happen. + A :py:class:`docker.types.daemon.CancellableStream` generator Raises: :py:class:`docker.errors.APIError` @@ -50,6 +49,14 @@ def events(self, since=None, until=None, filters=None, decode=None): u'status': u'start', u'time': 1423339459} ... + + or + + >>> events = client.events() + >>> for event in events: + ... print event + >>> # and cancel from another thread + >>> events.close() """ if isinstance(since, datetime): @@ -68,10 +75,10 @@ def events(self, since=None, until=None, filters=None, decode=None): } url = self._url('/events') - return self._stream_helper( - self._get(url, params=params, stream=True, timeout=None), - decode=decode - ) + response = self._get(url, params=params, stream=True, timeout=None) + stream = self._stream_helper(response, decode=decode) + + return types.CancellableStream(stream, response) def info(self): """ diff --git a/docker/types/__init__.py b/docker/types/__init__.py index 39c93e344d..0b0d847fe9 100644 --- a/docker/types/__init__.py +++ b/docker/types/__init__.py @@ -1,5 +1,6 @@ # flake8: noqa from .containers import ContainerConfig, HostConfig, LogConfig, Ulimit +from .daemon import CancellableStream from .healthcheck import Healthcheck from .networks import EndpointConfig, IPAMConfig, IPAMPool, NetworkingConfig from .services import ( diff --git a/docker/types/daemon.py b/docker/types/daemon.py new file mode 100644 index 0000000000..ba0334d0cd --- /dev/null +++ b/docker/types/daemon.py @@ -0,0 +1,63 @@ +import socket + +try: + import requests.packages.urllib3 as urllib3 +except ImportError: + import urllib3 + + +class CancellableStream(object): + """ + Stream wrapper for real-time events, logs, etc. from the server. + + Example: + >>> events = client.events() + >>> for event in events: + ... print event + >>> # and cancel from another thread + >>> events.close() + """ + + def __init__(self, stream, response): + self._stream = stream + self._response = response + + def __iter__(self): + return self + + def __next__(self): + try: + return next(self._stream) + except urllib3.exceptions.ProtocolError: + raise StopIteration + except socket.error: + raise StopIteration + + next = __next__ + + def close(self): + """ + Closes the event streaming. + """ + + if not self._response.raw.closed: + # find the underlying socket object + # based on api.client._get_raw_response_socket + + sock_fp = self._response.raw._fp.fp + + if hasattr(sock_fp, 'raw'): + sock_raw = sock_fp.raw + + if hasattr(sock_raw, 'sock'): + sock = sock_raw.sock + + elif hasattr(sock_raw, '_sock'): + sock = sock_raw._sock + + else: + sock = sock_fp._sock + + sock.shutdown(socket.SHUT_RDWR) + sock.makefile().close() + sock.close() diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index 8447aa5f05..cc2c07198b 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -2,6 +2,7 @@ import re import signal import tempfile +import threading from datetime import datetime import docker @@ -880,6 +881,30 @@ def test_logs_streaming_and_follow(self): assert logs == (snippet + '\n').encode(encoding='ascii') + def test_logs_streaming_and_follow_and_cancel(self): + snippet = 'Flowering Nights (Sakuya Iyazoi)' + container = self.client.create_container( + BUSYBOX, 'sh -c "echo \\"{0}\\" && sleep 3"'.format(snippet) + ) + id = container['Id'] + self.tmp_containers.append(id) + self.client.start(id) + logs = six.binary_type() + + generator = self.client.logs(id, stream=True, follow=True) + + exit_timer = threading.Timer(3, os._exit, args=[1]) + exit_timer.start() + + threading.Timer(1, generator.close).start() + + for chunk in generator: + logs += chunk + + exit_timer.cancel() + + assert logs == (snippet + '\n').encode(encoding='ascii') + def test_logs_with_dict_instead_of_id(self): snippet = 'Flowering Nights (Sakuya Iyazoi)' container = self.client.create_container( @@ -1226,6 +1251,29 @@ def test_attach_no_stream(self): output = self.client.attach(container, stream=False, logs=True) assert output == 'hello\n'.encode(encoding='ascii') + def test_attach_stream_and_cancel(self): + container = self.client.create_container( + BUSYBOX, 'sh -c "echo hello && sleep 60"', + tty=True + ) + self.tmp_containers.append(container) + self.client.start(container) + output = self.client.attach(container, stream=True, logs=True) + + exit_timer = threading.Timer(3, os._exit, args=[1]) + exit_timer.start() + + threading.Timer(1, output.close).start() + + lines = [] + for line in output: + lines.append(line) + + exit_timer.cancel() + + assert len(lines) == 1 + assert lines[0] == 'hello\r\n'.encode(encoding='ascii') + def test_detach_with_default(self): container = self.client.create_container( BUSYBOX, 'cat', diff --git a/tests/integration/client_test.py b/tests/integration/client_test.py index 8f6bd86b84..7df172c885 100644 --- a/tests/integration/client_test.py +++ b/tests/integration/client_test.py @@ -1,7 +1,10 @@ +import threading import unittest import docker +from datetime import datetime, timedelta + from ..helpers import requires_api_version from .base import TEST_API_VERSION @@ -27,3 +30,20 @@ def test_df(self): assert 'Containers' in data assert 'Volumes' in data assert 'Images' in data + + +class CancellableEventsTest(unittest.TestCase): + client = docker.from_env(version=TEST_API_VERSION) + + def test_cancel_events(self): + start = datetime.now() + + events = self.client.events(until=start + timedelta(seconds=5)) + + cancel_thread = threading.Timer(2, events.close) + cancel_thread.start() + + for _ in events: + pass + + self.assertLess(datetime.now() - start, timedelta(seconds=3)) diff --git a/tests/integration/models_containers_test.py b/tests/integration/models_containers_test.py index 38aae4d2e5..41faff3580 100644 --- a/tests/integration/models_containers_test.py +++ b/tests/integration/models_containers_test.py @@ -1,4 +1,6 @@ +import os import tempfile +import threading import docker import pytest @@ -141,6 +143,25 @@ def test_run_with_streamed_logs(self): assert logs[0] == b'hello\n' assert logs[1] == b'world\n' + def test_run_with_streamed_logs_and_cancel(self): + client = docker.from_env(version=TEST_API_VERSION) + out = client.containers.run( + 'alpine', 'sh -c "echo hello && echo world"', stream=True + ) + + exit_timer = threading.Timer(3, os._exit, args=[1]) + exit_timer.start() + + threading.Timer(1, out.close).start() + + logs = [line for line in out] + + exit_timer.cancel() + + assert len(logs) == 2 + assert logs[0] == b'hello\n' + assert logs[1] == b'world\n' + def test_get(self): client = docker.from_env(version=TEST_API_VERSION) container = client.containers.run("alpine", "sleep 300", detach=True) From e9f31e1e277a39eb275cdd3cc29a42adf0bc5094 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 19 Mar 2018 14:40:49 +0100 Subject: [PATCH 0648/1301] Remove redundant single-socket select call Clean up + use pytest-timeout Signed-off-by: Joffrey F --- docker/types/daemon.py | 1 - docker/utils/socket.py | 3 +-- test-requirements.txt | 5 +++-- tests/integration/api_container_test.py | 13 ++----------- tests/integration/models_containers_test.py | 7 +------ 5 files changed, 7 insertions(+), 22 deletions(-) diff --git a/docker/types/daemon.py b/docker/types/daemon.py index ba0334d0cd..852f3d8292 100644 --- a/docker/types/daemon.py +++ b/docker/types/daemon.py @@ -59,5 +59,4 @@ def close(self): sock = sock_fp._sock sock.shutdown(socket.SHUT_RDWR) - sock.makefile().close() sock.close() diff --git a/docker/utils/socket.py b/docker/utils/socket.py index c3a5f90fc3..0945f0a694 100644 --- a/docker/utils/socket.py +++ b/docker/utils/socket.py @@ -22,8 +22,7 @@ def read(socket, n=4096): recoverable_errors = (errno.EINTR, errno.EDEADLK, errno.EWOULDBLOCK) - # wait for data to become available - if not isinstance(socket, NpipeSocket): + if six.PY3 and not isinstance(socket, NpipeSocket): select.select([socket], [], []) try: diff --git a/test-requirements.txt b/test-requirements.txt index f79e815907..09680b6897 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,5 +1,6 @@ +coverage==3.7.1 +flake8==3.4.1 mock==1.0.1 pytest==2.9.1 -coverage==3.7.1 pytest-cov==2.1.0 -flake8==3.4.1 +pytest-timeout==1.2.1 diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index cc2c07198b..e2125186d1 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -881,6 +881,7 @@ def test_logs_streaming_and_follow(self): assert logs == (snippet + '\n').encode(encoding='ascii') + @pytest.mark.timeout(5) def test_logs_streaming_and_follow_and_cancel(self): snippet = 'Flowering Nights (Sakuya Iyazoi)' container = self.client.create_container( @@ -892,17 +893,11 @@ def test_logs_streaming_and_follow_and_cancel(self): logs = six.binary_type() generator = self.client.logs(id, stream=True, follow=True) - - exit_timer = threading.Timer(3, os._exit, args=[1]) - exit_timer.start() - threading.Timer(1, generator.close).start() for chunk in generator: logs += chunk - exit_timer.cancel() - assert logs == (snippet + '\n').encode(encoding='ascii') def test_logs_with_dict_instead_of_id(self): @@ -1251,6 +1246,7 @@ def test_attach_no_stream(self): output = self.client.attach(container, stream=False, logs=True) assert output == 'hello\n'.encode(encoding='ascii') + @pytest.mark.timeout(5) def test_attach_stream_and_cancel(self): container = self.client.create_container( BUSYBOX, 'sh -c "echo hello && sleep 60"', @@ -1260,17 +1256,12 @@ def test_attach_stream_and_cancel(self): self.client.start(container) output = self.client.attach(container, stream=True, logs=True) - exit_timer = threading.Timer(3, os._exit, args=[1]) - exit_timer.start() - threading.Timer(1, output.close).start() lines = [] for line in output: lines.append(line) - exit_timer.cancel() - assert len(lines) == 1 assert lines[0] == 'hello\r\n'.encode(encoding='ascii') diff --git a/tests/integration/models_containers_test.py b/tests/integration/models_containers_test.py index 41faff3580..6ddb034b41 100644 --- a/tests/integration/models_containers_test.py +++ b/tests/integration/models_containers_test.py @@ -1,4 +1,3 @@ -import os import tempfile import threading @@ -143,21 +142,17 @@ def test_run_with_streamed_logs(self): assert logs[0] == b'hello\n' assert logs[1] == b'world\n' + @pytest.mark.timeout(5) def test_run_with_streamed_logs_and_cancel(self): client = docker.from_env(version=TEST_API_VERSION) out = client.containers.run( 'alpine', 'sh -c "echo hello && echo world"', stream=True ) - exit_timer = threading.Timer(3, os._exit, args=[1]) - exit_timer.start() - threading.Timer(1, out.close).start() logs = [line for line in out] - exit_timer.cancel() - assert len(logs) == 2 assert logs[0] == b'hello\n' assert logs[1] == b'world\n' From 73a9003758d60226879f06c630c01389f3dd0fe7 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 22 Mar 2018 13:10:35 +0100 Subject: [PATCH 0649/1301] Generate test engines list dynamically Signed-off-by: Joffrey F --- Jenkinsfile | 29 +++++++++++++----- scripts/versions.py | 71 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 92 insertions(+), 8 deletions(-) create mode 100644 scripts/versions.py diff --git a/Jenkinsfile b/Jenkinsfile index 1323f4b827..211159bc28 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -5,13 +5,6 @@ def imageNamePy2 def imageNamePy3 def images = [:] -def dockerVersions = [ - "17.06.2-ce", // Latest EE - "17.12.1-ce", // Latest CE stable - "18.02.0-ce", // Latest CE edge - "18.03.0-ce-rc4" // Latest CE RC -] - def buildImage = { name, buildargs, pyTag -> img = docker.image(name) try { @@ -37,9 +30,27 @@ def buildImages = { -> } } +def getDockerVersions = { -> + def dockerVersions = ["17.06.2-ce"] + wrappedNode(label: "ubuntu && !zfs") { + def result = sh(script: """docker run --rm \\ + --entrypoint=python \\ + ${imageNamePy3} \\ + /src/scripts/versions.py + """, returnStdout: true + ) + dockerVersions = dockerVersions + result.trim().tokenize(' ') + } + return dockerVersions +} + def getAPIVersion = { engineVersion -> def versionMap = ['17.06': '1.30', '17.12': '1.35', '18.02': '1.36', '18.03': '1.37'] - return versionMap[engineVersion.substring(0, 5)] + def result = versionMap[engineVersion.substring(0, 5)] + if (!result) { + return '1.37' + } + return result } def runTests = { Map settings -> @@ -94,6 +105,8 @@ def runTests = { Map settings -> buildImages() +def dockerVersions = getDockerVersions() + def testMatrix = [failFast: false] for (imgKey in new ArrayList(images.keySet())) { diff --git a/scripts/versions.py b/scripts/versions.py new file mode 100644 index 0000000000..77aaf4f182 --- /dev/null +++ b/scripts/versions.py @@ -0,0 +1,71 @@ +import operator +import re +from collections import namedtuple + +import requests + +base_url = 'https://download.docker.com/linux/static/{0}/x86_64/' +categories = [ + 'edge', + 'stable', + 'test' +] + + +class Version(namedtuple('_Version', 'major minor patch rc edition')): + + @classmethod + def parse(cls, version): + edition = None + version = version.lstrip('v') + version, _, rc = version.partition('-') + if rc: + if 'rc' not in rc: + edition = rc + rc = None + elif '-' in rc: + edition, rc = rc.split('-') + + major, minor, patch = version.split('.', 3) + return cls(major, minor, patch, rc, edition) + + @property + def major_minor(self): + return self.major, self.minor + + @property + def order(self): + """Return a representation that allows this object to be sorted + correctly with the default comparator. + """ + # rc releases should appear before official releases + rc = (0, self.rc) if self.rc else (1, ) + return (int(self.major), int(self.minor), int(self.patch)) + rc + + def __str__(self): + rc = '-{}'.format(self.rc) if self.rc else '' + edition = '-{}'.format(self.edition) if self.edition else '' + return '.'.join(map(str, self[:3])) + edition + rc + + +def main(): + results = set() + for url in [base_url.format(cat) for cat in categories]: + res = requests.get(url) + content = res.text + versions = [ + Version.parse( + v.strip('"').lstrip('docker-').rstrip('.tgz').rstrip('-x86_64') + ) for v in re.findall( + r'"docker-[0-9]+\.[0-9]+\.[0-9]+-.*tgz"', content + ) + ] + sorted_versions = sorted( + versions, reverse=True, key=operator.attrgetter('order') + ) + latest = sorted_versions[0] + results.add(str(latest)) + print(' '.join(results)) + +if __name__ == '__main__': + main() From 27322fede7fcba1e81e447722d8708de9dfc7406 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 22 Mar 2018 09:51:10 +0100 Subject: [PATCH 0650/1301] Add isolation param to build Signed-off-by: Joffrey F --- docker/api/build.py | 11 ++++++++++- docker/models/images.py | 2 ++ tests/integration/api_build_test.py | 15 +++++++++++++++ 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/docker/api/build.py b/docker/api/build.py index e136a6eedb..3067c1051a 100644 --- a/docker/api/build.py +++ b/docker/api/build.py @@ -18,7 +18,7 @@ def build(self, path=None, tag=None, quiet=False, fileobj=None, forcerm=False, dockerfile=None, container_limits=None, decode=False, buildargs=None, gzip=False, shmsize=None, labels=None, cache_from=None, target=None, network_mode=None, - squash=None, extra_hosts=None, platform=None): + squash=None, extra_hosts=None, platform=None, isolation=None): """ Similar to the ``docker build`` command. Either ``path`` or ``fileobj`` needs to be set. ``path`` can be a local path (to a directory @@ -100,6 +100,8 @@ def build(self, path=None, tag=None, quiet=False, fileobj=None, extra_hosts (dict): Extra hosts to add to /etc/hosts in building containers, as a mapping of hostname to IP address. platform (str): Platform in the format ``os[/arch[/variant]]`` + isolation (str): Isolation technology used during build. + Default: `None`. Returns: A generator for the build output. @@ -232,6 +234,13 @@ def build(self, path=None, tag=None, quiet=False, fileobj=None, ) params['platform'] = platform + if isolation is not None: + if utils.version_lt(self._version, '1.24'): + raise errors.InvalidVersion( + 'isolation was only introduced in API version 1.24' + ) + params['isolation'] = isolation + if context is not None: headers = {'Content-Type': 'application/tar'} if encoding: diff --git a/docker/models/images.py b/docker/models/images.py index d4c281322f..bb24eb5c7e 100644 --- a/docker/models/images.py +++ b/docker/models/images.py @@ -164,6 +164,8 @@ def build(self, **kwargs): extra_hosts (dict): Extra hosts to add to /etc/hosts in building containers, as a mapping of hostname to IP address. platform (str): Platform in the format ``os[/arch[/variant]]``. + isolation (str): Isolation technology used during build. + Default: `None`. Returns: (tuple): The first item is the :py:class:`Image` object for the diff --git a/tests/integration/api_build_test.py b/tests/integration/api_build_test.py index ce587d54c1..13bd8ac576 100644 --- a/tests/integration/api_build_test.py +++ b/tests/integration/api_build_test.py @@ -138,6 +138,21 @@ def test_build_shmsize(self): # There is currently no way to get the shmsize # that was used to build the image + @requires_api_version('1.24') + def test_build_isolation(self): + script = io.BytesIO('\n'.join([ + 'FROM scratch', + 'CMD sh -c "echo \'Deaf To All But The Song\'' + ]).encode('ascii')) + + stream = self.client.build( + fileobj=script, tag='isolation', + isolation='default' + ) + + for chunk in stream: + pass + @requires_api_version('1.23') def test_build_labels(self): script = io.BytesIO('\n'.join([ From 20939d06819447ad8a745e5d5c0b4bfc41f9792b Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 26 Mar 2018 14:12:42 -0700 Subject: [PATCH 0651/1301] Update MAINTAINERS file Signed-off-by: Joffrey F --- MAINTAINERS | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/MAINTAINERS b/MAINTAINERS index 76aafd8876..b857d13dc0 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -10,13 +10,16 @@ # [Org] [Org."Core maintainers"] + people = [ + "shin-", + ] + [Org.Alumni] people = [ "aanand", "bfirsh", "dnephin", "mnowster", "mpetazzoni", - "shin-", ] [people] From 77c3e57dcfa5d5e8cf05038861b2037dafe2d0e6 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 26 Mar 2018 13:36:45 -0700 Subject: [PATCH 0652/1301] Support building with Dockerfile outside of context Signed-off-by: Joffrey F --- docker/api/build.py | 10 +++++++++ docker/utils/build.py | 22 +++++++++++++------ docker/utils/utils.py | 12 ++++++++++- tests/integration/api_build_test.py | 33 +++++++++++++++++++++++++++++ 4 files changed, 70 insertions(+), 7 deletions(-) diff --git a/docker/api/build.py b/docker/api/build.py index 3067c1051a..2a227591fa 100644 --- a/docker/api/build.py +++ b/docker/api/build.py @@ -1,6 +1,7 @@ import json import logging import os +import random from .. import auth from .. import constants @@ -148,6 +149,15 @@ def build(self, path=None, tag=None, quiet=False, fileobj=None, lambda x: x != '' and x[0] != '#', [l.strip() for l in f.read().splitlines()] )) + if dockerfile and os.path.relpath(dockerfile, path).startswith( + '..'): + with open(dockerfile, 'r') as df: + dockerfile = ( + '.dockerfile.{0:x}'.format(random.getrandbits(160)), + df.read() + ) + else: + dockerfile = (dockerfile, None) context = utils.tar( path, exclude=exclude, dockerfile=dockerfile, gzip=gzip ) diff --git a/docker/utils/build.py b/docker/utils/build.py index 894b29936b..0f173476f2 100644 --- a/docker/utils/build.py +++ b/docker/utils/build.py @@ -2,23 +2,33 @@ import re from ..constants import IS_WINDOWS_PLATFORM +from .utils import create_archive from fnmatch import fnmatch from itertools import chain -from .utils import create_archive + + +_SEP = re.compile('/|\\\\') if IS_WINDOWS_PLATFORM else re.compile('/') def tar(path, exclude=None, dockerfile=None, fileobj=None, gzip=False): root = os.path.abspath(path) exclude = exclude or [] + dockerfile = dockerfile or (None, None) + extra_files = [] + if dockerfile[1] is not None: + dockerignore_contents = '\n'.join( + (exclude or ['.dockerignore']) + [dockerfile[0]] + ) + extra_files = [ + ('.dockerignore', dockerignore_contents), + dockerfile, + ] return create_archive( - files=sorted(exclude_paths(root, exclude, dockerfile=dockerfile)), - root=root, fileobj=fileobj, gzip=gzip + files=sorted(exclude_paths(root, exclude, dockerfile=dockerfile[0])), + root=root, fileobj=fileobj, gzip=gzip, extra_files=extra_files ) -_SEP = re.compile('/|\\\\') if IS_WINDOWS_PLATFORM else re.compile('/') - - def exclude_paths(root, patterns, dockerfile=None): """ Given a root directory path and a list of .dockerignore patterns, return diff --git a/docker/utils/utils.py b/docker/utils/utils.py index 3cd2be8169..5024e471b2 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -88,13 +88,17 @@ def build_file_list(root): return files -def create_archive(root, files=None, fileobj=None, gzip=False): +def create_archive(root, files=None, fileobj=None, gzip=False, + extra_files=None): if not fileobj: fileobj = tempfile.NamedTemporaryFile() t = tarfile.open(mode='w:gz' if gzip else 'w', fileobj=fileobj) if files is None: files = build_file_list(root) for path in files: + if path in [e[0] for e in extra_files]: + # Extra files override context files with the same name + continue full_path = os.path.join(root, path) i = t.gettarinfo(full_path, arcname=path) @@ -123,6 +127,12 @@ def create_archive(root, files=None, fileobj=None, gzip=False): else: # Directories, FIFOs, symlinks... don't need to be read. t.addfile(i, None) + + for name, contents in extra_files: + info = tarfile.TarInfo(name) + info.size = len(contents) + t.addfile(info, io.BytesIO(contents.encode('utf-8'))) + t.close() fileobj.seek(0) return fileobj diff --git a/tests/integration/api_build_test.py b/tests/integration/api_build_test.py index 13bd8ac576..f411efc490 100644 --- a/tests/integration/api_build_test.py +++ b/tests/integration/api_build_test.py @@ -407,3 +407,36 @@ def test_build_invalid_platform(self): assert excinfo.value.status_code == 400 assert 'invalid platform' in excinfo.exconly() + + def test_build_out_of_context_dockerfile(self): + base_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, base_dir) + with open(os.path.join(base_dir, 'file.txt'), 'w') as f: + f.write('hello world') + with open(os.path.join(base_dir, '.dockerignore'), 'w') as f: + f.write('.dockerignore\n') + df = tempfile.NamedTemporaryFile() + self.addCleanup(df.close) + df.write(('\n'.join([ + 'FROM busybox', + 'COPY . /src', + 'WORKDIR /src', + ])).encode('utf-8')) + df.flush() + img_name = random_name() + self.tmp_imgs.append(img_name) + stream = self.client.build( + path=base_dir, dockerfile=df.name, tag=img_name, + decode=True + ) + lines = [] + for chunk in stream: + lines.append(chunk) + assert 'Successfully tagged' in lines[-1]['stream'] + + ctnr = self.client.create_container(img_name, 'ls -a') + self.tmp_containers.append(ctnr) + self.client.start(ctnr) + lsdata = self.client.logs(ctnr).strip().split(b'\n') + assert len(lsdata) == 3 + assert sorted([b'.', b'..', b'file.txt']) == sorted(lsdata) From fce99c329fe4157bda209b5dfb44b0c2f8fe037e Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 26 Mar 2018 13:38:13 -0700 Subject: [PATCH 0653/1301] Move build utils to appropriate file Signed-off-by: Joffrey F --- docker/utils/__init__.py | 6 +-- docker/utils/build.py | 91 +++++++++++++++++++++++++++++++++++++++- docker/utils/utils.py | 89 --------------------------------------- 3 files changed, 93 insertions(+), 93 deletions(-) diff --git a/docker/utils/__init__.py b/docker/utils/__init__.py index e70a5e615d..81c8186c84 100644 --- a/docker/utils/__init__.py +++ b/docker/utils/__init__.py @@ -1,13 +1,13 @@ # flake8: noqa -from .build import tar, exclude_paths +from .build import create_archive, exclude_paths, mkbuildcontext, tar from .decorators import check_resource, minimum_version, update_headers from .utils import ( compare_version, convert_port_bindings, convert_volume_binds, - mkbuildcontext, parse_repository_tag, parse_host, + parse_repository_tag, parse_host, kwargs_from_env, convert_filters, datetime_to_timestamp, create_host_config, parse_bytes, parse_env_file, version_lt, version_gte, decode_json_header, split_command, create_ipam_config, create_ipam_pool, parse_devices, normalize_links, convert_service_networks, - format_environment, create_archive, format_extra_hosts + format_environment, format_extra_hosts ) diff --git a/docker/utils/build.py b/docker/utils/build.py index 0f173476f2..783273ee8b 100644 --- a/docker/utils/build.py +++ b/docker/utils/build.py @@ -1,8 +1,11 @@ +import io import os import re +import six +import tarfile +import tempfile from ..constants import IS_WINDOWS_PLATFORM -from .utils import create_archive from fnmatch import fnmatch from itertools import chain @@ -127,3 +130,89 @@ def match(p): yield f elif matched: yield f + + +def build_file_list(root): + files = [] + for dirname, dirnames, fnames in os.walk(root): + for filename in fnames + dirnames: + longpath = os.path.join(dirname, filename) + files.append( + longpath.replace(root, '', 1).lstrip('/') + ) + + return files + + +def create_archive(root, files=None, fileobj=None, gzip=False, + extra_files=None): + extra_files = extra_files or [] + if not fileobj: + fileobj = tempfile.NamedTemporaryFile() + t = tarfile.open(mode='w:gz' if gzip else 'w', fileobj=fileobj) + if files is None: + files = build_file_list(root) + for path in files: + if path in [e[0] for e in extra_files]: + # Extra files override context files with the same name + continue + full_path = os.path.join(root, path) + + i = t.gettarinfo(full_path, arcname=path) + if i is None: + # This happens when we encounter a socket file. We can safely + # ignore it and proceed. + continue + + # Workaround https://bugs.python.org/issue32713 + if i.mtime < 0 or i.mtime > 8**11 - 1: + i.mtime = int(i.mtime) + + if IS_WINDOWS_PLATFORM: + # Windows doesn't keep track of the execute bit, so we make files + # and directories executable by default. + i.mode = i.mode & 0o755 | 0o111 + + if i.isfile(): + try: + with open(full_path, 'rb') as f: + t.addfile(i, f) + except IOError: + raise IOError( + 'Can not read file in context: {}'.format(full_path) + ) + else: + # Directories, FIFOs, symlinks... don't need to be read. + t.addfile(i, None) + + for name, contents in extra_files: + info = tarfile.TarInfo(name) + info.size = len(contents) + t.addfile(info, io.BytesIO(contents.encode('utf-8'))) + + t.close() + fileobj.seek(0) + return fileobj + + +def mkbuildcontext(dockerfile): + f = tempfile.NamedTemporaryFile() + t = tarfile.open(mode='w', fileobj=f) + if isinstance(dockerfile, io.StringIO): + dfinfo = tarfile.TarInfo('Dockerfile') + if six.PY3: + raise TypeError('Please use io.BytesIO to create in-memory ' + 'Dockerfiles with Python 3') + else: + dfinfo.size = len(dockerfile.getvalue()) + dockerfile.seek(0) + elif isinstance(dockerfile, io.BytesIO): + dfinfo = tarfile.TarInfo('Dockerfile') + dfinfo.size = len(dockerfile.getvalue()) + dockerfile.seek(0) + else: + dfinfo = t.gettarinfo(fileobj=dockerfile, arcname='Dockerfile') + t.addfile(dfinfo, dockerfile) + t.close() + f.seek(0) + return f diff --git a/docker/utils/utils.py b/docker/utils/utils.py index 5024e471b2..fe3b9a5767 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -1,17 +1,13 @@ import base64 -import io import os import os.path import json import shlex -import tarfile -import tempfile from distutils.version import StrictVersion from datetime import datetime import six -from .. import constants from .. import errors from .. import tls @@ -46,29 +42,6 @@ def create_ipam_config(*args, **kwargs): ) -def mkbuildcontext(dockerfile): - f = tempfile.NamedTemporaryFile() - t = tarfile.open(mode='w', fileobj=f) - if isinstance(dockerfile, io.StringIO): - dfinfo = tarfile.TarInfo('Dockerfile') - if six.PY3: - raise TypeError('Please use io.BytesIO to create in-memory ' - 'Dockerfiles with Python 3') - else: - dfinfo.size = len(dockerfile.getvalue()) - dockerfile.seek(0) - elif isinstance(dockerfile, io.BytesIO): - dfinfo = tarfile.TarInfo('Dockerfile') - dfinfo.size = len(dockerfile.getvalue()) - dockerfile.seek(0) - else: - dfinfo = t.gettarinfo(fileobj=dockerfile, arcname='Dockerfile') - t.addfile(dfinfo, dockerfile) - t.close() - f.seek(0) - return f - - def decode_json_header(header): data = base64.b64decode(header) if six.PY3: @@ -76,68 +49,6 @@ def decode_json_header(header): return json.loads(data) -def build_file_list(root): - files = [] - for dirname, dirnames, fnames in os.walk(root): - for filename in fnames + dirnames: - longpath = os.path.join(dirname, filename) - files.append( - longpath.replace(root, '', 1).lstrip('/') - ) - - return files - - -def create_archive(root, files=None, fileobj=None, gzip=False, - extra_files=None): - if not fileobj: - fileobj = tempfile.NamedTemporaryFile() - t = tarfile.open(mode='w:gz' if gzip else 'w', fileobj=fileobj) - if files is None: - files = build_file_list(root) - for path in files: - if path in [e[0] for e in extra_files]: - # Extra files override context files with the same name - continue - full_path = os.path.join(root, path) - - i = t.gettarinfo(full_path, arcname=path) - if i is None: - # This happens when we encounter a socket file. We can safely - # ignore it and proceed. - continue - - # Workaround https://bugs.python.org/issue32713 - if i.mtime < 0 or i.mtime > 8**11 - 1: - i.mtime = int(i.mtime) - - if constants.IS_WINDOWS_PLATFORM: - # Windows doesn't keep track of the execute bit, so we make files - # and directories executable by default. - i.mode = i.mode & 0o755 | 0o111 - - if i.isfile(): - try: - with open(full_path, 'rb') as f: - t.addfile(i, f) - except IOError: - raise IOError( - 'Can not read file in context: {}'.format(full_path) - ) - else: - # Directories, FIFOs, symlinks... don't need to be read. - t.addfile(i, None) - - for name, contents in extra_files: - info = tarfile.TarInfo(name) - info.size = len(contents) - t.addfile(info, io.BytesIO(contents.encode('utf-8'))) - - t.close() - fileobj.seek(0) - return fileobj - - def compare_version(v1, v2): """Compare docker versions From f39c0dc18d820392e5e1b32e30bc0764bf8e0714 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 27 Mar 2018 10:22:17 -0700 Subject: [PATCH 0654/1301] Improve extra_files override check Signed-off-by: Joffrey F --- docker/utils/build.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/utils/build.py b/docker/utils/build.py index 783273ee8b..b644c9fca1 100644 --- a/docker/utils/build.py +++ b/docker/utils/build.py @@ -152,8 +152,9 @@ def create_archive(root, files=None, fileobj=None, gzip=False, t = tarfile.open(mode='w:gz' if gzip else 'w', fileobj=fileobj) if files is None: files = build_file_list(root) + extra_names = set(e[0] for e in extra_files) for path in files: - if path in [e[0] for e in extra_files]: + if path in extra_names: # Extra files override context files with the same name continue full_path = os.path.join(root, path) From bedabbfa35c36c5a29c8d4a8b888f398caa8e0e2 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 26 Mar 2018 19:01:50 -0700 Subject: [PATCH 0655/1301] Add methods for /distribution//json endpoint Signed-off-by: Joffrey F --- docker/api/image.py | 21 ++++++ docker/models/images.py | 107 +++++++++++++++++++++++++++- docs/images.rst | 19 +++++ tests/integration/api_image_test.py | 9 +++ 4 files changed, 155 insertions(+), 1 deletion(-) diff --git a/docker/api/image.py b/docker/api/image.py index 3ebca32e59..5f05d8877e 100644 --- a/docker/api/image.py +++ b/docker/api/image.py @@ -245,6 +245,27 @@ def inspect_image(self, image): self._get(self._url("/images/{0}/json", image)), True ) + @utils.minimum_version('1.30') + @utils.check_resource('image') + def inspect_distribution(self, image): + """ + Get image digest and platform information by contacting the registry. + + Args: + image (str): The image name to inspect + + Returns: + (dict): A dict containing distribution data + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + + return self._result( + self._get(self._url("/distribution/{0}/json", image)), True + ) + def load_image(self, data, quiet=None): """ Load an image that was previously saved using diff --git a/docker/models/images.py b/docker/models/images.py index bb24eb5c7e..d4893bb6a1 100644 --- a/docker/models/images.py +++ b/docker/models/images.py @@ -5,7 +5,7 @@ from ..api import APIClient from ..constants import DEFAULT_DATA_CHUNK_SIZE -from ..errors import BuildError, ImageLoadError +from ..errors import BuildError, ImageLoadError, InvalidArgument from ..utils import parse_repository_tag from ..utils.json_stream import json_stream from .resource import Collection, Model @@ -105,6 +105,81 @@ def tag(self, repository, tag=None, **kwargs): return self.client.api.tag(self.id, repository, tag=tag, **kwargs) +class RegistryData(Model): + """ + Image metadata stored on the registry, including available platforms. + """ + def __init__(self, image_name, *args, **kwargs): + super(RegistryData, self).__init__(*args, **kwargs) + self.image_name = image_name + + @property + def id(self): + """ + The ID of the object. + """ + return self.attrs['Descriptor']['digest'] + + @property + def short_id(self): + """ + The ID of the image truncated to 10 characters, plus the ``sha256:`` + prefix. + """ + return self.id[:17] + + def pull(self, platform=None): + """ + Pull the image digest. + + Args: + platform (str): The platform to pull the image for. + Default: ``None`` + + Returns: + (:py:class:`Image`): A reference to the pulled image. + """ + repository, _ = parse_repository_tag(self.image_name) + return self.collection.pull(repository, tag=self.id, platform=platform) + + def has_platform(self, platform): + """ + Check whether the given platform identifier is available for this + digest. + + Args: + platform (str or dict): A string using the ``os[/arch[/variant]]`` + format, or a platform dictionary. + + Returns: + (bool): ``True`` if the platform is recognized as available, + ``False`` otherwise. + + Raises: + :py:class:`docker.errors.InvalidArgument` + If the platform argument is not a valid descriptor. + """ + if platform and not isinstance(platform, dict): + parts = platform.split('/') + if len(parts) > 3 or len(parts) < 1: + raise InvalidArgument( + '"{0}" is not a valid platform descriptor'.format(platform) + ) + platform = {'os': parts[0]} + if len(parts) > 2: + platform['variant'] = parts[2] + if len(parts) > 1: + platform['architecture'] = parts[1] + return normalize_platform( + platform, self.client.version() + ) in self.attrs['Platforms'] + + def reload(self): + self.attrs = self.client.api.inspect_distribution(self.image_name) + + reload.__doc__ = Model.reload.__doc__ + + class ImageCollection(Collection): model = Image @@ -219,6 +294,26 @@ def get(self, name): """ return self.prepare_model(self.client.api.inspect_image(name)) + def get_registry_data(self, name): + """ + Gets the registry data for an image. + + Args: + name (str): The name of the image. + + Returns: + (:py:class:`RegistryData`): The data object. + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + return RegistryData( + image_name=name, + attrs=self.client.api.inspect_distribution(name), + client=self.client, + collection=self, + ) + def list(self, name=None, all=False, filters=None): """ List images on the server. @@ -336,3 +431,13 @@ def search(self, *args, **kwargs): def prune(self, filters=None): return self.client.api.prune_images(filters=filters) prune.__doc__ = APIClient.prune_images.__doc__ + + +def normalize_platform(platform, engine_info): + if platform is None: + platform = {} + if 'os' not in platform: + platform['os'] = engine_info['Os'] + if 'architecture' not in platform: + platform['architecture'] = engine_info['Arch'] + return platform diff --git a/docs/images.rst b/docs/images.rst index 12b0fd1842..4d425e95a1 100644 --- a/docs/images.rst +++ b/docs/images.rst @@ -12,6 +12,7 @@ Methods available on ``client.images``: .. automethod:: build .. automethod:: get + .. automethod:: get_registry_data .. automethod:: list(**kwargs) .. automethod:: load .. automethod:: prune @@ -41,3 +42,21 @@ Image objects .. automethod:: reload .. automethod:: save .. automethod:: tag + +RegistryData objects +-------------------- + +.. autoclass:: RegistryData() + + .. py:attribute:: attrs + + The raw representation of this object from the server. + + .. autoattribute:: id + .. autoattribute:: short_id + + + + .. automethod:: has_platform + .. automethod:: pull + .. automethod:: reload diff --git a/tests/integration/api_image_test.py b/tests/integration/api_image_test.py index ab638c9e46..050e7f339b 100644 --- a/tests/integration/api_image_test.py +++ b/tests/integration/api_image_test.py @@ -357,3 +357,12 @@ def test_get_image_load_image(self): success = True break assert success is True + + +@requires_api_version('1.30') +class InspectDistributionTest(BaseAPIIntegrationTest): + def test_inspect_distribution(self): + data = self.client.inspect_distribution('busybox:latest') + assert data is not None + assert 'Platforms' in data + assert {'os': 'linux', 'architecture': 'amd64'} in data['Platforms'] From 2ecc3adcd4b7edcaa1f8eb4aa288089feebba42e Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 29 Mar 2018 15:58:42 -0700 Subject: [PATCH 0656/1301] Bump 3.2.0 Signed-off-by: Joffrey F --- docker/version.py | 2 +- docs/change-log.md | 19 +++++++++++++++++++ docs/client.rst | 1 + 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index 0233d23777..5460e16572 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "3.1.4" +version = "3.2.0" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) diff --git a/docs/change-log.md b/docs/change-log.md index 908519fb22..4715c52408 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -1,11 +1,30 @@ Change log ========== +3.2.0 +----- + +[List of PRs/ issues for this release](https://github.com/docker/docker-py/milestone/45?closed=1) + +### Features + +* Generators returned by `attach()`, `logs()` and `events()` now have a + `cancel()` method to let consumers stop the iteration client-side. +* `build()` methods can now handle Dockerfiles supplied outside of the + build context. +* Added `sparse` argument to `DockerClient.containers.list()` +* Added `isolation` parameter to `build()` methods. +* Added `close()` method to `DockerClient` +* Added `APIClient.inspect_distribution()` method and + `DockerClient.images.get_registry_data()` + * The latter returns an instance of the new `RegistryData` class + 3.1.4 ----- [List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/48?closed=1) +### Bugfixes * Fixed a bug where build contexts containing directory symlinks would produce invalid tar archives diff --git a/docs/client.rst b/docs/client.rst index 43d7c63be7..85a1396f63 100644 --- a/docs/client.rst +++ b/docs/client.rst @@ -26,6 +26,7 @@ Client reference .. autoattribute:: swarm .. autoattribute:: volumes + .. automethod:: close() .. automethod:: df() .. automethod:: events() .. automethod:: info() From bdee6e308734dfb0d1cd959575222b081e150d2f Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 29 Mar 2018 16:37:52 -0700 Subject: [PATCH 0657/1301] dev version Signed-off-by: Joffrey F --- docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index 5460e16572..c949131684 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "3.2.0" +version = "3.3.0-dev" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) From 16751ac509b4bbe75293847fe87099ff51a74013 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 30 Mar 2018 10:22:39 -0700 Subject: [PATCH 0658/1301] Properly handle relative Dockerfile paths and Dockerfile on different drives Signed-off-by: Joffrey F --- docker/api/build.py | 29 ++++++++++++++++++-------- tests/integration/api_build_test.py | 32 +++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 9 deletions(-) diff --git a/docker/api/build.py b/docker/api/build.py index 2a227591fa..d69985e68b 100644 --- a/docker/api/build.py +++ b/docker/api/build.py @@ -149,15 +149,7 @@ def build(self, path=None, tag=None, quiet=False, fileobj=None, lambda x: x != '' and x[0] != '#', [l.strip() for l in f.read().splitlines()] )) - if dockerfile and os.path.relpath(dockerfile, path).startswith( - '..'): - with open(dockerfile, 'r') as df: - dockerfile = ( - '.dockerfile.{0:x}'.format(random.getrandbits(160)), - df.read() - ) - else: - dockerfile = (dockerfile, None) + dockerfile = process_dockerfile(dockerfile, path) context = utils.tar( path, exclude=exclude, dockerfile=dockerfile, gzip=gzip ) @@ -312,3 +304,22 @@ def _set_auth_headers(self, headers): ) else: log.debug('No auth config found') + + +def process_dockerfile(dockerfile, path): + if not dockerfile: + return (None, None) + + abs_dockerfile = dockerfile + if not os.path.isabs(dockerfile): + abs_dockerfile = os.path.join(path, dockerfile) + + if (os.path.splitdrive(path)[0] != os.path.splitdrive(abs_dockerfile)[0] or + os.path.relpath(abs_dockerfile, path).startswith('..')): + with open(abs_dockerfile, 'r') as df: + return ( + '.dockerfile.{0:x}'.format(random.getrandbits(160)), + df.read() + ) + else: + return (dockerfile, None) diff --git a/tests/integration/api_build_test.py b/tests/integration/api_build_test.py index f411efc490..8910eb758f 100644 --- a/tests/integration/api_build_test.py +++ b/tests/integration/api_build_test.py @@ -440,3 +440,35 @@ def test_build_out_of_context_dockerfile(self): lsdata = self.client.logs(ctnr).strip().split(b'\n') assert len(lsdata) == 3 assert sorted([b'.', b'..', b'file.txt']) == sorted(lsdata) + + def test_build_in_context_dockerfile(self): + base_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, base_dir) + with open(os.path.join(base_dir, 'file.txt'), 'w') as f: + f.write('hello world') + with open(os.path.join(base_dir, 'custom.dockerfile'), 'w') as df: + df.write('\n'.join([ + 'FROM busybox', + 'COPY . /src', + 'WORKDIR /src', + ])) + print(os.path.join(base_dir, 'custom.dockerfile')) + img_name = random_name() + self.tmp_imgs.append(img_name) + stream = self.client.build( + path=base_dir, dockerfile='custom.dockerfile', tag=img_name, + decode=True + ) + lines = [] + for chunk in stream: + lines.append(chunk) + assert 'Successfully tagged' in lines[-1]['stream'] + + ctnr = self.client.create_container(img_name, 'ls -a') + self.tmp_containers.append(ctnr) + self.client.start(ctnr) + lsdata = self.client.logs(ctnr).strip().split(b'\n') + assert len(lsdata) == 4 + assert sorted( + [b'.', b'..', b'file.txt', b'custom.dockerfile'] + ) == sorted(lsdata) From 06f5a47d91dcdbef5d1c55800426a87c17034b77 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 30 Mar 2018 10:22:39 -0700 Subject: [PATCH 0659/1301] Properly handle relative Dockerfile paths and Dockerfile on different drives Signed-off-by: Joffrey F --- docker/api/build.py | 29 ++++++++++++++++++-------- tests/integration/api_build_test.py | 32 +++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 9 deletions(-) diff --git a/docker/api/build.py b/docker/api/build.py index 2a227591fa..d69985e68b 100644 --- a/docker/api/build.py +++ b/docker/api/build.py @@ -149,15 +149,7 @@ def build(self, path=None, tag=None, quiet=False, fileobj=None, lambda x: x != '' and x[0] != '#', [l.strip() for l in f.read().splitlines()] )) - if dockerfile and os.path.relpath(dockerfile, path).startswith( - '..'): - with open(dockerfile, 'r') as df: - dockerfile = ( - '.dockerfile.{0:x}'.format(random.getrandbits(160)), - df.read() - ) - else: - dockerfile = (dockerfile, None) + dockerfile = process_dockerfile(dockerfile, path) context = utils.tar( path, exclude=exclude, dockerfile=dockerfile, gzip=gzip ) @@ -312,3 +304,22 @@ def _set_auth_headers(self, headers): ) else: log.debug('No auth config found') + + +def process_dockerfile(dockerfile, path): + if not dockerfile: + return (None, None) + + abs_dockerfile = dockerfile + if not os.path.isabs(dockerfile): + abs_dockerfile = os.path.join(path, dockerfile) + + if (os.path.splitdrive(path)[0] != os.path.splitdrive(abs_dockerfile)[0] or + os.path.relpath(abs_dockerfile, path).startswith('..')): + with open(abs_dockerfile, 'r') as df: + return ( + '.dockerfile.{0:x}'.format(random.getrandbits(160)), + df.read() + ) + else: + return (dockerfile, None) diff --git a/tests/integration/api_build_test.py b/tests/integration/api_build_test.py index f411efc490..8910eb758f 100644 --- a/tests/integration/api_build_test.py +++ b/tests/integration/api_build_test.py @@ -440,3 +440,35 @@ def test_build_out_of_context_dockerfile(self): lsdata = self.client.logs(ctnr).strip().split(b'\n') assert len(lsdata) == 3 assert sorted([b'.', b'..', b'file.txt']) == sorted(lsdata) + + def test_build_in_context_dockerfile(self): + base_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, base_dir) + with open(os.path.join(base_dir, 'file.txt'), 'w') as f: + f.write('hello world') + with open(os.path.join(base_dir, 'custom.dockerfile'), 'w') as df: + df.write('\n'.join([ + 'FROM busybox', + 'COPY . /src', + 'WORKDIR /src', + ])) + print(os.path.join(base_dir, 'custom.dockerfile')) + img_name = random_name() + self.tmp_imgs.append(img_name) + stream = self.client.build( + path=base_dir, dockerfile='custom.dockerfile', tag=img_name, + decode=True + ) + lines = [] + for chunk in stream: + lines.append(chunk) + assert 'Successfully tagged' in lines[-1]['stream'] + + ctnr = self.client.create_container(img_name, 'ls -a') + self.tmp_containers.append(ctnr) + self.client.start(ctnr) + lsdata = self.client.logs(ctnr).strip().split(b'\n') + assert len(lsdata) == 4 + assert sorted( + [b'.', b'..', b'file.txt', b'custom.dockerfile'] + ) == sorted(lsdata) From d2d998281b5b94ac2b214719ed19037d7ba3a18c Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 30 Mar 2018 11:07:16 -0700 Subject: [PATCH 0660/1301] Bump 3.2.1 Signed-off-by: Joffrey F --- docker/version.py | 2 +- docs/change-log.md | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index 5460e16572..28dd1ea486 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "3.2.0" +version = "3.2.1" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) diff --git a/docs/change-log.md b/docs/change-log.md index 4715c52408..912ec5ff80 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -1,6 +1,18 @@ Change log ========== +3.2.1 +----- + +[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/50?closed=1) + +### Bugfixes + +* Fixed a bug with builds not properly identifying Dockerfile paths relative + to the build context +* Fixed an issue where builds would raise a `ValueError` when attempting to + build with a Dockerfile on a different Windows drive. + 3.2.0 ----- From 1d6f8ecf9277ef1c77f3d530efaafd39323cc8e7 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 12 Apr 2018 12:38:27 -0700 Subject: [PATCH 0661/1301] Support absolute paths for in-context Dockerfiles Signed-off-by: Joffrey F --- docker/api/build.py | 6 ++++-- setup.py | 16 ++++++++------ tests/integration/api_build_test.py | 33 ++++++++++++++++++++++++++++- 3 files changed, 46 insertions(+), 9 deletions(-) diff --git a/docker/api/build.py b/docker/api/build.py index d69985e68b..a76e32c121 100644 --- a/docker/api/build.py +++ b/docker/api/build.py @@ -316,10 +316,12 @@ def process_dockerfile(dockerfile, path): if (os.path.splitdrive(path)[0] != os.path.splitdrive(abs_dockerfile)[0] or os.path.relpath(abs_dockerfile, path).startswith('..')): + # Dockerfile not in context - read data to insert into tar later with open(abs_dockerfile, 'r') as df: return ( '.dockerfile.{0:x}'.format(random.getrandbits(160)), df.read() ) - else: - return (dockerfile, None) + + # Dockerfile is inside the context - return path relative to context root + return (os.path.relpath(abs_dockerfile, path), None) diff --git a/setup.py b/setup.py index 271d94f268..1153f784bd 100644 --- a/setup.py +++ b/setup.py @@ -9,12 +9,16 @@ from setuptools import setup, find_packages -if 'docker-py' in [x.project_name for x in pip.get_installed_distributions()]: - print( - 'ERROR: "docker-py" needs to be uninstalled before installing this' - ' package:\npip uninstall docker-py', file=sys.stderr - ) - sys.exit(1) +try: + if 'docker-py' in [ + x.project_name for x in pip.get_installed_distributions()]: + print( + 'ERROR: "docker-py" needs to be uninstalled before installing this' + ' package:\npip uninstall docker-py', file=sys.stderr + ) + sys.exit(1) +except AttributeError: + pass ROOT_DIR = os.path.dirname(__file__) SOURCE_DIR = os.path.join(ROOT_DIR) diff --git a/tests/integration/api_build_test.py b/tests/integration/api_build_test.py index 8910eb758f..9423012734 100644 --- a/tests/integration/api_build_test.py +++ b/tests/integration/api_build_test.py @@ -452,7 +452,6 @@ def test_build_in_context_dockerfile(self): 'COPY . /src', 'WORKDIR /src', ])) - print(os.path.join(base_dir, 'custom.dockerfile')) img_name = random_name() self.tmp_imgs.append(img_name) stream = self.client.build( @@ -472,3 +471,35 @@ def test_build_in_context_dockerfile(self): assert sorted( [b'.', b'..', b'file.txt', b'custom.dockerfile'] ) == sorted(lsdata) + + def test_build_in_context_abs_dockerfile(self): + base_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, base_dir) + abs_dockerfile_path = os.path.join(base_dir, 'custom.dockerfile') + with open(os.path.join(base_dir, 'file.txt'), 'w') as f: + f.write('hello world') + with open(abs_dockerfile_path, 'w') as df: + df.write('\n'.join([ + 'FROM busybox', + 'COPY . /src', + 'WORKDIR /src', + ])) + img_name = random_name() + self.tmp_imgs.append(img_name) + stream = self.client.build( + path=base_dir, dockerfile=abs_dockerfile_path, tag=img_name, + decode=True + ) + lines = [] + for chunk in stream: + lines.append(chunk) + assert 'Successfully tagged' in lines[-1]['stream'] + + ctnr = self.client.create_container(img_name, 'ls -a') + self.tmp_containers.append(ctnr) + self.client.start(ctnr) + lsdata = self.client.logs(ctnr).strip().split(b'\n') + assert len(lsdata) == 4 + assert sorted( + [b'.', b'..', b'file.txt', b'custom.dockerfile'] + ) == sorted(lsdata) From e1ab5457ca05e22c4e1f80b60c3a038c0da6d19e Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 25 Apr 2018 14:02:53 -0700 Subject: [PATCH 0662/1301] Bump docker-pycreds dependency Signed-off-by: Joffrey F --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 2b281ae82f..9079315d40 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ asn1crypto==0.22.0 backports.ssl-match-hostname==3.5.0.1 cffi==1.10.0 cryptography==1.9 -docker-pycreds==0.2.2 +docker-pycreds==0.2.3 enum34==1.1.6 idna==2.5 ipaddress==1.0.18 diff --git a/setup.py b/setup.py index 1153f784bd..a65437efea 100644 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ 'requests >= 2.14.2, != 2.18.0', 'six >= 1.4.0', 'websocket-client >= 0.32.0', - 'docker-pycreds >= 0.2.2' + 'docker-pycreds >= 0.2.3' ] extras_require = { From accb9de52f6e383ad0335807f73c8c35bd6e7426 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 25 Apr 2018 14:49:18 -0700 Subject: [PATCH 0663/1301] Remove obsolete docker-py check in setup.py Signed-off-by: Joffrey F --- setup.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/setup.py b/setup.py index a65437efea..c1eabcf056 100644 --- a/setup.py +++ b/setup.py @@ -3,23 +3,9 @@ import codecs import os -import sys - -import pip from setuptools import setup, find_packages -try: - if 'docker-py' in [ - x.project_name for x in pip.get_installed_distributions()]: - print( - 'ERROR: "docker-py" needs to be uninstalled before installing this' - ' package:\npip uninstall docker-py', file=sys.stderr - ) - sys.exit(1) -except AttributeError: - pass - ROOT_DIR = os.path.dirname(__file__) SOURCE_DIR = os.path.join(ROOT_DIR) From caf0f37927a296174c3f00e9505d87f70ac8fa0d Mon Sep 17 00:00:00 2001 From: John Hu Date: Fri, 13 Apr 2018 14:40:39 +0800 Subject: [PATCH 0664/1301] Set minimum version for configs api to 1.30 See: https://docs.docker.com/engine/reference/commandline/config/ https://docs.docker.com/engine/api/v1.30/ Signed-off-by: John Hu --- docker/api/config.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docker/api/config.py b/docker/api/config.py index b46b09c7c1..767bef263a 100644 --- a/docker/api/config.py +++ b/docker/api/config.py @@ -6,7 +6,7 @@ class ConfigApiMixin(object): - @utils.minimum_version('1.25') + @utils.minimum_version('1.30') def create_config(self, name, data, labels=None): """ Create a config @@ -35,7 +35,7 @@ def create_config(self, name, data, labels=None): self._post_json(url, data=body), True ) - @utils.minimum_version('1.25') + @utils.minimum_version('1.30') @utils.check_resource('id') def inspect_config(self, id): """ @@ -53,7 +53,7 @@ def inspect_config(self, id): url = self._url('/configs/{0}', id) return self._result(self._get(url), True) - @utils.minimum_version('1.25') + @utils.minimum_version('1.30') @utils.check_resource('id') def remove_config(self, id): """ @@ -73,7 +73,7 @@ def remove_config(self, id): self._raise_for_status(res) return True - @utils.minimum_version('1.25') + @utils.minimum_version('1.30') def configs(self, filters=None): """ List configs From cef9940ed3d993145c6db075b2f7f0f005415ff2 Mon Sep 17 00:00:00 2001 From: Matthieu Nottale Date: Tue, 13 Mar 2018 15:31:36 +0100 Subject: [PATCH 0665/1301] stop(), restart(): Adjust request timeout. Signed-off-by: Matthieu Nottale --- docker/api/container.py | 11 ++++++++--- tests/integration/api_container_test.py | 11 +++++++++++ tests/unit/api_container_test.py | 4 ++-- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/docker/api/container.py b/docker/api/container.py index cb97b7940c..144a6d90d6 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -1018,7 +1018,10 @@ def restart(self, container, timeout=10): """ params = {'t': timeout} url = self._url("/containers/{0}/restart", container) - res = self._post(url, params=params) + conn_timeout = self.timeout + if conn_timeout: + conn_timeout = max(conn_timeout, timeout+15) + res = self._post(url, params=params, timeout=conn_timeout) self._raise_for_status(res) @utils.check_resource('container') @@ -1107,9 +1110,11 @@ def stop(self, container, timeout=None): else: params = {'t': timeout} url = self._url("/containers/{0}/stop", container) - + conn_timeout = self.timeout + if conn_timeout: + conn_timeout = max(conn_timeout, timeout + 15) res = self._post(url, params=params, - timeout=(timeout + (self.timeout or 0))) + timeout=conn_timeout) self._raise_for_status(res) @utils.check_resource('container') diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index e2125186d1..3d985a455b 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -1165,6 +1165,17 @@ def test_restart(self): assert info2['State']['Running'] is True self.client.kill(id) + def test_restart_with_hight_timeout(self): + container = self.client.create_container(BUSYBOX, ['sleep', '9999']) + id = container['Id'] + self.client.start(id) + self.client.timeout = 1 + self.client.restart(id, timeout=3) + self.client.timeout = None + self.client.restart(id, timeout=3) + self.client.timeout = 1 + self.client.stop(id, timeout=3) + def test_restart_with_dict_instead_of_id(self): container = self.client.create_container(BUSYBOX, ['sleep', '9999']) assert 'Id' in container diff --git a/tests/unit/api_container_test.py b/tests/unit/api_container_test.py index c33f129eff..1aed96774f 100644 --- a/tests/unit/api_container_test.py +++ b/tests/unit/api_container_test.py @@ -1264,7 +1264,7 @@ def test_stop_container(self): 'POST', url_prefix + 'containers/3cc2351ab11b/stop', params={'t': timeout}, - timeout=(DEFAULT_TIMEOUT_SECONDS + timeout) + timeout=(DEFAULT_TIMEOUT_SECONDS) ) def test_stop_container_with_dict_instead_of_id(self): @@ -1277,7 +1277,7 @@ def test_stop_container_with_dict_instead_of_id(self): 'POST', url_prefix + 'containers/3cc2351ab11b/stop', params={'t': timeout}, - timeout=(DEFAULT_TIMEOUT_SECONDS + timeout) + timeout=(DEFAULT_TIMEOUT_SECONDS) ) def test_pause_container(self): From da028d88a2f133d038fa4a651318b60bed770ba5 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 25 Apr 2018 14:36:41 -0700 Subject: [PATCH 0666/1301] Total timeout should be HTTP timeout + operation timeout Signed-off-by: Joffrey F --- docker/api/container.py | 12 ++++++------ tests/integration/api_container_test.py | 2 +- tests/unit/api_container_test.py | 8 ++++---- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/docker/api/container.py b/docker/api/container.py index 144a6d90d6..4a49bab883 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -1018,10 +1018,9 @@ def restart(self, container, timeout=10): """ params = {'t': timeout} url = self._url("/containers/{0}/restart", container) - conn_timeout = self.timeout - if conn_timeout: - conn_timeout = max(conn_timeout, timeout+15) - res = self._post(url, params=params, timeout=conn_timeout) + res = self._post( + url, params=params, timeout=timeout + (self.timeout or 0) + ) self._raise_for_status(res) @utils.check_resource('container') @@ -1113,8 +1112,9 @@ def stop(self, container, timeout=None): conn_timeout = self.timeout if conn_timeout: conn_timeout = max(conn_timeout, timeout + 15) - res = self._post(url, params=params, - timeout=conn_timeout) + res = self._post( + url, params=params, timeout=timeout + (self.timeout or 0) + ) self._raise_for_status(res) @utils.check_resource('container') diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index 3d985a455b..da9b3ec62c 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -1165,7 +1165,7 @@ def test_restart(self): assert info2['State']['Running'] is True self.client.kill(id) - def test_restart_with_hight_timeout(self): + def test_restart_with_high_timeout(self): container = self.client.create_container(BUSYBOX, ['sleep', '9999']) id = container['Id'] self.client.start(id) diff --git a/tests/unit/api_container_test.py b/tests/unit/api_container_test.py index 1aed96774f..a7e183c839 100644 --- a/tests/unit/api_container_test.py +++ b/tests/unit/api_container_test.py @@ -1264,7 +1264,7 @@ def test_stop_container(self): 'POST', url_prefix + 'containers/3cc2351ab11b/stop', params={'t': timeout}, - timeout=(DEFAULT_TIMEOUT_SECONDS) + timeout=(DEFAULT_TIMEOUT_SECONDS + timeout) ) def test_stop_container_with_dict_instead_of_id(self): @@ -1277,7 +1277,7 @@ def test_stop_container_with_dict_instead_of_id(self): 'POST', url_prefix + 'containers/3cc2351ab11b/stop', params={'t': timeout}, - timeout=(DEFAULT_TIMEOUT_SECONDS) + timeout=(DEFAULT_TIMEOUT_SECONDS + timeout) ) def test_pause_container(self): @@ -1335,7 +1335,7 @@ def test_restart_container(self): 'POST', url_prefix + 'containers/3cc2351ab11b/restart', params={'t': 2}, - timeout=DEFAULT_TIMEOUT_SECONDS + timeout=(DEFAULT_TIMEOUT_SECONDS + 2) ) def test_restart_container_with_dict_instead_of_id(self): @@ -1345,7 +1345,7 @@ def test_restart_container_with_dict_instead_of_id(self): 'POST', url_prefix + 'containers/3cc2351ab11b/restart', params={'t': 2}, - timeout=DEFAULT_TIMEOUT_SECONDS + timeout=(DEFAULT_TIMEOUT_SECONDS + 2) ) def test_remove_container(self): From ae8f77737c164d2474681f839c43f51400b9e119 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 25 Apr 2018 15:12:49 -0700 Subject: [PATCH 0667/1301] Fix session timeout = None case Signed-off-by: Joffrey F --- docker/api/container.py | 15 +++++++-------- tests/integration/api_container_test.py | 12 +++++------- 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/docker/api/container.py b/docker/api/container.py index 4a49bab883..05676f11e9 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -1018,9 +1018,10 @@ def restart(self, container, timeout=10): """ params = {'t': timeout} url = self._url("/containers/{0}/restart", container) - res = self._post( - url, params=params, timeout=timeout + (self.timeout or 0) - ) + conn_timeout = self.timeout + if conn_timeout is not None: + conn_timeout += timeout + res = self._post(url, params=params, timeout=conn_timeout) self._raise_for_status(res) @utils.check_resource('container') @@ -1110,11 +1111,9 @@ def stop(self, container, timeout=None): params = {'t': timeout} url = self._url("/containers/{0}/stop", container) conn_timeout = self.timeout - if conn_timeout: - conn_timeout = max(conn_timeout, timeout + 15) - res = self._post( - url, params=params, timeout=timeout + (self.timeout or 0) - ) + if conn_timeout is not None: + conn_timeout += timeout + res = self._post(url, params=params, timeout=conn_timeout) self._raise_for_status(res) @utils.check_resource('container') diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index da9b3ec62c..afd439f9ce 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -1165,16 +1165,14 @@ def test_restart(self): assert info2['State']['Running'] is True self.client.kill(id) - def test_restart_with_high_timeout(self): + def test_restart_with_low_timeout(self): container = self.client.create_container(BUSYBOX, ['sleep', '9999']) - id = container['Id'] - self.client.start(id) + self.client.start(container) self.client.timeout = 1 - self.client.restart(id, timeout=3) + self.client.restart(container, timeout=3) self.client.timeout = None - self.client.restart(id, timeout=3) - self.client.timeout = 1 - self.client.stop(id, timeout=3) + self.client.restart(container, timeout=3) + self.client.kill(container) def test_restart_with_dict_instead_of_id(self): container = self.client.create_container(BUSYBOX, ['sleep', '9999']) From 8360ecae973a84bbd203cfe145657618a5659415 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 25 Apr 2018 16:29:41 -0700 Subject: [PATCH 0668/1301] prune_builds test Signed-off-by: Joffrey F --- docker/api/build.py | 33 +++++++++++++++-------------- tests/integration/api_build_test.py | 6 ++++++ 2 files changed, 23 insertions(+), 16 deletions(-) diff --git a/docker/api/build.py b/docker/api/build.py index 3d83b982c8..f62a7319af 100644 --- a/docker/api/build.py +++ b/docker/api/build.py @@ -264,6 +264,23 @@ def build(self, path=None, tag=None, quiet=False, fileobj=None, return self._stream_helper(response, decode=decode) + @utils.minimum_version('1.31') + def prune_builds(self): + """ + Delete the builder cache + + Returns: + (dict): A dictionary containing information about the operation's + result. The ``SpaceReclaimed`` key indicates the amount of + bytes of disk space reclaimed. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + url = self._url("/build/prune") + return self._result(self._post(url), True) + def _set_auth_headers(self, headers): log.debug('Looking for auth config') @@ -305,22 +322,6 @@ def _set_auth_headers(self, headers): else: log.debug('No auth config found') - @utils.minimum_version('1.31') - def prune_build(self): - """ - Delete builder cache - - Returns: - (dict): A dict containing - the amount of disk space reclaimed in bytes. - - Raises: - :py:class:`docker.errors.APIError` - If the server returns an error. - """ - url = self._url("/build/prune") - return self._result(self._post(url), True) - def process_dockerfile(dockerfile, path): if not dockerfile: diff --git a/tests/integration/api_build_test.py b/tests/integration/api_build_test.py index 9423012734..92e0062544 100644 --- a/tests/integration/api_build_test.py +++ b/tests/integration/api_build_test.py @@ -503,3 +503,9 @@ def test_build_in_context_abs_dockerfile(self): assert sorted( [b'.', b'..', b'file.txt', b'custom.dockerfile'] ) == sorted(lsdata) + + @requires_api_version('1.31') + def test_prune_builds(self): + prune_result = self.client.prune_builds() + assert 'SpaceReclaimed' in prune_result + assert isinstance(prune_result['SpaceReclaimed'], int) From b3ae4d6ebd0674b9dff0abfef001e8fe47ccfd22 Mon Sep 17 00:00:00 2001 From: Ben Doan Date: Thu, 15 Mar 2018 18:21:52 -0500 Subject: [PATCH 0669/1301] avoid race condition in containers.list Signed-off-by: Ben Doan --- docker/models/containers.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/docker/models/containers.py b/docker/models/containers.py index 1e06ed6099..789fa93f6f 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -6,7 +6,7 @@ from ..constants import DEFAULT_DATA_CHUNK_SIZE from ..errors import ( ContainerError, DockerException, ImageNotFound, - create_unexpected_kwargs_error + NotFound, create_unexpected_kwargs_error ) from ..types import HostConfig from ..utils import version_gte @@ -896,7 +896,14 @@ def list(self, all=False, before=None, filters=None, limit=-1, since=None, if sparse: return [self.prepare_model(r) for r in resp] else: - return [self.get(r['Id']) for r in resp] + containers = [] + for r in resp: + try: + containers.append(self.get(r['Id'])) + # a container may have been removed while iterating + except NotFound: + pass + return containers def prune(self, filters=None): return self.client.api.prune_containers(filters=filters) From 9709dd454b0ce23db5af55ad4b1d35a2fb67cc45 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 25 Apr 2018 16:55:40 -0700 Subject: [PATCH 0670/1301] Add ignore_removed param to containers.list() to control whether to raise or ignore NotFound Signed-off-by: Joffrey F --- docker/models/containers.py | 9 +++++++-- tests/unit/fake_api_client.py | 16 +++++++++++----- tests/unit/models_containers_test.py | 12 ++++++++++++ 3 files changed, 30 insertions(+), 7 deletions(-) diff --git a/docker/models/containers.py b/docker/models/containers.py index 789fa93f6f..b33a718f75 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -844,7 +844,7 @@ def get(self, container_id): return self.prepare_model(resp) def list(self, all=False, before=None, filters=None, limit=-1, since=None, - sparse=False): + sparse=False, ignore_removed=False): """ List containers. Similar to the ``docker ps`` command. @@ -882,6 +882,10 @@ def list(self, all=False, before=None, filters=None, limit=-1, since=None, information, but guaranteed not to block. Use :py:meth:`Container.reload` on resulting objects to retrieve all attributes. Default: ``False`` + ignore_removed (bool): Ignore failures due to missing containers + when attempting to inspect containers from the original list. + Set to ``True`` if race conditions are likely. Has no effect + if ``sparse=True``. Default: ``False`` Returns: (list of :py:class:`Container`) @@ -902,7 +906,8 @@ def list(self, all=False, before=None, filters=None, limit=-1, since=None, containers.append(self.get(r['Id'])) # a container may have been removed while iterating except NotFound: - pass + if not ignore_removed: + raise return containers def prune(self, filters=None): diff --git a/tests/unit/fake_api_client.py b/tests/unit/fake_api_client.py index 15b60eaadc..2147bfdfa1 100644 --- a/tests/unit/fake_api_client.py +++ b/tests/unit/fake_api_client.py @@ -20,15 +20,18 @@ def _mock_call(self, *args, **kwargs): return ret -def make_fake_api_client(): +def make_fake_api_client(overrides=None): """ Returns non-complete fake APIClient. This returns most of the default cases correctly, but most arguments that change behaviour will not work. """ + + if overrides is None: + overrides = {} api_client = docker.APIClient() - mock_client = CopyReturnMagicMock(**{ + mock_attrs = { 'build.return_value': fake_api.FAKE_IMAGE_ID, 'commit.return_value': fake_api.post_fake_commit()[1], 'containers.return_value': fake_api.get_fake_containers()[1], @@ -47,15 +50,18 @@ def make_fake_api_client(): 'networks.return_value': fake_api.get_fake_network_list()[1], 'start.return_value': None, 'wait.return_value': {'StatusCode': 0}, - }) + } + mock_attrs.update(overrides) + mock_client = CopyReturnMagicMock(**mock_attrs) + mock_client._version = docker.constants.DEFAULT_DOCKER_API_VERSION return mock_client -def make_fake_client(): +def make_fake_client(overrides=None): """ Returns a Client with a fake APIClient. """ client = docker.DockerClient() - client.api = make_fake_api_client() + client.api = make_fake_api_client(overrides) return client diff --git a/tests/unit/models_containers_test.py b/tests/unit/models_containers_test.py index 2b0b499ef3..48a5288869 100644 --- a/tests/unit/models_containers_test.py +++ b/tests/unit/models_containers_test.py @@ -359,6 +359,18 @@ def test_list(self): assert isinstance(containers[0], Container) assert containers[0].id == FAKE_CONTAINER_ID + def test_list_ignore_removed(self): + def side_effect(*args, **kwargs): + raise docker.errors.NotFound('Container not found') + client = make_fake_client({ + 'inspect_container.side_effect': side_effect + }) + + with pytest.raises(docker.errors.NotFound): + client.containers.list(all=True, ignore_removed=False) + + assert client.containers.list(all=True, ignore_removed=True) == [] + class ContainerTest(unittest.TestCase): def test_name(self): From d5693ed903217950f37c32ef6f35a1a0a10f55d9 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 25 Apr 2018 17:28:51 -0700 Subject: [PATCH 0671/1301] Add prune_builds to DockerClient Signed-off-by: Joffrey F --- docker/models/images.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docker/models/images.py b/docker/models/images.py index d4893bb6a1..41632c6a36 100644 --- a/docker/models/images.py +++ b/docker/models/images.py @@ -432,6 +432,10 @@ def prune(self, filters=None): return self.client.api.prune_images(filters=filters) prune.__doc__ = APIClient.prune_images.__doc__ + def prune_builds(self, *args, **kwargs): + return self.client.api.prune_builds(*args, **kwargs) + prune_builds.__doc__ = APIClient.prune_builds.__doc__ + def normalize_platform(platform, engine_info): if platform is None: From 8b5f281e0031b37484e10ec646698fd403f275ce Mon Sep 17 00:00:00 2001 From: Ronald van Zantvoort Date: Sat, 24 Feb 2018 13:07:29 +0100 Subject: [PATCH 0672/1301] build_prune Signed-off-by: Ronald van Zantvoort --- docker/api/build.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docker/api/build.py b/docker/api/build.py index d69985e68b..32328ab053 100644 --- a/docker/api/build.py +++ b/docker/api/build.py @@ -305,6 +305,22 @@ def _set_auth_headers(self, headers): else: log.debug('No auth config found') + @utils.minimum_version('1.31') + def prune_build(self): + """ + Delete builder cache + + Returns: + (dict): A dict containing + the amount of disk space reclaimed in bytes. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + url = self._url("/build/prune") + return self._result(self._post(url), True) + def process_dockerfile(dockerfile, path): if not dockerfile: From a1587e77c5d47a1e4815f9c29a6eb36db8156cb4 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 5 Mar 2018 15:26:59 -0800 Subject: [PATCH 0673/1301] Bump test engine versions Signed-off-by: Joffrey F --- Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Jenkinsfile b/Jenkinsfile index 211159bc28..52c7f92779 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -5,6 +5,7 @@ def imageNamePy2 def imageNamePy3 def images = [:] + def buildImage = { name, buildargs, pyTag -> img = docker.image(name) try { From 0b289bc79e5e887d348d2a20918d8a5661063563 Mon Sep 17 00:00:00 2001 From: Matthieu Nottale Date: Wed, 14 Mar 2018 15:30:30 +0100 Subject: [PATCH 0674/1301] Add sparse argument to DockerClient.containers.list(). Signed-off-by: Matthieu Nottale --- docker/models/containers.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker/models/containers.py b/docker/models/containers.py index 1e06ed6099..2a46a69a43 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -873,6 +873,9 @@ def list(self, all=False, before=None, filters=None, limit=-1, since=None, container. Give the container name or id. - `since` (str): Only containers created after a particular container. Give container name or id. + sparse (bool): Do not inspect containers. Returns partial + informations, but guaranteed not to block. Use reload() on + each container to get the full list of attributes. A comprehensive list can be found in the documentation for `docker ps From 23e6b5c46bff30812dea733e8e616d10f8e30825 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 14 Mar 2018 16:53:56 -0700 Subject: [PATCH 0675/1301] Add test for container list with sparse=True Signed-off-by: Joffrey F --- docker/models/containers.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/docker/models/containers.py b/docker/models/containers.py index 2a46a69a43..1e06ed6099 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -873,9 +873,6 @@ def list(self, all=False, before=None, filters=None, limit=-1, since=None, container. Give the container name or id. - `since` (str): Only containers created after a particular container. Give container name or id. - sparse (bool): Do not inspect containers. Returns partial - informations, but guaranteed not to block. Use reload() on - each container to get the full list of attributes. A comprehensive list can be found in the documentation for `docker ps From 298b7e14743b9cf2111bfbf792e97968dc0097d1 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 15 Mar 2018 14:37:02 -0700 Subject: [PATCH 0676/1301] Use networks instead of legacy links for test setup Signed-off-by: Joffrey F --- Jenkinsfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 52c7f92779..211159bc28 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -5,7 +5,6 @@ def imageNamePy2 def imageNamePy3 def images = [:] - def buildImage = { name, buildargs, pyTag -> img = docker.image(name) try { From 33cae41d90533236643a339d75786da9a0b1a538 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 12 Apr 2018 12:38:27 -0700 Subject: [PATCH 0677/1301] Support absolute paths for in-context Dockerfiles Signed-off-by: Joffrey F --- docker/api/build.py | 6 ++++-- setup.py | 16 ++++++++------ tests/integration/api_build_test.py | 33 ++++++++++++++++++++++++++++- 3 files changed, 46 insertions(+), 9 deletions(-) diff --git a/docker/api/build.py b/docker/api/build.py index 32328ab053..3d83b982c8 100644 --- a/docker/api/build.py +++ b/docker/api/build.py @@ -332,10 +332,12 @@ def process_dockerfile(dockerfile, path): if (os.path.splitdrive(path)[0] != os.path.splitdrive(abs_dockerfile)[0] or os.path.relpath(abs_dockerfile, path).startswith('..')): + # Dockerfile not in context - read data to insert into tar later with open(abs_dockerfile, 'r') as df: return ( '.dockerfile.{0:x}'.format(random.getrandbits(160)), df.read() ) - else: - return (dockerfile, None) + + # Dockerfile is inside the context - return path relative to context root + return (os.path.relpath(abs_dockerfile, path), None) diff --git a/setup.py b/setup.py index 271d94f268..1153f784bd 100644 --- a/setup.py +++ b/setup.py @@ -9,12 +9,16 @@ from setuptools import setup, find_packages -if 'docker-py' in [x.project_name for x in pip.get_installed_distributions()]: - print( - 'ERROR: "docker-py" needs to be uninstalled before installing this' - ' package:\npip uninstall docker-py', file=sys.stderr - ) - sys.exit(1) +try: + if 'docker-py' in [ + x.project_name for x in pip.get_installed_distributions()]: + print( + 'ERROR: "docker-py" needs to be uninstalled before installing this' + ' package:\npip uninstall docker-py', file=sys.stderr + ) + sys.exit(1) +except AttributeError: + pass ROOT_DIR = os.path.dirname(__file__) SOURCE_DIR = os.path.join(ROOT_DIR) diff --git a/tests/integration/api_build_test.py b/tests/integration/api_build_test.py index 8910eb758f..9423012734 100644 --- a/tests/integration/api_build_test.py +++ b/tests/integration/api_build_test.py @@ -452,7 +452,6 @@ def test_build_in_context_dockerfile(self): 'COPY . /src', 'WORKDIR /src', ])) - print(os.path.join(base_dir, 'custom.dockerfile')) img_name = random_name() self.tmp_imgs.append(img_name) stream = self.client.build( @@ -472,3 +471,35 @@ def test_build_in_context_dockerfile(self): assert sorted( [b'.', b'..', b'file.txt', b'custom.dockerfile'] ) == sorted(lsdata) + + def test_build_in_context_abs_dockerfile(self): + base_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, base_dir) + abs_dockerfile_path = os.path.join(base_dir, 'custom.dockerfile') + with open(os.path.join(base_dir, 'file.txt'), 'w') as f: + f.write('hello world') + with open(abs_dockerfile_path, 'w') as df: + df.write('\n'.join([ + 'FROM busybox', + 'COPY . /src', + 'WORKDIR /src', + ])) + img_name = random_name() + self.tmp_imgs.append(img_name) + stream = self.client.build( + path=base_dir, dockerfile=abs_dockerfile_path, tag=img_name, + decode=True + ) + lines = [] + for chunk in stream: + lines.append(chunk) + assert 'Successfully tagged' in lines[-1]['stream'] + + ctnr = self.client.create_container(img_name, 'ls -a') + self.tmp_containers.append(ctnr) + self.client.start(ctnr) + lsdata = self.client.logs(ctnr).strip().split(b'\n') + assert len(lsdata) == 4 + assert sorted( + [b'.', b'..', b'file.txt', b'custom.dockerfile'] + ) == sorted(lsdata) From 0dbb5da9223ac11062292e38683e3a4a68029002 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 25 Apr 2018 14:02:53 -0700 Subject: [PATCH 0678/1301] Bump docker-pycreds dependency Signed-off-by: Joffrey F --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 2b281ae82f..9079315d40 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ asn1crypto==0.22.0 backports.ssl-match-hostname==3.5.0.1 cffi==1.10.0 cryptography==1.9 -docker-pycreds==0.2.2 +docker-pycreds==0.2.3 enum34==1.1.6 idna==2.5 ipaddress==1.0.18 diff --git a/setup.py b/setup.py index 1153f784bd..a65437efea 100644 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ 'requests >= 2.14.2, != 2.18.0', 'six >= 1.4.0', 'websocket-client >= 0.32.0', - 'docker-pycreds >= 0.2.2' + 'docker-pycreds >= 0.2.3' ] extras_require = { From d14dfedcadf0236f2820583b3bcd04b2fed35049 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 25 Apr 2018 14:49:18 -0700 Subject: [PATCH 0679/1301] Remove obsolete docker-py check in setup.py Signed-off-by: Joffrey F --- setup.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/setup.py b/setup.py index a65437efea..c1eabcf056 100644 --- a/setup.py +++ b/setup.py @@ -3,23 +3,9 @@ import codecs import os -import sys - -import pip from setuptools import setup, find_packages -try: - if 'docker-py' in [ - x.project_name for x in pip.get_installed_distributions()]: - print( - 'ERROR: "docker-py" needs to be uninstalled before installing this' - ' package:\npip uninstall docker-py', file=sys.stderr - ) - sys.exit(1) -except AttributeError: - pass - ROOT_DIR = os.path.dirname(__file__) SOURCE_DIR = os.path.join(ROOT_DIR) From c0c46a3d14b6dc0980bb235eb1c4089f7bcbe5ce Mon Sep 17 00:00:00 2001 From: John Hu Date: Fri, 13 Apr 2018 14:40:39 +0800 Subject: [PATCH 0680/1301] Set minimum version for configs api to 1.30 See: https://docs.docker.com/engine/reference/commandline/config/ https://docs.docker.com/engine/api/v1.30/ Signed-off-by: John Hu --- docker/api/config.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docker/api/config.py b/docker/api/config.py index b46b09c7c1..767bef263a 100644 --- a/docker/api/config.py +++ b/docker/api/config.py @@ -6,7 +6,7 @@ class ConfigApiMixin(object): - @utils.minimum_version('1.25') + @utils.minimum_version('1.30') def create_config(self, name, data, labels=None): """ Create a config @@ -35,7 +35,7 @@ def create_config(self, name, data, labels=None): self._post_json(url, data=body), True ) - @utils.minimum_version('1.25') + @utils.minimum_version('1.30') @utils.check_resource('id') def inspect_config(self, id): """ @@ -53,7 +53,7 @@ def inspect_config(self, id): url = self._url('/configs/{0}', id) return self._result(self._get(url), True) - @utils.minimum_version('1.25') + @utils.minimum_version('1.30') @utils.check_resource('id') def remove_config(self, id): """ @@ -73,7 +73,7 @@ def remove_config(self, id): self._raise_for_status(res) return True - @utils.minimum_version('1.25') + @utils.minimum_version('1.30') def configs(self, filters=None): """ List configs From 1848d2c9492a7d97d1eec6baa95d7678d8d6a382 Mon Sep 17 00:00:00 2001 From: Matthieu Nottale Date: Tue, 13 Mar 2018 15:31:36 +0100 Subject: [PATCH 0681/1301] stop(), restart(): Adjust request timeout. Signed-off-by: Matthieu Nottale --- docker/api/container.py | 11 ++++++++--- tests/integration/api_container_test.py | 11 +++++++++++ tests/unit/api_container_test.py | 4 ++-- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/docker/api/container.py b/docker/api/container.py index cb97b7940c..144a6d90d6 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -1018,7 +1018,10 @@ def restart(self, container, timeout=10): """ params = {'t': timeout} url = self._url("/containers/{0}/restart", container) - res = self._post(url, params=params) + conn_timeout = self.timeout + if conn_timeout: + conn_timeout = max(conn_timeout, timeout+15) + res = self._post(url, params=params, timeout=conn_timeout) self._raise_for_status(res) @utils.check_resource('container') @@ -1107,9 +1110,11 @@ def stop(self, container, timeout=None): else: params = {'t': timeout} url = self._url("/containers/{0}/stop", container) - + conn_timeout = self.timeout + if conn_timeout: + conn_timeout = max(conn_timeout, timeout + 15) res = self._post(url, params=params, - timeout=(timeout + (self.timeout or 0))) + timeout=conn_timeout) self._raise_for_status(res) @utils.check_resource('container') diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index e2125186d1..3d985a455b 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -1165,6 +1165,17 @@ def test_restart(self): assert info2['State']['Running'] is True self.client.kill(id) + def test_restart_with_hight_timeout(self): + container = self.client.create_container(BUSYBOX, ['sleep', '9999']) + id = container['Id'] + self.client.start(id) + self.client.timeout = 1 + self.client.restart(id, timeout=3) + self.client.timeout = None + self.client.restart(id, timeout=3) + self.client.timeout = 1 + self.client.stop(id, timeout=3) + def test_restart_with_dict_instead_of_id(self): container = self.client.create_container(BUSYBOX, ['sleep', '9999']) assert 'Id' in container diff --git a/tests/unit/api_container_test.py b/tests/unit/api_container_test.py index c33f129eff..1aed96774f 100644 --- a/tests/unit/api_container_test.py +++ b/tests/unit/api_container_test.py @@ -1264,7 +1264,7 @@ def test_stop_container(self): 'POST', url_prefix + 'containers/3cc2351ab11b/stop', params={'t': timeout}, - timeout=(DEFAULT_TIMEOUT_SECONDS + timeout) + timeout=(DEFAULT_TIMEOUT_SECONDS) ) def test_stop_container_with_dict_instead_of_id(self): @@ -1277,7 +1277,7 @@ def test_stop_container_with_dict_instead_of_id(self): 'POST', url_prefix + 'containers/3cc2351ab11b/stop', params={'t': timeout}, - timeout=(DEFAULT_TIMEOUT_SECONDS + timeout) + timeout=(DEFAULT_TIMEOUT_SECONDS) ) def test_pause_container(self): From 2454e810268f154c7384054382b71becc2dce30d Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 25 Apr 2018 14:36:41 -0700 Subject: [PATCH 0682/1301] Total timeout should be HTTP timeout + operation timeout Signed-off-by: Joffrey F --- docker/api/container.py | 12 ++++++------ tests/integration/api_container_test.py | 2 +- tests/unit/api_container_test.py | 8 ++++---- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/docker/api/container.py b/docker/api/container.py index 144a6d90d6..4a49bab883 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -1018,10 +1018,9 @@ def restart(self, container, timeout=10): """ params = {'t': timeout} url = self._url("/containers/{0}/restart", container) - conn_timeout = self.timeout - if conn_timeout: - conn_timeout = max(conn_timeout, timeout+15) - res = self._post(url, params=params, timeout=conn_timeout) + res = self._post( + url, params=params, timeout=timeout + (self.timeout or 0) + ) self._raise_for_status(res) @utils.check_resource('container') @@ -1113,8 +1112,9 @@ def stop(self, container, timeout=None): conn_timeout = self.timeout if conn_timeout: conn_timeout = max(conn_timeout, timeout + 15) - res = self._post(url, params=params, - timeout=conn_timeout) + res = self._post( + url, params=params, timeout=timeout + (self.timeout or 0) + ) self._raise_for_status(res) @utils.check_resource('container') diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index 3d985a455b..da9b3ec62c 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -1165,7 +1165,7 @@ def test_restart(self): assert info2['State']['Running'] is True self.client.kill(id) - def test_restart_with_hight_timeout(self): + def test_restart_with_high_timeout(self): container = self.client.create_container(BUSYBOX, ['sleep', '9999']) id = container['Id'] self.client.start(id) diff --git a/tests/unit/api_container_test.py b/tests/unit/api_container_test.py index 1aed96774f..a7e183c839 100644 --- a/tests/unit/api_container_test.py +++ b/tests/unit/api_container_test.py @@ -1264,7 +1264,7 @@ def test_stop_container(self): 'POST', url_prefix + 'containers/3cc2351ab11b/stop', params={'t': timeout}, - timeout=(DEFAULT_TIMEOUT_SECONDS) + timeout=(DEFAULT_TIMEOUT_SECONDS + timeout) ) def test_stop_container_with_dict_instead_of_id(self): @@ -1277,7 +1277,7 @@ def test_stop_container_with_dict_instead_of_id(self): 'POST', url_prefix + 'containers/3cc2351ab11b/stop', params={'t': timeout}, - timeout=(DEFAULT_TIMEOUT_SECONDS) + timeout=(DEFAULT_TIMEOUT_SECONDS + timeout) ) def test_pause_container(self): @@ -1335,7 +1335,7 @@ def test_restart_container(self): 'POST', url_prefix + 'containers/3cc2351ab11b/restart', params={'t': 2}, - timeout=DEFAULT_TIMEOUT_SECONDS + timeout=(DEFAULT_TIMEOUT_SECONDS + 2) ) def test_restart_container_with_dict_instead_of_id(self): @@ -1345,7 +1345,7 @@ def test_restart_container_with_dict_instead_of_id(self): 'POST', url_prefix + 'containers/3cc2351ab11b/restart', params={'t': 2}, - timeout=DEFAULT_TIMEOUT_SECONDS + timeout=(DEFAULT_TIMEOUT_SECONDS + 2) ) def test_remove_container(self): From f2c1ae37dd7eca4dde9054178388aec6b15c5229 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 25 Apr 2018 15:12:49 -0700 Subject: [PATCH 0683/1301] Fix session timeout = None case Signed-off-by: Joffrey F --- docker/api/container.py | 15 +++++++-------- tests/integration/api_container_test.py | 12 +++++------- 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/docker/api/container.py b/docker/api/container.py index 4a49bab883..05676f11e9 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -1018,9 +1018,10 @@ def restart(self, container, timeout=10): """ params = {'t': timeout} url = self._url("/containers/{0}/restart", container) - res = self._post( - url, params=params, timeout=timeout + (self.timeout or 0) - ) + conn_timeout = self.timeout + if conn_timeout is not None: + conn_timeout += timeout + res = self._post(url, params=params, timeout=conn_timeout) self._raise_for_status(res) @utils.check_resource('container') @@ -1110,11 +1111,9 @@ def stop(self, container, timeout=None): params = {'t': timeout} url = self._url("/containers/{0}/stop", container) conn_timeout = self.timeout - if conn_timeout: - conn_timeout = max(conn_timeout, timeout + 15) - res = self._post( - url, params=params, timeout=timeout + (self.timeout or 0) - ) + if conn_timeout is not None: + conn_timeout += timeout + res = self._post(url, params=params, timeout=conn_timeout) self._raise_for_status(res) @utils.check_resource('container') diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index da9b3ec62c..afd439f9ce 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -1165,16 +1165,14 @@ def test_restart(self): assert info2['State']['Running'] is True self.client.kill(id) - def test_restart_with_high_timeout(self): + def test_restart_with_low_timeout(self): container = self.client.create_container(BUSYBOX, ['sleep', '9999']) - id = container['Id'] - self.client.start(id) + self.client.start(container) self.client.timeout = 1 - self.client.restart(id, timeout=3) + self.client.restart(container, timeout=3) self.client.timeout = None - self.client.restart(id, timeout=3) - self.client.timeout = 1 - self.client.stop(id, timeout=3) + self.client.restart(container, timeout=3) + self.client.kill(container) def test_restart_with_dict_instead_of_id(self): container = self.client.create_container(BUSYBOX, ['sleep', '9999']) From cc953cf8bf2049493c2eab252fd9fe22de8af62b Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 25 Apr 2018 16:29:41 -0700 Subject: [PATCH 0684/1301] prune_builds test Signed-off-by: Joffrey F --- docker/api/build.py | 33 +++++++++++++++-------------- tests/integration/api_build_test.py | 6 ++++++ 2 files changed, 23 insertions(+), 16 deletions(-) diff --git a/docker/api/build.py b/docker/api/build.py index 3d83b982c8..f62a7319af 100644 --- a/docker/api/build.py +++ b/docker/api/build.py @@ -264,6 +264,23 @@ def build(self, path=None, tag=None, quiet=False, fileobj=None, return self._stream_helper(response, decode=decode) + @utils.minimum_version('1.31') + def prune_builds(self): + """ + Delete the builder cache + + Returns: + (dict): A dictionary containing information about the operation's + result. The ``SpaceReclaimed`` key indicates the amount of + bytes of disk space reclaimed. + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + url = self._url("/build/prune") + return self._result(self._post(url), True) + def _set_auth_headers(self, headers): log.debug('Looking for auth config') @@ -305,22 +322,6 @@ def _set_auth_headers(self, headers): else: log.debug('No auth config found') - @utils.minimum_version('1.31') - def prune_build(self): - """ - Delete builder cache - - Returns: - (dict): A dict containing - the amount of disk space reclaimed in bytes. - - Raises: - :py:class:`docker.errors.APIError` - If the server returns an error. - """ - url = self._url("/build/prune") - return self._result(self._post(url), True) - def process_dockerfile(dockerfile, path): if not dockerfile: diff --git a/tests/integration/api_build_test.py b/tests/integration/api_build_test.py index 9423012734..92e0062544 100644 --- a/tests/integration/api_build_test.py +++ b/tests/integration/api_build_test.py @@ -503,3 +503,9 @@ def test_build_in_context_abs_dockerfile(self): assert sorted( [b'.', b'..', b'file.txt', b'custom.dockerfile'] ) == sorted(lsdata) + + @requires_api_version('1.31') + def test_prune_builds(self): + prune_result = self.client.prune_builds() + assert 'SpaceReclaimed' in prune_result + assert isinstance(prune_result['SpaceReclaimed'], int) From eacf9f6d08537be2fad6549af4b050fdd65fbef0 Mon Sep 17 00:00:00 2001 From: Ben Doan Date: Thu, 15 Mar 2018 18:21:52 -0500 Subject: [PATCH 0685/1301] avoid race condition in containers.list Signed-off-by: Ben Doan --- docker/models/containers.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/docker/models/containers.py b/docker/models/containers.py index 1e06ed6099..789fa93f6f 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -6,7 +6,7 @@ from ..constants import DEFAULT_DATA_CHUNK_SIZE from ..errors import ( ContainerError, DockerException, ImageNotFound, - create_unexpected_kwargs_error + NotFound, create_unexpected_kwargs_error ) from ..types import HostConfig from ..utils import version_gte @@ -896,7 +896,14 @@ def list(self, all=False, before=None, filters=None, limit=-1, since=None, if sparse: return [self.prepare_model(r) for r in resp] else: - return [self.get(r['Id']) for r in resp] + containers = [] + for r in resp: + try: + containers.append(self.get(r['Id'])) + # a container may have been removed while iterating + except NotFound: + pass + return containers def prune(self, filters=None): return self.client.api.prune_containers(filters=filters) From 94e9108441936b4cd8afa66e91397baee29e6cfd Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 25 Apr 2018 16:55:40 -0700 Subject: [PATCH 0686/1301] Add ignore_removed param to containers.list() to control whether to raise or ignore NotFound Signed-off-by: Joffrey F --- docker/models/containers.py | 9 +++++++-- tests/unit/fake_api_client.py | 16 +++++++++++----- tests/unit/models_containers_test.py | 12 ++++++++++++ 3 files changed, 30 insertions(+), 7 deletions(-) diff --git a/docker/models/containers.py b/docker/models/containers.py index 789fa93f6f..b33a718f75 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -844,7 +844,7 @@ def get(self, container_id): return self.prepare_model(resp) def list(self, all=False, before=None, filters=None, limit=-1, since=None, - sparse=False): + sparse=False, ignore_removed=False): """ List containers. Similar to the ``docker ps`` command. @@ -882,6 +882,10 @@ def list(self, all=False, before=None, filters=None, limit=-1, since=None, information, but guaranteed not to block. Use :py:meth:`Container.reload` on resulting objects to retrieve all attributes. Default: ``False`` + ignore_removed (bool): Ignore failures due to missing containers + when attempting to inspect containers from the original list. + Set to ``True`` if race conditions are likely. Has no effect + if ``sparse=True``. Default: ``False`` Returns: (list of :py:class:`Container`) @@ -902,7 +906,8 @@ def list(self, all=False, before=None, filters=None, limit=-1, since=None, containers.append(self.get(r['Id'])) # a container may have been removed while iterating except NotFound: - pass + if not ignore_removed: + raise return containers def prune(self, filters=None): diff --git a/tests/unit/fake_api_client.py b/tests/unit/fake_api_client.py index 15b60eaadc..2147bfdfa1 100644 --- a/tests/unit/fake_api_client.py +++ b/tests/unit/fake_api_client.py @@ -20,15 +20,18 @@ def _mock_call(self, *args, **kwargs): return ret -def make_fake_api_client(): +def make_fake_api_client(overrides=None): """ Returns non-complete fake APIClient. This returns most of the default cases correctly, but most arguments that change behaviour will not work. """ + + if overrides is None: + overrides = {} api_client = docker.APIClient() - mock_client = CopyReturnMagicMock(**{ + mock_attrs = { 'build.return_value': fake_api.FAKE_IMAGE_ID, 'commit.return_value': fake_api.post_fake_commit()[1], 'containers.return_value': fake_api.get_fake_containers()[1], @@ -47,15 +50,18 @@ def make_fake_api_client(): 'networks.return_value': fake_api.get_fake_network_list()[1], 'start.return_value': None, 'wait.return_value': {'StatusCode': 0}, - }) + } + mock_attrs.update(overrides) + mock_client = CopyReturnMagicMock(**mock_attrs) + mock_client._version = docker.constants.DEFAULT_DOCKER_API_VERSION return mock_client -def make_fake_client(): +def make_fake_client(overrides=None): """ Returns a Client with a fake APIClient. """ client = docker.DockerClient() - client.api = make_fake_api_client() + client.api = make_fake_api_client(overrides) return client diff --git a/tests/unit/models_containers_test.py b/tests/unit/models_containers_test.py index 2b0b499ef3..48a5288869 100644 --- a/tests/unit/models_containers_test.py +++ b/tests/unit/models_containers_test.py @@ -359,6 +359,18 @@ def test_list(self): assert isinstance(containers[0], Container) assert containers[0].id == FAKE_CONTAINER_ID + def test_list_ignore_removed(self): + def side_effect(*args, **kwargs): + raise docker.errors.NotFound('Container not found') + client = make_fake_client({ + 'inspect_container.side_effect': side_effect + }) + + with pytest.raises(docker.errors.NotFound): + client.containers.list(all=True, ignore_removed=False) + + assert client.containers.list(all=True, ignore_removed=True) == [] + class ContainerTest(unittest.TestCase): def test_name(self): From 9d86194ddaa77c9c4911cc2324653639d4f376ed Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 25 Apr 2018 17:28:51 -0700 Subject: [PATCH 0687/1301] Add prune_builds to DockerClient Signed-off-by: Joffrey F --- docker/models/images.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docker/models/images.py b/docker/models/images.py index d4893bb6a1..41632c6a36 100644 --- a/docker/models/images.py +++ b/docker/models/images.py @@ -432,6 +432,10 @@ def prune(self, filters=None): return self.client.api.prune_images(filters=filters) prune.__doc__ = APIClient.prune_images.__doc__ + def prune_builds(self, *args, **kwargs): + return self.client.api.prune_builds(*args, **kwargs) + prune_builds.__doc__ = APIClient.prune_builds.__doc__ + def normalize_platform(platform, engine_info): if platform is None: From e88751cb9a235f31ec946c199b952b69dcc4cc0b Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 25 Apr 2018 17:35:11 -0700 Subject: [PATCH 0688/1301] Bump 3.3.0 Signed-off-by: Joffrey F --- docker/version.py | 2 +- docs/change-log.md | 22 +++++++++++++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/docker/version.py b/docker/version.py index 28dd1ea486..8f6e6514aa 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "3.2.1" +version = "3.3.0" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) diff --git a/docs/change-log.md b/docs/change-log.md index 912ec5ff80..0065c62d09 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -1,6 +1,26 @@ Change log ========== +3.3.0 +----- + +[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/49?closed=1) + +### Features + +* Added support for `prune_builds` in `APIClient` and `DockerClient.images` +* Added support for `ignore_removed` parameter in + `DockerClient.containers.list` + +### Bugfixes + +* Fixed an issue that caused builds to fail when an in-context Dockerfile + would be specified using its absolute path +* Installation with pip 10.0.0 and above no longer fails +* Connection timeout for `stop` and `restart` now gets properly adjusted to + allow for the operation to finish in the specified time +* Improved docker credential store support on Windows + 3.2.1 ----- @@ -16,7 +36,7 @@ Change log 3.2.0 ----- -[List of PRs/ issues for this release](https://github.com/docker/docker-py/milestone/45?closed=1) +[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/45?closed=1) ### Features From 467cacb00d8dce68aa8ff2bdacc85acecd2d1207 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 25 Apr 2018 18:03:15 -0700 Subject: [PATCH 0689/1301] 3.4.0-dev Signed-off-by: Joffrey F --- docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index 8f6e6514aa..04fd3c28d4 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "3.3.0" +version = "3.4.0-dev" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) From 8228059f1e070e490b690dea539b065c449b0194 Mon Sep 17 00:00:00 2001 From: Srinivas Reddy Thatiparthy Date: Thu, 24 May 2018 23:31:31 +0530 Subject: [PATCH 0690/1301] return the pruned networks Signed-off-by: Srinivas Reddy Thatiparthy --- docker/models/networks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/models/networks.py b/docker/models/networks.py index 1c2fbf2465..be3291a417 100644 --- a/docker/models/networks.py +++ b/docker/models/networks.py @@ -211,5 +211,5 @@ def list(self, *args, **kwargs): return networks def prune(self, filters=None): - self.client.api.prune_networks(filters=filters) + return self.client.api.prune_networks(filters=filters) prune.__doc__ = APIClient.prune_networks.__doc__ From 17f41b56726957177724711b5bff6d51b02e6d93 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 24 May 2018 17:19:18 -0700 Subject: [PATCH 0691/1301] Avoid unwanted modification of dockerfile path Signed-off-by: Joffrey F --- docker/api/build.py | 7 +++- tests/integration/api_build_test.py | 53 ++++++++++++++++++++++++----- 2 files changed, 50 insertions(+), 10 deletions(-) diff --git a/docker/api/build.py b/docker/api/build.py index f62a7319af..3c3f130f98 100644 --- a/docker/api/build.py +++ b/docker/api/build.py @@ -341,4 +341,9 @@ def process_dockerfile(dockerfile, path): ) # Dockerfile is inside the context - return path relative to context root - return (os.path.relpath(abs_dockerfile, path), None) + if dockerfile == abs_dockerfile: + # Only calculate relpath if necessary to avoid errors + # on Windows client -> Linux Docker + # see https://github.com/docker/compose/issues/5969 + dockerfile = os.path.relpath(abs_dockerfile, path) + return (dockerfile, None) diff --git a/tests/integration/api_build_test.py b/tests/integration/api_build_test.py index 92e0062544..baaf33e3d9 100644 --- a/tests/integration/api_build_test.py +++ b/tests/integration/api_build_test.py @@ -415,18 +415,20 @@ def test_build_out_of_context_dockerfile(self): f.write('hello world') with open(os.path.join(base_dir, '.dockerignore'), 'w') as f: f.write('.dockerignore\n') - df = tempfile.NamedTemporaryFile() - self.addCleanup(df.close) - df.write(('\n'.join([ - 'FROM busybox', - 'COPY . /src', - 'WORKDIR /src', - ])).encode('utf-8')) - df.flush() + df_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, df_dir) + df_name = os.path.join(df_dir, 'Dockerfile') + with open(df_name, 'wb') as df: + df.write(('\n'.join([ + 'FROM busybox', + 'COPY . /src', + 'WORKDIR /src', + ])).encode('utf-8')) + df.flush() img_name = random_name() self.tmp_imgs.append(img_name) stream = self.client.build( - path=base_dir, dockerfile=df.name, tag=img_name, + path=base_dir, dockerfile=df_name, tag=img_name, decode=True ) lines = [] @@ -472,6 +474,39 @@ def test_build_in_context_dockerfile(self): [b'.', b'..', b'file.txt', b'custom.dockerfile'] ) == sorted(lsdata) + def test_build_in_context_nested_dockerfile(self): + base_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, base_dir) + with open(os.path.join(base_dir, 'file.txt'), 'w') as f: + f.write('hello world') + subdir = os.path.join(base_dir, 'hello', 'world') + os.makedirs(subdir) + with open(os.path.join(subdir, 'custom.dockerfile'), 'w') as df: + df.write('\n'.join([ + 'FROM busybox', + 'COPY . /src', + 'WORKDIR /src', + ])) + img_name = random_name() + self.tmp_imgs.append(img_name) + stream = self.client.build( + path=base_dir, dockerfile='hello/world/custom.dockerfile', + tag=img_name, decode=True + ) + lines = [] + for chunk in stream: + lines.append(chunk) + assert 'Successfully tagged' in lines[-1]['stream'] + + ctnr = self.client.create_container(img_name, 'ls -a') + self.tmp_containers.append(ctnr) + self.client.start(ctnr) + lsdata = self.client.logs(ctnr).strip().split(b'\n') + assert len(lsdata) == 4 + assert sorted( + [b'.', b'..', b'file.txt', b'hello'] + ) == sorted(lsdata) + def test_build_in_context_abs_dockerfile(self): base_dir = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, base_dir) From 95ad903c35fc6781c18191b5cbc586ed4abd1d41 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 24 May 2018 17:20:45 -0700 Subject: [PATCH 0692/1301] Fix create_plugin on Windows Signed-off-by: Joffrey F --- docker/api/plugin.py | 5 ++++- tests/integration/api_plugin_test.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/docker/api/plugin.py b/docker/api/plugin.py index 73f185251e..33d7419964 100644 --- a/docker/api/plugin.py +++ b/docker/api/plugin.py @@ -44,7 +44,10 @@ def create_plugin(self, name, plugin_data_dir, gzip=False): """ url = self._url('/plugins/create') - with utils.create_archive(root=plugin_data_dir, gzip=gzip) as archv: + with utils.create_archive( + root=plugin_data_dir, gzip=gzip, + files=set(utils.build.walk(plugin_data_dir, [])) + ) as archv: res = self._post(url, params={'name': name}, data=archv) self._raise_for_status(res) return True diff --git a/tests/integration/api_plugin_test.py b/tests/integration/api_plugin_test.py index 433d44d101..1150b0957a 100644 --- a/tests/integration/api_plugin_test.py +++ b/tests/integration/api_plugin_test.py @@ -135,7 +135,7 @@ def test_upgrade_plugin(self): def test_create_plugin(self): plugin_data_dir = os.path.join( - os.path.dirname(__file__), 'testdata/dummy-plugin' + os.path.dirname(__file__), os.path.join('testdata', 'dummy-plugin') ) assert self.client.create_plugin( 'docker-sdk-py/dummy', plugin_data_dir From 40711cb50189c4c39ed5a60c16910646a00f9acc Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 24 May 2018 17:21:17 -0700 Subject: [PATCH 0693/1301] Fix cancellable streams on Windows clients + HTTPS transport Signed-off-by: Joffrey F --- docker/types/daemon.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/types/daemon.py b/docker/types/daemon.py index 852f3d8292..ee8624e80a 100644 --- a/docker/types/daemon.py +++ b/docker/types/daemon.py @@ -57,6 +57,8 @@ def close(self): else: sock = sock_fp._sock + if isinstance(sock, urllib3.contrib.pyopenssl.WrappedSocket): + sock = sock.socket sock.shutdown(socket.SHUT_RDWR) sock.close() From b4efdc1b28062c835d04ac56995cb293d74de92b Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 24 May 2018 17:21:53 -0700 Subject: [PATCH 0694/1301] Fix several integration tests on Windows Signed-off-by: Joffrey F --- tests/integration/api_container_test.py | 19 ++++--------------- tests/integration/models_containers_test.py | 3 +++ 2 files changed, 7 insertions(+), 15 deletions(-) diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index afd439f9ce..ff7014879c 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -491,6 +491,9 @@ def test_create_with_device_cgroup_rules(self): assert rule in self.client.logs(ctnr).decode('utf-8') +@pytest.mark.xfail( + IS_WINDOWS_PLATFORM, reason='Test not designed for Windows platform' +) class VolumeBindTest(BaseAPIIntegrationTest): def setUp(self): super(VolumeBindTest, self).setUp() @@ -507,9 +510,6 @@ def setUp(self): ['touch', os.path.join(self.mount_dest, self.filename)], ) - @pytest.mark.xfail( - IS_WINDOWS_PLATFORM, reason='Test not designed for Windows platform' - ) def test_create_with_binds_rw(self): container = self.run_with_volume( @@ -525,9 +525,6 @@ def test_create_with_binds_rw(self): inspect_data = self.client.inspect_container(container) self.check_container_data(inspect_data, True) - @pytest.mark.xfail( - IS_WINDOWS_PLATFORM, reason='Test not designed for Windows platform' - ) def test_create_with_binds_ro(self): self.run_with_volume( False, @@ -548,9 +545,6 @@ def test_create_with_binds_ro(self): inspect_data = self.client.inspect_container(container) self.check_container_data(inspect_data, False) - @pytest.mark.xfail( - IS_WINDOWS_PLATFORM, reason='Test not designed for Windows platform' - ) @requires_api_version('1.30') def test_create_with_mounts(self): mount = docker.types.Mount( @@ -569,9 +563,6 @@ def test_create_with_mounts(self): inspect_data = self.client.inspect_container(container) self.check_container_data(inspect_data, True) - @pytest.mark.xfail( - IS_WINDOWS_PLATFORM, reason='Test not designed for Windows platform' - ) @requires_api_version('1.30') def test_create_with_mounts_ro(self): mount = docker.types.Mount( @@ -1116,9 +1107,7 @@ def test_top(self): self.client.start(container) res = self.client.top(container) - if IS_WINDOWS_PLATFORM: - assert res['Titles'] == ['PID', 'USER', 'TIME', 'COMMAND'] - else: + if not IS_WINDOWS_PLATFORM: assert res['Titles'] == [ 'UID', 'PID', 'PPID', 'C', 'STIME', 'TTY', 'TIME', 'CMD' ] diff --git a/tests/integration/models_containers_test.py b/tests/integration/models_containers_test.py index 6ddb034b41..ab41ea57de 100644 --- a/tests/integration/models_containers_test.py +++ b/tests/integration/models_containers_test.py @@ -36,6 +36,9 @@ def test_run_with_image_that_does_not_exist(self): with pytest.raises(docker.errors.ImageNotFound): client.containers.run("dockerpytest_does_not_exist") + @pytest.mark.skipif( + docker.constants.IS_WINDOWS_PLATFORM, reason="host mounts on Windows" + ) def test_run_with_volume(self): client = docker.from_env(version=TEST_API_VERSION) path = tempfile.mkdtemp() From 22b7b76142bd735c6be4f678dda8cf9d413e9f1c Mon Sep 17 00:00:00 2001 From: Anthony Sottile Date: Tue, 1 May 2018 11:55:48 -0400 Subject: [PATCH 0695/1301] Use six.moves to handle a py2+py3 import Signed-off-by: Anthony Sottile --- docker/transport/unixconn.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/docker/transport/unixconn.py b/docker/transport/unixconn.py index cc35d00466..c59821a849 100644 --- a/docker/transport/unixconn.py +++ b/docker/transport/unixconn.py @@ -1,14 +1,10 @@ import six import requests.adapters import socket +from six.moves import http_client as httplib from .. import constants -if six.PY3: - import http.client as httplib -else: - import httplib - try: import requests.packages.urllib3 as urllib3 except ImportError: From 49bb7386a3a3752dca64b411a6996663ed04ea1e Mon Sep 17 00:00:00 2001 From: Mike Lee Date: Wed, 6 Jun 2018 02:57:16 +0800 Subject: [PATCH 0696/1301] query plugin privilege with registry auth header Signed-off-by: Mike Lee --- docker/api/plugin.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docker/api/plugin.py b/docker/api/plugin.py index 33d7419964..f6c0b13387 100644 --- a/docker/api/plugin.py +++ b/docker/api/plugin.py @@ -170,8 +170,16 @@ def plugin_privileges(self, name): 'remote': name, } + headers = {} + registry, repo_name = auth.resolve_repository_name(name) + header = auth.get_config_header(self, registry) + if header: + headers['X-Registry-Auth'] = header + url = self._url('/plugins/privileges') - return self._result(self._get(url, params=params), True) + return self._result( + self._get(url, params=params, headers=headers), True + ) @utils.minimum_version('1.25') @utils.check_resource('name') From dbe52dcb7d5765352faa43ab0210fddbcb546431 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 5 Jun 2018 10:51:54 -0700 Subject: [PATCH 0697/1301] Fix socket reading function for TCP (non-HTTPS) connections on Windows Signed-off-by: Joffrey F --- docker/utils/socket.py | 3 +++ tests/unit/api_test.py | 58 ++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 59 insertions(+), 2 deletions(-) diff --git a/docker/utils/socket.py b/docker/utils/socket.py index 0945f0a694..7b96d4fce6 100644 --- a/docker/utils/socket.py +++ b/docker/utils/socket.py @@ -1,6 +1,7 @@ import errno import os import select +import socket as pysocket import struct import six @@ -28,6 +29,8 @@ def read(socket, n=4096): try: if hasattr(socket, 'recv'): return socket.recv(n) + if six.PY3 and isinstance(socket, getattr(pysocket, 'SocketIO')): + return socket.read(n) return os.read(socket.fileno(), n) except EnvironmentError as e: if e.errno not in recoverable_errors: diff --git a/tests/unit/api_test.py b/tests/unit/api_test.py index 46cbd68de2..ba8084023d 100644 --- a/tests/unit/api_test.py +++ b/tests/unit/api_test.py @@ -365,7 +365,7 @@ def test_stream_helper_decoding(self): assert result == content -class StreamTest(unittest.TestCase): +class UnixSocketStreamTest(unittest.TestCase): def setUp(self): socket_dir = tempfile.mkdtemp() self.build_context = tempfile.mkdtemp() @@ -462,7 +462,61 @@ def test_early_stream_response(self): raise e assert list(stream) == [ - str(i).encode() for i in range(50)] + str(i).encode() for i in range(50) + ] + + +class TCPSocketStreamTest(unittest.TestCase): + text_data = b''' + Now, those children out there, they're jumping through the + flames in the hope that the god of the fire will make them fruitful. + Really, you can't blame them. After all, what girl would not prefer the + child of a god to that of some acne-scarred artisan? + ''' + + def setUp(self): + + self.server = six.moves.socketserver.ThreadingTCPServer( + ('', 0), self.get_handler_class() + ) + self.thread = threading.Thread(target=self.server.serve_forever) + self.thread.setDaemon(True) + self.thread.start() + self.address = 'http://{}:{}'.format( + socket.gethostname(), self.server.server_address[1] + ) + + def tearDown(self): + self.server.shutdown() + self.server.server_close() + self.thread.join() + + def get_handler_class(self): + text_data = self.text_data + + class Handler(six.moves.BaseHTTPServer.BaseHTTPRequestHandler, object): + def do_POST(self): + self.send_response(101) + self.send_header( + 'Content-Type', 'application/vnd.docker.raw-stream' + ) + self.send_header('Connection', 'Upgrade') + self.send_header('Upgrade', 'tcp') + self.end_headers() + self.wfile.flush() + time.sleep(0.2) + self.wfile.write(text_data) + self.wfile.flush() + + return Handler + + def test_read_from_socket(self): + with APIClient(base_url=self.address) as client: + resp = client._post(client._url('/dummy'), stream=True) + data = client._read_from_socket(resp, stream=True, tty=True) + results = b''.join(data) + + assert results == self.text_data class UserAgentTest(unittest.TestCase): From 2d0c5dd484e7621a9859ab40ac43d25a1f5f5078 Mon Sep 17 00:00:00 2001 From: Chris Mark Date: Tue, 5 Jun 2018 13:22:07 +0000 Subject: [PATCH 0698/1301] Adding missing comma in spec list. Fixing #2046, syntax error caused by missing comma on CONTAINER_SPEC_KWARGS list. Signed-off-by: Chris Mark --- docker/models/services.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/models/services.py b/docker/models/services.py index 125896bab9..2834dd7419 100644 --- a/docker/models/services.py +++ b/docker/models/services.py @@ -276,7 +276,7 @@ def list(self, **kwargs): 'labels', 'mounts', 'open_stdin', - 'privileges' + 'privileges', 'read_only', 'secrets', 'stop_grace_period', From f1189bfb4b1f2ecb6adc77f7349a085bdca1a824 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 8 Jun 2018 15:14:06 -0700 Subject: [PATCH 0699/1301] Allow passing of env overrides to credstore through APIClient ctor Signed-off-by: Joffrey F --- docker/api/build.py | 3 ++- docker/api/client.py | 6 +++++- docker/api/daemon.py | 4 +++- docker/auth.py | 13 ++++++++----- docker/client.py | 9 +++++++-- requirements.txt | 2 +- setup.py | 2 +- tests/unit/api_test.py | 2 +- 8 files changed, 28 insertions(+), 13 deletions(-) diff --git a/docker/api/build.py b/docker/api/build.py index 3c3f130f98..419255fc62 100644 --- a/docker/api/build.py +++ b/docker/api/build.py @@ -302,7 +302,8 @@ def _set_auth_headers(self, headers): # credentials/native_store.go#L68-L83 for registry in self._auth_configs.get('auths', {}).keys(): auth_data[registry] = auth.resolve_authconfig( - self._auth_configs, registry + self._auth_configs, registry, + credstore_env=self.credstore_env, ) else: auth_data = self._auth_configs.get('auths', {}).copy() diff --git a/docker/api/client.py b/docker/api/client.py index 13c292a0a5..91da1c893b 100644 --- a/docker/api/client.py +++ b/docker/api/client.py @@ -83,6 +83,8 @@ class APIClient( :py:class:`~docker.tls.TLSConfig` object to use custom configuration. user_agent (str): Set a custom user agent for requests to the server. + credstore_env (dict): Override environment variables when calling the + credential store process. """ __attrs__ = requests.Session.__attrs__ + ['_auth_configs', @@ -93,7 +95,8 @@ class APIClient( def __init__(self, base_url=None, version=None, timeout=DEFAULT_TIMEOUT_SECONDS, tls=False, - user_agent=DEFAULT_USER_AGENT, num_pools=DEFAULT_NUM_POOLS): + user_agent=DEFAULT_USER_AGENT, num_pools=DEFAULT_NUM_POOLS, + credstore_env=None): super(APIClient, self).__init__() if tls and not base_url: @@ -109,6 +112,7 @@ def __init__(self, base_url=None, version=None, self._auth_configs = auth.load_config( config_dict=self._general_configs ) + self.credstore_env = credstore_env base_url = utils.parse_host( base_url, IS_WINDOWS_PLATFORM, tls=bool(tls) diff --git a/docker/api/daemon.py b/docker/api/daemon.py index fc3692c248..76a94cf034 100644 --- a/docker/api/daemon.py +++ b/docker/api/daemon.py @@ -128,7 +128,9 @@ def login(self, username, password=None, email=None, registry=None, elif not self._auth_configs: self._auth_configs = auth.load_config() - authcfg = auth.resolve_authconfig(self._auth_configs, registry) + authcfg = auth.resolve_authconfig( + self._auth_configs, registry, credstore_env=self.credstore_env, + ) # If we found an existing auth config for this registry and username # combination, we can return it immediately unless reauth is requested. if authcfg and authcfg.get('username', None) == username \ diff --git a/docker/auth.py b/docker/auth.py index 48fcd8b504..0c0cb204d9 100644 --- a/docker/auth.py +++ b/docker/auth.py @@ -44,7 +44,9 @@ def get_config_header(client, registry): "No auth config in memory - loading from filesystem" ) client._auth_configs = load_config() - authcfg = resolve_authconfig(client._auth_configs, registry) + authcfg = resolve_authconfig( + client._auth_configs, registry, credstore_env=client.credstore_env + ) # Do not fail here if no authentication exists for this # specific registry as we can have a readonly pull. Just # put the header if we can. @@ -76,7 +78,7 @@ def get_credential_store(authconfig, registry): ) -def resolve_authconfig(authconfig, registry=None): +def resolve_authconfig(authconfig, registry=None, credstore_env=None): """ Returns the authentication data from the given auth configuration for a specific registry. As with the Docker client, legacy entries in the config @@ -91,7 +93,7 @@ def resolve_authconfig(authconfig, registry=None): 'Using credentials store "{0}"'.format(store_name) ) cfg = _resolve_authconfig_credstore( - authconfig, registry, store_name + authconfig, registry, store_name, env=credstore_env ) if cfg is not None: return cfg @@ -115,13 +117,14 @@ def resolve_authconfig(authconfig, registry=None): return None -def _resolve_authconfig_credstore(authconfig, registry, credstore_name): +def _resolve_authconfig_credstore(authconfig, registry, credstore_name, + env=None): if not registry or registry == INDEX_NAME: # The ecosystem is a little schizophrenic with index.docker.io VS # docker.io - in that case, it seems the full URL is necessary. registry = INDEX_URL log.debug("Looking for auth entry for {0}".format(repr(registry))) - store = dockerpycreds.Store(credstore_name) + store = dockerpycreds.Store(credstore_name, environment=env) try: data = store.get(registry) res = { diff --git a/docker/client.py b/docker/client.py index b4364c3c07..8d4a52b2ef 100644 --- a/docker/client.py +++ b/docker/client.py @@ -33,6 +33,8 @@ class DockerClient(object): :py:class:`~docker.tls.TLSConfig` object to use custom configuration. user_agent (str): Set a custom user agent for requests to the server. + credstore_env (dict): Override environment variables when calling the + credential store process. """ def __init__(self, *args, **kwargs): self.api = APIClient(*args, **kwargs) @@ -66,6 +68,8 @@ def from_env(cls, **kwargs): assert_hostname (bool): Verify the hostname of the server. environment (dict): The environment to read environment variables from. Default: the value of ``os.environ`` + credstore_env (dict): Override environment variables when calling + the credential store process. Example: @@ -77,8 +81,9 @@ def from_env(cls, **kwargs): """ timeout = kwargs.pop('timeout', DEFAULT_TIMEOUT_SECONDS) version = kwargs.pop('version', None) - return cls(timeout=timeout, version=version, - **kwargs_from_env(**kwargs)) + return cls( + timeout=timeout, version=version, **kwargs_from_env(**kwargs) + ) # Resources @property diff --git a/requirements.txt b/requirements.txt index 9079315d40..6c5e7d03be 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ asn1crypto==0.22.0 backports.ssl-match-hostname==3.5.0.1 cffi==1.10.0 cryptography==1.9 -docker-pycreds==0.2.3 +docker-pycreds==0.3.0 enum34==1.1.6 idna==2.5 ipaddress==1.0.18 diff --git a/setup.py b/setup.py index c1eabcf056..57b2b5a813 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ 'requests >= 2.14.2, != 2.18.0', 'six >= 1.4.0', 'websocket-client >= 0.32.0', - 'docker-pycreds >= 0.2.3' + 'docker-pycreds >= 0.3.0' ] extras_require = { diff --git a/tests/unit/api_test.py b/tests/unit/api_test.py index ba8084023d..af2bb1c202 100644 --- a/tests/unit/api_test.py +++ b/tests/unit/api_test.py @@ -44,7 +44,7 @@ def response(status_code=200, content='', headers=None, reason=None, elapsed=0, return res -def fake_resolve_authconfig(authconfig, registry=None): +def fake_resolve_authconfig(authconfig, registry=None, *args, **kwargs): return None From 76471c6519204e2c761a57fbefc565f0ea23dc21 Mon Sep 17 00:00:00 2001 From: Alex Lloyd Date: Tue, 12 Jun 2018 12:45:33 +0100 Subject: [PATCH 0700/1301] Fixed typo in ContainerSpec Docs Signed-off-by: Alexander Lloyd --- docker/types/services.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/types/services.py b/docker/types/services.py index 09eb05edbd..31f4750f45 100644 --- a/docker/types/services.py +++ b/docker/types/services.py @@ -82,7 +82,7 @@ class ContainerSpec(dict): args (:py:class:`list`): Arguments to the command. hostname (string): The hostname to set on the container. env (dict): Environment variables. - dir (string): The working directory for commands to run in. + workdir (string): The working directory for commands to run in. user (string): The user inside the container. labels (dict): A map of labels to associate with the service. mounts (:py:class:`list`): A list of specifications for mounts to be From 000331cfc1443c61e1a7ac58b9ea8dbeb09d110d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=BCray=20Y=C4=B1ld=C4=B1r=C4=B1m?= Date: Sun, 29 Apr 2018 04:43:11 +0300 Subject: [PATCH 0701/1301] Swarm Mode service scaling parameter mistake is fixed. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Güray Yıldırım --- docker/models/services.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/models/services.py b/docker/models/services.py index 2834dd7419..458d2c8730 100644 --- a/docker/models/services.py +++ b/docker/models/services.py @@ -126,7 +126,7 @@ def scale(self, replicas): service_mode = ServiceMode('replicated', replicas) return self.client.api.update_service(self.id, self.version, - service_mode, + mode=service_mode, fetch_current_spec=True) def force_update(self): From e5f56247e3d6f6f0f325aab507d9845ad2c4c097 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 18 Jun 2018 15:11:12 -0700 Subject: [PATCH 0702/1301] Bump 3.4.0 Signed-off-by: Joffrey F --- docker/version.py | 2 +- docs/change-log.md | 25 +++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index 04fd3c28d4..c504327323 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "3.4.0-dev" +version = "3.4.0" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) diff --git a/docs/change-log.md b/docs/change-log.md index 0065c62d09..5a0d55a372 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -1,6 +1,31 @@ Change log ========== +3.4.0 +----- + +[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/51?closed=1) + +### Features + +* The `APIClient` and `DockerClient` constructors now accept a `credstore_env` + parameter. When set, values in this dictionary are added to the environment + when executing the credential store process. + +### Bugfixes + +* `DockerClient.networks.prune` now properly returns the operation's result +* Fixed a bug that caused custom Dockerfile paths in a subfolder of the build + context to be invalidated, preventing these builds from working +* The `plugin_privileges` method can now be called for plugins requiring + authentication to access +* Fixed a bug that caused attempts to read a data stream over an unsecured TCP + socket to crash on Windows clients +* Fixed a bug where using the `read_only` parameter when creating a service using + the `DockerClient` was being ignored +* Fixed an issue where `Service.scale` would not properly update the service's + mode, causing the operation to fail silently + 3.3.0 ----- From 5a85cad54785bae45787b2584476c286301329e2 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 18 Jun 2018 15:26:13 -0700 Subject: [PATCH 0703/1301] 3.5.0-dev Signed-off-by: Joffrey F --- docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index c504327323..0866ca1c78 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "3.4.0" +version = "3.5.0-dev" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) From 8c35eee0fba1ac403a7f498f76eb99d5ba71387d Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 26 Jun 2018 17:40:49 -0700 Subject: [PATCH 0704/1301] Fix support for legacy .dockercfg auth config format Signed-off-by: Joffrey F --- docker/api/container.py | 5 ++- docker/auth.py | 6 +-- tests/integration/api_client_test.py | 40 ----------------- tests/unit/auth_test.py | 66 +++++++++++++++++++++++----- 4 files changed, 60 insertions(+), 57 deletions(-) diff --git a/docker/api/container.py b/docker/api/container.py index 05676f11e9..d4f75f54b2 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -139,8 +139,9 @@ def commit(self, container, repository=None, tag=None, message=None, 'changes': changes } u = self._url("/commit") - return self._result(self._post_json(u, data=conf, params=params), - json=True) + return self._result( + self._post_json(u, data=conf, params=params), json=True + ) def containers(self, quiet=False, all=False, trunc=False, latest=False, since=None, before=None, limit=-1, size=False, diff --git a/docker/auth.py b/docker/auth.py index 0c0cb204d9..9635f933ec 100644 --- a/docker/auth.py +++ b/docker/auth.py @@ -270,7 +270,7 @@ def load_config(config_path=None, config_dict=None): "Couldn't find auth-related section ; attempting to interpret" "as auth-only file" ) - return parse_auth(config_dict) + return {'auths': parse_auth(config_dict)} def _load_legacy_config(config_file): @@ -287,14 +287,14 @@ def _load_legacy_config(config_file): ) username, password = decode_auth(data[0]) - return { + return {'auths': { INDEX_NAME: { 'username': username, 'password': password, 'email': data[1], 'serveraddress': INDEX_URL, } - } + }} except Exception as e: log.debug(e) pass diff --git a/tests/integration/api_client_test.py b/tests/integration/api_client_test.py index 05281f8849..905e06484d 100644 --- a/tests/integration/api_client_test.py +++ b/tests/integration/api_client_test.py @@ -1,6 +1,3 @@ -import base64 -import os -import tempfile import time import unittest import warnings @@ -24,43 +21,6 @@ def test_info(self): assert 'Debug' in res -class LoadConfigTest(BaseAPIIntegrationTest): - def test_load_legacy_config(self): - folder = tempfile.mkdtemp() - self.tmp_folders.append(folder) - cfg_path = os.path.join(folder, '.dockercfg') - f = open(cfg_path, 'w') - auth_ = base64.b64encode(b'sakuya:izayoi').decode('ascii') - f.write('auth = {0}\n'.format(auth_)) - f.write('email = sakuya@scarlet.net') - f.close() - cfg = docker.auth.load_config(cfg_path) - assert cfg[docker.auth.INDEX_NAME] is not None - cfg = cfg[docker.auth.INDEX_NAME] - assert cfg['username'] == 'sakuya' - assert cfg['password'] == 'izayoi' - assert cfg['email'] == 'sakuya@scarlet.net' - assert cfg.get('Auth') is None - - def test_load_json_config(self): - folder = tempfile.mkdtemp() - self.tmp_folders.append(folder) - cfg_path = os.path.join(folder, '.dockercfg') - f = open(os.path.join(folder, '.dockercfg'), 'w') - auth_ = base64.b64encode(b'sakuya:izayoi').decode('ascii') - email_ = 'sakuya@scarlet.net' - f.write('{{"{0}": {{"auth": "{1}", "email": "{2}"}}}}\n'.format( - docker.auth.INDEX_URL, auth_, email_)) - f.close() - cfg = docker.auth.load_config(cfg_path) - assert cfg[docker.auth.INDEX_URL] is not None - cfg = cfg[docker.auth.INDEX_URL] - assert cfg['username'] == 'sakuya' - assert cfg['password'] == 'izayoi' - assert cfg['email'] == 'sakuya@scarlet.net' - assert cfg.get('Auth') is None - - class AutoDetectVersionTest(unittest.TestCase): def test_client_init(self): client = docker.APIClient(version='auto', **kwargs_from_env()) diff --git a/tests/unit/auth_test.py b/tests/unit/auth_test.py index ee32ca08a9..947d680018 100644 --- a/tests/unit/auth_test.py +++ b/tests/unit/auth_test.py @@ -282,22 +282,64 @@ def test_load_config_no_file(self): cfg = auth.load_config(folder) assert cfg is not None - def test_load_config(self): + def test_load_legacy_config(self): folder = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, folder) - dockercfg_path = os.path.join(folder, '.dockercfg') - with open(dockercfg_path, 'w') as f: - auth_ = base64.b64encode(b'sakuya:izayoi').decode('ascii') + cfg_path = os.path.join(folder, '.dockercfg') + auth_ = base64.b64encode(b'sakuya:izayoi').decode('ascii') + with open(cfg_path, 'w') as f: f.write('auth = {0}\n'.format(auth_)) f.write('email = sakuya@scarlet.net') - cfg = auth.load_config(dockercfg_path) - assert auth.INDEX_NAME in cfg - assert cfg[auth.INDEX_NAME] is not None - cfg = cfg[auth.INDEX_NAME] + + cfg = auth.load_config(cfg_path) + assert auth.resolve_authconfig(cfg) is not None + assert cfg['auths'][auth.INDEX_NAME] is not None + cfg = cfg['auths'][auth.INDEX_NAME] assert cfg['username'] == 'sakuya' assert cfg['password'] == 'izayoi' assert cfg['email'] == 'sakuya@scarlet.net' - assert cfg.get('auth') is None + assert cfg.get('Auth') is None + + def test_load_json_config(self): + folder = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, folder) + cfg_path = os.path.join(folder, '.dockercfg') + auth_ = base64.b64encode(b'sakuya:izayoi').decode('ascii') + email = 'sakuya@scarlet.net' + with open(cfg_path, 'w') as f: + json.dump( + {auth.INDEX_URL: {'auth': auth_, 'email': email}}, f + ) + cfg = auth.load_config(cfg_path) + assert auth.resolve_authconfig(cfg) is not None + assert cfg['auths'][auth.INDEX_URL] is not None + cfg = cfg['auths'][auth.INDEX_URL] + assert cfg['username'] == 'sakuya' + assert cfg['password'] == 'izayoi' + assert cfg['email'] == email + assert cfg.get('Auth') is None + + def test_load_modern_json_config(self): + folder = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, folder) + cfg_path = os.path.join(folder, 'config.json') + auth_ = base64.b64encode(b'sakuya:izayoi').decode('ascii') + email = 'sakuya@scarlet.net' + with open(cfg_path, 'w') as f: + json.dump({ + 'auths': { + auth.INDEX_URL: { + 'auth': auth_, 'email': email + } + } + }, f) + cfg = auth.load_config(cfg_path) + assert auth.resolve_authconfig(cfg) is not None + assert cfg['auths'][auth.INDEX_URL] is not None + cfg = cfg['auths'][auth.INDEX_URL] + assert cfg['username'] == 'sakuya' + assert cfg['password'] == 'izayoi' + assert cfg['email'] == email def test_load_config_with_random_name(self): folder = tempfile.mkdtemp() @@ -318,7 +360,7 @@ def test_load_config_with_random_name(self): with open(dockercfg_path, 'w') as f: json.dump(config, f) - cfg = auth.load_config(dockercfg_path) + cfg = auth.load_config(dockercfg_path)['auths'] assert registry in cfg assert cfg[registry] is not None cfg = cfg[registry] @@ -345,7 +387,7 @@ def test_load_config_custom_config_env(self): json.dump(config, f) with mock.patch.dict(os.environ, {'DOCKER_CONFIG': folder}): - cfg = auth.load_config(None) + cfg = auth.load_config(None)['auths'] assert registry in cfg assert cfg[registry] is not None cfg = cfg[registry] @@ -422,7 +464,7 @@ def test_load_config_unknown_keys(self): json.dump(config, f) cfg = auth.load_config(dockercfg_path) - assert cfg == {} + assert cfg == {'auths': {}} def test_load_config_invalid_auth_dict(self): folder = tempfile.mkdtemp() From e195e022cf854d1487fd87d796d8391221797388 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 28 Jun 2018 16:33:06 -0700 Subject: [PATCH 0705/1301] Fix detach assert function to account for new behavior in engine 18.06 Signed-off-by: Joffrey F --- tests/helpers.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/helpers.py b/tests/helpers.py index b6b493b385..b36d6d786f 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -123,7 +123,12 @@ def assert_cat_socket_detached_with_keys(sock, inputs): sock.sendall(b'make sure the socket is closed\n') else: sock.sendall(b"make sure the socket is closed\n") - assert sock.recv(32) == b'' + data = sock.recv(128) + # New in 18.06: error message is broadcast over the socket when reading + # after detach + assert data == b'' or data.startswith( + b'exec attach failed: error on attach stdin: read escape sequence' + ) def ctrl_with(char): From 81b7d48ad6eb3e2275a0585421b3ed0af53e9f21 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 21 Jun 2018 00:15:55 -0700 Subject: [PATCH 0706/1301] Improved .dockerignore pattern processing to better match Docker CLI behavior Signed-off-by: Joffrey F --- docker/utils/build.py | 199 ++++++++++++++++++++++----------------- docker/utils/fnmatch.py | 1 + tests/unit/utils_test.py | 12 ++- 3 files changed, 123 insertions(+), 89 deletions(-) diff --git a/docker/utils/build.py b/docker/utils/build.py index b644c9fca1..9ce0095ce6 100644 --- a/docker/utils/build.py +++ b/docker/utils/build.py @@ -6,8 +6,7 @@ import tempfile from ..constants import IS_WINDOWS_PLATFORM -from fnmatch import fnmatch -from itertools import chain +from .fnmatch import fnmatch _SEP = re.compile('/|\\\\') if IS_WINDOWS_PLATFORM else re.compile('/') @@ -44,92 +43,9 @@ def exclude_paths(root, patterns, dockerfile=None): if dockerfile is None: dockerfile = 'Dockerfile' - def split_path(p): - return [pt for pt in re.split(_SEP, p) if pt and pt != '.'] - - def normalize(p): - # Leading and trailing slashes are not relevant. Yes, - # "foo.py/" must exclude the "foo.py" regular file. "." - # components are not relevant either, even if the whole - # pattern is only ".", as the Docker reference states: "For - # historical reasons, the pattern . is ignored." - # ".." component must be cleared with the potential previous - # component, regardless of whether it exists: "A preprocessing - # step [...] eliminates . and .. elements using Go's - # filepath.". - i = 0 - split = split_path(p) - while i < len(split): - if split[i] == '..': - del split[i] - if i > 0: - del split[i - 1] - i -= 1 - else: - i += 1 - return split - - patterns = ( - (True, normalize(p[1:])) - if p.startswith('!') else - (False, normalize(p)) - for p in patterns) - patterns = list(reversed(list(chain( - # Exclude empty patterns such as "." or the empty string. - filter(lambda p: p[1], patterns), - # Always include the Dockerfile and .dockerignore - [(True, split_path(dockerfile)), (True, ['.dockerignore'])])))) - return set(walk(root, patterns)) - - -def walk(root, patterns, default=True): - """ - A collection of file lying below root that should be included according to - patterns. - """ - - def match(p): - if p[1][0] == '**': - rec = (p[0], p[1][1:]) - return [p] + (match(rec) if rec[1] else [rec]) - elif fnmatch(f, p[1][0]): - return [(p[0], p[1][1:])] - else: - return [] - - for f in os.listdir(root): - cur = os.path.join(root, f) - # The patterns if recursing in that directory. - sub = list(chain(*(match(p) for p in patterns))) - # Whether this file is explicitely included / excluded. - hit = next((p[0] for p in sub if not p[1]), None) - # Whether this file is implicitely included / excluded. - matched = default if hit is None else hit - sub = list(filter(lambda p: p[1], sub)) - if os.path.isdir(cur) and not os.path.islink(cur): - # Entirely skip directories if there are no chance any subfile will - # be included. - if all(not p[0] for p in sub) and not matched: - continue - # I think this would greatly speed up dockerignore handling by not - # recursing into directories we are sure would be entirely - # included, and only yielding the directory itself, which will be - # recursively archived anyway. However the current unit test expect - # the full list of subfiles and I'm not 100% sure it would make no - # difference yet. - # if all(p[0] for p in sub) and matched: - # yield f - # continue - children = False - for r in (os.path.join(f, p) for p in walk(cur, sub, matched)): - yield r - children = True - # The current unit tests expect directories only under those - # conditions. It might be simplifiable though. - if (not sub or not children) and hit or hit is None and default: - yield f - elif matched: - yield f + patterns.append('!' + dockerfile) + pm = PatternMatcher(patterns) + return set(pm.walk(root)) def build_file_list(root): @@ -217,3 +133,110 @@ def mkbuildcontext(dockerfile): t.close() f.seek(0) return f + + +def split_path(p): + return [pt for pt in re.split(_SEP, p) if pt and pt != '.'] + + +# Heavily based on +# https://github.com/moby/moby/blob/master/pkg/fileutils/fileutils.go +class PatternMatcher(object): + def __init__(self, patterns): + self.patterns = list(filter( + lambda p: p.dirs, [Pattern(p) for p in patterns] + )) + self.patterns.append(Pattern('!.dockerignore')) + + def matches(self, filepath): + matched = False + parent_path = os.path.dirname(filepath) + parent_path_dirs = split_path(parent_path) + + for pattern in self.patterns: + negative = pattern.exclusion + match = pattern.match(filepath) + if not match and parent_path != '': + if len(pattern.dirs) <= len(parent_path_dirs): + match = pattern.match( + os.path.sep.join(parent_path_dirs[:len(pattern.dirs)]) + ) + + if match: + matched = not negative + + return matched + + def walk(self, root): + def rec_walk(current_dir): + for f in os.listdir(current_dir): + fpath = os.path.join( + os.path.relpath(current_dir, root), f + ) + if fpath.startswith('.' + os.path.sep): + fpath = fpath[2:] + match = self.matches(fpath) + if not match: + yield fpath + + cur = os.path.join(root, fpath) + if not os.path.isdir(cur) or os.path.islink(cur): + continue + + if match: + # If we want to skip this file and its a directory + # then we should first check to see if there's an + # excludes pattern (e.g. !dir/file) that starts with this + # dir. If so then we can't skip this dir. + skip = True + + for pat in self.patterns: + if not pat.exclusion: + continue + if pat.cleaned_pattern.startswith(fpath): + skip = False + break + if skip: + continue + for sub in rec_walk(cur): + yield sub + + return rec_walk(root) + + +class Pattern(object): + def __init__(self, pattern_str): + self.exclusion = False + if pattern_str.startswith('!'): + self.exclusion = True + pattern_str = pattern_str[1:] + + self.dirs = self.normalize(pattern_str) + self.cleaned_pattern = '/'.join(self.dirs) + + @classmethod + def normalize(cls, p): + + # Leading and trailing slashes are not relevant. Yes, + # "foo.py/" must exclude the "foo.py" regular file. "." + # components are not relevant either, even if the whole + # pattern is only ".", as the Docker reference states: "For + # historical reasons, the pattern . is ignored." + # ".." component must be cleared with the potential previous + # component, regardless of whether it exists: "A preprocessing + # step [...] eliminates . and .. elements using Go's + # filepath.". + i = 0 + split = split_path(p) + while i < len(split): + if split[i] == '..': + del split[i] + if i > 0: + del split[i - 1] + i -= 1 + else: + i += 1 + return split + + def match(self, filepath): + return fnmatch(filepath, self.cleaned_pattern) diff --git a/docker/utils/fnmatch.py b/docker/utils/fnmatch.py index 42461dd7d3..cc940a2e65 100644 --- a/docker/utils/fnmatch.py +++ b/docker/utils/fnmatch.py @@ -111,4 +111,5 @@ def translate(pat): res = '%s[%s]' % (res, stuff) else: res = res + re.escape(c) + return res + '$' diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index 00456e8c11..467e835c56 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -887,12 +887,22 @@ def test_trailing_double_wildcard(self): ) ) + def test_double_wildcard_with_exception(self): + assert self.exclude(['**', '!bar', '!foo/bar']) == convert_paths( + set([ + 'foo/bar', 'foo/bar/a.py', 'bar', 'bar/a.py', 'Dockerfile', + '.dockerignore', + ]) + ) + def test_include_wildcard(self): + # This may be surprising but it matches the CLI's behavior + # (tested with 18.05.0-ce on linux) base = make_tree(['a'], ['a/b.py']) assert exclude_paths( base, ['*', '!*/b.py'] - ) == convert_paths(['a/b.py']) + ) == set() def test_last_line_precedence(self): base = make_tree( From ced86ec81329e063550933abb90c940dceb24620 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 28 Jun 2018 14:07:38 -0700 Subject: [PATCH 0707/1301] On Windows, convert paths to use forward slashes before fnmatch call Signed-off-by: Joffrey F --- docker/utils/build.py | 18 +- tests/unit/utils_build_test.py | 493 +++++++++++++++++++++++++++++++++ tests/unit/utils_test.py | 490 +------------------------------- 3 files changed, 511 insertions(+), 490 deletions(-) create mode 100644 tests/unit/utils_build_test.py diff --git a/docker/utils/build.py b/docker/utils/build.py index 9ce0095ce6..6f6241e933 100644 --- a/docker/utils/build.py +++ b/docker/utils/build.py @@ -1,12 +1,13 @@ import io import os import re -import six import tarfile import tempfile -from ..constants import IS_WINDOWS_PLATFORM +import six + from .fnmatch import fnmatch +from ..constants import IS_WINDOWS_PLATFORM _SEP = re.compile('/|\\\\') if IS_WINDOWS_PLATFORM else re.compile('/') @@ -139,6 +140,12 @@ def split_path(p): return [pt for pt in re.split(_SEP, p) if pt and pt != '.'] +def normalize_slashes(p): + if IS_WINDOWS_PLATFORM: + return '/'.join(split_path(p)) + return p + + # Heavily based on # https://github.com/moby/moby/blob/master/pkg/fileutils/fileutils.go class PatternMatcher(object): @@ -184,7 +191,7 @@ def rec_walk(current_dir): continue if match: - # If we want to skip this file and its a directory + # If we want to skip this file and it's a directory # then we should first check to see if there's an # excludes pattern (e.g. !dir/file) that starts with this # dir. If so then we can't skip this dir. @@ -193,7 +200,8 @@ def rec_walk(current_dir): for pat in self.patterns: if not pat.exclusion: continue - if pat.cleaned_pattern.startswith(fpath): + if pat.cleaned_pattern.startswith( + normalize_slashes(fpath)): skip = False break if skip: @@ -239,4 +247,4 @@ def normalize(cls, p): return split def match(self, filepath): - return fnmatch(filepath, self.cleaned_pattern) + return fnmatch(normalize_slashes(filepath), self.cleaned_pattern) diff --git a/tests/unit/utils_build_test.py b/tests/unit/utils_build_test.py new file mode 100644 index 0000000000..012f15b46a --- /dev/null +++ b/tests/unit/utils_build_test.py @@ -0,0 +1,493 @@ +# -*- coding: utf-8 -*- + +import os +import os.path +import shutil +import socket +import tarfile +import tempfile +import unittest + + +from docker.constants import IS_WINDOWS_PLATFORM +from docker.utils import exclude_paths, tar + +import pytest + +from ..helpers import make_tree + + +def convert_paths(collection): + return set(map(convert_path, collection)) + + +def convert_path(path): + return path.replace('/', os.path.sep) + + +class ExcludePathsTest(unittest.TestCase): + dirs = [ + 'foo', + 'foo/bar', + 'bar', + 'target', + 'target/subdir', + 'subdir', + 'subdir/target', + 'subdir/target/subdir', + 'subdir/subdir2', + 'subdir/subdir2/target', + 'subdir/subdir2/target/subdir' + ] + + files = [ + 'Dockerfile', + 'Dockerfile.alt', + '.dockerignore', + 'a.py', + 'a.go', + 'b.py', + 'cde.py', + 'foo/a.py', + 'foo/b.py', + 'foo/bar/a.py', + 'bar/a.py', + 'foo/Dockerfile3', + 'target/file.txt', + 'target/subdir/file.txt', + 'subdir/file.txt', + 'subdir/target/file.txt', + 'subdir/target/subdir/file.txt', + 'subdir/subdir2/file.txt', + 'subdir/subdir2/target/file.txt', + 'subdir/subdir2/target/subdir/file.txt', + ] + + all_paths = set(dirs + files) + + def setUp(self): + self.base = make_tree(self.dirs, self.files) + + def tearDown(self): + shutil.rmtree(self.base) + + def exclude(self, patterns, dockerfile=None): + return set(exclude_paths(self.base, patterns, dockerfile=dockerfile)) + + def test_no_excludes(self): + assert self.exclude(['']) == convert_paths(self.all_paths) + + def test_no_dupes(self): + paths = exclude_paths(self.base, ['!a.py']) + assert sorted(paths) == sorted(set(paths)) + + def test_wildcard_exclude(self): + assert self.exclude(['*']) == set(['Dockerfile', '.dockerignore']) + + def test_exclude_dockerfile_dockerignore(self): + """ + Even if the .dockerignore file explicitly says to exclude + Dockerfile and/or .dockerignore, don't exclude them from + the actual tar file. + """ + assert self.exclude(['Dockerfile', '.dockerignore']) == convert_paths( + self.all_paths + ) + + def test_exclude_custom_dockerfile(self): + """ + If we're using a custom Dockerfile, make sure that's not + excluded. + """ + assert self.exclude(['*'], dockerfile='Dockerfile.alt') == set( + ['Dockerfile.alt', '.dockerignore'] + ) + + assert self.exclude( + ['*'], dockerfile='foo/Dockerfile3' + ) == convert_paths(set(['foo/Dockerfile3', '.dockerignore'])) + + # https://github.com/docker/docker-py/issues/1956 + assert self.exclude( + ['*'], dockerfile='./foo/Dockerfile3' + ) == convert_paths(set(['foo/Dockerfile3', '.dockerignore'])) + + def test_exclude_dockerfile_child(self): + includes = self.exclude(['foo/'], dockerfile='foo/Dockerfile3') + assert convert_path('foo/Dockerfile3') in includes + assert convert_path('foo/a.py') not in includes + + def test_single_filename(self): + assert self.exclude(['a.py']) == convert_paths( + self.all_paths - set(['a.py']) + ) + + def test_single_filename_leading_dot_slash(self): + assert self.exclude(['./a.py']) == convert_paths( + self.all_paths - set(['a.py']) + ) + + # As odd as it sounds, a filename pattern with a trailing slash on the + # end *will* result in that file being excluded. + def test_single_filename_trailing_slash(self): + assert self.exclude(['a.py/']) == convert_paths( + self.all_paths - set(['a.py']) + ) + + def test_wildcard_filename_start(self): + assert self.exclude(['*.py']) == convert_paths( + self.all_paths - set(['a.py', 'b.py', 'cde.py']) + ) + + def test_wildcard_with_exception(self): + assert self.exclude(['*.py', '!b.py']) == convert_paths( + self.all_paths - set(['a.py', 'cde.py']) + ) + + def test_wildcard_with_wildcard_exception(self): + assert self.exclude(['*.*', '!*.go']) == convert_paths( + self.all_paths - set([ + 'a.py', 'b.py', 'cde.py', 'Dockerfile.alt', + ]) + ) + + def test_wildcard_filename_end(self): + assert self.exclude(['a.*']) == convert_paths( + self.all_paths - set(['a.py', 'a.go']) + ) + + def test_question_mark(self): + assert self.exclude(['?.py']) == convert_paths( + self.all_paths - set(['a.py', 'b.py']) + ) + + def test_single_subdir_single_filename(self): + assert self.exclude(['foo/a.py']) == convert_paths( + self.all_paths - set(['foo/a.py']) + ) + + def test_single_subdir_single_filename_leading_slash(self): + assert self.exclude(['/foo/a.py']) == convert_paths( + self.all_paths - set(['foo/a.py']) + ) + + def test_exclude_include_absolute_path(self): + base = make_tree([], ['a.py', 'b.py']) + assert exclude_paths( + base, + ['/*', '!/*.py'] + ) == set(['a.py', 'b.py']) + + def test_single_subdir_with_path_traversal(self): + assert self.exclude(['foo/whoops/../a.py']) == convert_paths( + self.all_paths - set(['foo/a.py']) + ) + + def test_single_subdir_wildcard_filename(self): + assert self.exclude(['foo/*.py']) == convert_paths( + self.all_paths - set(['foo/a.py', 'foo/b.py']) + ) + + def test_wildcard_subdir_single_filename(self): + assert self.exclude(['*/a.py']) == convert_paths( + self.all_paths - set(['foo/a.py', 'bar/a.py']) + ) + + def test_wildcard_subdir_wildcard_filename(self): + assert self.exclude(['*/*.py']) == convert_paths( + self.all_paths - set(['foo/a.py', 'foo/b.py', 'bar/a.py']) + ) + + def test_directory(self): + assert self.exclude(['foo']) == convert_paths( + self.all_paths - set([ + 'foo', 'foo/a.py', 'foo/b.py', 'foo/bar', 'foo/bar/a.py', + 'foo/Dockerfile3' + ]) + ) + + def test_directory_with_trailing_slash(self): + assert self.exclude(['foo']) == convert_paths( + self.all_paths - set([ + 'foo', 'foo/a.py', 'foo/b.py', + 'foo/bar', 'foo/bar/a.py', 'foo/Dockerfile3' + ]) + ) + + def test_directory_with_single_exception(self): + assert self.exclude(['foo', '!foo/bar/a.py']) == convert_paths( + self.all_paths - set([ + 'foo/a.py', 'foo/b.py', 'foo', 'foo/bar', + 'foo/Dockerfile3' + ]) + ) + + def test_directory_with_subdir_exception(self): + assert self.exclude(['foo', '!foo/bar']) == convert_paths( + self.all_paths - set([ + 'foo/a.py', 'foo/b.py', 'foo', 'foo/Dockerfile3' + ]) + ) + + @pytest.mark.skipif( + not IS_WINDOWS_PLATFORM, reason='Backslash patterns only on Windows' + ) + def test_directory_with_subdir_exception_win32_pathsep(self): + assert self.exclude(['foo', '!foo\\bar']) == convert_paths( + self.all_paths - set([ + 'foo/a.py', 'foo/b.py', 'foo', 'foo/Dockerfile3' + ]) + ) + + def test_directory_with_wildcard_exception(self): + assert self.exclude(['foo', '!foo/*.py']) == convert_paths( + self.all_paths - set([ + 'foo/bar', 'foo/bar/a.py', 'foo', 'foo/Dockerfile3' + ]) + ) + + def test_subdirectory(self): + assert self.exclude(['foo/bar']) == convert_paths( + self.all_paths - set(['foo/bar', 'foo/bar/a.py']) + ) + + @pytest.mark.skipif( + not IS_WINDOWS_PLATFORM, reason='Backslash patterns only on Windows' + ) + def test_subdirectory_win32_pathsep(self): + assert self.exclude(['foo\\bar']) == convert_paths( + self.all_paths - set(['foo/bar', 'foo/bar/a.py']) + ) + + def test_double_wildcard(self): + assert self.exclude(['**/a.py']) == convert_paths( + self.all_paths - set( + ['a.py', 'foo/a.py', 'foo/bar/a.py', 'bar/a.py'] + ) + ) + + assert self.exclude(['foo/**/bar']) == convert_paths( + self.all_paths - set(['foo/bar', 'foo/bar/a.py']) + ) + + def test_single_and_double_wildcard(self): + assert self.exclude(['**/target/*/*']) == convert_paths( + self.all_paths - set( + ['target/subdir/file.txt', + 'subdir/target/subdir/file.txt', + 'subdir/subdir2/target/subdir/file.txt'] + ) + ) + + def test_trailing_double_wildcard(self): + assert self.exclude(['subdir/**']) == convert_paths( + self.all_paths - set( + ['subdir/file.txt', + 'subdir/target/file.txt', + 'subdir/target/subdir/file.txt', + 'subdir/subdir2/file.txt', + 'subdir/subdir2/target/file.txt', + 'subdir/subdir2/target/subdir/file.txt', + 'subdir/target', + 'subdir/target/subdir', + 'subdir/subdir2', + 'subdir/subdir2/target', + 'subdir/subdir2/target/subdir'] + ) + ) + + def test_double_wildcard_with_exception(self): + assert self.exclude(['**', '!bar', '!foo/bar']) == convert_paths( + set([ + 'foo/bar', 'foo/bar/a.py', 'bar', 'bar/a.py', 'Dockerfile', + '.dockerignore', + ]) + ) + + def test_include_wildcard(self): + # This may be surprising but it matches the CLI's behavior + # (tested with 18.05.0-ce on linux) + base = make_tree(['a'], ['a/b.py']) + assert exclude_paths( + base, + ['*', '!*/b.py'] + ) == set() + + def test_last_line_precedence(self): + base = make_tree( + [], + ['garbage.md', + 'trash.md', + 'README.md', + 'README-bis.md', + 'README-secret.md']) + assert exclude_paths( + base, + ['*.md', '!README*.md', 'README-secret.md'] + ) == set(['README.md', 'README-bis.md']) + + def test_parent_directory(self): + base = make_tree( + [], + ['a.py', + 'b.py', + 'c.py']) + # Dockerignore reference stipulates that absolute paths are + # equivalent to relative paths, hence /../foo should be + # equivalent to ../foo. It also stipulates that paths are run + # through Go's filepath.Clean, which explicitely "replace + # "/.." by "/" at the beginning of a path". + assert exclude_paths( + base, + ['../a.py', '/../b.py'] + ) == set(['c.py']) + + +class TarTest(unittest.TestCase): + def test_tar_with_excludes(self): + dirs = [ + 'foo', + 'foo/bar', + 'bar', + ] + + files = [ + 'Dockerfile', + 'Dockerfile.alt', + '.dockerignore', + 'a.py', + 'a.go', + 'b.py', + 'cde.py', + 'foo/a.py', + 'foo/b.py', + 'foo/bar/a.py', + 'bar/a.py', + ] + + exclude = [ + '*.py', + '!b.py', + '!a.go', + 'foo', + 'Dockerfile*', + '.dockerignore', + ] + + expected_names = set([ + 'Dockerfile', + '.dockerignore', + 'a.go', + 'b.py', + 'bar', + 'bar/a.py', + ]) + + base = make_tree(dirs, files) + self.addCleanup(shutil.rmtree, base) + + with tar(base, exclude=exclude) as archive: + tar_data = tarfile.open(fileobj=archive) + assert sorted(tar_data.getnames()) == sorted(expected_names) + + def test_tar_with_empty_directory(self): + base = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, base) + for d in ['foo', 'bar']: + os.makedirs(os.path.join(base, d)) + with tar(base) as archive: + tar_data = tarfile.open(fileobj=archive) + assert sorted(tar_data.getnames()) == ['bar', 'foo'] + + @pytest.mark.skipif( + IS_WINDOWS_PLATFORM or os.geteuid() == 0, + reason='root user always has access ; no chmod on Windows' + ) + def test_tar_with_inaccessible_file(self): + base = tempfile.mkdtemp() + full_path = os.path.join(base, 'foo') + self.addCleanup(shutil.rmtree, base) + with open(full_path, 'w') as f: + f.write('content') + os.chmod(full_path, 0o222) + with pytest.raises(IOError) as ei: + tar(base) + + assert 'Can not read file in context: {}'.format(full_path) in ( + ei.exconly() + ) + + @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='No symlinks on Windows') + def test_tar_with_file_symlinks(self): + base = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, base) + with open(os.path.join(base, 'foo'), 'w') as f: + f.write("content") + os.makedirs(os.path.join(base, 'bar')) + os.symlink('../foo', os.path.join(base, 'bar/foo')) + with tar(base) as archive: + tar_data = tarfile.open(fileobj=archive) + assert sorted(tar_data.getnames()) == ['bar', 'bar/foo', 'foo'] + + @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='No symlinks on Windows') + def test_tar_with_directory_symlinks(self): + base = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, base) + for d in ['foo', 'bar']: + os.makedirs(os.path.join(base, d)) + os.symlink('../foo', os.path.join(base, 'bar/foo')) + with tar(base) as archive: + tar_data = tarfile.open(fileobj=archive) + assert sorted(tar_data.getnames()) == ['bar', 'bar/foo', 'foo'] + + @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='No symlinks on Windows') + def test_tar_with_broken_symlinks(self): + base = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, base) + for d in ['foo', 'bar']: + os.makedirs(os.path.join(base, d)) + + os.symlink('../baz', os.path.join(base, 'bar/foo')) + with tar(base) as archive: + tar_data = tarfile.open(fileobj=archive) + assert sorted(tar_data.getnames()) == ['bar', 'bar/foo', 'foo'] + + @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='No UNIX sockets on Win32') + def test_tar_socket_file(self): + base = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, base) + for d in ['foo', 'bar']: + os.makedirs(os.path.join(base, d)) + sock = socket.socket(socket.AF_UNIX) + self.addCleanup(sock.close) + sock.bind(os.path.join(base, 'test.sock')) + with tar(base) as archive: + tar_data = tarfile.open(fileobj=archive) + assert sorted(tar_data.getnames()) == ['bar', 'foo'] + + def tar_test_negative_mtime_bug(self): + base = tempfile.mkdtemp() + filename = os.path.join(base, 'th.txt') + self.addCleanup(shutil.rmtree, base) + with open(filename, 'w') as f: + f.write('Invisible Full Moon') + os.utime(filename, (12345, -3600.0)) + with tar(base) as archive: + tar_data = tarfile.open(fileobj=archive) + assert tar_data.getnames() == ['th.txt'] + assert tar_data.getmember('th.txt').mtime == -3600 + + @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='No symlinks on Windows') + def test_tar_directory_link(self): + dirs = ['a', 'b', 'a/c'] + files = ['a/hello.py', 'b/utils.py', 'a/c/descend.py'] + base = make_tree(dirs, files) + self.addCleanup(shutil.rmtree, base) + os.symlink(os.path.join(base, 'b'), os.path.join(base, 'a/c/b')) + with tar(base) as archive: + tar_data = tarfile.open(fileobj=archive) + names = tar_data.getnames() + for member in dirs + files: + assert member in names + assert 'a/c/b' in names + assert 'a/c/b/utils.py' not in names diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index 467e835c56..8880cfef0f 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -5,29 +5,25 @@ import os import os.path import shutil -import socket import sys -import tarfile import tempfile import unittest -import pytest -import six from docker.api.client import APIClient -from docker.constants import IS_WINDOWS_PLATFORM from docker.errors import DockerException from docker.utils import ( - parse_repository_tag, parse_host, convert_filters, kwargs_from_env, - parse_bytes, parse_env_file, exclude_paths, convert_volume_binds, - decode_json_header, tar, split_command, parse_devices, update_headers, + convert_filters, convert_volume_binds, decode_json_header, kwargs_from_env, + parse_bytes, parse_devices, parse_env_file, parse_host, + parse_repository_tag, split_command, update_headers, ) from docker.utils.ports import build_port_bindings, split_port from docker.utils.utils import format_environment -from ..helpers import make_tree +import pytest +import six TEST_CERT_DIR = os.path.join( os.path.dirname(__file__), @@ -608,482 +604,6 @@ def test_build_port_bindings_with_nonmatching_internal_port_ranges(self): assert port_bindings["2000"] == [("127.0.0.1", "2000")] -def convert_paths(collection): - return set(map(convert_path, collection)) - - -def convert_path(path): - return path.replace('/', os.path.sep) - - -class ExcludePathsTest(unittest.TestCase): - dirs = [ - 'foo', - 'foo/bar', - 'bar', - 'target', - 'target/subdir', - 'subdir', - 'subdir/target', - 'subdir/target/subdir', - 'subdir/subdir2', - 'subdir/subdir2/target', - 'subdir/subdir2/target/subdir' - ] - - files = [ - 'Dockerfile', - 'Dockerfile.alt', - '.dockerignore', - 'a.py', - 'a.go', - 'b.py', - 'cde.py', - 'foo/a.py', - 'foo/b.py', - 'foo/bar/a.py', - 'bar/a.py', - 'foo/Dockerfile3', - 'target/file.txt', - 'target/subdir/file.txt', - 'subdir/file.txt', - 'subdir/target/file.txt', - 'subdir/target/subdir/file.txt', - 'subdir/subdir2/file.txt', - 'subdir/subdir2/target/file.txt', - 'subdir/subdir2/target/subdir/file.txt', - ] - - all_paths = set(dirs + files) - - def setUp(self): - self.base = make_tree(self.dirs, self.files) - - def tearDown(self): - shutil.rmtree(self.base) - - def exclude(self, patterns, dockerfile=None): - return set(exclude_paths(self.base, patterns, dockerfile=dockerfile)) - - def test_no_excludes(self): - assert self.exclude(['']) == convert_paths(self.all_paths) - - def test_no_dupes(self): - paths = exclude_paths(self.base, ['!a.py']) - assert sorted(paths) == sorted(set(paths)) - - def test_wildcard_exclude(self): - assert self.exclude(['*']) == set(['Dockerfile', '.dockerignore']) - - def test_exclude_dockerfile_dockerignore(self): - """ - Even if the .dockerignore file explicitly says to exclude - Dockerfile and/or .dockerignore, don't exclude them from - the actual tar file. - """ - assert self.exclude(['Dockerfile', '.dockerignore']) == convert_paths( - self.all_paths - ) - - def test_exclude_custom_dockerfile(self): - """ - If we're using a custom Dockerfile, make sure that's not - excluded. - """ - assert self.exclude(['*'], dockerfile='Dockerfile.alt') == set( - ['Dockerfile.alt', '.dockerignore'] - ) - - assert self.exclude( - ['*'], dockerfile='foo/Dockerfile3' - ) == convert_paths(set(['foo/Dockerfile3', '.dockerignore'])) - - # https://github.com/docker/docker-py/issues/1956 - assert self.exclude( - ['*'], dockerfile='./foo/Dockerfile3' - ) == convert_paths(set(['foo/Dockerfile3', '.dockerignore'])) - - def test_exclude_dockerfile_child(self): - includes = self.exclude(['foo/'], dockerfile='foo/Dockerfile3') - assert convert_path('foo/Dockerfile3') in includes - assert convert_path('foo/a.py') not in includes - - def test_single_filename(self): - assert self.exclude(['a.py']) == convert_paths( - self.all_paths - set(['a.py']) - ) - - def test_single_filename_leading_dot_slash(self): - assert self.exclude(['./a.py']) == convert_paths( - self.all_paths - set(['a.py']) - ) - - # As odd as it sounds, a filename pattern with a trailing slash on the - # end *will* result in that file being excluded. - def test_single_filename_trailing_slash(self): - assert self.exclude(['a.py/']) == convert_paths( - self.all_paths - set(['a.py']) - ) - - def test_wildcard_filename_start(self): - assert self.exclude(['*.py']) == convert_paths( - self.all_paths - set(['a.py', 'b.py', 'cde.py']) - ) - - def test_wildcard_with_exception(self): - assert self.exclude(['*.py', '!b.py']) == convert_paths( - self.all_paths - set(['a.py', 'cde.py']) - ) - - def test_wildcard_with_wildcard_exception(self): - assert self.exclude(['*.*', '!*.go']) == convert_paths( - self.all_paths - set([ - 'a.py', 'b.py', 'cde.py', 'Dockerfile.alt', - ]) - ) - - def test_wildcard_filename_end(self): - assert self.exclude(['a.*']) == convert_paths( - self.all_paths - set(['a.py', 'a.go']) - ) - - def test_question_mark(self): - assert self.exclude(['?.py']) == convert_paths( - self.all_paths - set(['a.py', 'b.py']) - ) - - def test_single_subdir_single_filename(self): - assert self.exclude(['foo/a.py']) == convert_paths( - self.all_paths - set(['foo/a.py']) - ) - - def test_single_subdir_single_filename_leading_slash(self): - assert self.exclude(['/foo/a.py']) == convert_paths( - self.all_paths - set(['foo/a.py']) - ) - - def test_exclude_include_absolute_path(self): - base = make_tree([], ['a.py', 'b.py']) - assert exclude_paths( - base, - ['/*', '!/*.py'] - ) == set(['a.py', 'b.py']) - - def test_single_subdir_with_path_traversal(self): - assert self.exclude(['foo/whoops/../a.py']) == convert_paths( - self.all_paths - set(['foo/a.py']) - ) - - def test_single_subdir_wildcard_filename(self): - assert self.exclude(['foo/*.py']) == convert_paths( - self.all_paths - set(['foo/a.py', 'foo/b.py']) - ) - - def test_wildcard_subdir_single_filename(self): - assert self.exclude(['*/a.py']) == convert_paths( - self.all_paths - set(['foo/a.py', 'bar/a.py']) - ) - - def test_wildcard_subdir_wildcard_filename(self): - assert self.exclude(['*/*.py']) == convert_paths( - self.all_paths - set(['foo/a.py', 'foo/b.py', 'bar/a.py']) - ) - - def test_directory(self): - assert self.exclude(['foo']) == convert_paths( - self.all_paths - set([ - 'foo', 'foo/a.py', 'foo/b.py', 'foo/bar', 'foo/bar/a.py', - 'foo/Dockerfile3' - ]) - ) - - def test_directory_with_trailing_slash(self): - assert self.exclude(['foo']) == convert_paths( - self.all_paths - set([ - 'foo', 'foo/a.py', 'foo/b.py', - 'foo/bar', 'foo/bar/a.py', 'foo/Dockerfile3' - ]) - ) - - def test_directory_with_single_exception(self): - assert self.exclude(['foo', '!foo/bar/a.py']) == convert_paths( - self.all_paths - set([ - 'foo/a.py', 'foo/b.py', 'foo', 'foo/bar', - 'foo/Dockerfile3' - ]) - ) - - def test_directory_with_subdir_exception(self): - assert self.exclude(['foo', '!foo/bar']) == convert_paths( - self.all_paths - set([ - 'foo/a.py', 'foo/b.py', 'foo', 'foo/Dockerfile3' - ]) - ) - - @pytest.mark.skipif( - not IS_WINDOWS_PLATFORM, reason='Backslash patterns only on Windows' - ) - def test_directory_with_subdir_exception_win32_pathsep(self): - assert self.exclude(['foo', '!foo\\bar']) == convert_paths( - self.all_paths - set([ - 'foo/a.py', 'foo/b.py', 'foo', 'foo/Dockerfile3' - ]) - ) - - def test_directory_with_wildcard_exception(self): - assert self.exclude(['foo', '!foo/*.py']) == convert_paths( - self.all_paths - set([ - 'foo/bar', 'foo/bar/a.py', 'foo', 'foo/Dockerfile3' - ]) - ) - - def test_subdirectory(self): - assert self.exclude(['foo/bar']) == convert_paths( - self.all_paths - set(['foo/bar', 'foo/bar/a.py']) - ) - - @pytest.mark.skipif( - not IS_WINDOWS_PLATFORM, reason='Backslash patterns only on Windows' - ) - def test_subdirectory_win32_pathsep(self): - assert self.exclude(['foo\\bar']) == convert_paths( - self.all_paths - set(['foo/bar', 'foo/bar/a.py']) - ) - - def test_double_wildcard(self): - assert self.exclude(['**/a.py']) == convert_paths( - self.all_paths - set( - ['a.py', 'foo/a.py', 'foo/bar/a.py', 'bar/a.py'] - ) - ) - - assert self.exclude(['foo/**/bar']) == convert_paths( - self.all_paths - set(['foo/bar', 'foo/bar/a.py']) - ) - - def test_single_and_double_wildcard(self): - assert self.exclude(['**/target/*/*']) == convert_paths( - self.all_paths - set( - ['target/subdir/file.txt', - 'subdir/target/subdir/file.txt', - 'subdir/subdir2/target/subdir/file.txt'] - ) - ) - - def test_trailing_double_wildcard(self): - assert self.exclude(['subdir/**']) == convert_paths( - self.all_paths - set( - ['subdir/file.txt', - 'subdir/target/file.txt', - 'subdir/target/subdir/file.txt', - 'subdir/subdir2/file.txt', - 'subdir/subdir2/target/file.txt', - 'subdir/subdir2/target/subdir/file.txt', - 'subdir/target', - 'subdir/target/subdir', - 'subdir/subdir2', - 'subdir/subdir2/target', - 'subdir/subdir2/target/subdir'] - ) - ) - - def test_double_wildcard_with_exception(self): - assert self.exclude(['**', '!bar', '!foo/bar']) == convert_paths( - set([ - 'foo/bar', 'foo/bar/a.py', 'bar', 'bar/a.py', 'Dockerfile', - '.dockerignore', - ]) - ) - - def test_include_wildcard(self): - # This may be surprising but it matches the CLI's behavior - # (tested with 18.05.0-ce on linux) - base = make_tree(['a'], ['a/b.py']) - assert exclude_paths( - base, - ['*', '!*/b.py'] - ) == set() - - def test_last_line_precedence(self): - base = make_tree( - [], - ['garbage.md', - 'thrash.md', - 'README.md', - 'README-bis.md', - 'README-secret.md']) - assert exclude_paths( - base, - ['*.md', '!README*.md', 'README-secret.md'] - ) == set(['README.md', 'README-bis.md']) - - def test_parent_directory(self): - base = make_tree( - [], - ['a.py', - 'b.py', - 'c.py']) - # Dockerignore reference stipulates that absolute paths are - # equivalent to relative paths, hence /../foo should be - # equivalent to ../foo. It also stipulates that paths are run - # through Go's filepath.Clean, which explicitely "replace - # "/.." by "/" at the beginning of a path". - assert exclude_paths( - base, - ['../a.py', '/../b.py'] - ) == set(['c.py']) - - -class TarTest(unittest.TestCase): - def test_tar_with_excludes(self): - dirs = [ - 'foo', - 'foo/bar', - 'bar', - ] - - files = [ - 'Dockerfile', - 'Dockerfile.alt', - '.dockerignore', - 'a.py', - 'a.go', - 'b.py', - 'cde.py', - 'foo/a.py', - 'foo/b.py', - 'foo/bar/a.py', - 'bar/a.py', - ] - - exclude = [ - '*.py', - '!b.py', - '!a.go', - 'foo', - 'Dockerfile*', - '.dockerignore', - ] - - expected_names = set([ - 'Dockerfile', - '.dockerignore', - 'a.go', - 'b.py', - 'bar', - 'bar/a.py', - ]) - - base = make_tree(dirs, files) - self.addCleanup(shutil.rmtree, base) - - with tar(base, exclude=exclude) as archive: - tar_data = tarfile.open(fileobj=archive) - assert sorted(tar_data.getnames()) == sorted(expected_names) - - def test_tar_with_empty_directory(self): - base = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, base) - for d in ['foo', 'bar']: - os.makedirs(os.path.join(base, d)) - with tar(base) as archive: - tar_data = tarfile.open(fileobj=archive) - assert sorted(tar_data.getnames()) == ['bar', 'foo'] - - @pytest.mark.skipif( - IS_WINDOWS_PLATFORM or os.geteuid() == 0, - reason='root user always has access ; no chmod on Windows' - ) - def test_tar_with_inaccessible_file(self): - base = tempfile.mkdtemp() - full_path = os.path.join(base, 'foo') - self.addCleanup(shutil.rmtree, base) - with open(full_path, 'w') as f: - f.write('content') - os.chmod(full_path, 0o222) - with pytest.raises(IOError) as ei: - tar(base) - - assert 'Can not read file in context: {}'.format(full_path) in ( - ei.exconly() - ) - - @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='No symlinks on Windows') - def test_tar_with_file_symlinks(self): - base = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, base) - with open(os.path.join(base, 'foo'), 'w') as f: - f.write("content") - os.makedirs(os.path.join(base, 'bar')) - os.symlink('../foo', os.path.join(base, 'bar/foo')) - with tar(base) as archive: - tar_data = tarfile.open(fileobj=archive) - assert sorted(tar_data.getnames()) == ['bar', 'bar/foo', 'foo'] - - @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='No symlinks on Windows') - def test_tar_with_directory_symlinks(self): - base = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, base) - for d in ['foo', 'bar']: - os.makedirs(os.path.join(base, d)) - os.symlink('../foo', os.path.join(base, 'bar/foo')) - with tar(base) as archive: - tar_data = tarfile.open(fileobj=archive) - assert sorted(tar_data.getnames()) == ['bar', 'bar/foo', 'foo'] - - @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='No symlinks on Windows') - def test_tar_with_broken_symlinks(self): - base = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, base) - for d in ['foo', 'bar']: - os.makedirs(os.path.join(base, d)) - - os.symlink('../baz', os.path.join(base, 'bar/foo')) - with tar(base) as archive: - tar_data = tarfile.open(fileobj=archive) - assert sorted(tar_data.getnames()) == ['bar', 'bar/foo', 'foo'] - - @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='No UNIX sockets on Win32') - def test_tar_socket_file(self): - base = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, base) - for d in ['foo', 'bar']: - os.makedirs(os.path.join(base, d)) - sock = socket.socket(socket.AF_UNIX) - self.addCleanup(sock.close) - sock.bind(os.path.join(base, 'test.sock')) - with tar(base) as archive: - tar_data = tarfile.open(fileobj=archive) - assert sorted(tar_data.getnames()) == ['bar', 'foo'] - - def tar_test_negative_mtime_bug(self): - base = tempfile.mkdtemp() - filename = os.path.join(base, 'th.txt') - self.addCleanup(shutil.rmtree, base) - with open(filename, 'w') as f: - f.write('Invisible Full Moon') - os.utime(filename, (12345, -3600.0)) - with tar(base) as archive: - tar_data = tarfile.open(fileobj=archive) - assert tar_data.getnames() == ['th.txt'] - assert tar_data.getmember('th.txt').mtime == -3600 - - @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='No symlinks on Windows') - def test_tar_directory_link(self): - dirs = ['a', 'b', 'a/c'] - files = ['a/hello.py', 'b/utils.py', 'a/c/descend.py'] - base = make_tree(dirs, files) - self.addCleanup(shutil.rmtree, base) - os.symlink(os.path.join(base, 'b'), os.path.join(base, 'a/c/b')) - with tar(base) as archive: - tar_data = tarfile.open(fileobj=archive) - names = tar_data.getnames() - for member in dirs + files: - assert member in names - assert 'a/c/b' in names - assert 'a/c/b/utils.py' not in names - - class FormatEnvironmentTest(unittest.TestCase): def test_format_env_binary_unicode_value(self): env_dict = { From 37ba1c1eac97b5b3fcddb98f4ff6e1698985bc79 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 28 Jun 2018 14:30:52 -0700 Subject: [PATCH 0708/1301] Re-add walk method to utils.build Signed-off-by: Joffrey F --- docker/utils/build.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docker/utils/build.py b/docker/utils/build.py index 6f6241e933..4fa5751870 100644 --- a/docker/utils/build.py +++ b/docker/utils/build.py @@ -146,6 +146,11 @@ def normalize_slashes(p): return p +def walk(root, patterns, default=True): + pm = PatternMatcher(patterns) + return pm.walk(root) + + # Heavily based on # https://github.com/moby/moby/blob/master/pkg/fileutils/fileutils.go class PatternMatcher(object): From 098318ad95bd3c1330d63af378397b0990ab03a5 Mon Sep 17 00:00:00 2001 From: Marco Trillo Date: Fri, 29 Jun 2018 14:54:48 +0200 Subject: [PATCH 0709/1301] Add support for `uts_mode` parameter in `Client.create_host_config`. This parameter allows to set the UTS namespace of the container, as in the `--uts=X` Docker CLI parameter: The only allowed value, if set, is "host". Signed-off-by: Marco Trillo Signed-off-by: Diego Alvarez --- docker/api/container.py | 2 ++ docker/models/containers.py | 1 + docker/types/containers.py | 15 ++++++++++----- tests/integration/api_container_test.py | 10 ++++++++++ tests/unit/dockertypes_test.py | 6 ++++++ tests/unit/models_containers_test.py | 2 ++ 6 files changed, 31 insertions(+), 5 deletions(-) diff --git a/docker/api/container.py b/docker/api/container.py index d4f75f54b2..d8416066d5 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -547,6 +547,8 @@ def create_host_config(self, *args, **kwargs): userns_mode (str): Sets the user namespace mode for the container when user namespace remapping option is enabled. Supported values are: ``host`` + uts_mode (str): Sets the UTS namespace mode for the container. + Supported values are: ``host`` volumes_from (:py:class:`list`): List of container names or IDs to get volumes from. runtime (str): Runtime to use with this container. diff --git a/docker/models/containers.py b/docker/models/containers.py index b33a718f75..de6222ec49 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -995,6 +995,7 @@ def prune(self, filters=None): 'tmpfs', 'ulimits', 'userns_mode', + 'uts_mode', 'version', 'volumes_from', 'runtime' diff --git a/docker/types/containers.py b/docker/types/containers.py index 252142073f..e7841bcb46 100644 --- a/docker/types/containers.py +++ b/docker/types/containers.py @@ -115,11 +115,11 @@ def __init__(self, version, binds=None, port_bindings=None, device_read_iops=None, device_write_iops=None, oom_kill_disable=False, shm_size=None, sysctls=None, tmpfs=None, oom_score_adj=None, dns_opt=None, cpu_shares=None, - cpuset_cpus=None, userns_mode=None, pids_limit=None, - isolation=None, auto_remove=False, storage_opt=None, - init=None, init_path=None, volume_driver=None, - cpu_count=None, cpu_percent=None, nano_cpus=None, - cpuset_mems=None, runtime=None, mounts=None, + cpuset_cpus=None, userns_mode=None, uts_mode=None, + pids_limit=None, isolation=None, auto_remove=False, + storage_opt=None, init=None, init_path=None, + volume_driver=None, cpu_count=None, cpu_percent=None, + nano_cpus=None, cpuset_mems=None, runtime=None, mounts=None, cpu_rt_period=None, cpu_rt_runtime=None, device_cgroup_rules=None): @@ -392,6 +392,11 @@ def __init__(self, version, binds=None, port_bindings=None, raise host_config_value_error("userns_mode", userns_mode) self['UsernsMode'] = userns_mode + if uts_mode: + if uts_mode != "host": + raise host_config_value_error("uts_mode", uts_mode) + self['UTSMode'] = uts_mode + if pids_limit: if not isinstance(pids_limit, int): raise host_config_type_error('pids_limit', pids_limit, 'int') diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index ff7014879c..6ce846bb20 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -490,6 +490,16 @@ def test_create_with_device_cgroup_rules(self): self.client.start(ctnr) assert rule in self.client.logs(ctnr).decode('utf-8') + def test_create_with_uts_mode(self): + container = self.client.create_container( + BUSYBOX, ['echo'], host_config=self.client.create_host_config( + uts_mode='host' + ) + ) + self.tmp_containers.append(container) + config = self.client.inspect_container(container) + assert config['HostConfig']['UTSMode'] == 'host' + @pytest.mark.xfail( IS_WINDOWS_PLATFORM, reason='Test not designed for Windows platform' diff --git a/tests/unit/dockertypes_test.py b/tests/unit/dockertypes_test.py index 2be05784bb..cdacf8cd5b 100644 --- a/tests/unit/dockertypes_test.py +++ b/tests/unit/dockertypes_test.py @@ -85,6 +85,12 @@ def test_create_host_config_with_userns_mode(self): with pytest.raises(ValueError): create_host_config(version='1.23', userns_mode='host12') + def test_create_host_config_with_uts(self): + config = create_host_config(version='1.15', uts_mode='host') + assert config.get('UTSMode') == 'host' + with pytest.raises(ValueError): + create_host_config(version='1.15', uts_mode='host12') + def test_create_host_config_with_oom_score_adj(self): config = create_host_config(version='1.22', oom_score_adj=100) assert config.get('OomScoreAdj') == 100 diff --git a/tests/unit/models_containers_test.py b/tests/unit/models_containers_test.py index 48a5288869..22dd241064 100644 --- a/tests/unit/models_containers_test.py +++ b/tests/unit/models_containers_test.py @@ -95,6 +95,7 @@ def test_create_container_args(self): ulimits=[{"Name": "nofile", "Soft": 1024, "Hard": 2048}], user='bob', userns_mode='host', + uts_mode='host', version='1.23', volume_driver='some_driver', volumes=[ @@ -174,6 +175,7 @@ def test_create_container_args(self): 'Tmpfs': {'/blah': ''}, 'Ulimits': [{"Name": "nofile", "Soft": 1024, "Hard": 2048}], 'UsernsMode': 'host', + 'UTSMode': 'host', 'VolumesFrom': ['container'], }, healthcheck={'test': 'true'}, From 310ea913835be232d3e15b19ad212a48a3906be7 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 26 Jun 2018 17:40:49 -0700 Subject: [PATCH 0710/1301] Fix support for legacy .dockercfg auth config format Signed-off-by: Joffrey F --- docker/api/container.py | 5 ++- docker/auth.py | 6 +-- tests/integration/api_client_test.py | 40 ----------------- tests/unit/auth_test.py | 66 +++++++++++++++++++++++----- 4 files changed, 60 insertions(+), 57 deletions(-) diff --git a/docker/api/container.py b/docker/api/container.py index 05676f11e9..d4f75f54b2 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -139,8 +139,9 @@ def commit(self, container, repository=None, tag=None, message=None, 'changes': changes } u = self._url("/commit") - return self._result(self._post_json(u, data=conf, params=params), - json=True) + return self._result( + self._post_json(u, data=conf, params=params), json=True + ) def containers(self, quiet=False, all=False, trunc=False, latest=False, since=None, before=None, limit=-1, size=False, diff --git a/docker/auth.py b/docker/auth.py index 0c0cb204d9..9635f933ec 100644 --- a/docker/auth.py +++ b/docker/auth.py @@ -270,7 +270,7 @@ def load_config(config_path=None, config_dict=None): "Couldn't find auth-related section ; attempting to interpret" "as auth-only file" ) - return parse_auth(config_dict) + return {'auths': parse_auth(config_dict)} def _load_legacy_config(config_file): @@ -287,14 +287,14 @@ def _load_legacy_config(config_file): ) username, password = decode_auth(data[0]) - return { + return {'auths': { INDEX_NAME: { 'username': username, 'password': password, 'email': data[1], 'serveraddress': INDEX_URL, } - } + }} except Exception as e: log.debug(e) pass diff --git a/tests/integration/api_client_test.py b/tests/integration/api_client_test.py index 05281f8849..905e06484d 100644 --- a/tests/integration/api_client_test.py +++ b/tests/integration/api_client_test.py @@ -1,6 +1,3 @@ -import base64 -import os -import tempfile import time import unittest import warnings @@ -24,43 +21,6 @@ def test_info(self): assert 'Debug' in res -class LoadConfigTest(BaseAPIIntegrationTest): - def test_load_legacy_config(self): - folder = tempfile.mkdtemp() - self.tmp_folders.append(folder) - cfg_path = os.path.join(folder, '.dockercfg') - f = open(cfg_path, 'w') - auth_ = base64.b64encode(b'sakuya:izayoi').decode('ascii') - f.write('auth = {0}\n'.format(auth_)) - f.write('email = sakuya@scarlet.net') - f.close() - cfg = docker.auth.load_config(cfg_path) - assert cfg[docker.auth.INDEX_NAME] is not None - cfg = cfg[docker.auth.INDEX_NAME] - assert cfg['username'] == 'sakuya' - assert cfg['password'] == 'izayoi' - assert cfg['email'] == 'sakuya@scarlet.net' - assert cfg.get('Auth') is None - - def test_load_json_config(self): - folder = tempfile.mkdtemp() - self.tmp_folders.append(folder) - cfg_path = os.path.join(folder, '.dockercfg') - f = open(os.path.join(folder, '.dockercfg'), 'w') - auth_ = base64.b64encode(b'sakuya:izayoi').decode('ascii') - email_ = 'sakuya@scarlet.net' - f.write('{{"{0}": {{"auth": "{1}", "email": "{2}"}}}}\n'.format( - docker.auth.INDEX_URL, auth_, email_)) - f.close() - cfg = docker.auth.load_config(cfg_path) - assert cfg[docker.auth.INDEX_URL] is not None - cfg = cfg[docker.auth.INDEX_URL] - assert cfg['username'] == 'sakuya' - assert cfg['password'] == 'izayoi' - assert cfg['email'] == 'sakuya@scarlet.net' - assert cfg.get('Auth') is None - - class AutoDetectVersionTest(unittest.TestCase): def test_client_init(self): client = docker.APIClient(version='auto', **kwargs_from_env()) diff --git a/tests/unit/auth_test.py b/tests/unit/auth_test.py index ee32ca08a9..947d680018 100644 --- a/tests/unit/auth_test.py +++ b/tests/unit/auth_test.py @@ -282,22 +282,64 @@ def test_load_config_no_file(self): cfg = auth.load_config(folder) assert cfg is not None - def test_load_config(self): + def test_load_legacy_config(self): folder = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, folder) - dockercfg_path = os.path.join(folder, '.dockercfg') - with open(dockercfg_path, 'w') as f: - auth_ = base64.b64encode(b'sakuya:izayoi').decode('ascii') + cfg_path = os.path.join(folder, '.dockercfg') + auth_ = base64.b64encode(b'sakuya:izayoi').decode('ascii') + with open(cfg_path, 'w') as f: f.write('auth = {0}\n'.format(auth_)) f.write('email = sakuya@scarlet.net') - cfg = auth.load_config(dockercfg_path) - assert auth.INDEX_NAME in cfg - assert cfg[auth.INDEX_NAME] is not None - cfg = cfg[auth.INDEX_NAME] + + cfg = auth.load_config(cfg_path) + assert auth.resolve_authconfig(cfg) is not None + assert cfg['auths'][auth.INDEX_NAME] is not None + cfg = cfg['auths'][auth.INDEX_NAME] assert cfg['username'] == 'sakuya' assert cfg['password'] == 'izayoi' assert cfg['email'] == 'sakuya@scarlet.net' - assert cfg.get('auth') is None + assert cfg.get('Auth') is None + + def test_load_json_config(self): + folder = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, folder) + cfg_path = os.path.join(folder, '.dockercfg') + auth_ = base64.b64encode(b'sakuya:izayoi').decode('ascii') + email = 'sakuya@scarlet.net' + with open(cfg_path, 'w') as f: + json.dump( + {auth.INDEX_URL: {'auth': auth_, 'email': email}}, f + ) + cfg = auth.load_config(cfg_path) + assert auth.resolve_authconfig(cfg) is not None + assert cfg['auths'][auth.INDEX_URL] is not None + cfg = cfg['auths'][auth.INDEX_URL] + assert cfg['username'] == 'sakuya' + assert cfg['password'] == 'izayoi' + assert cfg['email'] == email + assert cfg.get('Auth') is None + + def test_load_modern_json_config(self): + folder = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, folder) + cfg_path = os.path.join(folder, 'config.json') + auth_ = base64.b64encode(b'sakuya:izayoi').decode('ascii') + email = 'sakuya@scarlet.net' + with open(cfg_path, 'w') as f: + json.dump({ + 'auths': { + auth.INDEX_URL: { + 'auth': auth_, 'email': email + } + } + }, f) + cfg = auth.load_config(cfg_path) + assert auth.resolve_authconfig(cfg) is not None + assert cfg['auths'][auth.INDEX_URL] is not None + cfg = cfg['auths'][auth.INDEX_URL] + assert cfg['username'] == 'sakuya' + assert cfg['password'] == 'izayoi' + assert cfg['email'] == email def test_load_config_with_random_name(self): folder = tempfile.mkdtemp() @@ -318,7 +360,7 @@ def test_load_config_with_random_name(self): with open(dockercfg_path, 'w') as f: json.dump(config, f) - cfg = auth.load_config(dockercfg_path) + cfg = auth.load_config(dockercfg_path)['auths'] assert registry in cfg assert cfg[registry] is not None cfg = cfg[registry] @@ -345,7 +387,7 @@ def test_load_config_custom_config_env(self): json.dump(config, f) with mock.patch.dict(os.environ, {'DOCKER_CONFIG': folder}): - cfg = auth.load_config(None) + cfg = auth.load_config(None)['auths'] assert registry in cfg assert cfg[registry] is not None cfg = cfg[registry] @@ -422,7 +464,7 @@ def test_load_config_unknown_keys(self): json.dump(config, f) cfg = auth.load_config(dockercfg_path) - assert cfg == {} + assert cfg == {'auths': {}} def test_load_config_invalid_auth_dict(self): folder = tempfile.mkdtemp() From 180a0fd08510776699ad50e0b2bf655450ad7777 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 28 Jun 2018 16:33:06 -0700 Subject: [PATCH 0711/1301] Fix detach assert function to account for new behavior in engine 18.06 Signed-off-by: Joffrey F --- tests/helpers.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/helpers.py b/tests/helpers.py index b6b493b385..b36d6d786f 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -123,7 +123,12 @@ def assert_cat_socket_detached_with_keys(sock, inputs): sock.sendall(b'make sure the socket is closed\n') else: sock.sendall(b"make sure the socket is closed\n") - assert sock.recv(32) == b'' + data = sock.recv(128) + # New in 18.06: error message is broadcast over the socket when reading + # after detach + assert data == b'' or data.startswith( + b'exec attach failed: error on attach stdin: read escape sequence' + ) def ctrl_with(char): From 5017de498e498cb7c0f6ef57fdf7e772a208481f Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 21 Jun 2018 00:15:55 -0700 Subject: [PATCH 0712/1301] Improved .dockerignore pattern processing to better match Docker CLI behavior Signed-off-by: Joffrey F --- docker/utils/build.py | 199 ++++++++++++++++++++++----------------- docker/utils/fnmatch.py | 1 + tests/unit/utils_test.py | 12 ++- 3 files changed, 123 insertions(+), 89 deletions(-) diff --git a/docker/utils/build.py b/docker/utils/build.py index b644c9fca1..9ce0095ce6 100644 --- a/docker/utils/build.py +++ b/docker/utils/build.py @@ -6,8 +6,7 @@ import tempfile from ..constants import IS_WINDOWS_PLATFORM -from fnmatch import fnmatch -from itertools import chain +from .fnmatch import fnmatch _SEP = re.compile('/|\\\\') if IS_WINDOWS_PLATFORM else re.compile('/') @@ -44,92 +43,9 @@ def exclude_paths(root, patterns, dockerfile=None): if dockerfile is None: dockerfile = 'Dockerfile' - def split_path(p): - return [pt for pt in re.split(_SEP, p) if pt and pt != '.'] - - def normalize(p): - # Leading and trailing slashes are not relevant. Yes, - # "foo.py/" must exclude the "foo.py" regular file. "." - # components are not relevant either, even if the whole - # pattern is only ".", as the Docker reference states: "For - # historical reasons, the pattern . is ignored." - # ".." component must be cleared with the potential previous - # component, regardless of whether it exists: "A preprocessing - # step [...] eliminates . and .. elements using Go's - # filepath.". - i = 0 - split = split_path(p) - while i < len(split): - if split[i] == '..': - del split[i] - if i > 0: - del split[i - 1] - i -= 1 - else: - i += 1 - return split - - patterns = ( - (True, normalize(p[1:])) - if p.startswith('!') else - (False, normalize(p)) - for p in patterns) - patterns = list(reversed(list(chain( - # Exclude empty patterns such as "." or the empty string. - filter(lambda p: p[1], patterns), - # Always include the Dockerfile and .dockerignore - [(True, split_path(dockerfile)), (True, ['.dockerignore'])])))) - return set(walk(root, patterns)) - - -def walk(root, patterns, default=True): - """ - A collection of file lying below root that should be included according to - patterns. - """ - - def match(p): - if p[1][0] == '**': - rec = (p[0], p[1][1:]) - return [p] + (match(rec) if rec[1] else [rec]) - elif fnmatch(f, p[1][0]): - return [(p[0], p[1][1:])] - else: - return [] - - for f in os.listdir(root): - cur = os.path.join(root, f) - # The patterns if recursing in that directory. - sub = list(chain(*(match(p) for p in patterns))) - # Whether this file is explicitely included / excluded. - hit = next((p[0] for p in sub if not p[1]), None) - # Whether this file is implicitely included / excluded. - matched = default if hit is None else hit - sub = list(filter(lambda p: p[1], sub)) - if os.path.isdir(cur) and not os.path.islink(cur): - # Entirely skip directories if there are no chance any subfile will - # be included. - if all(not p[0] for p in sub) and not matched: - continue - # I think this would greatly speed up dockerignore handling by not - # recursing into directories we are sure would be entirely - # included, and only yielding the directory itself, which will be - # recursively archived anyway. However the current unit test expect - # the full list of subfiles and I'm not 100% sure it would make no - # difference yet. - # if all(p[0] for p in sub) and matched: - # yield f - # continue - children = False - for r in (os.path.join(f, p) for p in walk(cur, sub, matched)): - yield r - children = True - # The current unit tests expect directories only under those - # conditions. It might be simplifiable though. - if (not sub or not children) and hit or hit is None and default: - yield f - elif matched: - yield f + patterns.append('!' + dockerfile) + pm = PatternMatcher(patterns) + return set(pm.walk(root)) def build_file_list(root): @@ -217,3 +133,110 @@ def mkbuildcontext(dockerfile): t.close() f.seek(0) return f + + +def split_path(p): + return [pt for pt in re.split(_SEP, p) if pt and pt != '.'] + + +# Heavily based on +# https://github.com/moby/moby/blob/master/pkg/fileutils/fileutils.go +class PatternMatcher(object): + def __init__(self, patterns): + self.patterns = list(filter( + lambda p: p.dirs, [Pattern(p) for p in patterns] + )) + self.patterns.append(Pattern('!.dockerignore')) + + def matches(self, filepath): + matched = False + parent_path = os.path.dirname(filepath) + parent_path_dirs = split_path(parent_path) + + for pattern in self.patterns: + negative = pattern.exclusion + match = pattern.match(filepath) + if not match and parent_path != '': + if len(pattern.dirs) <= len(parent_path_dirs): + match = pattern.match( + os.path.sep.join(parent_path_dirs[:len(pattern.dirs)]) + ) + + if match: + matched = not negative + + return matched + + def walk(self, root): + def rec_walk(current_dir): + for f in os.listdir(current_dir): + fpath = os.path.join( + os.path.relpath(current_dir, root), f + ) + if fpath.startswith('.' + os.path.sep): + fpath = fpath[2:] + match = self.matches(fpath) + if not match: + yield fpath + + cur = os.path.join(root, fpath) + if not os.path.isdir(cur) or os.path.islink(cur): + continue + + if match: + # If we want to skip this file and its a directory + # then we should first check to see if there's an + # excludes pattern (e.g. !dir/file) that starts with this + # dir. If so then we can't skip this dir. + skip = True + + for pat in self.patterns: + if not pat.exclusion: + continue + if pat.cleaned_pattern.startswith(fpath): + skip = False + break + if skip: + continue + for sub in rec_walk(cur): + yield sub + + return rec_walk(root) + + +class Pattern(object): + def __init__(self, pattern_str): + self.exclusion = False + if pattern_str.startswith('!'): + self.exclusion = True + pattern_str = pattern_str[1:] + + self.dirs = self.normalize(pattern_str) + self.cleaned_pattern = '/'.join(self.dirs) + + @classmethod + def normalize(cls, p): + + # Leading and trailing slashes are not relevant. Yes, + # "foo.py/" must exclude the "foo.py" regular file. "." + # components are not relevant either, even if the whole + # pattern is only ".", as the Docker reference states: "For + # historical reasons, the pattern . is ignored." + # ".." component must be cleared with the potential previous + # component, regardless of whether it exists: "A preprocessing + # step [...] eliminates . and .. elements using Go's + # filepath.". + i = 0 + split = split_path(p) + while i < len(split): + if split[i] == '..': + del split[i] + if i > 0: + del split[i - 1] + i -= 1 + else: + i += 1 + return split + + def match(self, filepath): + return fnmatch(filepath, self.cleaned_pattern) diff --git a/docker/utils/fnmatch.py b/docker/utils/fnmatch.py index 42461dd7d3..cc940a2e65 100644 --- a/docker/utils/fnmatch.py +++ b/docker/utils/fnmatch.py @@ -111,4 +111,5 @@ def translate(pat): res = '%s[%s]' % (res, stuff) else: res = res + re.escape(c) + return res + '$' diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index 00456e8c11..467e835c56 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -887,12 +887,22 @@ def test_trailing_double_wildcard(self): ) ) + def test_double_wildcard_with_exception(self): + assert self.exclude(['**', '!bar', '!foo/bar']) == convert_paths( + set([ + 'foo/bar', 'foo/bar/a.py', 'bar', 'bar/a.py', 'Dockerfile', + '.dockerignore', + ]) + ) + def test_include_wildcard(self): + # This may be surprising but it matches the CLI's behavior + # (tested with 18.05.0-ce on linux) base = make_tree(['a'], ['a/b.py']) assert exclude_paths( base, ['*', '!*/b.py'] - ) == convert_paths(['a/b.py']) + ) == set() def test_last_line_precedence(self): base = make_tree( From 088094c64458948ec001aefd2cb9a24f3ced33fc Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 28 Jun 2018 14:07:38 -0700 Subject: [PATCH 0713/1301] On Windows, convert paths to use forward slashes before fnmatch call Signed-off-by: Joffrey F --- docker/utils/build.py | 18 +- tests/unit/utils_build_test.py | 493 +++++++++++++++++++++++++++++++++ tests/unit/utils_test.py | 490 +------------------------------- 3 files changed, 511 insertions(+), 490 deletions(-) create mode 100644 tests/unit/utils_build_test.py diff --git a/docker/utils/build.py b/docker/utils/build.py index 9ce0095ce6..6f6241e933 100644 --- a/docker/utils/build.py +++ b/docker/utils/build.py @@ -1,12 +1,13 @@ import io import os import re -import six import tarfile import tempfile -from ..constants import IS_WINDOWS_PLATFORM +import six + from .fnmatch import fnmatch +from ..constants import IS_WINDOWS_PLATFORM _SEP = re.compile('/|\\\\') if IS_WINDOWS_PLATFORM else re.compile('/') @@ -139,6 +140,12 @@ def split_path(p): return [pt for pt in re.split(_SEP, p) if pt and pt != '.'] +def normalize_slashes(p): + if IS_WINDOWS_PLATFORM: + return '/'.join(split_path(p)) + return p + + # Heavily based on # https://github.com/moby/moby/blob/master/pkg/fileutils/fileutils.go class PatternMatcher(object): @@ -184,7 +191,7 @@ def rec_walk(current_dir): continue if match: - # If we want to skip this file and its a directory + # If we want to skip this file and it's a directory # then we should first check to see if there's an # excludes pattern (e.g. !dir/file) that starts with this # dir. If so then we can't skip this dir. @@ -193,7 +200,8 @@ def rec_walk(current_dir): for pat in self.patterns: if not pat.exclusion: continue - if pat.cleaned_pattern.startswith(fpath): + if pat.cleaned_pattern.startswith( + normalize_slashes(fpath)): skip = False break if skip: @@ -239,4 +247,4 @@ def normalize(cls, p): return split def match(self, filepath): - return fnmatch(filepath, self.cleaned_pattern) + return fnmatch(normalize_slashes(filepath), self.cleaned_pattern) diff --git a/tests/unit/utils_build_test.py b/tests/unit/utils_build_test.py new file mode 100644 index 0000000000..012f15b46a --- /dev/null +++ b/tests/unit/utils_build_test.py @@ -0,0 +1,493 @@ +# -*- coding: utf-8 -*- + +import os +import os.path +import shutil +import socket +import tarfile +import tempfile +import unittest + + +from docker.constants import IS_WINDOWS_PLATFORM +from docker.utils import exclude_paths, tar + +import pytest + +from ..helpers import make_tree + + +def convert_paths(collection): + return set(map(convert_path, collection)) + + +def convert_path(path): + return path.replace('/', os.path.sep) + + +class ExcludePathsTest(unittest.TestCase): + dirs = [ + 'foo', + 'foo/bar', + 'bar', + 'target', + 'target/subdir', + 'subdir', + 'subdir/target', + 'subdir/target/subdir', + 'subdir/subdir2', + 'subdir/subdir2/target', + 'subdir/subdir2/target/subdir' + ] + + files = [ + 'Dockerfile', + 'Dockerfile.alt', + '.dockerignore', + 'a.py', + 'a.go', + 'b.py', + 'cde.py', + 'foo/a.py', + 'foo/b.py', + 'foo/bar/a.py', + 'bar/a.py', + 'foo/Dockerfile3', + 'target/file.txt', + 'target/subdir/file.txt', + 'subdir/file.txt', + 'subdir/target/file.txt', + 'subdir/target/subdir/file.txt', + 'subdir/subdir2/file.txt', + 'subdir/subdir2/target/file.txt', + 'subdir/subdir2/target/subdir/file.txt', + ] + + all_paths = set(dirs + files) + + def setUp(self): + self.base = make_tree(self.dirs, self.files) + + def tearDown(self): + shutil.rmtree(self.base) + + def exclude(self, patterns, dockerfile=None): + return set(exclude_paths(self.base, patterns, dockerfile=dockerfile)) + + def test_no_excludes(self): + assert self.exclude(['']) == convert_paths(self.all_paths) + + def test_no_dupes(self): + paths = exclude_paths(self.base, ['!a.py']) + assert sorted(paths) == sorted(set(paths)) + + def test_wildcard_exclude(self): + assert self.exclude(['*']) == set(['Dockerfile', '.dockerignore']) + + def test_exclude_dockerfile_dockerignore(self): + """ + Even if the .dockerignore file explicitly says to exclude + Dockerfile and/or .dockerignore, don't exclude them from + the actual tar file. + """ + assert self.exclude(['Dockerfile', '.dockerignore']) == convert_paths( + self.all_paths + ) + + def test_exclude_custom_dockerfile(self): + """ + If we're using a custom Dockerfile, make sure that's not + excluded. + """ + assert self.exclude(['*'], dockerfile='Dockerfile.alt') == set( + ['Dockerfile.alt', '.dockerignore'] + ) + + assert self.exclude( + ['*'], dockerfile='foo/Dockerfile3' + ) == convert_paths(set(['foo/Dockerfile3', '.dockerignore'])) + + # https://github.com/docker/docker-py/issues/1956 + assert self.exclude( + ['*'], dockerfile='./foo/Dockerfile3' + ) == convert_paths(set(['foo/Dockerfile3', '.dockerignore'])) + + def test_exclude_dockerfile_child(self): + includes = self.exclude(['foo/'], dockerfile='foo/Dockerfile3') + assert convert_path('foo/Dockerfile3') in includes + assert convert_path('foo/a.py') not in includes + + def test_single_filename(self): + assert self.exclude(['a.py']) == convert_paths( + self.all_paths - set(['a.py']) + ) + + def test_single_filename_leading_dot_slash(self): + assert self.exclude(['./a.py']) == convert_paths( + self.all_paths - set(['a.py']) + ) + + # As odd as it sounds, a filename pattern with a trailing slash on the + # end *will* result in that file being excluded. + def test_single_filename_trailing_slash(self): + assert self.exclude(['a.py/']) == convert_paths( + self.all_paths - set(['a.py']) + ) + + def test_wildcard_filename_start(self): + assert self.exclude(['*.py']) == convert_paths( + self.all_paths - set(['a.py', 'b.py', 'cde.py']) + ) + + def test_wildcard_with_exception(self): + assert self.exclude(['*.py', '!b.py']) == convert_paths( + self.all_paths - set(['a.py', 'cde.py']) + ) + + def test_wildcard_with_wildcard_exception(self): + assert self.exclude(['*.*', '!*.go']) == convert_paths( + self.all_paths - set([ + 'a.py', 'b.py', 'cde.py', 'Dockerfile.alt', + ]) + ) + + def test_wildcard_filename_end(self): + assert self.exclude(['a.*']) == convert_paths( + self.all_paths - set(['a.py', 'a.go']) + ) + + def test_question_mark(self): + assert self.exclude(['?.py']) == convert_paths( + self.all_paths - set(['a.py', 'b.py']) + ) + + def test_single_subdir_single_filename(self): + assert self.exclude(['foo/a.py']) == convert_paths( + self.all_paths - set(['foo/a.py']) + ) + + def test_single_subdir_single_filename_leading_slash(self): + assert self.exclude(['/foo/a.py']) == convert_paths( + self.all_paths - set(['foo/a.py']) + ) + + def test_exclude_include_absolute_path(self): + base = make_tree([], ['a.py', 'b.py']) + assert exclude_paths( + base, + ['/*', '!/*.py'] + ) == set(['a.py', 'b.py']) + + def test_single_subdir_with_path_traversal(self): + assert self.exclude(['foo/whoops/../a.py']) == convert_paths( + self.all_paths - set(['foo/a.py']) + ) + + def test_single_subdir_wildcard_filename(self): + assert self.exclude(['foo/*.py']) == convert_paths( + self.all_paths - set(['foo/a.py', 'foo/b.py']) + ) + + def test_wildcard_subdir_single_filename(self): + assert self.exclude(['*/a.py']) == convert_paths( + self.all_paths - set(['foo/a.py', 'bar/a.py']) + ) + + def test_wildcard_subdir_wildcard_filename(self): + assert self.exclude(['*/*.py']) == convert_paths( + self.all_paths - set(['foo/a.py', 'foo/b.py', 'bar/a.py']) + ) + + def test_directory(self): + assert self.exclude(['foo']) == convert_paths( + self.all_paths - set([ + 'foo', 'foo/a.py', 'foo/b.py', 'foo/bar', 'foo/bar/a.py', + 'foo/Dockerfile3' + ]) + ) + + def test_directory_with_trailing_slash(self): + assert self.exclude(['foo']) == convert_paths( + self.all_paths - set([ + 'foo', 'foo/a.py', 'foo/b.py', + 'foo/bar', 'foo/bar/a.py', 'foo/Dockerfile3' + ]) + ) + + def test_directory_with_single_exception(self): + assert self.exclude(['foo', '!foo/bar/a.py']) == convert_paths( + self.all_paths - set([ + 'foo/a.py', 'foo/b.py', 'foo', 'foo/bar', + 'foo/Dockerfile3' + ]) + ) + + def test_directory_with_subdir_exception(self): + assert self.exclude(['foo', '!foo/bar']) == convert_paths( + self.all_paths - set([ + 'foo/a.py', 'foo/b.py', 'foo', 'foo/Dockerfile3' + ]) + ) + + @pytest.mark.skipif( + not IS_WINDOWS_PLATFORM, reason='Backslash patterns only on Windows' + ) + def test_directory_with_subdir_exception_win32_pathsep(self): + assert self.exclude(['foo', '!foo\\bar']) == convert_paths( + self.all_paths - set([ + 'foo/a.py', 'foo/b.py', 'foo', 'foo/Dockerfile3' + ]) + ) + + def test_directory_with_wildcard_exception(self): + assert self.exclude(['foo', '!foo/*.py']) == convert_paths( + self.all_paths - set([ + 'foo/bar', 'foo/bar/a.py', 'foo', 'foo/Dockerfile3' + ]) + ) + + def test_subdirectory(self): + assert self.exclude(['foo/bar']) == convert_paths( + self.all_paths - set(['foo/bar', 'foo/bar/a.py']) + ) + + @pytest.mark.skipif( + not IS_WINDOWS_PLATFORM, reason='Backslash patterns only on Windows' + ) + def test_subdirectory_win32_pathsep(self): + assert self.exclude(['foo\\bar']) == convert_paths( + self.all_paths - set(['foo/bar', 'foo/bar/a.py']) + ) + + def test_double_wildcard(self): + assert self.exclude(['**/a.py']) == convert_paths( + self.all_paths - set( + ['a.py', 'foo/a.py', 'foo/bar/a.py', 'bar/a.py'] + ) + ) + + assert self.exclude(['foo/**/bar']) == convert_paths( + self.all_paths - set(['foo/bar', 'foo/bar/a.py']) + ) + + def test_single_and_double_wildcard(self): + assert self.exclude(['**/target/*/*']) == convert_paths( + self.all_paths - set( + ['target/subdir/file.txt', + 'subdir/target/subdir/file.txt', + 'subdir/subdir2/target/subdir/file.txt'] + ) + ) + + def test_trailing_double_wildcard(self): + assert self.exclude(['subdir/**']) == convert_paths( + self.all_paths - set( + ['subdir/file.txt', + 'subdir/target/file.txt', + 'subdir/target/subdir/file.txt', + 'subdir/subdir2/file.txt', + 'subdir/subdir2/target/file.txt', + 'subdir/subdir2/target/subdir/file.txt', + 'subdir/target', + 'subdir/target/subdir', + 'subdir/subdir2', + 'subdir/subdir2/target', + 'subdir/subdir2/target/subdir'] + ) + ) + + def test_double_wildcard_with_exception(self): + assert self.exclude(['**', '!bar', '!foo/bar']) == convert_paths( + set([ + 'foo/bar', 'foo/bar/a.py', 'bar', 'bar/a.py', 'Dockerfile', + '.dockerignore', + ]) + ) + + def test_include_wildcard(self): + # This may be surprising but it matches the CLI's behavior + # (tested with 18.05.0-ce on linux) + base = make_tree(['a'], ['a/b.py']) + assert exclude_paths( + base, + ['*', '!*/b.py'] + ) == set() + + def test_last_line_precedence(self): + base = make_tree( + [], + ['garbage.md', + 'trash.md', + 'README.md', + 'README-bis.md', + 'README-secret.md']) + assert exclude_paths( + base, + ['*.md', '!README*.md', 'README-secret.md'] + ) == set(['README.md', 'README-bis.md']) + + def test_parent_directory(self): + base = make_tree( + [], + ['a.py', + 'b.py', + 'c.py']) + # Dockerignore reference stipulates that absolute paths are + # equivalent to relative paths, hence /../foo should be + # equivalent to ../foo. It also stipulates that paths are run + # through Go's filepath.Clean, which explicitely "replace + # "/.." by "/" at the beginning of a path". + assert exclude_paths( + base, + ['../a.py', '/../b.py'] + ) == set(['c.py']) + + +class TarTest(unittest.TestCase): + def test_tar_with_excludes(self): + dirs = [ + 'foo', + 'foo/bar', + 'bar', + ] + + files = [ + 'Dockerfile', + 'Dockerfile.alt', + '.dockerignore', + 'a.py', + 'a.go', + 'b.py', + 'cde.py', + 'foo/a.py', + 'foo/b.py', + 'foo/bar/a.py', + 'bar/a.py', + ] + + exclude = [ + '*.py', + '!b.py', + '!a.go', + 'foo', + 'Dockerfile*', + '.dockerignore', + ] + + expected_names = set([ + 'Dockerfile', + '.dockerignore', + 'a.go', + 'b.py', + 'bar', + 'bar/a.py', + ]) + + base = make_tree(dirs, files) + self.addCleanup(shutil.rmtree, base) + + with tar(base, exclude=exclude) as archive: + tar_data = tarfile.open(fileobj=archive) + assert sorted(tar_data.getnames()) == sorted(expected_names) + + def test_tar_with_empty_directory(self): + base = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, base) + for d in ['foo', 'bar']: + os.makedirs(os.path.join(base, d)) + with tar(base) as archive: + tar_data = tarfile.open(fileobj=archive) + assert sorted(tar_data.getnames()) == ['bar', 'foo'] + + @pytest.mark.skipif( + IS_WINDOWS_PLATFORM or os.geteuid() == 0, + reason='root user always has access ; no chmod on Windows' + ) + def test_tar_with_inaccessible_file(self): + base = tempfile.mkdtemp() + full_path = os.path.join(base, 'foo') + self.addCleanup(shutil.rmtree, base) + with open(full_path, 'w') as f: + f.write('content') + os.chmod(full_path, 0o222) + with pytest.raises(IOError) as ei: + tar(base) + + assert 'Can not read file in context: {}'.format(full_path) in ( + ei.exconly() + ) + + @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='No symlinks on Windows') + def test_tar_with_file_symlinks(self): + base = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, base) + with open(os.path.join(base, 'foo'), 'w') as f: + f.write("content") + os.makedirs(os.path.join(base, 'bar')) + os.symlink('../foo', os.path.join(base, 'bar/foo')) + with tar(base) as archive: + tar_data = tarfile.open(fileobj=archive) + assert sorted(tar_data.getnames()) == ['bar', 'bar/foo', 'foo'] + + @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='No symlinks on Windows') + def test_tar_with_directory_symlinks(self): + base = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, base) + for d in ['foo', 'bar']: + os.makedirs(os.path.join(base, d)) + os.symlink('../foo', os.path.join(base, 'bar/foo')) + with tar(base) as archive: + tar_data = tarfile.open(fileobj=archive) + assert sorted(tar_data.getnames()) == ['bar', 'bar/foo', 'foo'] + + @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='No symlinks on Windows') + def test_tar_with_broken_symlinks(self): + base = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, base) + for d in ['foo', 'bar']: + os.makedirs(os.path.join(base, d)) + + os.symlink('../baz', os.path.join(base, 'bar/foo')) + with tar(base) as archive: + tar_data = tarfile.open(fileobj=archive) + assert sorted(tar_data.getnames()) == ['bar', 'bar/foo', 'foo'] + + @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='No UNIX sockets on Win32') + def test_tar_socket_file(self): + base = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, base) + for d in ['foo', 'bar']: + os.makedirs(os.path.join(base, d)) + sock = socket.socket(socket.AF_UNIX) + self.addCleanup(sock.close) + sock.bind(os.path.join(base, 'test.sock')) + with tar(base) as archive: + tar_data = tarfile.open(fileobj=archive) + assert sorted(tar_data.getnames()) == ['bar', 'foo'] + + def tar_test_negative_mtime_bug(self): + base = tempfile.mkdtemp() + filename = os.path.join(base, 'th.txt') + self.addCleanup(shutil.rmtree, base) + with open(filename, 'w') as f: + f.write('Invisible Full Moon') + os.utime(filename, (12345, -3600.0)) + with tar(base) as archive: + tar_data = tarfile.open(fileobj=archive) + assert tar_data.getnames() == ['th.txt'] + assert tar_data.getmember('th.txt').mtime == -3600 + + @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='No symlinks on Windows') + def test_tar_directory_link(self): + dirs = ['a', 'b', 'a/c'] + files = ['a/hello.py', 'b/utils.py', 'a/c/descend.py'] + base = make_tree(dirs, files) + self.addCleanup(shutil.rmtree, base) + os.symlink(os.path.join(base, 'b'), os.path.join(base, 'a/c/b')) + with tar(base) as archive: + tar_data = tarfile.open(fileobj=archive) + names = tar_data.getnames() + for member in dirs + files: + assert member in names + assert 'a/c/b' in names + assert 'a/c/b/utils.py' not in names diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index 467e835c56..8880cfef0f 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -5,29 +5,25 @@ import os import os.path import shutil -import socket import sys -import tarfile import tempfile import unittest -import pytest -import six from docker.api.client import APIClient -from docker.constants import IS_WINDOWS_PLATFORM from docker.errors import DockerException from docker.utils import ( - parse_repository_tag, parse_host, convert_filters, kwargs_from_env, - parse_bytes, parse_env_file, exclude_paths, convert_volume_binds, - decode_json_header, tar, split_command, parse_devices, update_headers, + convert_filters, convert_volume_binds, decode_json_header, kwargs_from_env, + parse_bytes, parse_devices, parse_env_file, parse_host, + parse_repository_tag, split_command, update_headers, ) from docker.utils.ports import build_port_bindings, split_port from docker.utils.utils import format_environment -from ..helpers import make_tree +import pytest +import six TEST_CERT_DIR = os.path.join( os.path.dirname(__file__), @@ -608,482 +604,6 @@ def test_build_port_bindings_with_nonmatching_internal_port_ranges(self): assert port_bindings["2000"] == [("127.0.0.1", "2000")] -def convert_paths(collection): - return set(map(convert_path, collection)) - - -def convert_path(path): - return path.replace('/', os.path.sep) - - -class ExcludePathsTest(unittest.TestCase): - dirs = [ - 'foo', - 'foo/bar', - 'bar', - 'target', - 'target/subdir', - 'subdir', - 'subdir/target', - 'subdir/target/subdir', - 'subdir/subdir2', - 'subdir/subdir2/target', - 'subdir/subdir2/target/subdir' - ] - - files = [ - 'Dockerfile', - 'Dockerfile.alt', - '.dockerignore', - 'a.py', - 'a.go', - 'b.py', - 'cde.py', - 'foo/a.py', - 'foo/b.py', - 'foo/bar/a.py', - 'bar/a.py', - 'foo/Dockerfile3', - 'target/file.txt', - 'target/subdir/file.txt', - 'subdir/file.txt', - 'subdir/target/file.txt', - 'subdir/target/subdir/file.txt', - 'subdir/subdir2/file.txt', - 'subdir/subdir2/target/file.txt', - 'subdir/subdir2/target/subdir/file.txt', - ] - - all_paths = set(dirs + files) - - def setUp(self): - self.base = make_tree(self.dirs, self.files) - - def tearDown(self): - shutil.rmtree(self.base) - - def exclude(self, patterns, dockerfile=None): - return set(exclude_paths(self.base, patterns, dockerfile=dockerfile)) - - def test_no_excludes(self): - assert self.exclude(['']) == convert_paths(self.all_paths) - - def test_no_dupes(self): - paths = exclude_paths(self.base, ['!a.py']) - assert sorted(paths) == sorted(set(paths)) - - def test_wildcard_exclude(self): - assert self.exclude(['*']) == set(['Dockerfile', '.dockerignore']) - - def test_exclude_dockerfile_dockerignore(self): - """ - Even if the .dockerignore file explicitly says to exclude - Dockerfile and/or .dockerignore, don't exclude them from - the actual tar file. - """ - assert self.exclude(['Dockerfile', '.dockerignore']) == convert_paths( - self.all_paths - ) - - def test_exclude_custom_dockerfile(self): - """ - If we're using a custom Dockerfile, make sure that's not - excluded. - """ - assert self.exclude(['*'], dockerfile='Dockerfile.alt') == set( - ['Dockerfile.alt', '.dockerignore'] - ) - - assert self.exclude( - ['*'], dockerfile='foo/Dockerfile3' - ) == convert_paths(set(['foo/Dockerfile3', '.dockerignore'])) - - # https://github.com/docker/docker-py/issues/1956 - assert self.exclude( - ['*'], dockerfile='./foo/Dockerfile3' - ) == convert_paths(set(['foo/Dockerfile3', '.dockerignore'])) - - def test_exclude_dockerfile_child(self): - includes = self.exclude(['foo/'], dockerfile='foo/Dockerfile3') - assert convert_path('foo/Dockerfile3') in includes - assert convert_path('foo/a.py') not in includes - - def test_single_filename(self): - assert self.exclude(['a.py']) == convert_paths( - self.all_paths - set(['a.py']) - ) - - def test_single_filename_leading_dot_slash(self): - assert self.exclude(['./a.py']) == convert_paths( - self.all_paths - set(['a.py']) - ) - - # As odd as it sounds, a filename pattern with a trailing slash on the - # end *will* result in that file being excluded. - def test_single_filename_trailing_slash(self): - assert self.exclude(['a.py/']) == convert_paths( - self.all_paths - set(['a.py']) - ) - - def test_wildcard_filename_start(self): - assert self.exclude(['*.py']) == convert_paths( - self.all_paths - set(['a.py', 'b.py', 'cde.py']) - ) - - def test_wildcard_with_exception(self): - assert self.exclude(['*.py', '!b.py']) == convert_paths( - self.all_paths - set(['a.py', 'cde.py']) - ) - - def test_wildcard_with_wildcard_exception(self): - assert self.exclude(['*.*', '!*.go']) == convert_paths( - self.all_paths - set([ - 'a.py', 'b.py', 'cde.py', 'Dockerfile.alt', - ]) - ) - - def test_wildcard_filename_end(self): - assert self.exclude(['a.*']) == convert_paths( - self.all_paths - set(['a.py', 'a.go']) - ) - - def test_question_mark(self): - assert self.exclude(['?.py']) == convert_paths( - self.all_paths - set(['a.py', 'b.py']) - ) - - def test_single_subdir_single_filename(self): - assert self.exclude(['foo/a.py']) == convert_paths( - self.all_paths - set(['foo/a.py']) - ) - - def test_single_subdir_single_filename_leading_slash(self): - assert self.exclude(['/foo/a.py']) == convert_paths( - self.all_paths - set(['foo/a.py']) - ) - - def test_exclude_include_absolute_path(self): - base = make_tree([], ['a.py', 'b.py']) - assert exclude_paths( - base, - ['/*', '!/*.py'] - ) == set(['a.py', 'b.py']) - - def test_single_subdir_with_path_traversal(self): - assert self.exclude(['foo/whoops/../a.py']) == convert_paths( - self.all_paths - set(['foo/a.py']) - ) - - def test_single_subdir_wildcard_filename(self): - assert self.exclude(['foo/*.py']) == convert_paths( - self.all_paths - set(['foo/a.py', 'foo/b.py']) - ) - - def test_wildcard_subdir_single_filename(self): - assert self.exclude(['*/a.py']) == convert_paths( - self.all_paths - set(['foo/a.py', 'bar/a.py']) - ) - - def test_wildcard_subdir_wildcard_filename(self): - assert self.exclude(['*/*.py']) == convert_paths( - self.all_paths - set(['foo/a.py', 'foo/b.py', 'bar/a.py']) - ) - - def test_directory(self): - assert self.exclude(['foo']) == convert_paths( - self.all_paths - set([ - 'foo', 'foo/a.py', 'foo/b.py', 'foo/bar', 'foo/bar/a.py', - 'foo/Dockerfile3' - ]) - ) - - def test_directory_with_trailing_slash(self): - assert self.exclude(['foo']) == convert_paths( - self.all_paths - set([ - 'foo', 'foo/a.py', 'foo/b.py', - 'foo/bar', 'foo/bar/a.py', 'foo/Dockerfile3' - ]) - ) - - def test_directory_with_single_exception(self): - assert self.exclude(['foo', '!foo/bar/a.py']) == convert_paths( - self.all_paths - set([ - 'foo/a.py', 'foo/b.py', 'foo', 'foo/bar', - 'foo/Dockerfile3' - ]) - ) - - def test_directory_with_subdir_exception(self): - assert self.exclude(['foo', '!foo/bar']) == convert_paths( - self.all_paths - set([ - 'foo/a.py', 'foo/b.py', 'foo', 'foo/Dockerfile3' - ]) - ) - - @pytest.mark.skipif( - not IS_WINDOWS_PLATFORM, reason='Backslash patterns only on Windows' - ) - def test_directory_with_subdir_exception_win32_pathsep(self): - assert self.exclude(['foo', '!foo\\bar']) == convert_paths( - self.all_paths - set([ - 'foo/a.py', 'foo/b.py', 'foo', 'foo/Dockerfile3' - ]) - ) - - def test_directory_with_wildcard_exception(self): - assert self.exclude(['foo', '!foo/*.py']) == convert_paths( - self.all_paths - set([ - 'foo/bar', 'foo/bar/a.py', 'foo', 'foo/Dockerfile3' - ]) - ) - - def test_subdirectory(self): - assert self.exclude(['foo/bar']) == convert_paths( - self.all_paths - set(['foo/bar', 'foo/bar/a.py']) - ) - - @pytest.mark.skipif( - not IS_WINDOWS_PLATFORM, reason='Backslash patterns only on Windows' - ) - def test_subdirectory_win32_pathsep(self): - assert self.exclude(['foo\\bar']) == convert_paths( - self.all_paths - set(['foo/bar', 'foo/bar/a.py']) - ) - - def test_double_wildcard(self): - assert self.exclude(['**/a.py']) == convert_paths( - self.all_paths - set( - ['a.py', 'foo/a.py', 'foo/bar/a.py', 'bar/a.py'] - ) - ) - - assert self.exclude(['foo/**/bar']) == convert_paths( - self.all_paths - set(['foo/bar', 'foo/bar/a.py']) - ) - - def test_single_and_double_wildcard(self): - assert self.exclude(['**/target/*/*']) == convert_paths( - self.all_paths - set( - ['target/subdir/file.txt', - 'subdir/target/subdir/file.txt', - 'subdir/subdir2/target/subdir/file.txt'] - ) - ) - - def test_trailing_double_wildcard(self): - assert self.exclude(['subdir/**']) == convert_paths( - self.all_paths - set( - ['subdir/file.txt', - 'subdir/target/file.txt', - 'subdir/target/subdir/file.txt', - 'subdir/subdir2/file.txt', - 'subdir/subdir2/target/file.txt', - 'subdir/subdir2/target/subdir/file.txt', - 'subdir/target', - 'subdir/target/subdir', - 'subdir/subdir2', - 'subdir/subdir2/target', - 'subdir/subdir2/target/subdir'] - ) - ) - - def test_double_wildcard_with_exception(self): - assert self.exclude(['**', '!bar', '!foo/bar']) == convert_paths( - set([ - 'foo/bar', 'foo/bar/a.py', 'bar', 'bar/a.py', 'Dockerfile', - '.dockerignore', - ]) - ) - - def test_include_wildcard(self): - # This may be surprising but it matches the CLI's behavior - # (tested with 18.05.0-ce on linux) - base = make_tree(['a'], ['a/b.py']) - assert exclude_paths( - base, - ['*', '!*/b.py'] - ) == set() - - def test_last_line_precedence(self): - base = make_tree( - [], - ['garbage.md', - 'thrash.md', - 'README.md', - 'README-bis.md', - 'README-secret.md']) - assert exclude_paths( - base, - ['*.md', '!README*.md', 'README-secret.md'] - ) == set(['README.md', 'README-bis.md']) - - def test_parent_directory(self): - base = make_tree( - [], - ['a.py', - 'b.py', - 'c.py']) - # Dockerignore reference stipulates that absolute paths are - # equivalent to relative paths, hence /../foo should be - # equivalent to ../foo. It also stipulates that paths are run - # through Go's filepath.Clean, which explicitely "replace - # "/.." by "/" at the beginning of a path". - assert exclude_paths( - base, - ['../a.py', '/../b.py'] - ) == set(['c.py']) - - -class TarTest(unittest.TestCase): - def test_tar_with_excludes(self): - dirs = [ - 'foo', - 'foo/bar', - 'bar', - ] - - files = [ - 'Dockerfile', - 'Dockerfile.alt', - '.dockerignore', - 'a.py', - 'a.go', - 'b.py', - 'cde.py', - 'foo/a.py', - 'foo/b.py', - 'foo/bar/a.py', - 'bar/a.py', - ] - - exclude = [ - '*.py', - '!b.py', - '!a.go', - 'foo', - 'Dockerfile*', - '.dockerignore', - ] - - expected_names = set([ - 'Dockerfile', - '.dockerignore', - 'a.go', - 'b.py', - 'bar', - 'bar/a.py', - ]) - - base = make_tree(dirs, files) - self.addCleanup(shutil.rmtree, base) - - with tar(base, exclude=exclude) as archive: - tar_data = tarfile.open(fileobj=archive) - assert sorted(tar_data.getnames()) == sorted(expected_names) - - def test_tar_with_empty_directory(self): - base = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, base) - for d in ['foo', 'bar']: - os.makedirs(os.path.join(base, d)) - with tar(base) as archive: - tar_data = tarfile.open(fileobj=archive) - assert sorted(tar_data.getnames()) == ['bar', 'foo'] - - @pytest.mark.skipif( - IS_WINDOWS_PLATFORM or os.geteuid() == 0, - reason='root user always has access ; no chmod on Windows' - ) - def test_tar_with_inaccessible_file(self): - base = tempfile.mkdtemp() - full_path = os.path.join(base, 'foo') - self.addCleanup(shutil.rmtree, base) - with open(full_path, 'w') as f: - f.write('content') - os.chmod(full_path, 0o222) - with pytest.raises(IOError) as ei: - tar(base) - - assert 'Can not read file in context: {}'.format(full_path) in ( - ei.exconly() - ) - - @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='No symlinks on Windows') - def test_tar_with_file_symlinks(self): - base = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, base) - with open(os.path.join(base, 'foo'), 'w') as f: - f.write("content") - os.makedirs(os.path.join(base, 'bar')) - os.symlink('../foo', os.path.join(base, 'bar/foo')) - with tar(base) as archive: - tar_data = tarfile.open(fileobj=archive) - assert sorted(tar_data.getnames()) == ['bar', 'bar/foo', 'foo'] - - @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='No symlinks on Windows') - def test_tar_with_directory_symlinks(self): - base = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, base) - for d in ['foo', 'bar']: - os.makedirs(os.path.join(base, d)) - os.symlink('../foo', os.path.join(base, 'bar/foo')) - with tar(base) as archive: - tar_data = tarfile.open(fileobj=archive) - assert sorted(tar_data.getnames()) == ['bar', 'bar/foo', 'foo'] - - @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='No symlinks on Windows') - def test_tar_with_broken_symlinks(self): - base = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, base) - for d in ['foo', 'bar']: - os.makedirs(os.path.join(base, d)) - - os.symlink('../baz', os.path.join(base, 'bar/foo')) - with tar(base) as archive: - tar_data = tarfile.open(fileobj=archive) - assert sorted(tar_data.getnames()) == ['bar', 'bar/foo', 'foo'] - - @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='No UNIX sockets on Win32') - def test_tar_socket_file(self): - base = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, base) - for d in ['foo', 'bar']: - os.makedirs(os.path.join(base, d)) - sock = socket.socket(socket.AF_UNIX) - self.addCleanup(sock.close) - sock.bind(os.path.join(base, 'test.sock')) - with tar(base) as archive: - tar_data = tarfile.open(fileobj=archive) - assert sorted(tar_data.getnames()) == ['bar', 'foo'] - - def tar_test_negative_mtime_bug(self): - base = tempfile.mkdtemp() - filename = os.path.join(base, 'th.txt') - self.addCleanup(shutil.rmtree, base) - with open(filename, 'w') as f: - f.write('Invisible Full Moon') - os.utime(filename, (12345, -3600.0)) - with tar(base) as archive: - tar_data = tarfile.open(fileobj=archive) - assert tar_data.getnames() == ['th.txt'] - assert tar_data.getmember('th.txt').mtime == -3600 - - @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='No symlinks on Windows') - def test_tar_directory_link(self): - dirs = ['a', 'b', 'a/c'] - files = ['a/hello.py', 'b/utils.py', 'a/c/descend.py'] - base = make_tree(dirs, files) - self.addCleanup(shutil.rmtree, base) - os.symlink(os.path.join(base, 'b'), os.path.join(base, 'a/c/b')) - with tar(base) as archive: - tar_data = tarfile.open(fileobj=archive) - names = tar_data.getnames() - for member in dirs + files: - assert member in names - assert 'a/c/b' in names - assert 'a/c/b/utils.py' not in names - - class FormatEnvironmentTest(unittest.TestCase): def test_format_env_binary_unicode_value(self): env_dict = { From e6b58410bd47445c96412efdf04c04dbb9e745d1 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 28 Jun 2018 14:30:52 -0700 Subject: [PATCH 0714/1301] Re-add walk method to utils.build Signed-off-by: Joffrey F --- docker/utils/build.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docker/utils/build.py b/docker/utils/build.py index 6f6241e933..4fa5751870 100644 --- a/docker/utils/build.py +++ b/docker/utils/build.py @@ -146,6 +146,11 @@ def normalize_slashes(p): return p +def walk(root, patterns, default=True): + pm = PatternMatcher(patterns) + return pm.walk(root) + + # Heavily based on # https://github.com/moby/moby/blob/master/pkg/fileutils/fileutils.go class PatternMatcher(object): From bc28fd0ee241b27ea4863c7b33d614a065276965 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 29 Jun 2018 11:50:42 -0700 Subject: [PATCH 0715/1301] Bump 3.4.1 Signed-off-by: Joffrey F --- docker/version.py | 2 +- docs/change-log.md | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index c504327323..d45137440c 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "3.4.0" +version = "3.4.1" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) diff --git a/docs/change-log.md b/docs/change-log.md index 5a0d55a372..2bd11a743f 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -1,6 +1,18 @@ Change log ========== +3.4.1 +----- + +[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/52?closed=1) + +### Bugfixes + +* Fixed a bug that caused auth values in config files written using one of the + legacy formats to be ignored +* Fixed issues with handling of double-wildcard `**` patterns in + `.dockerignore` files + 3.4.0 ----- From 6152dc8dad000f24386f99db16494e69455332fa Mon Sep 17 00:00:00 2001 From: Aron Parsons Date: Wed, 18 Jul 2018 19:02:44 -0400 Subject: [PATCH 0716/1301] honor placement preferences via services.create() this allows creating a service with placement preferences when calling services.create(). only constraints were being honored before. related to https://github.com/docker/docker-py/pull/1615 Signed-off-by: Aron Parsons --- docker/models/services.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/docker/models/services.py b/docker/models/services.py index 458d2c8730..612f34544e 100644 --- a/docker/models/services.py +++ b/docker/models/services.py @@ -321,10 +321,15 @@ def _get_create_service_kwargs(func_name, kwargs): if 'container_labels' in kwargs: container_spec_kwargs['labels'] = kwargs.pop('container_labels') + placement = {} + if 'constraints' in kwargs: - task_template_kwargs['placement'] = { - 'Constraints': kwargs.pop('constraints') - } + placement['Constraints'] = kwargs.pop('constraints') + + if 'preferences' in kwargs: + placement['Preferences'] = kwargs.pop('preferences') + + task_template_kwargs['placement'] = placement if 'log_driver' in kwargs: task_template_kwargs['log_driver'] = { From d7bb808ca63b4ef8175e404f722a77be57de0f0e Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 18 Jul 2018 16:59:09 -0700 Subject: [PATCH 0717/1301] Update deps for 3.3 & 3.7 support Signed-off-by: Joffrey F --- .travis.yml | 4 ++++ requirements.txt | 6 ++++-- setup.py | 11 ++++++++--- test-requirements.txt | 3 ++- tox.ini | 2 +- 5 files changed, 19 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 842e352836..1c837a2634 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,6 +10,10 @@ matrix: env: TOXENV=py35 - python: 3.6 env: TOXENV=py36 + - python: 3.7 + env: TOXENV=py37 + dist: xenial + sudo: true - env: TOXENV=flake8 install: diff --git a/requirements.txt b/requirements.txt index 6c5e7d03be..289dea9150 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,8 @@ appdirs==1.4.3 asn1crypto==0.22.0 backports.ssl-match-hostname==3.5.0.1 cffi==1.10.0 -cryptography==1.9 +cryptography==1.9; python_version == '3.3' +cryptography==2.3; python_version > '3.3' docker-pycreds==0.3.0 enum34==1.1.6 idna==2.5 @@ -12,7 +13,8 @@ pycparser==2.17 pyOpenSSL==17.0.0 pyparsing==2.2.0 pypiwin32==219; sys_platform == 'win32' and python_version < '3.6' -pypiwin32==220; sys_platform == 'win32' and python_version >= '3.6' +pypiwin32==223; sys_platform == 'win32' and python_version >= '3.6' requests==2.14.2 six==1.10.0 websocket-client==0.40.0 +urllib3==1.21.1; python_version == '3.3' \ No newline at end of file diff --git a/setup.py b/setup.py index 57b2b5a813..1b208e5c66 100644 --- a/setup.py +++ b/setup.py @@ -10,10 +10,10 @@ SOURCE_DIR = os.path.join(ROOT_DIR) requirements = [ - 'requests >= 2.14.2, != 2.18.0', 'six >= 1.4.0', 'websocket-client >= 0.32.0', - 'docker-pycreds >= 0.3.0' + 'docker-pycreds >= 0.3.0', + 'requests >= 2.14.2, != 2.18.0', ] extras_require = { @@ -27,7 +27,10 @@ # Python 3.6 is only compatible with v220 ; Python < 3.5 is not supported # on v220 ; ALL versions are broken for v222 (as of 2018-01-26) ':sys_platform == "win32" and python_version < "3.6"': 'pypiwin32==219', - ':sys_platform == "win32" and python_version >= "3.6"': 'pypiwin32==220', + ':sys_platform == "win32" and python_version >= "3.6"': 'pypiwin32==223', + + # urllib3 drops support for Python 3.3 in 1.23 + ':python_version == "3.3"': 'urllib3 < 1.23', # If using docker-py over TLS, highly recommend this option is # pip-installed or pinned. @@ -38,6 +41,7 @@ # installing the extra dependencies, install the following instead: # 'requests[security] >= 2.5.2, != 2.11.0, != 2.12.2' 'tls': ['pyOpenSSL>=0.14', 'cryptography>=1.3.4', 'idna>=2.0.0'], + } version = None @@ -81,6 +85,7 @@ 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', 'Topic :: Utilities', 'License :: OSI Approved :: Apache Software License', ], diff --git a/test-requirements.txt b/test-requirements.txt index 09680b6897..9ad59cc664 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,6 +1,7 @@ coverage==3.7.1 flake8==3.4.1 mock==1.0.1 -pytest==2.9.1 +pytest==2.9.1; python_version == '3.3' +pytest==3.6.3; python_version > '3.3' pytest-cov==2.1.0 pytest-timeout==1.2.1 diff --git a/tox.ini b/tox.ini index 41d88605d3..5396147ecc 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27, py33, py34, py35, py36, flake8 +envlist = py27, py34, py35, py36, py37, flake8 skipsdist=True [testenv] From 87f8956a3206b8b5c16dfb5c1df68e216131f024 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 10 Jul 2018 15:51:17 -0400 Subject: [PATCH 0718/1301] Add credHelpers support to set_auth_headers in build Signed-off-by: Joffrey F --- docker/api/build.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/docker/api/build.py b/docker/api/build.py index 419255fc62..0486dce62d 100644 --- a/docker/api/build.py +++ b/docker/api/build.py @@ -293,20 +293,28 @@ def _set_auth_headers(self, headers): # Send the full auth configuration (if any exists), since the build # could use any (or all) of the registries. if self._auth_configs: + auth_cfgs = self._auth_configs auth_data = {} - if self._auth_configs.get('credsStore'): + if auth_cfgs.get('credsStore'): # Using a credentials store, we need to retrieve the # credentials for each registry listed in the config.json file # Matches CLI behavior: https://github.com/docker/docker/blob/ # 67b85f9d26f1b0b2b240f2d794748fac0f45243c/cliconfig/ # credentials/native_store.go#L68-L83 - for registry in self._auth_configs.get('auths', {}).keys(): + for registry in auth_cfgs.get('auths', {}).keys(): auth_data[registry] = auth.resolve_authconfig( - self._auth_configs, registry, + auth_cfgs, registry, credstore_env=self.credstore_env, ) else: - auth_data = self._auth_configs.get('auths', {}).copy() + for registry in auth_cfgs.get('credHelpers', {}).keys(): + auth_data[registry] = auth.resolve_authconfig( + auth_cfgs, registry, + credstore_env=self.credstore_env + ) + for registry, creds in auth_cfgs.get('auths', {}).items(): + if registry not in auth_data: + auth_data[registry] = creds # See https://github.com/docker/docker-py/issues/1683 if auth.INDEX_NAME in auth_data: auth_data[auth.INDEX_URL] = auth_data[auth.INDEX_NAME] From 3112d3920988fde5afa35d2dadbb5298db3a77da Mon Sep 17 00:00:00 2001 From: Nikolay Murga Date: Fri, 20 Jul 2018 12:53:44 +0300 Subject: [PATCH 0719/1301] Add 'rollback' command as allowed for failure_action Signed-off-by: Nikolay Murga --- docker/types/services.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/types/services.py b/docker/types/services.py index 31f4750f45..a914cef691 100644 --- a/docker/types/services.py +++ b/docker/types/services.py @@ -371,7 +371,7 @@ class UpdateConfig(dict): delay (int): Amount of time between updates. failure_action (string): Action to take if an updated task fails to run, or stops running during the update. Acceptable values are - ``continue`` and ``pause``. Default: ``continue`` + ``continue``, ``rollback`` and ``pause``. Default: ``continue`` monitor (int): Amount of time to monitor each updated task for failures, in nanoseconds. max_failure_ratio (float): The fraction of tasks that may fail during @@ -385,7 +385,7 @@ def __init__(self, parallelism=0, delay=None, failure_action='continue', self['Parallelism'] = parallelism if delay is not None: self['Delay'] = delay - if failure_action not in ('pause', 'continue'): + if failure_action not in ('pause', 'continue', 'rollback'): raise errors.InvalidArgument( 'failure_action must be either `pause` or `continue`.' ) From 24fff59bd95d077a523496a942225a074707bc08 Mon Sep 17 00:00:00 2001 From: Nikolay Murga Date: Fri, 20 Jul 2018 13:20:19 +0300 Subject: [PATCH 0720/1301] Add documentation for delay property Signed-off-by: Nikolay Murga --- docker/types/services.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/types/services.py b/docker/types/services.py index a914cef691..294076384b 100644 --- a/docker/types/services.py +++ b/docker/types/services.py @@ -368,7 +368,7 @@ class UpdateConfig(dict): parallelism (int): Maximum number of tasks to be updated in one iteration (0 means unlimited parallelism). Default: 0. - delay (int): Amount of time between updates. + delay (int): Amount of time between updates, in nanoseconds. failure_action (string): Action to take if an updated task fails to run, or stops running during the update. Acceptable values are ``continue``, ``rollback`` and ``pause``. Default: ``continue`` From 3c9738a584ad2ff549bf95372758d41fe71adc2e Mon Sep 17 00:00:00 2001 From: Anthony Sottile Date: Thu, 2 Aug 2018 12:00:11 -0700 Subject: [PATCH 0721/1301] Allow user=0 to be passed in create_container Signed-off-by: Anthony Sottile --- docker/types/containers.py | 2 +- tests/unit/types_containers_test.py | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 tests/unit/types_containers_test.py diff --git a/docker/types/containers.py b/docker/types/containers.py index e7841bcb46..9dfea8ceb8 100644 --- a/docker/types/containers.py +++ b/docker/types/containers.py @@ -578,7 +578,7 @@ def __init__( 'Hostname': hostname, 'Domainname': domainname, 'ExposedPorts': ports, - 'User': six.text_type(user) if user else None, + 'User': six.text_type(user) if user is not None else None, 'Tty': tty, 'OpenStdin': stdin_open, 'StdinOnce': stdin_once, diff --git a/tests/unit/types_containers_test.py b/tests/unit/types_containers_test.py new file mode 100644 index 0000000000..b0ad0a71ac --- /dev/null +++ b/tests/unit/types_containers_test.py @@ -0,0 +1,6 @@ +from docker.types.containers import ContainerConfig + + +def test_uid_0_is_not_elided(): + x = ContainerConfig(image='i', version='v', command='true', user=0) + assert x['User'] == '0' From c28ff855429ca50804945d1c3c274fb89aca2ef3 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 3 Aug 2018 14:04:04 -0700 Subject: [PATCH 0722/1301] Improve placement handling in DockerClient.services.create Signed-off-by: Joffrey F --- docker/models/services.py | 22 ++++++++++++++-------- tests/unit/models_services_test.py | 8 +++++++- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/docker/models/services.py b/docker/models/services.py index 612f34544e..7fbd165102 100644 --- a/docker/models/services.py +++ b/docker/models/services.py @@ -1,6 +1,6 @@ import copy from docker.errors import create_unexpected_kwargs_error, InvalidArgument -from docker.types import TaskTemplate, ContainerSpec, ServiceMode +from docker.types import TaskTemplate, ContainerSpec, Placement, ServiceMode from .resource import Model, Collection @@ -153,6 +153,9 @@ def create(self, image, command=None, **kwargs): command (list of str or str): Command to run. args (list of str): Arguments to the command. constraints (list of str): Placement constraints. + preferences (list of str): Placement preferences. + platforms (list of tuple): A list of platforms constraints + expressed as ``(arch, os)`` tuples container_labels (dict): Labels to apply to the container. endpoint_spec (EndpointSpec): Properties that can be configured to access and load balance a service. Default: ``None``. @@ -302,6 +305,12 @@ def list(self, **kwargs): 'endpoint_spec', ] +PLACEMENT_KWARGS = [ + 'constraints', + 'preferences', + 'platforms', +] + def _get_create_service_kwargs(func_name, kwargs): # Copy over things which can be copied directly @@ -322,13 +331,10 @@ def _get_create_service_kwargs(func_name, kwargs): container_spec_kwargs['labels'] = kwargs.pop('container_labels') placement = {} - - if 'constraints' in kwargs: - placement['Constraints'] = kwargs.pop('constraints') - - if 'preferences' in kwargs: - placement['Preferences'] = kwargs.pop('preferences') - + for key in copy.copy(kwargs): + if key in PLACEMENT_KWARGS: + placement[key] = kwargs.pop(key) + placement = Placement(**placement) task_template_kwargs['placement'] = placement if 'log_driver' in kwargs: diff --git a/tests/unit/models_services_test.py b/tests/unit/models_services_test.py index 247bb4a4aa..a4ac50c3fe 100644 --- a/tests/unit/models_services_test.py +++ b/tests/unit/models_services_test.py @@ -26,6 +26,8 @@ def test_get_create_service_kwargs(self): 'mounts': [{'some': 'mounts'}], 'stop_grace_period': 5, 'constraints': ['foo=bar'], + 'preferences': ['bar=baz'], + 'platforms': [('x86_64', 'linux')], }) task_template = kwargs.pop('task_template') @@ -41,7 +43,11 @@ def test_get_create_service_kwargs(self): 'ContainerSpec', 'Resources', 'RestartPolicy', 'Placement', 'LogDriver', 'Networks' ]) - assert task_template['Placement'] == {'Constraints': ['foo=bar']} + assert task_template['Placement'] == { + 'Constraints': ['foo=bar'], + 'Preferences': ['bar=baz'], + 'Platforms': [{'Architecture': 'x86_64', 'OS': 'linux'}], + } assert task_template['LogDriver'] == { 'Name': 'logdriver', 'Options': {'foo': 'bar'} From 14524f19e2fd2d1c570453d530ae71b1d39dc9fb Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 9 Aug 2018 15:56:11 -0700 Subject: [PATCH 0723/1301] Add version checks and test Signed-off-by: Joffrey F --- docker/api/service.py | 6 ++++++ docker/types/services.py | 5 +++-- tests/integration/api_service_test.py | 14 ++++++++++++++ 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/docker/api/service.py b/docker/api/service.py index 03b0ca6ea2..1dbe2697f1 100644 --- a/docker/api/service.py +++ b/docker/api/service.py @@ -18,6 +18,12 @@ def raise_version_error(param, min_version): if 'Monitor' in update_config: raise_version_error('UpdateConfig.monitor', '1.25') + if utils.version_lt(version, '1.28'): + if update_config.get('FailureAction') == 'rollback': + raise_version_error( + 'UpdateConfig.failure_action rollback', '1.28' + ) + if utils.version_lt(version, '1.29'): if 'Order' in update_config: raise_version_error('UpdateConfig.order', '1.29') diff --git a/docker/types/services.py b/docker/types/services.py index 294076384b..a883f3ff7d 100644 --- a/docker/types/services.py +++ b/docker/types/services.py @@ -371,7 +371,8 @@ class UpdateConfig(dict): delay (int): Amount of time between updates, in nanoseconds. failure_action (string): Action to take if an updated task fails to run, or stops running during the update. Acceptable values are - ``continue``, ``rollback`` and ``pause``. Default: ``continue`` + ``continue``, ``pause``, as well as ``rollback`` since API v1.28. + Default: ``continue`` monitor (int): Amount of time to monitor each updated task for failures, in nanoseconds. max_failure_ratio (float): The fraction of tasks that may fail during @@ -387,7 +388,7 @@ def __init__(self, parallelism=0, delay=None, failure_action='continue', self['Delay'] = delay if failure_action not in ('pause', 'continue', 'rollback'): raise errors.InvalidArgument( - 'failure_action must be either `pause` or `continue`.' + 'failure_action must be one of `pause`, `continue`, `rollback`' ) self['FailureAction'] = failure_action diff --git a/tests/integration/api_service_test.py b/tests/integration/api_service_test.py index 85f9dccf26..ba2ed91f72 100644 --- a/tests/integration/api_service_test.py +++ b/tests/integration/api_service_test.py @@ -281,6 +281,20 @@ def test_create_service_with_update_config(self): assert update_config['Delay'] == uc['Delay'] assert update_config['FailureAction'] == uc['FailureAction'] + @requires_api_version('1.28') + def test_create_service_with_failure_action_rollback(self): + container_spec = docker.types.ContainerSpec(BUSYBOX, ['true']) + task_tmpl = docker.types.TaskTemplate(container_spec) + update_config = docker.types.UpdateConfig(failure_action='rollback') + name = self.get_service_name() + svc_id = self.client.create_service( + task_tmpl, update_config=update_config, name=name + ) + svc_info = self.client.inspect_service(svc_id) + assert 'UpdateConfig' in svc_info['Spec'] + uc = svc_info['Spec']['UpdateConfig'] + assert update_config['FailureAction'] == uc['FailureAction'] + @requires_api_version('1.25') def test_create_service_with_update_config_monitor(self): container_spec = docker.types.ContainerSpec('busybox', ['true']) From 82445764e0499f605ec6222ce3341511436e96b3 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 9 Aug 2018 16:41:25 -0700 Subject: [PATCH 0724/1301] Add support for RollbackConfig Signed-off-by: Joffrey F --- docker/api/service.py | 34 +++++++++++++++++++++++---- docker/models/services.py | 2 ++ docker/types/__init__.py | 4 ++-- docker/types/services.py | 24 +++++++++++++++++++ tests/integration/api_service_test.py | 21 +++++++++++++++++ 5 files changed, 78 insertions(+), 7 deletions(-) diff --git a/docker/api/service.py b/docker/api/service.py index 1dbe2697f1..8b956b63e1 100644 --- a/docker/api/service.py +++ b/docker/api/service.py @@ -2,7 +2,8 @@ from ..types import ServiceMode -def _check_api_features(version, task_template, update_config, endpoint_spec): +def _check_api_features(version, task_template, update_config, endpoint_spec, + rollback_config): def raise_version_error(param, min_version): raise errors.InvalidVersion( @@ -28,6 +29,14 @@ def raise_version_error(param, min_version): if 'Order' in update_config: raise_version_error('UpdateConfig.order', '1.29') + if rollback_config is not None: + if utils.version_lt(version, '1.28'): + raise_version_error('rollback_config', '1.28') + + if utils.version_lt(version, '1.29'): + if 'Order' in update_config: + raise_version_error('RollbackConfig.order', '1.29') + if endpoint_spec is not None: if utils.version_lt(version, '1.32') and 'Ports' in endpoint_spec: if any(p.get('PublishMode') for p in endpoint_spec['Ports']): @@ -105,7 +114,7 @@ class ServiceApiMixin(object): def create_service( self, task_template, name=None, labels=None, mode=None, update_config=None, networks=None, endpoint_config=None, - endpoint_spec=None + endpoint_spec=None, rollback_config=None ): """ Create a service. @@ -120,6 +129,8 @@ def create_service( or global). Defaults to replicated. update_config (UpdateConfig): Specification for the update strategy of the service. Default: ``None`` + rollback_config (RollbackConfig): Specification for the rollback + strategy of the service. Default: ``None`` networks (:py:class:`list`): List of network names or IDs to attach the service to. Default: ``None``. endpoint_spec (EndpointSpec): Properties that can be configured to @@ -135,7 +146,8 @@ def create_service( """ _check_api_features( - self._version, task_template, update_config, endpoint_spec + self._version, task_template, update_config, endpoint_spec, + rollback_config ) url = self._url('/services/create') @@ -166,6 +178,9 @@ def create_service( if update_config is not None: data['UpdateConfig'] = update_config + if rollback_config is not None: + data['RollbackConfig'] = rollback_config + return self._result( self._post_json(url, data=data, headers=headers), True ) @@ -342,7 +357,8 @@ def tasks(self, filters=None): def update_service(self, service, version, task_template=None, name=None, labels=None, mode=None, update_config=None, networks=None, endpoint_config=None, - endpoint_spec=None, fetch_current_spec=False): + endpoint_spec=None, fetch_current_spec=False, + rollback_config=None): """ Update a service. @@ -360,6 +376,8 @@ def update_service(self, service, version, task_template=None, name=None, or global). Defaults to replicated. update_config (UpdateConfig): Specification for the update strategy of the service. Default: ``None``. + rollback_config (RollbackConfig): Specification for the rollback + strategy of the service. Default: ``None`` networks (:py:class:`list`): List of network names or IDs to attach the service to. Default: ``None``. endpoint_spec (EndpointSpec): Properties that can be configured to @@ -376,7 +394,8 @@ def update_service(self, service, version, task_template=None, name=None, """ _check_api_features( - self._version, task_template, update_config, endpoint_spec + self._version, task_template, update_config, endpoint_spec, + rollback_config ) if fetch_current_spec: @@ -422,6 +441,11 @@ def update_service(self, service, version, task_template=None, name=None, else: data['UpdateConfig'] = current.get('UpdateConfig') + if rollback_config is not None: + data['RollbackConfig'] = rollback_config + else: + data['RollbackConfig'] = current.get('RollbackConfig') + if networks is not None: converted_networks = utils.convert_service_networks(networks) if utils.version_lt(self._version, '1.25'): diff --git a/docker/models/services.py b/docker/models/services.py index 7fbd165102..fa029f36ea 100644 --- a/docker/models/services.py +++ b/docker/models/services.py @@ -183,6 +183,8 @@ def create(self, image, command=None, **kwargs): containers to terminate before forcefully killing them. update_config (UpdateConfig): Specification for the update strategy of the service. Default: ``None`` + rollback_config (RollbackConfig): Specification for the rollback + strategy of the service. Default: ``None`` user (str): User to run commands as. workdir (str): Working directory for commands to run. tty (boolean): Whether a pseudo-TTY should be allocated. diff --git a/docker/types/__init__.py b/docker/types/__init__.py index 0b0d847fe9..64512333df 100644 --- a/docker/types/__init__.py +++ b/docker/types/__init__.py @@ -5,7 +5,7 @@ from .networks import EndpointConfig, IPAMConfig, IPAMPool, NetworkingConfig from .services import ( ConfigReference, ContainerSpec, DNSConfig, DriverConfig, EndpointSpec, - Mount, Placement, Privileges, Resources, RestartPolicy, SecretReference, - ServiceMode, TaskTemplate, UpdateConfig + Mount, Placement, Privileges, Resources, RestartPolicy, RollbackConfig, + SecretReference, ServiceMode, TaskTemplate, UpdateConfig ) from .swarm import SwarmSpec, SwarmExternalCA diff --git a/docker/types/services.py b/docker/types/services.py index a883f3ff7d..c66d41a167 100644 --- a/docker/types/services.py +++ b/docker/types/services.py @@ -414,6 +414,30 @@ def __init__(self, parallelism=0, delay=None, failure_action='continue', self['Order'] = order +class RollbackConfig(UpdateConfig): + """ + Used to specify the way containe rollbacks should be performed by a service + + Args: + parallelism (int): Maximum number of tasks to be rolled back in one + iteration (0 means unlimited parallelism). Default: 0 + delay (int): Amount of time between rollbacks, in nanoseconds. + failure_action (string): Action to take if a rolled back task fails to + run, or stops running during the rollback. Acceptable values are + ``continue``, ``pause`` or ``rollback``. + Default: ``continue`` + monitor (int): Amount of time to monitor each rolled back task for + failures, in nanoseconds. + max_failure_ratio (float): The fraction of tasks that may fail during + a rollback before the failure action is invoked, specified as a + floating point number between 0 and 1. Default: 0 + order (string): Specifies the order of operations when rolling out a + rolled back task. Either ``start_first`` or ``stop_first`` are + accepted. + """ + pass + + class RestartConditionTypesEnum(object): _values = ( 'none', diff --git a/tests/integration/api_service_test.py b/tests/integration/api_service_test.py index ba2ed91f72..a53ca1c836 100644 --- a/tests/integration/api_service_test.py +++ b/tests/integration/api_service_test.py @@ -312,6 +312,27 @@ def test_create_service_with_update_config_monitor(self): assert update_config['Monitor'] == uc['Monitor'] assert update_config['MaxFailureRatio'] == uc['MaxFailureRatio'] + @requires_api_version('1.28') + def test_create_service_with_rollback_config(self): + container_spec = docker.types.ContainerSpec(BUSYBOX, ['true']) + task_tmpl = docker.types.TaskTemplate(container_spec) + rollback_cfg = docker.types.RollbackConfig( + parallelism=10, delay=5, failure_action='pause', + monitor=300000000, max_failure_ratio=0.4 + ) + name = self.get_service_name() + svc_id = self.client.create_service( + task_tmpl, rollback_config=rollback_cfg, name=name + ) + svc_info = self.client.inspect_service(svc_id) + assert 'RollbackConfig' in svc_info['Spec'] + rc = svc_info['Spec']['RollbackConfig'] + assert rollback_cfg['Parallelism'] == rc['Parallelism'] + assert rollback_cfg['Delay'] == rc['Delay'] + assert rollback_cfg['FailureAction'] == rc['FailureAction'] + assert rollback_cfg['Monitor'] == rc['Monitor'] + assert rollback_cfg['MaxFailureRatio'] == rc['MaxFailureRatio'] + def test_create_service_with_restart_policy(self): container_spec = docker.types.ContainerSpec(BUSYBOX, ['true']) policy = docker.types.RestartPolicy( From 29dee5ac2e6a952b4d66090549a5a443e15c29d6 Mon Sep 17 00:00:00 2001 From: Marco Trillo Date: Fri, 29 Jun 2018 14:54:48 +0200 Subject: [PATCH 0725/1301] Add support for `uts_mode` parameter in `Client.create_host_config`. This parameter allows to set the UTS namespace of the container, as in the `--uts=X` Docker CLI parameter: The only allowed value, if set, is "host". Signed-off-by: Marco Trillo Signed-off-by: Diego Alvarez --- docker/api/container.py | 2 ++ docker/models/containers.py | 1 + docker/types/containers.py | 15 ++++++++++----- tests/integration/api_container_test.py | 10 ++++++++++ tests/unit/dockertypes_test.py | 6 ++++++ tests/unit/models_containers_test.py | 2 ++ 6 files changed, 31 insertions(+), 5 deletions(-) diff --git a/docker/api/container.py b/docker/api/container.py index d4f75f54b2..d8416066d5 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -547,6 +547,8 @@ def create_host_config(self, *args, **kwargs): userns_mode (str): Sets the user namespace mode for the container when user namespace remapping option is enabled. Supported values are: ``host`` + uts_mode (str): Sets the UTS namespace mode for the container. + Supported values are: ``host`` volumes_from (:py:class:`list`): List of container names or IDs to get volumes from. runtime (str): Runtime to use with this container. diff --git a/docker/models/containers.py b/docker/models/containers.py index b33a718f75..de6222ec49 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -995,6 +995,7 @@ def prune(self, filters=None): 'tmpfs', 'ulimits', 'userns_mode', + 'uts_mode', 'version', 'volumes_from', 'runtime' diff --git a/docker/types/containers.py b/docker/types/containers.py index 252142073f..e7841bcb46 100644 --- a/docker/types/containers.py +++ b/docker/types/containers.py @@ -115,11 +115,11 @@ def __init__(self, version, binds=None, port_bindings=None, device_read_iops=None, device_write_iops=None, oom_kill_disable=False, shm_size=None, sysctls=None, tmpfs=None, oom_score_adj=None, dns_opt=None, cpu_shares=None, - cpuset_cpus=None, userns_mode=None, pids_limit=None, - isolation=None, auto_remove=False, storage_opt=None, - init=None, init_path=None, volume_driver=None, - cpu_count=None, cpu_percent=None, nano_cpus=None, - cpuset_mems=None, runtime=None, mounts=None, + cpuset_cpus=None, userns_mode=None, uts_mode=None, + pids_limit=None, isolation=None, auto_remove=False, + storage_opt=None, init=None, init_path=None, + volume_driver=None, cpu_count=None, cpu_percent=None, + nano_cpus=None, cpuset_mems=None, runtime=None, mounts=None, cpu_rt_period=None, cpu_rt_runtime=None, device_cgroup_rules=None): @@ -392,6 +392,11 @@ def __init__(self, version, binds=None, port_bindings=None, raise host_config_value_error("userns_mode", userns_mode) self['UsernsMode'] = userns_mode + if uts_mode: + if uts_mode != "host": + raise host_config_value_error("uts_mode", uts_mode) + self['UTSMode'] = uts_mode + if pids_limit: if not isinstance(pids_limit, int): raise host_config_type_error('pids_limit', pids_limit, 'int') diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index ff7014879c..6ce846bb20 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -490,6 +490,16 @@ def test_create_with_device_cgroup_rules(self): self.client.start(ctnr) assert rule in self.client.logs(ctnr).decode('utf-8') + def test_create_with_uts_mode(self): + container = self.client.create_container( + BUSYBOX, ['echo'], host_config=self.client.create_host_config( + uts_mode='host' + ) + ) + self.tmp_containers.append(container) + config = self.client.inspect_container(container) + assert config['HostConfig']['UTSMode'] == 'host' + @pytest.mark.xfail( IS_WINDOWS_PLATFORM, reason='Test not designed for Windows platform' diff --git a/tests/unit/dockertypes_test.py b/tests/unit/dockertypes_test.py index 2be05784bb..cdacf8cd5b 100644 --- a/tests/unit/dockertypes_test.py +++ b/tests/unit/dockertypes_test.py @@ -85,6 +85,12 @@ def test_create_host_config_with_userns_mode(self): with pytest.raises(ValueError): create_host_config(version='1.23', userns_mode='host12') + def test_create_host_config_with_uts(self): + config = create_host_config(version='1.15', uts_mode='host') + assert config.get('UTSMode') == 'host' + with pytest.raises(ValueError): + create_host_config(version='1.15', uts_mode='host12') + def test_create_host_config_with_oom_score_adj(self): config = create_host_config(version='1.22', oom_score_adj=100) assert config.get('OomScoreAdj') == 100 diff --git a/tests/unit/models_containers_test.py b/tests/unit/models_containers_test.py index 48a5288869..22dd241064 100644 --- a/tests/unit/models_containers_test.py +++ b/tests/unit/models_containers_test.py @@ -95,6 +95,7 @@ def test_create_container_args(self): ulimits=[{"Name": "nofile", "Soft": 1024, "Hard": 2048}], user='bob', userns_mode='host', + uts_mode='host', version='1.23', volume_driver='some_driver', volumes=[ @@ -174,6 +175,7 @@ def test_create_container_args(self): 'Tmpfs': {'/blah': ''}, 'Ulimits': [{"Name": "nofile", "Soft": 1024, "Hard": 2048}], 'UsernsMode': 'host', + 'UTSMode': 'host', 'VolumesFrom': ['container'], }, healthcheck={'test': 'true'}, From dac943a91dfc6622ff77bae5521214a4faaa3f02 Mon Sep 17 00:00:00 2001 From: Aron Parsons Date: Wed, 18 Jul 2018 19:02:44 -0400 Subject: [PATCH 0726/1301] honor placement preferences via services.create() this allows creating a service with placement preferences when calling services.create(). only constraints were being honored before. related to https://github.com/docker/docker-py/pull/1615 Signed-off-by: Aron Parsons --- docker/models/services.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/docker/models/services.py b/docker/models/services.py index 458d2c8730..612f34544e 100644 --- a/docker/models/services.py +++ b/docker/models/services.py @@ -321,10 +321,15 @@ def _get_create_service_kwargs(func_name, kwargs): if 'container_labels' in kwargs: container_spec_kwargs['labels'] = kwargs.pop('container_labels') + placement = {} + if 'constraints' in kwargs: - task_template_kwargs['placement'] = { - 'Constraints': kwargs.pop('constraints') - } + placement['Constraints'] = kwargs.pop('constraints') + + if 'preferences' in kwargs: + placement['Preferences'] = kwargs.pop('preferences') + + task_template_kwargs['placement'] = placement if 'log_driver' in kwargs: task_template_kwargs['log_driver'] = { From f71d1cf3cfdce2cce77edc9dcdc33879175d44f7 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 18 Jul 2018 16:59:09 -0700 Subject: [PATCH 0727/1301] Update deps for 3.3 & 3.7 support Signed-off-by: Joffrey F --- .travis.yml | 4 ++++ requirements.txt | 6 ++++-- setup.py | 11 ++++++++--- test-requirements.txt | 3 ++- tox.ini | 2 +- 5 files changed, 19 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 842e352836..1c837a2634 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,6 +10,10 @@ matrix: env: TOXENV=py35 - python: 3.6 env: TOXENV=py36 + - python: 3.7 + env: TOXENV=py37 + dist: xenial + sudo: true - env: TOXENV=flake8 install: diff --git a/requirements.txt b/requirements.txt index 6c5e7d03be..289dea9150 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,8 @@ appdirs==1.4.3 asn1crypto==0.22.0 backports.ssl-match-hostname==3.5.0.1 cffi==1.10.0 -cryptography==1.9 +cryptography==1.9; python_version == '3.3' +cryptography==2.3; python_version > '3.3' docker-pycreds==0.3.0 enum34==1.1.6 idna==2.5 @@ -12,7 +13,8 @@ pycparser==2.17 pyOpenSSL==17.0.0 pyparsing==2.2.0 pypiwin32==219; sys_platform == 'win32' and python_version < '3.6' -pypiwin32==220; sys_platform == 'win32' and python_version >= '3.6' +pypiwin32==223; sys_platform == 'win32' and python_version >= '3.6' requests==2.14.2 six==1.10.0 websocket-client==0.40.0 +urllib3==1.21.1; python_version == '3.3' \ No newline at end of file diff --git a/setup.py b/setup.py index 57b2b5a813..1b208e5c66 100644 --- a/setup.py +++ b/setup.py @@ -10,10 +10,10 @@ SOURCE_DIR = os.path.join(ROOT_DIR) requirements = [ - 'requests >= 2.14.2, != 2.18.0', 'six >= 1.4.0', 'websocket-client >= 0.32.0', - 'docker-pycreds >= 0.3.0' + 'docker-pycreds >= 0.3.0', + 'requests >= 2.14.2, != 2.18.0', ] extras_require = { @@ -27,7 +27,10 @@ # Python 3.6 is only compatible with v220 ; Python < 3.5 is not supported # on v220 ; ALL versions are broken for v222 (as of 2018-01-26) ':sys_platform == "win32" and python_version < "3.6"': 'pypiwin32==219', - ':sys_platform == "win32" and python_version >= "3.6"': 'pypiwin32==220', + ':sys_platform == "win32" and python_version >= "3.6"': 'pypiwin32==223', + + # urllib3 drops support for Python 3.3 in 1.23 + ':python_version == "3.3"': 'urllib3 < 1.23', # If using docker-py over TLS, highly recommend this option is # pip-installed or pinned. @@ -38,6 +41,7 @@ # installing the extra dependencies, install the following instead: # 'requests[security] >= 2.5.2, != 2.11.0, != 2.12.2' 'tls': ['pyOpenSSL>=0.14', 'cryptography>=1.3.4', 'idna>=2.0.0'], + } version = None @@ -81,6 +85,7 @@ 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', 'Topic :: Utilities', 'License :: OSI Approved :: Apache Software License', ], diff --git a/test-requirements.txt b/test-requirements.txt index 09680b6897..9ad59cc664 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,6 +1,7 @@ coverage==3.7.1 flake8==3.4.1 mock==1.0.1 -pytest==2.9.1 +pytest==2.9.1; python_version == '3.3' +pytest==3.6.3; python_version > '3.3' pytest-cov==2.1.0 pytest-timeout==1.2.1 diff --git a/tox.ini b/tox.ini index 41d88605d3..5396147ecc 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27, py33, py34, py35, py36, flake8 +envlist = py27, py34, py35, py36, py37, flake8 skipsdist=True [testenv] From d4345b5824a23d088bf9c44d9fb87382f159a43a Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 10 Jul 2018 15:51:17 -0400 Subject: [PATCH 0728/1301] Add credHelpers support to set_auth_headers in build Signed-off-by: Joffrey F --- docker/api/build.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/docker/api/build.py b/docker/api/build.py index 419255fc62..0486dce62d 100644 --- a/docker/api/build.py +++ b/docker/api/build.py @@ -293,20 +293,28 @@ def _set_auth_headers(self, headers): # Send the full auth configuration (if any exists), since the build # could use any (or all) of the registries. if self._auth_configs: + auth_cfgs = self._auth_configs auth_data = {} - if self._auth_configs.get('credsStore'): + if auth_cfgs.get('credsStore'): # Using a credentials store, we need to retrieve the # credentials for each registry listed in the config.json file # Matches CLI behavior: https://github.com/docker/docker/blob/ # 67b85f9d26f1b0b2b240f2d794748fac0f45243c/cliconfig/ # credentials/native_store.go#L68-L83 - for registry in self._auth_configs.get('auths', {}).keys(): + for registry in auth_cfgs.get('auths', {}).keys(): auth_data[registry] = auth.resolve_authconfig( - self._auth_configs, registry, + auth_cfgs, registry, credstore_env=self.credstore_env, ) else: - auth_data = self._auth_configs.get('auths', {}).copy() + for registry in auth_cfgs.get('credHelpers', {}).keys(): + auth_data[registry] = auth.resolve_authconfig( + auth_cfgs, registry, + credstore_env=self.credstore_env + ) + for registry, creds in auth_cfgs.get('auths', {}).items(): + if registry not in auth_data: + auth_data[registry] = creds # See https://github.com/docker/docker-py/issues/1683 if auth.INDEX_NAME in auth_data: auth_data[auth.INDEX_URL] = auth_data[auth.INDEX_NAME] From 185f72723ab8bac589e73e58be64f1665472762e Mon Sep 17 00:00:00 2001 From: Nikolay Murga Date: Fri, 20 Jul 2018 12:53:44 +0300 Subject: [PATCH 0729/1301] Add 'rollback' command as allowed for failure_action Signed-off-by: Nikolay Murga --- docker/types/services.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/types/services.py b/docker/types/services.py index 31f4750f45..a914cef691 100644 --- a/docker/types/services.py +++ b/docker/types/services.py @@ -371,7 +371,7 @@ class UpdateConfig(dict): delay (int): Amount of time between updates. failure_action (string): Action to take if an updated task fails to run, or stops running during the update. Acceptable values are - ``continue`` and ``pause``. Default: ``continue`` + ``continue``, ``rollback`` and ``pause``. Default: ``continue`` monitor (int): Amount of time to monitor each updated task for failures, in nanoseconds. max_failure_ratio (float): The fraction of tasks that may fail during @@ -385,7 +385,7 @@ def __init__(self, parallelism=0, delay=None, failure_action='continue', self['Parallelism'] = parallelism if delay is not None: self['Delay'] = delay - if failure_action not in ('pause', 'continue'): + if failure_action not in ('pause', 'continue', 'rollback'): raise errors.InvalidArgument( 'failure_action must be either `pause` or `continue`.' ) From 8ee446631d6da7ca8e52a4f65aafecf2de1c8ea6 Mon Sep 17 00:00:00 2001 From: Nikolay Murga Date: Fri, 20 Jul 2018 13:20:19 +0300 Subject: [PATCH 0730/1301] Add documentation for delay property Signed-off-by: Nikolay Murga --- docker/types/services.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/types/services.py b/docker/types/services.py index a914cef691..294076384b 100644 --- a/docker/types/services.py +++ b/docker/types/services.py @@ -368,7 +368,7 @@ class UpdateConfig(dict): parallelism (int): Maximum number of tasks to be updated in one iteration (0 means unlimited parallelism). Default: 0. - delay (int): Amount of time between updates. + delay (int): Amount of time between updates, in nanoseconds. failure_action (string): Action to take if an updated task fails to run, or stops running during the update. Acceptable values are ``continue``, ``rollback`` and ``pause``. Default: ``continue`` From cbc7623ea09fd63a75f74378da11bd7cf266e32c Mon Sep 17 00:00:00 2001 From: Anthony Sottile Date: Thu, 2 Aug 2018 12:00:11 -0700 Subject: [PATCH 0731/1301] Allow user=0 to be passed in create_container Signed-off-by: Anthony Sottile --- docker/types/containers.py | 2 +- tests/unit/types_containers_test.py | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 tests/unit/types_containers_test.py diff --git a/docker/types/containers.py b/docker/types/containers.py index e7841bcb46..9dfea8ceb8 100644 --- a/docker/types/containers.py +++ b/docker/types/containers.py @@ -578,7 +578,7 @@ def __init__( 'Hostname': hostname, 'Domainname': domainname, 'ExposedPorts': ports, - 'User': six.text_type(user) if user else None, + 'User': six.text_type(user) if user is not None else None, 'Tty': tty, 'OpenStdin': stdin_open, 'StdinOnce': stdin_once, diff --git a/tests/unit/types_containers_test.py b/tests/unit/types_containers_test.py new file mode 100644 index 0000000000..b0ad0a71ac --- /dev/null +++ b/tests/unit/types_containers_test.py @@ -0,0 +1,6 @@ +from docker.types.containers import ContainerConfig + + +def test_uid_0_is_not_elided(): + x = ContainerConfig(image='i', version='v', command='true', user=0) + assert x['User'] == '0' From c2c9ccdd803352fef42eaf9efedf9238e2613a85 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 3 Aug 2018 14:04:04 -0700 Subject: [PATCH 0732/1301] Improve placement handling in DockerClient.services.create Signed-off-by: Joffrey F --- docker/models/services.py | 22 ++++++++++++++-------- tests/unit/models_services_test.py | 8 +++++++- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/docker/models/services.py b/docker/models/services.py index 612f34544e..7fbd165102 100644 --- a/docker/models/services.py +++ b/docker/models/services.py @@ -1,6 +1,6 @@ import copy from docker.errors import create_unexpected_kwargs_error, InvalidArgument -from docker.types import TaskTemplate, ContainerSpec, ServiceMode +from docker.types import TaskTemplate, ContainerSpec, Placement, ServiceMode from .resource import Model, Collection @@ -153,6 +153,9 @@ def create(self, image, command=None, **kwargs): command (list of str or str): Command to run. args (list of str): Arguments to the command. constraints (list of str): Placement constraints. + preferences (list of str): Placement preferences. + platforms (list of tuple): A list of platforms constraints + expressed as ``(arch, os)`` tuples container_labels (dict): Labels to apply to the container. endpoint_spec (EndpointSpec): Properties that can be configured to access and load balance a service. Default: ``None``. @@ -302,6 +305,12 @@ def list(self, **kwargs): 'endpoint_spec', ] +PLACEMENT_KWARGS = [ + 'constraints', + 'preferences', + 'platforms', +] + def _get_create_service_kwargs(func_name, kwargs): # Copy over things which can be copied directly @@ -322,13 +331,10 @@ def _get_create_service_kwargs(func_name, kwargs): container_spec_kwargs['labels'] = kwargs.pop('container_labels') placement = {} - - if 'constraints' in kwargs: - placement['Constraints'] = kwargs.pop('constraints') - - if 'preferences' in kwargs: - placement['Preferences'] = kwargs.pop('preferences') - + for key in copy.copy(kwargs): + if key in PLACEMENT_KWARGS: + placement[key] = kwargs.pop(key) + placement = Placement(**placement) task_template_kwargs['placement'] = placement if 'log_driver' in kwargs: diff --git a/tests/unit/models_services_test.py b/tests/unit/models_services_test.py index 247bb4a4aa..a4ac50c3fe 100644 --- a/tests/unit/models_services_test.py +++ b/tests/unit/models_services_test.py @@ -26,6 +26,8 @@ def test_get_create_service_kwargs(self): 'mounts': [{'some': 'mounts'}], 'stop_grace_period': 5, 'constraints': ['foo=bar'], + 'preferences': ['bar=baz'], + 'platforms': [('x86_64', 'linux')], }) task_template = kwargs.pop('task_template') @@ -41,7 +43,11 @@ def test_get_create_service_kwargs(self): 'ContainerSpec', 'Resources', 'RestartPolicy', 'Placement', 'LogDriver', 'Networks' ]) - assert task_template['Placement'] == {'Constraints': ['foo=bar']} + assert task_template['Placement'] == { + 'Constraints': ['foo=bar'], + 'Preferences': ['bar=baz'], + 'Platforms': [{'Architecture': 'x86_64', 'OS': 'linux'}], + } assert task_template['LogDriver'] == { 'Name': 'logdriver', 'Options': {'foo': 'bar'} From e4b509ecace12dab1244b3dbcc33196d5e59e9d3 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 9 Aug 2018 15:56:11 -0700 Subject: [PATCH 0733/1301] Add version checks and test Signed-off-by: Joffrey F --- docker/api/service.py | 6 ++++++ docker/types/services.py | 5 +++-- tests/integration/api_service_test.py | 14 ++++++++++++++ 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/docker/api/service.py b/docker/api/service.py index 03b0ca6ea2..1dbe2697f1 100644 --- a/docker/api/service.py +++ b/docker/api/service.py @@ -18,6 +18,12 @@ def raise_version_error(param, min_version): if 'Monitor' in update_config: raise_version_error('UpdateConfig.monitor', '1.25') + if utils.version_lt(version, '1.28'): + if update_config.get('FailureAction') == 'rollback': + raise_version_error( + 'UpdateConfig.failure_action rollback', '1.28' + ) + if utils.version_lt(version, '1.29'): if 'Order' in update_config: raise_version_error('UpdateConfig.order', '1.29') diff --git a/docker/types/services.py b/docker/types/services.py index 294076384b..a883f3ff7d 100644 --- a/docker/types/services.py +++ b/docker/types/services.py @@ -371,7 +371,8 @@ class UpdateConfig(dict): delay (int): Amount of time between updates, in nanoseconds. failure_action (string): Action to take if an updated task fails to run, or stops running during the update. Acceptable values are - ``continue``, ``rollback`` and ``pause``. Default: ``continue`` + ``continue``, ``pause``, as well as ``rollback`` since API v1.28. + Default: ``continue`` monitor (int): Amount of time to monitor each updated task for failures, in nanoseconds. max_failure_ratio (float): The fraction of tasks that may fail during @@ -387,7 +388,7 @@ def __init__(self, parallelism=0, delay=None, failure_action='continue', self['Delay'] = delay if failure_action not in ('pause', 'continue', 'rollback'): raise errors.InvalidArgument( - 'failure_action must be either `pause` or `continue`.' + 'failure_action must be one of `pause`, `continue`, `rollback`' ) self['FailureAction'] = failure_action diff --git a/tests/integration/api_service_test.py b/tests/integration/api_service_test.py index 85f9dccf26..ba2ed91f72 100644 --- a/tests/integration/api_service_test.py +++ b/tests/integration/api_service_test.py @@ -281,6 +281,20 @@ def test_create_service_with_update_config(self): assert update_config['Delay'] == uc['Delay'] assert update_config['FailureAction'] == uc['FailureAction'] + @requires_api_version('1.28') + def test_create_service_with_failure_action_rollback(self): + container_spec = docker.types.ContainerSpec(BUSYBOX, ['true']) + task_tmpl = docker.types.TaskTemplate(container_spec) + update_config = docker.types.UpdateConfig(failure_action='rollback') + name = self.get_service_name() + svc_id = self.client.create_service( + task_tmpl, update_config=update_config, name=name + ) + svc_info = self.client.inspect_service(svc_id) + assert 'UpdateConfig' in svc_info['Spec'] + uc = svc_info['Spec']['UpdateConfig'] + assert update_config['FailureAction'] == uc['FailureAction'] + @requires_api_version('1.25') def test_create_service_with_update_config_monitor(self): container_spec = docker.types.ContainerSpec('busybox', ['true']) From 91e9258659410f54e6f4bfa621cf5ebc854e0e41 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 9 Aug 2018 16:41:25 -0700 Subject: [PATCH 0734/1301] Add support for RollbackConfig Signed-off-by: Joffrey F --- docker/api/service.py | 34 +++++++++++++++++++++++---- docker/models/services.py | 2 ++ docker/types/__init__.py | 4 ++-- docker/types/services.py | 24 +++++++++++++++++++ tests/integration/api_service_test.py | 21 +++++++++++++++++ 5 files changed, 78 insertions(+), 7 deletions(-) diff --git a/docker/api/service.py b/docker/api/service.py index 1dbe2697f1..8b956b63e1 100644 --- a/docker/api/service.py +++ b/docker/api/service.py @@ -2,7 +2,8 @@ from ..types import ServiceMode -def _check_api_features(version, task_template, update_config, endpoint_spec): +def _check_api_features(version, task_template, update_config, endpoint_spec, + rollback_config): def raise_version_error(param, min_version): raise errors.InvalidVersion( @@ -28,6 +29,14 @@ def raise_version_error(param, min_version): if 'Order' in update_config: raise_version_error('UpdateConfig.order', '1.29') + if rollback_config is not None: + if utils.version_lt(version, '1.28'): + raise_version_error('rollback_config', '1.28') + + if utils.version_lt(version, '1.29'): + if 'Order' in update_config: + raise_version_error('RollbackConfig.order', '1.29') + if endpoint_spec is not None: if utils.version_lt(version, '1.32') and 'Ports' in endpoint_spec: if any(p.get('PublishMode') for p in endpoint_spec['Ports']): @@ -105,7 +114,7 @@ class ServiceApiMixin(object): def create_service( self, task_template, name=None, labels=None, mode=None, update_config=None, networks=None, endpoint_config=None, - endpoint_spec=None + endpoint_spec=None, rollback_config=None ): """ Create a service. @@ -120,6 +129,8 @@ def create_service( or global). Defaults to replicated. update_config (UpdateConfig): Specification for the update strategy of the service. Default: ``None`` + rollback_config (RollbackConfig): Specification for the rollback + strategy of the service. Default: ``None`` networks (:py:class:`list`): List of network names or IDs to attach the service to. Default: ``None``. endpoint_spec (EndpointSpec): Properties that can be configured to @@ -135,7 +146,8 @@ def create_service( """ _check_api_features( - self._version, task_template, update_config, endpoint_spec + self._version, task_template, update_config, endpoint_spec, + rollback_config ) url = self._url('/services/create') @@ -166,6 +178,9 @@ def create_service( if update_config is not None: data['UpdateConfig'] = update_config + if rollback_config is not None: + data['RollbackConfig'] = rollback_config + return self._result( self._post_json(url, data=data, headers=headers), True ) @@ -342,7 +357,8 @@ def tasks(self, filters=None): def update_service(self, service, version, task_template=None, name=None, labels=None, mode=None, update_config=None, networks=None, endpoint_config=None, - endpoint_spec=None, fetch_current_spec=False): + endpoint_spec=None, fetch_current_spec=False, + rollback_config=None): """ Update a service. @@ -360,6 +376,8 @@ def update_service(self, service, version, task_template=None, name=None, or global). Defaults to replicated. update_config (UpdateConfig): Specification for the update strategy of the service. Default: ``None``. + rollback_config (RollbackConfig): Specification for the rollback + strategy of the service. Default: ``None`` networks (:py:class:`list`): List of network names or IDs to attach the service to. Default: ``None``. endpoint_spec (EndpointSpec): Properties that can be configured to @@ -376,7 +394,8 @@ def update_service(self, service, version, task_template=None, name=None, """ _check_api_features( - self._version, task_template, update_config, endpoint_spec + self._version, task_template, update_config, endpoint_spec, + rollback_config ) if fetch_current_spec: @@ -422,6 +441,11 @@ def update_service(self, service, version, task_template=None, name=None, else: data['UpdateConfig'] = current.get('UpdateConfig') + if rollback_config is not None: + data['RollbackConfig'] = rollback_config + else: + data['RollbackConfig'] = current.get('RollbackConfig') + if networks is not None: converted_networks = utils.convert_service_networks(networks) if utils.version_lt(self._version, '1.25'): diff --git a/docker/models/services.py b/docker/models/services.py index 7fbd165102..fa029f36ea 100644 --- a/docker/models/services.py +++ b/docker/models/services.py @@ -183,6 +183,8 @@ def create(self, image, command=None, **kwargs): containers to terminate before forcefully killing them. update_config (UpdateConfig): Specification for the update strategy of the service. Default: ``None`` + rollback_config (RollbackConfig): Specification for the rollback + strategy of the service. Default: ``None`` user (str): User to run commands as. workdir (str): Working directory for commands to run. tty (boolean): Whether a pseudo-TTY should be allocated. diff --git a/docker/types/__init__.py b/docker/types/__init__.py index 0b0d847fe9..64512333df 100644 --- a/docker/types/__init__.py +++ b/docker/types/__init__.py @@ -5,7 +5,7 @@ from .networks import EndpointConfig, IPAMConfig, IPAMPool, NetworkingConfig from .services import ( ConfigReference, ContainerSpec, DNSConfig, DriverConfig, EndpointSpec, - Mount, Placement, Privileges, Resources, RestartPolicy, SecretReference, - ServiceMode, TaskTemplate, UpdateConfig + Mount, Placement, Privileges, Resources, RestartPolicy, RollbackConfig, + SecretReference, ServiceMode, TaskTemplate, UpdateConfig ) from .swarm import SwarmSpec, SwarmExternalCA diff --git a/docker/types/services.py b/docker/types/services.py index a883f3ff7d..c66d41a167 100644 --- a/docker/types/services.py +++ b/docker/types/services.py @@ -414,6 +414,30 @@ def __init__(self, parallelism=0, delay=None, failure_action='continue', self['Order'] = order +class RollbackConfig(UpdateConfig): + """ + Used to specify the way containe rollbacks should be performed by a service + + Args: + parallelism (int): Maximum number of tasks to be rolled back in one + iteration (0 means unlimited parallelism). Default: 0 + delay (int): Amount of time between rollbacks, in nanoseconds. + failure_action (string): Action to take if a rolled back task fails to + run, or stops running during the rollback. Acceptable values are + ``continue``, ``pause`` or ``rollback``. + Default: ``continue`` + monitor (int): Amount of time to monitor each rolled back task for + failures, in nanoseconds. + max_failure_ratio (float): The fraction of tasks that may fail during + a rollback before the failure action is invoked, specified as a + floating point number between 0 and 1. Default: 0 + order (string): Specifies the order of operations when rolling out a + rolled back task. Either ``start_first`` or ``stop_first`` are + accepted. + """ + pass + + class RestartConditionTypesEnum(object): _values = ( 'none', diff --git a/tests/integration/api_service_test.py b/tests/integration/api_service_test.py index ba2ed91f72..a53ca1c836 100644 --- a/tests/integration/api_service_test.py +++ b/tests/integration/api_service_test.py @@ -312,6 +312,27 @@ def test_create_service_with_update_config_monitor(self): assert update_config['Monitor'] == uc['Monitor'] assert update_config['MaxFailureRatio'] == uc['MaxFailureRatio'] + @requires_api_version('1.28') + def test_create_service_with_rollback_config(self): + container_spec = docker.types.ContainerSpec(BUSYBOX, ['true']) + task_tmpl = docker.types.TaskTemplate(container_spec) + rollback_cfg = docker.types.RollbackConfig( + parallelism=10, delay=5, failure_action='pause', + monitor=300000000, max_failure_ratio=0.4 + ) + name = self.get_service_name() + svc_id = self.client.create_service( + task_tmpl, rollback_config=rollback_cfg, name=name + ) + svc_info = self.client.inspect_service(svc_id) + assert 'RollbackConfig' in svc_info['Spec'] + rc = svc_info['Spec']['RollbackConfig'] + assert rollback_cfg['Parallelism'] == rc['Parallelism'] + assert rollback_cfg['Delay'] == rc['Delay'] + assert rollback_cfg['FailureAction'] == rc['FailureAction'] + assert rollback_cfg['Monitor'] == rc['Monitor'] + assert rollback_cfg['MaxFailureRatio'] == rc['MaxFailureRatio'] + def test_create_service_with_restart_policy(self): container_spec = docker.types.ContainerSpec(BUSYBOX, ['true']) policy = docker.types.RestartPolicy( From 05fa0be8ef5541462b59da9658c617c48bd60924 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 9 Aug 2018 17:10:07 -0700 Subject: [PATCH 0735/1301] 3.5.0 release Signed-off-by: Joffrey F --- docker/version.py | 2 +- docs/change-log.md | 27 +++++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index d45137440c..022daffdd2 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "3.4.1" +version = "3.5.0" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) diff --git a/docs/change-log.md b/docs/change-log.md index 2bd11a743f..1b2d620fb8 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -1,6 +1,33 @@ Change log ========== +3.5.0 +----- + +[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone=53?closed=1) + +### Deprecation warning + +* Support for Python 3.3 will be dropped in the 4.0.0 release + +### Features + +* Updated dependencies to ensure support for Python 3.7 environments +* Added support for the `uts_mode` parameter in `HostConfig` +* The `UpdateConfig` constructor now allows `rollback` as a valid + value for `failure_action` +* Added support for `rollback_config` in `APIClient.create_service`, + `APIClient.update_service`, `DockerClient.services.create` and + `Service.update`. + +### Bugfixes + +* Credential helpers are now properly leveraged by the `build` method +* Fixed a bug that caused placement preferences to be ignored when provided + to `DockerClient.services.create` +* Fixed a bug that caused a `user` value of `0` to be ignored in + `APIClient.create_container` and `DockerClient.containers.create` + 3.4.1 ----- From 205a2f76bd7791dcef52e8b1f3acd9b95bc29ad3 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 9 Aug 2018 17:28:06 -0700 Subject: [PATCH 0736/1301] Bump dev version Signed-off-by: Joffrey F --- docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index 022daffdd2..a274307cfc 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "3.5.0" +version = "3.6.0-dev" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) From 87d72c0f6cf726764a1cf8aebc527c64d46cecfb Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 9 Aug 2018 17:28:35 -0700 Subject: [PATCH 0737/1301] Misc release script improvements Signed-off-by: Joffrey F --- scripts/release.sh | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/scripts/release.sh b/scripts/release.sh index f36efff97a..5b37b6d083 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -22,7 +22,15 @@ echo "##> Removing stale build files" rm -rf ./build || exit 1 echo "##> Tagging the release as $VERSION" -git tag $VERSION || exit 1 +git tag $VERSION +if [[ $? != 0 ]]; then + head_commit=$(git show --pretty=format:%H HEAD) + tag_commit=$(git show --pretty=format:%H $VERSION) + if [[ $head_commit != $tag_commit ]]; then + echo "ERROR: tag already exists, but isn't the current HEAD" + exit 1 + fi +fi if [[ $2 == 'upload' ]]; then echo "##> Pushing tag to github" git push $GITHUB_REPO $VERSION || exit 1 @@ -30,10 +38,10 @@ fi pandoc -f markdown -t rst README.md -o README.rst || exit 1 +echo "##> sdist & wheel" +python setup.py sdist bdist_wheel + if [[ $2 == 'upload' ]]; then - echo "##> Uploading sdist to pypi" - python setup.py sdist bdist_wheel upload -else - echo "##> sdist & wheel" - python setup.py sdist bdist_wheel -fi + echo '##> Uploading sdist to pypi' + twine upload dist/docker-$VERSION* +fi \ No newline at end of file From e78e4e7491da7055151bfe454282770786a8c270 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 9 Aug 2018 17:33:22 -0700 Subject: [PATCH 0738/1301] Add RollbackConfig to API docs Signed-off-by: Joffrey F --- docs/api.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/api.rst b/docs/api.rst index ff466a1763..6931245716 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -145,6 +145,7 @@ Configuration types .. autoclass:: Privileges .. autoclass:: Resources .. autoclass:: RestartPolicy +.. autoclass:: RollbackConfig .. autoclass:: SecretReference .. autoclass:: ServiceMode .. autoclass:: SwarmExternalCA From 67308c1e55b8cc93276a52a2306d174e52b04cd4 Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Sun, 12 Aug 2018 13:00:16 +0400 Subject: [PATCH 0739/1301] Document defaults of logs() This is not obvious because some are True by default. Signed-off-by: Ben Firshman --- docker/api/container.py | 10 +++++----- docker/models/containers.py | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docker/api/container.py b/docker/api/container.py index d8416066d5..d02ad78d4a 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -765,16 +765,16 @@ def logs(self, container, stdout=True, stderr=True, stream=False, Args: container (str): The container to get logs from - stdout (bool): Get ``STDOUT`` - stderr (bool): Get ``STDERR`` - stream (bool): Stream the response - timestamps (bool): Show timestamps + stdout (bool): Get ``STDOUT``. Default ``True`` + stderr (bool): Get ``STDERR``. Default ``True`` + stream (bool): Stream the response. Default ``False`` + timestamps (bool): Show timestamps. Default ``False`` tail (str or int): Output specified number of lines at the end of logs. Either an integer of number of lines or the string ``all``. Default ``all`` since (datetime or int): Show logs since a given datetime or integer epoch (in seconds) - follow (bool): Follow log output + follow (bool): Follow log output. Default ``False`` until (datetime or int): Show logs that occurred before the given datetime or integer epoch (in seconds) diff --git a/docker/models/containers.py b/docker/models/containers.py index de6222ec49..14545a7688 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -253,16 +253,16 @@ def logs(self, **kwargs): generator you can iterate over to retrieve log output as it happens. Args: - stdout (bool): Get ``STDOUT`` - stderr (bool): Get ``STDERR`` - stream (bool): Stream the response - timestamps (bool): Show timestamps + stdout (bool): Get ``STDOUT``. Default ``True`` + stderr (bool): Get ``STDERR``. Default ``True`` + stream (bool): Stream the response. Default ``False`` + timestamps (bool): Show timestamps. Default ``False`` tail (str or int): Output specified number of lines at the end of logs. Either an integer of number of lines or the string ``all``. Default ``all`` since (datetime or int): Show logs since a given datetime or integer epoch (in seconds) - follow (bool): Follow log output + follow (bool): Follow log output. Default ``False`` until (datetime or int): Show logs that occurred before the given datetime or integer epoch (in seconds) From 74a293a9c92e30ff8e7927694a147ca293fbf686 Mon Sep 17 00:00:00 2001 From: adw1n Date: Mon, 3 Sep 2018 02:49:16 +0200 Subject: [PATCH 0740/1301] Fix docs for `chunk_size` parameter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #2122 Signed-off-by: Przemysław Adamek --- docker/models/images.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/models/images.py b/docker/models/images.py index 41632c6a36..7d9ab70b56 100644 --- a/docker/models/images.py +++ b/docker/models/images.py @@ -64,9 +64,9 @@ def save(self, chunk_size=DEFAULT_DATA_CHUNK_SIZE): Get a tarball of an image. Similar to the ``docker save`` command. Args: - chunk_size (int): The number of bytes returned by each iteration - of the generator. If ``None``, data will be streamed as it is - received. Default: 2 MB + chunk_size (int): The generator will return up to that much data + per iteration, but may return less. If ``None``, data will be + streamed as it is received. Default: 2 MB Returns: (generator): A stream of raw archive data. From 2b10c3773c1f48d16d395f1f08a7a93419ab0790 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 14 Sep 2018 16:58:11 -0700 Subject: [PATCH 0741/1301] Fix docs for Service objects Signed-off-by: Joffrey F --- docker/models/services.py | 53 ++++++++++++++++++++------------------- docs/services.rst | 3 +++ 2 files changed, 30 insertions(+), 26 deletions(-) diff --git a/docker/models/services.py b/docker/models/services.py index fa029f36ea..a2a3ed011f 100644 --- a/docker/models/services.py +++ b/docker/models/services.py @@ -42,7 +42,7 @@ def tasks(self, filters=None): ``label``, and ``desired-state``. Returns: - (:py:class:`list`): List of task dictionaries. + :py:class:`list`: List of task dictionaries. Raises: :py:class:`docker.errors.APIError` @@ -84,26 +84,27 @@ def update(self, **kwargs): def logs(self, **kwargs): """ - Get log stream for the service. - Note: This method works only for services with the ``json-file`` - or ``journald`` logging drivers. - - Args: - details (bool): Show extra details provided to logs. - Default: ``False`` - follow (bool): Keep connection open to read logs as they are - sent by the Engine. Default: ``False`` - stdout (bool): Return logs from ``stdout``. Default: ``False`` - stderr (bool): Return logs from ``stderr``. Default: ``False`` - since (int): UNIX timestamp for the logs staring point. - Default: 0 - timestamps (bool): Add timestamps to every log line. - tail (string or int): Number of log lines to be returned, - counting from the current end of the logs. Specify an - integer or ``'all'`` to output all log lines. - Default: ``all`` - - Returns (generator): Logs for the service. + Get log stream for the service. + Note: This method works only for services with the ``json-file`` + or ``journald`` logging drivers. + + Args: + details (bool): Show extra details provided to logs. + Default: ``False`` + follow (bool): Keep connection open to read logs as they are + sent by the Engine. Default: ``False`` + stdout (bool): Return logs from ``stdout``. Default: ``False`` + stderr (bool): Return logs from ``stderr``. Default: ``False`` + since (int): UNIX timestamp for the logs staring point. + Default: 0 + timestamps (bool): Add timestamps to every log line. + tail (string or int): Number of log lines to be returned, + counting from the current end of the logs. Specify an + integer or ``'all'`` to output all log lines. + Default: ``all`` + + Returns: + generator: Logs for the service. """ is_tty = self.attrs['Spec']['TaskTemplate']['ContainerSpec'].get( 'TTY', False @@ -118,7 +119,7 @@ def scale(self, replicas): replicas (int): The number of containers that should be running. Returns: - ``True``if successful. + bool: ``True`` if successful. """ if 'Global' in self.attrs['Spec']['Mode'].keys(): @@ -134,7 +135,7 @@ def force_update(self): Force update the service even if no changes require it. Returns: - ``True``if successful. + bool: ``True`` if successful. """ return self.update(force_update=True, fetch_current_spec=True) @@ -206,7 +207,7 @@ def create(self, image, command=None, **kwargs): containers. Returns: - (:py:class:`Service`) The created service. + :py:class:`Service`: The created service. Raises: :py:class:`docker.errors.APIError` @@ -228,7 +229,7 @@ def get(self, service_id, insert_defaults=None): into the output. Returns: - (:py:class:`Service`): The service. + :py:class:`Service`: The service. Raises: :py:class:`docker.errors.NotFound` @@ -253,7 +254,7 @@ def list(self, **kwargs): Default: ``None``. Returns: - (list of :py:class:`Service`): The services. + list of :py:class:`Service`: The services. Raises: :py:class:`docker.errors.APIError` diff --git a/docs/services.rst b/docs/services.rst index d8e528545a..8f44428872 100644 --- a/docs/services.rst +++ b/docs/services.rst @@ -30,7 +30,10 @@ Service objects The raw representation of this object from the server. + .. automethod:: force_update + .. automethod:: logs .. automethod:: reload .. automethod:: remove + .. automethod:: scale .. automethod:: tasks .. automethod:: update From 46a9b10b634f70d56662c9e5f834ba35b238321a Mon Sep 17 00:00:00 2001 From: Rui Cao Date: Thu, 27 Sep 2018 21:10:36 +0800 Subject: [PATCH 0742/1301] Fix typo: Addtional -> Additional Signed-off-by: Rui Cao --- docker/api/container.py | 2 +- docker/models/containers.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/api/container.py b/docker/api/container.py index d02ad78d4a..c59a6d01a5 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -465,7 +465,7 @@ def create_host_config(self, *args, **kwargs): dns_opt (:py:class:`list`): Additional options to be added to the container's ``resolv.conf`` file dns_search (:py:class:`list`): DNS search domains. - extra_hosts (dict): Addtional hostnames to resolve inside the + extra_hosts (dict): Additional hostnames to resolve inside the container, as a mapping of hostname to IP address. group_add (:py:class:`list`): List of additional group names and/or IDs that the container process will run as. diff --git a/docker/models/containers.py b/docker/models/containers.py index 14545a7688..f60ba6e225 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -558,7 +558,7 @@ def run(self, image, command=None, stdout=True, stderr=False, environment (dict or list): Environment variables to set inside the container, as a dictionary or a list of strings in the format ``["SOMEVARIABLE=xxx"]``. - extra_hosts (dict): Addtional hostnames to resolve inside the + extra_hosts (dict): Additional hostnames to resolve inside the container, as a mapping of hostname to IP address. group_add (:py:class:`list`): List of additional group names and/or IDs that the container process will run as. From 609045f343ac628f953bb3a8fe5b201700929b5c Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 17 Oct 2018 13:52:39 -0700 Subject: [PATCH 0743/1301] Bump pyopenssl to prevent installation of vulnerable version CVE refs: CVE-2018-1000807 CVE-2018-1000808 Signed-off-by: Joffrey F --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 289dea9150..c46a021e2c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,7 @@ idna==2.5 ipaddress==1.0.18 packaging==16.8 pycparser==2.17 -pyOpenSSL==17.0.0 +pyOpenSSL==18.0.0 pyparsing==2.2.0 pypiwin32==219; sys_platform == 'win32' and python_version < '3.6' pypiwin32==223; sys_platform == 'win32' and python_version >= '3.6' diff --git a/setup.py b/setup.py index 1b208e5c66..390783d50f 100644 --- a/setup.py +++ b/setup.py @@ -40,7 +40,7 @@ # https://github.com/pypa/pip/issues/4391). Once that's fixed, instead of # installing the extra dependencies, install the following instead: # 'requests[security] >= 2.5.2, != 2.11.0, != 2.12.2' - 'tls': ['pyOpenSSL>=0.14', 'cryptography>=1.3.4', 'idna>=2.0.0'], + 'tls': ['pyOpenSSL>=17.5.0', 'cryptography>=1.3.4', 'idna>=2.0.0'], } From f097ea5b9846471437ed06750dfc0bc52e82e055 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 17 Oct 2018 14:38:02 -0700 Subject: [PATCH 0744/1301] Bump 3.5.1 Signed-off-by: Joffrey F --- docker/version.py | 2 +- docs/change-log.md | 14 +++++++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/docker/version.py b/docker/version.py index a274307cfc..ef6b491c54 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "3.6.0-dev" +version = "3.5.1" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) diff --git a/docs/change-log.md b/docs/change-log.md index 1b2d620fb8..750afb9b69 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -1,10 +1,22 @@ Change log ========== +3.5.1 +----- + +[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/54?closed=1) + +### Miscellaneous + +* Bumped version of `pyOpenSSL` in `requirements.txt` and `setup.py` to prevent + installation of a vulnerable version + +* Docs fixes + 3.5.0 ----- -[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone=53?closed=1) +[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/53?closed=1) ### Deprecation warning From e688c09d68604298bfaf0660f8e4b71d7549e52c Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 29 Oct 2018 14:46:27 -0700 Subject: [PATCH 0745/1301] Bump requests dependency in requirements.txt (CVE-2018-18074) Signed-off-by: Joffrey F --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c46a021e2c..c0ce59aeb1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,7 +14,7 @@ pyOpenSSL==18.0.0 pyparsing==2.2.0 pypiwin32==219; sys_platform == 'win32' and python_version < '3.6' pypiwin32==223; sys_platform == 'win32' and python_version >= '3.6' -requests==2.14.2 +requests==2.20.0 six==1.10.0 websocket-client==0.40.0 urllib3==1.21.1; python_version == '3.3' \ No newline at end of file From a3111d9e00c15f957af19f5bb5e301cc606f9aeb Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 31 Oct 2018 18:05:26 -0700 Subject: [PATCH 0746/1301] Add xfail to ignore 18.09 beta bug Signed-off-by: Joffrey F --- tests/integration/api_build_test.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/integration/api_build_test.py b/tests/integration/api_build_test.py index baaf33e3d9..bad411beec 100644 --- a/tests/integration/api_build_test.py +++ b/tests/integration/api_build_test.py @@ -540,6 +540,11 @@ def test_build_in_context_abs_dockerfile(self): ) == sorted(lsdata) @requires_api_version('1.31') + @pytest.mark.xfail( + True, + reason='Currently fails on 18.09: ' + 'https://github.com/moby/moby/issues/37920' + ) def test_prune_builds(self): prune_result = self.client.prune_builds() assert 'SpaceReclaimed' in prune_result From dd7386de30c028300fd3ac9721cc0841fcd065c8 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 1 Nov 2018 15:23:21 -0700 Subject: [PATCH 0747/1301] Update version detection script for CI Signed-off-by: Joffrey F --- scripts/versions.py | 39 ++++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/scripts/versions.py b/scripts/versions.py index 77aaf4f182..7212195dec 100644 --- a/scripts/versions.py +++ b/scripts/versions.py @@ -11,23 +11,24 @@ 'test' ] +STAGES = ['tp', 'beta', 'rc'] -class Version(namedtuple('_Version', 'major minor patch rc edition')): + +class Version(namedtuple('_Version', 'major minor patch stage edition')): @classmethod def parse(cls, version): edition = None version = version.lstrip('v') - version, _, rc = version.partition('-') - if rc: - if 'rc' not in rc: - edition = rc - rc = None - elif '-' in rc: - edition, rc = rc.split('-') - + version, _, stage = version.partition('-') + if stage: + if not any(marker in stage for marker in STAGES): + edition = stage + stage = None + elif '-' in stage: + edition, stage = stage.split('-') major, minor, patch = version.split('.', 3) - return cls(major, minor, patch, rc, edition) + return cls(major, minor, patch, stage, edition) @property def major_minor(self): @@ -38,14 +39,22 @@ def order(self): """Return a representation that allows this object to be sorted correctly with the default comparator. """ - # rc releases should appear before official releases - rc = (0, self.rc) if self.rc else (1, ) - return (int(self.major), int(self.minor), int(self.patch)) + rc + # non-GA releases should appear before GA releases + # Order: tp -> beta -> rc -> GA + if self.stage: + for st in STAGES: + if st in self.stage: + stage = (STAGES.index(st), self.stage) + break + else: + stage = (len(STAGES),) + + return (int(self.major), int(self.minor), int(self.patch)) + stage def __str__(self): - rc = '-{}'.format(self.rc) if self.rc else '' + stage = '-{}'.format(self.stage) if self.stage else '' edition = '-{}'.format(self.edition) if self.edition else '' - return '.'.join(map(str, self[:3])) + edition + rc + return '.'.join(map(str, self[:3])) + edition + stage def main(): From 479f13eff1293731c3cf32db04f191f302fca090 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 31 Oct 2018 17:04:05 -0700 Subject: [PATCH 0748/1301] Add paramiko requirement for SSH transport Signed-off-by: Joffrey F --- requirements.txt | 1 + setup.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/requirements.txt b/requirements.txt index c0ce59aeb1..d13e9d6cad 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,6 +9,7 @@ enum34==1.1.6 idna==2.5 ipaddress==1.0.18 packaging==16.8 +paramiko==2.4.2 pycparser==2.17 pyOpenSSL==18.0.0 pyparsing==2.2.0 diff --git a/setup.py b/setup.py index 390783d50f..3ad572b3ec 100644 --- a/setup.py +++ b/setup.py @@ -42,6 +42,9 @@ # 'requests[security] >= 2.5.2, != 2.11.0, != 2.12.2' 'tls': ['pyOpenSSL>=17.5.0', 'cryptography>=1.3.4', 'idna>=2.0.0'], + # Only required when connecting using the ssh:// protocol + 'ssh': ['paramiko>=2.4.2'], + } version = None From 338dfb00b1c82d6fbd1bdd6a8acc16f7a4c920fa Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 31 Oct 2018 17:06:23 -0700 Subject: [PATCH 0749/1301] Add support for SSH protocol in base_url Signed-off-by: Joffrey F --- docker/api/client.py | 19 ++++++ docker/transport/__init__.py | 5 ++ docker/transport/sshconn.py | 110 +++++++++++++++++++++++++++++++++++ docker/utils/utils.py | 16 +++-- 4 files changed, 145 insertions(+), 5 deletions(-) create mode 100644 docker/transport/sshconn.py diff --git a/docker/api/client.py b/docker/api/client.py index 91da1c893b..197846d105 100644 --- a/docker/api/client.py +++ b/docker/api/client.py @@ -39,6 +39,11 @@ except ImportError: pass +try: + from ..transport import SSHAdapter +except ImportError: + pass + class APIClient( requests.Session, @@ -141,6 +146,18 @@ def __init__(self, base_url=None, version=None, ) self.mount('http+docker://', self._custom_adapter) self.base_url = 'http+docker://localnpipe' + elif base_url.startswith('ssh://'): + try: + self._custom_adapter = SSHAdapter( + base_url, timeout, pool_connections=num_pools + ) + except NameError: + raise DockerException( + 'Install paramiko package to enable ssh:// support' + ) + self.mount('http+docker://ssh', self._custom_adapter) + self._unmount('http://', 'https://') + self.base_url = 'http+docker://ssh' else: # Use SSLAdapter for the ability to specify SSL version if isinstance(tls, TLSConfig): @@ -279,6 +296,8 @@ def _get_raw_response_socket(self, response): self._raise_for_status(response) if self.base_url == "http+docker://localnpipe": sock = response.raw._fp.fp.raw.sock + elif self.base_url.startswith('http+docker://ssh'): + sock = response.raw._fp.fp.channel elif six.PY3: sock = response.raw._fp.fp.raw if self.base_url.startswith("https://"): diff --git a/docker/transport/__init__.py b/docker/transport/__init__.py index abbee182fc..d2cf2a7af3 100644 --- a/docker/transport/__init__.py +++ b/docker/transport/__init__.py @@ -6,3 +6,8 @@ from .npipesocket import NpipeSocket except ImportError: pass + +try: + from .sshconn import SSHAdapter +except ImportError: + pass diff --git a/docker/transport/sshconn.py b/docker/transport/sshconn.py new file mode 100644 index 0000000000..6c9c1196d2 --- /dev/null +++ b/docker/transport/sshconn.py @@ -0,0 +1,110 @@ +import urllib.parse + +import paramiko +import requests.adapters +import six + + +from .. import constants + +if six.PY3: + import http.client as httplib +else: + import httplib + +try: + import requests.packages.urllib3 as urllib3 +except ImportError: + import urllib3 + +RecentlyUsedContainer = urllib3._collections.RecentlyUsedContainer + + +class SSHConnection(httplib.HTTPConnection, object): + def __init__(self, ssh_transport, timeout=60): + super(SSHConnection, self).__init__( + 'localhost', timeout=timeout + ) + self.ssh_transport = ssh_transport + self.timeout = timeout + + def connect(self): + sock = self.ssh_transport.open_session() + sock.settimeout(self.timeout) + sock.exec_command('docker system dial-stdio') + self.sock = sock + + +class SSHConnectionPool(urllib3.connectionpool.HTTPConnectionPool): + scheme = 'ssh' + + def __init__(self, ssh_client, timeout=60, maxsize=10): + super(SSHConnectionPool, self).__init__( + 'localhost', timeout=timeout, maxsize=maxsize + ) + self.ssh_transport = ssh_client.get_transport() + self.timeout = timeout + + def _new_conn(self): + return SSHConnection(self.ssh_transport, self.timeout) + + # When re-using connections, urllib3 calls fileno() on our + # SSH channel instance, quickly overloading our fd limit. To avoid this, + # we override _get_conn + def _get_conn(self, timeout): + conn = None + try: + conn = self.pool.get(block=self.block, timeout=timeout) + + except AttributeError: # self.pool is None + raise urllib3.exceptions.ClosedPoolError(self, "Pool is closed.") + + except six.moves.queue.Empty: + if self.block: + raise urllib3.exceptions.EmptyPoolError( + self, + "Pool reached maximum size and no more " + "connections are allowed." + ) + pass # Oh well, we'll create a new connection then + + return conn or self._new_conn() + + +class SSHAdapter(requests.adapters.HTTPAdapter): + + __attrs__ = requests.adapters.HTTPAdapter.__attrs__ + [ + 'pools', 'timeout', 'ssh_client', + ] + + def __init__(self, base_url, timeout=60, + pool_connections=constants.DEFAULT_NUM_POOLS): + self.ssh_client = paramiko.SSHClient() + self.ssh_client.load_system_host_keys() + + parsed = urllib.parse.urlparse(base_url) + self.ssh_client.connect( + parsed.hostname, parsed.port, parsed.username, + ) + self.timeout = timeout + self.pools = RecentlyUsedContainer( + pool_connections, dispose_func=lambda p: p.close() + ) + super(SSHAdapter, self).__init__() + + def get_connection(self, url, proxies=None): + with self.pools.lock: + pool = self.pools.get(url) + if pool: + return pool + + pool = SSHConnectionPool( + self.ssh_client, self.timeout + ) + self.pools[url] = pool + + return pool + + def close(self): + self.pools.clear() + self.ssh_client.close() diff --git a/docker/utils/utils.py b/docker/utils/utils.py index fe3b9a5767..f8f712349d 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -250,6 +250,9 @@ def parse_host(addr, is_win32=False, tls=False): addr = addr[8:] elif addr.startswith('fd://'): raise errors.DockerException("fd protocol is not implemented") + elif addr.startswith('ssh://'): + proto = 'ssh' + addr = addr[6:] else: if "://" in addr: raise errors.DockerException( @@ -257,17 +260,20 @@ def parse_host(addr, is_win32=False, tls=False): ) proto = "https" if tls else "http" - if proto in ("http", "https"): + if proto in ("http", "https", "ssh"): address_parts = addr.split('/', 1) host = address_parts[0] if len(address_parts) == 2: path = '/' + address_parts[1] host, port = splitnport(host) - if port is None: - raise errors.DockerException( - "Invalid port: {0}".format(addr) - ) + if port is None or port < 0: + if proto == 'ssh': + port = 22 + else: + raise errors.DockerException( + "Invalid port: {0}".format(addr) + ) if not host: host = DEFAULT_HTTP_HOST From f4e9a1dc2a5a781f5768920b7677b7b8627b5675 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 31 Oct 2018 17:07:01 -0700 Subject: [PATCH 0750/1301] Remove misleading fileno method from NpipeSocket class Signed-off-by: Joffrey F --- docker/transport/npipesocket.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docker/transport/npipesocket.py b/docker/transport/npipesocket.py index c04b39dd66..ef02031640 100644 --- a/docker/transport/npipesocket.py +++ b/docker/transport/npipesocket.py @@ -87,10 +87,6 @@ def detach(self): def dup(self): return NpipeSocket(self._handle) - @check_closed - def fileno(self): - return int(self._handle) - def getpeername(self): return self._address From 1df021ee240140d4d713108822473572c24d5feb Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 31 Oct 2018 17:08:52 -0700 Subject: [PATCH 0751/1301] Update tests for ssh protocol compatibility Signed-off-by: Joffrey F --- tests/helpers.py | 4 ++++ tests/integration/api_container_test.py | 1 + 2 files changed, 5 insertions(+) diff --git a/tests/helpers.py b/tests/helpers.py index b36d6d786f..f912bd8d43 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -10,6 +10,7 @@ import socket import docker +import paramiko import pytest @@ -121,6 +122,9 @@ def assert_cat_socket_detached_with_keys(sock, inputs): if getattr(sock, 'family', -9) == getattr(socket, 'AF_UNIX', -1): with pytest.raises(socket.error): sock.sendall(b'make sure the socket is closed\n') + elif isinstance(sock, paramiko.Channel): + with pytest.raises(OSError): + sock.sendall(b'make sure the socket is closed\n') else: sock.sendall(b"make sure the socket is closed\n") data = sock.recv(128) diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index 6ce846bb20..249ef7cfcc 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -1255,6 +1255,7 @@ def test_attach_no_stream(self): assert output == 'hello\n'.encode(encoding='ascii') @pytest.mark.timeout(5) + @pytest.mark.xfail(True, reason='Cancellable events broken over SSH') def test_attach_stream_and_cancel(self): container = self.client.create_container( BUSYBOX, 'sh -c "echo hello && sleep 60"', From 94aa9a89f73e4873350349e79d79b638e101e491 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 31 Oct 2018 17:09:33 -0700 Subject: [PATCH 0752/1301] Update tests to properly dispose of client instances in tearDown Signed-off-by: Joffrey F --- tests/integration/api_network_test.py | 2 +- tests/integration/api_plugin_test.py | 16 +++--- tests/integration/api_swarm_test.py | 3 +- tests/integration/base.py | 73 ++++++++++++++------------- 4 files changed, 48 insertions(+), 46 deletions(-) diff --git a/tests/integration/api_network_test.py b/tests/integration/api_network_test.py index b6726d0242..db37cbd974 100644 --- a/tests/integration/api_network_test.py +++ b/tests/integration/api_network_test.py @@ -8,8 +8,8 @@ class TestNetworks(BaseAPIIntegrationTest): def tearDown(self): - super(TestNetworks, self).tearDown() self.client.leave_swarm(force=True) + super(TestNetworks, self).tearDown() def create_network(self, *args, **kwargs): net_name = random_name() diff --git a/tests/integration/api_plugin_test.py b/tests/integration/api_plugin_test.py index 1150b0957a..38f9d12dad 100644 --- a/tests/integration/api_plugin_test.py +++ b/tests/integration/api_plugin_test.py @@ -3,7 +3,7 @@ import docker import pytest -from .base import BaseAPIIntegrationTest, TEST_API_VERSION +from .base import BaseAPIIntegrationTest from ..helpers import requires_api_version SSHFS = 'vieux/sshfs:latest' @@ -13,27 +13,27 @@ class PluginTest(BaseAPIIntegrationTest): @classmethod def teardown_class(cls): - c = docker.APIClient( - version=TEST_API_VERSION, timeout=60, - **docker.utils.kwargs_from_env() - ) + client = cls.get_client_instance() try: - c.remove_plugin(SSHFS, force=True) + client.remove_plugin(SSHFS, force=True) except docker.errors.APIError: pass def teardown_method(self, method): + client = self.get_client_instance() try: - self.client.disable_plugin(SSHFS) + client.disable_plugin(SSHFS) except docker.errors.APIError: pass for p in self.tmp_plugins: try: - self.client.remove_plugin(p, force=True) + client.remove_plugin(p, force=True) except docker.errors.APIError: pass + client.close() + def ensure_plugin_installed(self, plugin_name): try: return self.client.inspect_plugin(plugin_name) diff --git a/tests/integration/api_swarm_test.py b/tests/integration/api_swarm_test.py index dbf3786eb0..b58dabc639 100644 --- a/tests/integration/api_swarm_test.py +++ b/tests/integration/api_swarm_test.py @@ -13,14 +13,13 @@ def setUp(self): self._unlock_key = None def tearDown(self): - super(SwarmTest, self).tearDown() try: if self._unlock_key: self.client.unlock_swarm(self._unlock_key) except docker.errors.APIError: pass - force_leave_swarm(self.client) + super(SwarmTest, self).tearDown() @requires_api_version('1.24') def test_init_swarm_simple(self): diff --git a/tests/integration/base.py b/tests/integration/base.py index 56c23ed4af..262769de40 100644 --- a/tests/integration/base.py +++ b/tests/integration/base.py @@ -29,41 +29,44 @@ def setUp(self): def tearDown(self): client = docker.from_env(version=TEST_API_VERSION) - for img in self.tmp_imgs: - try: - client.api.remove_image(img) - except docker.errors.APIError: - pass - for container in self.tmp_containers: - try: - client.api.remove_container(container, force=True, v=True) - except docker.errors.APIError: - pass - for network in self.tmp_networks: - try: - client.api.remove_network(network) - except docker.errors.APIError: - pass - for volume in self.tmp_volumes: - try: - client.api.remove_volume(volume) - except docker.errors.APIError: - pass - - for secret in self.tmp_secrets: - try: - client.api.remove_secret(secret) - except docker.errors.APIError: - pass - - for config in self.tmp_configs: - try: - client.api.remove_config(config) - except docker.errors.APIError: - pass - - for folder in self.tmp_folders: - shutil.rmtree(folder) + try: + for img in self.tmp_imgs: + try: + client.api.remove_image(img) + except docker.errors.APIError: + pass + for container in self.tmp_containers: + try: + client.api.remove_container(container, force=True, v=True) + except docker.errors.APIError: + pass + for network in self.tmp_networks: + try: + client.api.remove_network(network) + except docker.errors.APIError: + pass + for volume in self.tmp_volumes: + try: + client.api.remove_volume(volume) + except docker.errors.APIError: + pass + + for secret in self.tmp_secrets: + try: + client.api.remove_secret(secret) + except docker.errors.APIError: + pass + + for config in self.tmp_configs: + try: + client.api.remove_config(config) + except docker.errors.APIError: + pass + + for folder in self.tmp_folders: + shutil.rmtree(folder) + finally: + client.close() class BaseAPIIntegrationTest(BaseIntegrationTest): From 6bfe2005e0a700621c094a01b42db39e7c6408de Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 31 Oct 2018 17:59:26 -0700 Subject: [PATCH 0753/1301] Clear error for cancellable streams over SSH Signed-off-by: Joffrey F --- docker/types/daemon.py | 12 +++++++++++- tests/integration/api_container_test.py | 5 ++++- tests/integration/models_containers_test.py | 3 +++ 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/docker/types/daemon.py b/docker/types/daemon.py index ee8624e80a..700f9a90c4 100644 --- a/docker/types/daemon.py +++ b/docker/types/daemon.py @@ -5,6 +5,8 @@ except ImportError: import urllib3 +from ..errors import DockerException + class CancellableStream(object): """ @@ -55,9 +57,17 @@ def close(self): elif hasattr(sock_raw, '_sock'): sock = sock_raw._sock + elif hasattr(sock_fp, 'channel'): + # We're working with a paramiko (SSH) channel, which doesn't + # support cancelable streams with the current implementation + raise DockerException( + 'Cancellable streams not supported for the SSH protocol' + ) else: sock = sock_fp._sock - if isinstance(sock, urllib3.contrib.pyopenssl.WrappedSocket): + + if hasattr(urllib3.contrib, 'pyopenssl') and isinstance( + sock, urllib3.contrib.pyopenssl.WrappedSocket): sock = sock.socket sock.shutdown(socket.SHUT_RDWR) diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index 249ef7cfcc..02f3603374 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -883,6 +883,8 @@ def test_logs_streaming_and_follow(self): assert logs == (snippet + '\n').encode(encoding='ascii') @pytest.mark.timeout(5) + @pytest.mark.skipif(os.environ.get('DOCKER_HOST', '').startswith('ssh://'), + reason='No cancellable streams over SSH') def test_logs_streaming_and_follow_and_cancel(self): snippet = 'Flowering Nights (Sakuya Iyazoi)' container = self.client.create_container( @@ -1255,7 +1257,8 @@ def test_attach_no_stream(self): assert output == 'hello\n'.encode(encoding='ascii') @pytest.mark.timeout(5) - @pytest.mark.xfail(True, reason='Cancellable events broken over SSH') + @pytest.mark.skipif(os.environ.get('DOCKER_HOST', '').startswith('ssh://'), + reason='No cancellable streams over SSH') def test_attach_stream_and_cancel(self): container = self.client.create_container( BUSYBOX, 'sh -c "echo hello && sleep 60"', diff --git a/tests/integration/models_containers_test.py b/tests/integration/models_containers_test.py index ab41ea57de..b48f6fb6ce 100644 --- a/tests/integration/models_containers_test.py +++ b/tests/integration/models_containers_test.py @@ -1,3 +1,4 @@ +import os import tempfile import threading @@ -146,6 +147,8 @@ def test_run_with_streamed_logs(self): assert logs[1] == b'world\n' @pytest.mark.timeout(5) + @pytest.mark.skipif(os.environ.get('DOCKER_HOST', '').startswith('ssh://'), + reason='No cancellable streams over SSH') def test_run_with_streamed_logs_and_cancel(self): client = docker.from_env(version=TEST_API_VERSION) out = client.containers.run( From f302756599a61d6775fbdf2beab8f1de7e0022c4 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 1 Nov 2018 14:57:29 -0700 Subject: [PATCH 0754/1301] Rewrite utils.parse_host to detect more invalid addresses. The method now uses parsing methods from urllib to better split provided URLs. Addresses containing query strings, parameters, passwords or fragments no longer fail silently. SSH addresses containing paths are no longer accepted. Signed-off-by: Joffrey F --- docker/transport/sshconn.py | 5 +- docker/utils/utils.py | 130 ++++++++++++++++++++---------------- tests/unit/utils_test.py | 12 +++- 3 files changed, 83 insertions(+), 64 deletions(-) diff --git a/docker/transport/sshconn.py b/docker/transport/sshconn.py index 6c9c1196d2..0f6bb51fc2 100644 --- a/docker/transport/sshconn.py +++ b/docker/transport/sshconn.py @@ -1,10 +1,7 @@ -import urllib.parse - import paramiko import requests.adapters import six - from .. import constants if six.PY3: @@ -82,7 +79,7 @@ def __init__(self, base_url, timeout=60, self.ssh_client = paramiko.SSHClient() self.ssh_client.load_system_host_keys() - parsed = urllib.parse.urlparse(base_url) + parsed = six.moves.urllib_parse.urlparse(base_url) self.ssh_client.connect( parsed.hostname, parsed.port, parsed.username, ) diff --git a/docker/utils/utils.py b/docker/utils/utils.py index f8f712349d..4e04cafdb4 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -1,10 +1,11 @@ import base64 +import json import os import os.path -import json import shlex -from distutils.version import StrictVersion +import string from datetime import datetime +from distutils.version import StrictVersion import six @@ -13,11 +14,12 @@ if six.PY2: from urllib import splitnport + from urlparse import urlparse else: - from urllib.parse import splitnport + from urllib.parse import splitnport, urlparse DEFAULT_HTTP_HOST = "127.0.0.1" -DEFAULT_UNIX_SOCKET = "http+unix://var/run/docker.sock" +DEFAULT_UNIX_SOCKET = "http+unix:///var/run/docker.sock" DEFAULT_NPIPE = 'npipe:////./pipe/docker_engine' BYTE_UNITS = { @@ -212,81 +214,93 @@ def parse_repository_tag(repo_name): return repo_name, None -# Based on utils.go:ParseHost http://tinyurl.com/nkahcfh -# fd:// protocol unsupported (for obvious reasons) -# Added support for http and https -# Protocol translation: tcp -> http, unix -> http+unix def parse_host(addr, is_win32=False, tls=False): - proto = "http+unix" - port = None path = '' + port = None + host = None + # Sensible defaults if not addr and is_win32: - addr = DEFAULT_NPIPE - + return DEFAULT_NPIPE if not addr or addr.strip() == 'unix://': return DEFAULT_UNIX_SOCKET addr = addr.strip() - if addr.startswith('http://'): - addr = addr.replace('http://', 'tcp://') - if addr.startswith('http+unix://'): - addr = addr.replace('http+unix://', 'unix://') - if addr == 'tcp://': + parsed_url = urlparse(addr) + proto = parsed_url.scheme + if not proto or any([x not in string.ascii_letters + '+' for x in proto]): + # https://bugs.python.org/issue754016 + parsed_url = urlparse('//' + addr, 'tcp') + proto = 'tcp' + + if proto == 'fd': + raise errors.DockerException('fd protocol is not implemented') + + # These protos are valid aliases for our library but not for the + # official spec + if proto == 'http' or proto == 'https': + tls = proto == 'https' + proto = 'tcp' + elif proto == 'http+unix': + proto = 'unix' + + if proto not in ('tcp', 'unix', 'npipe', 'ssh'): raise errors.DockerException( - "Invalid bind address format: {0}".format(addr) + "Invalid bind address protocol: {}".format(addr) ) - elif addr.startswith('unix://'): - addr = addr[7:] - elif addr.startswith('tcp://'): - proto = 'http{0}'.format('s' if tls else '') - addr = addr[6:] - elif addr.startswith('https://'): - proto = "https" - addr = addr[8:] - elif addr.startswith('npipe://'): - proto = 'npipe' - addr = addr[8:] - elif addr.startswith('fd://'): - raise errors.DockerException("fd protocol is not implemented") - elif addr.startswith('ssh://'): - proto = 'ssh' - addr = addr[6:] - else: - if "://" in addr: - raise errors.DockerException( - "Invalid bind address protocol: {0}".format(addr) - ) - proto = "https" if tls else "http" - if proto in ("http", "https", "ssh"): - address_parts = addr.split('/', 1) - host = address_parts[0] - if len(address_parts) == 2: - path = '/' + address_parts[1] - host, port = splitnport(host) + if proto == 'tcp' and not parsed_url.netloc: + # "tcp://" is exceptionally disallowed by convention; + # omitting a hostname for other protocols is fine + raise errors.DockerException( + 'Invalid bind address format: {}'.format(addr) + ) + if any([ + parsed_url.params, parsed_url.query, parsed_url.fragment, + parsed_url.password + ]): + raise errors.DockerException( + 'Invalid bind address format: {}'.format(addr) + ) + + if parsed_url.path and proto == 'ssh': + raise errors.DockerException( + 'Invalid bind address format: no path allowed for this protocol:' + ' {}'.format(addr) + ) + else: + path = parsed_url.path + if proto == 'unix' and parsed_url.hostname is not None: + # For legacy reasons, we consider unix://path + # to be valid and equivalent to unix:///path + path = '/'.join((parsed_url.hostname, path)) + + if proto in ('tcp', 'ssh'): + # parsed_url.hostname strips brackets from IPv6 addresses, + # which can be problematic hence our use of splitnport() instead. + host, port = splitnport(parsed_url.netloc) if port is None or port < 0: - if proto == 'ssh': - port = 22 - else: + if proto != 'ssh': raise errors.DockerException( - "Invalid port: {0}".format(addr) + 'Invalid bind address format: port is required:' + ' {}'.format(addr) ) + port = 22 if not host: host = DEFAULT_HTTP_HOST - else: - host = addr - if proto in ("http", "https") and port == -1: - raise errors.DockerException( - "Bind address needs a port: {0}".format(addr)) + # Rewrite schemes to fit library internals (requests adapters) + if proto == 'tcp': + proto = 'http{}'.format('s' if tls else '') + elif proto == 'unix': + proto = 'http+unix' - if proto == "http+unix" or proto == 'npipe': - return "{0}://{1}".format(proto, host).rstrip('/') - return "{0}://{1}:{2}{3}".format(proto, host, port, path).rstrip('/') + if proto in ('http+unix', 'npipe'): + return "{}://{}".format(proto, path).rstrip('/') + return '{0}://{1}:{2}{3}'.format(proto, host, port, path).rstrip('/') def parse_devices(devices): diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index 8880cfef0f..c862a1cec9 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -272,6 +272,11 @@ def test_parse_host(self): 'tcp://', 'udp://127.0.0.1', 'udp://127.0.0.1:2375', + 'ssh://:22/path', + 'tcp://netloc:3333/path?q=1', + 'unix:///sock/path#fragment', + 'https://netloc:3333/path;params', + 'ssh://:clearpassword@host:22', ] valid_hosts = { @@ -281,7 +286,7 @@ def test_parse_host(self): 'http://:7777': 'http://127.0.0.1:7777', 'https://kokia.jp:2375': 'https://kokia.jp:2375', 'unix:///var/run/docker.sock': 'http+unix:///var/run/docker.sock', - 'unix://': 'http+unix://var/run/docker.sock', + 'unix://': 'http+unix:///var/run/docker.sock', '12.234.45.127:2375/docker/engine': ( 'http://12.234.45.127:2375/docker/engine' ), @@ -294,6 +299,9 @@ def test_parse_host(self): '[fd12::82d1]:2375/docker/engine': ( 'http://[fd12::82d1]:2375/docker/engine' ), + 'ssh://': 'ssh://127.0.0.1:22', + 'ssh://user@localhost:22': 'ssh://user@localhost:22', + 'ssh://user@remote': 'ssh://user@remote:22', } for host in invalid_hosts: @@ -304,7 +312,7 @@ def test_parse_host(self): assert parse_host(host, None) == expected def test_parse_host_empty_value(self): - unix_socket = 'http+unix://var/run/docker.sock' + unix_socket = 'http+unix:///var/run/docker.sock' npipe = 'npipe:////./pipe/docker_engine' for val in [None, '']: From 490b2db3ae3ed35bd4f57be08ce554bfacff508e Mon Sep 17 00:00:00 2001 From: Adam Dangoor Date: Mon, 5 Nov 2018 00:04:43 +0000 Subject: [PATCH 0755/1301] Add a missing space in a log message Signed-off-by: Adam Dangoor --- docker/auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/auth.py b/docker/auth.py index 9635f933ec..17158f4ae3 100644 --- a/docker/auth.py +++ b/docker/auth.py @@ -267,7 +267,7 @@ def load_config(config_path=None, config_dict=None): return res log.debug( - "Couldn't find auth-related section ; attempting to interpret" + "Couldn't find auth-related section ; attempting to interpret " "as auth-only file" ) return {'auths': parse_auth(config_dict)} From e237c0ea165e485f23b02b0d598c6a65fd8deb94 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 6 Nov 2018 14:46:37 -0800 Subject: [PATCH 0756/1301] Add named parameter to image.save to identify which repository name to use in the resulting tarball Signed-off-by: Joffrey F --- docker/models/images.py | 20 ++++++++++++++++-- tests/integration/models_images_test.py | 27 +++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/docker/models/images.py b/docker/models/images.py index 7d9ab70b56..28b1fd3ffd 100644 --- a/docker/models/images.py +++ b/docker/models/images.py @@ -59,7 +59,7 @@ def history(self): """ return self.client.api.history(self.id) - def save(self, chunk_size=DEFAULT_DATA_CHUNK_SIZE): + def save(self, chunk_size=DEFAULT_DATA_CHUNK_SIZE, named=False): """ Get a tarball of an image. Similar to the ``docker save`` command. @@ -67,6 +67,12 @@ def save(self, chunk_size=DEFAULT_DATA_CHUNK_SIZE): chunk_size (int): The generator will return up to that much data per iteration, but may return less. If ``None``, data will be streamed as it is received. Default: 2 MB + named (str or bool): If ``False`` (default), the tarball will not + retain repository and tag information for this image. If set + to ``True``, the first tag in the :py:attr:`~tags` list will + be used to identify the image. Alternatively, any element of + the :py:attr:`~tags` list can be used as an argument to use + that specific tag as the saved identifier. Returns: (generator): A stream of raw archive data. @@ -83,7 +89,17 @@ def save(self, chunk_size=DEFAULT_DATA_CHUNK_SIZE): >>> f.write(chunk) >>> f.close() """ - return self.client.api.get_image(self.id, chunk_size) + img = self.id + if named: + img = self.tags[0] if self.tags else img + if isinstance(named, six.string_types): + if named not in self.tags: + raise InvalidArgument( + "{} is not a valid tag for this image".format(named) + ) + img = named + + return self.client.api.get_image(img, chunk_size) def tag(self, repository, tag=None, **kwargs): """ diff --git a/tests/integration/models_images_test.py b/tests/integration/models_images_test.py index ae735baafb..31fab10968 100644 --- a/tests/integration/models_images_test.py +++ b/tests/integration/models_images_test.py @@ -5,6 +5,7 @@ import pytest from .base import BaseIntegrationTest, BUSYBOX, TEST_API_VERSION +from ..helpers import random_name class ImageCollectionTest(BaseIntegrationTest): @@ -108,6 +109,32 @@ def test_save_and_load(self): assert len(result) == 1 assert result[0].id == image.id + def test_save_and_load_repo_name(self): + client = docker.from_env(version=TEST_API_VERSION) + image = client.images.get(BUSYBOX) + additional_tag = random_name() + image.tag(additional_tag) + self.tmp_imgs.append(additional_tag) + image.reload() + with tempfile.TemporaryFile() as f: + stream = image.save(named='{}:latest'.format(additional_tag)) + for chunk in stream: + f.write(chunk) + + f.seek(0) + client.images.remove(additional_tag, force=True) + result = client.images.load(f.read()) + + assert len(result) == 1 + assert result[0].id == image.id + assert '{}:latest'.format(additional_tag) in result[0].tags + + def test_save_name_error(self): + client = docker.from_env(version=TEST_API_VERSION) + image = client.images.get(BUSYBOX) + with pytest.raises(docker.errors.InvalidArgument): + image.save(named='sakuya/izayoi') + class ImageTest(BaseIntegrationTest): From 9987c1bc42bc36160bc7fbdf17849a32fcf11808 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 8 Nov 2018 15:05:22 -0800 Subject: [PATCH 0757/1301] Fix docs examples to work with Python 3 Signed-off-by: Joffrey F --- docker/api/daemon.py | 6 ++--- docker/api/image.py | 16 ++++++------ docker/api/mixin.py | 57 ++++++++++++++++++++++++++++++++++++++++++ docker/types/daemon.py | 2 +- 4 files changed, 69 insertions(+), 12 deletions(-) create mode 100644 docker/api/mixin.py diff --git a/docker/api/daemon.py b/docker/api/daemon.py index 76a94cf034..431e7d41cd 100644 --- a/docker/api/daemon.py +++ b/docker/api/daemon.py @@ -42,8 +42,8 @@ def events(self, since=None, until=None, filters=None, decode=None): Example: - >>> for event in client.events() - ... print event + >>> for event in client.events(decode=True) + ... print(event) {u'from': u'image/with:tag', u'id': u'container-id', u'status': u'start', @@ -54,7 +54,7 @@ def events(self, since=None, until=None, filters=None, decode=None): >>> events = client.events() >>> for event in events: - ... print event + ... print(event) >>> # and cancel from another thread >>> events.close() """ diff --git a/docker/api/image.py b/docker/api/image.py index 5f05d8877e..1c1fcde84c 100644 --- a/docker/api/image.py +++ b/docker/api/image.py @@ -352,8 +352,8 @@ def pull(self, repository, tag=None, stream=False, auth_config=None, Example: - >>> for line in cli.pull('busybox', stream=True): - ... print(json.dumps(json.loads(line), indent=4)) + >>> for line in cli.pull('busybox', stream=True, decode=True): + ... print(json.dumps(line, indent=4)) { "status": "Pulling image (latest) from busybox", "progressDetail": {}, @@ -428,12 +428,12 @@ def push(self, repository, tag=None, stream=False, auth_config=None, If the server returns an error. Example: - >>> for line in cli.push('yourname/app', stream=True): - ... print line - {"status":"Pushing repository yourname/app (1 tags)"} - {"status":"Pushing","progressDetail":{},"id":"511136ea3c5a"} - {"status":"Image already pushed, skipping","progressDetail":{}, - "id":"511136ea3c5a"} + >>> for line in cli.push('yourname/app', stream=True, decode=True): + ... print(line) + {'status': 'Pushing repository yourname/app (1 tags)'} + {'status': 'Pushing','progressDetail': {}, 'id': '511136ea3c5a'} + {'status': 'Image already pushed, skipping', 'progressDetail':{}, + 'id': '511136ea3c5a'} ... """ diff --git a/docker/api/mixin.py b/docker/api/mixin.py new file mode 100644 index 0000000000..1353568167 --- /dev/null +++ b/docker/api/mixin.py @@ -0,0 +1,57 @@ +from typing import Any, Dict, Iterable, List, Optional, Union + +import requests + +from ..constants import DEFAULT_DOCKER_API_VERSION, DEFAULT_TIMEOUT_SECONDS + +class BaseMixin(object): + base_url: str = '' + credstore_env: Optional[Dict[str, str]] = None + timeout: int = DEFAULT_TIMEOUT_SECONDS + _auth_configs: Dict[str, Dict] + _general_configs: Dict[str, Dict] + _version: str = DEFAULT_DOCKER_API_VERSION + + def _url(self, pathfmt: str, *args, **kwargs) -> str: + raise NotImplemented + + def _post(self, url: str, **kwargs) -> requests.Response: + raise NotImplemented + + def _get(self, url: str, **kwargs) -> requests.Response: + raise NotImplemented + + def _put(self, url: str, **kwargs) -> requests.Response: + raise NotImplemented + + def _delete(self, url: str, **kwargs) -> requests.Response: + raise NotImplemented + + def _post_json(self, url: str, data: Optional[Union[Dict[str, Any], List[Any]]], **kwargs) -> requests.Response: + raise NotImplemented + + def _raise_for_status(self, response: requests.Response) -> None: + raise NotImplemented + + def _result(self, response: requests.Response, json: bool=False, binary: bool=False) -> Any: + raise NotImplemented + + def _stream_helper(self, response: requests.Response, decode: bool = False) -> Iterable: + raise NotImplemented + + def _get_raw_response_socket(self, response: requests.Response) -> Iterable: + raise NotImplemented + + def _read_from_socket( + self, + response: requests.Response, + stream: bool, + tty: bool = False) -> Union[Iterable[bytes], bytes]: + raise NotImplemented + + def _stream_raw_result( + self, + response: requests.Response, + chunk_size: int = 1, + decode: bool = True) -> Iterable[bytes]: + raise NotImplemented diff --git a/docker/types/daemon.py b/docker/types/daemon.py index 700f9a90c4..af3e5bcb5e 100644 --- a/docker/types/daemon.py +++ b/docker/types/daemon.py @@ -15,7 +15,7 @@ class CancellableStream(object): Example: >>> events = client.events() >>> for event in events: - ... print event + ... print(event) >>> # and cancel from another thread >>> events.close() """ From 1d124a1262c73a85ff6601e8fdef9c7e48b33543 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 8 Nov 2018 15:32:10 -0800 Subject: [PATCH 0758/1301] Improve ulimits documentation Signed-off-by: Joffrey F --- docker/api/container.py | 2 +- docker/models/containers.py | 4 ++-- docker/types/containers.py | 17 +++++++++++++++++ docs/api.rst | 1 + 4 files changed, 21 insertions(+), 3 deletions(-) diff --git a/docker/api/container.py b/docker/api/container.py index c59a6d01a5..6967a13acf 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -543,7 +543,7 @@ def create_host_config(self, *args, **kwargs): } ulimits (:py:class:`list`): Ulimits to set inside the container, - as a list of dicts. + as a list of :py:class:`docker.types.Ulimit` instances. userns_mode (str): Sets the user namespace mode for the container when user namespace remapping option is enabled. Supported values are: ``host`` diff --git a/docker/models/containers.py b/docker/models/containers.py index f60ba6e225..98c717421f 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -691,8 +691,8 @@ def run(self, image, command=None, stdout=True, stderr=False, } tty (bool): Allocate a pseudo-TTY. - ulimits (:py:class:`list`): Ulimits to set inside the container, as - a list of dicts. + ulimits (:py:class:`list`): Ulimits to set inside the container, + as a list of :py:class:`docker.types.Ulimit` instances. user (str or int): Username or UID to run commands as inside the container. userns_mode (str): Sets the user namespace mode for the container diff --git a/docker/types/containers.py b/docker/types/containers.py index 9dfea8ceb8..13eb4ef37b 100644 --- a/docker/types/containers.py +++ b/docker/types/containers.py @@ -58,6 +58,23 @@ def unset_config(self, key): class Ulimit(DictType): + """ + Create a ulimit declaration to be used with + :py:meth:`~docker.api.container.ContainerApiMixin.create_host_config`. + + Args: + + name (str): Which ulimit will this apply to. A list of valid names can + be found `here `_. + soft (int): The soft limit for this ulimit. Optional. + hard (int): The hard limit for this ulimit. Optional. + + Example: + + nproc_limit = docker.types.Ulimit(name='nproc', soft=1024) + hc = client.create_host_config(ulimits=[nproc_limit]) + container = client.create_container('busybox', 'true', host_config=hc) + """ def __init__(self, **kwargs): name = kwargs.get('name', kwargs.get('Name')) soft = kwargs.get('soft', kwargs.get('Soft')) diff --git a/docs/api.rst b/docs/api.rst index 6931245716..2c2391a803 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -151,4 +151,5 @@ Configuration types .. autoclass:: SwarmExternalCA .. autoclass:: SwarmSpec(*args, **kwargs) .. autoclass:: TaskTemplate +.. autoclass:: Ulimit .. autoclass:: UpdateConfig From d5bc46ad456ca7f8c008baa31f6623d4f58ccdcd Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 8 Nov 2018 16:20:28 -0800 Subject: [PATCH 0759/1301] Improved LogConfig documentation Signed-off-by: Joffrey F --- docker/api/container.py | 8 +------ docker/models/containers.py | 8 +------ docker/types/containers.py | 45 ++++++++++++++++++++++++++++++++++--- docs/api.rst | 1 + 4 files changed, 45 insertions(+), 17 deletions(-) diff --git a/docker/api/container.py b/docker/api/container.py index 6967a13acf..39478329ae 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -476,13 +476,7 @@ def create_host_config(self, *args, **kwargs): isolation (str): Isolation technology to use. Default: `None`. links (dict or list of tuples): Either a dictionary mapping name to alias or as a list of ``(name, alias)`` tuples. - log_config (dict): Logging configuration, as a dictionary with - keys: - - - ``type`` The logging driver name. - - ``config`` A dictionary of configuration for the logging - driver. - + log_config (LogConfig): Logging configuration lxc_conf (dict): LXC config. mem_limit (float or str): Memory limit. Accepts float values (which represent the memory limit of the created container in diff --git a/docker/models/containers.py b/docker/models/containers.py index 98c717421f..4cd7d13f41 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -576,13 +576,7 @@ def run(self, image, command=None, stdout=True, stderr=False, ``["label1", "label2"]``) links (dict or list of tuples): Either a dictionary mapping name to alias or as a list of ``(name, alias)`` tuples. - log_config (dict): Logging configuration, as a dictionary with - keys: - - - ``type`` The logging driver name. - - ``config`` A dictionary of configuration for the logging - driver. - + log_config (LogConfig): Logging configuration. mac_address (str): MAC address to assign to the container. mem_limit (int or str): Memory limit. Accepts float values (which represent the memory limit of the created container in diff --git a/docker/types/containers.py b/docker/types/containers.py index 13eb4ef37b..d040c0fb5e 100644 --- a/docker/types/containers.py +++ b/docker/types/containers.py @@ -23,6 +23,36 @@ class LogConfigTypesEnum(object): class LogConfig(DictType): + """ + Configure logging for a container, when provided as an argument to + :py:meth:`~docker.api.container.ContainerApiMixin.create_host_config`. + You may refer to the + `official logging driver documentation `_ + for more information. + + Args: + type (str): Indicate which log driver to use. A set of valid drivers + is provided as part of the :py:attr:`LogConfig.types` + enum. Other values may be accepted depending on the engine version + and available logging plugins. + config (dict): A driver-dependent configuration dictionary. Please + refer to the driver's documentation for a list of valid config + keys. + + Example: + + >>> from docker.types import LogConfig + >>> lc = LogConfig(type=LogConfig.types.JSON, config={ + ... 'max-size': '1g', + ... 'labels': 'production_status,geo' + ... }) + >>> hc = client.create_host_config(log_config=lc) + >>> container = client.create_container('busybox', 'true', + ... host_config=hc) + >>> client.inspect_container(container)['HostConfig']['LogConfig'] + {'Type': 'json-file', 'Config': {'labels': 'production_status,geo', 'max-size': '1g'}} + + """ # flake8: noqa types = LogConfigTypesEnum def __init__(self, **kwargs): @@ -50,9 +80,13 @@ def config(self): return self['Config'] def set_config_value(self, key, value): + """ Set a the value for ``key`` to ``value`` inside the ``config`` + dict. + """ self.config[key] = value def unset_config(self, key): + """ Remove the ``key`` property from the ``config`` dict. """ if key in self.config: del self.config[key] @@ -71,9 +105,14 @@ class Ulimit(DictType): Example: - nproc_limit = docker.types.Ulimit(name='nproc', soft=1024) - hc = client.create_host_config(ulimits=[nproc_limit]) - container = client.create_container('busybox', 'true', host_config=hc) + >>> nproc_limit = docker.types.Ulimit(name='nproc', soft=1024) + >>> hc = client.create_host_config(ulimits=[nproc_limit]) + >>> container = client.create_container( + 'busybox', 'true', host_config=hc + ) + >>> client.inspect_container(container)['HostConfig']['Ulimits'] + [{'Name': 'nproc', 'Hard': 0, 'Soft': 1024}] + """ def __init__(self, **kwargs): name = kwargs.get('name', kwargs.get('Name')) diff --git a/docs/api.rst b/docs/api.rst index 2c2391a803..1682128951 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -140,6 +140,7 @@ Configuration types .. autoclass:: Healthcheck .. autoclass:: IPAMConfig .. autoclass:: IPAMPool +.. autoclass:: LogConfig .. autoclass:: Mount .. autoclass:: Placement .. autoclass:: Privileges From 6064947431dd36b8aa33a5f2eb850b03302de5ee Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 8 Nov 2018 16:52:32 -0800 Subject: [PATCH 0760/1301] Update links docs and fix bug in normalize_links Signed-off-by: Joffrey F --- docker/api/container.py | 17 ++++++++++------- docker/models/containers.py | 6 ++++-- docker/utils/utils.py | 2 +- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/docker/api/container.py b/docker/api/container.py index 39478329ae..8858aa68a3 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -473,9 +473,11 @@ def create_host_config(self, *args, **kwargs): signals and reaps processes init_path (str): Path to the docker-init binary ipc_mode (str): Set the IPC mode for the container. - isolation (str): Isolation technology to use. Default: `None`. - links (dict or list of tuples): Either a dictionary mapping name - to alias or as a list of ``(name, alias)`` tuples. + isolation (str): Isolation technology to use. Default: ``None``. + links (dict): Mapping of links using the + ``{'container': 'alias'}`` format. The alias is optional. + Containers declared in this dict will be linked to the new + container using the provided alias. Default: ``None``. log_config (LogConfig): Logging configuration lxc_conf (dict): LXC config. mem_limit (float or str): Memory limit. Accepts float values @@ -605,9 +607,10 @@ def create_endpoint_config(self, *args, **kwargs): aliases (:py:class:`list`): A list of aliases for this endpoint. Names in that list can be used within the network to reach the container. Defaults to ``None``. - links (:py:class:`list`): A list of links for this endpoint. - Containers declared in this list will be linked to this - container. Defaults to ``None``. + links (dict): Mapping of links for this endpoint using the + ``{'container': 'alias'}`` format. The alias is optional. + Containers declared in this dict will be linked to this + container using the provided alias. Defaults to ``None``. ipv4_address (str): The IP address of this container on the network, using the IPv4 protocol. Defaults to ``None``. ipv6_address (str): The IP address of this container on the @@ -622,7 +625,7 @@ def create_endpoint_config(self, *args, **kwargs): >>> endpoint_config = client.create_endpoint_config( aliases=['web', 'app'], - links=['app_db'], + links={'app_db': 'db', 'another': None}, ipv4_address='132.65.0.123' ) diff --git a/docker/models/containers.py b/docker/models/containers.py index 4cd7d13f41..bba0395eed 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -574,8 +574,10 @@ def run(self, image, command=None, stdout=True, stderr=False, ``{"label1": "value1", "label2": "value2"}``) or a list of names of labels to set with empty values (e.g. ``["label1", "label2"]``) - links (dict or list of tuples): Either a dictionary mapping name - to alias or as a list of ``(name, alias)`` tuples. + links (dict): Mapping of links using the + ``{'container': 'alias'}`` format. The alias is optional. + Containers declared in this dict will be linked to the new + container using the provided alias. Default: ``None``. log_config (LogConfig): Logging configuration. mac_address (str): MAC address to assign to the container. mem_limit (int or str): Memory limit. Accepts float values diff --git a/docker/utils/utils.py b/docker/utils/utils.py index 4e04cafdb4..a8e814d7d5 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -441,7 +441,7 @@ def normalize_links(links): if isinstance(links, dict): links = six.iteritems(links) - return ['{0}:{1}'.format(k, v) for k, v in sorted(links)] + return ['{0}:{1}'.format(k, v) if v else k for k, v in sorted(links)] def parse_env_file(env_file): From 6bfe4c90906b3ff737b256825b5059c893a6a4d1 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 8 Nov 2018 17:05:42 -0800 Subject: [PATCH 0761/1301] Document attr caching for Container objects Signed-off-by: Joffrey F --- docker/models/containers.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docker/models/containers.py b/docker/models/containers.py index bba0395eed..a3fd1a8edf 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -15,7 +15,12 @@ class Container(Model): - + """ Local representation of a container object. Detailed configuration may + be accessed through the :py:attr:`attrs` attribute. Note that local + attributes are cached; users may call :py:meth:`reload` to + query the Docker daemon for the current properties, causing + :py:attr:`attrs` to be refreshed. + """ @property def name(self): """ From b927a5f62cf2d66209244129ed3434575c4045f5 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 8 Nov 2018 17:08:41 -0800 Subject: [PATCH 0762/1301] Fix incorrect return info for inspect_service Signed-off-by: Joffrey F --- docker/api/service.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/api/service.py b/docker/api/service.py index 8b956b63e1..08e2591730 100644 --- a/docker/api/service.py +++ b/docker/api/service.py @@ -197,7 +197,8 @@ def inspect_service(self, service, insert_defaults=None): into the service inspect output. Returns: - ``True`` if successful. + (dict): A dictionary of the server-side representation of the + service, including all relevant properties. Raises: :py:class:`docker.errors.APIError` From 89ee08f511d8a0882d5b034fc5be670f8987a802 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 8 Nov 2018 17:13:19 -0800 Subject: [PATCH 0763/1301] Disallow incompatible combination stats(decode=True, stream=False) Signed-off-by: Joffrey F --- docker/api/container.py | 7 ++++++- docker/models/containers.py | 3 ++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/docker/api/container.py b/docker/api/container.py index 8858aa68a3..753e0a57fe 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -1071,7 +1071,8 @@ def stats(self, container, decode=None, stream=True): Args: container (str): The container to stream statistics from decode (bool): If set to true, stream will be decoded into dicts - on the fly. False by default. + on the fly. Only applicable if ``stream`` is True. + False by default. stream (bool): If set to false, only the current stats will be returned instead of a stream. True by default. @@ -1085,6 +1086,10 @@ def stats(self, container, decode=None, stream=True): return self._stream_helper(self._get(url, stream=True), decode=decode) else: + if decode: + raise errors.InvalidArgument( + "decode is only available in conjuction with stream=True" + ) return self._result(self._get(url, params={'stream': False}), json=True) diff --git a/docker/models/containers.py b/docker/models/containers.py index a3fd1a8edf..493b9fc732 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -385,7 +385,8 @@ def stats(self, **kwargs): Args: decode (bool): If set to true, stream will be decoded into dicts - on the fly. False by default. + on the fly. Only applicable if ``stream`` is True. + False by default. stream (bool): If set to false, only the current stats will be returned instead of a stream. True by default. From f83fe7c9594e72cddf1d89031603c3d246c4c101 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 8 Nov 2018 17:22:24 -0800 Subject: [PATCH 0764/1301] Properly convert non-string filters to expected string format Signed-off-by: Joffrey F --- docker/utils/utils.py | 5 ++++- tests/unit/utils_test.py | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/docker/utils/utils.py b/docker/utils/utils.py index a8e814d7d5..61e307adc7 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -386,7 +386,10 @@ def convert_filters(filters): v = 'true' if v else 'false' if not isinstance(v, list): v = [v, ] - result[k] = v + result[k] = [ + str(item) if not isinstance(item, six.string_types) else item + for item in v + ] return json.dumps(result) diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index c862a1cec9..a4e9c9c53e 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -457,8 +457,8 @@ def test_convert_filters(self): tests = [ ({'dangling': True}, '{"dangling": ["true"]}'), ({'dangling': "true"}, '{"dangling": ["true"]}'), - ({'exited': 0}, '{"exited": [0]}'), - ({'exited': [0, 1]}, '{"exited": [0, 1]}'), + ({'exited': 0}, '{"exited": ["0"]}'), + ({'exited': [0, 1]}, '{"exited": ["0", "1"]}'), ] for filters, expected in tests: From cebdee4aefd6c17eeb3542f1c53f1c461b95ea61 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 8 Nov 2018 17:31:22 -0800 Subject: [PATCH 0765/1301] Add doc example for get_archive Signed-off-by: Joffrey F --- docker/api/container.py | 12 ++++++++++++ docker/models/containers.py | 11 +++++++++++ 2 files changed, 23 insertions(+) diff --git a/docker/api/container.py b/docker/api/container.py index 753e0a57fe..fce73af640 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -694,6 +694,18 @@ def get_archive(self, container, path, chunk_size=DEFAULT_DATA_CHUNK_SIZE): Raises: :py:class:`docker.errors.APIError` If the server returns an error. + + Example: + + >>> c = docker.APIClient() + >>> f = open('./sh_bin.tar', 'wb') + >>> bits, stat = c.get_archive(container, '/bin/sh') + >>> print(stat) + {'name': 'sh', 'size': 1075464, 'mode': 493, + 'mtime': '2018-10-01T15:37:48-07:00', 'linkTarget': ''} + >>> for chunk in bits: + ... f.write(chunk) + >>> f.close() """ params = { 'path': path diff --git a/docker/models/containers.py b/docker/models/containers.py index 493b9fc732..9d6f2cc6af 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -233,6 +233,17 @@ def get_archive(self, path, chunk_size=DEFAULT_DATA_CHUNK_SIZE): Raises: :py:class:`docker.errors.APIError` If the server returns an error. + + Example: + + >>> f = open('./sh_bin.tar', 'wb') + >>> bits, stat = container.get_archive('/bin/sh') + >>> print(stat) + {'name': 'sh', 'size': 1075464, 'mode': 493, + 'mtime': '2018-10-01T15:37:48-07:00', 'linkTarget': ''} + >>> for chunk in bits: + ... f.write(chunk) + >>> f.close() """ return self.client.api.get_archive(self.id, path, chunk_size) From 852d79b08d708ff25c1f8bee3b68ffb11ea98dc0 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 8 Nov 2018 17:32:33 -0800 Subject: [PATCH 0766/1301] Fix file mode in image.save examples Signed-off-by: Joffrey F --- docker/api/image.py | 2 +- docker/models/images.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/api/image.py b/docker/api/image.py index 1c1fcde84c..a9f801e93b 100644 --- a/docker/api/image.py +++ b/docker/api/image.py @@ -32,7 +32,7 @@ def get_image(self, image, chunk_size=DEFAULT_DATA_CHUNK_SIZE): Example: >>> image = cli.get_image("busybox:latest") - >>> f = open('/tmp/busybox-latest.tar', 'w') + >>> f = open('/tmp/busybox-latest.tar', 'wb') >>> for chunk in image: >>> f.write(chunk) >>> f.close() diff --git a/docker/models/images.py b/docker/models/images.py index 28b1fd3ffd..4578c0bd89 100644 --- a/docker/models/images.py +++ b/docker/models/images.py @@ -84,7 +84,7 @@ def save(self, chunk_size=DEFAULT_DATA_CHUNK_SIZE, named=False): Example: >>> image = cli.get_image("busybox:latest") - >>> f = open('/tmp/busybox-latest.tar', 'w') + >>> f = open('/tmp/busybox-latest.tar', 'wb') >>> for chunk in image: >>> f.write(chunk) >>> f.close() From 35b9460748d792670d815496767fab8b396d4300 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 8 Nov 2018 17:38:59 -0800 Subject: [PATCH 0767/1301] Remove prematurely committed file Signed-off-by: Joffrey F --- docker/api/mixin.py | 57 --------------------------------------------- 1 file changed, 57 deletions(-) delete mode 100644 docker/api/mixin.py diff --git a/docker/api/mixin.py b/docker/api/mixin.py deleted file mode 100644 index 1353568167..0000000000 --- a/docker/api/mixin.py +++ /dev/null @@ -1,57 +0,0 @@ -from typing import Any, Dict, Iterable, List, Optional, Union - -import requests - -from ..constants import DEFAULT_DOCKER_API_VERSION, DEFAULT_TIMEOUT_SECONDS - -class BaseMixin(object): - base_url: str = '' - credstore_env: Optional[Dict[str, str]] = None - timeout: int = DEFAULT_TIMEOUT_SECONDS - _auth_configs: Dict[str, Dict] - _general_configs: Dict[str, Dict] - _version: str = DEFAULT_DOCKER_API_VERSION - - def _url(self, pathfmt: str, *args, **kwargs) -> str: - raise NotImplemented - - def _post(self, url: str, **kwargs) -> requests.Response: - raise NotImplemented - - def _get(self, url: str, **kwargs) -> requests.Response: - raise NotImplemented - - def _put(self, url: str, **kwargs) -> requests.Response: - raise NotImplemented - - def _delete(self, url: str, **kwargs) -> requests.Response: - raise NotImplemented - - def _post_json(self, url: str, data: Optional[Union[Dict[str, Any], List[Any]]], **kwargs) -> requests.Response: - raise NotImplemented - - def _raise_for_status(self, response: requests.Response) -> None: - raise NotImplemented - - def _result(self, response: requests.Response, json: bool=False, binary: bool=False) -> Any: - raise NotImplemented - - def _stream_helper(self, response: requests.Response, decode: bool = False) -> Iterable: - raise NotImplemented - - def _get_raw_response_socket(self, response: requests.Response) -> Iterable: - raise NotImplemented - - def _read_from_socket( - self, - response: requests.Response, - stream: bool, - tty: bool = False) -> Union[Iterable[bytes], bytes]: - raise NotImplemented - - def _stream_raw_result( - self, - response: requests.Response, - chunk_size: int = 1, - decode: bool = True) -> Iterable[bytes]: - raise NotImplemented From f7a1052b2ba5b5ac308a9ab5c218057ab569a204 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 8 Nov 2018 18:58:06 -0800 Subject: [PATCH 0768/1301] Fix versions script to accept versions without -ce suffix Signed-off-by: Joffrey F --- scripts/versions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/versions.py b/scripts/versions.py index 7212195dec..7ad1d56a61 100644 --- a/scripts/versions.py +++ b/scripts/versions.py @@ -66,7 +66,7 @@ def main(): Version.parse( v.strip('"').lstrip('docker-').rstrip('.tgz').rstrip('-x86_64') ) for v in re.findall( - r'"docker-[0-9]+\.[0-9]+\.[0-9]+-.*tgz"', content + r'"docker-[0-9]+\.[0-9]+\.[0-9]+-?.*tgz"', content ) ] sorted_versions = sorted( From 47c10aa383b87e0cb47904bd912839c13d902924 Mon Sep 17 00:00:00 2001 From: Corentin Henry Date: Wed, 21 Nov 2018 17:12:01 -0800 Subject: [PATCH 0769/1301] tests: fix failure due to pytest deprecation Signed-off-by: Corentin Henry --- tests/unit/utils_config_test.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/unit/utils_config_test.py b/tests/unit/utils_config_test.py index 50ba3831db..b0934f9568 100644 --- a/tests/unit/utils_config_test.py +++ b/tests/unit/utils_config_test.py @@ -4,8 +4,8 @@ import tempfile import json -from py.test import ensuretemp -from pytest import mark +from pytest import mark, fixture + from docker.utils import config try: @@ -15,25 +15,25 @@ class FindConfigFileTest(unittest.TestCase): - def tmpdir(self, name): - tmpdir = ensuretemp(name) - self.addCleanup(tmpdir.remove) - return tmpdir + + @fixture(autouse=True) + def tmpdir(self, tmpdir): + self.mkdir = tmpdir.mkdir def test_find_config_fallback(self): - tmpdir = self.tmpdir('test_find_config_fallback') + tmpdir = self.mkdir('test_find_config_fallback') with mock.patch.dict(os.environ, {'HOME': str(tmpdir)}): assert config.find_config_file() is None def test_find_config_from_explicit_path(self): - tmpdir = self.tmpdir('test_find_config_from_explicit_path') + tmpdir = self.mkdir('test_find_config_from_explicit_path') config_path = tmpdir.ensure('my-config-file.json') assert config.find_config_file(str(config_path)) == str(config_path) def test_find_config_from_environment(self): - tmpdir = self.tmpdir('test_find_config_from_environment') + tmpdir = self.mkdir('test_find_config_from_environment') config_path = tmpdir.ensure('config.json') with mock.patch.dict(os.environ, {'DOCKER_CONFIG': str(tmpdir)}): @@ -41,7 +41,7 @@ def test_find_config_from_environment(self): @mark.skipif("sys.platform == 'win32'") def test_find_config_from_home_posix(self): - tmpdir = self.tmpdir('test_find_config_from_home_posix') + tmpdir = self.mkdir('test_find_config_from_home_posix') config_path = tmpdir.ensure('.docker', 'config.json') with mock.patch.dict(os.environ, {'HOME': str(tmpdir)}): @@ -49,7 +49,7 @@ def test_find_config_from_home_posix(self): @mark.skipif("sys.platform == 'win32'") def test_find_config_from_home_legacy_name(self): - tmpdir = self.tmpdir('test_find_config_from_home_legacy_name') + tmpdir = self.mkdir('test_find_config_from_home_legacy_name') config_path = tmpdir.ensure('.dockercfg') with mock.patch.dict(os.environ, {'HOME': str(tmpdir)}): @@ -57,7 +57,7 @@ def test_find_config_from_home_legacy_name(self): @mark.skipif("sys.platform != 'win32'") def test_find_config_from_home_windows(self): - tmpdir = self.tmpdir('test_find_config_from_home_windows') + tmpdir = self.mkdir('test_find_config_from_home_windows') config_path = tmpdir.ensure('.docker', 'config.json') with mock.patch.dict(os.environ, {'USERPROFILE': str(tmpdir)}): From 493d7f0f3041eb6f7170fa1edce2e8aa2740bd41 Mon Sep 17 00:00:00 2001 From: Corentin Henry Date: Wed, 21 Nov 2018 17:52:35 -0800 Subject: [PATCH 0770/1301] tests: bump pytest-timeout Signed-off-by: Corentin Henry pytest-timeout 1.2.1 seems to be incompatible with pytest 3.6.3: INTERNALERROR> Traceback (most recent call last): INTERNALERROR> File "/usr/local/lib/python2.7/site-packages/_pytest/main.py", line 185, in wrap_session INTERNALERROR> session.exitstatus = doit(config, session) or 0 INTERNALERROR> File "/usr/local/lib/python2.7/site-packages/_pytest/main.py", line 225, in _main INTERNALERROR> config.hook.pytest_runtestloop(session=session) INTERNALERROR> File "/usr/local/lib/python2.7/site-packages/pluggy/hooks.py", line 284, in __call__ INTERNALERROR> return self._hookexec(self, self.get_hookimpls(), kwargs) INTERNALERROR> File "/usr/local/lib/python2.7/site-packages/pluggy/manager.py", line 67, in _hookexec INTERNALERROR> return self._inner_hookexec(hook, methods, kwargs) INTERNALERROR> File "/usr/local/lib/python2.7/site-packages/pluggy/manager.py", line 61, in INTERNALERROR> firstresult=hook.spec.opts.get("firstresult") if hook.spec else False, INTERNALERROR> File "/usr/local/lib/python2.7/site-packages/pluggy/callers.py", line 208, in _multicall INTERNALERROR> return outcome.get_result() INTERNALERROR> File "/usr/local/lib/python2.7/site-packages/pluggy/callers.py", line 81, in get_result INTERNALERROR> _reraise(*ex) # noqa INTERNALERROR> File "/usr/local/lib/python2.7/site-packages/pluggy/callers.py", line 187, in _multicall INTERNALERROR> res = hook_impl.function(*args) INTERNALERROR> File "/usr/local/lib/python2.7/site-packages/_pytest/main.py", line 246, in pytest_runtestloop INTERNALERROR> item.config.hook.pytest_runtest_protocol(item=item, nextitem=nextitem) INTERNALERROR> File "/usr/local/lib/python2.7/site-packages/pluggy/hooks.py", line 284, in __call__ INTERNALERROR> return self._hookexec(self, self.get_hookimpls(), kwargs) INTERNALERROR> File "/usr/local/lib/python2.7/site-packages/pluggy/manager.py", line 67, in _hookexec INTERNALERROR> return self._inner_hookexec(hook, methods, kwargs) INTERNALERROR> File "/usr/local/lib/python2.7/site-packages/pluggy/manager.py", line 61, in INTERNALERROR> firstresult=hook.spec.opts.get("firstresult") if hook.spec else False, INTERNALERROR> File "/usr/local/lib/python2.7/site-packages/pluggy/callers.py", line 208, in _multicall INTERNALERROR> return outcome.get_result() INTERNALERROR> File "/usr/local/lib/python2.7/site-packages/pluggy/callers.py", line 81, in get_result INTERNALERROR> _reraise(*ex) # noqa INTERNALERROR> File "/usr/local/lib/python2.7/site-packages/pluggy/callers.py", line 182, in _multicall INTERNALERROR> next(gen) # first yield INTERNALERROR> File "/usr/local/lib/python2.7/site-packages/pytest_timeout.py", line 76, in pytest_runtest_protocol INTERNALERROR> timeout_setup(item) INTERNALERROR> File "/usr/local/lib/python2.7/site-packages/pytest_timeout.py", line 104, in timeout_setup INTERNALERROR> timeout, method = get_params(item) INTERNALERROR> File "/usr/local/lib/python2.7/site-packages/pytest_timeout.py", line 162, in get_params INTERNALERROR> timeout, method = _parse_marker(item.keywords['timeout']) INTERNALERROR> File "/usr/local/lib/python2.7/site-packages/pytest_timeout.py", line 178, in _parse_marker INTERNALERROR> if not marker.args and not marker.kwargs: INTERNALERROR> File "/usr/local/lib/python2.7/site-packages/_pytest/mark/structures.py", line 25, in warned INTERNALERROR> warnings.warn(warning, stacklevel=2) INTERNALERROR> RemovedInPytest4Warning: MarkInfo objects are deprecated as they contain merged marks which are hard to deal with correctly. INTERNALERROR> Please use node.get_closest_marker(name) or node.iter_markers(name). INTERNALERROR> Docs: https://docs.pytest.org/en/latest/mark.html#updating-code --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 9ad59cc664..07e1a900db 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -4,4 +4,4 @@ mock==1.0.1 pytest==2.9.1; python_version == '3.3' pytest==3.6.3; python_version > '3.3' pytest-cov==2.1.0 -pytest-timeout==1.2.1 +pytest-timeout==1.3.3 From 3fca75ffef3513a0ab8243c3d688b1dff384fd2e Mon Sep 17 00:00:00 2001 From: Frank Sachsenheim Date: Tue, 27 Nov 2018 01:06:02 +0100 Subject: [PATCH 0771/1301] Fixes return value models.containers.Container.exec_run.__doc__ Signed-off-by: Frank Sachsenheim --- docker/models/containers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/models/containers.py b/docker/models/containers.py index 9d6f2cc6af..34996cec61 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -172,10 +172,10 @@ def exec_run(self, cmd, stdout=True, stderr=True, stdin=False, tty=False, exit_code: (int): Exit code for the executed command or ``None`` if either ``stream```or ``socket`` is ``True``. - output: (generator or str): + output: (generator or bytes): If ``stream=True``, a generator yielding response chunks. If ``socket=True``, a socket object for the connection. - A string containing response data otherwise. + A bytestring containing response data otherwise. Raises: :py:class:`docker.errors.APIError` From 114630161a08b19489e464ad2c1a70ccccc9cc74 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 26 Nov 2018 17:33:30 -0800 Subject: [PATCH 0772/1301] Correctly handle longpath prefix in process_dockerfile when joining paths Signed-off-by: Joffrey F --- docker/api/build.py | 9 ++++- docker/constants.py | 1 + tests/unit/api_build_test.py | 64 +++++++++++++++++++++++++++++++++++- 3 files changed, 72 insertions(+), 2 deletions(-) diff --git a/docker/api/build.py b/docker/api/build.py index 0486dce62d..3a67ff8b28 100644 --- a/docker/api/build.py +++ b/docker/api/build.py @@ -339,7 +339,14 @@ def process_dockerfile(dockerfile, path): abs_dockerfile = dockerfile if not os.path.isabs(dockerfile): abs_dockerfile = os.path.join(path, dockerfile) - + if constants.IS_WINDOWS_PLATFORM and path.startswith( + constants.WINDOWS_LONGPATH_PREFIX): + abs_dockerfile = '{}{}'.format( + constants.WINDOWS_LONGPATH_PREFIX, + os.path.normpath( + abs_dockerfile[len(constants.WINDOWS_LONGPATH_PREFIX):] + ) + ) if (os.path.splitdrive(path)[0] != os.path.splitdrive(abs_dockerfile)[0] or os.path.relpath(abs_dockerfile, path).startswith('..')): # Dockerfile not in context - read data to insert into tar later diff --git a/docker/constants.py b/docker/constants.py index 7565a76889..1ab11ec051 100644 --- a/docker/constants.py +++ b/docker/constants.py @@ -14,6 +14,7 @@ 'is deprecated and non-functional. Please remove it.' IS_WINDOWS_PLATFORM = (sys.platform == 'win32') +WINDOWS_LONGPATH_PREFIX = '\\\\?\\' DEFAULT_USER_AGENT = "docker-sdk-python/{0}".format(version) DEFAULT_NUM_POOLS = 25 diff --git a/tests/unit/api_build_test.py b/tests/unit/api_build_test.py index a7f34fd3f2..59470caa5f 100644 --- a/tests/unit/api_build_test.py +++ b/tests/unit/api_build_test.py @@ -1,12 +1,16 @@ import gzip import io +import shutil import docker from docker import auth +from docker.api.build import process_dockerfile -from .api_test import BaseAPIClientTest, fake_request, url_prefix import pytest +from ..helpers import make_tree +from .api_test import BaseAPIClientTest, fake_request, url_prefix + class BuildTest(BaseAPIClientTest): def test_build_container(self): @@ -161,3 +165,61 @@ def test_set_auth_headers_with_dict_and_no_auth_configs(self): self.client._set_auth_headers(headers) assert headers == expected_headers + + @pytest.mark.skipif( + not docker.constants.IS_WINDOWS_PLATFORM, + reason='Windows-specific syntax') + def test_process_dockerfile_win_longpath_prefix(self): + dirs = [ + 'foo', 'foo/bar', 'baz', + ] + + files = [ + 'Dockerfile', 'foo/Dockerfile.foo', 'foo/bar/Dockerfile.bar', + 'baz/Dockerfile.baz', + ] + + base = make_tree(dirs, files) + self.addCleanup(shutil.rmtree, base) + + def pre(path): + return docker.constants.WINDOWS_LONGPATH_PREFIX + path + + assert process_dockerfile(None, pre(base)) == (None, None) + assert process_dockerfile('Dockerfile', pre(base)) == ( + 'Dockerfile', None + ) + assert process_dockerfile('foo/Dockerfile.foo', pre(base)) == ( + 'foo/Dockerfile.foo', None + ) + assert process_dockerfile( + '../Dockerfile', pre(base + '\\foo') + )[1] is not None + assert process_dockerfile( + '../baz/Dockerfile.baz', pre(base + '/baz') + ) == ('../baz/Dockerfile.baz', None) + + def test_process_dockerfile(self): + dirs = [ + 'foo', 'foo/bar', 'baz', + ] + + files = [ + 'Dockerfile', 'foo/Dockerfile.foo', 'foo/bar/Dockerfile.bar', + 'baz/Dockerfile.baz', + ] + + base = make_tree(dirs, files) + self.addCleanup(shutil.rmtree, base) + + assert process_dockerfile(None, base) == (None, None) + assert process_dockerfile('Dockerfile', base) == ('Dockerfile', None) + assert process_dockerfile('foo/Dockerfile.foo', base) == ( + 'foo/Dockerfile.foo', None + ) + assert process_dockerfile( + '../Dockerfile', base + '/foo' + )[1] is not None + assert process_dockerfile('../baz/Dockerfile.baz', base + '/baz') == ( + '../baz/Dockerfile.baz', None + ) From 5f157bbaca5ae62a5bb71547106beb6ef02bc485 Mon Sep 17 00:00:00 2001 From: Corentin Henry Date: Mon, 8 Oct 2018 11:02:31 -0700 Subject: [PATCH 0773/1301] implement stream demultiplexing for exec commands fixes https://github.com/docker/docker-py/issues/1952 Signed-off-by: Corentin Henry --- docker/api/client.py | 18 +++-- docker/api/container.py | 13 ++-- docker/api/exec_api.py | 13 ++-- docker/models/containers.py | 70 ++++++++++++++++++- docker/utils/socket.py | 89 +++++++++++++++++++++---- tests/integration/api_container_test.py | 5 +- tests/integration/api_exec_test.py | 5 +- tests/unit/api_test.py | 2 +- tests/unit/models_containers_test.py | 6 +- 9 files changed, 181 insertions(+), 40 deletions(-) diff --git a/docker/api/client.py b/docker/api/client.py index 197846d105..8a5a60b0b2 100644 --- a/docker/api/client.py +++ b/docker/api/client.py @@ -32,7 +32,7 @@ from ..tls import TLSConfig from ..transport import SSLAdapter, UnixAdapter from ..utils import utils, check_resource, update_headers, config -from ..utils.socket import frames_iter, socket_raw_iter +from ..utils.socket import frames_iter, consume_socket_output, demux_adaptor from ..utils.json_stream import json_stream try: from ..transport import NpipeAdapter @@ -381,19 +381,23 @@ def _stream_raw_result(self, response, chunk_size=1, decode=True): for out in response.iter_content(chunk_size, decode): yield out - def _read_from_socket(self, response, stream, tty=False): + def _read_from_socket(self, response, stream, tty=True, demux=False): socket = self._get_raw_response_socket(response) - gen = None - if tty is False: - gen = frames_iter(socket) + gen = frames_iter(socket, tty) + + if demux: + # The generator will output tuples (stdout, stderr) + gen = (demux_adaptor(*frame) for frame in gen) else: - gen = socket_raw_iter(socket) + # The generator will output strings + gen = (data for (_, data) in gen) if stream: return gen else: - return six.binary_type().join(gen) + # Wait for all the frames, concatenate them, and return the result + return consume_socket_output(gen, demux=demux) def _disable_socket_timeout(self, socket): """ Depending on the combination of python version and whether we're diff --git a/docker/api/container.py b/docker/api/container.py index fce73af640..ab3b1cf410 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -13,7 +13,7 @@ class ContainerApiMixin(object): @utils.check_resource('container') def attach(self, container, stdout=True, stderr=True, - stream=False, logs=False): + stream=False, logs=False, demux=False): """ Attach to a container. @@ -28,11 +28,15 @@ def attach(self, container, stdout=True, stderr=True, stream (bool): Return container output progressively as an iterator of strings, rather than a single string. logs (bool): Include the container's previous output. + demux (bool): Keep stdout and stderr separate. Returns: - By default, the container's output as a single string. + By default, the container's output as a single string (two if + ``demux=True``: one for stdout and one for stderr). - If ``stream=True``, an iterator of output strings. + If ``stream=True``, an iterator of output strings. If + ``demux=True``, two iterators are returned: one for stdout and one + for stderr. Raises: :py:class:`docker.errors.APIError` @@ -54,8 +58,7 @@ def attach(self, container, stdout=True, stderr=True, response = self._post(u, headers=headers, params=params, stream=True) output = self._read_from_socket( - response, stream, self._check_is_tty(container) - ) + response, stream, self._check_is_tty(container), demux=demux) if stream: return CancellableStream(output, response) diff --git a/docker/api/exec_api.py b/docker/api/exec_api.py index 986d87f21c..3950991972 100644 --- a/docker/api/exec_api.py +++ b/docker/api/exec_api.py @@ -118,7 +118,7 @@ def exec_resize(self, exec_id, height=None, width=None): @utils.check_resource('exec_id') def exec_start(self, exec_id, detach=False, tty=False, stream=False, - socket=False): + socket=False, demux=False): """ Start a previously set up exec instance. @@ -130,11 +130,14 @@ def exec_start(self, exec_id, detach=False, tty=False, stream=False, stream (bool): Stream response data. Default: False socket (bool): Return the connection socket to allow custom read/write operations. + demux (bool): Separate return stdin, stdout and stderr separately Returns: - (generator or str): If ``stream=True``, a generator yielding - response chunks. If ``socket=True``, a socket object for the - connection. A string containing response data otherwise. + + (generator or str or tuple): If ``stream=True``, a generator + yielding response chunks. If ``socket=True``, a socket object for + the connection. A string containing response data otherwise. If + ``demux=True``, stdin, stdout and stderr are separated. Raises: :py:class:`docker.errors.APIError` @@ -162,4 +165,4 @@ def exec_start(self, exec_id, detach=False, tty=False, stream=False, return self._result(res) if socket: return self._get_raw_response_socket(res) - return self._read_from_socket(res, stream, tty) + return self._read_from_socket(res, stream, tty=tty, demux=demux) diff --git a/docker/models/containers.py b/docker/models/containers.py index 9d6f2cc6af..15cc212531 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -144,7 +144,7 @@ def diff(self): def exec_run(self, cmd, stdout=True, stderr=True, stdin=False, tty=False, privileged=False, user='', detach=False, stream=False, - socket=False, environment=None, workdir=None): + socket=False, environment=None, workdir=None, demux=False): """ Run a command inside this container. Similar to ``docker exec``. @@ -166,6 +166,7 @@ def exec_run(self, cmd, stdout=True, stderr=True, stdin=False, tty=False, the following format ``["PASSWORD=xxx"]`` or ``{"PASSWORD": "xxx"}``. workdir (str): Path to working directory for this exec session + demux (bool): Return stdout and stderr separately Returns: (ExecResult): A tuple of (exit_code, output) @@ -180,6 +181,70 @@ def exec_run(self, cmd, stdout=True, stderr=True, stdin=False, tty=False, Raises: :py:class:`docker.errors.APIError` If the server returns an error. + + Example: + + Create a container that runs in the background + + >>> client = docker.from_env() + >>> container = client.containers.run( + ... 'bfirsh/reticulate-splines', detach=True) + + Prepare the command we are going to use. It prints "hello stdout" + in `stdout`, followed by "hello stderr" in `stderr`: + + >>> cmd = '/bin/sh -c "echo hello stdout ; echo hello stderr >&2"' + + We'll run this command with all four the combinations of ``stream`` + and ``demux``. + + With ``stream=False`` and ``demux=False``, the output is a string + that contains both the `stdout` and the `stderr` output: + + >>> res = container.exec_run(cmd, stream=False, demux=False) + >>> res.output + b'hello stderr\nhello stdout\n' + + With ``stream=True``, and ``demux=False``, the output is a + generator that yields strings containing the output of both + `stdout` and `stderr`: + + >>> res = container.exec_run(cmd, stream=True, demux=False) + >>> next(res.output) + b'hello stdout\n' + >>> next(res.output) + b'hello stderr\n' + >>> next(res.output) + Traceback (most recent call last): + File "", line 1, in + StopIteration + + With ``stream=True`` and ``demux=True``, the generator now + separates the streams, and yield tuples + ``(stdout, stderr)``: + + >>> res = container.exec_run(cmd, stream=True, demux=True) + >>> next(res.output) + (b'hello stdout\n', None) + >>> next(res.output) + (None, b'hello stderr\n') + >>> next(res.output) + Traceback (most recent call last): + File "", line 1, in + StopIteration + + Finally, with ``stream=False`` and ``demux=True``, the whole output + is returned, but the streams are still separated: + + >>> res = container.exec_run(cmd, stream=True, demux=True) + >>> next(res.output) + (b'hello stdout\n', None) + >>> next(res.output) + (None, b'hello stderr\n') + >>> next(res.output) + Traceback (most recent call last): + File "", line 1, in + StopIteration """ resp = self.client.api.exec_create( self.id, cmd, stdout=stdout, stderr=stderr, stdin=stdin, tty=tty, @@ -187,7 +252,8 @@ def exec_run(self, cmd, stdout=True, stderr=True, stdin=False, tty=False, workdir=workdir ) exec_output = self.client.api.exec_start( - resp['Id'], detach=detach, tty=tty, stream=stream, socket=socket + resp['Id'], detach=detach, tty=tty, stream=stream, socket=socket, + demux=demux ) if socket or stream: return ExecResult(None, exec_output) diff --git a/docker/utils/socket.py b/docker/utils/socket.py index 7b96d4fce6..fe4a33266c 100644 --- a/docker/utils/socket.py +++ b/docker/utils/socket.py @@ -12,6 +12,10 @@ NpipeSocket = type(None) +STDOUT = 1 +STDERR = 2 + + class SocketError(Exception): pass @@ -51,28 +55,43 @@ def read_exactly(socket, n): return data -def next_frame_size(socket): +def next_frame_header(socket): """ - Returns the size of the next frame of data waiting to be read from socket, - according to the protocol defined here: + Returns the stream and size of the next frame of data waiting to be read + from socket, according to the protocol defined here: - https://docs.docker.com/engine/reference/api/docker_remote_api_v1.24/#/attach-to-a-container + https://docs.docker.com/engine/api/v1.24/#attach-to-a-container """ try: data = read_exactly(socket, 8) except SocketError: - return -1 + return (-1, -1) + + stream, actual = struct.unpack('>BxxxL', data) + return (stream, actual) + - _, actual = struct.unpack('>BxxxL', data) - return actual +def frames_iter(socket, tty): + """ + Return a generator of frames read from socket. A frame is a tuple where + the first item is the stream number and the second item is a chunk of data. + + If the tty setting is enabled, the streams are multiplexed into the stdout + stream. + """ + if tty: + return ((STDOUT, frame) for frame in frames_iter_tty(socket)) + else: + return frames_iter_no_tty(socket) -def frames_iter(socket): +def frames_iter_no_tty(socket): """ - Returns a generator of frames read from socket + Returns a generator of data read from the socket when the tty setting is + not enabled. """ while True: - n = next_frame_size(socket) + (stream, n) = next_frame_header(socket) if n < 0: break while n > 0: @@ -84,13 +103,13 @@ def frames_iter(socket): # We have reached EOF return n -= data_length - yield result + yield (stream, result) -def socket_raw_iter(socket): +def frames_iter_tty(socket): """ - Returns a generator of data read from the socket. - This is used for non-multiplexed streams. + Return a generator of data read from the socket when the tty setting is + enabled. """ while True: result = read(socket) @@ -98,3 +117,45 @@ def socket_raw_iter(socket): # We have reached EOF return yield result + + +def consume_socket_output(frames, demux=False): + """ + Iterate through frames read from the socket and return the result. + + Args: + + demux (bool): + If False, stdout and stderr are multiplexed, and the result is the + concatenation of all the frames. If True, the streams are + demultiplexed, and the result is a 2-tuple where each item is the + concatenation of frames belonging to the same stream. + """ + if demux is False: + # If the streams are multiplexed, the generator returns strings, that + # we just need to concatenate. + return six.binary_type().join(frames) + + # If the streams are demultiplexed, the generator returns tuples + # (stdin, stdout, stderr) + out = [six.binary_type(), six.binary_type()] + for frame in frames: + for stream_id in [STDOUT, STDERR]: + # It is guaranteed that for each frame, one and only one stream + # is not None. + if frame[stream_id] is not None: + out[stream_id] += frame[stream_id] + return tuple(out) + + +def demux_adaptor(stream_id, data): + """ + Utility to demultiplex stdout and stderr when reading frames from the + socket. + """ + if stream_id == STDOUT: + return (data, None) + elif stream_id == STDERR: + return (None, data) + else: + raise ValueError('{0} is not a valid stream'.format(stream_id)) diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index 02f3603374..83df3424a9 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -7,7 +7,7 @@ import docker from docker.constants import IS_WINDOWS_PLATFORM -from docker.utils.socket import next_frame_size +from docker.utils.socket import next_frame_header from docker.utils.socket import read_exactly import pytest @@ -1242,7 +1242,8 @@ def test_run_container_reading_socket(self): self.client.start(container) - next_size = next_frame_size(pty_stdout) + (stream, next_size) = next_frame_header(pty_stdout) + assert stream == 1 # correspond to stdout assert next_size == len(line) data = read_exactly(pty_stdout, next_size) assert data.decode('utf-8') == line diff --git a/tests/integration/api_exec_test.py b/tests/integration/api_exec_test.py index 1a5a4e5472..ac64af7724 100644 --- a/tests/integration/api_exec_test.py +++ b/tests/integration/api_exec_test.py @@ -1,4 +1,4 @@ -from docker.utils.socket import next_frame_size +from docker.utils.socket import next_frame_header from docker.utils.socket import read_exactly from .base import BaseAPIIntegrationTest, BUSYBOX @@ -91,7 +91,8 @@ def test_exec_start_socket(self): socket = self.client.exec_start(exec_id, socket=True) self.addCleanup(socket.close) - next_size = next_frame_size(socket) + (stream, next_size) = next_frame_header(socket) + assert stream == 1 # stdout (0 = stdin, 1 = stdout, 2 = stderr) assert next_size == len(line) data = read_exactly(socket, next_size) assert data.decode('utf-8') == line diff --git a/tests/unit/api_test.py b/tests/unit/api_test.py index af2bb1c202..ccddbb16de 100644 --- a/tests/unit/api_test.py +++ b/tests/unit/api_test.py @@ -83,7 +83,7 @@ def fake_delete(self, url, *args, **kwargs): return fake_request('DELETE', url, *args, **kwargs) -def fake_read_from_socket(self, response, stream, tty=False): +def fake_read_from_socket(self, response, stream, tty=False, demux=False): return six.binary_type() diff --git a/tests/unit/models_containers_test.py b/tests/unit/models_containers_test.py index 22dd241064..24f6316acf 100644 --- a/tests/unit/models_containers_test.py +++ b/tests/unit/models_containers_test.py @@ -417,7 +417,8 @@ def test_exec_run(self): workdir=None ) client.api.exec_start.assert_called_with( - FAKE_EXEC_ID, detach=False, tty=False, stream=True, socket=False + FAKE_EXEC_ID, detach=False, tty=False, stream=True, socket=False, + demux=False, ) def test_exec_run_failure(self): @@ -430,7 +431,8 @@ def test_exec_run_failure(self): workdir=None ) client.api.exec_start.assert_called_with( - FAKE_EXEC_ID, detach=False, tty=False, stream=False, socket=False + FAKE_EXEC_ID, detach=False, tty=False, stream=False, socket=False, + demux=False, ) def test_export(self): From a74d546864b64ba03dce1e3407d3b0cd5badee9f Mon Sep 17 00:00:00 2001 From: adw1n Date: Mon, 3 Sep 2018 05:54:12 +0200 Subject: [PATCH 0774/1301] Fix pulling images with `stream=True` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pulling an image with option `stream=True` like this: ``` client.api.pull('docker.io/user/repo_name', tag='latest', stream=True) ``` without consuming the generator oftentimes results in premature drop of the connection. Docker daemon tries to send progress of pulling the image to the client, but it encounters an error (broken pipe) and therefore cancells the pull action: ``` Thread 1 "dockerd-dev" received signal SIGPIPE, Broken pipe. ERRO[2018-09-03T05:12:35.746497638+02:00] Not continuing with pull after error: context canceled ``` As described in issue #2116, even though client receives response with status code 200, image is not pulled. Closes #2116 Signed-off-by: Przemysław Adamek --- docker/api/image.py | 3 ++- docker/models/images.py | 1 + tests/unit/models_containers_test.py | 3 ++- tests/unit/models_images_test.py | 6 ++++-- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/docker/api/image.py b/docker/api/image.py index a9f801e93b..d3fed5c0cc 100644 --- a/docker/api/image.py +++ b/docker/api/image.py @@ -334,7 +334,8 @@ def pull(self, repository, tag=None, stream=False, auth_config=None, Args: repository (str): The repository to pull tag (str): The tag to pull - stream (bool): Stream the output as a generator + stream (bool): Stream the output as a generator. Make sure to + consume the generator, otherwise pull might get cancelled. auth_config (dict): Override the credentials that :py:meth:`~docker.api.daemon.DaemonApiMixin.login` has set for this request. ``auth_config`` should contain the ``username`` diff --git a/docker/models/images.py b/docker/models/images.py index 4578c0bd89..f8b842a943 100644 --- a/docker/models/images.py +++ b/docker/models/images.py @@ -425,6 +425,7 @@ def pull(self, repository, tag=None, **kwargs): if not tag: repository, tag = parse_repository_tag(repository) + kwargs['stream'] = False self.client.api.pull(repository, tag=tag, **kwargs) if tag: return self.get('{0}{2}{1}'.format( diff --git a/tests/unit/models_containers_test.py b/tests/unit/models_containers_test.py index 22dd241064..957035af0a 100644 --- a/tests/unit/models_containers_test.py +++ b/tests/unit/models_containers_test.py @@ -232,7 +232,8 @@ def test_run_pull(self): container = client.containers.run('alpine', 'sleep 300', detach=True) assert container.id == FAKE_CONTAINER_ID - client.api.pull.assert_called_with('alpine', platform=None, tag=None) + client.api.pull.assert_called_with('alpine', platform=None, tag=None, + stream=False) def test_run_with_error(self): client = make_fake_client() diff --git a/tests/unit/models_images_test.py b/tests/unit/models_images_test.py index 67832795fe..ef81a1599d 100644 --- a/tests/unit/models_images_test.py +++ b/tests/unit/models_images_test.py @@ -43,7 +43,8 @@ def test_load(self): def test_pull(self): client = make_fake_client() image = client.images.pull('test_image:latest') - client.api.pull.assert_called_with('test_image', tag='latest') + client.api.pull.assert_called_with('test_image', tag='latest', + stream=False) client.api.inspect_image.assert_called_with('test_image:latest') assert isinstance(image, Image) assert image.id == FAKE_IMAGE_ID @@ -51,7 +52,8 @@ def test_pull(self): def test_pull_multiple(self): client = make_fake_client() images = client.images.pull('test_image') - client.api.pull.assert_called_with('test_image', tag=None) + client.api.pull.assert_called_with('test_image', tag=None, + stream=False) client.api.images.assert_called_with( all=False, name='test_image', filters=None ) From c53423f11851a69fa18783b5b0b801a24d8d2f82 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 28 Nov 2018 11:24:21 -0800 Subject: [PATCH 0775/1301] Update DockerClient.images.pull to always stream response Also raise a warning when users attempt to specify the "stream" parameter Signed-off-by: Joffrey F --- docker/models/images.py | 18 ++++++++++++++++-- tests/unit/models_containers_test.py | 5 +++-- tests/unit/models_images_test.py | 24 +++++++++++++++++++----- 3 files changed, 38 insertions(+), 9 deletions(-) diff --git a/docker/models/images.py b/docker/models/images.py index f8b842a943..30e86f109e 100644 --- a/docker/models/images.py +++ b/docker/models/images.py @@ -1,5 +1,6 @@ import itertools import re +import warnings import six @@ -425,8 +426,21 @@ def pull(self, repository, tag=None, **kwargs): if not tag: repository, tag = parse_repository_tag(repository) - kwargs['stream'] = False - self.client.api.pull(repository, tag=tag, **kwargs) + if 'stream' in kwargs: + warnings.warn( + '`stream` is not a valid parameter for this method' + ' and will be overridden' + ) + del kwargs['stream'] + + pull_log = self.client.api.pull( + repository, tag=tag, stream=True, **kwargs + ) + for _ in pull_log: + # We don't do anything with the logs, but we need + # to keep the connection alive and wait for the image + # to be pulled. + pass if tag: return self.get('{0}{2}{1}'.format( repository, tag, '@' if tag.startswith('sha256:') else ':' diff --git a/tests/unit/models_containers_test.py b/tests/unit/models_containers_test.py index 957035af0a..39e409e4bf 100644 --- a/tests/unit/models_containers_test.py +++ b/tests/unit/models_containers_test.py @@ -232,8 +232,9 @@ def test_run_pull(self): container = client.containers.run('alpine', 'sleep 300', detach=True) assert container.id == FAKE_CONTAINER_ID - client.api.pull.assert_called_with('alpine', platform=None, tag=None, - stream=False) + client.api.pull.assert_called_with( + 'alpine', platform=None, tag=None, stream=True + ) def test_run_with_error(self): client = make_fake_client() diff --git a/tests/unit/models_images_test.py b/tests/unit/models_images_test.py index ef81a1599d..fd894ab71d 100644 --- a/tests/unit/models_images_test.py +++ b/tests/unit/models_images_test.py @@ -1,6 +1,8 @@ +import unittest +import warnings + from docker.constants import DEFAULT_DATA_CHUNK_SIZE from docker.models.images import Image -import unittest from .fake_api import FAKE_IMAGE_ID from .fake_api_client import make_fake_client @@ -43,8 +45,9 @@ def test_load(self): def test_pull(self): client = make_fake_client() image = client.images.pull('test_image:latest') - client.api.pull.assert_called_with('test_image', tag='latest', - stream=False) + client.api.pull.assert_called_with( + 'test_image', tag='latest', stream=True + ) client.api.inspect_image.assert_called_with('test_image:latest') assert isinstance(image, Image) assert image.id == FAKE_IMAGE_ID @@ -52,8 +55,9 @@ def test_pull(self): def test_pull_multiple(self): client = make_fake_client() images = client.images.pull('test_image') - client.api.pull.assert_called_with('test_image', tag=None, - stream=False) + client.api.pull.assert_called_with( + 'test_image', tag=None, stream=True + ) client.api.images.assert_called_with( all=False, name='test_image', filters=None ) @@ -63,6 +67,16 @@ def test_pull_multiple(self): assert isinstance(image, Image) assert image.id == FAKE_IMAGE_ID + def test_pull_with_stream_param(self): + client = make_fake_client() + with warnings.catch_warnings(record=True) as w: + client.images.pull('test_image', stream=True) + + assert len(w) == 1 + assert str(w[0].message).startswith( + '`stream` is not a valid parameter' + ) + def test_push(self): client = make_fake_client() client.images.push('foobar', insecure_registry=True) From a422924a5eaaadef80dc3cc6ccf6e5a9afeb31e3 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 29 Oct 2018 14:46:27 -0700 Subject: [PATCH 0776/1301] Bump requests dependency in requirements.txt (CVE-2018-18074) Signed-off-by: Joffrey F --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c46a021e2c..c0ce59aeb1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,7 +14,7 @@ pyOpenSSL==18.0.0 pyparsing==2.2.0 pypiwin32==219; sys_platform == 'win32' and python_version < '3.6' pypiwin32==223; sys_platform == 'win32' and python_version >= '3.6' -requests==2.14.2 +requests==2.20.0 six==1.10.0 websocket-client==0.40.0 urllib3==1.21.1; python_version == '3.3' \ No newline at end of file From 26daa6194a934d11bc92490582e722e618ab9853 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 31 Oct 2018 18:05:26 -0700 Subject: [PATCH 0777/1301] Add xfail to ignore 18.09 beta bug Signed-off-by: Joffrey F --- tests/integration/api_build_test.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/integration/api_build_test.py b/tests/integration/api_build_test.py index baaf33e3d9..bad411beec 100644 --- a/tests/integration/api_build_test.py +++ b/tests/integration/api_build_test.py @@ -540,6 +540,11 @@ def test_build_in_context_abs_dockerfile(self): ) == sorted(lsdata) @requires_api_version('1.31') + @pytest.mark.xfail( + True, + reason='Currently fails on 18.09: ' + 'https://github.com/moby/moby/issues/37920' + ) def test_prune_builds(self): prune_result = self.client.prune_builds() assert 'SpaceReclaimed' in prune_result From 2e4a45358ebf827ca18b06e5a5193fd2d40b35c4 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 1 Nov 2018 15:23:21 -0700 Subject: [PATCH 0778/1301] Update version detection script for CI Signed-off-by: Joffrey F --- scripts/versions.py | 39 ++++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/scripts/versions.py b/scripts/versions.py index 77aaf4f182..7212195dec 100644 --- a/scripts/versions.py +++ b/scripts/versions.py @@ -11,23 +11,24 @@ 'test' ] +STAGES = ['tp', 'beta', 'rc'] -class Version(namedtuple('_Version', 'major minor patch rc edition')): + +class Version(namedtuple('_Version', 'major minor patch stage edition')): @classmethod def parse(cls, version): edition = None version = version.lstrip('v') - version, _, rc = version.partition('-') - if rc: - if 'rc' not in rc: - edition = rc - rc = None - elif '-' in rc: - edition, rc = rc.split('-') - + version, _, stage = version.partition('-') + if stage: + if not any(marker in stage for marker in STAGES): + edition = stage + stage = None + elif '-' in stage: + edition, stage = stage.split('-') major, minor, patch = version.split('.', 3) - return cls(major, minor, patch, rc, edition) + return cls(major, minor, patch, stage, edition) @property def major_minor(self): @@ -38,14 +39,22 @@ def order(self): """Return a representation that allows this object to be sorted correctly with the default comparator. """ - # rc releases should appear before official releases - rc = (0, self.rc) if self.rc else (1, ) - return (int(self.major), int(self.minor), int(self.patch)) + rc + # non-GA releases should appear before GA releases + # Order: tp -> beta -> rc -> GA + if self.stage: + for st in STAGES: + if st in self.stage: + stage = (STAGES.index(st), self.stage) + break + else: + stage = (len(STAGES),) + + return (int(self.major), int(self.minor), int(self.patch)) + stage def __str__(self): - rc = '-{}'.format(self.rc) if self.rc else '' + stage = '-{}'.format(self.stage) if self.stage else '' edition = '-{}'.format(self.edition) if self.edition else '' - return '.'.join(map(str, self[:3])) + edition + rc + return '.'.join(map(str, self[:3])) + edition + stage def main(): From e4368fb0f3b99df7df76cd0e655e7bbc783eb293 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 31 Oct 2018 17:04:05 -0700 Subject: [PATCH 0779/1301] Add paramiko requirement for SSH transport Signed-off-by: Joffrey F --- requirements.txt | 1 + setup.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/requirements.txt b/requirements.txt index c0ce59aeb1..d13e9d6cad 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,6 +9,7 @@ enum34==1.1.6 idna==2.5 ipaddress==1.0.18 packaging==16.8 +paramiko==2.4.2 pycparser==2.17 pyOpenSSL==18.0.0 pyparsing==2.2.0 diff --git a/setup.py b/setup.py index 390783d50f..3ad572b3ec 100644 --- a/setup.py +++ b/setup.py @@ -42,6 +42,9 @@ # 'requests[security] >= 2.5.2, != 2.11.0, != 2.12.2' 'tls': ['pyOpenSSL>=17.5.0', 'cryptography>=1.3.4', 'idna>=2.0.0'], + # Only required when connecting using the ssh:// protocol + 'ssh': ['paramiko>=2.4.2'], + } version = None From 956474a1bf1426cb808eae69f4b811e5599a820d Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 31 Oct 2018 17:06:23 -0700 Subject: [PATCH 0780/1301] Add support for SSH protocol in base_url Signed-off-by: Joffrey F --- docker/api/client.py | 19 ++++++ docker/transport/__init__.py | 5 ++ docker/transport/sshconn.py | 110 +++++++++++++++++++++++++++++++++++ docker/utils/utils.py | 16 +++-- 4 files changed, 145 insertions(+), 5 deletions(-) create mode 100644 docker/transport/sshconn.py diff --git a/docker/api/client.py b/docker/api/client.py index 91da1c893b..197846d105 100644 --- a/docker/api/client.py +++ b/docker/api/client.py @@ -39,6 +39,11 @@ except ImportError: pass +try: + from ..transport import SSHAdapter +except ImportError: + pass + class APIClient( requests.Session, @@ -141,6 +146,18 @@ def __init__(self, base_url=None, version=None, ) self.mount('http+docker://', self._custom_adapter) self.base_url = 'http+docker://localnpipe' + elif base_url.startswith('ssh://'): + try: + self._custom_adapter = SSHAdapter( + base_url, timeout, pool_connections=num_pools + ) + except NameError: + raise DockerException( + 'Install paramiko package to enable ssh:// support' + ) + self.mount('http+docker://ssh', self._custom_adapter) + self._unmount('http://', 'https://') + self.base_url = 'http+docker://ssh' else: # Use SSLAdapter for the ability to specify SSL version if isinstance(tls, TLSConfig): @@ -279,6 +296,8 @@ def _get_raw_response_socket(self, response): self._raise_for_status(response) if self.base_url == "http+docker://localnpipe": sock = response.raw._fp.fp.raw.sock + elif self.base_url.startswith('http+docker://ssh'): + sock = response.raw._fp.fp.channel elif six.PY3: sock = response.raw._fp.fp.raw if self.base_url.startswith("https://"): diff --git a/docker/transport/__init__.py b/docker/transport/__init__.py index abbee182fc..d2cf2a7af3 100644 --- a/docker/transport/__init__.py +++ b/docker/transport/__init__.py @@ -6,3 +6,8 @@ from .npipesocket import NpipeSocket except ImportError: pass + +try: + from .sshconn import SSHAdapter +except ImportError: + pass diff --git a/docker/transport/sshconn.py b/docker/transport/sshconn.py new file mode 100644 index 0000000000..6c9c1196d2 --- /dev/null +++ b/docker/transport/sshconn.py @@ -0,0 +1,110 @@ +import urllib.parse + +import paramiko +import requests.adapters +import six + + +from .. import constants + +if six.PY3: + import http.client as httplib +else: + import httplib + +try: + import requests.packages.urllib3 as urllib3 +except ImportError: + import urllib3 + +RecentlyUsedContainer = urllib3._collections.RecentlyUsedContainer + + +class SSHConnection(httplib.HTTPConnection, object): + def __init__(self, ssh_transport, timeout=60): + super(SSHConnection, self).__init__( + 'localhost', timeout=timeout + ) + self.ssh_transport = ssh_transport + self.timeout = timeout + + def connect(self): + sock = self.ssh_transport.open_session() + sock.settimeout(self.timeout) + sock.exec_command('docker system dial-stdio') + self.sock = sock + + +class SSHConnectionPool(urllib3.connectionpool.HTTPConnectionPool): + scheme = 'ssh' + + def __init__(self, ssh_client, timeout=60, maxsize=10): + super(SSHConnectionPool, self).__init__( + 'localhost', timeout=timeout, maxsize=maxsize + ) + self.ssh_transport = ssh_client.get_transport() + self.timeout = timeout + + def _new_conn(self): + return SSHConnection(self.ssh_transport, self.timeout) + + # When re-using connections, urllib3 calls fileno() on our + # SSH channel instance, quickly overloading our fd limit. To avoid this, + # we override _get_conn + def _get_conn(self, timeout): + conn = None + try: + conn = self.pool.get(block=self.block, timeout=timeout) + + except AttributeError: # self.pool is None + raise urllib3.exceptions.ClosedPoolError(self, "Pool is closed.") + + except six.moves.queue.Empty: + if self.block: + raise urllib3.exceptions.EmptyPoolError( + self, + "Pool reached maximum size and no more " + "connections are allowed." + ) + pass # Oh well, we'll create a new connection then + + return conn or self._new_conn() + + +class SSHAdapter(requests.adapters.HTTPAdapter): + + __attrs__ = requests.adapters.HTTPAdapter.__attrs__ + [ + 'pools', 'timeout', 'ssh_client', + ] + + def __init__(self, base_url, timeout=60, + pool_connections=constants.DEFAULT_NUM_POOLS): + self.ssh_client = paramiko.SSHClient() + self.ssh_client.load_system_host_keys() + + parsed = urllib.parse.urlparse(base_url) + self.ssh_client.connect( + parsed.hostname, parsed.port, parsed.username, + ) + self.timeout = timeout + self.pools = RecentlyUsedContainer( + pool_connections, dispose_func=lambda p: p.close() + ) + super(SSHAdapter, self).__init__() + + def get_connection(self, url, proxies=None): + with self.pools.lock: + pool = self.pools.get(url) + if pool: + return pool + + pool = SSHConnectionPool( + self.ssh_client, self.timeout + ) + self.pools[url] = pool + + return pool + + def close(self): + self.pools.clear() + self.ssh_client.close() diff --git a/docker/utils/utils.py b/docker/utils/utils.py index fe3b9a5767..f8f712349d 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -250,6 +250,9 @@ def parse_host(addr, is_win32=False, tls=False): addr = addr[8:] elif addr.startswith('fd://'): raise errors.DockerException("fd protocol is not implemented") + elif addr.startswith('ssh://'): + proto = 'ssh' + addr = addr[6:] else: if "://" in addr: raise errors.DockerException( @@ -257,17 +260,20 @@ def parse_host(addr, is_win32=False, tls=False): ) proto = "https" if tls else "http" - if proto in ("http", "https"): + if proto in ("http", "https", "ssh"): address_parts = addr.split('/', 1) host = address_parts[0] if len(address_parts) == 2: path = '/' + address_parts[1] host, port = splitnport(host) - if port is None: - raise errors.DockerException( - "Invalid port: {0}".format(addr) - ) + if port is None or port < 0: + if proto == 'ssh': + port = 22 + else: + raise errors.DockerException( + "Invalid port: {0}".format(addr) + ) if not host: host = DEFAULT_HTTP_HOST From 2b2d711a5ba8be5cebe5913870c4dea1b9498af1 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 31 Oct 2018 17:07:01 -0700 Subject: [PATCH 0781/1301] Remove misleading fileno method from NpipeSocket class Signed-off-by: Joffrey F --- docker/transport/npipesocket.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docker/transport/npipesocket.py b/docker/transport/npipesocket.py index c04b39dd66..ef02031640 100644 --- a/docker/transport/npipesocket.py +++ b/docker/transport/npipesocket.py @@ -87,10 +87,6 @@ def detach(self): def dup(self): return NpipeSocket(self._handle) - @check_closed - def fileno(self): - return int(self._handle) - def getpeername(self): return self._address From c9edc9c748ca2eea97ed130fd526aeb4eabae80d Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 31 Oct 2018 17:08:52 -0700 Subject: [PATCH 0782/1301] Update tests for ssh protocol compatibility Signed-off-by: Joffrey F --- tests/helpers.py | 4 ++++ tests/integration/api_container_test.py | 1 + 2 files changed, 5 insertions(+) diff --git a/tests/helpers.py b/tests/helpers.py index b36d6d786f..f912bd8d43 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -10,6 +10,7 @@ import socket import docker +import paramiko import pytest @@ -121,6 +122,9 @@ def assert_cat_socket_detached_with_keys(sock, inputs): if getattr(sock, 'family', -9) == getattr(socket, 'AF_UNIX', -1): with pytest.raises(socket.error): sock.sendall(b'make sure the socket is closed\n') + elif isinstance(sock, paramiko.Channel): + with pytest.raises(OSError): + sock.sendall(b'make sure the socket is closed\n') else: sock.sendall(b"make sure the socket is closed\n") data = sock.recv(128) diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index 6ce846bb20..249ef7cfcc 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -1255,6 +1255,7 @@ def test_attach_no_stream(self): assert output == 'hello\n'.encode(encoding='ascii') @pytest.mark.timeout(5) + @pytest.mark.xfail(True, reason='Cancellable events broken over SSH') def test_attach_stream_and_cancel(self): container = self.client.create_container( BUSYBOX, 'sh -c "echo hello && sleep 60"', From 8c86aa90b1353bb9aa74f2a6ff19f9913104f42b Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 31 Oct 2018 17:09:33 -0700 Subject: [PATCH 0783/1301] Update tests to properly dispose of client instances in tearDown Signed-off-by: Joffrey F --- tests/integration/api_network_test.py | 2 +- tests/integration/api_plugin_test.py | 16 +++--- tests/integration/api_swarm_test.py | 3 +- tests/integration/base.py | 73 ++++++++++++++------------- 4 files changed, 48 insertions(+), 46 deletions(-) diff --git a/tests/integration/api_network_test.py b/tests/integration/api_network_test.py index b6726d0242..db37cbd974 100644 --- a/tests/integration/api_network_test.py +++ b/tests/integration/api_network_test.py @@ -8,8 +8,8 @@ class TestNetworks(BaseAPIIntegrationTest): def tearDown(self): - super(TestNetworks, self).tearDown() self.client.leave_swarm(force=True) + super(TestNetworks, self).tearDown() def create_network(self, *args, **kwargs): net_name = random_name() diff --git a/tests/integration/api_plugin_test.py b/tests/integration/api_plugin_test.py index 1150b0957a..38f9d12dad 100644 --- a/tests/integration/api_plugin_test.py +++ b/tests/integration/api_plugin_test.py @@ -3,7 +3,7 @@ import docker import pytest -from .base import BaseAPIIntegrationTest, TEST_API_VERSION +from .base import BaseAPIIntegrationTest from ..helpers import requires_api_version SSHFS = 'vieux/sshfs:latest' @@ -13,27 +13,27 @@ class PluginTest(BaseAPIIntegrationTest): @classmethod def teardown_class(cls): - c = docker.APIClient( - version=TEST_API_VERSION, timeout=60, - **docker.utils.kwargs_from_env() - ) + client = cls.get_client_instance() try: - c.remove_plugin(SSHFS, force=True) + client.remove_plugin(SSHFS, force=True) except docker.errors.APIError: pass def teardown_method(self, method): + client = self.get_client_instance() try: - self.client.disable_plugin(SSHFS) + client.disable_plugin(SSHFS) except docker.errors.APIError: pass for p in self.tmp_plugins: try: - self.client.remove_plugin(p, force=True) + client.remove_plugin(p, force=True) except docker.errors.APIError: pass + client.close() + def ensure_plugin_installed(self, plugin_name): try: return self.client.inspect_plugin(plugin_name) diff --git a/tests/integration/api_swarm_test.py b/tests/integration/api_swarm_test.py index dbf3786eb0..b58dabc639 100644 --- a/tests/integration/api_swarm_test.py +++ b/tests/integration/api_swarm_test.py @@ -13,14 +13,13 @@ def setUp(self): self._unlock_key = None def tearDown(self): - super(SwarmTest, self).tearDown() try: if self._unlock_key: self.client.unlock_swarm(self._unlock_key) except docker.errors.APIError: pass - force_leave_swarm(self.client) + super(SwarmTest, self).tearDown() @requires_api_version('1.24') def test_init_swarm_simple(self): diff --git a/tests/integration/base.py b/tests/integration/base.py index 56c23ed4af..262769de40 100644 --- a/tests/integration/base.py +++ b/tests/integration/base.py @@ -29,41 +29,44 @@ def setUp(self): def tearDown(self): client = docker.from_env(version=TEST_API_VERSION) - for img in self.tmp_imgs: - try: - client.api.remove_image(img) - except docker.errors.APIError: - pass - for container in self.tmp_containers: - try: - client.api.remove_container(container, force=True, v=True) - except docker.errors.APIError: - pass - for network in self.tmp_networks: - try: - client.api.remove_network(network) - except docker.errors.APIError: - pass - for volume in self.tmp_volumes: - try: - client.api.remove_volume(volume) - except docker.errors.APIError: - pass - - for secret in self.tmp_secrets: - try: - client.api.remove_secret(secret) - except docker.errors.APIError: - pass - - for config in self.tmp_configs: - try: - client.api.remove_config(config) - except docker.errors.APIError: - pass - - for folder in self.tmp_folders: - shutil.rmtree(folder) + try: + for img in self.tmp_imgs: + try: + client.api.remove_image(img) + except docker.errors.APIError: + pass + for container in self.tmp_containers: + try: + client.api.remove_container(container, force=True, v=True) + except docker.errors.APIError: + pass + for network in self.tmp_networks: + try: + client.api.remove_network(network) + except docker.errors.APIError: + pass + for volume in self.tmp_volumes: + try: + client.api.remove_volume(volume) + except docker.errors.APIError: + pass + + for secret in self.tmp_secrets: + try: + client.api.remove_secret(secret) + except docker.errors.APIError: + pass + + for config in self.tmp_configs: + try: + client.api.remove_config(config) + except docker.errors.APIError: + pass + + for folder in self.tmp_folders: + shutil.rmtree(folder) + finally: + client.close() class BaseAPIIntegrationTest(BaseIntegrationTest): From 6777c28dee66915dea8facca577ebf42dce1b0bf Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 31 Oct 2018 17:59:26 -0700 Subject: [PATCH 0784/1301] Clear error for cancellable streams over SSH Signed-off-by: Joffrey F --- docker/types/daemon.py | 12 +++++++++++- tests/integration/api_container_test.py | 5 ++++- tests/integration/models_containers_test.py | 3 +++ 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/docker/types/daemon.py b/docker/types/daemon.py index ee8624e80a..700f9a90c4 100644 --- a/docker/types/daemon.py +++ b/docker/types/daemon.py @@ -5,6 +5,8 @@ except ImportError: import urllib3 +from ..errors import DockerException + class CancellableStream(object): """ @@ -55,9 +57,17 @@ def close(self): elif hasattr(sock_raw, '_sock'): sock = sock_raw._sock + elif hasattr(sock_fp, 'channel'): + # We're working with a paramiko (SSH) channel, which doesn't + # support cancelable streams with the current implementation + raise DockerException( + 'Cancellable streams not supported for the SSH protocol' + ) else: sock = sock_fp._sock - if isinstance(sock, urllib3.contrib.pyopenssl.WrappedSocket): + + if hasattr(urllib3.contrib, 'pyopenssl') and isinstance( + sock, urllib3.contrib.pyopenssl.WrappedSocket): sock = sock.socket sock.shutdown(socket.SHUT_RDWR) diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index 249ef7cfcc..02f3603374 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -883,6 +883,8 @@ def test_logs_streaming_and_follow(self): assert logs == (snippet + '\n').encode(encoding='ascii') @pytest.mark.timeout(5) + @pytest.mark.skipif(os.environ.get('DOCKER_HOST', '').startswith('ssh://'), + reason='No cancellable streams over SSH') def test_logs_streaming_and_follow_and_cancel(self): snippet = 'Flowering Nights (Sakuya Iyazoi)' container = self.client.create_container( @@ -1255,7 +1257,8 @@ def test_attach_no_stream(self): assert output == 'hello\n'.encode(encoding='ascii') @pytest.mark.timeout(5) - @pytest.mark.xfail(True, reason='Cancellable events broken over SSH') + @pytest.mark.skipif(os.environ.get('DOCKER_HOST', '').startswith('ssh://'), + reason='No cancellable streams over SSH') def test_attach_stream_and_cancel(self): container = self.client.create_container( BUSYBOX, 'sh -c "echo hello && sleep 60"', diff --git a/tests/integration/models_containers_test.py b/tests/integration/models_containers_test.py index ab41ea57de..b48f6fb6ce 100644 --- a/tests/integration/models_containers_test.py +++ b/tests/integration/models_containers_test.py @@ -1,3 +1,4 @@ +import os import tempfile import threading @@ -146,6 +147,8 @@ def test_run_with_streamed_logs(self): assert logs[1] == b'world\n' @pytest.mark.timeout(5) + @pytest.mark.skipif(os.environ.get('DOCKER_HOST', '').startswith('ssh://'), + reason='No cancellable streams over SSH') def test_run_with_streamed_logs_and_cancel(self): client = docker.from_env(version=TEST_API_VERSION) out = client.containers.run( From 3d5313a5fd1a6fced07a7a993d49c64aaa0aaf57 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 1 Nov 2018 14:57:29 -0700 Subject: [PATCH 0785/1301] Rewrite utils.parse_host to detect more invalid addresses. The method now uses parsing methods from urllib to better split provided URLs. Addresses containing query strings, parameters, passwords or fragments no longer fail silently. SSH addresses containing paths are no longer accepted. Signed-off-by: Joffrey F --- docker/transport/sshconn.py | 5 +- docker/utils/utils.py | 130 ++++++++++++++++++++---------------- tests/unit/utils_test.py | 12 +++- 3 files changed, 83 insertions(+), 64 deletions(-) diff --git a/docker/transport/sshconn.py b/docker/transport/sshconn.py index 6c9c1196d2..0f6bb51fc2 100644 --- a/docker/transport/sshconn.py +++ b/docker/transport/sshconn.py @@ -1,10 +1,7 @@ -import urllib.parse - import paramiko import requests.adapters import six - from .. import constants if six.PY3: @@ -82,7 +79,7 @@ def __init__(self, base_url, timeout=60, self.ssh_client = paramiko.SSHClient() self.ssh_client.load_system_host_keys() - parsed = urllib.parse.urlparse(base_url) + parsed = six.moves.urllib_parse.urlparse(base_url) self.ssh_client.connect( parsed.hostname, parsed.port, parsed.username, ) diff --git a/docker/utils/utils.py b/docker/utils/utils.py index f8f712349d..4e04cafdb4 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -1,10 +1,11 @@ import base64 +import json import os import os.path -import json import shlex -from distutils.version import StrictVersion +import string from datetime import datetime +from distutils.version import StrictVersion import six @@ -13,11 +14,12 @@ if six.PY2: from urllib import splitnport + from urlparse import urlparse else: - from urllib.parse import splitnport + from urllib.parse import splitnport, urlparse DEFAULT_HTTP_HOST = "127.0.0.1" -DEFAULT_UNIX_SOCKET = "http+unix://var/run/docker.sock" +DEFAULT_UNIX_SOCKET = "http+unix:///var/run/docker.sock" DEFAULT_NPIPE = 'npipe:////./pipe/docker_engine' BYTE_UNITS = { @@ -212,81 +214,93 @@ def parse_repository_tag(repo_name): return repo_name, None -# Based on utils.go:ParseHost http://tinyurl.com/nkahcfh -# fd:// protocol unsupported (for obvious reasons) -# Added support for http and https -# Protocol translation: tcp -> http, unix -> http+unix def parse_host(addr, is_win32=False, tls=False): - proto = "http+unix" - port = None path = '' + port = None + host = None + # Sensible defaults if not addr and is_win32: - addr = DEFAULT_NPIPE - + return DEFAULT_NPIPE if not addr or addr.strip() == 'unix://': return DEFAULT_UNIX_SOCKET addr = addr.strip() - if addr.startswith('http://'): - addr = addr.replace('http://', 'tcp://') - if addr.startswith('http+unix://'): - addr = addr.replace('http+unix://', 'unix://') - if addr == 'tcp://': + parsed_url = urlparse(addr) + proto = parsed_url.scheme + if not proto or any([x not in string.ascii_letters + '+' for x in proto]): + # https://bugs.python.org/issue754016 + parsed_url = urlparse('//' + addr, 'tcp') + proto = 'tcp' + + if proto == 'fd': + raise errors.DockerException('fd protocol is not implemented') + + # These protos are valid aliases for our library but not for the + # official spec + if proto == 'http' or proto == 'https': + tls = proto == 'https' + proto = 'tcp' + elif proto == 'http+unix': + proto = 'unix' + + if proto not in ('tcp', 'unix', 'npipe', 'ssh'): raise errors.DockerException( - "Invalid bind address format: {0}".format(addr) + "Invalid bind address protocol: {}".format(addr) ) - elif addr.startswith('unix://'): - addr = addr[7:] - elif addr.startswith('tcp://'): - proto = 'http{0}'.format('s' if tls else '') - addr = addr[6:] - elif addr.startswith('https://'): - proto = "https" - addr = addr[8:] - elif addr.startswith('npipe://'): - proto = 'npipe' - addr = addr[8:] - elif addr.startswith('fd://'): - raise errors.DockerException("fd protocol is not implemented") - elif addr.startswith('ssh://'): - proto = 'ssh' - addr = addr[6:] - else: - if "://" in addr: - raise errors.DockerException( - "Invalid bind address protocol: {0}".format(addr) - ) - proto = "https" if tls else "http" - if proto in ("http", "https", "ssh"): - address_parts = addr.split('/', 1) - host = address_parts[0] - if len(address_parts) == 2: - path = '/' + address_parts[1] - host, port = splitnport(host) + if proto == 'tcp' and not parsed_url.netloc: + # "tcp://" is exceptionally disallowed by convention; + # omitting a hostname for other protocols is fine + raise errors.DockerException( + 'Invalid bind address format: {}'.format(addr) + ) + if any([ + parsed_url.params, parsed_url.query, parsed_url.fragment, + parsed_url.password + ]): + raise errors.DockerException( + 'Invalid bind address format: {}'.format(addr) + ) + + if parsed_url.path and proto == 'ssh': + raise errors.DockerException( + 'Invalid bind address format: no path allowed for this protocol:' + ' {}'.format(addr) + ) + else: + path = parsed_url.path + if proto == 'unix' and parsed_url.hostname is not None: + # For legacy reasons, we consider unix://path + # to be valid and equivalent to unix:///path + path = '/'.join((parsed_url.hostname, path)) + + if proto in ('tcp', 'ssh'): + # parsed_url.hostname strips brackets from IPv6 addresses, + # which can be problematic hence our use of splitnport() instead. + host, port = splitnport(parsed_url.netloc) if port is None or port < 0: - if proto == 'ssh': - port = 22 - else: + if proto != 'ssh': raise errors.DockerException( - "Invalid port: {0}".format(addr) + 'Invalid bind address format: port is required:' + ' {}'.format(addr) ) + port = 22 if not host: host = DEFAULT_HTTP_HOST - else: - host = addr - if proto in ("http", "https") and port == -1: - raise errors.DockerException( - "Bind address needs a port: {0}".format(addr)) + # Rewrite schemes to fit library internals (requests adapters) + if proto == 'tcp': + proto = 'http{}'.format('s' if tls else '') + elif proto == 'unix': + proto = 'http+unix' - if proto == "http+unix" or proto == 'npipe': - return "{0}://{1}".format(proto, host).rstrip('/') - return "{0}://{1}:{2}{3}".format(proto, host, port, path).rstrip('/') + if proto in ('http+unix', 'npipe'): + return "{}://{}".format(proto, path).rstrip('/') + return '{0}://{1}:{2}{3}'.format(proto, host, port, path).rstrip('/') def parse_devices(devices): diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index 8880cfef0f..c862a1cec9 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -272,6 +272,11 @@ def test_parse_host(self): 'tcp://', 'udp://127.0.0.1', 'udp://127.0.0.1:2375', + 'ssh://:22/path', + 'tcp://netloc:3333/path?q=1', + 'unix:///sock/path#fragment', + 'https://netloc:3333/path;params', + 'ssh://:clearpassword@host:22', ] valid_hosts = { @@ -281,7 +286,7 @@ def test_parse_host(self): 'http://:7777': 'http://127.0.0.1:7777', 'https://kokia.jp:2375': 'https://kokia.jp:2375', 'unix:///var/run/docker.sock': 'http+unix:///var/run/docker.sock', - 'unix://': 'http+unix://var/run/docker.sock', + 'unix://': 'http+unix:///var/run/docker.sock', '12.234.45.127:2375/docker/engine': ( 'http://12.234.45.127:2375/docker/engine' ), @@ -294,6 +299,9 @@ def test_parse_host(self): '[fd12::82d1]:2375/docker/engine': ( 'http://[fd12::82d1]:2375/docker/engine' ), + 'ssh://': 'ssh://127.0.0.1:22', + 'ssh://user@localhost:22': 'ssh://user@localhost:22', + 'ssh://user@remote': 'ssh://user@remote:22', } for host in invalid_hosts: @@ -304,7 +312,7 @@ def test_parse_host(self): assert parse_host(host, None) == expected def test_parse_host_empty_value(self): - unix_socket = 'http+unix://var/run/docker.sock' + unix_socket = 'http+unix:///var/run/docker.sock' npipe = 'npipe:////./pipe/docker_engine' for val in [None, '']: From 8f3dc4740e94085d4b14b2847ec0349f075daed8 Mon Sep 17 00:00:00 2001 From: Adam Dangoor Date: Mon, 5 Nov 2018 00:04:43 +0000 Subject: [PATCH 0786/1301] Add a missing space in a log message Signed-off-by: Adam Dangoor --- docker/auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/auth.py b/docker/auth.py index 9635f933ec..17158f4ae3 100644 --- a/docker/auth.py +++ b/docker/auth.py @@ -267,7 +267,7 @@ def load_config(config_path=None, config_dict=None): return res log.debug( - "Couldn't find auth-related section ; attempting to interpret" + "Couldn't find auth-related section ; attempting to interpret " "as auth-only file" ) return {'auths': parse_auth(config_dict)} From f1d629fb5c3259c4e1b63cf3adf6ddf4f383153f Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 6 Nov 2018 14:46:37 -0800 Subject: [PATCH 0787/1301] Add named parameter to image.save to identify which repository name to use in the resulting tarball Signed-off-by: Joffrey F --- docker/models/images.py | 20 ++++++++++++++++-- tests/integration/models_images_test.py | 27 +++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/docker/models/images.py b/docker/models/images.py index 7d9ab70b56..28b1fd3ffd 100644 --- a/docker/models/images.py +++ b/docker/models/images.py @@ -59,7 +59,7 @@ def history(self): """ return self.client.api.history(self.id) - def save(self, chunk_size=DEFAULT_DATA_CHUNK_SIZE): + def save(self, chunk_size=DEFAULT_DATA_CHUNK_SIZE, named=False): """ Get a tarball of an image. Similar to the ``docker save`` command. @@ -67,6 +67,12 @@ def save(self, chunk_size=DEFAULT_DATA_CHUNK_SIZE): chunk_size (int): The generator will return up to that much data per iteration, but may return less. If ``None``, data will be streamed as it is received. Default: 2 MB + named (str or bool): If ``False`` (default), the tarball will not + retain repository and tag information for this image. If set + to ``True``, the first tag in the :py:attr:`~tags` list will + be used to identify the image. Alternatively, any element of + the :py:attr:`~tags` list can be used as an argument to use + that specific tag as the saved identifier. Returns: (generator): A stream of raw archive data. @@ -83,7 +89,17 @@ def save(self, chunk_size=DEFAULT_DATA_CHUNK_SIZE): >>> f.write(chunk) >>> f.close() """ - return self.client.api.get_image(self.id, chunk_size) + img = self.id + if named: + img = self.tags[0] if self.tags else img + if isinstance(named, six.string_types): + if named not in self.tags: + raise InvalidArgument( + "{} is not a valid tag for this image".format(named) + ) + img = named + + return self.client.api.get_image(img, chunk_size) def tag(self, repository, tag=None, **kwargs): """ diff --git a/tests/integration/models_images_test.py b/tests/integration/models_images_test.py index ae735baafb..31fab10968 100644 --- a/tests/integration/models_images_test.py +++ b/tests/integration/models_images_test.py @@ -5,6 +5,7 @@ import pytest from .base import BaseIntegrationTest, BUSYBOX, TEST_API_VERSION +from ..helpers import random_name class ImageCollectionTest(BaseIntegrationTest): @@ -108,6 +109,32 @@ def test_save_and_load(self): assert len(result) == 1 assert result[0].id == image.id + def test_save_and_load_repo_name(self): + client = docker.from_env(version=TEST_API_VERSION) + image = client.images.get(BUSYBOX) + additional_tag = random_name() + image.tag(additional_tag) + self.tmp_imgs.append(additional_tag) + image.reload() + with tempfile.TemporaryFile() as f: + stream = image.save(named='{}:latest'.format(additional_tag)) + for chunk in stream: + f.write(chunk) + + f.seek(0) + client.images.remove(additional_tag, force=True) + result = client.images.load(f.read()) + + assert len(result) == 1 + assert result[0].id == image.id + assert '{}:latest'.format(additional_tag) in result[0].tags + + def test_save_name_error(self): + client = docker.from_env(version=TEST_API_VERSION) + image = client.images.get(BUSYBOX) + with pytest.raises(docker.errors.InvalidArgument): + image.save(named='sakuya/izayoi') + class ImageTest(BaseIntegrationTest): From 9467fa480902d06e9f5a59d3c3b5770c8940e15c Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 8 Nov 2018 15:32:10 -0800 Subject: [PATCH 0788/1301] Improve ulimits documentation Signed-off-by: Joffrey F --- docker/api/container.py | 2 +- docker/models/containers.py | 4 ++-- docker/types/containers.py | 17 +++++++++++++++++ docs/api.rst | 1 + 4 files changed, 21 insertions(+), 3 deletions(-) diff --git a/docker/api/container.py b/docker/api/container.py index c59a6d01a5..6967a13acf 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -543,7 +543,7 @@ def create_host_config(self, *args, **kwargs): } ulimits (:py:class:`list`): Ulimits to set inside the container, - as a list of dicts. + as a list of :py:class:`docker.types.Ulimit` instances. userns_mode (str): Sets the user namespace mode for the container when user namespace remapping option is enabled. Supported values are: ``host`` diff --git a/docker/models/containers.py b/docker/models/containers.py index f60ba6e225..98c717421f 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -691,8 +691,8 @@ def run(self, image, command=None, stdout=True, stderr=False, } tty (bool): Allocate a pseudo-TTY. - ulimits (:py:class:`list`): Ulimits to set inside the container, as - a list of dicts. + ulimits (:py:class:`list`): Ulimits to set inside the container, + as a list of :py:class:`docker.types.Ulimit` instances. user (str or int): Username or UID to run commands as inside the container. userns_mode (str): Sets the user namespace mode for the container diff --git a/docker/types/containers.py b/docker/types/containers.py index 9dfea8ceb8..13eb4ef37b 100644 --- a/docker/types/containers.py +++ b/docker/types/containers.py @@ -58,6 +58,23 @@ def unset_config(self, key): class Ulimit(DictType): + """ + Create a ulimit declaration to be used with + :py:meth:`~docker.api.container.ContainerApiMixin.create_host_config`. + + Args: + + name (str): Which ulimit will this apply to. A list of valid names can + be found `here `_. + soft (int): The soft limit for this ulimit. Optional. + hard (int): The hard limit for this ulimit. Optional. + + Example: + + nproc_limit = docker.types.Ulimit(name='nproc', soft=1024) + hc = client.create_host_config(ulimits=[nproc_limit]) + container = client.create_container('busybox', 'true', host_config=hc) + """ def __init__(self, **kwargs): name = kwargs.get('name', kwargs.get('Name')) soft = kwargs.get('soft', kwargs.get('Soft')) diff --git a/docs/api.rst b/docs/api.rst index 6931245716..2c2391a803 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -151,4 +151,5 @@ Configuration types .. autoclass:: SwarmExternalCA .. autoclass:: SwarmSpec(*args, **kwargs) .. autoclass:: TaskTemplate +.. autoclass:: Ulimit .. autoclass:: UpdateConfig From 049e7e16d4759407055720fa427472dcb5509cc8 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 8 Nov 2018 16:20:28 -0800 Subject: [PATCH 0789/1301] Improved LogConfig documentation Signed-off-by: Joffrey F --- docker/api/container.py | 8 +------ docker/models/containers.py | 8 +------ docker/types/containers.py | 45 ++++++++++++++++++++++++++++++++++--- docs/api.rst | 1 + 4 files changed, 45 insertions(+), 17 deletions(-) diff --git a/docker/api/container.py b/docker/api/container.py index 6967a13acf..39478329ae 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -476,13 +476,7 @@ def create_host_config(self, *args, **kwargs): isolation (str): Isolation technology to use. Default: `None`. links (dict or list of tuples): Either a dictionary mapping name to alias or as a list of ``(name, alias)`` tuples. - log_config (dict): Logging configuration, as a dictionary with - keys: - - - ``type`` The logging driver name. - - ``config`` A dictionary of configuration for the logging - driver. - + log_config (LogConfig): Logging configuration lxc_conf (dict): LXC config. mem_limit (float or str): Memory limit. Accepts float values (which represent the memory limit of the created container in diff --git a/docker/models/containers.py b/docker/models/containers.py index 98c717421f..4cd7d13f41 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -576,13 +576,7 @@ def run(self, image, command=None, stdout=True, stderr=False, ``["label1", "label2"]``) links (dict or list of tuples): Either a dictionary mapping name to alias or as a list of ``(name, alias)`` tuples. - log_config (dict): Logging configuration, as a dictionary with - keys: - - - ``type`` The logging driver name. - - ``config`` A dictionary of configuration for the logging - driver. - + log_config (LogConfig): Logging configuration. mac_address (str): MAC address to assign to the container. mem_limit (int or str): Memory limit. Accepts float values (which represent the memory limit of the created container in diff --git a/docker/types/containers.py b/docker/types/containers.py index 13eb4ef37b..d040c0fb5e 100644 --- a/docker/types/containers.py +++ b/docker/types/containers.py @@ -23,6 +23,36 @@ class LogConfigTypesEnum(object): class LogConfig(DictType): + """ + Configure logging for a container, when provided as an argument to + :py:meth:`~docker.api.container.ContainerApiMixin.create_host_config`. + You may refer to the + `official logging driver documentation `_ + for more information. + + Args: + type (str): Indicate which log driver to use. A set of valid drivers + is provided as part of the :py:attr:`LogConfig.types` + enum. Other values may be accepted depending on the engine version + and available logging plugins. + config (dict): A driver-dependent configuration dictionary. Please + refer to the driver's documentation for a list of valid config + keys. + + Example: + + >>> from docker.types import LogConfig + >>> lc = LogConfig(type=LogConfig.types.JSON, config={ + ... 'max-size': '1g', + ... 'labels': 'production_status,geo' + ... }) + >>> hc = client.create_host_config(log_config=lc) + >>> container = client.create_container('busybox', 'true', + ... host_config=hc) + >>> client.inspect_container(container)['HostConfig']['LogConfig'] + {'Type': 'json-file', 'Config': {'labels': 'production_status,geo', 'max-size': '1g'}} + + """ # flake8: noqa types = LogConfigTypesEnum def __init__(self, **kwargs): @@ -50,9 +80,13 @@ def config(self): return self['Config'] def set_config_value(self, key, value): + """ Set a the value for ``key`` to ``value`` inside the ``config`` + dict. + """ self.config[key] = value def unset_config(self, key): + """ Remove the ``key`` property from the ``config`` dict. """ if key in self.config: del self.config[key] @@ -71,9 +105,14 @@ class Ulimit(DictType): Example: - nproc_limit = docker.types.Ulimit(name='nproc', soft=1024) - hc = client.create_host_config(ulimits=[nproc_limit]) - container = client.create_container('busybox', 'true', host_config=hc) + >>> nproc_limit = docker.types.Ulimit(name='nproc', soft=1024) + >>> hc = client.create_host_config(ulimits=[nproc_limit]) + >>> container = client.create_container( + 'busybox', 'true', host_config=hc + ) + >>> client.inspect_container(container)['HostConfig']['Ulimits'] + [{'Name': 'nproc', 'Hard': 0, 'Soft': 1024}] + """ def __init__(self, **kwargs): name = kwargs.get('name', kwargs.get('Name')) diff --git a/docs/api.rst b/docs/api.rst index 2c2391a803..1682128951 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -140,6 +140,7 @@ Configuration types .. autoclass:: Healthcheck .. autoclass:: IPAMConfig .. autoclass:: IPAMPool +.. autoclass:: LogConfig .. autoclass:: Mount .. autoclass:: Placement .. autoclass:: Privileges From d4b1c259a24de084a5a65aa8a56cdd13d3764db2 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 8 Nov 2018 16:52:32 -0800 Subject: [PATCH 0790/1301] Update links docs and fix bug in normalize_links Signed-off-by: Joffrey F --- docker/api/container.py | 17 ++++++++++------- docker/models/containers.py | 6 ++++-- docker/utils/utils.py | 2 +- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/docker/api/container.py b/docker/api/container.py index 39478329ae..8858aa68a3 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -473,9 +473,11 @@ def create_host_config(self, *args, **kwargs): signals and reaps processes init_path (str): Path to the docker-init binary ipc_mode (str): Set the IPC mode for the container. - isolation (str): Isolation technology to use. Default: `None`. - links (dict or list of tuples): Either a dictionary mapping name - to alias or as a list of ``(name, alias)`` tuples. + isolation (str): Isolation technology to use. Default: ``None``. + links (dict): Mapping of links using the + ``{'container': 'alias'}`` format. The alias is optional. + Containers declared in this dict will be linked to the new + container using the provided alias. Default: ``None``. log_config (LogConfig): Logging configuration lxc_conf (dict): LXC config. mem_limit (float or str): Memory limit. Accepts float values @@ -605,9 +607,10 @@ def create_endpoint_config(self, *args, **kwargs): aliases (:py:class:`list`): A list of aliases for this endpoint. Names in that list can be used within the network to reach the container. Defaults to ``None``. - links (:py:class:`list`): A list of links for this endpoint. - Containers declared in this list will be linked to this - container. Defaults to ``None``. + links (dict): Mapping of links for this endpoint using the + ``{'container': 'alias'}`` format. The alias is optional. + Containers declared in this dict will be linked to this + container using the provided alias. Defaults to ``None``. ipv4_address (str): The IP address of this container on the network, using the IPv4 protocol. Defaults to ``None``. ipv6_address (str): The IP address of this container on the @@ -622,7 +625,7 @@ def create_endpoint_config(self, *args, **kwargs): >>> endpoint_config = client.create_endpoint_config( aliases=['web', 'app'], - links=['app_db'], + links={'app_db': 'db', 'another': None}, ipv4_address='132.65.0.123' ) diff --git a/docker/models/containers.py b/docker/models/containers.py index 4cd7d13f41..bba0395eed 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -574,8 +574,10 @@ def run(self, image, command=None, stdout=True, stderr=False, ``{"label1": "value1", "label2": "value2"}``) or a list of names of labels to set with empty values (e.g. ``["label1", "label2"]``) - links (dict or list of tuples): Either a dictionary mapping name - to alias or as a list of ``(name, alias)`` tuples. + links (dict): Mapping of links using the + ``{'container': 'alias'}`` format. The alias is optional. + Containers declared in this dict will be linked to the new + container using the provided alias. Default: ``None``. log_config (LogConfig): Logging configuration. mac_address (str): MAC address to assign to the container. mem_limit (int or str): Memory limit. Accepts float values diff --git a/docker/utils/utils.py b/docker/utils/utils.py index 4e04cafdb4..a8e814d7d5 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -441,7 +441,7 @@ def normalize_links(links): if isinstance(links, dict): links = six.iteritems(links) - return ['{0}:{1}'.format(k, v) for k, v in sorted(links)] + return ['{0}:{1}'.format(k, v) if v else k for k, v in sorted(links)] def parse_env_file(env_file): From bc45e71c55f02ece4b448da21b94e7c5b719ea4b Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 8 Nov 2018 17:05:42 -0800 Subject: [PATCH 0791/1301] Document attr caching for Container objects Signed-off-by: Joffrey F --- docker/models/containers.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docker/models/containers.py b/docker/models/containers.py index bba0395eed..a3fd1a8edf 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -15,7 +15,12 @@ class Container(Model): - + """ Local representation of a container object. Detailed configuration may + be accessed through the :py:attr:`attrs` attribute. Note that local + attributes are cached; users may call :py:meth:`reload` to + query the Docker daemon for the current properties, causing + :py:attr:`attrs` to be refreshed. + """ @property def name(self): """ From ebfba8d4c3136015c73cd164202099cc1787c0e5 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 8 Nov 2018 17:08:41 -0800 Subject: [PATCH 0792/1301] Fix incorrect return info for inspect_service Signed-off-by: Joffrey F --- docker/api/service.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/api/service.py b/docker/api/service.py index 8b956b63e1..08e2591730 100644 --- a/docker/api/service.py +++ b/docker/api/service.py @@ -197,7 +197,8 @@ def inspect_service(self, service, insert_defaults=None): into the service inspect output. Returns: - ``True`` if successful. + (dict): A dictionary of the server-side representation of the + service, including all relevant properties. Raises: :py:class:`docker.errors.APIError` From d9e08aedc3743c133bc4bb87917bc0914f9ff63d Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 8 Nov 2018 17:13:19 -0800 Subject: [PATCH 0793/1301] Disallow incompatible combination stats(decode=True, stream=False) Signed-off-by: Joffrey F --- docker/api/container.py | 7 ++++++- docker/models/containers.py | 3 ++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/docker/api/container.py b/docker/api/container.py index 8858aa68a3..753e0a57fe 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -1071,7 +1071,8 @@ def stats(self, container, decode=None, stream=True): Args: container (str): The container to stream statistics from decode (bool): If set to true, stream will be decoded into dicts - on the fly. False by default. + on the fly. Only applicable if ``stream`` is True. + False by default. stream (bool): If set to false, only the current stats will be returned instead of a stream. True by default. @@ -1085,6 +1086,10 @@ def stats(self, container, decode=None, stream=True): return self._stream_helper(self._get(url, stream=True), decode=decode) else: + if decode: + raise errors.InvalidArgument( + "decode is only available in conjuction with stream=True" + ) return self._result(self._get(url, params={'stream': False}), json=True) diff --git a/docker/models/containers.py b/docker/models/containers.py index a3fd1a8edf..493b9fc732 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -385,7 +385,8 @@ def stats(self, **kwargs): Args: decode (bool): If set to true, stream will be decoded into dicts - on the fly. False by default. + on the fly. Only applicable if ``stream`` is True. + False by default. stream (bool): If set to false, only the current stats will be returned instead of a stream. True by default. From 66666f9824a500e402e16c78866ad2a37438448d Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 8 Nov 2018 17:22:24 -0800 Subject: [PATCH 0794/1301] Properly convert non-string filters to expected string format Signed-off-by: Joffrey F --- docker/utils/utils.py | 5 ++++- tests/unit/utils_test.py | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/docker/utils/utils.py b/docker/utils/utils.py index a8e814d7d5..61e307adc7 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -386,7 +386,10 @@ def convert_filters(filters): v = 'true' if v else 'false' if not isinstance(v, list): v = [v, ] - result[k] = v + result[k] = [ + str(item) if not isinstance(item, six.string_types) else item + for item in v + ] return json.dumps(result) diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index c862a1cec9..a4e9c9c53e 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -457,8 +457,8 @@ def test_convert_filters(self): tests = [ ({'dangling': True}, '{"dangling": ["true"]}'), ({'dangling': "true"}, '{"dangling": ["true"]}'), - ({'exited': 0}, '{"exited": [0]}'), - ({'exited': [0, 1]}, '{"exited": [0, 1]}'), + ({'exited': 0}, '{"exited": ["0"]}'), + ({'exited': [0, 1]}, '{"exited": ["0", "1"]}'), ] for filters, expected in tests: From 584204bbddd4f2ad36142cbf2b8b27887ba5c685 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 8 Nov 2018 17:31:22 -0800 Subject: [PATCH 0795/1301] Add doc example for get_archive Signed-off-by: Joffrey F --- docker/api/container.py | 12 ++++++++++++ docker/models/containers.py | 11 +++++++++++ 2 files changed, 23 insertions(+) diff --git a/docker/api/container.py b/docker/api/container.py index 753e0a57fe..fce73af640 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -694,6 +694,18 @@ def get_archive(self, container, path, chunk_size=DEFAULT_DATA_CHUNK_SIZE): Raises: :py:class:`docker.errors.APIError` If the server returns an error. + + Example: + + >>> c = docker.APIClient() + >>> f = open('./sh_bin.tar', 'wb') + >>> bits, stat = c.get_archive(container, '/bin/sh') + >>> print(stat) + {'name': 'sh', 'size': 1075464, 'mode': 493, + 'mtime': '2018-10-01T15:37:48-07:00', 'linkTarget': ''} + >>> for chunk in bits: + ... f.write(chunk) + >>> f.close() """ params = { 'path': path diff --git a/docker/models/containers.py b/docker/models/containers.py index 493b9fc732..9d6f2cc6af 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -233,6 +233,17 @@ def get_archive(self, path, chunk_size=DEFAULT_DATA_CHUNK_SIZE): Raises: :py:class:`docker.errors.APIError` If the server returns an error. + + Example: + + >>> f = open('./sh_bin.tar', 'wb') + >>> bits, stat = container.get_archive('/bin/sh') + >>> print(stat) + {'name': 'sh', 'size': 1075464, 'mode': 493, + 'mtime': '2018-10-01T15:37:48-07:00', 'linkTarget': ''} + >>> for chunk in bits: + ... f.write(chunk) + >>> f.close() """ return self.client.api.get_archive(self.id, path, chunk_size) From e6889eb9d652a03af555a4f9dbd17f27fdd1116b Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 8 Nov 2018 17:32:33 -0800 Subject: [PATCH 0796/1301] Fix file mode in image.save examples Signed-off-by: Joffrey F --- docker/api/image.py | 2 +- docker/models/images.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/api/image.py b/docker/api/image.py index 5f05d8877e..e9e61db130 100644 --- a/docker/api/image.py +++ b/docker/api/image.py @@ -32,7 +32,7 @@ def get_image(self, image, chunk_size=DEFAULT_DATA_CHUNK_SIZE): Example: >>> image = cli.get_image("busybox:latest") - >>> f = open('/tmp/busybox-latest.tar', 'w') + >>> f = open('/tmp/busybox-latest.tar', 'wb') >>> for chunk in image: >>> f.write(chunk) >>> f.close() diff --git a/docker/models/images.py b/docker/models/images.py index 28b1fd3ffd..4578c0bd89 100644 --- a/docker/models/images.py +++ b/docker/models/images.py @@ -84,7 +84,7 @@ def save(self, chunk_size=DEFAULT_DATA_CHUNK_SIZE, named=False): Example: >>> image = cli.get_image("busybox:latest") - >>> f = open('/tmp/busybox-latest.tar', 'w') + >>> f = open('/tmp/busybox-latest.tar', 'wb') >>> for chunk in image: >>> f.write(chunk) >>> f.close() From cafb802c51e0c0b8ff58cb749fa30b99cd7182b4 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 8 Nov 2018 18:58:06 -0800 Subject: [PATCH 0797/1301] Fix versions script to accept versions without -ce suffix Signed-off-by: Joffrey F --- scripts/versions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/versions.py b/scripts/versions.py index 7212195dec..7ad1d56a61 100644 --- a/scripts/versions.py +++ b/scripts/versions.py @@ -66,7 +66,7 @@ def main(): Version.parse( v.strip('"').lstrip('docker-').rstrip('.tgz').rstrip('-x86_64') ) for v in re.findall( - r'"docker-[0-9]+\.[0-9]+\.[0-9]+-.*tgz"', content + r'"docker-[0-9]+\.[0-9]+\.[0-9]+-?.*tgz"', content ) ] sorted_versions = sorted( From ad4f5f9d0c4668a67a5513f05811335c519b1bec Mon Sep 17 00:00:00 2001 From: Corentin Henry Date: Wed, 21 Nov 2018 17:12:01 -0800 Subject: [PATCH 0798/1301] tests: fix failure due to pytest deprecation Signed-off-by: Corentin Henry --- tests/unit/utils_config_test.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/unit/utils_config_test.py b/tests/unit/utils_config_test.py index 50ba3831db..b0934f9568 100644 --- a/tests/unit/utils_config_test.py +++ b/tests/unit/utils_config_test.py @@ -4,8 +4,8 @@ import tempfile import json -from py.test import ensuretemp -from pytest import mark +from pytest import mark, fixture + from docker.utils import config try: @@ -15,25 +15,25 @@ class FindConfigFileTest(unittest.TestCase): - def tmpdir(self, name): - tmpdir = ensuretemp(name) - self.addCleanup(tmpdir.remove) - return tmpdir + + @fixture(autouse=True) + def tmpdir(self, tmpdir): + self.mkdir = tmpdir.mkdir def test_find_config_fallback(self): - tmpdir = self.tmpdir('test_find_config_fallback') + tmpdir = self.mkdir('test_find_config_fallback') with mock.patch.dict(os.environ, {'HOME': str(tmpdir)}): assert config.find_config_file() is None def test_find_config_from_explicit_path(self): - tmpdir = self.tmpdir('test_find_config_from_explicit_path') + tmpdir = self.mkdir('test_find_config_from_explicit_path') config_path = tmpdir.ensure('my-config-file.json') assert config.find_config_file(str(config_path)) == str(config_path) def test_find_config_from_environment(self): - tmpdir = self.tmpdir('test_find_config_from_environment') + tmpdir = self.mkdir('test_find_config_from_environment') config_path = tmpdir.ensure('config.json') with mock.patch.dict(os.environ, {'DOCKER_CONFIG': str(tmpdir)}): @@ -41,7 +41,7 @@ def test_find_config_from_environment(self): @mark.skipif("sys.platform == 'win32'") def test_find_config_from_home_posix(self): - tmpdir = self.tmpdir('test_find_config_from_home_posix') + tmpdir = self.mkdir('test_find_config_from_home_posix') config_path = tmpdir.ensure('.docker', 'config.json') with mock.patch.dict(os.environ, {'HOME': str(tmpdir)}): @@ -49,7 +49,7 @@ def test_find_config_from_home_posix(self): @mark.skipif("sys.platform == 'win32'") def test_find_config_from_home_legacy_name(self): - tmpdir = self.tmpdir('test_find_config_from_home_legacy_name') + tmpdir = self.mkdir('test_find_config_from_home_legacy_name') config_path = tmpdir.ensure('.dockercfg') with mock.patch.dict(os.environ, {'HOME': str(tmpdir)}): @@ -57,7 +57,7 @@ def test_find_config_from_home_legacy_name(self): @mark.skipif("sys.platform != 'win32'") def test_find_config_from_home_windows(self): - tmpdir = self.tmpdir('test_find_config_from_home_windows') + tmpdir = self.mkdir('test_find_config_from_home_windows') config_path = tmpdir.ensure('.docker', 'config.json') with mock.patch.dict(os.environ, {'USERPROFILE': str(tmpdir)}): From 2d5a7c3894efcb29333cd934692fbf5082672445 Mon Sep 17 00:00:00 2001 From: Corentin Henry Date: Wed, 21 Nov 2018 17:52:35 -0800 Subject: [PATCH 0799/1301] tests: bump pytest-timeout Signed-off-by: Corentin Henry pytest-timeout 1.2.1 seems to be incompatible with pytest 3.6.3: INTERNALERROR> Traceback (most recent call last): INTERNALERROR> File "/usr/local/lib/python2.7/site-packages/_pytest/main.py", line 185, in wrap_session INTERNALERROR> session.exitstatus = doit(config, session) or 0 INTERNALERROR> File "/usr/local/lib/python2.7/site-packages/_pytest/main.py", line 225, in _main INTERNALERROR> config.hook.pytest_runtestloop(session=session) INTERNALERROR> File "/usr/local/lib/python2.7/site-packages/pluggy/hooks.py", line 284, in __call__ INTERNALERROR> return self._hookexec(self, self.get_hookimpls(), kwargs) INTERNALERROR> File "/usr/local/lib/python2.7/site-packages/pluggy/manager.py", line 67, in _hookexec INTERNALERROR> return self._inner_hookexec(hook, methods, kwargs) INTERNALERROR> File "/usr/local/lib/python2.7/site-packages/pluggy/manager.py", line 61, in INTERNALERROR> firstresult=hook.spec.opts.get("firstresult") if hook.spec else False, INTERNALERROR> File "/usr/local/lib/python2.7/site-packages/pluggy/callers.py", line 208, in _multicall INTERNALERROR> return outcome.get_result() INTERNALERROR> File "/usr/local/lib/python2.7/site-packages/pluggy/callers.py", line 81, in get_result INTERNALERROR> _reraise(*ex) # noqa INTERNALERROR> File "/usr/local/lib/python2.7/site-packages/pluggy/callers.py", line 187, in _multicall INTERNALERROR> res = hook_impl.function(*args) INTERNALERROR> File "/usr/local/lib/python2.7/site-packages/_pytest/main.py", line 246, in pytest_runtestloop INTERNALERROR> item.config.hook.pytest_runtest_protocol(item=item, nextitem=nextitem) INTERNALERROR> File "/usr/local/lib/python2.7/site-packages/pluggy/hooks.py", line 284, in __call__ INTERNALERROR> return self._hookexec(self, self.get_hookimpls(), kwargs) INTERNALERROR> File "/usr/local/lib/python2.7/site-packages/pluggy/manager.py", line 67, in _hookexec INTERNALERROR> return self._inner_hookexec(hook, methods, kwargs) INTERNALERROR> File "/usr/local/lib/python2.7/site-packages/pluggy/manager.py", line 61, in INTERNALERROR> firstresult=hook.spec.opts.get("firstresult") if hook.spec else False, INTERNALERROR> File "/usr/local/lib/python2.7/site-packages/pluggy/callers.py", line 208, in _multicall INTERNALERROR> return outcome.get_result() INTERNALERROR> File "/usr/local/lib/python2.7/site-packages/pluggy/callers.py", line 81, in get_result INTERNALERROR> _reraise(*ex) # noqa INTERNALERROR> File "/usr/local/lib/python2.7/site-packages/pluggy/callers.py", line 182, in _multicall INTERNALERROR> next(gen) # first yield INTERNALERROR> File "/usr/local/lib/python2.7/site-packages/pytest_timeout.py", line 76, in pytest_runtest_protocol INTERNALERROR> timeout_setup(item) INTERNALERROR> File "/usr/local/lib/python2.7/site-packages/pytest_timeout.py", line 104, in timeout_setup INTERNALERROR> timeout, method = get_params(item) INTERNALERROR> File "/usr/local/lib/python2.7/site-packages/pytest_timeout.py", line 162, in get_params INTERNALERROR> timeout, method = _parse_marker(item.keywords['timeout']) INTERNALERROR> File "/usr/local/lib/python2.7/site-packages/pytest_timeout.py", line 178, in _parse_marker INTERNALERROR> if not marker.args and not marker.kwargs: INTERNALERROR> File "/usr/local/lib/python2.7/site-packages/_pytest/mark/structures.py", line 25, in warned INTERNALERROR> warnings.warn(warning, stacklevel=2) INTERNALERROR> RemovedInPytest4Warning: MarkInfo objects are deprecated as they contain merged marks which are hard to deal with correctly. INTERNALERROR> Please use node.get_closest_marker(name) or node.iter_markers(name). INTERNALERROR> Docs: https://docs.pytest.org/en/latest/mark.html#updating-code --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 9ad59cc664..07e1a900db 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -4,4 +4,4 @@ mock==1.0.1 pytest==2.9.1; python_version == '3.3' pytest==3.6.3; python_version > '3.3' pytest-cov==2.1.0 -pytest-timeout==1.2.1 +pytest-timeout==1.3.3 From f9505da1d604f2c4046b8116a80e62efe5937e87 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 26 Nov 2018 17:33:30 -0800 Subject: [PATCH 0800/1301] Correctly handle longpath prefix in process_dockerfile when joining paths Signed-off-by: Joffrey F --- docker/api/build.py | 9 ++++- docker/constants.py | 1 + tests/unit/api_build_test.py | 64 +++++++++++++++++++++++++++++++++++- 3 files changed, 72 insertions(+), 2 deletions(-) diff --git a/docker/api/build.py b/docker/api/build.py index 0486dce62d..3a67ff8b28 100644 --- a/docker/api/build.py +++ b/docker/api/build.py @@ -339,7 +339,14 @@ def process_dockerfile(dockerfile, path): abs_dockerfile = dockerfile if not os.path.isabs(dockerfile): abs_dockerfile = os.path.join(path, dockerfile) - + if constants.IS_WINDOWS_PLATFORM and path.startswith( + constants.WINDOWS_LONGPATH_PREFIX): + abs_dockerfile = '{}{}'.format( + constants.WINDOWS_LONGPATH_PREFIX, + os.path.normpath( + abs_dockerfile[len(constants.WINDOWS_LONGPATH_PREFIX):] + ) + ) if (os.path.splitdrive(path)[0] != os.path.splitdrive(abs_dockerfile)[0] or os.path.relpath(abs_dockerfile, path).startswith('..')): # Dockerfile not in context - read data to insert into tar later diff --git a/docker/constants.py b/docker/constants.py index 7565a76889..1ab11ec051 100644 --- a/docker/constants.py +++ b/docker/constants.py @@ -14,6 +14,7 @@ 'is deprecated and non-functional. Please remove it.' IS_WINDOWS_PLATFORM = (sys.platform == 'win32') +WINDOWS_LONGPATH_PREFIX = '\\\\?\\' DEFAULT_USER_AGENT = "docker-sdk-python/{0}".format(version) DEFAULT_NUM_POOLS = 25 diff --git a/tests/unit/api_build_test.py b/tests/unit/api_build_test.py index a7f34fd3f2..59470caa5f 100644 --- a/tests/unit/api_build_test.py +++ b/tests/unit/api_build_test.py @@ -1,12 +1,16 @@ import gzip import io +import shutil import docker from docker import auth +from docker.api.build import process_dockerfile -from .api_test import BaseAPIClientTest, fake_request, url_prefix import pytest +from ..helpers import make_tree +from .api_test import BaseAPIClientTest, fake_request, url_prefix + class BuildTest(BaseAPIClientTest): def test_build_container(self): @@ -161,3 +165,61 @@ def test_set_auth_headers_with_dict_and_no_auth_configs(self): self.client._set_auth_headers(headers) assert headers == expected_headers + + @pytest.mark.skipif( + not docker.constants.IS_WINDOWS_PLATFORM, + reason='Windows-specific syntax') + def test_process_dockerfile_win_longpath_prefix(self): + dirs = [ + 'foo', 'foo/bar', 'baz', + ] + + files = [ + 'Dockerfile', 'foo/Dockerfile.foo', 'foo/bar/Dockerfile.bar', + 'baz/Dockerfile.baz', + ] + + base = make_tree(dirs, files) + self.addCleanup(shutil.rmtree, base) + + def pre(path): + return docker.constants.WINDOWS_LONGPATH_PREFIX + path + + assert process_dockerfile(None, pre(base)) == (None, None) + assert process_dockerfile('Dockerfile', pre(base)) == ( + 'Dockerfile', None + ) + assert process_dockerfile('foo/Dockerfile.foo', pre(base)) == ( + 'foo/Dockerfile.foo', None + ) + assert process_dockerfile( + '../Dockerfile', pre(base + '\\foo') + )[1] is not None + assert process_dockerfile( + '../baz/Dockerfile.baz', pre(base + '/baz') + ) == ('../baz/Dockerfile.baz', None) + + def test_process_dockerfile(self): + dirs = [ + 'foo', 'foo/bar', 'baz', + ] + + files = [ + 'Dockerfile', 'foo/Dockerfile.foo', 'foo/bar/Dockerfile.bar', + 'baz/Dockerfile.baz', + ] + + base = make_tree(dirs, files) + self.addCleanup(shutil.rmtree, base) + + assert process_dockerfile(None, base) == (None, None) + assert process_dockerfile('Dockerfile', base) == ('Dockerfile', None) + assert process_dockerfile('foo/Dockerfile.foo', base) == ( + 'foo/Dockerfile.foo', None + ) + assert process_dockerfile( + '../Dockerfile', base + '/foo' + )[1] is not None + assert process_dockerfile('../baz/Dockerfile.baz', base + '/baz') == ( + '../baz/Dockerfile.baz', None + ) From 7117855f6e577678ba47536d860d193fcb131a75 Mon Sep 17 00:00:00 2001 From: adw1n Date: Mon, 3 Sep 2018 05:54:12 +0200 Subject: [PATCH 0801/1301] Fix pulling images with `stream=True` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pulling an image with option `stream=True` like this: ``` client.api.pull('docker.io/user/repo_name', tag='latest', stream=True) ``` without consuming the generator oftentimes results in premature drop of the connection. Docker daemon tries to send progress of pulling the image to the client, but it encounters an error (broken pipe) and therefore cancells the pull action: ``` Thread 1 "dockerd-dev" received signal SIGPIPE, Broken pipe. ERRO[2018-09-03T05:12:35.746497638+02:00] Not continuing with pull after error: context canceled ``` As described in issue #2116, even though client receives response with status code 200, image is not pulled. Closes #2116 Signed-off-by: Przemysław Adamek --- docker/api/image.py | 3 ++- docker/models/images.py | 1 + tests/unit/models_containers_test.py | 3 ++- tests/unit/models_images_test.py | 6 ++++-- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/docker/api/image.py b/docker/api/image.py index e9e61db130..5a6537e797 100644 --- a/docker/api/image.py +++ b/docker/api/image.py @@ -334,7 +334,8 @@ def pull(self, repository, tag=None, stream=False, auth_config=None, Args: repository (str): The repository to pull tag (str): The tag to pull - stream (bool): Stream the output as a generator + stream (bool): Stream the output as a generator. Make sure to + consume the generator, otherwise pull might get cancelled. auth_config (dict): Override the credentials that :py:meth:`~docker.api.daemon.DaemonApiMixin.login` has set for this request. ``auth_config`` should contain the ``username`` diff --git a/docker/models/images.py b/docker/models/images.py index 4578c0bd89..f8b842a943 100644 --- a/docker/models/images.py +++ b/docker/models/images.py @@ -425,6 +425,7 @@ def pull(self, repository, tag=None, **kwargs): if not tag: repository, tag = parse_repository_tag(repository) + kwargs['stream'] = False self.client.api.pull(repository, tag=tag, **kwargs) if tag: return self.get('{0}{2}{1}'.format( diff --git a/tests/unit/models_containers_test.py b/tests/unit/models_containers_test.py index 22dd241064..957035af0a 100644 --- a/tests/unit/models_containers_test.py +++ b/tests/unit/models_containers_test.py @@ -232,7 +232,8 @@ def test_run_pull(self): container = client.containers.run('alpine', 'sleep 300', detach=True) assert container.id == FAKE_CONTAINER_ID - client.api.pull.assert_called_with('alpine', platform=None, tag=None) + client.api.pull.assert_called_with('alpine', platform=None, tag=None, + stream=False) def test_run_with_error(self): client = make_fake_client() diff --git a/tests/unit/models_images_test.py b/tests/unit/models_images_test.py index 67832795fe..ef81a1599d 100644 --- a/tests/unit/models_images_test.py +++ b/tests/unit/models_images_test.py @@ -43,7 +43,8 @@ def test_load(self): def test_pull(self): client = make_fake_client() image = client.images.pull('test_image:latest') - client.api.pull.assert_called_with('test_image', tag='latest') + client.api.pull.assert_called_with('test_image', tag='latest', + stream=False) client.api.inspect_image.assert_called_with('test_image:latest') assert isinstance(image, Image) assert image.id == FAKE_IMAGE_ID @@ -51,7 +52,8 @@ def test_pull(self): def test_pull_multiple(self): client = make_fake_client() images = client.images.pull('test_image') - client.api.pull.assert_called_with('test_image', tag=None) + client.api.pull.assert_called_with('test_image', tag=None, + stream=False) client.api.images.assert_called_with( all=False, name='test_image', filters=None ) From 30d16ce89ae745154027da2419cca06fdf7d5fb2 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 28 Nov 2018 11:24:21 -0800 Subject: [PATCH 0802/1301] Update DockerClient.images.pull to always stream response Also raise a warning when users attempt to specify the "stream" parameter Signed-off-by: Joffrey F --- docker/models/images.py | 18 ++++++++++++++++-- tests/unit/models_containers_test.py | 5 +++-- tests/unit/models_images_test.py | 24 +++++++++++++++++++----- 3 files changed, 38 insertions(+), 9 deletions(-) diff --git a/docker/models/images.py b/docker/models/images.py index f8b842a943..30e86f109e 100644 --- a/docker/models/images.py +++ b/docker/models/images.py @@ -1,5 +1,6 @@ import itertools import re +import warnings import six @@ -425,8 +426,21 @@ def pull(self, repository, tag=None, **kwargs): if not tag: repository, tag = parse_repository_tag(repository) - kwargs['stream'] = False - self.client.api.pull(repository, tag=tag, **kwargs) + if 'stream' in kwargs: + warnings.warn( + '`stream` is not a valid parameter for this method' + ' and will be overridden' + ) + del kwargs['stream'] + + pull_log = self.client.api.pull( + repository, tag=tag, stream=True, **kwargs + ) + for _ in pull_log: + # We don't do anything with the logs, but we need + # to keep the connection alive and wait for the image + # to be pulled. + pass if tag: return self.get('{0}{2}{1}'.format( repository, tag, '@' if tag.startswith('sha256:') else ':' diff --git a/tests/unit/models_containers_test.py b/tests/unit/models_containers_test.py index 957035af0a..39e409e4bf 100644 --- a/tests/unit/models_containers_test.py +++ b/tests/unit/models_containers_test.py @@ -232,8 +232,9 @@ def test_run_pull(self): container = client.containers.run('alpine', 'sleep 300', detach=True) assert container.id == FAKE_CONTAINER_ID - client.api.pull.assert_called_with('alpine', platform=None, tag=None, - stream=False) + client.api.pull.assert_called_with( + 'alpine', platform=None, tag=None, stream=True + ) def test_run_with_error(self): client = make_fake_client() diff --git a/tests/unit/models_images_test.py b/tests/unit/models_images_test.py index ef81a1599d..fd894ab71d 100644 --- a/tests/unit/models_images_test.py +++ b/tests/unit/models_images_test.py @@ -1,6 +1,8 @@ +import unittest +import warnings + from docker.constants import DEFAULT_DATA_CHUNK_SIZE from docker.models.images import Image -import unittest from .fake_api import FAKE_IMAGE_ID from .fake_api_client import make_fake_client @@ -43,8 +45,9 @@ def test_load(self): def test_pull(self): client = make_fake_client() image = client.images.pull('test_image:latest') - client.api.pull.assert_called_with('test_image', tag='latest', - stream=False) + client.api.pull.assert_called_with( + 'test_image', tag='latest', stream=True + ) client.api.inspect_image.assert_called_with('test_image:latest') assert isinstance(image, Image) assert image.id == FAKE_IMAGE_ID @@ -52,8 +55,9 @@ def test_pull(self): def test_pull_multiple(self): client = make_fake_client() images = client.images.pull('test_image') - client.api.pull.assert_called_with('test_image', tag=None, - stream=False) + client.api.pull.assert_called_with( + 'test_image', tag=None, stream=True + ) client.api.images.assert_called_with( all=False, name='test_image', filters=None ) @@ -63,6 +67,16 @@ def test_pull_multiple(self): assert isinstance(image, Image) assert image.id == FAKE_IMAGE_ID + def test_pull_with_stream_param(self): + client = make_fake_client() + with warnings.catch_warnings(record=True) as w: + client.images.pull('test_image', stream=True) + + assert len(w) == 1 + assert str(w[0].message).startswith( + '`stream` is not a valid parameter' + ) + def test_push(self): client = make_fake_client() client.images.push('foobar', insecure_registry=True) From 24ed2f356bd522f413a5e01765f63f2da3975f31 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 28 Nov 2018 11:48:15 -0800 Subject: [PATCH 0803/1301] Release 3.6.0 Signed-off-by: Joffrey F --- docker/version.py | 2 +- docs/change-log.md | 25 +++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index ef6b491c54..0b27a263a6 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "3.5.1" +version = "3.6.0" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) diff --git a/docs/change-log.md b/docs/change-log.md index 750afb9b69..873db8cef5 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -1,6 +1,31 @@ Change log ========== +3.6.0 +----- + +[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone=55?closed=1) + +### Features + +* Added support for connecting to the Docker Engine over SSH. Additional + dependencies for this feature can be installed with + `pip install "docker[ssh]"` +* Added support for the `named` parameter in `Image.save`, which may be + used to ensure the resulting tarball retains the image's name on save. + +### Bugfixes + +* Fixed a bug where builds on Windows with a context path using the `\\?\` + prefix would fail with some relative Dockerfile paths. +* Fixed an issue where pulls made with the `DockerClient` would fail when + setting the `stream` parameter to `True`. + +### Miscellaneous + +* The minimum requirement for the `requests` dependency has been bumped + to 2.20.0 + 3.5.1 ----- From 6540900dae21571a60fd92037e926cd6599a52eb Mon Sep 17 00:00:00 2001 From: Corentin Henry Date: Tue, 27 Nov 2018 17:01:06 -0800 Subject: [PATCH 0804/1301] add tests for _read_from_socket Check that the return value against the various combination of parameters this function can take (tty, stream, and demux). This commit also fixes a bug that the tests uncovered a bug in consume_socket_output. Signed-off-by: Corentin Henry --- docker/utils/socket.py | 16 ++++--- tests/unit/api_test.py | 98 +++++++++++++++++++++++++++++++++++------- 2 files changed, 91 insertions(+), 23 deletions(-) diff --git a/docker/utils/socket.py b/docker/utils/socket.py index fe4a33266c..4b32853688 100644 --- a/docker/utils/socket.py +++ b/docker/utils/socket.py @@ -136,15 +136,17 @@ def consume_socket_output(frames, demux=False): # we just need to concatenate. return six.binary_type().join(frames) - # If the streams are demultiplexed, the generator returns tuples - # (stdin, stdout, stderr) + # If the streams are demultiplexed, the generator yields tuples + # (stdout, stderr) out = [six.binary_type(), six.binary_type()] for frame in frames: - for stream_id in [STDOUT, STDERR]: - # It is guaranteed that for each frame, one and only one stream - # is not None. - if frame[stream_id] is not None: - out[stream_id] += frame[stream_id] + # It is guaranteed that for each frame, one and only one stream + # is not None. + assert frame != (None, None) + if frame[0] is not None: + out[0] += frame[0] + else: + out[1] += frame[1] return tuple(out) diff --git a/tests/unit/api_test.py b/tests/unit/api_test.py index ccddbb16de..0f5ad7c49e 100644 --- a/tests/unit/api_test.py +++ b/tests/unit/api_test.py @@ -15,6 +15,7 @@ import requests from requests.packages import urllib3 import six +import struct from . import fake_api @@ -467,24 +468,25 @@ def test_early_stream_response(self): class TCPSocketStreamTest(unittest.TestCase): - text_data = b''' + stdout_data = b''' Now, those children out there, they're jumping through the flames in the hope that the god of the fire will make them fruitful. Really, you can't blame them. After all, what girl would not prefer the child of a god to that of some acne-scarred artisan? ''' + stderr_data = b''' + And what of the true God? To whose glory churches and monasteries have been + built on these islands for generations past? Now shall what of Him? + ''' def setUp(self): - self.server = six.moves.socketserver.ThreadingTCPServer( - ('', 0), self.get_handler_class() - ) + ('', 0), self.get_handler_class()) self.thread = threading.Thread(target=self.server.serve_forever) self.thread.setDaemon(True) self.thread.start() self.address = 'http://{}:{}'.format( - socket.gethostname(), self.server.server_address[1] - ) + socket.gethostname(), self.server.server_address[1]) def tearDown(self): self.server.shutdown() @@ -492,31 +494,95 @@ def tearDown(self): self.thread.join() def get_handler_class(self): - text_data = self.text_data + stdout_data = self.stdout_data + stderr_data = self.stderr_data class Handler(six.moves.BaseHTTPServer.BaseHTTPRequestHandler, object): def do_POST(self): + resp_data = self.get_resp_data() self.send_response(101) self.send_header( - 'Content-Type', 'application/vnd.docker.raw-stream' - ) + 'Content-Type', 'application/vnd.docker.raw-stream') self.send_header('Connection', 'Upgrade') self.send_header('Upgrade', 'tcp') self.end_headers() self.wfile.flush() time.sleep(0.2) - self.wfile.write(text_data) + self.wfile.write(resp_data) self.wfile.flush() + def get_resp_data(self): + path = self.path.split('/')[-1] + if path == 'tty': + return stdout_data + stderr_data + elif path == 'no-tty': + data = b'' + data += self.frame_header(1, stdout_data) + data += stdout_data + data += self.frame_header(2, stderr_data) + data += stderr_data + return data + else: + raise Exception('Unknown path {0}'.format(path)) + + @staticmethod + def frame_header(stream, data): + return struct.pack('>BxxxL', stream, len(data)) + return Handler - def test_read_from_socket(self): + def request(self, stream=None, tty=None, demux=None): + assert stream is not None and tty is not None and demux is not None with APIClient(base_url=self.address) as client: - resp = client._post(client._url('/dummy'), stream=True) - data = client._read_from_socket(resp, stream=True, tty=True) - results = b''.join(data) - - assert results == self.text_data + if tty: + url = client._url('/tty') + else: + url = client._url('/no-tty') + resp = client._post(url, stream=True) + return client._read_from_socket( + resp, stream=stream, tty=tty, demux=demux) + + def test_read_from_socket_1(self): + res = self.request(stream=True, tty=True, demux=False) + assert next(res) == self.stdout_data + self.stderr_data + with self.assertRaises(StopIteration): + next(res) + + def test_read_from_socket_2(self): + res = self.request(stream=True, tty=True, demux=True) + assert next(res) == (self.stdout_data + self.stderr_data, None) + with self.assertRaises(StopIteration): + next(res) + + def test_read_from_socket_3(self): + res = self.request(stream=True, tty=False, demux=False) + assert next(res) == self.stdout_data + assert next(res) == self.stderr_data + with self.assertRaises(StopIteration): + next(res) + + def test_read_from_socket_4(self): + res = self.request(stream=True, tty=False, demux=True) + assert (self.stdout_data, None) == next(res) + assert (None, self.stderr_data) == next(res) + with self.assertRaises(StopIteration): + next(res) + + def test_read_from_socket_5(self): + res = self.request(stream=False, tty=True, demux=False) + assert res == self.stdout_data + self.stderr_data + + def test_read_from_socket_6(self): + res = self.request(stream=False, tty=True, demux=True) + assert res == (self.stdout_data + self.stderr_data, b'') + + def test_read_from_socket_7(self): + res = self.request(stream=False, tty=False, demux=False) + res == self.stdout_data + self.stderr_data + + def test_read_from_socket_8(self): + res = self.request(stream=False, tty=False, demux=True) + assert res == (self.stdout_data, self.stderr_data) class UserAgentTest(unittest.TestCase): From 76447d0ca330e32811c95fe7fecdebc8cd5c71c2 Mon Sep 17 00:00:00 2001 From: Corentin Henry Date: Wed, 28 Nov 2018 13:36:28 -0800 Subject: [PATCH 0805/1301] tests various exec_create/exec_start combinations Test the interation of the tty, demux and stream parameters Signed-off-by: Corentin Henry --- tests/integration/api_exec_test.py | 69 ++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/tests/integration/api_exec_test.py b/tests/integration/api_exec_test.py index ac64af7724..fd2f0eae31 100644 --- a/tests/integration/api_exec_test.py +++ b/tests/integration/api_exec_test.py @@ -75,6 +75,75 @@ def test_exec_command_streaming(self): res += chunk assert res == b'hello\nworld\n' + def test_exec_command_demux(self): + container = self.client.create_container( + BUSYBOX, 'cat', detach=True, stdin_open=True) + id = container['Id'] + self.client.start(id) + self.tmp_containers.append(id) + + script = ' ; '.join([ + # Write something on stdout + 'echo hello out', + # Busybox's sleep does not handle sub-second times. + # This loops takes ~0.3 second to execute on my machine. + 'for i in $(seq 1 50000); do echo $i>/dev/null; done', + # Write something on stderr + 'echo hello err >&2']) + cmd = 'sh -c "{}"'.format(script) + + # tty=False, stream=False, demux=False + res = self.client.exec_create(id, cmd) + exec_log = self.client.exec_start(res) + assert exec_log == b'hello out\nhello err\n' + + # tty=False, stream=True, demux=False + res = self.client.exec_create(id, cmd) + exec_log = self.client.exec_start(res, stream=True) + assert next(exec_log) == b'hello out\n' + assert next(exec_log) == b'hello err\n' + with self.assertRaises(StopIteration): + next(exec_log) + + # tty=False, stream=False, demux=True + res = self.client.exec_create(id, cmd) + exec_log = self.client.exec_start(res, demux=True) + assert exec_log == (b'hello out\n', b'hello err\n') + + # tty=False, stream=True, demux=True + res = self.client.exec_create(id, cmd) + exec_log = self.client.exec_start(res, demux=True, stream=True) + assert next(exec_log) == (b'hello out\n', None) + assert next(exec_log) == (None, b'hello err\n') + with self.assertRaises(StopIteration): + next(exec_log) + + # tty=True, stream=False, demux=False + res = self.client.exec_create(id, cmd, tty=True) + exec_log = self.client.exec_start(res) + assert exec_log == b'hello out\r\nhello err\r\n' + + # tty=True, stream=True, demux=False + res = self.client.exec_create(id, cmd, tty=True) + exec_log = self.client.exec_start(res, stream=True) + assert next(exec_log) == b'hello out\r\n' + assert next(exec_log) == b'hello err\r\n' + with self.assertRaises(StopIteration): + next(exec_log) + + # tty=True, stream=False, demux=True + res = self.client.exec_create(id, cmd, tty=True) + exec_log = self.client.exec_start(res, demux=True) + assert exec_log == (b'hello out\r\nhello err\r\n', b'') + + # tty=True, stream=True, demux=True + res = self.client.exec_create(id, cmd, tty=True) + exec_log = self.client.exec_start(res, demux=True, stream=True) + assert next(exec_log) == (b'hello out\r\n', None) + assert next(exec_log) == (b'hello err\r\n', None) + with self.assertRaises(StopIteration): + next(exec_log) + def test_exec_start_socket(self): container = self.client.create_container(BUSYBOX, 'cat', detach=True, stdin_open=True) From 9a67e2032e68eea397f3bf6b727464ed39a02123 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 28 Nov 2018 14:31:28 -0800 Subject: [PATCH 0806/1301] Next dev version Signed-off-by: Joffrey F --- docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index 0b27a263a6..b4cf22bbb2 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "3.6.0" +version = "3.7.0-dev" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) From 41c0eb7e804ef428da27509714df34d26a56b9a6 Mon Sep 17 00:00:00 2001 From: Corentin Henry Date: Wed, 28 Nov 2018 13:59:19 -0800 Subject: [PATCH 0807/1301] fix exec_start() documentation Signed-off-by: Corentin Henry --- docker/api/exec_api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/api/exec_api.py b/docker/api/exec_api.py index 3950991972..d13b128998 100644 --- a/docker/api/exec_api.py +++ b/docker/api/exec_api.py @@ -130,14 +130,14 @@ def exec_start(self, exec_id, detach=False, tty=False, stream=False, stream (bool): Stream response data. Default: False socket (bool): Return the connection socket to allow custom read/write operations. - demux (bool): Separate return stdin, stdout and stderr separately + demux (bool): Return stdout and stderr separately Returns: (generator or str or tuple): If ``stream=True``, a generator yielding response chunks. If ``socket=True``, a socket object for the connection. A string containing response data otherwise. If - ``demux=True``, stdin, stdout and stderr are separated. + ``demux=True``, stdout and stderr are separated. Raises: :py:class:`docker.errors.APIError` From 7b3b83dfdbb9f4270dcf54e1449645efc045dfd3 Mon Sep 17 00:00:00 2001 From: Corentin Henry Date: Wed, 28 Nov 2018 14:32:12 -0800 Subject: [PATCH 0808/1301] fix exec api inconsistency Signed-off-by: Corentin Henry --- docker/utils/socket.py | 12 +++++++++--- tests/integration/api_exec_test.py | 2 +- tests/unit/api_test.py | 2 +- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/docker/utils/socket.py b/docker/utils/socket.py index 4b32853688..7ba9505538 100644 --- a/docker/utils/socket.py +++ b/docker/utils/socket.py @@ -138,15 +138,21 @@ def consume_socket_output(frames, demux=False): # If the streams are demultiplexed, the generator yields tuples # (stdout, stderr) - out = [six.binary_type(), six.binary_type()] + out = [None, None] for frame in frames: # It is guaranteed that for each frame, one and only one stream # is not None. assert frame != (None, None) if frame[0] is not None: - out[0] += frame[0] + if out[0] is None: + out[0] = frame[0] + else: + out[0] += frame[0] else: - out[1] += frame[1] + if out[1] is None: + out[1] = frame[1] + else: + out[1] += frame[1] return tuple(out) diff --git a/tests/integration/api_exec_test.py b/tests/integration/api_exec_test.py index fd2f0eae31..857a18cb3f 100644 --- a/tests/integration/api_exec_test.py +++ b/tests/integration/api_exec_test.py @@ -134,7 +134,7 @@ def test_exec_command_demux(self): # tty=True, stream=False, demux=True res = self.client.exec_create(id, cmd, tty=True) exec_log = self.client.exec_start(res, demux=True) - assert exec_log == (b'hello out\r\nhello err\r\n', b'') + assert exec_log == (b'hello out\r\nhello err\r\n', None) # tty=True, stream=True, demux=True res = self.client.exec_create(id, cmd, tty=True) diff --git a/tests/unit/api_test.py b/tests/unit/api_test.py index 0f5ad7c49e..fac314d3d2 100644 --- a/tests/unit/api_test.py +++ b/tests/unit/api_test.py @@ -574,7 +574,7 @@ def test_read_from_socket_5(self): def test_read_from_socket_6(self): res = self.request(stream=False, tty=True, demux=True) - assert res == (self.stdout_data + self.stderr_data, b'') + assert res == (self.stdout_data + self.stderr_data, None) def test_read_from_socket_7(self): res = self.request(stream=False, tty=False, demux=False) From bc5d7c8cb676d54ad00b8cd8e731a9db049c392c Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 28 Nov 2018 19:32:01 -0800 Subject: [PATCH 0809/1301] Modernize auth management Signed-off-by: Joffrey F --- docker/api/build.py | 37 +--- docker/api/daemon.py | 16 +- docker/auth.py | 388 +++++++++++++++++++++-------------- requirements.txt | 2 +- setup.py | 2 +- tests/unit/api_build_test.py | 18 +- tests/unit/api_test.py | 12 +- tests/unit/auth_test.py | 56 ++--- 8 files changed, 294 insertions(+), 237 deletions(-) diff --git a/docker/api/build.py b/docker/api/build.py index 3a67ff8b28..1723083bc7 100644 --- a/docker/api/build.py +++ b/docker/api/build.py @@ -293,31 +293,11 @@ def _set_auth_headers(self, headers): # Send the full auth configuration (if any exists), since the build # could use any (or all) of the registries. if self._auth_configs: - auth_cfgs = self._auth_configs - auth_data = {} - if auth_cfgs.get('credsStore'): - # Using a credentials store, we need to retrieve the - # credentials for each registry listed in the config.json file - # Matches CLI behavior: https://github.com/docker/docker/blob/ - # 67b85f9d26f1b0b2b240f2d794748fac0f45243c/cliconfig/ - # credentials/native_store.go#L68-L83 - for registry in auth_cfgs.get('auths', {}).keys(): - auth_data[registry] = auth.resolve_authconfig( - auth_cfgs, registry, - credstore_env=self.credstore_env, - ) - else: - for registry in auth_cfgs.get('credHelpers', {}).keys(): - auth_data[registry] = auth.resolve_authconfig( - auth_cfgs, registry, - credstore_env=self.credstore_env - ) - for registry, creds in auth_cfgs.get('auths', {}).items(): - if registry not in auth_data: - auth_data[registry] = creds - # See https://github.com/docker/docker-py/issues/1683 - if auth.INDEX_NAME in auth_data: - auth_data[auth.INDEX_URL] = auth_data[auth.INDEX_NAME] + auth_data = self._auth_configs.get_all_credentials() + + # See https://github.com/docker/docker-py/issues/1683 + if auth.INDEX_URL not in auth_data and auth.INDEX_URL in auth_data: + auth_data[auth.INDEX_URL] = auth_data.get(auth.INDEX_NAME, {}) log.debug( 'Sending auth config ({0})'.format( @@ -325,9 +305,10 @@ def _set_auth_headers(self, headers): ) ) - headers['X-Registry-Config'] = auth.encode_header( - auth_data - ) + if auth_data: + headers['X-Registry-Config'] = auth.encode_header( + auth_data + ) else: log.debug('No auth config found') diff --git a/docker/api/daemon.py b/docker/api/daemon.py index 431e7d41cd..a2936f2a0e 100644 --- a/docker/api/daemon.py +++ b/docker/api/daemon.py @@ -124,13 +124,15 @@ def login(self, username, password=None, email=None, registry=None, # If dockercfg_path is passed check to see if the config file exists, # if so load that config. if dockercfg_path and os.path.exists(dockercfg_path): - self._auth_configs = auth.load_config(dockercfg_path) + self._auth_configs = auth.load_config( + dockercfg_path, credstore_env=self.credstore_env + ) elif not self._auth_configs: - self._auth_configs = auth.load_config() + self._auth_configs = auth.load_config( + credstore_env=self.credstore_env + ) - authcfg = auth.resolve_authconfig( - self._auth_configs, registry, credstore_env=self.credstore_env, - ) + authcfg = self._auth_configs.resolve_authconfig(registry) # If we found an existing auth config for this registry and username # combination, we can return it immediately unless reauth is requested. if authcfg and authcfg.get('username', None) == username \ @@ -146,9 +148,7 @@ def login(self, username, password=None, email=None, registry=None, response = self._post_json(self._url('/auth'), data=req_data) if response.status_code == 200: - if 'auths' not in self._auth_configs: - self._auth_configs['auths'] = {} - self._auth_configs['auths'][registry or auth.INDEX_NAME] = req_data + self._auth_configs.add_auth(registry or auth.INDEX_NAME, req_data) return self._result(response, json=True) def ping(self): diff --git a/docker/auth.py b/docker/auth.py index 17158f4ae3..a6c8ae16a3 100644 --- a/docker/auth.py +++ b/docker/auth.py @@ -70,81 +70,246 @@ def split_repo_name(repo_name): def get_credential_store(authconfig, registry): - if not registry or registry == INDEX_NAME: - registry = 'https://index.docker.io/v1/' + return authconfig.get_credential_store(registry) + + +class AuthConfig(object): + def __init__(self, dct, credstore_env=None): + if 'auths' not in dct: + dct['auths'] = {} + self._dct = dct + self._credstore_env = credstore_env + self._stores = {} + + @classmethod + def parse_auth(cls, entries, raise_on_error=False): + """ + Parses authentication entries + + Args: + entries: Dict of authentication entries. + raise_on_error: If set to true, an invalid format will raise + InvalidConfigFile + + Returns: + Authentication registry. + """ + + conf = {} + for registry, entry in six.iteritems(entries): + if not isinstance(entry, dict): + log.debug( + 'Config entry for key {0} is not auth config'.format( + registry + ) + ) + # We sometimes fall back to parsing the whole config as if it + # was the auth config by itself, for legacy purposes. In that + # case, we fail silently and return an empty conf if any of the + # keys is not formatted properly. + if raise_on_error: + raise errors.InvalidConfigFile( + 'Invalid configuration for registry {0}'.format( + registry + ) + ) + return {} + if 'identitytoken' in entry: + log.debug( + 'Found an IdentityToken entry for registry {0}'.format( + registry + ) + ) + conf[registry] = { + 'IdentityToken': entry['identitytoken'] + } + continue # Other values are irrelevant if we have a token + + if 'auth' not in entry: + # Starting with engine v1.11 (API 1.23), an empty dictionary is + # a valid value in the auths config. + # https://github.com/docker/compose/issues/3265 + log.debug( + 'Auth data for {0} is absent. Client might be using a ' + 'credentials store instead.'.format(registry) + ) + conf[registry] = {} + continue - return authconfig.get('credHelpers', {}).get(registry) or authconfig.get( - 'credsStore' - ) + username, password = decode_auth(entry['auth']) + log.debug( + 'Found entry (registry={0}, username={1})' + .format(repr(registry), repr(username)) + ) + conf[registry] = { + 'username': username, + 'password': password, + 'email': entry.get('email'), + 'serveraddress': registry, + } + return conf + + @classmethod + def load_config(cls, config_path, config_dict, credstore_env=None): + """ + Loads authentication data from a Docker configuration file in the given + root directory or if config_path is passed use given path. + Lookup priority: + explicit config_path parameter > DOCKER_CONFIG environment + variable > ~/.docker/config.json > ~/.dockercfg + """ + + if not config_dict: + config_file = config.find_config_file(config_path) + + if not config_file: + return cls({}, credstore_env) + try: + with open(config_file) as f: + config_dict = json.load(f) + except (IOError, KeyError, ValueError) as e: + # Likely missing new Docker config file or it's in an + # unknown format, continue to attempt to read old location + # and format. + log.debug(e) + return cls(_load_legacy_config(config_file), credstore_env) + + res = {} + if config_dict.get('auths'): + log.debug("Found 'auths' section") + res.update({ + 'auths': cls.parse_auth( + config_dict.pop('auths'), raise_on_error=True + ) + }) + if config_dict.get('credsStore'): + log.debug("Found 'credsStore' section") + res.update({'credsStore': config_dict.pop('credsStore')}) + if config_dict.get('credHelpers'): + log.debug("Found 'credHelpers' section") + res.update({'credHelpers': config_dict.pop('credHelpers')}) + if res: + return cls(res, credstore_env) -def resolve_authconfig(authconfig, registry=None, credstore_env=None): - """ - Returns the authentication data from the given auth configuration for a - specific registry. As with the Docker client, legacy entries in the config - with full URLs are stripped down to hostnames before checking for a match. - Returns None if no match was found. - """ + log.debug( + "Couldn't find auth-related section ; attempting to interpret " + "as auth-only file" + ) + return cls({'auths': cls.parse_auth(config_dict)}, credstore_env) + + @property + def auths(self): + return self._dct.get('auths', {}) + + @property + def creds_store(self): + return self._dct.get('credsStore', None) + + @property + def cred_helpers(self): + return self._dct.get('credHelpers', {}) + + def resolve_authconfig(self, registry=None): + """ + Returns the authentication data from the given auth configuration for a + specific registry. As with the Docker client, legacy entries in the + config with full URLs are stripped down to hostnames before checking + for a match. Returns None if no match was found. + """ + + if self.creds_store or self.cred_helpers: + store_name = self.get_credential_store(registry) + if store_name is not None: + log.debug( + 'Using credentials store "{0}"'.format(store_name) + ) + cfg = self._resolve_authconfig_credstore(registry, store_name) + if cfg is not None: + return cfg + log.debug('No entry in credstore - fetching from auth dict') - if 'credHelpers' in authconfig or 'credsStore' in authconfig: - store_name = get_credential_store(authconfig, registry) - if store_name is not None: - log.debug( - 'Using credentials store "{0}"'.format(store_name) - ) - cfg = _resolve_authconfig_credstore( - authconfig, registry, store_name, env=credstore_env + # Default to the public index server + registry = resolve_index_name(registry) if registry else INDEX_NAME + log.debug("Looking for auth entry for {0}".format(repr(registry))) + + if registry in self.auths: + log.debug("Found {0}".format(repr(registry))) + return self.auths[registry] + + for key, conf in six.iteritems(self.auths): + if resolve_index_name(key) == registry: + log.debug("Found {0}".format(repr(key))) + return conf + + log.debug("No entry found") + return None + + def _resolve_authconfig_credstore(self, registry, credstore_name): + if not registry or registry == INDEX_NAME: + # The ecosystem is a little schizophrenic with index.docker.io VS + # docker.io - in that case, it seems the full URL is necessary. + registry = INDEX_URL + log.debug("Looking for auth entry for {0}".format(repr(registry))) + store = self._get_store_instance(credstore_name) + try: + data = store.get(registry) + res = { + 'ServerAddress': registry, + } + if data['Username'] == TOKEN_USERNAME: + res['IdentityToken'] = data['Secret'] + else: + res.update({ + 'Username': data['Username'], + 'Password': data['Secret'], + }) + return res + except dockerpycreds.CredentialsNotFound as e: + log.debug('No entry found') + return None + except dockerpycreds.StoreError as e: + raise errors.DockerException( + 'Credentials store error: {0}'.format(repr(e)) ) - if cfg is not None: - return cfg - log.debug('No entry in credstore - fetching from auth dict') - # Default to the public index server - registry = resolve_index_name(registry) if registry else INDEX_NAME - log.debug("Looking for auth entry for {0}".format(repr(registry))) + def _get_store_instance(self, name): + if name not in self._stores: + self._stores[name] = dockerpycreds.Store( + name, environment=self._credstore_env + ) + return self._stores[name] + + def get_credential_store(self, registry): + if not registry or registry == INDEX_NAME: + registry = 'https://index.docker.io/v1/' + + return self.cred_helpers.get(registry) or self.creds_store + + def get_all_credentials(self): + auth_data = self.auths.copy() + if self.creds_store: + # Retrieve all credentials from the default store + store = self._get_store_instance(self.creds_store) + for k in store.list().keys(): + auth_data[k] = self._resolve_authconfig_credstore( + k, self.creds_store + ) - authdict = authconfig.get('auths', {}) - if registry in authdict: - log.debug("Found {0}".format(repr(registry))) - return authdict[registry] + # credHelpers entries take priority over all others + for reg, store_name in self.cred_helpers.items(): + auth_data[reg] = self._resolve_authconfig_credstore( + reg, store_name + ) - for key, conf in six.iteritems(authdict): - if resolve_index_name(key) == registry: - log.debug("Found {0}".format(repr(key))) - return conf + return auth_data - log.debug("No entry found") - return None + def add_auth(self, reg, data): + self._dct['auths'][reg] = data -def _resolve_authconfig_credstore(authconfig, registry, credstore_name, - env=None): - if not registry or registry == INDEX_NAME: - # The ecosystem is a little schizophrenic with index.docker.io VS - # docker.io - in that case, it seems the full URL is necessary. - registry = INDEX_URL - log.debug("Looking for auth entry for {0}".format(repr(registry))) - store = dockerpycreds.Store(credstore_name, environment=env) - try: - data = store.get(registry) - res = { - 'ServerAddress': registry, - } - if data['Username'] == TOKEN_USERNAME: - res['IdentityToken'] = data['Secret'] - else: - res.update({ - 'Username': data['Username'], - 'Password': data['Secret'], - }) - return res - except dockerpycreds.CredentialsNotFound as e: - log.debug('No entry found') - return None - except dockerpycreds.StoreError as e: - raise errors.DockerException( - 'Credentials store error: {0}'.format(repr(e)) - ) +def resolve_authconfig(authconfig, registry=None, credstore_env=None): + return authconfig.resolve_authconfig(registry) def convert_to_hostname(url): @@ -177,100 +342,11 @@ def parse_auth(entries, raise_on_error=False): Authentication registry. """ - conf = {} - for registry, entry in six.iteritems(entries): - if not isinstance(entry, dict): - log.debug( - 'Config entry for key {0} is not auth config'.format(registry) - ) - # We sometimes fall back to parsing the whole config as if it was - # the auth config by itself, for legacy purposes. In that case, we - # fail silently and return an empty conf if any of the keys is not - # formatted properly. - if raise_on_error: - raise errors.InvalidConfigFile( - 'Invalid configuration for registry {0}'.format(registry) - ) - return {} - if 'identitytoken' in entry: - log.debug('Found an IdentityToken entry for registry {0}'.format( - registry - )) - conf[registry] = { - 'IdentityToken': entry['identitytoken'] - } - continue # Other values are irrelevant if we have a token, skip. - - if 'auth' not in entry: - # Starting with engine v1.11 (API 1.23), an empty dictionary is - # a valid value in the auths config. - # https://github.com/docker/compose/issues/3265 - log.debug( - 'Auth data for {0} is absent. Client might be using a ' - 'credentials store instead.'.format(registry) - ) - conf[registry] = {} - continue - - username, password = decode_auth(entry['auth']) - log.debug( - 'Found entry (registry={0}, username={1})' - .format(repr(registry), repr(username)) - ) - - conf[registry] = { - 'username': username, - 'password': password, - 'email': entry.get('email'), - 'serveraddress': registry, - } - return conf - + return AuthConfig.parse_auth(entries, raise_on_error) -def load_config(config_path=None, config_dict=None): - """ - Loads authentication data from a Docker configuration file in the given - root directory or if config_path is passed use given path. - Lookup priority: - explicit config_path parameter > DOCKER_CONFIG environment variable > - ~/.docker/config.json > ~/.dockercfg - """ - if not config_dict: - config_file = config.find_config_file(config_path) - - if not config_file: - return {} - try: - with open(config_file) as f: - config_dict = json.load(f) - except (IOError, KeyError, ValueError) as e: - # Likely missing new Docker config file or it's in an - # unknown format, continue to attempt to read old location - # and format. - log.debug(e) - return _load_legacy_config(config_file) - - res = {} - if config_dict.get('auths'): - log.debug("Found 'auths' section") - res.update({ - 'auths': parse_auth(config_dict.pop('auths'), raise_on_error=True) - }) - if config_dict.get('credsStore'): - log.debug("Found 'credsStore' section") - res.update({'credsStore': config_dict.pop('credsStore')}) - if config_dict.get('credHelpers'): - log.debug("Found 'credHelpers' section") - res.update({'credHelpers': config_dict.pop('credHelpers')}) - if res: - return res - - log.debug( - "Couldn't find auth-related section ; attempting to interpret " - "as auth-only file" - ) - return {'auths': parse_auth(config_dict)} +def load_config(config_path=None, config_dict=None, credstore_env=None): + return AuthConfig.load_config(config_path, config_dict, credstore_env) def _load_legacy_config(config_file): diff --git a/requirements.txt b/requirements.txt index d13e9d6cad..f1c9bdbc76 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ backports.ssl-match-hostname==3.5.0.1 cffi==1.10.0 cryptography==1.9; python_version == '3.3' cryptography==2.3; python_version > '3.3' -docker-pycreds==0.3.0 +docker-pycreds==0.4.0 enum34==1.1.6 idna==2.5 ipaddress==1.0.18 diff --git a/setup.py b/setup.py index 3ad572b3ec..4ce55fe815 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ requirements = [ 'six >= 1.4.0', 'websocket-client >= 0.32.0', - 'docker-pycreds >= 0.3.0', + 'docker-pycreds >= 0.4.0', 'requests >= 2.14.2, != 2.18.0', ] diff --git a/tests/unit/api_build_test.py b/tests/unit/api_build_test.py index 59470caa5f..7e07a2695e 100644 --- a/tests/unit/api_build_test.py +++ b/tests/unit/api_build_test.py @@ -65,7 +65,7 @@ def test_build_container_custom_context_gzip(self): ) def test_build_remote_with_registry_auth(self): - self.client._auth_configs = { + self.client._auth_configs = auth.AuthConfig({ 'auths': { 'https://example.com': { 'user': 'example', @@ -73,7 +73,7 @@ def test_build_remote_with_registry_auth(self): 'email': 'example@example.com' } } - } + }) expected_params = {'t': None, 'q': False, 'dockerfile': None, 'rm': False, 'nocache': False, 'pull': False, @@ -81,7 +81,7 @@ def test_build_remote_with_registry_auth(self): 'remote': 'https://github.com/docker-library/mongo'} expected_headers = { 'X-Registry-Config': auth.encode_header( - self.client._auth_configs['auths'] + self.client._auth_configs.auths ) } @@ -115,7 +115,7 @@ def test_build_container_invalid_container_limits(self): }) def test_set_auth_headers_with_empty_dict_and_auth_configs(self): - self.client._auth_configs = { + self.client._auth_configs = auth.AuthConfig({ 'auths': { 'https://example.com': { 'user': 'example', @@ -123,12 +123,12 @@ def test_set_auth_headers_with_empty_dict_and_auth_configs(self): 'email': 'example@example.com' } } - } + }) headers = {} expected_headers = { 'X-Registry-Config': auth.encode_header( - self.client._auth_configs['auths'] + self.client._auth_configs.auths ) } @@ -136,7 +136,7 @@ def test_set_auth_headers_with_empty_dict_and_auth_configs(self): assert headers == expected_headers def test_set_auth_headers_with_dict_and_auth_configs(self): - self.client._auth_configs = { + self.client._auth_configs = auth.AuthConfig({ 'auths': { 'https://example.com': { 'user': 'example', @@ -144,12 +144,12 @@ def test_set_auth_headers_with_dict_and_auth_configs(self): 'email': 'example@example.com' } } - } + }) headers = {'foo': 'bar'} expected_headers = { 'X-Registry-Config': auth.encode_header( - self.client._auth_configs['auths'] + self.client._auth_configs.auths ), 'foo': 'bar' } diff --git a/tests/unit/api_test.py b/tests/unit/api_test.py index af2bb1c202..14399fe285 100644 --- a/tests/unit/api_test.py +++ b/tests/unit/api_test.py @@ -221,13 +221,11 @@ def test_login(self): 'username': 'sakuya', 'password': 'izayoi' } assert args[1]['headers'] == {'Content-Type': 'application/json'} - assert self.client._auth_configs['auths'] == { - 'docker.io': { - 'email': None, - 'password': 'izayoi', - 'username': 'sakuya', - 'serveraddress': None, - } + assert self.client._auth_configs.auths['docker.io'] == { + 'email': None, + 'password': 'izayoi', + 'username': 'sakuya', + 'serveraddress': None, } def test_events(self): diff --git a/tests/unit/auth_test.py b/tests/unit/auth_test.py index 947d680018..4ad74d5d68 100644 --- a/tests/unit/auth_test.py +++ b/tests/unit/auth_test.py @@ -106,13 +106,13 @@ class ResolveAuthTest(unittest.TestCase): private_config = {'auth': encode_auth({'username': 'privateuser'})} legacy_config = {'auth': encode_auth({'username': 'legacyauth'})} - auth_config = { + auth_config = auth.AuthConfig({ 'auths': auth.parse_auth({ 'https://index.docker.io/v1/': index_config, 'my.registry.net': private_config, 'http://legacy.registry.url/v1/': legacy_config, }) - } + }) def test_resolve_authconfig_hostname_only(self): assert auth.resolve_authconfig( @@ -211,13 +211,15 @@ def test_resolve_registry_and_auth_unauthenticated_registry(self): ) is None def test_resolve_auth_with_empty_credstore_and_auth_dict(self): - auth_config = { + auth_config = auth.AuthConfig({ 'auths': auth.parse_auth({ 'https://index.docker.io/v1/': self.index_config, }), 'credsStore': 'blackbox' - } - with mock.patch('docker.auth._resolve_authconfig_credstore') as m: + }) + with mock.patch( + 'docker.auth.AuthConfig._resolve_authconfig_credstore' + ) as m: m.return_value = None assert 'indexuser' == auth.resolve_authconfig( auth_config, None @@ -226,13 +228,13 @@ def test_resolve_auth_with_empty_credstore_and_auth_dict(self): class CredStoreTest(unittest.TestCase): def test_get_credential_store(self): - auth_config = { + auth_config = auth.AuthConfig({ 'credHelpers': { 'registry1.io': 'truesecret', 'registry2.io': 'powerlock' }, 'credsStore': 'blackbox', - } + }) assert auth.get_credential_store( auth_config, 'registry1.io' @@ -245,12 +247,12 @@ def test_get_credential_store(self): ) == 'blackbox' def test_get_credential_store_no_default(self): - auth_config = { + auth_config = auth.AuthConfig({ 'credHelpers': { 'registry1.io': 'truesecret', 'registry2.io': 'powerlock' }, - } + }) assert auth.get_credential_store( auth_config, 'registry2.io' ) == 'powerlock' @@ -259,12 +261,12 @@ def test_get_credential_store_no_default(self): ) is None def test_get_credential_store_default_index(self): - auth_config = { + auth_config = auth.AuthConfig({ 'credHelpers': { 'https://index.docker.io/v1/': 'powerlock' }, 'credsStore': 'truesecret' - } + }) assert auth.get_credential_store(auth_config, None) == 'powerlock' assert auth.get_credential_store( @@ -293,8 +295,8 @@ def test_load_legacy_config(self): cfg = auth.load_config(cfg_path) assert auth.resolve_authconfig(cfg) is not None - assert cfg['auths'][auth.INDEX_NAME] is not None - cfg = cfg['auths'][auth.INDEX_NAME] + assert cfg.auths[auth.INDEX_NAME] is not None + cfg = cfg.auths[auth.INDEX_NAME] assert cfg['username'] == 'sakuya' assert cfg['password'] == 'izayoi' assert cfg['email'] == 'sakuya@scarlet.net' @@ -312,8 +314,8 @@ def test_load_json_config(self): ) cfg = auth.load_config(cfg_path) assert auth.resolve_authconfig(cfg) is not None - assert cfg['auths'][auth.INDEX_URL] is not None - cfg = cfg['auths'][auth.INDEX_URL] + assert cfg.auths[auth.INDEX_URL] is not None + cfg = cfg.auths[auth.INDEX_URL] assert cfg['username'] == 'sakuya' assert cfg['password'] == 'izayoi' assert cfg['email'] == email @@ -335,8 +337,8 @@ def test_load_modern_json_config(self): }, f) cfg = auth.load_config(cfg_path) assert auth.resolve_authconfig(cfg) is not None - assert cfg['auths'][auth.INDEX_URL] is not None - cfg = cfg['auths'][auth.INDEX_URL] + assert cfg.auths[auth.INDEX_URL] is not None + cfg = cfg.auths[auth.INDEX_URL] assert cfg['username'] == 'sakuya' assert cfg['password'] == 'izayoi' assert cfg['email'] == email @@ -360,7 +362,7 @@ def test_load_config_with_random_name(self): with open(dockercfg_path, 'w') as f: json.dump(config, f) - cfg = auth.load_config(dockercfg_path)['auths'] + cfg = auth.load_config(dockercfg_path).auths assert registry in cfg assert cfg[registry] is not None cfg = cfg[registry] @@ -387,7 +389,7 @@ def test_load_config_custom_config_env(self): json.dump(config, f) with mock.patch.dict(os.environ, {'DOCKER_CONFIG': folder}): - cfg = auth.load_config(None)['auths'] + cfg = auth.load_config(None).auths assert registry in cfg assert cfg[registry] is not None cfg = cfg[registry] @@ -417,8 +419,8 @@ def test_load_config_custom_config_env_with_auths(self): with mock.patch.dict(os.environ, {'DOCKER_CONFIG': folder}): cfg = auth.load_config(None) - assert registry in cfg['auths'] - cfg = cfg['auths'][registry] + assert registry in cfg.auths + cfg = cfg.auths[registry] assert cfg['username'] == 'sakuya' assert cfg['password'] == 'izayoi' assert cfg['email'] == 'sakuya@scarlet.net' @@ -446,8 +448,8 @@ def test_load_config_custom_config_env_utf8(self): with mock.patch.dict(os.environ, {'DOCKER_CONFIG': folder}): cfg = auth.load_config(None) - assert registry in cfg['auths'] - cfg = cfg['auths'][registry] + assert registry in cfg.auths + cfg = cfg.auths[registry] assert cfg['username'] == b'sakuya\xc3\xa6'.decode('utf8') assert cfg['password'] == b'izayoi\xc3\xa6'.decode('utf8') assert cfg['email'] == 'sakuya@scarlet.net' @@ -464,7 +466,7 @@ def test_load_config_unknown_keys(self): json.dump(config, f) cfg = auth.load_config(dockercfg_path) - assert cfg == {'auths': {}} + assert cfg._dct == {'auths': {}} def test_load_config_invalid_auth_dict(self): folder = tempfile.mkdtemp() @@ -479,7 +481,7 @@ def test_load_config_invalid_auth_dict(self): json.dump(config, f) cfg = auth.load_config(dockercfg_path) - assert cfg == {'auths': {'scarlet.net': {}}} + assert cfg._dct == {'auths': {'scarlet.net': {}}} def test_load_config_identity_token(self): folder = tempfile.mkdtemp() @@ -500,7 +502,7 @@ def test_load_config_identity_token(self): json.dump(config, f) cfg = auth.load_config(dockercfg_path) - assert registry in cfg['auths'] - cfg = cfg['auths'][registry] + assert registry in cfg.auths + cfg = cfg.auths[registry] assert 'IdentityToken' in cfg assert cfg['IdentityToken'] == token From 01ccaa6af2106f01b9804177782622f12525b8a5 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 30 Nov 2018 13:51:01 -0800 Subject: [PATCH 0810/1301] Make AuthConfig a dict subclass for backward-compatibility Signed-off-by: Joffrey F --- docker/auth.py | 18 +++++++++++------- tests/unit/auth_test.py | 4 ++-- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/docker/auth.py b/docker/auth.py index a6c8ae16a3..462390b1d1 100644 --- a/docker/auth.py +++ b/docker/auth.py @@ -43,7 +43,7 @@ def get_config_header(client, registry): log.debug( "No auth config in memory - loading from filesystem" ) - client._auth_configs = load_config() + client._auth_configs = load_config(credstore_env=client.credstore_env) authcfg = resolve_authconfig( client._auth_configs, registry, credstore_env=client.credstore_env ) @@ -70,14 +70,16 @@ def split_repo_name(repo_name): def get_credential_store(authconfig, registry): + if not isinstance(authconfig, AuthConfig): + authconfig = AuthConfig(authconfig) return authconfig.get_credential_store(registry) -class AuthConfig(object): +class AuthConfig(dict): def __init__(self, dct, credstore_env=None): if 'auths' not in dct: dct['auths'] = {} - self._dct = dct + self.update(dct) self._credstore_env = credstore_env self._stores = {} @@ -200,15 +202,15 @@ def load_config(cls, config_path, config_dict, credstore_env=None): @property def auths(self): - return self._dct.get('auths', {}) + return self.get('auths', {}) @property def creds_store(self): - return self._dct.get('credsStore', None) + return self.get('credsStore', None) @property def cred_helpers(self): - return self._dct.get('credHelpers', {}) + return self.get('credHelpers', {}) def resolve_authconfig(self, registry=None): """ @@ -305,10 +307,12 @@ def get_all_credentials(self): return auth_data def add_auth(self, reg, data): - self._dct['auths'][reg] = data + self['auths'][reg] = data def resolve_authconfig(authconfig, registry=None, credstore_env=None): + if not isinstance(authconfig, AuthConfig): + authconfig = AuthConfig(authconfig, credstore_env) return authconfig.resolve_authconfig(registry) diff --git a/tests/unit/auth_test.py b/tests/unit/auth_test.py index 4ad74d5d68..d3c8eee621 100644 --- a/tests/unit/auth_test.py +++ b/tests/unit/auth_test.py @@ -466,7 +466,7 @@ def test_load_config_unknown_keys(self): json.dump(config, f) cfg = auth.load_config(dockercfg_path) - assert cfg._dct == {'auths': {}} + assert dict(cfg) == {'auths': {}} def test_load_config_invalid_auth_dict(self): folder = tempfile.mkdtemp() @@ -481,7 +481,7 @@ def test_load_config_invalid_auth_dict(self): json.dump(config, f) cfg = auth.load_config(dockercfg_path) - assert cfg._dct == {'auths': {'scarlet.net': {}}} + assert dict(cfg) == {'auths': {'scarlet.net': {}}} def test_load_config_identity_token(self): folder = tempfile.mkdtemp() From bef10ecac1692146fd770dcd0a098f28860bce13 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 30 Nov 2018 13:51:40 -0800 Subject: [PATCH 0811/1301] Add credstore_env to all load_config calls Signed-off-by: Joffrey F --- docker/api/build.py | 4 +++- docker/api/client.py | 6 ++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/docker/api/build.py b/docker/api/build.py index 1723083bc7..c4fc37ec98 100644 --- a/docker/api/build.py +++ b/docker/api/build.py @@ -288,7 +288,9 @@ def _set_auth_headers(self, headers): # file one more time in case anything showed up in there. if not self._auth_configs: log.debug("No auth config in memory - loading from filesystem") - self._auth_configs = auth.load_config() + self._auth_configs = auth.load_config( + credsore_env=self.credsore_env + ) # Send the full auth configuration (if any exists), since the build # could use any (or all) of the registries. diff --git a/docker/api/client.py b/docker/api/client.py index 197846d105..74c4698875 100644 --- a/docker/api/client.py +++ b/docker/api/client.py @@ -115,7 +115,7 @@ def __init__(self, base_url=None, version=None, self._general_configs = config.load_general_config() self._auth_configs = auth.load_config( - config_dict=self._general_configs + config_dict=self._general_configs, credstore_env=credstore_env, ) self.credstore_env = credstore_env @@ -476,4 +476,6 @@ def reload_config(self, dockercfg_path=None): Returns: None """ - self._auth_configs = auth.load_config(dockercfg_path) + self._auth_configs = auth.load_config( + dockercfg_path, credstore_env=self.credstore_env + ) From cc38efa68e6640933f19481b4caf5fb21c7b0564 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 30 Nov 2018 14:41:56 -0800 Subject: [PATCH 0812/1301] Add some credHelpers tests Signed-off-by: Joffrey F --- docker/auth.py | 2 +- tests/unit/auth_test.py | 281 ++++++++++++++++++++++++++++++++-------- 2 files changed, 231 insertions(+), 52 deletions(-) diff --git a/docker/auth.py b/docker/auth.py index 462390b1d1..c1b874f870 100644 --- a/docker/auth.py +++ b/docker/auth.py @@ -284,7 +284,7 @@ def _get_store_instance(self, name): def get_credential_store(self, registry): if not registry or registry == INDEX_NAME: - registry = 'https://index.docker.io/v1/' + registry = INDEX_URL return self.cred_helpers.get(registry) or self.creds_store diff --git a/tests/unit/auth_test.py b/tests/unit/auth_test.py index d3c8eee621..dc4d6f59ad 100644 --- a/tests/unit/auth_test.py +++ b/tests/unit/auth_test.py @@ -10,6 +10,7 @@ import unittest from docker import auth, errors +import dockerpycreds import pytest try: @@ -226,57 +227,6 @@ def test_resolve_auth_with_empty_credstore_and_auth_dict(self): )['username'] -class CredStoreTest(unittest.TestCase): - def test_get_credential_store(self): - auth_config = auth.AuthConfig({ - 'credHelpers': { - 'registry1.io': 'truesecret', - 'registry2.io': 'powerlock' - }, - 'credsStore': 'blackbox', - }) - - assert auth.get_credential_store( - auth_config, 'registry1.io' - ) == 'truesecret' - assert auth.get_credential_store( - auth_config, 'registry2.io' - ) == 'powerlock' - assert auth.get_credential_store( - auth_config, 'registry3.io' - ) == 'blackbox' - - def test_get_credential_store_no_default(self): - auth_config = auth.AuthConfig({ - 'credHelpers': { - 'registry1.io': 'truesecret', - 'registry2.io': 'powerlock' - }, - }) - assert auth.get_credential_store( - auth_config, 'registry2.io' - ) == 'powerlock' - assert auth.get_credential_store( - auth_config, 'registry3.io' - ) is None - - def test_get_credential_store_default_index(self): - auth_config = auth.AuthConfig({ - 'credHelpers': { - 'https://index.docker.io/v1/': 'powerlock' - }, - 'credsStore': 'truesecret' - }) - - assert auth.get_credential_store(auth_config, None) == 'powerlock' - assert auth.get_credential_store( - auth_config, 'docker.io' - ) == 'powerlock' - assert auth.get_credential_store( - auth_config, 'images.io' - ) == 'truesecret' - - class LoadConfigTest(unittest.TestCase): def test_load_config_no_file(self): folder = tempfile.mkdtemp() @@ -506,3 +456,232 @@ def test_load_config_identity_token(self): cfg = cfg.auths[registry] assert 'IdentityToken' in cfg assert cfg['IdentityToken'] == token + + +class CredstoreTest(unittest.TestCase): + def setUp(self): + self.authconfig = auth.AuthConfig({'credsStore': 'default'}) + self.default_store = InMemoryStore('default') + self.authconfig._stores['default'] = self.default_store + self.default_store.store( + 'https://gensokyo.jp/v2', 'sakuya', 'izayoi', + ) + self.default_store.store( + 'https://default.com/v2', 'user', 'hunter2', + ) + + def test_get_credential_store(self): + auth_config = auth.AuthConfig({ + 'credHelpers': { + 'registry1.io': 'truesecret', + 'registry2.io': 'powerlock' + }, + 'credsStore': 'blackbox', + }) + + assert auth_config.get_credential_store('registry1.io') == 'truesecret' + assert auth_config.get_credential_store('registry2.io') == 'powerlock' + assert auth_config.get_credential_store('registry3.io') == 'blackbox' + + def test_get_credential_store_no_default(self): + auth_config = auth.AuthConfig({ + 'credHelpers': { + 'registry1.io': 'truesecret', + 'registry2.io': 'powerlock' + }, + }) + assert auth_config.get_credential_store('registry2.io') == 'powerlock' + assert auth_config.get_credential_store('registry3.io') is None + + def test_get_credential_store_default_index(self): + auth_config = auth.AuthConfig({ + 'credHelpers': { + 'https://index.docker.io/v1/': 'powerlock' + }, + 'credsStore': 'truesecret' + }) + + assert auth_config.get_credential_store(None) == 'powerlock' + assert auth_config.get_credential_store('docker.io') == 'powerlock' + assert auth_config.get_credential_store('images.io') == 'truesecret' + + def test_get_credential_store_with_plain_dict(self): + auth_config = { + 'credHelpers': { + 'registry1.io': 'truesecret', + 'registry2.io': 'powerlock' + }, + 'credsStore': 'blackbox', + } + + assert auth.get_credential_store( + auth_config, 'registry1.io' + ) == 'truesecret' + assert auth.get_credential_store( + auth_config, 'registry2.io' + ) == 'powerlock' + assert auth.get_credential_store( + auth_config, 'registry3.io' + ) == 'blackbox' + + def test_get_all_credentials_credstore_only(self): + assert self.authconfig.get_all_credentials() == { + 'https://gensokyo.jp/v2': { + 'Username': 'sakuya', + 'Password': 'izayoi', + 'ServerAddress': 'https://gensokyo.jp/v2', + }, + 'https://default.com/v2': { + 'Username': 'user', + 'Password': 'hunter2', + 'ServerAddress': 'https://default.com/v2', + }, + } + + def test_get_all_credentials_with_empty_credhelper(self): + self.authconfig['credHelpers'] = { + 'registry1.io': 'truesecret', + } + self.authconfig._stores['truesecret'] = InMemoryStore() + assert self.authconfig.get_all_credentials() == { + 'https://gensokyo.jp/v2': { + 'Username': 'sakuya', + 'Password': 'izayoi', + 'ServerAddress': 'https://gensokyo.jp/v2', + }, + 'https://default.com/v2': { + 'Username': 'user', + 'Password': 'hunter2', + 'ServerAddress': 'https://default.com/v2', + }, + 'registry1.io': None, + } + + def test_get_all_credentials_with_credhelpers_only(self): + del self.authconfig['credsStore'] + assert self.authconfig.get_all_credentials() == {} + + self.authconfig['credHelpers'] = { + 'https://gensokyo.jp/v2': 'default', + 'https://default.com/v2': 'default', + } + + assert self.authconfig.get_all_credentials() == { + 'https://gensokyo.jp/v2': { + 'Username': 'sakuya', + 'Password': 'izayoi', + 'ServerAddress': 'https://gensokyo.jp/v2', + }, + 'https://default.com/v2': { + 'Username': 'user', + 'Password': 'hunter2', + 'ServerAddress': 'https://default.com/v2', + }, + } + + def test_get_all_credentials_with_auths_entries(self): + self.authconfig.add_auth('registry1.io', { + 'ServerAddress': 'registry1.io', + 'Username': 'reimu', + 'Password': 'hakurei', + }) + + assert self.authconfig.get_all_credentials() == { + 'https://gensokyo.jp/v2': { + 'Username': 'sakuya', + 'Password': 'izayoi', + 'ServerAddress': 'https://gensokyo.jp/v2', + }, + 'https://default.com/v2': { + 'Username': 'user', + 'Password': 'hunter2', + 'ServerAddress': 'https://default.com/v2', + }, + 'registry1.io': { + 'ServerAddress': 'registry1.io', + 'Username': 'reimu', + 'Password': 'hakurei', + }, + } + + def test_get_all_credentials_helpers_override_default(self): + self.authconfig['credHelpers'] = { + 'https://default.com/v2': 'truesecret', + } + truesecret = InMemoryStore('truesecret') + truesecret.store('https://default.com/v2', 'reimu', 'hakurei') + self.authconfig._stores['truesecret'] = truesecret + assert self.authconfig.get_all_credentials() == { + 'https://gensokyo.jp/v2': { + 'Username': 'sakuya', + 'Password': 'izayoi', + 'ServerAddress': 'https://gensokyo.jp/v2', + }, + 'https://default.com/v2': { + 'Username': 'reimu', + 'Password': 'hakurei', + 'ServerAddress': 'https://default.com/v2', + }, + } + + def test_get_all_credentials_3_sources(self): + self.authconfig['credHelpers'] = { + 'registry1.io': 'truesecret', + } + truesecret = InMemoryStore('truesecret') + truesecret.store('registry1.io', 'reimu', 'hakurei') + self.authconfig._stores['truesecret'] = truesecret + self.authconfig.add_auth('registry2.io', { + 'ServerAddress': 'registry2.io', + 'Username': 'reimu', + 'Password': 'hakurei', + }) + + assert self.authconfig.get_all_credentials() == { + 'https://gensokyo.jp/v2': { + 'Username': 'sakuya', + 'Password': 'izayoi', + 'ServerAddress': 'https://gensokyo.jp/v2', + }, + 'https://default.com/v2': { + 'Username': 'user', + 'Password': 'hunter2', + 'ServerAddress': 'https://default.com/v2', + }, + 'registry1.io': { + 'ServerAddress': 'registry1.io', + 'Username': 'reimu', + 'Password': 'hakurei', + }, + 'registry2.io': { + 'ServerAddress': 'registry2.io', + 'Username': 'reimu', + 'Password': 'hakurei', + } + } + + +class InMemoryStore(dockerpycreds.Store): + def __init__(self, *args, **kwargs): + self.__store = {} + + def get(self, server): + try: + return self.__store[server] + except KeyError: + raise dockerpycreds.errors.CredentialsNotFound() + + def store(self, server, username, secret): + self.__store[server] = { + 'ServerURL': server, + 'Username': username, + 'Secret': secret, + } + + def list(self): + return dict( + [(k, v['Username']) for k, v in self.__store.items()] + ) + + def erase(self, server): + del self.__store[server] From b2ad302636bc845b17ef63785738a757aef099f7 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 30 Nov 2018 14:58:18 -0800 Subject: [PATCH 0813/1301] Fix test names Signed-off-by: Joffrey F --- tests/unit/api_test.py | 53 ++++++++++++++++++++++-------------------- 1 file changed, 28 insertions(+), 25 deletions(-) diff --git a/tests/unit/api_test.py b/tests/unit/api_test.py index fac314d3d2..d0f22a81c9 100644 --- a/tests/unit/api_test.py +++ b/tests/unit/api_test.py @@ -479,23 +479,26 @@ class TCPSocketStreamTest(unittest.TestCase): built on these islands for generations past? Now shall what of Him? ''' - def setUp(self): - self.server = six.moves.socketserver.ThreadingTCPServer( - ('', 0), self.get_handler_class()) - self.thread = threading.Thread(target=self.server.serve_forever) - self.thread.setDaemon(True) - self.thread.start() - self.address = 'http://{}:{}'.format( - socket.gethostname(), self.server.server_address[1]) - - def tearDown(self): - self.server.shutdown() - self.server.server_close() - self.thread.join() - - def get_handler_class(self): - stdout_data = self.stdout_data - stderr_data = self.stderr_data + @classmethod + def setup_class(cls): + cls.server = six.moves.socketserver.ThreadingTCPServer( + ('', 0), cls.get_handler_class()) + cls.thread = threading.Thread(target=cls.server.serve_forever) + cls.thread.setDaemon(True) + cls.thread.start() + cls.address = 'http://{}:{}'.format( + socket.gethostname(), cls.server.server_address[1]) + + @classmethod + def teardown_class(cls): + cls.server.shutdown() + cls.server.server_close() + cls.thread.join() + + @classmethod + def get_handler_class(cls): + stdout_data = cls.stdout_data + stderr_data = cls.stderr_data class Handler(six.moves.BaseHTTPServer.BaseHTTPRequestHandler, object): def do_POST(self): @@ -542,45 +545,45 @@ def request(self, stream=None, tty=None, demux=None): return client._read_from_socket( resp, stream=stream, tty=tty, demux=demux) - def test_read_from_socket_1(self): + def test_read_from_socket_tty(self): res = self.request(stream=True, tty=True, demux=False) assert next(res) == self.stdout_data + self.stderr_data with self.assertRaises(StopIteration): next(res) - def test_read_from_socket_2(self): + def test_read_from_socket_tty_demux(self): res = self.request(stream=True, tty=True, demux=True) assert next(res) == (self.stdout_data + self.stderr_data, None) with self.assertRaises(StopIteration): next(res) - def test_read_from_socket_3(self): + def test_read_from_socket_no_tty(self): res = self.request(stream=True, tty=False, demux=False) assert next(res) == self.stdout_data assert next(res) == self.stderr_data with self.assertRaises(StopIteration): next(res) - def test_read_from_socket_4(self): + def test_read_from_socket_no_tty_demux(self): res = self.request(stream=True, tty=False, demux=True) assert (self.stdout_data, None) == next(res) assert (None, self.stderr_data) == next(res) with self.assertRaises(StopIteration): next(res) - def test_read_from_socket_5(self): + def test_read_from_socket_no_stream_tty(self): res = self.request(stream=False, tty=True, demux=False) assert res == self.stdout_data + self.stderr_data - def test_read_from_socket_6(self): + def test_read_from_socket_no_stream_tty_demux(self): res = self.request(stream=False, tty=True, demux=True) assert res == (self.stdout_data + self.stderr_data, None) - def test_read_from_socket_7(self): + def test_read_from_socket_no_stream_no_tty(self): res = self.request(stream=False, tty=False, demux=False) res == self.stdout_data + self.stderr_data - def test_read_from_socket_8(self): + def test_read_from_socket_no_stream_no_tty_demux(self): res = self.request(stream=False, tty=False, demux=True) assert res == (self.stdout_data, self.stderr_data) From 16c28093b965e13ef8a394799f7a9aa1bd08307c Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 30 Nov 2018 15:26:51 -0800 Subject: [PATCH 0814/1301] Move exec_run example to user guides section of docs Signed-off-by: Joffrey F --- docker/models/containers.py | 64 ----------------------------- docs/index.rst | 1 + docs/user_guides/index.rst | 8 ++++ docs/user_guides/multiplex.rst | 66 ++++++++++++++++++++++++++++++ docs/user_guides/swarm_services.md | 4 ++ 5 files changed, 79 insertions(+), 64 deletions(-) create mode 100644 docs/user_guides/index.rst create mode 100644 docs/user_guides/multiplex.rst diff --git a/docker/models/containers.py b/docker/models/containers.py index 75d8c2ebd7..41bc4da859 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -181,70 +181,6 @@ def exec_run(self, cmd, stdout=True, stderr=True, stdin=False, tty=False, Raises: :py:class:`docker.errors.APIError` If the server returns an error. - - Example: - - Create a container that runs in the background - - >>> client = docker.from_env() - >>> container = client.containers.run( - ... 'bfirsh/reticulate-splines', detach=True) - - Prepare the command we are going to use. It prints "hello stdout" - in `stdout`, followed by "hello stderr" in `stderr`: - - >>> cmd = '/bin/sh -c "echo hello stdout ; echo hello stderr >&2"' - - We'll run this command with all four the combinations of ``stream`` - and ``demux``. - - With ``stream=False`` and ``demux=False``, the output is a string - that contains both the `stdout` and the `stderr` output: - - >>> res = container.exec_run(cmd, stream=False, demux=False) - >>> res.output - b'hello stderr\nhello stdout\n' - - With ``stream=True``, and ``demux=False``, the output is a - generator that yields strings containing the output of both - `stdout` and `stderr`: - - >>> res = container.exec_run(cmd, stream=True, demux=False) - >>> next(res.output) - b'hello stdout\n' - >>> next(res.output) - b'hello stderr\n' - >>> next(res.output) - Traceback (most recent call last): - File "", line 1, in - StopIteration - - With ``stream=True`` and ``demux=True``, the generator now - separates the streams, and yield tuples - ``(stdout, stderr)``: - - >>> res = container.exec_run(cmd, stream=True, demux=True) - >>> next(res.output) - (b'hello stdout\n', None) - >>> next(res.output) - (None, b'hello stderr\n') - >>> next(res.output) - Traceback (most recent call last): - File "", line 1, in - StopIteration - - Finally, with ``stream=False`` and ``demux=True``, the whole output - is returned, but the streams are still separated: - - >>> res = container.exec_run(cmd, stream=True, demux=True) - >>> next(res.output) - (b'hello stdout\n', None) - >>> next(res.output) - (None, b'hello stderr\n') - >>> next(res.output) - Traceback (most recent call last): - File "", line 1, in - StopIteration """ resp = self.client.api.exec_create( self.id, cmd, stdout=stdout, stderr=stderr, stdin=stdin, tty=tty, diff --git a/docs/index.rst b/docs/index.rst index 39426b6819..63e85d3635 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -92,4 +92,5 @@ That's just a taste of what you can do with the Docker SDK for Python. For more, volumes api tls + user_guides/index change-log diff --git a/docs/user_guides/index.rst b/docs/user_guides/index.rst new file mode 100644 index 0000000000..79b3a909e3 --- /dev/null +++ b/docs/user_guides/index.rst @@ -0,0 +1,8 @@ +User guides and tutorials +========================= + +.. toctree:: + :maxdepth: 2 + + multiplex + swarm_services \ No newline at end of file diff --git a/docs/user_guides/multiplex.rst b/docs/user_guides/multiplex.rst new file mode 100644 index 0000000000..78d7e3728d --- /dev/null +++ b/docs/user_guides/multiplex.rst @@ -0,0 +1,66 @@ +Handling multiplexed streams +============================ + +.. note:: + The following instruction assume you're interested in getting output from + an ``exec`` command. These instruction are similarly applicable to the + output of ``attach``. + +First create a container that runs in the background: + +>>> client = docker.from_env() +>>> container = client.containers.run( +... 'bfirsh/reticulate-splines', detach=True) + +Prepare the command we are going to use. It prints "hello stdout" +in `stdout`, followed by "hello stderr" in `stderr`: + +>>> cmd = '/bin/sh -c "echo hello stdout ; echo hello stderr >&2"' +We'll run this command with all four the combinations of ``stream`` +and ``demux``. +With ``stream=False`` and ``demux=False``, the output is a string +that contains both the `stdout` and the `stderr` output: +>>> res = container.exec_run(cmd, stream=False, demux=False) +>>> res.output +b'hello stderr\nhello stdout\n' + +With ``stream=True``, and ``demux=False``, the output is a +generator that yields strings containing the output of both +`stdout` and `stderr`: + +>>> res = container.exec_run(cmd, stream=True, demux=False) +>>> next(res.output) +b'hello stdout\n' +>>> next(res.output) +b'hello stderr\n' +>>> next(res.output) +Traceback (most recent call last): + File "", line 1, in +StopIteration + +With ``stream=True`` and ``demux=True``, the generator now +separates the streams, and yield tuples +``(stdout, stderr)``: + +>>> res = container.exec_run(cmd, stream=True, demux=True) +>>> next(res.output) +(b'hello stdout\n', None) +>>> next(res.output) +(None, b'hello stderr\n') +>>> next(res.output) +Traceback (most recent call last): + File "", line 1, in +StopIteration + +Finally, with ``stream=False`` and ``demux=True``, the whole output +is returned, but the streams are still separated: + +>>> res = container.exec_run(cmd, stream=True, demux=True) +>>> next(res.output) +(b'hello stdout\n', None) +>>> next(res.output) +(None, b'hello stderr\n') +>>> next(res.output) +Traceback (most recent call last): + File "", line 1, in +StopIteration diff --git a/docs/user_guides/swarm_services.md b/docs/user_guides/swarm_services.md index 9bd4dca3fb..369fbed00e 100644 --- a/docs/user_guides/swarm_services.md +++ b/docs/user_guides/swarm_services.md @@ -1,5 +1,9 @@ # Swarm services +> Warning: +> This is a stale document and may contain outdated information. +> Refer to the API docs for updated classes and method signatures. + Starting with Engine version 1.12 (API 1.24), it is possible to manage services using the Docker Engine API. Note that the engine needs to be part of a [Swarm cluster](../swarm.rst) before you can use the service-related methods. From bc84ed11ec7f7e20f27139a5e44ae0862d5a1f7b Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 30 Nov 2018 15:40:14 -0800 Subject: [PATCH 0815/1301] Fix empty authconfig detection Signed-off-by: Joffrey F --- docker/api/build.py | 4 ++-- docker/api/daemon.py | 2 +- docker/auth.py | 8 +++++++- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/docker/api/build.py b/docker/api/build.py index c4fc37ec98..5db58382ba 100644 --- a/docker/api/build.py +++ b/docker/api/build.py @@ -286,10 +286,10 @@ def _set_auth_headers(self, headers): # If we don't have any auth data so far, try reloading the config # file one more time in case anything showed up in there. - if not self._auth_configs: + if not self._auth_configs or self._auth_configs.is_empty: log.debug("No auth config in memory - loading from filesystem") self._auth_configs = auth.load_config( - credsore_env=self.credsore_env + credstore_env=self.credstore_env ) # Send the full auth configuration (if any exists), since the build diff --git a/docker/api/daemon.py b/docker/api/daemon.py index a2936f2a0e..f715a131ad 100644 --- a/docker/api/daemon.py +++ b/docker/api/daemon.py @@ -127,7 +127,7 @@ def login(self, username, password=None, email=None, registry=None, self._auth_configs = auth.load_config( dockercfg_path, credstore_env=self.credstore_env ) - elif not self._auth_configs: + elif not self._auth_configs or self._auth_configs.is_empty: self._auth_configs = auth.load_config( credstore_env=self.credstore_env ) diff --git a/docker/auth.py b/docker/auth.py index c1b874f870..58b35eb491 100644 --- a/docker/auth.py +++ b/docker/auth.py @@ -39,7 +39,7 @@ def resolve_index_name(index_name): def get_config_header(client, registry): log.debug('Looking for auth config') - if not client._auth_configs: + if not client._auth_configs or client._auth_configs.is_empty: log.debug( "No auth config in memory - loading from filesystem" ) @@ -212,6 +212,12 @@ def creds_store(self): def cred_helpers(self): return self.get('credHelpers', {}) + @property + def is_empty(self): + return ( + not self.auths and not self.creds_store and not self.cred_helpers + ) + def resolve_authconfig(self, registry=None): """ Returns the authentication data from the given auth configuration for a From 3381f7be151400c973e26a32f194e1ef869f6db8 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 30 Nov 2018 17:26:50 -0800 Subject: [PATCH 0816/1301] Update setup.py for modern pypi / setuptools Signed-off-by: Joffrey F --- scripts/release.sh | 9 +-------- setup.py | 18 +++++++++++------- 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/scripts/release.sh b/scripts/release.sh index 5b37b6d083..f3ace27bdf 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -3,12 +3,6 @@ # Create the official release # -if [ -z "$(command -v pandoc 2> /dev/null)" ]; then - >&2 echo "$0 requires http://pandoc.org/" - >&2 echo "Please install it and make sure it is available on your \$PATH." - exit 2 -fi - VERSION=$1 REPO=docker/docker-py GITHUB_REPO=git@github.com:$REPO @@ -37,11 +31,10 @@ if [[ $2 == 'upload' ]]; then fi -pandoc -f markdown -t rst README.md -o README.rst || exit 1 echo "##> sdist & wheel" python setup.py sdist bdist_wheel if [[ $2 == 'upload' ]]; then echo '##> Uploading sdist to pypi' twine upload dist/docker-$VERSION* -fi \ No newline at end of file +fi diff --git a/setup.py b/setup.py index 4ce55fe815..f1c3c204b0 100644 --- a/setup.py +++ b/setup.py @@ -55,24 +55,27 @@ long_description = '' -try: - with codecs.open('./README.rst', encoding='utf-8') as readme_rst: - long_description = readme_rst.read() -except IOError: - # README.rst is only generated on release. Its absence should not prevent - # setup.py from working properly. - pass +with codecs.open('./README.md', encoding='utf-8') as readme_md: + long_description = readme_md.read() setup( name="docker", version=version, description="A Python library for the Docker Engine API.", long_description=long_description, + long_description_content_type='text/markdown', url='https://github.com/docker/docker-py', + project_urls={ + 'Documentation': 'https://docker-py.readthedocs.io', + 'Changelog': 'https://docker-py.readthedocs.io/en/stable/change-log.html', # flake8: noqa + 'Source': 'https://github.com/docker/docker-py', + 'Tracker': 'https://github.com/docker/docker-py/issues', + }, packages=find_packages(exclude=["tests.*", "tests"]), install_requires=requirements, tests_require=test_requirements, extras_require=extras_require, + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*', zip_safe=False, test_suite='tests', classifiers=[ @@ -89,6 +92,7 @@ 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Topic :: Software Development', 'Topic :: Utilities', 'License :: OSI Approved :: Apache Software License', ], From 1bc5783a3d253021f82d21f123b00a8fe45d08e3 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 7 Dec 2018 15:30:25 -0800 Subject: [PATCH 0817/1301] Prevent untracked files in releases Signed-off-by: Joffrey F --- scripts/release.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/release.sh b/scripts/release.sh index f3ace27bdf..d9e7a055a1 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -12,8 +12,9 @@ if [ -z $VERSION ]; then exit 1 fi -echo "##> Removing stale build files" -rm -rf ./build || exit 1 +echo "##> Removing stale build files and other untracked files" +git clean -x -d -i +test -z "$(git clean -x -d -n)" || exit 1 echo "##> Tagging the release as $VERSION" git tag $VERSION From e15db4cb20b22c97e5df5f4fc2fdb54c18e785e9 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 7 Dec 2018 16:56:45 -0800 Subject: [PATCH 0818/1301] Improve handling of placement preferences; improve docs Signed-off-by: Joffrey F --- docker/models/services.py | 10 ++++++---- docker/types/__init__.py | 5 +++-- docker/types/services.py | 41 ++++++++++++++++++++++++++++++++------- docs/api.rst | 1 + 4 files changed, 44 insertions(+), 13 deletions(-) diff --git a/docker/models/services.py b/docker/models/services.py index a2a3ed011f..5d2bd9b3ec 100644 --- a/docker/models/services.py +++ b/docker/models/services.py @@ -153,10 +153,12 @@ def create(self, image, command=None, **kwargs): image (str): The image name to use for the containers. command (list of str or str): Command to run. args (list of str): Arguments to the command. - constraints (list of str): Placement constraints. - preferences (list of str): Placement preferences. - platforms (list of tuple): A list of platforms constraints - expressed as ``(arch, os)`` tuples + constraints (list of str): :py:class:`~docker.types.Placement` + constraints. + preferences (list of tuple): :py:class:`~docker.types.Placement` + preferences. + platforms (list of tuple): A list of platform constraints + expressed as ``(arch, os)`` tuples. container_labels (dict): Labels to apply to the container. endpoint_spec (EndpointSpec): Properties that can be configured to access and load balance a service. Default: ``None``. diff --git a/docker/types/__init__.py b/docker/types/__init__.py index 64512333df..f3cac1bc17 100644 --- a/docker/types/__init__.py +++ b/docker/types/__init__.py @@ -5,7 +5,8 @@ from .networks import EndpointConfig, IPAMConfig, IPAMPool, NetworkingConfig from .services import ( ConfigReference, ContainerSpec, DNSConfig, DriverConfig, EndpointSpec, - Mount, Placement, Privileges, Resources, RestartPolicy, RollbackConfig, - SecretReference, ServiceMode, TaskTemplate, UpdateConfig + Mount, Placement, PlacementPreference, Privileges, Resources, + RestartPolicy, RollbackConfig, SecretReference, ServiceMode, TaskTemplate, + UpdateConfig ) from .swarm import SwarmSpec, SwarmExternalCA diff --git a/docker/types/services.py b/docker/types/services.py index c66d41a167..79794d7eb2 100644 --- a/docker/types/services.py +++ b/docker/types/services.py @@ -648,18 +648,24 @@ class Placement(dict): Placement constraints to be used as part of a :py:class:`TaskTemplate` Args: - constraints (:py:class:`list`): A list of constraints - preferences (:py:class:`list`): Preferences provide a way to make - the scheduler aware of factors such as topology. They are - provided in order from highest to lowest precedence. - platforms (:py:class:`list`): A list of platforms expressed as - ``(arch, os)`` tuples + constraints (:py:class:`list` of str): A list of constraints + preferences (:py:class:`list` of tuple): Preferences provide a way + to make the scheduler aware of factors such as topology. They + are provided in order from highest to lowest precedence and + are expressed as ``(strategy, descriptor)`` tuples. See + :py:class:`PlacementPreference` for details. + platforms (:py:class:`list` of tuple): A list of platforms + expressed as ``(arch, os)`` tuples """ def __init__(self, constraints=None, preferences=None, platforms=None): if constraints is not None: self['Constraints'] = constraints if preferences is not None: - self['Preferences'] = preferences + self['Preferences'] = [] + for pref in preferences: + if isinstance(pref, tuple): + pref = PlacementPreference(*pref) + self['Preferences'].append(pref) if platforms: self['Platforms'] = [] for plat in platforms: @@ -668,6 +674,27 @@ def __init__(self, constraints=None, preferences=None, platforms=None): }) +class PlacementPreference(dict): + """ + Placement preference to be used as an element in the list of + preferences for :py:class:`Placement` objects. + + Args: + strategy (string): The placement strategy to implement. Currently, + the only supported strategy is ``spread``. + descriptor (string): A label descriptor. For the spread strategy, + the scheduler will try to spread tasks evenly over groups of + nodes identified by this label. + """ + def __init__(self, strategy, descriptor): + if strategy != 'spread': + raise errors.InvalidArgument( + 'PlacementPreference strategy value is invalid ({}):' + ' must be "spread".'.format(strategy) + ) + self['SpreadOver'] = descriptor + + class DNSConfig(dict): """ Specification for DNS related configurations in resolver configuration diff --git a/docs/api.rst b/docs/api.rst index 1682128951..edb8fffadc 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -143,6 +143,7 @@ Configuration types .. autoclass:: LogConfig .. autoclass:: Mount .. autoclass:: Placement +.. autoclass:: PlacementPreference .. autoclass:: Privileges .. autoclass:: Resources .. autoclass:: RestartPolicy From b297b837df511178a69f29a9f8461aee0193e278 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 7 Dec 2018 16:57:40 -0800 Subject: [PATCH 0819/1301] Dynamically retrieve version information for generated docs Signed-off-by: Joffrey F --- docs/conf.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 3e17678a83..f46d1f76ea 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -69,10 +69,12 @@ # |version| and |release|, also used in various other places throughout the # built documents. # -# The short X.Y version. -version = u'2.0' +with open('../docker/version.py', 'r') as vfile: + exec(vfile.read()) # The full version, including alpha/beta/rc tags. -release = u'2.0' +release = version +# The short X.Y version. +version = '{}.{}'.format(version_info[0], version_info[1]) # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. From a207122c0d6ce4cbd32c8887533018593dbdeae5 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 7 Dec 2018 17:04:54 -0800 Subject: [PATCH 0820/1301] Update Jenkinsfile version map Signed-off-by: Joffrey F --- Jenkinsfile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 211159bc28..33a0fc3136 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -45,10 +45,13 @@ def getDockerVersions = { -> } def getAPIVersion = { engineVersion -> - def versionMap = ['17.06': '1.30', '17.12': '1.35', '18.02': '1.36', '18.03': '1.37'] + def versionMap = [ + '17.06': '1.30', '17.12': '1.35', '18.02': '1.36', '18.03': '1.37', + '18.06': '1.38', '18.09': '1.39' + ] def result = versionMap[engineVersion.substring(0, 5)] if (!result) { - return '1.37' + return '1.39' } return result } From 543d83cb094ef7b1fe9f5cd61fcc978b99b14ca3 Mon Sep 17 00:00:00 2001 From: Maximilian Bischoff Date: Fri, 14 Dec 2018 17:50:20 +0100 Subject: [PATCH 0821/1301] Fixed a typo in the configs api doc The documentation for id in ConfigApiMixin inspect_config was wrongly mentioning removal of a config Signed-off-by: Maximilian Bischoff --- docker/api/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/api/config.py b/docker/api/config.py index 767bef263a..93e5168f64 100644 --- a/docker/api/config.py +++ b/docker/api/config.py @@ -42,7 +42,7 @@ def inspect_config(self, id): Retrieve config metadata Args: - id (string): Full ID of the config to remove + id (string): Full ID of the config to inspect Returns (dict): A dictionary of metadata From 341e2580aa5c1b278d93f382517a48fc177c64bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Schoentgen?= Date: Thu, 20 Dec 2018 14:43:00 +0100 Subject: [PATCH 0822/1301] Fix DeprecationWarning: invalid escape sequence in services.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Mickaël Schoentgen --- docker/types/services.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/types/services.py b/docker/types/services.py index 79794d7eb2..ac1c181a90 100644 --- a/docker/types/services.py +++ b/docker/types/services.py @@ -714,7 +714,7 @@ def __init__(self, nameservers=None, search=None, options=None): class Privileges(dict): - """ + r""" Security options for a service's containers. Part of a :py:class:`ContainerSpec` definition. From e99ce1e3593c64e6a6cae08a0af99ccb05a005e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Schoentgen?= Date: Thu, 20 Dec 2018 14:47:28 +0100 Subject: [PATCH 0823/1301] Fix DeprecationWarning: invalid escape sequence in ports.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Mickaël Schoentgen --- docker/utils/ports.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/utils/ports.py b/docker/utils/ports.py index bf7d697271..cf5987c94f 100644 --- a/docker/utils/ports.py +++ b/docker/utils/ports.py @@ -3,10 +3,10 @@ PORT_SPEC = re.compile( "^" # Match full string "(" # External part - "((?P[a-fA-F\d.:]+):)?" # Address - "(?P[\d]*)(-(?P[\d]+))?:" # External range + r"((?P[a-fA-F\d.:]+):)?" # Address + r"(?P[\d]*)(-(?P[\d]+))?:" # External range ")?" - "(?P[\d]+)(-(?P[\d]+))?" # Internal range + r"(?P[\d]+)(-(?P[\d]+))?" # Internal range "(?P/(udp|tcp))?" # Protocol "$" # Match full string ) From 3cda1e8bbd20900e9eadd9b88459d16c86d93648 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 28 Dec 2018 15:45:54 -0800 Subject: [PATCH 0824/1301] Make swarm.init() return value match documentation Signed-off-by: Joffrey F --- docker/models/swarm.py | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/models/swarm.py b/docker/models/swarm.py index 7396e730d7..3a02ae3707 100644 --- a/docker/models/swarm.py +++ b/docker/models/swarm.py @@ -112,6 +112,7 @@ def init(self, advertise_addr=None, listen_addr='0.0.0.0:2377', init_kwargs['swarm_spec'] = self.client.api.create_swarm_spec(**kwargs) self.client.api.init_swarm(**init_kwargs) self.reload() + return True def join(self, *args, **kwargs): return self.client.api.join_swarm(*args, **kwargs) From 72f4f527ad60b2a677b883dc54669ebe98f0879f Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 9 Jan 2019 11:14:08 -0800 Subject: [PATCH 0825/1301] Update test dependencies to latest version, fix some flake8 errors Signed-off-by: Joffrey F --- docker/auth.py | 2 +- docker/types/containers.py | 11 +++++------ setup.py | 2 +- test-requirements.txt | 10 ++++++---- tests/integration/api_client_test.py | 2 +- tests/unit/dockertypes_test.py | 2 +- 6 files changed, 15 insertions(+), 14 deletions(-) diff --git a/docker/auth.py b/docker/auth.py index 58b35eb491..638ab9b0a9 100644 --- a/docker/auth.py +++ b/docker/auth.py @@ -273,7 +273,7 @@ def _resolve_authconfig_credstore(self, registry, credstore_name): 'Password': data['Secret'], }) return res - except dockerpycreds.CredentialsNotFound as e: + except dockerpycreds.CredentialsNotFound: log.debug('No entry found') return None except dockerpycreds.StoreError as e: diff --git a/docker/types/containers.py b/docker/types/containers.py index d040c0fb5e..fd8cab4979 100644 --- a/docker/types/containers.py +++ b/docker/types/containers.py @@ -51,8 +51,7 @@ class LogConfig(DictType): ... host_config=hc) >>> client.inspect_container(container)['HostConfig']['LogConfig'] {'Type': 'json-file', 'Config': {'labels': 'production_status,geo', 'max-size': '1g'}} - - """ # flake8: noqa + """ # noqa: E501 types = LogConfigTypesEnum def __init__(self, **kwargs): @@ -320,10 +319,10 @@ def __init__(self, version, binds=None, port_bindings=None, if not isinstance(ulimits, list): raise host_config_type_error('ulimits', ulimits, 'list') self['Ulimits'] = [] - for l in ulimits: - if not isinstance(l, Ulimit): - l = Ulimit(**l) - self['Ulimits'].append(l) + for lmt in ulimits: + if not isinstance(lmt, Ulimit): + lmt = Ulimit(**lmt) + self['Ulimits'].append(lmt) if log_config is not None: if not isinstance(log_config, LogConfig): diff --git a/setup.py b/setup.py index f1c3c204b0..94fbdf444e 100644 --- a/setup.py +++ b/setup.py @@ -67,7 +67,7 @@ url='https://github.com/docker/docker-py', project_urls={ 'Documentation': 'https://docker-py.readthedocs.io', - 'Changelog': 'https://docker-py.readthedocs.io/en/stable/change-log.html', # flake8: noqa + 'Changelog': 'https://docker-py.readthedocs.io/en/stable/change-log.html', # noqa: E501 'Source': 'https://github.com/docker/docker-py', 'Tracker': 'https://github.com/docker/docker-py/issues', }, diff --git a/test-requirements.txt b/test-requirements.txt index 07e1a900db..510fa295ec 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,7 +1,9 @@ -coverage==3.7.1 -flake8==3.4.1 +coverage==4.5.2 +flake8==3.6.0; python_version != '3.3' +flake8==3.4.1; python_version == '3.3' mock==1.0.1 pytest==2.9.1; python_version == '3.3' -pytest==3.6.3; python_version > '3.3' -pytest-cov==2.1.0 +pytest==4.1.0; python_version != '3.3' +pytest-cov==2.6.1; python_version != '3.3' +pytest-cov==2.5.1; python_version == '3.3' pytest-timeout==1.3.3 diff --git a/tests/integration/api_client_test.py b/tests/integration/api_client_test.py index 905e06484d..9e348f3e3f 100644 --- a/tests/integration/api_client_test.py +++ b/tests/integration/api_client_test.py @@ -47,7 +47,7 @@ def test_timeout(self): # This call isn't supposed to complete, and it should fail fast. try: res = self.client.inspect_container('id') - except: + except: # noqa: E722 pass end = time.time() assert res is None diff --git a/tests/unit/dockertypes_test.py b/tests/unit/dockertypes_test.py index cdacf8cd5b..0689d07b32 100644 --- a/tests/unit/dockertypes_test.py +++ b/tests/unit/dockertypes_test.py @@ -14,7 +14,7 @@ try: from unittest import mock -except: +except: # noqa: E722 import mock From bfdd0a881ea10e0f09a90a5282ccb1e023e1ba75 Mon Sep 17 00:00:00 2001 From: Corentin Henry Date: Wed, 12 Dec 2018 12:29:06 +0100 Subject: [PATCH 0826/1301] add support for proxies Signed-off-by: Corentin Henry --- docker/api/build.py | 5 ++- docker/api/client.py | 7 +++++ docker/api/container.py | 4 +++ docker/api/exec_api.py | 1 + docker/utils/proxy.py | 69 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 85 insertions(+), 1 deletion(-) create mode 100644 docker/utils/proxy.py diff --git a/docker/api/build.py b/docker/api/build.py index 5db58382ba..0871df8c60 100644 --- a/docker/api/build.py +++ b/docker/api/build.py @@ -168,8 +168,11 @@ def build(self, path=None, tag=None, quiet=False, fileobj=None, } params.update(container_limits) + final_buildargs = self._proxy_configs.get_environment() if buildargs: - params.update({'buildargs': json.dumps(buildargs)}) + final_buildargs.update(buildargs) + if final_buildargs: + params.update({'buildargs': json.dumps(final_buildargs)}) if shmsize: if utils.version_gte(self._version, '1.22'): diff --git a/docker/api/client.py b/docker/api/client.py index 265dfdcef5..b12398e5ed 100644 --- a/docker/api/client.py +++ b/docker/api/client.py @@ -34,6 +34,7 @@ from ..utils import utils, check_resource, update_headers, config from ..utils.socket import frames_iter, consume_socket_output, demux_adaptor from ..utils.json_stream import json_stream +from ..utils.proxy import ProxyConfig try: from ..transport import NpipeAdapter except ImportError: @@ -114,6 +115,12 @@ def __init__(self, base_url=None, version=None, self.headers['User-Agent'] = user_agent self._general_configs = config.load_general_config() + try: + proxies = self._general_configs['proxies']['default'] + except KeyError: + proxies = {} + self._proxy_configs = ProxyConfig.from_dict(proxies) + self._auth_configs = auth.load_config( config_dict=self._general_configs, credstore_env=credstore_env, ) diff --git a/docker/api/container.py b/docker/api/container.py index ab3b1cf410..2a80ff7dc7 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -403,6 +403,10 @@ def create_container(self, image, command=None, hostname=None, user=None, if isinstance(volumes, six.string_types): volumes = [volumes, ] + if isinstance(environment, dict): + environment = utils.utils.format_environment(environment) + environment = self._proxy_configs.inject_proxy_environment(environment) + config = self.create_container_config( image, command, hostname, user, detach, stdin_open, tty, ports, environment, volumes, diff --git a/docker/api/exec_api.py b/docker/api/exec_api.py index d13b128998..0dabdd3078 100644 --- a/docker/api/exec_api.py +++ b/docker/api/exec_api.py @@ -50,6 +50,7 @@ def exec_create(self, container, cmd, stdout=True, stderr=True, if isinstance(environment, dict): environment = utils.utils.format_environment(environment) + environment = self._proxy_configs.inject_proxy_environment(environment) data = { 'Container': container, diff --git a/docker/utils/proxy.py b/docker/utils/proxy.py new file mode 100644 index 0000000000..3f55a3c13d --- /dev/null +++ b/docker/utils/proxy.py @@ -0,0 +1,69 @@ +from .utils import format_environment + + +class ProxyConfig(): + ''' + Hold the client's proxy configuration + ''' + + def __init__(self, http=None, https=None, ftp=None, no_proxy=None): + self.http = http + self.https = https + self.ftp = ftp + self.no_proxy = no_proxy + + @staticmethod + def from_dict(config): + ''' + Instantiate a new ProxyConfig from a dictionary that represents a + client configuration, as described in `the documentation`_. + + .. _the documentation: + https://docs.docker.com/network/proxy/#configure-the-docker-client + ''' + return ProxyConfig( + http=config.get('httpProxy', None), + https=config.get('httpsProxy', None), + ftp=config.get('ftpProxy', None), + no_proxy=config.get('noProxy', None)) + + def get_environment(self): + ''' + Return a dictionary representing the environment variables used to + set the proxy settings. + ''' + env = {} + if self.http: + env['http_proxy'] = env['HTTP_PROXY'] = self.http + if self.https: + env['https_proxy'] = env['HTTPS_PROXY'] = self.https + if self.ftp: + env['ftp_proxy'] = env['FTP_PROXY'] = self.ftp + if self.no_proxy: + env['no_proxy'] = env['NO_PROXY'] = self.no_proxy + return env + + def inject_proxy_environment(self, environment): + ''' + Given a list of strings representing environment variables, prepend the + environemt variables corresponding to the proxy settings. + ''' + if not self: + return environment + + proxy_env = format_environment(self.get_environment()) + if not environment: + return proxy_env + # It is important to prepend our variables, because we want the + # variables defined in "environment" to take precedence. + return proxy_env + environment + + def __bool__(self): + return bool(self.http or self.https or self.ftp or self.no_proxy) + + def __nonzero__(self): + return self.__bool__() + + def __str__(self): + return 'ProxyConfig(http={}, https={}, ftp={}, no_proxy={})'.format( + self.http, self.https, self.ftp, self.no_proxy) From 6e227895d3ed2a0d41409f7e2bdd7d82bf6a8068 Mon Sep 17 00:00:00 2001 From: Corentin Henry Date: Wed, 12 Dec 2018 15:14:49 +0100 Subject: [PATCH 0827/1301] tests: remove outdated code the _cfg attribute does not exist anymore Signed-off-by: Corentin Henry --- tests/unit/api_test.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/unit/api_test.py b/tests/unit/api_test.py index 203caf3fbb..f4d220a2c6 100644 --- a/tests/unit/api_test.py +++ b/tests/unit/api_test.py @@ -106,8 +106,6 @@ def setUp(self): ) self.patcher.start() self.client = APIClient() - # Force-clear authconfig to avoid tampering with the tests - self.client._cfg = {'Configs': {}} def tearDown(self): self.client.close() From 545adc2a59193cbdf4fc79bfe761828229d1dc0f Mon Sep 17 00:00:00 2001 From: Corentin Henry Date: Wed, 12 Dec 2018 15:58:00 +0100 Subject: [PATCH 0828/1301] add unit tests for ProxyConfig Signed-off-by: Corentin Henry --- tests/unit/utils_proxy_test.py | 84 ++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 tests/unit/utils_proxy_test.py diff --git a/tests/unit/utils_proxy_test.py b/tests/unit/utils_proxy_test.py new file mode 100644 index 0000000000..ff0e14ba74 --- /dev/null +++ b/tests/unit/utils_proxy_test.py @@ -0,0 +1,84 @@ +# -*- coding: utf-8 -*- + +import unittest +import six + +from docker.utils.proxy import ProxyConfig + +HTTP = 'http://test:80' +HTTPS = 'https://test:443' +FTP = 'ftp://user:password@host:23' +NO_PROXY = 'localhost,.localdomain' +CONFIG = ProxyConfig(http=HTTP, https=HTTPS, ftp=FTP, no_proxy=NO_PROXY) +ENV = { + 'http_proxy': HTTP, + 'HTTP_PROXY': HTTP, + 'https_proxy': HTTPS, + 'HTTPS_PROXY': HTTPS, + 'ftp_proxy': FTP, + 'FTP_PROXY': FTP, + 'no_proxy': NO_PROXY, + 'NO_PROXY': NO_PROXY, +} + + +class ProxyConfigTest(unittest.TestCase): + + def test_from_dict(self): + config = ProxyConfig.from_dict({ + 'httpProxy': HTTP, + 'httpsProxy': HTTPS, + 'ftpProxy': FTP, + 'noProxy': NO_PROXY + }) + self.assertEqual(CONFIG.http, config.http) + self.assertEqual(CONFIG.https, config.https) + self.assertEqual(CONFIG.ftp, config.ftp) + self.assertEqual(CONFIG.no_proxy, config.no_proxy) + + def test_new(self): + config = ProxyConfig() + self.assertIsNone(config.http) + self.assertIsNone(config.https) + self.assertIsNone(config.ftp) + self.assertIsNone(config.no_proxy) + + config = ProxyConfig(http='a', https='b', ftp='c', no_proxy='d') + self.assertEqual(config.http, 'a') + self.assertEqual(config.https, 'b') + self.assertEqual(config.ftp, 'c') + self.assertEqual(config.no_proxy, 'd') + + def test_truthiness(self): + assert not ProxyConfig() + assert ProxyConfig(http='non-zero') + assert ProxyConfig(https='non-zero') + assert ProxyConfig(ftp='non-zero') + assert ProxyConfig(no_proxy='non-zero') + + def test_environment(self): + self.assertDictEqual(CONFIG.get_environment(), ENV) + empty = ProxyConfig() + self.assertDictEqual(empty.get_environment(), {}) + + def test_inject_proxy_environment(self): + # Proxy config is non null, env is None. + self.assertSetEqual( + set(CONFIG.inject_proxy_environment(None)), + set(['{}={}'.format(k, v) for k, v in six.iteritems(ENV)])) + + # Proxy config is null, env is None. + self.assertIsNone(ProxyConfig().inject_proxy_environment(None), None) + + env = ['FOO=BAR', 'BAR=BAZ'] + + # Proxy config is non null, env is non null + actual = CONFIG.inject_proxy_environment(env) + expected = ['{}={}'.format(k, v) for k, v in six.iteritems(ENV)] + env + # It's important that the first 8 variables are the ones from the proxy + # config, and the last 2 are the ones from the input environment + self.assertSetEqual(set(actual[:8]), set(expected[:8])) + self.assertSetEqual(set(actual[-2:]), set(expected[-2:])) + + # Proxy is null, and is non null + self.assertListEqual(ProxyConfig().inject_proxy_environment(env), env) From 4f79ba160e78f5327c683b6a7797bde5dfb6dc32 Mon Sep 17 00:00:00 2001 From: Corentin Henry Date: Wed, 12 Dec 2018 17:10:04 +0100 Subject: [PATCH 0829/1301] make the integration tests more verbose Signed-off-by: Corentin Henry --- Jenkinsfile | 2 +- Makefile | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 33a0fc3136..387e147036 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -91,7 +91,7 @@ def runTests = { Map settings -> --network ${testNetwork} \\ --volumes-from ${dindContainerName} \\ ${testImage} \\ - py.test -v -rxs tests/integration + py.test -vv -rxs tests/integration """ } finally { sh """ diff --git a/Makefile b/Makefile index 434d40e1cc..684716af2f 100644 --- a/Makefile +++ b/Makefile @@ -35,11 +35,11 @@ unit-test-py3: build-py3 .PHONY: integration-test integration-test: build - docker run -t --rm -v /var/run/docker.sock:/var/run/docker.sock docker-sdk-python py.test -v tests/integration/${file} + docker run -t --rm -v /var/run/docker.sock:/var/run/docker.sock docker-sdk-python py.test -vv tests/integration/${file} .PHONY: integration-test-py3 integration-test-py3: build-py3 - docker run -t --rm -v /var/run/docker.sock:/var/run/docker.sock docker-sdk-python3 py.test tests/integration/${file} + docker run -t --rm -v /var/run/docker.sock:/var/run/docker.sock docker-sdk-python3 py.test -vv tests/integration/${file} TEST_API_VERSION ?= 1.35 TEST_ENGINE_VERSION ?= 17.12.0-ce @@ -57,7 +57,7 @@ integration-dind-py2: build setup-network docker run -d --network dpy-tests --name dpy-dind-py2 --privileged\ dockerswarm/dind:${TEST_ENGINE_VERSION} dockerd -H tcp://0.0.0.0:2375 --experimental docker run -t --rm --env="DOCKER_HOST=tcp://dpy-dind-py2:2375" --env="DOCKER_TEST_API_VERSION=${TEST_API_VERSION}"\ - --network dpy-tests docker-sdk-python py.test tests/integration + --network dpy-tests docker-sdk-python py.test -vv tests/integration docker rm -vf dpy-dind-py2 .PHONY: integration-dind-py3 @@ -66,7 +66,7 @@ integration-dind-py3: build-py3 setup-network docker run -d --network dpy-tests --name dpy-dind-py3 --privileged\ dockerswarm/dind:${TEST_ENGINE_VERSION} dockerd -H tcp://0.0.0.0:2375 --experimental docker run -t --rm --env="DOCKER_HOST=tcp://dpy-dind-py3:2375" --env="DOCKER_TEST_API_VERSION=${TEST_API_VERSION}"\ - --network dpy-tests docker-sdk-python3 py.test tests/integration + --network dpy-tests docker-sdk-python3 py.test -vv tests/integration docker rm -vf dpy-dind-py3 .PHONY: integration-dind-ssl @@ -81,10 +81,10 @@ integration-dind-ssl: build-dind-certs build build-py3 --tlskey=/certs/server-key.pem -H tcp://0.0.0.0:2375 --experimental docker run -t --rm --volumes-from dpy-dind-ssl --env="DOCKER_HOST=tcp://docker:2375"\ --env="DOCKER_TLS_VERIFY=1" --env="DOCKER_CERT_PATH=/certs" --env="DOCKER_TEST_API_VERSION=${TEST_API_VERSION}"\ - --network dpy-tests docker-sdk-python py.test tests/integration + --network dpy-tests docker-sdk-python py.test -vv tests/integration docker run -t --rm --volumes-from dpy-dind-ssl --env="DOCKER_HOST=tcp://docker:2375"\ --env="DOCKER_TLS_VERIFY=1" --env="DOCKER_CERT_PATH=/certs" --env="DOCKER_TEST_API_VERSION=${TEST_API_VERSION}"\ - --network dpy-tests docker-sdk-python3 py.test tests/integration + --network dpy-tests docker-sdk-python3 py.test -vv tests/integration docker rm -vf dpy-dind-ssl dpy-dind-certs .PHONY: flake8 From 0d37390c463b64b3ec3f95831921ff19eaf1a598 Mon Sep 17 00:00:00 2001 From: Corentin Henry Date: Wed, 12 Dec 2018 16:39:24 +0100 Subject: [PATCH 0830/1301] add integration tests for proxy support Signed-off-by: Corentin Henry --- tests/integration/api_build_test.py | 38 +++++++++++++++++++++++++++++ tests/integration/api_exec_test.py | 34 ++++++++++++++++++++++++++ 2 files changed, 72 insertions(+) diff --git a/tests/integration/api_build_test.py b/tests/integration/api_build_test.py index bad411beec..3c4e982048 100644 --- a/tests/integration/api_build_test.py +++ b/tests/integration/api_build_test.py @@ -4,6 +4,7 @@ import tempfile from docker import errors +from docker.utils.proxy import ProxyConfig import pytest import six @@ -13,6 +14,43 @@ class BuildTest(BaseAPIIntegrationTest): + def test_build_with_proxy(self): + self.client._proxy_configs = ProxyConfig( + ftp='a', http='b', https='c', no_proxy='d') + + script = io.BytesIO('\n'.join([ + 'FROM busybox', + 'RUN env | grep "FTP_PROXY=a"', + 'RUN env | grep "ftp_proxy=a"', + 'RUN env | grep "HTTP_PROXY=b"', + 'RUN env | grep "http_proxy=b"', + 'RUN env | grep "HTTPS_PROXY=c"', + 'RUN env | grep "https_proxy=c"', + 'RUN env | grep "NO_PROXY=d"', + 'RUN env | grep "no_proxy=d"', + ]).encode('ascii')) + self.client.build(fileobj=script, decode=True) + + def test_build_with_proxy_and_buildargs(self): + self.client._proxy_configs = ProxyConfig( + ftp='a', http='b', https='c', no_proxy='d') + + script = io.BytesIO('\n'.join([ + 'FROM busybox', + 'RUN env | grep "FTP_PROXY=XXX"', + 'RUN env | grep "ftp_proxy=xxx"', + 'RUN env | grep "HTTP_PROXY=b"', + 'RUN env | grep "http_proxy=b"', + 'RUN env | grep "HTTPS_PROXY=c"', + 'RUN env | grep "https_proxy=c"', + 'RUN env | grep "NO_PROXY=d"', + 'RUN env | grep "no_proxy=d"', + ]).encode('ascii')) + self.client.build( + fileobj=script, + decode=True, + buildargs={'FTP_PROXY': 'XXX', 'ftp_proxy': 'xxx'}) + def test_build_streaming(self): script = io.BytesIO('\n'.join([ 'FROM busybox', diff --git a/tests/integration/api_exec_test.py b/tests/integration/api_exec_test.py index 857a18cb3f..6d4f97daab 100644 --- a/tests/integration/api_exec_test.py +++ b/tests/integration/api_exec_test.py @@ -1,5 +1,6 @@ from docker.utils.socket import next_frame_header from docker.utils.socket import read_exactly +from docker.utils.proxy import ProxyConfig from .base import BaseAPIIntegrationTest, BUSYBOX from ..helpers import ( @@ -8,6 +9,39 @@ class ExecTest(BaseAPIIntegrationTest): + def test_execute_proxy_env(self): + # Set a custom proxy config on the client + self.client._proxy_configs = ProxyConfig( + ftp='a', https='b', http='c', no_proxy='d') + + container = self.client.create_container( + BUSYBOX, 'cat', detach=True, stdin_open=True) + id = container['Id'] + self.client.start(id) + self.tmp_containers.append(id) + + # First, just make sure the environment variables from the custom + # config are set + res = self.client.exec_create( + id, cmd='sh -c "env | grep -i proxy"') + output = self.client.exec_start(res).decode('utf-8').split('\n') + expected = [ + 'ftp_proxy=a', 'https_proxy=b', 'http_proxy=c', 'no_proxy=d', + 'FTP_PROXY=a', 'HTTPS_PROXY=b', 'HTTP_PROXY=c', 'NO_PROXY=d'] + for item in expected: + assert item in output + + # Overwrite some variables with a custom environment + env = {'https_proxy': 'xxx', 'HTTPS_PROXY': 'XXX'} + res = self.client.exec_create( + id, cmd='sh -c "env | grep -i proxy"', environment=env) + output = self.client.exec_start(res).decode('utf-8').split('\n') + expected = [ + 'ftp_proxy=a', 'https_proxy=xxx', 'http_proxy=c', 'no_proxy=d', + 'FTP_PROXY=a', 'HTTPS_PROXY=XXX', 'HTTP_PROXY=c', 'NO_PROXY=d'] + for item in expected: + assert item in output + def test_execute_command(self): container = self.client.create_container(BUSYBOX, 'cat', detach=True, stdin_open=True) From 708ef6d81ebd12d997b8bca5481541655cbfa61d Mon Sep 17 00:00:00 2001 From: Corentin Henry Date: Sat, 15 Dec 2018 14:55:43 +0100 Subject: [PATCH 0831/1301] Revert "make the integration tests more verbose" This reverts commit 7584e5d7f068400d33d6658ae706c79db1aab2c5. Signed-off-by: Corentin Henry --- Jenkinsfile | 2 +- Makefile | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 387e147036..33a0fc3136 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -91,7 +91,7 @@ def runTests = { Map settings -> --network ${testNetwork} \\ --volumes-from ${dindContainerName} \\ ${testImage} \\ - py.test -vv -rxs tests/integration + py.test -v -rxs tests/integration """ } finally { sh """ diff --git a/Makefile b/Makefile index 684716af2f..434d40e1cc 100644 --- a/Makefile +++ b/Makefile @@ -35,11 +35,11 @@ unit-test-py3: build-py3 .PHONY: integration-test integration-test: build - docker run -t --rm -v /var/run/docker.sock:/var/run/docker.sock docker-sdk-python py.test -vv tests/integration/${file} + docker run -t --rm -v /var/run/docker.sock:/var/run/docker.sock docker-sdk-python py.test -v tests/integration/${file} .PHONY: integration-test-py3 integration-test-py3: build-py3 - docker run -t --rm -v /var/run/docker.sock:/var/run/docker.sock docker-sdk-python3 py.test -vv tests/integration/${file} + docker run -t --rm -v /var/run/docker.sock:/var/run/docker.sock docker-sdk-python3 py.test tests/integration/${file} TEST_API_VERSION ?= 1.35 TEST_ENGINE_VERSION ?= 17.12.0-ce @@ -57,7 +57,7 @@ integration-dind-py2: build setup-network docker run -d --network dpy-tests --name dpy-dind-py2 --privileged\ dockerswarm/dind:${TEST_ENGINE_VERSION} dockerd -H tcp://0.0.0.0:2375 --experimental docker run -t --rm --env="DOCKER_HOST=tcp://dpy-dind-py2:2375" --env="DOCKER_TEST_API_VERSION=${TEST_API_VERSION}"\ - --network dpy-tests docker-sdk-python py.test -vv tests/integration + --network dpy-tests docker-sdk-python py.test tests/integration docker rm -vf dpy-dind-py2 .PHONY: integration-dind-py3 @@ -66,7 +66,7 @@ integration-dind-py3: build-py3 setup-network docker run -d --network dpy-tests --name dpy-dind-py3 --privileged\ dockerswarm/dind:${TEST_ENGINE_VERSION} dockerd -H tcp://0.0.0.0:2375 --experimental docker run -t --rm --env="DOCKER_HOST=tcp://dpy-dind-py3:2375" --env="DOCKER_TEST_API_VERSION=${TEST_API_VERSION}"\ - --network dpy-tests docker-sdk-python3 py.test -vv tests/integration + --network dpy-tests docker-sdk-python3 py.test tests/integration docker rm -vf dpy-dind-py3 .PHONY: integration-dind-ssl @@ -81,10 +81,10 @@ integration-dind-ssl: build-dind-certs build build-py3 --tlskey=/certs/server-key.pem -H tcp://0.0.0.0:2375 --experimental docker run -t --rm --volumes-from dpy-dind-ssl --env="DOCKER_HOST=tcp://docker:2375"\ --env="DOCKER_TLS_VERIFY=1" --env="DOCKER_CERT_PATH=/certs" --env="DOCKER_TEST_API_VERSION=${TEST_API_VERSION}"\ - --network dpy-tests docker-sdk-python py.test -vv tests/integration + --network dpy-tests docker-sdk-python py.test tests/integration docker run -t --rm --volumes-from dpy-dind-ssl --env="DOCKER_HOST=tcp://docker:2375"\ --env="DOCKER_TLS_VERIFY=1" --env="DOCKER_CERT_PATH=/certs" --env="DOCKER_TEST_API_VERSION=${TEST_API_VERSION}"\ - --network dpy-tests docker-sdk-python3 py.test -vv tests/integration + --network dpy-tests docker-sdk-python3 py.test tests/integration docker rm -vf dpy-dind-ssl dpy-dind-certs .PHONY: flake8 From 9146dd57d7a71f830a4845b9ecd52d561dee29ef Mon Sep 17 00:00:00 2001 From: Corentin Henry Date: Sat, 15 Dec 2018 14:16:56 +0100 Subject: [PATCH 0832/1301] code style improvement Signed-off-by: Corentin Henry --- docker/api/build.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docker/api/build.py b/docker/api/build.py index 0871df8c60..dab06d5d57 100644 --- a/docker/api/build.py +++ b/docker/api/build.py @@ -168,11 +168,11 @@ def build(self, path=None, tag=None, quiet=False, fileobj=None, } params.update(container_limits) - final_buildargs = self._proxy_configs.get_environment() + proxy_args = self._proxy_configs.get_environment() + for k, v in proxy_args.items(): + buildargs.setdefault(k, v) if buildargs: - final_buildargs.update(buildargs) - if final_buildargs: - params.update({'buildargs': json.dumps(final_buildargs)}) + params.update({'buildargs': json.dumps(buildargs)}) if shmsize: if utils.version_gte(self._version, '1.22'): From f97f71342ff07038ee56fac030daef6f568fa4b3 Mon Sep 17 00:00:00 2001 From: Corentin Henry Date: Sat, 15 Dec 2018 14:27:09 +0100 Subject: [PATCH 0833/1301] refactor ProxyConfig Signed-off-by: Corentin Henry --- docker/utils/proxy.py | 52 ++++++++++++++++++++++++++++++------------- 1 file changed, 36 insertions(+), 16 deletions(-) diff --git a/docker/utils/proxy.py b/docker/utils/proxy.py index 3f55a3c13d..943c25b02f 100644 --- a/docker/utils/proxy.py +++ b/docker/utils/proxy.py @@ -1,16 +1,41 @@ from .utils import format_environment -class ProxyConfig(): +class ProxyConfig(dict): ''' Hold the client's proxy configuration ''' + @property + def http(self): + return self['http'] - def __init__(self, http=None, https=None, ftp=None, no_proxy=None): - self.http = http - self.https = https - self.ftp = ftp - self.no_proxy = no_proxy + @http.setter + def http(self, value): + self['http'] = value + + @property + def https(self): + return self['https'] + + @https.setter + def https(self, value): + self['https'] = value + + @property + def ftp(self): + return self['ftp'] + + @ftp.setter + def ftp(self, value): + self['ftp'] = value + + @property + def no_proxy(self): + return self['no_proxy'] + + @no_proxy.setter + def no_proxy(self, value): + self['no_proxy'] = value @staticmethod def from_dict(config): @@ -22,10 +47,11 @@ def from_dict(config): https://docs.docker.com/network/proxy/#configure-the-docker-client ''' return ProxyConfig( - http=config.get('httpProxy', None), - https=config.get('httpsProxy', None), - ftp=config.get('ftpProxy', None), - no_proxy=config.get('noProxy', None)) + http=config.get('httpProxy'), + https=config.get('httpsProxy'), + ftp=config.get('ftpProxy'), + no_proxy=config.get('noProxy'), + ) def get_environment(self): ''' @@ -58,12 +84,6 @@ def inject_proxy_environment(self, environment): # variables defined in "environment" to take precedence. return proxy_env + environment - def __bool__(self): - return bool(self.http or self.https or self.ftp or self.no_proxy) - - def __nonzero__(self): - return self.__bool__() - def __str__(self): return 'ProxyConfig(http={}, https={}, ftp={}, no_proxy={})'.format( self.http, self.https, self.ftp, self.no_proxy) From 73c17f85e5844accc401dc5ca4ae879bafb7f9e2 Mon Sep 17 00:00:00 2001 From: Corentin Henry Date: Sat, 15 Dec 2018 14:28:01 +0100 Subject: [PATCH 0834/1301] fix typo in docstring Signed-off-by: Corentin Henry --- docker/utils/proxy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/utils/proxy.py b/docker/utils/proxy.py index 943c25b02f..1c4fb248ef 100644 --- a/docker/utils/proxy.py +++ b/docker/utils/proxy.py @@ -72,7 +72,7 @@ def get_environment(self): def inject_proxy_environment(self, environment): ''' Given a list of strings representing environment variables, prepend the - environemt variables corresponding to the proxy settings. + environment variables corresponding to the proxy settings. ''' if not self: return environment From 2c4a8651a82016bd422e09688542872d3e236958 Mon Sep 17 00:00:00 2001 From: Corentin Henry Date: Sat, 15 Dec 2018 14:32:10 +0100 Subject: [PATCH 0835/1301] By default, disable proxy support This is to avoid a breaking change Signed-off-by: Corentin Henry --- docker/api/build.py | 14 ++++++++++---- docker/api/container.py | 11 +++++++++-- docker/api/exec_api.py | 11 +++++++++-- docker/models/containers.py | 9 +++++++-- docker/utils/proxy.py | 8 ++++---- tests/integration/api_exec_test.py | 7 ++++--- tests/unit/models_containers_test.py | 4 ++-- 7 files changed, 45 insertions(+), 19 deletions(-) diff --git a/docker/api/build.py b/docker/api/build.py index dab06d5d57..53c94b0dcf 100644 --- a/docker/api/build.py +++ b/docker/api/build.py @@ -19,7 +19,8 @@ def build(self, path=None, tag=None, quiet=False, fileobj=None, forcerm=False, dockerfile=None, container_limits=None, decode=False, buildargs=None, gzip=False, shmsize=None, labels=None, cache_from=None, target=None, network_mode=None, - squash=None, extra_hosts=None, platform=None, isolation=None): + squash=None, extra_hosts=None, platform=None, isolation=None, + use_config_proxy=False): """ Similar to the ``docker build`` command. Either ``path`` or ``fileobj`` needs to be set. ``path`` can be a local path (to a directory @@ -103,6 +104,10 @@ def build(self, path=None, tag=None, quiet=False, fileobj=None, platform (str): Platform in the format ``os[/arch[/variant]]`` isolation (str): Isolation technology used during build. Default: `None`. + use_config_proxy (bool): If ``True``, and if the docker client + configuration file (``~/.docker/config.json`` by default) + contains a proxy configuration, the corresponding environment + variables will be set in the container being built. Returns: A generator for the build output. @@ -168,9 +173,10 @@ def build(self, path=None, tag=None, quiet=False, fileobj=None, } params.update(container_limits) - proxy_args = self._proxy_configs.get_environment() - for k, v in proxy_args.items(): - buildargs.setdefault(k, v) + if use_config_proxy: + proxy_args = self._proxy_configs.get_environment() + for k, v in proxy_args.items(): + buildargs.setdefault(k, v) if buildargs: params.update({'buildargs': json.dumps(buildargs)}) diff --git a/docker/api/container.py b/docker/api/container.py index 2a80ff7dc7..54f9950ffd 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -221,7 +221,8 @@ def create_container(self, image, command=None, hostname=None, user=None, working_dir=None, domainname=None, host_config=None, mac_address=None, labels=None, stop_signal=None, networking_config=None, healthcheck=None, - stop_timeout=None, runtime=None): + stop_timeout=None, runtime=None, + use_config_proxy=False): """ Creates a container. Parameters are similar to those for the ``docker run`` command except it doesn't support the attach options (``-a``). @@ -390,6 +391,10 @@ def create_container(self, image, command=None, hostname=None, user=None, runtime (str): Runtime to use with this container. healthcheck (dict): Specify a test to perform to check that the container is healthy. + use_config_proxy (bool): If ``True``, and if the docker client + configuration file (``~/.docker/config.json`` by default) + contains a proxy configuration, the corresponding environment + variables will be set in the container being created. Returns: A dictionary with an image 'Id' key and a 'Warnings' key. @@ -405,7 +410,9 @@ def create_container(self, image, command=None, hostname=None, user=None, if isinstance(environment, dict): environment = utils.utils.format_environment(environment) - environment = self._proxy_configs.inject_proxy_environment(environment) + if use_config_proxy: + environment = \ + self._proxy_configs.inject_proxy_environment(environment) config = self.create_container_config( image, command, hostname, user, detach, stdin_open, tty, diff --git a/docker/api/exec_api.py b/docker/api/exec_api.py index 0dabdd3078..830432a72c 100644 --- a/docker/api/exec_api.py +++ b/docker/api/exec_api.py @@ -8,7 +8,8 @@ class ExecApiMixin(object): @utils.check_resource('container') def exec_create(self, container, cmd, stdout=True, stderr=True, stdin=False, tty=False, privileged=False, user='', - environment=None, workdir=None, detach_keys=None): + environment=None, workdir=None, detach_keys=None, + use_config_proxy=False): """ Sets up an exec instance in a running container. @@ -31,6 +32,10 @@ def exec_create(self, container, cmd, stdout=True, stderr=True, or `ctrl-` where `` is one of: `a-z`, `@`, `^`, `[`, `,` or `_`. ~/.docker/config.json is used by default. + use_config_proxy (bool): If ``True``, and if the docker client + configuration file (``~/.docker/config.json`` by default) + contains a proxy configuration, the corresponding environment + variables will be set in the container being created. Returns: (dict): A dictionary with an exec ``Id`` key. @@ -50,7 +55,9 @@ def exec_create(self, container, cmd, stdout=True, stderr=True, if isinstance(environment, dict): environment = utils.utils.format_environment(environment) - environment = self._proxy_configs.inject_proxy_environment(environment) + if use_config_proxy: + environment = \ + self._proxy_configs.inject_proxy_environment(environment) data = { 'Container': container, diff --git a/docker/models/containers.py b/docker/models/containers.py index 41bc4da859..817d5db5f5 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -144,7 +144,8 @@ def diff(self): def exec_run(self, cmd, stdout=True, stderr=True, stdin=False, tty=False, privileged=False, user='', detach=False, stream=False, - socket=False, environment=None, workdir=None, demux=False): + socket=False, environment=None, workdir=None, demux=False, + use_config_proxy=False): """ Run a command inside this container. Similar to ``docker exec``. @@ -167,6 +168,10 @@ def exec_run(self, cmd, stdout=True, stderr=True, stdin=False, tty=False, ``{"PASSWORD": "xxx"}``. workdir (str): Path to working directory for this exec session demux (bool): Return stdout and stderr separately + use_config_proxy (bool): If ``True``, and if the docker client + configuration file (``~/.docker/config.json`` by default) + contains a proxy configuration, the corresponding environment + variables will be set in the command's environment. Returns: (ExecResult): A tuple of (exit_code, output) @@ -185,7 +190,7 @@ def exec_run(self, cmd, stdout=True, stderr=True, stdin=False, tty=False, resp = self.client.api.exec_create( self.id, cmd, stdout=stdout, stderr=stderr, stdin=stdin, tty=tty, privileged=privileged, user=user, environment=environment, - workdir=workdir + workdir=workdir, use_config_proxy=use_config_proxy, ) exec_output = self.client.api.exec_start( resp['Id'], detach=detach, tty=tty, stream=stream, socket=socket, diff --git a/docker/utils/proxy.py b/docker/utils/proxy.py index 1c4fb248ef..c368116eab 100644 --- a/docker/utils/proxy.py +++ b/docker/utils/proxy.py @@ -7,7 +7,7 @@ class ProxyConfig(dict): ''' @property def http(self): - return self['http'] + return self.get('http') @http.setter def http(self, value): @@ -15,7 +15,7 @@ def http(self, value): @property def https(self): - return self['https'] + return self.get('https') @https.setter def https(self, value): @@ -23,7 +23,7 @@ def https(self, value): @property def ftp(self): - return self['ftp'] + return self.get('ftp') @ftp.setter def ftp(self, value): @@ -31,7 +31,7 @@ def ftp(self, value): @property def no_proxy(self): - return self['no_proxy'] + return self.get('no_proxy') @no_proxy.setter def no_proxy(self, value): diff --git a/tests/integration/api_exec_test.py b/tests/integration/api_exec_test.py index 6d4f97daab..8947b41328 100644 --- a/tests/integration/api_exec_test.py +++ b/tests/integration/api_exec_test.py @@ -20,10 +20,11 @@ def test_execute_proxy_env(self): self.client.start(id) self.tmp_containers.append(id) + cmd = 'sh -c "env | grep -i proxy"' + # First, just make sure the environment variables from the custom # config are set - res = self.client.exec_create( - id, cmd='sh -c "env | grep -i proxy"') + res = self.client.exec_create(id, cmd=cmd, use_config_proxy=True) output = self.client.exec_start(res).decode('utf-8').split('\n') expected = [ 'ftp_proxy=a', 'https_proxy=b', 'http_proxy=c', 'no_proxy=d', @@ -34,7 +35,7 @@ def test_execute_proxy_env(self): # Overwrite some variables with a custom environment env = {'https_proxy': 'xxx', 'HTTPS_PROXY': 'XXX'} res = self.client.exec_create( - id, cmd='sh -c "env | grep -i proxy"', environment=env) + id, cmd=cmd, environment=env, use_config_proxy=True) output = self.client.exec_start(res).decode('utf-8').split('\n') expected = [ 'ftp_proxy=a', 'https_proxy=xxx', 'http_proxy=c', 'no_proxy=d', diff --git a/tests/unit/models_containers_test.py b/tests/unit/models_containers_test.py index cb92c62be0..b35aeb6a38 100644 --- a/tests/unit/models_containers_test.py +++ b/tests/unit/models_containers_test.py @@ -416,7 +416,7 @@ def test_exec_run(self): client.api.exec_create.assert_called_with( FAKE_CONTAINER_ID, "echo hello world", stdout=True, stderr=True, stdin=False, tty=False, privileged=True, user='', environment=None, - workdir=None + workdir=None, use_config_proxy=False, ) client.api.exec_start.assert_called_with( FAKE_EXEC_ID, detach=False, tty=False, stream=True, socket=False, @@ -430,7 +430,7 @@ def test_exec_run_failure(self): client.api.exec_create.assert_called_with( FAKE_CONTAINER_ID, "docker ps", stdout=True, stderr=True, stdin=False, tty=False, privileged=True, user='', environment=None, - workdir=None + workdir=None, use_config_proxy=False, ) client.api.exec_start.assert_called_with( FAKE_EXEC_ID, detach=False, tty=False, stream=False, socket=False, From 6969e8becde593ee3dc1d3093d9299502ccc10ed Mon Sep 17 00:00:00 2001 From: Corentin Henry Date: Mon, 17 Dec 2018 10:44:37 +0100 Subject: [PATCH 0836/1301] handle url-based proxy configurations Signed-off-by: Corentin Henry --- docker/api/client.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docker/api/client.py b/docker/api/client.py index b12398e5ed..668dfeef86 100644 --- a/docker/api/client.py +++ b/docker/api/client.py @@ -115,10 +115,13 @@ def __init__(self, base_url=None, version=None, self.headers['User-Agent'] = user_agent self._general_configs = config.load_general_config() + + proxy_config = self._general_configs.get('proxies', {}) try: - proxies = self._general_configs['proxies']['default'] + proxies = proxy_config[base_url] except KeyError: - proxies = {} + proxies = proxy_config.get('default', {}) + self._proxy_configs = ProxyConfig.from_dict(proxies) self._auth_configs = auth.load_config( From 65bebc085ab8bb36e5d6de44399b8480e7ae3e68 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 9 Jan 2019 10:52:50 -0800 Subject: [PATCH 0837/1301] Style fixes and removed some unused code Signed-off-by: Joffrey F --- docker/api/container.py | 6 ++++-- docker/utils/proxy.py | 16 ---------------- tests/integration/api_build_test.py | 11 ++++++++--- tests/integration/api_exec_test.py | 27 ++++++++++++++++----------- 4 files changed, 28 insertions(+), 32 deletions(-) diff --git a/docker/api/container.py b/docker/api/container.py index 54f9950ffd..43ae5320ff 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -410,9 +410,11 @@ def create_container(self, image, command=None, hostname=None, user=None, if isinstance(environment, dict): environment = utils.utils.format_environment(environment) + if use_config_proxy: - environment = \ - self._proxy_configs.inject_proxy_environment(environment) + environment = self._proxy_configs.inject_proxy_environment( + environment + ) config = self.create_container_config( image, command, hostname, user, detach, stdin_open, tty, diff --git a/docker/utils/proxy.py b/docker/utils/proxy.py index c368116eab..49e98ed912 100644 --- a/docker/utils/proxy.py +++ b/docker/utils/proxy.py @@ -9,34 +9,18 @@ class ProxyConfig(dict): def http(self): return self.get('http') - @http.setter - def http(self, value): - self['http'] = value - @property def https(self): return self.get('https') - @https.setter - def https(self, value): - self['https'] = value - @property def ftp(self): return self.get('ftp') - @ftp.setter - def ftp(self, value): - self['ftp'] = value - @property def no_proxy(self): return self.get('no_proxy') - @no_proxy.setter - def no_proxy(self, value): - self['no_proxy'] = value - @staticmethod def from_dict(config): ''' diff --git a/tests/integration/api_build_test.py b/tests/integration/api_build_test.py index 3c4e982048..8bfc7960fc 100644 --- a/tests/integration/api_build_test.py +++ b/tests/integration/api_build_test.py @@ -16,7 +16,8 @@ class BuildTest(BaseAPIIntegrationTest): def test_build_with_proxy(self): self.client._proxy_configs = ProxyConfig( - ftp='a', http='b', https='c', no_proxy='d') + ftp='a', http='b', https='c', no_proxy='d' + ) script = io.BytesIO('\n'.join([ 'FROM busybox', @@ -29,11 +30,13 @@ def test_build_with_proxy(self): 'RUN env | grep "NO_PROXY=d"', 'RUN env | grep "no_proxy=d"', ]).encode('ascii')) + self.client.build(fileobj=script, decode=True) def test_build_with_proxy_and_buildargs(self): self.client._proxy_configs = ProxyConfig( - ftp='a', http='b', https='c', no_proxy='d') + ftp='a', http='b', https='c', no_proxy='d' + ) script = io.BytesIO('\n'.join([ 'FROM busybox', @@ -46,10 +49,12 @@ def test_build_with_proxy_and_buildargs(self): 'RUN env | grep "NO_PROXY=d"', 'RUN env | grep "no_proxy=d"', ]).encode('ascii')) + self.client.build( fileobj=script, decode=True, - buildargs={'FTP_PROXY': 'XXX', 'ftp_proxy': 'xxx'}) + buildargs={'FTP_PROXY': 'XXX', 'ftp_proxy': 'xxx'} + ) def test_build_streaming(self): script = io.BytesIO('\n'.join([ diff --git a/tests/integration/api_exec_test.py b/tests/integration/api_exec_test.py index 8947b41328..e6079eb337 100644 --- a/tests/integration/api_exec_test.py +++ b/tests/integration/api_exec_test.py @@ -9,37 +9,42 @@ class ExecTest(BaseAPIIntegrationTest): - def test_execute_proxy_env(self): + def test_execute_command_with_proxy_env(self): # Set a custom proxy config on the client self.client._proxy_configs = ProxyConfig( - ftp='a', https='b', http='c', no_proxy='d') + ftp='a', https='b', http='c', no_proxy='d' + ) container = self.client.create_container( - BUSYBOX, 'cat', detach=True, stdin_open=True) - id = container['Id'] - self.client.start(id) - self.tmp_containers.append(id) + BUSYBOX, 'cat', detach=True, stdin_open=True, + use_config_proxy=True, + ) + self.client.start(container) + self.tmp_containers.append(container) cmd = 'sh -c "env | grep -i proxy"' # First, just make sure the environment variables from the custom # config are set - res = self.client.exec_create(id, cmd=cmd, use_config_proxy=True) + + res = self.client.exec_create(container, cmd=cmd) output = self.client.exec_start(res).decode('utf-8').split('\n') expected = [ 'ftp_proxy=a', 'https_proxy=b', 'http_proxy=c', 'no_proxy=d', - 'FTP_PROXY=a', 'HTTPS_PROXY=b', 'HTTP_PROXY=c', 'NO_PROXY=d'] + 'FTP_PROXY=a', 'HTTPS_PROXY=b', 'HTTP_PROXY=c', 'NO_PROXY=d' + ] for item in expected: assert item in output # Overwrite some variables with a custom environment env = {'https_proxy': 'xxx', 'HTTPS_PROXY': 'XXX'} - res = self.client.exec_create( - id, cmd=cmd, environment=env, use_config_proxy=True) + + res = self.client.exec_create(container, cmd=cmd, environment=env) output = self.client.exec_start(res).decode('utf-8').split('\n') expected = [ 'ftp_proxy=a', 'https_proxy=xxx', 'http_proxy=c', 'no_proxy=d', - 'FTP_PROXY=a', 'HTTPS_PROXY=XXX', 'HTTP_PROXY=c', 'NO_PROXY=d'] + 'FTP_PROXY=a', 'HTTPS_PROXY=XXX', 'HTTP_PROXY=c', 'NO_PROXY=d' + ] for item in expected: assert item in output From 219c52141e3cd15db3348c5420220f640323499f Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Wed, 9 Jan 2019 00:35:03 +0100 Subject: [PATCH 0838/1301] Regression 443 test: relax status-code check This test was testing for a 500 status, but this status is actually a bug in the API (as it's due to an invalid request), and the API should actually return a 400 status. To make this test handle both situations, relax the test to accept either a 4xx or 5xx status. Signed-off-by: Sebastiaan van Stijn --- docker/errors.py | 3 +++ tests/integration/regression_test.py | 2 +- tests/unit/errors_test.py | 21 +++++++++++++++++++++ 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/docker/errors.py b/docker/errors.py index 0253695a5f..c340dcb123 100644 --- a/docker/errors.py +++ b/docker/errors.py @@ -63,6 +63,9 @@ def status_code(self): if self.response is not None: return self.response.status_code + def is_error(self): + return self.is_client_error() or self.is_server_error() + def is_client_error(self): if self.status_code is None: return False diff --git a/tests/integration/regression_test.py b/tests/integration/regression_test.py index 0fd4e43104..9aab076e30 100644 --- a/tests/integration/regression_test.py +++ b/tests/integration/regression_test.py @@ -14,7 +14,7 @@ def test_443_handle_nonchunked_response_in_stream(self): with pytest.raises(docker.errors.APIError) as exc: for line in self.client.build(fileobj=dfile, tag="a/b/c"): pass - assert exc.value.response.status_code == 500 + assert exc.value.is_error() dfile.close() def test_542_truncate_ids_client_side(self): diff --git a/tests/unit/errors_test.py b/tests/unit/errors_test.py index e27a9b1975..2134f86f04 100644 --- a/tests/unit/errors_test.py +++ b/tests/unit/errors_test.py @@ -79,6 +79,27 @@ def test_is_client_error_400(self): err = APIError('', response=resp) assert err.is_client_error() is True + def test_is_error_300(self): + """Report no error on 300 response.""" + resp = requests.Response() + resp.status_code = 300 + err = APIError('', response=resp) + assert err.is_error() is False + + def test_is_error_400(self): + """Report error on 400 response.""" + resp = requests.Response() + resp.status_code = 400 + err = APIError('', response=resp) + assert err.is_error() is True + + def test_is_error_500(self): + """Report error on 500 response.""" + resp = requests.Response() + resp.status_code = 500 + err = APIError('', response=resp) + assert err.is_error() is True + def test_create_error_from_exception(self): resp = requests.Response() resp.status_code = 500 From a579e9e20578ca9a074b28865ff594ed02fba6b3 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 9 Jan 2019 14:38:53 -0800 Subject: [PATCH 0839/1301] Remove use_config_proxy from exec. Add use_config_proxy docs to DockerClient Signed-off-by: Joffrey F --- Jenkinsfile | 2 +- docker/api/exec_api.py | 10 +--------- docker/models/containers.py | 16 ++++++++-------- docker/models/images.py | 4 ++++ tests/integration/models_containers_test.py | 13 +++++++++++++ tests/unit/models_containers_test.py | 4 ++-- 6 files changed, 29 insertions(+), 20 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 33a0fc3136..8724c10fb3 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -91,7 +91,7 @@ def runTests = { Map settings -> --network ${testNetwork} \\ --volumes-from ${dindContainerName} \\ ${testImage} \\ - py.test -v -rxs tests/integration + py.test -v -rxs --cov=docker tests/ """ } finally { sh """ diff --git a/docker/api/exec_api.py b/docker/api/exec_api.py index 830432a72c..d13b128998 100644 --- a/docker/api/exec_api.py +++ b/docker/api/exec_api.py @@ -8,8 +8,7 @@ class ExecApiMixin(object): @utils.check_resource('container') def exec_create(self, container, cmd, stdout=True, stderr=True, stdin=False, tty=False, privileged=False, user='', - environment=None, workdir=None, detach_keys=None, - use_config_proxy=False): + environment=None, workdir=None, detach_keys=None): """ Sets up an exec instance in a running container. @@ -32,10 +31,6 @@ def exec_create(self, container, cmd, stdout=True, stderr=True, or `ctrl-` where `` is one of: `a-z`, `@`, `^`, `[`, `,` or `_`. ~/.docker/config.json is used by default. - use_config_proxy (bool): If ``True``, and if the docker client - configuration file (``~/.docker/config.json`` by default) - contains a proxy configuration, the corresponding environment - variables will be set in the container being created. Returns: (dict): A dictionary with an exec ``Id`` key. @@ -55,9 +50,6 @@ def exec_create(self, container, cmd, stdout=True, stderr=True, if isinstance(environment, dict): environment = utils.utils.format_environment(environment) - if use_config_proxy: - environment = \ - self._proxy_configs.inject_proxy_environment(environment) data = { 'Container': container, diff --git a/docker/models/containers.py b/docker/models/containers.py index 817d5db5f5..10f667d706 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -144,8 +144,7 @@ def diff(self): def exec_run(self, cmd, stdout=True, stderr=True, stdin=False, tty=False, privileged=False, user='', detach=False, stream=False, - socket=False, environment=None, workdir=None, demux=False, - use_config_proxy=False): + socket=False, environment=None, workdir=None, demux=False): """ Run a command inside this container. Similar to ``docker exec``. @@ -168,10 +167,6 @@ def exec_run(self, cmd, stdout=True, stderr=True, stdin=False, tty=False, ``{"PASSWORD": "xxx"}``. workdir (str): Path to working directory for this exec session demux (bool): Return stdout and stderr separately - use_config_proxy (bool): If ``True``, and if the docker client - configuration file (``~/.docker/config.json`` by default) - contains a proxy configuration, the corresponding environment - variables will be set in the command's environment. Returns: (ExecResult): A tuple of (exit_code, output) @@ -190,7 +185,7 @@ def exec_run(self, cmd, stdout=True, stderr=True, stdin=False, tty=False, resp = self.client.api.exec_create( self.id, cmd, stdout=stdout, stderr=stderr, stdin=stdin, tty=tty, privileged=privileged, user=user, environment=environment, - workdir=workdir, use_config_proxy=use_config_proxy, + workdir=workdir, ) exec_output = self.client.api.exec_start( resp['Id'], detach=detach, tty=tty, stream=stream, socket=socket, @@ -682,6 +677,7 @@ def run(self, image, command=None, stdout=True, stderr=False, For example: ``{"Name": "on-failure", "MaximumRetryCount": 5}`` + runtime (str): Runtime to use with this container. security_opt (:py:class:`list`): A list of string values to customize labels for MLS systems, such as SELinux. shm_size (str or int): Size of /dev/shm (e.g. ``1G``). @@ -713,6 +709,10 @@ def run(self, image, command=None, stdout=True, stderr=False, tty (bool): Allocate a pseudo-TTY. ulimits (:py:class:`list`): Ulimits to set inside the container, as a list of :py:class:`docker.types.Ulimit` instances. + use_config_proxy (bool): If ``True``, and if the docker client + configuration file (``~/.docker/config.json`` by default) + contains a proxy configuration, the corresponding environment + variables will be set in the container being built. user (str or int): Username or UID to run commands as inside the container. userns_mode (str): Sets the user namespace mode for the container @@ -737,7 +737,6 @@ def run(self, image, command=None, stdout=True, stderr=False, volumes_from (:py:class:`list`): List of container names or IDs to get volumes from. working_dir (str): Path to the working directory. - runtime (str): Runtime to use with this container. Returns: The container logs, either ``STDOUT``, ``STDERR``, or both, @@ -952,6 +951,7 @@ def prune(self, filters=None): 'stdin_open', 'stop_signal', 'tty', + 'use_config_proxy', 'user', 'volume_driver', 'working_dir', diff --git a/docker/models/images.py b/docker/models/images.py index 30e86f109e..af94520d9b 100644 --- a/docker/models/images.py +++ b/docker/models/images.py @@ -258,6 +258,10 @@ def build(self, **kwargs): platform (str): Platform in the format ``os[/arch[/variant]]``. isolation (str): Isolation technology used during build. Default: `None`. + use_config_proxy (bool): If ``True``, and if the docker client + configuration file (``~/.docker/config.json`` by default) + contains a proxy configuration, the corresponding environment + variables will be set in the container being built. Returns: (tuple): The first item is the :py:class:`Image` object for the diff --git a/tests/integration/models_containers_test.py b/tests/integration/models_containers_test.py index b48f6fb6ce..92eca36d1a 100644 --- a/tests/integration/models_containers_test.py +++ b/tests/integration/models_containers_test.py @@ -163,6 +163,19 @@ def test_run_with_streamed_logs_and_cancel(self): assert logs[0] == b'hello\n' assert logs[1] == b'world\n' + def test_run_with_proxy_config(self): + client = docker.from_env(version=TEST_API_VERSION) + client.api._proxy_configs = docker.utils.proxy.ProxyConfig( + ftp='sakuya.jp:4967' + ) + + out = client.containers.run( + 'alpine', 'sh -c "env"', use_config_proxy=True + ) + + assert b'FTP_PROXY=sakuya.jp:4967\n' in out + assert b'ftp_proxy=sakuya.jp:4967\n' in out + def test_get(self): client = docker.from_env(version=TEST_API_VERSION) container = client.containers.run("alpine", "sleep 300", detach=True) diff --git a/tests/unit/models_containers_test.py b/tests/unit/models_containers_test.py index b35aeb6a38..f44e365851 100644 --- a/tests/unit/models_containers_test.py +++ b/tests/unit/models_containers_test.py @@ -416,7 +416,7 @@ def test_exec_run(self): client.api.exec_create.assert_called_with( FAKE_CONTAINER_ID, "echo hello world", stdout=True, stderr=True, stdin=False, tty=False, privileged=True, user='', environment=None, - workdir=None, use_config_proxy=False, + workdir=None, ) client.api.exec_start.assert_called_with( FAKE_EXEC_ID, detach=False, tty=False, stream=True, socket=False, @@ -430,7 +430,7 @@ def test_exec_run_failure(self): client.api.exec_create.assert_called_with( FAKE_CONTAINER_ID, "docker ps", stdout=True, stderr=True, stdin=False, tty=False, privileged=True, user='', environment=None, - workdir=None, use_config_proxy=False, + workdir=None, ) client.api.exec_start.assert_called_with( FAKE_EXEC_ID, detach=False, tty=False, stream=False, socket=False, From e6783d8cf33d0bd2654da01904b986a4af134daa Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Thu, 10 Jan 2019 17:58:39 +0100 Subject: [PATCH 0840/1301] Bump 3.7.0 Signed-off-by: Ulysses Souza --- docker/version.py | 2 +- docs/change-log.md | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index b4cf22bbb2..c3edb8a35e 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "3.7.0-dev" +version = "3.7.0" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) diff --git a/docs/change-log.md b/docs/change-log.md index 873db8cef5..008a2ad270 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -1,6 +1,28 @@ Change log ========== +3.7.0 +----- + +[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/56?closed=1) + +### Features + +* Added support for multiplexed streams (for `attach` and `exec_start`). Learn + more at https://docker-py.readthedocs.io/en/stable/user_guides/multiplex.html +* Added the `use_config_proxy` parameter to the following methods: + `APIClient.build`, `APIClient.create_container`, `DockerClient.images.build` + and `DockerClient.containers.run` (`False` by default). **This parameter** + **will become `True` by default in the 4.0.0 release.** +* Placement preferences for Swarm services are better validated on the client + and documentation has been updated accordingly + +### Bugfixes + +* Fixed a bug where credential stores weren't queried for relevant registry + credentials with certain variations of the `config.json` file. +* `DockerClient.swarm.init` now returns a boolean value as advertised. + 3.6.0 ----- From 28c9100a7c57c856686e2db0a83853343f6d03bb Mon Sep 17 00:00:00 2001 From: Christopher Crone Date: Thu, 10 Jan 2019 18:26:02 +0100 Subject: [PATCH 0841/1301] Bump to next dev version Signed-off-by: Christopher Crone --- docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index c3edb8a35e..b6302bad11 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "3.7.0" +version = "3.8.0-dev" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) From b6f6e7270ef1acfe7398b99b575d22d0d37ae8bf Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 11 Jan 2019 16:39:16 -0800 Subject: [PATCH 0842/1301] Add registry auth header to inspect_distribution requests Update docstring for auth_config parameter in pull, push, and inspect_distribution Signed-off-by: Joffrey F --- docker/api/image.py | 33 +++++++++++++++++++++++---------- docker/models/images.py | 15 +++++++++------ 2 files changed, 32 insertions(+), 16 deletions(-) diff --git a/docker/api/image.py b/docker/api/image.py index d3fed5c0cc..b370b7d83b 100644 --- a/docker/api/image.py +++ b/docker/api/image.py @@ -247,12 +247,15 @@ def inspect_image(self, image): @utils.minimum_version('1.30') @utils.check_resource('image') - def inspect_distribution(self, image): + def inspect_distribution(self, image, auth_config=None): """ Get image digest and platform information by contacting the registry. Args: image (str): The image name to inspect + auth_config (dict): Override the credentials that are found in the + config for this request. ``auth_config`` should contain the + ``username`` and ``password`` keys to be valid. Returns: (dict): A dict containing distribution data @@ -261,9 +264,21 @@ def inspect_distribution(self, image): :py:class:`docker.errors.APIError` If the server returns an error. """ + registry, _ = auth.resolve_repository_name(image) + + headers = {} + if auth_config is None: + header = auth.get_config_header(self, registry) + if header: + headers['X-Registry-Auth'] = header + else: + log.debug('Sending supplied auth config') + headers['X-Registry-Auth'] = auth.encode_header(auth_config) + + url = self._url("/distribution/{0}/json", image) return self._result( - self._get(self._url("/distribution/{0}/json", image)), True + self._get(url, headers=headers), True ) def load_image(self, data, quiet=None): @@ -336,10 +351,9 @@ def pull(self, repository, tag=None, stream=False, auth_config=None, tag (str): The tag to pull stream (bool): Stream the output as a generator. Make sure to consume the generator, otherwise pull might get cancelled. - auth_config (dict): Override the credentials that - :py:meth:`~docker.api.daemon.DaemonApiMixin.login` has set for - this request. ``auth_config`` should contain the ``username`` - and ``password`` keys to be valid. + auth_config (dict): Override the credentials that are found in the + config for this request. ``auth_config`` should contain the + ``username`` and ``password`` keys to be valid. decode (bool): Decode the JSON data from the server into dicts. Only applies with ``stream=True`` platform (str): Platform in the format ``os[/arch[/variant]]`` @@ -414,10 +428,9 @@ def push(self, repository, tag=None, stream=False, auth_config=None, repository (str): The repository to push to tag (str): An optional tag to push stream (bool): Stream the output as a blocking generator - auth_config (dict): Override the credentials that - :py:meth:`~docker.api.daemon.DaemonApiMixin.login` has set for - this request. ``auth_config`` should contain the ``username`` - and ``password`` keys to be valid. + auth_config (dict): Override the credentials that are found in the + config for this request. ``auth_config`` should contain the + ``username`` and ``password`` keys to be valid. decode (bool): Decode the JSON data from the server into dicts. Only applies with ``stream=True`` diff --git a/docker/models/images.py b/docker/models/images.py index af94520d9b..5419682940 100644 --- a/docker/models/images.py +++ b/docker/models/images.py @@ -315,22 +315,26 @@ def get(self, name): """ return self.prepare_model(self.client.api.inspect_image(name)) - def get_registry_data(self, name): + def get_registry_data(self, name, auth_config=None): """ Gets the registry data for an image. Args: name (str): The name of the image. + auth_config (dict): Override the credentials that are found in the + config for this request. ``auth_config`` should contain the + ``username`` and ``password`` keys to be valid. Returns: (:py:class:`RegistryData`): The data object. + Raises: :py:class:`docker.errors.APIError` If the server returns an error. """ return RegistryData( image_name=name, - attrs=self.client.api.inspect_distribution(name), + attrs=self.client.api.inspect_distribution(name, auth_config), client=self.client, collection=self, ) @@ -404,10 +408,9 @@ def pull(self, repository, tag=None, **kwargs): Args: repository (str): The repository to pull tag (str): The tag to pull - auth_config (dict): Override the credentials that - :py:meth:`~docker.client.DockerClient.login` has set for - this request. ``auth_config`` should contain the ``username`` - and ``password`` keys to be valid. + auth_config (dict): Override the credentials that are found in the + config for this request. ``auth_config`` should contain the + ``username`` and ``password`` keys to be valid. platform (str): Platform in the format ``os[/arch[/variant]]`` Returns: From 24f7c6db669de9bd01baf449d82d0d264316ada1 Mon Sep 17 00:00:00 2001 From: wvaske Date: Thu, 17 Jan 2019 10:40:06 -0600 Subject: [PATCH 0843/1301] Added missing options from RUN_HOST_CONFIG_KWARGS list in docker.models.containers to the docstring for client.containers.run() Signed-off-by: wvaske --- docker/models/containers.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docker/models/containers.py b/docker/models/containers.py index 10f667d706..86cb1535a2 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -540,12 +540,15 @@ def run(self, image, command=None, stdout=True, stderr=False, cap_add (list of str): Add kernel capabilities. For example, ``["SYS_ADMIN", "MKNOD"]``. cap_drop (list of str): Drop kernel capabilities. + cgroup_parent (str): Override the default parent cgroup. cpu_count (int): Number of usable CPUs (Windows only). cpu_percent (int): Usable percentage of the available CPUs (Windows only). cpu_period (int): The length of a CPU period in microseconds. cpu_quota (int): Microseconds of CPU time that the container can get in a CPU period. + cpu_rt_period (int): Limit CPU real-time period in microseconds. + cpu_rt_runtime (int): Limit CPU real-time runtime in microseconds. cpu_shares (int): CPU shares (relative weight). cpuset_cpus (str): CPUs in which to allow execution (``0-3``, ``0,1``). @@ -589,6 +592,7 @@ def run(self, image, command=None, stdout=True, stderr=False, init_path (str): Path to the docker-init binary ipc_mode (str): Set the IPC mode for the container. isolation (str): Isolation technology to use. Default: `None`. + kernel_memory (int or str): Kernel memory limit labels (dict or list): A dictionary of name-value labels (e.g. ``{"label1": "value1", "label2": "value2"}``) or a list of names of labels to set with empty values (e.g. @@ -598,6 +602,7 @@ def run(self, image, command=None, stdout=True, stderr=False, Containers declared in this dict will be linked to the new container using the provided alias. Default: ``None``. log_config (LogConfig): Logging configuration. + lxc_conf (dict): LXC config. mac_address (str): MAC address to assign to the container. mem_limit (int or str): Memory limit. Accepts float values (which represent the memory limit of the created container in @@ -605,6 +610,7 @@ def run(self, image, command=None, stdout=True, stderr=False, (``100000b``, ``1000k``, ``128m``, ``1g``). If a string is specified without a units character, bytes are assumed as an intended unit. + mem_reservation (int or str): Memory soft limit mem_swappiness (int): Tune a container's memory swappiness behavior. Accepts number between 0 and 100. memswap_limit (str or int): Maximum amount of memory + swap a @@ -718,6 +724,10 @@ def run(self, image, command=None, stdout=True, stderr=False, userns_mode (str): Sets the user namespace mode for the container when user namespace remapping option is enabled. Supported values are: ``host`` + uts_mode (str): Sets the UTS namespace mode for the container. + Supported values are: ``host`` + version (str): The version of the API to use. Set to ``auto`` to + automatically detect the server's version. Default: ``1.30`` volume_driver (str): The name of a volume driver/plugin. volumes (dict or list): A dictionary to configure volumes mounted inside the container. The key is either the host path or a From d429a823ed33032d1980903922251996c551ca5c Mon Sep 17 00:00:00 2001 From: Hannes Ljungberg Date: Fri, 18 Jan 2019 21:50:31 +0100 Subject: [PATCH 0844/1301] Make PlacementPreference build correct context Signed-off-by: Hannes Ljungberg --- docker/types/services.py | 2 +- tests/integration/api_service_test.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/docker/types/services.py b/docker/types/services.py index ac1c181a90..a0721f607b 100644 --- a/docker/types/services.py +++ b/docker/types/services.py @@ -692,7 +692,7 @@ def __init__(self, strategy, descriptor): 'PlacementPreference strategy value is invalid ({}):' ' must be "spread".'.format(strategy) ) - self['SpreadOver'] = descriptor + self['Spread'] = {'SpreadDescriptor': descriptor} class DNSConfig(dict): diff --git a/tests/integration/api_service_test.py b/tests/integration/api_service_test.py index a53ca1c836..57a8d331ce 100644 --- a/tests/integration/api_service_test.py +++ b/tests/integration/api_service_test.py @@ -427,6 +427,21 @@ def test_create_service_with_placement_preferences(self): assert 'Placement' in svc_info['Spec']['TaskTemplate'] assert svc_info['Spec']['TaskTemplate']['Placement'] == placemt + @requires_api_version('1.27') + def test_create_service_with_placement_preferences_tuple(self): + container_spec = docker.types.ContainerSpec(BUSYBOX, ['true']) + placemt = docker.types.Placement(preferences=( + ('spread', 'com.dockerpy.test'), + )) + task_tmpl = docker.types.TaskTemplate( + container_spec, placement=placemt + ) + name = self.get_service_name() + svc_id = self.client.create_service(task_tmpl, name=name) + svc_info = self.client.inspect_service(svc_id) + assert 'Placement' in svc_info['Spec']['TaskTemplate'] + assert svc_info['Spec']['TaskTemplate']['Placement'] == placemt + def test_create_service_with_endpoint_spec(self): container_spec = docker.types.ContainerSpec(BUSYBOX, ['true']) task_tmpl = docker.types.TaskTemplate(container_spec) From 6935ce88192db5b0465d53e3005e15be47f4ed58 Mon Sep 17 00:00:00 2001 From: Tsuyoshi Hombashi Date: Sat, 26 Jan 2019 12:39:14 +0900 Subject: [PATCH 0845/1301] Fix descriptions of the default API version in docs 1.30 -> 1.35 Signed-off-by: Tsuyoshi Hombashi --- docker/api/client.py | 2 +- docker/client.py | 4 ++-- docker/models/containers.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docker/api/client.py b/docker/api/client.py index 668dfeef86..9b70554902 100644 --- a/docker/api/client.py +++ b/docker/api/client.py @@ -82,7 +82,7 @@ class APIClient( base_url (str): URL to the Docker server. For example, ``unix:///var/run/docker.sock`` or ``tcp://127.0.0.1:1234``. version (str): The version of the API to use. Set to ``auto`` to - automatically detect the server's version. Default: ``1.30`` + automatically detect the server's version. Default: ``1.35`` timeout (int): Default timeout for API calls, in seconds. tls (bool or :py:class:`~docker.tls.TLSConfig`): Enable TLS. Pass ``True`` to enable it with default options, or pass a diff --git a/docker/client.py b/docker/client.py index 8d4a52b2ef..99ae1962c4 100644 --- a/docker/client.py +++ b/docker/client.py @@ -26,7 +26,7 @@ class DockerClient(object): base_url (str): URL to the Docker server. For example, ``unix:///var/run/docker.sock`` or ``tcp://127.0.0.1:1234``. version (str): The version of the API to use. Set to ``auto`` to - automatically detect the server's version. Default: ``1.30`` + automatically detect the server's version. Default: ``1.35`` timeout (int): Default timeout for API calls, in seconds. tls (bool or :py:class:`~docker.tls.TLSConfig`): Enable TLS. Pass ``True`` to enable it with default options, or pass a @@ -62,7 +62,7 @@ def from_env(cls, **kwargs): Args: version (str): The version of the API to use. Set to ``auto`` to - automatically detect the server's version. Default: ``1.30`` + automatically detect the server's version. Default: ``1.35`` timeout (int): Default timeout for API calls, in seconds. ssl_version (int): A valid `SSL version`_. assert_hostname (bool): Verify the hostname of the server. diff --git a/docker/models/containers.py b/docker/models/containers.py index 86cb1535a2..089e78c756 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -727,7 +727,7 @@ def run(self, image, command=None, stdout=True, stderr=False, uts_mode (str): Sets the UTS namespace mode for the container. Supported values are: ``host`` version (str): The version of the API to use. Set to ``auto`` to - automatically detect the server's version. Default: ``1.30`` + automatically detect the server's version. Default: ``1.35`` volume_driver (str): The name of a volume driver/plugin. volumes (dict or list): A dictionary to configure volumes mounted inside the container. The key is either the host path or a From 189552eb57016c5a49a1ca6f0f48d616fe5c04d7 Mon Sep 17 00:00:00 2001 From: p1100i Date: Thu, 21 Feb 2019 07:55:38 +0100 Subject: [PATCH 0846/1301] Fix `network_mode` API documentation wording Signed-off-by: p1100i --- docker/api/container.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/api/container.py b/docker/api/container.py index 43ae5320ff..83e965778d 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -512,7 +512,7 @@ def create_host_config(self, *args, **kwargs): network_mode (str): One of: - ``bridge`` Create a new network stack for the container on - on the bridge network. + the bridge network. - ``none`` No networking for this container. - ``container:`` Reuse another container's network stack. From 37e096f6add7e26ada3d6840ce9a9ce341bbdf23 Mon Sep 17 00:00:00 2001 From: Leks Date: Fri, 1 Mar 2019 14:05:39 +0300 Subject: [PATCH 0847/1301] set buildargs default value if None Signed-off-by: Leks --- docker/api/build.py | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/api/build.py b/docker/api/build.py index 53c94b0dcf..5176afb3a9 100644 --- a/docker/api/build.py +++ b/docker/api/build.py @@ -121,6 +121,7 @@ def build(self, path=None, tag=None, quiet=False, fileobj=None, remote = context = None headers = {} container_limits = container_limits or {} + buildargs = buildargs or {} if path is None and fileobj is None: raise TypeError("Either path or fileobj needs to be provided.") if gzip and encoding is not None: From 8d1e9670b1b4a5ee4ea3881236988bfbe40792be Mon Sep 17 00:00:00 2001 From: Hannes Ljungberg Date: Wed, 13 Mar 2019 10:12:17 +0100 Subject: [PATCH 0848/1301] Return API response on service update Signed-off-by: Hannes Ljungberg --- docker/api/service.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docker/api/service.py b/docker/api/service.py index 08e2591730..02f3380e87 100644 --- a/docker/api/service.py +++ b/docker/api/service.py @@ -387,7 +387,7 @@ def update_service(self, service, version, task_template=None, name=None, current specification of the service. Default: ``False`` Returns: - ``True`` if successful. + A dictionary containing a ``Warnings`` key. Raises: :py:class:`docker.errors.APIError` @@ -471,5 +471,4 @@ def update_service(self, service, version, task_template=None, name=None, resp = self._post_json( url, data=data, params={'version': version}, headers=headers ) - self._raise_for_status(resp) - return True + return self._result(resp, json=True) From e48a1a94e6f76f9fd1b2882522eb714b1f70d5d6 Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Tue, 12 Mar 2019 15:36:58 +0100 Subject: [PATCH 0849/1301] Sets a different default number of pools to SSH This is because default the number of connections in OpenSSH is 10 Signed-off-by: Ulysses Souza --- docker/api/client.py | 10 +++++++--- docker/constants.py | 6 ++++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/docker/api/client.py b/docker/api/client.py index 9b70554902..7609651502 100644 --- a/docker/api/client.py +++ b/docker/api/client.py @@ -22,8 +22,8 @@ from .. import auth from ..constants import ( DEFAULT_TIMEOUT_SECONDS, DEFAULT_USER_AGENT, IS_WINDOWS_PLATFORM, - DEFAULT_DOCKER_API_VERSION, STREAM_HEADER_SIZE_BYTES, DEFAULT_NUM_POOLS, - MINIMUM_DOCKER_API_VERSION + DEFAULT_DOCKER_API_VERSION, MINIMUM_DOCKER_API_VERSION, + STREAM_HEADER_SIZE_BYTES, DEFAULT_NUM_POOLS_SSH, DEFAULT_NUM_POOLS ) from ..errors import ( DockerException, InvalidVersion, TLSParameterError, @@ -101,7 +101,7 @@ class APIClient( def __init__(self, base_url=None, version=None, timeout=DEFAULT_TIMEOUT_SECONDS, tls=False, - user_agent=DEFAULT_USER_AGENT, num_pools=DEFAULT_NUM_POOLS, + user_agent=DEFAULT_USER_AGENT, num_pools=None, credstore_env=None): super(APIClient, self).__init__() @@ -132,6 +132,10 @@ def __init__(self, base_url=None, version=None, base_url = utils.parse_host( base_url, IS_WINDOWS_PLATFORM, tls=bool(tls) ) + # SSH has a different default for num_pools to all other adapters + num_pools = num_pools or DEFAULT_NUM_POOLS_SSH if \ + base_url.startswith('ssh://') else DEFAULT_NUM_POOLS + if base_url.startswith('http+unix://'): self._custom_adapter = UnixAdapter( base_url, timeout, pool_connections=num_pools diff --git a/docker/constants.py b/docker/constants.py index 1ab11ec051..dcba0de262 100644 --- a/docker/constants.py +++ b/docker/constants.py @@ -18,4 +18,10 @@ DEFAULT_USER_AGENT = "docker-sdk-python/{0}".format(version) DEFAULT_NUM_POOLS = 25 + +# The OpenSSH server default value for MaxSessions is 10 which means we can +# use up to 9, leaving the final session for the underlying SSH connection. +# For more details see: https://github.com/docker/docker-py/issues/2246 +DEFAULT_NUM_POOLS_SSH = 9 + DEFAULT_DATA_CHUNK_SIZE = 1024 * 2048 From 4d7d4084138fa6161ef5f31d410b0d326d41f777 Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Wed, 13 Mar 2019 19:07:50 +0100 Subject: [PATCH 0850/1301] Homogenize adapters close() behaviour. - Adds a BaseHTTPAdapter with a close method to ensure that the pools is clean on close() - Makes SSHHTTPAdapter reopen a closed connection when needed like the others Signed-off-by: Ulysses Souza --- docker/api/client.py | 15 ++++++++------- docker/tls.py | 4 ++-- docker/transport/__init__.py | 8 ++++---- docker/transport/basehttpadapter.py | 6 ++++++ docker/transport/npipeconn.py | 8 +++----- docker/transport/sshconn.py | 23 ++++++++++++++++------- docker/transport/ssladapter.py | 8 +++++--- docker/transport/unixconn.py | 8 +++----- 8 files changed, 47 insertions(+), 33 deletions(-) create mode 100644 docker/transport/basehttpadapter.py diff --git a/docker/api/client.py b/docker/api/client.py index 7609651502..35dc84e71f 100644 --- a/docker/api/client.py +++ b/docker/api/client.py @@ -30,18 +30,18 @@ create_api_error_from_http_exception ) from ..tls import TLSConfig -from ..transport import SSLAdapter, UnixAdapter +from ..transport import SSLHTTPAdapter, UnixHTTPAdapter from ..utils import utils, check_resource, update_headers, config from ..utils.socket import frames_iter, consume_socket_output, demux_adaptor from ..utils.json_stream import json_stream from ..utils.proxy import ProxyConfig try: - from ..transport import NpipeAdapter + from ..transport import NpipeHTTPAdapter except ImportError: pass try: - from ..transport import SSHAdapter + from ..transport import SSHHTTPAdapter except ImportError: pass @@ -137,7 +137,7 @@ def __init__(self, base_url=None, version=None, base_url.startswith('ssh://') else DEFAULT_NUM_POOLS if base_url.startswith('http+unix://'): - self._custom_adapter = UnixAdapter( + self._custom_adapter = UnixHTTPAdapter( base_url, timeout, pool_connections=num_pools ) self.mount('http+docker://', self._custom_adapter) @@ -151,7 +151,7 @@ def __init__(self, base_url=None, version=None, 'The npipe:// protocol is only supported on Windows' ) try: - self._custom_adapter = NpipeAdapter( + self._custom_adapter = NpipeHTTPAdapter( base_url, timeout, pool_connections=num_pools ) except NameError: @@ -162,7 +162,7 @@ def __init__(self, base_url=None, version=None, self.base_url = 'http+docker://localnpipe' elif base_url.startswith('ssh://'): try: - self._custom_adapter = SSHAdapter( + self._custom_adapter = SSHHTTPAdapter( base_url, timeout, pool_connections=num_pools ) except NameError: @@ -177,7 +177,8 @@ def __init__(self, base_url=None, version=None, if isinstance(tls, TLSConfig): tls.configure_client(self) elif tls: - self._custom_adapter = SSLAdapter(pool_connections=num_pools) + self._custom_adapter = SSLHTTPAdapter( + pool_connections=num_pools) self.mount('https://', self._custom_adapter) self.base_url = base_url diff --git a/docker/tls.py b/docker/tls.py index 4900e9fdf7..d4671d126a 100644 --- a/docker/tls.py +++ b/docker/tls.py @@ -2,7 +2,7 @@ import ssl from . import errors -from .transport import SSLAdapter +from .transport import SSLHTTPAdapter class TLSConfig(object): @@ -105,7 +105,7 @@ def configure_client(self, client): if self.cert: client.cert = self.cert - client.mount('https://', SSLAdapter( + client.mount('https://', SSLHTTPAdapter( ssl_version=self.ssl_version, assert_hostname=self.assert_hostname, assert_fingerprint=self.assert_fingerprint, diff --git a/docker/transport/__init__.py b/docker/transport/__init__.py index d2cf2a7af3..e37fc3ba21 100644 --- a/docker/transport/__init__.py +++ b/docker/transport/__init__.py @@ -1,13 +1,13 @@ # flake8: noqa -from .unixconn import UnixAdapter -from .ssladapter import SSLAdapter +from .unixconn import UnixHTTPAdapter +from .ssladapter import SSLHTTPAdapter try: - from .npipeconn import NpipeAdapter + from .npipeconn import NpipeHTTPAdapter from .npipesocket import NpipeSocket except ImportError: pass try: - from .sshconn import SSHAdapter + from .sshconn import SSHHTTPAdapter except ImportError: pass diff --git a/docker/transport/basehttpadapter.py b/docker/transport/basehttpadapter.py new file mode 100644 index 0000000000..d10c115bc1 --- /dev/null +++ b/docker/transport/basehttpadapter.py @@ -0,0 +1,6 @@ +import requests.adapters + + +class BaseHTTPAdapter(requests.adapters.HTTPAdapter): + def close(self): + self.pools.clear() diff --git a/docker/transport/npipeconn.py b/docker/transport/npipeconn.py index ab9b90480a..aa05538ddf 100644 --- a/docker/transport/npipeconn.py +++ b/docker/transport/npipeconn.py @@ -1,6 +1,7 @@ import six import requests.adapters +from docker.transport.basehttpadapter import BaseHTTPAdapter from .. import constants from .npipesocket import NpipeSocket @@ -68,7 +69,7 @@ def _get_conn(self, timeout): return conn or self._new_conn() -class NpipeAdapter(requests.adapters.HTTPAdapter): +class NpipeHTTPAdapter(BaseHTTPAdapter): __attrs__ = requests.adapters.HTTPAdapter.__attrs__ + ['npipe_path', 'pools', @@ -81,7 +82,7 @@ def __init__(self, base_url, timeout=60, self.pools = RecentlyUsedContainer( pool_connections, dispose_func=lambda p: p.close() ) - super(NpipeAdapter, self).__init__() + super(NpipeHTTPAdapter, self).__init__() def get_connection(self, url, proxies=None): with self.pools.lock: @@ -103,6 +104,3 @@ def request_url(self, request, proxies): # anyway, we simply return the path URL directly. # See also: https://github.com/docker/docker-sdk-python/issues/811 return request.path_url - - def close(self): - self.pools.clear() diff --git a/docker/transport/sshconn.py b/docker/transport/sshconn.py index 0f6bb51fc2..5a8ceb08b3 100644 --- a/docker/transport/sshconn.py +++ b/docker/transport/sshconn.py @@ -2,6 +2,7 @@ import requests.adapters import six +from docker.transport.basehttpadapter import BaseHTTPAdapter from .. import constants if six.PY3: @@ -68,7 +69,7 @@ def _get_conn(self, timeout): return conn or self._new_conn() -class SSHAdapter(requests.adapters.HTTPAdapter): +class SSHHTTPAdapter(BaseHTTPAdapter): __attrs__ = requests.adapters.HTTPAdapter.__attrs__ + [ 'pools', 'timeout', 'ssh_client', @@ -79,15 +80,19 @@ def __init__(self, base_url, timeout=60, self.ssh_client = paramiko.SSHClient() self.ssh_client.load_system_host_keys() - parsed = six.moves.urllib_parse.urlparse(base_url) - self.ssh_client.connect( - parsed.hostname, parsed.port, parsed.username, - ) + self.base_url = base_url + self._connect() self.timeout = timeout self.pools = RecentlyUsedContainer( pool_connections, dispose_func=lambda p: p.close() ) - super(SSHAdapter, self).__init__() + super(SSHHTTPAdapter, self).__init__() + + def _connect(self): + parsed = six.moves.urllib_parse.urlparse(self.base_url) + self.ssh_client.connect( + parsed.hostname, parsed.port, parsed.username, + ) def get_connection(self, url, proxies=None): with self.pools.lock: @@ -95,6 +100,10 @@ def get_connection(self, url, proxies=None): if pool: return pool + # Connection is closed try a reconnect + if not self.ssh_client.get_transport(): + self._connect() + pool = SSHConnectionPool( self.ssh_client, self.timeout ) @@ -103,5 +112,5 @@ def get_connection(self, url, proxies=None): return pool def close(self): - self.pools.clear() + super(SSHHTTPAdapter, self).close() self.ssh_client.close() diff --git a/docker/transport/ssladapter.py b/docker/transport/ssladapter.py index 8fafec3550..12de76cdca 100644 --- a/docker/transport/ssladapter.py +++ b/docker/transport/ssladapter.py @@ -7,6 +7,8 @@ from distutils.version import StrictVersion from requests.adapters import HTTPAdapter +from docker.transport.basehttpadapter import BaseHTTPAdapter + try: import requests.packages.urllib3 as urllib3 except ImportError: @@ -22,7 +24,7 @@ urllib3.connection.match_hostname = match_hostname -class SSLAdapter(HTTPAdapter): +class SSLHTTPAdapter(BaseHTTPAdapter): '''An HTTPS Transport Adapter that uses an arbitrary SSL version.''' __attrs__ = HTTPAdapter.__attrs__ + ['assert_fingerprint', @@ -34,7 +36,7 @@ def __init__(self, ssl_version=None, assert_hostname=None, self.ssl_version = ssl_version self.assert_hostname = assert_hostname self.assert_fingerprint = assert_fingerprint - super(SSLAdapter, self).__init__(**kwargs) + super(SSLHTTPAdapter, self).__init__(**kwargs) def init_poolmanager(self, connections, maxsize, block=False): kwargs = { @@ -57,7 +59,7 @@ def get_connection(self, *args, **kwargs): But we still need to take care of when there is a proxy poolmanager """ - conn = super(SSLAdapter, self).get_connection(*args, **kwargs) + conn = super(SSLHTTPAdapter, self).get_connection(*args, **kwargs) if conn.assert_hostname != self.assert_hostname: conn.assert_hostname = self.assert_hostname return conn diff --git a/docker/transport/unixconn.py b/docker/transport/unixconn.py index c59821a849..b619103247 100644 --- a/docker/transport/unixconn.py +++ b/docker/transport/unixconn.py @@ -3,6 +3,7 @@ import socket from six.moves import http_client as httplib +from docker.transport.basehttpadapter import BaseHTTPAdapter from .. import constants try: @@ -69,7 +70,7 @@ def _new_conn(self): ) -class UnixAdapter(requests.adapters.HTTPAdapter): +class UnixHTTPAdapter(BaseHTTPAdapter): __attrs__ = requests.adapters.HTTPAdapter.__attrs__ + ['pools', 'socket_path', @@ -85,7 +86,7 @@ def __init__(self, socket_url, timeout=60, self.pools = RecentlyUsedContainer( pool_connections, dispose_func=lambda p: p.close() ) - super(UnixAdapter, self).__init__() + super(UnixHTTPAdapter, self).__init__() def get_connection(self, url, proxies=None): with self.pools.lock: @@ -107,6 +108,3 @@ def request_url(self, request, proxies): # anyway, we simply return the path URL directly. # See also: https://github.com/docker/docker-py/issues/811 return request.path_url - - def close(self): - self.pools.clear() From 55ffb761bffd8b6382332faf3e3375817706c690 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 13 Dec 2018 14:53:09 -0800 Subject: [PATCH 0851/1301] Terminate support for Python 3.3 (EOL in 2018) Signed-off-by: Joffrey F --- requirements.txt | 5 ++--- setup.py | 6 +----- test-requirements.txt | 6 ++---- 3 files changed, 5 insertions(+), 12 deletions(-) diff --git a/requirements.txt b/requirements.txt index f1c9bdbc76..461bf530e2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,8 +2,7 @@ appdirs==1.4.3 asn1crypto==0.22.0 backports.ssl-match-hostname==3.5.0.1 cffi==1.10.0 -cryptography==1.9; python_version == '3.3' -cryptography==2.3; python_version > '3.3' +cryptography==2.3 docker-pycreds==0.4.0 enum34==1.1.6 idna==2.5 @@ -17,5 +16,5 @@ pypiwin32==219; sys_platform == 'win32' and python_version < '3.6' pypiwin32==223; sys_platform == 'win32' and python_version >= '3.6' requests==2.20.0 six==1.10.0 +urllib3==1.24.1 websocket-client==0.40.0 -urllib3==1.21.1; python_version == '3.3' \ No newline at end of file diff --git a/setup.py b/setup.py index 94fbdf444e..677bc204ee 100644 --- a/setup.py +++ b/setup.py @@ -29,9 +29,6 @@ ':sys_platform == "win32" and python_version < "3.6"': 'pypiwin32==219', ':sys_platform == "win32" and python_version >= "3.6"': 'pypiwin32==223', - # urllib3 drops support for Python 3.3 in 1.23 - ':python_version == "3.3"': 'urllib3 < 1.23', - # If using docker-py over TLS, highly recommend this option is # pip-installed or pinned. @@ -75,7 +72,7 @@ install_requires=requirements, tests_require=test_requirements, extras_require=extras_require, - python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', zip_safe=False, test_suite='tests', classifiers=[ @@ -87,7 +84,6 @@ 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/test-requirements.txt b/test-requirements.txt index 510fa295ec..df369881c9 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -2,8 +2,6 @@ coverage==4.5.2 flake8==3.6.0; python_version != '3.3' flake8==3.4.1; python_version == '3.3' mock==1.0.1 -pytest==2.9.1; python_version == '3.3' -pytest==4.1.0; python_version != '3.3' -pytest-cov==2.6.1; python_version != '3.3' -pytest-cov==2.5.1; python_version == '3.3' +pytest==4.1.0 +pytest-cov==2.6.1 pytest-timeout==1.3.3 From 5d76e8e13ea85dc583c805120db349a96917f312 Mon Sep 17 00:00:00 2001 From: Hannes Ljungberg Date: Mon, 18 Mar 2019 15:34:36 +0100 Subject: [PATCH 0852/1301] Support sctp as protocol Signed-off-by: Hannes Ljungberg --- docker/api/container.py | 7 ++++--- docker/models/containers.py | 4 ++-- docker/utils/ports.py | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/docker/api/container.py b/docker/api/container.py index 83e965778d..6069181720 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -915,9 +915,10 @@ def port(self, container, private_port): if '/' in private_port: return port_settings.get(private_port) - h_ports = port_settings.get(private_port + '/tcp') - if h_ports is None: - h_ports = port_settings.get(private_port + '/udp') + for protocol in ['tcp', 'udp', 'sctp']: + h_ports = port_settings.get(private_port + '/' + protocol) + if h_ports: + break return h_ports diff --git a/docker/models/containers.py b/docker/models/containers.py index 089e78c756..effa1073ed 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -649,8 +649,8 @@ def run(self, image, command=None, stdout=True, stderr=False, The keys of the dictionary are the ports to bind inside the container, either as an integer or a string in the form - ``port/protocol``, where the protocol is either ``tcp`` or - ``udp``. + ``port/protocol``, where the protocol is either ``tcp``, + ``udp``, or ``sctp``. The values of the dictionary are the corresponding ports to open on the host, which can be either: diff --git a/docker/utils/ports.py b/docker/utils/ports.py index cf5987c94f..a50cc029f2 100644 --- a/docker/utils/ports.py +++ b/docker/utils/ports.py @@ -7,7 +7,7 @@ r"(?P[\d]*)(-(?P[\d]+))?:" # External range ")?" r"(?P[\d]+)(-(?P[\d]+))?" # Internal range - "(?P/(udp|tcp))?" # Protocol + "(?P/(udp|tcp|sctp))?" # Protocol "$" # Match full string ) From 35714c46b10e9900c127341e5a91cabddbb4271d Mon Sep 17 00:00:00 2001 From: Hannes Ljungberg Date: Mon, 18 Mar 2019 15:35:22 +0100 Subject: [PATCH 0853/1301] Test all split_port with all valid protocols Signed-off-by: Hannes Ljungberg --- tests/unit/utils_test.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index a4e9c9c53e..3cb3be91b2 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -491,9 +491,12 @@ def test_split_port_with_host_ip(self): assert external_port == [("127.0.0.1", "1000")] def test_split_port_with_protocol(self): - internal_port, external_port = split_port("127.0.0.1:1000:2000/udp") - assert internal_port == ["2000/udp"] - assert external_port == [("127.0.0.1", "1000")] + for protocol in ['tcp', 'udp', 'sctp']: + internal_port, external_port = split_port( + "127.0.0.1:1000:2000/" + protocol + ) + assert internal_port == ["2000/" + protocol] + assert external_port == [("127.0.0.1", "1000")] def test_split_port_with_host_ip_no_port(self): internal_port, external_port = split_port("127.0.0.1::2000") @@ -546,6 +549,10 @@ def test_split_port_invalid(self): with pytest.raises(ValueError): split_port("0.0.0.0:1000:2000:tcp") + def test_split_port_invalid_protocol(self): + with pytest.raises(ValueError): + split_port("0.0.0.0:1000:2000/ftp") + def test_non_matching_length_port_ranges(self): with pytest.raises(ValueError): split_port("0.0.0.0:1000-1010:2000-2002/tcp") From 7143cf02abece116df63de8a63956cfff185177c Mon Sep 17 00:00:00 2001 From: Hannes Ljungberg Date: Mon, 18 Mar 2019 15:36:02 +0100 Subject: [PATCH 0854/1301] Test port lookup with protocols Signed-off-by: Hannes Ljungberg --- tests/integration/api_container_test.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index 83df3424a9..eb3fd66117 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -1083,11 +1083,17 @@ def test_port(self): port_bindings = { '1111': ('127.0.0.1', '4567'), - '2222': ('127.0.0.1', '4568') + '2222': ('127.0.0.1', '4568'), + '3333/udp': ('127.0.0.1', '4569'), } + ports = [ + 1111, + 2222, + (3333, 'udp'), + ] container = self.client.create_container( - BUSYBOX, ['sleep', '60'], ports=list(port_bindings.keys()), + BUSYBOX, ['sleep', '60'], ports=ports, host_config=self.client.create_host_config( port_bindings=port_bindings, network_mode='bridge' ) @@ -1098,13 +1104,15 @@ def test_port(self): # Call the port function on each biding and compare expected vs actual for port in port_bindings: + port, _, protocol = port.partition('/') actual_bindings = self.client.port(container, port) port_binding = actual_bindings.pop() ip, host_port = port_binding['HostIp'], port_binding['HostPort'] - assert ip == port_bindings[port][0] - assert host_port == port_bindings[port][1] + port_binding = port if not protocol else port + "/" + protocol + assert ip == port_bindings[port_binding][0] + assert host_port == port_bindings[port_binding][1] self.client.kill(id) From 0f7af860d8df01d1c614b20d687ff6d0393d6938 Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Mon, 18 Mar 2019 15:42:54 +0100 Subject: [PATCH 0855/1301] Fix BaseHTTPAdapter for the SSL case Signed-off-by: Ulysses Souza --- docker/transport/basehttpadapter.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker/transport/basehttpadapter.py b/docker/transport/basehttpadapter.py index d10c115bc1..4d819b669c 100644 --- a/docker/transport/basehttpadapter.py +++ b/docker/transport/basehttpadapter.py @@ -3,4 +3,6 @@ class BaseHTTPAdapter(requests.adapters.HTTPAdapter): def close(self): - self.pools.clear() + super(BaseHTTPAdapter, self).close() + if hasattr(self, 'pools'): + self.pools.clear() From 729c2e783079a9c2948318c70fe7aa22681f1ebe Mon Sep 17 00:00:00 2001 From: Yincen Xia Date: Tue, 19 Mar 2019 21:27:13 +0800 Subject: [PATCH 0856/1301] Update doc for container.exec_run & exec_api about demux Signed-off-by: Yincen Xia --- docker/api/exec_api.py | 3 ++- docker/models/containers.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docker/api/exec_api.py b/docker/api/exec_api.py index d13b128998..4c49ac3380 100644 --- a/docker/api/exec_api.py +++ b/docker/api/exec_api.py @@ -137,7 +137,8 @@ def exec_start(self, exec_id, detach=False, tty=False, stream=False, (generator or str or tuple): If ``stream=True``, a generator yielding response chunks. If ``socket=True``, a socket object for the connection. A string containing response data otherwise. If - ``demux=True``, stdout and stderr are separated. + ``demux=True``, a tuple with two elements of type byte: stdout and + stderr. Raises: :py:class:`docker.errors.APIError` diff --git a/docker/models/containers.py b/docker/models/containers.py index 089e78c756..502251d54f 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -173,9 +173,10 @@ def exec_run(self, cmd, stdout=True, stderr=True, stdin=False, tty=False, exit_code: (int): Exit code for the executed command or ``None`` if either ``stream```or ``socket`` is ``True``. - output: (generator or bytes): + output: (generator, bytes, or tuple): If ``stream=True``, a generator yielding response chunks. If ``socket=True``, a socket object for the connection. + If ``demux=True``, a tuple of two bytes: stdout and stderr. A bytestring containing response data otherwise. Raises: From e577f5d61d96a0673b97e92cf60873fcbce57426 Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Tue, 12 Mar 2019 15:36:58 +0100 Subject: [PATCH 0857/1301] Sets a different default number of pools to SSH This is because default the number of connections in OpenSSH is 10 Signed-off-by: Ulysses Souza --- docker/api/client.py | 10 +++++++--- docker/constants.py | 6 ++++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/docker/api/client.py b/docker/api/client.py index 668dfeef86..dc1d01559a 100644 --- a/docker/api/client.py +++ b/docker/api/client.py @@ -22,8 +22,8 @@ from .. import auth from ..constants import ( DEFAULT_TIMEOUT_SECONDS, DEFAULT_USER_AGENT, IS_WINDOWS_PLATFORM, - DEFAULT_DOCKER_API_VERSION, STREAM_HEADER_SIZE_BYTES, DEFAULT_NUM_POOLS, - MINIMUM_DOCKER_API_VERSION + DEFAULT_DOCKER_API_VERSION, MINIMUM_DOCKER_API_VERSION, + STREAM_HEADER_SIZE_BYTES, DEFAULT_NUM_POOLS_SSH, DEFAULT_NUM_POOLS ) from ..errors import ( DockerException, InvalidVersion, TLSParameterError, @@ -101,7 +101,7 @@ class APIClient( def __init__(self, base_url=None, version=None, timeout=DEFAULT_TIMEOUT_SECONDS, tls=False, - user_agent=DEFAULT_USER_AGENT, num_pools=DEFAULT_NUM_POOLS, + user_agent=DEFAULT_USER_AGENT, num_pools=None, credstore_env=None): super(APIClient, self).__init__() @@ -132,6 +132,10 @@ def __init__(self, base_url=None, version=None, base_url = utils.parse_host( base_url, IS_WINDOWS_PLATFORM, tls=bool(tls) ) + # SSH has a different default for num_pools to all other adapters + num_pools = num_pools or DEFAULT_NUM_POOLS_SSH if \ + base_url.startswith('ssh://') else DEFAULT_NUM_POOLS + if base_url.startswith('http+unix://'): self._custom_adapter = UnixAdapter( base_url, timeout, pool_connections=num_pools diff --git a/docker/constants.py b/docker/constants.py index 1ab11ec051..dcba0de262 100644 --- a/docker/constants.py +++ b/docker/constants.py @@ -18,4 +18,10 @@ DEFAULT_USER_AGENT = "docker-sdk-python/{0}".format(version) DEFAULT_NUM_POOLS = 25 + +# The OpenSSH server default value for MaxSessions is 10 which means we can +# use up to 9, leaving the final session for the underlying SSH connection. +# For more details see: https://github.com/docker/docker-py/issues/2246 +DEFAULT_NUM_POOLS_SSH = 9 + DEFAULT_DATA_CHUNK_SIZE = 1024 * 2048 From 313f73648842b486417b2736d97381bb07a9a627 Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Wed, 13 Mar 2019 19:07:50 +0100 Subject: [PATCH 0858/1301] Homogenize adapters close() behaviour. - Adds a BaseHTTPAdapter with a close method to ensure that the pools is clean on close() - Makes SSHHTTPAdapter reopen a closed connection when needed like the others Signed-off-by: Ulysses Souza --- docker/api/client.py | 15 ++++++++------- docker/tls.py | 4 ++-- docker/transport/__init__.py | 8 ++++---- docker/transport/basehttpadapter.py | 6 ++++++ docker/transport/npipeconn.py | 8 +++----- docker/transport/sshconn.py | 23 ++++++++++++++++------- docker/transport/ssladapter.py | 8 +++++--- docker/transport/unixconn.py | 8 +++----- 8 files changed, 47 insertions(+), 33 deletions(-) create mode 100644 docker/transport/basehttpadapter.py diff --git a/docker/api/client.py b/docker/api/client.py index dc1d01559a..b8ae484262 100644 --- a/docker/api/client.py +++ b/docker/api/client.py @@ -30,18 +30,18 @@ create_api_error_from_http_exception ) from ..tls import TLSConfig -from ..transport import SSLAdapter, UnixAdapter +from ..transport import SSLHTTPAdapter, UnixHTTPAdapter from ..utils import utils, check_resource, update_headers, config from ..utils.socket import frames_iter, consume_socket_output, demux_adaptor from ..utils.json_stream import json_stream from ..utils.proxy import ProxyConfig try: - from ..transport import NpipeAdapter + from ..transport import NpipeHTTPAdapter except ImportError: pass try: - from ..transport import SSHAdapter + from ..transport import SSHHTTPAdapter except ImportError: pass @@ -137,7 +137,7 @@ def __init__(self, base_url=None, version=None, base_url.startswith('ssh://') else DEFAULT_NUM_POOLS if base_url.startswith('http+unix://'): - self._custom_adapter = UnixAdapter( + self._custom_adapter = UnixHTTPAdapter( base_url, timeout, pool_connections=num_pools ) self.mount('http+docker://', self._custom_adapter) @@ -151,7 +151,7 @@ def __init__(self, base_url=None, version=None, 'The npipe:// protocol is only supported on Windows' ) try: - self._custom_adapter = NpipeAdapter( + self._custom_adapter = NpipeHTTPAdapter( base_url, timeout, pool_connections=num_pools ) except NameError: @@ -162,7 +162,7 @@ def __init__(self, base_url=None, version=None, self.base_url = 'http+docker://localnpipe' elif base_url.startswith('ssh://'): try: - self._custom_adapter = SSHAdapter( + self._custom_adapter = SSHHTTPAdapter( base_url, timeout, pool_connections=num_pools ) except NameError: @@ -177,7 +177,8 @@ def __init__(self, base_url=None, version=None, if isinstance(tls, TLSConfig): tls.configure_client(self) elif tls: - self._custom_adapter = SSLAdapter(pool_connections=num_pools) + self._custom_adapter = SSLHTTPAdapter( + pool_connections=num_pools) self.mount('https://', self._custom_adapter) self.base_url = base_url diff --git a/docker/tls.py b/docker/tls.py index 4900e9fdf7..d4671d126a 100644 --- a/docker/tls.py +++ b/docker/tls.py @@ -2,7 +2,7 @@ import ssl from . import errors -from .transport import SSLAdapter +from .transport import SSLHTTPAdapter class TLSConfig(object): @@ -105,7 +105,7 @@ def configure_client(self, client): if self.cert: client.cert = self.cert - client.mount('https://', SSLAdapter( + client.mount('https://', SSLHTTPAdapter( ssl_version=self.ssl_version, assert_hostname=self.assert_hostname, assert_fingerprint=self.assert_fingerprint, diff --git a/docker/transport/__init__.py b/docker/transport/__init__.py index d2cf2a7af3..e37fc3ba21 100644 --- a/docker/transport/__init__.py +++ b/docker/transport/__init__.py @@ -1,13 +1,13 @@ # flake8: noqa -from .unixconn import UnixAdapter -from .ssladapter import SSLAdapter +from .unixconn import UnixHTTPAdapter +from .ssladapter import SSLHTTPAdapter try: - from .npipeconn import NpipeAdapter + from .npipeconn import NpipeHTTPAdapter from .npipesocket import NpipeSocket except ImportError: pass try: - from .sshconn import SSHAdapter + from .sshconn import SSHHTTPAdapter except ImportError: pass diff --git a/docker/transport/basehttpadapter.py b/docker/transport/basehttpadapter.py new file mode 100644 index 0000000000..d10c115bc1 --- /dev/null +++ b/docker/transport/basehttpadapter.py @@ -0,0 +1,6 @@ +import requests.adapters + + +class BaseHTTPAdapter(requests.adapters.HTTPAdapter): + def close(self): + self.pools.clear() diff --git a/docker/transport/npipeconn.py b/docker/transport/npipeconn.py index ab9b90480a..aa05538ddf 100644 --- a/docker/transport/npipeconn.py +++ b/docker/transport/npipeconn.py @@ -1,6 +1,7 @@ import six import requests.adapters +from docker.transport.basehttpadapter import BaseHTTPAdapter from .. import constants from .npipesocket import NpipeSocket @@ -68,7 +69,7 @@ def _get_conn(self, timeout): return conn or self._new_conn() -class NpipeAdapter(requests.adapters.HTTPAdapter): +class NpipeHTTPAdapter(BaseHTTPAdapter): __attrs__ = requests.adapters.HTTPAdapter.__attrs__ + ['npipe_path', 'pools', @@ -81,7 +82,7 @@ def __init__(self, base_url, timeout=60, self.pools = RecentlyUsedContainer( pool_connections, dispose_func=lambda p: p.close() ) - super(NpipeAdapter, self).__init__() + super(NpipeHTTPAdapter, self).__init__() def get_connection(self, url, proxies=None): with self.pools.lock: @@ -103,6 +104,3 @@ def request_url(self, request, proxies): # anyway, we simply return the path URL directly. # See also: https://github.com/docker/docker-sdk-python/issues/811 return request.path_url - - def close(self): - self.pools.clear() diff --git a/docker/transport/sshconn.py b/docker/transport/sshconn.py index 0f6bb51fc2..5a8ceb08b3 100644 --- a/docker/transport/sshconn.py +++ b/docker/transport/sshconn.py @@ -2,6 +2,7 @@ import requests.adapters import six +from docker.transport.basehttpadapter import BaseHTTPAdapter from .. import constants if six.PY3: @@ -68,7 +69,7 @@ def _get_conn(self, timeout): return conn or self._new_conn() -class SSHAdapter(requests.adapters.HTTPAdapter): +class SSHHTTPAdapter(BaseHTTPAdapter): __attrs__ = requests.adapters.HTTPAdapter.__attrs__ + [ 'pools', 'timeout', 'ssh_client', @@ -79,15 +80,19 @@ def __init__(self, base_url, timeout=60, self.ssh_client = paramiko.SSHClient() self.ssh_client.load_system_host_keys() - parsed = six.moves.urllib_parse.urlparse(base_url) - self.ssh_client.connect( - parsed.hostname, parsed.port, parsed.username, - ) + self.base_url = base_url + self._connect() self.timeout = timeout self.pools = RecentlyUsedContainer( pool_connections, dispose_func=lambda p: p.close() ) - super(SSHAdapter, self).__init__() + super(SSHHTTPAdapter, self).__init__() + + def _connect(self): + parsed = six.moves.urllib_parse.urlparse(self.base_url) + self.ssh_client.connect( + parsed.hostname, parsed.port, parsed.username, + ) def get_connection(self, url, proxies=None): with self.pools.lock: @@ -95,6 +100,10 @@ def get_connection(self, url, proxies=None): if pool: return pool + # Connection is closed try a reconnect + if not self.ssh_client.get_transport(): + self._connect() + pool = SSHConnectionPool( self.ssh_client, self.timeout ) @@ -103,5 +112,5 @@ def get_connection(self, url, proxies=None): return pool def close(self): - self.pools.clear() + super(SSHHTTPAdapter, self).close() self.ssh_client.close() diff --git a/docker/transport/ssladapter.py b/docker/transport/ssladapter.py index 8fafec3550..12de76cdca 100644 --- a/docker/transport/ssladapter.py +++ b/docker/transport/ssladapter.py @@ -7,6 +7,8 @@ from distutils.version import StrictVersion from requests.adapters import HTTPAdapter +from docker.transport.basehttpadapter import BaseHTTPAdapter + try: import requests.packages.urllib3 as urllib3 except ImportError: @@ -22,7 +24,7 @@ urllib3.connection.match_hostname = match_hostname -class SSLAdapter(HTTPAdapter): +class SSLHTTPAdapter(BaseHTTPAdapter): '''An HTTPS Transport Adapter that uses an arbitrary SSL version.''' __attrs__ = HTTPAdapter.__attrs__ + ['assert_fingerprint', @@ -34,7 +36,7 @@ def __init__(self, ssl_version=None, assert_hostname=None, self.ssl_version = ssl_version self.assert_hostname = assert_hostname self.assert_fingerprint = assert_fingerprint - super(SSLAdapter, self).__init__(**kwargs) + super(SSLHTTPAdapter, self).__init__(**kwargs) def init_poolmanager(self, connections, maxsize, block=False): kwargs = { @@ -57,7 +59,7 @@ def get_connection(self, *args, **kwargs): But we still need to take care of when there is a proxy poolmanager """ - conn = super(SSLAdapter, self).get_connection(*args, **kwargs) + conn = super(SSLHTTPAdapter, self).get_connection(*args, **kwargs) if conn.assert_hostname != self.assert_hostname: conn.assert_hostname = self.assert_hostname return conn diff --git a/docker/transport/unixconn.py b/docker/transport/unixconn.py index c59821a849..b619103247 100644 --- a/docker/transport/unixconn.py +++ b/docker/transport/unixconn.py @@ -3,6 +3,7 @@ import socket from six.moves import http_client as httplib +from docker.transport.basehttpadapter import BaseHTTPAdapter from .. import constants try: @@ -69,7 +70,7 @@ def _new_conn(self): ) -class UnixAdapter(requests.adapters.HTTPAdapter): +class UnixHTTPAdapter(BaseHTTPAdapter): __attrs__ = requests.adapters.HTTPAdapter.__attrs__ + ['pools', 'socket_path', @@ -85,7 +86,7 @@ def __init__(self, socket_url, timeout=60, self.pools = RecentlyUsedContainer( pool_connections, dispose_func=lambda p: p.close() ) - super(UnixAdapter, self).__init__() + super(UnixHTTPAdapter, self).__init__() def get_connection(self, url, proxies=None): with self.pools.lock: @@ -107,6 +108,3 @@ def request_url(self, request, proxies): # anyway, we simply return the path URL directly. # See also: https://github.com/docker/docker-py/issues/811 return request.path_url - - def close(self): - self.pools.clear() From 89485bf26ebb7cf4570549cda42543061d6b4fd7 Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Mon, 18 Mar 2019 15:42:54 +0100 Subject: [PATCH 0859/1301] Fix BaseHTTPAdapter for the SSL case Signed-off-by: Ulysses Souza --- docker/transport/basehttpadapter.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker/transport/basehttpadapter.py b/docker/transport/basehttpadapter.py index d10c115bc1..4d819b669c 100644 --- a/docker/transport/basehttpadapter.py +++ b/docker/transport/basehttpadapter.py @@ -3,4 +3,6 @@ class BaseHTTPAdapter(requests.adapters.HTTPAdapter): def close(self): - self.pools.clear() + super(BaseHTTPAdapter, self).close() + if hasattr(self, 'pools'): + self.pools.clear() From 963818a4d21509696988bffdd755214d8268a75f Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Wed, 20 Mar 2019 11:56:24 +0100 Subject: [PATCH 0860/1301] Bump 3.7.1 Signed-off-by: Ulysses Souza --- docker/version.py | 2 +- docs/change-log.md | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index c3edb8a35e..249475f457 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "3.7.0" +version = "3.7.1" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) diff --git a/docs/change-log.md b/docs/change-log.md index 008a2ad270..9edfee2f87 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -1,6 +1,19 @@ Change log ========== +3.7.1 +----- + +[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/58?closed=1) + +### Bugfixes + +* Set a different default number (which is now 9) for SSH pools +* Adds a BaseHTTPAdapter with a close method to ensure that the +pools is clean on close() +* Makes SSHHTTPAdapter reopen a closed connection when needed +like the others + 3.7.0 ----- From 5d69a0a62e9294bdcedfa9cc6d6d4d1f7c4fe961 Mon Sep 17 00:00:00 2001 From: Barry Shapira Date: Tue, 11 Dec 2018 22:06:59 -0800 Subject: [PATCH 0861/1301] Added arguments to creeate a swarm with a custom address pool and subnet size. Signed-off-by: Barry Shapira --- docker/api/swarm.py | 9 +++++++++ docker/models/swarm.py | 10 ++++++++++ tests/integration/api_swarm_test.py | 23 +++++++++++++++++++++++ 3 files changed, 42 insertions(+) diff --git a/docker/api/swarm.py b/docker/api/swarm.py index 04595da139..bec3efdf23 100644 --- a/docker/api/swarm.py +++ b/docker/api/swarm.py @@ -82,6 +82,7 @@ def get_unlock_key(self): @utils.minimum_version('1.24') def init_swarm(self, advertise_addr=None, listen_addr='0.0.0.0:2377', + default_addr_pool=[], subnet_size=24, force_new_cluster=False, swarm_spec=None): """ Initialize a new Swarm using the current connected engine as the first @@ -102,6 +103,12 @@ def init_swarm(self, advertise_addr=None, listen_addr='0.0.0.0:2377', or an interface followed by a port number, like ``eth0:4567``. If the port number is omitted, the default swarm listening port is used. Default: '0.0.0.0:2377' + default_addr_pool (list of strings): Default Address Pool specifies + default subnet pools for global scope networks. Each pool + should be specified as a CIDR block, like '10.0.0.0/16'. + Default: [] + subnet_size (int): SubnetSize specifies the subnet size of the + networks created from the default subnet pool. Default: 24 force_new_cluster (bool): Force creating a new Swarm, even if already part of one. Default: False swarm_spec (dict): Configuration settings of the new Swarm. Use @@ -122,6 +129,8 @@ def init_swarm(self, advertise_addr=None, listen_addr='0.0.0.0:2377', data = { 'AdvertiseAddr': advertise_addr, 'ListenAddr': listen_addr, + 'DefaultAddrPool': default_addr_pool, + 'SubnetSize': subnet_size, 'ForceNewCluster': force_new_cluster, 'Spec': swarm_spec, } diff --git a/docker/models/swarm.py b/docker/models/swarm.py index 3a02ae3707..e39e6f35ad 100644 --- a/docker/models/swarm.py +++ b/docker/models/swarm.py @@ -34,6 +34,7 @@ def get_unlock_key(self): get_unlock_key.__doc__ = APIClient.get_unlock_key.__doc__ def init(self, advertise_addr=None, listen_addr='0.0.0.0:2377', + default_addr_pool=[], subnet_size=24, force_new_cluster=False, **kwargs): """ Initialize a new swarm on this Engine. @@ -54,6 +55,12 @@ def init(self, advertise_addr=None, listen_addr='0.0.0.0:2377', or an interface followed by a port number, like ``eth0:4567``. If the port number is omitted, the default swarm listening port is used. Default: ``0.0.0.0:2377`` + default_addr_pool (list of str): Default Address Pool specifies + default subnet pools for global scope networks. Each pool + should be specified as a CIDR block, like '10.0.0.0/16'. + Default: [] + subnet_size (int): SubnetSize specifies the subnet size of the + networks created from the default subnet pool. Default: 24 force_new_cluster (bool): Force creating a new Swarm, even if already part of one. Default: False task_history_retention_limit (int): Maximum number of tasks @@ -99,6 +106,7 @@ def init(self, advertise_addr=None, listen_addr='0.0.0.0:2377', >>> client.swarm.init( advertise_addr='eth0', listen_addr='0.0.0.0:5000', + default_addr_pool=['10.20.0.0/16], subnet_size=24, force_new_cluster=False, snapshot_interval=5000, log_entries_for_slow_followers=1200 ) @@ -107,6 +115,8 @@ def init(self, advertise_addr=None, listen_addr='0.0.0.0:2377', init_kwargs = { 'advertise_addr': advertise_addr, 'listen_addr': listen_addr, + 'default_addr_pool': default_addr_pool, + 'subnet_size': subnet_size, 'force_new_cluster': force_new_cluster } init_kwargs['swarm_spec'] = self.client.api.create_swarm_spec(**kwargs) diff --git a/tests/integration/api_swarm_test.py b/tests/integration/api_swarm_test.py index b58dabc639..5ef651d2ad 100644 --- a/tests/integration/api_swarm_test.py +++ b/tests/integration/api_swarm_test.py @@ -35,6 +35,29 @@ def test_init_swarm_force_new_cluster(self): version_2 = self.client.inspect_swarm()['Version']['Index'] assert version_2 != version_1 + @requires_api_version('1.39') + def test_init_swarm_custom_addr_pool(self): + assert self.init_swarm() + results_1 = self.client.inspect_swarm() + assert results_1['DefaultAddrPool'] is None + assert results_1['SubnetSize'] == 24 + + assert self.init_swarm(default_addr_pool=['2.0.0.0/16'], + force_new_cluster=True) + results_2 = self.client.inspect_swarm() + assert set(results_2['DefaultAddrPool']) == ( + {'2.0.0.0/16'} + ) + assert results_2['SubnetSize'] == 24 + + assert self.init_swarm(default_addr_pool=['2.0.0.0/16', '3.0.0.0/16'], + subnet_size=28, force_new_cluster=True) + results_3 = self.client.inspect_swarm() + assert set(results_3['DefaultAddrPool']) == ( + {'2.0.0.0/16', '3.0.0.0/16'} + ) + assert results_3['SubnetSize'] == 28 + @requires_api_version('1.24') def test_init_already_in_cluster(self): assert self.init_swarm() From 781dc30ad425286ede981d639647cae6afd1a2e9 Mon Sep 17 00:00:00 2001 From: Barry Shapira Date: Fri, 14 Dec 2018 07:30:55 +0000 Subject: [PATCH 0862/1301] Check API version before setting swarm addr pool. Also corrected a documentation error: the default API version from constants is currently 1.35, not 1.30 as was sometimes listed. Signed-off-by: Barry Shapira Removed accidental whitespace. Signed-off-by: Barry Shapira --- docker/api/swarm.py | 27 ++++++++++++++++++++++++--- docker/constants.py | 3 +++ docker/models/swarm.py | 6 +++--- tests/integration/api_swarm_test.py | 23 ++++++++++++++--------- 4 files changed, 44 insertions(+), 15 deletions(-) diff --git a/docker/api/swarm.py b/docker/api/swarm.py index bec3efdf23..4a39782aaa 100644 --- a/docker/api/swarm.py +++ b/docker/api/swarm.py @@ -1,5 +1,6 @@ import logging from six.moves import http_client +from ..constants import DEFAULT_SWARM_ADDR_POOL, DEFAULT_SWARM_SUBNET_SIZE from .. import errors from .. import types from .. import utils @@ -82,7 +83,7 @@ def get_unlock_key(self): @utils.minimum_version('1.24') def init_swarm(self, advertise_addr=None, listen_addr='0.0.0.0:2377', - default_addr_pool=[], subnet_size=24, + default_addr_pool=None, subnet_size=None, force_new_cluster=False, swarm_spec=None): """ Initialize a new Swarm using the current connected engine as the first @@ -106,9 +107,9 @@ def init_swarm(self, advertise_addr=None, listen_addr='0.0.0.0:2377', default_addr_pool (list of strings): Default Address Pool specifies default subnet pools for global scope networks. Each pool should be specified as a CIDR block, like '10.0.0.0/16'. - Default: [] + Default: None subnet_size (int): SubnetSize specifies the subnet size of the - networks created from the default subnet pool. Default: 24 + networks created from the default subnet pool. Default: None force_new_cluster (bool): Force creating a new Swarm, even if already part of one. Default: False swarm_spec (dict): Configuration settings of the new Swarm. Use @@ -124,8 +125,28 @@ def init_swarm(self, advertise_addr=None, listen_addr='0.0.0.0:2377', """ url = self._url('/swarm/init') + if swarm_spec is not None and not isinstance(swarm_spec, dict): raise TypeError('swarm_spec must be a dictionary') + + if default_addr_pool is not None: + if utils.version_lt(self._version, '1.39'): + raise errors.InvalidVersion( + 'Address pool is only available for API version >= 1.39' + ) + # subnet_size becomes 0 if not set with default_addr_pool + if subnet_size is None: + subnet_size = DEFAULT_SWARM_SUBNET_SIZE + + if subnet_size is not None: + if utils.version_lt(self._version, '1.39'): + raise errors.InvalidVersion( + 'Subnet size is only available for API version >= 1.39' + ) + # subnet_size is ignored if set without default_addr_pool + if default_addr_pool is None: + default_addr_pool = DEFAULT_SWARM_ADDR_POOL + data = { 'AdvertiseAddr': advertise_addr, 'ListenAddr': listen_addr, diff --git a/docker/constants.py b/docker/constants.py index dcba0de262..4b96e1ce52 100644 --- a/docker/constants.py +++ b/docker/constants.py @@ -25,3 +25,6 @@ DEFAULT_NUM_POOLS_SSH = 9 DEFAULT_DATA_CHUNK_SIZE = 1024 * 2048 + +DEFAULT_SWARM_ADDR_POOL = ['10.0.0.0/8'] +DEFAULT_SWARM_SUBNET_SIZE = 24 diff --git a/docker/models/swarm.py b/docker/models/swarm.py index e39e6f35ad..1106ce26cd 100644 --- a/docker/models/swarm.py +++ b/docker/models/swarm.py @@ -34,7 +34,7 @@ def get_unlock_key(self): get_unlock_key.__doc__ = APIClient.get_unlock_key.__doc__ def init(self, advertise_addr=None, listen_addr='0.0.0.0:2377', - default_addr_pool=[], subnet_size=24, + default_addr_pool=None, subnet_size=None, force_new_cluster=False, **kwargs): """ Initialize a new swarm on this Engine. @@ -58,9 +58,9 @@ def init(self, advertise_addr=None, listen_addr='0.0.0.0:2377', default_addr_pool (list of str): Default Address Pool specifies default subnet pools for global scope networks. Each pool should be specified as a CIDR block, like '10.0.0.0/16'. - Default: [] + Default: None subnet_size (int): SubnetSize specifies the subnet size of the - networks created from the default subnet pool. Default: 24 + networks created from the default subnet pool. Default: None force_new_cluster (bool): Force creating a new Swarm, even if already part of one. Default: False task_history_retention_limit (int): Maximum number of tasks diff --git a/tests/integration/api_swarm_test.py b/tests/integration/api_swarm_test.py index 5ef651d2ad..41fae578be 100644 --- a/tests/integration/api_swarm_test.py +++ b/tests/integration/api_swarm_test.py @@ -37,26 +37,31 @@ def test_init_swarm_force_new_cluster(self): @requires_api_version('1.39') def test_init_swarm_custom_addr_pool(self): + # test defaults assert self.init_swarm() results_1 = self.client.inspect_swarm() - assert results_1['DefaultAddrPool'] is None + assert set(results_1['DefaultAddrPool']) == {'10.0.0.0/8'} assert results_1['SubnetSize'] == 24 - + # test addr pool alone assert self.init_swarm(default_addr_pool=['2.0.0.0/16'], force_new_cluster=True) results_2 = self.client.inspect_swarm() - assert set(results_2['DefaultAddrPool']) == ( - {'2.0.0.0/16'} - ) + assert set(results_2['DefaultAddrPool']) == {'2.0.0.0/16'} assert results_2['SubnetSize'] == 24 - + # test subnet size alone + assert self.init_swarm(subnet_size=26, + force_new_cluster=True) + results_3 = self.client.inspect_swarm() + assert set(results_3['DefaultAddrPool']) == {'10.0.0.0/8'} + assert results_3['SubnetSize'] == 26 + # test both arguments together assert self.init_swarm(default_addr_pool=['2.0.0.0/16', '3.0.0.0/16'], subnet_size=28, force_new_cluster=True) - results_3 = self.client.inspect_swarm() - assert set(results_3['DefaultAddrPool']) == ( + results_4 = self.client.inspect_swarm() + assert set(results_4['DefaultAddrPool']) == ( {'2.0.0.0/16', '3.0.0.0/16'} ) - assert results_3['SubnetSize'] == 28 + assert results_4['SubnetSize'] == 28 @requires_api_version('1.24') def test_init_already_in_cluster(self): From d6cc972cd9955b1aadd373391673314f79e82679 Mon Sep 17 00:00:00 2001 From: Barry Shapira Date: Thu, 3 Jan 2019 17:31:06 -0800 Subject: [PATCH 0863/1301] Split monolithic integration tests into individual tests. The integration tests require restarting the swarm once for each test. I had done so manually with self.init_swarm(force_new_cluster=True) but that wasn't resetting the swarm state correctly. The usual test teardown procedure cleans up correctly. Signed-off-by: Barry Shapira --- tests/integration/api_swarm_test.py | 49 +++++++++++++++-------------- 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/tests/integration/api_swarm_test.py b/tests/integration/api_swarm_test.py index 41fae578be..37f5fa7959 100644 --- a/tests/integration/api_swarm_test.py +++ b/tests/integration/api_swarm_test.py @@ -36,32 +36,33 @@ def test_init_swarm_force_new_cluster(self): assert version_2 != version_1 @requires_api_version('1.39') - def test_init_swarm_custom_addr_pool(self): - # test defaults + def test_init_swarm_custom_addr_pool_defaults(self): assert self.init_swarm() - results_1 = self.client.inspect_swarm() - assert set(results_1['DefaultAddrPool']) == {'10.0.0.0/8'} - assert results_1['SubnetSize'] == 24 - # test addr pool alone - assert self.init_swarm(default_addr_pool=['2.0.0.0/16'], - force_new_cluster=True) - results_2 = self.client.inspect_swarm() - assert set(results_2['DefaultAddrPool']) == {'2.0.0.0/16'} - assert results_2['SubnetSize'] == 24 - # test subnet size alone - assert self.init_swarm(subnet_size=26, - force_new_cluster=True) - results_3 = self.client.inspect_swarm() - assert set(results_3['DefaultAddrPool']) == {'10.0.0.0/8'} - assert results_3['SubnetSize'] == 26 - # test both arguments together + results = self.client.inspect_swarm() + assert set(results['DefaultAddrPool']) == {'10.0.0.0/8'} + assert results['SubnetSize'] == 24 + + @requires_api_version('1.39') + def test_init_swarm_custom_addr_pool_only_pool(self): + assert self.init_swarm(default_addr_pool=['2.0.0.0/16']) + results = self.client.inspect_swarm() + assert set(results['DefaultAddrPool']) == {'2.0.0.0/16'} + assert results['SubnetSize'] == 24 + + @requires_api_version('1.39') + def test_init_swarm_custom_addr_pool_only_subnet_size(self): + assert self.init_swarm(subnet_size=26) + results = self.client.inspect_swarm() + assert set(results['DefaultAddrPool']) == {'10.0.0.0/8'} + assert results['SubnetSize'] == 26 + + @requires_api_version('1.39') + def test_init_swarm_custom_addr_pool_both_args(self): assert self.init_swarm(default_addr_pool=['2.0.0.0/16', '3.0.0.0/16'], - subnet_size=28, force_new_cluster=True) - results_4 = self.client.inspect_swarm() - assert set(results_4['DefaultAddrPool']) == ( - {'2.0.0.0/16', '3.0.0.0/16'} - ) - assert results_4['SubnetSize'] == 28 + subnet_size=28) + results = self.client.inspect_swarm() + assert set(results['DefaultAddrPool']) == {'2.0.0.0/16', '3.0.0.0/16'} + assert results['SubnetSize'] == 28 @requires_api_version('1.24') def test_init_already_in_cluster(self): From 68a271cef4e2afe881d5c4dfe18a97496dc3adb0 Mon Sep 17 00:00:00 2001 From: Hannes Ljungberg Date: Fri, 22 Mar 2019 16:55:10 +0100 Subject: [PATCH 0864/1301] Fix documentation and order of arguments Following https://github.com/docker/docker-py/pull/2201#pullrequestreview-192571911 Signed-off-by: Hannes Ljungberg Co-authored-by: Hannes Ljungberg Co-authored-by: bluikko <14869000+bluikko@users.noreply.github.com> --- docker/api/swarm.py | 17 ++++++++--------- docker/models/swarm.py | 18 +++++++++--------- 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/docker/api/swarm.py b/docker/api/swarm.py index 4a39782aaa..e7db5e29e4 100644 --- a/docker/api/swarm.py +++ b/docker/api/swarm.py @@ -83,8 +83,8 @@ def get_unlock_key(self): @utils.minimum_version('1.24') def init_swarm(self, advertise_addr=None, listen_addr='0.0.0.0:2377', - default_addr_pool=None, subnet_size=None, - force_new_cluster=False, swarm_spec=None): + force_new_cluster=False, swarm_spec=None, + default_addr_pool=None, subnet_size=None): """ Initialize a new Swarm using the current connected engine as the first node. @@ -104,17 +104,17 @@ def init_swarm(self, advertise_addr=None, listen_addr='0.0.0.0:2377', or an interface followed by a port number, like ``eth0:4567``. If the port number is omitted, the default swarm listening port is used. Default: '0.0.0.0:2377' - default_addr_pool (list of strings): Default Address Pool specifies - default subnet pools for global scope networks. Each pool - should be specified as a CIDR block, like '10.0.0.0/16'. - Default: None - subnet_size (int): SubnetSize specifies the subnet size of the - networks created from the default subnet pool. Default: None force_new_cluster (bool): Force creating a new Swarm, even if already part of one. Default: False swarm_spec (dict): Configuration settings of the new Swarm. Use ``APIClient.create_swarm_spec`` to generate a valid configuration. Default: None + default_addr_pool (list of strings): Default Address Pool specifies + default subnet pools for global scope networks. Each pool + should be specified as a CIDR block, like '10.0.0.0/8'. + Default: None + subnet_size (int): SubnetSize specifies the subnet size of the + networks created from the default subnet pool. Default: None Returns: ``True`` if successful. @@ -125,7 +125,6 @@ def init_swarm(self, advertise_addr=None, listen_addr='0.0.0.0:2377', """ url = self._url('/swarm/init') - if swarm_spec is not None and not isinstance(swarm_spec, dict): raise TypeError('swarm_spec must be a dictionary') diff --git a/docker/models/swarm.py b/docker/models/swarm.py index 1106ce26cd..cb27467d32 100644 --- a/docker/models/swarm.py +++ b/docker/models/swarm.py @@ -34,8 +34,8 @@ def get_unlock_key(self): get_unlock_key.__doc__ = APIClient.get_unlock_key.__doc__ def init(self, advertise_addr=None, listen_addr='0.0.0.0:2377', - default_addr_pool=None, subnet_size=None, - force_new_cluster=False, **kwargs): + force_new_cluster=False, default_addr_pool=None, + subnet_size=None, **kwargs): """ Initialize a new swarm on this Engine. @@ -55,14 +55,14 @@ def init(self, advertise_addr=None, listen_addr='0.0.0.0:2377', or an interface followed by a port number, like ``eth0:4567``. If the port number is omitted, the default swarm listening port is used. Default: ``0.0.0.0:2377`` + force_new_cluster (bool): Force creating a new Swarm, even if + already part of one. Default: False default_addr_pool (list of str): Default Address Pool specifies default subnet pools for global scope networks. Each pool - should be specified as a CIDR block, like '10.0.0.0/16'. + should be specified as a CIDR block, like '10.0.0.0/8'. Default: None subnet_size (int): SubnetSize specifies the subnet size of the networks created from the default subnet pool. Default: None - force_new_cluster (bool): Force creating a new Swarm, even if - already part of one. Default: False task_history_retention_limit (int): Maximum number of tasks history stored. snapshot_interval (int): Number of logs entries between snapshot. @@ -106,8 +106,8 @@ def init(self, advertise_addr=None, listen_addr='0.0.0.0:2377', >>> client.swarm.init( advertise_addr='eth0', listen_addr='0.0.0.0:5000', - default_addr_pool=['10.20.0.0/16], subnet_size=24, - force_new_cluster=False, snapshot_interval=5000, + force_new_cluster=False, default_addr_pool=['10.20.0.0/16], + subnet_size=24, snapshot_interval=5000, log_entries_for_slow_followers=1200 ) @@ -115,9 +115,9 @@ def init(self, advertise_addr=None, listen_addr='0.0.0.0:2377', init_kwargs = { 'advertise_addr': advertise_addr, 'listen_addr': listen_addr, + 'force_new_cluster': force_new_cluster, 'default_addr_pool': default_addr_pool, - 'subnet_size': subnet_size, - 'force_new_cluster': force_new_cluster + 'subnet_size': subnet_size } init_kwargs['swarm_spec'] = self.client.api.create_swarm_spec(**kwargs) self.client.api.init_swarm(**init_kwargs) From 523371e21d41e5afdb800aadec123853b8c37f2b Mon Sep 17 00:00:00 2001 From: Hannes Ljungberg Date: Sat, 23 Mar 2019 20:57:14 +0100 Subject: [PATCH 0865/1301] Move volume_driver to RUN_HOST_CONFIG_KWARGS Fixes #2271 Signed-off-by: Hannes Ljungberg --- docker/models/containers.py | 2 +- tests/integration/models_containers_test.py | 10 ++++++++++ tests/unit/models_containers_test.py | 2 +- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/docker/models/containers.py b/docker/models/containers.py index 502251d54f..6659e6cfaa 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -964,7 +964,6 @@ def prune(self, filters=None): 'tty', 'use_config_proxy', 'user', - 'volume_driver', 'working_dir', ] @@ -1028,6 +1027,7 @@ def prune(self, filters=None): 'userns_mode', 'uts_mode', 'version', + 'volume_driver', 'volumes_from', 'runtime' ] diff --git a/tests/integration/models_containers_test.py b/tests/integration/models_containers_test.py index 92eca36d1a..cbc5746154 100644 --- a/tests/integration/models_containers_test.py +++ b/tests/integration/models_containers_test.py @@ -378,3 +378,13 @@ def test_wait(self): detach=True) self.tmp_containers.append(container.id) assert container.wait()['StatusCode'] == 1 + + def test_create_with_volume_driver(self): + client = docker.from_env(version=TEST_API_VERSION) + container = client.containers.create( + 'alpine', + 'sleep 300', + volume_driver='foo' + ) + self.tmp_containers.append(container.id) + assert container.attrs['HostConfig']['VolumeDriver'] == 'foo' diff --git a/tests/unit/models_containers_test.py b/tests/unit/models_containers_test.py index f44e365851..da5f0ab9d9 100644 --- a/tests/unit/models_containers_test.py +++ b/tests/unit/models_containers_test.py @@ -176,6 +176,7 @@ def test_create_container_args(self): 'Ulimits': [{"Name": "nofile", "Soft": 1024, "Hard": 2048}], 'UsernsMode': 'host', 'UTSMode': 'host', + 'VolumeDriver': 'some_driver', 'VolumesFrom': ['container'], }, healthcheck={'test': 'true'}, @@ -190,7 +191,6 @@ def test_create_container_args(self): stop_signal=9, tty=True, user='bob', - volume_driver='some_driver', volumes=[ '/mnt/vol2', '/mnt/vol1', From 8f42dd14841c43aa8081fe67c9af305391e4952b Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Tue, 19 Mar 2019 17:38:24 +0100 Subject: [PATCH 0866/1301] Avoid race condition on short execution - Add a sleep of 2 seconds to be sure the logs can be requested before the daemon removes the container when run with auto_remove=True Signed-off-by: Ulysses Souza --- tests/integration/models_containers_test.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/integration/models_containers_test.py b/tests/integration/models_containers_test.py index 92eca36d1a..872f753e77 100644 --- a/tests/integration/models_containers_test.py +++ b/tests/integration/models_containers_test.py @@ -123,7 +123,9 @@ def test_run_with_json_file_driver(self): def test_run_with_auto_remove(self): client = docker.from_env(version=TEST_API_VERSION) out = client.containers.run( - 'alpine', 'echo hello', auto_remove=True + # sleep(2) to allow any communication with the container + # before it gets removed by the host. + 'alpine', 'sh -c "echo hello && sleep 2"', auto_remove=True ) assert out == b'hello\n' @@ -132,7 +134,10 @@ def test_run_with_auto_remove_error(self): client = docker.from_env(version=TEST_API_VERSION) with pytest.raises(docker.errors.ContainerError) as e: client.containers.run( - 'alpine', 'sh -c ">&2 echo error && exit 1"', auto_remove=True + # sleep(2) to allow any communication with the container + # before it gets removed by the host. + 'alpine', 'sh -c ">&2 echo error && sleep 2 && exit 1"', + auto_remove=True ) assert e.value.exit_status == 1 assert e.value.stderr is None From 15862eacbf863cb7371a6629a9ab951bc05d86a3 Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Tue, 26 Mar 2019 15:15:40 +0100 Subject: [PATCH 0867/1301] Xfail test_attach_stream_and_cancel on TLS This test is quite flaky on ssl integration test Signed-off-by: Ulysses Souza --- tests/integration/api_container_test.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index eb3fd66117..5a8ba5a844 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -1080,7 +1080,6 @@ def test_kill_with_signal_integer(self): class PortTest(BaseAPIIntegrationTest): def test_port(self): - port_bindings = { '1111': ('127.0.0.1', '4567'), '2222': ('127.0.0.1', '4568'), @@ -1268,6 +1267,9 @@ def test_attach_no_stream(self): @pytest.mark.timeout(5) @pytest.mark.skipif(os.environ.get('DOCKER_HOST', '').startswith('ssh://'), reason='No cancellable streams over SSH') + @pytest.mark.xfail(condition=os.environ.get('DOCKER_TLS_VERIFY') or + os.environ.get('DOCKER_CERT_PATH'), + reason='Flaky test on TLS') def test_attach_stream_and_cancel(self): container = self.client.create_container( BUSYBOX, 'sh -c "echo hello && sleep 60"', From b2175c9475b0c3bffd268768136fa30fba8ecf96 Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Tue, 26 Mar 2019 12:07:25 +0100 Subject: [PATCH 0868/1301] Fix base_url to keep TCP protocol This fix lets the responsability of changing the protocol to `parse_host` afterwards, letting `base_url` with the original value. Signed-off-by: Ulysses Souza --- docker/utils/utils.py | 4 +--- tests/unit/utils_test.py | 12 ++++++++---- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/docker/utils/utils.py b/docker/utils/utils.py index 61e307adc7..7819ace4f4 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -352,9 +352,7 @@ def kwargs_from_env(ssl_version=None, assert_hostname=None, environment=None): params = {} if host: - params['base_url'] = ( - host.replace('tcp://', 'https://') if enable_tls else host - ) + params['base_url'] = host if not enable_tls: return params diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index 3cb3be91b2..d9cb002809 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -11,6 +11,7 @@ from docker.api.client import APIClient +from docker.constants import IS_WINDOWS_PLATFORM from docker.errors import DockerException from docker.utils import ( convert_filters, convert_volume_binds, decode_json_header, kwargs_from_env, @@ -83,15 +84,17 @@ def test_kwargs_from_env_tls(self): DOCKER_CERT_PATH=TEST_CERT_DIR, DOCKER_TLS_VERIFY='1') kwargs = kwargs_from_env(assert_hostname=False) - assert 'https://192.168.59.103:2376' == kwargs['base_url'] + assert 'tcp://192.168.59.103:2376' == kwargs['base_url'] assert 'ca.pem' in kwargs['tls'].ca_cert assert 'cert.pem' in kwargs['tls'].cert[0] assert 'key.pem' in kwargs['tls'].cert[1] assert kwargs['tls'].assert_hostname is False assert kwargs['tls'].verify + + parsed_host = parse_host(kwargs['base_url'], IS_WINDOWS_PLATFORM, True) try: client = APIClient(**kwargs) - assert kwargs['base_url'] == client.base_url + assert parsed_host == client.base_url assert kwargs['tls'].ca_cert == client.verify assert kwargs['tls'].cert == client.cert except TypeError as e: @@ -102,15 +105,16 @@ def test_kwargs_from_env_tls_verify_false(self): DOCKER_CERT_PATH=TEST_CERT_DIR, DOCKER_TLS_VERIFY='') kwargs = kwargs_from_env(assert_hostname=True) - assert 'https://192.168.59.103:2376' == kwargs['base_url'] + assert 'tcp://192.168.59.103:2376' == kwargs['base_url'] assert 'ca.pem' in kwargs['tls'].ca_cert assert 'cert.pem' in kwargs['tls'].cert[0] assert 'key.pem' in kwargs['tls'].cert[1] assert kwargs['tls'].assert_hostname is True assert kwargs['tls'].verify is False + parsed_host = parse_host(kwargs['base_url'], IS_WINDOWS_PLATFORM, True) try: client = APIClient(**kwargs) - assert kwargs['base_url'] == client.base_url + assert parsed_host == client.base_url assert kwargs['tls'].cert == client.cert assert not kwargs['tls'].verify except TypeError as e: From 4890864d65a427847cca8d58b9281e3eaab82994 Mon Sep 17 00:00:00 2001 From: Karl Kuehn Date: Tue, 30 Jan 2018 14:28:37 -0800 Subject: [PATCH 0869/1301] add ports to containers Signed-off-by: Karl Kuehn --- docker/models/containers.py | 7 +++++++ tests/integration/models_containers_test.py | 22 +++++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/docker/models/containers.py b/docker/models/containers.py index 6cd33a61b3..11d8f0a3ee 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -62,6 +62,13 @@ def status(self): return self.attrs['State']['Status'] return self.attrs['State'] + @property + def ports(self): + """ + The ports that the container exposes as a dictionary. + """ + return self.attrs.get('NetworkSettings', {}).get('Ports', {}) + def attach(self, **kwargs): """ Attach to this container. diff --git a/tests/integration/models_containers_test.py b/tests/integration/models_containers_test.py index c7d897eb0b..f0c3083be0 100644 --- a/tests/integration/models_containers_test.py +++ b/tests/integration/models_containers_test.py @@ -346,6 +346,28 @@ def test_stats(self): 'memory_stats', 'blkio_stats']: assert key in stats + def test_ports(self): + client = docker.from_env(version=TEST_API_VERSION) + target_ports = {'2222/tcp': None} + container = client.containers.run( + "alpine", "sleep 100", detach=True, + ports=target_ports + ) + self.tmp_containers.append(container.id) + container.reload() # required to get auto-assigned ports + actual_ports = container.ports + assert sorted(target_ports.keys()) == sorted(actual_ports.keys()) + for target_client, target_host in target_ports.items(): + for actual_port in actual_ports[target_client]: + actual_keys = sorted(actual_port.keys()) + assert sorted(['HostIp', 'HostPort']) == actual_keys + if target_host is None: + int(actual_port['HostPort']) + elif isinstance(target_host, (list, tuple)): + raise NotImplementedError() + else: + assert actual_port['HostPort'] == target_host.split('/', 1) + def test_stop(self): client = docker.from_env(version=TEST_API_VERSION) container = client.containers.run("alpine", "top", detach=True) From d1f7979f24fbc2ad0d33fbce6399ff60d791eca2 Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Tue, 26 Mar 2019 17:28:49 +0100 Subject: [PATCH 0870/1301] Refactor and add tests Signed-off-by: Ulysses Souza --- tests/integration/models_containers_test.py | 54 ++++++++++++++++++--- 1 file changed, 46 insertions(+), 8 deletions(-) diff --git a/tests/integration/models_containers_test.py b/tests/integration/models_containers_test.py index f0c3083be0..951a08ae68 100644 --- a/tests/integration/models_containers_test.py +++ b/tests/integration/models_containers_test.py @@ -346,9 +346,10 @@ def test_stats(self): 'memory_stats', 'blkio_stats']: assert key in stats - def test_ports(self): + def test_ports_target_none(self): client = docker.from_env(version=TEST_API_VERSION) - target_ports = {'2222/tcp': None} + ports = None + target_ports = {'2222/tcp': ports} container = client.containers.run( "alpine", "sleep 100", detach=True, ports=target_ports @@ -361,12 +362,49 @@ def test_ports(self): for actual_port in actual_ports[target_client]: actual_keys = sorted(actual_port.keys()) assert sorted(['HostIp', 'HostPort']) == actual_keys - if target_host is None: - int(actual_port['HostPort']) - elif isinstance(target_host, (list, tuple)): - raise NotImplementedError() - else: - assert actual_port['HostPort'] == target_host.split('/', 1) + assert target_host is ports + assert int(actual_port['HostPort']) > 0 + client.close() + + def test_ports_target_tuple(self): + client = docker.from_env(version=TEST_API_VERSION) + ports = ('127.0.0.1', 1111) + target_ports = {'2222/tcp': ports} + container = client.containers.run( + "alpine", "sleep 100", detach=True, + ports=target_ports + ) + self.tmp_containers.append(container.id) + container.reload() # required to get auto-assigned ports + actual_ports = container.ports + assert sorted(target_ports.keys()) == sorted(actual_ports.keys()) + for target_client, target_host in target_ports.items(): + for actual_port in actual_ports[target_client]: + actual_keys = sorted(actual_port.keys()) + assert sorted(['HostIp', 'HostPort']) == actual_keys + assert target_host == ports + assert int(actual_port['HostPort']) > 0 + client.close() + + def test_ports_target_list(self): + client = docker.from_env(version=TEST_API_VERSION) + ports = [1234, 4567] + target_ports = {'2222/tcp': ports} + container = client.containers.run( + "alpine", "sleep 100", detach=True, + ports=target_ports + ) + self.tmp_containers.append(container.id) + container.reload() # required to get auto-assigned ports + actual_ports = container.ports + assert sorted(target_ports.keys()) == sorted(actual_ports.keys()) + for target_client, target_host in target_ports.items(): + for actual_port in actual_ports[target_client]: + actual_keys = sorted(actual_port.keys()) + assert sorted(['HostIp', 'HostPort']) == actual_keys + assert target_host == ports + assert int(actual_port['HostPort']) > 0 + client.close() def test_stop(self): client = docker.from_env(version=TEST_API_VERSION) From 33b8fd6eecae3a5dc2c9476a409cf894354bf994 Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Tue, 26 Mar 2019 15:15:40 +0100 Subject: [PATCH 0871/1301] Xfail test_attach_stream_and_cancel on TLS This test is quite flaky on ssl integration test Signed-off-by: Ulysses Souza --- tests/integration/api_container_test.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index 83df3424a9..558441e0f9 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -1080,7 +1080,6 @@ def test_kill_with_signal_integer(self): class PortTest(BaseAPIIntegrationTest): def test_port(self): - port_bindings = { '1111': ('127.0.0.1', '4567'), '2222': ('127.0.0.1', '4568') @@ -1260,6 +1259,9 @@ def test_attach_no_stream(self): @pytest.mark.timeout(5) @pytest.mark.skipif(os.environ.get('DOCKER_HOST', '').startswith('ssh://'), reason='No cancellable streams over SSH') + @pytest.mark.xfail(condition=os.environ.get('DOCKER_TLS_VERIFY') or + os.environ.get('DOCKER_CERT_PATH'), + reason='Flaky test on TLS') def test_attach_stream_and_cancel(self): container = self.client.create_container( BUSYBOX, 'sh -c "echo hello && sleep 60"', From b05bfd7b22dd23e425cbe9838957e0f3460d2417 Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Tue, 26 Mar 2019 12:07:25 +0100 Subject: [PATCH 0872/1301] Fix base_url to keep TCP protocol This fix lets the responsability of changing the protocol to `parse_host` afterwards, letting `base_url` with the original value. Signed-off-by: Ulysses Souza --- docker/utils/utils.py | 4 +--- tests/unit/utils_test.py | 12 ++++++++---- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/docker/utils/utils.py b/docker/utils/utils.py index 61e307adc7..7819ace4f4 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -352,9 +352,7 @@ def kwargs_from_env(ssl_version=None, assert_hostname=None, environment=None): params = {} if host: - params['base_url'] = ( - host.replace('tcp://', 'https://') if enable_tls else host - ) + params['base_url'] = host if not enable_tls: return params diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index a4e9c9c53e..ee660a3614 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -11,6 +11,7 @@ from docker.api.client import APIClient +from docker.constants import IS_WINDOWS_PLATFORM from docker.errors import DockerException from docker.utils import ( convert_filters, convert_volume_binds, decode_json_header, kwargs_from_env, @@ -83,15 +84,17 @@ def test_kwargs_from_env_tls(self): DOCKER_CERT_PATH=TEST_CERT_DIR, DOCKER_TLS_VERIFY='1') kwargs = kwargs_from_env(assert_hostname=False) - assert 'https://192.168.59.103:2376' == kwargs['base_url'] + assert 'tcp://192.168.59.103:2376' == kwargs['base_url'] assert 'ca.pem' in kwargs['tls'].ca_cert assert 'cert.pem' in kwargs['tls'].cert[0] assert 'key.pem' in kwargs['tls'].cert[1] assert kwargs['tls'].assert_hostname is False assert kwargs['tls'].verify + + parsed_host = parse_host(kwargs['base_url'], IS_WINDOWS_PLATFORM, True) try: client = APIClient(**kwargs) - assert kwargs['base_url'] == client.base_url + assert parsed_host == client.base_url assert kwargs['tls'].ca_cert == client.verify assert kwargs['tls'].cert == client.cert except TypeError as e: @@ -102,15 +105,16 @@ def test_kwargs_from_env_tls_verify_false(self): DOCKER_CERT_PATH=TEST_CERT_DIR, DOCKER_TLS_VERIFY='') kwargs = kwargs_from_env(assert_hostname=True) - assert 'https://192.168.59.103:2376' == kwargs['base_url'] + assert 'tcp://192.168.59.103:2376' == kwargs['base_url'] assert 'ca.pem' in kwargs['tls'].ca_cert assert 'cert.pem' in kwargs['tls'].cert[0] assert 'key.pem' in kwargs['tls'].cert[1] assert kwargs['tls'].assert_hostname is True assert kwargs['tls'].verify is False + parsed_host = parse_host(kwargs['base_url'], IS_WINDOWS_PLATFORM, True) try: client = APIClient(**kwargs) - assert kwargs['base_url'] == client.base_url + assert parsed_host == client.base_url assert kwargs['tls'].cert == client.cert assert not kwargs['tls'].verify except TypeError as e: From b0abdac90c7e6aef444368781bcc3df24f69cab0 Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Thu, 28 Mar 2019 11:42:02 +0000 Subject: [PATCH 0873/1301] scripts/version.py: Use regex grouping to extract the version The `lstrip` and `rstrip` functions take a set of characters to remove, not a prefix/suffix. Thus `rstrip('-x86_64')` will remove any trailing characters in the string `'-x86_64'` in any order (in effect it strips the suffix matching the regex `[-_x468]*`). So with `18.09.4` it removes the `4` suffix resulting in trying to `int('')` later on: Traceback (most recent call last): File "/src/scripts/versions.py", line 80, in main() File "/src/scripts/versions.py", line 73, in main versions, reverse=True, key=operator.attrgetter('order') File "/src/scripts/versions.py", line 52, in order return (int(self.major), int(self.minor), int(self.patch)) + stage ValueError: invalid literal for int() with base 10: '' Since we no longer need to check for the arch suffix (since it no longer appears in the URLs we are traversing) we could just drop the `rstrip` and invent a local prefix stripping helper to replace `lstrip('docker-')`. Instead lets take advantage of the behaviour of `re.findall` which is that if the regex contains a single `()` match that will be returned. This lets us match exactly the sub-section of the regex we require. While editing the regex, also ensure that the suffix is precisely `.tgz` and not merely `tgz` by adding an explicit `\.`, previously the literal `.` would be swallowed by the `.*` instead. Signed-off-by: Ian Campbell --- scripts/versions.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/scripts/versions.py b/scripts/versions.py index 7ad1d56a61..93fe0d7fe8 100644 --- a/scripts/versions.py +++ b/scripts/versions.py @@ -62,13 +62,9 @@ def main(): for url in [base_url.format(cat) for cat in categories]: res = requests.get(url) content = res.text - versions = [ - Version.parse( - v.strip('"').lstrip('docker-').rstrip('.tgz').rstrip('-x86_64') - ) for v in re.findall( - r'"docker-[0-9]+\.[0-9]+\.[0-9]+-?.*tgz"', content - ) - ] + versions = [Version.parse(v) for v in re.findall( + r'"docker-([0-9]+\.[0-9]+\.[0-9]+)-?.*tgz"', content + )] sorted_versions = sorted( versions, reverse=True, key=operator.attrgetter('order') ) From cd59491b9a595a70e9a5b33cd0e09d540c738ee2 Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Thu, 28 Mar 2019 11:42:02 +0000 Subject: [PATCH 0874/1301] scripts/version.py: Use regex grouping to extract the version The `lstrip` and `rstrip` functions take a set of characters to remove, not a prefix/suffix. Thus `rstrip('-x86_64')` will remove any trailing characters in the string `'-x86_64'` in any order (in effect it strips the suffix matching the regex `[-_x468]*`). So with `18.09.4` it removes the `4` suffix resulting in trying to `int('')` later on: Traceback (most recent call last): File "/src/scripts/versions.py", line 80, in main() File "/src/scripts/versions.py", line 73, in main versions, reverse=True, key=operator.attrgetter('order') File "/src/scripts/versions.py", line 52, in order return (int(self.major), int(self.minor), int(self.patch)) + stage ValueError: invalid literal for int() with base 10: '' Since we no longer need to check for the arch suffix (since it no longer appears in the URLs we are traversing) we could just drop the `rstrip` and invent a local prefix stripping helper to replace `lstrip('docker-')`. Instead lets take advantage of the behaviour of `re.findall` which is that if the regex contains a single `()` match that will be returned. This lets us match exactly the sub-section of the regex we require. While editing the regex, also ensure that the suffix is precisely `.tgz` and not merely `tgz` by adding an explicit `\.`, previously the literal `.` would be swallowed by the `.*` instead. Signed-off-by: Ian Campbell --- scripts/versions.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/scripts/versions.py b/scripts/versions.py index 7ad1d56a61..93fe0d7fe8 100644 --- a/scripts/versions.py +++ b/scripts/versions.py @@ -62,13 +62,9 @@ def main(): for url in [base_url.format(cat) for cat in categories]: res = requests.get(url) content = res.text - versions = [ - Version.parse( - v.strip('"').lstrip('docker-').rstrip('.tgz').rstrip('-x86_64') - ) for v in re.findall( - r'"docker-[0-9]+\.[0-9]+\.[0-9]+-?.*tgz"', content - ) - ] + versions = [Version.parse(v) for v in re.findall( + r'"docker-([0-9]+\.[0-9]+\.[0-9]+)-?.*tgz"', content + )] sorted_versions = sorted( versions, reverse=True, key=operator.attrgetter('order') ) From 8f2d9a5687c2358cea364a1d4d64fb1c8d0af7c4 Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Thu, 28 Mar 2019 14:23:19 +0100 Subject: [PATCH 0875/1301] Bump 3.7.2 Signed-off-by: Ulysses Souza --- docker/version.py | 2 +- docs/change-log.md | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index 249475f457..8f81f0d5ef 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "3.7.1" +version = "3.7.2" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) diff --git a/docs/change-log.md b/docs/change-log.md index 9edfee2f87..d7c336112f 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -1,6 +1,17 @@ Change log ========== +3.7.2 +----- + +[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/59?closed=1) + +### Bugfixes + +* Fix base_url to keep TCP protocol on utils.py by letting the responsability of changing the +protocol to `parse_host` afterwards, letting `base_url` with the original value. +* XFAIL test_attach_stream_and_cancel on TLS + 3.7.1 ----- From 0d5aacc464df9765bbc54e9aaaaedb14e51b78f7 Mon Sep 17 00:00:00 2001 From: Hannes Ljungberg Date: Thu, 28 Mar 2019 11:31:28 +0100 Subject: [PATCH 0876/1301] Add support for setting init on services Signed-off-by: Hannes Ljungberg --- docker/api/service.py | 4 ++++ docker/models/services.py | 3 +++ docker/types/services.py | 7 ++++++- tests/integration/api_service_test.py | 14 ++++++++++++++ 4 files changed, 27 insertions(+), 1 deletion(-) diff --git a/docker/api/service.py b/docker/api/service.py index 02f3380e87..372dd10b5c 100644 --- a/docker/api/service.py +++ b/docker/api/service.py @@ -88,6 +88,10 @@ def raise_version_error(param, min_version): if container_spec.get('Isolation') is not None: raise_version_error('ContainerSpec.isolation', '1.35') + if utils.version_lt(version, '1.38'): + if container_spec.get('Init') is not None: + raise_version_error('ContainerSpec.init', '1.38') + if task_template.get('Resources'): if utils.version_lt(version, '1.32'): if task_template['Resources'].get('GenericResources'): diff --git a/docker/models/services.py b/docker/models/services.py index 5d2bd9b3ec..2b6479f2a1 100644 --- a/docker/models/services.py +++ b/docker/models/services.py @@ -165,6 +165,8 @@ def create(self, image, command=None, **kwargs): env (list of str): Environment variables, in the form ``KEY=val``. hostname (string): Hostname to set on the container. + init (boolean): Run an init inside the container that forwards + signals and reaps processes isolation (string): Isolation technology used by the service's containers. Only used for Windows containers. labels (dict): Labels to apply to the service. @@ -280,6 +282,7 @@ def list(self, **kwargs): 'hostname', 'hosts', 'image', + 'init', 'isolation', 'labels', 'mounts', diff --git a/docker/types/services.py b/docker/types/services.py index a0721f607b..5722b0e33d 100644 --- a/docker/types/services.py +++ b/docker/types/services.py @@ -110,13 +110,15 @@ class ContainerSpec(dict): privileges (Privileges): Security options for the service's containers. isolation (string): Isolation technology used by the service's containers. Only used for Windows containers. + init (boolean): Run an init inside the container that forwards signals + and reaps processes. """ def __init__(self, image, command=None, args=None, hostname=None, env=None, workdir=None, user=None, labels=None, mounts=None, stop_grace_period=None, secrets=None, tty=None, groups=None, open_stdin=None, read_only=None, stop_signal=None, healthcheck=None, hosts=None, dns_config=None, configs=None, - privileges=None, isolation=None): + privileges=None, isolation=None, init=None): self['Image'] = image if isinstance(command, six.string_types): @@ -183,6 +185,9 @@ def __init__(self, image, command=None, args=None, hostname=None, env=None, if isolation is not None: self['Isolation'] = isolation + if init is not None: + self['Init'] = init + class Mount(dict): """ diff --git a/tests/integration/api_service_test.py b/tests/integration/api_service_test.py index 57a8d331ce..71e0869e9f 100644 --- a/tests/integration/api_service_test.py +++ b/tests/integration/api_service_test.py @@ -850,6 +850,20 @@ def test_create_service_with_privileges(self): ) assert privileges['SELinuxContext']['Disable'] is True + @requires_api_version('1.38') + def test_create_service_with_init(self): + container_spec = docker.types.ContainerSpec( + 'busybox', ['sleep', '999'], init=True + ) + task_tmpl = docker.types.TaskTemplate(container_spec) + name = self.get_service_name() + svc_id = self.client.create_service(task_tmpl, name=name) + svc_info = self.client.inspect_service(svc_id) + assert 'Init' in svc_info['Spec']['TaskTemplate']['ContainerSpec'] + assert ( + svc_info['Spec']['TaskTemplate']['ContainerSpec']['Init'] is True + ) + @requires_api_version('1.25') def test_update_service_with_defaults_name(self): container_spec = docker.types.ContainerSpec( From 8010d8ba1e0119351a1b5e864ce466882be11bc7 Mon Sep 17 00:00:00 2001 From: Hannes Ljungberg Date: Sun, 31 Mar 2019 23:17:51 +0200 Subject: [PATCH 0877/1301] Document correct listen_addr on join swarm Signed-off-by: Hannes Ljungberg --- docker/api/swarm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/api/swarm.py b/docker/api/swarm.py index e7db5e29e4..bab91ee453 100644 --- a/docker/api/swarm.py +++ b/docker/api/swarm.py @@ -205,7 +205,7 @@ def join_swarm(self, remote_addrs, join_token, listen_addr='0.0.0.0:2377', listen_addr (string): Listen address used for inter-manager communication if the node gets promoted to manager, as well as determining the networking interface used for the VXLAN Tunnel - Endpoint (VTEP). Default: ``None`` + Endpoint (VTEP). Default: ``'0.0.0.0:2377`` advertise_addr (string): Externally reachable address advertised to other nodes. This can either be an address/port combination in the form ``192.168.1.1:4567``, or an interface followed by a From ef043559c4bbd3d1fbc06277160c253fab6df879 Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Thu, 4 Apr 2019 10:29:36 +0200 Subject: [PATCH 0878/1301] Add 'sleep 2' to avoid race condition on attach Signed-off-by: Ulysses Souza --- tests/integration/api_container_test.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index 5a8ba5a844..b364f94c00 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -1267,19 +1267,16 @@ def test_attach_no_stream(self): @pytest.mark.timeout(5) @pytest.mark.skipif(os.environ.get('DOCKER_HOST', '').startswith('ssh://'), reason='No cancellable streams over SSH') - @pytest.mark.xfail(condition=os.environ.get('DOCKER_TLS_VERIFY') or - os.environ.get('DOCKER_CERT_PATH'), - reason='Flaky test on TLS') def test_attach_stream_and_cancel(self): container = self.client.create_container( - BUSYBOX, 'sh -c "echo hello && sleep 60"', + BUSYBOX, 'sh -c "sleep 2 && echo hello && sleep 60"', tty=True ) self.tmp_containers.append(container) self.client.start(container) output = self.client.attach(container, stream=True, logs=True) - threading.Timer(1, output.close).start() + threading.Timer(3, output.close).start() lines = [] for line in output: From 50d475797a3866767289d81a74c1f720662004a5 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Wed, 10 Apr 2019 02:42:23 +0200 Subject: [PATCH 0879/1301] Remove init_path from create This option was never functional, and was not intended to be added to the "container create" API, so let's remove it, because it has been removed in Docker 17.05, and was broken in versions before that; see - https://github.com/moby/moby/issues/32355 --init-path does not seem to work - https://github.com/moby/moby/pull/32470 remove --init-path from client Signed-off-by: Sebastiaan van Stijn --- docker/api/container.py | 1 - tests/integration/api_container_test.py | 13 ------------- 2 files changed, 14 deletions(-) diff --git a/docker/api/container.py b/docker/api/container.py index 6069181720..94f53ff279 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -487,7 +487,6 @@ def create_host_config(self, *args, **kwargs): IDs that the container process will run as. init (bool): Run an init inside the container that forwards signals and reaps processes - init_path (str): Path to the docker-init binary ipc_mode (str): Set the IPC mode for the container. isolation (str): Isolation technology to use. Default: ``None``. links (dict): Mapping of links using the diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index b364f94c00..730c9eebd6 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -448,19 +448,6 @@ def test_create_with_init(self): config = self.client.inspect_container(ctnr) assert config['HostConfig']['Init'] is True - @pytest.mark.xfail(True, reason='init-path removed in 17.05.0') - @requires_api_version('1.25') - def test_create_with_init_path(self): - ctnr = self.client.create_container( - BUSYBOX, 'true', - host_config=self.client.create_host_config( - init_path="/usr/libexec/docker-init" - ) - ) - self.tmp_containers.append(ctnr['Id']) - config = self.client.inspect_container(ctnr) - assert config['HostConfig']['InitPath'] == "/usr/libexec/docker-init" - @requires_api_version('1.24') @pytest.mark.xfail(not os.path.exists('/sys/fs/cgroup/cpu.rt_runtime_us'), reason='CONFIG_RT_GROUP_SCHED isn\'t enabled') From 221d64f427e5578faba604dd03376aa52ed08f6d Mon Sep 17 00:00:00 2001 From: Adam Dangoor Date: Sat, 27 Apr 2019 09:14:00 +0100 Subject: [PATCH 0880/1301] Replace triple backtick in exec_run documentation which caused a rendering error. Signed-off-by: Adam Dangoor --- docker/models/containers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/models/containers.py b/docker/models/containers.py index 11d8f0a3ee..d321a58022 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -179,7 +179,7 @@ def exec_run(self, cmd, stdout=True, stderr=True, stdin=False, tty=False, (ExecResult): A tuple of (exit_code, output) exit_code: (int): Exit code for the executed command or ``None`` if - either ``stream```or ``socket`` is ``True``. + either ``stream`` or ``socket`` is ``True``. output: (generator, bytes, or tuple): If ``stream=True``, a generator yielding response chunks. If ``socket=True``, a socket object for the connection. From 20a5c067243bb8736595e92addb873f828fb4e1b Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 30 Apr 2019 23:16:09 -0700 Subject: [PATCH 0881/1301] Fix versions.py to include release stage Signed-off-by: Joffrey F --- scripts/versions.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) mode change 100644 => 100755 scripts/versions.py diff --git a/scripts/versions.py b/scripts/versions.py old mode 100644 new mode 100755 index 93fe0d7fe8..4bdcb74de2 --- a/scripts/versions.py +++ b/scripts/versions.py @@ -26,8 +26,8 @@ def parse(cls, version): edition = stage stage = None elif '-' in stage: - edition, stage = stage.split('-') - major, minor, patch = version.split('.', 3) + edition, stage = stage.split('-', 1) + major, minor, patch = version.split('.', 2) return cls(major, minor, patch, stage, edition) @property @@ -63,7 +63,7 @@ def main(): res = requests.get(url) content = res.text versions = [Version.parse(v) for v in re.findall( - r'"docker-([0-9]+\.[0-9]+\.[0-9]+)-?.*tgz"', content + r'"docker-([0-9]+\.[0-9]+\.[0-9]+-?.*)\.tgz"', content )] sorted_versions = sorted( versions, reverse=True, key=operator.attrgetter('order') From a823acc2cae10c4635db2fb963cc37d8a23cc0c4 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 9 Jan 2019 16:18:21 -0800 Subject: [PATCH 0882/1301] Make dockerpycreds part of the SDK under docker.credentials Signed-off-by: Joffrey F --- MANIFEST.in | 1 + Makefile | 6 +- docker/auth.py | 8 +- docker/credentials/__init__.py | 4 + docker/credentials/constants.py | 4 + docker/credentials/errors.py | 25 ++++ docker/credentials/store.py | 107 ++++++++++++++++++ docker/credentials/utils.py | 38 +++++++ requirements.txt | 1 - setup.py | 1 - tests/Dockerfile | 28 +++++ tests/gpg-keys/ownertrust | 3 + tests/gpg-keys/secret | Bin 0 -> 966 bytes tests/integration/credentials/__init__.py | 0 .../integration/credentials/create_gpg_key.sh | 12 ++ tests/integration/credentials/store_test.py | 87 ++++++++++++++ tests/integration/credentials/utils_test.py | 22 ++++ tests/unit/auth_test.py | 7 +- 18 files changed, 341 insertions(+), 13 deletions(-) create mode 100644 docker/credentials/__init__.py create mode 100644 docker/credentials/constants.py create mode 100644 docker/credentials/errors.py create mode 100644 docker/credentials/store.py create mode 100644 docker/credentials/utils.py create mode 100644 tests/Dockerfile create mode 100644 tests/gpg-keys/ownertrust create mode 100644 tests/gpg-keys/secret create mode 100644 tests/integration/credentials/__init__.py create mode 100644 tests/integration/credentials/create_gpg_key.sh create mode 100644 tests/integration/credentials/store_test.py create mode 100644 tests/integration/credentials/utils_test.py diff --git a/MANIFEST.in b/MANIFEST.in index 41b3fa9f8b..2ba6e0274c 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -6,3 +6,4 @@ include LICENSE recursive-include tests *.py recursive-include tests/unit/testdata * recursive-include tests/integration/testdata * +recursive-include tests/gpg-keys * diff --git a/Makefile b/Makefile index 434d40e1cc..8cf2b74ded 100644 --- a/Makefile +++ b/Makefile @@ -8,11 +8,11 @@ clean: .PHONY: build build: - docker build -t docker-sdk-python . + docker build -t docker-sdk-python -f tests/Dockerfile --build-arg PYTHON_VERSION=2.7 . .PHONY: build-py3 build-py3: - docker build -t docker-sdk-python3 -f Dockerfile-py3 . + docker build -t docker-sdk-python3 -f tests/Dockerfile . .PHONY: build-docs build-docs: @@ -39,7 +39,7 @@ integration-test: build .PHONY: integration-test-py3 integration-test-py3: build-py3 - docker run -t --rm -v /var/run/docker.sock:/var/run/docker.sock docker-sdk-python3 py.test tests/integration/${file} + docker run -t --rm -v /var/run/docker.sock:/var/run/docker.sock docker-sdk-python3 py.test -v tests/integration/${file} TEST_API_VERSION ?= 1.35 TEST_ENGINE_VERSION ?= 17.12.0-ce diff --git a/docker/auth.py b/docker/auth.py index 638ab9b0a9..5f34ac087d 100644 --- a/docker/auth.py +++ b/docker/auth.py @@ -2,9 +2,9 @@ import json import logging -import dockerpycreds import six +from . import credentials from . import errors from .utils import config @@ -273,17 +273,17 @@ def _resolve_authconfig_credstore(self, registry, credstore_name): 'Password': data['Secret'], }) return res - except dockerpycreds.CredentialsNotFound: + except credentials.CredentialsNotFound: log.debug('No entry found') return None - except dockerpycreds.StoreError as e: + except credentials.StoreError as e: raise errors.DockerException( 'Credentials store error: {0}'.format(repr(e)) ) def _get_store_instance(self, name): if name not in self._stores: - self._stores[name] = dockerpycreds.Store( + self._stores[name] = credentials.Store( name, environment=self._credstore_env ) return self._stores[name] diff --git a/docker/credentials/__init__.py b/docker/credentials/__init__.py new file mode 100644 index 0000000000..31ad28e34d --- /dev/null +++ b/docker/credentials/__init__.py @@ -0,0 +1,4 @@ +# flake8: noqa +from .store import Store +from .errors import StoreError, CredentialsNotFound +from .constants import * diff --git a/docker/credentials/constants.py b/docker/credentials/constants.py new file mode 100644 index 0000000000..6a82d8da42 --- /dev/null +++ b/docker/credentials/constants.py @@ -0,0 +1,4 @@ +PROGRAM_PREFIX = 'docker-credential-' +DEFAULT_LINUX_STORE = 'secretservice' +DEFAULT_OSX_STORE = 'osxkeychain' +DEFAULT_WIN32_STORE = 'wincred' diff --git a/docker/credentials/errors.py b/docker/credentials/errors.py new file mode 100644 index 0000000000..42a1bc1a50 --- /dev/null +++ b/docker/credentials/errors.py @@ -0,0 +1,25 @@ +class StoreError(RuntimeError): + pass + + +class CredentialsNotFound(StoreError): + pass + + +class InitializationError(StoreError): + pass + + +def process_store_error(cpe, program): + message = cpe.output.decode('utf-8') + if 'credentials not found in native keychain' in message: + return CredentialsNotFound( + 'No matching credentials in {}'.format( + program + ) + ) + return StoreError( + 'Credentials store {} exited with "{}".'.format( + program, cpe.output.decode('utf-8').strip() + ) + ) diff --git a/docker/credentials/store.py b/docker/credentials/store.py new file mode 100644 index 0000000000..3f51e4a7eb --- /dev/null +++ b/docker/credentials/store.py @@ -0,0 +1,107 @@ +import json +import os +import subprocess + +import six + +from . import constants +from . import errors +from .utils import create_environment_dict +from .utils import find_executable + + +class Store(object): + def __init__(self, program, environment=None): + """ Create a store object that acts as an interface to + perform the basic operations for storing, retrieving + and erasing credentials using `program`. + """ + self.program = constants.PROGRAM_PREFIX + program + self.exe = find_executable(self.program) + self.environment = environment + if self.exe is None: + raise errors.InitializationError( + '{} not installed or not available in PATH'.format( + self.program + ) + ) + + def get(self, server): + """ Retrieve credentials for `server`. If no credentials are found, + a `StoreError` will be raised. + """ + if not isinstance(server, six.binary_type): + server = server.encode('utf-8') + data = self._execute('get', server) + result = json.loads(data.decode('utf-8')) + + # docker-credential-pass will return an object for inexistent servers + # whereas other helpers will exit with returncode != 0. For + # consistency, if no significant data is returned, + # raise CredentialsNotFound + if result['Username'] == '' and result['Secret'] == '': + raise errors.CredentialsNotFound( + 'No matching credentials in {}'.format(self.program) + ) + + return result + + def store(self, server, username, secret): + """ Store credentials for `server`. Raises a `StoreError` if an error + occurs. + """ + data_input = json.dumps({ + 'ServerURL': server, + 'Username': username, + 'Secret': secret + }).encode('utf-8') + return self._execute('store', data_input) + + def erase(self, server): + """ Erase credentials for `server`. Raises a `StoreError` if an error + occurs. + """ + if not isinstance(server, six.binary_type): + server = server.encode('utf-8') + self._execute('erase', server) + + def list(self): + """ List stored credentials. Requires v0.4.0+ of the helper. + """ + data = self._execute('list', None) + return json.loads(data.decode('utf-8')) + + def _execute(self, subcmd, data_input): + output = None + env = create_environment_dict(self.environment) + try: + if six.PY3: + output = subprocess.check_output( + [self.exe, subcmd], input=data_input, env=env, + ) + else: + process = subprocess.Popen( + [self.exe, subcmd], stdin=subprocess.PIPE, + stdout=subprocess.PIPE, env=env, + ) + output, err = process.communicate(data_input) + if process.returncode != 0: + raise subprocess.CalledProcessError( + returncode=process.returncode, cmd='', output=output + ) + except subprocess.CalledProcessError as e: + raise errors.process_store_error(e, self.program) + except OSError as e: + if e.errno == os.errno.ENOENT: + raise errors.StoreError( + '{} not installed or not available in PATH'.format( + self.program + ) + ) + else: + raise errors.StoreError( + 'Unexpected OS error "{}", errno={}'.format( + e.strerror, e.errno + ) + ) + return output diff --git a/docker/credentials/utils.py b/docker/credentials/utils.py new file mode 100644 index 0000000000..3f720ef1a7 --- /dev/null +++ b/docker/credentials/utils.py @@ -0,0 +1,38 @@ +import distutils.spawn +import os +import sys + + +def find_executable(executable, path=None): + """ + As distutils.spawn.find_executable, but on Windows, look up + every extension declared in PATHEXT instead of just `.exe` + """ + if sys.platform != 'win32': + return distutils.spawn.find_executable(executable, path) + + if path is None: + path = os.environ['PATH'] + + paths = path.split(os.pathsep) + extensions = os.environ.get('PATHEXT', '.exe').split(os.pathsep) + base, ext = os.path.splitext(executable) + + if not os.path.isfile(executable): + for p in paths: + for ext in extensions: + f = os.path.join(p, base + ext) + if os.path.isfile(f): + return f + return None + else: + return executable + + +def create_environment_dict(overrides): + """ + Create and return a copy of os.environ with the specified overrides + """ + result = os.environ.copy() + result.update(overrides or {}) + return result diff --git a/requirements.txt b/requirements.txt index 461bf530e2..eb66c9f592 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,6 @@ asn1crypto==0.22.0 backports.ssl-match-hostname==3.5.0.1 cffi==1.10.0 cryptography==2.3 -docker-pycreds==0.4.0 enum34==1.1.6 idna==2.5 ipaddress==1.0.18 diff --git a/setup.py b/setup.py index 677bc204ee..3e1afcbe99 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,6 @@ requirements = [ 'six >= 1.4.0', 'websocket-client >= 0.32.0', - 'docker-pycreds >= 0.4.0', 'requests >= 2.14.2, != 2.18.0', ] diff --git a/tests/Dockerfile b/tests/Dockerfile new file mode 100644 index 0000000000..042fc7038a --- /dev/null +++ b/tests/Dockerfile @@ -0,0 +1,28 @@ +ARG PYTHON_VERSION=3.6 +FROM python:$PYTHON_VERSION-jessie +RUN apt-get update && apt-get -y install \ + gnupg2 \ + pass \ + curl + +COPY ./tests/gpg-keys /gpg-keys +RUN gpg2 --import gpg-keys/secret +RUN gpg2 --import-ownertrust gpg-keys/ownertrust +RUN yes | pass init $(gpg2 --no-auto-check-trustdb --list-secret-keys | grep ^sec | cut -d/ -f2 | cut -d" " -f1) +RUN gpg2 --check-trustdb +ARG CREDSTORE_VERSION=v0.6.0 +RUN curl -sSL -o /opt/docker-credential-pass.tar.gz \ + https://github.com/docker/docker-credential-helpers/releases/download/$CREDSTORE_VERSION/docker-credential-pass-$CREDSTORE_VERSION-amd64.tar.gz && \ + tar -xf /opt/docker-credential-pass.tar.gz -O > /usr/local/bin/docker-credential-pass && \ + rm -rf /opt/docker-credential-pass.tar.gz && \ + chmod +x /usr/local/bin/docker-credential-pass + +WORKDIR /src +COPY requirements.txt /src/requirements.txt +RUN pip install -r requirements.txt + +COPY test-requirements.txt /src/test-requirements.txt +RUN pip install -r test-requirements.txt + +COPY . /src +RUN pip install . diff --git a/tests/gpg-keys/ownertrust b/tests/gpg-keys/ownertrust new file mode 100644 index 0000000000..141ea57e8d --- /dev/null +++ b/tests/gpg-keys/ownertrust @@ -0,0 +1,3 @@ +# List of assigned trustvalues, created Wed 25 Apr 2018 01:28:17 PM PDT +# (Use "gpg --import-ownertrust" to restore them) +9781B87DAB042E6FD51388A5464ED987A7B21401:6: diff --git a/tests/gpg-keys/secret b/tests/gpg-keys/secret new file mode 100644 index 0000000000000000000000000000000000000000..412294db8492a86a109545e31888b5b230017b4d GIT binary patch literal 966 zcmV;%13CPa0lNfR;MwsJ1OT9MDfJwS(eFF4u%|6O+V;@nXyh}N3JQBjI;zcUW-XT9 zaG7iCNTsErINPbwhKkhqBEL7K%)>|l%|76`yGw3PkhkWv4a3 zE`6J5cQlHWX7-66CA@P)lWU%iSfo}pZkc13MBQb^3>ts8S&yoPUuJd5Mx$jnZAeL7 z0zAcmWp+hXz~f4lPQ{t3ERNjVp!@|-+Tf5|795qpWtFb$+R+36ipYm_E{{YIj=r~P zhDgerPA@%P9tyu^R;9JfI%G&`CN3RPT++*Iok+3wqDJ1MI#nH8bX*ba7E;mD3oKWx z4F)RmD9X8Gek8e@9)D0vFD16xns4w9V?I0;7-YhszQ6_J9jFZstH=sIsFyTOh7(9 zAUtznYjt^HKxbucb8l;TZ!T(ZK8Ruj6A=OcA_W3k;Mws48zTk_2?z%R0tOWb0tpHW z1Qr4V0RkQY0vCV)3JDNKPT7a2vJ?SJ!vLNlQ2c^gnajF&6@3oWB3pxV$5IRcpLjI| zaM)KtsV+Ogrz|M%1rjc@)SUq`1X|$P@el+6*?$94?5D}G;(n*OHUX4mi&Uz(Ekblp zx#Z#D2=83>X=T>HpnuCrx0%>I6aHAJdUBQ5OT?3M@xWi{Qhp7n_^wx*ZKjl?)arA# z!%d7}b^_>O3L)$l(zL3NZ3NBX|2Y=a1RpjR*Z0_r`V_7_DtROw1Yd~QdJwNW-&Y_f z00RXB{0M;!W|?O7&g9-Kp;{9790`zvjA5%QL168tfB>R_(~>4ekzQsQyfA| zp#lMz98D=hpP1>ygj2Rt)a9rmuw9RtI<#`f)c^qc0%I1YGSqC3F1t6W2UVO?<>3=n zob?W;+amTvGA{v(4#|i~1Q-zl00{*GTHx970vikf3JDNKPT7a2vJ?UJWB{P14n<>s o>?00~Fi*{Ffy<{9 literal 0 HcmV?d00001 diff --git a/tests/integration/credentials/__init__.py b/tests/integration/credentials/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/credentials/create_gpg_key.sh b/tests/integration/credentials/create_gpg_key.sh new file mode 100644 index 0000000000..b276c20dc5 --- /dev/null +++ b/tests/integration/credentials/create_gpg_key.sh @@ -0,0 +1,12 @@ +#!/usr/bin/sh +haveged +gpg --batch --gen-key <<-EOF +%echo Generating a standard key +Key-Type: DSA +Key-Length: 1024 +Subkey-Type: ELG-E +Subkey-Length: 1024 +Name-Real: Sakuya Izayoi +Name-Email: sakuya@gensokyo.jp +Expire-Date: 0 +EOF \ No newline at end of file diff --git a/tests/integration/credentials/store_test.py b/tests/integration/credentials/store_test.py new file mode 100644 index 0000000000..dd543e24ad --- /dev/null +++ b/tests/integration/credentials/store_test.py @@ -0,0 +1,87 @@ +import os +import random +import sys + +import pytest +import six +from distutils.spawn import find_executable + +from docker.credentials import ( + CredentialsNotFound, Store, StoreError, DEFAULT_LINUX_STORE, + DEFAULT_OSX_STORE +) + + +class TestStore(object): + def teardown_method(self): + for server in self.tmp_keys: + try: + self.store.erase(server) + except StoreError: + pass + + def setup_method(self): + self.tmp_keys = [] + if sys.platform.startswith('linux'): + if find_executable('docker-credential-' + DEFAULT_LINUX_STORE): + self.store = Store(DEFAULT_LINUX_STORE) + elif find_executable('docker-credential-pass'): + self.store = Store('pass') + else: + raise Exception('No supported docker-credential store in PATH') + elif sys.platform.startswith('darwin'): + self.store = Store(DEFAULT_OSX_STORE) + + def get_random_servername(self): + res = 'pycreds_test_{:x}'.format(random.getrandbits(32)) + self.tmp_keys.append(res) + return res + + def test_store_and_get(self): + key = self.get_random_servername() + self.store.store(server=key, username='user', secret='pass') + data = self.store.get(key) + assert data == { + 'ServerURL': key, + 'Username': 'user', + 'Secret': 'pass' + } + + def test_get_nonexistent(self): + key = self.get_random_servername() + with pytest.raises(CredentialsNotFound): + self.store.get(key) + + def test_store_and_erase(self): + key = self.get_random_servername() + self.store.store(server=key, username='user', secret='pass') + self.store.erase(key) + with pytest.raises(CredentialsNotFound): + self.store.get(key) + + def test_unicode_strings(self): + key = self.get_random_servername() + key = six.u(key) + self.store.store(server=key, username='user', secret='pass') + data = self.store.get(key) + assert data + self.store.erase(key) + with pytest.raises(CredentialsNotFound): + self.store.get(key) + + def test_list(self): + names = (self.get_random_servername(), self.get_random_servername()) + self.store.store(names[0], username='sakuya', secret='izayoi') + self.store.store(names[1], username='reimu', secret='hakurei') + data = self.store.list() + assert names[0] in data + assert data[names[0]] == 'sakuya' + assert names[1] in data + assert data[names[1]] == 'reimu' + + def test_execute_with_env_override(self): + self.store.exe = 'env' + self.store.environment = {'FOO': 'bar'} + data = self.store._execute('--null', '') + assert b'\0FOO=bar\0' in data + assert 'FOO' not in os.environ diff --git a/tests/integration/credentials/utils_test.py b/tests/integration/credentials/utils_test.py new file mode 100644 index 0000000000..ad55f3216b --- /dev/null +++ b/tests/integration/credentials/utils_test.py @@ -0,0 +1,22 @@ +import os + +from docker.credentials.utils import create_environment_dict + +try: + from unittest import mock +except ImportError: + import mock + + +@mock.patch.dict(os.environ) +def test_create_environment_dict(): + base = {'FOO': 'bar', 'BAZ': 'foobar'} + os.environ = base + assert create_environment_dict({'FOO': 'baz'}) == { + 'FOO': 'baz', 'BAZ': 'foobar', + } + assert create_environment_dict({'HELLO': 'world'}) == { + 'FOO': 'bar', 'BAZ': 'foobar', 'HELLO': 'world', + } + + assert os.environ == base diff --git a/tests/unit/auth_test.py b/tests/unit/auth_test.py index dc4d6f59ad..d46da503e3 100644 --- a/tests/unit/auth_test.py +++ b/tests/unit/auth_test.py @@ -9,8 +9,7 @@ import tempfile import unittest -from docker import auth, errors -import dockerpycreds +from docker import auth, credentials, errors import pytest try: @@ -661,7 +660,7 @@ def test_get_all_credentials_3_sources(self): } -class InMemoryStore(dockerpycreds.Store): +class InMemoryStore(credentials.Store): def __init__(self, *args, **kwargs): self.__store = {} @@ -669,7 +668,7 @@ def get(self, server): try: return self.__store[server] except KeyError: - raise dockerpycreds.errors.CredentialsNotFound() + raise credentials.errors.CredentialsNotFound() def store(self, server, username, secret): self.__store[server] = { From 4c45067df9a71181b8cc090ef562f48842ded80b Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 9 Jan 2019 16:27:55 -0800 Subject: [PATCH 0883/1301] New Jenkinsfile build instructions Signed-off-by: Joffrey F --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 8724c10fb3..e618c5dd77 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -24,8 +24,8 @@ def buildImages = { -> imageNamePy2 = "${imageNameBase}:py2-${gitCommit()}" imageNamePy3 = "${imageNameBase}:py3-${gitCommit()}" - buildImage(imageNamePy2, ".", "py2.7") - buildImage(imageNamePy3, "-f Dockerfile-py3 .", "py3.6") + buildImage(imageNamePy2, "-f tests/Dockerfile --build-arg PYTHON_VERSION=2.7 .", "py2.7") + buildImage(imageNamePy3, "-f tests/Dockerfile --build-arg PYTHON_VERSION=3.6 .", "py3.6") } } } From b06e437da89ec391d72de2158f40f2b1a37dbc43 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 30 Apr 2019 23:47:09 -0700 Subject: [PATCH 0884/1301] Avoid demux test flakiness Signed-off-by: Joffrey F --- tests/integration/api_exec_test.py | 48 +++++++++++++----------------- 1 file changed, 21 insertions(+), 27 deletions(-) diff --git a/tests/integration/api_exec_test.py b/tests/integration/api_exec_test.py index e6079eb337..602e69a3b4 100644 --- a/tests/integration/api_exec_test.py +++ b/tests/integration/api_exec_test.py @@ -1,10 +1,10 @@ +from docker.utils.proxy import ProxyConfig from docker.utils.socket import next_frame_header from docker.utils.socket import read_exactly -from docker.utils.proxy import ProxyConfig -from .base import BaseAPIIntegrationTest, BUSYBOX +from .base import BUSYBOX, BaseAPIIntegrationTest from ..helpers import ( - requires_api_version, ctrl_with, assert_cat_socket_detached_with_keys + assert_cat_socket_detached_with_keys, ctrl_with, requires_api_version, ) @@ -125,9 +125,6 @@ def test_exec_command_demux(self): script = ' ; '.join([ # Write something on stdout 'echo hello out', - # Busybox's sleep does not handle sub-second times. - # This loops takes ~0.3 second to execute on my machine. - 'for i in $(seq 1 50000); do echo $i>/dev/null; done', # Write something on stderr 'echo hello err >&2']) cmd = 'sh -c "{}"'.format(script) @@ -135,15 +132,15 @@ def test_exec_command_demux(self): # tty=False, stream=False, demux=False res = self.client.exec_create(id, cmd) exec_log = self.client.exec_start(res) - assert exec_log == b'hello out\nhello err\n' + assert 'hello out\n' in exec_log + assert 'hello err\n' in exec_log # tty=False, stream=True, demux=False res = self.client.exec_create(id, cmd) - exec_log = self.client.exec_start(res, stream=True) - assert next(exec_log) == b'hello out\n' - assert next(exec_log) == b'hello err\n' - with self.assertRaises(StopIteration): - next(exec_log) + exec_log = list(self.client.exec_start(res, stream=True)) + assert len(exec_log) == 2 + assert 'hello out\n' in exec_log + assert 'hello err\n' in exec_log # tty=False, stream=False, demux=True res = self.client.exec_create(id, cmd) @@ -152,11 +149,10 @@ def test_exec_command_demux(self): # tty=False, stream=True, demux=True res = self.client.exec_create(id, cmd) - exec_log = self.client.exec_start(res, demux=True, stream=True) - assert next(exec_log) == (b'hello out\n', None) - assert next(exec_log) == (None, b'hello err\n') - with self.assertRaises(StopIteration): - next(exec_log) + exec_log = list(self.client.exec_start(res, demux=True, stream=True)) + assert len(exec_log) == 2 + assert (b'hello out\n', None) in exec_log + assert (None, b'hello err\n') in exec_log # tty=True, stream=False, demux=False res = self.client.exec_create(id, cmd, tty=True) @@ -165,11 +161,10 @@ def test_exec_command_demux(self): # tty=True, stream=True, demux=False res = self.client.exec_create(id, cmd, tty=True) - exec_log = self.client.exec_start(res, stream=True) - assert next(exec_log) == b'hello out\r\n' - assert next(exec_log) == b'hello err\r\n' - with self.assertRaises(StopIteration): - next(exec_log) + exec_log = list(self.client.exec_start(res, stream=True)) + assert len(exec_log) == 2 + assert 'hello out\r\n' in exec_log + assert 'hello err\r\n' in exec_log # tty=True, stream=False, demux=True res = self.client.exec_create(id, cmd, tty=True) @@ -178,11 +173,10 @@ def test_exec_command_demux(self): # tty=True, stream=True, demux=True res = self.client.exec_create(id, cmd, tty=True) - exec_log = self.client.exec_start(res, demux=True, stream=True) - assert next(exec_log) == (b'hello out\r\n', None) - assert next(exec_log) == (b'hello err\r\n', None) - with self.assertRaises(StopIteration): - next(exec_log) + exec_log = list(self.client.exec_start(res, demux=True, stream=True)) + assert len(exec_log) == 2 + assert (b'hello out\r\n', None) in exec_log + assert (b'hello err\r\n', None) in exec_log def test_exec_start_socket(self): container = self.client.create_container(BUSYBOX, 'cat', From 073a21c28a11c980ec43018b12677fbcecbdc90d Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 1 May 2019 00:20:40 -0700 Subject: [PATCH 0885/1301] Separate into individual tests Signed-off-by: Joffrey F --- tests/integration/api_exec_test.py | 140 ++++++++++++++++------------- 1 file changed, 77 insertions(+), 63 deletions(-) diff --git a/tests/integration/api_exec_test.py b/tests/integration/api_exec_test.py index 602e69a3b4..b9310d6518 100644 --- a/tests/integration/api_exec_test.py +++ b/tests/integration/api_exec_test.py @@ -115,69 +115,6 @@ def test_exec_command_streaming(self): res += chunk assert res == b'hello\nworld\n' - def test_exec_command_demux(self): - container = self.client.create_container( - BUSYBOX, 'cat', detach=True, stdin_open=True) - id = container['Id'] - self.client.start(id) - self.tmp_containers.append(id) - - script = ' ; '.join([ - # Write something on stdout - 'echo hello out', - # Write something on stderr - 'echo hello err >&2']) - cmd = 'sh -c "{}"'.format(script) - - # tty=False, stream=False, demux=False - res = self.client.exec_create(id, cmd) - exec_log = self.client.exec_start(res) - assert 'hello out\n' in exec_log - assert 'hello err\n' in exec_log - - # tty=False, stream=True, demux=False - res = self.client.exec_create(id, cmd) - exec_log = list(self.client.exec_start(res, stream=True)) - assert len(exec_log) == 2 - assert 'hello out\n' in exec_log - assert 'hello err\n' in exec_log - - # tty=False, stream=False, demux=True - res = self.client.exec_create(id, cmd) - exec_log = self.client.exec_start(res, demux=True) - assert exec_log == (b'hello out\n', b'hello err\n') - - # tty=False, stream=True, demux=True - res = self.client.exec_create(id, cmd) - exec_log = list(self.client.exec_start(res, demux=True, stream=True)) - assert len(exec_log) == 2 - assert (b'hello out\n', None) in exec_log - assert (None, b'hello err\n') in exec_log - - # tty=True, stream=False, demux=False - res = self.client.exec_create(id, cmd, tty=True) - exec_log = self.client.exec_start(res) - assert exec_log == b'hello out\r\nhello err\r\n' - - # tty=True, stream=True, demux=False - res = self.client.exec_create(id, cmd, tty=True) - exec_log = list(self.client.exec_start(res, stream=True)) - assert len(exec_log) == 2 - assert 'hello out\r\n' in exec_log - assert 'hello err\r\n' in exec_log - - # tty=True, stream=False, demux=True - res = self.client.exec_create(id, cmd, tty=True) - exec_log = self.client.exec_start(res, demux=True) - assert exec_log == (b'hello out\r\nhello err\r\n', None) - - # tty=True, stream=True, demux=True - res = self.client.exec_create(id, cmd, tty=True) - exec_log = list(self.client.exec_start(res, demux=True, stream=True)) - assert len(exec_log) == 2 - assert (b'hello out\r\n', None) in exec_log - assert (b'hello err\r\n', None) in exec_log - def test_exec_start_socket(self): container = self.client.create_container(BUSYBOX, 'cat', detach=True, stdin_open=True) @@ -307,3 +244,80 @@ def test_detach_with_arg(self): self.addCleanup(sock.close) assert_cat_socket_detached_with_keys(sock, [ctrl_with('x')]) + + +class ExecDemuxTest(BaseAPIIntegrationTest): + cmd = 'sh -c "{}"'.format(' ; '.join([ + # Write something on stdout + 'echo hello out', + # Busybox's sleep does not handle sub-second times. + # This loops takes ~0.3 second to execute on my machine. + 'for i in $(seq 1 50000); do echo $i>/dev/null; done', + # Write something on stderr + 'echo hello err >&2']) + ) + + def setUp(self): + super(ExecDemuxTest, self).setUp() + self.container = self.client.create_container( + BUSYBOX, 'cat', detach=True, stdin_open=True + ) + self.client.start(self.container) + self.tmp_containers.append(self.container) + + def test_exec_command_no_stream_no_demux(self): + # tty=False, stream=False, demux=False + res = self.client.exec_create(self.container, self.cmd) + exec_log = self.client.exec_start(res) + assert b'hello out\n' in exec_log + assert b'hello err\n' in exec_log + + def test_exec_command_stream_no_demux(self): + # tty=False, stream=True, demux=False + res = self.client.exec_create(self.container, self.cmd) + exec_log = list(self.client.exec_start(res, stream=True)) + assert len(exec_log) == 2 + assert b'hello out\n' in exec_log + assert b'hello err\n' in exec_log + + def test_exec_command_no_stream_demux(self): + # tty=False, stream=False, demux=True + res = self.client.exec_create(self.container, self.cmd) + exec_log = self.client.exec_start(res, demux=True) + assert exec_log == (b'hello out\n', b'hello err\n') + + def test_exec_command_stream_demux(self): + # tty=False, stream=True, demux=True + res = self.client.exec_create(self.container, self.cmd) + exec_log = list(self.client.exec_start(res, demux=True, stream=True)) + assert len(exec_log) == 2 + assert (b'hello out\n', None) in exec_log + assert (None, b'hello err\n') in exec_log + + def test_exec_command_tty_no_stream_no_demux(self): + # tty=True, stream=False, demux=False + res = self.client.exec_create(self.container, self.cmd, tty=True) + exec_log = self.client.exec_start(res) + assert exec_log == b'hello out\r\nhello err\r\n' + + def test_exec_command_tty_stream_no_demux(self): + # tty=True, stream=True, demux=False + res = self.client.exec_create(self.container, self.cmd, tty=True) + exec_log = list(self.client.exec_start(res, stream=True)) + assert len(exec_log) == 2 + assert b'hello out\r\n' in exec_log + assert b'hello err\r\n' in exec_log + + def test_exec_command_tty_no_stream_demux(self): + # tty=True, stream=False, demux=True + res = self.client.exec_create(self.container, self.cmd, tty=True) + exec_log = self.client.exec_start(res, demux=True) + assert exec_log == (b'hello out\r\nhello err\r\n', None) + + def test_exec_command_tty_stream_demux(self): + # tty=True, stream=True, demux=True + res = self.client.exec_create(self.container, self.cmd, tty=True) + exec_log = list(self.client.exec_start(res, demux=True, stream=True)) + assert len(exec_log) == 2 + assert (b'hello out\r\n', None) in exec_log + assert (b'hello err\r\n', None) in exec_log From 2e67cd1cc7ec4b00afadb9609bb235e3a2f3a0e3 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 22 Apr 2019 18:01:11 -0700 Subject: [PATCH 0886/1301] Improve socket_detached test helper to support future versions of the daemon Signed-off-by: Joffrey F --- tests/helpers.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tests/helpers.py b/tests/helpers.py index f912bd8d43..9e5d2ab4ed 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -119,13 +119,18 @@ def assert_cat_socket_detached_with_keys(sock, inputs): # If we're using a Unix socket, the sock.send call will fail with a # BrokenPipeError ; INET sockets will just stop receiving / sending data # but will not raise an error - if getattr(sock, 'family', -9) == getattr(socket, 'AF_UNIX', -1): - with pytest.raises(socket.error): - sock.sendall(b'make sure the socket is closed\n') - elif isinstance(sock, paramiko.Channel): + if isinstance(sock, paramiko.Channel): with pytest.raises(OSError): sock.sendall(b'make sure the socket is closed\n') else: + if getattr(sock, 'family', -9) == getattr(socket, 'AF_UNIX', -1): + # We do not want to use pytest.raises here because future versions + # of the daemon no longer cause this to raise an error. + try: + sock.sendall(b'make sure the socket is closed\n') + except socket.error: + return + sock.sendall(b"make sure the socket is closed\n") data = sock.recv(128) # New in 18.06: error message is broadcast over the socket when reading From a2a2d100e841b7bb37b9c3d805ca25260f0b3bda Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 22 Apr 2019 18:03:19 -0700 Subject: [PATCH 0887/1301] Reorder imports Signed-off-by: Joffrey F --- tests/helpers.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/helpers.py b/tests/helpers.py index 9e5d2ab4ed..f344e1c333 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -2,16 +2,16 @@ import os import os.path import random +import re +import socket import tarfile import tempfile import time -import re -import six -import socket import docker import paramiko import pytest +import six def make_tree(dirs, files): From 62c8bcbbb600cbe26e3e12ed95207ffe63c40fc8 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 1 May 2019 00:47:16 -0700 Subject: [PATCH 0888/1301] Increase timeout on test with long sleeps Signed-off-by: Joffrey F --- tests/integration/api_container_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index 730c9eebd6..9b770c715a 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -1251,7 +1251,7 @@ def test_attach_no_stream(self): output = self.client.attach(container, stream=False, logs=True) assert output == 'hello\n'.encode(encoding='ascii') - @pytest.mark.timeout(5) + @pytest.mark.timeout(10) @pytest.mark.skipif(os.environ.get('DOCKER_HOST', '').startswith('ssh://'), reason='No cancellable streams over SSH') def test_attach_stream_and_cancel(self): From 34ffc5686546343eaa27d1fb8f9432237bdd1886 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 1 May 2019 00:59:35 -0700 Subject: [PATCH 0889/1301] Streaming TTY messages sometimes get truncated. Handle gracefully in demux tests Signed-off-by: Joffrey F --- tests/integration/api_exec_test.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/tests/integration/api_exec_test.py b/tests/integration/api_exec_test.py index b9310d6518..c7e7799b92 100644 --- a/tests/integration/api_exec_test.py +++ b/tests/integration/api_exec_test.py @@ -304,9 +304,13 @@ def test_exec_command_tty_stream_no_demux(self): # tty=True, stream=True, demux=False res = self.client.exec_create(self.container, self.cmd, tty=True) exec_log = list(self.client.exec_start(res, stream=True)) - assert len(exec_log) == 2 assert b'hello out\r\n' in exec_log - assert b'hello err\r\n' in exec_log + if len(exec_log) == 2: + assert b'hello err\r\n' in exec_log + else: + assert len(exec_log) == 3 + assert b'hello err' in exec_log + assert b'\r\n' in exec_log def test_exec_command_tty_no_stream_demux(self): # tty=True, stream=False, demux=True @@ -318,6 +322,10 @@ def test_exec_command_tty_stream_demux(self): # tty=True, stream=True, demux=True res = self.client.exec_create(self.container, self.cmd, tty=True) exec_log = list(self.client.exec_start(res, demux=True, stream=True)) - assert len(exec_log) == 2 assert (b'hello out\r\n', None) in exec_log - assert (b'hello err\r\n', None) in exec_log + if len(exec_log) == 2: + assert (b'hello err\r\n', None) in exec_log + else: + assert len(exec_log) == 3 + assert (b'hello err', None) in exec_log + assert (b'\r\n', None) in exec_log From 1a4881acd9d8b84135ce9d71ff01325308d8a6b0 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 1 May 2019 01:15:41 -0700 Subject: [PATCH 0890/1301] Improve low_timeout test resilience Signed-off-by: Joffrey F --- tests/integration/api_container_test.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index 9b770c715a..df405ef94a 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -18,7 +18,7 @@ from .base import BUSYBOX, BaseAPIIntegrationTest from .. import helpers from ..helpers import ( - requires_api_version, ctrl_with, assert_cat_socket_detached_with_keys + assert_cat_socket_detached_with_keys, ctrl_with, requires_api_version, ) @@ -1163,10 +1163,10 @@ def test_restart(self): def test_restart_with_low_timeout(self): container = self.client.create_container(BUSYBOX, ['sleep', '9999']) self.client.start(container) - self.client.timeout = 1 - self.client.restart(container, timeout=3) + self.client.timeout = 3 + self.client.restart(container, timeout=1) self.client.timeout = None - self.client.restart(container, timeout=3) + self.client.restart(container, timeout=1) self.client.kill(container) def test_restart_with_dict_instead_of_id(self): From acd7a8f43056007d8ae5df3d8156c34b837d97b1 Mon Sep 17 00:00:00 2001 From: Hannes Ljungberg Date: Thu, 28 Mar 2019 10:48:22 +0100 Subject: [PATCH 0891/1301] Return node id on swarm init Signed-off-by: Hannes Ljungberg --- docker/api/swarm.py | 5 ++--- docker/models/swarm.py | 6 +++--- tests/integration/api_swarm_test.py | 4 +++- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/docker/api/swarm.py b/docker/api/swarm.py index bab91ee453..ea4c1e7128 100644 --- a/docker/api/swarm.py +++ b/docker/api/swarm.py @@ -117,7 +117,7 @@ def init_swarm(self, advertise_addr=None, listen_addr='0.0.0.0:2377', networks created from the default subnet pool. Default: None Returns: - ``True`` if successful. + (str): The ID of the created node. Raises: :py:class:`docker.errors.APIError` @@ -155,8 +155,7 @@ def init_swarm(self, advertise_addr=None, listen_addr='0.0.0.0:2377', 'Spec': swarm_spec, } response = self._post_json(url, data=data) - self._raise_for_status(response) - return True + return self._result(response, json=True) @utils.minimum_version('1.24') def inspect_swarm(self): diff --git a/docker/models/swarm.py b/docker/models/swarm.py index cb27467d32..f78e8e1672 100644 --- a/docker/models/swarm.py +++ b/docker/models/swarm.py @@ -96,7 +96,7 @@ def init(self, advertise_addr=None, listen_addr='0.0.0.0:2377', created in the orchestrator. Returns: - ``True`` if the request went through. + (str): The ID of the created node. Raises: :py:class:`docker.errors.APIError` @@ -120,9 +120,9 @@ def init(self, advertise_addr=None, listen_addr='0.0.0.0:2377', 'subnet_size': subnet_size } init_kwargs['swarm_spec'] = self.client.api.create_swarm_spec(**kwargs) - self.client.api.init_swarm(**init_kwargs) + node_id = self.client.api.init_swarm(**init_kwargs) self.reload() - return True + return node_id def join(self, *args, **kwargs): return self.client.api.join_swarm(*args, **kwargs) diff --git a/tests/integration/api_swarm_test.py b/tests/integration/api_swarm_test.py index 37f5fa7959..94ab2a63c8 100644 --- a/tests/integration/api_swarm_test.py +++ b/tests/integration/api_swarm_test.py @@ -186,12 +186,14 @@ def test_list_nodes(self): @requires_api_version('1.24') def test_inspect_node(self): - assert self.init_swarm() + node_id = self.init_swarm() + assert node_id nodes_list = self.client.nodes() assert len(nodes_list) == 1 node = nodes_list[0] node_data = self.client.inspect_node(node['ID']) assert node['ID'] == node_data['ID'] + assert node_id == node['ID'] assert node['Version'] == node_data['Version'] @requires_api_version('1.24') From c7b9cae0a0430ec9e9ce95bb872755dfe61d4f87 Mon Sep 17 00:00:00 2001 From: Hannes Ljungberg Date: Sun, 31 Mar 2019 23:10:09 +0200 Subject: [PATCH 0892/1301] Add swarm support for data_addr_path Signed-off-by: Hannes Ljungberg --- docker/api/swarm.py | 35 ++++++++++++++++++++++++----- docker/models/swarm.py | 7 ++++-- tests/integration/api_swarm_test.py | 4 ++++ 3 files changed, 38 insertions(+), 8 deletions(-) diff --git a/docker/api/swarm.py b/docker/api/swarm.py index bab91ee453..0bd6d12816 100644 --- a/docker/api/swarm.py +++ b/docker/api/swarm.py @@ -84,7 +84,8 @@ def get_unlock_key(self): @utils.minimum_version('1.24') def init_swarm(self, advertise_addr=None, listen_addr='0.0.0.0:2377', force_new_cluster=False, swarm_spec=None, - default_addr_pool=None, subnet_size=None): + default_addr_pool=None, subnet_size=None, + data_path_addr=None): """ Initialize a new Swarm using the current connected engine as the first node. @@ -115,6 +116,8 @@ def init_swarm(self, advertise_addr=None, listen_addr='0.0.0.0:2377', Default: None subnet_size (int): SubnetSize specifies the subnet size of the networks created from the default subnet pool. Default: None + data_path_addr (string): Address or interface to use for data path + traffic. For example, 192.168.1.1, or an interface, like eth0. Returns: ``True`` if successful. @@ -154,6 +157,15 @@ def init_swarm(self, advertise_addr=None, listen_addr='0.0.0.0:2377', 'ForceNewCluster': force_new_cluster, 'Spec': swarm_spec, } + + if data_path_addr is not None: + if utils.version_lt(self._version, '1.30'): + raise errors.InvalidVersion( + 'Data address path is only available for ' + 'API version >= 1.30' + ) + data['DataPathAddr'] = data_path_addr + response = self._post_json(url, data=data) self._raise_for_status(response) return True @@ -194,7 +206,7 @@ def inspect_node(self, node_id): @utils.minimum_version('1.24') def join_swarm(self, remote_addrs, join_token, listen_addr='0.0.0.0:2377', - advertise_addr=None): + advertise_addr=None, data_path_addr=None): """ Make this Engine join a swarm that has already been created. @@ -213,6 +225,8 @@ def join_swarm(self, remote_addrs, join_token, listen_addr='0.0.0.0:2377', the port number from the listen address is used. If AdvertiseAddr is not specified, it will be automatically detected when possible. Default: ``None`` + data_path_addr (string): Address or interface to use for data path + traffic. For example, 192.168.1.1, or an interface, like eth0. Returns: ``True`` if the request went through. @@ -222,11 +236,20 @@ def join_swarm(self, remote_addrs, join_token, listen_addr='0.0.0.0:2377', If the server returns an error. """ data = { - "RemoteAddrs": remote_addrs, - "ListenAddr": listen_addr, - "JoinToken": join_token, - "AdvertiseAddr": advertise_addr, + 'RemoteAddrs': remote_addrs, + 'ListenAddr': listen_addr, + 'JoinToken': join_token, + 'AdvertiseAddr': advertise_addr, } + + if data_path_addr is not None: + if utils.version_lt(self._version, '1.30'): + raise errors.InvalidVersion( + 'Data address path is only available for ' + 'API version >= 1.30' + ) + data['DataPathAddr'] = data_path_addr + url = self._url('/swarm/join') response = self._post_json(url, data=data) self._raise_for_status(response) diff --git a/docker/models/swarm.py b/docker/models/swarm.py index cb27467d32..386d23d3fe 100644 --- a/docker/models/swarm.py +++ b/docker/models/swarm.py @@ -35,7 +35,7 @@ def get_unlock_key(self): def init(self, advertise_addr=None, listen_addr='0.0.0.0:2377', force_new_cluster=False, default_addr_pool=None, - subnet_size=None, **kwargs): + subnet_size=None, data_path_addr=None, **kwargs): """ Initialize a new swarm on this Engine. @@ -63,6 +63,8 @@ def init(self, advertise_addr=None, listen_addr='0.0.0.0:2377', Default: None subnet_size (int): SubnetSize specifies the subnet size of the networks created from the default subnet pool. Default: None + data_path_addr (string): Address or interface to use for data path + traffic. For example, 192.168.1.1, or an interface, like eth0. task_history_retention_limit (int): Maximum number of tasks history stored. snapshot_interval (int): Number of logs entries between snapshot. @@ -117,7 +119,8 @@ def init(self, advertise_addr=None, listen_addr='0.0.0.0:2377', 'listen_addr': listen_addr, 'force_new_cluster': force_new_cluster, 'default_addr_pool': default_addr_pool, - 'subnet_size': subnet_size + 'subnet_size': subnet_size, + 'data_path_addr': data_path_addr, } init_kwargs['swarm_spec'] = self.client.api.create_swarm_spec(**kwargs) self.client.api.init_swarm(**init_kwargs) diff --git a/tests/integration/api_swarm_test.py b/tests/integration/api_swarm_test.py index 37f5fa7959..5e9aea1e1f 100644 --- a/tests/integration/api_swarm_test.py +++ b/tests/integration/api_swarm_test.py @@ -233,3 +233,7 @@ def test_remove_main_node(self): self.client.remove_node(node_id, True) assert e.value.response.status_code >= 400 + + @requires_api_version('1.30') + def test_init_swarm_data_path_addr(self): + assert self.init_swarm(data_path_addr='eth0') From 110c6769c93cd9c8bf20cc88c520a9f97afc040e Mon Sep 17 00:00:00 2001 From: Hannes Ljungberg Date: Sun, 31 Mar 2019 23:10:23 +0200 Subject: [PATCH 0893/1301] Add test for join on already joined swarm Signed-off-by: Hannes Ljungberg --- tests/integration/models_swarm_test.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/integration/models_swarm_test.py b/tests/integration/models_swarm_test.py index f39f0d34cf..6c1836dc60 100644 --- a/tests/integration/models_swarm_test.py +++ b/tests/integration/models_swarm_test.py @@ -31,3 +31,15 @@ def test_init_update_leave(self): cm.value.response.status_code == 406 or cm.value.response.status_code == 503 ) + + def test_join_on_already_joined_swarm(self): + client = docker.from_env(version=TEST_API_VERSION) + client.swarm.init() + join_token = client.swarm.attrs['JoinTokens']['Manager'] + with pytest.raises(docker.errors.APIError) as cm: + client.swarm.join( + remote_addrs=['127.0.0.1'], + join_token=join_token, + ) + assert cm.value.response.status_code == 503 + assert 'This node is already part of a swarm.' in cm.value.explanation From eba8345c3726ea0ea40436507264229bf4ab56d0 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 1 May 2019 02:16:42 -0700 Subject: [PATCH 0894/1301] Update some test dependencies / default values with newer versions Signed-off-by: Joffrey F --- Makefile | 2 +- tests/integration/api_container_test.py | 28 ++++++++++++------------- tests/integration/api_exec_test.py | 20 +++++++++--------- tests/integration/base.py | 5 ++--- 4 files changed, 26 insertions(+), 29 deletions(-) diff --git a/Makefile b/Makefile index 434d40e1cc..d64e618e8b 100644 --- a/Makefile +++ b/Makefile @@ -42,7 +42,7 @@ integration-test-py3: build-py3 docker run -t --rm -v /var/run/docker.sock:/var/run/docker.sock docker-sdk-python3 py.test tests/integration/${file} TEST_API_VERSION ?= 1.35 -TEST_ENGINE_VERSION ?= 17.12.0-ce +TEST_ENGINE_VERSION ?= 18.09.5 .PHONY: setup-network setup-network: diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index df405ef94a..1190d91e8f 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -5,21 +5,20 @@ import threading from datetime import datetime -import docker -from docker.constants import IS_WINDOWS_PLATFORM -from docker.utils.socket import next_frame_header -from docker.utils.socket import read_exactly - import pytest - import requests import six -from .base import BUSYBOX, BaseAPIIntegrationTest +import docker from .. import helpers -from ..helpers import ( - assert_cat_socket_detached_with_keys, ctrl_with, requires_api_version, -) +from ..helpers import assert_cat_socket_detached_with_keys +from ..helpers import ctrl_with +from ..helpers import requires_api_version +from .base import BaseAPIIntegrationTest +from .base import BUSYBOX +from docker.constants import IS_WINDOWS_PLATFORM +from docker.utils.socket import next_frame_header +from docker.utils.socket import read_exactly class ListContainersTest(BaseAPIIntegrationTest): @@ -38,7 +37,7 @@ def test_list_containers(self): assert 'Command' in retrieved assert retrieved['Command'] == six.text_type('true') assert 'Image' in retrieved - assert re.search(r'busybox:.*', retrieved['Image']) + assert re.search(r'alpine:.*', retrieved['Image']) assert 'Status' in retrieved @@ -368,10 +367,9 @@ def test_create_with_environment_variable_no_value(self): ) self.tmp_containers.append(container['Id']) config = self.client.inspect_container(container['Id']) - assert ( - sorted(config['Config']['Env']) == - sorted(['Foo', 'Other=one', 'Blank=']) - ) + assert 'Foo' in config['Config']['Env'] + assert 'Other=one' in config['Config']['Env'] + assert 'Blank=' in config['Config']['Env'] @requires_api_version('1.22') def test_create_with_tmpfs(self): diff --git a/tests/integration/api_exec_test.py b/tests/integration/api_exec_test.py index c7e7799b92..80b63ffc1c 100644 --- a/tests/integration/api_exec_test.py +++ b/tests/integration/api_exec_test.py @@ -1,12 +1,12 @@ +from ..helpers import assert_cat_socket_detached_with_keys +from ..helpers import ctrl_with +from ..helpers import requires_api_version +from .base import BaseAPIIntegrationTest +from .base import BUSYBOX from docker.utils.proxy import ProxyConfig from docker.utils.socket import next_frame_header from docker.utils.socket import read_exactly -from .base import BUSYBOX, BaseAPIIntegrationTest -from ..helpers import ( - assert_cat_socket_detached_with_keys, ctrl_with, requires_api_version, -) - class ExecTest(BaseAPIIntegrationTest): def test_execute_command_with_proxy_env(self): @@ -81,11 +81,11 @@ def test_exec_command_as_user(self): self.client.start(id) self.tmp_containers.append(id) - res = self.client.exec_create(id, 'whoami', user='default') + res = self.client.exec_create(id, 'whoami', user='postgres') assert 'Id' in res exec_log = self.client.exec_start(res) - assert exec_log == b'default\n' + assert exec_log == b'postgres\n' def test_exec_command_as_root(self): container = self.client.create_container(BUSYBOX, 'cat', @@ -188,9 +188,9 @@ def test_exec_command_with_workdir(self): self.tmp_containers.append(container) self.client.start(container) - res = self.client.exec_create(container, 'pwd', workdir='/var/www') + res = self.client.exec_create(container, 'pwd', workdir='/var/opt') exec_log = self.client.exec_start(res) - assert exec_log == b'/var/www\n' + assert exec_log == b'/var/opt\n' def test_detach_with_default(self): container = self.client.create_container( @@ -252,7 +252,7 @@ class ExecDemuxTest(BaseAPIIntegrationTest): 'echo hello out', # Busybox's sleep does not handle sub-second times. # This loops takes ~0.3 second to execute on my machine. - 'for i in $(seq 1 50000); do echo $i>/dev/null; done', + 'sleep 0.5', # Write something on stderr 'echo hello err >&2']) ) diff --git a/tests/integration/base.py b/tests/integration/base.py index 262769de40..0ebf5b9911 100644 --- a/tests/integration/base.py +++ b/tests/integration/base.py @@ -3,11 +3,10 @@ import unittest import docker -from docker.utils import kwargs_from_env - from .. import helpers +from docker.utils import kwargs_from_env -BUSYBOX = 'busybox:buildroot-2014.02' +BUSYBOX = 'alpine:3.9.3' # FIXME: this should probably be renamed TEST_API_VERSION = os.environ.get('DOCKER_TEST_API_VERSION') From 532c62ee51aacbf0416ae2bc2cf53212ad6eb9db Mon Sep 17 00:00:00 2001 From: Hannes Ljungberg Date: Thu, 28 Mar 2019 10:04:18 +0100 Subject: [PATCH 0895/1301] Add support for rotate_manager_unlock_key Signed-off-by: Hannes Ljungberg --- docker/api/swarm.py | 22 +++++++++++++++++----- docker/models/swarm.py | 8 +++++--- tests/integration/api_swarm_test.py | 13 +++++++++++++ 3 files changed, 35 insertions(+), 8 deletions(-) diff --git a/docker/api/swarm.py b/docker/api/swarm.py index 13e4fd5c6d..897f08e423 100644 --- a/docker/api/swarm.py +++ b/docker/api/swarm.py @@ -406,8 +406,10 @@ def update_node(self, node_id, version, node_spec=None): return True @utils.minimum_version('1.24') - def update_swarm(self, version, swarm_spec=None, rotate_worker_token=False, - rotate_manager_token=False): + def update_swarm(self, version, swarm_spec=None, + rotate_worker_token=False, + rotate_manager_token=False, + rotate_manager_unlock_key=False): """ Update the Swarm's configuration @@ -421,6 +423,8 @@ def update_swarm(self, version, swarm_spec=None, rotate_worker_token=False, ``False``. rotate_manager_token (bool): Rotate the manager join token. Default: ``False``. + rotate_manager_unlock_key (bool): Rotate the manager unlock key. + Default: ``False``. Returns: ``True`` if the request went through. @@ -429,12 +433,20 @@ def update_swarm(self, version, swarm_spec=None, rotate_worker_token=False, :py:class:`docker.errors.APIError` If the server returns an error. """ - url = self._url('/swarm/update') - response = self._post_json(url, data=swarm_spec, params={ + params = { 'rotateWorkerToken': rotate_worker_token, 'rotateManagerToken': rotate_manager_token, 'version': version - }) + } + if rotate_manager_unlock_key: + if utils.version_lt(self._version, '1.25'): + raise errors.InvalidVersion( + 'Rotate manager unlock key ' + 'is only available for API version >= 1.25' + ) + params['rotateManagerUnlockKey'] = rotate_manager_unlock_key + + response = self._post_json(url, data=swarm_spec, params=params) self._raise_for_status(response) return True diff --git a/docker/models/swarm.py b/docker/models/swarm.py index f8c5fff6f3..755c17db43 100644 --- a/docker/models/swarm.py +++ b/docker/models/swarm.py @@ -151,7 +151,7 @@ def unlock(self, key): unlock.__doc__ = APIClient.unlock_swarm.__doc__ def update(self, rotate_worker_token=False, rotate_manager_token=False, - **kwargs): + rotate_manager_unlock_key=False, **kwargs): """ Update the swarm's configuration. @@ -164,7 +164,8 @@ def update(self, rotate_worker_token=False, rotate_manager_token=False, ``False``. rotate_manager_token (bool): Rotate the manager join token. Default: ``False``. - + rotate_manager_unlock_key (bool): Rotate the manager unlock key. + Default: ``False``. Raises: :py:class:`docker.errors.APIError` If the server returns an error. @@ -178,5 +179,6 @@ def update(self, rotate_worker_token=False, rotate_manager_token=False, version=self.version, swarm_spec=self.client.api.create_swarm_spec(**kwargs), rotate_worker_token=rotate_worker_token, - rotate_manager_token=rotate_manager_token + rotate_manager_token=rotate_manager_token, + rotate_manager_unlock_key=rotate_manager_unlock_key ) diff --git a/tests/integration/api_swarm_test.py b/tests/integration/api_swarm_test.py index 5d4086a60b..bf809bd0c1 100644 --- a/tests/integration/api_swarm_test.py +++ b/tests/integration/api_swarm_test.py @@ -236,6 +236,19 @@ def test_remove_main_node(self): assert e.value.response.status_code >= 400 + @requires_api_version('1.25') + def test_rotate_manager_unlock_key(self): + spec = self.client.create_swarm_spec(autolock_managers=True) + assert self.init_swarm(swarm_spec=spec) + swarm_info = self.client.inspect_swarm() + key_1 = self.client.get_unlock_key() + assert self.client.update_swarm( + version=swarm_info['Version']['Index'], + rotate_manager_unlock_key=True + ) + key_2 = self.client.get_unlock_key() + assert key_1['UnlockKey'] != key_2['UnlockKey'] + @requires_api_version('1.30') def test_init_swarm_data_path_addr(self): assert self.init_swarm(data_path_addr='eth0') From 1aae20d13abde8fa4747d5f9c4f5b5c22d534925 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 1 May 2019 19:44:23 -0700 Subject: [PATCH 0896/1301] Remove obsolete win32-requirements file Signed-off-by: Joffrey F --- win32-requirements.txt | 1 - 1 file changed, 1 deletion(-) delete mode 100644 win32-requirements.txt diff --git a/win32-requirements.txt b/win32-requirements.txt deleted file mode 100644 index bc04b4960a..0000000000 --- a/win32-requirements.txt +++ /dev/null @@ -1 +0,0 @@ --r requirements.txt From bdc954b00963f0b8b6b98f03d7939bd9184d548a Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 1 May 2019 19:49:01 -0700 Subject: [PATCH 0897/1301] Stop supporting EOL Python 3.4 Signed-off-by: Joffrey F --- .travis.yml | 2 -- setup.py | 6 +++--- test-requirements.txt | 3 +-- tox.ini | 2 +- 4 files changed, 5 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1c837a2634..577b893f22 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,8 +4,6 @@ matrix: include: - python: 2.7 env: TOXENV=py27 - - python: 3.4 - env: TOXENV=py34 - python: 3.5 env: TOXENV=py35 - python: 3.6 diff --git a/setup.py b/setup.py index 3e1afcbe99..c29787b679 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,8 @@ import codecs import os -from setuptools import setup, find_packages +from setuptools import find_packages +from setuptools import setup ROOT_DIR = os.path.dirname(__file__) SOURCE_DIR = os.path.join(ROOT_DIR) @@ -71,7 +72,7 @@ install_requires=requirements, tests_require=test_requirements, extras_require=extras_require, - python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*', zip_safe=False, test_suite='tests', classifiers=[ @@ -83,7 +84,6 @@ 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', diff --git a/test-requirements.txt b/test-requirements.txt index df369881c9..b89f64622d 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,6 +1,5 @@ coverage==4.5.2 -flake8==3.6.0; python_version != '3.3' -flake8==3.4.1; python_version == '3.3' +flake8==3.6.0 mock==1.0.1 pytest==4.1.0 pytest-cov==2.6.1 diff --git a/tox.ini b/tox.ini index 5396147ecc..df797f4113 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27, py34, py35, py36, py37, flake8 +envlist = py27, py35, py36, py37, flake8 skipsdist=True [testenv] From 87ee18aa39698f91b24116592abbf817a0d2b738 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 1 May 2019 20:47:41 -0700 Subject: [PATCH 0898/1301] Change use_config_proxy default value to True to match CLI behavior Signed-off-by: Joffrey F --- docker/api/build.py | 2 +- docker/api/container.py | 14 ++++++++------ tests/integration/api_exec_test.py | 1 - tests/integration/models_containers_test.py | 13 +++++++------ 4 files changed, 16 insertions(+), 14 deletions(-) diff --git a/docker/api/build.py b/docker/api/build.py index 5176afb3a9..e0a4ac969d 100644 --- a/docker/api/build.py +++ b/docker/api/build.py @@ -20,7 +20,7 @@ def build(self, path=None, tag=None, quiet=False, fileobj=None, decode=False, buildargs=None, gzip=False, shmsize=None, labels=None, cache_from=None, target=None, network_mode=None, squash=None, extra_hosts=None, platform=None, isolation=None, - use_config_proxy=False): + use_config_proxy=True): """ Similar to the ``docker build`` command. Either ``path`` or ``fileobj`` needs to be set. ``path`` can be a local path (to a directory diff --git a/docker/api/container.py b/docker/api/container.py index 94f53ff279..2977f28276 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -1,13 +1,15 @@ -import six from datetime import datetime +import six + from .. import errors from .. import utils from ..constants import DEFAULT_DATA_CHUNK_SIZE -from ..types import ( - CancellableStream, ContainerConfig, EndpointConfig, HostConfig, - NetworkingConfig -) +from ..types import CancellableStream +from ..types import ContainerConfig +from ..types import EndpointConfig +from ..types import HostConfig +from ..types import NetworkingConfig class ContainerApiMixin(object): @@ -222,7 +224,7 @@ def create_container(self, image, command=None, hostname=None, user=None, mac_address=None, labels=None, stop_signal=None, networking_config=None, healthcheck=None, stop_timeout=None, runtime=None, - use_config_proxy=False): + use_config_proxy=True): """ Creates a container. Parameters are similar to those for the ``docker run`` command except it doesn't support the attach options (``-a``). diff --git a/tests/integration/api_exec_test.py b/tests/integration/api_exec_test.py index 80b63ffc1c..dda0ed9051 100644 --- a/tests/integration/api_exec_test.py +++ b/tests/integration/api_exec_test.py @@ -17,7 +17,6 @@ def test_execute_command_with_proxy_env(self): container = self.client.create_container( BUSYBOX, 'cat', detach=True, stdin_open=True, - use_config_proxy=True, ) self.client.start(container) self.tmp_containers.append(container) diff --git a/tests/integration/models_containers_test.py b/tests/integration/models_containers_test.py index 951a08ae68..eac4c97909 100644 --- a/tests/integration/models_containers_test.py +++ b/tests/integration/models_containers_test.py @@ -2,10 +2,13 @@ import tempfile import threading -import docker import pytest -from .base import BaseIntegrationTest, TEST_API_VERSION -from ..helpers import random_name, requires_api_version + +import docker +from ..helpers import random_name +from ..helpers import requires_api_version +from .base import BaseIntegrationTest +from .base import TEST_API_VERSION class ContainerCollectionTest(BaseIntegrationTest): @@ -174,9 +177,7 @@ def test_run_with_proxy_config(self): ftp='sakuya.jp:4967' ) - out = client.containers.run( - 'alpine', 'sh -c "env"', use_config_proxy=True - ) + out = client.containers.run('alpine', 'sh -c "env"') assert b'FTP_PROXY=sakuya.jp:4967\n' in out assert b'ftp_proxy=sakuya.jp:4967\n' in out From 7f56f7057c3448373600534fa4cdb1fb04d51524 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 2 May 2019 12:46:56 -0700 Subject: [PATCH 0899/1301] Don't add superfluous arguments Signed-off-by: Joffrey F --- docker/api/container.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/api/container.py b/docker/api/container.py index 2977f28276..2dca68a144 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -416,7 +416,7 @@ def create_container(self, image, command=None, hostname=None, user=None, if use_config_proxy: environment = self._proxy_configs.inject_proxy_environment( environment - ) + ) or None config = self.create_container_config( image, command, hostname, user, detach, stdin_open, tty, From 0ddf428b6ce7accdac3506b45047df2cb72941ec Mon Sep 17 00:00:00 2001 From: Hannes Ljungberg Date: Fri, 3 May 2019 21:53:36 +0200 Subject: [PATCH 0900/1301] Add NetworkAttachmentConfig type Signed-off-by: Hannes Ljungberg --- docker/api/service.py | 10 +++++---- docker/models/services.py | 5 +++-- docker/types/__init__.py | 2 +- docker/types/services.py | 22 ++++++++++++++++++-- docs/api.rst | 1 + tests/integration/api_service_test.py | 29 +++++++++++++++++++++++++++ 6 files changed, 60 insertions(+), 9 deletions(-) diff --git a/docker/api/service.py b/docker/api/service.py index 372dd10b5c..e9027bfa21 100644 --- a/docker/api/service.py +++ b/docker/api/service.py @@ -135,8 +135,9 @@ def create_service( of the service. Default: ``None`` rollback_config (RollbackConfig): Specification for the rollback strategy of the service. Default: ``None`` - networks (:py:class:`list`): List of network names or IDs to attach - the service to. Default: ``None``. + networks (:py:class:`list`): List of network names or IDs or + :py:class:`~docker.types.NetworkAttachmentConfig` to attach the + service to. Default: ``None``. endpoint_spec (EndpointSpec): Properties that can be configured to access and load balance a service. Default: ``None``. @@ -383,8 +384,9 @@ def update_service(self, service, version, task_template=None, name=None, of the service. Default: ``None``. rollback_config (RollbackConfig): Specification for the rollback strategy of the service. Default: ``None`` - networks (:py:class:`list`): List of network names or IDs to attach - the service to. Default: ``None``. + networks (:py:class:`list`): List of network names or IDs or + :py:class:`~docker.types.NetworkAttachmentConfig` to attach the + service to. Default: ``None``. endpoint_spec (EndpointSpec): Properties that can be configured to access and load balance a service. Default: ``None``. fetch_current_spec (boolean): Use the undefined settings from the diff --git a/docker/models/services.py b/docker/models/services.py index 2b6479f2a1..5eff8c88b5 100644 --- a/docker/models/services.py +++ b/docker/models/services.py @@ -178,8 +178,9 @@ def create(self, image, command=None, **kwargs): ``source:target:options``, where options is either ``ro`` or ``rw``. name (str): Name to give to the service. - networks (list of str): List of network names or IDs to attach - the service to. Default: ``None``. + networks (:py:class:`list`): List of network names or IDs or + :py:class:`~docker.types.NetworkAttachmentConfig` to attach the + service to. Default: ``None``. resources (Resources): Resource limits and reservations. restart_policy (RestartPolicy): Restart policy for containers. secrets (list of :py:class:`docker.types.SecretReference`): List diff --git a/docker/types/__init__.py b/docker/types/__init__.py index f3cac1bc17..5db330e284 100644 --- a/docker/types/__init__.py +++ b/docker/types/__init__.py @@ -7,6 +7,6 @@ ConfigReference, ContainerSpec, DNSConfig, DriverConfig, EndpointSpec, Mount, Placement, PlacementPreference, Privileges, Resources, RestartPolicy, RollbackConfig, SecretReference, ServiceMode, TaskTemplate, - UpdateConfig + UpdateConfig, NetworkAttachmentConfig ) from .swarm import SwarmSpec, SwarmExternalCA diff --git a/docker/types/services.py b/docker/types/services.py index 5722b0e33d..05dda15d75 100644 --- a/docker/types/services.py +++ b/docker/types/services.py @@ -26,8 +26,8 @@ class TaskTemplate(dict): placement (Placement): Placement instructions for the scheduler. If a list is passed instead, it is assumed to be a list of constraints as part of a :py:class:`Placement` object. - networks (:py:class:`list`): List of network names or IDs to attach - the containers to. + networks (:py:class:`list`): List of network names or IDs or + :py:class:`NetworkAttachmentConfig` to attach the service to. force_update (int): A counter that triggers an update even if no relevant parameters have been changed. """ @@ -770,3 +770,21 @@ def __init__(self, credentialspec_file=None, credentialspec_registry=None, if len(selinux_context) > 0: self['SELinuxContext'] = selinux_context + + +class NetworkAttachmentConfig(dict): + """ + Network attachment options for a service. + + Args: + target (str): The target network for attachment. + Can be a network name or ID. + aliases (:py:class:`list`): A list of discoverable alternate names + for the service. + options (:py:class:`dict`): Driver attachment options for the + network target. + """ + def __init__(self, target, aliases=None, options=None): + self['Target'] = target + self['Aliases'] = aliases + self['DriverOpts'] = options diff --git a/docs/api.rst b/docs/api.rst index edb8fffadc..bd0466143d 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -142,6 +142,7 @@ Configuration types .. autoclass:: IPAMPool .. autoclass:: LogConfig .. autoclass:: Mount +.. autoclass:: NetworkAttachmentConfig .. autoclass:: Placement .. autoclass:: PlacementPreference .. autoclass:: Privileges diff --git a/tests/integration/api_service_test.py b/tests/integration/api_service_test.py index 71e0869e9f..520c0d6825 100644 --- a/tests/integration/api_service_test.py +++ b/tests/integration/api_service_test.py @@ -371,6 +371,35 @@ def test_create_service_with_custom_networks(self): {'Target': net1['Id']}, {'Target': net2['Id']} ] + def test_create_service_with_network_attachment_config(self): + network = self.client.create_network( + 'dockerpytest_1', driver='overlay', ipam={'Driver': 'default'} + ) + self.tmp_networks.append(network['Id']) + container_spec = docker.types.ContainerSpec(BUSYBOX, ['true']) + network_config = docker.types.NetworkAttachmentConfig( + target='dockerpytest_1', + aliases=['dockerpytest_1_alias'], + options={ + 'foo': 'bar' + } + ) + task_tmpl = docker.types.TaskTemplate( + container_spec, + networks=[network_config] + ) + name = self.get_service_name() + svc_id = self.client.create_service( + task_tmpl, name=name + ) + svc_info = self.client.inspect_service(svc_id) + assert 'Networks' in svc_info['Spec']['TaskTemplate'] + service_networks_info = svc_info['Spec']['TaskTemplate']['Networks'] + assert len(service_networks_info) == 1 + assert service_networks_info[0]['Target'] == network['Id'] + assert service_networks_info[0]['Aliases'] == ['dockerpytest_1_alias'] + assert service_networks_info[0]['DriverOpts'] == {'foo': 'bar'} + def test_create_service_with_placement(self): node_id = self.client.nodes()[0]['ID'] container_spec = docker.types.ContainerSpec(BUSYBOX, ['true']) From ff0cbe4c7966b6d034dbe4ef39ddab47a9a490fe Mon Sep 17 00:00:00 2001 From: Hannes Ljungberg Date: Fri, 3 May 2019 22:09:48 +0200 Subject: [PATCH 0901/1301] Correctly reference ConfigReference Signed-off-by: Hannes Ljungberg --- docker/models/services.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docker/models/services.py b/docker/models/services.py index 5eff8c88b5..e866545da8 100644 --- a/docker/models/services.py +++ b/docker/models/services.py @@ -206,8 +206,9 @@ def create(self, image, command=None, **kwargs): the container's `hosts` file. dns_config (DNSConfig): Specification for DNS related configurations in resolver configuration file. - configs (:py:class:`list`): List of :py:class:`ConfigReference` - that will be exposed to the service. + configs (:py:class:`list`): List of + :py:class:`~docker.types.ConfigReference` that will be exposed + to the service. privileges (Privileges): Security options for the service's containers. From bcd61e40ddb80735fd6b479c96aaa18cf392020d Mon Sep 17 00:00:00 2001 From: Hannes Ljungberg Date: Fri, 3 May 2019 22:24:33 +0200 Subject: [PATCH 0902/1301] Correctly reference SecretReference Signed-off-by: Hannes Ljungberg --- docker/models/services.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/models/services.py b/docker/models/services.py index e866545da8..a35687b3ca 100644 --- a/docker/models/services.py +++ b/docker/models/services.py @@ -183,7 +183,7 @@ def create(self, image, command=None, **kwargs): service to. Default: ``None``. resources (Resources): Resource limits and reservations. restart_policy (RestartPolicy): Restart policy for containers. - secrets (list of :py:class:`docker.types.SecretReference`): List + secrets (list of :py:class:`~docker.types.SecretReference`): List of secrets accessible to containers for this service. stop_grace_period (int): Amount of time to wait for containers to terminate before forcefully killing them. From 384043229bd2bd5150b8040c27f6c0d5fefe0ea0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Batuhan=20Ta=C5=9Fkaya?= Date: Thu, 9 May 2019 20:29:11 +0300 Subject: [PATCH 0903/1301] reference swarm page correctly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Batuhan Taşkaya --- docs/user_guides/swarm_services.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/user_guides/swarm_services.md b/docs/user_guides/swarm_services.md index 369fbed00e..5c3a80d2d4 100644 --- a/docs/user_guides/swarm_services.md +++ b/docs/user_guides/swarm_services.md @@ -6,7 +6,7 @@ Starting with Engine version 1.12 (API 1.24), it is possible to manage services using the Docker Engine API. Note that the engine needs to be part of a -[Swarm cluster](../swarm.rst) before you can use the service-related methods. +[Swarm cluster](../swarm.html) before you can use the service-related methods. ## Creating a service @@ -66,4 +66,4 @@ Either the service name or service ID can be used as argument. ```python client.remove_service('my_service_name') -``` \ No newline at end of file +``` From 12d73c6d381760840256a4cbc768933555abb512 Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Tue, 26 Mar 2019 15:15:40 +0100 Subject: [PATCH 0904/1301] Xfail test_attach_stream_and_cancel on TLS This test is quite flaky on ssl integration test Signed-off-by: Ulysses Souza --- tests/integration/api_container_test.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index 1190d91e8f..26245c1fa9 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -1252,6 +1252,9 @@ def test_attach_no_stream(self): @pytest.mark.timeout(10) @pytest.mark.skipif(os.environ.get('DOCKER_HOST', '').startswith('ssh://'), reason='No cancellable streams over SSH') + @pytest.mark.xfail(condition=os.environ.get('DOCKER_TLS_VERIFY') or + os.environ.get('DOCKER_CERT_PATH'), + reason='Flaky test on TLS') def test_attach_stream_and_cancel(self): container = self.client.create_container( BUSYBOX, 'sh -c "sleep 2 && echo hello && sleep 60"', From d863f729398911e2f918a1bc822b8f4f32151783 Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Thu, 28 Mar 2019 14:23:19 +0100 Subject: [PATCH 0905/1301] Bump 3.7.2 Signed-off-by: Ulysses Souza --- docs/change-log.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/change-log.md b/docs/change-log.md index 9edfee2f87..d7c336112f 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -1,6 +1,17 @@ Change log ========== +3.7.2 +----- + +[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/59?closed=1) + +### Bugfixes + +* Fix base_url to keep TCP protocol on utils.py by letting the responsability of changing the +protocol to `parse_host` afterwards, letting `base_url` with the original value. +* XFAIL test_attach_stream_and_cancel on TLS + 3.7.1 ----- From f6781575c12a8a9aebe1e1ccee4716eaabf88b3d Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 1 May 2019 19:58:18 -0700 Subject: [PATCH 0906/1301] Bump version 4.0.0 Signed-off-by: Joffrey F --- docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index 93d068ebb5..68d64c8aa8 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "4.0.0-dev" +version = "4.0.0" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) From 589e76ea3c13d469f141bf89aba3f142789da0ba Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 1 May 2019 20:38:33 -0700 Subject: [PATCH 0907/1301] Update changelog for 4.0.0 Signed-off-by: Joffrey F --- docs/change-log.md | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/docs/change-log.md b/docs/change-log.md index d7c336112f..53e9f207ba 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -1,6 +1,43 @@ Change log ========== +4.0.0 +----- + +[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/57?closed=1) + +### Breaking changes + +- Support for Python 3.3 and Python 3.4 has been dropped +- `APIClient.update_service`, `APIClient.init_swarm`, and + `DockerClient.swarm.init` now return a `dict` from the API's response body +- In `APIClient.build` and `DockerClient.images.build`, the `use_config_proxy` + parameter now defaults to True +- `init_path` is no longer a valid parameter for `HostConfig` + +### Features + +- It is now possible to provide `SCTP` ports for port mappings +- `ContainerSpec`s now support the `init` parameter +- `DockerClient.swarm.init` and `APIClient.init_swarm` now support the + `data_path_addr` parameter +- `APIClient.update_swarm` and `DockerClient.swarm.update` now support the + `rotate_manager_unlock_key` parameter +- `APIClient.update_service` returns the API's response body as a `dict` +- `APIClient.init_swarm`, and `DockerClient.swarm.init` now return the API's + response body as a `dict` + +### Bugfixes + +- Fixed `PlacementPreference` instances to produce a valid API type +- Fixed a bug where not setting a value for `buildargs` in `build` could cause + the library to attempt accessing attributes of a `None` value +- Fixed a bug where setting the `volume_driver` parameter in + `DockerClient.containers.create` would result in an error +- `APIClient.inspect_distribution` now correctly sets the authentication + headers on the request, allowing it to be used with private repositories + This change also applies to `DockerClient.get_registry_data` + 3.7.2 ----- From 690b0ce9c4002ff98cc583c842094266cd954729 Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Tue, 14 May 2019 13:16:27 +0200 Subject: [PATCH 0908/1301] Bump urllib3 -> 1.24.3 Signed-off-by: Ulysses Souza --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index eb66c9f592..70f37e2069 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,5 +15,5 @@ pypiwin32==219; sys_platform == 'win32' and python_version < '3.6' pypiwin32==223; sys_platform == 'win32' and python_version >= '3.6' requests==2.20.0 six==1.10.0 -urllib3==1.24.1 +urllib3==1.24.3 websocket-client==0.40.0 From 5de5af115563d2f9a647bdeb4234fa440b2da58c Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Tue, 14 May 2019 13:16:27 +0200 Subject: [PATCH 0909/1301] Bump urllib3 -> 1.24.3 Signed-off-by: Ulysses Souza --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index eb66c9f592..70f37e2069 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,5 +15,5 @@ pypiwin32==219; sys_platform == 'win32' and python_version < '3.6' pypiwin32==223; sys_platform == 'win32' and python_version >= '3.6' requests==2.20.0 six==1.10.0 -urllib3==1.24.1 +urllib3==1.24.3 websocket-client==0.40.0 From 31236ba5a786c2c6c78ab2847db49a1e98ed7528 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Sat, 18 May 2019 18:57:30 -0700 Subject: [PATCH 0910/1301] Add readthedocs config Signed-off-by: Joffrey F --- .readthedocs.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 .readthedocs.yml diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 0000000000..7679f80ab1 --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,10 @@ +version: 2 + +sphinx: + configuration: docs/conf.py + +python: + version: 3.5 + install: + - requirements: docs-requirements.txt + - requirements: requirements.txt From 31fec93872790c10fc2428f4c8b2dbbf115ee621 Mon Sep 17 00:00:00 2001 From: Simon Gurcke Date: Sun, 19 May 2019 12:20:12 +1000 Subject: [PATCH 0911/1301] Change os.errno to errno for py3.7 compatibility Signed-off-by: Simon Gurcke --- docker/credentials/store.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/credentials/store.py b/docker/credentials/store.py index 3f51e4a7eb..0017888978 100644 --- a/docker/credentials/store.py +++ b/docker/credentials/store.py @@ -1,5 +1,5 @@ +import errno import json -import os import subprocess import six @@ -84,7 +84,7 @@ def _execute(self, subcmd, data_input): [self.exe, subcmd], stdin=subprocess.PIPE, stdout=subprocess.PIPE, env=env, ) - output, err = process.communicate(data_input) + output, _ = process.communicate(data_input) if process.returncode != 0: raise subprocess.CalledProcessError( returncode=process.returncode, cmd='', output=output @@ -92,7 +92,7 @@ def _execute(self, subcmd, data_input): except subprocess.CalledProcessError as e: raise errors.process_store_error(e, self.program) except OSError as e: - if e.errno == os.errno.ENOENT: + if e.errno == errno.ENOENT: raise errors.StoreError( '{} not installed or not available in PATH'.format( self.program From ccd9ca494710cc77b100f89b3423ba99603fa030 Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Tue, 26 Mar 2019 15:15:40 +0100 Subject: [PATCH 0912/1301] Xfail test_attach_stream_and_cancel on TLS This test is quite flaky on ssl integration test Signed-off-by: Ulysses Souza --- tests/integration/api_container_test.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index 1190d91e8f..26245c1fa9 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -1252,6 +1252,9 @@ def test_attach_no_stream(self): @pytest.mark.timeout(10) @pytest.mark.skipif(os.environ.get('DOCKER_HOST', '').startswith('ssh://'), reason='No cancellable streams over SSH') + @pytest.mark.xfail(condition=os.environ.get('DOCKER_TLS_VERIFY') or + os.environ.get('DOCKER_CERT_PATH'), + reason='Flaky test on TLS') def test_attach_stream_and_cancel(self): container = self.client.create_container( BUSYBOX, 'sh -c "sleep 2 && echo hello && sleep 60"', From efdac34ef4a99494d09f19832d5bc1c25523ff49 Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Thu, 28 Mar 2019 14:23:19 +0100 Subject: [PATCH 0913/1301] Bump 3.7.2 Signed-off-by: Ulysses Souza --- docs/change-log.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/change-log.md b/docs/change-log.md index 9edfee2f87..d7c336112f 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -1,6 +1,17 @@ Change log ========== +3.7.2 +----- + +[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/59?closed=1) + +### Bugfixes + +* Fix base_url to keep TCP protocol on utils.py by letting the responsability of changing the +protocol to `parse_host` afterwards, letting `base_url` with the original value. +* XFAIL test_attach_stream_and_cancel on TLS + 3.7.1 ----- From 3267d1f0cc46f390d87d1b5a20af1fa5309fbd12 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 1 May 2019 19:58:18 -0700 Subject: [PATCH 0914/1301] Bump version 4.0.0 Signed-off-by: Joffrey F --- docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index 93d068ebb5..68d64c8aa8 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "4.0.0-dev" +version = "4.0.0" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) From b406bfb463dee7f40e92ea37d2ecedde0387ebb3 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 1 May 2019 20:38:33 -0700 Subject: [PATCH 0915/1301] Update changelog for 4.0.0 Signed-off-by: Joffrey F --- docs/change-log.md | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/docs/change-log.md b/docs/change-log.md index d7c336112f..53e9f207ba 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -1,6 +1,43 @@ Change log ========== +4.0.0 +----- + +[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/57?closed=1) + +### Breaking changes + +- Support for Python 3.3 and Python 3.4 has been dropped +- `APIClient.update_service`, `APIClient.init_swarm`, and + `DockerClient.swarm.init` now return a `dict` from the API's response body +- In `APIClient.build` and `DockerClient.images.build`, the `use_config_proxy` + parameter now defaults to True +- `init_path` is no longer a valid parameter for `HostConfig` + +### Features + +- It is now possible to provide `SCTP` ports for port mappings +- `ContainerSpec`s now support the `init` parameter +- `DockerClient.swarm.init` and `APIClient.init_swarm` now support the + `data_path_addr` parameter +- `APIClient.update_swarm` and `DockerClient.swarm.update` now support the + `rotate_manager_unlock_key` parameter +- `APIClient.update_service` returns the API's response body as a `dict` +- `APIClient.init_swarm`, and `DockerClient.swarm.init` now return the API's + response body as a `dict` + +### Bugfixes + +- Fixed `PlacementPreference` instances to produce a valid API type +- Fixed a bug where not setting a value for `buildargs` in `build` could cause + the library to attempt accessing attributes of a `None` value +- Fixed a bug where setting the `volume_driver` parameter in + `DockerClient.containers.create` would result in an error +- `APIClient.inspect_distribution` now correctly sets the authentication + headers on the request, allowing it to be used with private repositories + This change also applies to `DockerClient.get_registry_data` + 3.7.2 ----- From eee115c2b8de639b07d4c2fa43a5a82fbe41de47 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Sat, 18 May 2019 19:28:39 -0700 Subject: [PATCH 0916/1301] Version bump Signed-off-by: Joffrey F --- docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index 68d64c8aa8..21249253e6 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "4.0.0" +version = "4.1.0-dev" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) From 80f68c81cd86447d3a9ed01049f353c32bb6adb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Batuhan=20Ta=C5=9Fkaya?= Date: Thu, 9 May 2019 20:29:11 +0300 Subject: [PATCH 0917/1301] reference swarm page correctly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Batuhan Taşkaya --- docs/user_guides/swarm_services.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/user_guides/swarm_services.md b/docs/user_guides/swarm_services.md index 369fbed00e..5c3a80d2d4 100644 --- a/docs/user_guides/swarm_services.md +++ b/docs/user_guides/swarm_services.md @@ -6,7 +6,7 @@ Starting with Engine version 1.12 (API 1.24), it is possible to manage services using the Docker Engine API. Note that the engine needs to be part of a -[Swarm cluster](../swarm.rst) before you can use the service-related methods. +[Swarm cluster](../swarm.html) before you can use the service-related methods. ## Creating a service @@ -66,4 +66,4 @@ Either the service name or service ID can be used as argument. ```python client.remove_service('my_service_name') -``` \ No newline at end of file +``` From 4a8a86eed41926781a0409b743c59f1fc675a497 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Sat, 18 May 2019 18:57:30 -0700 Subject: [PATCH 0918/1301] Add readthedocs config Signed-off-by: Joffrey F --- .readthedocs.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 .readthedocs.yml diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 0000000000..7679f80ab1 --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,10 @@ +version: 2 + +sphinx: + configuration: docs/conf.py + +python: + version: 3.5 + install: + - requirements: docs-requirements.txt + - requirements: requirements.txt From df182fd42d0919fe5812ffcce15c54499c594c44 Mon Sep 17 00:00:00 2001 From: Simon Gurcke Date: Sun, 19 May 2019 12:20:12 +1000 Subject: [PATCH 0919/1301] Change os.errno to errno for py3.7 compatibility Signed-off-by: Simon Gurcke --- docker/credentials/store.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/credentials/store.py b/docker/credentials/store.py index 3f51e4a7eb..0017888978 100644 --- a/docker/credentials/store.py +++ b/docker/credentials/store.py @@ -1,5 +1,5 @@ +import errno import json -import os import subprocess import six @@ -84,7 +84,7 @@ def _execute(self, subcmd, data_input): [self.exe, subcmd], stdin=subprocess.PIPE, stdout=subprocess.PIPE, env=env, ) - output, err = process.communicate(data_input) + output, _ = process.communicate(data_input) if process.returncode != 0: raise subprocess.CalledProcessError( returncode=process.returncode, cmd='', output=output @@ -92,7 +92,7 @@ def _execute(self, subcmd, data_input): except subprocess.CalledProcessError as e: raise errors.process_store_error(e, self.program) except OSError as e: - if e.errno == os.errno.ENOENT: + if e.errno == errno.ENOENT: raise errors.StoreError( '{} not installed or not available in PATH'.format( self.program From fc0285c09b4285129a5273cc01356e6178f7fbff Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Sat, 18 May 2019 19:28:39 -0700 Subject: [PATCH 0920/1301] Version bump Signed-off-by: Joffrey F --- docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index 68d64c8aa8..21249253e6 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "4.0.0" +version = "4.1.0-dev" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) From 4d08f2c33d0cd15413c5dd0fd5a16e22436b3e81 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Sat, 18 May 2019 19:51:14 -0700 Subject: [PATCH 0921/1301] Bump 4.0.1 Signed-off-by: Joffrey F --- docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index 21249253e6..247312639f 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "4.1.0-dev" +version = "4.0.1" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) From 307e2b3eda7198cbc991a8fbc2bce25b08e9eb9f Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Sat, 18 May 2019 19:53:53 -0700 Subject: [PATCH 0922/1301] Changelog 4.0.1 Signed-off-by: Joffrey F --- docs/change-log.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/change-log.md b/docs/change-log.md index 53e9f207ba..4032249b97 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -1,6 +1,20 @@ Change log ========== +4.0.1 +----- + +[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/60?closed=1) + +### Bugfixes + +- Fixed an obsolete import in the `credentials` subpackage that caused import errors in + Python 3.7 + +### Miscellaneous + +- Docs building has been repaired + 4.0.0 ----- From bc827a2ea9fcf0aa42fe1b93e08de2c3d3c6fab0 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Sat, 18 May 2019 19:51:14 -0700 Subject: [PATCH 0923/1301] Bump 4.0.1 Signed-off-by: Joffrey F --- docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index 21249253e6..247312639f 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "4.1.0-dev" +version = "4.0.1" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) From 4828138f5037aed8c8664c580eef50c7ac099053 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Sat, 18 May 2019 19:53:53 -0700 Subject: [PATCH 0924/1301] Changelog 4.0.1 Signed-off-by: Joffrey F --- docs/change-log.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/change-log.md b/docs/change-log.md index 53e9f207ba..4032249b97 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -1,6 +1,20 @@ Change log ========== +4.0.1 +----- + +[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/60?closed=1) + +### Bugfixes + +- Fixed an obsolete import in the `credentials` subpackage that caused import errors in + Python 3.7 + +### Miscellaneous + +- Docs building has been repaired + 4.0.0 ----- From 0200e051c0f8a83dcd203d0108f67992dabedb25 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Sat, 18 May 2019 20:28:19 -0700 Subject: [PATCH 0925/1301] dev version Signed-off-by: Joffrey F --- docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index 247312639f..21249253e6 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "4.0.1" +version = "4.1.0-dev" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) From 0624ccc9ccd18244622404aaa7ff8717e437d4c5 Mon Sep 17 00:00:00 2001 From: Kajetan Champlewski Date: Fri, 24 May 2019 15:34:30 +0000 Subject: [PATCH 0926/1301] Fix documentation for inspect_secret referring to removal. Signed-off-by: Kajetan Champlewski --- docker/api/secret.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/api/secret.py b/docker/api/secret.py index fa4c2ab81d..e57952b53b 100644 --- a/docker/api/secret.py +++ b/docker/api/secret.py @@ -53,7 +53,7 @@ def inspect_secret(self, id): Retrieve secret metadata Args: - id (string): Full ID of the secret to remove + id (string): Full ID of the secret to inspect Returns (dict): A dictionary of metadata From 4b924dbaf4ebe27f991bc6ad12c21f149e243879 Mon Sep 17 00:00:00 2001 From: Frank Sachsenheim Date: Mon, 27 May 2019 22:07:24 +0200 Subject: [PATCH 0927/1301] Amends the docs concerning multiple label filters Closes #2338 Signed-off-by: Frank Sachsenheim --- docker/api/container.py | 3 ++- docker/api/image.py | 3 ++- docker/api/network.py | 3 ++- docker/models/containers.py | 3 ++- docker/models/images.py | 3 ++- docker/models/networks.py | 3 ++- 6 files changed, 12 insertions(+), 6 deletions(-) diff --git a/docker/api/container.py b/docker/api/container.py index 2dca68a144..302b1d7d91 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -174,7 +174,8 @@ def containers(self, quiet=False, all=False, trunc=False, latest=False, - `exited` (int): Only containers with specified exit code - `status` (str): One of ``restarting``, ``running``, ``paused``, ``exited`` - - `label` (str): format either ``"key"`` or ``"key=value"`` + - `label` (str|list): format either ``"key"``, ``"key=value"`` + or a list of such. - `id` (str): The id of the container. - `name` (str): The name of the container. - `ancestor` (str): Filter by container ancestor. Format of diff --git a/docker/api/image.py b/docker/api/image.py index b370b7d83b..11c8cf7547 100644 --- a/docker/api/image.py +++ b/docker/api/image.py @@ -70,7 +70,8 @@ def images(self, name=None, quiet=False, all=False, filters=None): filters (dict): Filters to be processed on the image list. Available filters: - ``dangling`` (bool) - - ``label`` (str): format either ``key`` or ``key=value`` + - `label` (str|list): format either ``"key"``, ``"key=value"`` + or a list of such. Returns: (dict or list): A list if ``quiet=True``, otherwise a dict. diff --git a/docker/api/network.py b/docker/api/network.py index 57ed8d3b75..b0b7f18c9a 100644 --- a/docker/api/network.py +++ b/docker/api/network.py @@ -15,7 +15,8 @@ def networks(self, names=None, ids=None, filters=None): filters (dict): Filters to be processed on the network list. Available filters: - ``driver=[]`` Matches a network's driver. - - ``label=[]`` or ``label=[=]``. + - ``label=[]``, ``label=[=]`` or a list of + such. - ``type=["custom"|"builtin"]`` Filters networks by type. Returns: diff --git a/docker/models/containers.py b/docker/models/containers.py index d321a58022..4ce2367324 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -900,7 +900,8 @@ def list(self, all=False, before=None, filters=None, limit=-1, since=None, - `exited` (int): Only containers with specified exit code - `status` (str): One of ``restarting``, ``running``, ``paused``, ``exited`` - - `label` (str): format either ``"key"`` or ``"key=value"`` + - `label` (str|list): format either ``"key"``, ``"key=value"`` + or a list of such. - `id` (str): The id of the container. - `name` (str): The name of the container. - `ancestor` (str): Filter by container ancestor. Format of diff --git a/docker/models/images.py b/docker/models/images.py index 5419682940..757a5a4750 100644 --- a/docker/models/images.py +++ b/docker/models/images.py @@ -350,7 +350,8 @@ def list(self, name=None, all=False, filters=None): filters (dict): Filters to be processed on the image list. Available filters: - ``dangling`` (bool) - - ``label`` (str): format either ``key`` or ``key=value`` + - `label` (str|list): format either ``"key"``, ``"key=value"`` + or a list of such. Returns: (list of :py:class:`Image`): The images. diff --git a/docker/models/networks.py b/docker/models/networks.py index be3291a417..f944c8e299 100644 --- a/docker/models/networks.py +++ b/docker/models/networks.py @@ -190,7 +190,8 @@ def list(self, *args, **kwargs): filters (dict): Filters to be processed on the network list. Available filters: - ``driver=[]`` Matches a network's driver. - - ``label=[]`` or ``label=[=]``. + - `label` (str|list): format either ``"key"``, ``"key=value"`` + or a list of such. - ``type=["custom"|"builtin"]`` Filters networks by type. greedy (bool): Fetch more details for each network individually. You might want this to get the containers attached to them. From 7302d1af04d014b9c264bce30915ef411382ef8e Mon Sep 17 00:00:00 2001 From: Kajetan Champlewski Date: Fri, 31 May 2019 09:11:20 +0000 Subject: [PATCH 0928/1301] Handle str in setter for test. Signed-off-by: Kajetan Champlewski --- docker/types/healthcheck.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/types/healthcheck.py b/docker/types/healthcheck.py index 61857c21ce..919c4fd752 100644 --- a/docker/types/healthcheck.py +++ b/docker/types/healthcheck.py @@ -53,6 +53,8 @@ def test(self): @test.setter def test(self, value): + if isinstance(value, six.string_types): + value = ["CMD-SHELL", value] self['Test'] = value @property From dcff8876b1d2eed158e7b80000d82018aa3d8e34 Mon Sep 17 00:00:00 2001 From: Kajetan Champlewski Date: Fri, 31 May 2019 09:13:30 +0000 Subject: [PATCH 0929/1301] Clean up healtcheck.py docs Signed-off-by: Kajetan Champlewski --- docker/types/healthcheck.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/types/healthcheck.py b/docker/types/healthcheck.py index 919c4fd752..9815018db8 100644 --- a/docker/types/healthcheck.py +++ b/docker/types/healthcheck.py @@ -14,7 +14,7 @@ class Healthcheck(DictType): - Empty list: Inherit healthcheck from parent image - ``["NONE"]``: Disable healthcheck - ``["CMD", args...]``: exec arguments directly. - - ``["CMD-SHELL", command]``: RUn command in the system's + - ``["CMD-SHELL", command]``: Run command in the system's default shell. If a string is provided, it will be used as a ``CMD-SHELL`` @@ -23,9 +23,9 @@ class Healthcheck(DictType): should be 0 or at least 1000000 (1 ms). timeout (int): The time to wait before considering the check to have hung. It should be 0 or at least 1000000 (1 ms). - retries (integer): The number of consecutive failures needed to + retries (int): The number of consecutive failures needed to consider a container as unhealthy. - start_period (integer): Start period for the container to + start_period (int): Start period for the container to initialize before starting health-retries countdown in nanoseconds. It should be 0 or at least 1000000 (1 ms). """ From 7dd3d563f6662b8c082e8b49c93676b4c96b7e8b Mon Sep 17 00:00:00 2001 From: Djordje Lukic Date: Wed, 19 Jun 2019 14:09:47 +0200 Subject: [PATCH 0930/1301] Bump websocket-client -> 0.56.0 Signed-off-by: Djordje Lukic --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 70f37e2069..804a78a0ce 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,4 +16,4 @@ pypiwin32==223; sys_platform == 'win32' and python_version >= '3.6' requests==2.20.0 six==1.10.0 urllib3==1.24.3 -websocket-client==0.40.0 +websocket-client==0.56.0 From 1ef822afee898b968476d0a8fff31e8455f4066d Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Sat, 18 May 2019 20:28:19 -0700 Subject: [PATCH 0931/1301] dev version Signed-off-by: Joffrey F --- docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index 247312639f..21249253e6 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "4.0.1" +version = "4.1.0-dev" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) From c5ca2ef85eac15ef38d98abaf28e97dd2f04822f Mon Sep 17 00:00:00 2001 From: Kajetan Champlewski Date: Fri, 24 May 2019 15:34:30 +0000 Subject: [PATCH 0932/1301] Fix documentation for inspect_secret referring to removal. Signed-off-by: Kajetan Champlewski --- docker/api/secret.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/api/secret.py b/docker/api/secret.py index fa4c2ab81d..e57952b53b 100644 --- a/docker/api/secret.py +++ b/docker/api/secret.py @@ -53,7 +53,7 @@ def inspect_secret(self, id): Retrieve secret metadata Args: - id (string): Full ID of the secret to remove + id (string): Full ID of the secret to inspect Returns (dict): A dictionary of metadata From 241aaaab238c54b07575c1c7bef8b321f4cd0fc3 Mon Sep 17 00:00:00 2001 From: Kajetan Champlewski Date: Fri, 31 May 2019 09:11:20 +0000 Subject: [PATCH 0933/1301] Handle str in setter for test. Signed-off-by: Kajetan Champlewski --- docker/types/healthcheck.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/types/healthcheck.py b/docker/types/healthcheck.py index 61857c21ce..919c4fd752 100644 --- a/docker/types/healthcheck.py +++ b/docker/types/healthcheck.py @@ -53,6 +53,8 @@ def test(self): @test.setter def test(self, value): + if isinstance(value, six.string_types): + value = ["CMD-SHELL", value] self['Test'] = value @property From 1f38d270e0ba0219ad8400c8f02f678ed3b90b47 Mon Sep 17 00:00:00 2001 From: Kajetan Champlewski Date: Fri, 31 May 2019 09:13:30 +0000 Subject: [PATCH 0934/1301] Clean up healtcheck.py docs Signed-off-by: Kajetan Champlewski --- docker/types/healthcheck.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/types/healthcheck.py b/docker/types/healthcheck.py index 919c4fd752..9815018db8 100644 --- a/docker/types/healthcheck.py +++ b/docker/types/healthcheck.py @@ -14,7 +14,7 @@ class Healthcheck(DictType): - Empty list: Inherit healthcheck from parent image - ``["NONE"]``: Disable healthcheck - ``["CMD", args...]``: exec arguments directly. - - ``["CMD-SHELL", command]``: RUn command in the system's + - ``["CMD-SHELL", command]``: Run command in the system's default shell. If a string is provided, it will be used as a ``CMD-SHELL`` @@ -23,9 +23,9 @@ class Healthcheck(DictType): should be 0 or at least 1000000 (1 ms). timeout (int): The time to wait before considering the check to have hung. It should be 0 or at least 1000000 (1 ms). - retries (integer): The number of consecutive failures needed to + retries (int): The number of consecutive failures needed to consider a container as unhealthy. - start_period (integer): Start period for the container to + start_period (int): Start period for the container to initialize before starting health-retries countdown in nanoseconds. It should be 0 or at least 1000000 (1 ms). """ From a821502b9ecf4dad26df0201a5c95111b00e42c3 Mon Sep 17 00:00:00 2001 From: Djordje Lukic Date: Wed, 19 Jun 2019 14:09:47 +0200 Subject: [PATCH 0935/1301] Bump websocket-client -> 0.56.0 Signed-off-by: Djordje Lukic --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 70f37e2069..804a78a0ce 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,4 +16,4 @@ pypiwin32==223; sys_platform == 'win32' and python_version >= '3.6' requests==2.20.0 six==1.10.0 urllib3==1.24.3 -websocket-client==0.40.0 +websocket-client==0.56.0 From 805f5f4b38cedce36b2037d66bf1a6f99982c017 Mon Sep 17 00:00:00 2001 From: Djordje Lukic Date: Thu, 20 Jun 2019 12:58:09 +0200 Subject: [PATCH 0936/1301] Bump 4.0.2 Signed-off-by: Djordje Lukic --- docker/version.py | 2 +- docs/change-log.md | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index 21249253e6..25c92501f4 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "4.1.0-dev" +version = "4.0.2" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) diff --git a/docs/change-log.md b/docs/change-log.md index 4032249b97..b10cfd544c 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -1,6 +1,20 @@ Change log ========== +4.0.2 +----- + +[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/62?closed=1) + +### Bugfixes + +- Unified the way `HealthCheck` is created/configured + +### Miscellaneous + +- Bumped version of websocket-client + + 4.0.1 ----- From 46fdeffb103af59b526e0d1da840efaba93679c3 Mon Sep 17 00:00:00 2001 From: Djordje Lukic Date: Thu, 20 Jun 2019 12:58:09 +0200 Subject: [PATCH 0937/1301] Bump 4.0.2 Signed-off-by: Djordje Lukic --- docker/version.py | 2 +- docs/change-log.md | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index 21249253e6..25c92501f4 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "4.1.0-dev" +version = "4.0.2" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) diff --git a/docs/change-log.md b/docs/change-log.md index 4032249b97..b10cfd544c 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -1,6 +1,20 @@ Change log ========== +4.0.2 +----- + +[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/62?closed=1) + +### Bugfixes + +- Unified the way `HealthCheck` is created/configured + +### Miscellaneous + +- Bumped version of websocket-client + + 4.0.1 ----- From 4db37a12676e106b5f2980e959c9baa623ff6231 Mon Sep 17 00:00:00 2001 From: Djordje Lukic Date: Thu, 20 Jun 2019 13:34:03 +0200 Subject: [PATCH 0938/1301] Bump dev Signed-off-by: Djordje Lukic --- docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index 25c92501f4..21249253e6 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "4.0.2" +version = "4.1.0-dev" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) From 8303884612cd235727434b81e5ff28b7f999be81 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Tue, 25 Jun 2019 13:08:39 -0400 Subject: [PATCH 0939/1301] Remove exec detach test Forking off an exec process and detaching isn't a supported method Signed-off-by: Michael Crosby --- tests/integration/api_exec_test.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/tests/integration/api_exec_test.py b/tests/integration/api_exec_test.py index dda0ed9051..53b7e22fea 100644 --- a/tests/integration/api_exec_test.py +++ b/tests/integration/api_exec_test.py @@ -226,24 +226,6 @@ def test_detach_with_config_file(self): assert_cat_socket_detached_with_keys(sock, [ctrl_with('p')]) - def test_detach_with_arg(self): - self.client._general_configs['detachKeys'] = 'ctrl-p' - container = self.client.create_container( - BUSYBOX, 'cat', detach=True, stdin_open=True - ) - id = container['Id'] - self.client.start(id) - self.tmp_containers.append(id) - - exec_id = self.client.exec_create( - id, 'cat', - stdin=True, tty=True, detach_keys='ctrl-x', stdout=True - ) - sock = self.client.exec_start(exec_id, tty=True, socket=True) - self.addCleanup(sock.close) - - assert_cat_socket_detached_with_keys(sock, [ctrl_with('x')]) - class ExecDemuxTest(BaseAPIIntegrationTest): cmd = 'sh -c "{}"'.format(' ; '.join([ From df340bea60d7623c130f18a7602e7086dfa65044 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Fri, 12 Jul 2019 01:28:41 +0200 Subject: [PATCH 0940/1301] Update credentials-helpers to v0.6.2 Signed-off-by: Sebastiaan van Stijn --- tests/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Dockerfile b/tests/Dockerfile index 042fc7038a..8f49cd2ce0 100644 --- a/tests/Dockerfile +++ b/tests/Dockerfile @@ -10,7 +10,7 @@ RUN gpg2 --import gpg-keys/secret RUN gpg2 --import-ownertrust gpg-keys/ownertrust RUN yes | pass init $(gpg2 --no-auto-check-trustdb --list-secret-keys | grep ^sec | cut -d/ -f2 | cut -d" " -f1) RUN gpg2 --check-trustdb -ARG CREDSTORE_VERSION=v0.6.0 +ARG CREDSTORE_VERSION=v0.6.2 RUN curl -sSL -o /opt/docker-credential-pass.tar.gz \ https://github.com/docker/docker-credential-helpers/releases/download/$CREDSTORE_VERSION/docker-credential-pass-$CREDSTORE_VERSION-amd64.tar.gz && \ tar -xf /opt/docker-credential-pass.tar.gz -O > /usr/local/bin/docker-credential-pass && \ From 1126ea9d6f981bfa8663cab8b13ece8b501cd8b3 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Fri, 12 Jul 2019 22:50:35 +0200 Subject: [PATCH 0941/1301] xfail test_init_swarm_data_path_addr This test can fail if `eth0` has multiple IP addresses; E docker.errors.APIError: 400 Client Error: Bad Request ("interface eth0 has more than one IPv6 address (2001:db8:1::242:ac11:2 and fe80::42:acff:fe11:2)") Which is not a failiure, but depends on the environment that the test is run in. Signed-off-by: Sebastiaan van Stijn --- tests/integration/api_swarm_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/api_swarm_test.py b/tests/integration/api_swarm_test.py index bf809bd0c1..f1cbc264e2 100644 --- a/tests/integration/api_swarm_test.py +++ b/tests/integration/api_swarm_test.py @@ -250,5 +250,6 @@ def test_rotate_manager_unlock_key(self): assert key_1['UnlockKey'] != key_2['UnlockKey'] @requires_api_version('1.30') + @pytest.mark.xfail(reason='Can fail if eth0 has multiple IP addresses') def test_init_swarm_data_path_addr(self): assert self.init_swarm(data_path_addr='eth0') From 6f6572bb8a786e33da4fada6c14ddd65e70426bb Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Fri, 12 Jul 2019 18:53:34 +0200 Subject: [PATCH 0942/1301] Update to python 3.7 (buster) and use build-args The build arg can be used to either test different versions, but also makes it easier to "grep" when upgrading versions. The output format of `gpg2 --list-secret-keys` changed in the version installed on Buster, so `grep` was replaced with `awk` to address the new output format; Debian Jessie: gpg2 --no-auto-check-trustdb --list-secret-keys /root/.gnupg/secring.gpg ------------------------ sec 1024D/A7B21401 2018-04-25 uid Sakuya Izayoi ssb 1024g/C235E4CE 2018-04-25 Debian Buster: gpg2 --no-auto-check-trustdb --list-secret-keys /root/.gnupg/pubring.kbx ------------------------ sec dsa1024 2018-04-25 [SCA] 9781B87DAB042E6FD51388A5464ED987A7B21401 uid [ultimate] Sakuya Izayoi ssb elg1024 2018-04-25 [E] Signed-off-by: Sebastiaan van Stijn --- Dockerfile | 4 +++- Dockerfile-docs | 4 +++- Dockerfile-py3 | 4 +++- tests/Dockerfile | 7 ++++--- tests/Dockerfile-dind-certs | 4 +++- 5 files changed, 16 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 82758daf97..124f68cdd0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,6 @@ -FROM python:2.7 +ARG PYTHON_VERSION=2.7 + +FROM python:${PYTHON_VERSION} RUN mkdir /src WORKDIR /src diff --git a/Dockerfile-docs b/Dockerfile-docs index 105083e8cb..9d11312fca 100644 --- a/Dockerfile-docs +++ b/Dockerfile-docs @@ -1,4 +1,6 @@ -FROM python:3.5 +ARG PYTHON_VERSION=3.7 + +FROM python:${PYTHON_VERSION} ARG uid=1000 ARG gid=1000 diff --git a/Dockerfile-py3 b/Dockerfile-py3 index d558ba3e4f..22732dec5c 100644 --- a/Dockerfile-py3 +++ b/Dockerfile-py3 @@ -1,4 +1,6 @@ -FROM python:3.6 +ARG PYTHON_VERSION=3.7 + +FROM python:${PYTHON_VERSION} RUN mkdir /src WORKDIR /src diff --git a/tests/Dockerfile b/tests/Dockerfile index 8f49cd2ce0..f2f36b4471 100644 --- a/tests/Dockerfile +++ b/tests/Dockerfile @@ -1,5 +1,6 @@ -ARG PYTHON_VERSION=3.6 -FROM python:$PYTHON_VERSION-jessie +ARG PYTHON_VERSION=3.7 + +FROM python:${PYTHON_VERSION} RUN apt-get update && apt-get -y install \ gnupg2 \ pass \ @@ -8,7 +9,7 @@ RUN apt-get update && apt-get -y install \ COPY ./tests/gpg-keys /gpg-keys RUN gpg2 --import gpg-keys/secret RUN gpg2 --import-ownertrust gpg-keys/ownertrust -RUN yes | pass init $(gpg2 --no-auto-check-trustdb --list-secret-keys | grep ^sec | cut -d/ -f2 | cut -d" " -f1) +RUN yes | pass init $(gpg2 --no-auto-check-trustdb --list-secret-key | awk '/^sec/{getline; $1=$1; print}') RUN gpg2 --check-trustdb ARG CREDSTORE_VERSION=v0.6.2 RUN curl -sSL -o /opt/docker-credential-pass.tar.gz \ diff --git a/tests/Dockerfile-dind-certs b/tests/Dockerfile-dind-certs index 9e8c042b63..2ab87ef732 100644 --- a/tests/Dockerfile-dind-certs +++ b/tests/Dockerfile-dind-certs @@ -1,4 +1,6 @@ -FROM python:2.7 +ARG PYTHON_VERSION=2.7 + +FROM python:${PYTHON_VERSION} RUN mkdir /tmp/certs VOLUME /certs From bc46490a6883267785403ec3f741127c024dd814 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Mon, 15 Jul 2019 15:04:31 +0200 Subject: [PATCH 0943/1301] Adjust `--platform` tests for changes in docker engine These tests started failing on recent versions of the engine because the error string changed, and due to a regression, the status code for one endpoint changed from a 400 to a 500. On Docker 18.03: The `docker build` case properly returns a 400, and "invalid platform" as error string; ```bash docker build --platform=foobar -<}] module=grpc INFO[2019-07-15T11:59:20.688270160Z] ClientConn switching balancer to "pick_first" module=grpc INFO[2019-07-15T11:59:20.688353083Z] pickfirstBalancer: HandleSubConnStateChange: 0xc4209b0630, CONNECTING module=grpc INFO[2019-07-15T11:59:20.688985698Z] pickfirstBalancer: HandleSubConnStateChange: 0xc4209b0630, READY module=grpc DEBU[2019-07-15T11:59:20.812700550Z] client is session enabled DEBU[2019-07-15T11:59:20.813139288Z] FIXME: Got an API for which error does not match any expected type!!!: invalid argument github.com/docker/docker/vendor/github.com/containerd/containerd/errdefs.init /go/src/github.com/docker/docker/vendor/github.com/containerd/containerd/errdefs/errors.go:40 github.com/docker/docker/vendor/github.com/containerd/containerd/content.init :1 github.com/docker/docker/builder/builder-next.init :1 github.com/docker/docker/api/server/backend/build.init :1 main.init :1 runtime.main /usr/local/go/src/runtime/proc.go:186 runtime.goexit /usr/local/go/src/runtime/asm_amd64.s:2361 error_type="*errors.fundamental" module=api ERRO[2019-07-15T11:59:20.813210677Z] Handler for POST /v1.39/build returned error: "foobar": unknown operating system or architecture: invalid argument DEBU[2019-07-15T11:59:20.813276737Z] FIXME: Got an API for which error does not match any expected type!!!: invalid argument github.com/docker/docker/vendor/github.com/containerd/containerd/errdefs.init /go/src/github.com/docker/docker/vendor/github.com/containerd/containerd/errdefs/errors.go:40 github.com/docker/docker/vendor/github.com/containerd/containerd/content.init :1 github.com/docker/docker/builder/builder-next.init :1 github.com/docker/docker/api/server/backend/build.init :1 main.init :1 runtime.main /usr/local/go/src/runtime/proc.go:186 runtime.goexit /usr/local/go/src/runtime/asm_amd64.s:2361 error_type="*errors.fundamental" module=api ``` Same for the `docker pull --platform=foobar hello-world:latest` case: ```bash docker pull --platform=foobar hello-world:latest Error response from daemon: "foobar": unknown operating system or architecture: invalid argument ``` ``` DEBU[2019-07-15T12:00:18.812995330Z] Calling POST /v1.39/images/create?fromImage=hello-world&platform=foobar&tag=latest DEBU[2019-07-15T12:00:18.813229172Z] FIXME: Got an API for which error does not match any expected type!!!: invalid argument github.com/docker/docker/vendor/github.com/containerd/containerd/errdefs.init /go/src/github.com/docker/docker/vendor/github.com/containerd/containerd/errdefs/errors.go:40 github.com/docker/docker/vendor/github.com/containerd/containerd/content.init :1 github.com/docker/docker/builder/builder-next.init :1 github.com/docker/docker/api/server/backend/build.init :1 main.init :1 runtime.main /usr/local/go/src/runtime/proc.go:186 runtime.goexit /usr/local/go/src/runtime/asm_amd64.s:2361 error_type="*errors.fundamental" module=api ERRO[2019-07-15T12:00:18.813365546Z] Handler for POST /v1.39/images/create returned error: "foobar": unknown operating system or architecture: invalid argument DEBU[2019-07-15T12:00:18.813461428Z] FIXME: Got an API for which error does not match any expected type!!!: invalid argument github.com/docker/docker/vendor/github.com/containerd/containerd/errdefs.init /go/src/github.com/docker/docker/vendor/github.com/containerd/containerd/errdefs/errors.go:40 github.com/docker/docker/vendor/github.com/containerd/containerd/content.init :1 github.com/docker/docker/builder/builder-next.init :1 github.com/docker/docker/api/server/backend/build.init :1 main.init :1 runtime.main /usr/local/go/src/runtime/proc.go:186 runtime.goexit /usr/local/go/src/runtime/asm_amd64.s:2361 error_type="*errors.fundamental" module=api ``` Signed-off-by: Sebastiaan van Stijn --- tests/integration/api_build_test.py | 6 ++++-- tests/integration/api_image_test.py | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/integration/api_build_test.py b/tests/integration/api_build_test.py index 8bfc7960fc..4776f45385 100644 --- a/tests/integration/api_build_test.py +++ b/tests/integration/api_build_test.py @@ -448,8 +448,10 @@ def test_build_invalid_platform(self): for _ in stream: pass - assert excinfo.value.status_code == 400 - assert 'invalid platform' in excinfo.exconly() + # Some API versions incorrectly returns 500 status; assert 4xx or 5xx + assert excinfo.value.is_error() + assert 'unknown operating system' in excinfo.exconly() \ + or 'invalid platform' in excinfo.exconly() def test_build_out_of_context_dockerfile(self): base_dir = tempfile.mkdtemp() diff --git a/tests/integration/api_image_test.py b/tests/integration/api_image_test.py index 050e7f339b..56a7692489 100644 --- a/tests/integration/api_image_test.py +++ b/tests/integration/api_image_test.py @@ -69,8 +69,10 @@ def test_pull_invalid_platform(self): with pytest.raises(docker.errors.APIError) as excinfo: self.client.pull('hello-world', platform='foobar') - assert excinfo.value.status_code == 500 - assert 'invalid platform' in excinfo.exconly() + # Some API versions incorrectly returns 500 status; assert 4xx or 5xx + assert excinfo.value.is_error() + assert 'unknown operating system' in excinfo.exconly() \ + or 'invalid platform' in excinfo.exconly() class CommitTest(BaseAPIIntegrationTest): From 5a91c2e83ef5d2d46b65bc12b8ed62ea8c033c68 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Tue, 16 Jul 2019 12:51:01 +0200 Subject: [PATCH 0944/1301] test/Dockerfile: allow using a mirror for the apt repository With this change applied, the default debian package repository can be replaced with a mirror; ``` make APT_MIRROR=cdn-fastly.deb.debian.org build-py3 ... Step 5/19 : RUN apt-get update && apt-get -y install gnupg2 pass curl ---> Running in 01c1101a0bd0 Get:1 http://cdn-fastly.deb.debian.org/debian buster InRelease [118 kB] Get:2 http://cdn-fastly.deb.debian.org/debian-security buster/updates InRelease [39.1 kB] Get:3 http://cdn-fastly.deb.debian.org/debian buster-updates InRelease [46.8 kB] Get:4 http://cdn-fastly.deb.debian.org/debian buster/main amd64 Packages [7897 kB] Get:5 http://cdn-fastly.deb.debian.org/debian-security buster/updates/main amd64 Packages [22.8 kB] ``` Signed-off-by: Sebastiaan van Stijn --- Makefile | 4 ++-- tests/Dockerfile | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index ad643e8051..db103f5bdf 100644 --- a/Makefile +++ b/Makefile @@ -8,11 +8,11 @@ clean: .PHONY: build build: - docker build -t docker-sdk-python -f tests/Dockerfile --build-arg PYTHON_VERSION=2.7 . + docker build -t docker-sdk-python -f tests/Dockerfile --build-arg PYTHON_VERSION=2.7 --build-arg APT_MIRROR . .PHONY: build-py3 build-py3: - docker build -t docker-sdk-python3 -f tests/Dockerfile . + docker build -t docker-sdk-python3 -f tests/Dockerfile --build-arg APT_MIRROR . .PHONY: build-docs build-docs: diff --git a/tests/Dockerfile b/tests/Dockerfile index f2f36b4471..2fc06840e9 100644 --- a/tests/Dockerfile +++ b/tests/Dockerfile @@ -1,6 +1,11 @@ ARG PYTHON_VERSION=3.7 FROM python:${PYTHON_VERSION} + +ARG APT_MIRROR +RUN sed -ri "s/(httpredir|deb).debian.org/${APT_MIRROR:-deb.debian.org}/g" /etc/apt/sources.list \ + && sed -ri "s/(security).debian.org/${APT_MIRROR:-security.debian.org}/g" /etc/apt/sources.list + RUN apt-get update && apt-get -y install \ gnupg2 \ pass \ From a1bc6c289bbecfbe7cc379a16aa9703e600a560c Mon Sep 17 00:00:00 2001 From: Francis Laniel Date: Wed, 9 Jan 2019 19:31:56 +0100 Subject: [PATCH 0945/1301] Add documentation to argument 'mem_reservation'. The documentation was added for function ContainerCollection::run and ContainerApiMixin::create_host_config. Signed-off-by: Francis Laniel Add documentation to argument 'mem_reservation'. The documentation was added for function ContainerCollection::run and ContainerApiMixin::create_host_config. Signed-off-by: Francis Laniel --- docker/api/container.py | 1 + docker/models/containers.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/api/container.py b/docker/api/container.py index 2dca68a144..326e7679f1 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -502,6 +502,7 @@ def create_host_config(self, *args, **kwargs): bytes) or a string with a units identification char (``100000b``, ``1000k``, ``128m``, ``1g``). If a string is specified without a units character, bytes are assumed as an + mem_reservation (int or str): Memory soft limit. mem_swappiness (int): Tune a container's memory swappiness behavior. Accepts number between 0 and 100. memswap_limit (str or int): Maximum amount of memory + swap a diff --git a/docker/models/containers.py b/docker/models/containers.py index d321a58022..999851ec13 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -618,7 +618,7 @@ def run(self, image, command=None, stdout=True, stderr=False, (``100000b``, ``1000k``, ``128m``, ``1g``). If a string is specified without a units character, bytes are assumed as an intended unit. - mem_reservation (int or str): Memory soft limit + mem_reservation (int or str): Memory soft limit. mem_swappiness (int): Tune a container's memory swappiness behavior. Accepts number between 0 and 100. memswap_limit (str or int): Maximum amount of memory + swap a From b2a1b0316304d46a6b19dafe638855cd05ff3b01 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Tue, 16 Jul 2019 16:04:38 +0200 Subject: [PATCH 0946/1301] Update credentials-helpers to v0.6.3 full diff: https://github.com/docker/docker-credential-helpers/compare/v0.6.2...v0.6.3 Signed-off-by: Sebastiaan van Stijn --- tests/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Dockerfile b/tests/Dockerfile index f2f36b4471..4bd98f8733 100644 --- a/tests/Dockerfile +++ b/tests/Dockerfile @@ -11,7 +11,7 @@ RUN gpg2 --import gpg-keys/secret RUN gpg2 --import-ownertrust gpg-keys/ownertrust RUN yes | pass init $(gpg2 --no-auto-check-trustdb --list-secret-key | awk '/^sec/{getline; $1=$1; print}') RUN gpg2 --check-trustdb -ARG CREDSTORE_VERSION=v0.6.2 +ARG CREDSTORE_VERSION=v0.6.3 RUN curl -sSL -o /opt/docker-credential-pass.tar.gz \ https://github.com/docker/docker-credential-helpers/releases/download/$CREDSTORE_VERSION/docker-credential-pass-$CREDSTORE_VERSION-amd64.tar.gz && \ tar -xf /opt/docker-credential-pass.tar.gz -O > /usr/local/bin/docker-credential-pass && \ From d7caa6039d7b084baebd9795ffc9b43098deda3b Mon Sep 17 00:00:00 2001 From: Matt Fluet Date: Mon, 5 Aug 2019 18:31:56 -0400 Subject: [PATCH 0947/1301] Correct INDEX_URL logic in build.py _set_auth_headers Signed-off-by: Matt Fluet --- docker/api/build.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/api/build.py b/docker/api/build.py index e0a4ac969d..365129a064 100644 --- a/docker/api/build.py +++ b/docker/api/build.py @@ -308,7 +308,8 @@ def _set_auth_headers(self, headers): auth_data = self._auth_configs.get_all_credentials() # See https://github.com/docker/docker-py/issues/1683 - if auth.INDEX_URL not in auth_data and auth.INDEX_URL in auth_data: + if (auth.INDEX_URL not in auth_data and + auth.INDEX_NAME in auth_data): auth_data[auth.INDEX_URL] = auth_data.get(auth.INDEX_NAME, {}) log.debug( From 7b22b147159f440ca468280d2438ea8073e36faf Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Sat, 10 Aug 2019 18:50:36 +0200 Subject: [PATCH 0948/1301] pytest: set junitxml suite name to "docker-py" Signed-off-by: Sebastiaan van Stijn --- pytest.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pytest.ini b/pytest.ini index 21b47a6aaa..d233c56f18 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,2 +1,4 @@ [pytest] addopts = --tb=short -rxs + +junit_suite_name = docker-py From 54b48a9b7ab59b4dcf49acf49ddf52035ba3ea08 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Sat, 10 Aug 2019 18:30:40 +0200 Subject: [PATCH 0949/1301] Update alpine version to 3.10, and rename BUSYBOX variable Signed-off-by: Sebastiaan van Stijn --- tests/integration/api_build_test.py | 4 +- tests/integration/api_container_test.py | 190 +++++++++++----------- tests/integration/api_exec_test.py | 30 ++-- tests/integration/api_healthcheck_test.py | 10 +- tests/integration/api_image_test.py | 18 +- tests/integration/api_network_test.py | 18 +- tests/integration/api_service_test.py | 68 ++++---- tests/integration/base.py | 4 +- tests/integration/conftest.py | 10 +- tests/integration/errors_test.py | 4 +- tests/integration/models_images_test.py | 12 +- tests/integration/regression_test.py | 10 +- 12 files changed, 189 insertions(+), 189 deletions(-) diff --git a/tests/integration/api_build_test.py b/tests/integration/api_build_test.py index 4776f45385..57128124ef 100644 --- a/tests/integration/api_build_test.py +++ b/tests/integration/api_build_test.py @@ -9,7 +9,7 @@ import pytest import six -from .base import BaseAPIIntegrationTest, BUSYBOX +from .base import BaseAPIIntegrationTest, TEST_IMG from ..helpers import random_name, requires_api_version, requires_experimental @@ -277,7 +277,7 @@ def test_build_with_network_mode(self): # Set up pingable endpoint on custom network network = self.client.create_network(random_name())['Id'] self.tmp_networks.append(network) - container = self.client.create_container(BUSYBOX, 'top') + container = self.client.create_container(TEST_IMG, 'top') self.tmp_containers.append(container) self.client.start(container) self.client.connect_container_to_network( diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index 26245c1fa9..1ba3eaa583 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -15,7 +15,7 @@ from ..helpers import ctrl_with from ..helpers import requires_api_version from .base import BaseAPIIntegrationTest -from .base import BUSYBOX +from .base import TEST_IMG from docker.constants import IS_WINDOWS_PLATFORM from docker.utils.socket import next_frame_header from docker.utils.socket import read_exactly @@ -25,7 +25,7 @@ class ListContainersTest(BaseAPIIntegrationTest): def test_list_containers(self): res0 = self.client.containers(all=True) size = len(res0) - res1 = self.client.create_container(BUSYBOX, 'true') + res1 = self.client.create_container(TEST_IMG, 'true') assert 'Id' in res1 self.client.start(res1['Id']) self.tmp_containers.append(res1['Id']) @@ -44,13 +44,13 @@ def test_list_containers(self): class CreateContainerTest(BaseAPIIntegrationTest): def test_create(self): - res = self.client.create_container(BUSYBOX, 'true') + res = self.client.create_container(TEST_IMG, 'true') assert 'Id' in res self.tmp_containers.append(res['Id']) def test_create_with_host_pid_mode(self): ctnr = self.client.create_container( - BUSYBOX, 'true', host_config=self.client.create_host_config( + TEST_IMG, 'true', host_config=self.client.create_host_config( pid_mode='host', network_mode='none' ) ) @@ -65,7 +65,7 @@ def test_create_with_host_pid_mode(self): def test_create_with_links(self): res0 = self.client.create_container( - BUSYBOX, 'cat', + TEST_IMG, 'cat', detach=True, stdin_open=True, environment={'FOO': '1'}) @@ -75,7 +75,7 @@ def test_create_with_links(self): self.client.start(container1_id) res1 = self.client.create_container( - BUSYBOX, 'cat', + TEST_IMG, 'cat', detach=True, stdin_open=True, environment={'FOO': '1'}) @@ -94,7 +94,7 @@ def test_create_with_links(self): link_env_prefix2 = link_alias2.upper() res2 = self.client.create_container( - BUSYBOX, 'env', host_config=self.client.create_host_config( + TEST_IMG, 'env', host_config=self.client.create_host_config( links={link_path1: link_alias1, link_path2: link_alias2}, network_mode='bridge' ) @@ -114,7 +114,7 @@ def test_create_with_links(self): def test_create_with_restart_policy(self): container = self.client.create_container( - BUSYBOX, ['sleep', '2'], + TEST_IMG, ['sleep', '2'], host_config=self.client.create_host_config( restart_policy={"Name": "always", "MaximumRetryCount": 0}, network_mode='none' @@ -133,21 +133,21 @@ def test_create_container_with_volumes_from(self): vol_names = ['foobar_vol0', 'foobar_vol1'] res0 = self.client.create_container( - BUSYBOX, 'true', name=vol_names[0] + TEST_IMG, 'true', name=vol_names[0] ) container1_id = res0['Id'] self.tmp_containers.append(container1_id) self.client.start(container1_id) res1 = self.client.create_container( - BUSYBOX, 'true', name=vol_names[1] + TEST_IMG, 'true', name=vol_names[1] ) container2_id = res1['Id'] self.tmp_containers.append(container2_id) self.client.start(container2_id) res = self.client.create_container( - BUSYBOX, 'cat', detach=True, stdin_open=True, + TEST_IMG, 'cat', detach=True, stdin_open=True, host_config=self.client.create_host_config( volumes_from=vol_names, network_mode='none' ) @@ -161,7 +161,7 @@ def test_create_container_with_volumes_from(self): def create_container_readonly_fs(self): ctnr = self.client.create_container( - BUSYBOX, ['mkdir', '/shrine'], + TEST_IMG, ['mkdir', '/shrine'], host_config=self.client.create_host_config( read_only=True, network_mode='none' ) @@ -173,7 +173,7 @@ def create_container_readonly_fs(self): assert res != 0 def create_container_with_name(self): - res = self.client.create_container(BUSYBOX, 'true', name='foobar') + res = self.client.create_container(TEST_IMG, 'true', name='foobar') assert 'Id' in res self.tmp_containers.append(res['Id']) inspect = self.client.inspect_container(res['Id']) @@ -182,7 +182,7 @@ def create_container_with_name(self): def create_container_privileged(self): res = self.client.create_container( - BUSYBOX, 'true', host_config=self.client.create_host_config( + TEST_IMG, 'true', host_config=self.client.create_host_config( privileged=True, network_mode='none' ) ) @@ -208,7 +208,7 @@ def create_container_privileged(self): def test_create_with_mac_address(self): mac_address_expected = "02:42:ac:11:00:0a" container = self.client.create_container( - BUSYBOX, ['sleep', '60'], mac_address=mac_address_expected) + TEST_IMG, ['sleep', '60'], mac_address=mac_address_expected) id = container['Id'] @@ -220,7 +220,7 @@ def test_create_with_mac_address(self): def test_group_id_ints(self): container = self.client.create_container( - BUSYBOX, 'id -G', + TEST_IMG, 'id -G', host_config=self.client.create_host_config(group_add=[1000, 1001]) ) self.tmp_containers.append(container) @@ -236,7 +236,7 @@ def test_group_id_ints(self): def test_group_id_strings(self): container = self.client.create_container( - BUSYBOX, 'id -G', host_config=self.client.create_host_config( + TEST_IMG, 'id -G', host_config=self.client.create_host_config( group_add=['1000', '1001'] ) ) @@ -259,7 +259,7 @@ def test_valid_log_driver_and_log_opt(self): ) container = self.client.create_container( - BUSYBOX, ['true'], + TEST_IMG, ['true'], host_config=self.client.create_host_config(log_config=log_config) ) self.tmp_containers.append(container['Id']) @@ -281,7 +281,7 @@ def test_invalid_log_driver_raises_exception(self): with pytest.raises(docker.errors.APIError) as excinfo: # raises an internal server error 500 container = self.client.create_container( - BUSYBOX, ['true'], host_config=self.client.create_host_config( + TEST_IMG, ['true'], host_config=self.client.create_host_config( log_config=log_config ) ) @@ -296,7 +296,7 @@ def test_valid_no_log_driver_specified(self): ) container = self.client.create_container( - BUSYBOX, ['true'], + TEST_IMG, ['true'], host_config=self.client.create_host_config(log_config=log_config) ) self.tmp_containers.append(container['Id']) @@ -315,7 +315,7 @@ def test_valid_no_config_specified(self): ) container = self.client.create_container( - BUSYBOX, ['true'], + TEST_IMG, ['true'], host_config=self.client.create_host_config(log_config=log_config) ) self.tmp_containers.append(container['Id']) @@ -329,7 +329,7 @@ def test_valid_no_config_specified(self): def test_create_with_memory_constraints_with_str(self): ctnr = self.client.create_container( - BUSYBOX, 'true', + TEST_IMG, 'true', host_config=self.client.create_host_config( memswap_limit='1G', mem_limit='700M' @@ -347,7 +347,7 @@ def test_create_with_memory_constraints_with_str(self): def test_create_with_memory_constraints_with_int(self): ctnr = self.client.create_container( - BUSYBOX, 'true', + TEST_IMG, 'true', host_config=self.client.create_host_config(mem_swappiness=40) ) assert 'Id' in ctnr @@ -361,7 +361,7 @@ def test_create_with_memory_constraints_with_int(self): def test_create_with_environment_variable_no_value(self): container = self.client.create_container( - BUSYBOX, + TEST_IMG, ['echo'], environment={'Foo': None, 'Other': 'one', 'Blank': ''}, ) @@ -378,7 +378,7 @@ def test_create_with_tmpfs(self): } container = self.client.create_container( - BUSYBOX, + TEST_IMG, ['echo'], host_config=self.client.create_host_config( tmpfs=tmpfs)) @@ -390,7 +390,7 @@ def test_create_with_tmpfs(self): @requires_api_version('1.24') def test_create_with_isolation(self): container = self.client.create_container( - BUSYBOX, ['echo'], host_config=self.client.create_host_config( + TEST_IMG, ['echo'], host_config=self.client.create_host_config( isolation='default' ) ) @@ -404,7 +404,7 @@ def test_create_with_auto_remove(self): auto_remove=True ) container = self.client.create_container( - BUSYBOX, ['echo', 'test'], host_config=host_config + TEST_IMG, ['echo', 'test'], host_config=host_config ) self.tmp_containers.append(container['Id']) config = self.client.inspect_container(container) @@ -413,7 +413,7 @@ def test_create_with_auto_remove(self): @requires_api_version('1.25') def test_create_with_stop_timeout(self): container = self.client.create_container( - BUSYBOX, ['echo', 'test'], stop_timeout=25 + TEST_IMG, ['echo', 'test'], stop_timeout=25 ) self.tmp_containers.append(container['Id']) config = self.client.inspect_container(container) @@ -426,7 +426,7 @@ def test_create_with_storage_opt(self): storage_opt={'size': '120G'} ) container = self.client.create_container( - BUSYBOX, ['echo', 'test'], host_config=host_config + TEST_IMG, ['echo', 'test'], host_config=host_config ) self.tmp_containers.append(container) config = self.client.inspect_container(container) @@ -437,7 +437,7 @@ def test_create_with_storage_opt(self): @requires_api_version('1.25') def test_create_with_init(self): ctnr = self.client.create_container( - BUSYBOX, 'true', + TEST_IMG, 'true', host_config=self.client.create_host_config( init=True ) @@ -451,7 +451,7 @@ def test_create_with_init(self): reason='CONFIG_RT_GROUP_SCHED isn\'t enabled') def test_create_with_cpu_rt_options(self): ctnr = self.client.create_container( - BUSYBOX, 'true', host_config=self.client.create_host_config( + TEST_IMG, 'true', host_config=self.client.create_host_config( cpu_rt_period=1000, cpu_rt_runtime=500 ) ) @@ -464,7 +464,7 @@ def test_create_with_cpu_rt_options(self): def test_create_with_device_cgroup_rules(self): rule = 'c 7:128 rwm' ctnr = self.client.create_container( - BUSYBOX, 'cat /sys/fs/cgroup/devices/devices.list', + TEST_IMG, 'cat /sys/fs/cgroup/devices/devices.list', host_config=self.client.create_host_config( device_cgroup_rules=[rule] ) @@ -477,7 +477,7 @@ def test_create_with_device_cgroup_rules(self): def test_create_with_uts_mode(self): container = self.client.create_container( - BUSYBOX, ['echo'], host_config=self.client.create_host_config( + TEST_IMG, ['echo'], host_config=self.client.create_host_config( uts_mode='host' ) ) @@ -501,7 +501,7 @@ def setUp(self): self.run_with_volume( False, - BUSYBOX, + TEST_IMG, ['touch', os.path.join(self.mount_dest, self.filename)], ) @@ -509,7 +509,7 @@ def test_create_with_binds_rw(self): container = self.run_with_volume( False, - BUSYBOX, + TEST_IMG, ['ls', self.mount_dest], ) logs = self.client.logs(container) @@ -523,12 +523,12 @@ def test_create_with_binds_rw(self): def test_create_with_binds_ro(self): self.run_with_volume( False, - BUSYBOX, + TEST_IMG, ['touch', os.path.join(self.mount_dest, self.filename)], ) container = self.run_with_volume( True, - BUSYBOX, + TEST_IMG, ['ls', self.mount_dest], ) logs = self.client.logs(container) @@ -547,7 +547,7 @@ def test_create_with_mounts(self): ) host_config = self.client.create_host_config(mounts=[mount]) container = self.run_container( - BUSYBOX, ['ls', self.mount_dest], + TEST_IMG, ['ls', self.mount_dest], host_config=host_config ) assert container @@ -566,7 +566,7 @@ def test_create_with_mounts_ro(self): ) host_config = self.client.create_host_config(mounts=[mount]) container = self.run_container( - BUSYBOX, ['ls', self.mount_dest], + TEST_IMG, ['ls', self.mount_dest], host_config=host_config ) assert container @@ -585,7 +585,7 @@ def test_create_with_volume_mount(self): ) host_config = self.client.create_host_config(mounts=[mount]) container = self.client.create_container( - BUSYBOX, ['true'], host_config=host_config, + TEST_IMG, ['true'], host_config=host_config, ) assert container inspect_data = self.client.inspect_container(container) @@ -631,7 +631,7 @@ class ArchiveTest(BaseAPIIntegrationTest): def test_get_file_archive_from_container(self): data = 'The Maid and the Pocket Watch of Blood' ctnr = self.client.create_container( - BUSYBOX, 'sh -c "echo {0} > /vol1/data.txt"'.format(data), + TEST_IMG, 'sh -c "echo {0} > /vol1/data.txt"'.format(data), volumes=['/vol1'] ) self.tmp_containers.append(ctnr) @@ -650,7 +650,7 @@ def test_get_file_archive_from_container(self): def test_get_file_stat_from_container(self): data = 'The Maid and the Pocket Watch of Blood' ctnr = self.client.create_container( - BUSYBOX, 'sh -c "echo -n {0} > /vol1/data.txt"'.format(data), + TEST_IMG, 'sh -c "echo -n {0} > /vol1/data.txt"'.format(data), volumes=['/vol1'] ) self.tmp_containers.append(ctnr) @@ -668,7 +668,7 @@ def test_copy_file_to_container(self): test_file.write(data) test_file.seek(0) ctnr = self.client.create_container( - BUSYBOX, + TEST_IMG, 'cat {0}'.format( os.path.join('/vol1/', os.path.basename(test_file.name)) ), @@ -690,7 +690,7 @@ def test_copy_directory_to_container(self): dirs = ['foo', 'bar'] base = helpers.make_tree(dirs, files) ctnr = self.client.create_container( - BUSYBOX, 'ls -p /vol1', volumes=['/vol1'] + TEST_IMG, 'ls -p /vol1', volumes=['/vol1'] ) self.tmp_containers.append(ctnr) with docker.utils.tar(base) as test_tar: @@ -711,7 +711,7 @@ class RenameContainerTest(BaseAPIIntegrationTest): def test_rename_container(self): version = self.client.version()['Version'] name = 'hong_meiling' - res = self.client.create_container(BUSYBOX, 'true') + res = self.client.create_container(TEST_IMG, 'true') assert 'Id' in res self.tmp_containers.append(res['Id']) self.client.rename(res, name) @@ -725,7 +725,7 @@ def test_rename_container(self): class StartContainerTest(BaseAPIIntegrationTest): def test_start_container(self): - res = self.client.create_container(BUSYBOX, 'true') + res = self.client.create_container(TEST_IMG, 'true') assert 'Id' in res self.tmp_containers.append(res['Id']) self.client.start(res['Id']) @@ -741,7 +741,7 @@ def test_start_container(self): assert inspect['State']['ExitCode'] == 0 def test_start_container_with_dict_instead_of_id(self): - res = self.client.create_container(BUSYBOX, 'true') + res = self.client.create_container(TEST_IMG, 'true') assert 'Id' in res self.tmp_containers.append(res['Id']) self.client.start(res) @@ -769,7 +769,7 @@ def test_run_shlex_commands(self): 'true && echo "Night of Nights"' ] for cmd in commands: - container = self.client.create_container(BUSYBOX, cmd) + container = self.client.create_container(TEST_IMG, cmd) id = container['Id'] self.client.start(id) self.tmp_containers.append(id) @@ -779,7 +779,7 @@ def test_run_shlex_commands(self): class WaitTest(BaseAPIIntegrationTest): def test_wait(self): - res = self.client.create_container(BUSYBOX, ['sleep', '3']) + res = self.client.create_container(TEST_IMG, ['sleep', '3']) id = res['Id'] self.tmp_containers.append(id) self.client.start(id) @@ -792,7 +792,7 @@ def test_wait(self): assert inspect['State']['ExitCode'] == exitcode def test_wait_with_dict_instead_of_id(self): - res = self.client.create_container(BUSYBOX, ['sleep', '3']) + res = self.client.create_container(TEST_IMG, ['sleep', '3']) id = res['Id'] self.tmp_containers.append(id) self.client.start(res) @@ -806,13 +806,13 @@ def test_wait_with_dict_instead_of_id(self): @requires_api_version('1.30') def test_wait_with_condition(self): - ctnr = self.client.create_container(BUSYBOX, 'true') + ctnr = self.client.create_container(TEST_IMG, 'true') self.tmp_containers.append(ctnr) with pytest.raises(requests.exceptions.ConnectionError): self.client.wait(ctnr, condition='removed', timeout=1) ctnr = self.client.create_container( - BUSYBOX, ['sleep', '3'], + TEST_IMG, ['sleep', '3'], host_config=self.client.create_host_config(auto_remove=True) ) self.tmp_containers.append(ctnr) @@ -826,7 +826,7 @@ class LogsTest(BaseAPIIntegrationTest): def test_logs(self): snippet = 'Flowering Nights (Sakuya Iyazoi)' container = self.client.create_container( - BUSYBOX, 'echo {0}'.format(snippet) + TEST_IMG, 'echo {0}'.format(snippet) ) id = container['Id'] self.tmp_containers.append(id) @@ -840,7 +840,7 @@ def test_logs_tail_option(self): snippet = '''Line1 Line2''' container = self.client.create_container( - BUSYBOX, 'echo "{0}"'.format(snippet) + TEST_IMG, 'echo "{0}"'.format(snippet) ) id = container['Id'] self.tmp_containers.append(id) @@ -853,7 +853,7 @@ def test_logs_tail_option(self): def test_logs_streaming_and_follow(self): snippet = 'Flowering Nights (Sakuya Iyazoi)' container = self.client.create_container( - BUSYBOX, 'echo {0}'.format(snippet) + TEST_IMG, 'echo {0}'.format(snippet) ) id = container['Id'] self.tmp_containers.append(id) @@ -873,7 +873,7 @@ def test_logs_streaming_and_follow(self): def test_logs_streaming_and_follow_and_cancel(self): snippet = 'Flowering Nights (Sakuya Iyazoi)' container = self.client.create_container( - BUSYBOX, 'sh -c "echo \\"{0}\\" && sleep 3"'.format(snippet) + TEST_IMG, 'sh -c "echo \\"{0}\\" && sleep 3"'.format(snippet) ) id = container['Id'] self.tmp_containers.append(id) @@ -891,7 +891,7 @@ def test_logs_streaming_and_follow_and_cancel(self): def test_logs_with_dict_instead_of_id(self): snippet = 'Flowering Nights (Sakuya Iyazoi)' container = self.client.create_container( - BUSYBOX, 'echo {0}'.format(snippet) + TEST_IMG, 'echo {0}'.format(snippet) ) id = container['Id'] self.tmp_containers.append(id) @@ -904,7 +904,7 @@ def test_logs_with_dict_instead_of_id(self): def test_logs_with_tail_0(self): snippet = 'Flowering Nights (Sakuya Iyazoi)' container = self.client.create_container( - BUSYBOX, 'echo "{0}"'.format(snippet) + TEST_IMG, 'echo "{0}"'.format(snippet) ) id = container['Id'] self.tmp_containers.append(id) @@ -918,7 +918,7 @@ def test_logs_with_tail_0(self): def test_logs_with_until(self): snippet = 'Shanghai Teahouse (Hong Meiling)' container = self.client.create_container( - BUSYBOX, 'echo "{0}"'.format(snippet) + TEST_IMG, 'echo "{0}"'.format(snippet) ) self.tmp_containers.append(container) @@ -933,7 +933,7 @@ def test_logs_with_until(self): class DiffTest(BaseAPIIntegrationTest): def test_diff(self): - container = self.client.create_container(BUSYBOX, ['touch', '/test']) + container = self.client.create_container(TEST_IMG, ['touch', '/test']) id = container['Id'] self.client.start(id) self.tmp_containers.append(id) @@ -946,7 +946,7 @@ def test_diff(self): assert test_diff[0]['Kind'] == 1 def test_diff_with_dict_instead_of_id(self): - container = self.client.create_container(BUSYBOX, ['touch', '/test']) + container = self.client.create_container(TEST_IMG, ['touch', '/test']) id = container['Id'] self.client.start(id) self.tmp_containers.append(id) @@ -961,7 +961,7 @@ def test_diff_with_dict_instead_of_id(self): class StopTest(BaseAPIIntegrationTest): def test_stop(self): - container = self.client.create_container(BUSYBOX, ['sleep', '9999']) + container = self.client.create_container(TEST_IMG, ['sleep', '9999']) id = container['Id'] self.client.start(id) self.tmp_containers.append(id) @@ -973,7 +973,7 @@ def test_stop(self): assert state['Running'] is False def test_stop_with_dict_instead_of_id(self): - container = self.client.create_container(BUSYBOX, ['sleep', '9999']) + container = self.client.create_container(TEST_IMG, ['sleep', '9999']) assert 'Id' in container id = container['Id'] self.client.start(container) @@ -988,7 +988,7 @@ def test_stop_with_dict_instead_of_id(self): class KillTest(BaseAPIIntegrationTest): def test_kill(self): - container = self.client.create_container(BUSYBOX, ['sleep', '9999']) + container = self.client.create_container(TEST_IMG, ['sleep', '9999']) id = container['Id'] self.client.start(id) self.tmp_containers.append(id) @@ -1002,7 +1002,7 @@ def test_kill(self): assert state['Running'] is False def test_kill_with_dict_instead_of_id(self): - container = self.client.create_container(BUSYBOX, ['sleep', '9999']) + container = self.client.create_container(TEST_IMG, ['sleep', '9999']) id = container['Id'] self.client.start(id) self.tmp_containers.append(id) @@ -1016,7 +1016,7 @@ def test_kill_with_dict_instead_of_id(self): assert state['Running'] is False def test_kill_with_signal(self): - id = self.client.create_container(BUSYBOX, ['sleep', '60']) + id = self.client.create_container(TEST_IMG, ['sleep', '60']) self.tmp_containers.append(id) self.client.start(id) self.client.kill( @@ -1033,7 +1033,7 @@ def test_kill_with_signal(self): assert state['Running'] is False, state def test_kill_with_signal_name(self): - id = self.client.create_container(BUSYBOX, ['sleep', '60']) + id = self.client.create_container(TEST_IMG, ['sleep', '60']) self.client.start(id) self.tmp_containers.append(id) self.client.kill(id, signal='SIGKILL') @@ -1048,7 +1048,7 @@ def test_kill_with_signal_name(self): assert state['Running'] is False, state def test_kill_with_signal_integer(self): - id = self.client.create_container(BUSYBOX, ['sleep', '60']) + id = self.client.create_container(TEST_IMG, ['sleep', '60']) self.client.start(id) self.tmp_containers.append(id) self.client.kill(id, signal=9) @@ -1077,7 +1077,7 @@ def test_port(self): ] container = self.client.create_container( - BUSYBOX, ['sleep', '60'], ports=ports, + TEST_IMG, ['sleep', '60'], ports=ports, host_config=self.client.create_host_config( port_bindings=port_bindings, network_mode='bridge' ) @@ -1104,7 +1104,7 @@ def test_port(self): class ContainerTopTest(BaseAPIIntegrationTest): def test_top(self): container = self.client.create_container( - BUSYBOX, ['sleep', '60'] + TEST_IMG, ['sleep', '60'] ) self.tmp_containers.append(container) @@ -1124,7 +1124,7 @@ def test_top(self): ) def test_top_with_psargs(self): container = self.client.create_container( - BUSYBOX, ['sleep', '60']) + TEST_IMG, ['sleep', '60']) self.tmp_containers.append(container) @@ -1140,7 +1140,7 @@ def test_top_with_psargs(self): class RestartContainerTest(BaseAPIIntegrationTest): def test_restart(self): - container = self.client.create_container(BUSYBOX, ['sleep', '9999']) + container = self.client.create_container(TEST_IMG, ['sleep', '9999']) id = container['Id'] self.client.start(id) self.tmp_containers.append(id) @@ -1159,7 +1159,7 @@ def test_restart(self): self.client.kill(id) def test_restart_with_low_timeout(self): - container = self.client.create_container(BUSYBOX, ['sleep', '9999']) + container = self.client.create_container(TEST_IMG, ['sleep', '9999']) self.client.start(container) self.client.timeout = 3 self.client.restart(container, timeout=1) @@ -1168,7 +1168,7 @@ def test_restart_with_low_timeout(self): self.client.kill(container) def test_restart_with_dict_instead_of_id(self): - container = self.client.create_container(BUSYBOX, ['sleep', '9999']) + container = self.client.create_container(TEST_IMG, ['sleep', '9999']) assert 'Id' in container id = container['Id'] self.client.start(container) @@ -1190,7 +1190,7 @@ def test_restart_with_dict_instead_of_id(self): class RemoveContainerTest(BaseAPIIntegrationTest): def test_remove(self): - container = self.client.create_container(BUSYBOX, ['true']) + container = self.client.create_container(TEST_IMG, ['true']) id = container['Id'] self.client.start(id) self.client.wait(id) @@ -1200,7 +1200,7 @@ def test_remove(self): assert len(res) == 0 def test_remove_with_dict_instead_of_id(self): - container = self.client.create_container(BUSYBOX, ['true']) + container = self.client.create_container(TEST_IMG, ['true']) id = container['Id'] self.client.start(id) self.client.wait(id) @@ -1212,7 +1212,7 @@ def test_remove_with_dict_instead_of_id(self): class AttachContainerTest(BaseAPIIntegrationTest): def test_run_container_streaming(self): - container = self.client.create_container(BUSYBOX, '/bin/sh', + container = self.client.create_container(TEST_IMG, '/bin/sh', detach=True, stdin_open=True) id = container['Id'] self.tmp_containers.append(id) @@ -1224,7 +1224,7 @@ def test_run_container_reading_socket(self): line = 'hi there and stuff and things, words!' # `echo` appends CRLF, `printf` doesn't command = "printf '{0}'".format(line) - container = self.client.create_container(BUSYBOX, command, + container = self.client.create_container(TEST_IMG, command, detach=True, tty=False) self.tmp_containers.append(container) @@ -1242,7 +1242,7 @@ def test_run_container_reading_socket(self): def test_attach_no_stream(self): container = self.client.create_container( - BUSYBOX, 'echo hello' + TEST_IMG, 'echo hello' ) self.tmp_containers.append(container) self.client.start(container) @@ -1257,7 +1257,7 @@ def test_attach_no_stream(self): reason='Flaky test on TLS') def test_attach_stream_and_cancel(self): container = self.client.create_container( - BUSYBOX, 'sh -c "sleep 2 && echo hello && sleep 60"', + TEST_IMG, 'sh -c "sleep 2 && echo hello && sleep 60"', tty=True ) self.tmp_containers.append(container) @@ -1275,7 +1275,7 @@ def test_attach_stream_and_cancel(self): def test_detach_with_default(self): container = self.client.create_container( - BUSYBOX, 'cat', + TEST_IMG, 'cat', detach=True, stdin_open=True, tty=True ) self.tmp_containers.append(container) @@ -1294,7 +1294,7 @@ def test_detach_with_config_file(self): self.client._general_configs['detachKeys'] = 'ctrl-p' container = self.client.create_container( - BUSYBOX, 'cat', + TEST_IMG, 'cat', detach=True, stdin_open=True, tty=True ) self.tmp_containers.append(container) @@ -1311,7 +1311,7 @@ def test_detach_with_arg(self): self.client._general_configs['detachKeys'] = 'ctrl-p' container = self.client.create_container( - BUSYBOX, 'cat', + TEST_IMG, 'cat', detach=True, stdin_open=True, tty=True ) self.tmp_containers.append(container) @@ -1327,7 +1327,7 @@ def test_detach_with_arg(self): class PauseTest(BaseAPIIntegrationTest): def test_pause_unpause(self): - container = self.client.create_container(BUSYBOX, ['sleep', '9999']) + container = self.client.create_container(TEST_IMG, ['sleep', '9999']) id = container['Id'] self.tmp_containers.append(id) self.client.start(container) @@ -1358,9 +1358,9 @@ class PruneTest(BaseAPIIntegrationTest): @requires_api_version('1.25') def test_prune_containers(self): container1 = self.client.create_container( - BUSYBOX, ['sh', '-c', 'echo hello > /data.txt'] + TEST_IMG, ['sh', '-c', 'echo hello > /data.txt'] ) - container2 = self.client.create_container(BUSYBOX, ['sleep', '9999']) + container2 = self.client.create_container(TEST_IMG, ['sleep', '9999']) self.client.start(container1) self.client.start(container2) self.client.wait(container1) @@ -1373,7 +1373,7 @@ def test_prune_containers(self): class GetContainerStatsTest(BaseAPIIntegrationTest): def test_get_container_stats_no_stream(self): container = self.client.create_container( - BUSYBOX, ['sleep', '60'], + TEST_IMG, ['sleep', '60'], ) self.tmp_containers.append(container) self.client.start(container) @@ -1387,7 +1387,7 @@ def test_get_container_stats_no_stream(self): def test_get_container_stats_stream(self): container = self.client.create_container( - BUSYBOX, ['sleep', '60'], + TEST_IMG, ['sleep', '60'], ) self.tmp_containers.append(container) self.client.start(container) @@ -1405,7 +1405,7 @@ def test_update_container(self): old_mem_limit = 400 * 1024 * 1024 new_mem_limit = 300 * 1024 * 1024 container = self.client.create_container( - BUSYBOX, 'top', host_config=self.client.create_host_config( + TEST_IMG, 'top', host_config=self.client.create_host_config( mem_limit=old_mem_limit ) ) @@ -1426,7 +1426,7 @@ def test_restart_policy_update(self): 'Name': 'on-failure' } container = self.client.create_container( - BUSYBOX, ['sleep', '60'], + TEST_IMG, ['sleep', '60'], host_config=self.client.create_host_config( restart_policy=old_restart_policy ) @@ -1450,7 +1450,7 @@ class ContainerCPUTest(BaseAPIIntegrationTest): def test_container_cpu_shares(self): cpu_shares = 512 container = self.client.create_container( - BUSYBOX, 'ls', host_config=self.client.create_host_config( + TEST_IMG, 'ls', host_config=self.client.create_host_config( cpu_shares=cpu_shares ) ) @@ -1462,7 +1462,7 @@ def test_container_cpu_shares(self): def test_container_cpuset(self): cpuset_cpus = "0,1" container = self.client.create_container( - BUSYBOX, 'ls', host_config=self.client.create_host_config( + TEST_IMG, 'ls', host_config=self.client.create_host_config( cpuset_cpus=cpuset_cpus ) ) @@ -1474,7 +1474,7 @@ def test_container_cpuset(self): @requires_api_version('1.25') def test_create_with_runtime(self): container = self.client.create_container( - BUSYBOX, ['echo', 'test'], runtime='runc' + TEST_IMG, ['echo', 'test'], runtime='runc' ) self.tmp_containers.append(container['Id']) config = self.client.inspect_container(container) @@ -1485,7 +1485,7 @@ class LinkTest(BaseAPIIntegrationTest): def test_remove_link(self): # Create containers container1 = self.client.create_container( - BUSYBOX, 'cat', detach=True, stdin_open=True + TEST_IMG, 'cat', detach=True, stdin_open=True ) container1_id = container1['Id'] self.tmp_containers.append(container1_id) @@ -1497,7 +1497,7 @@ def test_remove_link(self): link_alias = 'mylink' container2 = self.client.create_container( - BUSYBOX, 'cat', host_config=self.client.create_host_config( + TEST_IMG, 'cat', host_config=self.client.create_host_config( links={link_path: link_alias} ) ) diff --git a/tests/integration/api_exec_test.py b/tests/integration/api_exec_test.py index 53b7e22fea..554e8629e5 100644 --- a/tests/integration/api_exec_test.py +++ b/tests/integration/api_exec_test.py @@ -2,7 +2,7 @@ from ..helpers import ctrl_with from ..helpers import requires_api_version from .base import BaseAPIIntegrationTest -from .base import BUSYBOX +from .base import TEST_IMG from docker.utils.proxy import ProxyConfig from docker.utils.socket import next_frame_header from docker.utils.socket import read_exactly @@ -16,7 +16,7 @@ def test_execute_command_with_proxy_env(self): ) container = self.client.create_container( - BUSYBOX, 'cat', detach=True, stdin_open=True, + TEST_IMG, 'cat', detach=True, stdin_open=True, ) self.client.start(container) self.tmp_containers.append(container) @@ -48,7 +48,7 @@ def test_execute_command_with_proxy_env(self): assert item in output def test_execute_command(self): - container = self.client.create_container(BUSYBOX, 'cat', + container = self.client.create_container(TEST_IMG, 'cat', detach=True, stdin_open=True) id = container['Id'] self.client.start(id) @@ -61,7 +61,7 @@ def test_execute_command(self): assert exec_log == b'hello\n' def test_exec_command_string(self): - container = self.client.create_container(BUSYBOX, 'cat', + container = self.client.create_container(TEST_IMG, 'cat', detach=True, stdin_open=True) id = container['Id'] self.client.start(id) @@ -74,7 +74,7 @@ def test_exec_command_string(self): assert exec_log == b'hello world\n' def test_exec_command_as_user(self): - container = self.client.create_container(BUSYBOX, 'cat', + container = self.client.create_container(TEST_IMG, 'cat', detach=True, stdin_open=True) id = container['Id'] self.client.start(id) @@ -87,7 +87,7 @@ def test_exec_command_as_user(self): assert exec_log == b'postgres\n' def test_exec_command_as_root(self): - container = self.client.create_container(BUSYBOX, 'cat', + container = self.client.create_container(TEST_IMG, 'cat', detach=True, stdin_open=True) id = container['Id'] self.client.start(id) @@ -100,7 +100,7 @@ def test_exec_command_as_root(self): assert exec_log == b'root\n' def test_exec_command_streaming(self): - container = self.client.create_container(BUSYBOX, 'cat', + container = self.client.create_container(TEST_IMG, 'cat', detach=True, stdin_open=True) id = container['Id'] self.tmp_containers.append(id) @@ -115,7 +115,7 @@ def test_exec_command_streaming(self): assert res == b'hello\nworld\n' def test_exec_start_socket(self): - container = self.client.create_container(BUSYBOX, 'cat', + container = self.client.create_container(TEST_IMG, 'cat', detach=True, stdin_open=True) container_id = container['Id'] self.client.start(container_id) @@ -137,7 +137,7 @@ def test_exec_start_socket(self): assert data.decode('utf-8') == line def test_exec_start_detached(self): - container = self.client.create_container(BUSYBOX, 'cat', + container = self.client.create_container(TEST_IMG, 'cat', detach=True, stdin_open=True) container_id = container['Id'] self.client.start(container_id) @@ -152,7 +152,7 @@ def test_exec_start_detached(self): assert response == "" def test_exec_inspect(self): - container = self.client.create_container(BUSYBOX, 'cat', + container = self.client.create_container(TEST_IMG, 'cat', detach=True, stdin_open=True) id = container['Id'] self.client.start(id) @@ -167,7 +167,7 @@ def test_exec_inspect(self): @requires_api_version('1.25') def test_exec_command_with_env(self): - container = self.client.create_container(BUSYBOX, 'cat', + container = self.client.create_container(TEST_IMG, 'cat', detach=True, stdin_open=True) id = container['Id'] self.client.start(id) @@ -182,7 +182,7 @@ def test_exec_command_with_env(self): @requires_api_version('1.35') def test_exec_command_with_workdir(self): container = self.client.create_container( - BUSYBOX, 'cat', detach=True, stdin_open=True + TEST_IMG, 'cat', detach=True, stdin_open=True ) self.tmp_containers.append(container) self.client.start(container) @@ -193,7 +193,7 @@ def test_exec_command_with_workdir(self): def test_detach_with_default(self): container = self.client.create_container( - BUSYBOX, 'cat', detach=True, stdin_open=True + TEST_IMG, 'cat', detach=True, stdin_open=True ) id = container['Id'] self.client.start(id) @@ -212,7 +212,7 @@ def test_detach_with_default(self): def test_detach_with_config_file(self): self.client._general_configs['detachKeys'] = 'ctrl-p' container = self.client.create_container( - BUSYBOX, 'cat', detach=True, stdin_open=True + TEST_IMG, 'cat', detach=True, stdin_open=True ) id = container['Id'] self.client.start(id) @@ -241,7 +241,7 @@ class ExecDemuxTest(BaseAPIIntegrationTest): def setUp(self): super(ExecDemuxTest, self).setUp() self.container = self.client.create_container( - BUSYBOX, 'cat', detach=True, stdin_open=True + TEST_IMG, 'cat', detach=True, stdin_open=True ) self.client.start(self.container) self.tmp_containers.append(self.container) diff --git a/tests/integration/api_healthcheck_test.py b/tests/integration/api_healthcheck_test.py index 5dbac3769f..c54583b0be 100644 --- a/tests/integration/api_healthcheck_test.py +++ b/tests/integration/api_healthcheck_test.py @@ -1,4 +1,4 @@ -from .base import BaseAPIIntegrationTest, BUSYBOX +from .base import BaseAPIIntegrationTest, TEST_IMG from .. import helpers SECOND = 1000000000 @@ -16,7 +16,7 @@ class HealthcheckTest(BaseAPIIntegrationTest): @helpers.requires_api_version('1.24') def test_healthcheck_shell_command(self): container = self.client.create_container( - BUSYBOX, 'top', healthcheck=dict(test='echo "hello world"')) + TEST_IMG, 'top', healthcheck=dict(test='echo "hello world"')) self.tmp_containers.append(container) res = self.client.inspect_container(container) @@ -27,7 +27,7 @@ def test_healthcheck_shell_command(self): @helpers.requires_api_version('1.24') def test_healthcheck_passes(self): container = self.client.create_container( - BUSYBOX, 'top', healthcheck=dict( + TEST_IMG, 'top', healthcheck=dict( test="true", interval=1 * SECOND, timeout=1 * SECOND, @@ -40,7 +40,7 @@ def test_healthcheck_passes(self): @helpers.requires_api_version('1.24') def test_healthcheck_fails(self): container = self.client.create_container( - BUSYBOX, 'top', healthcheck=dict( + TEST_IMG, 'top', healthcheck=dict( test="false", interval=1 * SECOND, timeout=1 * SECOND, @@ -53,7 +53,7 @@ def test_healthcheck_fails(self): @helpers.requires_api_version('1.29') def test_healthcheck_start_period(self): container = self.client.create_container( - BUSYBOX, 'top', healthcheck=dict( + TEST_IMG, 'top', healthcheck=dict( test="echo 'x' >> /counter.txt && " "test `cat /counter.txt | wc -l` -ge 3", interval=1 * SECOND, diff --git a/tests/integration/api_image_test.py b/tests/integration/api_image_test.py index 56a7692489..2bc96abf46 100644 --- a/tests/integration/api_image_test.py +++ b/tests/integration/api_image_test.py @@ -15,7 +15,7 @@ import docker from ..helpers import requires_api_version, requires_experimental -from .base import BaseAPIIntegrationTest, BUSYBOX +from .base import BaseAPIIntegrationTest, TEST_IMG class ListImagesTest(BaseAPIIntegrationTest): @@ -77,7 +77,7 @@ def test_pull_invalid_platform(self): class CommitTest(BaseAPIIntegrationTest): def test_commit(self): - container = self.client.create_container(BUSYBOX, ['touch', '/test']) + container = self.client.create_container(TEST_IMG, ['touch', '/test']) id = container['Id'] self.client.start(id) self.tmp_containers.append(id) @@ -90,13 +90,13 @@ def test_commit(self): assert img['Container'].startswith(id) assert 'ContainerConfig' in img assert 'Image' in img['ContainerConfig'] - assert BUSYBOX == img['ContainerConfig']['Image'] - busybox_id = self.client.inspect_image(BUSYBOX)['Id'] + assert TEST_IMG == img['ContainerConfig']['Image'] + busybox_id = self.client.inspect_image(TEST_IMG)['Id'] assert 'Parent' in img assert img['Parent'] == busybox_id def test_commit_with_changes(self): - cid = self.client.create_container(BUSYBOX, ['touch', '/test']) + cid = self.client.create_container(TEST_IMG, ['touch', '/test']) self.tmp_containers.append(cid) self.client.start(cid) img_id = self.client.commit( @@ -112,7 +112,7 @@ def test_commit_with_changes(self): class RemoveImageTest(BaseAPIIntegrationTest): def test_remove(self): - container = self.client.create_container(BUSYBOX, ['touch', '/test']) + container = self.client.create_container(TEST_IMG, ['touch', '/test']) id = container['Id'] self.client.start(id) self.tmp_containers.append(id) @@ -319,7 +319,7 @@ def test_prune_images(self): pass # Ensure busybox does not get pruned - ctnr = self.client.create_container(BUSYBOX, ['sleep', '9999']) + ctnr = self.client.create_container(TEST_IMG, ['sleep', '9999']) self.tmp_containers.append(ctnr) self.client.pull('hello-world', tag='latest') @@ -343,7 +343,7 @@ class SaveLoadImagesTest(BaseAPIIntegrationTest): @requires_api_version('1.23') def test_get_image_load_image(self): with tempfile.TemporaryFile() as f: - stream = self.client.get_image(BUSYBOX) + stream = self.client.get_image(TEST_IMG) for chunk in stream: f.write(chunk) @@ -351,7 +351,7 @@ def test_get_image_load_image(self): result = self.client.load_image(f.read()) success = False - result_line = 'Loaded image: {}\n'.format(BUSYBOX) + result_line = 'Loaded image: {}\n'.format(TEST_IMG) for data in result: print(data) if 'stream' in data: diff --git a/tests/integration/api_network_test.py b/tests/integration/api_network_test.py index db37cbd974..0f26827b17 100644 --- a/tests/integration/api_network_test.py +++ b/tests/integration/api_network_test.py @@ -3,7 +3,7 @@ import pytest from ..helpers import random_name, requires_api_version -from .base import BaseAPIIntegrationTest, BUSYBOX +from .base import BaseAPIIntegrationTest, TEST_IMG class TestNetworks(BaseAPIIntegrationTest): @@ -92,7 +92,7 @@ def test_remove_network(self): def test_connect_and_disconnect_container(self): net_name, net_id = self.create_network() - container = self.client.create_container(BUSYBOX, 'top') + container = self.client.create_container(TEST_IMG, 'top') self.tmp_containers.append(container) self.client.start(container) @@ -119,7 +119,7 @@ def test_connect_and_disconnect_container(self): def test_connect_and_force_disconnect_container(self): net_name, net_id = self.create_network() - container = self.client.create_container(BUSYBOX, 'top') + container = self.client.create_container(TEST_IMG, 'top') self.tmp_containers.append(container) self.client.start(container) @@ -144,7 +144,7 @@ def test_connect_and_force_disconnect_container(self): def test_connect_with_aliases(self): net_name, net_id = self.create_network() - container = self.client.create_container(BUSYBOX, 'top') + container = self.client.create_container(TEST_IMG, 'top') self.tmp_containers.append(container) self.client.start(container) @@ -161,7 +161,7 @@ def test_connect_on_container_create(self): net_name, net_id = self.create_network() container = self.client.create_container( - image=BUSYBOX, + image=TEST_IMG, command='top', host_config=self.client.create_host_config(network_mode=net_name), ) @@ -181,7 +181,7 @@ def test_create_with_aliases(self): net_name, net_id = self.create_network() container = self.client.create_container( - image=BUSYBOX, + image=TEST_IMG, command='top', host_config=self.client.create_host_config( network_mode=net_name, @@ -211,7 +211,7 @@ def test_create_with_ipv4_address(self): ), ) container = self.client.create_container( - image=BUSYBOX, command='top', + image=TEST_IMG, command='top', host_config=self.client.create_host_config(network_mode=net_name), networking_config=self.client.create_networking_config({ net_name: self.client.create_endpoint_config( @@ -237,7 +237,7 @@ def test_create_with_ipv6_address(self): ), ) container = self.client.create_container( - image=BUSYBOX, command='top', + image=TEST_IMG, command='top', host_config=self.client.create_host_config(network_mode=net_name), networking_config=self.client.create_networking_config({ net_name: self.client.create_endpoint_config( @@ -257,7 +257,7 @@ def test_create_with_ipv6_address(self): @requires_api_version('1.24') def test_create_with_linklocal_ips(self): container = self.client.create_container( - BUSYBOX, 'top', + TEST_IMG, 'top', networking_config=self.client.create_networking_config( { 'bridge': self.client.create_endpoint_config( diff --git a/tests/integration/api_service_test.py b/tests/integration/api_service_test.py index 71e0869e9f..c170a0a88f 100644 --- a/tests/integration/api_service_test.py +++ b/tests/integration/api_service_test.py @@ -10,7 +10,7 @@ from ..helpers import ( force_leave_swarm, requires_api_version, requires_experimental ) -from .base import BaseAPIIntegrationTest, BUSYBOX +from .base import BaseAPIIntegrationTest, TEST_IMG class ServiceTest(BaseAPIIntegrationTest): @@ -60,7 +60,7 @@ def create_simple_service(self, name=None, labels=None): name = self.get_service_name() container_spec = docker.types.ContainerSpec( - BUSYBOX, ['echo', 'hello'] + TEST_IMG, ['echo', 'hello'] ) task_tmpl = docker.types.TaskTemplate(container_spec) return name, self.client.create_service( @@ -156,7 +156,7 @@ def test_service_logs(self): def test_create_service_custom_log_driver(self): container_spec = docker.types.ContainerSpec( - BUSYBOX, ['echo', 'hello'] + TEST_IMG, ['echo', 'hello'] ) log_cfg = docker.types.DriverConfig('none') task_tmpl = docker.types.TaskTemplate( @@ -174,7 +174,7 @@ def test_create_service_custom_log_driver(self): def test_create_service_with_volume_mount(self): vol_name = self.get_service_name() container_spec = docker.types.ContainerSpec( - BUSYBOX, ['ls'], + TEST_IMG, ['ls'], mounts=[ docker.types.Mount(target='/test', source=vol_name) ] @@ -194,7 +194,7 @@ def test_create_service_with_volume_mount(self): assert mount['Type'] == 'volume' def test_create_service_with_resources_constraints(self): - container_spec = docker.types.ContainerSpec(BUSYBOX, ['true']) + container_spec = docker.types.ContainerSpec(TEST_IMG, ['true']) resources = docker.types.Resources( cpu_limit=4000000, mem_limit=3 * 1024 * 1024 * 1024, cpu_reservation=3500000, mem_reservation=2 * 1024 * 1024 * 1024 @@ -214,7 +214,7 @@ def test_create_service_with_resources_constraints(self): ] def _create_service_with_generic_resources(self, generic_resources): - container_spec = docker.types.ContainerSpec(BUSYBOX, ['true']) + container_spec = docker.types.ContainerSpec(TEST_IMG, ['true']) resources = docker.types.Resources( generic_resources=generic_resources @@ -265,7 +265,7 @@ def test_create_service_with_invalid_generic_resources(self): self._create_service_with_generic_resources(test_input) def test_create_service_with_update_config(self): - container_spec = docker.types.ContainerSpec(BUSYBOX, ['true']) + container_spec = docker.types.ContainerSpec(TEST_IMG, ['true']) task_tmpl = docker.types.TaskTemplate(container_spec) update_config = docker.types.UpdateConfig( parallelism=10, delay=5, failure_action='pause' @@ -283,7 +283,7 @@ def test_create_service_with_update_config(self): @requires_api_version('1.28') def test_create_service_with_failure_action_rollback(self): - container_spec = docker.types.ContainerSpec(BUSYBOX, ['true']) + container_spec = docker.types.ContainerSpec(TEST_IMG, ['true']) task_tmpl = docker.types.TaskTemplate(container_spec) update_config = docker.types.UpdateConfig(failure_action='rollback') name = self.get_service_name() @@ -314,7 +314,7 @@ def test_create_service_with_update_config_monitor(self): @requires_api_version('1.28') def test_create_service_with_rollback_config(self): - container_spec = docker.types.ContainerSpec(BUSYBOX, ['true']) + container_spec = docker.types.ContainerSpec(TEST_IMG, ['true']) task_tmpl = docker.types.TaskTemplate(container_spec) rollback_cfg = docker.types.RollbackConfig( parallelism=10, delay=5, failure_action='pause', @@ -334,7 +334,7 @@ def test_create_service_with_rollback_config(self): assert rollback_cfg['MaxFailureRatio'] == rc['MaxFailureRatio'] def test_create_service_with_restart_policy(self): - container_spec = docker.types.ContainerSpec(BUSYBOX, ['true']) + container_spec = docker.types.ContainerSpec(TEST_IMG, ['true']) policy = docker.types.RestartPolicy( docker.types.RestartPolicy.condition_types.ANY, delay=5, max_attempts=5 @@ -357,7 +357,7 @@ def test_create_service_with_custom_networks(self): 'dockerpytest_2', driver='overlay', ipam={'Driver': 'default'} ) self.tmp_networks.append(net2['Id']) - container_spec = docker.types.ContainerSpec(BUSYBOX, ['true']) + container_spec = docker.types.ContainerSpec(TEST_IMG, ['true']) task_tmpl = docker.types.TaskTemplate(container_spec) name = self.get_service_name() svc_id = self.client.create_service( @@ -373,7 +373,7 @@ def test_create_service_with_custom_networks(self): def test_create_service_with_placement(self): node_id = self.client.nodes()[0]['ID'] - container_spec = docker.types.ContainerSpec(BUSYBOX, ['true']) + container_spec = docker.types.ContainerSpec(TEST_IMG, ['true']) task_tmpl = docker.types.TaskTemplate( container_spec, placement=['node.id=={}'.format(node_id)] ) @@ -386,7 +386,7 @@ def test_create_service_with_placement(self): def test_create_service_with_placement_object(self): node_id = self.client.nodes()[0]['ID'] - container_spec = docker.types.ContainerSpec(BUSYBOX, ['true']) + container_spec = docker.types.ContainerSpec(TEST_IMG, ['true']) placemt = docker.types.Placement( constraints=['node.id=={}'.format(node_id)] ) @@ -401,7 +401,7 @@ def test_create_service_with_placement_object(self): @requires_api_version('1.30') def test_create_service_with_placement_platform(self): - container_spec = docker.types.ContainerSpec(BUSYBOX, ['true']) + container_spec = docker.types.ContainerSpec(TEST_IMG, ['true']) placemt = docker.types.Placement(platforms=[('x86_64', 'linux')]) task_tmpl = docker.types.TaskTemplate( container_spec, placement=placemt @@ -414,7 +414,7 @@ def test_create_service_with_placement_platform(self): @requires_api_version('1.27') def test_create_service_with_placement_preferences(self): - container_spec = docker.types.ContainerSpec(BUSYBOX, ['true']) + container_spec = docker.types.ContainerSpec(TEST_IMG, ['true']) placemt = docker.types.Placement(preferences=[ {'Spread': {'SpreadDescriptor': 'com.dockerpy.test'}} ]) @@ -429,7 +429,7 @@ def test_create_service_with_placement_preferences(self): @requires_api_version('1.27') def test_create_service_with_placement_preferences_tuple(self): - container_spec = docker.types.ContainerSpec(BUSYBOX, ['true']) + container_spec = docker.types.ContainerSpec(TEST_IMG, ['true']) placemt = docker.types.Placement(preferences=( ('spread', 'com.dockerpy.test'), )) @@ -443,7 +443,7 @@ def test_create_service_with_placement_preferences_tuple(self): assert svc_info['Spec']['TaskTemplate']['Placement'] == placemt def test_create_service_with_endpoint_spec(self): - container_spec = docker.types.ContainerSpec(BUSYBOX, ['true']) + container_spec = docker.types.ContainerSpec(TEST_IMG, ['true']) task_tmpl = docker.types.TaskTemplate(container_spec) name = self.get_service_name() endpoint_spec = docker.types.EndpointSpec(ports={ @@ -473,7 +473,7 @@ def test_create_service_with_endpoint_spec(self): @requires_api_version('1.32') def test_create_service_with_endpoint_spec_host_publish_mode(self): - container_spec = docker.types.ContainerSpec(BUSYBOX, ['true']) + container_spec = docker.types.ContainerSpec(TEST_IMG, ['true']) task_tmpl = docker.types.TaskTemplate(container_spec) name = self.get_service_name() endpoint_spec = docker.types.EndpointSpec(ports={ @@ -493,7 +493,7 @@ def test_create_service_with_endpoint_spec_host_publish_mode(self): def test_create_service_with_env(self): container_spec = docker.types.ContainerSpec( - BUSYBOX, ['true'], env={'DOCKER_PY_TEST': 1} + TEST_IMG, ['true'], env={'DOCKER_PY_TEST': 1} ) task_tmpl = docker.types.TaskTemplate( container_spec, @@ -509,7 +509,7 @@ def test_create_service_with_env(self): @requires_api_version('1.29') def test_create_service_with_update_order(self): - container_spec = docker.types.ContainerSpec(BUSYBOX, ['true']) + container_spec = docker.types.ContainerSpec(TEST_IMG, ['true']) task_tmpl = docker.types.TaskTemplate(container_spec) update_config = docker.types.UpdateConfig( parallelism=10, delay=5, order='start-first' @@ -528,7 +528,7 @@ def test_create_service_with_update_order(self): @requires_api_version('1.25') def test_create_service_with_tty(self): container_spec = docker.types.ContainerSpec( - BUSYBOX, ['true'], tty=True + TEST_IMG, ['true'], tty=True ) task_tmpl = docker.types.TaskTemplate( container_spec, @@ -545,7 +545,7 @@ def test_create_service_with_tty(self): @requires_api_version('1.25') def test_create_service_with_tty_dict(self): container_spec = { - 'Image': BUSYBOX, + 'Image': TEST_IMG, 'Command': ['true'], 'TTY': True } @@ -561,7 +561,7 @@ def test_create_service_with_tty_dict(self): def test_create_service_global_mode(self): container_spec = docker.types.ContainerSpec( - BUSYBOX, ['echo', 'hello'] + TEST_IMG, ['echo', 'hello'] ) task_tmpl = docker.types.TaskTemplate(container_spec) name = self.get_service_name() @@ -574,7 +574,7 @@ def test_create_service_global_mode(self): def test_create_service_replicated_mode(self): container_spec = docker.types.ContainerSpec( - BUSYBOX, ['echo', 'hello'] + TEST_IMG, ['echo', 'hello'] ) task_tmpl = docker.types.TaskTemplate(container_spec) name = self.get_service_name() @@ -767,7 +767,7 @@ def test_create_service_with_dns_config(self): search=['local'], options=['debug'] ) container_spec = docker.types.ContainerSpec( - BUSYBOX, ['sleep', '999'], dns_config=dns_config + TEST_IMG, ['sleep', '999'], dns_config=dns_config ) task_tmpl = docker.types.TaskTemplate(container_spec) name = self.get_service_name() @@ -787,7 +787,7 @@ def test_create_service_with_healthcheck(self): start_period=3 * second, interval=int(second / 2), ) container_spec = docker.types.ContainerSpec( - BUSYBOX, ['sleep', '999'], healthcheck=hc + TEST_IMG, ['sleep', '999'], healthcheck=hc ) task_tmpl = docker.types.TaskTemplate(container_spec) name = self.get_service_name() @@ -804,7 +804,7 @@ def test_create_service_with_healthcheck(self): @requires_api_version('1.28') def test_create_service_with_readonly(self): container_spec = docker.types.ContainerSpec( - BUSYBOX, ['sleep', '999'], read_only=True + TEST_IMG, ['sleep', '999'], read_only=True ) task_tmpl = docker.types.TaskTemplate(container_spec) name = self.get_service_name() @@ -818,7 +818,7 @@ def test_create_service_with_readonly(self): @requires_api_version('1.28') def test_create_service_with_stop_signal(self): container_spec = docker.types.ContainerSpec( - BUSYBOX, ['sleep', '999'], stop_signal='SIGINT' + TEST_IMG, ['sleep', '999'], stop_signal='SIGINT' ) task_tmpl = docker.types.TaskTemplate(container_spec) name = self.get_service_name() @@ -836,7 +836,7 @@ def test_create_service_with_stop_signal(self): def test_create_service_with_privileges(self): priv = docker.types.Privileges(selinux_disable=True) container_spec = docker.types.ContainerSpec( - BUSYBOX, ['sleep', '999'], privileges=priv + TEST_IMG, ['sleep', '999'], privileges=priv ) task_tmpl = docker.types.TaskTemplate(container_spec) name = self.get_service_name() @@ -992,7 +992,7 @@ def test_update_service_with_defaults_container_labels(self): assert labels['container.label'] == 'SampleLabel' def test_update_service_with_defaults_update_config(self): - container_spec = docker.types.ContainerSpec(BUSYBOX, ['true']) + container_spec = docker.types.ContainerSpec(TEST_IMG, ['true']) task_tmpl = docker.types.TaskTemplate(container_spec) update_config = docker.types.UpdateConfig( parallelism=10, delay=5, failure_action='pause' @@ -1031,7 +1031,7 @@ def test_update_service_with_defaults_networks(self): 'dockerpytest_2', driver='overlay', ipam={'Driver': 'default'} ) self.tmp_networks.append(net2['Id']) - container_spec = docker.types.ContainerSpec(BUSYBOX, ['true']) + container_spec = docker.types.ContainerSpec(TEST_IMG, ['true']) task_tmpl = docker.types.TaskTemplate(container_spec) name = self.get_service_name() svc_id = self.client.create_service( @@ -1070,7 +1070,7 @@ def test_update_service_with_defaults_networks(self): ] def test_update_service_with_defaults_endpoint_spec(self): - container_spec = docker.types.ContainerSpec(BUSYBOX, ['true']) + container_spec = docker.types.ContainerSpec(TEST_IMG, ['true']) task_tmpl = docker.types.TaskTemplate(container_spec) name = self.get_service_name() endpoint_spec = docker.types.EndpointSpec(ports={ @@ -1134,7 +1134,7 @@ def test_update_service_remove_healthcheck(self): start_period=3 * second, interval=int(second / 2), ) container_spec = docker.types.ContainerSpec( - BUSYBOX, ['sleep', '999'], healthcheck=hc + TEST_IMG, ['sleep', '999'], healthcheck=hc ) task_tmpl = docker.types.TaskTemplate(container_spec) name = self.get_service_name() @@ -1149,7 +1149,7 @@ def test_update_service_remove_healthcheck(self): ) container_spec = docker.types.ContainerSpec( - BUSYBOX, ['sleep', '999'], healthcheck={} + TEST_IMG, ['sleep', '999'], healthcheck={} ) task_tmpl = docker.types.TaskTemplate(container_spec) diff --git a/tests/integration/base.py b/tests/integration/base.py index 0ebf5b9911..a7613f6917 100644 --- a/tests/integration/base.py +++ b/tests/integration/base.py @@ -6,7 +6,7 @@ from .. import helpers from docker.utils import kwargs_from_env -BUSYBOX = 'alpine:3.9.3' # FIXME: this should probably be renamed +TEST_IMG = 'alpine:3.10' TEST_API_VERSION = os.environ.get('DOCKER_TEST_API_VERSION') @@ -108,7 +108,7 @@ def run_container(self, *args, **kwargs): return container - def create_and_start(self, image=BUSYBOX, command='top', **kwargs): + def create_and_start(self, image=TEST_IMG, command='top', **kwargs): container = self.client.create_container( image=image, command=command, **kwargs) self.tmp_containers.append(container) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 4e8d26831d..ec48835dcd 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -7,7 +7,7 @@ from docker.utils import kwargs_from_env import pytest -from .base import BUSYBOX +from .base import TEST_IMG @pytest.fixture(autouse=True, scope='session') @@ -15,15 +15,15 @@ def setup_test_session(): warnings.simplefilter('error') c = docker.APIClient(version='auto', **kwargs_from_env()) try: - c.inspect_image(BUSYBOX) + c.inspect_image(TEST_IMG) except docker.errors.NotFound: - print("\npulling {0}".format(BUSYBOX), file=sys.stderr) - for data in c.pull(BUSYBOX, stream=True, decode=True): + print("\npulling {0}".format(TEST_IMG), file=sys.stderr) + for data in c.pull(TEST_IMG, stream=True, decode=True): status = data.get("status") progress = data.get("progress") detail = "{0} - {1}".format(status, progress) print(detail, file=sys.stderr) # Double make sure we now have busybox - c.inspect_image(BUSYBOX) + c.inspect_image(TEST_IMG) c.close() diff --git a/tests/integration/errors_test.py b/tests/integration/errors_test.py index ac74d72100..7bf156afb0 100644 --- a/tests/integration/errors_test.py +++ b/tests/integration/errors_test.py @@ -1,11 +1,11 @@ from docker.errors import APIError -from .base import BaseAPIIntegrationTest, BUSYBOX +from .base import BaseAPIIntegrationTest, TEST_IMG import pytest class ErrorsTest(BaseAPIIntegrationTest): def test_api_error_parses_json(self): - container = self.client.create_container(BUSYBOX, ['sleep', '10']) + container = self.client.create_container(TEST_IMG, ['sleep', '10']) self.client.start(container['Id']) with pytest.raises(APIError) as cm: self.client.remove_container(container['Id']) diff --git a/tests/integration/models_images_test.py b/tests/integration/models_images_test.py index 31fab10968..375d972df9 100644 --- a/tests/integration/models_images_test.py +++ b/tests/integration/models_images_test.py @@ -4,7 +4,7 @@ import docker import pytest -from .base import BaseIntegrationTest, BUSYBOX, TEST_API_VERSION +from .base import BaseIntegrationTest, TEST_IMG, TEST_API_VERSION from ..helpers import random_name @@ -72,8 +72,8 @@ def test_pull(self): def test_pull_with_tag(self): client = docker.from_env(version=TEST_API_VERSION) - image = client.images.pull('alpine', tag='3.3') - assert 'alpine:3.3' in image.attrs['RepoTags'] + image = client.images.pull('alpine', tag='3.10') + assert 'alpine:3.10' in image.attrs['RepoTags'] def test_pull_with_sha(self): image_ref = ( @@ -97,7 +97,7 @@ def test_load_error(self): def test_save_and_load(self): client = docker.from_env(version=TEST_API_VERSION) - image = client.images.get(BUSYBOX) + image = client.images.get(TEST_IMG) with tempfile.TemporaryFile() as f: stream = image.save() for chunk in stream: @@ -111,7 +111,7 @@ def test_save_and_load(self): def test_save_and_load_repo_name(self): client = docker.from_env(version=TEST_API_VERSION) - image = client.images.get(BUSYBOX) + image = client.images.get(TEST_IMG) additional_tag = random_name() image.tag(additional_tag) self.tmp_imgs.append(additional_tag) @@ -131,7 +131,7 @@ def test_save_and_load_repo_name(self): def test_save_name_error(self): client = docker.from_env(version=TEST_API_VERSION) - image = client.images.get(BUSYBOX) + image = client.images.get(TEST_IMG) with pytest.raises(docker.errors.InvalidArgument): image.save(named='sakuya/izayoi') diff --git a/tests/integration/regression_test.py b/tests/integration/regression_test.py index 9aab076e30..a63883c4f5 100644 --- a/tests/integration/regression_test.py +++ b/tests/integration/regression_test.py @@ -4,7 +4,7 @@ import docker import six -from .base import BaseAPIIntegrationTest, BUSYBOX +from .base import BaseAPIIntegrationTest, TEST_IMG import pytest @@ -19,7 +19,7 @@ def test_443_handle_nonchunked_response_in_stream(self): def test_542_truncate_ids_client_side(self): self.client.start( - self.client.create_container(BUSYBOX, ['true']) + self.client.create_container(TEST_IMG, ['true']) ) result = self.client.containers(all=True, trunc=True) assert len(result[0]['Id']) == 12 @@ -30,12 +30,12 @@ def test_647_support_doubleslash_in_image_names(self): def test_649_handle_timeout_value_none(self): self.client.timeout = None - ctnr = self.client.create_container(BUSYBOX, ['sleep', '2']) + ctnr = self.client.create_container(TEST_IMG, ['sleep', '2']) self.client.start(ctnr) self.client.stop(ctnr) def test_715_handle_user_param_as_int_value(self): - ctnr = self.client.create_container(BUSYBOX, ['id', '-u'], user=1000) + ctnr = self.client.create_container(TEST_IMG, ['id', '-u'], user=1000) self.client.start(ctnr) self.client.wait(ctnr) logs = self.client.logs(ctnr) @@ -47,7 +47,7 @@ def test_792_explicit_port_protocol(self): tcp_port, udp_port = random.sample(range(9999, 32000), 2) ctnr = self.client.create_container( - BUSYBOX, ['sleep', '9999'], ports=[2000, (2000, 'udp')], + TEST_IMG, ['sleep', '9999'], ports=[2000, (2000, 'udp')], host_config=self.client.create_host_config( port_bindings={'2000/tcp': tcp_port, '2000/udp': udp_port} ) From 16794d1d236e7a2576af4fc1493dd3644e3820fb Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Sat, 10 Aug 2019 19:19:32 +0200 Subject: [PATCH 0950/1301] Jenkinsfile: update API version matrix; set default to v1.40 - Added new entry for Docker 19.03 - Removed obsolete engine versions that reached EOL (both as Community Edition and Enterprise Edition) - Set the fallback/default API version to v1.40, which corresponds with Docker 19.03 (current release) Signed-off-by: Sebastiaan van Stijn --- Jenkinsfile | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index e618c5dd77..a0f983c29d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -46,12 +46,14 @@ def getDockerVersions = { -> def getAPIVersion = { engineVersion -> def versionMap = [ - '17.06': '1.30', '17.12': '1.35', '18.02': '1.36', '18.03': '1.37', - '18.06': '1.38', '18.09': '1.39' + '17.06': '1.30', + '18.03': '1.37', + '18.09': '1.39', + '19.03': '1.40' ] def result = versionMap[engineVersion.substring(0, 5)] if (!result) { - return '1.39' + return '1.40' } return result } From f0fc266eb503eab979492bcfaf504348dc8ca239 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Sat, 10 Aug 2019 19:22:52 +0200 Subject: [PATCH 0951/1301] Jenkinsfile: update python 3.6 -> 3.7 Signed-off-by: Sebastiaan van Stijn --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index a0f983c29d..e879eb43e7 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -25,7 +25,7 @@ def buildImages = { -> imageNamePy3 = "${imageNameBase}:py3-${gitCommit()}" buildImage(imageNamePy2, "-f tests/Dockerfile --build-arg PYTHON_VERSION=2.7 .", "py2.7") - buildImage(imageNamePy3, "-f tests/Dockerfile --build-arg PYTHON_VERSION=3.6 .", "py3.6") + buildImage(imageNamePy3, "-f tests/Dockerfile --build-arg PYTHON_VERSION=3.7 .", "py3.7") } } } From 9ea3da37a7a03ceb23b992cba269eb3123614f46 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Sat, 10 Aug 2019 19:04:54 +0200 Subject: [PATCH 0952/1301] pytest: update to v4.2.1 - use xunit2 for compatibility with Jenkins - pytest-dev/pytest#3547: `--junitxml` can emit XML compatible with Jenkins xUnit. `junit_family` INI option accepts `legacy|xunit1`, which produces old style output, and `xunit2` that conforms more strictly to https://github.com/jenkinsci/xunit-plugin/blob/xunit-2.3.2/src/main/resources/org/jenkinsci/plugins/xunit/types/model/xsd/junit-10.xsd Signed-off-by: Sebastiaan van Stijn --- pytest.ini | 1 + test-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/pytest.ini b/pytest.ini index d233c56f18..d4f718e782 100644 --- a/pytest.ini +++ b/pytest.ini @@ -2,3 +2,4 @@ addopts = --tb=short -rxs junit_suite_name = docker-py +junit_family = xunit2 diff --git a/test-requirements.txt b/test-requirements.txt index b89f64622d..bebfee8618 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,6 +1,6 @@ coverage==4.5.2 flake8==3.6.0 mock==1.0.1 -pytest==4.1.0 +pytest==4.2.1 pytest-cov==2.6.1 pytest-timeout==1.3.3 From aa13df40b15b7fabafd3ee443ff691b44fab04de Mon Sep 17 00:00:00 2001 From: Matt Fluet Date: Wed, 7 Aug 2019 17:32:41 -0400 Subject: [PATCH 0953/1301] Fix for empty auth keys in config.json Signed-off-by: Matt Fluet --- docker/auth.py | 2 + tests/unit/auth_test.py | 116 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 118 insertions(+) diff --git a/docker/auth.py b/docker/auth.py index 5f34ac087d..6a07ea2059 100644 --- a/docker/auth.py +++ b/docker/auth.py @@ -303,12 +303,14 @@ def get_all_credentials(self): auth_data[k] = self._resolve_authconfig_credstore( k, self.creds_store ) + auth_data[convert_to_hostname(k)] = auth_data[k] # credHelpers entries take priority over all others for reg, store_name in self.cred_helpers.items(): auth_data[reg] = self._resolve_authconfig_credstore( reg, store_name ) + auth_data[convert_to_hostname(reg)] = auth_data[reg] return auth_data diff --git a/tests/unit/auth_test.py b/tests/unit/auth_test.py index d46da503e3..aac8910911 100644 --- a/tests/unit/auth_test.py +++ b/tests/unit/auth_test.py @@ -530,11 +530,21 @@ def test_get_all_credentials_credstore_only(self): 'Password': 'izayoi', 'ServerAddress': 'https://gensokyo.jp/v2', }, + 'gensokyo.jp': { + 'Username': 'sakuya', + 'Password': 'izayoi', + 'ServerAddress': 'https://gensokyo.jp/v2', + }, 'https://default.com/v2': { 'Username': 'user', 'Password': 'hunter2', 'ServerAddress': 'https://default.com/v2', }, + 'default.com': { + 'Username': 'user', + 'Password': 'hunter2', + 'ServerAddress': 'https://default.com/v2', + }, } def test_get_all_credentials_with_empty_credhelper(self): @@ -548,11 +558,21 @@ def test_get_all_credentials_with_empty_credhelper(self): 'Password': 'izayoi', 'ServerAddress': 'https://gensokyo.jp/v2', }, + 'gensokyo.jp': { + 'Username': 'sakuya', + 'Password': 'izayoi', + 'ServerAddress': 'https://gensokyo.jp/v2', + }, 'https://default.com/v2': { 'Username': 'user', 'Password': 'hunter2', 'ServerAddress': 'https://default.com/v2', }, + 'default.com': { + 'Username': 'user', + 'Password': 'hunter2', + 'ServerAddress': 'https://default.com/v2', + }, 'registry1.io': None, } @@ -571,11 +591,21 @@ def test_get_all_credentials_with_credhelpers_only(self): 'Password': 'izayoi', 'ServerAddress': 'https://gensokyo.jp/v2', }, + 'gensokyo.jp': { + 'Username': 'sakuya', + 'Password': 'izayoi', + 'ServerAddress': 'https://gensokyo.jp/v2', + }, 'https://default.com/v2': { 'Username': 'user', 'Password': 'hunter2', 'ServerAddress': 'https://default.com/v2', }, + 'default.com': { + 'Username': 'user', + 'Password': 'hunter2', + 'ServerAddress': 'https://default.com/v2', + }, } def test_get_all_credentials_with_auths_entries(self): @@ -591,11 +621,21 @@ def test_get_all_credentials_with_auths_entries(self): 'Password': 'izayoi', 'ServerAddress': 'https://gensokyo.jp/v2', }, + 'gensokyo.jp': { + 'Username': 'sakuya', + 'Password': 'izayoi', + 'ServerAddress': 'https://gensokyo.jp/v2', + }, 'https://default.com/v2': { 'Username': 'user', 'Password': 'hunter2', 'ServerAddress': 'https://default.com/v2', }, + 'default.com': { + 'Username': 'user', + 'Password': 'hunter2', + 'ServerAddress': 'https://default.com/v2', + }, 'registry1.io': { 'ServerAddress': 'registry1.io', 'Username': 'reimu', @@ -603,6 +643,62 @@ def test_get_all_credentials_with_auths_entries(self): }, } + def test_get_all_credentials_with_empty_auths_entry(self): + self.authconfig.add_auth('default.com', {}) + + assert self.authconfig.get_all_credentials() == { + 'https://gensokyo.jp/v2': { + 'Username': 'sakuya', + 'Password': 'izayoi', + 'ServerAddress': 'https://gensokyo.jp/v2', + }, + 'gensokyo.jp': { + 'Username': 'sakuya', + 'Password': 'izayoi', + 'ServerAddress': 'https://gensokyo.jp/v2', + }, + 'https://default.com/v2': { + 'Username': 'user', + 'Password': 'hunter2', + 'ServerAddress': 'https://default.com/v2', + }, + 'default.com': { + 'Username': 'user', + 'Password': 'hunter2', + 'ServerAddress': 'https://default.com/v2', + }, + } + + def test_get_all_credentials_credstore_overrides_auth_entry(self): + self.authconfig.add_auth('default.com', { + 'Username': 'shouldnotsee', + 'Password': 'thisentry', + 'ServerAddress': 'https://default.com/v2', + }) + + assert self.authconfig.get_all_credentials() == { + 'https://gensokyo.jp/v2': { + 'Username': 'sakuya', + 'Password': 'izayoi', + 'ServerAddress': 'https://gensokyo.jp/v2', + }, + 'gensokyo.jp': { + 'Username': 'sakuya', + 'Password': 'izayoi', + 'ServerAddress': 'https://gensokyo.jp/v2', + }, + 'https://default.com/v2': { + 'Username': 'user', + 'Password': 'hunter2', + 'ServerAddress': 'https://default.com/v2', + }, + 'default.com': { + 'Username': 'user', + 'Password': 'hunter2', + 'ServerAddress': 'https://default.com/v2', + }, + } + def test_get_all_credentials_helpers_override_default(self): self.authconfig['credHelpers'] = { 'https://default.com/v2': 'truesecret', @@ -616,11 +712,21 @@ def test_get_all_credentials_helpers_override_default(self): 'Password': 'izayoi', 'ServerAddress': 'https://gensokyo.jp/v2', }, + 'gensokyo.jp': { + 'Username': 'sakuya', + 'Password': 'izayoi', + 'ServerAddress': 'https://gensokyo.jp/v2', + }, 'https://default.com/v2': { 'Username': 'reimu', 'Password': 'hakurei', 'ServerAddress': 'https://default.com/v2', }, + 'default.com': { + 'Username': 'reimu', + 'Password': 'hakurei', + 'ServerAddress': 'https://default.com/v2', + }, } def test_get_all_credentials_3_sources(self): @@ -642,11 +748,21 @@ def test_get_all_credentials_3_sources(self): 'Password': 'izayoi', 'ServerAddress': 'https://gensokyo.jp/v2', }, + 'gensokyo.jp': { + 'Username': 'sakuya', + 'Password': 'izayoi', + 'ServerAddress': 'https://gensokyo.jp/v2', + }, 'https://default.com/v2': { 'Username': 'user', 'Password': 'hunter2', 'ServerAddress': 'https://default.com/v2', }, + 'default.com': { + 'Username': 'user', + 'Password': 'hunter2', + 'ServerAddress': 'https://default.com/v2', + }, 'registry1.io': { 'ServerAddress': 'registry1.io', 'Username': 'reimu', From 93dc5082de24226d65b6ca695f4e20a5e4bc6f8f Mon Sep 17 00:00:00 2001 From: Ryan McCullagh Date: Fri, 23 Aug 2019 10:36:58 -0500 Subject: [PATCH 0954/1301] Fix typo in comment. networks => network Signed-off-by: Ryan McCullagh --- docker/api/network.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/api/network.py b/docker/api/network.py index 57ed8d3b75..c56a8d0bd7 100644 --- a/docker/api/network.py +++ b/docker/api/network.py @@ -7,7 +7,7 @@ class NetworkApiMixin(object): def networks(self, names=None, ids=None, filters=None): """ - List networks. Similar to the ``docker networks ls`` command. + List networks. Similar to the ``docker network ls`` command. Args: names (:py:class:`list`): List of names to filter by From 53469e0dd37abcd3f164c34aa2c83cfba2b9e1e0 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Fri, 30 Aug 2019 00:14:20 +0200 Subject: [PATCH 0955/1301] Fix broken test due to BUSYBOX -> TEST_IMG rename The BUSYBOX variable was renamed to TEST_IMG in 54b48a9b7ab59b4dcf49acf49ddf52035ba3ea08, however 0ddf428b6ce7accdac3506b45047df2cb72941ec got merged after that change, but was out of date, and therefore caused the tests to fail: ``` =================================== FAILURES =================================== ________ ServiceTest.test_create_service_with_network_attachment_config ________ tests/integration/api_service_test.py:379: in test_create_service_with_network_attachment_config container_spec = docker.types.ContainerSpec(BUSYBOX, ['true']) E NameError: global name 'BUSYBOX' is not defined ``` Fix the test by using the correct variable name. Signed-off-by: Sebastiaan van Stijn --- tests/integration/api_service_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/api_service_test.py b/tests/integration/api_service_test.py index 784d1e377d..b6b7ec538d 100644 --- a/tests/integration/api_service_test.py +++ b/tests/integration/api_service_test.py @@ -376,7 +376,7 @@ def test_create_service_with_network_attachment_config(self): 'dockerpytest_1', driver='overlay', ipam={'Driver': 'default'} ) self.tmp_networks.append(network['Id']) - container_spec = docker.types.ContainerSpec(BUSYBOX, ['true']) + container_spec = docker.types.ContainerSpec(TEST_IMG, ['true']) network_config = docker.types.NetworkAttachmentConfig( target='dockerpytest_1', aliases=['dockerpytest_1_alias'], From 015f44d8f833df64e326d08f347b7f54e3c0d99f Mon Sep 17 00:00:00 2001 From: rentu Date: Fri, 30 Aug 2019 09:35:46 +0100 Subject: [PATCH 0956/1301] Fix win32pipe.WaitNamedPipe throw exception in windows container. Signed-off-by: Renlong Tu --- docker/transport/npipesocket.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/docker/transport/npipesocket.py b/docker/transport/npipesocket.py index ef02031640..176b5c87a9 100644 --- a/docker/transport/npipesocket.py +++ b/docker/transport/npipesocket.py @@ -1,4 +1,5 @@ import functools +import time import io import six @@ -9,7 +10,7 @@ cSECURITY_SQOS_PRESENT = 0x100000 cSECURITY_ANONYMOUS = 0 -RETRY_WAIT_TIMEOUT = 10000 +MAXIMUM_RETRY_COUNT = 10 def check_closed(f): @@ -46,8 +47,7 @@ def close(self): self._closed = True @check_closed - def connect(self, address): - win32pipe.WaitNamedPipe(address, self._timeout) + def connect(self, address, retry_count=0): try: handle = win32file.CreateFile( address, @@ -65,8 +65,10 @@ def connect(self, address): # Another program or thread has grabbed our pipe instance # before we got to it. Wait for availability and attempt to # connect again. - win32pipe.WaitNamedPipe(address, RETRY_WAIT_TIMEOUT) - return self.connect(address) + retry_count = retry_count + 1 + if (retry_count < MAXIMUM_RETRY_COUNT): + time.sleep(1) + return self.connect(address, retry_count) raise e self.flags = win32pipe.GetNamedPipeInfo(handle)[0] From 44904f696f639fa69cf7b01d1200042fab1d9aad Mon Sep 17 00:00:00 2001 From: Kir Kolyshkin Date: Tue, 1 Oct 2019 17:21:38 -0700 Subject: [PATCH 0957/1301] Bump pytest to 4.3.1 Pytest 4.3.1 includes the fix from https://github.com/pytest-dev/pytest/pull/4795 which should fix the following failure: > INFO: Building docker-sdk-python3:4.0.2... > sha256:c7a40413c985b6e75df324fae39b1c30cb78a25df71b7892f1a4a15449537fb3 > INFO: Starting docker-py tests... > Traceback (most recent call last): > File "/usr/local/bin/pytest", line 10, in > sys.exit(main()) > File "/usr/local/lib/python3.6/site-packages/_pytest/config/__init__.py", line 61, in main > config = _prepareconfig(args, plugins) > File "/usr/local/lib/python3.6/site-packages/_pytest/config/__init__.py", line 182, in _prepareconfig > config = get_config() > File "/usr/local/lib/python3.6/site-packages/_pytest/config/__init__.py", line 156, in get_config > pluginmanager.import_plugin(spec) > File "/usr/local/lib/python3.6/site-packages/_pytest/config/__init__.py", line 530, in import_plugin > __import__(importspec) > File "/usr/local/lib/python3.6/site-packages/_pytest/tmpdir.py", line 25, in > class TempPathFactory(object): > File "/usr/local/lib/python3.6/site-packages/_pytest/tmpdir.py", line 35, in TempPathFactory > lambda p: Path(os.path.abspath(six.text_type(p))) > TypeError: attrib() got an unexpected keyword argument 'convert' > Sending interrupt signal to process > Terminated > script returned exit code 143 Signed-off-by: Kir Kolyshkin --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index bebfee8618..0b01e569e6 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,6 +1,6 @@ coverage==4.5.2 flake8==3.6.0 mock==1.0.1 -pytest==4.2.1 +pytest==4.3.1 pytest-cov==2.6.1 pytest-timeout==1.3.3 From c2ed66552bf5ff49199670f920ec5034f0a4c6ae Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Tue, 25 Jun 2019 13:08:39 -0400 Subject: [PATCH 0958/1301] Remove exec detach test Forking off an exec process and detaching isn't a supported method Signed-off-by: Michael Crosby --- tests/integration/api_exec_test.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/tests/integration/api_exec_test.py b/tests/integration/api_exec_test.py index dda0ed9051..53b7e22fea 100644 --- a/tests/integration/api_exec_test.py +++ b/tests/integration/api_exec_test.py @@ -226,24 +226,6 @@ def test_detach_with_config_file(self): assert_cat_socket_detached_with_keys(sock, [ctrl_with('p')]) - def test_detach_with_arg(self): - self.client._general_configs['detachKeys'] = 'ctrl-p' - container = self.client.create_container( - BUSYBOX, 'cat', detach=True, stdin_open=True - ) - id = container['Id'] - self.client.start(id) - self.tmp_containers.append(id) - - exec_id = self.client.exec_create( - id, 'cat', - stdin=True, tty=True, detach_keys='ctrl-x', stdout=True - ) - sock = self.client.exec_start(exec_id, tty=True, socket=True) - self.addCleanup(sock.close) - - assert_cat_socket_detached_with_keys(sock, [ctrl_with('x')]) - class ExecDemuxTest(BaseAPIIntegrationTest): cmd = 'sh -c "{}"'.format(' ; '.join([ From f3961244a0473305b56e8ffe27d53a6e7902e8bc Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Fri, 12 Jul 2019 01:28:41 +0200 Subject: [PATCH 0959/1301] Update credentials-helpers to v0.6.2 Signed-off-by: Sebastiaan van Stijn --- tests/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Dockerfile b/tests/Dockerfile index 042fc7038a..8f49cd2ce0 100644 --- a/tests/Dockerfile +++ b/tests/Dockerfile @@ -10,7 +10,7 @@ RUN gpg2 --import gpg-keys/secret RUN gpg2 --import-ownertrust gpg-keys/ownertrust RUN yes | pass init $(gpg2 --no-auto-check-trustdb --list-secret-keys | grep ^sec | cut -d/ -f2 | cut -d" " -f1) RUN gpg2 --check-trustdb -ARG CREDSTORE_VERSION=v0.6.0 +ARG CREDSTORE_VERSION=v0.6.2 RUN curl -sSL -o /opt/docker-credential-pass.tar.gz \ https://github.com/docker/docker-credential-helpers/releases/download/$CREDSTORE_VERSION/docker-credential-pass-$CREDSTORE_VERSION-amd64.tar.gz && \ tar -xf /opt/docker-credential-pass.tar.gz -O > /usr/local/bin/docker-credential-pass && \ From 546bc63244941e8aa22a408635d0bff554b1702b Mon Sep 17 00:00:00 2001 From: Djordje Lukic Date: Thu, 20 Jun 2019 13:34:03 +0200 Subject: [PATCH 0960/1301] Bump dev Signed-off-by: Djordje Lukic --- docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index 25c92501f4..21249253e6 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "4.0.2" +version = "4.1.0-dev" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) From ea4fbd7ddf5ff1b7a9b4a1900522d51537387156 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Fri, 12 Jul 2019 18:53:34 +0200 Subject: [PATCH 0961/1301] Update to python 3.7 (buster) and use build-args The build arg can be used to either test different versions, but also makes it easier to "grep" when upgrading versions. The output format of `gpg2 --list-secret-keys` changed in the version installed on Buster, so `grep` was replaced with `awk` to address the new output format; Debian Jessie: gpg2 --no-auto-check-trustdb --list-secret-keys /root/.gnupg/secring.gpg ------------------------ sec 1024D/A7B21401 2018-04-25 uid Sakuya Izayoi ssb 1024g/C235E4CE 2018-04-25 Debian Buster: gpg2 --no-auto-check-trustdb --list-secret-keys /root/.gnupg/pubring.kbx ------------------------ sec dsa1024 2018-04-25 [SCA] 9781B87DAB042E6FD51388A5464ED987A7B21401 uid [ultimate] Sakuya Izayoi ssb elg1024 2018-04-25 [E] Signed-off-by: Sebastiaan van Stijn --- Dockerfile | 4 +++- Dockerfile-docs | 4 +++- Dockerfile-py3 | 4 +++- tests/Dockerfile | 7 ++++--- tests/Dockerfile-dind-certs | 4 +++- 5 files changed, 16 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 82758daf97..124f68cdd0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,6 @@ -FROM python:2.7 +ARG PYTHON_VERSION=2.7 + +FROM python:${PYTHON_VERSION} RUN mkdir /src WORKDIR /src diff --git a/Dockerfile-docs b/Dockerfile-docs index 105083e8cb..9d11312fca 100644 --- a/Dockerfile-docs +++ b/Dockerfile-docs @@ -1,4 +1,6 @@ -FROM python:3.5 +ARG PYTHON_VERSION=3.7 + +FROM python:${PYTHON_VERSION} ARG uid=1000 ARG gid=1000 diff --git a/Dockerfile-py3 b/Dockerfile-py3 index d558ba3e4f..22732dec5c 100644 --- a/Dockerfile-py3 +++ b/Dockerfile-py3 @@ -1,4 +1,6 @@ -FROM python:3.6 +ARG PYTHON_VERSION=3.7 + +FROM python:${PYTHON_VERSION} RUN mkdir /src WORKDIR /src diff --git a/tests/Dockerfile b/tests/Dockerfile index 8f49cd2ce0..f2f36b4471 100644 --- a/tests/Dockerfile +++ b/tests/Dockerfile @@ -1,5 +1,6 @@ -ARG PYTHON_VERSION=3.6 -FROM python:$PYTHON_VERSION-jessie +ARG PYTHON_VERSION=3.7 + +FROM python:${PYTHON_VERSION} RUN apt-get update && apt-get -y install \ gnupg2 \ pass \ @@ -8,7 +9,7 @@ RUN apt-get update && apt-get -y install \ COPY ./tests/gpg-keys /gpg-keys RUN gpg2 --import gpg-keys/secret RUN gpg2 --import-ownertrust gpg-keys/ownertrust -RUN yes | pass init $(gpg2 --no-auto-check-trustdb --list-secret-keys | grep ^sec | cut -d/ -f2 | cut -d" " -f1) +RUN yes | pass init $(gpg2 --no-auto-check-trustdb --list-secret-key | awk '/^sec/{getline; $1=$1; print}') RUN gpg2 --check-trustdb ARG CREDSTORE_VERSION=v0.6.2 RUN curl -sSL -o /opt/docker-credential-pass.tar.gz \ diff --git a/tests/Dockerfile-dind-certs b/tests/Dockerfile-dind-certs index 9e8c042b63..2ab87ef732 100644 --- a/tests/Dockerfile-dind-certs +++ b/tests/Dockerfile-dind-certs @@ -1,4 +1,6 @@ -FROM python:2.7 +ARG PYTHON_VERSION=2.7 + +FROM python:${PYTHON_VERSION} RUN mkdir /tmp/certs VOLUME /certs From a316e6a9274cc3e153305b443aa77ef531c4e4a9 Mon Sep 17 00:00:00 2001 From: Francis Laniel Date: Wed, 9 Jan 2019 19:31:56 +0100 Subject: [PATCH 0962/1301] Add documentation to argument 'mem_reservation'. The documentation was added for function ContainerCollection::run and ContainerApiMixin::create_host_config. Signed-off-by: Francis Laniel Add documentation to argument 'mem_reservation'. The documentation was added for function ContainerCollection::run and ContainerApiMixin::create_host_config. Signed-off-by: Francis Laniel --- docker/api/container.py | 1 + docker/models/containers.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/api/container.py b/docker/api/container.py index 2dca68a144..326e7679f1 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -502,6 +502,7 @@ def create_host_config(self, *args, **kwargs): bytes) or a string with a units identification char (``100000b``, ``1000k``, ``128m``, ``1g``). If a string is specified without a units character, bytes are assumed as an + mem_reservation (int or str): Memory soft limit. mem_swappiness (int): Tune a container's memory swappiness behavior. Accepts number between 0 and 100. memswap_limit (str or int): Maximum amount of memory + swap a diff --git a/docker/models/containers.py b/docker/models/containers.py index d321a58022..999851ec13 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -618,7 +618,7 @@ def run(self, image, command=None, stdout=True, stderr=False, (``100000b``, ``1000k``, ``128m``, ``1g``). If a string is specified without a units character, bytes are assumed as an intended unit. - mem_reservation (int or str): Memory soft limit + mem_reservation (int or str): Memory soft limit. mem_swappiness (int): Tune a container's memory swappiness behavior. Accepts number between 0 and 100. memswap_limit (str or int): Maximum amount of memory + swap a From 38d18a2d1f53fd674447812b28fef3b3b5f81301 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Tue, 16 Jul 2019 16:04:38 +0200 Subject: [PATCH 0963/1301] Update credentials-helpers to v0.6.3 full diff: https://github.com/docker/docker-credential-helpers/compare/v0.6.2...v0.6.3 Signed-off-by: Sebastiaan van Stijn --- tests/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Dockerfile b/tests/Dockerfile index f2f36b4471..4bd98f8733 100644 --- a/tests/Dockerfile +++ b/tests/Dockerfile @@ -11,7 +11,7 @@ RUN gpg2 --import gpg-keys/secret RUN gpg2 --import-ownertrust gpg-keys/ownertrust RUN yes | pass init $(gpg2 --no-auto-check-trustdb --list-secret-key | awk '/^sec/{getline; $1=$1; print}') RUN gpg2 --check-trustdb -ARG CREDSTORE_VERSION=v0.6.2 +ARG CREDSTORE_VERSION=v0.6.3 RUN curl -sSL -o /opt/docker-credential-pass.tar.gz \ https://github.com/docker/docker-credential-helpers/releases/download/$CREDSTORE_VERSION/docker-credential-pass-$CREDSTORE_VERSION-amd64.tar.gz && \ tar -xf /opt/docker-credential-pass.tar.gz -O > /usr/local/bin/docker-credential-pass && \ From cd3a696603d2fa985415aeb45896ab8bceccb2b7 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Fri, 12 Jul 2019 22:50:35 +0200 Subject: [PATCH 0964/1301] xfail test_init_swarm_data_path_addr This test can fail if `eth0` has multiple IP addresses; E docker.errors.APIError: 400 Client Error: Bad Request ("interface eth0 has more than one IPv6 address (2001:db8:1::242:ac11:2 and fe80::42:acff:fe11:2)") Which is not a failiure, but depends on the environment that the test is run in. Signed-off-by: Sebastiaan van Stijn --- tests/integration/api_swarm_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/api_swarm_test.py b/tests/integration/api_swarm_test.py index bf809bd0c1..f1cbc264e2 100644 --- a/tests/integration/api_swarm_test.py +++ b/tests/integration/api_swarm_test.py @@ -250,5 +250,6 @@ def test_rotate_manager_unlock_key(self): assert key_1['UnlockKey'] != key_2['UnlockKey'] @requires_api_version('1.30') + @pytest.mark.xfail(reason='Can fail if eth0 has multiple IP addresses') def test_init_swarm_data_path_addr(self): assert self.init_swarm(data_path_addr='eth0') From 23635d43abe7ac0c3512e03292ee786a07e543a8 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Mon, 15 Jul 2019 15:04:31 +0200 Subject: [PATCH 0965/1301] Adjust `--platform` tests for changes in docker engine These tests started failing on recent versions of the engine because the error string changed, and due to a regression, the status code for one endpoint changed from a 400 to a 500. On Docker 18.03: The `docker build` case properly returns a 400, and "invalid platform" as error string; ```bash docker build --platform=foobar -<}] module=grpc INFO[2019-07-15T11:59:20.688270160Z] ClientConn switching balancer to "pick_first" module=grpc INFO[2019-07-15T11:59:20.688353083Z] pickfirstBalancer: HandleSubConnStateChange: 0xc4209b0630, CONNECTING module=grpc INFO[2019-07-15T11:59:20.688985698Z] pickfirstBalancer: HandleSubConnStateChange: 0xc4209b0630, READY module=grpc DEBU[2019-07-15T11:59:20.812700550Z] client is session enabled DEBU[2019-07-15T11:59:20.813139288Z] FIXME: Got an API for which error does not match any expected type!!!: invalid argument github.com/docker/docker/vendor/github.com/containerd/containerd/errdefs.init /go/src/github.com/docker/docker/vendor/github.com/containerd/containerd/errdefs/errors.go:40 github.com/docker/docker/vendor/github.com/containerd/containerd/content.init :1 github.com/docker/docker/builder/builder-next.init :1 github.com/docker/docker/api/server/backend/build.init :1 main.init :1 runtime.main /usr/local/go/src/runtime/proc.go:186 runtime.goexit /usr/local/go/src/runtime/asm_amd64.s:2361 error_type="*errors.fundamental" module=api ERRO[2019-07-15T11:59:20.813210677Z] Handler for POST /v1.39/build returned error: "foobar": unknown operating system or architecture: invalid argument DEBU[2019-07-15T11:59:20.813276737Z] FIXME: Got an API for which error does not match any expected type!!!: invalid argument github.com/docker/docker/vendor/github.com/containerd/containerd/errdefs.init /go/src/github.com/docker/docker/vendor/github.com/containerd/containerd/errdefs/errors.go:40 github.com/docker/docker/vendor/github.com/containerd/containerd/content.init :1 github.com/docker/docker/builder/builder-next.init :1 github.com/docker/docker/api/server/backend/build.init :1 main.init :1 runtime.main /usr/local/go/src/runtime/proc.go:186 runtime.goexit /usr/local/go/src/runtime/asm_amd64.s:2361 error_type="*errors.fundamental" module=api ``` Same for the `docker pull --platform=foobar hello-world:latest` case: ```bash docker pull --platform=foobar hello-world:latest Error response from daemon: "foobar": unknown operating system or architecture: invalid argument ``` ``` DEBU[2019-07-15T12:00:18.812995330Z] Calling POST /v1.39/images/create?fromImage=hello-world&platform=foobar&tag=latest DEBU[2019-07-15T12:00:18.813229172Z] FIXME: Got an API for which error does not match any expected type!!!: invalid argument github.com/docker/docker/vendor/github.com/containerd/containerd/errdefs.init /go/src/github.com/docker/docker/vendor/github.com/containerd/containerd/errdefs/errors.go:40 github.com/docker/docker/vendor/github.com/containerd/containerd/content.init :1 github.com/docker/docker/builder/builder-next.init :1 github.com/docker/docker/api/server/backend/build.init :1 main.init :1 runtime.main /usr/local/go/src/runtime/proc.go:186 runtime.goexit /usr/local/go/src/runtime/asm_amd64.s:2361 error_type="*errors.fundamental" module=api ERRO[2019-07-15T12:00:18.813365546Z] Handler for POST /v1.39/images/create returned error: "foobar": unknown operating system or architecture: invalid argument DEBU[2019-07-15T12:00:18.813461428Z] FIXME: Got an API for which error does not match any expected type!!!: invalid argument github.com/docker/docker/vendor/github.com/containerd/containerd/errdefs.init /go/src/github.com/docker/docker/vendor/github.com/containerd/containerd/errdefs/errors.go:40 github.com/docker/docker/vendor/github.com/containerd/containerd/content.init :1 github.com/docker/docker/builder/builder-next.init :1 github.com/docker/docker/api/server/backend/build.init :1 main.init :1 runtime.main /usr/local/go/src/runtime/proc.go:186 runtime.goexit /usr/local/go/src/runtime/asm_amd64.s:2361 error_type="*errors.fundamental" module=api ``` Signed-off-by: Sebastiaan van Stijn --- tests/integration/api_build_test.py | 6 ++++-- tests/integration/api_image_test.py | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/integration/api_build_test.py b/tests/integration/api_build_test.py index 8bfc7960fc..4776f45385 100644 --- a/tests/integration/api_build_test.py +++ b/tests/integration/api_build_test.py @@ -448,8 +448,10 @@ def test_build_invalid_platform(self): for _ in stream: pass - assert excinfo.value.status_code == 400 - assert 'invalid platform' in excinfo.exconly() + # Some API versions incorrectly returns 500 status; assert 4xx or 5xx + assert excinfo.value.is_error() + assert 'unknown operating system' in excinfo.exconly() \ + or 'invalid platform' in excinfo.exconly() def test_build_out_of_context_dockerfile(self): base_dir = tempfile.mkdtemp() diff --git a/tests/integration/api_image_test.py b/tests/integration/api_image_test.py index 050e7f339b..56a7692489 100644 --- a/tests/integration/api_image_test.py +++ b/tests/integration/api_image_test.py @@ -69,8 +69,10 @@ def test_pull_invalid_platform(self): with pytest.raises(docker.errors.APIError) as excinfo: self.client.pull('hello-world', platform='foobar') - assert excinfo.value.status_code == 500 - assert 'invalid platform' in excinfo.exconly() + # Some API versions incorrectly returns 500 status; assert 4xx or 5xx + assert excinfo.value.is_error() + assert 'unknown operating system' in excinfo.exconly() \ + or 'invalid platform' in excinfo.exconly() class CommitTest(BaseAPIIntegrationTest): From 73ad8b8f1909a2f1191605f2204f44dcac90c104 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Sat, 10 Aug 2019 18:30:40 +0200 Subject: [PATCH 0966/1301] Update alpine version to 3.10, and rename BUSYBOX variable Signed-off-by: Sebastiaan van Stijn --- tests/integration/api_build_test.py | 4 +- tests/integration/api_container_test.py | 190 +++++++++++----------- tests/integration/api_exec_test.py | 30 ++-- tests/integration/api_healthcheck_test.py | 10 +- tests/integration/api_image_test.py | 18 +- tests/integration/api_network_test.py | 18 +- tests/integration/api_service_test.py | 68 ++++---- tests/integration/base.py | 4 +- tests/integration/conftest.py | 10 +- tests/integration/errors_test.py | 4 +- tests/integration/models_images_test.py | 12 +- tests/integration/regression_test.py | 10 +- 12 files changed, 189 insertions(+), 189 deletions(-) diff --git a/tests/integration/api_build_test.py b/tests/integration/api_build_test.py index 4776f45385..57128124ef 100644 --- a/tests/integration/api_build_test.py +++ b/tests/integration/api_build_test.py @@ -9,7 +9,7 @@ import pytest import six -from .base import BaseAPIIntegrationTest, BUSYBOX +from .base import BaseAPIIntegrationTest, TEST_IMG from ..helpers import random_name, requires_api_version, requires_experimental @@ -277,7 +277,7 @@ def test_build_with_network_mode(self): # Set up pingable endpoint on custom network network = self.client.create_network(random_name())['Id'] self.tmp_networks.append(network) - container = self.client.create_container(BUSYBOX, 'top') + container = self.client.create_container(TEST_IMG, 'top') self.tmp_containers.append(container) self.client.start(container) self.client.connect_container_to_network( diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index 26245c1fa9..1ba3eaa583 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -15,7 +15,7 @@ from ..helpers import ctrl_with from ..helpers import requires_api_version from .base import BaseAPIIntegrationTest -from .base import BUSYBOX +from .base import TEST_IMG from docker.constants import IS_WINDOWS_PLATFORM from docker.utils.socket import next_frame_header from docker.utils.socket import read_exactly @@ -25,7 +25,7 @@ class ListContainersTest(BaseAPIIntegrationTest): def test_list_containers(self): res0 = self.client.containers(all=True) size = len(res0) - res1 = self.client.create_container(BUSYBOX, 'true') + res1 = self.client.create_container(TEST_IMG, 'true') assert 'Id' in res1 self.client.start(res1['Id']) self.tmp_containers.append(res1['Id']) @@ -44,13 +44,13 @@ def test_list_containers(self): class CreateContainerTest(BaseAPIIntegrationTest): def test_create(self): - res = self.client.create_container(BUSYBOX, 'true') + res = self.client.create_container(TEST_IMG, 'true') assert 'Id' in res self.tmp_containers.append(res['Id']) def test_create_with_host_pid_mode(self): ctnr = self.client.create_container( - BUSYBOX, 'true', host_config=self.client.create_host_config( + TEST_IMG, 'true', host_config=self.client.create_host_config( pid_mode='host', network_mode='none' ) ) @@ -65,7 +65,7 @@ def test_create_with_host_pid_mode(self): def test_create_with_links(self): res0 = self.client.create_container( - BUSYBOX, 'cat', + TEST_IMG, 'cat', detach=True, stdin_open=True, environment={'FOO': '1'}) @@ -75,7 +75,7 @@ def test_create_with_links(self): self.client.start(container1_id) res1 = self.client.create_container( - BUSYBOX, 'cat', + TEST_IMG, 'cat', detach=True, stdin_open=True, environment={'FOO': '1'}) @@ -94,7 +94,7 @@ def test_create_with_links(self): link_env_prefix2 = link_alias2.upper() res2 = self.client.create_container( - BUSYBOX, 'env', host_config=self.client.create_host_config( + TEST_IMG, 'env', host_config=self.client.create_host_config( links={link_path1: link_alias1, link_path2: link_alias2}, network_mode='bridge' ) @@ -114,7 +114,7 @@ def test_create_with_links(self): def test_create_with_restart_policy(self): container = self.client.create_container( - BUSYBOX, ['sleep', '2'], + TEST_IMG, ['sleep', '2'], host_config=self.client.create_host_config( restart_policy={"Name": "always", "MaximumRetryCount": 0}, network_mode='none' @@ -133,21 +133,21 @@ def test_create_container_with_volumes_from(self): vol_names = ['foobar_vol0', 'foobar_vol1'] res0 = self.client.create_container( - BUSYBOX, 'true', name=vol_names[0] + TEST_IMG, 'true', name=vol_names[0] ) container1_id = res0['Id'] self.tmp_containers.append(container1_id) self.client.start(container1_id) res1 = self.client.create_container( - BUSYBOX, 'true', name=vol_names[1] + TEST_IMG, 'true', name=vol_names[1] ) container2_id = res1['Id'] self.tmp_containers.append(container2_id) self.client.start(container2_id) res = self.client.create_container( - BUSYBOX, 'cat', detach=True, stdin_open=True, + TEST_IMG, 'cat', detach=True, stdin_open=True, host_config=self.client.create_host_config( volumes_from=vol_names, network_mode='none' ) @@ -161,7 +161,7 @@ def test_create_container_with_volumes_from(self): def create_container_readonly_fs(self): ctnr = self.client.create_container( - BUSYBOX, ['mkdir', '/shrine'], + TEST_IMG, ['mkdir', '/shrine'], host_config=self.client.create_host_config( read_only=True, network_mode='none' ) @@ -173,7 +173,7 @@ def create_container_readonly_fs(self): assert res != 0 def create_container_with_name(self): - res = self.client.create_container(BUSYBOX, 'true', name='foobar') + res = self.client.create_container(TEST_IMG, 'true', name='foobar') assert 'Id' in res self.tmp_containers.append(res['Id']) inspect = self.client.inspect_container(res['Id']) @@ -182,7 +182,7 @@ def create_container_with_name(self): def create_container_privileged(self): res = self.client.create_container( - BUSYBOX, 'true', host_config=self.client.create_host_config( + TEST_IMG, 'true', host_config=self.client.create_host_config( privileged=True, network_mode='none' ) ) @@ -208,7 +208,7 @@ def create_container_privileged(self): def test_create_with_mac_address(self): mac_address_expected = "02:42:ac:11:00:0a" container = self.client.create_container( - BUSYBOX, ['sleep', '60'], mac_address=mac_address_expected) + TEST_IMG, ['sleep', '60'], mac_address=mac_address_expected) id = container['Id'] @@ -220,7 +220,7 @@ def test_create_with_mac_address(self): def test_group_id_ints(self): container = self.client.create_container( - BUSYBOX, 'id -G', + TEST_IMG, 'id -G', host_config=self.client.create_host_config(group_add=[1000, 1001]) ) self.tmp_containers.append(container) @@ -236,7 +236,7 @@ def test_group_id_ints(self): def test_group_id_strings(self): container = self.client.create_container( - BUSYBOX, 'id -G', host_config=self.client.create_host_config( + TEST_IMG, 'id -G', host_config=self.client.create_host_config( group_add=['1000', '1001'] ) ) @@ -259,7 +259,7 @@ def test_valid_log_driver_and_log_opt(self): ) container = self.client.create_container( - BUSYBOX, ['true'], + TEST_IMG, ['true'], host_config=self.client.create_host_config(log_config=log_config) ) self.tmp_containers.append(container['Id']) @@ -281,7 +281,7 @@ def test_invalid_log_driver_raises_exception(self): with pytest.raises(docker.errors.APIError) as excinfo: # raises an internal server error 500 container = self.client.create_container( - BUSYBOX, ['true'], host_config=self.client.create_host_config( + TEST_IMG, ['true'], host_config=self.client.create_host_config( log_config=log_config ) ) @@ -296,7 +296,7 @@ def test_valid_no_log_driver_specified(self): ) container = self.client.create_container( - BUSYBOX, ['true'], + TEST_IMG, ['true'], host_config=self.client.create_host_config(log_config=log_config) ) self.tmp_containers.append(container['Id']) @@ -315,7 +315,7 @@ def test_valid_no_config_specified(self): ) container = self.client.create_container( - BUSYBOX, ['true'], + TEST_IMG, ['true'], host_config=self.client.create_host_config(log_config=log_config) ) self.tmp_containers.append(container['Id']) @@ -329,7 +329,7 @@ def test_valid_no_config_specified(self): def test_create_with_memory_constraints_with_str(self): ctnr = self.client.create_container( - BUSYBOX, 'true', + TEST_IMG, 'true', host_config=self.client.create_host_config( memswap_limit='1G', mem_limit='700M' @@ -347,7 +347,7 @@ def test_create_with_memory_constraints_with_str(self): def test_create_with_memory_constraints_with_int(self): ctnr = self.client.create_container( - BUSYBOX, 'true', + TEST_IMG, 'true', host_config=self.client.create_host_config(mem_swappiness=40) ) assert 'Id' in ctnr @@ -361,7 +361,7 @@ def test_create_with_memory_constraints_with_int(self): def test_create_with_environment_variable_no_value(self): container = self.client.create_container( - BUSYBOX, + TEST_IMG, ['echo'], environment={'Foo': None, 'Other': 'one', 'Blank': ''}, ) @@ -378,7 +378,7 @@ def test_create_with_tmpfs(self): } container = self.client.create_container( - BUSYBOX, + TEST_IMG, ['echo'], host_config=self.client.create_host_config( tmpfs=tmpfs)) @@ -390,7 +390,7 @@ def test_create_with_tmpfs(self): @requires_api_version('1.24') def test_create_with_isolation(self): container = self.client.create_container( - BUSYBOX, ['echo'], host_config=self.client.create_host_config( + TEST_IMG, ['echo'], host_config=self.client.create_host_config( isolation='default' ) ) @@ -404,7 +404,7 @@ def test_create_with_auto_remove(self): auto_remove=True ) container = self.client.create_container( - BUSYBOX, ['echo', 'test'], host_config=host_config + TEST_IMG, ['echo', 'test'], host_config=host_config ) self.tmp_containers.append(container['Id']) config = self.client.inspect_container(container) @@ -413,7 +413,7 @@ def test_create_with_auto_remove(self): @requires_api_version('1.25') def test_create_with_stop_timeout(self): container = self.client.create_container( - BUSYBOX, ['echo', 'test'], stop_timeout=25 + TEST_IMG, ['echo', 'test'], stop_timeout=25 ) self.tmp_containers.append(container['Id']) config = self.client.inspect_container(container) @@ -426,7 +426,7 @@ def test_create_with_storage_opt(self): storage_opt={'size': '120G'} ) container = self.client.create_container( - BUSYBOX, ['echo', 'test'], host_config=host_config + TEST_IMG, ['echo', 'test'], host_config=host_config ) self.tmp_containers.append(container) config = self.client.inspect_container(container) @@ -437,7 +437,7 @@ def test_create_with_storage_opt(self): @requires_api_version('1.25') def test_create_with_init(self): ctnr = self.client.create_container( - BUSYBOX, 'true', + TEST_IMG, 'true', host_config=self.client.create_host_config( init=True ) @@ -451,7 +451,7 @@ def test_create_with_init(self): reason='CONFIG_RT_GROUP_SCHED isn\'t enabled') def test_create_with_cpu_rt_options(self): ctnr = self.client.create_container( - BUSYBOX, 'true', host_config=self.client.create_host_config( + TEST_IMG, 'true', host_config=self.client.create_host_config( cpu_rt_period=1000, cpu_rt_runtime=500 ) ) @@ -464,7 +464,7 @@ def test_create_with_cpu_rt_options(self): def test_create_with_device_cgroup_rules(self): rule = 'c 7:128 rwm' ctnr = self.client.create_container( - BUSYBOX, 'cat /sys/fs/cgroup/devices/devices.list', + TEST_IMG, 'cat /sys/fs/cgroup/devices/devices.list', host_config=self.client.create_host_config( device_cgroup_rules=[rule] ) @@ -477,7 +477,7 @@ def test_create_with_device_cgroup_rules(self): def test_create_with_uts_mode(self): container = self.client.create_container( - BUSYBOX, ['echo'], host_config=self.client.create_host_config( + TEST_IMG, ['echo'], host_config=self.client.create_host_config( uts_mode='host' ) ) @@ -501,7 +501,7 @@ def setUp(self): self.run_with_volume( False, - BUSYBOX, + TEST_IMG, ['touch', os.path.join(self.mount_dest, self.filename)], ) @@ -509,7 +509,7 @@ def test_create_with_binds_rw(self): container = self.run_with_volume( False, - BUSYBOX, + TEST_IMG, ['ls', self.mount_dest], ) logs = self.client.logs(container) @@ -523,12 +523,12 @@ def test_create_with_binds_rw(self): def test_create_with_binds_ro(self): self.run_with_volume( False, - BUSYBOX, + TEST_IMG, ['touch', os.path.join(self.mount_dest, self.filename)], ) container = self.run_with_volume( True, - BUSYBOX, + TEST_IMG, ['ls', self.mount_dest], ) logs = self.client.logs(container) @@ -547,7 +547,7 @@ def test_create_with_mounts(self): ) host_config = self.client.create_host_config(mounts=[mount]) container = self.run_container( - BUSYBOX, ['ls', self.mount_dest], + TEST_IMG, ['ls', self.mount_dest], host_config=host_config ) assert container @@ -566,7 +566,7 @@ def test_create_with_mounts_ro(self): ) host_config = self.client.create_host_config(mounts=[mount]) container = self.run_container( - BUSYBOX, ['ls', self.mount_dest], + TEST_IMG, ['ls', self.mount_dest], host_config=host_config ) assert container @@ -585,7 +585,7 @@ def test_create_with_volume_mount(self): ) host_config = self.client.create_host_config(mounts=[mount]) container = self.client.create_container( - BUSYBOX, ['true'], host_config=host_config, + TEST_IMG, ['true'], host_config=host_config, ) assert container inspect_data = self.client.inspect_container(container) @@ -631,7 +631,7 @@ class ArchiveTest(BaseAPIIntegrationTest): def test_get_file_archive_from_container(self): data = 'The Maid and the Pocket Watch of Blood' ctnr = self.client.create_container( - BUSYBOX, 'sh -c "echo {0} > /vol1/data.txt"'.format(data), + TEST_IMG, 'sh -c "echo {0} > /vol1/data.txt"'.format(data), volumes=['/vol1'] ) self.tmp_containers.append(ctnr) @@ -650,7 +650,7 @@ def test_get_file_archive_from_container(self): def test_get_file_stat_from_container(self): data = 'The Maid and the Pocket Watch of Blood' ctnr = self.client.create_container( - BUSYBOX, 'sh -c "echo -n {0} > /vol1/data.txt"'.format(data), + TEST_IMG, 'sh -c "echo -n {0} > /vol1/data.txt"'.format(data), volumes=['/vol1'] ) self.tmp_containers.append(ctnr) @@ -668,7 +668,7 @@ def test_copy_file_to_container(self): test_file.write(data) test_file.seek(0) ctnr = self.client.create_container( - BUSYBOX, + TEST_IMG, 'cat {0}'.format( os.path.join('/vol1/', os.path.basename(test_file.name)) ), @@ -690,7 +690,7 @@ def test_copy_directory_to_container(self): dirs = ['foo', 'bar'] base = helpers.make_tree(dirs, files) ctnr = self.client.create_container( - BUSYBOX, 'ls -p /vol1', volumes=['/vol1'] + TEST_IMG, 'ls -p /vol1', volumes=['/vol1'] ) self.tmp_containers.append(ctnr) with docker.utils.tar(base) as test_tar: @@ -711,7 +711,7 @@ class RenameContainerTest(BaseAPIIntegrationTest): def test_rename_container(self): version = self.client.version()['Version'] name = 'hong_meiling' - res = self.client.create_container(BUSYBOX, 'true') + res = self.client.create_container(TEST_IMG, 'true') assert 'Id' in res self.tmp_containers.append(res['Id']) self.client.rename(res, name) @@ -725,7 +725,7 @@ def test_rename_container(self): class StartContainerTest(BaseAPIIntegrationTest): def test_start_container(self): - res = self.client.create_container(BUSYBOX, 'true') + res = self.client.create_container(TEST_IMG, 'true') assert 'Id' in res self.tmp_containers.append(res['Id']) self.client.start(res['Id']) @@ -741,7 +741,7 @@ def test_start_container(self): assert inspect['State']['ExitCode'] == 0 def test_start_container_with_dict_instead_of_id(self): - res = self.client.create_container(BUSYBOX, 'true') + res = self.client.create_container(TEST_IMG, 'true') assert 'Id' in res self.tmp_containers.append(res['Id']) self.client.start(res) @@ -769,7 +769,7 @@ def test_run_shlex_commands(self): 'true && echo "Night of Nights"' ] for cmd in commands: - container = self.client.create_container(BUSYBOX, cmd) + container = self.client.create_container(TEST_IMG, cmd) id = container['Id'] self.client.start(id) self.tmp_containers.append(id) @@ -779,7 +779,7 @@ def test_run_shlex_commands(self): class WaitTest(BaseAPIIntegrationTest): def test_wait(self): - res = self.client.create_container(BUSYBOX, ['sleep', '3']) + res = self.client.create_container(TEST_IMG, ['sleep', '3']) id = res['Id'] self.tmp_containers.append(id) self.client.start(id) @@ -792,7 +792,7 @@ def test_wait(self): assert inspect['State']['ExitCode'] == exitcode def test_wait_with_dict_instead_of_id(self): - res = self.client.create_container(BUSYBOX, ['sleep', '3']) + res = self.client.create_container(TEST_IMG, ['sleep', '3']) id = res['Id'] self.tmp_containers.append(id) self.client.start(res) @@ -806,13 +806,13 @@ def test_wait_with_dict_instead_of_id(self): @requires_api_version('1.30') def test_wait_with_condition(self): - ctnr = self.client.create_container(BUSYBOX, 'true') + ctnr = self.client.create_container(TEST_IMG, 'true') self.tmp_containers.append(ctnr) with pytest.raises(requests.exceptions.ConnectionError): self.client.wait(ctnr, condition='removed', timeout=1) ctnr = self.client.create_container( - BUSYBOX, ['sleep', '3'], + TEST_IMG, ['sleep', '3'], host_config=self.client.create_host_config(auto_remove=True) ) self.tmp_containers.append(ctnr) @@ -826,7 +826,7 @@ class LogsTest(BaseAPIIntegrationTest): def test_logs(self): snippet = 'Flowering Nights (Sakuya Iyazoi)' container = self.client.create_container( - BUSYBOX, 'echo {0}'.format(snippet) + TEST_IMG, 'echo {0}'.format(snippet) ) id = container['Id'] self.tmp_containers.append(id) @@ -840,7 +840,7 @@ def test_logs_tail_option(self): snippet = '''Line1 Line2''' container = self.client.create_container( - BUSYBOX, 'echo "{0}"'.format(snippet) + TEST_IMG, 'echo "{0}"'.format(snippet) ) id = container['Id'] self.tmp_containers.append(id) @@ -853,7 +853,7 @@ def test_logs_tail_option(self): def test_logs_streaming_and_follow(self): snippet = 'Flowering Nights (Sakuya Iyazoi)' container = self.client.create_container( - BUSYBOX, 'echo {0}'.format(snippet) + TEST_IMG, 'echo {0}'.format(snippet) ) id = container['Id'] self.tmp_containers.append(id) @@ -873,7 +873,7 @@ def test_logs_streaming_and_follow(self): def test_logs_streaming_and_follow_and_cancel(self): snippet = 'Flowering Nights (Sakuya Iyazoi)' container = self.client.create_container( - BUSYBOX, 'sh -c "echo \\"{0}\\" && sleep 3"'.format(snippet) + TEST_IMG, 'sh -c "echo \\"{0}\\" && sleep 3"'.format(snippet) ) id = container['Id'] self.tmp_containers.append(id) @@ -891,7 +891,7 @@ def test_logs_streaming_and_follow_and_cancel(self): def test_logs_with_dict_instead_of_id(self): snippet = 'Flowering Nights (Sakuya Iyazoi)' container = self.client.create_container( - BUSYBOX, 'echo {0}'.format(snippet) + TEST_IMG, 'echo {0}'.format(snippet) ) id = container['Id'] self.tmp_containers.append(id) @@ -904,7 +904,7 @@ def test_logs_with_dict_instead_of_id(self): def test_logs_with_tail_0(self): snippet = 'Flowering Nights (Sakuya Iyazoi)' container = self.client.create_container( - BUSYBOX, 'echo "{0}"'.format(snippet) + TEST_IMG, 'echo "{0}"'.format(snippet) ) id = container['Id'] self.tmp_containers.append(id) @@ -918,7 +918,7 @@ def test_logs_with_tail_0(self): def test_logs_with_until(self): snippet = 'Shanghai Teahouse (Hong Meiling)' container = self.client.create_container( - BUSYBOX, 'echo "{0}"'.format(snippet) + TEST_IMG, 'echo "{0}"'.format(snippet) ) self.tmp_containers.append(container) @@ -933,7 +933,7 @@ def test_logs_with_until(self): class DiffTest(BaseAPIIntegrationTest): def test_diff(self): - container = self.client.create_container(BUSYBOX, ['touch', '/test']) + container = self.client.create_container(TEST_IMG, ['touch', '/test']) id = container['Id'] self.client.start(id) self.tmp_containers.append(id) @@ -946,7 +946,7 @@ def test_diff(self): assert test_diff[0]['Kind'] == 1 def test_diff_with_dict_instead_of_id(self): - container = self.client.create_container(BUSYBOX, ['touch', '/test']) + container = self.client.create_container(TEST_IMG, ['touch', '/test']) id = container['Id'] self.client.start(id) self.tmp_containers.append(id) @@ -961,7 +961,7 @@ def test_diff_with_dict_instead_of_id(self): class StopTest(BaseAPIIntegrationTest): def test_stop(self): - container = self.client.create_container(BUSYBOX, ['sleep', '9999']) + container = self.client.create_container(TEST_IMG, ['sleep', '9999']) id = container['Id'] self.client.start(id) self.tmp_containers.append(id) @@ -973,7 +973,7 @@ def test_stop(self): assert state['Running'] is False def test_stop_with_dict_instead_of_id(self): - container = self.client.create_container(BUSYBOX, ['sleep', '9999']) + container = self.client.create_container(TEST_IMG, ['sleep', '9999']) assert 'Id' in container id = container['Id'] self.client.start(container) @@ -988,7 +988,7 @@ def test_stop_with_dict_instead_of_id(self): class KillTest(BaseAPIIntegrationTest): def test_kill(self): - container = self.client.create_container(BUSYBOX, ['sleep', '9999']) + container = self.client.create_container(TEST_IMG, ['sleep', '9999']) id = container['Id'] self.client.start(id) self.tmp_containers.append(id) @@ -1002,7 +1002,7 @@ def test_kill(self): assert state['Running'] is False def test_kill_with_dict_instead_of_id(self): - container = self.client.create_container(BUSYBOX, ['sleep', '9999']) + container = self.client.create_container(TEST_IMG, ['sleep', '9999']) id = container['Id'] self.client.start(id) self.tmp_containers.append(id) @@ -1016,7 +1016,7 @@ def test_kill_with_dict_instead_of_id(self): assert state['Running'] is False def test_kill_with_signal(self): - id = self.client.create_container(BUSYBOX, ['sleep', '60']) + id = self.client.create_container(TEST_IMG, ['sleep', '60']) self.tmp_containers.append(id) self.client.start(id) self.client.kill( @@ -1033,7 +1033,7 @@ def test_kill_with_signal(self): assert state['Running'] is False, state def test_kill_with_signal_name(self): - id = self.client.create_container(BUSYBOX, ['sleep', '60']) + id = self.client.create_container(TEST_IMG, ['sleep', '60']) self.client.start(id) self.tmp_containers.append(id) self.client.kill(id, signal='SIGKILL') @@ -1048,7 +1048,7 @@ def test_kill_with_signal_name(self): assert state['Running'] is False, state def test_kill_with_signal_integer(self): - id = self.client.create_container(BUSYBOX, ['sleep', '60']) + id = self.client.create_container(TEST_IMG, ['sleep', '60']) self.client.start(id) self.tmp_containers.append(id) self.client.kill(id, signal=9) @@ -1077,7 +1077,7 @@ def test_port(self): ] container = self.client.create_container( - BUSYBOX, ['sleep', '60'], ports=ports, + TEST_IMG, ['sleep', '60'], ports=ports, host_config=self.client.create_host_config( port_bindings=port_bindings, network_mode='bridge' ) @@ -1104,7 +1104,7 @@ def test_port(self): class ContainerTopTest(BaseAPIIntegrationTest): def test_top(self): container = self.client.create_container( - BUSYBOX, ['sleep', '60'] + TEST_IMG, ['sleep', '60'] ) self.tmp_containers.append(container) @@ -1124,7 +1124,7 @@ def test_top(self): ) def test_top_with_psargs(self): container = self.client.create_container( - BUSYBOX, ['sleep', '60']) + TEST_IMG, ['sleep', '60']) self.tmp_containers.append(container) @@ -1140,7 +1140,7 @@ def test_top_with_psargs(self): class RestartContainerTest(BaseAPIIntegrationTest): def test_restart(self): - container = self.client.create_container(BUSYBOX, ['sleep', '9999']) + container = self.client.create_container(TEST_IMG, ['sleep', '9999']) id = container['Id'] self.client.start(id) self.tmp_containers.append(id) @@ -1159,7 +1159,7 @@ def test_restart(self): self.client.kill(id) def test_restart_with_low_timeout(self): - container = self.client.create_container(BUSYBOX, ['sleep', '9999']) + container = self.client.create_container(TEST_IMG, ['sleep', '9999']) self.client.start(container) self.client.timeout = 3 self.client.restart(container, timeout=1) @@ -1168,7 +1168,7 @@ def test_restart_with_low_timeout(self): self.client.kill(container) def test_restart_with_dict_instead_of_id(self): - container = self.client.create_container(BUSYBOX, ['sleep', '9999']) + container = self.client.create_container(TEST_IMG, ['sleep', '9999']) assert 'Id' in container id = container['Id'] self.client.start(container) @@ -1190,7 +1190,7 @@ def test_restart_with_dict_instead_of_id(self): class RemoveContainerTest(BaseAPIIntegrationTest): def test_remove(self): - container = self.client.create_container(BUSYBOX, ['true']) + container = self.client.create_container(TEST_IMG, ['true']) id = container['Id'] self.client.start(id) self.client.wait(id) @@ -1200,7 +1200,7 @@ def test_remove(self): assert len(res) == 0 def test_remove_with_dict_instead_of_id(self): - container = self.client.create_container(BUSYBOX, ['true']) + container = self.client.create_container(TEST_IMG, ['true']) id = container['Id'] self.client.start(id) self.client.wait(id) @@ -1212,7 +1212,7 @@ def test_remove_with_dict_instead_of_id(self): class AttachContainerTest(BaseAPIIntegrationTest): def test_run_container_streaming(self): - container = self.client.create_container(BUSYBOX, '/bin/sh', + container = self.client.create_container(TEST_IMG, '/bin/sh', detach=True, stdin_open=True) id = container['Id'] self.tmp_containers.append(id) @@ -1224,7 +1224,7 @@ def test_run_container_reading_socket(self): line = 'hi there and stuff and things, words!' # `echo` appends CRLF, `printf` doesn't command = "printf '{0}'".format(line) - container = self.client.create_container(BUSYBOX, command, + container = self.client.create_container(TEST_IMG, command, detach=True, tty=False) self.tmp_containers.append(container) @@ -1242,7 +1242,7 @@ def test_run_container_reading_socket(self): def test_attach_no_stream(self): container = self.client.create_container( - BUSYBOX, 'echo hello' + TEST_IMG, 'echo hello' ) self.tmp_containers.append(container) self.client.start(container) @@ -1257,7 +1257,7 @@ def test_attach_no_stream(self): reason='Flaky test on TLS') def test_attach_stream_and_cancel(self): container = self.client.create_container( - BUSYBOX, 'sh -c "sleep 2 && echo hello && sleep 60"', + TEST_IMG, 'sh -c "sleep 2 && echo hello && sleep 60"', tty=True ) self.tmp_containers.append(container) @@ -1275,7 +1275,7 @@ def test_attach_stream_and_cancel(self): def test_detach_with_default(self): container = self.client.create_container( - BUSYBOX, 'cat', + TEST_IMG, 'cat', detach=True, stdin_open=True, tty=True ) self.tmp_containers.append(container) @@ -1294,7 +1294,7 @@ def test_detach_with_config_file(self): self.client._general_configs['detachKeys'] = 'ctrl-p' container = self.client.create_container( - BUSYBOX, 'cat', + TEST_IMG, 'cat', detach=True, stdin_open=True, tty=True ) self.tmp_containers.append(container) @@ -1311,7 +1311,7 @@ def test_detach_with_arg(self): self.client._general_configs['detachKeys'] = 'ctrl-p' container = self.client.create_container( - BUSYBOX, 'cat', + TEST_IMG, 'cat', detach=True, stdin_open=True, tty=True ) self.tmp_containers.append(container) @@ -1327,7 +1327,7 @@ def test_detach_with_arg(self): class PauseTest(BaseAPIIntegrationTest): def test_pause_unpause(self): - container = self.client.create_container(BUSYBOX, ['sleep', '9999']) + container = self.client.create_container(TEST_IMG, ['sleep', '9999']) id = container['Id'] self.tmp_containers.append(id) self.client.start(container) @@ -1358,9 +1358,9 @@ class PruneTest(BaseAPIIntegrationTest): @requires_api_version('1.25') def test_prune_containers(self): container1 = self.client.create_container( - BUSYBOX, ['sh', '-c', 'echo hello > /data.txt'] + TEST_IMG, ['sh', '-c', 'echo hello > /data.txt'] ) - container2 = self.client.create_container(BUSYBOX, ['sleep', '9999']) + container2 = self.client.create_container(TEST_IMG, ['sleep', '9999']) self.client.start(container1) self.client.start(container2) self.client.wait(container1) @@ -1373,7 +1373,7 @@ def test_prune_containers(self): class GetContainerStatsTest(BaseAPIIntegrationTest): def test_get_container_stats_no_stream(self): container = self.client.create_container( - BUSYBOX, ['sleep', '60'], + TEST_IMG, ['sleep', '60'], ) self.tmp_containers.append(container) self.client.start(container) @@ -1387,7 +1387,7 @@ def test_get_container_stats_no_stream(self): def test_get_container_stats_stream(self): container = self.client.create_container( - BUSYBOX, ['sleep', '60'], + TEST_IMG, ['sleep', '60'], ) self.tmp_containers.append(container) self.client.start(container) @@ -1405,7 +1405,7 @@ def test_update_container(self): old_mem_limit = 400 * 1024 * 1024 new_mem_limit = 300 * 1024 * 1024 container = self.client.create_container( - BUSYBOX, 'top', host_config=self.client.create_host_config( + TEST_IMG, 'top', host_config=self.client.create_host_config( mem_limit=old_mem_limit ) ) @@ -1426,7 +1426,7 @@ def test_restart_policy_update(self): 'Name': 'on-failure' } container = self.client.create_container( - BUSYBOX, ['sleep', '60'], + TEST_IMG, ['sleep', '60'], host_config=self.client.create_host_config( restart_policy=old_restart_policy ) @@ -1450,7 +1450,7 @@ class ContainerCPUTest(BaseAPIIntegrationTest): def test_container_cpu_shares(self): cpu_shares = 512 container = self.client.create_container( - BUSYBOX, 'ls', host_config=self.client.create_host_config( + TEST_IMG, 'ls', host_config=self.client.create_host_config( cpu_shares=cpu_shares ) ) @@ -1462,7 +1462,7 @@ def test_container_cpu_shares(self): def test_container_cpuset(self): cpuset_cpus = "0,1" container = self.client.create_container( - BUSYBOX, 'ls', host_config=self.client.create_host_config( + TEST_IMG, 'ls', host_config=self.client.create_host_config( cpuset_cpus=cpuset_cpus ) ) @@ -1474,7 +1474,7 @@ def test_container_cpuset(self): @requires_api_version('1.25') def test_create_with_runtime(self): container = self.client.create_container( - BUSYBOX, ['echo', 'test'], runtime='runc' + TEST_IMG, ['echo', 'test'], runtime='runc' ) self.tmp_containers.append(container['Id']) config = self.client.inspect_container(container) @@ -1485,7 +1485,7 @@ class LinkTest(BaseAPIIntegrationTest): def test_remove_link(self): # Create containers container1 = self.client.create_container( - BUSYBOX, 'cat', detach=True, stdin_open=True + TEST_IMG, 'cat', detach=True, stdin_open=True ) container1_id = container1['Id'] self.tmp_containers.append(container1_id) @@ -1497,7 +1497,7 @@ def test_remove_link(self): link_alias = 'mylink' container2 = self.client.create_container( - BUSYBOX, 'cat', host_config=self.client.create_host_config( + TEST_IMG, 'cat', host_config=self.client.create_host_config( links={link_path: link_alias} ) ) diff --git a/tests/integration/api_exec_test.py b/tests/integration/api_exec_test.py index 53b7e22fea..554e8629e5 100644 --- a/tests/integration/api_exec_test.py +++ b/tests/integration/api_exec_test.py @@ -2,7 +2,7 @@ from ..helpers import ctrl_with from ..helpers import requires_api_version from .base import BaseAPIIntegrationTest -from .base import BUSYBOX +from .base import TEST_IMG from docker.utils.proxy import ProxyConfig from docker.utils.socket import next_frame_header from docker.utils.socket import read_exactly @@ -16,7 +16,7 @@ def test_execute_command_with_proxy_env(self): ) container = self.client.create_container( - BUSYBOX, 'cat', detach=True, stdin_open=True, + TEST_IMG, 'cat', detach=True, stdin_open=True, ) self.client.start(container) self.tmp_containers.append(container) @@ -48,7 +48,7 @@ def test_execute_command_with_proxy_env(self): assert item in output def test_execute_command(self): - container = self.client.create_container(BUSYBOX, 'cat', + container = self.client.create_container(TEST_IMG, 'cat', detach=True, stdin_open=True) id = container['Id'] self.client.start(id) @@ -61,7 +61,7 @@ def test_execute_command(self): assert exec_log == b'hello\n' def test_exec_command_string(self): - container = self.client.create_container(BUSYBOX, 'cat', + container = self.client.create_container(TEST_IMG, 'cat', detach=True, stdin_open=True) id = container['Id'] self.client.start(id) @@ -74,7 +74,7 @@ def test_exec_command_string(self): assert exec_log == b'hello world\n' def test_exec_command_as_user(self): - container = self.client.create_container(BUSYBOX, 'cat', + container = self.client.create_container(TEST_IMG, 'cat', detach=True, stdin_open=True) id = container['Id'] self.client.start(id) @@ -87,7 +87,7 @@ def test_exec_command_as_user(self): assert exec_log == b'postgres\n' def test_exec_command_as_root(self): - container = self.client.create_container(BUSYBOX, 'cat', + container = self.client.create_container(TEST_IMG, 'cat', detach=True, stdin_open=True) id = container['Id'] self.client.start(id) @@ -100,7 +100,7 @@ def test_exec_command_as_root(self): assert exec_log == b'root\n' def test_exec_command_streaming(self): - container = self.client.create_container(BUSYBOX, 'cat', + container = self.client.create_container(TEST_IMG, 'cat', detach=True, stdin_open=True) id = container['Id'] self.tmp_containers.append(id) @@ -115,7 +115,7 @@ def test_exec_command_streaming(self): assert res == b'hello\nworld\n' def test_exec_start_socket(self): - container = self.client.create_container(BUSYBOX, 'cat', + container = self.client.create_container(TEST_IMG, 'cat', detach=True, stdin_open=True) container_id = container['Id'] self.client.start(container_id) @@ -137,7 +137,7 @@ def test_exec_start_socket(self): assert data.decode('utf-8') == line def test_exec_start_detached(self): - container = self.client.create_container(BUSYBOX, 'cat', + container = self.client.create_container(TEST_IMG, 'cat', detach=True, stdin_open=True) container_id = container['Id'] self.client.start(container_id) @@ -152,7 +152,7 @@ def test_exec_start_detached(self): assert response == "" def test_exec_inspect(self): - container = self.client.create_container(BUSYBOX, 'cat', + container = self.client.create_container(TEST_IMG, 'cat', detach=True, stdin_open=True) id = container['Id'] self.client.start(id) @@ -167,7 +167,7 @@ def test_exec_inspect(self): @requires_api_version('1.25') def test_exec_command_with_env(self): - container = self.client.create_container(BUSYBOX, 'cat', + container = self.client.create_container(TEST_IMG, 'cat', detach=True, stdin_open=True) id = container['Id'] self.client.start(id) @@ -182,7 +182,7 @@ def test_exec_command_with_env(self): @requires_api_version('1.35') def test_exec_command_with_workdir(self): container = self.client.create_container( - BUSYBOX, 'cat', detach=True, stdin_open=True + TEST_IMG, 'cat', detach=True, stdin_open=True ) self.tmp_containers.append(container) self.client.start(container) @@ -193,7 +193,7 @@ def test_exec_command_with_workdir(self): def test_detach_with_default(self): container = self.client.create_container( - BUSYBOX, 'cat', detach=True, stdin_open=True + TEST_IMG, 'cat', detach=True, stdin_open=True ) id = container['Id'] self.client.start(id) @@ -212,7 +212,7 @@ def test_detach_with_default(self): def test_detach_with_config_file(self): self.client._general_configs['detachKeys'] = 'ctrl-p' container = self.client.create_container( - BUSYBOX, 'cat', detach=True, stdin_open=True + TEST_IMG, 'cat', detach=True, stdin_open=True ) id = container['Id'] self.client.start(id) @@ -241,7 +241,7 @@ class ExecDemuxTest(BaseAPIIntegrationTest): def setUp(self): super(ExecDemuxTest, self).setUp() self.container = self.client.create_container( - BUSYBOX, 'cat', detach=True, stdin_open=True + TEST_IMG, 'cat', detach=True, stdin_open=True ) self.client.start(self.container) self.tmp_containers.append(self.container) diff --git a/tests/integration/api_healthcheck_test.py b/tests/integration/api_healthcheck_test.py index 5dbac3769f..c54583b0be 100644 --- a/tests/integration/api_healthcheck_test.py +++ b/tests/integration/api_healthcheck_test.py @@ -1,4 +1,4 @@ -from .base import BaseAPIIntegrationTest, BUSYBOX +from .base import BaseAPIIntegrationTest, TEST_IMG from .. import helpers SECOND = 1000000000 @@ -16,7 +16,7 @@ class HealthcheckTest(BaseAPIIntegrationTest): @helpers.requires_api_version('1.24') def test_healthcheck_shell_command(self): container = self.client.create_container( - BUSYBOX, 'top', healthcheck=dict(test='echo "hello world"')) + TEST_IMG, 'top', healthcheck=dict(test='echo "hello world"')) self.tmp_containers.append(container) res = self.client.inspect_container(container) @@ -27,7 +27,7 @@ def test_healthcheck_shell_command(self): @helpers.requires_api_version('1.24') def test_healthcheck_passes(self): container = self.client.create_container( - BUSYBOX, 'top', healthcheck=dict( + TEST_IMG, 'top', healthcheck=dict( test="true", interval=1 * SECOND, timeout=1 * SECOND, @@ -40,7 +40,7 @@ def test_healthcheck_passes(self): @helpers.requires_api_version('1.24') def test_healthcheck_fails(self): container = self.client.create_container( - BUSYBOX, 'top', healthcheck=dict( + TEST_IMG, 'top', healthcheck=dict( test="false", interval=1 * SECOND, timeout=1 * SECOND, @@ -53,7 +53,7 @@ def test_healthcheck_fails(self): @helpers.requires_api_version('1.29') def test_healthcheck_start_period(self): container = self.client.create_container( - BUSYBOX, 'top', healthcheck=dict( + TEST_IMG, 'top', healthcheck=dict( test="echo 'x' >> /counter.txt && " "test `cat /counter.txt | wc -l` -ge 3", interval=1 * SECOND, diff --git a/tests/integration/api_image_test.py b/tests/integration/api_image_test.py index 56a7692489..2bc96abf46 100644 --- a/tests/integration/api_image_test.py +++ b/tests/integration/api_image_test.py @@ -15,7 +15,7 @@ import docker from ..helpers import requires_api_version, requires_experimental -from .base import BaseAPIIntegrationTest, BUSYBOX +from .base import BaseAPIIntegrationTest, TEST_IMG class ListImagesTest(BaseAPIIntegrationTest): @@ -77,7 +77,7 @@ def test_pull_invalid_platform(self): class CommitTest(BaseAPIIntegrationTest): def test_commit(self): - container = self.client.create_container(BUSYBOX, ['touch', '/test']) + container = self.client.create_container(TEST_IMG, ['touch', '/test']) id = container['Id'] self.client.start(id) self.tmp_containers.append(id) @@ -90,13 +90,13 @@ def test_commit(self): assert img['Container'].startswith(id) assert 'ContainerConfig' in img assert 'Image' in img['ContainerConfig'] - assert BUSYBOX == img['ContainerConfig']['Image'] - busybox_id = self.client.inspect_image(BUSYBOX)['Id'] + assert TEST_IMG == img['ContainerConfig']['Image'] + busybox_id = self.client.inspect_image(TEST_IMG)['Id'] assert 'Parent' in img assert img['Parent'] == busybox_id def test_commit_with_changes(self): - cid = self.client.create_container(BUSYBOX, ['touch', '/test']) + cid = self.client.create_container(TEST_IMG, ['touch', '/test']) self.tmp_containers.append(cid) self.client.start(cid) img_id = self.client.commit( @@ -112,7 +112,7 @@ def test_commit_with_changes(self): class RemoveImageTest(BaseAPIIntegrationTest): def test_remove(self): - container = self.client.create_container(BUSYBOX, ['touch', '/test']) + container = self.client.create_container(TEST_IMG, ['touch', '/test']) id = container['Id'] self.client.start(id) self.tmp_containers.append(id) @@ -319,7 +319,7 @@ def test_prune_images(self): pass # Ensure busybox does not get pruned - ctnr = self.client.create_container(BUSYBOX, ['sleep', '9999']) + ctnr = self.client.create_container(TEST_IMG, ['sleep', '9999']) self.tmp_containers.append(ctnr) self.client.pull('hello-world', tag='latest') @@ -343,7 +343,7 @@ class SaveLoadImagesTest(BaseAPIIntegrationTest): @requires_api_version('1.23') def test_get_image_load_image(self): with tempfile.TemporaryFile() as f: - stream = self.client.get_image(BUSYBOX) + stream = self.client.get_image(TEST_IMG) for chunk in stream: f.write(chunk) @@ -351,7 +351,7 @@ def test_get_image_load_image(self): result = self.client.load_image(f.read()) success = False - result_line = 'Loaded image: {}\n'.format(BUSYBOX) + result_line = 'Loaded image: {}\n'.format(TEST_IMG) for data in result: print(data) if 'stream' in data: diff --git a/tests/integration/api_network_test.py b/tests/integration/api_network_test.py index db37cbd974..0f26827b17 100644 --- a/tests/integration/api_network_test.py +++ b/tests/integration/api_network_test.py @@ -3,7 +3,7 @@ import pytest from ..helpers import random_name, requires_api_version -from .base import BaseAPIIntegrationTest, BUSYBOX +from .base import BaseAPIIntegrationTest, TEST_IMG class TestNetworks(BaseAPIIntegrationTest): @@ -92,7 +92,7 @@ def test_remove_network(self): def test_connect_and_disconnect_container(self): net_name, net_id = self.create_network() - container = self.client.create_container(BUSYBOX, 'top') + container = self.client.create_container(TEST_IMG, 'top') self.tmp_containers.append(container) self.client.start(container) @@ -119,7 +119,7 @@ def test_connect_and_disconnect_container(self): def test_connect_and_force_disconnect_container(self): net_name, net_id = self.create_network() - container = self.client.create_container(BUSYBOX, 'top') + container = self.client.create_container(TEST_IMG, 'top') self.tmp_containers.append(container) self.client.start(container) @@ -144,7 +144,7 @@ def test_connect_and_force_disconnect_container(self): def test_connect_with_aliases(self): net_name, net_id = self.create_network() - container = self.client.create_container(BUSYBOX, 'top') + container = self.client.create_container(TEST_IMG, 'top') self.tmp_containers.append(container) self.client.start(container) @@ -161,7 +161,7 @@ def test_connect_on_container_create(self): net_name, net_id = self.create_network() container = self.client.create_container( - image=BUSYBOX, + image=TEST_IMG, command='top', host_config=self.client.create_host_config(network_mode=net_name), ) @@ -181,7 +181,7 @@ def test_create_with_aliases(self): net_name, net_id = self.create_network() container = self.client.create_container( - image=BUSYBOX, + image=TEST_IMG, command='top', host_config=self.client.create_host_config( network_mode=net_name, @@ -211,7 +211,7 @@ def test_create_with_ipv4_address(self): ), ) container = self.client.create_container( - image=BUSYBOX, command='top', + image=TEST_IMG, command='top', host_config=self.client.create_host_config(network_mode=net_name), networking_config=self.client.create_networking_config({ net_name: self.client.create_endpoint_config( @@ -237,7 +237,7 @@ def test_create_with_ipv6_address(self): ), ) container = self.client.create_container( - image=BUSYBOX, command='top', + image=TEST_IMG, command='top', host_config=self.client.create_host_config(network_mode=net_name), networking_config=self.client.create_networking_config({ net_name: self.client.create_endpoint_config( @@ -257,7 +257,7 @@ def test_create_with_ipv6_address(self): @requires_api_version('1.24') def test_create_with_linklocal_ips(self): container = self.client.create_container( - BUSYBOX, 'top', + TEST_IMG, 'top', networking_config=self.client.create_networking_config( { 'bridge': self.client.create_endpoint_config( diff --git a/tests/integration/api_service_test.py b/tests/integration/api_service_test.py index 71e0869e9f..c170a0a88f 100644 --- a/tests/integration/api_service_test.py +++ b/tests/integration/api_service_test.py @@ -10,7 +10,7 @@ from ..helpers import ( force_leave_swarm, requires_api_version, requires_experimental ) -from .base import BaseAPIIntegrationTest, BUSYBOX +from .base import BaseAPIIntegrationTest, TEST_IMG class ServiceTest(BaseAPIIntegrationTest): @@ -60,7 +60,7 @@ def create_simple_service(self, name=None, labels=None): name = self.get_service_name() container_spec = docker.types.ContainerSpec( - BUSYBOX, ['echo', 'hello'] + TEST_IMG, ['echo', 'hello'] ) task_tmpl = docker.types.TaskTemplate(container_spec) return name, self.client.create_service( @@ -156,7 +156,7 @@ def test_service_logs(self): def test_create_service_custom_log_driver(self): container_spec = docker.types.ContainerSpec( - BUSYBOX, ['echo', 'hello'] + TEST_IMG, ['echo', 'hello'] ) log_cfg = docker.types.DriverConfig('none') task_tmpl = docker.types.TaskTemplate( @@ -174,7 +174,7 @@ def test_create_service_custom_log_driver(self): def test_create_service_with_volume_mount(self): vol_name = self.get_service_name() container_spec = docker.types.ContainerSpec( - BUSYBOX, ['ls'], + TEST_IMG, ['ls'], mounts=[ docker.types.Mount(target='/test', source=vol_name) ] @@ -194,7 +194,7 @@ def test_create_service_with_volume_mount(self): assert mount['Type'] == 'volume' def test_create_service_with_resources_constraints(self): - container_spec = docker.types.ContainerSpec(BUSYBOX, ['true']) + container_spec = docker.types.ContainerSpec(TEST_IMG, ['true']) resources = docker.types.Resources( cpu_limit=4000000, mem_limit=3 * 1024 * 1024 * 1024, cpu_reservation=3500000, mem_reservation=2 * 1024 * 1024 * 1024 @@ -214,7 +214,7 @@ def test_create_service_with_resources_constraints(self): ] def _create_service_with_generic_resources(self, generic_resources): - container_spec = docker.types.ContainerSpec(BUSYBOX, ['true']) + container_spec = docker.types.ContainerSpec(TEST_IMG, ['true']) resources = docker.types.Resources( generic_resources=generic_resources @@ -265,7 +265,7 @@ def test_create_service_with_invalid_generic_resources(self): self._create_service_with_generic_resources(test_input) def test_create_service_with_update_config(self): - container_spec = docker.types.ContainerSpec(BUSYBOX, ['true']) + container_spec = docker.types.ContainerSpec(TEST_IMG, ['true']) task_tmpl = docker.types.TaskTemplate(container_spec) update_config = docker.types.UpdateConfig( parallelism=10, delay=5, failure_action='pause' @@ -283,7 +283,7 @@ def test_create_service_with_update_config(self): @requires_api_version('1.28') def test_create_service_with_failure_action_rollback(self): - container_spec = docker.types.ContainerSpec(BUSYBOX, ['true']) + container_spec = docker.types.ContainerSpec(TEST_IMG, ['true']) task_tmpl = docker.types.TaskTemplate(container_spec) update_config = docker.types.UpdateConfig(failure_action='rollback') name = self.get_service_name() @@ -314,7 +314,7 @@ def test_create_service_with_update_config_monitor(self): @requires_api_version('1.28') def test_create_service_with_rollback_config(self): - container_spec = docker.types.ContainerSpec(BUSYBOX, ['true']) + container_spec = docker.types.ContainerSpec(TEST_IMG, ['true']) task_tmpl = docker.types.TaskTemplate(container_spec) rollback_cfg = docker.types.RollbackConfig( parallelism=10, delay=5, failure_action='pause', @@ -334,7 +334,7 @@ def test_create_service_with_rollback_config(self): assert rollback_cfg['MaxFailureRatio'] == rc['MaxFailureRatio'] def test_create_service_with_restart_policy(self): - container_spec = docker.types.ContainerSpec(BUSYBOX, ['true']) + container_spec = docker.types.ContainerSpec(TEST_IMG, ['true']) policy = docker.types.RestartPolicy( docker.types.RestartPolicy.condition_types.ANY, delay=5, max_attempts=5 @@ -357,7 +357,7 @@ def test_create_service_with_custom_networks(self): 'dockerpytest_2', driver='overlay', ipam={'Driver': 'default'} ) self.tmp_networks.append(net2['Id']) - container_spec = docker.types.ContainerSpec(BUSYBOX, ['true']) + container_spec = docker.types.ContainerSpec(TEST_IMG, ['true']) task_tmpl = docker.types.TaskTemplate(container_spec) name = self.get_service_name() svc_id = self.client.create_service( @@ -373,7 +373,7 @@ def test_create_service_with_custom_networks(self): def test_create_service_with_placement(self): node_id = self.client.nodes()[0]['ID'] - container_spec = docker.types.ContainerSpec(BUSYBOX, ['true']) + container_spec = docker.types.ContainerSpec(TEST_IMG, ['true']) task_tmpl = docker.types.TaskTemplate( container_spec, placement=['node.id=={}'.format(node_id)] ) @@ -386,7 +386,7 @@ def test_create_service_with_placement(self): def test_create_service_with_placement_object(self): node_id = self.client.nodes()[0]['ID'] - container_spec = docker.types.ContainerSpec(BUSYBOX, ['true']) + container_spec = docker.types.ContainerSpec(TEST_IMG, ['true']) placemt = docker.types.Placement( constraints=['node.id=={}'.format(node_id)] ) @@ -401,7 +401,7 @@ def test_create_service_with_placement_object(self): @requires_api_version('1.30') def test_create_service_with_placement_platform(self): - container_spec = docker.types.ContainerSpec(BUSYBOX, ['true']) + container_spec = docker.types.ContainerSpec(TEST_IMG, ['true']) placemt = docker.types.Placement(platforms=[('x86_64', 'linux')]) task_tmpl = docker.types.TaskTemplate( container_spec, placement=placemt @@ -414,7 +414,7 @@ def test_create_service_with_placement_platform(self): @requires_api_version('1.27') def test_create_service_with_placement_preferences(self): - container_spec = docker.types.ContainerSpec(BUSYBOX, ['true']) + container_spec = docker.types.ContainerSpec(TEST_IMG, ['true']) placemt = docker.types.Placement(preferences=[ {'Spread': {'SpreadDescriptor': 'com.dockerpy.test'}} ]) @@ -429,7 +429,7 @@ def test_create_service_with_placement_preferences(self): @requires_api_version('1.27') def test_create_service_with_placement_preferences_tuple(self): - container_spec = docker.types.ContainerSpec(BUSYBOX, ['true']) + container_spec = docker.types.ContainerSpec(TEST_IMG, ['true']) placemt = docker.types.Placement(preferences=( ('spread', 'com.dockerpy.test'), )) @@ -443,7 +443,7 @@ def test_create_service_with_placement_preferences_tuple(self): assert svc_info['Spec']['TaskTemplate']['Placement'] == placemt def test_create_service_with_endpoint_spec(self): - container_spec = docker.types.ContainerSpec(BUSYBOX, ['true']) + container_spec = docker.types.ContainerSpec(TEST_IMG, ['true']) task_tmpl = docker.types.TaskTemplate(container_spec) name = self.get_service_name() endpoint_spec = docker.types.EndpointSpec(ports={ @@ -473,7 +473,7 @@ def test_create_service_with_endpoint_spec(self): @requires_api_version('1.32') def test_create_service_with_endpoint_spec_host_publish_mode(self): - container_spec = docker.types.ContainerSpec(BUSYBOX, ['true']) + container_spec = docker.types.ContainerSpec(TEST_IMG, ['true']) task_tmpl = docker.types.TaskTemplate(container_spec) name = self.get_service_name() endpoint_spec = docker.types.EndpointSpec(ports={ @@ -493,7 +493,7 @@ def test_create_service_with_endpoint_spec_host_publish_mode(self): def test_create_service_with_env(self): container_spec = docker.types.ContainerSpec( - BUSYBOX, ['true'], env={'DOCKER_PY_TEST': 1} + TEST_IMG, ['true'], env={'DOCKER_PY_TEST': 1} ) task_tmpl = docker.types.TaskTemplate( container_spec, @@ -509,7 +509,7 @@ def test_create_service_with_env(self): @requires_api_version('1.29') def test_create_service_with_update_order(self): - container_spec = docker.types.ContainerSpec(BUSYBOX, ['true']) + container_spec = docker.types.ContainerSpec(TEST_IMG, ['true']) task_tmpl = docker.types.TaskTemplate(container_spec) update_config = docker.types.UpdateConfig( parallelism=10, delay=5, order='start-first' @@ -528,7 +528,7 @@ def test_create_service_with_update_order(self): @requires_api_version('1.25') def test_create_service_with_tty(self): container_spec = docker.types.ContainerSpec( - BUSYBOX, ['true'], tty=True + TEST_IMG, ['true'], tty=True ) task_tmpl = docker.types.TaskTemplate( container_spec, @@ -545,7 +545,7 @@ def test_create_service_with_tty(self): @requires_api_version('1.25') def test_create_service_with_tty_dict(self): container_spec = { - 'Image': BUSYBOX, + 'Image': TEST_IMG, 'Command': ['true'], 'TTY': True } @@ -561,7 +561,7 @@ def test_create_service_with_tty_dict(self): def test_create_service_global_mode(self): container_spec = docker.types.ContainerSpec( - BUSYBOX, ['echo', 'hello'] + TEST_IMG, ['echo', 'hello'] ) task_tmpl = docker.types.TaskTemplate(container_spec) name = self.get_service_name() @@ -574,7 +574,7 @@ def test_create_service_global_mode(self): def test_create_service_replicated_mode(self): container_spec = docker.types.ContainerSpec( - BUSYBOX, ['echo', 'hello'] + TEST_IMG, ['echo', 'hello'] ) task_tmpl = docker.types.TaskTemplate(container_spec) name = self.get_service_name() @@ -767,7 +767,7 @@ def test_create_service_with_dns_config(self): search=['local'], options=['debug'] ) container_spec = docker.types.ContainerSpec( - BUSYBOX, ['sleep', '999'], dns_config=dns_config + TEST_IMG, ['sleep', '999'], dns_config=dns_config ) task_tmpl = docker.types.TaskTemplate(container_spec) name = self.get_service_name() @@ -787,7 +787,7 @@ def test_create_service_with_healthcheck(self): start_period=3 * second, interval=int(second / 2), ) container_spec = docker.types.ContainerSpec( - BUSYBOX, ['sleep', '999'], healthcheck=hc + TEST_IMG, ['sleep', '999'], healthcheck=hc ) task_tmpl = docker.types.TaskTemplate(container_spec) name = self.get_service_name() @@ -804,7 +804,7 @@ def test_create_service_with_healthcheck(self): @requires_api_version('1.28') def test_create_service_with_readonly(self): container_spec = docker.types.ContainerSpec( - BUSYBOX, ['sleep', '999'], read_only=True + TEST_IMG, ['sleep', '999'], read_only=True ) task_tmpl = docker.types.TaskTemplate(container_spec) name = self.get_service_name() @@ -818,7 +818,7 @@ def test_create_service_with_readonly(self): @requires_api_version('1.28') def test_create_service_with_stop_signal(self): container_spec = docker.types.ContainerSpec( - BUSYBOX, ['sleep', '999'], stop_signal='SIGINT' + TEST_IMG, ['sleep', '999'], stop_signal='SIGINT' ) task_tmpl = docker.types.TaskTemplate(container_spec) name = self.get_service_name() @@ -836,7 +836,7 @@ def test_create_service_with_stop_signal(self): def test_create_service_with_privileges(self): priv = docker.types.Privileges(selinux_disable=True) container_spec = docker.types.ContainerSpec( - BUSYBOX, ['sleep', '999'], privileges=priv + TEST_IMG, ['sleep', '999'], privileges=priv ) task_tmpl = docker.types.TaskTemplate(container_spec) name = self.get_service_name() @@ -992,7 +992,7 @@ def test_update_service_with_defaults_container_labels(self): assert labels['container.label'] == 'SampleLabel' def test_update_service_with_defaults_update_config(self): - container_spec = docker.types.ContainerSpec(BUSYBOX, ['true']) + container_spec = docker.types.ContainerSpec(TEST_IMG, ['true']) task_tmpl = docker.types.TaskTemplate(container_spec) update_config = docker.types.UpdateConfig( parallelism=10, delay=5, failure_action='pause' @@ -1031,7 +1031,7 @@ def test_update_service_with_defaults_networks(self): 'dockerpytest_2', driver='overlay', ipam={'Driver': 'default'} ) self.tmp_networks.append(net2['Id']) - container_spec = docker.types.ContainerSpec(BUSYBOX, ['true']) + container_spec = docker.types.ContainerSpec(TEST_IMG, ['true']) task_tmpl = docker.types.TaskTemplate(container_spec) name = self.get_service_name() svc_id = self.client.create_service( @@ -1070,7 +1070,7 @@ def test_update_service_with_defaults_networks(self): ] def test_update_service_with_defaults_endpoint_spec(self): - container_spec = docker.types.ContainerSpec(BUSYBOX, ['true']) + container_spec = docker.types.ContainerSpec(TEST_IMG, ['true']) task_tmpl = docker.types.TaskTemplate(container_spec) name = self.get_service_name() endpoint_spec = docker.types.EndpointSpec(ports={ @@ -1134,7 +1134,7 @@ def test_update_service_remove_healthcheck(self): start_period=3 * second, interval=int(second / 2), ) container_spec = docker.types.ContainerSpec( - BUSYBOX, ['sleep', '999'], healthcheck=hc + TEST_IMG, ['sleep', '999'], healthcheck=hc ) task_tmpl = docker.types.TaskTemplate(container_spec) name = self.get_service_name() @@ -1149,7 +1149,7 @@ def test_update_service_remove_healthcheck(self): ) container_spec = docker.types.ContainerSpec( - BUSYBOX, ['sleep', '999'], healthcheck={} + TEST_IMG, ['sleep', '999'], healthcheck={} ) task_tmpl = docker.types.TaskTemplate(container_spec) diff --git a/tests/integration/base.py b/tests/integration/base.py index 0ebf5b9911..a7613f6917 100644 --- a/tests/integration/base.py +++ b/tests/integration/base.py @@ -6,7 +6,7 @@ from .. import helpers from docker.utils import kwargs_from_env -BUSYBOX = 'alpine:3.9.3' # FIXME: this should probably be renamed +TEST_IMG = 'alpine:3.10' TEST_API_VERSION = os.environ.get('DOCKER_TEST_API_VERSION') @@ -108,7 +108,7 @@ def run_container(self, *args, **kwargs): return container - def create_and_start(self, image=BUSYBOX, command='top', **kwargs): + def create_and_start(self, image=TEST_IMG, command='top', **kwargs): container = self.client.create_container( image=image, command=command, **kwargs) self.tmp_containers.append(container) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 4e8d26831d..ec48835dcd 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -7,7 +7,7 @@ from docker.utils import kwargs_from_env import pytest -from .base import BUSYBOX +from .base import TEST_IMG @pytest.fixture(autouse=True, scope='session') @@ -15,15 +15,15 @@ def setup_test_session(): warnings.simplefilter('error') c = docker.APIClient(version='auto', **kwargs_from_env()) try: - c.inspect_image(BUSYBOX) + c.inspect_image(TEST_IMG) except docker.errors.NotFound: - print("\npulling {0}".format(BUSYBOX), file=sys.stderr) - for data in c.pull(BUSYBOX, stream=True, decode=True): + print("\npulling {0}".format(TEST_IMG), file=sys.stderr) + for data in c.pull(TEST_IMG, stream=True, decode=True): status = data.get("status") progress = data.get("progress") detail = "{0} - {1}".format(status, progress) print(detail, file=sys.stderr) # Double make sure we now have busybox - c.inspect_image(BUSYBOX) + c.inspect_image(TEST_IMG) c.close() diff --git a/tests/integration/errors_test.py b/tests/integration/errors_test.py index ac74d72100..7bf156afb0 100644 --- a/tests/integration/errors_test.py +++ b/tests/integration/errors_test.py @@ -1,11 +1,11 @@ from docker.errors import APIError -from .base import BaseAPIIntegrationTest, BUSYBOX +from .base import BaseAPIIntegrationTest, TEST_IMG import pytest class ErrorsTest(BaseAPIIntegrationTest): def test_api_error_parses_json(self): - container = self.client.create_container(BUSYBOX, ['sleep', '10']) + container = self.client.create_container(TEST_IMG, ['sleep', '10']) self.client.start(container['Id']) with pytest.raises(APIError) as cm: self.client.remove_container(container['Id']) diff --git a/tests/integration/models_images_test.py b/tests/integration/models_images_test.py index 31fab10968..375d972df9 100644 --- a/tests/integration/models_images_test.py +++ b/tests/integration/models_images_test.py @@ -4,7 +4,7 @@ import docker import pytest -from .base import BaseIntegrationTest, BUSYBOX, TEST_API_VERSION +from .base import BaseIntegrationTest, TEST_IMG, TEST_API_VERSION from ..helpers import random_name @@ -72,8 +72,8 @@ def test_pull(self): def test_pull_with_tag(self): client = docker.from_env(version=TEST_API_VERSION) - image = client.images.pull('alpine', tag='3.3') - assert 'alpine:3.3' in image.attrs['RepoTags'] + image = client.images.pull('alpine', tag='3.10') + assert 'alpine:3.10' in image.attrs['RepoTags'] def test_pull_with_sha(self): image_ref = ( @@ -97,7 +97,7 @@ def test_load_error(self): def test_save_and_load(self): client = docker.from_env(version=TEST_API_VERSION) - image = client.images.get(BUSYBOX) + image = client.images.get(TEST_IMG) with tempfile.TemporaryFile() as f: stream = image.save() for chunk in stream: @@ -111,7 +111,7 @@ def test_save_and_load(self): def test_save_and_load_repo_name(self): client = docker.from_env(version=TEST_API_VERSION) - image = client.images.get(BUSYBOX) + image = client.images.get(TEST_IMG) additional_tag = random_name() image.tag(additional_tag) self.tmp_imgs.append(additional_tag) @@ -131,7 +131,7 @@ def test_save_and_load_repo_name(self): def test_save_name_error(self): client = docker.from_env(version=TEST_API_VERSION) - image = client.images.get(BUSYBOX) + image = client.images.get(TEST_IMG) with pytest.raises(docker.errors.InvalidArgument): image.save(named='sakuya/izayoi') diff --git a/tests/integration/regression_test.py b/tests/integration/regression_test.py index 9aab076e30..a63883c4f5 100644 --- a/tests/integration/regression_test.py +++ b/tests/integration/regression_test.py @@ -4,7 +4,7 @@ import docker import six -from .base import BaseAPIIntegrationTest, BUSYBOX +from .base import BaseAPIIntegrationTest, TEST_IMG import pytest @@ -19,7 +19,7 @@ def test_443_handle_nonchunked_response_in_stream(self): def test_542_truncate_ids_client_side(self): self.client.start( - self.client.create_container(BUSYBOX, ['true']) + self.client.create_container(TEST_IMG, ['true']) ) result = self.client.containers(all=True, trunc=True) assert len(result[0]['Id']) == 12 @@ -30,12 +30,12 @@ def test_647_support_doubleslash_in_image_names(self): def test_649_handle_timeout_value_none(self): self.client.timeout = None - ctnr = self.client.create_container(BUSYBOX, ['sleep', '2']) + ctnr = self.client.create_container(TEST_IMG, ['sleep', '2']) self.client.start(ctnr) self.client.stop(ctnr) def test_715_handle_user_param_as_int_value(self): - ctnr = self.client.create_container(BUSYBOX, ['id', '-u'], user=1000) + ctnr = self.client.create_container(TEST_IMG, ['id', '-u'], user=1000) self.client.start(ctnr) self.client.wait(ctnr) logs = self.client.logs(ctnr) @@ -47,7 +47,7 @@ def test_792_explicit_port_protocol(self): tcp_port, udp_port = random.sample(range(9999, 32000), 2) ctnr = self.client.create_container( - BUSYBOX, ['sleep', '9999'], ports=[2000, (2000, 'udp')], + TEST_IMG, ['sleep', '9999'], ports=[2000, (2000, 'udp')], host_config=self.client.create_host_config( port_bindings={'2000/tcp': tcp_port, '2000/udp': udp_port} ) From cce095408927836712cf78346df8e2b5460e00ad Mon Sep 17 00:00:00 2001 From: Matt Fluet Date: Wed, 7 Aug 2019 17:32:41 -0400 Subject: [PATCH 0967/1301] Fix for empty auth keys in config.json Signed-off-by: Matt Fluet --- docker/auth.py | 2 + tests/unit/auth_test.py | 116 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 118 insertions(+) diff --git a/docker/auth.py b/docker/auth.py index 5f34ac087d..6a07ea2059 100644 --- a/docker/auth.py +++ b/docker/auth.py @@ -303,12 +303,14 @@ def get_all_credentials(self): auth_data[k] = self._resolve_authconfig_credstore( k, self.creds_store ) + auth_data[convert_to_hostname(k)] = auth_data[k] # credHelpers entries take priority over all others for reg, store_name in self.cred_helpers.items(): auth_data[reg] = self._resolve_authconfig_credstore( reg, store_name ) + auth_data[convert_to_hostname(reg)] = auth_data[reg] return auth_data diff --git a/tests/unit/auth_test.py b/tests/unit/auth_test.py index d46da503e3..aac8910911 100644 --- a/tests/unit/auth_test.py +++ b/tests/unit/auth_test.py @@ -530,11 +530,21 @@ def test_get_all_credentials_credstore_only(self): 'Password': 'izayoi', 'ServerAddress': 'https://gensokyo.jp/v2', }, + 'gensokyo.jp': { + 'Username': 'sakuya', + 'Password': 'izayoi', + 'ServerAddress': 'https://gensokyo.jp/v2', + }, 'https://default.com/v2': { 'Username': 'user', 'Password': 'hunter2', 'ServerAddress': 'https://default.com/v2', }, + 'default.com': { + 'Username': 'user', + 'Password': 'hunter2', + 'ServerAddress': 'https://default.com/v2', + }, } def test_get_all_credentials_with_empty_credhelper(self): @@ -548,11 +558,21 @@ def test_get_all_credentials_with_empty_credhelper(self): 'Password': 'izayoi', 'ServerAddress': 'https://gensokyo.jp/v2', }, + 'gensokyo.jp': { + 'Username': 'sakuya', + 'Password': 'izayoi', + 'ServerAddress': 'https://gensokyo.jp/v2', + }, 'https://default.com/v2': { 'Username': 'user', 'Password': 'hunter2', 'ServerAddress': 'https://default.com/v2', }, + 'default.com': { + 'Username': 'user', + 'Password': 'hunter2', + 'ServerAddress': 'https://default.com/v2', + }, 'registry1.io': None, } @@ -571,11 +591,21 @@ def test_get_all_credentials_with_credhelpers_only(self): 'Password': 'izayoi', 'ServerAddress': 'https://gensokyo.jp/v2', }, + 'gensokyo.jp': { + 'Username': 'sakuya', + 'Password': 'izayoi', + 'ServerAddress': 'https://gensokyo.jp/v2', + }, 'https://default.com/v2': { 'Username': 'user', 'Password': 'hunter2', 'ServerAddress': 'https://default.com/v2', }, + 'default.com': { + 'Username': 'user', + 'Password': 'hunter2', + 'ServerAddress': 'https://default.com/v2', + }, } def test_get_all_credentials_with_auths_entries(self): @@ -591,11 +621,21 @@ def test_get_all_credentials_with_auths_entries(self): 'Password': 'izayoi', 'ServerAddress': 'https://gensokyo.jp/v2', }, + 'gensokyo.jp': { + 'Username': 'sakuya', + 'Password': 'izayoi', + 'ServerAddress': 'https://gensokyo.jp/v2', + }, 'https://default.com/v2': { 'Username': 'user', 'Password': 'hunter2', 'ServerAddress': 'https://default.com/v2', }, + 'default.com': { + 'Username': 'user', + 'Password': 'hunter2', + 'ServerAddress': 'https://default.com/v2', + }, 'registry1.io': { 'ServerAddress': 'registry1.io', 'Username': 'reimu', @@ -603,6 +643,62 @@ def test_get_all_credentials_with_auths_entries(self): }, } + def test_get_all_credentials_with_empty_auths_entry(self): + self.authconfig.add_auth('default.com', {}) + + assert self.authconfig.get_all_credentials() == { + 'https://gensokyo.jp/v2': { + 'Username': 'sakuya', + 'Password': 'izayoi', + 'ServerAddress': 'https://gensokyo.jp/v2', + }, + 'gensokyo.jp': { + 'Username': 'sakuya', + 'Password': 'izayoi', + 'ServerAddress': 'https://gensokyo.jp/v2', + }, + 'https://default.com/v2': { + 'Username': 'user', + 'Password': 'hunter2', + 'ServerAddress': 'https://default.com/v2', + }, + 'default.com': { + 'Username': 'user', + 'Password': 'hunter2', + 'ServerAddress': 'https://default.com/v2', + }, + } + + def test_get_all_credentials_credstore_overrides_auth_entry(self): + self.authconfig.add_auth('default.com', { + 'Username': 'shouldnotsee', + 'Password': 'thisentry', + 'ServerAddress': 'https://default.com/v2', + }) + + assert self.authconfig.get_all_credentials() == { + 'https://gensokyo.jp/v2': { + 'Username': 'sakuya', + 'Password': 'izayoi', + 'ServerAddress': 'https://gensokyo.jp/v2', + }, + 'gensokyo.jp': { + 'Username': 'sakuya', + 'Password': 'izayoi', + 'ServerAddress': 'https://gensokyo.jp/v2', + }, + 'https://default.com/v2': { + 'Username': 'user', + 'Password': 'hunter2', + 'ServerAddress': 'https://default.com/v2', + }, + 'default.com': { + 'Username': 'user', + 'Password': 'hunter2', + 'ServerAddress': 'https://default.com/v2', + }, + } + def test_get_all_credentials_helpers_override_default(self): self.authconfig['credHelpers'] = { 'https://default.com/v2': 'truesecret', @@ -616,11 +712,21 @@ def test_get_all_credentials_helpers_override_default(self): 'Password': 'izayoi', 'ServerAddress': 'https://gensokyo.jp/v2', }, + 'gensokyo.jp': { + 'Username': 'sakuya', + 'Password': 'izayoi', + 'ServerAddress': 'https://gensokyo.jp/v2', + }, 'https://default.com/v2': { 'Username': 'reimu', 'Password': 'hakurei', 'ServerAddress': 'https://default.com/v2', }, + 'default.com': { + 'Username': 'reimu', + 'Password': 'hakurei', + 'ServerAddress': 'https://default.com/v2', + }, } def test_get_all_credentials_3_sources(self): @@ -642,11 +748,21 @@ def test_get_all_credentials_3_sources(self): 'Password': 'izayoi', 'ServerAddress': 'https://gensokyo.jp/v2', }, + 'gensokyo.jp': { + 'Username': 'sakuya', + 'Password': 'izayoi', + 'ServerAddress': 'https://gensokyo.jp/v2', + }, 'https://default.com/v2': { 'Username': 'user', 'Password': 'hunter2', 'ServerAddress': 'https://default.com/v2', }, + 'default.com': { + 'Username': 'user', + 'Password': 'hunter2', + 'ServerAddress': 'https://default.com/v2', + }, 'registry1.io': { 'ServerAddress': 'registry1.io', 'Username': 'reimu', From 2d327bf74304170d86ae63f18ed91a7fa3ac1a79 Mon Sep 17 00:00:00 2001 From: Ryan McCullagh Date: Fri, 23 Aug 2019 10:36:58 -0500 Subject: [PATCH 0968/1301] Fix typo in comment. networks => network Signed-off-by: Ryan McCullagh --- docker/api/network.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/api/network.py b/docker/api/network.py index 57ed8d3b75..c56a8d0bd7 100644 --- a/docker/api/network.py +++ b/docker/api/network.py @@ -7,7 +7,7 @@ class NetworkApiMixin(object): def networks(self, names=None, ids=None, filters=None): """ - List networks. Similar to the ``docker networks ls`` command. + List networks. Similar to the ``docker network ls`` command. Args: names (:py:class:`list`): List of names to filter by From 57c2193f6d0a6be6240d4fee9793e59ee7a9a2de Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Sat, 10 Aug 2019 18:50:36 +0200 Subject: [PATCH 0969/1301] pytest: set junitxml suite name to "docker-py" Signed-off-by: Sebastiaan van Stijn --- pytest.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pytest.ini b/pytest.ini index 21b47a6aaa..d233c56f18 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,2 +1,4 @@ [pytest] addopts = --tb=short -rxs + +junit_suite_name = docker-py From c238315c64ba3a0b1d3252e7965a94bb3618f94f Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Sat, 10 Aug 2019 19:04:54 +0200 Subject: [PATCH 0970/1301] pytest: update to v4.2.1 - use xunit2 for compatibility with Jenkins - pytest-dev/pytest#3547: `--junitxml` can emit XML compatible with Jenkins xUnit. `junit_family` INI option accepts `legacy|xunit1`, which produces old style output, and `xunit2` that conforms more strictly to https://github.com/jenkinsci/xunit-plugin/blob/xunit-2.3.2/src/main/resources/org/jenkinsci/plugins/xunit/types/model/xsd/junit-10.xsd Signed-off-by: Sebastiaan van Stijn --- pytest.ini | 1 + test-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/pytest.ini b/pytest.ini index d233c56f18..d4f718e782 100644 --- a/pytest.ini +++ b/pytest.ini @@ -2,3 +2,4 @@ addopts = --tb=short -rxs junit_suite_name = docker-py +junit_family = xunit2 diff --git a/test-requirements.txt b/test-requirements.txt index b89f64622d..bebfee8618 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,6 +1,6 @@ coverage==4.5.2 flake8==3.6.0 mock==1.0.1 -pytest==4.1.0 +pytest==4.2.1 pytest-cov==2.6.1 pytest-timeout==1.3.3 From 06c606300c5f6f16cc83e10edb261b81d9ab4133 Mon Sep 17 00:00:00 2001 From: Matt Fluet Date: Mon, 5 Aug 2019 18:31:56 -0400 Subject: [PATCH 0971/1301] Correct INDEX_URL logic in build.py _set_auth_headers Signed-off-by: Matt Fluet --- docker/api/build.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/api/build.py b/docker/api/build.py index e0a4ac969d..365129a064 100644 --- a/docker/api/build.py +++ b/docker/api/build.py @@ -308,7 +308,8 @@ def _set_auth_headers(self, headers): auth_data = self._auth_configs.get_all_credentials() # See https://github.com/docker/docker-py/issues/1683 - if auth.INDEX_URL not in auth_data and auth.INDEX_URL in auth_data: + if (auth.INDEX_URL not in auth_data and + auth.INDEX_NAME in auth_data): auth_data[auth.INDEX_URL] = auth_data.get(auth.INDEX_NAME, {}) log.debug( From 63760b192228725ac6c2808c996ccf6f45aff7e4 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Tue, 16 Jul 2019 12:51:01 +0200 Subject: [PATCH 0972/1301] test/Dockerfile: allow using a mirror for the apt repository With this change applied, the default debian package repository can be replaced with a mirror; ``` make APT_MIRROR=cdn-fastly.deb.debian.org build-py3 ... Step 5/19 : RUN apt-get update && apt-get -y install gnupg2 pass curl ---> Running in 01c1101a0bd0 Get:1 http://cdn-fastly.deb.debian.org/debian buster InRelease [118 kB] Get:2 http://cdn-fastly.deb.debian.org/debian-security buster/updates InRelease [39.1 kB] Get:3 http://cdn-fastly.deb.debian.org/debian buster-updates InRelease [46.8 kB] Get:4 http://cdn-fastly.deb.debian.org/debian buster/main amd64 Packages [7897 kB] Get:5 http://cdn-fastly.deb.debian.org/debian-security buster/updates/main amd64 Packages [22.8 kB] ``` Signed-off-by: Sebastiaan van Stijn --- Makefile | 4 ++-- tests/Dockerfile | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index ad643e8051..db103f5bdf 100644 --- a/Makefile +++ b/Makefile @@ -8,11 +8,11 @@ clean: .PHONY: build build: - docker build -t docker-sdk-python -f tests/Dockerfile --build-arg PYTHON_VERSION=2.7 . + docker build -t docker-sdk-python -f tests/Dockerfile --build-arg PYTHON_VERSION=2.7 --build-arg APT_MIRROR . .PHONY: build-py3 build-py3: - docker build -t docker-sdk-python3 -f tests/Dockerfile . + docker build -t docker-sdk-python3 -f tests/Dockerfile --build-arg APT_MIRROR . .PHONY: build-docs build-docs: diff --git a/tests/Dockerfile b/tests/Dockerfile index 4bd98f8733..df8468abab 100644 --- a/tests/Dockerfile +++ b/tests/Dockerfile @@ -1,6 +1,11 @@ ARG PYTHON_VERSION=3.7 FROM python:${PYTHON_VERSION} + +ARG APT_MIRROR +RUN sed -ri "s/(httpredir|deb).debian.org/${APT_MIRROR:-deb.debian.org}/g" /etc/apt/sources.list \ + && sed -ri "s/(security).debian.org/${APT_MIRROR:-security.debian.org}/g" /etc/apt/sources.list + RUN apt-get update && apt-get -y install \ gnupg2 \ pass \ From c88205c5cea0e04d67b5604d130f22cc3c833c66 Mon Sep 17 00:00:00 2001 From: Frank Sachsenheim Date: Mon, 27 May 2019 22:07:24 +0200 Subject: [PATCH 0973/1301] Amends the docs concerning multiple label filters Closes #2338 Signed-off-by: Frank Sachsenheim --- docker/api/container.py | 3 ++- docker/api/image.py | 3 ++- docker/api/network.py | 3 ++- docker/models/containers.py | 3 ++- docker/models/images.py | 3 ++- docker/models/networks.py | 3 ++- 6 files changed, 12 insertions(+), 6 deletions(-) diff --git a/docker/api/container.py b/docker/api/container.py index 326e7679f1..45bd3528ba 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -174,7 +174,8 @@ def containers(self, quiet=False, all=False, trunc=False, latest=False, - `exited` (int): Only containers with specified exit code - `status` (str): One of ``restarting``, ``running``, ``paused``, ``exited`` - - `label` (str): format either ``"key"`` or ``"key=value"`` + - `label` (str|list): format either ``"key"``, ``"key=value"`` + or a list of such. - `id` (str): The id of the container. - `name` (str): The name of the container. - `ancestor` (str): Filter by container ancestor. Format of diff --git a/docker/api/image.py b/docker/api/image.py index b370b7d83b..11c8cf7547 100644 --- a/docker/api/image.py +++ b/docker/api/image.py @@ -70,7 +70,8 @@ def images(self, name=None, quiet=False, all=False, filters=None): filters (dict): Filters to be processed on the image list. Available filters: - ``dangling`` (bool) - - ``label`` (str): format either ``key`` or ``key=value`` + - `label` (str|list): format either ``"key"``, ``"key=value"`` + or a list of such. Returns: (dict or list): A list if ``quiet=True``, otherwise a dict. diff --git a/docker/api/network.py b/docker/api/network.py index c56a8d0bd7..750b91b200 100644 --- a/docker/api/network.py +++ b/docker/api/network.py @@ -15,7 +15,8 @@ def networks(self, names=None, ids=None, filters=None): filters (dict): Filters to be processed on the network list. Available filters: - ``driver=[]`` Matches a network's driver. - - ``label=[]`` or ``label=[=]``. + - ``label=[]``, ``label=[=]`` or a list of + such. - ``type=["custom"|"builtin"]`` Filters networks by type. Returns: diff --git a/docker/models/containers.py b/docker/models/containers.py index 999851ec13..d1f275f74f 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -900,7 +900,8 @@ def list(self, all=False, before=None, filters=None, limit=-1, since=None, - `exited` (int): Only containers with specified exit code - `status` (str): One of ``restarting``, ``running``, ``paused``, ``exited`` - - `label` (str): format either ``"key"`` or ``"key=value"`` + - `label` (str|list): format either ``"key"``, ``"key=value"`` + or a list of such. - `id` (str): The id of the container. - `name` (str): The name of the container. - `ancestor` (str): Filter by container ancestor. Format of diff --git a/docker/models/images.py b/docker/models/images.py index 5419682940..757a5a4750 100644 --- a/docker/models/images.py +++ b/docker/models/images.py @@ -350,7 +350,8 @@ def list(self, name=None, all=False, filters=None): filters (dict): Filters to be processed on the image list. Available filters: - ``dangling`` (bool) - - ``label`` (str): format either ``key`` or ``key=value`` + - `label` (str|list): format either ``"key"``, ``"key=value"`` + or a list of such. Returns: (list of :py:class:`Image`): The images. diff --git a/docker/models/networks.py b/docker/models/networks.py index be3291a417..f944c8e299 100644 --- a/docker/models/networks.py +++ b/docker/models/networks.py @@ -190,7 +190,8 @@ def list(self, *args, **kwargs): filters (dict): Filters to be processed on the network list. Available filters: - ``driver=[]`` Matches a network's driver. - - ``label=[]`` or ``label=[=]``. + - `label` (str|list): format either ``"key"``, ``"key=value"`` + or a list of such. - ``type=["custom"|"builtin"]`` Filters networks by type. greedy (bool): Fetch more details for each network individually. You might want this to get the containers attached to them. From 38fe3983ba7dd22ef34c4612e3aacdab82fbe08a Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Sat, 10 Aug 2019 19:19:32 +0200 Subject: [PATCH 0974/1301] Jenkinsfile: update API version matrix; set default to v1.40 - Added new entry for Docker 19.03 - Removed obsolete engine versions that reached EOL (both as Community Edition and Enterprise Edition) - Set the fallback/default API version to v1.40, which corresponds with Docker 19.03 (current release) Signed-off-by: Sebastiaan van Stijn --- Jenkinsfile | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index e618c5dd77..a0f983c29d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -46,12 +46,14 @@ def getDockerVersions = { -> def getAPIVersion = { engineVersion -> def versionMap = [ - '17.06': '1.30', '17.12': '1.35', '18.02': '1.36', '18.03': '1.37', - '18.06': '1.38', '18.09': '1.39' + '17.06': '1.30', + '18.03': '1.37', + '18.09': '1.39', + '19.03': '1.40' ] def result = versionMap[engineVersion.substring(0, 5)] if (!result) { - return '1.39' + return '1.40' } return result } From 0be550dcf059efb28d27813b2a4486fc02d7b688 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Sat, 10 Aug 2019 19:22:52 +0200 Subject: [PATCH 0975/1301] Jenkinsfile: update python 3.6 -> 3.7 Signed-off-by: Sebastiaan van Stijn --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index a0f983c29d..e879eb43e7 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -25,7 +25,7 @@ def buildImages = { -> imageNamePy3 = "${imageNameBase}:py3-${gitCommit()}" buildImage(imageNamePy2, "-f tests/Dockerfile --build-arg PYTHON_VERSION=2.7 .", "py2.7") - buildImage(imageNamePy3, "-f tests/Dockerfile --build-arg PYTHON_VERSION=3.6 .", "py3.6") + buildImage(imageNamePy3, "-f tests/Dockerfile --build-arg PYTHON_VERSION=3.7 .", "py3.7") } } } From 934072a5e7fd73f395a788ac4200e42883ab327f Mon Sep 17 00:00:00 2001 From: Hannes Ljungberg Date: Fri, 3 May 2019 21:53:36 +0200 Subject: [PATCH 0976/1301] Add NetworkAttachmentConfig type Signed-off-by: Hannes Ljungberg --- docker/api/service.py | 10 +++++---- docker/models/services.py | 5 +++-- docker/types/__init__.py | 2 +- docker/types/services.py | 22 ++++++++++++++++++-- docs/api.rst | 1 + tests/integration/api_service_test.py | 29 +++++++++++++++++++++++++++ 6 files changed, 60 insertions(+), 9 deletions(-) diff --git a/docker/api/service.py b/docker/api/service.py index 372dd10b5c..e9027bfa21 100644 --- a/docker/api/service.py +++ b/docker/api/service.py @@ -135,8 +135,9 @@ def create_service( of the service. Default: ``None`` rollback_config (RollbackConfig): Specification for the rollback strategy of the service. Default: ``None`` - networks (:py:class:`list`): List of network names or IDs to attach - the service to. Default: ``None``. + networks (:py:class:`list`): List of network names or IDs or + :py:class:`~docker.types.NetworkAttachmentConfig` to attach the + service to. Default: ``None``. endpoint_spec (EndpointSpec): Properties that can be configured to access and load balance a service. Default: ``None``. @@ -383,8 +384,9 @@ def update_service(self, service, version, task_template=None, name=None, of the service. Default: ``None``. rollback_config (RollbackConfig): Specification for the rollback strategy of the service. Default: ``None`` - networks (:py:class:`list`): List of network names or IDs to attach - the service to. Default: ``None``. + networks (:py:class:`list`): List of network names or IDs or + :py:class:`~docker.types.NetworkAttachmentConfig` to attach the + service to. Default: ``None``. endpoint_spec (EndpointSpec): Properties that can be configured to access and load balance a service. Default: ``None``. fetch_current_spec (boolean): Use the undefined settings from the diff --git a/docker/models/services.py b/docker/models/services.py index 2b6479f2a1..5eff8c88b5 100644 --- a/docker/models/services.py +++ b/docker/models/services.py @@ -178,8 +178,9 @@ def create(self, image, command=None, **kwargs): ``source:target:options``, where options is either ``ro`` or ``rw``. name (str): Name to give to the service. - networks (list of str): List of network names or IDs to attach - the service to. Default: ``None``. + networks (:py:class:`list`): List of network names or IDs or + :py:class:`~docker.types.NetworkAttachmentConfig` to attach the + service to. Default: ``None``. resources (Resources): Resource limits and reservations. restart_policy (RestartPolicy): Restart policy for containers. secrets (list of :py:class:`docker.types.SecretReference`): List diff --git a/docker/types/__init__.py b/docker/types/__init__.py index f3cac1bc17..5db330e284 100644 --- a/docker/types/__init__.py +++ b/docker/types/__init__.py @@ -7,6 +7,6 @@ ConfigReference, ContainerSpec, DNSConfig, DriverConfig, EndpointSpec, Mount, Placement, PlacementPreference, Privileges, Resources, RestartPolicy, RollbackConfig, SecretReference, ServiceMode, TaskTemplate, - UpdateConfig + UpdateConfig, NetworkAttachmentConfig ) from .swarm import SwarmSpec, SwarmExternalCA diff --git a/docker/types/services.py b/docker/types/services.py index 5722b0e33d..05dda15d75 100644 --- a/docker/types/services.py +++ b/docker/types/services.py @@ -26,8 +26,8 @@ class TaskTemplate(dict): placement (Placement): Placement instructions for the scheduler. If a list is passed instead, it is assumed to be a list of constraints as part of a :py:class:`Placement` object. - networks (:py:class:`list`): List of network names or IDs to attach - the containers to. + networks (:py:class:`list`): List of network names or IDs or + :py:class:`NetworkAttachmentConfig` to attach the service to. force_update (int): A counter that triggers an update even if no relevant parameters have been changed. """ @@ -770,3 +770,21 @@ def __init__(self, credentialspec_file=None, credentialspec_registry=None, if len(selinux_context) > 0: self['SELinuxContext'] = selinux_context + + +class NetworkAttachmentConfig(dict): + """ + Network attachment options for a service. + + Args: + target (str): The target network for attachment. + Can be a network name or ID. + aliases (:py:class:`list`): A list of discoverable alternate names + for the service. + options (:py:class:`dict`): Driver attachment options for the + network target. + """ + def __init__(self, target, aliases=None, options=None): + self['Target'] = target + self['Aliases'] = aliases + self['DriverOpts'] = options diff --git a/docs/api.rst b/docs/api.rst index edb8fffadc..bd0466143d 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -142,6 +142,7 @@ Configuration types .. autoclass:: IPAMPool .. autoclass:: LogConfig .. autoclass:: Mount +.. autoclass:: NetworkAttachmentConfig .. autoclass:: Placement .. autoclass:: PlacementPreference .. autoclass:: Privileges diff --git a/tests/integration/api_service_test.py b/tests/integration/api_service_test.py index c170a0a88f..784d1e377d 100644 --- a/tests/integration/api_service_test.py +++ b/tests/integration/api_service_test.py @@ -371,6 +371,35 @@ def test_create_service_with_custom_networks(self): {'Target': net1['Id']}, {'Target': net2['Id']} ] + def test_create_service_with_network_attachment_config(self): + network = self.client.create_network( + 'dockerpytest_1', driver='overlay', ipam={'Driver': 'default'} + ) + self.tmp_networks.append(network['Id']) + container_spec = docker.types.ContainerSpec(BUSYBOX, ['true']) + network_config = docker.types.NetworkAttachmentConfig( + target='dockerpytest_1', + aliases=['dockerpytest_1_alias'], + options={ + 'foo': 'bar' + } + ) + task_tmpl = docker.types.TaskTemplate( + container_spec, + networks=[network_config] + ) + name = self.get_service_name() + svc_id = self.client.create_service( + task_tmpl, name=name + ) + svc_info = self.client.inspect_service(svc_id) + assert 'Networks' in svc_info['Spec']['TaskTemplate'] + service_networks_info = svc_info['Spec']['TaskTemplate']['Networks'] + assert len(service_networks_info) == 1 + assert service_networks_info[0]['Target'] == network['Id'] + assert service_networks_info[0]['Aliases'] == ['dockerpytest_1_alias'] + assert service_networks_info[0]['DriverOpts'] == {'foo': 'bar'} + def test_create_service_with_placement(self): node_id = self.client.nodes()[0]['ID'] container_spec = docker.types.ContainerSpec(TEST_IMG, ['true']) From ec63237da0edb2fd7180d14183c3f2bcfe12cc0c Mon Sep 17 00:00:00 2001 From: Hannes Ljungberg Date: Fri, 3 May 2019 22:09:48 +0200 Subject: [PATCH 0977/1301] Correctly reference ConfigReference Signed-off-by: Hannes Ljungberg --- docker/models/services.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docker/models/services.py b/docker/models/services.py index 5eff8c88b5..e866545da8 100644 --- a/docker/models/services.py +++ b/docker/models/services.py @@ -206,8 +206,9 @@ def create(self, image, command=None, **kwargs): the container's `hosts` file. dns_config (DNSConfig): Specification for DNS related configurations in resolver configuration file. - configs (:py:class:`list`): List of :py:class:`ConfigReference` - that will be exposed to the service. + configs (:py:class:`list`): List of + :py:class:`~docker.types.ConfigReference` that will be exposed + to the service. privileges (Privileges): Security options for the service's containers. From 7c8264ce9629be771a8245ef702e8608fc528544 Mon Sep 17 00:00:00 2001 From: Hannes Ljungberg Date: Fri, 3 May 2019 22:24:33 +0200 Subject: [PATCH 0978/1301] Correctly reference SecretReference Signed-off-by: Hannes Ljungberg --- docker/models/services.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/models/services.py b/docker/models/services.py index e866545da8..a35687b3ca 100644 --- a/docker/models/services.py +++ b/docker/models/services.py @@ -183,7 +183,7 @@ def create(self, image, command=None, **kwargs): service to. Default: ``None``. resources (Resources): Resource limits and reservations. restart_policy (RestartPolicy): Restart policy for containers. - secrets (list of :py:class:`docker.types.SecretReference`): List + secrets (list of :py:class:`~docker.types.SecretReference`): List of secrets accessible to containers for this service. stop_grace_period (int): Amount of time to wait for containers to terminate before forcefully killing them. From bc89de6047a771b778d6e7d85fba401befd80675 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Fri, 30 Aug 2019 00:14:20 +0200 Subject: [PATCH 0979/1301] Fix broken test due to BUSYBOX -> TEST_IMG rename The BUSYBOX variable was renamed to TEST_IMG in 54b48a9b7ab59b4dcf49acf49ddf52035ba3ea08, however 0ddf428b6ce7accdac3506b45047df2cb72941ec got merged after that change, but was out of date, and therefore caused the tests to fail: ``` =================================== FAILURES =================================== ________ ServiceTest.test_create_service_with_network_attachment_config ________ tests/integration/api_service_test.py:379: in test_create_service_with_network_attachment_config container_spec = docker.types.ContainerSpec(BUSYBOX, ['true']) E NameError: global name 'BUSYBOX' is not defined ``` Fix the test by using the correct variable name. Signed-off-by: Sebastiaan van Stijn --- tests/integration/api_service_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/api_service_test.py b/tests/integration/api_service_test.py index 784d1e377d..b6b7ec538d 100644 --- a/tests/integration/api_service_test.py +++ b/tests/integration/api_service_test.py @@ -376,7 +376,7 @@ def test_create_service_with_network_attachment_config(self): 'dockerpytest_1', driver='overlay', ipam={'Driver': 'default'} ) self.tmp_networks.append(network['Id']) - container_spec = docker.types.ContainerSpec(BUSYBOX, ['true']) + container_spec = docker.types.ContainerSpec(TEST_IMG, ['true']) network_config = docker.types.NetworkAttachmentConfig( target='dockerpytest_1', aliases=['dockerpytest_1_alias'], From 88219c682c9ba673a03197191233cb3a7c69167d Mon Sep 17 00:00:00 2001 From: Kir Kolyshkin Date: Tue, 1 Oct 2019 17:21:38 -0700 Subject: [PATCH 0980/1301] Bump pytest to 4.3.1 Pytest 4.3.1 includes the fix from https://github.com/pytest-dev/pytest/pull/4795 which should fix the following failure: > INFO: Building docker-sdk-python3:4.0.2... > sha256:c7a40413c985b6e75df324fae39b1c30cb78a25df71b7892f1a4a15449537fb3 > INFO: Starting docker-py tests... > Traceback (most recent call last): > File "/usr/local/bin/pytest", line 10, in > sys.exit(main()) > File "/usr/local/lib/python3.6/site-packages/_pytest/config/__init__.py", line 61, in main > config = _prepareconfig(args, plugins) > File "/usr/local/lib/python3.6/site-packages/_pytest/config/__init__.py", line 182, in _prepareconfig > config = get_config() > File "/usr/local/lib/python3.6/site-packages/_pytest/config/__init__.py", line 156, in get_config > pluginmanager.import_plugin(spec) > File "/usr/local/lib/python3.6/site-packages/_pytest/config/__init__.py", line 530, in import_plugin > __import__(importspec) > File "/usr/local/lib/python3.6/site-packages/_pytest/tmpdir.py", line 25, in > class TempPathFactory(object): > File "/usr/local/lib/python3.6/site-packages/_pytest/tmpdir.py", line 35, in TempPathFactory > lambda p: Path(os.path.abspath(six.text_type(p))) > TypeError: attrib() got an unexpected keyword argument 'convert' > Sending interrupt signal to process > Terminated > script returned exit code 143 Signed-off-by: Kir Kolyshkin --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index bebfee8618..0b01e569e6 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,6 +1,6 @@ coverage==4.5.2 flake8==3.6.0 mock==1.0.1 -pytest==4.2.1 +pytest==4.3.1 pytest-cov==2.6.1 pytest-timeout==1.3.3 From 2bb08b3985fbde794a75fbf321872c9c4d84abf9 Mon Sep 17 00:00:00 2001 From: Christopher Crone Date: Thu, 3 Oct 2019 15:44:27 +0200 Subject: [PATCH 0981/1301] Bump 4.1.0 Signed-off-by: Christopher Crone --- docker/version.py | 2 +- docs/change-log.md | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index 21249253e6..99a8b424be 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "4.1.0-dev" +version = "4.1.0" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) diff --git a/docs/change-log.md b/docs/change-log.md index b10cfd544c..7cc05068e1 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -1,6 +1,27 @@ Change log ========== +4.1.0 +----- + +[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/61?closed=1) + +### Bugfixes + +- Correct `INDEX_URL` logic in build.py _set_auth_headers +- Fix for empty auth keys in config.json + +### Features + +- Add `NetworkAttachmentConfig` for service create/update + +### Miscellaneous + +- Bump pytest to 4.3.1 +- Adjust `--platform` tests for changes in docker engine +- Update credentials-helpers to v0.6.3 + + 4.0.2 ----- From c81200a483824d14f73f71537ec2e4f7b171db4d Mon Sep 17 00:00:00 2001 From: Christopher Crone Date: Thu, 3 Oct 2019 15:44:27 +0200 Subject: [PATCH 0982/1301] Bump 4.1.0 Signed-off-by: Christopher Crone --- docker/version.py | 2 +- docs/change-log.md | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index 21249253e6..99a8b424be 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "4.1.0-dev" +version = "4.1.0" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) diff --git a/docs/change-log.md b/docs/change-log.md index b10cfd544c..7cc05068e1 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -1,6 +1,27 @@ Change log ========== +4.1.0 +----- + +[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/61?closed=1) + +### Bugfixes + +- Correct `INDEX_URL` logic in build.py _set_auth_headers +- Fix for empty auth keys in config.json + +### Features + +- Add `NetworkAttachmentConfig` for service create/update + +### Miscellaneous + +- Bump pytest to 4.3.1 +- Adjust `--platform` tests for changes in docker engine +- Update credentials-helpers to v0.6.3 + + 4.0.2 ----- From efc7e3c4b0ba5f522b6338b225711d17e2a72017 Mon Sep 17 00:00:00 2001 From: Christopher Crone Date: Thu, 3 Oct 2019 16:40:23 +0200 Subject: [PATCH 0983/1301] Version bump Signed-off-by: Christopher Crone --- docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index 99a8b424be..0c9ec47c23 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "4.1.0" +version = "4.2.0-dev" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) From 2dc569a232cf45ddf10265f8778d70587112cc95 Mon Sep 17 00:00:00 2001 From: Till Riedel Date: Sun, 14 Apr 2019 10:38:07 +0200 Subject: [PATCH 0984/1301] set logging level of paramiko to warn Signed-off-by: Till Riedel --- docker/transport/sshconn.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/transport/sshconn.py b/docker/transport/sshconn.py index 5a8ceb08b3..6d2b1af60f 100644 --- a/docker/transport/sshconn.py +++ b/docker/transport/sshconn.py @@ -1,6 +1,7 @@ import paramiko import requests.adapters import six +import logging from docker.transport.basehttpadapter import BaseHTTPAdapter from .. import constants @@ -77,6 +78,7 @@ class SSHHTTPAdapter(BaseHTTPAdapter): def __init__(self, base_url, timeout=60, pool_connections=constants.DEFAULT_NUM_POOLS): + logging.getLogger("paramiko").setLevel(logging.WARNING) self.ssh_client = paramiko.SSHClient() self.ssh_client.load_system_host_keys() From eb8c78c3b3234c61abbc5cddf2c1e49243118a6b Mon Sep 17 00:00:00 2001 From: Till Riedel Date: Sun, 14 Apr 2019 10:40:15 +0200 Subject: [PATCH 0985/1301] set host key policy for ssh transport to WarningPolicy() Signed-off-by: Till Riedel --- docker/transport/sshconn.py | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/transport/sshconn.py b/docker/transport/sshconn.py index 6d2b1af60f..79aefcecb8 100644 --- a/docker/transport/sshconn.py +++ b/docker/transport/sshconn.py @@ -81,6 +81,7 @@ def __init__(self, base_url, timeout=60, logging.getLogger("paramiko").setLevel(logging.WARNING) self.ssh_client = paramiko.SSHClient() self.ssh_client.load_system_host_keys() + self.ssh_client.set_missing_host_key_policy(paramiko.WarningPolicy()) self.base_url = base_url self._connect() From c285bee1bc59f6b2d65cee952b5522c88047a3bc Mon Sep 17 00:00:00 2001 From: Till Riedel Date: Sun, 14 Apr 2019 13:52:12 +0200 Subject: [PATCH 0986/1301] obey Hostname Username Port and ProxyCommand settings from .ssh/config Signed-off-by: Till Riedel --- docker/transport/sshconn.py | 33 +++++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/docker/transport/sshconn.py b/docker/transport/sshconn.py index 79aefcecb8..7de0e59087 100644 --- a/docker/transport/sshconn.py +++ b/docker/transport/sshconn.py @@ -2,6 +2,7 @@ import requests.adapters import six import logging +import os from docker.transport.basehttpadapter import BaseHTTPAdapter from .. import constants @@ -73,17 +74,40 @@ def _get_conn(self, timeout): class SSHHTTPAdapter(BaseHTTPAdapter): __attrs__ = requests.adapters.HTTPAdapter.__attrs__ + [ - 'pools', 'timeout', 'ssh_client', + 'pools', 'timeout', 'ssh_client', 'ssh_params' ] def __init__(self, base_url, timeout=60, pool_connections=constants.DEFAULT_NUM_POOLS): logging.getLogger("paramiko").setLevel(logging.WARNING) self.ssh_client = paramiko.SSHClient() + base_url = six.moves.urllib_parse.urlparse(base_url) + self.ssh_params = { + "hostname": base_url.hostname, + "port": base_url.port, + "username": base_url.username + } + ssh_config_file = os.path.expanduser("~/.ssh/config") + if os.path.exists(ssh_config_file): + conf = paramiko.SSHConfig() + with open(ssh_config_file) as f: + conf.parse(f) + host_config = conf.lookup(base_url.hostname) + self.ssh_conf = host_config + if 'proxycommand' in host_config: + self.ssh_params["sock"] = paramiko.ProxyCommand( + self.ssh_conf['proxycommand'] + ) + if 'hostname' in host_config: + self.ssh_params['hostname'] = host_config['hostname'] + if base_url.port is None and 'port' in host_config: + self.ssh_params['port'] = self.ssh_conf['port'] + if base_url.username is None and 'user' in host_config: + self.ssh_params['username'] = self.ssh_conf['user'] + self.ssh_client.load_system_host_keys() self.ssh_client.set_missing_host_key_policy(paramiko.WarningPolicy()) - self.base_url = base_url self._connect() self.timeout = timeout self.pools = RecentlyUsedContainer( @@ -92,10 +116,7 @@ def __init__(self, base_url, timeout=60, super(SSHHTTPAdapter, self).__init__() def _connect(self): - parsed = six.moves.urllib_parse.urlparse(self.base_url) - self.ssh_client.connect( - parsed.hostname, parsed.port, parsed.username, - ) + self.ssh_client.connect(**self.ssh_params) def get_connection(self, url, proxies=None): with self.pools.lock: From 1d8aa3019ec3299e1f76d2d92cd86790061fe96c Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Mon, 14 Oct 2019 12:24:55 +0200 Subject: [PATCH 0987/1301] Fix CI labels so we run on amd64 nodes Signed-off-by: Nicolas De Loof --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index e879eb43e7..7af23e9cef 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -32,7 +32,7 @@ def buildImages = { -> def getDockerVersions = { -> def dockerVersions = ["17.06.2-ce"] - wrappedNode(label: "ubuntu && !zfs") { + wrappedNode(label: "ubuntu && !zfs && amd64") { def result = sh(script: """docker run --rm \\ --entrypoint=python \\ ${imageNamePy3} \\ From 19171d0e1e81c1070f213a576e9a37ef0df1209d Mon Sep 17 00:00:00 2001 From: Jack Laxson Date: Mon, 28 Oct 2019 05:41:15 -0400 Subject: [PATCH 0988/1301] remove hyphens in literals Signed-off-by: Jack Laxson --- docs/_static/custom.css | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/_static/custom.css b/docs/_static/custom.css index 5d711eeffb..fb6d3af4ba 100644 --- a/docs/_static/custom.css +++ b/docs/_static/custom.css @@ -1,3 +1,6 @@ dl.hide-signature > dt { display: none; } +code.literal{ + hyphens: none; +} From 755fd735667e0777c1e98c988ad1b9506aec3444 Mon Sep 17 00:00:00 2001 From: Hongbin Lu Date: Sun, 24 Sep 2017 16:47:45 +0000 Subject: [PATCH 0989/1301] Add mac_address to connect_container_to_network Signed-off-by: Hongbin Lu --- docker/api/network.py | 4 +++- docker/types/networks.py | 9 ++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/docker/api/network.py b/docker/api/network.py index 750b91b200..19407bf39f 100644 --- a/docker/api/network.py +++ b/docker/api/network.py @@ -216,7 +216,7 @@ def inspect_network(self, net_id, verbose=None, scope=None): def connect_container_to_network(self, container, net_id, ipv4_address=None, ipv6_address=None, aliases=None, links=None, - link_local_ips=None): + link_local_ips=None, mac_address=None): """ Connect a container to a network. @@ -235,6 +235,8 @@ def connect_container_to_network(self, container, net_id, network, using the IPv6 protocol. Defaults to ``None``. link_local_ips (:py:class:`list`): A list of link-local (IPv4/IPv6) addresses. + mac_address (str): The MAC address of this container on the + network. Defaults to ``None``. """ data = { "Container": container, diff --git a/docker/types/networks.py b/docker/types/networks.py index 1c7b2c9e69..f6db26c2fa 100644 --- a/docker/types/networks.py +++ b/docker/types/networks.py @@ -4,7 +4,7 @@ class EndpointConfig(dict): def __init__(self, version, aliases=None, links=None, ipv4_address=None, - ipv6_address=None, link_local_ips=None): + ipv6_address=None, link_local_ips=None, mac_address=None): if version_lt(version, '1.22'): raise errors.InvalidVersion( 'Endpoint config is not supported for API version < 1.22' @@ -23,6 +23,13 @@ def __init__(self, version, aliases=None, links=None, ipv4_address=None, if ipv6_address: ipam_config['IPv6Address'] = ipv6_address + if mac_address: + if version_lt(version, '1.25'): + raise errors.InvalidVersion( + 'mac_address is not supported for API version < 1.25' + ) + ipam_config['MacAddress'] = mac_address + if link_local_ips is not None: if version_lt(version, '1.24'): raise errors.InvalidVersion( From 656db96b4a8b0db28d4b19ca60c95036c995175b Mon Sep 17 00:00:00 2001 From: Yuval Goldberg Date: Thu, 19 Dec 2019 15:35:08 +0200 Subject: [PATCH 0990/1301] Fix mac_address connect usage in network functions && addind appropriate test Signed-off-by: Yuval Goldberg --- docker/api/network.py | 3 ++- docker/types/networks.py | 2 +- tests/integration/api_network_test.py | 16 ++++++++++++++++ 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/docker/api/network.py b/docker/api/network.py index 19407bf39f..1709b62185 100644 --- a/docker/api/network.py +++ b/docker/api/network.py @@ -242,7 +242,8 @@ def connect_container_to_network(self, container, net_id, "Container": container, "EndpointConfig": self.create_endpoint_config( aliases=aliases, links=links, ipv4_address=ipv4_address, - ipv6_address=ipv6_address, link_local_ips=link_local_ips + ipv6_address=ipv6_address, link_local_ips=link_local_ips, + mac_address=mac_address ), } diff --git a/docker/types/networks.py b/docker/types/networks.py index f6db26c2fa..442adb1ead 100644 --- a/docker/types/networks.py +++ b/docker/types/networks.py @@ -28,7 +28,7 @@ def __init__(self, version, aliases=None, links=None, ipv4_address=None, raise errors.InvalidVersion( 'mac_address is not supported for API version < 1.25' ) - ipam_config['MacAddress'] = mac_address + self['MacAddress'] = mac_address if link_local_ips is not None: if version_lt(version, '1.24'): diff --git a/tests/integration/api_network_test.py b/tests/integration/api_network_test.py index 0f26827b17..4b5e6fcfa3 100644 --- a/tests/integration/api_network_test.py +++ b/tests/integration/api_network_test.py @@ -387,6 +387,22 @@ def test_connect_with_ipv6_address(self): net_data = container_data['NetworkSettings']['Networks'][net_name] assert net_data['IPAMConfig']['IPv6Address'] == '2001:389::f00d' + @requires_api_version('1.25') + def test_connect_with_mac_address(self): + net_name, net_id = self.create_network() + + container = self.client.create_container(TEST_IMG, 'top') + self.tmp_containers.append(container) + + self.client.connect_container_to_network( + container, net_name, mac_address='02:42:ac:11:00:02' + ) + + container_data = self.client.inspect_container(container) + + net_data = container_data['NetworkSettings']['Networks'][net_name] + assert net_data['MacAddress'] == '02:42:ac:11:00:02' + @requires_api_version('1.23') def test_create_internal_networks(self): _, net_id = self.create_network(internal=True) From 940805dde6982ff75d80f5257a68f93b725ba17b Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Fri, 3 Jan 2020 15:20:20 +0100 Subject: [PATCH 0991/1301] Fix ImageCollectionTest.test_pull_multiple flakiness The ImageCollectionTest.test_pull_multiple test performs a `docker pull` without a `:tag` specified) to pull all tags of the given repository (image). After pulling the image, the image(s) pulled are checked to verify if the list of images contains the `:latest` tag. However, the test assumes that all tags of the image are tags for the same version of the image (same digest), and thus a *single* image is returned, which is not always the case. Currently, the `hello-world:latest` and `hello-world:linux` tags point to a different digest, therefore the `client.images.pull()` returns multiple images: one image for digest, making the test fail: =================================== FAILURES =================================== ____________________ ImageCollectionTest.test_pull_multiple ____________________ tests/integration/models_images_test.py:90: in test_pull_multiple assert len(images) == 1 E AssertionError: assert 2 == 1 E + where 2 = len([, ]) This patch updates the test to not assume a single image is returned, and instead loop through the list of images and check if any of the images contains the `:latest` tag. Signed-off-by: Sebastiaan van Stijn --- tests/integration/models_images_test.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/integration/models_images_test.py b/tests/integration/models_images_test.py index 375d972df9..223d102f2b 100644 --- a/tests/integration/models_images_test.py +++ b/tests/integration/models_images_test.py @@ -87,8 +87,10 @@ def test_pull_with_sha(self): def test_pull_multiple(self): client = docker.from_env(version=TEST_API_VERSION) images = client.images.pull('hello-world') - assert len(images) == 1 - assert 'hello-world:latest' in images[0].attrs['RepoTags'] + assert len(images) >= 1 + assert any([ + 'hello-world:latest' in img.attrs['RepoTags'] for img in images + ]) def test_load_error(self): client = docker.from_env(version=TEST_API_VERSION) From 64fdb32ae801ef5a49541e421f05678767677bae Mon Sep 17 00:00:00 2001 From: Anca Iordache Date: Fri, 17 Jan 2020 19:25:55 +0100 Subject: [PATCH 0992/1301] Implement context management, lifecycle and unittests. Signed-off-by: Anca Iordache --- .travis.yml | 2 +- appveyor.yml | 3 +- docker/__init__.py | 3 + docker/constants.py | 12 ++ docker/context/__init__.py | 3 + docker/context/api.py | 205 +++++++++++++++++++++++++ docker/context/config.py | 81 ++++++++++ docker/context/context.py | 208 ++++++++++++++++++++++++++ docker/errors.py | 32 ++++ docker/utils/utils.py | 15 +- test-requirements.txt | 1 + tests/integration/context_api_test.py | 52 +++++++ tests/unit/context_test.py | 45 ++++++ tests/unit/errors_test.py | 20 +-- 14 files changed, 659 insertions(+), 23 deletions(-) create mode 100644 docker/context/__init__.py create mode 100644 docker/context/api.py create mode 100644 docker/context/config.py create mode 100644 docker/context/context.py create mode 100644 tests/integration/context_api_test.py create mode 100644 tests/unit/context_test.py diff --git a/.travis.yml b/.travis.yml index 577b893f22..7b3d7248d9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,6 +15,6 @@ matrix: - env: TOXENV=flake8 install: - - pip install tox + - pip install tox==2.9.1 script: - tox diff --git a/appveyor.yml b/appveyor.yml index d659b586ee..144ab35289 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,8 +1,9 @@ version: '{branch}-{build}' install: - - "SET PATH=C:\\Python27-x64;C:\\Python27-x64\\Scripts;%PATH%" + - "SET PATH=C:\\Python37-x64;C:\\Python37-x64\\Scripts;%PATH%" - "python --version" + - "python -m pip install --upgrade pip" - "pip install tox==2.9.1" # Build the binary after tests diff --git a/docker/__init__.py b/docker/__init__.py index cf732e137b..e5c1a8f6e0 100644 --- a/docker/__init__.py +++ b/docker/__init__.py @@ -1,6 +1,9 @@ # flake8: noqa from .api import APIClient from .client import DockerClient, from_env +from .context import Context +from .context import ContextAPI +from .tls import TLSConfig from .version import version, version_info __version__ = version diff --git a/docker/constants.py b/docker/constants.py index 4b96e1ce52..e4daed5d54 100644 --- a/docker/constants.py +++ b/docker/constants.py @@ -9,6 +9,18 @@ 'memory', 'memswap', 'cpushares', 'cpusetcpus' ] +DEFAULT_HTTP_HOST = "127.0.0.1" +DEFAULT_UNIX_SOCKET = "http+unix:///var/run/docker.sock" +DEFAULT_NPIPE = 'npipe:////./pipe/docker_engine' + +BYTE_UNITS = { + 'b': 1, + 'k': 1024, + 'm': 1024 * 1024, + 'g': 1024 * 1024 * 1024 +} + + INSECURE_REGISTRY_DEPRECATION_WARNING = \ 'The `insecure_registry` argument to {} ' \ 'is deprecated and non-functional. Please remove it.' diff --git a/docker/context/__init__.py b/docker/context/__init__.py new file mode 100644 index 0000000000..0a6707f997 --- /dev/null +++ b/docker/context/__init__.py @@ -0,0 +1,3 @@ +# flake8: noqa +from .context import Context +from .api import ContextAPI diff --git a/docker/context/api.py b/docker/context/api.py new file mode 100644 index 0000000000..fc7e8940c0 --- /dev/null +++ b/docker/context/api.py @@ -0,0 +1,205 @@ +import json +import os + +from docker import errors +from docker.context.config import get_meta_dir +from docker.context.config import METAFILE +from docker.context.config import get_current_context_name +from docker.context.config import write_context_name_to_docker_config +from docker.context import Context + + +class ContextAPI(object): + """Context API. + Contains methods for context management: + create, list, remove, get, inspect. + """ + DEFAULT_CONTEXT = Context("default") + + @classmethod + def create_context( + cls, name, orchestrator="swarm", host=None, tls_cfg=None, + default_namespace=None, skip_tls_verify=False): + """Creates a new context. + Returns: + (Context): a Context object. + Raises: + :py:class:`docker.errors.MissingContextParameter` + If a context name is not provided. + :py:class:`docker.errors.ContextAlreadyExists` + If a context with the name already exists. + :py:class:`docker.errors.ContextException` + If name is default. + + Example: + + >>> from docker.context import ContextAPI + >>> ctx = ContextAPI.create_context(name='test') + >>> print(ctx.Metadata) + { + "Name": "test", + "Metadata": { + "StackOrchestrator": "swarm" + }, + "Endpoints": { + "docker": { + "Host": "unix:///var/run/docker.sock", + "SkipTLSVerify": false + } + } + } + """ + if not name: + raise errors.MissingContextParameter("name") + if name == "default": + raise errors.ContextException( + '"default" is a reserved context name') + ctx = Context.load_context(name) + if ctx: + raise errors.ContextAlreadyExists(name) + endpoint = "docker" if orchestrator == "swarm" else orchestrator + ctx = Context(name, orchestrator) + ctx.set_endpoint( + endpoint, host, tls_cfg, + skip_tls_verify=skip_tls_verify, + def_namespace=default_namespace) + ctx.save() + return ctx + + @classmethod + def get_context(cls, name=None): + """Retrieves a context object. + Args: + name (str): The name of the context + + Example: + + >>> from docker.context import ContextAPI + >>> ctx = ContextAPI.get_context(name='test') + >>> print(ctx.Metadata) + { + "Name": "test", + "Metadata": { + "StackOrchestrator": "swarm" + }, + "Endpoints": { + "docker": { + "Host": "unix:///var/run/docker.sock", + "SkipTLSVerify": false + } + } + } + """ + if not name: + name = get_current_context_name() + if name == "default": + return cls.DEFAULT_CONTEXT + return Context.load_context(name) + + @classmethod + def contexts(cls): + """Context list. + Returns: + (Context): List of context objects. + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + names = [] + for dirname, dirnames, fnames in os.walk(get_meta_dir()): + for filename in fnames + dirnames: + if filename == METAFILE: + try: + data = json.load( + open(os.path.join(dirname, filename), "r")) + names.append(data["Name"]) + except Exception as e: + raise errors.ContextException( + "Failed to load metafile {}: {}".format( + filename, e)) + + contexts = [cls.DEFAULT_CONTEXT] + for name in names: + contexts.append(Context.load_context(name)) + return contexts + + @classmethod + def get_current_context(cls): + """Get current context. + Returns: + (Context): current context object. + """ + return cls.get_context() + + @classmethod + def set_current_context(cls, name="default"): + ctx = cls.get_context(name) + if not ctx: + raise errors.ContextNotFound(name) + + err = write_context_name_to_docker_config(name) + if err: + raise errors.ContextException( + 'Failed to set current context: {}'.format(err)) + + @classmethod + def remove_context(cls, name): + """Remove a context. Similar to the ``docker context rm`` command. + + Args: + name (str): The name of the context + + Raises: + :py:class:`docker.errors.MissingContextParameter` + If a context name is not provided. + :py:class:`docker.errors.ContextNotFound` + If a context with the name does not exist. + :py:class:`docker.errors.ContextException` + If name is default. + + Example: + + >>> from docker.context import ContextAPI + >>> ContextAPI.remove_context(name='test') + >>> + """ + if not name: + raise errors.MissingContextParameter("name") + if name == "default": + raise errors.ContextException( + 'context "default" cannot be removed') + ctx = Context.load_context(name) + if not ctx: + raise errors.ContextNotFound(name) + if name == get_current_context_name(): + write_context_name_to_docker_config(None) + ctx.remove() + + @classmethod + def inspect_context(cls, name="default"): + """Remove a context. Similar to the ``docker context inspect`` command. + + Args: + name (str): The name of the context + + Raises: + :py:class:`docker.errors.MissingContextParameter` + If a context name is not provided. + :py:class:`docker.errors.ContextNotFound` + If a context with the name does not exist. + + Example: + + >>> from docker.context import ContextAPI + >>> ContextAPI.remove_context(name='test') + >>> + """ + if not name: + raise errors.MissingContextParameter("name") + if name == "default": + return cls.DEFAULT_CONTEXT() + ctx = Context.load_context(name) + if not ctx: + raise errors.ContextNotFound(name) + + return ctx() diff --git a/docker/context/config.py b/docker/context/config.py new file mode 100644 index 0000000000..ac9a342ef9 --- /dev/null +++ b/docker/context/config.py @@ -0,0 +1,81 @@ +import os +import json +import hashlib + +from docker import utils +from docker.constants import IS_WINDOWS_PLATFORM +from docker.constants import DEFAULT_UNIX_SOCKET +from docker.utils.config import find_config_file + +METAFILE = "meta.json" + + +def get_current_context_name(): + name = "default" + docker_cfg_path = find_config_file() + if docker_cfg_path: + try: + with open(docker_cfg_path, "r") as f: + name = json.load(f).get("currentContext", "default") + except Exception: + return "default" + return name + + +def write_context_name_to_docker_config(name=None): + if name == 'default': + name = None + docker_cfg_path = find_config_file() + config = {} + if docker_cfg_path: + try: + with open(docker_cfg_path, "r") as f: + config = json.load(f) + except Exception as e: + return e + current_context = config.get("currentContext", None) + if current_context and not name: + del config["currentContext"] + elif name: + config["currentContext"] = name + else: + return + try: + with open(docker_cfg_path, "w") as f: + json.dump(config, f, indent=4) + except Exception as e: + return e + + +def get_context_id(name): + return hashlib.sha256(name.encode('utf-8')).hexdigest() + + +def get_context_dir(): + return os.path.join(os.path.dirname(find_config_file() or ""), "contexts") + + +def get_meta_dir(name=None): + meta_dir = os.path.join(get_context_dir(), "meta") + if name: + return os.path.join(meta_dir, get_context_id(name)) + return meta_dir + + +def get_meta_file(name): + return os.path.join(get_meta_dir(name), METAFILE) + + +def get_tls_dir(name=None, endpoint=""): + context_dir = get_context_dir() + if name: + return os.path.join(context_dir, "tls", get_context_id(name), endpoint) + return os.path.join(context_dir, "tls") + + +def get_context_host(path=None): + host = utils.parse_host(path, IS_WINDOWS_PLATFORM) + if host == DEFAULT_UNIX_SOCKET: + # remove http+ from default docker socket url + return host.strip("http+") + return host diff --git a/docker/context/context.py b/docker/context/context.py new file mode 100644 index 0000000000..4a0549ca9e --- /dev/null +++ b/docker/context/context.py @@ -0,0 +1,208 @@ +import os +import json +from shutil import copyfile, rmtree +from docker.tls import TLSConfig +from docker.errors import ContextException +from docker.context.config import get_meta_dir +from docker.context.config import get_meta_file +from docker.context.config import get_tls_dir +from docker.context.config import get_context_host + + +class Context: + """A context.""" + def __init__(self, name, orchestrator="swarm", host=None, endpoints=None): + if not name: + raise Exception("Name not provided") + self.name = name + self.orchestrator = orchestrator + if not endpoints: + default_endpoint = "docker" if ( + orchestrator == "swarm" + ) else orchestrator + self.endpoints = { + default_endpoint: { + "Host": get_context_host(host), + "SkipTLSVerify": False + } + } + else: + for k, v in endpoints.items(): + ekeys = v.keys() + for param in ["Host", "SkipTLSVerify"]: + if param not in ekeys: + raise ContextException( + "Missing parameter {} from endpoint {}".format( + param, k)) + self.endpoints = endpoints + + self.tls_cfg = {} + self.meta_path = "IN MEMORY" + self.tls_path = "IN MEMORY" + + def set_endpoint( + self, name="docker", host=None, tls_cfg=None, + skip_tls_verify=False, def_namespace=None): + self.endpoints[name] = { + "Host": get_context_host(host), + "SkipTLSVerify": skip_tls_verify + } + if def_namespace: + self.endpoints[name]["DefaultNamespace"] = def_namespace + + if tls_cfg: + self.tls_cfg[name] = tls_cfg + + def inspect(self): + return self.__call__() + + @classmethod + def load_context(cls, name): + name, orchestrator, endpoints = Context._load_meta(name) + if name: + instance = cls(name, orchestrator, endpoints=endpoints) + instance._load_certs() + instance.meta_path = get_meta_dir(name) + return instance + return None + + @classmethod + def _load_meta(cls, name): + metadata = {} + meta_file = get_meta_file(name) + if os.path.isfile(meta_file): + with open(meta_file) as f: + try: + with open(meta_file) as f: + metadata = json.load(f) + for k, v in metadata["Endpoints"].items(): + metadata["Endpoints"][k]["SkipTLSVerify"] = bool( + v["SkipTLSVerify"]) + except (IOError, KeyError, ValueError) as e: + # unknown format + raise Exception("""Detected corrupted meta file for + context {} : {}""".format(name, e)) + + return ( + metadata["Name"], metadata["Metadata"]["StackOrchestrator"], + metadata["Endpoints"]) + return None, None, None + + def _load_certs(self): + certs = {} + tls_dir = get_tls_dir(self.name) + for endpoint in self.endpoints.keys(): + if not os.path.isdir(os.path.join(tls_dir, endpoint)): + continue + ca_cert = None + cert = None + key = None + for filename in os.listdir(os.path.join(tls_dir, endpoint)): + if filename.startswith("ca"): + ca_cert = os.path.join(tls_dir, endpoint, filename) + elif filename.startswith("cert"): + cert = os.path.join(tls_dir, endpoint, filename) + elif filename.startswith("key"): + key = os.path.join(tls_dir, endpoint, filename) + if all([ca_cert, cert, key]): + certs[endpoint] = TLSConfig( + client_cert=(cert, key), ca_cert=ca_cert) + self.tls_cfg = certs + self.tls_path = tls_dir + + def save(self): + meta_dir = get_meta_dir(self.name) + if not os.path.isdir(meta_dir): + os.makedirs(meta_dir) + with open(get_meta_file(self.name), "w") as f: + f.write(json.dumps(self.Metadata)) + + tls_dir = get_tls_dir(self.name) + for endpoint, tls in self.tls_cfg.items(): + if not os.path.isdir(os.path.join(tls_dir, endpoint)): + os.makedirs(os.path.join(tls_dir, endpoint)) + + ca_file = tls.ca_cert + if ca_file: + copyfile(ca_file, os.path.join( + tls_dir, endpoint, os.path.basename(ca_file))) + + if tls.cert: + cert_file, key_file = tls.cert + copyfile(cert_file, os.path.join( + tls_dir, endpoint, os.path.basename(cert_file))) + copyfile(key_file, os.path.join( + tls_dir, endpoint, os.path.basename(key_file))) + + self.meta_path = get_meta_dir(self.name) + self.tls_path = get_tls_dir(self.name) + + def remove(self): + if os.path.isdir(self.meta_path): + rmtree(self.meta_path) + if os.path.isdir(self.tls_path): + rmtree(self.tls_path) + + def __repr__(self): + return "<%s: '%s'>" % (self.__class__.__name__, self.name) + + def __str__(self): + return json.dumps(self.__call__(), indent=2) + + def __call__(self): + result = self.Metadata + result.update(self.TLSMaterial) + result.update(self.Storage) + return result + + @property + def Name(self): + return self.name + + @property + def Host(self): + if self.orchestrator == "swarm": + return self.endpoints["docker"]["Host"] + return self.endpoints[self.orchestrator]["Host"] + + @property + def Orchestrator(self): + return self.orchestrator + + @property + def Metadata(self): + return { + "Name": self.name, + "Metadata": { + "StackOrchestrator": self.orchestrator + }, + "Endpoints": self.endpoints + } + + @property + def TLSConfig(self): + key = self.orchestrator + if key == "swarm": + key = "docker" + if key in self.tls_cfg.keys(): + return self.tls_cfg[key] + return None + + @property + def TLSMaterial(self): + certs = {} + for endpoint, tls in self.tls_cfg.items(): + cert, key = tls.cert + certs[endpoint] = list( + map(os.path.basename, [tls.ca_cert, cert, key])) + return { + "TLSMaterial": certs + } + + @property + def Storage(self): + return { + "Storage": { + "MetadataPath": self.meta_path, + "TLSPath": self.tls_path + }} diff --git a/docker/errors.py b/docker/errors.py index c340dcb123..e5d07a5bfe 100644 --- a/docker/errors.py +++ b/docker/errors.py @@ -163,3 +163,35 @@ def create_unexpected_kwargs_error(name, kwargs): text.append("got unexpected keyword arguments ") text.append(', '.join(quoted_kwargs)) return TypeError(''.join(text)) + + +class MissingContextParameter(DockerException): + def __init__(self, param): + self.param = param + + def __str__(self): + return ("missing parameter: {}".format(self.param)) + + +class ContextAlreadyExists(DockerException): + def __init__(self, name): + self.name = name + + def __str__(self): + return ("context {} already exists".format(self.name)) + + +class ContextException(DockerException): + def __init__(self, msg): + self.msg = msg + + def __str__(self): + return (self.msg) + + +class ContextNotFound(DockerException): + def __init__(self, name): + self.name = name + + def __str__(self): + return ("context '{}' not found".format(self.name)) diff --git a/docker/utils/utils.py b/docker/utils/utils.py index 7819ace4f4..447760b483 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -11,6 +11,10 @@ from .. import errors from .. import tls +from ..constants import DEFAULT_HTTP_HOST +from ..constants import DEFAULT_UNIX_SOCKET +from ..constants import DEFAULT_NPIPE +from ..constants import BYTE_UNITS if six.PY2: from urllib import splitnport @@ -18,17 +22,6 @@ else: from urllib.parse import splitnport, urlparse -DEFAULT_HTTP_HOST = "127.0.0.1" -DEFAULT_UNIX_SOCKET = "http+unix:///var/run/docker.sock" -DEFAULT_NPIPE = 'npipe:////./pipe/docker_engine' - -BYTE_UNITS = { - 'b': 1, - 'k': 1024, - 'm': 1024 * 1024, - 'g': 1024 * 1024 * 1024 -} - def create_ipam_pool(*args, **kwargs): raise errors.DeprecatedMethod( diff --git a/test-requirements.txt b/test-requirements.txt index 0b01e569e6..24078e27a8 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,3 +1,4 @@ +setuptools==44.0.0 # last version with python 2.7 support coverage==4.5.2 flake8==3.6.0 mock==1.0.1 diff --git a/tests/integration/context_api_test.py b/tests/integration/context_api_test.py new file mode 100644 index 0000000000..60235ee7ba --- /dev/null +++ b/tests/integration/context_api_test.py @@ -0,0 +1,52 @@ +import os +import tempfile +import pytest +from docker import errors +from docker.context import ContextAPI +from docker.tls import TLSConfig +from .base import BaseAPIIntegrationTest + + +class ContextLifecycleTest(BaseAPIIntegrationTest): + def test_lifecycle(self): + assert ContextAPI.get_context().Name == "default" + assert not ContextAPI.get_context("test") + assert ContextAPI.get_current_context().Name == "default" + + dirpath = tempfile.mkdtemp() + ca = tempfile.NamedTemporaryFile( + prefix=os.path.join(dirpath, "ca.pem"), mode="r") + cert = tempfile.NamedTemporaryFile( + prefix=os.path.join(dirpath, "cert.pem"), mode="r") + key = tempfile.NamedTemporaryFile( + prefix=os.path.join(dirpath, "key.pem"), mode="r") + + # create context 'test + docker_tls = TLSConfig( + client_cert=(cert.name, key.name), + ca_cert=ca.name) + ContextAPI.create_context( + "test", tls_cfg=docker_tls) + + # check for a context 'test' in the context store + assert any([ctx.Name == "test" for ctx in ContextAPI.contexts()]) + # retrieve a context object for 'test' + assert ContextAPI.get_context("test") + # remove context + ContextAPI.remove_context("test") + with pytest.raises(errors.ContextNotFound): + ContextAPI.inspect_context("test") + # check there is no 'test' context in store + assert not ContextAPI.get_context("test") + + ca.close() + key.close() + cert.close() + + def test_context_remove(self): + ContextAPI.create_context("test") + assert ContextAPI.inspect_context("test")["Name"] == "test" + + ContextAPI.remove_context("test") + with pytest.raises(errors.ContextNotFound): + ContextAPI.inspect_context("test") diff --git a/tests/unit/context_test.py b/tests/unit/context_test.py new file mode 100644 index 0000000000..5e88c69139 --- /dev/null +++ b/tests/unit/context_test.py @@ -0,0 +1,45 @@ +import unittest +import docker +import pytest +from docker.constants import DEFAULT_UNIX_SOCKET +from docker.constants import DEFAULT_NPIPE +from docker.constants import IS_WINDOWS_PLATFORM +from docker.context import ContextAPI, Context + + +class BaseContextTest(unittest.TestCase): + @pytest.mark.skipif( + IS_WINDOWS_PLATFORM, reason='Linux specific path check' + ) + def test_url_compatibility_on_linux(self): + c = Context("test") + assert c.Host == DEFAULT_UNIX_SOCKET.strip("http+") + + @pytest.mark.skipif( + not IS_WINDOWS_PLATFORM, reason='Windows specific path check' + ) + def test_url_compatibility_on_windows(self): + c = Context("test") + assert c.Host == DEFAULT_NPIPE + + def test_fail_on_default_context_create(self): + with pytest.raises(docker.errors.ContextException): + ContextAPI.create_context("default") + + def test_default_in_context_list(self): + found = False + ctx = ContextAPI.contexts() + for c in ctx: + if c.Name == "default": + found = True + assert found is True + + def test_get_current_context(self): + assert ContextAPI.get_current_context().Name == "default" + + def test_context_inspect_without_params(self): + ctx = ContextAPI.inspect_context() + assert ctx["Name"] == "default" + assert ctx["Metadata"]["StackOrchestrator"] == "swarm" + assert ctx["Endpoints"]["docker"]["Host"] in [ + DEFAULT_NPIPE, DEFAULT_UNIX_SOCKET.strip("http+")] diff --git a/tests/unit/errors_test.py b/tests/unit/errors_test.py index 2134f86f04..54c2ba8f66 100644 --- a/tests/unit/errors_test.py +++ b/tests/unit/errors_test.py @@ -101,17 +101,17 @@ def test_is_error_500(self): assert err.is_error() is True def test_create_error_from_exception(self): - resp = requests.Response() - resp.status_code = 500 - err = APIError('') + resp = requests.Response() + resp.status_code = 500 + err = APIError('') + try: + resp.raise_for_status() + except requests.exceptions.HTTPError as e: try: - resp.raise_for_status() - except requests.exceptions.HTTPError as e: - try: - create_api_error_from_http_exception(e) - except APIError as e: - err = e - assert err.is_server_error() is True + create_api_error_from_http_exception(e) + except APIError as e: + err = e + assert err.is_server_error() is True class ContainerErrorTest(unittest.TestCase): From 1e567223ef4bc23b85e0f19da89ea910aebf46ca Mon Sep 17 00:00:00 2001 From: Till Riedel Date: Sun, 14 Apr 2019 10:38:07 +0200 Subject: [PATCH 0993/1301] set logging level of paramiko to warn Signed-off-by: Till Riedel --- docker/transport/sshconn.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/transport/sshconn.py b/docker/transport/sshconn.py index 5a8ceb08b3..6d2b1af60f 100644 --- a/docker/transport/sshconn.py +++ b/docker/transport/sshconn.py @@ -1,6 +1,7 @@ import paramiko import requests.adapters import six +import logging from docker.transport.basehttpadapter import BaseHTTPAdapter from .. import constants @@ -77,6 +78,7 @@ class SSHHTTPAdapter(BaseHTTPAdapter): def __init__(self, base_url, timeout=60, pool_connections=constants.DEFAULT_NUM_POOLS): + logging.getLogger("paramiko").setLevel(logging.WARNING) self.ssh_client = paramiko.SSHClient() self.ssh_client.load_system_host_keys() From bc6777eb01d34d5aacc38c6579ce6ae3a1569e8d Mon Sep 17 00:00:00 2001 From: Till Riedel Date: Sun, 14 Apr 2019 10:40:15 +0200 Subject: [PATCH 0994/1301] set host key policy for ssh transport to WarningPolicy() Signed-off-by: Till Riedel --- docker/transport/sshconn.py | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/transport/sshconn.py b/docker/transport/sshconn.py index 6d2b1af60f..79aefcecb8 100644 --- a/docker/transport/sshconn.py +++ b/docker/transport/sshconn.py @@ -81,6 +81,7 @@ def __init__(self, base_url, timeout=60, logging.getLogger("paramiko").setLevel(logging.WARNING) self.ssh_client = paramiko.SSHClient() self.ssh_client.load_system_host_keys() + self.ssh_client.set_missing_host_key_policy(paramiko.WarningPolicy()) self.base_url = base_url self._connect() From ed9b208e156a2685b032d934c02621f54caf7608 Mon Sep 17 00:00:00 2001 From: Till Riedel Date: Sun, 14 Apr 2019 13:52:12 +0200 Subject: [PATCH 0995/1301] obey Hostname Username Port and ProxyCommand settings from .ssh/config Signed-off-by: Till Riedel --- docker/transport/sshconn.py | 33 +++++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/docker/transport/sshconn.py b/docker/transport/sshconn.py index 79aefcecb8..7de0e59087 100644 --- a/docker/transport/sshconn.py +++ b/docker/transport/sshconn.py @@ -2,6 +2,7 @@ import requests.adapters import six import logging +import os from docker.transport.basehttpadapter import BaseHTTPAdapter from .. import constants @@ -73,17 +74,40 @@ def _get_conn(self, timeout): class SSHHTTPAdapter(BaseHTTPAdapter): __attrs__ = requests.adapters.HTTPAdapter.__attrs__ + [ - 'pools', 'timeout', 'ssh_client', + 'pools', 'timeout', 'ssh_client', 'ssh_params' ] def __init__(self, base_url, timeout=60, pool_connections=constants.DEFAULT_NUM_POOLS): logging.getLogger("paramiko").setLevel(logging.WARNING) self.ssh_client = paramiko.SSHClient() + base_url = six.moves.urllib_parse.urlparse(base_url) + self.ssh_params = { + "hostname": base_url.hostname, + "port": base_url.port, + "username": base_url.username + } + ssh_config_file = os.path.expanduser("~/.ssh/config") + if os.path.exists(ssh_config_file): + conf = paramiko.SSHConfig() + with open(ssh_config_file) as f: + conf.parse(f) + host_config = conf.lookup(base_url.hostname) + self.ssh_conf = host_config + if 'proxycommand' in host_config: + self.ssh_params["sock"] = paramiko.ProxyCommand( + self.ssh_conf['proxycommand'] + ) + if 'hostname' in host_config: + self.ssh_params['hostname'] = host_config['hostname'] + if base_url.port is None and 'port' in host_config: + self.ssh_params['port'] = self.ssh_conf['port'] + if base_url.username is None and 'user' in host_config: + self.ssh_params['username'] = self.ssh_conf['user'] + self.ssh_client.load_system_host_keys() self.ssh_client.set_missing_host_key_policy(paramiko.WarningPolicy()) - self.base_url = base_url self._connect() self.timeout = timeout self.pools = RecentlyUsedContainer( @@ -92,10 +116,7 @@ def __init__(self, base_url, timeout=60, super(SSHHTTPAdapter, self).__init__() def _connect(self): - parsed = six.moves.urllib_parse.urlparse(self.base_url) - self.ssh_client.connect( - parsed.hostname, parsed.port, parsed.username, - ) + self.ssh_client.connect(**self.ssh_params) def get_connection(self, url, proxies=None): with self.pools.lock: From a67d180e2c4346aedc44a066ff9d95a0f59155c8 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Mon, 14 Oct 2019 12:24:55 +0200 Subject: [PATCH 0996/1301] Fix CI labels so we run on amd64 nodes Signed-off-by: Nicolas De Loof --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index e879eb43e7..7af23e9cef 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -32,7 +32,7 @@ def buildImages = { -> def getDockerVersions = { -> def dockerVersions = ["17.06.2-ce"] - wrappedNode(label: "ubuntu && !zfs") { + wrappedNode(label: "ubuntu && !zfs && amd64") { def result = sh(script: """docker run --rm \\ --entrypoint=python \\ ${imageNamePy3} \\ From 61e2d5f69bd189679306e8e3b52d9c109b339f6f Mon Sep 17 00:00:00 2001 From: rentu Date: Fri, 30 Aug 2019 09:35:46 +0100 Subject: [PATCH 0997/1301] Fix win32pipe.WaitNamedPipe throw exception in windows container. Signed-off-by: Renlong Tu --- docker/transport/npipesocket.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/docker/transport/npipesocket.py b/docker/transport/npipesocket.py index ef02031640..176b5c87a9 100644 --- a/docker/transport/npipesocket.py +++ b/docker/transport/npipesocket.py @@ -1,4 +1,5 @@ import functools +import time import io import six @@ -9,7 +10,7 @@ cSECURITY_SQOS_PRESENT = 0x100000 cSECURITY_ANONYMOUS = 0 -RETRY_WAIT_TIMEOUT = 10000 +MAXIMUM_RETRY_COUNT = 10 def check_closed(f): @@ -46,8 +47,7 @@ def close(self): self._closed = True @check_closed - def connect(self, address): - win32pipe.WaitNamedPipe(address, self._timeout) + def connect(self, address, retry_count=0): try: handle = win32file.CreateFile( address, @@ -65,8 +65,10 @@ def connect(self, address): # Another program or thread has grabbed our pipe instance # before we got to it. Wait for availability and attempt to # connect again. - win32pipe.WaitNamedPipe(address, RETRY_WAIT_TIMEOUT) - return self.connect(address) + retry_count = retry_count + 1 + if (retry_count < MAXIMUM_RETRY_COUNT): + time.sleep(1) + return self.connect(address, retry_count) raise e self.flags = win32pipe.GetNamedPipeInfo(handle)[0] From 9b0d07f9a8ced5f5f0100fc579a25b82a585c20b Mon Sep 17 00:00:00 2001 From: Christopher Crone Date: Thu, 3 Oct 2019 16:40:23 +0200 Subject: [PATCH 0998/1301] Version bump Signed-off-by: Christopher Crone --- docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index 99a8b424be..0c9ec47c23 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "4.1.0" +version = "4.2.0-dev" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) From 6c29375fd13bcaccc3d9f88a1449d83aed322794 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Fri, 3 Jan 2020 15:20:20 +0100 Subject: [PATCH 0999/1301] Fix ImageCollectionTest.test_pull_multiple flakiness The ImageCollectionTest.test_pull_multiple test performs a `docker pull` without a `:tag` specified) to pull all tags of the given repository (image). After pulling the image, the image(s) pulled are checked to verify if the list of images contains the `:latest` tag. However, the test assumes that all tags of the image are tags for the same version of the image (same digest), and thus a *single* image is returned, which is not always the case. Currently, the `hello-world:latest` and `hello-world:linux` tags point to a different digest, therefore the `client.images.pull()` returns multiple images: one image for digest, making the test fail: =================================== FAILURES =================================== ____________________ ImageCollectionTest.test_pull_multiple ____________________ tests/integration/models_images_test.py:90: in test_pull_multiple assert len(images) == 1 E AssertionError: assert 2 == 1 E + where 2 = len([, ]) This patch updates the test to not assume a single image is returned, and instead loop through the list of images and check if any of the images contains the `:latest` tag. Signed-off-by: Sebastiaan van Stijn --- tests/integration/models_images_test.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/integration/models_images_test.py b/tests/integration/models_images_test.py index 375d972df9..223d102f2b 100644 --- a/tests/integration/models_images_test.py +++ b/tests/integration/models_images_test.py @@ -87,8 +87,10 @@ def test_pull_with_sha(self): def test_pull_multiple(self): client = docker.from_env(version=TEST_API_VERSION) images = client.images.pull('hello-world') - assert len(images) == 1 - assert 'hello-world:latest' in images[0].attrs['RepoTags'] + assert len(images) >= 1 + assert any([ + 'hello-world:latest' in img.attrs['RepoTags'] for img in images + ]) def test_load_error(self): client = docker.from_env(version=TEST_API_VERSION) From 6e44d8422c3bd74578787582fa73cba73184c7f5 Mon Sep 17 00:00:00 2001 From: Anca Iordache Date: Fri, 17 Jan 2020 19:25:55 +0100 Subject: [PATCH 1000/1301] Implement context management, lifecycle and unittests. Signed-off-by: Anca Iordache --- .travis.yml | 2 +- appveyor.yml | 3 +- docker/__init__.py | 3 + docker/constants.py | 12 ++ docker/context/__init__.py | 3 + docker/context/api.py | 205 +++++++++++++++++++++++++ docker/context/config.py | 81 ++++++++++ docker/context/context.py | 208 ++++++++++++++++++++++++++ docker/errors.py | 32 ++++ docker/utils/utils.py | 15 +- test-requirements.txt | 1 + tests/integration/context_api_test.py | 52 +++++++ tests/unit/context_test.py | 45 ++++++ tests/unit/errors_test.py | 20 +-- 14 files changed, 659 insertions(+), 23 deletions(-) create mode 100644 docker/context/__init__.py create mode 100644 docker/context/api.py create mode 100644 docker/context/config.py create mode 100644 docker/context/context.py create mode 100644 tests/integration/context_api_test.py create mode 100644 tests/unit/context_test.py diff --git a/.travis.yml b/.travis.yml index 577b893f22..7b3d7248d9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,6 +15,6 @@ matrix: - env: TOXENV=flake8 install: - - pip install tox + - pip install tox==2.9.1 script: - tox diff --git a/appveyor.yml b/appveyor.yml index d659b586ee..144ab35289 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,8 +1,9 @@ version: '{branch}-{build}' install: - - "SET PATH=C:\\Python27-x64;C:\\Python27-x64\\Scripts;%PATH%" + - "SET PATH=C:\\Python37-x64;C:\\Python37-x64\\Scripts;%PATH%" - "python --version" + - "python -m pip install --upgrade pip" - "pip install tox==2.9.1" # Build the binary after tests diff --git a/docker/__init__.py b/docker/__init__.py index cf732e137b..e5c1a8f6e0 100644 --- a/docker/__init__.py +++ b/docker/__init__.py @@ -1,6 +1,9 @@ # flake8: noqa from .api import APIClient from .client import DockerClient, from_env +from .context import Context +from .context import ContextAPI +from .tls import TLSConfig from .version import version, version_info __version__ = version diff --git a/docker/constants.py b/docker/constants.py index 4b96e1ce52..e4daed5d54 100644 --- a/docker/constants.py +++ b/docker/constants.py @@ -9,6 +9,18 @@ 'memory', 'memswap', 'cpushares', 'cpusetcpus' ] +DEFAULT_HTTP_HOST = "127.0.0.1" +DEFAULT_UNIX_SOCKET = "http+unix:///var/run/docker.sock" +DEFAULT_NPIPE = 'npipe:////./pipe/docker_engine' + +BYTE_UNITS = { + 'b': 1, + 'k': 1024, + 'm': 1024 * 1024, + 'g': 1024 * 1024 * 1024 +} + + INSECURE_REGISTRY_DEPRECATION_WARNING = \ 'The `insecure_registry` argument to {} ' \ 'is deprecated and non-functional. Please remove it.' diff --git a/docker/context/__init__.py b/docker/context/__init__.py new file mode 100644 index 0000000000..0a6707f997 --- /dev/null +++ b/docker/context/__init__.py @@ -0,0 +1,3 @@ +# flake8: noqa +from .context import Context +from .api import ContextAPI diff --git a/docker/context/api.py b/docker/context/api.py new file mode 100644 index 0000000000..fc7e8940c0 --- /dev/null +++ b/docker/context/api.py @@ -0,0 +1,205 @@ +import json +import os + +from docker import errors +from docker.context.config import get_meta_dir +from docker.context.config import METAFILE +from docker.context.config import get_current_context_name +from docker.context.config import write_context_name_to_docker_config +from docker.context import Context + + +class ContextAPI(object): + """Context API. + Contains methods for context management: + create, list, remove, get, inspect. + """ + DEFAULT_CONTEXT = Context("default") + + @classmethod + def create_context( + cls, name, orchestrator="swarm", host=None, tls_cfg=None, + default_namespace=None, skip_tls_verify=False): + """Creates a new context. + Returns: + (Context): a Context object. + Raises: + :py:class:`docker.errors.MissingContextParameter` + If a context name is not provided. + :py:class:`docker.errors.ContextAlreadyExists` + If a context with the name already exists. + :py:class:`docker.errors.ContextException` + If name is default. + + Example: + + >>> from docker.context import ContextAPI + >>> ctx = ContextAPI.create_context(name='test') + >>> print(ctx.Metadata) + { + "Name": "test", + "Metadata": { + "StackOrchestrator": "swarm" + }, + "Endpoints": { + "docker": { + "Host": "unix:///var/run/docker.sock", + "SkipTLSVerify": false + } + } + } + """ + if not name: + raise errors.MissingContextParameter("name") + if name == "default": + raise errors.ContextException( + '"default" is a reserved context name') + ctx = Context.load_context(name) + if ctx: + raise errors.ContextAlreadyExists(name) + endpoint = "docker" if orchestrator == "swarm" else orchestrator + ctx = Context(name, orchestrator) + ctx.set_endpoint( + endpoint, host, tls_cfg, + skip_tls_verify=skip_tls_verify, + def_namespace=default_namespace) + ctx.save() + return ctx + + @classmethod + def get_context(cls, name=None): + """Retrieves a context object. + Args: + name (str): The name of the context + + Example: + + >>> from docker.context import ContextAPI + >>> ctx = ContextAPI.get_context(name='test') + >>> print(ctx.Metadata) + { + "Name": "test", + "Metadata": { + "StackOrchestrator": "swarm" + }, + "Endpoints": { + "docker": { + "Host": "unix:///var/run/docker.sock", + "SkipTLSVerify": false + } + } + } + """ + if not name: + name = get_current_context_name() + if name == "default": + return cls.DEFAULT_CONTEXT + return Context.load_context(name) + + @classmethod + def contexts(cls): + """Context list. + Returns: + (Context): List of context objects. + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + names = [] + for dirname, dirnames, fnames in os.walk(get_meta_dir()): + for filename in fnames + dirnames: + if filename == METAFILE: + try: + data = json.load( + open(os.path.join(dirname, filename), "r")) + names.append(data["Name"]) + except Exception as e: + raise errors.ContextException( + "Failed to load metafile {}: {}".format( + filename, e)) + + contexts = [cls.DEFAULT_CONTEXT] + for name in names: + contexts.append(Context.load_context(name)) + return contexts + + @classmethod + def get_current_context(cls): + """Get current context. + Returns: + (Context): current context object. + """ + return cls.get_context() + + @classmethod + def set_current_context(cls, name="default"): + ctx = cls.get_context(name) + if not ctx: + raise errors.ContextNotFound(name) + + err = write_context_name_to_docker_config(name) + if err: + raise errors.ContextException( + 'Failed to set current context: {}'.format(err)) + + @classmethod + def remove_context(cls, name): + """Remove a context. Similar to the ``docker context rm`` command. + + Args: + name (str): The name of the context + + Raises: + :py:class:`docker.errors.MissingContextParameter` + If a context name is not provided. + :py:class:`docker.errors.ContextNotFound` + If a context with the name does not exist. + :py:class:`docker.errors.ContextException` + If name is default. + + Example: + + >>> from docker.context import ContextAPI + >>> ContextAPI.remove_context(name='test') + >>> + """ + if not name: + raise errors.MissingContextParameter("name") + if name == "default": + raise errors.ContextException( + 'context "default" cannot be removed') + ctx = Context.load_context(name) + if not ctx: + raise errors.ContextNotFound(name) + if name == get_current_context_name(): + write_context_name_to_docker_config(None) + ctx.remove() + + @classmethod + def inspect_context(cls, name="default"): + """Remove a context. Similar to the ``docker context inspect`` command. + + Args: + name (str): The name of the context + + Raises: + :py:class:`docker.errors.MissingContextParameter` + If a context name is not provided. + :py:class:`docker.errors.ContextNotFound` + If a context with the name does not exist. + + Example: + + >>> from docker.context import ContextAPI + >>> ContextAPI.remove_context(name='test') + >>> + """ + if not name: + raise errors.MissingContextParameter("name") + if name == "default": + return cls.DEFAULT_CONTEXT() + ctx = Context.load_context(name) + if not ctx: + raise errors.ContextNotFound(name) + + return ctx() diff --git a/docker/context/config.py b/docker/context/config.py new file mode 100644 index 0000000000..ac9a342ef9 --- /dev/null +++ b/docker/context/config.py @@ -0,0 +1,81 @@ +import os +import json +import hashlib + +from docker import utils +from docker.constants import IS_WINDOWS_PLATFORM +from docker.constants import DEFAULT_UNIX_SOCKET +from docker.utils.config import find_config_file + +METAFILE = "meta.json" + + +def get_current_context_name(): + name = "default" + docker_cfg_path = find_config_file() + if docker_cfg_path: + try: + with open(docker_cfg_path, "r") as f: + name = json.load(f).get("currentContext", "default") + except Exception: + return "default" + return name + + +def write_context_name_to_docker_config(name=None): + if name == 'default': + name = None + docker_cfg_path = find_config_file() + config = {} + if docker_cfg_path: + try: + with open(docker_cfg_path, "r") as f: + config = json.load(f) + except Exception as e: + return e + current_context = config.get("currentContext", None) + if current_context and not name: + del config["currentContext"] + elif name: + config["currentContext"] = name + else: + return + try: + with open(docker_cfg_path, "w") as f: + json.dump(config, f, indent=4) + except Exception as e: + return e + + +def get_context_id(name): + return hashlib.sha256(name.encode('utf-8')).hexdigest() + + +def get_context_dir(): + return os.path.join(os.path.dirname(find_config_file() or ""), "contexts") + + +def get_meta_dir(name=None): + meta_dir = os.path.join(get_context_dir(), "meta") + if name: + return os.path.join(meta_dir, get_context_id(name)) + return meta_dir + + +def get_meta_file(name): + return os.path.join(get_meta_dir(name), METAFILE) + + +def get_tls_dir(name=None, endpoint=""): + context_dir = get_context_dir() + if name: + return os.path.join(context_dir, "tls", get_context_id(name), endpoint) + return os.path.join(context_dir, "tls") + + +def get_context_host(path=None): + host = utils.parse_host(path, IS_WINDOWS_PLATFORM) + if host == DEFAULT_UNIX_SOCKET: + # remove http+ from default docker socket url + return host.strip("http+") + return host diff --git a/docker/context/context.py b/docker/context/context.py new file mode 100644 index 0000000000..4a0549ca9e --- /dev/null +++ b/docker/context/context.py @@ -0,0 +1,208 @@ +import os +import json +from shutil import copyfile, rmtree +from docker.tls import TLSConfig +from docker.errors import ContextException +from docker.context.config import get_meta_dir +from docker.context.config import get_meta_file +from docker.context.config import get_tls_dir +from docker.context.config import get_context_host + + +class Context: + """A context.""" + def __init__(self, name, orchestrator="swarm", host=None, endpoints=None): + if not name: + raise Exception("Name not provided") + self.name = name + self.orchestrator = orchestrator + if not endpoints: + default_endpoint = "docker" if ( + orchestrator == "swarm" + ) else orchestrator + self.endpoints = { + default_endpoint: { + "Host": get_context_host(host), + "SkipTLSVerify": False + } + } + else: + for k, v in endpoints.items(): + ekeys = v.keys() + for param in ["Host", "SkipTLSVerify"]: + if param not in ekeys: + raise ContextException( + "Missing parameter {} from endpoint {}".format( + param, k)) + self.endpoints = endpoints + + self.tls_cfg = {} + self.meta_path = "IN MEMORY" + self.tls_path = "IN MEMORY" + + def set_endpoint( + self, name="docker", host=None, tls_cfg=None, + skip_tls_verify=False, def_namespace=None): + self.endpoints[name] = { + "Host": get_context_host(host), + "SkipTLSVerify": skip_tls_verify + } + if def_namespace: + self.endpoints[name]["DefaultNamespace"] = def_namespace + + if tls_cfg: + self.tls_cfg[name] = tls_cfg + + def inspect(self): + return self.__call__() + + @classmethod + def load_context(cls, name): + name, orchestrator, endpoints = Context._load_meta(name) + if name: + instance = cls(name, orchestrator, endpoints=endpoints) + instance._load_certs() + instance.meta_path = get_meta_dir(name) + return instance + return None + + @classmethod + def _load_meta(cls, name): + metadata = {} + meta_file = get_meta_file(name) + if os.path.isfile(meta_file): + with open(meta_file) as f: + try: + with open(meta_file) as f: + metadata = json.load(f) + for k, v in metadata["Endpoints"].items(): + metadata["Endpoints"][k]["SkipTLSVerify"] = bool( + v["SkipTLSVerify"]) + except (IOError, KeyError, ValueError) as e: + # unknown format + raise Exception("""Detected corrupted meta file for + context {} : {}""".format(name, e)) + + return ( + metadata["Name"], metadata["Metadata"]["StackOrchestrator"], + metadata["Endpoints"]) + return None, None, None + + def _load_certs(self): + certs = {} + tls_dir = get_tls_dir(self.name) + for endpoint in self.endpoints.keys(): + if not os.path.isdir(os.path.join(tls_dir, endpoint)): + continue + ca_cert = None + cert = None + key = None + for filename in os.listdir(os.path.join(tls_dir, endpoint)): + if filename.startswith("ca"): + ca_cert = os.path.join(tls_dir, endpoint, filename) + elif filename.startswith("cert"): + cert = os.path.join(tls_dir, endpoint, filename) + elif filename.startswith("key"): + key = os.path.join(tls_dir, endpoint, filename) + if all([ca_cert, cert, key]): + certs[endpoint] = TLSConfig( + client_cert=(cert, key), ca_cert=ca_cert) + self.tls_cfg = certs + self.tls_path = tls_dir + + def save(self): + meta_dir = get_meta_dir(self.name) + if not os.path.isdir(meta_dir): + os.makedirs(meta_dir) + with open(get_meta_file(self.name), "w") as f: + f.write(json.dumps(self.Metadata)) + + tls_dir = get_tls_dir(self.name) + for endpoint, tls in self.tls_cfg.items(): + if not os.path.isdir(os.path.join(tls_dir, endpoint)): + os.makedirs(os.path.join(tls_dir, endpoint)) + + ca_file = tls.ca_cert + if ca_file: + copyfile(ca_file, os.path.join( + tls_dir, endpoint, os.path.basename(ca_file))) + + if tls.cert: + cert_file, key_file = tls.cert + copyfile(cert_file, os.path.join( + tls_dir, endpoint, os.path.basename(cert_file))) + copyfile(key_file, os.path.join( + tls_dir, endpoint, os.path.basename(key_file))) + + self.meta_path = get_meta_dir(self.name) + self.tls_path = get_tls_dir(self.name) + + def remove(self): + if os.path.isdir(self.meta_path): + rmtree(self.meta_path) + if os.path.isdir(self.tls_path): + rmtree(self.tls_path) + + def __repr__(self): + return "<%s: '%s'>" % (self.__class__.__name__, self.name) + + def __str__(self): + return json.dumps(self.__call__(), indent=2) + + def __call__(self): + result = self.Metadata + result.update(self.TLSMaterial) + result.update(self.Storage) + return result + + @property + def Name(self): + return self.name + + @property + def Host(self): + if self.orchestrator == "swarm": + return self.endpoints["docker"]["Host"] + return self.endpoints[self.orchestrator]["Host"] + + @property + def Orchestrator(self): + return self.orchestrator + + @property + def Metadata(self): + return { + "Name": self.name, + "Metadata": { + "StackOrchestrator": self.orchestrator + }, + "Endpoints": self.endpoints + } + + @property + def TLSConfig(self): + key = self.orchestrator + if key == "swarm": + key = "docker" + if key in self.tls_cfg.keys(): + return self.tls_cfg[key] + return None + + @property + def TLSMaterial(self): + certs = {} + for endpoint, tls in self.tls_cfg.items(): + cert, key = tls.cert + certs[endpoint] = list( + map(os.path.basename, [tls.ca_cert, cert, key])) + return { + "TLSMaterial": certs + } + + @property + def Storage(self): + return { + "Storage": { + "MetadataPath": self.meta_path, + "TLSPath": self.tls_path + }} diff --git a/docker/errors.py b/docker/errors.py index c340dcb123..e5d07a5bfe 100644 --- a/docker/errors.py +++ b/docker/errors.py @@ -163,3 +163,35 @@ def create_unexpected_kwargs_error(name, kwargs): text.append("got unexpected keyword arguments ") text.append(', '.join(quoted_kwargs)) return TypeError(''.join(text)) + + +class MissingContextParameter(DockerException): + def __init__(self, param): + self.param = param + + def __str__(self): + return ("missing parameter: {}".format(self.param)) + + +class ContextAlreadyExists(DockerException): + def __init__(self, name): + self.name = name + + def __str__(self): + return ("context {} already exists".format(self.name)) + + +class ContextException(DockerException): + def __init__(self, msg): + self.msg = msg + + def __str__(self): + return (self.msg) + + +class ContextNotFound(DockerException): + def __init__(self, name): + self.name = name + + def __str__(self): + return ("context '{}' not found".format(self.name)) diff --git a/docker/utils/utils.py b/docker/utils/utils.py index 7819ace4f4..447760b483 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -11,6 +11,10 @@ from .. import errors from .. import tls +from ..constants import DEFAULT_HTTP_HOST +from ..constants import DEFAULT_UNIX_SOCKET +from ..constants import DEFAULT_NPIPE +from ..constants import BYTE_UNITS if six.PY2: from urllib import splitnport @@ -18,17 +22,6 @@ else: from urllib.parse import splitnport, urlparse -DEFAULT_HTTP_HOST = "127.0.0.1" -DEFAULT_UNIX_SOCKET = "http+unix:///var/run/docker.sock" -DEFAULT_NPIPE = 'npipe:////./pipe/docker_engine' - -BYTE_UNITS = { - 'b': 1, - 'k': 1024, - 'm': 1024 * 1024, - 'g': 1024 * 1024 * 1024 -} - def create_ipam_pool(*args, **kwargs): raise errors.DeprecatedMethod( diff --git a/test-requirements.txt b/test-requirements.txt index 0b01e569e6..24078e27a8 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,3 +1,4 @@ +setuptools==44.0.0 # last version with python 2.7 support coverage==4.5.2 flake8==3.6.0 mock==1.0.1 diff --git a/tests/integration/context_api_test.py b/tests/integration/context_api_test.py new file mode 100644 index 0000000000..60235ee7ba --- /dev/null +++ b/tests/integration/context_api_test.py @@ -0,0 +1,52 @@ +import os +import tempfile +import pytest +from docker import errors +from docker.context import ContextAPI +from docker.tls import TLSConfig +from .base import BaseAPIIntegrationTest + + +class ContextLifecycleTest(BaseAPIIntegrationTest): + def test_lifecycle(self): + assert ContextAPI.get_context().Name == "default" + assert not ContextAPI.get_context("test") + assert ContextAPI.get_current_context().Name == "default" + + dirpath = tempfile.mkdtemp() + ca = tempfile.NamedTemporaryFile( + prefix=os.path.join(dirpath, "ca.pem"), mode="r") + cert = tempfile.NamedTemporaryFile( + prefix=os.path.join(dirpath, "cert.pem"), mode="r") + key = tempfile.NamedTemporaryFile( + prefix=os.path.join(dirpath, "key.pem"), mode="r") + + # create context 'test + docker_tls = TLSConfig( + client_cert=(cert.name, key.name), + ca_cert=ca.name) + ContextAPI.create_context( + "test", tls_cfg=docker_tls) + + # check for a context 'test' in the context store + assert any([ctx.Name == "test" for ctx in ContextAPI.contexts()]) + # retrieve a context object for 'test' + assert ContextAPI.get_context("test") + # remove context + ContextAPI.remove_context("test") + with pytest.raises(errors.ContextNotFound): + ContextAPI.inspect_context("test") + # check there is no 'test' context in store + assert not ContextAPI.get_context("test") + + ca.close() + key.close() + cert.close() + + def test_context_remove(self): + ContextAPI.create_context("test") + assert ContextAPI.inspect_context("test")["Name"] == "test" + + ContextAPI.remove_context("test") + with pytest.raises(errors.ContextNotFound): + ContextAPI.inspect_context("test") diff --git a/tests/unit/context_test.py b/tests/unit/context_test.py new file mode 100644 index 0000000000..5e88c69139 --- /dev/null +++ b/tests/unit/context_test.py @@ -0,0 +1,45 @@ +import unittest +import docker +import pytest +from docker.constants import DEFAULT_UNIX_SOCKET +from docker.constants import DEFAULT_NPIPE +from docker.constants import IS_WINDOWS_PLATFORM +from docker.context import ContextAPI, Context + + +class BaseContextTest(unittest.TestCase): + @pytest.mark.skipif( + IS_WINDOWS_PLATFORM, reason='Linux specific path check' + ) + def test_url_compatibility_on_linux(self): + c = Context("test") + assert c.Host == DEFAULT_UNIX_SOCKET.strip("http+") + + @pytest.mark.skipif( + not IS_WINDOWS_PLATFORM, reason='Windows specific path check' + ) + def test_url_compatibility_on_windows(self): + c = Context("test") + assert c.Host == DEFAULT_NPIPE + + def test_fail_on_default_context_create(self): + with pytest.raises(docker.errors.ContextException): + ContextAPI.create_context("default") + + def test_default_in_context_list(self): + found = False + ctx = ContextAPI.contexts() + for c in ctx: + if c.Name == "default": + found = True + assert found is True + + def test_get_current_context(self): + assert ContextAPI.get_current_context().Name == "default" + + def test_context_inspect_without_params(self): + ctx = ContextAPI.inspect_context() + assert ctx["Name"] == "default" + assert ctx["Metadata"]["StackOrchestrator"] == "swarm" + assert ctx["Endpoints"]["docker"]["Host"] in [ + DEFAULT_NPIPE, DEFAULT_UNIX_SOCKET.strip("http+")] diff --git a/tests/unit/errors_test.py b/tests/unit/errors_test.py index 2134f86f04..54c2ba8f66 100644 --- a/tests/unit/errors_test.py +++ b/tests/unit/errors_test.py @@ -101,17 +101,17 @@ def test_is_error_500(self): assert err.is_error() is True def test_create_error_from_exception(self): - resp = requests.Response() - resp.status_code = 500 - err = APIError('') + resp = requests.Response() + resp.status_code = 500 + err = APIError('') + try: + resp.raise_for_status() + except requests.exceptions.HTTPError as e: try: - resp.raise_for_status() - except requests.exceptions.HTTPError as e: - try: - create_api_error_from_http_exception(e) - except APIError as e: - err = e - assert err.is_server_error() is True + create_api_error_from_http_exception(e) + except APIError as e: + err = e + assert err.is_server_error() is True class ContainerErrorTest(unittest.TestCase): From ab5678469c7d2dc73367a63b947aa84d16f36591 Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Thu, 6 Feb 2020 10:23:58 +0100 Subject: [PATCH 1001/1301] Bump 4.2.0 Signed-off-by: Ulysses Souza --- docker/version.py | 2 +- docs/change-log.md | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index 0c9ec47c23..f0a3170952 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "4.2.0-dev" +version = "4.2.0" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) diff --git a/docs/change-log.md b/docs/change-log.md index 7cc05068e1..2f0a9ed60c 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -1,6 +1,23 @@ Change log ========== +4.2.0 +----- + +[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/63?closed=1) + +### Bugfixes + +- Fix `win32pipe.WaitNamedPipe` throw exception in Windows containers +- Use `Hostname`, `Username`, `Port` and `ProxyCommand` settings from `.ssh/config` when on SSH +- Set host key policy for ssh transport to `paramiko.WarningPolicy()` +- Set logging level of `paramiko` to warn + +### Features + +- Add support for docker contexts through `docker.ContextAPI` + + 4.1.0 ----- From 7c4194ce5d132d7d9dc513a6e082cc3063e59990 Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Fri, 7 Feb 2020 01:00:18 +0100 Subject: [PATCH 1002/1301] Post release 4.2.0 update: - Changelog - Next Version Signed-off-by: Ulysses Souza --- docker/version.py | 2 +- docs/change-log.md | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index 0c9ec47c23..a75460921a 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "4.2.0-dev" +version = "4.3.0-dev" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) diff --git a/docs/change-log.md b/docs/change-log.md index 7cc05068e1..23330c1247 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -1,6 +1,22 @@ Change log ========== +4.2.0 +----- + +[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/63?closed=1) + +### Bugfixes + +- Fix `win32pipe.WaitNamedPipe` throw exception in Windows containers +- Use `Hostname`, `Username`, `Port` and `ProxyCommand` settings from `.ssh/config` when on SSH +- Set host key policy for ssh transport to `paramiko.WarningPolicy()` +- Set logging level of `paramiko` to warn + +### Features + +- Add support for docker contexts through `docker.ContextAPI` + 4.1.0 ----- From 789b6715cabe26fbe1061c411bc0646c582251dc Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Fri, 14 Feb 2020 23:51:44 +0100 Subject: [PATCH 1003/1301] Jenkinsfile: remove obsolete engine versions Signed-off-by: Sebastiaan van Stijn --- Jenkinsfile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 7af23e9cef..88a27c319c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -31,7 +31,7 @@ def buildImages = { -> } def getDockerVersions = { -> - def dockerVersions = ["17.06.2-ce"] + def dockerVersions = ["19.03.5"] wrappedNode(label: "ubuntu && !zfs && amd64") { def result = sh(script: """docker run --rm \\ --entrypoint=python \\ @@ -46,8 +46,6 @@ def getDockerVersions = { -> def getAPIVersion = { engineVersion -> def versionMap = [ - '17.06': '1.30', - '18.03': '1.37', '18.09': '1.39', '19.03': '1.40' ] From 7bef5e867676999bf1c1a7be8d5084bf16dd2764 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Fri, 14 Feb 2020 23:54:20 +0100 Subject: [PATCH 1004/1301] Update test engine version to 19.03.5 Signed-off-by: Sebastiaan van Stijn --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index db103f5bdf..a41c46b87c 100644 --- a/Makefile +++ b/Makefile @@ -42,7 +42,7 @@ integration-test-py3: build-py3 docker run -t --rm -v /var/run/docker.sock:/var/run/docker.sock docker-sdk-python3 py.test -v tests/integration/${file} TEST_API_VERSION ?= 1.35 -TEST_ENGINE_VERSION ?= 18.09.5 +TEST_ENGINE_VERSION ?= 19.03.5 .PHONY: setup-network setup-network: From da90bb325924ff5ca7f01d5ddf9180cddb2cee4e Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Mon, 17 Feb 2020 10:19:56 +0100 Subject: [PATCH 1005/1301] xfail "docker top" tests, and adjust for alpine image Signed-off-by: Sebastiaan van Stijn --- tests/integration/api_container_test.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index 1ba3eaa583..c503a3674e 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -1102,6 +1102,8 @@ def test_port(self): class ContainerTopTest(BaseAPIIntegrationTest): + @pytest.mark.xfail(reason='Output of docker top depends on host distro, ' + 'and is not formalized.') def test_top(self): container = self.client.create_container( TEST_IMG, ['sleep', '60'] @@ -1112,9 +1114,7 @@ def test_top(self): self.client.start(container) res = self.client.top(container) if not IS_WINDOWS_PLATFORM: - assert res['Titles'] == [ - 'UID', 'PID', 'PPID', 'C', 'STIME', 'TTY', 'TIME', 'CMD' - ] + assert res['Titles'] == [u'PID', u'USER', u'TIME', u'COMMAND'] assert len(res['Processes']) == 1 assert res['Processes'][0][-1] == 'sleep 60' self.client.kill(container) @@ -1122,6 +1122,8 @@ def test_top(self): @pytest.mark.skipif( IS_WINDOWS_PLATFORM, reason='No psargs support on windows' ) + @pytest.mark.xfail(reason='Output of docker top depends on host distro, ' + 'and is not formalized.') def test_top_with_psargs(self): container = self.client.create_container( TEST_IMG, ['sleep', '60']) @@ -1129,11 +1131,8 @@ def test_top_with_psargs(self): self.tmp_containers.append(container) self.client.start(container) - res = self.client.top(container, 'waux') - assert res['Titles'] == [ - 'USER', 'PID', '%CPU', '%MEM', 'VSZ', 'RSS', - 'TTY', 'STAT', 'START', 'TIME', 'COMMAND' - ] + res = self.client.top(container, '-eopid,user') + assert res['Titles'] == [u'PID', u'USER'] assert len(res['Processes']) == 1 assert res['Processes'][0][10] == 'sleep 60' From 8ced47dca9edb78200e9f6f9e45bb332e688dd87 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Sat, 15 Feb 2020 00:13:16 +0100 Subject: [PATCH 1006/1301] Use official docker:dind image instead of custom image This replaces the custom dockerswarm/dind image with the official dind images, which should provide the same functionality. Signed-off-by: Sebastiaan van Stijn --- Jenkinsfile | 2 +- Makefile | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 7af23e9cef..28511b2289 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -84,7 +84,7 @@ def runTests = { Map settings -> try { sh """docker network create ${testNetwork}""" sh """docker run -d --name ${dindContainerName} -v /tmp --privileged --network ${testNetwork} \\ - dockerswarm/dind:${dockerVersion} dockerd -H tcp://0.0.0.0:2375 + docker:${dockerVersion}-dind dockerd -H tcp://0.0.0.0:2375 """ sh """docker run \\ --name ${testContainerName} \\ diff --git a/Makefile b/Makefile index db103f5bdf..f456283fdf 100644 --- a/Makefile +++ b/Makefile @@ -55,7 +55,7 @@ integration-dind: integration-dind-py2 integration-dind-py3 integration-dind-py2: build setup-network docker rm -vf dpy-dind-py2 || : docker run -d --network dpy-tests --name dpy-dind-py2 --privileged\ - dockerswarm/dind:${TEST_ENGINE_VERSION} dockerd -H tcp://0.0.0.0:2375 --experimental + docker:${TEST_ENGINE_VERSION}-dind dockerd -H tcp://0.0.0.0:2375 --experimental docker run -t --rm --env="DOCKER_HOST=tcp://dpy-dind-py2:2375" --env="DOCKER_TEST_API_VERSION=${TEST_API_VERSION}"\ --network dpy-tests docker-sdk-python py.test tests/integration docker rm -vf dpy-dind-py2 @@ -64,7 +64,7 @@ integration-dind-py2: build setup-network integration-dind-py3: build-py3 setup-network docker rm -vf dpy-dind-py3 || : docker run -d --network dpy-tests --name dpy-dind-py3 --privileged\ - dockerswarm/dind:${TEST_ENGINE_VERSION} dockerd -H tcp://0.0.0.0:2375 --experimental + docker:${TEST_ENGINE_VERSION}-dind dockerd -H tcp://0.0.0.0:2375 --experimental docker run -t --rm --env="DOCKER_HOST=tcp://dpy-dind-py3:2375" --env="DOCKER_TEST_API_VERSION=${TEST_API_VERSION}"\ --network dpy-tests docker-sdk-python3 py.test tests/integration docker rm -vf dpy-dind-py3 @@ -76,7 +76,7 @@ integration-dind-ssl: build-dind-certs build build-py3 docker run -d --env="DOCKER_HOST=tcp://localhost:2375" --env="DOCKER_TLS_VERIFY=1"\ --env="DOCKER_CERT_PATH=/certs" --volumes-from dpy-dind-certs --name dpy-dind-ssl\ --network dpy-tests --network-alias docker -v /tmp --privileged\ - dockerswarm/dind:${TEST_ENGINE_VERSION}\ + docker:${TEST_ENGINE_VERSION}-dind\ dockerd --tlsverify --tlscacert=/certs/ca.pem --tlscert=/certs/server-cert.pem\ --tlskey=/certs/server-key.pem -H tcp://0.0.0.0:2375 --experimental docker run -t --rm --volumes-from dpy-dind-ssl --env="DOCKER_HOST=tcp://docker:2375"\ From 51fd6dd1ced0b16486ca66a52baf64515e63131f Mon Sep 17 00:00:00 2001 From: Niklas Saari Date: Wed, 26 Feb 2020 22:34:40 +0200 Subject: [PATCH 1007/1301] Disable compression by default when using get_archive method Signed-off-by: Niklas Saari --- docker/api/container.py | 12 ++++++++++-- docker/models/containers.py | 8 ++++++-- tests/unit/models_containers_test.py | 2 +- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/docker/api/container.py b/docker/api/container.py index 45bd3528ba..391832af65 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -694,7 +694,8 @@ def export(self, container, chunk_size=DEFAULT_DATA_CHUNK_SIZE): return self._stream_raw_result(res, chunk_size, False) @utils.check_resource('container') - def get_archive(self, container, path, chunk_size=DEFAULT_DATA_CHUNK_SIZE): + def get_archive(self, container, path, chunk_size=DEFAULT_DATA_CHUNK_SIZE, + encode_stream=False): """ Retrieve a file or folder from a container in the form of a tar archive. @@ -705,6 +706,8 @@ def get_archive(self, container, path, chunk_size=DEFAULT_DATA_CHUNK_SIZE): chunk_size (int): The number of bytes returned by each iteration of the generator. If ``None``, data will be streamed as it is received. Default: 2 MB + encode_stream (bool): Determines if data should be encoded + (gzip-compressed) during transmission. Default: False Returns: (tuple): First element is a raw tar data stream. Second element is @@ -729,8 +732,13 @@ def get_archive(self, container, path, chunk_size=DEFAULT_DATA_CHUNK_SIZE): params = { 'path': path } + headers = { + "Accept-Encoding": "gzip, deflate" + } if encode_stream else { + "Accept-Encoding": "identity" + } url = self._url('/containers/{0}/archive', container) - res = self._get(url, params=params, stream=True) + res = self._get(url, params=params, stream=True, headers=headers) self._raise_for_status(res) encoded_stat = res.headers.get('x-docker-container-path-stat') return ( diff --git a/docker/models/containers.py b/docker/models/containers.py index d1f275f74f..f143d4244b 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -225,7 +225,8 @@ def export(self, chunk_size=DEFAULT_DATA_CHUNK_SIZE): """ return self.client.api.export(self.id, chunk_size) - def get_archive(self, path, chunk_size=DEFAULT_DATA_CHUNK_SIZE): + def get_archive(self, path, chunk_size=DEFAULT_DATA_CHUNK_SIZE, + encode_stream=False): """ Retrieve a file or folder from the container in the form of a tar archive. @@ -235,6 +236,8 @@ def get_archive(self, path, chunk_size=DEFAULT_DATA_CHUNK_SIZE): chunk_size (int): The number of bytes returned by each iteration of the generator. If ``None``, data will be streamed as it is received. Default: 2 MB + encode_stream (bool): Determines if data should be encoded + (gzip-compressed) during transmission. Default: False Returns: (tuple): First element is a raw tar data stream. Second element is @@ -255,7 +258,8 @@ def get_archive(self, path, chunk_size=DEFAULT_DATA_CHUNK_SIZE): ... f.write(chunk) >>> f.close() """ - return self.client.api.get_archive(self.id, path, chunk_size) + return self.client.api.get_archive(self.id, path, + chunk_size, encode_stream) def kill(self, signal=None): """ diff --git a/tests/unit/models_containers_test.py b/tests/unit/models_containers_test.py index da5f0ab9d9..c9f73f3737 100644 --- a/tests/unit/models_containers_test.py +++ b/tests/unit/models_containers_test.py @@ -450,7 +450,7 @@ def test_get_archive(self): container = client.containers.get(FAKE_CONTAINER_ID) container.get_archive('foo') client.api.get_archive.assert_called_with( - FAKE_CONTAINER_ID, 'foo', DEFAULT_DATA_CHUNK_SIZE + FAKE_CONTAINER_ID, 'foo', DEFAULT_DATA_CHUNK_SIZE, False ) def test_image(self): From dac038aca2fd47328846a7f98457b574d31b33ab Mon Sep 17 00:00:00 2001 From: Leo Hanisch <23164374+HaaLeo@users.noreply.github.com> Date: Fri, 20 Mar 2020 12:40:58 +0100 Subject: [PATCH 1008/1301] Fixes docker/docker-py#2533 Signed-off-by: Leo Hanisch <23164374+HaaLeo@users.noreply.github.com> --- docker/transport/sshconn.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/transport/sshconn.py b/docker/transport/sshconn.py index 7de0e59087..9cfd99809b 100644 --- a/docker/transport/sshconn.py +++ b/docker/transport/sshconn.py @@ -100,6 +100,8 @@ def __init__(self, base_url, timeout=60, ) if 'hostname' in host_config: self.ssh_params['hostname'] = host_config['hostname'] + if 'identityfile' in host_config: + self.ssh_params['key_filename'] = host_config['identityfile'] if base_url.port is None and 'port' in host_config: self.ssh_params['port'] = self.ssh_conf['port'] if base_url.username is None and 'user' in host_config: From 7d92fbdee1b8621f54faa595ba53d7ef78ef1acc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Fri, 17 Apr 2020 09:37:34 -0300 Subject: [PATCH 1009/1301] Fix tests to support both log plugin feedbacks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Wilson Júnior Docker-DCO-1.1-Signed-off-by: Wilson Júnior (github: wpjunior) --- tests/integration/api_container_test.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index c503a3674e..411d4c2e2f 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -273,11 +273,14 @@ def test_valid_log_driver_and_log_opt(self): def test_invalid_log_driver_raises_exception(self): log_config = docker.types.LogConfig( - type='asdf-nope', + type='asdf', config={} ) - expected_msg = "logger: no log driver named 'asdf-nope' is registered" + expected_msgs = [ + "logger: no log driver named 'asdf' is registered", + "looking up logging plugin asdf: plugin \"asdf\" not found", + ] with pytest.raises(docker.errors.APIError) as excinfo: # raises an internal server error 500 container = self.client.create_container( @@ -287,7 +290,7 @@ def test_invalid_log_driver_raises_exception(self): ) self.client.start(container) - assert excinfo.value.explanation == expected_msg + assert excinfo.value.explanation in expected_msgs def test_valid_no_log_driver_specified(self): log_config = docker.types.LogConfig( From a07b5ee16c1368a5873cfa08e5f407cbe7d275f5 Mon Sep 17 00:00:00 2001 From: fengbaolong Date: Tue, 28 Apr 2020 16:37:02 +0800 Subject: [PATCH 1010/1301] fix docker build error when dockerfile contains unicode character. if dockerfile contains unicode character,len(contents) will return character length,this length will less than len(contents_encoded) length,so contants data will be truncated. Signed-off-by: fengbaolong --- docker/utils/build.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docker/utils/build.py b/docker/utils/build.py index 4fa5751870..5787cab0fd 100644 --- a/docker/utils/build.py +++ b/docker/utils/build.py @@ -105,8 +105,9 @@ def create_archive(root, files=None, fileobj=None, gzip=False, for name, contents in extra_files: info = tarfile.TarInfo(name) - info.size = len(contents) - t.addfile(info, io.BytesIO(contents.encode('utf-8'))) + contents_encoded = contents.encode('utf-8') + info.size = len(contents_encoded) + t.addfile(info, io.BytesIO(contents_encoded)) t.close() fileobj.seek(0) From df7bf5f5e0a5baafbbf5b88638c09abfd288f686 Mon Sep 17 00:00:00 2001 From: Mike Haboustak Date: Fri, 24 Apr 2020 06:42:59 -0400 Subject: [PATCH 1011/1301] Add support for DriverOpts in EndpointConfig Docker API 1.32 added support for providing options to a network driver via EndpointConfig when connecting a container to a network. Signed-off-by: Mike Haboustak --- docker/api/container.py | 2 ++ docker/api/network.py | 5 +++-- docker/models/networks.py | 2 ++ docker/types/networks.py | 11 ++++++++++- tests/integration/api_network_test.py | 21 +++++++++++++++++++++ tests/unit/api_network_test.py | 4 +++- 6 files changed, 41 insertions(+), 4 deletions(-) diff --git a/docker/api/container.py b/docker/api/container.py index 45bd3528ba..9df22a5217 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -636,6 +636,8 @@ def create_endpoint_config(self, *args, **kwargs): network, using the IPv6 protocol. Defaults to ``None``. link_local_ips (:py:class:`list`): A list of link-local (IPv4/IPv6) addresses. + driver_opt (dict): A dictionary of options to provide to the + network driver. Defaults to ``None``. Returns: (dict) An endpoint config. diff --git a/docker/api/network.py b/docker/api/network.py index 750b91b200..139c2d1a82 100644 --- a/docker/api/network.py +++ b/docker/api/network.py @@ -216,7 +216,7 @@ def inspect_network(self, net_id, verbose=None, scope=None): def connect_container_to_network(self, container, net_id, ipv4_address=None, ipv6_address=None, aliases=None, links=None, - link_local_ips=None): + link_local_ips=None, driver_opt=None): """ Connect a container to a network. @@ -240,7 +240,8 @@ def connect_container_to_network(self, container, net_id, "Container": container, "EndpointConfig": self.create_endpoint_config( aliases=aliases, links=links, ipv4_address=ipv4_address, - ipv6_address=ipv6_address, link_local_ips=link_local_ips + ipv6_address=ipv6_address, link_local_ips=link_local_ips, + driver_opt=driver_opt ), } diff --git a/docker/models/networks.py b/docker/models/networks.py index f944c8e299..093deb7fe3 100644 --- a/docker/models/networks.py +++ b/docker/models/networks.py @@ -46,6 +46,8 @@ def connect(self, container, *args, **kwargs): network, using the IPv6 protocol. Defaults to ``None``. link_local_ips (:py:class:`list`): A list of link-local (IPv4/IPv6) addresses. + driver_opt (dict): A dictionary of options to provide to the + network driver. Defaults to ``None``. Raises: :py:class:`docker.errors.APIError` diff --git a/docker/types/networks.py b/docker/types/networks.py index 1c7b2c9e69..1370dc19fd 100644 --- a/docker/types/networks.py +++ b/docker/types/networks.py @@ -4,7 +4,7 @@ class EndpointConfig(dict): def __init__(self, version, aliases=None, links=None, ipv4_address=None, - ipv6_address=None, link_local_ips=None): + ipv6_address=None, link_local_ips=None, driver_opt=None): if version_lt(version, '1.22'): raise errors.InvalidVersion( 'Endpoint config is not supported for API version < 1.22' @@ -33,6 +33,15 @@ def __init__(self, version, aliases=None, links=None, ipv4_address=None, if ipam_config: self['IPAMConfig'] = ipam_config + if driver_opt: + if version_lt(version, '1.32'): + raise errors.InvalidVersion( + 'DriverOpts is not supported for API version < 1.32' + ) + if not isinstance(driver_opt, dict): + raise TypeError('driver_opt must be a dictionary') + self['DriverOpts'] = driver_opt + class NetworkingConfig(dict): def __init__(self, endpoints_config=None): diff --git a/tests/integration/api_network_test.py b/tests/integration/api_network_test.py index 0f26827b17..af22da8d2d 100644 --- a/tests/integration/api_network_test.py +++ b/tests/integration/api_network_test.py @@ -275,6 +275,27 @@ def test_create_with_linklocal_ips(self): assert 'LinkLocalIPs' in net_cfg['IPAMConfig'] assert net_cfg['IPAMConfig']['LinkLocalIPs'] == ['169.254.8.8'] + @requires_api_version('1.32') + def test_create_with_driveropt(self): + container = self.client.create_container( + TEST_IMG, 'top', + networking_config=self.client.create_networking_config( + { + 'bridge': self.client.create_endpoint_config( + driver_opt={'com.docker-py.setting': 'on'} + ) + } + ), + host_config=self.client.create_host_config(network_mode='bridge') + ) + self.tmp_containers.append(container) + self.client.start(container) + container_data = self.client.inspect_container(container) + net_cfg = container_data['NetworkSettings']['Networks']['bridge'] + assert 'DriverOpts' in net_cfg + assert 'com.docker-py.setting' in net_cfg['DriverOpts'] + assert net_cfg['DriverOpts']['com.docker-py.setting'] == 'on' + @requires_api_version('1.22') def test_create_with_links(self): net_name, net_id = self.create_network() diff --git a/tests/unit/api_network_test.py b/tests/unit/api_network_test.py index c78554da67..758f013230 100644 --- a/tests/unit/api_network_test.py +++ b/tests/unit/api_network_test.py @@ -136,7 +136,8 @@ def test_connect_container_to_network(self): container={'Id': container_id}, net_id=network_id, aliases=['foo', 'bar'], - links=[('baz', 'quux')] + links=[('baz', 'quux')], + driver_opt={'com.docker-py.setting': 'yes'}, ) assert post.call_args[0][0] == ( @@ -148,6 +149,7 @@ def test_connect_container_to_network(self): 'EndpointConfig': { 'Aliases': ['foo', 'bar'], 'Links': ['baz:quux'], + 'DriverOpts': {'com.docker-py.setting': 'yes'}, }, } From 81eb5d42c99b8fc5cae975e8beeadc716caf349a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ville=20Skytt=C3=A4?= Date: Mon, 25 May 2020 08:31:24 +0300 Subject: [PATCH 1012/1301] Fix parameter names in TLSConfig error messages and comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Ville Skyttä --- docker/tls.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docker/tls.py b/docker/tls.py index d4671d126a..1b297ab666 100644 --- a/docker/tls.py +++ b/docker/tls.py @@ -32,7 +32,7 @@ def __init__(self, client_cert=None, ca_cert=None, verify=None, # https://docs.docker.com/engine/articles/https/ # This diverges from the Docker CLI in that users can specify 'tls' # here, but also disable any public/default CA pool verification by - # leaving tls_verify=False + # leaving verify=False self.assert_hostname = assert_hostname self.assert_fingerprint = assert_fingerprint @@ -62,7 +62,7 @@ def __init__(self, client_cert=None, ca_cert=None, verify=None, # https://github.com/docker/docker-py/issues/963 self.ssl_version = ssl.PROTOCOL_TLSv1 - # "tls" and "tls_verify" must have both or neither cert/key files In + # "client_cert" must have both or neither cert/key files. In # either case, Alert the user when both are expected, but any are # missing. @@ -71,7 +71,7 @@ def __init__(self, client_cert=None, ca_cert=None, verify=None, tls_cert, tls_key = client_cert except ValueError: raise errors.TLSParameterError( - 'client_config must be a tuple of' + 'client_cert must be a tuple of' ' (client certificate, key file)' ) @@ -79,7 +79,7 @@ def __init__(self, client_cert=None, ca_cert=None, verify=None, not os.path.isfile(tls_key)): raise errors.TLSParameterError( 'Path to a certificate and key files must be provided' - ' through the client_config param' + ' through the client_cert param' ) self.cert = (tls_cert, tls_key) @@ -88,7 +88,7 @@ def __init__(self, client_cert=None, ca_cert=None, verify=None, self.ca_cert = ca_cert if self.verify and self.ca_cert and not os.path.isfile(self.ca_cert): raise errors.TLSParameterError( - 'Invalid CA certificate provided for `tls_ca_cert`.' + 'Invalid CA certificate provided for `ca_cert`.' ) def configure_client(self, client): From 3ce2d8959da21484a531f87117c65187bcfed0ea Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Thu, 28 May 2020 20:53:45 +0200 Subject: [PATCH 1013/1301] Specify when to use `tls` on Context constructor Signed-off-by: Ulysses Souza --- docker/context/config.py | 4 ++-- docker/context/context.py | 9 +++++---- tests/unit/context_test.py | 4 ++++ 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/docker/context/config.py b/docker/context/config.py index ac9a342ef9..baf54f797e 100644 --- a/docker/context/config.py +++ b/docker/context/config.py @@ -73,8 +73,8 @@ def get_tls_dir(name=None, endpoint=""): return os.path.join(context_dir, "tls") -def get_context_host(path=None): - host = utils.parse_host(path, IS_WINDOWS_PLATFORM) +def get_context_host(path=None, tls=False): + host = utils.parse_host(path, IS_WINDOWS_PLATFORM, tls) if host == DEFAULT_UNIX_SOCKET: # remove http+ from default docker socket url return host.strip("http+") diff --git a/docker/context/context.py b/docker/context/context.py index 4a0549ca9e..fdc290a00a 100644 --- a/docker/context/context.py +++ b/docker/context/context.py @@ -11,7 +11,8 @@ class Context: """A context.""" - def __init__(self, name, orchestrator="swarm", host=None, endpoints=None): + def __init__(self, name, orchestrator="swarm", host=None, endpoints=None, + tls=False): if not name: raise Exception("Name not provided") self.name = name @@ -22,8 +23,8 @@ def __init__(self, name, orchestrator="swarm", host=None, endpoints=None): ) else orchestrator self.endpoints = { default_endpoint: { - "Host": get_context_host(host), - "SkipTLSVerify": False + "Host": get_context_host(host, tls), + "SkipTLSVerify": not tls } } else: @@ -44,7 +45,7 @@ def set_endpoint( self, name="docker", host=None, tls_cfg=None, skip_tls_verify=False, def_namespace=None): self.endpoints[name] = { - "Host": get_context_host(host), + "Host": get_context_host(host, not skip_tls_verify), "SkipTLSVerify": skip_tls_verify } if def_namespace: diff --git a/tests/unit/context_test.py b/tests/unit/context_test.py index 5e88c69139..6d6d6726bc 100644 --- a/tests/unit/context_test.py +++ b/tests/unit/context_test.py @@ -37,6 +37,10 @@ def test_default_in_context_list(self): def test_get_current_context(self): assert ContextAPI.get_current_context().Name == "default" + def test_https_host(self): + c = Context("test", host="tcp://testdomain:8080", tls=True) + assert c.Host == "https://testdomain:8080" + def test_context_inspect_without_params(self): ctx = ContextAPI.inspect_context() assert ctx["Name"] == "default" From 1e11ecec34788aea040590b2db76c236ec715e05 Mon Sep 17 00:00:00 2001 From: aiordache Date: Sat, 30 May 2020 11:01:22 +0200 Subject: [PATCH 1014/1301] Make orchestrator field optional Signed-off-by: aiordache --- docker/context/api.py | 16 +++++++--------- docker/context/context.py | 18 ++++++++++-------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/docker/context/api.py b/docker/context/api.py index fc7e8940c0..c45115bce5 100644 --- a/docker/context/api.py +++ b/docker/context/api.py @@ -14,11 +14,11 @@ class ContextAPI(object): Contains methods for context management: create, list, remove, get, inspect. """ - DEFAULT_CONTEXT = Context("default") + DEFAULT_CONTEXT = Context("default", "swarm") @classmethod def create_context( - cls, name, orchestrator="swarm", host=None, tls_cfg=None, + cls, name, orchestrator=None, host=None, tls_cfg=None, default_namespace=None, skip_tls_verify=False): """Creates a new context. Returns: @@ -38,9 +38,7 @@ def create_context( >>> print(ctx.Metadata) { "Name": "test", - "Metadata": { - "StackOrchestrator": "swarm" - }, + "Metadata": {}, "Endpoints": { "docker": { "Host": "unix:///var/run/docker.sock", @@ -57,7 +55,9 @@ def create_context( ctx = Context.load_context(name) if ctx: raise errors.ContextAlreadyExists(name) - endpoint = "docker" if orchestrator == "swarm" else orchestrator + endpoint = "docker" + if orchestrator and orchestrator != "swarm": + endpoint = orchestrator ctx = Context(name, orchestrator) ctx.set_endpoint( endpoint, host, tls_cfg, @@ -79,9 +79,7 @@ def get_context(cls, name=None): >>> print(ctx.Metadata) { "Name": "test", - "Metadata": { - "StackOrchestrator": "swarm" - }, + "Metadata": {}, "Endpoints": { "docker": { "Host": "unix:///var/run/docker.sock", diff --git a/docker/context/context.py b/docker/context/context.py index fdc290a00a..b2af20c61a 100644 --- a/docker/context/context.py +++ b/docker/context/context.py @@ -11,7 +11,7 @@ class Context: """A context.""" - def __init__(self, name, orchestrator="swarm", host=None, endpoints=None, + def __init__(self, name, orchestrator=None, host=None, endpoints=None, tls=False): if not name: raise Exception("Name not provided") @@ -19,7 +19,7 @@ def __init__(self, name, orchestrator="swarm", host=None, endpoints=None, self.orchestrator = orchestrator if not endpoints: default_endpoint = "docker" if ( - orchestrator == "swarm" + not orchestrator or orchestrator == "swarm" ) else orchestrator self.endpoints = { default_endpoint: { @@ -85,7 +85,8 @@ def _load_meta(cls, name): context {} : {}""".format(name, e)) return ( - metadata["Name"], metadata["Metadata"]["StackOrchestrator"], + metadata["Name"], + metadata["Metadata"].get("StackOrchestrator", None), metadata["Endpoints"]) return None, None, None @@ -162,7 +163,7 @@ def Name(self): @property def Host(self): - if self.orchestrator == "swarm": + if not self.orchestrator or self.orchestrator == "swarm": return self.endpoints["docker"]["Host"] return self.endpoints[self.orchestrator]["Host"] @@ -172,18 +173,19 @@ def Orchestrator(self): @property def Metadata(self): + meta = {} + if self.orchestrator: + meta = {"StackOrchestrator": self.orchestrator} return { "Name": self.name, - "Metadata": { - "StackOrchestrator": self.orchestrator - }, + "Metadata": meta, "Endpoints": self.endpoints } @property def TLSConfig(self): key = self.orchestrator - if key == "swarm": + if not key or key == "swarm": key = "docker" if key in self.tls_cfg.keys(): return self.tls_cfg[key] From 71339167981b154a595869cb3d634d2c1c07741e Mon Sep 17 00:00:00 2001 From: aiordache Date: Tue, 2 Jun 2020 10:45:52 +0200 Subject: [PATCH 1015/1301] add test for context load without orchestrator Signed-off-by: aiordache --- tests/integration/context_api_test.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/integration/context_api_test.py b/tests/integration/context_api_test.py index 60235ee7ba..a2a12a5cb0 100644 --- a/tests/integration/context_api_test.py +++ b/tests/integration/context_api_test.py @@ -50,3 +50,10 @@ def test_context_remove(self): ContextAPI.remove_context("test") with pytest.raises(errors.ContextNotFound): ContextAPI.inspect_context("test") + + def test_load_context_without_orchestrator(self): + ContextAPI.create_context("test") + ctx = ContextAPI.get_context("test") + assert ctx + assert ctx.Name == "test" + assert ctx.Orchestrator is None From fd4526a7d34a08d55532dac34d0e94804176de10 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Mon, 17 Feb 2020 10:19:56 +0100 Subject: [PATCH 1016/1301] xfail "docker top" tests, and adjust for alpine image Signed-off-by: Sebastiaan van Stijn --- tests/integration/api_container_test.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index 1ba3eaa583..c503a3674e 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -1102,6 +1102,8 @@ def test_port(self): class ContainerTopTest(BaseAPIIntegrationTest): + @pytest.mark.xfail(reason='Output of docker top depends on host distro, ' + 'and is not formalized.') def test_top(self): container = self.client.create_container( TEST_IMG, ['sleep', '60'] @@ -1112,9 +1114,7 @@ def test_top(self): self.client.start(container) res = self.client.top(container) if not IS_WINDOWS_PLATFORM: - assert res['Titles'] == [ - 'UID', 'PID', 'PPID', 'C', 'STIME', 'TTY', 'TIME', 'CMD' - ] + assert res['Titles'] == [u'PID', u'USER', u'TIME', u'COMMAND'] assert len(res['Processes']) == 1 assert res['Processes'][0][-1] == 'sleep 60' self.client.kill(container) @@ -1122,6 +1122,8 @@ def test_top(self): @pytest.mark.skipif( IS_WINDOWS_PLATFORM, reason='No psargs support on windows' ) + @pytest.mark.xfail(reason='Output of docker top depends on host distro, ' + 'and is not formalized.') def test_top_with_psargs(self): container = self.client.create_container( TEST_IMG, ['sleep', '60']) @@ -1129,11 +1131,8 @@ def test_top_with_psargs(self): self.tmp_containers.append(container) self.client.start(container) - res = self.client.top(container, 'waux') - assert res['Titles'] == [ - 'USER', 'PID', '%CPU', '%MEM', 'VSZ', 'RSS', - 'TTY', 'STAT', 'START', 'TIME', 'COMMAND' - ] + res = self.client.top(container, '-eopid,user') + assert res['Titles'] == [u'PID', u'USER'] assert len(res['Processes']) == 1 assert res['Processes'][0][10] == 'sleep 60' From db6a2471f527c69b33840ed1121114dd526a0134 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Sat, 15 Feb 2020 00:13:16 +0100 Subject: [PATCH 1017/1301] Use official docker:dind image instead of custom image This replaces the custom dockerswarm/dind image with the official dind images, which should provide the same functionality. Signed-off-by: Sebastiaan van Stijn --- Jenkinsfile | 2 +- Makefile | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 7af23e9cef..28511b2289 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -84,7 +84,7 @@ def runTests = { Map settings -> try { sh """docker network create ${testNetwork}""" sh """docker run -d --name ${dindContainerName} -v /tmp --privileged --network ${testNetwork} \\ - dockerswarm/dind:${dockerVersion} dockerd -H tcp://0.0.0.0:2375 + docker:${dockerVersion}-dind dockerd -H tcp://0.0.0.0:2375 """ sh """docker run \\ --name ${testContainerName} \\ diff --git a/Makefile b/Makefile index db103f5bdf..f456283fdf 100644 --- a/Makefile +++ b/Makefile @@ -55,7 +55,7 @@ integration-dind: integration-dind-py2 integration-dind-py3 integration-dind-py2: build setup-network docker rm -vf dpy-dind-py2 || : docker run -d --network dpy-tests --name dpy-dind-py2 --privileged\ - dockerswarm/dind:${TEST_ENGINE_VERSION} dockerd -H tcp://0.0.0.0:2375 --experimental + docker:${TEST_ENGINE_VERSION}-dind dockerd -H tcp://0.0.0.0:2375 --experimental docker run -t --rm --env="DOCKER_HOST=tcp://dpy-dind-py2:2375" --env="DOCKER_TEST_API_VERSION=${TEST_API_VERSION}"\ --network dpy-tests docker-sdk-python py.test tests/integration docker rm -vf dpy-dind-py2 @@ -64,7 +64,7 @@ integration-dind-py2: build setup-network integration-dind-py3: build-py3 setup-network docker rm -vf dpy-dind-py3 || : docker run -d --network dpy-tests --name dpy-dind-py3 --privileged\ - dockerswarm/dind:${TEST_ENGINE_VERSION} dockerd -H tcp://0.0.0.0:2375 --experimental + docker:${TEST_ENGINE_VERSION}-dind dockerd -H tcp://0.0.0.0:2375 --experimental docker run -t --rm --env="DOCKER_HOST=tcp://dpy-dind-py3:2375" --env="DOCKER_TEST_API_VERSION=${TEST_API_VERSION}"\ --network dpy-tests docker-sdk-python3 py.test tests/integration docker rm -vf dpy-dind-py3 @@ -76,7 +76,7 @@ integration-dind-ssl: build-dind-certs build build-py3 docker run -d --env="DOCKER_HOST=tcp://localhost:2375" --env="DOCKER_TLS_VERIFY=1"\ --env="DOCKER_CERT_PATH=/certs" --volumes-from dpy-dind-certs --name dpy-dind-ssl\ --network dpy-tests --network-alias docker -v /tmp --privileged\ - dockerswarm/dind:${TEST_ENGINE_VERSION}\ + docker:${TEST_ENGINE_VERSION}-dind\ dockerd --tlsverify --tlscacert=/certs/ca.pem --tlscert=/certs/server-cert.pem\ --tlskey=/certs/server-key.pem -H tcp://0.0.0.0:2375 --experimental docker run -t --rm --volumes-from dpy-dind-ssl --env="DOCKER_HOST=tcp://docker:2375"\ From 9713227d7bca2ae37357a500a4e80e6bab152b16 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Fri, 14 Feb 2020 23:51:44 +0100 Subject: [PATCH 1018/1301] Jenkinsfile: remove obsolete engine versions Signed-off-by: Sebastiaan van Stijn --- Jenkinsfile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 28511b2289..f905325c68 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -31,7 +31,7 @@ def buildImages = { -> } def getDockerVersions = { -> - def dockerVersions = ["17.06.2-ce"] + def dockerVersions = ["19.03.5"] wrappedNode(label: "ubuntu && !zfs && amd64") { def result = sh(script: """docker run --rm \\ --entrypoint=python \\ @@ -46,8 +46,6 @@ def getDockerVersions = { -> def getAPIVersion = { engineVersion -> def versionMap = [ - '17.06': '1.30', - '18.03': '1.37', '18.09': '1.39', '19.03': '1.40' ] From 913d129dc9e5cb84bfe385a1b58badfae48e1344 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Fri, 14 Feb 2020 23:54:20 +0100 Subject: [PATCH 1019/1301] Update test engine version to 19.03.5 Signed-off-by: Sebastiaan van Stijn --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index f456283fdf..551868eccf 100644 --- a/Makefile +++ b/Makefile @@ -42,7 +42,7 @@ integration-test-py3: build-py3 docker run -t --rm -v /var/run/docker.sock:/var/run/docker.sock docker-sdk-python3 py.test -v tests/integration/${file} TEST_API_VERSION ?= 1.35 -TEST_ENGINE_VERSION ?= 18.09.5 +TEST_ENGINE_VERSION ?= 19.03.5 .PHONY: setup-network setup-network: From 9b59e4911309dc1e9ff9017f2adad8bad8060e80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Fri, 17 Apr 2020 09:37:34 -0300 Subject: [PATCH 1020/1301] Fix tests to support both log plugin feedbacks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Wilson Júnior Docker-DCO-1.1-Signed-off-by: Wilson Júnior (github: wpjunior) --- tests/integration/api_container_test.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index c503a3674e..411d4c2e2f 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -273,11 +273,14 @@ def test_valid_log_driver_and_log_opt(self): def test_invalid_log_driver_raises_exception(self): log_config = docker.types.LogConfig( - type='asdf-nope', + type='asdf', config={} ) - expected_msg = "logger: no log driver named 'asdf-nope' is registered" + expected_msgs = [ + "logger: no log driver named 'asdf' is registered", + "looking up logging plugin asdf: plugin \"asdf\" not found", + ] with pytest.raises(docker.errors.APIError) as excinfo: # raises an internal server error 500 container = self.client.create_container( @@ -287,7 +290,7 @@ def test_invalid_log_driver_raises_exception(self): ) self.client.start(container) - assert excinfo.value.explanation == expected_msg + assert excinfo.value.explanation in expected_msgs def test_valid_no_log_driver_specified(self): log_config = docker.types.LogConfig( From 105efa02a9016646998400efe3cb4f0c7dcce16b Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Thu, 28 May 2020 20:53:45 +0200 Subject: [PATCH 1021/1301] Specify when to use `tls` on Context constructor Signed-off-by: Ulysses Souza --- docker/context/config.py | 4 ++-- docker/context/context.py | 9 +++++---- tests/unit/context_test.py | 4 ++++ 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/docker/context/config.py b/docker/context/config.py index ac9a342ef9..baf54f797e 100644 --- a/docker/context/config.py +++ b/docker/context/config.py @@ -73,8 +73,8 @@ def get_tls_dir(name=None, endpoint=""): return os.path.join(context_dir, "tls") -def get_context_host(path=None): - host = utils.parse_host(path, IS_WINDOWS_PLATFORM) +def get_context_host(path=None, tls=False): + host = utils.parse_host(path, IS_WINDOWS_PLATFORM, tls) if host == DEFAULT_UNIX_SOCKET: # remove http+ from default docker socket url return host.strip("http+") diff --git a/docker/context/context.py b/docker/context/context.py index 4a0549ca9e..fdc290a00a 100644 --- a/docker/context/context.py +++ b/docker/context/context.py @@ -11,7 +11,8 @@ class Context: """A context.""" - def __init__(self, name, orchestrator="swarm", host=None, endpoints=None): + def __init__(self, name, orchestrator="swarm", host=None, endpoints=None, + tls=False): if not name: raise Exception("Name not provided") self.name = name @@ -22,8 +23,8 @@ def __init__(self, name, orchestrator="swarm", host=None, endpoints=None): ) else orchestrator self.endpoints = { default_endpoint: { - "Host": get_context_host(host), - "SkipTLSVerify": False + "Host": get_context_host(host, tls), + "SkipTLSVerify": not tls } } else: @@ -44,7 +45,7 @@ def set_endpoint( self, name="docker", host=None, tls_cfg=None, skip_tls_verify=False, def_namespace=None): self.endpoints[name] = { - "Host": get_context_host(host), + "Host": get_context_host(host, not skip_tls_verify), "SkipTLSVerify": skip_tls_verify } if def_namespace: diff --git a/tests/unit/context_test.py b/tests/unit/context_test.py index 5e88c69139..6d6d6726bc 100644 --- a/tests/unit/context_test.py +++ b/tests/unit/context_test.py @@ -37,6 +37,10 @@ def test_default_in_context_list(self): def test_get_current_context(self): assert ContextAPI.get_current_context().Name == "default" + def test_https_host(self): + c = Context("test", host="tcp://testdomain:8080", tls=True) + assert c.Host == "https://testdomain:8080" + def test_context_inspect_without_params(self): ctx = ContextAPI.inspect_context() assert ctx["Name"] == "default" From 31276df6a31511f5d1654b98112f2ea02dea4a91 Mon Sep 17 00:00:00 2001 From: aiordache Date: Sat, 30 May 2020 11:01:22 +0200 Subject: [PATCH 1022/1301] Make orchestrator field optional Signed-off-by: aiordache --- docker/context/api.py | 16 +++++++--------- docker/context/context.py | 18 ++++++++++-------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/docker/context/api.py b/docker/context/api.py index fc7e8940c0..c45115bce5 100644 --- a/docker/context/api.py +++ b/docker/context/api.py @@ -14,11 +14,11 @@ class ContextAPI(object): Contains methods for context management: create, list, remove, get, inspect. """ - DEFAULT_CONTEXT = Context("default") + DEFAULT_CONTEXT = Context("default", "swarm") @classmethod def create_context( - cls, name, orchestrator="swarm", host=None, tls_cfg=None, + cls, name, orchestrator=None, host=None, tls_cfg=None, default_namespace=None, skip_tls_verify=False): """Creates a new context. Returns: @@ -38,9 +38,7 @@ def create_context( >>> print(ctx.Metadata) { "Name": "test", - "Metadata": { - "StackOrchestrator": "swarm" - }, + "Metadata": {}, "Endpoints": { "docker": { "Host": "unix:///var/run/docker.sock", @@ -57,7 +55,9 @@ def create_context( ctx = Context.load_context(name) if ctx: raise errors.ContextAlreadyExists(name) - endpoint = "docker" if orchestrator == "swarm" else orchestrator + endpoint = "docker" + if orchestrator and orchestrator != "swarm": + endpoint = orchestrator ctx = Context(name, orchestrator) ctx.set_endpoint( endpoint, host, tls_cfg, @@ -79,9 +79,7 @@ def get_context(cls, name=None): >>> print(ctx.Metadata) { "Name": "test", - "Metadata": { - "StackOrchestrator": "swarm" - }, + "Metadata": {}, "Endpoints": { "docker": { "Host": "unix:///var/run/docker.sock", diff --git a/docker/context/context.py b/docker/context/context.py index fdc290a00a..b2af20c61a 100644 --- a/docker/context/context.py +++ b/docker/context/context.py @@ -11,7 +11,7 @@ class Context: """A context.""" - def __init__(self, name, orchestrator="swarm", host=None, endpoints=None, + def __init__(self, name, orchestrator=None, host=None, endpoints=None, tls=False): if not name: raise Exception("Name not provided") @@ -19,7 +19,7 @@ def __init__(self, name, orchestrator="swarm", host=None, endpoints=None, self.orchestrator = orchestrator if not endpoints: default_endpoint = "docker" if ( - orchestrator == "swarm" + not orchestrator or orchestrator == "swarm" ) else orchestrator self.endpoints = { default_endpoint: { @@ -85,7 +85,8 @@ def _load_meta(cls, name): context {} : {}""".format(name, e)) return ( - metadata["Name"], metadata["Metadata"]["StackOrchestrator"], + metadata["Name"], + metadata["Metadata"].get("StackOrchestrator", None), metadata["Endpoints"]) return None, None, None @@ -162,7 +163,7 @@ def Name(self): @property def Host(self): - if self.orchestrator == "swarm": + if not self.orchestrator or self.orchestrator == "swarm": return self.endpoints["docker"]["Host"] return self.endpoints[self.orchestrator]["Host"] @@ -172,18 +173,19 @@ def Orchestrator(self): @property def Metadata(self): + meta = {} + if self.orchestrator: + meta = {"StackOrchestrator": self.orchestrator} return { "Name": self.name, - "Metadata": { - "StackOrchestrator": self.orchestrator - }, + "Metadata": meta, "Endpoints": self.endpoints } @property def TLSConfig(self): key = self.orchestrator - if key == "swarm": + if not key or key == "swarm": key = "docker" if key in self.tls_cfg.keys(): return self.tls_cfg[key] From 67cad6842ceb9a49fbff70faa8dbff8b7ef20134 Mon Sep 17 00:00:00 2001 From: aiordache Date: Tue, 2 Jun 2020 10:45:52 +0200 Subject: [PATCH 1023/1301] add test for context load without orchestrator Signed-off-by: aiordache --- tests/integration/context_api_test.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/integration/context_api_test.py b/tests/integration/context_api_test.py index 60235ee7ba..a2a12a5cb0 100644 --- a/tests/integration/context_api_test.py +++ b/tests/integration/context_api_test.py @@ -50,3 +50,10 @@ def test_context_remove(self): ContextAPI.remove_context("test") with pytest.raises(errors.ContextNotFound): ContextAPI.inspect_context("test") + + def test_load_context_without_orchestrator(self): + ContextAPI.create_context("test") + ctx = ContextAPI.get_context("test") + assert ctx + assert ctx.Name == "test" + assert ctx.Orchestrator is None From 9923746095d9fd9a8fabf4a8ce5e895ad5a3e48c Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Tue, 2 Jun 2020 15:47:10 +0200 Subject: [PATCH 1024/1301] Bump 4.2.1 Signed-off-by: Ulysses Souza --- docker/version.py | 2 +- docs/change-log.md | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index f0a3170952..d69fbd0d59 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "4.2.0" +version = "4.2.1" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) diff --git a/docs/change-log.md b/docs/change-log.md index 2f0a9ed60c..4a37b59482 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -1,6 +1,16 @@ Change log ========== +4.2.1 +----- + +[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/65?closed=1) + +### Features + +- Add option on when to use `tls` on Context constructor +- Make context orchestrator field optional + 4.2.0 ----- From df08c14c87ee0fa1de4ffaf00c5c26425860a4a1 Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Thu, 28 Mar 2019 14:23:19 +0100 Subject: [PATCH 1025/1301] Bump 3.7.2 Signed-off-by: Ulysses Souza --- docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index a75460921a..8f81f0d5ef 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "4.3.0-dev" +version = "3.7.2" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) From 88163c00f2d222a71fa2d976cb31d811a3f45c96 Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Thu, 6 Feb 2020 10:23:58 +0100 Subject: [PATCH 1026/1301] Bump 4.2.0 Signed-off-by: Ulysses Souza --- docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index 8f81f0d5ef..f0a3170952 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "3.7.2" +version = "4.2.0" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) From 525ff592ee9bf7b0ead948086c5b36cf9a68cd10 Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Tue, 2 Jun 2020 15:47:10 +0200 Subject: [PATCH 1027/1301] Bump 4.2.1 Signed-off-by: Ulysses Souza --- docker/version.py | 2 +- docs/change-log.md | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index f0a3170952..d69fbd0d59 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "4.2.0" +version = "4.2.1" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) diff --git a/docs/change-log.md b/docs/change-log.md index 23330c1247..ab7065a9c0 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -1,6 +1,16 @@ Change log ========== +4.2.1 +----- + +[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/65?closed=1) + +### Features + +- Add option on when to use `tls` on Context constructor +- Make context orchestrator field optional + 4.2.0 ----- From 57a8a0c561025a38b2cf5b90d430fd8a5c9f4649 Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Tue, 2 Jun 2020 16:36:14 +0200 Subject: [PATCH 1028/1301] Update version after 4.2.1 release Signed-off-by: Ulysses Souza --- docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index d69fbd0d59..a75460921a 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "4.2.1" +version = "4.3.0-dev" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) From fefa96cd0ea968bfe1f0e98c0f3c455d82042020 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Wed, 3 Jun 2020 10:43:42 +0200 Subject: [PATCH 1029/1301] Jenkinsfile: update node selection labels Make sure we use the LTS nodes, to prevent using machines that we prepared with cgroups v2 (which is not yet supported by docker v19.03) Signed-off-by: Sebastiaan van Stijn --- Jenkinsfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index f905325c68..8777214ca6 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -17,7 +17,7 @@ def buildImage = { name, buildargs, pyTag -> } def buildImages = { -> - wrappedNode(label: "ubuntu && !zfs && amd64", cleanWorkspace: true) { + wrappedNode(label: "amd64 && ubuntu-1804 && overlay2", cleanWorkspace: true) { stage("build image") { checkout(scm) @@ -32,7 +32,7 @@ def buildImages = { -> def getDockerVersions = { -> def dockerVersions = ["19.03.5"] - wrappedNode(label: "ubuntu && !zfs && amd64") { + wrappedNode(label: "amd64 && ubuntu-1804 && overlay2") { def result = sh(script: """docker run --rm \\ --entrypoint=python \\ ${imageNamePy3} \\ @@ -73,7 +73,7 @@ def runTests = { Map settings -> } { -> - wrappedNode(label: "ubuntu && !zfs && amd64", cleanWorkspace: true) { + wrappedNode(label: "amd64 && ubuntu-1804 && overlay2", cleanWorkspace: true) { stage("test python=${pythonVersion} / docker=${dockerVersion}") { checkout(scm) def dindContainerName = "dpy-dind-\$BUILD_NUMBER-\$EXECUTOR_NUMBER-${pythonVersion}-${dockerVersion}" From bf1a3518f92eb845d1e39c8c18d9ee137f896c32 Mon Sep 17 00:00:00 2001 From: Janosch Deurer Date: Mon, 15 Jun 2020 16:37:54 +0200 Subject: [PATCH 1030/1301] Add healthcheck doc for container.run Signed-off-by: Janosch Deurer --- docker/models/containers.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/docker/models/containers.py b/docker/models/containers.py index d1f275f74f..19477fe6bf 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -593,7 +593,27 @@ def run(self, image, command=None, stdout=True, stderr=False, group_add (:py:class:`list`): List of additional group names and/or IDs that the container process will run as. healthcheck (dict): Specify a test to perform to check that the - container is healthy. + container is healthy. The dict takes the following keys: + - test (:py:class:`list` or str): Test to perform to determine + container health. Possible values: + + - Empty list: Inherit healthcheck from parent image + - ``["NONE"]``: Disable healthcheck + - ``["CMD", args...]``: exec arguments directly. + - ``["CMD-SHELL", command]``: Run command in the system's + default shell. + + If a string is provided, it will be used as a ``CMD-SHELL`` + command. + - interval (int): The time to wait between checks in + nanoseconds. It should be 0 or at least 1000000 (1 ms). + - timeout (int): The time to wait before considering the check + to have hung. It should be 0 or at least 1000000 (1 ms). + - retries (int): The number of consecutive failures needed to + consider a container as unhealthy. + - start_period (int): Start period for the container to + initialize before starting health-retries countdown in + nanoseconds. It should be 0 or at least 1000000 (1 ms). hostname (str): Optional hostname for the container. init (bool): Run an init inside the container that forwards signals and reaps processes From 309ce44052223d374d4e0174e163d28fe195fc5b Mon Sep 17 00:00:00 2001 From: aiordache Date: Thu, 25 Jun 2020 16:27:07 +0200 Subject: [PATCH 1031/1301] Skip parsing non-docker endpoints Signed-off-by: aiordache --- docker/context/context.py | 97 +++++++++++++++++++++++++-------------- 1 file changed, 62 insertions(+), 35 deletions(-) diff --git a/docker/context/context.py b/docker/context/context.py index b2af20c61a..2413b2ecbf 100644 --- a/docker/context/context.py +++ b/docker/context/context.py @@ -16,30 +16,42 @@ def __init__(self, name, orchestrator=None, host=None, endpoints=None, if not name: raise Exception("Name not provided") self.name = name + self.context_type = None self.orchestrator = orchestrator + self.endpoints = {} + self.tls_cfg = {} + self.meta_path = "IN MEMORY" + self.tls_path = "IN MEMORY" + if not endpoints: + # set default docker endpoint if no endpoint is set default_endpoint = "docker" if ( not orchestrator or orchestrator == "swarm" ) else orchestrator + self.endpoints = { default_endpoint: { "Host": get_context_host(host, tls), "SkipTLSVerify": not tls } } - else: - for k, v in endpoints.items(): - ekeys = v.keys() - for param in ["Host", "SkipTLSVerify"]: - if param not in ekeys: - raise ContextException( - "Missing parameter {} from endpoint {}".format( - param, k)) - self.endpoints = endpoints + return - self.tls_cfg = {} - self.meta_path = "IN MEMORY" - self.tls_path = "IN MEMORY" + # check docker endpoints + for k, v in endpoints.items(): + if not isinstance(v, dict): + # unknown format + raise ContextException("""Unknown endpoint format for + context {}: {}""".format(name, v)) + + self.endpoints[k] = v + if k != "docker": + continue + + self.endpoints[k]["Host"] = v.get("Host", get_context_host( + host, tls)) + self.endpoints[k]["SkipTLSVerify"] = bool(v.get( + "SkipTLSVerify", not tls)) def set_endpoint( self, name="docker", host=None, tls_cfg=None, @@ -59,9 +71,13 @@ def inspect(self): @classmethod def load_context(cls, name): - name, orchestrator, endpoints = Context._load_meta(name) - if name: - instance = cls(name, orchestrator, endpoints=endpoints) + meta = Context._load_meta(name) + if meta: + instance = cls( + meta["Name"], + orchestrator=meta["Metadata"].get("StackOrchestrator", None), + endpoints=meta.get("Endpoints", None)) + instance.context_type = meta["Metadata"].get("Type", None) instance._load_certs() instance.meta_path = get_meta_dir(name) return instance @@ -69,26 +85,30 @@ def load_context(cls, name): @classmethod def _load_meta(cls, name): - metadata = {} meta_file = get_meta_file(name) - if os.path.isfile(meta_file): + if not os.path.isfile(meta_file): + return None + + metadata = {} + try: with open(meta_file) as f: - try: - with open(meta_file) as f: - metadata = json.load(f) - for k, v in metadata["Endpoints"].items(): - metadata["Endpoints"][k]["SkipTLSVerify"] = bool( - v["SkipTLSVerify"]) - except (IOError, KeyError, ValueError) as e: - # unknown format - raise Exception("""Detected corrupted meta file for - context {} : {}""".format(name, e)) - - return ( - metadata["Name"], - metadata["Metadata"].get("StackOrchestrator", None), - metadata["Endpoints"]) - return None, None, None + metadata = json.load(f) + except (IOError, KeyError, ValueError) as e: + # unknown format + raise Exception("""Detected corrupted meta file for + context {} : {}""".format(name, e)) + + # for docker endpoints, set defaults for + # Host and SkipTLSVerify fields + for k, v in metadata["Endpoints"].items(): + if k != "docker": + continue + metadata["Endpoints"][k]["Host"] = v.get( + "Host", get_context_host(None, False)) + metadata["Endpoints"][k]["SkipTLSVerify"] = bool( + v.get("SkipTLSVerify", True)) + + return metadata def _load_certs(self): certs = {} @@ -157,6 +177,9 @@ def __call__(self): result.update(self.Storage) return result + def is_docker_host(self): + return self.context_type is None + @property def Name(self): return self.name @@ -164,8 +187,12 @@ def Name(self): @property def Host(self): if not self.orchestrator or self.orchestrator == "swarm": - return self.endpoints["docker"]["Host"] - return self.endpoints[self.orchestrator]["Host"] + endpoint = self.endpoints.get("docker", None) + if endpoint: + return endpoint.get("Host", None) + return None + + return self.endpoints[self.orchestrator].get("Host", None) @property def Orchestrator(self): From 2c68b382a8847118fb11f40675823602d653357d Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Wed, 3 Jun 2020 10:26:41 +0200 Subject: [PATCH 1032/1301] Update test engine version to 19.03.12 Signed-off-by: Sebastiaan van Stijn --- Jenkinsfile | 4 ++-- Makefile | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 8777214ca6..88c21592c0 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -31,7 +31,7 @@ def buildImages = { -> } def getDockerVersions = { -> - def dockerVersions = ["19.03.5"] + def dockerVersions = ["19.03.12"] wrappedNode(label: "amd64 && ubuntu-1804 && overlay2") { def result = sh(script: """docker run --rm \\ --entrypoint=python \\ @@ -66,7 +66,7 @@ def runTests = { Map settings -> throw new Exception("Need test image object, e.g.: `runTests(testImage: img)`") } if (!dockerVersion) { - throw new Exception("Need Docker version to test, e.g.: `runTests(dockerVersion: '1.12.3')`") + throw new Exception("Need Docker version to test, e.g.: `runTests(dockerVersion: '19.03.12')`") } if (!pythonVersion) { throw new Exception("Need Python version being tested, e.g.: `runTests(pythonVersion: 'py2.7')`") diff --git a/Makefile b/Makefile index 551868eccf..4795c63c8a 100644 --- a/Makefile +++ b/Makefile @@ -42,7 +42,7 @@ integration-test-py3: build-py3 docker run -t --rm -v /var/run/docker.sock:/var/run/docker.sock docker-sdk-python3 py.test -v tests/integration/${file} TEST_API_VERSION ?= 1.35 -TEST_ENGINE_VERSION ?= 19.03.5 +TEST_ENGINE_VERSION ?= 19.03.12 .PHONY: setup-network setup-network: From e18a64b6302a24d18f291d14af102917304f330f Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Tue, 30 Jun 2020 17:22:00 +0200 Subject: [PATCH 1033/1301] Bump 4.2.2 Signed-off-by: Ulysses Souza --- docker/version.py | 2 +- docs/change-log.md | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index a75460921a..06d6cc730f 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "4.3.0-dev" +version = "4.2.2" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) diff --git a/docs/change-log.md b/docs/change-log.md index ab7065a9c0..84ed013fb6 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -1,6 +1,15 @@ Change log ========== +4.2.2 +----- + +[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/66?closed=1) + +### Bugfixes + +- Fix context load for non-docker endpoints + 4.2.1 ----- From 6d9847838aec3895a00caf0963c5b5b33b303aab Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Tue, 30 Jun 2020 18:30:49 +0200 Subject: [PATCH 1034/1301] Update version to 4.3.0-dev Signed-off-by: Ulysses Souza --- docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index 06d6cc730f..a75460921a 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "4.2.2" +version = "4.3.0-dev" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) From c65d437843310b46ccb93b1c418ff7da547a5fec Mon Sep 17 00:00:00 2001 From: Ofek Lev Date: Wed, 10 Jun 2020 15:31:19 -0400 Subject: [PATCH 1035/1301] Upgrade Windows dependency Signed-off-by: Ofek Lev --- requirements.txt | 3 +-- setup.py | 5 +---- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/requirements.txt b/requirements.txt index 804a78a0ce..340e431285 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,8 +11,7 @@ paramiko==2.4.2 pycparser==2.17 pyOpenSSL==18.0.0 pyparsing==2.2.0 -pypiwin32==219; sys_platform == 'win32' and python_version < '3.6' -pypiwin32==223; sys_platform == 'win32' and python_version >= '3.6' +pywin32==227; sys_platform == 'win32' requests==2.20.0 six==1.10.0 urllib3==1.24.3 diff --git a/setup.py b/setup.py index c29787b679..c702295080 100644 --- a/setup.py +++ b/setup.py @@ -24,10 +24,7 @@ ':python_version < "3.3"': 'ipaddress >= 1.0.16', # win32 APIs if on Windows (required for npipe support) - # Python 3.6 is only compatible with v220 ; Python < 3.5 is not supported - # on v220 ; ALL versions are broken for v222 (as of 2018-01-26) - ':sys_platform == "win32" and python_version < "3.6"': 'pypiwin32==219', - ':sys_platform == "win32" and python_version >= "3.6"': 'pypiwin32==223', + ':sys_platform == "win32"': 'pywin32==227', # If using docker-py over TLS, highly recommend this option is # pip-installed or pinned. From 26d8045ffa99ec402e451cde67415b14b13cc95a Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Fri, 17 Jul 2020 14:25:27 +0200 Subject: [PATCH 1036/1301] Fix CreateContainerTest.test_invalid_log_driver_raises_exception This test was updated in 7d92fbdee1b8621f54faa595ba53d7ef78ef1acc, but omitted the "error" prefix in the message, causing the test to fail; _________ CreateContainerTest.test_invalid_log_driver_raises_exception _________ tests/integration/api_container_test.py:293: in test_invalid_log_driver_raises_exception assert excinfo.value.explanation in expected_msgs E AssertionError: assert 'error looking up logging plugin asdf: plugin "asdf" not found' in ["logger: no log driver named 'asdf' is registered", 'looking up logging plugin asdf: plugin "asdf" not found'] E + where 'error looking up logging plugin asdf: plugin "asdf" not found' = APIError(HTTPError('400 Client Error: Bad Request for url: http+docker://localhost/v1.39/containers/create')).explanation E + where APIError(HTTPError('400 Client Error: Bad Request for url: http+docker://localhost/v1.39/containers/create')) = .value Signed-off-by: Sebastiaan van Stijn --- tests/integration/api_container_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index 411d4c2e2f..65e611b2f5 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -279,7 +279,7 @@ def test_invalid_log_driver_raises_exception(self): expected_msgs = [ "logger: no log driver named 'asdf' is registered", - "looking up logging plugin asdf: plugin \"asdf\" not found", + "error looking up logging plugin asdf: plugin \"asdf\" not found", ] with pytest.raises(docker.errors.APIError) as excinfo: # raises an internal server error 500 From dd0450a14c407050db141af486cc2ed9639ffd8d Mon Sep 17 00:00:00 2001 From: Lucidiot Date: Fri, 7 Aug 2020 13:58:35 +0200 Subject: [PATCH 1037/1301] Add device requests (#2471) * Add DeviceRequest type Signed-off-by: Erwan Rouchet * Add device_requests kwarg in host config Signed-off-by: Erwan Rouchet * Add unit test for device requests Signed-off-by: Erwan Rouchet * Fix unit test Signed-off-by: Erwan Rouchet * Use parentheses for multiline import Signed-off-by: Erwan Rouchet * Create 1.40 client for device-requests test Signed-off-by: Laurie O Co-authored-by: Laurie O Co-authored-by: Bastien Abadie --- docker/api/container.py | 3 + docker/models/containers.py | 4 ++ docker/types/__init__.py | 4 +- docker/types/containers.py | 113 ++++++++++++++++++++++++++++++- tests/unit/api_container_test.py | 64 ++++++++++++++++- 5 files changed, 185 insertions(+), 3 deletions(-) diff --git a/docker/api/container.py b/docker/api/container.py index 9df22a5217..2ba08e536d 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -480,6 +480,9 @@ def create_host_config(self, *args, **kwargs): For example, ``/dev/sda:/dev/xvda:rwm`` allows the container to have read-write access to the host's ``/dev/sda`` via a node named ``/dev/xvda`` inside the container. + device_requests (:py:class:`list`): Expose host resources such as + GPUs to the container, as a list of + :py:class:`docker.types.DeviceRequest` instances. dns (:py:class:`list`): Set custom DNS servers. dns_opt (:py:class:`list`): Additional options to be added to the container's ``resolv.conf`` file diff --git a/docker/models/containers.py b/docker/models/containers.py index d1f275f74f..e8082ba41a 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -579,6 +579,9 @@ def run(self, image, command=None, stdout=True, stderr=False, For example, ``/dev/sda:/dev/xvda:rwm`` allows the container to have read-write access to the host's ``/dev/sda`` via a node named ``/dev/xvda`` inside the container. + device_requests (:py:class:`list`): Expose host resources such as + GPUs to the container, as a list of + :py:class:`docker.types.DeviceRequest` instances. dns (:py:class:`list`): Set custom DNS servers. dns_opt (:py:class:`list`): Additional options to be added to the container's ``resolv.conf`` file. @@ -998,6 +1001,7 @@ def prune(self, filters=None): 'device_write_bps', 'device_write_iops', 'devices', + 'device_requests', 'dns_opt', 'dns_search', 'dns', diff --git a/docker/types/__init__.py b/docker/types/__init__.py index 5db330e284..b425746e78 100644 --- a/docker/types/__init__.py +++ b/docker/types/__init__.py @@ -1,5 +1,7 @@ # flake8: noqa -from .containers import ContainerConfig, HostConfig, LogConfig, Ulimit +from .containers import ( + ContainerConfig, HostConfig, LogConfig, Ulimit, DeviceRequest +) from .daemon import CancellableStream from .healthcheck import Healthcheck from .networks import EndpointConfig, IPAMConfig, IPAMPool, NetworkingConfig diff --git a/docker/types/containers.py b/docker/types/containers.py index fd8cab4979..149b85dfc9 100644 --- a/docker/types/containers.py +++ b/docker/types/containers.py @@ -154,6 +154,104 @@ def hard(self, value): self['Hard'] = value +class DeviceRequest(DictType): + """ + Create a device request to be used with + :py:meth:`~docker.api.container.ContainerApiMixin.create_host_config`. + + Args: + + driver (str): Which driver to use for this device. Optional. + count (int): Number or devices to request. Optional. + Set to -1 to request all available devices. + device_ids (list): List of strings for device IDs. Optional. + Set either ``count`` or ``device_ids``. + capabilities (list): List of lists of strings to request + capabilities. Optional. The global list acts like an OR, + and the sub-lists are AND. The driver will try to satisfy + one of the sub-lists. + Available capabilities for the ``nvidia`` driver can be found + `here `_. + options (dict): Driver-specific options. Optional. + """ + + def __init__(self, **kwargs): + driver = kwargs.get('driver', kwargs.get('Driver')) + count = kwargs.get('count', kwargs.get('Count')) + device_ids = kwargs.get('device_ids', kwargs.get('DeviceIDs')) + capabilities = kwargs.get('capabilities', kwargs.get('Capabilities')) + options = kwargs.get('options', kwargs.get('Options')) + + if driver is None: + driver = '' + elif not isinstance(driver, six.string_types): + raise ValueError('DeviceRequest.driver must be a string') + if count is None: + count = 0 + elif not isinstance(count, int): + raise ValueError('DeviceRequest.count must be an integer') + if device_ids is None: + device_ids = [] + elif not isinstance(device_ids, list): + raise ValueError('DeviceRequest.device_ids must be a list') + if capabilities is None: + capabilities = [] + elif not isinstance(capabilities, list): + raise ValueError('DeviceRequest.capabilities must be a list') + if options is None: + options = {} + elif not isinstance(options, dict): + raise ValueError('DeviceRequest.options must be a dict') + + super(DeviceRequest, self).__init__({ + 'Driver': driver, + 'Count': count, + 'DeviceIDs': device_ids, + 'Capabilities': capabilities, + 'Options': options + }) + + @property + def driver(self): + return self['Driver'] + + @driver.setter + def driver(self, value): + self['Driver'] = value + + @property + def count(self): + return self['Count'] + + @count.setter + def count(self, value): + self['Count'] = value + + @property + def device_ids(self): + return self['DeviceIDs'] + + @device_ids.setter + def device_ids(self, value): + self['DeviceIDs'] = value + + @property + def capabilities(self): + return self['Capabilities'] + + @capabilities.setter + def capabilities(self, value): + self['Capabilities'] = value + + @property + def options(self): + return self['Options'] + + @options.setter + def options(self, value): + self['Options'] = value + + class HostConfig(dict): def __init__(self, version, binds=None, port_bindings=None, lxc_conf=None, publish_all_ports=False, links=None, @@ -176,7 +274,7 @@ def __init__(self, version, binds=None, port_bindings=None, volume_driver=None, cpu_count=None, cpu_percent=None, nano_cpus=None, cpuset_mems=None, runtime=None, mounts=None, cpu_rt_period=None, cpu_rt_runtime=None, - device_cgroup_rules=None): + device_cgroup_rules=None, device_requests=None): if mem_limit is not None: self['Memory'] = parse_bytes(mem_limit) @@ -536,6 +634,19 @@ def __init__(self, version, binds=None, port_bindings=None, ) self['DeviceCgroupRules'] = device_cgroup_rules + if device_requests is not None: + if version_lt(version, '1.40'): + raise host_config_version_error('device_requests', '1.40') + if not isinstance(device_requests, list): + raise host_config_type_error( + 'device_requests', device_requests, 'list' + ) + self['DeviceRequests'] = [] + for req in device_requests: + if not isinstance(req, DeviceRequest): + req = DeviceRequest(**req) + self['DeviceRequests'].append(req) + def host_config_type_error(param, param_value, expected): error_msg = 'Invalid type for {0} param: expected {1} but found {2}' diff --git a/tests/unit/api_container_test.py b/tests/unit/api_container_test.py index a7e183c839..8a0577e78f 100644 --- a/tests/unit/api_container_test.py +++ b/tests/unit/api_container_test.py @@ -5,6 +5,7 @@ import signal import docker +from docker.api import APIClient import pytest import six @@ -12,7 +13,7 @@ from ..helpers import requires_api_version from .api_test import ( BaseAPIClientTest, url_prefix, fake_request, DEFAULT_TIMEOUT_SECONDS, - fake_inspect_container + fake_inspect_container, url_base ) try: @@ -767,6 +768,67 @@ def test_create_container_with_devices(self): assert args[1]['headers'] == {'Content-Type': 'application/json'} assert args[1]['timeout'] == DEFAULT_TIMEOUT_SECONDS + def test_create_container_with_device_requests(self): + client = APIClient(version='1.40') + fake_api.fake_responses.setdefault( + '{0}/v1.40/containers/create'.format(fake_api.prefix), + fake_api.post_fake_create_container, + ) + client.create_container( + 'busybox', 'true', host_config=client.create_host_config( + device_requests=[ + { + 'device_ids': [ + '0', + 'GPU-3a23c669-1f69-c64e-cf85-44e9b07e7a2a' + ] + }, + { + 'driver': 'nvidia', + 'Count': -1, + 'capabilities': [ + ['gpu', 'utility'] + ], + 'options': { + 'key': 'value' + } + } + ] + ) + ) + + args = fake_request.call_args + assert args[0][1] == url_base + 'v1.40/' + 'containers/create' + expected_payload = self.base_create_payload() + expected_payload['HostConfig'] = client.create_host_config() + expected_payload['HostConfig']['DeviceRequests'] = [ + { + 'Driver': '', + 'Count': 0, + 'DeviceIDs': [ + '0', + 'GPU-3a23c669-1f69-c64e-cf85-44e9b07e7a2a' + ], + 'Capabilities': [], + 'Options': {} + }, + { + 'Driver': 'nvidia', + 'Count': -1, + 'DeviceIDs': [], + 'Capabilities': [ + ['gpu', 'utility'] + ], + 'Options': { + 'key': 'value' + } + } + ] + assert json.loads(args[1]['data']) == expected_payload + assert args[1]['headers']['Content-Type'] == 'application/json' + assert set(args[1]['headers']) <= {'Content-Type', 'User-Agent'} + assert args[1]['timeout'] == DEFAULT_TIMEOUT_SECONDS + def test_create_container_with_labels_dict(self): labels_dict = { six.text_type('foo'): six.text_type('1'), From 631abd156ad11433c9c09d957ebcb1868f738a58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ville=20Skytt=C3=A4?= Date: Fri, 7 Aug 2020 15:33:19 +0300 Subject: [PATCH 1038/1301] Spelling fixes (#2571) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Ville Skyttä --- docker/api/container.py | 2 +- docs/change-log.md | 4 ++-- tests/unit/utils_build_test.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docker/api/container.py b/docker/api/container.py index 2ba08e536d..ee3b4c3f28 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -1125,7 +1125,7 @@ def stats(self, container, decode=None, stream=True): else: if decode: raise errors.InvalidArgument( - "decode is only available in conjuction with stream=True" + "decode is only available in conjunction with stream=True" ) return self._result(self._get(url, params={'stream': False}), json=True) diff --git a/docs/change-log.md b/docs/change-log.md index 84ed013fb6..f0be8ac1c3 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -129,7 +129,7 @@ Change log ### Bugfixes -* Fix base_url to keep TCP protocol on utils.py by letting the responsability of changing the +* Fix base_url to keep TCP protocol on utils.py by letting the responsibility of changing the protocol to `parse_host` afterwards, letting `base_url` with the original value. * XFAIL test_attach_stream_and_cancel on TLS @@ -1233,7 +1233,7 @@ like the others (`Client.volumes`, `Client.create_volume`, `Client.inspect_volume`, `Client.remove_volume`). * Added support for the `group_add` parameter in `create_host_config`. -* Added support for the CPU CFS (`cpu_quota` and `cpu_period`) parameteres +* Added support for the CPU CFS (`cpu_quota` and `cpu_period`) parameters in `create_host_config`. * Added support for the archive API endpoint (`Client.get_archive`, `Client.put_archive`). diff --git a/tests/unit/utils_build_test.py b/tests/unit/utils_build_test.py index 012f15b46a..bc6fb5f409 100644 --- a/tests/unit/utils_build_test.py +++ b/tests/unit/utils_build_test.py @@ -335,7 +335,7 @@ def test_parent_directory(self): # Dockerignore reference stipulates that absolute paths are # equivalent to relative paths, hence /../foo should be # equivalent to ../foo. It also stipulates that paths are run - # through Go's filepath.Clean, which explicitely "replace + # through Go's filepath.Clean, which explicitly "replace # "/.." by "/" at the beginning of a path". assert exclude_paths( base, From b4beaaac8cafcec9fe9eb3d6903addd5d9bac4f2 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Fri, 7 Aug 2020 16:45:20 +0200 Subject: [PATCH 1039/1301] Update default API version to v1.39 (#2512) * Update default API version to v1.39 When running the docker-py integration tests in the Moby repository, some tests were skipped because the API version used was too low: SKIPPED [1] tests/integration/api_service_test.py:882: API version is too low (< 1.38) SKIPPED [1] tests/integration/api_swarm_test.py:59: API version is too low (< 1.39) SKIPPED [1] tests/integration/api_swarm_test.py:38: API version is too low (< 1.39) SKIPPED [1] tests/integration/api_swarm_test.py:45: API version is too low (< 1.39) SKIPPED [1] tests/integration/api_swarm_test.py:52: API version is too low (< 1.39) While it's possible to override the API version to use for testing using the `DOCKER_TEST_API_VERSION` environment variable, we may want to set the default to a version that supports all features that were added. This patch updates the default API version to v1.39, which is the minimum version required for those features, and corresponds with Docker 18.09. Note that the API version of the current (19.03) Docker release is v1.40, but using that version as default would exclude users that did not update their Docker version yet (and would not be needed yet for the features provided). Signed-off-by: Sebastiaan van Stijn * Makefile: set DOCKER_TEST_API_VERSION to v1.39 Signed-off-by: Sebastiaan van Stijn --- Makefile | 2 +- docker/constants.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 4795c63c8a..6765d4d77e 100644 --- a/Makefile +++ b/Makefile @@ -41,7 +41,7 @@ integration-test: build integration-test-py3: build-py3 docker run -t --rm -v /var/run/docker.sock:/var/run/docker.sock docker-sdk-python3 py.test -v tests/integration/${file} -TEST_API_VERSION ?= 1.35 +TEST_API_VERSION ?= 1.39 TEST_ENGINE_VERSION ?= 19.03.12 .PHONY: setup-network diff --git a/docker/constants.py b/docker/constants.py index e4daed5d54..c09eedab29 100644 --- a/docker/constants.py +++ b/docker/constants.py @@ -1,7 +1,7 @@ import sys from .version import version -DEFAULT_DOCKER_API_VERSION = '1.35' +DEFAULT_DOCKER_API_VERSION = '1.39' MINIMUM_DOCKER_API_VERSION = '1.21' DEFAULT_TIMEOUT_SECONDS = 60 STREAM_HEADER_SIZE_BYTES = 8 From 0be75d54cada06d1c9bd0ac66d118e9007defe09 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Fri, 12 Jul 2019 01:28:41 +0200 Subject: [PATCH 1040/1301] Update credentials-helpers to v0.6.2 Signed-off-by: Sebastiaan van Stijn --- tests/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Dockerfile b/tests/Dockerfile index df8468abab..27a1267310 100644 --- a/tests/Dockerfile +++ b/tests/Dockerfile @@ -4,7 +4,7 @@ FROM python:${PYTHON_VERSION} ARG APT_MIRROR RUN sed -ri "s/(httpredir|deb).debian.org/${APT_MIRROR:-deb.debian.org}/g" /etc/apt/sources.list \ - && sed -ri "s/(security).debian.org/${APT_MIRROR:-security.debian.org}/g" /etc/apt/sources.list + && sed -ri "s/(security).debian.org/${APT_MIRROR:-security.debian.org}/g" /etc/apt/sources.list RUN apt-get update && apt-get -y install \ gnupg2 \ From 70cdb08f9ad2e462423b0c4b42af3e6307cd392a Mon Sep 17 00:00:00 2001 From: Till Riedel Date: Sun, 14 Apr 2019 10:38:07 +0200 Subject: [PATCH 1041/1301] set logging level of paramiko to warn Signed-off-by: Till Riedel --- docker/transport/sshconn.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker/transport/sshconn.py b/docker/transport/sshconn.py index 7de0e59087..57b55c9eb5 100644 --- a/docker/transport/sshconn.py +++ b/docker/transport/sshconn.py @@ -2,7 +2,10 @@ import requests.adapters import six import logging +<<<<<<< HEAD import os +======= +>>>>>>> 2dc569a... set logging level of paramiko to warn from docker.transport.basehttpadapter import BaseHTTPAdapter from .. import constants From fcd0093050714de06bcaf05781a995f044f20ce2 Mon Sep 17 00:00:00 2001 From: Till Riedel Date: Sun, 14 Apr 2019 13:52:12 +0200 Subject: [PATCH 1042/1301] obey Hostname Username Port and ProxyCommand settings from .ssh/config Signed-off-by: Till Riedel --- docker/transport/sshconn.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/docker/transport/sshconn.py b/docker/transport/sshconn.py index 57b55c9eb5..7de0e59087 100644 --- a/docker/transport/sshconn.py +++ b/docker/transport/sshconn.py @@ -2,10 +2,7 @@ import requests.adapters import six import logging -<<<<<<< HEAD import os -======= ->>>>>>> 2dc569a... set logging level of paramiko to warn from docker.transport.basehttpadapter import BaseHTTPAdapter from .. import constants From 087b3f0a4956f059ea5998c6dfb34ec581c25c6b Mon Sep 17 00:00:00 2001 From: Anca Iordache Date: Fri, 17 Jan 2020 19:25:55 +0100 Subject: [PATCH 1043/1301] Implement context management, lifecycle and unittests. Signed-off-by: Anca Iordache --- docker/context/api.py | 16 ++++++++++++ docker/context/context.py | 53 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+) diff --git a/docker/context/api.py b/docker/context/api.py index c45115bce5..d903d9c6b9 100644 --- a/docker/context/api.py +++ b/docker/context/api.py @@ -38,7 +38,13 @@ def create_context( >>> print(ctx.Metadata) { "Name": "test", +<<<<<<< HEAD "Metadata": {}, +======= + "Metadata": { + "StackOrchestrator": "swarm" + }, +>>>>>>> 64fdb32... Implement context management, lifecycle and unittests. "Endpoints": { "docker": { "Host": "unix:///var/run/docker.sock", @@ -55,9 +61,13 @@ def create_context( ctx = Context.load_context(name) if ctx: raise errors.ContextAlreadyExists(name) +<<<<<<< HEAD endpoint = "docker" if orchestrator and orchestrator != "swarm": endpoint = orchestrator +======= + endpoint = "docker" if orchestrator == "swarm" else orchestrator +>>>>>>> 64fdb32... Implement context management, lifecycle and unittests. ctx = Context(name, orchestrator) ctx.set_endpoint( endpoint, host, tls_cfg, @@ -79,7 +89,13 @@ def get_context(cls, name=None): >>> print(ctx.Metadata) { "Name": "test", +<<<<<<< HEAD "Metadata": {}, +======= + "Metadata": { + "StackOrchestrator": "swarm" + }, +>>>>>>> 64fdb32... Implement context management, lifecycle and unittests. "Endpoints": { "docker": { "Host": "unix:///var/run/docker.sock", diff --git a/docker/context/context.py b/docker/context/context.py index 2413b2ecbf..3859db2b4c 100644 --- a/docker/context/context.py +++ b/docker/context/context.py @@ -57,7 +57,11 @@ def set_endpoint( self, name="docker", host=None, tls_cfg=None, skip_tls_verify=False, def_namespace=None): self.endpoints[name] = { +<<<<<<< HEAD "Host": get_context_host(host, not skip_tls_verify), +======= + "Host": get_context_host(host), +>>>>>>> 64fdb32... Implement context management, lifecycle and unittests. "SkipTLSVerify": skip_tls_verify } if def_namespace: @@ -71,6 +75,7 @@ def inspect(self): @classmethod def load_context(cls, name): +<<<<<<< HEAD meta = Context._load_meta(name) if meta: instance = cls( @@ -78,6 +83,11 @@ def load_context(cls, name): orchestrator=meta["Metadata"].get("StackOrchestrator", None), endpoints=meta.get("Endpoints", None)) instance.context_type = meta["Metadata"].get("Type", None) +======= + name, orchestrator, endpoints = Context._load_meta(name) + if name: + instance = cls(name, orchestrator, endpoints=endpoints) +>>>>>>> 64fdb32... Implement context management, lifecycle and unittests. instance._load_certs() instance.meta_path = get_meta_dir(name) return instance @@ -85,6 +95,7 @@ def load_context(cls, name): @classmethod def _load_meta(cls, name): +<<<<<<< HEAD meta_file = get_meta_file(name) if not os.path.isfile(meta_file): return None @@ -109,6 +120,27 @@ def _load_meta(cls, name): v.get("SkipTLSVerify", True)) return metadata +======= + metadata = {} + meta_file = get_meta_file(name) + if os.path.isfile(meta_file): + with open(meta_file) as f: + try: + with open(meta_file) as f: + metadata = json.load(f) + for k, v in metadata["Endpoints"].items(): + metadata["Endpoints"][k]["SkipTLSVerify"] = bool( + v["SkipTLSVerify"]) + except (IOError, KeyError, ValueError) as e: + # unknown format + raise Exception("""Detected corrupted meta file for + context {} : {}""".format(name, e)) + + return ( + metadata["Name"], metadata["Metadata"]["StackOrchestrator"], + metadata["Endpoints"]) + return None, None, None +>>>>>>> 64fdb32... Implement context management, lifecycle and unittests. def _load_certs(self): certs = {} @@ -177,15 +209,19 @@ def __call__(self): result.update(self.Storage) return result +<<<<<<< HEAD def is_docker_host(self): return self.context_type is None +======= +>>>>>>> 64fdb32... Implement context management, lifecycle and unittests. @property def Name(self): return self.name @property def Host(self): +<<<<<<< HEAD if not self.orchestrator or self.orchestrator == "swarm": endpoint = self.endpoints.get("docker", None) if endpoint: @@ -193,6 +229,11 @@ def Host(self): return None return self.endpoints[self.orchestrator].get("Host", None) +======= + if self.orchestrator == "swarm": + return self.endpoints["docker"]["Host"] + return self.endpoints[self.orchestrator]["Host"] +>>>>>>> 64fdb32... Implement context management, lifecycle and unittests. @property def Orchestrator(self): @@ -200,19 +241,31 @@ def Orchestrator(self): @property def Metadata(self): +<<<<<<< HEAD meta = {} if self.orchestrator: meta = {"StackOrchestrator": self.orchestrator} return { "Name": self.name, "Metadata": meta, +======= + return { + "Name": self.name, + "Metadata": { + "StackOrchestrator": self.orchestrator + }, +>>>>>>> 64fdb32... Implement context management, lifecycle and unittests. "Endpoints": self.endpoints } @property def TLSConfig(self): key = self.orchestrator +<<<<<<< HEAD if not key or key == "swarm": +======= + if key == "swarm": +>>>>>>> 64fdb32... Implement context management, lifecycle and unittests. key = "docker" if key in self.tls_cfg.keys(): return self.tls_cfg[key] From 67b77f2fa30ad97e8699c744ac43fafdbad6127c Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Fri, 7 Feb 2020 01:00:18 +0100 Subject: [PATCH 1044/1301] Post release 4.2.0 update: - Changelog - Next Version Signed-off-by: Ulysses Souza --- docs/change-log.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/change-log.md b/docs/change-log.md index f0be8ac1c3..829333b7e2 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -36,6 +36,7 @@ Change log - Add support for docker contexts through `docker.ContextAPI` + 4.1.0 ----- From 2e274d00b3009e800e4bfa135b0875eb8a7127a3 Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Thu, 28 May 2020 20:53:45 +0200 Subject: [PATCH 1045/1301] Specify when to use `tls` on Context constructor Signed-off-by: Ulysses Souza --- docker/context/context.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docker/context/context.py b/docker/context/context.py index 3859db2b4c..026a694144 100644 --- a/docker/context/context.py +++ b/docker/context/context.py @@ -57,11 +57,15 @@ def set_endpoint( self, name="docker", host=None, tls_cfg=None, skip_tls_verify=False, def_namespace=None): self.endpoints[name] = { +<<<<<<< HEAD <<<<<<< HEAD "Host": get_context_host(host, not skip_tls_verify), ======= "Host": get_context_host(host), >>>>>>> 64fdb32... Implement context management, lifecycle and unittests. +======= + "Host": get_context_host(host, not skip_tls_verify), +>>>>>>> 3ce2d89... Specify when to use `tls` on Context constructor "SkipTLSVerify": skip_tls_verify } if def_namespace: From 3999707fb3fb5f7cea7d3d92eeda5cf160abb496 Mon Sep 17 00:00:00 2001 From: aiordache Date: Sat, 30 May 2020 11:01:22 +0200 Subject: [PATCH 1046/1301] Make orchestrator field optional Signed-off-by: aiordache --- docker/context/api.py | 16 ----------- docker/context/context.py | 56 ++++++++++++--------------------------- 2 files changed, 17 insertions(+), 55 deletions(-) diff --git a/docker/context/api.py b/docker/context/api.py index d903d9c6b9..c45115bce5 100644 --- a/docker/context/api.py +++ b/docker/context/api.py @@ -38,13 +38,7 @@ def create_context( >>> print(ctx.Metadata) { "Name": "test", -<<<<<<< HEAD "Metadata": {}, -======= - "Metadata": { - "StackOrchestrator": "swarm" - }, ->>>>>>> 64fdb32... Implement context management, lifecycle and unittests. "Endpoints": { "docker": { "Host": "unix:///var/run/docker.sock", @@ -61,13 +55,9 @@ def create_context( ctx = Context.load_context(name) if ctx: raise errors.ContextAlreadyExists(name) -<<<<<<< HEAD endpoint = "docker" if orchestrator and orchestrator != "swarm": endpoint = orchestrator -======= - endpoint = "docker" if orchestrator == "swarm" else orchestrator ->>>>>>> 64fdb32... Implement context management, lifecycle and unittests. ctx = Context(name, orchestrator) ctx.set_endpoint( endpoint, host, tls_cfg, @@ -89,13 +79,7 @@ def get_context(cls, name=None): >>> print(ctx.Metadata) { "Name": "test", -<<<<<<< HEAD "Metadata": {}, -======= - "Metadata": { - "StackOrchestrator": "swarm" - }, ->>>>>>> 64fdb32... Implement context management, lifecycle and unittests. "Endpoints": { "docker": { "Host": "unix:///var/run/docker.sock", diff --git a/docker/context/context.py b/docker/context/context.py index 026a694144..8158803da4 100644 --- a/docker/context/context.py +++ b/docker/context/context.py @@ -57,15 +57,7 @@ def set_endpoint( self, name="docker", host=None, tls_cfg=None, skip_tls_verify=False, def_namespace=None): self.endpoints[name] = { -<<<<<<< HEAD -<<<<<<< HEAD - "Host": get_context_host(host, not skip_tls_verify), -======= - "Host": get_context_host(host), ->>>>>>> 64fdb32... Implement context management, lifecycle and unittests. -======= "Host": get_context_host(host, not skip_tls_verify), ->>>>>>> 3ce2d89... Specify when to use `tls` on Context constructor "SkipTLSVerify": skip_tls_verify } if def_namespace: @@ -79,7 +71,6 @@ def inspect(self): @classmethod def load_context(cls, name): -<<<<<<< HEAD meta = Context._load_meta(name) if meta: instance = cls( @@ -87,11 +78,6 @@ def load_context(cls, name): orchestrator=meta["Metadata"].get("StackOrchestrator", None), endpoints=meta.get("Endpoints", None)) instance.context_type = meta["Metadata"].get("Type", None) -======= - name, orchestrator, endpoints = Context._load_meta(name) - if name: - instance = cls(name, orchestrator, endpoints=endpoints) ->>>>>>> 64fdb32... Implement context management, lifecycle and unittests. instance._load_certs() instance.meta_path = get_meta_dir(name) return instance @@ -99,7 +85,6 @@ def load_context(cls, name): @classmethod def _load_meta(cls, name): -<<<<<<< HEAD meta_file = get_meta_file(name) if not os.path.isfile(meta_file): return None @@ -124,27 +109,6 @@ def _load_meta(cls, name): v.get("SkipTLSVerify", True)) return metadata -======= - metadata = {} - meta_file = get_meta_file(name) - if os.path.isfile(meta_file): - with open(meta_file) as f: - try: - with open(meta_file) as f: - metadata = json.load(f) - for k, v in metadata["Endpoints"].items(): - metadata["Endpoints"][k]["SkipTLSVerify"] = bool( - v["SkipTLSVerify"]) - except (IOError, KeyError, ValueError) as e: - # unknown format - raise Exception("""Detected corrupted meta file for - context {} : {}""".format(name, e)) - - return ( - metadata["Name"], metadata["Metadata"]["StackOrchestrator"], - metadata["Endpoints"]) - return None, None, None ->>>>>>> 64fdb32... Implement context management, lifecycle and unittests. def _load_certs(self): certs = {} @@ -213,18 +177,16 @@ def __call__(self): result.update(self.Storage) return result -<<<<<<< HEAD def is_docker_host(self): return self.context_type is None -======= ->>>>>>> 64fdb32... Implement context management, lifecycle and unittests. @property def Name(self): return self.name @property def Host(self): +<<<<<<< HEAD <<<<<<< HEAD if not self.orchestrator or self.orchestrator == "swarm": endpoint = self.endpoints.get("docker", None) @@ -235,6 +197,9 @@ def Host(self): return self.endpoints[self.orchestrator].get("Host", None) ======= if self.orchestrator == "swarm": +======= + if not self.orchestrator or self.orchestrator == "swarm": +>>>>>>> 1e11ece... Make orchestrator field optional return self.endpoints["docker"]["Host"] return self.endpoints[self.orchestrator]["Host"] >>>>>>> 64fdb32... Implement context management, lifecycle and unittests. @@ -245,6 +210,7 @@ def Orchestrator(self): @property def Metadata(self): +<<<<<<< HEAD <<<<<<< HEAD meta = {} if self.orchestrator: @@ -259,17 +225,29 @@ def Metadata(self): "StackOrchestrator": self.orchestrator }, >>>>>>> 64fdb32... Implement context management, lifecycle and unittests. +======= + meta = {} + if self.orchestrator: + meta = {"StackOrchestrator": self.orchestrator} + return { + "Name": self.name, + "Metadata": meta, +>>>>>>> 1e11ece... Make orchestrator field optional "Endpoints": self.endpoints } @property def TLSConfig(self): key = self.orchestrator +<<<<<<< HEAD <<<<<<< HEAD if not key or key == "swarm": ======= if key == "swarm": >>>>>>> 64fdb32... Implement context management, lifecycle and unittests. +======= + if not key or key == "swarm": +>>>>>>> 1e11ece... Make orchestrator field optional key = "docker" if key in self.tls_cfg.keys(): return self.tls_cfg[key] From 746a2509ab7cb0656f62de0440be9ce442775e8c Mon Sep 17 00:00:00 2001 From: aiordache Date: Mon, 10 Aug 2020 15:29:34 +0200 Subject: [PATCH 1047/1301] Prepare release 4.3.0 Signed-off-by: aiordache --- docker/version.py | 2 +- docs/change-log.md | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index a75460921a..29c6b00e25 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "4.3.0-dev" +version = "4.3.0" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) diff --git a/docs/change-log.md b/docs/change-log.md index 829333b7e2..c753ffd3ff 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -1,6 +1,21 @@ Change log ========== +4.3.0 +----- + +[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/66?closed=1) + +### Features +- Add `DeviceRequest` type to expose host resources such as GPUs +- Add support for `DriverOpts` in EndpointConfig +- Disable compression by default when using container.get_archive method + +### Miscellaneous +- Update default API version to v1.39 +- Update test engine version to 19.03.12 + + 4.2.2 ----- From 8080fbb4ed7a5c25fa0bf1deb39bcdf4cdfb7ddd Mon Sep 17 00:00:00 2001 From: aiordache Date: Mon, 10 Aug 2020 18:15:18 +0200 Subject: [PATCH 1048/1301] Fix merge Signed-off-by: aiordache --- docker/context/context.py | 35 ----------------------------------- 1 file changed, 35 deletions(-) diff --git a/docker/context/context.py b/docker/context/context.py index 8158803da4..2413b2ecbf 100644 --- a/docker/context/context.py +++ b/docker/context/context.py @@ -186,8 +186,6 @@ def Name(self): @property def Host(self): -<<<<<<< HEAD -<<<<<<< HEAD if not self.orchestrator or self.orchestrator == "swarm": endpoint = self.endpoints.get("docker", None) if endpoint: @@ -195,14 +193,6 @@ def Host(self): return None return self.endpoints[self.orchestrator].get("Host", None) -======= - if self.orchestrator == "swarm": -======= - if not self.orchestrator or self.orchestrator == "swarm": ->>>>>>> 1e11ece... Make orchestrator field optional - return self.endpoints["docker"]["Host"] - return self.endpoints[self.orchestrator]["Host"] ->>>>>>> 64fdb32... Implement context management, lifecycle and unittests. @property def Orchestrator(self): @@ -210,44 +200,19 @@ def Orchestrator(self): @property def Metadata(self): -<<<<<<< HEAD -<<<<<<< HEAD - meta = {} - if self.orchestrator: - meta = {"StackOrchestrator": self.orchestrator} - return { - "Name": self.name, - "Metadata": meta, -======= - return { - "Name": self.name, - "Metadata": { - "StackOrchestrator": self.orchestrator - }, ->>>>>>> 64fdb32... Implement context management, lifecycle and unittests. -======= meta = {} if self.orchestrator: meta = {"StackOrchestrator": self.orchestrator} return { "Name": self.name, "Metadata": meta, ->>>>>>> 1e11ece... Make orchestrator field optional "Endpoints": self.endpoints } @property def TLSConfig(self): key = self.orchestrator -<<<<<<< HEAD -<<<<<<< HEAD - if not key or key == "swarm": -======= - if key == "swarm": ->>>>>>> 64fdb32... Implement context management, lifecycle and unittests. -======= if not key or key == "swarm": ->>>>>>> 1e11ece... Make orchestrator field optional key = "docker" if key in self.tls_cfg.keys(): return self.tls_cfg[key] From 9579b7ac0e43eadf6e3a62d61ed67aff25839143 Mon Sep 17 00:00:00 2001 From: aiordache Date: Mon, 10 Aug 2020 18:21:57 +0200 Subject: [PATCH 1049/1301] Fix changelog merge Signed-off-by: aiordache --- docs/change-log.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/change-log.md b/docs/change-log.md index c753ffd3ff..cecce9d89b 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -4,7 +4,7 @@ Change log 4.3.0 ----- -[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/66?closed=1) +[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/64?closed=1) ### Features - Add `DeviceRequest` type to expose host resources such as GPUs @@ -51,7 +51,6 @@ Change log - Add support for docker contexts through `docker.ContextAPI` - 4.1.0 ----- From 5cdbbab3eedeea5b73151c604209d576722fb24d Mon Sep 17 00:00:00 2001 From: aiordache Date: Mon, 10 Aug 2020 19:21:44 +0200 Subject: [PATCH 1050/1301] Update version to the next dev version Signed-off-by: aiordache --- docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index 29c6b00e25..f40347aa54 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "4.3.0" +version = "4.4.0-dev" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) From 2c522fb362247a692c0493f0b47a33988eb2f3e3 Mon Sep 17 00:00:00 2001 From: Anca Iordache Date: Mon, 17 Aug 2020 18:32:48 +0200 Subject: [PATCH 1051/1301] Fix memory conversion to bytes (#2645) * Fix memory conversion to bytes Co-authored-by: Ulysses Souza Signed-off-by: aiordache --- docker/api/container.py | 6 +++--- docker/utils/utils.py | 2 +- tests/unit/utils_test.py | 24 +++++++----------------- 3 files changed, 11 insertions(+), 21 deletions(-) diff --git a/docker/api/container.py b/docker/api/container.py index cf5caebbf6..24eb9c1ca5 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -506,7 +506,7 @@ def create_host_config(self, *args, **kwargs): bytes) or a string with a units identification char (``100000b``, ``1000k``, ``128m``, ``1g``). If a string is specified without a units character, bytes are assumed as an - mem_reservation (int or str): Memory soft limit. + mem_reservation (float or str): Memory soft limit. mem_swappiness (int): Tune a container's memory swappiness behavior. Accepts number between 0 and 100. memswap_limit (str or int): Maximum amount of memory + swap a @@ -1219,8 +1219,8 @@ def update_container( cpu_shares (int): CPU shares (relative weight) cpuset_cpus (str): CPUs in which to allow execution cpuset_mems (str): MEMs in which to allow execution - mem_limit (int or str): Memory limit - mem_reservation (int or str): Memory soft limit + mem_limit (float or str): Memory limit + mem_reservation (float or str): Memory soft limit memswap_limit (int or str): Total memory (memory + swap), -1 to disable swap kernel_memory (int or str): Kernel memory limit diff --git a/docker/utils/utils.py b/docker/utils/utils.py index 447760b483..1b195e2787 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -412,7 +412,7 @@ def parse_bytes(s): if suffix in units.keys() or suffix.isdigit(): try: - digits = int(digits_part) + digits = float(digits_part) except ValueError: raise errors.DockerException( 'Failed converting the string value for memory ({0}) to' diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index d9cb002809..07209a1b13 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -5,27 +5,21 @@ import os import os.path import shutil -import sys import tempfile import unittest - +import pytest +import six from docker.api.client import APIClient from docker.constants import IS_WINDOWS_PLATFORM from docker.errors import DockerException -from docker.utils import ( - convert_filters, convert_volume_binds, decode_json_header, kwargs_from_env, - parse_bytes, parse_devices, parse_env_file, parse_host, - parse_repository_tag, split_command, update_headers, -) - +from docker.utils import (convert_filters, convert_volume_binds, + decode_json_header, kwargs_from_env, parse_bytes, + parse_devices, parse_env_file, parse_host, + parse_repository_tag, split_command, update_headers) from docker.utils.ports import build_port_bindings, split_port from docker.utils.utils import format_environment -import pytest - -import six - TEST_CERT_DIR = os.path.join( os.path.dirname(__file__), 'testdata/certs', @@ -447,11 +441,7 @@ def test_parse_bytes_invalid(self): parse_bytes("127.0.0.1K") def test_parse_bytes_float(self): - with pytest.raises(DockerException): - parse_bytes("1.5k") - - def test_parse_bytes_maxint(self): - assert parse_bytes("{0}k".format(sys.maxsize)) == sys.maxsize * 1024 + assert parse_bytes("1.5k") == 1536 class UtilsTest(unittest.TestCase): From 0dfae33ce800d17d23a43ebde064c146d2c99781 Mon Sep 17 00:00:00 2001 From: Yuval Goldberg Date: Sun, 16 Aug 2020 18:51:11 +0300 Subject: [PATCH 1052/1301] Add file environment variable to integration-dind Signed-off-by: Yuval Goldberg --- Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 6765d4d77e..9f30166d77 100644 --- a/Makefile +++ b/Makefile @@ -57,7 +57,7 @@ integration-dind-py2: build setup-network docker run -d --network dpy-tests --name dpy-dind-py2 --privileged\ docker:${TEST_ENGINE_VERSION}-dind dockerd -H tcp://0.0.0.0:2375 --experimental docker run -t --rm --env="DOCKER_HOST=tcp://dpy-dind-py2:2375" --env="DOCKER_TEST_API_VERSION=${TEST_API_VERSION}"\ - --network dpy-tests docker-sdk-python py.test tests/integration + --network dpy-tests docker-sdk-python py.test tests/integration/${file} docker rm -vf dpy-dind-py2 .PHONY: integration-dind-py3 @@ -66,7 +66,7 @@ integration-dind-py3: build-py3 setup-network docker run -d --network dpy-tests --name dpy-dind-py3 --privileged\ docker:${TEST_ENGINE_VERSION}-dind dockerd -H tcp://0.0.0.0:2375 --experimental docker run -t --rm --env="DOCKER_HOST=tcp://dpy-dind-py3:2375" --env="DOCKER_TEST_API_VERSION=${TEST_API_VERSION}"\ - --network dpy-tests docker-sdk-python3 py.test tests/integration + --network dpy-tests docker-sdk-python3 py.test tests/integration/${file} docker rm -vf dpy-dind-py3 .PHONY: integration-dind-ssl @@ -81,10 +81,10 @@ integration-dind-ssl: build-dind-certs build build-py3 --tlskey=/certs/server-key.pem -H tcp://0.0.0.0:2375 --experimental docker run -t --rm --volumes-from dpy-dind-ssl --env="DOCKER_HOST=tcp://docker:2375"\ --env="DOCKER_TLS_VERIFY=1" --env="DOCKER_CERT_PATH=/certs" --env="DOCKER_TEST_API_VERSION=${TEST_API_VERSION}"\ - --network dpy-tests docker-sdk-python py.test tests/integration + --network dpy-tests docker-sdk-python py.test tests/integration/${file} docker run -t --rm --volumes-from dpy-dind-ssl --env="DOCKER_HOST=tcp://docker:2375"\ --env="DOCKER_TLS_VERIFY=1" --env="DOCKER_CERT_PATH=/certs" --env="DOCKER_TEST_API_VERSION=${TEST_API_VERSION}"\ - --network dpy-tests docker-sdk-python3 py.test tests/integration + --network dpy-tests docker-sdk-python3 py.test tests/integration/${file} docker rm -vf dpy-dind-ssl dpy-dind-certs .PHONY: flake8 From 727080b3cca846a28d5436bed861359c9742c7e1 Mon Sep 17 00:00:00 2001 From: aiordache Date: Wed, 19 Aug 2020 14:19:29 +0200 Subject: [PATCH 1053/1301] set version to 'auto' to avoid breaking on old engine versions Signed-off-by: aiordache --- docker/api/client.py | 44 +++++++++++++++++++++----------------------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/docker/api/client.py b/docker/api/client.py index 35dc84e71f..51abaedf2c 100644 --- a/docker/api/client.py +++ b/docker/api/client.py @@ -7,6 +7,19 @@ import six import websocket +from .. import auth +from ..constants import (DEFAULT_DOCKER_API_VERSION, DEFAULT_NUM_POOLS, + DEFAULT_NUM_POOLS_SSH, DEFAULT_TIMEOUT_SECONDS, + DEFAULT_USER_AGENT, IS_WINDOWS_PLATFORM, + MINIMUM_DOCKER_API_VERSION, STREAM_HEADER_SIZE_BYTES) +from ..errors import (DockerException, InvalidVersion, TLSParameterError, + create_api_error_from_http_exception) +from ..tls import TLSConfig +from ..transport import SSLHTTPAdapter, UnixHTTPAdapter +from ..utils import check_resource, config, update_headers, utils +from ..utils.json_stream import json_stream +from ..utils.proxy import ProxyConfig +from ..utils.socket import consume_socket_output, demux_adaptor, frames_iter from .build import BuildApiMixin from .config import ConfigApiMixin from .container import ContainerApiMixin @@ -19,22 +32,7 @@ from .service import ServiceApiMixin from .swarm import SwarmApiMixin from .volume import VolumeApiMixin -from .. import auth -from ..constants import ( - DEFAULT_TIMEOUT_SECONDS, DEFAULT_USER_AGENT, IS_WINDOWS_PLATFORM, - DEFAULT_DOCKER_API_VERSION, MINIMUM_DOCKER_API_VERSION, - STREAM_HEADER_SIZE_BYTES, DEFAULT_NUM_POOLS_SSH, DEFAULT_NUM_POOLS -) -from ..errors import ( - DockerException, InvalidVersion, TLSParameterError, - create_api_error_from_http_exception -) -from ..tls import TLSConfig -from ..transport import SSLHTTPAdapter, UnixHTTPAdapter -from ..utils import utils, check_resource, update_headers, config -from ..utils.socket import frames_iter, consume_socket_output, demux_adaptor -from ..utils.json_stream import json_stream -from ..utils.proxy import ProxyConfig + try: from ..transport import NpipeHTTPAdapter except ImportError: @@ -183,14 +181,14 @@ def __init__(self, base_url=None, version=None, self.base_url = base_url # version detection needs to be after unix adapter mounting - if version is None: - self._version = DEFAULT_DOCKER_API_VERSION - elif isinstance(version, six.string_types): - if version.lower() == 'auto': - self._version = self._retrieve_server_version() - else: - self._version = version + if version is None or ( + isinstance(version, six.string_types) and + version.lower()) == 'auto': + self._version = self._retrieve_server_version() else: + self._version = version + + if not isinstance(self._version, six.string_types): raise DockerException( 'Version parameter must be a string or None. Found {0}'.format( type(version).__name__ From c7c5b551fcbbdcbf33eabf51007d0f9494637edb Mon Sep 17 00:00:00 2001 From: aiordache Date: Thu, 20 Aug 2020 14:59:36 +0200 Subject: [PATCH 1054/1301] set engine version for unit tests to avoid querying the engine Signed-off-by: aiordache --- docker/api/client.py | 16 +++++------ docker/client.py | 2 +- tests/unit/api_test.py | 51 +++++++++++++++++++++++------------ tests/unit/client_test.py | 32 +++++++++++----------- tests/unit/fake_api.py | 3 ++- tests/unit/fake_api_client.py | 8 +++--- tests/unit/utils_test.py | 6 +++-- 7 files changed, 71 insertions(+), 47 deletions(-) diff --git a/docker/api/client.py b/docker/api/client.py index 51abaedf2c..43e309b5f8 100644 --- a/docker/api/client.py +++ b/docker/api/client.py @@ -8,10 +8,10 @@ import websocket from .. import auth -from ..constants import (DEFAULT_DOCKER_API_VERSION, DEFAULT_NUM_POOLS, - DEFAULT_NUM_POOLS_SSH, DEFAULT_TIMEOUT_SECONDS, - DEFAULT_USER_AGENT, IS_WINDOWS_PLATFORM, - MINIMUM_DOCKER_API_VERSION, STREAM_HEADER_SIZE_BYTES) +from ..constants import (DEFAULT_NUM_POOLS, DEFAULT_NUM_POOLS_SSH, + DEFAULT_TIMEOUT_SECONDS, DEFAULT_USER_AGENT, + IS_WINDOWS_PLATFORM, MINIMUM_DOCKER_API_VERSION, + STREAM_HEADER_SIZE_BYTES) from ..errors import (DockerException, InvalidVersion, TLSParameterError, create_api_error_from_http_exception) from ..tls import TLSConfig @@ -181,13 +181,13 @@ def __init__(self, base_url=None, version=None, self.base_url = base_url # version detection needs to be after unix adapter mounting - if version is None or ( - isinstance(version, six.string_types) and - version.lower()) == 'auto': + if version is None or (isinstance( + version, + six.string_types + ) and version.lower() == 'auto'): self._version = self._retrieve_server_version() else: self._version = version - if not isinstance(self._version, six.string_types): raise DockerException( 'Version parameter must be a string or None. Found {0}'.format( diff --git a/docker/client.py b/docker/client.py index 99ae1962c4..6c397da0d0 100644 --- a/docker/client.py +++ b/docker/client.py @@ -62,7 +62,7 @@ def from_env(cls, **kwargs): Args: version (str): The version of the API to use. Set to ``auto`` to - automatically detect the server's version. Default: ``1.35`` + automatically detect the server's version. Default: ``auto`` timeout (int): Default timeout for API calls, in seconds. ssl_version (int): A valid `SSL version`_. assert_hostname (bool): Verify the hostname of the server. diff --git a/tests/unit/api_test.py b/tests/unit/api_test.py index f4d220a2c6..cb14b74e11 100644 --- a/tests/unit/api_test.py +++ b/tests/unit/api_test.py @@ -1,26 +1,26 @@ import datetime -import json import io +import json import os import re import shutil import socket +import struct import tempfile import threading import time import unittest import docker -from docker.api import APIClient +import pytest import requests -from requests.packages import urllib3 import six -import struct +from docker.api import APIClient +from docker.constants import DEFAULT_DOCKER_API_VERSION +from requests.packages import urllib3 from . import fake_api -import pytest - try: from unittest import mock except ImportError: @@ -105,7 +105,7 @@ def setUp(self): _read_from_socket=fake_read_from_socket ) self.patcher.start() - self.client = APIClient() + self.client = APIClient(version=DEFAULT_DOCKER_API_VERSION) def tearDown(self): self.client.close() @@ -282,27 +282,37 @@ def _socket_path_for_client_session(self, client): return socket_adapter.socket_path def test_url_compatibility_unix(self): - c = APIClient(base_url="unix://socket") + c = APIClient( + base_url="unix://socket", + version=DEFAULT_DOCKER_API_VERSION) assert self._socket_path_for_client_session(c) == '/socket' def test_url_compatibility_unix_triple_slash(self): - c = APIClient(base_url="unix:///socket") + c = APIClient( + base_url="unix:///socket", + version=DEFAULT_DOCKER_API_VERSION) assert self._socket_path_for_client_session(c) == '/socket' def test_url_compatibility_http_unix_triple_slash(self): - c = APIClient(base_url="http+unix:///socket") + c = APIClient( + base_url="http+unix:///socket", + version=DEFAULT_DOCKER_API_VERSION) assert self._socket_path_for_client_session(c) == '/socket' def test_url_compatibility_http(self): - c = APIClient(base_url="http://hostname:1234") + c = APIClient( + base_url="http://hostname:1234", + version=DEFAULT_DOCKER_API_VERSION) assert c.base_url == "http://hostname:1234" def test_url_compatibility_tcp(self): - c = APIClient(base_url="tcp://hostname:1234") + c = APIClient( + base_url="tcp://hostname:1234", + version=DEFAULT_DOCKER_API_VERSION) assert c.base_url == "http://hostname:1234" @@ -447,7 +457,9 @@ def test_early_stream_response(self): b'\r\n' ) + b'\r\n'.join(lines) - with APIClient(base_url="http+unix://" + self.socket_file) as client: + with APIClient( + base_url="http+unix://" + self.socket_file, + version=DEFAULT_DOCKER_API_VERSION) as client: for i in range(5): try: stream = client.build( @@ -532,7 +544,10 @@ def frame_header(stream, data): def request(self, stream=None, tty=None, demux=None): assert stream is not None and tty is not None and demux is not None - with APIClient(base_url=self.address) as client: + with APIClient( + base_url=self.address, + version=DEFAULT_DOCKER_API_VERSION + ) as client: if tty: url = client._url('/tty') else: @@ -597,7 +612,7 @@ def tearDown(self): self.patcher.stop() def test_default_user_agent(self): - client = APIClient() + client = APIClient(version=DEFAULT_DOCKER_API_VERSION) client.version() assert self.mock_send.call_count == 1 @@ -606,7 +621,9 @@ def test_default_user_agent(self): assert headers['User-Agent'] == expected def test_custom_user_agent(self): - client = APIClient(user_agent='foo/bar') + client = APIClient( + user_agent='foo/bar', + version=DEFAULT_DOCKER_API_VERSION) client.version() assert self.mock_send.call_count == 1 @@ -626,7 +643,7 @@ def gettimeout(self): return self.timeout def setUp(self): - self.client = APIClient() + self.client = APIClient(version=DEFAULT_DOCKER_API_VERSION) def test_disable_socket_timeout(self): """Test that the timeout is disabled on a generic socket object.""" diff --git a/tests/unit/client_test.py b/tests/unit/client_test.py index cce99c53ad..cc9ff8f24c 100644 --- a/tests/unit/client_test.py +++ b/tests/unit/client_test.py @@ -1,14 +1,14 @@ import datetime -import docker -from docker.utils import kwargs_from_env -from docker.constants import ( - DEFAULT_DOCKER_API_VERSION, DEFAULT_TIMEOUT_SECONDS -) import os import unittest -from . import fake_api +import docker import pytest +from docker.constants import ( + DEFAULT_DOCKER_API_VERSION, DEFAULT_TIMEOUT_SECONDS) +from docker.utils import kwargs_from_env + +from . import fake_api try: from unittest import mock @@ -25,33 +25,33 @@ class ClientTest(unittest.TestCase): def test_events(self, mock_func): since = datetime.datetime(2016, 1, 1, 0, 0) mock_func.return_value = fake_api.get_fake_events()[1] - client = docker.from_env() + client = docker.from_env(version=DEFAULT_DOCKER_API_VERSION) assert client.events(since=since) == mock_func.return_value mock_func.assert_called_with(since=since) @mock.patch('docker.api.APIClient.info') def test_info(self, mock_func): mock_func.return_value = fake_api.get_fake_info()[1] - client = docker.from_env() + client = docker.from_env(version=DEFAULT_DOCKER_API_VERSION) assert client.info() == mock_func.return_value mock_func.assert_called_with() @mock.patch('docker.api.APIClient.ping') def test_ping(self, mock_func): mock_func.return_value = True - client = docker.from_env() + client = docker.from_env(version=DEFAULT_DOCKER_API_VERSION) assert client.ping() is True mock_func.assert_called_with() @mock.patch('docker.api.APIClient.version') def test_version(self, mock_func): mock_func.return_value = fake_api.get_fake_version()[1] - client = docker.from_env() + client = docker.from_env(version=DEFAULT_DOCKER_API_VERSION) assert client.version() == mock_func.return_value mock_func.assert_called_with() def test_call_api_client_method(self): - client = docker.from_env() + client = docker.from_env(version=DEFAULT_DOCKER_API_VERSION) with pytest.raises(AttributeError) as cm: client.create_container() s = cm.exconly() @@ -65,7 +65,9 @@ def test_call_api_client_method(self): assert "this method is now on the object APIClient" not in s def test_call_containers(self): - client = docker.DockerClient(**kwargs_from_env()) + client = docker.DockerClient( + version=DEFAULT_DOCKER_API_VERSION, + **kwargs_from_env()) with pytest.raises(TypeError) as cm: client.containers() @@ -90,7 +92,7 @@ def test_from_env(self): os.environ.update(DOCKER_HOST='tcp://192.168.59.103:2376', DOCKER_CERT_PATH=TEST_CERT_DIR, DOCKER_TLS_VERIFY='1') - client = docker.from_env() + client = docker.from_env(version=DEFAULT_DOCKER_API_VERSION) assert client.api.base_url == "https://192.168.59.103:2376" def test_from_env_with_version(self): @@ -102,11 +104,11 @@ def test_from_env_with_version(self): assert client.api._version == '2.32' def test_from_env_without_version_uses_default(self): - client = docker.from_env() + client = docker.from_env(version=DEFAULT_DOCKER_API_VERSION) assert client.api._version == DEFAULT_DOCKER_API_VERSION def test_from_env_without_timeout_uses_default(self): - client = docker.from_env() + client = docker.from_env(version=DEFAULT_DOCKER_API_VERSION) assert client.api.timeout == DEFAULT_TIMEOUT_SECONDS diff --git a/tests/unit/fake_api.py b/tests/unit/fake_api.py index e609b64edd..27e463d27e 100644 --- a/tests/unit/fake_api.py +++ b/tests/unit/fake_api.py @@ -1,6 +1,7 @@ -from . import fake_stat from docker import constants +from . import fake_stat + CURRENT_VERSION = 'v{0}'.format(constants.DEFAULT_DOCKER_API_VERSION) FAKE_CONTAINER_ID = '3cc2351ab11b' diff --git a/tests/unit/fake_api_client.py b/tests/unit/fake_api_client.py index 2147bfdfa1..e85001dbbb 100644 --- a/tests/unit/fake_api_client.py +++ b/tests/unit/fake_api_client.py @@ -1,6 +1,7 @@ import copy -import docker +import docker +from docker.constants import DEFAULT_DOCKER_API_VERSION from . import fake_api try: @@ -30,7 +31,7 @@ def make_fake_api_client(overrides=None): if overrides is None: overrides = {} - api_client = docker.APIClient() + api_client = docker.APIClient(version=DEFAULT_DOCKER_API_VERSION) mock_attrs = { 'build.return_value': fake_api.FAKE_IMAGE_ID, 'commit.return_value': fake_api.post_fake_commit()[1], @@ -50,6 +51,7 @@ def make_fake_api_client(overrides=None): 'networks.return_value': fake_api.get_fake_network_list()[1], 'start.return_value': None, 'wait.return_value': {'StatusCode': 0}, + 'version.return_value': fake_api.get_fake_version() } mock_attrs.update(overrides) mock_client = CopyReturnMagicMock(**mock_attrs) @@ -62,6 +64,6 @@ def make_fake_client(overrides=None): """ Returns a Client with a fake APIClient. """ - client = docker.DockerClient() + client = docker.DockerClient(version=DEFAULT_DOCKER_API_VERSION) client.api = make_fake_api_client(overrides) return client diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index 07209a1b13..a53151cb3b 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -11,7 +11,7 @@ import pytest import six from docker.api.client import APIClient -from docker.constants import IS_WINDOWS_PLATFORM +from docker.constants import IS_WINDOWS_PLATFORM, DEFAULT_DOCKER_API_VERSION from docker.errors import DockerException from docker.utils import (convert_filters, convert_volume_binds, decode_json_header, kwargs_from_env, parse_bytes, @@ -35,7 +35,7 @@ def test_update_headers(self): def f(self, headers=None): return headers - client = APIClient() + client = APIClient(version=DEFAULT_DOCKER_API_VERSION) client._general_configs = {} g = update_headers(f) @@ -86,6 +86,7 @@ def test_kwargs_from_env_tls(self): assert kwargs['tls'].verify parsed_host = parse_host(kwargs['base_url'], IS_WINDOWS_PLATFORM, True) + kwargs['version'] = DEFAULT_DOCKER_API_VERSION try: client = APIClient(**kwargs) assert parsed_host == client.base_url @@ -106,6 +107,7 @@ def test_kwargs_from_env_tls_verify_false(self): assert kwargs['tls'].assert_hostname is True assert kwargs['tls'].verify is False parsed_host = parse_host(kwargs['base_url'], IS_WINDOWS_PLATFORM, True) + kwargs['version'] = DEFAULT_DOCKER_API_VERSION try: client = APIClient(**kwargs) assert parsed_host == client.base_url From ed46fb0143020621c68bd2e62bf5a0780552c1fb Mon Sep 17 00:00:00 2001 From: aiordache Date: Fri, 21 Aug 2020 10:43:12 +0200 Subject: [PATCH 1055/1301] Add release 4.3.1 information to changelog Signed-off-by: aiordache --- docs/change-log.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/change-log.md b/docs/change-log.md index cecce9d89b..11c055fda3 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -1,6 +1,16 @@ Change log ========== +4.3.1 +----- + +[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/68?closed=1) + +### Miscellaneous +- Set default API version to `auto` +- Fix conversion to bytes for `float` +- Support OpenSSH `identityfile` option + 4.3.0 ----- From 84857a896cf9ed7f67973db260ae5450ff79e394 Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Tue, 15 Sep 2020 15:33:04 +0200 Subject: [PATCH 1056/1301] Add github supported CODEOWNERS file Signed-off-by: Ulysses Souza --- .github/CODEOWNERS | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .github/CODEOWNERS diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000000..5df3014937 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,6 @@ +# GitHub code owners +# See https://help.github.com/articles/about-codeowners/ +# +# KEEP THIS FILE SORTED. Order is important. Last match takes precedence. + +* @aiordache @ulyssessouza From cec152db5f679bc61c2093959bd9109cb9abb169 Mon Sep 17 00:00:00 2001 From: aiordache Date: Tue, 15 Sep 2020 18:42:19 +0200 Subject: [PATCH 1057/1301] Set image default tag on pull Signed-off-by: aiordache --- docker/api/image.py | 11 ++++++++--- docker/models/images.py | 21 +++++++++++---------- tests/integration/api_image_test.py | 4 ++-- tests/integration/models_images_test.py | 2 +- tests/unit/api_image_test.py | 4 ++-- tests/unit/models_containers_test.py | 2 +- tests/unit/models_images_test.py | 24 ++++++++++++++++++++---- 7 files changed, 45 insertions(+), 23 deletions(-) diff --git a/docker/api/image.py b/docker/api/image.py index 11c8cf7547..dcce0acb41 100644 --- a/docker/api/image.py +++ b/docker/api/image.py @@ -343,7 +343,7 @@ def prune_images(self, filters=None): return self._result(self._post(url, params=params), True) def pull(self, repository, tag=None, stream=False, auth_config=None, - decode=False, platform=None): + decode=False, platform=None, all_tags=False): """ Pulls an image. Similar to the ``docker pull`` command. @@ -358,6 +358,7 @@ def pull(self, repository, tag=None, stream=False, auth_config=None, decode (bool): Decode the JSON data from the server into dicts. Only applies with ``stream=True`` platform (str): Platform in the format ``os[/arch[/variant]]`` + all_tags (bool): Pull all image tags. Returns: (generator or str): The output @@ -382,8 +383,12 @@ def pull(self, repository, tag=None, stream=False, auth_config=None, } """ - if not tag: - repository, tag = utils.parse_repository_tag(repository) + repository, image_tag = utils.parse_repository_tag(repository) + tag = tag or image_tag or 'latest' + + if all_tags: + tag = None + registry, repo_name = auth.resolve_repository_name(repository) params = { diff --git a/docker/models/images.py b/docker/models/images.py index 757a5a4750..d2c5835afc 100644 --- a/docker/models/images.py +++ b/docker/models/images.py @@ -395,12 +395,12 @@ def load(self, data): return [self.get(i) for i in images] - def pull(self, repository, tag=None, **kwargs): + def pull(self, repository, tag=None, all_tags=False, **kwargs): """ Pull an image of the given name and return it. Similar to the ``docker pull`` command. - If no tag is specified, all tags from that repository will be - pulled. + If ``all_tags`` is set, the ``tag`` parameter is ignored and all image + tags will be pulled. If you want to get the raw pull output, use the :py:meth:`~docker.api.image.ImageApiMixin.pull` method in the @@ -413,10 +413,11 @@ def pull(self, repository, tag=None, **kwargs): config for this request. ``auth_config`` should contain the ``username`` and ``password`` keys to be valid. platform (str): Platform in the format ``os[/arch[/variant]]`` + all_tags (bool): Pull all image tags Returns: (:py:class:`Image` or list): The image that has been pulled. - If no ``tag`` was specified, the method will return a list + If ``tag`` is None, the method will return a list of :py:class:`Image` objects belonging to this repository. Raises: @@ -426,13 +427,13 @@ def pull(self, repository, tag=None, **kwargs): Example: >>> # Pull the image tagged `latest` in the busybox repo - >>> image = client.images.pull('busybox:latest') + >>> image = client.images.pull('busybox') >>> # Pull all tags in the busybox repo - >>> images = client.images.pull('busybox') + >>> images = client.images.pull('busybox', all_tags=True) """ - if not tag: - repository, tag = parse_repository_tag(repository) + repository, image_tag = parse_repository_tag(repository) + tag = tag or image_tag or 'latest' if 'stream' in kwargs: warnings.warn( @@ -442,14 +443,14 @@ def pull(self, repository, tag=None, **kwargs): del kwargs['stream'] pull_log = self.client.api.pull( - repository, tag=tag, stream=True, **kwargs + repository, tag=tag, stream=True, all_tags=all_tags, **kwargs ) for _ in pull_log: # We don't do anything with the logs, but we need # to keep the connection alive and wait for the image # to be pulled. pass - if tag: + if not all_tags: return self.get('{0}{2}{1}'.format( repository, tag, '@' if tag.startswith('sha256:') else ':' )) diff --git a/tests/integration/api_image_test.py b/tests/integration/api_image_test.py index 2bc96abf46..37e26a3fd2 100644 --- a/tests/integration/api_image_test.py +++ b/tests/integration/api_image_test.py @@ -42,7 +42,7 @@ def test_pull(self): self.client.remove_image('hello-world') except docker.errors.APIError: pass - res = self.client.pull('hello-world', tag='latest') + res = self.client.pull('hello-world') self.tmp_imgs.append('hello-world') assert type(res) == six.text_type assert len(self.client.images('hello-world')) >= 1 @@ -55,7 +55,7 @@ def test_pull_streaming(self): except docker.errors.APIError: pass stream = self.client.pull( - 'hello-world', tag='latest', stream=True, decode=True) + 'hello-world', stream=True, decode=True) self.tmp_imgs.append('hello-world') for chunk in stream: assert isinstance(chunk, dict) diff --git a/tests/integration/models_images_test.py b/tests/integration/models_images_test.py index 223d102f2b..0d60f37b08 100644 --- a/tests/integration/models_images_test.py +++ b/tests/integration/models_images_test.py @@ -86,7 +86,7 @@ def test_pull_with_sha(self): def test_pull_multiple(self): client = docker.from_env(version=TEST_API_VERSION) - images = client.images.pull('hello-world') + images = client.images.pull('hello-world', all_tags=True) assert len(images) >= 1 assert any([ 'hello-world:latest' in img.attrs['RepoTags'] for img in images diff --git a/tests/unit/api_image_test.py b/tests/unit/api_image_test.py index 1e2315dbb1..4b4fb97765 100644 --- a/tests/unit/api_image_test.py +++ b/tests/unit/api_image_test.py @@ -67,7 +67,7 @@ def test_pull(self): args = fake_request.call_args assert args[0][1] == url_prefix + 'images/create' assert args[1]['params'] == { - 'tag': None, 'fromImage': 'joffrey/test001' + 'tag': 'latest', 'fromImage': 'joffrey/test001' } assert not args[1]['stream'] @@ -77,7 +77,7 @@ def test_pull_stream(self): args = fake_request.call_args assert args[0][1] == url_prefix + 'images/create' assert args[1]['params'] == { - 'tag': None, 'fromImage': 'joffrey/test001' + 'tag': 'latest', 'fromImage': 'joffrey/test001' } assert args[1]['stream'] diff --git a/tests/unit/models_containers_test.py b/tests/unit/models_containers_test.py index c9f73f3737..c7aa46b2a0 100644 --- a/tests/unit/models_containers_test.py +++ b/tests/unit/models_containers_test.py @@ -233,7 +233,7 @@ def test_run_pull(self): assert container.id == FAKE_CONTAINER_ID client.api.pull.assert_called_with( - 'alpine', platform=None, tag=None, stream=True + 'alpine', platform=None, tag='latest', all_tags=False, stream=True ) def test_run_with_error(self): diff --git a/tests/unit/models_images_test.py b/tests/unit/models_images_test.py index fd894ab71d..e3d070c048 100644 --- a/tests/unit/models_images_test.py +++ b/tests/unit/models_images_test.py @@ -44,9 +44,25 @@ def test_load(self): def test_pull(self): client = make_fake_client() - image = client.images.pull('test_image:latest') + image = client.images.pull('test_image:test') client.api.pull.assert_called_with( - 'test_image', tag='latest', stream=True + 'test_image', tag='test', all_tags=False, stream=True + ) + client.api.inspect_image.assert_called_with('test_image:test') + assert isinstance(image, Image) + assert image.id == FAKE_IMAGE_ID + + def test_pull_tag_precedence(self): + client = make_fake_client() + image = client.images.pull('test_image:latest', tag='test') + client.api.pull.assert_called_with( + 'test_image', tag='test', all_tags=False, stream=True + ) + client.api.inspect_image.assert_called_with('test_image:test') + + image = client.images.pull('test_image') + client.api.pull.assert_called_with( + 'test_image', tag='latest', all_tags=False, stream=True ) client.api.inspect_image.assert_called_with('test_image:latest') assert isinstance(image, Image) @@ -54,9 +70,9 @@ def test_pull(self): def test_pull_multiple(self): client = make_fake_client() - images = client.images.pull('test_image') + images = client.images.pull('test_image', all_tags=True) client.api.pull.assert_called_with( - 'test_image', tag=None, stream=True + 'test_image', tag='latest', all_tags=True, stream=True ) client.api.images.assert_called_with( all=False, name='test_image', filters=None From ea093a75dd5c8330d2267a7be9d513dad19f66b7 Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Thu, 17 Sep 2020 16:54:53 +0200 Subject: [PATCH 1058/1301] Fix url of examples in ulimits Signed-off-by: Ulysses Souza --- docker/types/containers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/types/containers.py b/docker/types/containers.py index 149b85dfc9..44bfcfd805 100644 --- a/docker/types/containers.py +++ b/docker/types/containers.py @@ -97,8 +97,8 @@ class Ulimit(DictType): Args: - name (str): Which ulimit will this apply to. A list of valid names can - be found `here `_. + name (str): Which ulimit will this apply to. The valid names can be + found in '/etc/security/limits.conf' on a gnu/linux system. soft (int): The soft limit for this ulimit. Optional. hard (int): The hard limit for this ulimit. Optional. From aed570098518cbc73dc1450b001520a0c5bf5046 Mon Sep 17 00:00:00 2001 From: aiordache Date: Mon, 21 Sep 2020 10:23:29 +0200 Subject: [PATCH 1059/1301] update `pull` method docs Signed-off-by: aiordache --- docker/api/image.py | 6 ++++-- docker/models/images.py | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/docker/api/image.py b/docker/api/image.py index dcce0acb41..4082bfb3d7 100644 --- a/docker/api/image.py +++ b/docker/api/image.py @@ -349,7 +349,8 @@ def pull(self, repository, tag=None, stream=False, auth_config=None, Args: repository (str): The repository to pull - tag (str): The tag to pull + tag (str): The tag to pull. If ``tag`` is ``None`` or empty, it + is set to ``latest``. stream (bool): Stream the output as a generator. Make sure to consume the generator, otherwise pull might get cancelled. auth_config (dict): Override the credentials that are found in the @@ -358,7 +359,8 @@ def pull(self, repository, tag=None, stream=False, auth_config=None, decode (bool): Decode the JSON data from the server into dicts. Only applies with ``stream=True`` platform (str): Platform in the format ``os[/arch[/variant]]`` - all_tags (bool): Pull all image tags. + all_tags (bool): Pull all image tags, the ``tag`` parameter is + ignored. Returns: (generator or str): The output diff --git a/docker/models/images.py b/docker/models/images.py index d2c5835afc..e63558859e 100644 --- a/docker/models/images.py +++ b/docker/models/images.py @@ -399,6 +399,7 @@ def pull(self, repository, tag=None, all_tags=False, **kwargs): """ Pull an image of the given name and return it. Similar to the ``docker pull`` command. + If ``tag`` is ``None`` or empty, it is set to ``latest``. If ``all_tags`` is set, the ``tag`` parameter is ignored and all image tags will be pulled. @@ -417,7 +418,7 @@ def pull(self, repository, tag=None, all_tags=False, **kwargs): Returns: (:py:class:`Image` or list): The image that has been pulled. - If ``tag`` is None, the method will return a list + If ``all_tags`` is True, the method will return a list of :py:class:`Image` objects belonging to this repository. Raises: From 910cc124238aa4c5b56bfafd7c9b72492b2df7d4 Mon Sep 17 00:00:00 2001 From: Ian Fijolek Date: Fri, 9 Oct 2020 18:12:00 -0700 Subject: [PATCH 1060/1301] Fix plugin model upgrade When upgrading a plugin via the model interface, it would yield the following error: AttributeError: 'Plugin' object has no attribute '_reload' It appears that the proper method is `self.reload()`. This is what is used by all other methods in the class and base. I'm not finding any references to `_reload` apart from this instance in the project either. I've verified that this patch fixes the issue on my machine and all tests pass. Signed-off-by: Ian Fijolek --- docker/models/plugins.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/models/plugins.py b/docker/models/plugins.py index 06880181f5..ae5851c919 100644 --- a/docker/models/plugins.py +++ b/docker/models/plugins.py @@ -119,7 +119,7 @@ def upgrade(self, remote=None): privileges = self.client.api.plugin_privileges(remote) for d in self.client.api.upgrade_plugin(self.name, remote, privileges): yield d - self._reload() + self.reload() class PluginCollection(Collection): From 180414dcbbde807f85695a03a5c12d5ffc3aa6f3 Mon Sep 17 00:00:00 2001 From: aiordache Date: Tue, 22 Sep 2020 10:20:18 +0200 Subject: [PATCH 1061/1301] Shell out to SSH client for an ssh connection Signed-off-by: aiordache --- Dockerfile | 4 + Jenkinsfile | 32 +- Makefile | 33 +- docker/api/client.py | 8 +- docker/client.py | 12 +- docker/transport/sshconn.py | 177 +++++++-- tests/Dockerfile | 9 +- tests/Dockerfile-ssh-dind | 23 ++ tests/integration/api_build_test.py | 1 - tests/ssh-keys/authorized_keys | 1 + tests/ssh-keys/config | 3 + tests/ssh-keys/id_rsa | 38 ++ tests/ssh-keys/id_rsa.pub | 1 + tests/ssh/__init__.py | 0 tests/ssh/api_build_test.py | 595 ++++++++++++++++++++++++++++ tests/ssh/base.py | 130 ++++++ 16 files changed, 1006 insertions(+), 61 deletions(-) create mode 100644 tests/Dockerfile-ssh-dind create mode 100755 tests/ssh-keys/authorized_keys create mode 100644 tests/ssh-keys/config create mode 100644 tests/ssh-keys/id_rsa create mode 100644 tests/ssh-keys/id_rsa.pub create mode 100644 tests/ssh/__init__.py create mode 100644 tests/ssh/api_build_test.py create mode 100644 tests/ssh/base.py diff --git a/Dockerfile b/Dockerfile index 124f68cdd0..7309a83e3f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,6 +2,10 @@ ARG PYTHON_VERSION=2.7 FROM python:${PYTHON_VERSION} +# Add SSH keys and set permissions +COPY tests/ssh-keys /root/.ssh +RUN chmod -R 600 /root/.ssh + RUN mkdir /src WORKDIR /src diff --git a/Jenkinsfile b/Jenkinsfile index 88c21592c0..fc716d80eb 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -3,6 +3,7 @@ def imageNameBase = "dockerbuildbot/docker-py" def imageNamePy2 def imageNamePy3 +def imageDindSSH def images = [:] def buildImage = { name, buildargs, pyTag -> @@ -13,7 +14,7 @@ def buildImage = { name, buildargs, pyTag -> img = docker.build(name, buildargs) img.push() } - images[pyTag] = img.id + if (pyTag?.trim()) images[pyTag] = img.id } def buildImages = { -> @@ -23,7 +24,9 @@ def buildImages = { -> imageNamePy2 = "${imageNameBase}:py2-${gitCommit()}" imageNamePy3 = "${imageNameBase}:py3-${gitCommit()}" + imageDindSSH = "${imageNameBase}:sshdind-${gitCommit()}" + buildImage(imageDindSSH, "-f tests/Dockerfile-ssh-dind .", "") buildImage(imageNamePy2, "-f tests/Dockerfile --build-arg PYTHON_VERSION=2.7 .", "py2.7") buildImage(imageNamePy3, "-f tests/Dockerfile --build-arg PYTHON_VERSION=3.7 .", "py3.7") } @@ -81,22 +84,37 @@ def runTests = { Map settings -> def testNetwork = "dpy-testnet-\$BUILD_NUMBER-\$EXECUTOR_NUMBER-${pythonVersion}-${dockerVersion}" try { sh """docker network create ${testNetwork}""" - sh """docker run -d --name ${dindContainerName} -v /tmp --privileged --network ${testNetwork} \\ - docker:${dockerVersion}-dind dockerd -H tcp://0.0.0.0:2375 + sh """docker run --rm -d --name ${dindContainerName} -v /tmp --privileged --network ${testNetwork} \\ + ${imageDindSSH} dockerd -H tcp://0.0.0.0:2375 """ - sh """docker run \\ + sh """docker run --rm \\ --name ${testContainerName} \\ -e "DOCKER_HOST=tcp://${dindContainerName}:2375" \\ -e 'DOCKER_TEST_API_VERSION=${apiVersion}' \\ --network ${testNetwork} \\ --volumes-from ${dindContainerName} \\ ${testImage} \\ - py.test -v -rxs --cov=docker tests/ + py.test -v -rxs --cov=docker --ignore=tests/ssh tests/ + """ + sh """docker stop ${dindContainerName}""" + + // start DIND container with SSH + sh """docker run --rm -d --name ${dindContainerName} -v /tmp --privileged --network ${testNetwork} \\ + ${imageDindSSH} dockerd --experimental""" + sh """docker exec ${dindContainerName} sh -c /usr/sbin/sshd """ + // run SSH tests only + sh """docker run --rm \\ + --name ${testContainerName} \\ + -e "DOCKER_HOST=ssh://${dindContainerName}:22" \\ + -e 'DOCKER_TEST_API_VERSION=${apiVersion}' \\ + --network ${testNetwork} \\ + --volumes-from ${dindContainerName} \\ + ${testImage} \\ + py.test -v -rxs --cov=docker tests/ssh """ } finally { sh """ - docker stop ${dindContainerName} ${testContainerName} - docker rm -vf ${dindContainerName} ${testContainerName} + docker stop ${dindContainerName} docker network rm ${testNetwork} """ } diff --git a/Makefile b/Makefile index 9f30166d77..70d7083e41 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,6 @@ +TEST_API_VERSION ?= 1.39 +TEST_ENGINE_VERSION ?= 19.03.13 + .PHONY: all all: test @@ -10,6 +13,10 @@ clean: build: docker build -t docker-sdk-python -f tests/Dockerfile --build-arg PYTHON_VERSION=2.7 --build-arg APT_MIRROR . +.PHONY: build-dind-ssh +build-dind-ssh: + docker build -t docker-dind-ssh -f tests/Dockerfile-ssh-dind --build-arg ENGINE_VERSION=${TEST_ENGINE_VERSION} --build-arg API_VERSION=${TEST_API_VERSION} --build-arg APT_MIRROR . + .PHONY: build-py3 build-py3: docker build -t docker-sdk-python3 -f tests/Dockerfile --build-arg APT_MIRROR . @@ -41,9 +48,6 @@ integration-test: build integration-test-py3: build-py3 docker run -t --rm -v /var/run/docker.sock:/var/run/docker.sock docker-sdk-python3 py.test -v tests/integration/${file} -TEST_API_VERSION ?= 1.39 -TEST_ENGINE_VERSION ?= 19.03.12 - .PHONY: setup-network setup-network: docker network inspect dpy-tests || docker network create dpy-tests @@ -69,6 +73,29 @@ integration-dind-py3: build-py3 setup-network --network dpy-tests docker-sdk-python3 py.test tests/integration/${file} docker rm -vf dpy-dind-py3 +.PHONY: integration-ssh-py2 +integration-ssh-py2: build-dind-ssh build setup-network + docker rm -vf dpy-dind-py2 || : + docker run -d --network dpy-tests --name dpy-dind-py2 --privileged\ + docker-dind-ssh dockerd --experimental + # start SSH daemon + docker exec dpy-dind-py2 sh -c "/usr/sbin/sshd" + docker run -t --rm --env="DOCKER_HOST=ssh://dpy-dind-py2" --env="DOCKER_TEST_API_VERSION=${TEST_API_VERSION}"\ + --network dpy-tests docker-sdk-python py.test tests/ssh/${file} + docker rm -vf dpy-dind-py2 + +.PHONY: integration-ssh-py3 +integration-ssh-py3: build-dind-ssh build-py3 setup-network + docker rm -vf dpy-dind-py3 || : + docker run -d --network dpy-tests --name dpy-dind-py3 --privileged\ + docker-dind-ssh dockerd --experimental + # start SSH daemon + docker exec dpy-dind-py3 sh -c "/usr/sbin/sshd" + docker run -t --rm --env="DOCKER_HOST=ssh://dpy-dind-py3" --env="DOCKER_TEST_API_VERSION=${TEST_API_VERSION}"\ + --network dpy-tests docker-sdk-python3 py.test tests/ssh/${file} + docker rm -vf dpy-dind-py3 + + .PHONY: integration-dind-ssl integration-dind-ssl: build-dind-certs build build-py3 docker rm -vf dpy-dind-certs dpy-dind-ssl || : diff --git a/docker/api/client.py b/docker/api/client.py index 43e309b5f8..1edd434558 100644 --- a/docker/api/client.py +++ b/docker/api/client.py @@ -89,6 +89,9 @@ class APIClient( user_agent (str): Set a custom user agent for requests to the server. credstore_env (dict): Override environment variables when calling the credential store process. + use_ssh_client (bool): If set to `True`, an ssh connection is made + via shelling out to the ssh client. Ensure the ssh client is + installed and configured on the host. """ __attrs__ = requests.Session.__attrs__ + ['_auth_configs', @@ -100,7 +103,7 @@ class APIClient( def __init__(self, base_url=None, version=None, timeout=DEFAULT_TIMEOUT_SECONDS, tls=False, user_agent=DEFAULT_USER_AGENT, num_pools=None, - credstore_env=None): + credstore_env=None, use_ssh_client=False): super(APIClient, self).__init__() if tls and not base_url: @@ -161,7 +164,8 @@ def __init__(self, base_url=None, version=None, elif base_url.startswith('ssh://'): try: self._custom_adapter = SSHHTTPAdapter( - base_url, timeout, pool_connections=num_pools + base_url, timeout, pool_connections=num_pools, + shell_out=use_ssh_client ) except NameError: raise DockerException( diff --git a/docker/client.py b/docker/client.py index 6c397da0d0..1fea69e660 100644 --- a/docker/client.py +++ b/docker/client.py @@ -35,6 +35,9 @@ class DockerClient(object): user_agent (str): Set a custom user agent for requests to the server. credstore_env (dict): Override environment variables when calling the credential store process. + use_ssh_client (bool): If set to `True`, an ssh connection is made + via shelling out to the ssh client. Ensure the ssh client is + installed and configured on the host. """ def __init__(self, *args, **kwargs): self.api = APIClient(*args, **kwargs) @@ -70,6 +73,9 @@ def from_env(cls, **kwargs): from. Default: the value of ``os.environ`` credstore_env (dict): Override environment variables when calling the credential store process. + use_ssh_client (bool): If set to `True`, an ssh connection is + made via shelling out to the ssh client. Ensure the ssh + client is installed and configured on the host. Example: @@ -81,8 +87,12 @@ def from_env(cls, **kwargs): """ timeout = kwargs.pop('timeout', DEFAULT_TIMEOUT_SECONDS) version = kwargs.pop('version', None) + use_ssh_client = kwargs.pop('use_ssh_client', False) return cls( - timeout=timeout, version=version, **kwargs_from_env(**kwargs) + timeout=timeout, + version=version, + use_ssh_client=use_ssh_client, + **kwargs_from_env(**kwargs) ) # Resources diff --git a/docker/transport/sshconn.py b/docker/transport/sshconn.py index 9cfd99809b..42d1ef9688 100644 --- a/docker/transport/sshconn.py +++ b/docker/transport/sshconn.py @@ -1,8 +1,11 @@ +import io import paramiko import requests.adapters import six import logging import os +import socket +import subprocess from docker.transport.basehttpadapter import BaseHTTPAdapter from .. import constants @@ -20,33 +23,140 @@ RecentlyUsedContainer = urllib3._collections.RecentlyUsedContainer +def create_paramiko_client(base_url): + logging.getLogger("paramiko").setLevel(logging.WARNING) + ssh_client = paramiko.SSHClient() + base_url = six.moves.urllib_parse.urlparse(base_url) + ssh_params = { + "hostname": base_url.hostname, + "port": base_url.port, + "username": base_url.username + } + ssh_config_file = os.path.expanduser("~/.ssh/config") + if os.path.exists(ssh_config_file): + conf = paramiko.SSHConfig() + with open(ssh_config_file) as f: + conf.parse(f) + host_config = conf.lookup(base_url.hostname) + ssh_conf = host_config + if 'proxycommand' in host_config: + ssh_params["sock"] = paramiko.ProxyCommand( + ssh_conf['proxycommand'] + ) + if 'hostname' in host_config: + ssh_params['hostname'] = host_config['hostname'] + if 'identityfile' in host_config: + ssh_params['key_filename'] = host_config['identityfile'] + if base_url.port is None and 'port' in host_config: + ssh_params['port'] = ssh_conf['port'] + if base_url.username is None and 'user' in host_config: + ssh_params['username'] = ssh_conf['user'] + + ssh_client.load_system_host_keys() + ssh_client.set_missing_host_key_policy(paramiko.WarningPolicy()) + return ssh_client, ssh_params + + +class SSHSocket(socket.socket): + def __init__(self, host): + super(SSHSocket, self).__init__( + socket.AF_INET, socket.SOCK_STREAM) + self.host = host + self.port = None + if ':' in host: + self.host, self.port = host.split(':') + self.proc = None + + def connect(self, **kwargs): + port = '' if not self.port else '-p {}'.format(self.port) + args = [ + 'ssh', + '-q', + self.host, + port, + 'docker system dial-stdio' + ] + self.proc = subprocess.Popen( + ' '.join(args), + shell=True, + stdout=subprocess.PIPE, + stdin=subprocess.PIPE) + + def _write(self, data): + if not self.proc or self.proc.stdin.closed: + raise Exception('SSH subprocess not initiated.' + 'connect() must be called first.') + written = self.proc.stdin.write(data) + self.proc.stdin.flush() + return written + + def sendall(self, data): + self._write(data) + + def send(self, data): + return self._write(data) + + def recv(self): + if not self.proc: + raise Exception('SSH subprocess not initiated.' + 'connect() must be called first.') + return self.proc.stdout.read() + + def makefile(self, mode): + if not self.proc or self.proc.stdout.closed: + buf = io.BytesIO() + buf.write(b'\n\n') + return buf + return self.proc.stdout + + def close(self): + if not self.proc or self.proc.stdin.closed: + return + self.proc.stdin.write(b'\n\n') + self.proc.stdin.flush() + self.proc.terminate() + + class SSHConnection(httplib.HTTPConnection, object): - def __init__(self, ssh_transport, timeout=60): + def __init__(self, ssh_transport=None, timeout=60, host=None): super(SSHConnection, self).__init__( 'localhost', timeout=timeout ) self.ssh_transport = ssh_transport self.timeout = timeout + self.host = host def connect(self): - sock = self.ssh_transport.open_session() - sock.settimeout(self.timeout) - sock.exec_command('docker system dial-stdio') + if self.ssh_transport: + sock = self.ssh_transport.open_session() + sock.settimeout(self.timeout) + sock.exec_command('docker system dial-stdio') + else: + sock = SSHSocket(self.host) + sock.settimeout(self.timeout) + sock.connect() + self.sock = sock class SSHConnectionPool(urllib3.connectionpool.HTTPConnectionPool): scheme = 'ssh' - def __init__(self, ssh_client, timeout=60, maxsize=10): + def __init__(self, ssh_client=None, timeout=60, maxsize=10, host=None): super(SSHConnectionPool, self).__init__( 'localhost', timeout=timeout, maxsize=maxsize ) - self.ssh_transport = ssh_client.get_transport() + self.ssh_transport = None + if ssh_client: + self.ssh_transport = ssh_client.get_transport() self.timeout = timeout + self.host = host + self.port = None + if ':' in host: + self.host, self.port = host.split(':') def _new_conn(self): - return SSHConnection(self.ssh_transport, self.timeout) + return SSHConnection(self.ssh_transport, self.timeout, self.host) # When re-using connections, urllib3 calls fileno() on our # SSH channel instance, quickly overloading our fd limit. To avoid this, @@ -78,39 +188,14 @@ class SSHHTTPAdapter(BaseHTTPAdapter): ] def __init__(self, base_url, timeout=60, - pool_connections=constants.DEFAULT_NUM_POOLS): - logging.getLogger("paramiko").setLevel(logging.WARNING) - self.ssh_client = paramiko.SSHClient() - base_url = six.moves.urllib_parse.urlparse(base_url) - self.ssh_params = { - "hostname": base_url.hostname, - "port": base_url.port, - "username": base_url.username - } - ssh_config_file = os.path.expanduser("~/.ssh/config") - if os.path.exists(ssh_config_file): - conf = paramiko.SSHConfig() - with open(ssh_config_file) as f: - conf.parse(f) - host_config = conf.lookup(base_url.hostname) - self.ssh_conf = host_config - if 'proxycommand' in host_config: - self.ssh_params["sock"] = paramiko.ProxyCommand( - self.ssh_conf['proxycommand'] - ) - if 'hostname' in host_config: - self.ssh_params['hostname'] = host_config['hostname'] - if 'identityfile' in host_config: - self.ssh_params['key_filename'] = host_config['identityfile'] - if base_url.port is None and 'port' in host_config: - self.ssh_params['port'] = self.ssh_conf['port'] - if base_url.username is None and 'user' in host_config: - self.ssh_params['username'] = self.ssh_conf['user'] - - self.ssh_client.load_system_host_keys() - self.ssh_client.set_missing_host_key_policy(paramiko.WarningPolicy()) - - self._connect() + pool_connections=constants.DEFAULT_NUM_POOLS, + shell_out=True): + self.ssh_client = None + if not shell_out: + self.ssh_client, self.ssh_params = create_paramiko_client(base_url) + self._connect() + base_url = base_url.lstrip('ssh://') + self.host = base_url self.timeout = timeout self.pools = RecentlyUsedContainer( pool_connections, dispose_func=lambda p: p.close() @@ -118,7 +203,8 @@ def __init__(self, base_url, timeout=60, super(SSHHTTPAdapter, self).__init__() def _connect(self): - self.ssh_client.connect(**self.ssh_params) + if self.ssh_client: + self.ssh_client.connect(**self.ssh_params) def get_connection(self, url, proxies=None): with self.pools.lock: @@ -127,11 +213,13 @@ def get_connection(self, url, proxies=None): return pool # Connection is closed try a reconnect - if not self.ssh_client.get_transport(): + if self.ssh_client and not self.ssh_client.get_transport(): self._connect() pool = SSHConnectionPool( - self.ssh_client, self.timeout + ssh_client=self.ssh_client, + timeout=self.timeout, + host=self.host ) self.pools[url] = pool @@ -139,4 +227,5 @@ def get_connection(self, url, proxies=None): def close(self): super(SSHHTTPAdapter, self).close() - self.ssh_client.close() + if self.ssh_client: + self.ssh_client.close() diff --git a/tests/Dockerfile b/tests/Dockerfile index 27a1267310..3236f3875e 100644 --- a/tests/Dockerfile +++ b/tests/Dockerfile @@ -6,10 +6,13 @@ ARG APT_MIRROR RUN sed -ri "s/(httpredir|deb).debian.org/${APT_MIRROR:-deb.debian.org}/g" /etc/apt/sources.list \ && sed -ri "s/(security).debian.org/${APT_MIRROR:-security.debian.org}/g" /etc/apt/sources.list -RUN apt-get update && apt-get -y install \ +RUN apt-get update && apt-get -y install --no-install-recommends \ gnupg2 \ - pass \ - curl + pass + +# Add SSH keys and set permissions +COPY tests/ssh-keys /root/.ssh +RUN chmod -R 600 /root/.ssh COPY ./tests/gpg-keys /gpg-keys RUN gpg2 --import gpg-keys/secret diff --git a/tests/Dockerfile-ssh-dind b/tests/Dockerfile-ssh-dind new file mode 100644 index 0000000000..9d8f0eab74 --- /dev/null +++ b/tests/Dockerfile-ssh-dind @@ -0,0 +1,23 @@ +ARG API_VERSION=1.39 +ARG ENGINE_VERSION=19.03.12 + +FROM docker:${ENGINE_VERSION}-dind + +RUN apk add --no-cache \ + openssh + +# Add the keys and set permissions +RUN ssh-keygen -A + +# copy the test SSH config +RUN echo "IgnoreUserKnownHosts yes" >> /etc/ssh/sshd_config && \ + echo "PubkeyAuthentication yes" >> /etc/ssh/sshd_config && \ + echo "PermitRootLogin yes" >> /etc/ssh/sshd_config + +# set authorized keys for client paswordless connection +COPY tests/ssh-keys/authorized_keys /root/.ssh/authorized_keys +RUN chmod 600 /root/.ssh/authorized_keys + +RUN echo "root:root" | chpasswd +RUN ln -s /usr/local/bin/docker /usr/bin/docker +EXPOSE 22 diff --git a/tests/integration/api_build_test.py b/tests/integration/api_build_test.py index 57128124ef..b830a106b9 100644 --- a/tests/integration/api_build_test.py +++ b/tests/integration/api_build_test.py @@ -339,7 +339,6 @@ def test_build_with_extra_hosts(self): assert self.client.inspect_image(img_name) ctnr = self.run_container(img_name, 'cat /hosts-file') - self.tmp_containers.append(ctnr) logs = self.client.logs(ctnr) if six.PY3: logs = logs.decode('utf-8') diff --git a/tests/ssh-keys/authorized_keys b/tests/ssh-keys/authorized_keys new file mode 100755 index 0000000000..33252fe503 --- /dev/null +++ b/tests/ssh-keys/authorized_keys @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC/BiXkbL9oEbE3PJv1S2p12XK5BHW3qQT5Rf+CYG0ATYyMPIVM6+IXVyf3QNxpnvPXvbPBQJCs0qHeuPwZy2Gsbt35QnmlgrczFPiXXosCD2N+wrcOQPZGuLjQyUUP2yJRVSTLpp8zk2F8w3laGIB3Jk1hUcMUExemKxQYk/L40b5rXKkarLk5awBuicjRStMrchPRHZ2n715TG+zSvf8tB/UHRXKYPqai/Je5eiH3yGUzCY4zn+uEoqAFb4V8lpIj8Rw3EXmCYVwG0vg+44QIQ2gJnIhTlcmxwkynvZn97nug4NLlGJQ+sDCnIvMapycHfGkNlBz3fFtu/ORsxPpZbTNg/9noa3Zf8OpIwvE/FHNPqDctGltwxEgQxj5fE34x0fYnF08tejAUJJCZE3YsGgNabsS4pD+kRhI83eFZvgj3Q1AeTK0V9bRM7jujcc9Rz+V9Gb5zYEHN/l8PxEVlj0OlURf9ZlknNQK8xRh597jDXTfVQKCMO/nRaWH2bq0= diff --git a/tests/ssh-keys/config b/tests/ssh-keys/config new file mode 100644 index 0000000000..8dd13540ff --- /dev/null +++ b/tests/ssh-keys/config @@ -0,0 +1,3 @@ +Host * + StrictHostKeyChecking no + UserKnownHostsFile=/dev/null diff --git a/tests/ssh-keys/id_rsa b/tests/ssh-keys/id_rsa new file mode 100644 index 0000000000..0ec063e2e4 --- /dev/null +++ b/tests/ssh-keys/id_rsa @@ -0,0 +1,38 @@ +-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAABlwAAAAdzc2gtcn +NhAAAAAwEAAQAAAYEAvwYl5Gy/aBGxNzyb9UtqddlyuQR1t6kE+UX/gmBtAE2MjDyFTOvi +F1cn90DcaZ7z172zwUCQrNKh3rj8GcthrG7d+UJ5pYK3MxT4l16LAg9jfsK3DkD2Rri40M +lFD9siUVUky6afM5NhfMN5WhiAdyZNYVHDFBMXpisUGJPy+NG+a1ypGqy5OWsAbonI0UrT +K3IT0R2dp+9eUxvs0r3/LQf1B0VymD6movyXuXoh98hlMwmOM5/rhKKgBW+FfJaSI/EcNx +F5gmFcBtL4PuOECENoCZyIU5XJscJMp72Z/e57oODS5RiUPrAwpyLzGqcnB3xpDZQc93xb +bvzkbMT6WW0zYP/Z6Gt2X/DqSMLxPxRzT6g3LRpbcMRIEMY+XxN+MdH2JxdPLXowFCSQmR +N2LBoDWm7EuKQ/pEYSPN3hWb4I90NQHkytFfW0TO47o3HPUc/lfRm+c2BBzf5fD8RFZY9D +pVEX/WZZJzUCvMUYefe4w1031UCgjDv50Wlh9m6tAAAFeM2kMyHNpDMhAAAAB3NzaC1yc2 +EAAAGBAL8GJeRsv2gRsTc8m/VLanXZcrkEdbepBPlF/4JgbQBNjIw8hUzr4hdXJ/dA3Gme +89e9s8FAkKzSod64/BnLYaxu3flCeaWCtzMU+JdeiwIPY37Ctw5A9ka4uNDJRQ/bIlFVJM +umnzOTYXzDeVoYgHcmTWFRwxQTF6YrFBiT8vjRvmtcqRqsuTlrAG6JyNFK0ytyE9Ednafv +XlMb7NK9/y0H9QdFcpg+pqL8l7l6IffIZTMJjjOf64SioAVvhXyWkiPxHDcReYJhXAbS+D +7jhAhDaAmciFOVybHCTKe9mf3ue6Dg0uUYlD6wMKci8xqnJwd8aQ2UHPd8W2785GzE+llt +M2D/2ehrdl/w6kjC8T8Uc0+oNy0aW3DESBDGPl8TfjHR9icXTy16MBQkkJkTdiwaA1puxL +ikP6RGEjzd4Vm+CPdDUB5MrRX1tEzuO6Nxz1HP5X0ZvnNgQc3+Xw/ERWWPQ6VRF/1mWSc1 +ArzFGHn3uMNdN9VAoIw7+dFpYfZurQAAAAMBAAEAAAGBAKtnotyiz+Vb6r57vh2OvEpfAd +gOrmpMWVArhSfBykz5SOIU9C+fgVIcPJpaMuz7WiX97Ku9eZP5tJGbP2sN2ejV2ovtICZp +cmV9rcp1ZRpGIKr/oS5DEDlJS1zdHQErSlHcqpWqPzQSTOmcpOk5Dxza25g1u2vp7dCG2x +NqvhySZ+ECViK/Vby1zL9jFzTlhTJ4vFtpzauA2AyPBCPdpHkNqMoLgNYncXLSYHpnos8p +m9T+AAFGwBhVrGz0Mr0mhRDnV/PgbKplKT7l+CGceb8LuWmj/vzuP5Wv6dglw3hJnT2V5p +nTBp3dJ6R006+yvr5T/Xb+ObGqFfgfenjLfHjqbJ/gZdGWt4Le84g8tmSkjJBJ2Yj3kynQ +sdfv9k7JJ4t5euoje0XW0YVN1ih5DdyO4hHDRD1lSTFYT5Gl2sCTt28qsMC12rWzFkezJo +Fhewq2Ddtg4AK6SxqH4rFQCmgOR/ci7jv9TXS9xEQxYliyN5aNymRTyXmwqBIzjNKR6QAA +AMEAxpme2upng9LS6Epa83d1gnWUilYPbpb1C8+1FgpnBv9zkjFE1vY0Vu4i9LcLGlCQ0x +PB1Z16TQlEluqiSuSA0eyaWSQBF9NyGsOCOZ63lpJs/2FRBfcbUvHhv8/g1fv/xvI+FnE+ +DoAhz8V3byU8HUZer7pQY3hSxisdYdsaromxC8DSSPFQoxpxwh7WuP4c3veWkdL13h4fSN +khGr3G1XGfsZOu6V6F1i7yMU6OcwBAxzPsHqZv66sT8lE6n4xjAAAAwQDzAaVaJqZ2ROoF +loltJZUtE7o+zpoDzjOJyGYaCYTU4dHPN1aeYBjw8QfmJhdmZfJp9AeJDB/W0wzoHi2ONI +chnQ1EdbCLk9pvA7rhfVdZaxPeHwniDp2iA/wZKTRG3hav9nEzS72uXuZprCsbBvGXeR0z +iuIx5odVXG8qyuI9lDY6B/IoLg7zd+V6iw9mqWYlLLsgHiAvg32LAT4j0KoTufOqpnxqTQ +P2EguTmxDWkfQmbEHdJvbD2tLQ90zMlwMAAADBAMk88wOA1i/TibH5gm/lAtKPcNKbrHfk +7O9gdSZd2HL0fLjptpOplS89Y7muTElsRDRGiKq+7KV/sxQRNcITkxdTKu8CKnftFWHrLk +9WHWVHXbu9h8ttsKeUr9i27ojxpe5I82of8k7fJTg1LxMnGzuDZfq1BGsQnOWrY7r1Yjcd +8EtSrwOB+J/S4U+rR6kwUEFYeBkhE599P1EtHTCm8kWh368di9Q+Y/VIOa3qRx4hxuiCLI +qj4ZpdVMk2cCNcjwAAAAAB +-----END OPENSSH PRIVATE KEY----- diff --git a/tests/ssh-keys/id_rsa.pub b/tests/ssh-keys/id_rsa.pub new file mode 100644 index 0000000000..33252fe503 --- /dev/null +++ b/tests/ssh-keys/id_rsa.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC/BiXkbL9oEbE3PJv1S2p12XK5BHW3qQT5Rf+CYG0ATYyMPIVM6+IXVyf3QNxpnvPXvbPBQJCs0qHeuPwZy2Gsbt35QnmlgrczFPiXXosCD2N+wrcOQPZGuLjQyUUP2yJRVSTLpp8zk2F8w3laGIB3Jk1hUcMUExemKxQYk/L40b5rXKkarLk5awBuicjRStMrchPRHZ2n715TG+zSvf8tB/UHRXKYPqai/Je5eiH3yGUzCY4zn+uEoqAFb4V8lpIj8Rw3EXmCYVwG0vg+44QIQ2gJnIhTlcmxwkynvZn97nug4NLlGJQ+sDCnIvMapycHfGkNlBz3fFtu/ORsxPpZbTNg/9noa3Zf8OpIwvE/FHNPqDctGltwxEgQxj5fE34x0fYnF08tejAUJJCZE3YsGgNabsS4pD+kRhI83eFZvgj3Q1AeTK0V9bRM7jujcc9Rz+V9Gb5zYEHN/l8PxEVlj0OlURf9ZlknNQK8xRh597jDXTfVQKCMO/nRaWH2bq0= diff --git a/tests/ssh/__init__.py b/tests/ssh/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/ssh/api_build_test.py b/tests/ssh/api_build_test.py new file mode 100644 index 0000000000..b830a106b9 --- /dev/null +++ b/tests/ssh/api_build_test.py @@ -0,0 +1,595 @@ +import io +import os +import shutil +import tempfile + +from docker import errors +from docker.utils.proxy import ProxyConfig + +import pytest +import six + +from .base import BaseAPIIntegrationTest, TEST_IMG +from ..helpers import random_name, requires_api_version, requires_experimental + + +class BuildTest(BaseAPIIntegrationTest): + def test_build_with_proxy(self): + self.client._proxy_configs = ProxyConfig( + ftp='a', http='b', https='c', no_proxy='d' + ) + + script = io.BytesIO('\n'.join([ + 'FROM busybox', + 'RUN env | grep "FTP_PROXY=a"', + 'RUN env | grep "ftp_proxy=a"', + 'RUN env | grep "HTTP_PROXY=b"', + 'RUN env | grep "http_proxy=b"', + 'RUN env | grep "HTTPS_PROXY=c"', + 'RUN env | grep "https_proxy=c"', + 'RUN env | grep "NO_PROXY=d"', + 'RUN env | grep "no_proxy=d"', + ]).encode('ascii')) + + self.client.build(fileobj=script, decode=True) + + def test_build_with_proxy_and_buildargs(self): + self.client._proxy_configs = ProxyConfig( + ftp='a', http='b', https='c', no_proxy='d' + ) + + script = io.BytesIO('\n'.join([ + 'FROM busybox', + 'RUN env | grep "FTP_PROXY=XXX"', + 'RUN env | grep "ftp_proxy=xxx"', + 'RUN env | grep "HTTP_PROXY=b"', + 'RUN env | grep "http_proxy=b"', + 'RUN env | grep "HTTPS_PROXY=c"', + 'RUN env | grep "https_proxy=c"', + 'RUN env | grep "NO_PROXY=d"', + 'RUN env | grep "no_proxy=d"', + ]).encode('ascii')) + + self.client.build( + fileobj=script, + decode=True, + buildargs={'FTP_PROXY': 'XXX', 'ftp_proxy': 'xxx'} + ) + + def test_build_streaming(self): + script = io.BytesIO('\n'.join([ + 'FROM busybox', + 'RUN mkdir -p /tmp/test', + 'EXPOSE 8080', + 'ADD https://dl.dropboxusercontent.com/u/20637798/silence.tar.gz' + ' /tmp/silence.tar.gz' + ]).encode('ascii')) + stream = self.client.build(fileobj=script, decode=True) + logs = [] + for chunk in stream: + logs.append(chunk) + assert len(logs) > 0 + + def test_build_from_stringio(self): + if six.PY3: + return + script = io.StringIO(six.text_type('\n').join([ + 'FROM busybox', + 'RUN mkdir -p /tmp/test', + 'EXPOSE 8080', + 'ADD https://dl.dropboxusercontent.com/u/20637798/silence.tar.gz' + ' /tmp/silence.tar.gz' + ])) + stream = self.client.build(fileobj=script) + logs = '' + for chunk in stream: + if six.PY3: + chunk = chunk.decode('utf-8') + logs += chunk + assert logs != '' + + def test_build_with_dockerignore(self): + base_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, base_dir) + + with open(os.path.join(base_dir, 'Dockerfile'), 'w') as f: + f.write("\n".join([ + 'FROM busybox', + 'ADD . /test', + ])) + + with open(os.path.join(base_dir, '.dockerignore'), 'w') as f: + f.write("\n".join([ + 'ignored', + 'Dockerfile', + '.dockerignore', + '!ignored/subdir/excepted-file', + '', # empty line, + '#*', # comment line + ])) + + with open(os.path.join(base_dir, 'not-ignored'), 'w') as f: + f.write("this file should not be ignored") + + with open(os.path.join(base_dir, '#file.txt'), 'w') as f: + f.write('this file should not be ignored') + + subdir = os.path.join(base_dir, 'ignored', 'subdir') + os.makedirs(subdir) + with open(os.path.join(subdir, 'file'), 'w') as f: + f.write("this file should be ignored") + + with open(os.path.join(subdir, 'excepted-file'), 'w') as f: + f.write("this file should not be ignored") + + tag = 'docker-py-test-build-with-dockerignore' + stream = self.client.build( + path=base_dir, + tag=tag, + ) + for chunk in stream: + pass + + c = self.client.create_container(tag, ['find', '/test', '-type', 'f']) + self.client.start(c) + self.client.wait(c) + logs = self.client.logs(c) + + if six.PY3: + logs = logs.decode('utf-8') + + assert sorted(list(filter(None, logs.split('\n')))) == sorted([ + '/test/#file.txt', + '/test/ignored/subdir/excepted-file', + '/test/not-ignored' + ]) + + def test_build_with_buildargs(self): + script = io.BytesIO('\n'.join([ + 'FROM scratch', + 'ARG test', + 'USER $test' + ]).encode('ascii')) + + stream = self.client.build( + fileobj=script, tag='buildargs', buildargs={'test': 'OK'} + ) + self.tmp_imgs.append('buildargs') + for chunk in stream: + pass + + info = self.client.inspect_image('buildargs') + assert info['Config']['User'] == 'OK' + + @requires_api_version('1.22') + def test_build_shmsize(self): + script = io.BytesIO('\n'.join([ + 'FROM scratch', + 'CMD sh -c "echo \'Hello, World!\'"', + ]).encode('ascii')) + + tag = 'shmsize' + shmsize = 134217728 + + stream = self.client.build( + fileobj=script, tag=tag, shmsize=shmsize + ) + self.tmp_imgs.append(tag) + for chunk in stream: + pass + + # There is currently no way to get the shmsize + # that was used to build the image + + @requires_api_version('1.24') + def test_build_isolation(self): + script = io.BytesIO('\n'.join([ + 'FROM scratch', + 'CMD sh -c "echo \'Deaf To All But The Song\'' + ]).encode('ascii')) + + stream = self.client.build( + fileobj=script, tag='isolation', + isolation='default' + ) + + for chunk in stream: + pass + + @requires_api_version('1.23') + def test_build_labels(self): + script = io.BytesIO('\n'.join([ + 'FROM scratch', + ]).encode('ascii')) + + labels = {'test': 'OK'} + + stream = self.client.build( + fileobj=script, tag='labels', labels=labels + ) + self.tmp_imgs.append('labels') + for chunk in stream: + pass + + info = self.client.inspect_image('labels') + assert info['Config']['Labels'] == labels + + @requires_api_version('1.25') + def test_build_with_cache_from(self): + script = io.BytesIO('\n'.join([ + 'FROM busybox', + 'ENV FOO=bar', + 'RUN touch baz', + 'RUN touch bax', + ]).encode('ascii')) + + stream = self.client.build(fileobj=script, tag='build1') + self.tmp_imgs.append('build1') + for chunk in stream: + pass + + stream = self.client.build( + fileobj=script, tag='build2', cache_from=['build1'], + decode=True + ) + self.tmp_imgs.append('build2') + counter = 0 + for chunk in stream: + if 'Using cache' in chunk.get('stream', ''): + counter += 1 + assert counter == 3 + self.client.remove_image('build2') + + counter = 0 + stream = self.client.build( + fileobj=script, tag='build2', cache_from=['nosuchtag'], + decode=True + ) + for chunk in stream: + if 'Using cache' in chunk.get('stream', ''): + counter += 1 + assert counter == 0 + + @requires_api_version('1.29') + def test_build_container_with_target(self): + script = io.BytesIO('\n'.join([ + 'FROM busybox as first', + 'RUN mkdir -p /tmp/test', + 'RUN touch /tmp/silence.tar.gz', + 'FROM alpine:latest', + 'WORKDIR /root/' + 'COPY --from=first /tmp/silence.tar.gz .', + 'ONBUILD RUN echo "This should not be in the final image"' + ]).encode('ascii')) + + stream = self.client.build( + fileobj=script, target='first', tag='build1' + ) + self.tmp_imgs.append('build1') + for chunk in stream: + pass + + info = self.client.inspect_image('build1') + assert not info['Config']['OnBuild'] + + @requires_api_version('1.25') + def test_build_with_network_mode(self): + # Set up pingable endpoint on custom network + network = self.client.create_network(random_name())['Id'] + self.tmp_networks.append(network) + container = self.client.create_container(TEST_IMG, 'top') + self.tmp_containers.append(container) + self.client.start(container) + self.client.connect_container_to_network( + container, network, aliases=['pingtarget.docker'] + ) + + script = io.BytesIO('\n'.join([ + 'FROM busybox', + 'RUN ping -c1 pingtarget.docker' + ]).encode('ascii')) + + stream = self.client.build( + fileobj=script, network_mode=network, + tag='dockerpytest_customnetbuild' + ) + + self.tmp_imgs.append('dockerpytest_customnetbuild') + for chunk in stream: + pass + + assert self.client.inspect_image('dockerpytest_customnetbuild') + + script.seek(0) + stream = self.client.build( + fileobj=script, network_mode='none', + tag='dockerpytest_nonebuild', nocache=True, decode=True + ) + + self.tmp_imgs.append('dockerpytest_nonebuild') + logs = [chunk for chunk in stream] + assert 'errorDetail' in logs[-1] + assert logs[-1]['errorDetail']['code'] == 1 + + with pytest.raises(errors.NotFound): + self.client.inspect_image('dockerpytest_nonebuild') + + @requires_api_version('1.27') + def test_build_with_extra_hosts(self): + img_name = 'dockerpytest_extrahost_build' + self.tmp_imgs.append(img_name) + + script = io.BytesIO('\n'.join([ + 'FROM busybox', + 'RUN ping -c1 hello.world.test', + 'RUN ping -c1 extrahost.local.test', + 'RUN cp /etc/hosts /hosts-file' + ]).encode('ascii')) + + stream = self.client.build( + fileobj=script, tag=img_name, + extra_hosts={ + 'extrahost.local.test': '127.0.0.1', + 'hello.world.test': '127.0.0.1', + }, decode=True + ) + for chunk in stream: + if 'errorDetail' in chunk: + pytest.fail(chunk) + + assert self.client.inspect_image(img_name) + ctnr = self.run_container(img_name, 'cat /hosts-file') + logs = self.client.logs(ctnr) + if six.PY3: + logs = logs.decode('utf-8') + assert '127.0.0.1\textrahost.local.test' in logs + assert '127.0.0.1\thello.world.test' in logs + + @requires_experimental(until=None) + @requires_api_version('1.25') + def test_build_squash(self): + script = io.BytesIO('\n'.join([ + 'FROM busybox', + 'RUN echo blah > /file_1', + 'RUN echo blahblah > /file_2', + 'RUN echo blahblahblah > /file_3' + ]).encode('ascii')) + + def build_squashed(squash): + tag = 'squash' if squash else 'nosquash' + stream = self.client.build( + fileobj=script, tag=tag, squash=squash + ) + self.tmp_imgs.append(tag) + for chunk in stream: + pass + + return self.client.inspect_image(tag) + + non_squashed = build_squashed(False) + squashed = build_squashed(True) + assert len(non_squashed['RootFS']['Layers']) == 4 + assert len(squashed['RootFS']['Layers']) == 2 + + def test_build_stderr_data(self): + control_chars = ['\x1b[91m', '\x1b[0m'] + snippet = 'Ancient Temple (Mystic Oriental Dream ~ Ancient Temple)' + script = io.BytesIO(b'\n'.join([ + b'FROM busybox', + 'RUN sh -c ">&2 echo \'{0}\'"'.format(snippet).encode('utf-8') + ])) + + stream = self.client.build( + fileobj=script, decode=True, nocache=True + ) + lines = [] + for chunk in stream: + lines.append(chunk.get('stream')) + expected = '{0}{2}\n{1}'.format( + control_chars[0], control_chars[1], snippet + ) + assert any([line == expected for line in lines]) + + def test_build_gzip_encoding(self): + base_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, base_dir) + + with open(os.path.join(base_dir, 'Dockerfile'), 'w') as f: + f.write("\n".join([ + 'FROM busybox', + 'ADD . /test', + ])) + + stream = self.client.build( + path=base_dir, decode=True, nocache=True, + gzip=True + ) + + lines = [] + for chunk in stream: + lines.append(chunk) + + assert 'Successfully built' in lines[-1]['stream'] + + def test_build_with_dockerfile_empty_lines(self): + base_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, base_dir) + with open(os.path.join(base_dir, 'Dockerfile'), 'w') as f: + f.write('FROM busybox\n') + with open(os.path.join(base_dir, '.dockerignore'), 'w') as f: + f.write('\n'.join([ + ' ', + '', + '\t\t', + '\t ', + ])) + + stream = self.client.build( + path=base_dir, decode=True, nocache=True + ) + + lines = [] + for chunk in stream: + lines.append(chunk) + assert 'Successfully built' in lines[-1]['stream'] + + def test_build_gzip_custom_encoding(self): + with pytest.raises(errors.DockerException): + self.client.build(path='.', gzip=True, encoding='text/html') + + @requires_api_version('1.32') + @requires_experimental(until=None) + def test_build_invalid_platform(self): + script = io.BytesIO('FROM busybox\n'.encode('ascii')) + + with pytest.raises(errors.APIError) as excinfo: + stream = self.client.build(fileobj=script, platform='foobar') + for _ in stream: + pass + + # Some API versions incorrectly returns 500 status; assert 4xx or 5xx + assert excinfo.value.is_error() + assert 'unknown operating system' in excinfo.exconly() \ + or 'invalid platform' in excinfo.exconly() + + def test_build_out_of_context_dockerfile(self): + base_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, base_dir) + with open(os.path.join(base_dir, 'file.txt'), 'w') as f: + f.write('hello world') + with open(os.path.join(base_dir, '.dockerignore'), 'w') as f: + f.write('.dockerignore\n') + df_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, df_dir) + df_name = os.path.join(df_dir, 'Dockerfile') + with open(df_name, 'wb') as df: + df.write(('\n'.join([ + 'FROM busybox', + 'COPY . /src', + 'WORKDIR /src', + ])).encode('utf-8')) + df.flush() + img_name = random_name() + self.tmp_imgs.append(img_name) + stream = self.client.build( + path=base_dir, dockerfile=df_name, tag=img_name, + decode=True + ) + lines = [] + for chunk in stream: + lines.append(chunk) + assert 'Successfully tagged' in lines[-1]['stream'] + + ctnr = self.client.create_container(img_name, 'ls -a') + self.tmp_containers.append(ctnr) + self.client.start(ctnr) + lsdata = self.client.logs(ctnr).strip().split(b'\n') + assert len(lsdata) == 3 + assert sorted([b'.', b'..', b'file.txt']) == sorted(lsdata) + + def test_build_in_context_dockerfile(self): + base_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, base_dir) + with open(os.path.join(base_dir, 'file.txt'), 'w') as f: + f.write('hello world') + with open(os.path.join(base_dir, 'custom.dockerfile'), 'w') as df: + df.write('\n'.join([ + 'FROM busybox', + 'COPY . /src', + 'WORKDIR /src', + ])) + img_name = random_name() + self.tmp_imgs.append(img_name) + stream = self.client.build( + path=base_dir, dockerfile='custom.dockerfile', tag=img_name, + decode=True + ) + lines = [] + for chunk in stream: + lines.append(chunk) + assert 'Successfully tagged' in lines[-1]['stream'] + + ctnr = self.client.create_container(img_name, 'ls -a') + self.tmp_containers.append(ctnr) + self.client.start(ctnr) + lsdata = self.client.logs(ctnr).strip().split(b'\n') + assert len(lsdata) == 4 + assert sorted( + [b'.', b'..', b'file.txt', b'custom.dockerfile'] + ) == sorted(lsdata) + + def test_build_in_context_nested_dockerfile(self): + base_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, base_dir) + with open(os.path.join(base_dir, 'file.txt'), 'w') as f: + f.write('hello world') + subdir = os.path.join(base_dir, 'hello', 'world') + os.makedirs(subdir) + with open(os.path.join(subdir, 'custom.dockerfile'), 'w') as df: + df.write('\n'.join([ + 'FROM busybox', + 'COPY . /src', + 'WORKDIR /src', + ])) + img_name = random_name() + self.tmp_imgs.append(img_name) + stream = self.client.build( + path=base_dir, dockerfile='hello/world/custom.dockerfile', + tag=img_name, decode=True + ) + lines = [] + for chunk in stream: + lines.append(chunk) + assert 'Successfully tagged' in lines[-1]['stream'] + + ctnr = self.client.create_container(img_name, 'ls -a') + self.tmp_containers.append(ctnr) + self.client.start(ctnr) + lsdata = self.client.logs(ctnr).strip().split(b'\n') + assert len(lsdata) == 4 + assert sorted( + [b'.', b'..', b'file.txt', b'hello'] + ) == sorted(lsdata) + + def test_build_in_context_abs_dockerfile(self): + base_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, base_dir) + abs_dockerfile_path = os.path.join(base_dir, 'custom.dockerfile') + with open(os.path.join(base_dir, 'file.txt'), 'w') as f: + f.write('hello world') + with open(abs_dockerfile_path, 'w') as df: + df.write('\n'.join([ + 'FROM busybox', + 'COPY . /src', + 'WORKDIR /src', + ])) + img_name = random_name() + self.tmp_imgs.append(img_name) + stream = self.client.build( + path=base_dir, dockerfile=abs_dockerfile_path, tag=img_name, + decode=True + ) + lines = [] + for chunk in stream: + lines.append(chunk) + assert 'Successfully tagged' in lines[-1]['stream'] + + ctnr = self.client.create_container(img_name, 'ls -a') + self.tmp_containers.append(ctnr) + self.client.start(ctnr) + lsdata = self.client.logs(ctnr).strip().split(b'\n') + assert len(lsdata) == 4 + assert sorted( + [b'.', b'..', b'file.txt', b'custom.dockerfile'] + ) == sorted(lsdata) + + @requires_api_version('1.31') + @pytest.mark.xfail( + True, + reason='Currently fails on 18.09: ' + 'https://github.com/moby/moby/issues/37920' + ) + def test_prune_builds(self): + prune_result = self.client.prune_builds() + assert 'SpaceReclaimed' in prune_result + assert isinstance(prune_result['SpaceReclaimed'], int) diff --git a/tests/ssh/base.py b/tests/ssh/base.py new file mode 100644 index 0000000000..c723d823bc --- /dev/null +++ b/tests/ssh/base.py @@ -0,0 +1,130 @@ +import os +import shutil +import unittest + +import docker +from .. import helpers +from docker.utils import kwargs_from_env + +TEST_IMG = 'alpine:3.10' +TEST_API_VERSION = os.environ.get('DOCKER_TEST_API_VERSION') + + +class BaseIntegrationTest(unittest.TestCase): + """ + A base class for integration test cases. It cleans up the Docker server + after itself. + """ + + def setUp(self): + self.tmp_imgs = [] + self.tmp_containers = [] + self.tmp_folders = [] + self.tmp_volumes = [] + self.tmp_networks = [] + self.tmp_plugins = [] + self.tmp_secrets = [] + self.tmp_configs = [] + + def tearDown(self): + client = docker.from_env(version=TEST_API_VERSION, use_ssh_client=True) + try: + for img in self.tmp_imgs: + try: + client.api.remove_image(img) + except docker.errors.APIError: + pass + for container in self.tmp_containers: + try: + client.api.remove_container(container, force=True, v=True) + except docker.errors.APIError: + pass + for network in self.tmp_networks: + try: + client.api.remove_network(network) + except docker.errors.APIError: + pass + for volume in self.tmp_volumes: + try: + client.api.remove_volume(volume) + except docker.errors.APIError: + pass + + for secret in self.tmp_secrets: + try: + client.api.remove_secret(secret) + except docker.errors.APIError: + pass + + for config in self.tmp_configs: + try: + client.api.remove_config(config) + except docker.errors.APIError: + pass + + for folder in self.tmp_folders: + shutil.rmtree(folder) + finally: + client.close() + + +class BaseAPIIntegrationTest(BaseIntegrationTest): + """ + A test case for `APIClient` integration tests. It sets up an `APIClient` + as `self.client`. + """ + @classmethod + def setUpClass(cls): + cls.client = cls.get_client_instance() + cls.client.pull(TEST_IMG) + + def tearDown(self): + super(BaseAPIIntegrationTest, self).tearDown() + self.client.close() + + @staticmethod + def get_client_instance(): + return docker.APIClient( + version=TEST_API_VERSION, + timeout=60, + use_ssh_client=True, + **kwargs_from_env() + ) + + @staticmethod + def _init_swarm(client, **kwargs): + return client.init_swarm( + '127.0.0.1', listen_addr=helpers.swarm_listen_addr(), **kwargs + ) + + def run_container(self, *args, **kwargs): + container = self.client.create_container(*args, **kwargs) + self.tmp_containers.append(container) + self.client.start(container) + exitcode = self.client.wait(container)['StatusCode'] + + if exitcode != 0: + output = self.client.logs(container) + raise Exception( + "Container exited with code {}:\n{}" + .format(exitcode, output)) + + return container + + def create_and_start(self, image=TEST_IMG, command='top', **kwargs): + container = self.client.create_container( + image=image, command=command, **kwargs) + self.tmp_containers.append(container) + self.client.start(container) + return container + + def execute(self, container, cmd, exit_code=0, **kwargs): + exc = self.client.exec_create(container, cmd, **kwargs) + output = self.client.exec_start(exc) + actual_exit_code = self.client.exec_inspect(exc)['ExitCode'] + msg = "Expected `{}` to exit with code {} but returned {}:\n{}".format( + " ".join(cmd), exit_code, actual_exit_code, output) + assert actual_exit_code == exit_code, msg + + def init_swarm(self, **kwargs): + return self._init_swarm(self.client, **kwargs) From 3766f77c20e1e14d5ad49bdcf7314f3f8459927d Mon Sep 17 00:00:00 2001 From: Yuval Goldberg Date: Sun, 16 Aug 2020 18:54:14 +0300 Subject: [PATCH 1062/1301] Add response url to Server Error and Client Error messages Signed-off-by: Yuval Goldberg --- docker/errors.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docker/errors.py b/docker/errors.py index e5d07a5bfe..ab30a2908e 100644 --- a/docker/errors.py +++ b/docker/errors.py @@ -46,12 +46,14 @@ def __str__(self): message = super(APIError, self).__str__() if self.is_client_error(): - message = '{0} Client Error: {1}'.format( - self.response.status_code, self.response.reason) + message = '{0} Client Error for {1}: {2}'.format( + self.response.status_code, self.response.url, + self.response.reason) elif self.is_server_error(): - message = '{0} Server Error: {1}'.format( - self.response.status_code, self.response.reason) + message = '{0} Server Error for {1}: {2}'.format( + self.response.status_code, self.response.url, + self.response.reason) if self.explanation: message = '{0} ("{1}")'.format(message, self.explanation) From 93bcc0497d8302aa2d78bd7ef756fc2ff3fd0912 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 27 Oct 2020 21:18:15 +0000 Subject: [PATCH 1063/1301] Bump cryptography from 2.3 to 3.2 Bumps [cryptography](https://github.com/pyca/cryptography) from 2.3 to 3.2. - [Release notes](https://github.com/pyca/cryptography/releases) - [Changelog](https://github.com/pyca/cryptography/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/2.3...3.2) Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 340e431285..0edd4e1e71 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ appdirs==1.4.3 asn1crypto==0.22.0 backports.ssl-match-hostname==3.5.0.1 cffi==1.10.0 -cryptography==2.3 +cryptography==3.2 enum34==1.1.6 idna==2.5 ipaddress==1.0.18 From 9c53024eade661d98d5a344bc08b2dc4ed386903 Mon Sep 17 00:00:00 2001 From: Daeseok Youn Date: Fri, 7 Feb 2020 18:11:00 +0900 Subject: [PATCH 1064/1301] raise an error for binding specific ports in 'host' mode of network The binding ports are ignored where the network mode is 'host'. It could be a problem in case of using these options together on Mac or Windows OS. Because the limitation that could not use the 'host' in network_mode on Mac and Windows. When 'host' mode is set on network_mode, the specific ports in 'ports' are ignored so the network is not able to be accessed through defined ports by developer. Signed-off-by: Daeseok Youn --- docker/api/container.py | 7 ++++++- docker/models/containers.py | 5 +++++ docker/types/containers.py | 12 ++++++++++++ 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/docker/api/container.py b/docker/api/container.py index 24eb9c1ca5..cfd514708c 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -523,6 +523,10 @@ def create_host_config(self, *args, **kwargs): - ``container:`` Reuse another container's network stack. - ``host`` Use the host network stack. + This mode is incompatible with ``port_bindings``. + If ``host`` is used as network_mode, all of listed up to + ``port_bindings``` are ignored in running container. + oom_kill_disable (bool): Whether to disable OOM killer. oom_score_adj (int): An integer value containing the score given to the container in order to tune OOM killer preferences. @@ -531,7 +535,8 @@ def create_host_config(self, *args, **kwargs): pids_limit (int): Tune a container's pids limit. Set ``-1`` for unlimited. port_bindings (dict): See :py:meth:`create_container` - for more information. + for more information. The binding ports are ignored in + ``host`` as network mode. privileged (bool): Give extended privileges to this container. publish_all_ports (bool): Publish all ports to the host. read_only (bool): Mount the container's root filesystem as read diff --git a/docker/models/containers.py b/docker/models/containers.py index 0c2b855a2b..bcd780172c 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -649,6 +649,9 @@ def run(self, image, command=None, stdout=True, stderr=False, - ``container:`` Reuse another container's network stack. - ``host`` Use the host network stack. + This mode is incompatible with ``ports``. If ``host`` is + used as network_mode, all of listed up to ``ports``` are + ignored in running container. Incompatible with ``network``. oom_kill_disable (bool): Whether to disable OOM killer. @@ -667,6 +670,8 @@ def run(self, image, command=None, stdout=True, stderr=False, ``port/protocol``, where the protocol is either ``tcp``, ``udp``, or ``sctp``. + Ports are ignored to bind with ``host`` as network mode. + The values of the dictionary are the corresponding ports to open on the host, which can be either: diff --git a/docker/types/containers.py b/docker/types/containers.py index 44bfcfd805..1df3fff1c7 100644 --- a/docker/types/containers.py +++ b/docker/types/containers.py @@ -334,6 +334,11 @@ def __init__(self, version, binds=None, port_bindings=None, if dns_search: self['DnsSearch'] = dns_search + if network_mode is 'host' and port_bindings is not None: + raise host_config_incompatible_error( + 'network_mode', 'host', 'port_bindings' + ) + if network_mode: self['NetworkMode'] = network_mode elif network_mode is None: @@ -664,6 +669,13 @@ def host_config_value_error(param, param_value): return ValueError(error_msg.format(param, param_value)) +def host_config_incompatible_error(param, param_value, incompatible_param): + error_msg = 'Incompatible {1} in {0} is not compatible with {2}' + return errors.InvalidArgument( + error_msg.format(param, param_value, incompatible_param) + ) + + class ContainerConfig(dict): def __init__( self, version, image, command, hostname=None, user=None, detach=False, From 433264d04b56ef234998be9ec33745d087d3caba Mon Sep 17 00:00:00 2001 From: Daeseok Youn Date: Tue, 17 Nov 2020 21:25:00 +0900 Subject: [PATCH 1065/1301] Correct comments on ports_binding and host mode as network_mode Signed-off-by: Daeseok Youn --- docker/api/container.py | 6 ++---- docker/models/containers.py | 7 ++----- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/docker/api/container.py b/docker/api/container.py index cfd514708c..754b5dc6a4 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -524,8 +524,6 @@ def create_host_config(self, *args, **kwargs): stack. - ``host`` Use the host network stack. This mode is incompatible with ``port_bindings``. - If ``host`` is used as network_mode, all of listed up to - ``port_bindings``` are ignored in running container. oom_kill_disable (bool): Whether to disable OOM killer. oom_score_adj (int): An integer value containing the score given @@ -535,8 +533,8 @@ def create_host_config(self, *args, **kwargs): pids_limit (int): Tune a container's pids limit. Set ``-1`` for unlimited. port_bindings (dict): See :py:meth:`create_container` - for more information. The binding ports are ignored in - ``host`` as network mode. + for more information. + Imcompatible with ``host`` in ``network_mode``. privileged (bool): Give extended privileges to this container. publish_all_ports (bool): Publish all ports to the host. read_only (bool): Mount the container's root filesystem as read diff --git a/docker/models/containers.py b/docker/models/containers.py index bcd780172c..120386a199 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -649,9 +649,7 @@ def run(self, image, command=None, stdout=True, stderr=False, - ``container:`` Reuse another container's network stack. - ``host`` Use the host network stack. - This mode is incompatible with ``ports``. If ``host`` is - used as network_mode, all of listed up to ``ports``` are - ignored in running container. + This mode is incompatible with ``ports``. Incompatible with ``network``. oom_kill_disable (bool): Whether to disable OOM killer. @@ -670,8 +668,6 @@ def run(self, image, command=None, stdout=True, stderr=False, ``port/protocol``, where the protocol is either ``tcp``, ``udp``, or ``sctp``. - Ports are ignored to bind with ``host`` as network mode. - The values of the dictionary are the corresponding ports to open on the host, which can be either: @@ -687,6 +683,7 @@ def run(self, image, command=None, stdout=True, stderr=False, to a single container port. For example, ``{'1111/tcp': [1234, 4567]}``. + Imcompatible with ``host`` in ``network_mode``. privileged (bool): Give extended privileges to this container. publish_all_ports (bool): Publish all ports to the host. read_only (bool): Mount the container's root filesystem as read From bb1c528ab3c67ac6ceb3f8a65a7cc0f919cf83fe Mon Sep 17 00:00:00 2001 From: Mariano Scazzariello Date: Tue, 17 Nov 2020 15:42:36 +0100 Subject: [PATCH 1066/1301] Add max_pool_size parameter (#2699) * Add max_pool_size parameter Signed-off-by: Mariano Scazzariello * Add client version to tests Signed-off-by: Mariano Scazzariello * Fix parameter position Signed-off-by: Mariano Scazzariello --- docker/api/client.py | 19 ++-- docker/client.py | 8 +- docker/constants.py | 2 + docker/transport/npipeconn.py | 10 ++- docker/transport/sshconn.py | 5 +- docker/transport/unixconn.py | 10 ++- tests/unit/client_test.py | 158 +++++++++++++++++++++++++++++++++- 7 files changed, 195 insertions(+), 17 deletions(-) diff --git a/docker/api/client.py b/docker/api/client.py index 1edd434558..fbf7ad455a 100644 --- a/docker/api/client.py +++ b/docker/api/client.py @@ -9,9 +9,9 @@ from .. import auth from ..constants import (DEFAULT_NUM_POOLS, DEFAULT_NUM_POOLS_SSH, - DEFAULT_TIMEOUT_SECONDS, DEFAULT_USER_AGENT, - IS_WINDOWS_PLATFORM, MINIMUM_DOCKER_API_VERSION, - STREAM_HEADER_SIZE_BYTES) + DEFAULT_MAX_POOL_SIZE, DEFAULT_TIMEOUT_SECONDS, + DEFAULT_USER_AGENT, IS_WINDOWS_PLATFORM, + MINIMUM_DOCKER_API_VERSION, STREAM_HEADER_SIZE_BYTES) from ..errors import (DockerException, InvalidVersion, TLSParameterError, create_api_error_from_http_exception) from ..tls import TLSConfig @@ -92,6 +92,8 @@ class APIClient( use_ssh_client (bool): If set to `True`, an ssh connection is made via shelling out to the ssh client. Ensure the ssh client is installed and configured on the host. + max_pool_size (int): The maximum number of connections + to save in the pool. """ __attrs__ = requests.Session.__attrs__ + ['_auth_configs', @@ -103,7 +105,8 @@ class APIClient( def __init__(self, base_url=None, version=None, timeout=DEFAULT_TIMEOUT_SECONDS, tls=False, user_agent=DEFAULT_USER_AGENT, num_pools=None, - credstore_env=None, use_ssh_client=False): + credstore_env=None, use_ssh_client=False, + max_pool_size=DEFAULT_MAX_POOL_SIZE): super(APIClient, self).__init__() if tls and not base_url: @@ -139,7 +142,8 @@ def __init__(self, base_url=None, version=None, if base_url.startswith('http+unix://'): self._custom_adapter = UnixHTTPAdapter( - base_url, timeout, pool_connections=num_pools + base_url, timeout, pool_connections=num_pools, + max_pool_size=max_pool_size ) self.mount('http+docker://', self._custom_adapter) self._unmount('http://', 'https://') @@ -153,7 +157,8 @@ def __init__(self, base_url=None, version=None, ) try: self._custom_adapter = NpipeHTTPAdapter( - base_url, timeout, pool_connections=num_pools + base_url, timeout, pool_connections=num_pools, + max_pool_size=max_pool_size ) except NameError: raise DockerException( @@ -165,7 +170,7 @@ def __init__(self, base_url=None, version=None, try: self._custom_adapter = SSHHTTPAdapter( base_url, timeout, pool_connections=num_pools, - shell_out=use_ssh_client + max_pool_size=max_pool_size, shell_out=use_ssh_client ) except NameError: raise DockerException( diff --git a/docker/client.py b/docker/client.py index 1fea69e660..5add5d730e 100644 --- a/docker/client.py +++ b/docker/client.py @@ -1,5 +1,5 @@ from .api.client import APIClient -from .constants import DEFAULT_TIMEOUT_SECONDS +from .constants import (DEFAULT_TIMEOUT_SECONDS, DEFAULT_MAX_POOL_SIZE) from .models.configs import ConfigCollection from .models.containers import ContainerCollection from .models.images import ImageCollection @@ -38,6 +38,8 @@ class DockerClient(object): use_ssh_client (bool): If set to `True`, an ssh connection is made via shelling out to the ssh client. Ensure the ssh client is installed and configured on the host. + max_pool_size (int): The maximum number of connections + to save in the pool. """ def __init__(self, *args, **kwargs): self.api = APIClient(*args, **kwargs) @@ -67,6 +69,8 @@ def from_env(cls, **kwargs): version (str): The version of the API to use. Set to ``auto`` to automatically detect the server's version. Default: ``auto`` timeout (int): Default timeout for API calls, in seconds. + max_pool_size (int): The maximum number of connections + to save in the pool. ssl_version (int): A valid `SSL version`_. assert_hostname (bool): Verify the hostname of the server. environment (dict): The environment to read environment variables @@ -86,10 +90,12 @@ def from_env(cls, **kwargs): https://docs.python.org/3.5/library/ssl.html#ssl.PROTOCOL_TLSv1 """ timeout = kwargs.pop('timeout', DEFAULT_TIMEOUT_SECONDS) + max_pool_size = kwargs.pop('max_pool_size', DEFAULT_MAX_POOL_SIZE) version = kwargs.pop('version', None) use_ssh_client = kwargs.pop('use_ssh_client', False) return cls( timeout=timeout, + max_pool_size=max_pool_size, version=version, use_ssh_client=use_ssh_client, **kwargs_from_env(**kwargs) diff --git a/docker/constants.py b/docker/constants.py index c09eedab29..43fce6138e 100644 --- a/docker/constants.py +++ b/docker/constants.py @@ -36,6 +36,8 @@ # For more details see: https://github.com/docker/docker-py/issues/2246 DEFAULT_NUM_POOLS_SSH = 9 +DEFAULT_MAX_POOL_SIZE = 10 + DEFAULT_DATA_CHUNK_SIZE = 1024 * 2048 DEFAULT_SWARM_ADDR_POOL = ['10.0.0.0/8'] diff --git a/docker/transport/npipeconn.py b/docker/transport/npipeconn.py index aa05538ddf..70d8519dc0 100644 --- a/docker/transport/npipeconn.py +++ b/docker/transport/npipeconn.py @@ -73,12 +73,15 @@ class NpipeHTTPAdapter(BaseHTTPAdapter): __attrs__ = requests.adapters.HTTPAdapter.__attrs__ + ['npipe_path', 'pools', - 'timeout'] + 'timeout', + 'max_pool_size'] def __init__(self, base_url, timeout=60, - pool_connections=constants.DEFAULT_NUM_POOLS): + pool_connections=constants.DEFAULT_NUM_POOLS, + max_pool_size=constants.DEFAULT_MAX_POOL_SIZE): self.npipe_path = base_url.replace('npipe://', '') self.timeout = timeout + self.max_pool_size = max_pool_size self.pools = RecentlyUsedContainer( pool_connections, dispose_func=lambda p: p.close() ) @@ -91,7 +94,8 @@ def get_connection(self, url, proxies=None): return pool pool = NpipeHTTPConnectionPool( - self.npipe_path, self.timeout + self.npipe_path, self.timeout, + maxsize=self.max_pool_size ) self.pools[url] = pool diff --git a/docker/transport/sshconn.py b/docker/transport/sshconn.py index 42d1ef9688..cdeeae4e75 100644 --- a/docker/transport/sshconn.py +++ b/docker/transport/sshconn.py @@ -184,11 +184,12 @@ def _get_conn(self, timeout): class SSHHTTPAdapter(BaseHTTPAdapter): __attrs__ = requests.adapters.HTTPAdapter.__attrs__ + [ - 'pools', 'timeout', 'ssh_client', 'ssh_params' + 'pools', 'timeout', 'ssh_client', 'ssh_params', 'max_pool_size' ] def __init__(self, base_url, timeout=60, pool_connections=constants.DEFAULT_NUM_POOLS, + max_pool_size=constants.DEFAULT_MAX_POOL_SIZE, shell_out=True): self.ssh_client = None if not shell_out: @@ -197,6 +198,7 @@ def __init__(self, base_url, timeout=60, base_url = base_url.lstrip('ssh://') self.host = base_url self.timeout = timeout + self.max_pool_size = max_pool_size self.pools = RecentlyUsedContainer( pool_connections, dispose_func=lambda p: p.close() ) @@ -219,6 +221,7 @@ def get_connection(self, url, proxies=None): pool = SSHConnectionPool( ssh_client=self.ssh_client, timeout=self.timeout, + maxsize=self.max_pool_size, host=self.host ) self.pools[url] = pool diff --git a/docker/transport/unixconn.py b/docker/transport/unixconn.py index b619103247..3e040c5af8 100644 --- a/docker/transport/unixconn.py +++ b/docker/transport/unixconn.py @@ -74,15 +74,18 @@ class UnixHTTPAdapter(BaseHTTPAdapter): __attrs__ = requests.adapters.HTTPAdapter.__attrs__ + ['pools', 'socket_path', - 'timeout'] + 'timeout', + 'max_pool_size'] def __init__(self, socket_url, timeout=60, - pool_connections=constants.DEFAULT_NUM_POOLS): + pool_connections=constants.DEFAULT_NUM_POOLS, + max_pool_size=constants.DEFAULT_MAX_POOL_SIZE): socket_path = socket_url.replace('http+unix://', '') if not socket_path.startswith('/'): socket_path = '/' + socket_path self.socket_path = socket_path self.timeout = timeout + self.max_pool_size = max_pool_size self.pools = RecentlyUsedContainer( pool_connections, dispose_func=lambda p: p.close() ) @@ -95,7 +98,8 @@ def get_connection(self, url, proxies=None): return pool pool = UnixHTTPConnectionPool( - url, self.socket_path, self.timeout + url, self.socket_path, self.timeout, + maxsize=self.max_pool_size ) self.pools[url] = pool diff --git a/tests/unit/client_test.py b/tests/unit/client_test.py index cc9ff8f24c..ad88e8456e 100644 --- a/tests/unit/client_test.py +++ b/tests/unit/client_test.py @@ -5,7 +5,9 @@ import docker import pytest from docker.constants import ( - DEFAULT_DOCKER_API_VERSION, DEFAULT_TIMEOUT_SECONDS) + DEFAULT_DOCKER_API_VERSION, DEFAULT_TIMEOUT_SECONDS, + DEFAULT_MAX_POOL_SIZE, IS_WINDOWS_PLATFORM +) from docker.utils import kwargs_from_env from . import fake_api @@ -15,8 +17,8 @@ except ImportError: import mock - TEST_CERT_DIR = os.path.join(os.path.dirname(__file__), 'testdata/certs') +POOL_SIZE = 20 class ClientTest(unittest.TestCase): @@ -76,6 +78,84 @@ def test_call_containers(self): assert "'ContainerCollection' object is not callable" in s assert "docker.APIClient" in s + @pytest.mark.skipif( + IS_WINDOWS_PLATFORM, reason='Unix Connection Pool only on Linux' + ) + @mock.patch("docker.transport.unixconn.UnixHTTPConnectionPool") + def test_default_pool_size_unix(self, mock_obj): + client = docker.DockerClient( + version=DEFAULT_DOCKER_API_VERSION + ) + mock_obj.return_value.urlopen.return_value.status = 200 + client.ping() + + base_url = "{base_url}/v{version}/_ping".format( + base_url=client.api.base_url, + version=client.api._version + ) + + mock_obj.assert_called_once_with(base_url, + "/var/run/docker.sock", + 60, + maxsize=DEFAULT_MAX_POOL_SIZE + ) + + @pytest.mark.skipif( + not IS_WINDOWS_PLATFORM, reason='Npipe Connection Pool only on Windows' + ) + @mock.patch("docker.transport.npipeconn.NpipeHTTPConnectionPool") + def test_default_pool_size_win(self, mock_obj): + client = docker.DockerClient( + version=DEFAULT_DOCKER_API_VERSION + ) + mock_obj.return_value.urlopen.return_value.status = 200 + client.ping() + + mock_obj.assert_called_once_with("//./pipe/docker_engine", + 60, + maxsize=DEFAULT_MAX_POOL_SIZE + ) + + @pytest.mark.skipif( + IS_WINDOWS_PLATFORM, reason='Unix Connection Pool only on Linux' + ) + @mock.patch("docker.transport.unixconn.UnixHTTPConnectionPool") + def test_pool_size_unix(self, mock_obj): + client = docker.DockerClient( + version=DEFAULT_DOCKER_API_VERSION, + max_pool_size=POOL_SIZE + ) + mock_obj.return_value.urlopen.return_value.status = 200 + client.ping() + + base_url = "{base_url}/v{version}/_ping".format( + base_url=client.api.base_url, + version=client.api._version + ) + + mock_obj.assert_called_once_with(base_url, + "/var/run/docker.sock", + 60, + maxsize=POOL_SIZE + ) + + @pytest.mark.skipif( + not IS_WINDOWS_PLATFORM, reason='Npipe Connection Pool only on Windows' + ) + @mock.patch("docker.transport.npipeconn.NpipeHTTPConnectionPool") + def test_pool_size_win(self, mock_obj): + client = docker.DockerClient( + version=DEFAULT_DOCKER_API_VERSION, + max_pool_size=POOL_SIZE + ) + mock_obj.return_value.urlopen.return_value.status = 200 + client.ping() + + mock_obj.assert_called_once_with("//./pipe/docker_engine", + 60, + maxsize=POOL_SIZE + ) + class FromEnvTest(unittest.TestCase): @@ -112,3 +192,77 @@ def test_from_env_without_timeout_uses_default(self): client = docker.from_env(version=DEFAULT_DOCKER_API_VERSION) assert client.api.timeout == DEFAULT_TIMEOUT_SECONDS + + @pytest.mark.skipif( + IS_WINDOWS_PLATFORM, reason='Unix Connection Pool only on Linux' + ) + @mock.patch("docker.transport.unixconn.UnixHTTPConnectionPool") + def test_default_pool_size_from_env_unix(self, mock_obj): + client = docker.from_env(version=DEFAULT_DOCKER_API_VERSION) + mock_obj.return_value.urlopen.return_value.status = 200 + client.ping() + + base_url = "{base_url}/v{version}/_ping".format( + base_url=client.api.base_url, + version=client.api._version + ) + + mock_obj.assert_called_once_with(base_url, + "/var/run/docker.sock", + 60, + maxsize=DEFAULT_MAX_POOL_SIZE + ) + + @pytest.mark.skipif( + not IS_WINDOWS_PLATFORM, reason='Npipe Connection Pool only on Windows' + ) + @mock.patch("docker.transport.npipeconn.NpipeHTTPConnectionPool") + def test_default_pool_size_from_env_win(self, mock_obj): + client = docker.from_env(version=DEFAULT_DOCKER_API_VERSION) + mock_obj.return_value.urlopen.return_value.status = 200 + client.ping() + + mock_obj.assert_called_once_with("//./pipe/docker_engine", + 60, + maxsize=DEFAULT_MAX_POOL_SIZE + ) + + @pytest.mark.skipif( + IS_WINDOWS_PLATFORM, reason='Unix Connection Pool only on Linux' + ) + @mock.patch("docker.transport.unixconn.UnixHTTPConnectionPool") + def test_pool_size_from_env_unix(self, mock_obj): + client = docker.from_env( + version=DEFAULT_DOCKER_API_VERSION, + max_pool_size=POOL_SIZE + ) + mock_obj.return_value.urlopen.return_value.status = 200 + client.ping() + + base_url = "{base_url}/v{version}/_ping".format( + base_url=client.api.base_url, + version=client.api._version + ) + + mock_obj.assert_called_once_with(base_url, + "/var/run/docker.sock", + 60, + maxsize=POOL_SIZE + ) + + @pytest.mark.skipif( + not IS_WINDOWS_PLATFORM, reason='Npipe Connection Pool only on Windows' + ) + @mock.patch("docker.transport.npipeconn.NpipeHTTPConnectionPool") + def test_pool_size_from_env_win(self, mock_obj): + client = docker.from_env( + version=DEFAULT_DOCKER_API_VERSION, + max_pool_size=POOL_SIZE + ) + mock_obj.return_value.urlopen.return_value.status = 200 + client.ping() + + mock_obj.assert_called_once_with("//./pipe/docker_engine", + 60, + maxsize=POOL_SIZE + ) From f5531a94e1096f4c8456264f6511dfe89e1c825e Mon Sep 17 00:00:00 2001 From: aiordache Date: Tue, 20 Oct 2020 10:05:07 +0200 Subject: [PATCH 1067/1301] Fix ssh connection - don't override the host and port of the http pool Signed-off-by: aiordache --- docker/transport/sshconn.py | 111 +++++++++++++++++++----------------- tests/Dockerfile-ssh-dind | 2 +- 2 files changed, 59 insertions(+), 54 deletions(-) diff --git a/docker/transport/sshconn.py b/docker/transport/sshconn.py index cdeeae4e75..5cdaa27573 100644 --- a/docker/transport/sshconn.py +++ b/docker/transport/sshconn.py @@ -1,9 +1,9 @@ -import io import paramiko import requests.adapters import six import logging import os +import signal import socket import subprocess @@ -23,40 +23,6 @@ RecentlyUsedContainer = urllib3._collections.RecentlyUsedContainer -def create_paramiko_client(base_url): - logging.getLogger("paramiko").setLevel(logging.WARNING) - ssh_client = paramiko.SSHClient() - base_url = six.moves.urllib_parse.urlparse(base_url) - ssh_params = { - "hostname": base_url.hostname, - "port": base_url.port, - "username": base_url.username - } - ssh_config_file = os.path.expanduser("~/.ssh/config") - if os.path.exists(ssh_config_file): - conf = paramiko.SSHConfig() - with open(ssh_config_file) as f: - conf.parse(f) - host_config = conf.lookup(base_url.hostname) - ssh_conf = host_config - if 'proxycommand' in host_config: - ssh_params["sock"] = paramiko.ProxyCommand( - ssh_conf['proxycommand'] - ) - if 'hostname' in host_config: - ssh_params['hostname'] = host_config['hostname'] - if 'identityfile' in host_config: - ssh_params['key_filename'] = host_config['identityfile'] - if base_url.port is None and 'port' in host_config: - ssh_params['port'] = ssh_conf['port'] - if base_url.username is None and 'user' in host_config: - ssh_params['username'] = ssh_conf['user'] - - ssh_client.load_system_host_keys() - ssh_client.set_missing_host_key_policy(paramiko.WarningPolicy()) - return ssh_client, ssh_params - - class SSHSocket(socket.socket): def __init__(self, host): super(SSHSocket, self).__init__( @@ -80,7 +46,8 @@ def connect(self, **kwargs): ' '.join(args), shell=True, stdout=subprocess.PIPE, - stdin=subprocess.PIPE) + stdin=subprocess.PIPE, + preexec_fn=lambda: signal.signal(signal.SIGINT, signal.SIG_IGN)) def _write(self, data): if not self.proc or self.proc.stdin.closed: @@ -96,17 +63,18 @@ def sendall(self, data): def send(self, data): return self._write(data) - def recv(self): + def recv(self, n): if not self.proc: raise Exception('SSH subprocess not initiated.' 'connect() must be called first.') - return self.proc.stdout.read() + return self.proc.stdout.read(n) def makefile(self, mode): - if not self.proc or self.proc.stdout.closed: - buf = io.BytesIO() - buf.write(b'\n\n') - return buf + if not self.proc: + self.connect() + if six.PY3: + self.proc.stdout.channel = self + return self.proc.stdout def close(self): @@ -124,7 +92,7 @@ def __init__(self, ssh_transport=None, timeout=60, host=None): ) self.ssh_transport = ssh_transport self.timeout = timeout - self.host = host + self.ssh_host = host def connect(self): if self.ssh_transport: @@ -132,7 +100,7 @@ def connect(self): sock.settimeout(self.timeout) sock.exec_command('docker system dial-stdio') else: - sock = SSHSocket(self.host) + sock = SSHSocket(self.ssh_host) sock.settimeout(self.timeout) sock.connect() @@ -147,16 +115,16 @@ def __init__(self, ssh_client=None, timeout=60, maxsize=10, host=None): 'localhost', timeout=timeout, maxsize=maxsize ) self.ssh_transport = None + self.timeout = timeout if ssh_client: self.ssh_transport = ssh_client.get_transport() - self.timeout = timeout - self.host = host - self.port = None + self.ssh_host = host + self.ssh_port = None if ':' in host: - self.host, self.port = host.split(':') + self.ssh_host, self.ssh_port = host.split(':') def _new_conn(self): - return SSHConnection(self.ssh_transport, self.timeout, self.host) + return SSHConnection(self.ssh_transport, self.timeout, self.ssh_host) # When re-using connections, urllib3 calls fileno() on our # SSH channel instance, quickly overloading our fd limit. To avoid this, @@ -193,10 +161,10 @@ def __init__(self, base_url, timeout=60, shell_out=True): self.ssh_client = None if not shell_out: - self.ssh_client, self.ssh_params = create_paramiko_client(base_url) + self._create_paramiko_client(base_url) self._connect() - base_url = base_url.lstrip('ssh://') - self.host = base_url + + self.ssh_host = base_url.lstrip('ssh://') self.timeout = timeout self.max_pool_size = max_pool_size self.pools = RecentlyUsedContainer( @@ -204,11 +172,48 @@ def __init__(self, base_url, timeout=60, ) super(SSHHTTPAdapter, self).__init__() + def _create_paramiko_client(self, base_url): + logging.getLogger("paramiko").setLevel(logging.WARNING) + self.ssh_client = paramiko.SSHClient() + base_url = six.moves.urllib_parse.urlparse(base_url) + self.ssh_params = { + "hostname": base_url.hostname, + "port": base_url.port, + "username": base_url.username + } + ssh_config_file = os.path.expanduser("~/.ssh/config") + if os.path.exists(ssh_config_file): + conf = paramiko.SSHConfig() + with open(ssh_config_file) as f: + conf.parse(f) + host_config = conf.lookup(base_url.hostname) + self.ssh_conf = host_config + if 'proxycommand' in host_config: + self.ssh_params["sock"] = paramiko.ProxyCommand( + self.ssh_conf['proxycommand'] + ) + if 'hostname' in host_config: + self.ssh_params['hostname'] = host_config['hostname'] + if base_url.port is None and 'port' in host_config: + self.ssh_params['port'] = self.ssh_conf['port'] + if base_url.username is None and 'user' in host_config: + self.ssh_params['username'] = self.ssh_conf['user'] + + self.ssh_client.load_system_host_keys() + self.ssh_client.set_missing_host_key_policy(paramiko.WarningPolicy()) + def _connect(self): if self.ssh_client: self.ssh_client.connect(**self.ssh_params) def get_connection(self, url, proxies=None): + if not self.ssh_client: + return SSHConnectionPool( + ssh_client=self.ssh_client, + timeout=self.timeout, + maxsize=self.max_pool_size, + host=self.ssh_host + ) with self.pools.lock: pool = self.pools.get(url) if pool: @@ -222,7 +227,7 @@ def get_connection(self, url, proxies=None): ssh_client=self.ssh_client, timeout=self.timeout, maxsize=self.max_pool_size, - host=self.host + host=self.ssh_host ) self.pools[url] = pool diff --git a/tests/Dockerfile-ssh-dind b/tests/Dockerfile-ssh-dind index 9d8f0eab74..aba9bb34b2 100644 --- a/tests/Dockerfile-ssh-dind +++ b/tests/Dockerfile-ssh-dind @@ -10,7 +10,7 @@ RUN apk add --no-cache \ RUN ssh-keygen -A # copy the test SSH config -RUN echo "IgnoreUserKnownHosts yes" >> /etc/ssh/sshd_config && \ +RUN echo "IgnoreUserKnownHosts yes" > /etc/ssh/sshd_config && \ echo "PubkeyAuthentication yes" >> /etc/ssh/sshd_config && \ echo "PermitRootLogin yes" >> /etc/ssh/sshd_config From db9af44e5b36b81d2dc6643dda5d674a1ae462c9 Mon Sep 17 00:00:00 2001 From: aiordache Date: Wed, 18 Nov 2020 19:00:27 +0100 Subject: [PATCH 1068/1301] Fix docs typo Signed-off-by: aiordache --- docker/models/containers.py | 2 +- docker/types/containers.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/models/containers.py b/docker/models/containers.py index 120386a199..36cbbc41ad 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -683,7 +683,7 @@ def run(self, image, command=None, stdout=True, stderr=False, to a single container port. For example, ``{'1111/tcp': [1234, 4567]}``. - Imcompatible with ``host`` in ``network_mode``. + Incompatible with ``host`` network mode. privileged (bool): Give extended privileges to this container. publish_all_ports (bool): Publish all ports to the host. read_only (bool): Mount the container's root filesystem as read diff --git a/docker/types/containers.py b/docker/types/containers.py index 1df3fff1c7..d1938c91d8 100644 --- a/docker/types/containers.py +++ b/docker/types/containers.py @@ -334,7 +334,7 @@ def __init__(self, version, binds=None, port_bindings=None, if dns_search: self['DnsSearch'] = dns_search - if network_mode is 'host' and port_bindings is not None: + if network_mode is 'host' and port_bindings: raise host_config_incompatible_error( 'network_mode', 'host', 'port_bindings' ) @@ -670,7 +670,7 @@ def host_config_value_error(param, param_value): def host_config_incompatible_error(param, param_value, incompatible_param): - error_msg = 'Incompatible {1} in {0} is not compatible with {2}' + error_msg = '\"{1}\" {0} is incompatible with {2}' return errors.InvalidArgument( error_msg.format(param, param_value, incompatible_param) ) From a0c51be2289dfa1ff05eb1834452e5808714b64a Mon Sep 17 00:00:00 2001 From: aiordache Date: Thu, 19 Nov 2020 15:38:26 +0100 Subject: [PATCH 1069/1301] Syntax warning fix Signed-off-by: aiordache --- docker/types/containers.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/docker/types/containers.py b/docker/types/containers.py index d1938c91d8..9fa4656ab8 100644 --- a/docker/types/containers.py +++ b/docker/types/containers.py @@ -334,15 +334,11 @@ def __init__(self, version, binds=None, port_bindings=None, if dns_search: self['DnsSearch'] = dns_search - if network_mode is 'host' and port_bindings: + if network_mode == 'host' and port_bindings: raise host_config_incompatible_error( 'network_mode', 'host', 'port_bindings' ) - - if network_mode: - self['NetworkMode'] = network_mode - elif network_mode is None: - self['NetworkMode'] = 'default' + self['NetworkMode'] = network_mode or 'default' if restart_policy: if not isinstance(restart_policy, dict): From 260114229a9adf47bf76ed0ed7e9da9364a7d30f Mon Sep 17 00:00:00 2001 From: aiordache Date: Thu, 19 Nov 2020 18:51:59 +0100 Subject: [PATCH 1070/1301] Update Jenkinsfile with docker registry credentials Signed-off-by: aiordache --- Jenkinsfile | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index fc716d80eb..b5ea7a4b90 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -25,10 +25,11 @@ def buildImages = { -> imageNamePy2 = "${imageNameBase}:py2-${gitCommit()}" imageNamePy3 = "${imageNameBase}:py3-${gitCommit()}" imageDindSSH = "${imageNameBase}:sshdind-${gitCommit()}" - - buildImage(imageDindSSH, "-f tests/Dockerfile-ssh-dind .", "") - buildImage(imageNamePy2, "-f tests/Dockerfile --build-arg PYTHON_VERSION=2.7 .", "py2.7") - buildImage(imageNamePy3, "-f tests/Dockerfile --build-arg PYTHON_VERSION=3.7 .", "py3.7") + withDockerRegistry(credentialsId:'dockerbuildbot-index.docker.io') { + buildImage(imageDindSSH, "-f tests/Dockerfile-ssh-dind .", "") + buildImage(imageNamePy2, "-f tests/Dockerfile --build-arg PYTHON_VERSION=2.7 .", "py2.7") + buildImage(imageNamePy3, "-f tests/Dockerfile --build-arg PYTHON_VERSION=3.7 .", "py3.7") + } } } } From c854aba15e14cdb6b84f34770e0ee8398f54b393 Mon Sep 17 00:00:00 2001 From: aiordache Date: Thu, 19 Nov 2020 19:33:24 +0100 Subject: [PATCH 1071/1301] Mount docker config to DIND containers for authentication Signed-off-by: aiordache --- Jenkinsfile | 73 ++++++++++++++++++++++++++++------------------------- 1 file changed, 38 insertions(+), 35 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index b5ea7a4b90..4f970770b6 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -83,41 +83,44 @@ def runTests = { Map settings -> def dindContainerName = "dpy-dind-\$BUILD_NUMBER-\$EXECUTOR_NUMBER-${pythonVersion}-${dockerVersion}" def testContainerName = "dpy-tests-\$BUILD_NUMBER-\$EXECUTOR_NUMBER-${pythonVersion}-${dockerVersion}" def testNetwork = "dpy-testnet-\$BUILD_NUMBER-\$EXECUTOR_NUMBER-${pythonVersion}-${dockerVersion}" - try { - sh """docker network create ${testNetwork}""" - sh """docker run --rm -d --name ${dindContainerName} -v /tmp --privileged --network ${testNetwork} \\ - ${imageDindSSH} dockerd -H tcp://0.0.0.0:2375 - """ - sh """docker run --rm \\ - --name ${testContainerName} \\ - -e "DOCKER_HOST=tcp://${dindContainerName}:2375" \\ - -e 'DOCKER_TEST_API_VERSION=${apiVersion}' \\ - --network ${testNetwork} \\ - --volumes-from ${dindContainerName} \\ - ${testImage} \\ - py.test -v -rxs --cov=docker --ignore=tests/ssh tests/ - """ - sh """docker stop ${dindContainerName}""" - - // start DIND container with SSH - sh """docker run --rm -d --name ${dindContainerName} -v /tmp --privileged --network ${testNetwork} \\ - ${imageDindSSH} dockerd --experimental""" - sh """docker exec ${dindContainerName} sh -c /usr/sbin/sshd """ - // run SSH tests only - sh """docker run --rm \\ - --name ${testContainerName} \\ - -e "DOCKER_HOST=ssh://${dindContainerName}:22" \\ - -e 'DOCKER_TEST_API_VERSION=${apiVersion}' \\ - --network ${testNetwork} \\ - --volumes-from ${dindContainerName} \\ - ${testImage} \\ - py.test -v -rxs --cov=docker tests/ssh - """ - } finally { - sh """ - docker stop ${dindContainerName} - docker network rm ${testNetwork} - """ + withDockerRegistry(credentialsId:'dockerbuildbot-index.docker.io') { + try { + sh """docker network create ${testNetwork}""" + sh """docker run --rm -d --name ${dindContainerName} -v /tmp --privileged --network ${testNetwork} \\ + ${imageDindSSH} dockerd -H tcp://0.0.0.0:2375 + """ + sh """docker run --rm \\ + --name ${testContainerName} \\ + -e "DOCKER_HOST=tcp://${dindContainerName}:2375" \\ + -e 'DOCKER_TEST_API_VERSION=${apiVersion}' \\ + --network ${testNetwork} \\ + --volumes-from ${dindContainerName} \\ + -v ~/.docker/config.json:/root/.docker/config.json \\ + ${testImage} \\ + py.test -v -rxs --cov=docker --ignore=tests/ssh tests/ + """ + sh """docker stop ${dindContainerName}""" + // start DIND container with SSH + sh """docker run --rm -d --name ${dindContainerName} -v /tmp --privileged --network ${testNetwork} \\ + ${imageDindSSH} dockerd --experimental""" + sh """docker exec ${dindContainerName} sh -c /usr/sbin/sshd """ + // run SSH tests only + sh """docker run --rm \\ + --name ${testContainerName} \\ + -e "DOCKER_HOST=ssh://${dindContainerName}:22" \\ + -e 'DOCKER_TEST_API_VERSION=${apiVersion}' \\ + --network ${testNetwork} \\ + --volumes-from ${dindContainerName} \\ + -v ~/.docker/config.json:/root/.docker/config.json \\ + ${testImage} \\ + py.test -v -rxs --cov=docker tests/ssh + """ + } finally { + sh """ + docker stop ${dindContainerName} + docker network rm ${testNetwork} + """ + } } } } From 990ef4904c22aac2d35a7ae3a583b57d170cdfd7 Mon Sep 17 00:00:00 2001 From: aiordache Date: Mon, 23 Nov 2020 13:16:23 +0100 Subject: [PATCH 1072/1301] Post-release v4.4.0 Signed-off-by: aiordache --- docker/version.py | 2 +- docs/change-log.md | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index f40347aa54..bc09e63709 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "4.4.0-dev" +version = "4.5.0-dev" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) diff --git a/docs/change-log.md b/docs/change-log.md index 11c055fda3..fe3dc71b2f 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -1,6 +1,24 @@ Change log ========== +4.4.0 +----- + +[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/67?closed=1) + +### Features +- Add an alternative SSH connection to the paramiko one, based on shelling out to the SSh client. Similar to the behaviour of Docker cli +- Default image tag to `latest` on `pull` + +### Bugfixes +- Fix plugin model upgrade +- Fix examples URL in ulimits + +### Miscellaneous +- Improve exception messages for server and client errors +- Bump cryptography from 2.3 to 3.2 + + 4.3.1 ----- From 1757c974fa3a05b0e9b783af85242b18df09d05d Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Tue, 24 Nov 2020 12:04:42 +0100 Subject: [PATCH 1073/1301] docker/api/image: replace use of deprecated "filter" argument The "filter" argument was deprecated in docker 1.13 (API version 1.25), and removed from API v1.41 and up. See https://github.com/docker/cli/blob/v20.10.0-rc1/docs/deprecated.md#filter-param-for-imagesjson-endpoint This patch applies the name as "reference" filter, instead of "filter" for API 1.25 and up. Signed-off-by: Sebastiaan van Stijn --- docker/api/image.py | 10 +++++++++- tests/unit/api_image_test.py | 19 +++++++++++++++---- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/docker/api/image.py b/docker/api/image.py index 4082bfb3d7..56c5448eb3 100644 --- a/docker/api/image.py +++ b/docker/api/image.py @@ -81,10 +81,18 @@ def images(self, name=None, quiet=False, all=False, filters=None): If the server returns an error. """ params = { - 'filter': name, 'only_ids': 1 if quiet else 0, 'all': 1 if all else 0, } + if name: + if utils.version_lt(self._version, '1.25'): + # only use "filter" on API 1.24 and under, as it is deprecated + params['filter'] = name + else: + if filters: + filters['reference'] = name + else: + filters = {'reference': name} if filters: params['filters'] = utils.convert_filters(filters) res = self._result(self._get(self._url("/images/json"), params=params), diff --git a/tests/unit/api_image_test.py b/tests/unit/api_image_test.py index 4b4fb97765..0b60df43a7 100644 --- a/tests/unit/api_image_test.py +++ b/tests/unit/api_image_test.py @@ -26,7 +26,18 @@ def test_images(self): fake_request.assert_called_with( 'GET', url_prefix + 'images/json', - params={'filter': None, 'only_ids': 0, 'all': 1}, + params={'only_ids': 0, 'all': 1}, + timeout=DEFAULT_TIMEOUT_SECONDS + ) + + def test_images_name(self): + self.client.images('foo:bar') + + fake_request.assert_called_with( + 'GET', + url_prefix + 'images/json', + params={'only_ids': 0, 'all': 0, + 'filters': '{"reference": ["foo:bar"]}'}, timeout=DEFAULT_TIMEOUT_SECONDS ) @@ -36,7 +47,7 @@ def test_images_quiet(self): fake_request.assert_called_with( 'GET', url_prefix + 'images/json', - params={'filter': None, 'only_ids': 1, 'all': 1}, + params={'only_ids': 1, 'all': 1}, timeout=DEFAULT_TIMEOUT_SECONDS ) @@ -46,7 +57,7 @@ def test_image_ids(self): fake_request.assert_called_with( 'GET', url_prefix + 'images/json', - params={'filter': None, 'only_ids': 1, 'all': 0}, + params={'only_ids': 1, 'all': 0}, timeout=DEFAULT_TIMEOUT_SECONDS ) @@ -56,7 +67,7 @@ def test_images_filters(self): fake_request.assert_called_with( 'GET', url_prefix + 'images/json', - params={'filter': None, 'only_ids': 0, 'all': 0, + params={'only_ids': 0, 'all': 0, 'filters': '{"dangling": ["true"]}'}, timeout=DEFAULT_TIMEOUT_SECONDS ) From d8bbbf23517007083a4e9f66e6e3028061c0e5ed Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Mon, 14 Dec 2020 14:06:23 -0300 Subject: [PATCH 1074/1301] Add Github Actions Signed-off-by: Ulysses Souza --- .github/workflows/ci.yml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000000..9fb9a455c1 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,27 @@ +name: Python package + +on: [push] + +jobs: + build: + runs-on: ubuntu-latest + strategy: + max-parallel: 1 + matrix: + python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9] + + steps: + - uses: actions/checkout@v1 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install tox tox-gh-actions==2.2.0 + - name: Test with tox + run: | + docker logout + rm -rf ~/.docker + tox From ab0d65e2e0f09d47886b79f874e344da9a523286 Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Mon, 14 Dec 2020 14:17:52 -0300 Subject: [PATCH 1075/1301] Remove travis Signed-off-by: Ulysses Souza --- .travis.yml | 20 -------------------- 1 file changed, 20 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 7b3d7248d9..0000000000 --- a/.travis.yml +++ /dev/null @@ -1,20 +0,0 @@ -sudo: false -language: python -matrix: - include: - - python: 2.7 - env: TOXENV=py27 - - python: 3.5 - env: TOXENV=py35 - - python: 3.6 - env: TOXENV=py36 - - python: 3.7 - env: TOXENV=py37 - dist: xenial - sudo: true - - env: TOXENV=flake8 - -install: - - pip install tox==2.9.1 -script: - - tox From 4757eea80c49b7d593537ea9a0b9e5b398570745 Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Fri, 18 Dec 2020 11:51:55 -0300 Subject: [PATCH 1076/1301] Trigger GHA on pull_request Signed-off-by: Ulysses Souza --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9fb9a455c1..c3c786a480 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,6 +1,6 @@ name: Python package -on: [push] +on: [push, pull_request] jobs: build: From 3ec7fee7362eecfd7bcfd62edcfac3380422fb64 Mon Sep 17 00:00:00 2001 From: aiordache Date: Thu, 17 Dec 2020 12:09:27 +0100 Subject: [PATCH 1077/1301] Avoid setting unsuported parameter for subprocess.Popen on Windows Signed-off-by: aiordache --- docker/transport/sshconn.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docker/transport/sshconn.py b/docker/transport/sshconn.py index 5cdaa27573..7f4b2a2d21 100644 --- a/docker/transport/sshconn.py +++ b/docker/transport/sshconn.py @@ -42,12 +42,17 @@ def connect(self, **kwargs): port, 'docker system dial-stdio' ] + + preexec_func = None + if not constants.IS_WINDOWS_PLATFORM: + preexec_func = lambda: signal.signal(signal.SIGINT, signal.SIG_IGN) + self.proc = subprocess.Popen( ' '.join(args), shell=True, stdout=subprocess.PIPE, stdin=subprocess.PIPE, - preexec_fn=lambda: signal.signal(signal.SIGINT, signal.SIG_IGN)) + preexec_fn=preexec_func) def _write(self, data): if not self.proc or self.proc.stdin.closed: From 2f3e0f9fc441d2a637bfc0816d7eb6d814a7cd72 Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Tue, 22 Dec 2020 17:19:24 -0300 Subject: [PATCH 1078/1301] Prepare release 4.4.1 Signed-off-by: Ulysses Souza --- docker/version.py | 2 +- docs/change-log.md | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index bc09e63709..600d2454d5 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "4.5.0-dev" +version = "4.4.1" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) diff --git a/docs/change-log.md b/docs/change-log.md index fe3dc71b2f..351e2c517f 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -1,6 +1,16 @@ Change log ========== +4.4.1 +----- + +[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/69?closed=1) + +### Bugfixes +- Avoid setting unsuported parameter for subprocess.Popen on Windows +- Replace use of deprecated "filter" argument on ""docker/api/image" + + 4.4.0 ----- From b72926b3822e018baa8d0a82a76c32881932d6a2 Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Tue, 22 Dec 2020 17:50:19 -0300 Subject: [PATCH 1079/1301] Post 4.4.1 release Signed-off-by: Ulysses Souza --- docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index 600d2454d5..bc09e63709 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "4.4.1" +version = "4.5.0-dev" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) From 2426a5ffd57c49c257988df498f402978e9d901a Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Thu, 24 Dec 2020 15:14:05 +0100 Subject: [PATCH 1080/1301] setup.py: Add support for Python 3.8 and 3.9 Signed-off-by: Christian Clauss --- setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.py b/setup.py index c702295080..330ab3e357 100644 --- a/setup.py +++ b/setup.py @@ -84,6 +84,8 @@ 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'Topic :: Software Development', 'Topic :: Utilities', 'License :: OSI Approved :: Apache Software License', From f0ab0ed25d7b031b13226f1b44ce8d33a56d1ffa Mon Sep 17 00:00:00 2001 From: Piotr Wojciechowski Date: Fri, 25 Dec 2020 16:39:44 +0100 Subject: [PATCH 1081/1301] Support for docker.types.Placement.MaxReplicas (new in API 1.40) in Docker Swarm Service Signed-off-by: WojciechowskiPiotr --- docker/models/services.py | 3 +++ docker/types/services.py | 6 +++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/docker/models/services.py b/docker/models/services.py index a35687b3ca..a29ff1326a 100644 --- a/docker/models/services.py +++ b/docker/models/services.py @@ -157,6 +157,8 @@ def create(self, image, command=None, **kwargs): constraints. preferences (list of tuple): :py:class:`~docker.types.Placement` preferences. + maxreplicas (int): :py:class:`~docker.types.Placement` maxreplicas + or (int) representing maximum number of replicas per node. platforms (list of tuple): A list of platform constraints expressed as ``(arch, os)`` tuples. container_labels (dict): Labels to apply to the container. @@ -319,6 +321,7 @@ def list(self, **kwargs): 'constraints', 'preferences', 'platforms', + 'maxreplicas', ] diff --git a/docker/types/services.py b/docker/types/services.py index 05dda15d75..3cde8592e0 100644 --- a/docker/types/services.py +++ b/docker/types/services.py @@ -659,10 +659,12 @@ class Placement(dict): are provided in order from highest to lowest precedence and are expressed as ``(strategy, descriptor)`` tuples. See :py:class:`PlacementPreference` for details. + maxreplicas (int): Maximum number of replicas per node platforms (:py:class:`list` of tuple): A list of platforms expressed as ``(arch, os)`` tuples """ - def __init__(self, constraints=None, preferences=None, platforms=None): + def __init__(self, constraints=None, preferences=None, maxreplicas=None, + platforms=None): if constraints is not None: self['Constraints'] = constraints if preferences is not None: @@ -671,6 +673,8 @@ def __init__(self, constraints=None, preferences=None, platforms=None): if isinstance(pref, tuple): pref = PlacementPreference(*pref) self['Preferences'].append(pref) + if maxreplicas is not None: + self['MaxReplicas'] = maxreplicas if platforms: self['Platforms'] = [] for plat in platforms: From ce2669e3edfe5d3215ba501cc9771fc0ffad680a Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Thu, 24 Dec 2020 15:29:54 +0100 Subject: [PATCH 1082/1301] print() is a function in Python 3 Signed-off-by: Christian Clauss --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3ff124d7a5..8ce684b553 100644 --- a/README.md +++ b/README.md @@ -58,7 +58,7 @@ You can stream logs: ```python >>> for line in container.logs(stream=True): -... print line.strip() +... print(line.strip()) Reticulating spline 2... Reticulating spline 3... ... From 10ff4030793ac93ef5f4cc079dc8c71c6fa60a74 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Mon, 28 Dec 2020 18:51:17 +0100 Subject: [PATCH 1083/1301] print() is a function in Python 3 Like #2740 but for the docs Signed-off-by: Christian Clauss --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index 63e85d3635..93b30d4a07 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -58,7 +58,7 @@ You can stream logs: .. code-block:: python >>> for line in container.logs(stream=True): - ... print line.strip() + ... print(line.strip()) Reticulating spline 2... Reticulating spline 3... ... From 0edea80c415652f13a25b20e5937cd9c41e35063 Mon Sep 17 00:00:00 2001 From: aiordache Date: Mon, 8 Feb 2021 20:04:14 +0100 Subject: [PATCH 1084/1301] Update base image to `dockerpinata/docker-py` in Jenkinsfile Signed-off-by: aiordache --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 4f970770b6..d99c605472 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,6 +1,6 @@ #!groovy -def imageNameBase = "dockerbuildbot/docker-py" +def imageNameBase = "dockerpinata/docker-py" def imageNamePy2 def imageNamePy3 def imageDindSSH From caab390696940dce51f93e04e43419a90595216e Mon Sep 17 00:00:00 2001 From: aiordache Date: Mon, 8 Feb 2021 19:03:58 +0100 Subject: [PATCH 1085/1301] Fix host trimming and remove quiet flag for the ssh connection Signed-off-by: aiordache --- docker/transport/sshconn.py | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/docker/transport/sshconn.py b/docker/transport/sshconn.py index 7f4b2a2d21..7593b5bd34 100644 --- a/docker/transport/sshconn.py +++ b/docker/transport/sshconn.py @@ -29,26 +29,33 @@ def __init__(self, host): socket.AF_INET, socket.SOCK_STREAM) self.host = host self.port = None + self.user = None if ':' in host: self.host, self.port = host.split(':') + if '@' in self.host: + self.user, self.host = host.split('@') + self.proc = None def connect(self, **kwargs): - port = '' if not self.port else '-p {}'.format(self.port) - args = [ - 'ssh', - '-q', - self.host, - port, - 'docker system dial-stdio' - ] + args = ['ssh'] + if self.user: + args = args + ['-l', self.user] + + if self.port: + args = args + ['-p', self.port] + + args = args + ['--', self.host, 'docker system dial-stdio'] preexec_func = None if not constants.IS_WINDOWS_PLATFORM: - preexec_func = lambda: signal.signal(signal.SIGINT, signal.SIG_IGN) + def f(): + signal.signal(signal.SIGINT, signal.SIG_IGN) + preexec_func = f self.proc = subprocess.Popen( ' '.join(args), + env=os.environ, shell=True, stdout=subprocess.PIPE, stdin=subprocess.PIPE, @@ -124,9 +131,6 @@ def __init__(self, ssh_client=None, timeout=60, maxsize=10, host=None): if ssh_client: self.ssh_transport = ssh_client.get_transport() self.ssh_host = host - self.ssh_port = None - if ':' in host: - self.ssh_host, self.ssh_port = host.split(':') def _new_conn(self): return SSHConnection(self.ssh_transport, self.timeout, self.ssh_host) @@ -169,7 +173,10 @@ def __init__(self, base_url, timeout=60, self._create_paramiko_client(base_url) self._connect() - self.ssh_host = base_url.lstrip('ssh://') + self.ssh_host = base_url + if base_url.startswith('ssh://'): + self.ssh_host = base_url[len('ssh://'):] + self.timeout = timeout self.max_pool_size = max_pool_size self.pools = RecentlyUsedContainer( From 514f98a0d69c9350d8f19088e571ddbebfe89b5e Mon Sep 17 00:00:00 2001 From: WojciechowskiPiotr Date: Tue, 9 Feb 2021 19:45:52 +0100 Subject: [PATCH 1086/1301] Support for docker.types.Placement.MaxReplicas (new in API 1.40) in Docker Swarm Service Signed-off-by: WojciechowskiPiotr --- docker/types/services.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/types/services.py b/docker/types/services.py index 3cde8592e0..29498e9715 100644 --- a/docker/types/services.py +++ b/docker/types/services.py @@ -663,8 +663,8 @@ class Placement(dict): platforms (:py:class:`list` of tuple): A list of platforms expressed as ``(arch, os)`` tuples """ - def __init__(self, constraints=None, preferences=None, maxreplicas=None, - platforms=None): + def __init__(self, constraints=None, preferences=None, platforms=None, + maxreplicas=None): if constraints is not None: self['Constraints'] = constraints if preferences is not None: From da32a2f1a2f21573627a5df0ea309048e4058b9f Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Thu, 24 Dec 2020 15:07:45 +0100 Subject: [PATCH 1087/1301] GitHub Actions: Upgrade actions/checkout https://github.com/actions/checkout/releases Signed-off-by: Christian Clauss --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c3c786a480..d8d55ea08d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,7 +11,7 @@ jobs: python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9] steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 with: From 9e007469ef0d206517618bcc66a379d024ed715c Mon Sep 17 00:00:00 2001 From: Stefan Scherer Date: Tue, 9 Feb 2021 20:39:54 +0100 Subject: [PATCH 1088/1301] Update CI to ubuntu-2004 Signed-off-by: Stefan Scherer --- Jenkinsfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index d99c605472..e7c332155f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -18,7 +18,7 @@ def buildImage = { name, buildargs, pyTag -> } def buildImages = { -> - wrappedNode(label: "amd64 && ubuntu-1804 && overlay2", cleanWorkspace: true) { + wrappedNode(label: "amd64 && ubuntu-2004 && overlay2", cleanWorkspace: true) { stage("build image") { checkout(scm) @@ -36,7 +36,7 @@ def buildImages = { -> def getDockerVersions = { -> def dockerVersions = ["19.03.12"] - wrappedNode(label: "amd64 && ubuntu-1804 && overlay2") { + wrappedNode(label: "amd64 && ubuntu-2004 && overlay2") { def result = sh(script: """docker run --rm \\ --entrypoint=python \\ ${imageNamePy3} \\ @@ -77,7 +77,7 @@ def runTests = { Map settings -> } { -> - wrappedNode(label: "amd64 && ubuntu-1804 && overlay2", cleanWorkspace: true) { + wrappedNode(label: "amd64 && ubuntu-2004 && overlay2", cleanWorkspace: true) { stage("test python=${pythonVersion} / docker=${dockerVersion}") { checkout(scm) def dindContainerName = "dpy-dind-\$BUILD_NUMBER-\$EXECUTOR_NUMBER-${pythonVersion}-${dockerVersion}" From 6d1dffe3e5738ebe02294c795a7e5e630f7913c2 Mon Sep 17 00:00:00 2001 From: WojciechowskiPiotr Date: Tue, 9 Feb 2021 21:37:26 +0100 Subject: [PATCH 1089/1301] Unit and integration tests added Signed-off-by: WojciechowskiPiotr --- tests/integration/api_service_test.py | 13 +++++++++++++ tests/unit/models_services_test.py | 2 ++ 2 files changed, 15 insertions(+) diff --git a/tests/integration/api_service_test.py b/tests/integration/api_service_test.py index b6b7ec538d..7e5336e2e5 100644 --- a/tests/integration/api_service_test.py +++ b/tests/integration/api_service_test.py @@ -471,6 +471,19 @@ def test_create_service_with_placement_preferences_tuple(self): assert 'Placement' in svc_info['Spec']['TaskTemplate'] assert svc_info['Spec']['TaskTemplate']['Placement'] == placemt + @requires_api_version('1.40') + def test_create_service_with_placement_maxreplicas(self): + container_spec = docker.types.ContainerSpec(TEST_IMG, ['true']) + placemt = docker.types.Placement(maxreplicas=1) + task_tmpl = docker.types.TaskTemplate( + container_spec, placement=placemt + ) + name = self.get_service_name() + svc_id = self.client.create_service(task_tmpl, name=name) + svc_info = self.client.inspect_service(svc_id) + assert 'Placement' in svc_info['Spec']['TaskTemplate'] + assert svc_info['Spec']['TaskTemplate']['Placement'] == placemt + def test_create_service_with_endpoint_spec(self): container_spec = docker.types.ContainerSpec(TEST_IMG, ['true']) task_tmpl = docker.types.TaskTemplate(container_spec) diff --git a/tests/unit/models_services_test.py b/tests/unit/models_services_test.py index a4ac50c3fe..07bb58970d 100644 --- a/tests/unit/models_services_test.py +++ b/tests/unit/models_services_test.py @@ -28,6 +28,7 @@ def test_get_create_service_kwargs(self): 'constraints': ['foo=bar'], 'preferences': ['bar=baz'], 'platforms': [('x86_64', 'linux')], + 'maxreplicas': 1 }) task_template = kwargs.pop('task_template') @@ -47,6 +48,7 @@ def test_get_create_service_kwargs(self): 'Constraints': ['foo=bar'], 'Preferences': ['bar=baz'], 'Platforms': [{'Architecture': 'x86_64', 'OS': 'linux'}], + 'MaxReplicas': 1, } assert task_template['LogDriver'] == { 'Name': 'logdriver', From f520b4c4ebfe01484563681e7d8411de44e5ee85 Mon Sep 17 00:00:00 2001 From: aiordache Date: Tue, 9 Feb 2021 18:55:25 +0100 Subject: [PATCH 1090/1301] Update GH action step Signed-off-by: aiordache --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d8d55ea08d..1f119c90e9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,9 +19,9 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install tox tox-gh-actions==2.2.0 - - name: Test with tox + pip install -r test-requirements.txt -r requirements.txt + - name: Test with pytest run: | docker logout rm -rf ~/.docker - tox + py.test -v --cov=docker tests/unit From ccab78840e78ff768d2559c8539660e83e679fe2 Mon Sep 17 00:00:00 2001 From: aiordache Date: Tue, 9 Feb 2021 19:55:35 +0100 Subject: [PATCH 1091/1301] Bump cffi to 1.14.4 Signed-off-by: aiordache --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 0edd4e1e71..43a688fd90 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ appdirs==1.4.3 asn1crypto==0.22.0 backports.ssl-match-hostname==3.5.0.1 -cffi==1.10.0 +cffi==1.14.4 cryptography==3.2 enum34==1.1.6 idna==2.5 From 9556b890f9a1a7488adac06792aecf767a20f1d3 Mon Sep 17 00:00:00 2001 From: Stefan Scherer Date: Wed, 10 Feb 2021 16:57:30 +0100 Subject: [PATCH 1092/1301] Remove wrappedNode Signed-off-by: Stefan Scherer --- Jenkinsfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index e7c332155f..d333f425b6 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -18,7 +18,7 @@ def buildImage = { name, buildargs, pyTag -> } def buildImages = { -> - wrappedNode(label: "amd64 && ubuntu-2004 && overlay2", cleanWorkspace: true) { + node("amd64 && ubuntu-2004 && overlay2") { stage("build image") { checkout(scm) @@ -36,7 +36,7 @@ def buildImages = { -> def getDockerVersions = { -> def dockerVersions = ["19.03.12"] - wrappedNode(label: "amd64 && ubuntu-2004 && overlay2") { + node("amd64 && ubuntu-2004 && overlay2") { def result = sh(script: """docker run --rm \\ --entrypoint=python \\ ${imageNamePy3} \\ @@ -77,7 +77,7 @@ def runTests = { Map settings -> } { -> - wrappedNode(label: "amd64 && ubuntu-2004 && overlay2", cleanWorkspace: true) { + node("amd64 && ubuntu-2004 && overlay2") { stage("test python=${pythonVersion} / docker=${dockerVersion}") { checkout(scm) def dindContainerName = "dpy-dind-\$BUILD_NUMBER-\$EXECUTOR_NUMBER-${pythonVersion}-${dockerVersion}" From 56d4b09700cdd9a7a99b5840601af946a63f3bfa Mon Sep 17 00:00:00 2001 From: Vlad Romanenko Date: Mon, 30 Nov 2020 14:18:54 +0000 Subject: [PATCH 1093/1301] Fix doc formatting Signed-off-by: Vlad Romanenko --- docker/api/client.py | 2 +- docker/api/daemon.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/api/client.py b/docker/api/client.py index fbf7ad455a..2b67291aa9 100644 --- a/docker/api/client.py +++ b/docker/api/client.py @@ -497,7 +497,7 @@ def reload_config(self, dockercfg_path=None): Args: dockercfg_path (str): Use a custom path for the Docker config file (default ``$HOME/.docker/config.json`` if present, - otherwise``$HOME/.dockercfg``) + otherwise ``$HOME/.dockercfg``) Returns: None diff --git a/docker/api/daemon.py b/docker/api/daemon.py index f715a131ad..6b719268ef 100644 --- a/docker/api/daemon.py +++ b/docker/api/daemon.py @@ -109,7 +109,7 @@ def login(self, username, password=None, email=None, registry=None, the Docker server. dockercfg_path (str): Use a custom path for the Docker config file (default ``$HOME/.docker/config.json`` if present, - otherwise``$HOME/.dockercfg``) + otherwise ``$HOME/.dockercfg``) Returns: (dict): The response from the login request From 94d7983ef0e0a421fd1a84320dea960e78605ab3 Mon Sep 17 00:00:00 2001 From: Stefan Scherer Date: Thu, 11 Feb 2021 10:24:57 +0100 Subject: [PATCH 1094/1301] Revert back to wrappedNode Signed-off-by: Stefan Scherer --- Jenkinsfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index d333f425b6..e7c332155f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -18,7 +18,7 @@ def buildImage = { name, buildargs, pyTag -> } def buildImages = { -> - node("amd64 && ubuntu-2004 && overlay2") { + wrappedNode(label: "amd64 && ubuntu-2004 && overlay2", cleanWorkspace: true) { stage("build image") { checkout(scm) @@ -36,7 +36,7 @@ def buildImages = { -> def getDockerVersions = { -> def dockerVersions = ["19.03.12"] - node("amd64 && ubuntu-2004 && overlay2") { + wrappedNode(label: "amd64 && ubuntu-2004 && overlay2") { def result = sh(script: """docker run --rm \\ --entrypoint=python \\ ${imageNamePy3} \\ @@ -77,7 +77,7 @@ def runTests = { Map settings -> } { -> - node("amd64 && ubuntu-2004 && overlay2") { + wrappedNode(label: "amd64 && ubuntu-2004 && overlay2", cleanWorkspace: true) { stage("test python=${pythonVersion} / docker=${dockerVersion}") { checkout(scm) def dindContainerName = "dpy-dind-\$BUILD_NUMBER-\$EXECUTOR_NUMBER-${pythonVersion}-${dockerVersion}" From 6de6936f5d2d4decb2150f1816d4dd94b73649f1 Mon Sep 17 00:00:00 2001 From: Stefan Scherer Date: Thu, 11 Feb 2021 17:52:56 +0100 Subject: [PATCH 1095/1301] Use DOCKER_CONFIG to have creds in dind environment Signed-off-by: Stefan Scherer --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index e7c332155f..b85598f85f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -95,7 +95,7 @@ def runTests = { Map settings -> -e 'DOCKER_TEST_API_VERSION=${apiVersion}' \\ --network ${testNetwork} \\ --volumes-from ${dindContainerName} \\ - -v ~/.docker/config.json:/root/.docker/config.json \\ + -v $DOCKER_CONFIG/config.json:/root/.docker/config.json \\ ${testImage} \\ py.test -v -rxs --cov=docker --ignore=tests/ssh tests/ """ @@ -111,7 +111,7 @@ def runTests = { Map settings -> -e 'DOCKER_TEST_API_VERSION=${apiVersion}' \\ --network ${testNetwork} \\ --volumes-from ${dindContainerName} \\ - -v ~/.docker/config.json:/root/.docker/config.json \\ + -v $DOCKER_CONFIG/config.json:/root/.docker/config.json \\ ${testImage} \\ py.test -v -rxs --cov=docker tests/ssh """ From 00da4dc0eae7d491c16384077f4d4da9a58836b1 Mon Sep 17 00:00:00 2001 From: aiordache Date: Thu, 11 Feb 2021 19:58:35 +0100 Subject: [PATCH 1096/1301] Run unit tests in a container with no .docker/config mount Signed-off-by: aiordache --- Jenkinsfile | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index b85598f85f..471072bf19 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -85,6 +85,13 @@ def runTests = { Map settings -> def testNetwork = "dpy-testnet-\$BUILD_NUMBER-\$EXECUTOR_NUMBER-${pythonVersion}-${dockerVersion}" withDockerRegistry(credentialsId:'dockerbuildbot-index.docker.io') { try { + // unit tests + sh """docker run --rm \\ + -e 'DOCKER_TEST_API_VERSION=${apiVersion}' \\ + ${testImage} \\ + py.test -v -rxs --cov=docker tests/unit + """ + // integration tests sh """docker network create ${testNetwork}""" sh """docker run --rm -d --name ${dindContainerName} -v /tmp --privileged --network ${testNetwork} \\ ${imageDindSSH} dockerd -H tcp://0.0.0.0:2375 @@ -97,7 +104,7 @@ def runTests = { Map settings -> --volumes-from ${dindContainerName} \\ -v $DOCKER_CONFIG/config.json:/root/.docker/config.json \\ ${testImage} \\ - py.test -v -rxs --cov=docker --ignore=tests/ssh tests/ + py.test -v -rxs --cov=docker tests/integration """ sh """docker stop ${dindContainerName}""" // start DIND container with SSH From 2807fde6c991596ac12853995c931c925128ee61 Mon Sep 17 00:00:00 2001 From: aiordache Date: Thu, 18 Feb 2021 12:56:46 +0100 Subject: [PATCH 1097/1301] Fix SSH port parsing and add regression tests Signed-off-by: aiordache --- docker/transport/sshconn.py | 8 ++++---- tests/unit/sshadapter_test.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 4 deletions(-) create mode 100644 tests/unit/sshadapter_test.py diff --git a/docker/transport/sshconn.py b/docker/transport/sshconn.py index 7593b5bd34..fbfdf4163c 100644 --- a/docker/transport/sshconn.py +++ b/docker/transport/sshconn.py @@ -30,10 +30,10 @@ def __init__(self, host): self.host = host self.port = None self.user = None - if ':' in host: - self.host, self.port = host.split(':') + if ':' in self.host: + self.host, self.port = self.host.split(':') if '@' in self.host: - self.user, self.host = host.split('@') + self.user, self.host = self.host.split('@') self.proc = None @@ -167,7 +167,7 @@ class SSHHTTPAdapter(BaseHTTPAdapter): def __init__(self, base_url, timeout=60, pool_connections=constants.DEFAULT_NUM_POOLS, max_pool_size=constants.DEFAULT_MAX_POOL_SIZE, - shell_out=True): + shell_out=False): self.ssh_client = None if not shell_out: self._create_paramiko_client(base_url) diff --git a/tests/unit/sshadapter_test.py b/tests/unit/sshadapter_test.py new file mode 100644 index 0000000000..ddee592029 --- /dev/null +++ b/tests/unit/sshadapter_test.py @@ -0,0 +1,32 @@ +import unittest +import docker +from docker.transport.sshconn import SSHSocket + +class SSHAdapterTest(unittest.TestCase): + def test_ssh_hostname_prefix_trim(self): + conn = docker.transport.SSHHTTPAdapter(base_url="ssh://user@hostname:1234", shell_out=True) + assert conn.ssh_host == "user@hostname:1234" + + def test_ssh_parse_url(self): + c = SSHSocket(host="user@hostname:1234") + assert c.host == "hostname" + assert c.port == "1234" + assert c.user == "user" + + def test_ssh_parse_hostname_only(self): + c = SSHSocket(host="hostname") + assert c.host == "hostname" + assert c.port == None + assert c.user == None + + def test_ssh_parse_user_and_hostname(self): + c = SSHSocket(host="user@hostname") + assert c.host == "hostname" + assert c.port == None + assert c.user == "user" + + def test_ssh_parse_hostname_and_port(self): + c = SSHSocket(host="hostname:22") + assert c.host == "hostname" + assert c.port == "22" + assert c.user == None \ No newline at end of file From e6689e0bb9af849bd9d1509fd3b2db52e0d6d776 Mon Sep 17 00:00:00 2001 From: aiordache Date: Thu, 18 Feb 2021 10:23:20 +0100 Subject: [PATCH 1098/1301] Post-release 4.4.2 changelog updates Signed-off-by: aiordache --- docs/change-log.md | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/docs/change-log.md b/docs/change-log.md index 351e2c517f..f666697c68 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -1,6 +1,18 @@ Change log ========== +4.4.2 +----- + +[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/71?closed=1) + +### Bugfixes +- Fix SSH connection bug where the hostname was incorrectly trimmed and the error was hidden +- Fix docs example + +### Miscellaneous +- Add Python3.8 and 3.9 in setup.py classifier list + 4.4.1 ----- @@ -10,7 +22,6 @@ Change log - Avoid setting unsuported parameter for subprocess.Popen on Windows - Replace use of deprecated "filter" argument on ""docker/api/image" - 4.4.0 ----- @@ -28,7 +39,6 @@ Change log - Improve exception messages for server and client errors - Bump cryptography from 2.3 to 3.2 - 4.3.1 ----- @@ -53,7 +63,6 @@ Change log - Update default API version to v1.39 - Update test engine version to 19.03.12 - 4.2.2 ----- @@ -109,7 +118,6 @@ Change log - Adjust `--platform` tests for changes in docker engine - Update credentials-helpers to v0.6.3 - 4.0.2 ----- @@ -123,7 +131,6 @@ Change log - Bumped version of websocket-client - 4.0.1 ----- From fe995ae79f9ae464b39f33e1ce474d33999e867f Mon Sep 17 00:00:00 2001 From: aiordache Date: Fri, 19 Feb 2021 10:07:52 +0100 Subject: [PATCH 1099/1301] Update changelog post-release 4.4.3 Signed-off-by: aiordache --- docs/change-log.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/change-log.md b/docs/change-log.md index f666697c68..546e071f98 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -1,6 +1,17 @@ Change log ========== +4.4.3 +----- + +[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/72?closed=1) + +### Features +- Add support for docker.types.Placement.MaxReplicas + +### Bugfixes +- Fix SSH port parsing when shelling out to the ssh client + 4.4.2 ----- From 43ca2f8ff958cc29d66ef6badae8121b81ee3434 Mon Sep 17 00:00:00 2001 From: aiordache Date: Tue, 23 Feb 2021 19:04:03 +0100 Subject: [PATCH 1100/1301] Drop LD_LIBRARY_PATH env var for SSH shellout Signed-off-by: aiordache --- docker/transport/sshconn.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docker/transport/sshconn.py b/docker/transport/sshconn.py index fbfdf4163c..a761ef517f 100644 --- a/docker/transport/sshconn.py +++ b/docker/transport/sshconn.py @@ -53,9 +53,15 @@ def f(): signal.signal(signal.SIGINT, signal.SIG_IGN) preexec_func = f + env = dict(os.environ) + + # drop LD_LIBRARY_PATH and SSL_CERT_FILE + env.pop('LD_LIBRARY_PATH', None) + env.pop('SSL_CERT_FILE', None) + self.proc = subprocess.Popen( ' '.join(args), - env=os.environ, + env=env, shell=True, stdout=subprocess.PIPE, stdin=subprocess.PIPE, From 148f9161e113a59914df941707c919b7319266dc Mon Sep 17 00:00:00 2001 From: aiordache Date: Wed, 24 Feb 2021 18:20:24 +0100 Subject: [PATCH 1101/1301] Update changelog for 4.4.4 Signed-off-by: aiordache --- docs/change-log.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/change-log.md b/docs/change-log.md index 546e071f98..8db3fc5821 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -1,6 +1,14 @@ Change log ========== +4.4.4 +----- + +[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/73?closed=1) + +### Bugfixes +- Remove `LD_LIBRARY_PATH` and `SSL_CERT_FILE` environment variables when shelling out to the ssh client + 4.4.3 ----- From 7d316641a3da5ece9a29471390ef965d13b160b7 Mon Sep 17 00:00:00 2001 From: Felipe Ruhland Date: Wed, 4 Dec 2019 19:44:27 -0300 Subject: [PATCH 1102/1301] Add limit parameter to image search endpoint Signed-off-by: Felipe Ruhland --- docker/api/image.py | 9 +++++++-- tests/unit/models_images_test.py | 5 +++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/docker/api/image.py b/docker/api/image.py index 56c5448eb3..4658cdee7c 100644 --- a/docker/api/image.py +++ b/docker/api/image.py @@ -509,13 +509,14 @@ def remove_image(self, image, force=False, noprune=False): res = self._delete(self._url("/images/{0}", image), params=params) return self._result(res, True) - def search(self, term): + def search(self, term, limit=None): """ Search for images on Docker Hub. Similar to the ``docker search`` command. Args: term (str): A term to search for. + limit (int): The maximum number of results to return. Returns: (list of dicts): The response of the search. @@ -524,8 +525,12 @@ def search(self, term): :py:class:`docker.errors.APIError` If the server returns an error. """ + params = {'term': term} + if limit is not None: + params['limit'] = limit + return self._result( - self._get(self._url("/images/search"), params={'term': term}), + self._get(self._url("/images/search"), params=params), True ) diff --git a/tests/unit/models_images_test.py b/tests/unit/models_images_test.py index e3d070c048..f3ca0be4e7 100644 --- a/tests/unit/models_images_test.py +++ b/tests/unit/models_images_test.py @@ -112,6 +112,11 @@ def test_search(self): client.images.search('test') client.api.search.assert_called_with('test') + def test_search_limit(self): + client = make_fake_client() + client.images.search('test', limit=5) + client.api.search.assert_called_with('test', limit=5) + class ImageTest(unittest.TestCase): def test_short_id(self): From d836bb8703959128e5ce274e3b5186a797760303 Mon Sep 17 00:00:00 2001 From: Felipe Ruhland Date: Fri, 26 Feb 2021 21:59:35 +0100 Subject: [PATCH 1103/1301] Fix continuous integration status badged https://docs.github.com/en/actions/managing-workflow-runs/ adding-a-workflow-status-badge Signed-off-by: Felipe Ruhland --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8ce684b553..4fc31f7d75 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Docker SDK for Python -[![Build Status](https://travis-ci.org/docker/docker-py.svg?branch=master)](https://travis-ci.org/docker/docker-py) +[![Build Status](https://github.com/docker/docker-py/actions/workflows/ci.yml/badge.svg?branch=master)](https://github.com/docker/docker-py/actions/workflows/ci.yml/) A Python library for the Docker Engine API. It lets you do anything the `docker` command does, but from within Python apps – run containers, manage containers, manage Swarms, etc. From c239d66d5d261f0f956925705c679fffb61bdb05 Mon Sep 17 00:00:00 2001 From: Hakan Ardo Date: Wed, 3 Mar 2021 09:27:21 +0100 Subject: [PATCH 1104/1301] Verify TLS keys loaded from docker contexts This maches the behaviour of the docker cli when using contexts. Signed-off-by: Hakan Ardo --- docker/context/context.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docker/context/context.py b/docker/context/context.py index 2413b2ecbf..b1cacf92ae 100644 --- a/docker/context/context.py +++ b/docker/context/context.py @@ -127,8 +127,12 @@ def _load_certs(self): elif filename.startswith("key"): key = os.path.join(tls_dir, endpoint, filename) if all([ca_cert, cert, key]): + verify = None + if endpoint == "docker": + if not self.endpoints["docker"].get("SkipTLSVerify", False): + verify = True certs[endpoint] = TLSConfig( - client_cert=(cert, key), ca_cert=ca_cert) + client_cert=(cert, key), ca_cert=ca_cert, verify=verify) self.tls_cfg = certs self.tls_path = tls_dir From 563124163a5d092e954846121150d896ddca0836 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Thu, 4 Mar 2021 11:37:07 +0100 Subject: [PATCH 1105/1301] relax PORT_SPEC regex so it accept and ignore square brackets Signed-off-by: Nicolas De Loof --- docker/utils/ports.py | 2 +- tests/unit/utils_test.py | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/docker/utils/ports.py b/docker/utils/ports.py index a50cc029f2..10b19d741c 100644 --- a/docker/utils/ports.py +++ b/docker/utils/ports.py @@ -3,7 +3,7 @@ PORT_SPEC = re.compile( "^" # Match full string "(" # External part - r"((?P[a-fA-F\d.:]+):)?" # Address + r"(\[?(?P[a-fA-F\d.:]+)\]?:)?" # Address r"(?P[\d]*)(-(?P[\d]+))?:" # External range ")?" r"(?P[\d]+)(-(?P[\d]+))?" # Internal range diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index a53151cb3b..0d6ff22d7e 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -541,6 +541,12 @@ def test_split_port_with_ipv6_address(self): assert internal_port == ["2000"] assert external_port == [("2001:abcd:ef00::2", "1000")] + def test_split_port_with_ipv6_square_brackets_address(self): + internal_port, external_port = split_port( + "[2001:abcd:ef00::2]:1000:2000") + assert internal_port == ["2000"] + assert external_port == [("2001:abcd:ef00::2", "1000")] + def test_split_port_invalid(self): with pytest.raises(ValueError): split_port("0.0.0.0:1000:2000:tcp") From c8fba210a222d4f7fde90da8f48db1e7faa637ec Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Wed, 10 Mar 2021 20:43:37 -0300 Subject: [PATCH 1106/1301] Remove support to pre python 3.6 Signed-off-by: Ulysses Souza --- .github/workflows/ci.yml | 6 +-- .readthedocs.yml | 2 +- Dockerfile | 6 +-- Dockerfile-py3 | 15 -------- Jenkinsfile | 5 +-- Makefile | 50 ++++--------------------- docker/api/client.py | 18 ++++----- docker/context/context.py | 7 ++-- docker/transport/sshconn.py | 4 +- docker/utils/utils.py | 49 ++++++++++-------------- requirements.txt | 1 - setup.py | 12 +----- test-requirements.txt | 2 +- tests/Dockerfile-dind-certs | 2 +- tests/integration/api_container_test.py | 48 +++++++----------------- tests/integration/api_image_test.py | 11 +++--- tests/integration/api_service_test.py | 3 +- tests/unit/sshadapter_test.py | 27 ++++++++----- tox.ini | 2 +- 19 files changed, 88 insertions(+), 182 deletions(-) delete mode 100644 Dockerfile-py3 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1f119c90e9..b692508220 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,7 +8,7 @@ jobs: strategy: max-parallel: 1 matrix: - python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9] + python-version: [3.6, 3.7, 3.8, 3.9] steps: - uses: actions/checkout@v2 @@ -18,8 +18,8 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | - python -m pip install --upgrade pip - pip install -r test-requirements.txt -r requirements.txt + python3 -m pip install --upgrade pip + pip3 install -r test-requirements.txt -r requirements.txt - name: Test with pytest run: | docker logout diff --git a/.readthedocs.yml b/.readthedocs.yml index 7679f80ab1..32113fedb4 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -4,7 +4,7 @@ sphinx: configuration: docs/conf.py python: - version: 3.5 + version: 3.6 install: - requirements: docs-requirements.txt - requirements: requirements.txt diff --git a/Dockerfile b/Dockerfile index 7309a83e3f..22732dec5c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,7 @@ -ARG PYTHON_VERSION=2.7 +ARG PYTHON_VERSION=3.7 FROM python:${PYTHON_VERSION} -# Add SSH keys and set permissions -COPY tests/ssh-keys /root/.ssh -RUN chmod -R 600 /root/.ssh - RUN mkdir /src WORKDIR /src diff --git a/Dockerfile-py3 b/Dockerfile-py3 deleted file mode 100644 index 22732dec5c..0000000000 --- a/Dockerfile-py3 +++ /dev/null @@ -1,15 +0,0 @@ -ARG PYTHON_VERSION=3.7 - -FROM python:${PYTHON_VERSION} - -RUN mkdir /src -WORKDIR /src - -COPY requirements.txt /src/requirements.txt -RUN pip install -r requirements.txt - -COPY test-requirements.txt /src/test-requirements.txt -RUN pip install -r test-requirements.txt - -COPY . /src -RUN pip install . diff --git a/Jenkinsfile b/Jenkinsfile index 471072bf19..f524ae7a14 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,7 +1,6 @@ #!groovy def imageNameBase = "dockerpinata/docker-py" -def imageNamePy2 def imageNamePy3 def imageDindSSH def images = [:] @@ -22,12 +21,10 @@ def buildImages = { -> stage("build image") { checkout(scm) - imageNamePy2 = "${imageNameBase}:py2-${gitCommit()}" imageNamePy3 = "${imageNameBase}:py3-${gitCommit()}" imageDindSSH = "${imageNameBase}:sshdind-${gitCommit()}" withDockerRegistry(credentialsId:'dockerbuildbot-index.docker.io') { buildImage(imageDindSSH, "-f tests/Dockerfile-ssh-dind .", "") - buildImage(imageNamePy2, "-f tests/Dockerfile --build-arg PYTHON_VERSION=2.7 .", "py2.7") buildImage(imageNamePy3, "-f tests/Dockerfile --build-arg PYTHON_VERSION=3.7 .", "py3.7") } } @@ -73,7 +70,7 @@ def runTests = { Map settings -> throw new Exception("Need Docker version to test, e.g.: `runTests(dockerVersion: '19.03.12')`") } if (!pythonVersion) { - throw new Exception("Need Python version being tested, e.g.: `runTests(pythonVersion: 'py2.7')`") + throw new Exception("Need Python version being tested, e.g.: `runTests(pythonVersion: 'py3.7')`") } { -> diff --git a/Makefile b/Makefile index 70d7083e41..60d99842c3 100644 --- a/Makefile +++ b/Makefile @@ -6,13 +6,9 @@ all: test .PHONY: clean clean: - -docker rm -f dpy-dind-py2 dpy-dind-py3 dpy-dind-certs dpy-dind-ssl + -docker rm -f dpy-dind-py3 dpy-dind-certs dpy-dind-ssl find -name "__pycache__" | xargs rm -rf -.PHONY: build -build: - docker build -t docker-sdk-python -f tests/Dockerfile --build-arg PYTHON_VERSION=2.7 --build-arg APT_MIRROR . - .PHONY: build-dind-ssh build-dind-ssh: docker build -t docker-dind-ssh -f tests/Dockerfile-ssh-dind --build-arg ENGINE_VERSION=${TEST_ENGINE_VERSION} --build-arg API_VERSION=${TEST_API_VERSION} --build-arg APT_MIRROR . @@ -30,20 +26,12 @@ build-dind-certs: docker build -t dpy-dind-certs -f tests/Dockerfile-dind-certs . .PHONY: test -test: flake8 unit-test unit-test-py3 integration-dind integration-dind-ssl - -.PHONY: unit-test -unit-test: build - docker run -t --rm docker-sdk-python py.test tests/unit +test: flake8 unit-test-py3 integration-dind integration-dind-ssl .PHONY: unit-test-py3 unit-test-py3: build-py3 docker run -t --rm docker-sdk-python3 py.test tests/unit -.PHONY: integration-test -integration-test: build - docker run -t --rm -v /var/run/docker.sock:/var/run/docker.sock docker-sdk-python py.test -v tests/integration/${file} - .PHONY: integration-test-py3 integration-test-py3: build-py3 docker run -t --rm -v /var/run/docker.sock:/var/run/docker.sock docker-sdk-python3 py.test -v tests/integration/${file} @@ -53,16 +41,7 @@ setup-network: docker network inspect dpy-tests || docker network create dpy-tests .PHONY: integration-dind -integration-dind: integration-dind-py2 integration-dind-py3 - -.PHONY: integration-dind-py2 -integration-dind-py2: build setup-network - docker rm -vf dpy-dind-py2 || : - docker run -d --network dpy-tests --name dpy-dind-py2 --privileged\ - docker:${TEST_ENGINE_VERSION}-dind dockerd -H tcp://0.0.0.0:2375 --experimental - docker run -t --rm --env="DOCKER_HOST=tcp://dpy-dind-py2:2375" --env="DOCKER_TEST_API_VERSION=${TEST_API_VERSION}"\ - --network dpy-tests docker-sdk-python py.test tests/integration/${file} - docker rm -vf dpy-dind-py2 +integration-dind: integration-dind-py3 .PHONY: integration-dind-py3 integration-dind-py3: build-py3 setup-network @@ -73,16 +52,6 @@ integration-dind-py3: build-py3 setup-network --network dpy-tests docker-sdk-python3 py.test tests/integration/${file} docker rm -vf dpy-dind-py3 -.PHONY: integration-ssh-py2 -integration-ssh-py2: build-dind-ssh build setup-network - docker rm -vf dpy-dind-py2 || : - docker run -d --network dpy-tests --name dpy-dind-py2 --privileged\ - docker-dind-ssh dockerd --experimental - # start SSH daemon - docker exec dpy-dind-py2 sh -c "/usr/sbin/sshd" - docker run -t --rm --env="DOCKER_HOST=ssh://dpy-dind-py2" --env="DOCKER_TEST_API_VERSION=${TEST_API_VERSION}"\ - --network dpy-tests docker-sdk-python py.test tests/ssh/${file} - docker rm -vf dpy-dind-py2 .PHONY: integration-ssh-py3 integration-ssh-py3: build-dind-ssh build-py3 setup-network @@ -97,7 +66,7 @@ integration-ssh-py3: build-dind-ssh build-py3 setup-network .PHONY: integration-dind-ssl -integration-dind-ssl: build-dind-certs build build-py3 +integration-dind-ssl: build-dind-certs build-py3 docker rm -vf dpy-dind-certs dpy-dind-ssl || : docker run -d --name dpy-dind-certs dpy-dind-certs docker run -d --env="DOCKER_HOST=tcp://localhost:2375" --env="DOCKER_TLS_VERIFY=1"\ @@ -106,22 +75,19 @@ integration-dind-ssl: build-dind-certs build build-py3 docker:${TEST_ENGINE_VERSION}-dind\ dockerd --tlsverify --tlscacert=/certs/ca.pem --tlscert=/certs/server-cert.pem\ --tlskey=/certs/server-key.pem -H tcp://0.0.0.0:2375 --experimental - docker run -t --rm --volumes-from dpy-dind-ssl --env="DOCKER_HOST=tcp://docker:2375"\ - --env="DOCKER_TLS_VERIFY=1" --env="DOCKER_CERT_PATH=/certs" --env="DOCKER_TEST_API_VERSION=${TEST_API_VERSION}"\ - --network dpy-tests docker-sdk-python py.test tests/integration/${file} docker run -t --rm --volumes-from dpy-dind-ssl --env="DOCKER_HOST=tcp://docker:2375"\ --env="DOCKER_TLS_VERIFY=1" --env="DOCKER_CERT_PATH=/certs" --env="DOCKER_TEST_API_VERSION=${TEST_API_VERSION}"\ --network dpy-tests docker-sdk-python3 py.test tests/integration/${file} docker rm -vf dpy-dind-ssl dpy-dind-certs .PHONY: flake8 -flake8: build - docker run -t --rm docker-sdk-python flake8 docker tests +flake8: build-py3 + docker run -t --rm docker-sdk-python3 flake8 docker tests .PHONY: docs docs: build-docs docker run --rm -t -v `pwd`:/src docker-sdk-python-docs sphinx-build docs docs/_build .PHONY: shell -shell: build - docker run -it -v /var/run/docker.sock:/var/run/docker.sock docker-sdk-python python +shell: build-py3 + docker run -it -v /var/run/docker.sock:/var/run/docker.sock docker-sdk-python3 python diff --git a/docker/api/client.py b/docker/api/client.py index 2b67291aa9..ee9ad9c3b4 100644 --- a/docker/api/client.py +++ b/docker/api/client.py @@ -1,10 +1,10 @@ import json import struct +import urllib from functools import partial import requests import requests.exceptions -import six import websocket from .. import auth @@ -192,12 +192,12 @@ def __init__(self, base_url=None, version=None, # version detection needs to be after unix adapter mounting if version is None or (isinstance( version, - six.string_types + str ) and version.lower() == 'auto'): self._version = self._retrieve_server_version() else: self._version = version - if not isinstance(self._version, six.string_types): + if not isinstance(self._version, str): raise DockerException( 'Version parameter must be a string or None. Found {0}'.format( type(version).__name__ @@ -246,13 +246,13 @@ def _delete(self, url, **kwargs): def _url(self, pathfmt, *args, **kwargs): for arg in args: - if not isinstance(arg, six.string_types): + if not isinstance(arg, str): raise ValueError( 'Expected a string but found {0} ({1}) ' 'instead'.format(arg, type(arg)) ) - quote_f = partial(six.moves.urllib.parse.quote, safe="/:") + quote_f = partial(urllib.parse.quote, safe="/:") args = map(quote_f, args) if kwargs.get('versioned_api', True): @@ -284,7 +284,7 @@ def _post_json(self, url, data, **kwargs): # so we do this disgusting thing here. data2 = {} if data is not None and isinstance(data, dict): - for k, v in six.iteritems(data): + for k, v in iter(data.items()): if v is not None: data2[k] = v elif data is not None: @@ -320,12 +320,10 @@ def _get_raw_response_socket(self, response): sock = response.raw._fp.fp.raw.sock elif self.base_url.startswith('http+docker://ssh'): sock = response.raw._fp.fp.channel - elif six.PY3: + else: sock = response.raw._fp.fp.raw if self.base_url.startswith("https://"): sock = sock._sock - else: - sock = response.raw._fp.fp._sock try: # Keep a reference to the response to stop it being garbage # collected. If the response is garbage collected, it will @@ -465,7 +463,7 @@ def _get_result_tty(self, stream, res, is_tty): self._result(res, binary=True) self._raise_for_status(res) - sep = six.binary_type() + sep = b'' if stream: return self._multiplexed_response_stream_helper(res) else: diff --git a/docker/context/context.py b/docker/context/context.py index b1cacf92ae..f4aff6b0d2 100644 --- a/docker/context/context.py +++ b/docker/context/context.py @@ -11,6 +11,7 @@ class Context: """A context.""" + def __init__(self, name, orchestrator=None, host=None, endpoints=None, tls=False): if not name: @@ -128,9 +129,9 @@ def _load_certs(self): key = os.path.join(tls_dir, endpoint, filename) if all([ca_cert, cert, key]): verify = None - if endpoint == "docker": - if not self.endpoints["docker"].get("SkipTLSVerify", False): - verify = True + if endpoint == "docker" and not self.endpoints["docker"].get( + "SkipTLSVerify", False): + verify = True certs[endpoint] = TLSConfig( client_cert=(cert, key), ca_cert=ca_cert, verify=verify) self.tls_cfg = certs diff --git a/docker/transport/sshconn.py b/docker/transport/sshconn.py index a761ef517f..fb5c6bbe8a 100644 --- a/docker/transport/sshconn.py +++ b/docker/transport/sshconn.py @@ -53,7 +53,7 @@ def f(): signal.signal(signal.SIGINT, signal.SIG_IGN) preexec_func = f - env = dict(os.environ) + env = dict(os.environ) # drop LD_LIBRARY_PATH and SSL_CERT_FILE env.pop('LD_LIBRARY_PATH', None) @@ -65,7 +65,7 @@ def f(): shell=True, stdout=subprocess.PIPE, stdin=subprocess.PIPE, - preexec_fn=preexec_func) + preexec_fn=None if constants.IS_WINDOWS_PLATFORM else preexec_func) def _write(self, data): if not self.proc or self.proc.stdin.closed: diff --git a/docker/utils/utils.py b/docker/utils/utils.py index 1b195e2787..f703cbd342 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -7,8 +7,6 @@ from datetime import datetime from distutils.version import StrictVersion -import six - from .. import errors from .. import tls from ..constants import DEFAULT_HTTP_HOST @@ -16,11 +14,7 @@ from ..constants import DEFAULT_NPIPE from ..constants import BYTE_UNITS -if six.PY2: - from urllib import splitnport - from urlparse import urlparse -else: - from urllib.parse import splitnport, urlparse +from urllib.parse import splitnport, urlparse def create_ipam_pool(*args, **kwargs): @@ -39,8 +33,7 @@ def create_ipam_config(*args, **kwargs): def decode_json_header(header): data = base64.b64decode(header) - if six.PY3: - data = data.decode('utf-8') + data = data.decode('utf-8') return json.loads(data) @@ -80,7 +73,7 @@ def _convert_port_binding(binding): if len(binding) == 2: result['HostPort'] = binding[1] result['HostIp'] = binding[0] - elif isinstance(binding[0], six.string_types): + elif isinstance(binding[0], str): result['HostIp'] = binding[0] else: result['HostPort'] = binding[0] @@ -104,7 +97,7 @@ def _convert_port_binding(binding): def convert_port_bindings(port_bindings): result = {} - for k, v in six.iteritems(port_bindings): + for k, v in iter(port_bindings.items()): key = str(k) if '/' not in key: key += '/tcp' @@ -121,7 +114,7 @@ def convert_volume_binds(binds): result = [] for k, v in binds.items(): - if isinstance(k, six.binary_type): + if isinstance(k, bytes): k = k.decode('utf-8') if isinstance(v, dict): @@ -132,7 +125,7 @@ def convert_volume_binds(binds): ) bind = v['bind'] - if isinstance(bind, six.binary_type): + if isinstance(bind, bytes): bind = bind.decode('utf-8') if 'ro' in v: @@ -143,13 +136,13 @@ def convert_volume_binds(binds): mode = 'rw' result.append( - six.text_type('{0}:{1}:{2}').format(k, bind, mode) + str('{0}:{1}:{2}').format(k, bind, mode) ) else: - if isinstance(v, six.binary_type): + if isinstance(v, bytes): v = v.decode('utf-8') result.append( - six.text_type('{0}:{1}:rw').format(k, v) + str('{0}:{1}:rw').format(k, v) ) return result @@ -166,7 +159,7 @@ def convert_tmpfs_mounts(tmpfs): result = {} for mount in tmpfs: - if isinstance(mount, six.string_types): + if isinstance(mount, str): if ":" in mount: name, options = mount.split(":", 1) else: @@ -191,7 +184,7 @@ def convert_service_networks(networks): result = [] for n in networks: - if isinstance(n, six.string_types): + if isinstance(n, str): n = {'Target': n} result.append(n) return result @@ -302,7 +295,7 @@ def parse_devices(devices): if isinstance(device, dict): device_list.append(device) continue - if not isinstance(device, six.string_types): + if not isinstance(device, str): raise errors.DockerException( 'Invalid device type {0}'.format(type(device)) ) @@ -372,13 +365,13 @@ def kwargs_from_env(ssl_version=None, assert_hostname=None, environment=None): def convert_filters(filters): result = {} - for k, v in six.iteritems(filters): + for k, v in iter(filters.items()): if isinstance(v, bool): v = 'true' if v else 'false' if not isinstance(v, list): v = [v, ] result[k] = [ - str(item) if not isinstance(item, six.string_types) else item + str(item) if not isinstance(item, str) else item for item in v ] return json.dumps(result) @@ -391,7 +384,7 @@ def datetime_to_timestamp(dt): def parse_bytes(s): - if isinstance(s, six.integer_types + (float,)): + if isinstance(s, (int, float,)): return s if len(s) == 0: return 0 @@ -433,7 +426,7 @@ def parse_bytes(s): def normalize_links(links): if isinstance(links, dict): - links = six.iteritems(links) + links = iter(links.items()) return ['{0}:{1}'.format(k, v) if v else k for k, v in sorted(links)] @@ -468,8 +461,6 @@ def parse_env_file(env_file): def split_command(command): - if six.PY2 and not isinstance(command, six.binary_type): - command = command.encode('utf-8') return shlex.split(command) @@ -477,22 +468,22 @@ def format_environment(environment): def format_env(key, value): if value is None: return key - if isinstance(value, six.binary_type): + if isinstance(value, bytes): value = value.decode('utf-8') return u'{key}={value}'.format(key=key, value=value) - return [format_env(*var) for var in six.iteritems(environment)] + return [format_env(*var) for var in iter(environment.items())] def format_extra_hosts(extra_hosts, task=False): # Use format dictated by Swarm API if container is part of a task if task: return [ - '{} {}'.format(v, k) for k, v in sorted(six.iteritems(extra_hosts)) + '{} {}'.format(v, k) for k, v in sorted(iter(extra_hosts.items())) ] return [ - '{}:{}'.format(k, v) for k, v in sorted(six.iteritems(extra_hosts)) + '{}:{}'.format(k, v) for k, v in sorted(iter(extra_hosts.items())) ] diff --git a/requirements.txt b/requirements.txt index 43a688fd90..f86a7bd7fe 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,6 +13,5 @@ pyOpenSSL==18.0.0 pyparsing==2.2.0 pywin32==227; sys_platform == 'win32' requests==2.20.0 -six==1.10.0 urllib3==1.24.3 websocket-client==0.56.0 diff --git a/setup.py b/setup.py index 330ab3e357..b86016effc 100644 --- a/setup.py +++ b/setup.py @@ -11,18 +11,11 @@ SOURCE_DIR = os.path.join(ROOT_DIR) requirements = [ - 'six >= 1.4.0', 'websocket-client >= 0.32.0', 'requests >= 2.14.2, != 2.18.0', ] extras_require = { - ':python_version < "3.5"': 'backports.ssl_match_hostname >= 3.5', - # While not imported explicitly, the ipaddress module is required for - # ssl_match_hostname to verify hosts match with certificates via - # ServerAltname: https://pypi.python.org/pypi/backports.ssl_match_hostname - ':python_version < "3.3"': 'ipaddress >= 1.0.16', - # win32 APIs if on Windows (required for npipe support) ':sys_platform == "win32"': 'pywin32==227', @@ -69,7 +62,7 @@ install_requires=requirements, tests_require=test_requirements, extras_require=extras_require, - python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*', + python_requires='>=3.6', zip_safe=False, test_suite='tests', classifiers=[ @@ -78,10 +71,7 @@ 'Intended Audience :: Developers', 'Operating System :: OS Independent', 'Programming Language :: Python', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', diff --git a/test-requirements.txt b/test-requirements.txt index 24078e27a8..40161bb8ec 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,4 +1,4 @@ -setuptools==44.0.0 # last version with python 2.7 support +setuptools==54.1.1 coverage==4.5.2 flake8==3.6.0 mock==1.0.1 diff --git a/tests/Dockerfile-dind-certs b/tests/Dockerfile-dind-certs index 2ab87ef732..8829ff7946 100644 --- a/tests/Dockerfile-dind-certs +++ b/tests/Dockerfile-dind-certs @@ -1,4 +1,4 @@ -ARG PYTHON_VERSION=2.7 +ARG PYTHON_VERSION=3.6 FROM python:${PYTHON_VERSION} RUN mkdir /tmp/certs diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index 65e611b2f5..3087045b20 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -7,7 +7,6 @@ import pytest import requests -import six import docker from .. import helpers @@ -35,7 +34,7 @@ def test_list_containers(self): assert len(retrieved) == 1 retrieved = retrieved[0] assert 'Command' in retrieved - assert retrieved['Command'] == six.text_type('true') + assert retrieved['Command'] == str('true') assert 'Image' in retrieved assert re.search(r'alpine:.*', retrieved['Image']) assert 'Status' in retrieved @@ -104,9 +103,7 @@ def test_create_with_links(self): self.client.start(container3_id) assert self.client.wait(container3_id)['StatusCode'] == 0 - logs = self.client.logs(container3_id) - if six.PY3: - logs = logs.decode('utf-8') + logs = self.client.logs(container3_id).decode('utf-8') assert '{0}_NAME='.format(link_env_prefix1) in logs assert '{0}_ENV_FOO=1'.format(link_env_prefix1) in logs assert '{0}_NAME='.format(link_env_prefix2) in logs @@ -227,9 +224,7 @@ def test_group_id_ints(self): self.client.start(container) self.client.wait(container) - logs = self.client.logs(container) - if six.PY3: - logs = logs.decode('utf-8') + logs = self.client.logs(container).decode('utf-8') groups = logs.strip().split(' ') assert '1000' in groups assert '1001' in groups @@ -244,9 +239,7 @@ def test_group_id_strings(self): self.client.start(container) self.client.wait(container) - logs = self.client.logs(container) - if six.PY3: - logs = logs.decode('utf-8') + logs = self.client.logs(container).decode('utf-8') groups = logs.strip().split(' ') assert '1000' in groups @@ -515,10 +508,7 @@ def test_create_with_binds_rw(self): TEST_IMG, ['ls', self.mount_dest], ) - logs = self.client.logs(container) - - if six.PY3: - logs = logs.decode('utf-8') + logs = self.client.logs(container).decode('utf-8') assert self.filename in logs inspect_data = self.client.inspect_container(container) self.check_container_data(inspect_data, True) @@ -534,10 +524,8 @@ def test_create_with_binds_ro(self): TEST_IMG, ['ls', self.mount_dest], ) - logs = self.client.logs(container) + logs = self.client.logs(container).decode('utf-8') - if six.PY3: - logs = logs.decode('utf-8') assert self.filename in logs inspect_data = self.client.inspect_container(container) @@ -554,9 +542,7 @@ def test_create_with_mounts(self): host_config=host_config ) assert container - logs = self.client.logs(container) - if six.PY3: - logs = logs.decode('utf-8') + logs = self.client.logs(container).decode('utf-8') assert self.filename in logs inspect_data = self.client.inspect_container(container) self.check_container_data(inspect_data, True) @@ -573,9 +559,7 @@ def test_create_with_mounts_ro(self): host_config=host_config ) assert container - logs = self.client.logs(container) - if six.PY3: - logs = logs.decode('utf-8') + logs = self.client.logs(container).decode('utf-8') assert self.filename in logs inspect_data = self.client.inspect_container(container) self.check_container_data(inspect_data, False) @@ -645,9 +629,8 @@ def test_get_file_archive_from_container(self): for d in strm: destination.write(d) destination.seek(0) - retrieved_data = helpers.untar_file(destination, 'data.txt') - if six.PY3: - retrieved_data = retrieved_data.decode('utf-8') + retrieved_data = helpers.untar_file(destination, 'data.txt')\ + .decode('utf-8') assert data == retrieved_data.strip() def test_get_file_stat_from_container(self): @@ -683,9 +666,6 @@ def test_copy_file_to_container(self): self.client.start(ctnr) self.client.wait(ctnr) logs = self.client.logs(ctnr) - if six.PY3: - logs = logs.decode('utf-8') - data = data.decode('utf-8') assert logs.strip() == data def test_copy_directory_to_container(self): @@ -700,9 +680,7 @@ def test_copy_directory_to_container(self): self.client.put_archive(ctnr, '/vol1', test_tar) self.client.start(ctnr) self.client.wait(ctnr) - logs = self.client.logs(ctnr) - if six.PY3: - logs = logs.decode('utf-8') + logs = self.client.logs(ctnr).decode('utf-8') results = logs.strip().split() assert 'a.py' in results assert 'b.py' in results @@ -861,7 +839,7 @@ def test_logs_streaming_and_follow(self): id = container['Id'] self.tmp_containers.append(id) self.client.start(id) - logs = six.binary_type() + logs = b'' for chunk in self.client.logs(id, stream=True, follow=True): logs += chunk @@ -881,7 +859,7 @@ def test_logs_streaming_and_follow_and_cancel(self): id = container['Id'] self.tmp_containers.append(id) self.client.start(id) - logs = six.binary_type() + logs = b'' generator = self.client.logs(id, stream=True, follow=True) threading.Timer(1, generator.close).start() diff --git a/tests/integration/api_image_test.py b/tests/integration/api_image_test.py index 37e26a3fd2..d5f8989304 100644 --- a/tests/integration/api_image_test.py +++ b/tests/integration/api_image_test.py @@ -7,9 +7,8 @@ import threading import pytest -import six -from six.moves import BaseHTTPServer -from six.moves import socketserver +from http.server import SimpleHTTPRequestHandler +import socketserver import docker @@ -33,7 +32,7 @@ def test_images(self): def test_images_quiet(self): res1 = self.client.images(quiet=True) - assert type(res1[0]) == six.text_type + assert type(res1[0]) == str class PullImageTest(BaseAPIIntegrationTest): @@ -44,7 +43,7 @@ def test_pull(self): pass res = self.client.pull('hello-world') self.tmp_imgs.append('hello-world') - assert type(res) == six.text_type + assert type(res) == str assert len(self.client.images('hello-world')) >= 1 img_info = self.client.inspect_image('hello-world') assert 'Id' in img_info @@ -273,7 +272,7 @@ def test_get_load_image(self): def temporary_http_file_server(self, stream): '''Serve data from an IO stream over HTTP.''' - class Handler(BaseHTTPServer.BaseHTTPRequestHandler): + class Handler(SimpleHTTPRequestHandler): def do_GET(self): self.send_response(200) self.send_header('Content-Type', 'application/x-tar') diff --git a/tests/integration/api_service_test.py b/tests/integration/api_service_test.py index 7e5336e2e5..1bee46e563 100644 --- a/tests/integration/api_service_test.py +++ b/tests/integration/api_service_test.py @@ -5,7 +5,6 @@ import docker import pytest -import six from ..helpers import ( force_leave_swarm, requires_api_version, requires_experimental @@ -150,7 +149,7 @@ def test_service_logs(self): else: break - if six.PY3: + if log_line is not None: log_line = log_line.decode('utf-8') assert 'hello\n' in log_line diff --git a/tests/unit/sshadapter_test.py b/tests/unit/sshadapter_test.py index ddee592029..874239ac8d 100644 --- a/tests/unit/sshadapter_test.py +++ b/tests/unit/sshadapter_test.py @@ -2,31 +2,38 @@ import docker from docker.transport.sshconn import SSHSocket + class SSHAdapterTest(unittest.TestCase): - def test_ssh_hostname_prefix_trim(self): - conn = docker.transport.SSHHTTPAdapter(base_url="ssh://user@hostname:1234", shell_out=True) + @staticmethod + def test_ssh_hostname_prefix_trim(): + conn = docker.transport.SSHHTTPAdapter( + base_url="ssh://user@hostname:1234", shell_out=True) assert conn.ssh_host == "user@hostname:1234" - def test_ssh_parse_url(self): + @staticmethod + def test_ssh_parse_url(): c = SSHSocket(host="user@hostname:1234") assert c.host == "hostname" assert c.port == "1234" assert c.user == "user" - def test_ssh_parse_hostname_only(self): + @staticmethod + def test_ssh_parse_hostname_only(): c = SSHSocket(host="hostname") assert c.host == "hostname" - assert c.port == None - assert c.user == None + assert c.port is None + assert c.user is None - def test_ssh_parse_user_and_hostname(self): + @staticmethod + def test_ssh_parse_user_and_hostname(): c = SSHSocket(host="user@hostname") assert c.host == "hostname" - assert c.port == None + assert c.port is None assert c.user == "user" - def test_ssh_parse_hostname_and_port(self): + @staticmethod + def test_ssh_parse_hostname_and_port(): c = SSHSocket(host="hostname:22") assert c.host == "hostname" assert c.port == "22" - assert c.user == None \ No newline at end of file + assert c.user is None diff --git a/tox.ini b/tox.ini index df797f4113..d35d41ae89 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27, py35, py36, py37, flake8 +envlist = py36, py37, flake8 skipsdist=True [testenv] From d4310b2db0e5fe5cb987a454f7e97c6e388a470e Mon Sep 17 00:00:00 2001 From: Felipe Ruhland Date: Wed, 24 Mar 2021 17:59:47 +0100 Subject: [PATCH 1107/1301] Fix `KeyError` when creating a new secret How to reproduce the issue: ```py >>> import docker >>> cli = docker.from_env() >>> cli.secrets.create(name="any_name", data="1") Traceback (most recent call last): File "", line 1, in File "/home/docker-py/docker/models/secrets.py", line 10, in __repr__ return "<%s: '%s'>" % (self.__class__.__name__, self.name) File "/home/docker-py/docker/models/secrets.py", line 14, in name return self.attrs['Spec']['Name'] KeyError: 'Spec' ``` The exception raises because create secrets API `/secrets/create` only return the `id` attribute: https://docs.docker.com/engine/api/v1.41/#operation/SecretCreate The secret model is created using just the `id` attribute and fails when looking for Spec.Name attribute. ```py def __repr__(self): return "<%s: '%s'>" % (self.__class__.__name__, self.name) ``` ```py @property def name(self): return self.attrs['Spec']['Name'] ``` I came up with a ugly solution but will prevent the problem to happen again: ```py def create(self, **kwargs): obj = self.client.api.create_secret(**kwargs) + obj.setdefault("Spec", {})["Name"] = kwargs.get("name") return self.prepare_model(obj) ``` After the API call, I added the name attribute to the right place to be used on the property name. ```py >>> import docker >>> cli = docker.from_env() >>> cli.secrets.create(name="any_name", data="1") ``` It isn't the most elegant solution, but it will do the trick. I had a previous PR #2517 when I propose using the `id` attribute instead of `name` on the `__repr__` method, but I think this one will be better. That fixes #2025 Signed-off-by: Felipe Ruhland --- docker/models/secrets.py | 1 + tests/unit/fake_api.py | 10 ++++++++++ tests/unit/fake_api_client.py | 1 + tests/unit/models_secrets_test.py | 11 +++++++++++ 4 files changed, 23 insertions(+) create mode 100644 tests/unit/models_secrets_test.py diff --git a/docker/models/secrets.py b/docker/models/secrets.py index ca11edeb08..e2ee88af02 100644 --- a/docker/models/secrets.py +++ b/docker/models/secrets.py @@ -30,6 +30,7 @@ class SecretCollection(Collection): def create(self, **kwargs): obj = self.client.api.create_secret(**kwargs) + obj.setdefault("Spec", {})["Name"] = kwargs.get("name") return self.prepare_model(obj) create.__doc__ = APIClient.create_secret.__doc__ diff --git a/tests/unit/fake_api.py b/tests/unit/fake_api.py index 27e463d27e..4fd4d11381 100644 --- a/tests/unit/fake_api.py +++ b/tests/unit/fake_api.py @@ -17,6 +17,8 @@ FAKE_PATH = '/path' FAKE_VOLUME_NAME = 'perfectcherryblossom' FAKE_NODE_ID = '24ifsmvkjbyhk' +FAKE_SECRET_ID = 'epdyrw4tsi03xy3deu8g8ly6o' +FAKE_SECRET_NAME = 'super_secret' # Each method is prefixed with HTTP method (get, post...) # for clarity and readability @@ -512,6 +514,12 @@ def post_fake_network_disconnect(): return 200, None +def post_fake_secret(): + status_code = 200 + response = {'ID': FAKE_SECRET_ID} + return status_code, response + + # Maps real api url to fake response callback prefix = 'http+docker://localhost' if constants.IS_WINDOWS_PLATFORM: @@ -643,4 +651,6 @@ def post_fake_network_disconnect(): CURRENT_VERSION, prefix, FAKE_NETWORK_ID ), 'POST'): post_fake_network_disconnect, + '{1}/{0}/secrets/create'.format(CURRENT_VERSION, prefix): + post_fake_secret, } diff --git a/tests/unit/fake_api_client.py b/tests/unit/fake_api_client.py index e85001dbbb..5825b6ec00 100644 --- a/tests/unit/fake_api_client.py +++ b/tests/unit/fake_api_client.py @@ -40,6 +40,7 @@ def make_fake_api_client(overrides=None): fake_api.post_fake_create_container()[1], 'create_host_config.side_effect': api_client.create_host_config, 'create_network.return_value': fake_api.post_fake_network()[1], + 'create_secret.return_value': fake_api.post_fake_secret()[1], 'exec_create.return_value': fake_api.post_fake_exec_create()[1], 'exec_start.return_value': fake_api.post_fake_exec_start()[1], 'images.return_value': fake_api.get_fake_images()[1], diff --git a/tests/unit/models_secrets_test.py b/tests/unit/models_secrets_test.py new file mode 100644 index 0000000000..4ccf4c6385 --- /dev/null +++ b/tests/unit/models_secrets_test.py @@ -0,0 +1,11 @@ +import unittest + +from .fake_api_client import make_fake_client +from .fake_api import FAKE_SECRET_NAME + + +class CreateServiceTest(unittest.TestCase): + def test_secrets_repr(self): + client = make_fake_client() + secret = client.secrets.create(name="super_secret", data="secret") + assert secret.__repr__() == "".format(FAKE_SECRET_NAME) From 2403774e76b279ecdd7238d35f04def6fb1ca8b8 Mon Sep 17 00:00:00 2001 From: Felipe Ruhland Date: Fri, 2 Apr 2021 02:30:22 +0200 Subject: [PATCH 1108/1301] Upgrade cryptography library to version 3.4.7 Dependabot opened a pull request 93bcc0497d8302aa2d78bd7ef756fc2ff3fd0912 to upgrade cryptography from 2.3 to 3.2. However, only `requirements.txt` was updated. The extra requirements were kept outdated. This commit was made to update the library to the last version. Fix #2791 Signed-off-by: Felipe Ruhland --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index f86a7bd7fe..1d0be30a16 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ appdirs==1.4.3 asn1crypto==0.22.0 backports.ssl-match-hostname==3.5.0.1 cffi==1.14.4 -cryptography==3.2 +cryptography==3.4.7 enum34==1.1.6 idna==2.5 ipaddress==1.0.18 diff --git a/setup.py b/setup.py index b86016effc..b692eabd8e 100644 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ # https://github.com/pypa/pip/issues/4391). Once that's fixed, instead of # installing the extra dependencies, install the following instead: # 'requests[security] >= 2.5.2, != 2.11.0, != 2.12.2' - 'tls': ['pyOpenSSL>=17.5.0', 'cryptography>=1.3.4', 'idna>=2.0.0'], + 'tls': ['pyOpenSSL>=17.5.0', 'cryptography>=3.4.7', 'idna>=2.0.0'], # Only required when connecting using the ssh:// protocol 'ssh': ['paramiko>=2.4.2'], From a34dd8b1a987bfd98882197c909a24a963b68a8f Mon Sep 17 00:00:00 2001 From: Felipe Ruhland Date: Mon, 5 Apr 2021 14:57:52 +0200 Subject: [PATCH 1109/1301] Fix images low-level documentation examples I realize that the documentation of low-level `images` was outdated when answering issue #2798 The issue can reproduce it with a simple test: ```py In [1]: import docker In [2]: client = docker.from_env() In [3]: client.pull --------------------------------------------------------------------------- AttributeError Traceback (most recent call last) in ----> 1 client.pull ~/docker-py/docker/client.py in __getattr__(self, name) 219 "object APIClient. See the low-level API section of the " 220 "documentation for more details.") --> 221 raise AttributeError(' '.join(s)) 222 223 AttributeError: 'DockerClient' object has no attribute 'pull' In Docker SDK for Python 2.0, this method is now on the object APIClient. See the low-level API section of the documentation for more details. In [4]: client.push --------------------------------------------------------------------------- AttributeError Traceback (most recent call last) in ----> 1 client.push ~/docker-py/docker/client.py in __getattr__(self, name) 219 "object APIClient. See the low-level API section of the " 220 "documentation for more details.") --> 221 raise AttributeError(' '.join(s)) 222 223 AttributeError: 'DockerClient' object has no attribute 'push' In Docker SDK for Python 2.0, this method is now on the object APIClient. See the low-level API section of the documentation for more details. In [5]: client.tag --------------------------------------------------------------------------- AttributeError Traceback (most recent call last) in ----> 1 client.tag ~/docker-py/docker/client.py in __getattr__(self, name) 219 "object APIClient. See the low-level API section of the " 220 "documentation for more details.") --> 221 raise AttributeError(' '.join(s)) 222 223 AttributeError: 'DockerClient' object has no attribute 'tag' In Docker SDK for Python 2.0, this method is now on the object APIClient. See the low-level API section of the documentation for more details. In [6]: client.get_image --------------------------------------------------------------------------- AttributeError Traceback (most recent call last) in ----> 1 client.get_image ~/docker-py/docker/client.py in __getattr__(self, name) 219 "object APIClient. See the low-level API section of the " 220 "documentation for more details.") --> 221 raise AttributeError(' '.join(s)) 222 223 AttributeError: 'DockerClient' object has no attribute 'get_image' In Docker SDK for Python 2.0, this method is now on the object APIClient. See the low-level API section of the documentation for more details. In [7]: client.api.get_image Out[7]: > In [8]: client.api.tag Out[8]: > In [9]: client.api.pull Out[9]: > In [10]: client.api.push Out[10]: > ``` Signed-off-by: Felipe Ruhland --- docker/api/image.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docker/api/image.py b/docker/api/image.py index 56c5448eb3..db806c4925 100644 --- a/docker/api/image.py +++ b/docker/api/image.py @@ -31,7 +31,7 @@ def get_image(self, image, chunk_size=DEFAULT_DATA_CHUNK_SIZE): Example: - >>> image = cli.get_image("busybox:latest") + >>> image = client.api.get_image("busybox:latest") >>> f = open('/tmp/busybox-latest.tar', 'wb') >>> for chunk in image: >>> f.write(chunk) @@ -379,7 +379,7 @@ def pull(self, repository, tag=None, stream=False, auth_config=None, Example: - >>> for line in cli.pull('busybox', stream=True, decode=True): + >>> for line in client.api.pull('busybox', stream=True, decode=True): ... print(json.dumps(line, indent=4)) { "status": "Pulling image (latest) from busybox", @@ -458,7 +458,7 @@ def push(self, repository, tag=None, stream=False, auth_config=None, If the server returns an error. Example: - >>> for line in cli.push('yourname/app', stream=True, decode=True): + >>> for line in client.api.push('yourname/app', stream=True, decode=True): ... print(line) {'status': 'Pushing repository yourname/app (1 tags)'} {'status': 'Pushing','progressDetail': {}, 'id': '511136ea3c5a'} @@ -549,7 +549,7 @@ def tag(self, image, repository, tag=None, force=False): Example: - >>> client.tag('ubuntu', 'localhost:5000/ubuntu', 'latest', + >>> client.api.tag('ubuntu', 'localhost:5000/ubuntu', 'latest', force=True) """ params = { From ac9ae1f249c7635b3fd12b12d293f1c9848aaef5 Mon Sep 17 00:00:00 2001 From: Felipe Ruhland Date: Mon, 5 Apr 2021 15:31:43 +0200 Subject: [PATCH 1110/1301] Fix containers low-level documentation examples I realize that low-level documentation has outdated examples, so I created issue #2800 to fix that Signed-off-by: Felipe Ruhland --- docker/api/container.py | 50 ++++++++++++++++++++--------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/docker/api/container.py b/docker/api/container.py index 754b5dc6a4..369eba952f 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -244,9 +244,9 @@ def create_container(self, image, command=None, hostname=None, user=None, .. code-block:: python - container_id = cli.create_container( + container_id = client.api.create_container( 'busybox', 'ls', ports=[1111, 2222], - host_config=cli.create_host_config(port_bindings={ + host_config=client.api.create_host_config(port_bindings={ 1111: 4567, 2222: None }) @@ -258,22 +258,22 @@ def create_container(self, image, command=None, hostname=None, user=None, .. code-block:: python - cli.create_host_config(port_bindings={1111: ('127.0.0.1', 4567)}) + client.api.create_host_config(port_bindings={1111: ('127.0.0.1', 4567)}) Or without host port assignment: .. code-block:: python - cli.create_host_config(port_bindings={1111: ('127.0.0.1',)}) + client.api.create_host_config(port_bindings={1111: ('127.0.0.1',)}) If you wish to use UDP instead of TCP (default), you need to declare ports as such in both the config and host config: .. code-block:: python - container_id = cli.create_container( + container_id = client.api.create_container( 'busybox', 'ls', ports=[(1111, 'udp'), 2222], - host_config=cli.create_host_config(port_bindings={ + host_config=client.api.create_host_config(port_bindings={ '1111/udp': 4567, 2222: None }) ) @@ -283,7 +283,7 @@ def create_container(self, image, command=None, hostname=None, user=None, .. code-block:: python - cli.create_host_config(port_bindings={ + client.api.create_host_config(port_bindings={ 1111: [1234, 4567] }) @@ -291,7 +291,7 @@ def create_container(self, image, command=None, hostname=None, user=None, .. code-block:: python - cli.create_host_config(port_bindings={ + client.api.create_host_config(port_bindings={ 1111: [ ('192.168.0.100', 1234), ('192.168.0.101', 1234) @@ -307,9 +307,9 @@ def create_container(self, image, command=None, hostname=None, user=None, .. code-block:: python - container_id = cli.create_container( + container_id = client.api.create_container( 'busybox', 'ls', volumes=['/mnt/vol1', '/mnt/vol2'], - host_config=cli.create_host_config(binds={ + host_config=client.api.create_host_config(binds={ '/home/user1/': { 'bind': '/mnt/vol2', 'mode': 'rw', @@ -326,9 +326,9 @@ def create_container(self, image, command=None, hostname=None, user=None, .. code-block:: python - container_id = cli.create_container( + container_id = client.api.create_container( 'busybox', 'ls', volumes=['/mnt/vol1', '/mnt/vol2'], - host_config=cli.create_host_config(binds=[ + host_config=client.api.create_host_config(binds=[ '/home/user1/:/mnt/vol2', '/var/www:/mnt/vol1:ro', ]) @@ -346,15 +346,15 @@ def create_container(self, image, command=None, hostname=None, user=None, .. code-block:: python - networking_config = docker_client.create_networking_config({ - 'network1': docker_client.create_endpoint_config( + networking_config = client.api.create_networking_config({ + 'network1': client.api.create_endpoint_config( ipv4_address='172.28.0.124', aliases=['foo', 'bar'], links=['container2'] ) }) - ctnr = docker_client.create_container( + ctnr = client.api.create_container( img, command, networking_config=networking_config ) @@ -581,7 +581,7 @@ def create_host_config(self, *args, **kwargs): Example: - >>> cli.create_host_config(privileged=True, cap_drop=['MKNOD'], + >>> client.api.create_host_config(privileged=True, cap_drop=['MKNOD'], volumes_from=['nostalgic_newton']) {'CapDrop': ['MKNOD'], 'LxcConf': None, 'Privileged': True, 'VolumesFrom': ['nostalgic_newton'], 'PublishAllPorts': False} @@ -612,11 +612,11 @@ def create_networking_config(self, *args, **kwargs): Example: - >>> docker_client.create_network('network1') - >>> networking_config = docker_client.create_networking_config({ - 'network1': docker_client.create_endpoint_config() + >>> client.api.create_network('network1') + >>> networking_config = client.api.create_networking_config({ + 'network1': client.api.create_endpoint_config() }) - >>> container = docker_client.create_container( + >>> container = client.api.create_container( img, command, networking_config=networking_config ) @@ -650,7 +650,7 @@ def create_endpoint_config(self, *args, **kwargs): Example: - >>> endpoint_config = client.create_endpoint_config( + >>> endpoint_config = client.api.create_endpoint_config( aliases=['web', 'app'], links={'app_db': 'db', 'another': None}, ipv4_address='132.65.0.123' @@ -729,7 +729,7 @@ def get_archive(self, container, path, chunk_size=DEFAULT_DATA_CHUNK_SIZE, >>> c = docker.APIClient() >>> f = open('./sh_bin.tar', 'wb') - >>> bits, stat = c.get_archive(container, '/bin/sh') + >>> bits, stat = c.api.get_archive(container, '/bin/sh') >>> print(stat) {'name': 'sh', 'size': 1075464, 'mode': 493, 'mtime': '2018-10-01T15:37:48-07:00', 'linkTarget': ''} @@ -916,7 +916,7 @@ def port(self, container, private_port): .. code-block:: python - >>> cli.port('7174d6347063', 80) + >>> client.api.port('7174d6347063', 80) [{'HostIp': '0.0.0.0', 'HostPort': '80'}] """ res = self._get(self._url("/containers/{0}/json", container)) @@ -1095,10 +1095,10 @@ def start(self, container, *args, **kwargs): Example: - >>> container = cli.create_container( + >>> container = client.api.create_container( ... image='busybox:latest', ... command='/bin/sleep 30') - >>> cli.start(container=container.get('Id')) + >>> client.api.start(container=container.get('Id')) """ if args or kwargs: raise errors.DeprecatedMethod( From 8945fda6be44f99cdbd68e5912eca4dffbb13acc Mon Sep 17 00:00:00 2001 From: Anca Iordache Date: Tue, 6 Apr 2021 16:01:16 +0200 Subject: [PATCH 1111/1301] Update maintainers Signed-off-by: Anca Iordache --- MAINTAINERS | 14 +++++++++++++- setup.py | 4 ++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/MAINTAINERS b/MAINTAINERS index b857d13dc0..b74cb28fd3 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -11,7 +11,8 @@ [Org] [Org."Core maintainers"] people = [ - "shin-", + "aiordache", + "ulyssessouza", ] [Org.Alumni] people = [ @@ -20,6 +21,7 @@ "dnephin", "mnowster", "mpetazzoni", + "shin-", ] [people] @@ -35,6 +37,11 @@ Email = "aanand@docker.com" GitHub = "aanand" + [people.aiordache] + Name = "Anca Iordache" + Email = "anca.iordache@docker.com" + GitHub = "aiordache" + [people.bfirsh] Name = "Ben Firshman" Email = "b@fir.sh" @@ -59,3 +66,8 @@ Name = "Joffrey F" Email = "joffrey@docker.com" GitHub = "shin-" + + [people.ulyssessouza] + Name = "Ulysses Domiciano Souza" + Email = "ulysses.souza@docker.com" + GitHub = "ulyssessouza" diff --git a/setup.py b/setup.py index b692eabd8e..ec1a51deb8 100644 --- a/setup.py +++ b/setup.py @@ -80,6 +80,6 @@ 'Topic :: Utilities', 'License :: OSI Approved :: Apache Software License', ], - maintainer='Joffrey F', - maintainer_email='joffrey@docker.com', + maintainer='Ulysses Souza', + maintainer_email='ulysses.souza@docker.com', ) From 4b44fa7e5db98af52fb9269422d05b9aa7e03f5c Mon Sep 17 00:00:00 2001 From: Felipe Ruhland Date: Tue, 6 Apr 2021 20:32:05 +0200 Subject: [PATCH 1112/1301] Fix volumes low-level documentation examples I realize that low-level documentation has outdated examples, so I created issue #2800 to fix that Signed-off-by: Felipe Ruhland --- docker/api/volume.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/api/volume.py b/docker/api/volume.py index 900a6086b5..9604554a32 100644 --- a/docker/api/volume.py +++ b/docker/api/volume.py @@ -21,7 +21,7 @@ def volumes(self, filters=None): Example: - >>> cli.volumes() + >>> client.api.volumes() {u'Volumes': [{u'Driver': u'local', u'Mountpoint': u'/var/lib/docker/volumes/foobar/_data', u'Name': u'foobar'}, @@ -56,7 +56,7 @@ def create_volume(self, name=None, driver=None, driver_opts=None, Example: - >>> volume = cli.create_volume(name='foobar', driver='local', + >>> volume = client.api.create_volume(name='foobar', driver='local', driver_opts={'foo': 'bar', 'baz': 'false'}, labels={"key": "value"}) >>> print(volume) @@ -104,7 +104,7 @@ def inspect_volume(self, name): Example: - >>> cli.inspect_volume('foobar') + >>> client.api.inspect_volume('foobar') {u'Driver': u'local', u'Mountpoint': u'/var/lib/docker/volumes/foobar/_data', u'Name': u'foobar'} From 50a0ff596fde9cea5acb5250a07de16ca584d0a1 Mon Sep 17 00:00:00 2001 From: Felipe Ruhland Date: Tue, 6 Apr 2021 20:27:07 +0200 Subject: [PATCH 1113/1301] Fix network low-level documentation examples I realize that low-level documentation has outdated examples, so I created issue #2800 to fix that Signed-off-by: Felipe Ruhland --- docker/api/network.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/api/network.py b/docker/api/network.py index 139c2d1a82..18419932ff 100644 --- a/docker/api/network.py +++ b/docker/api/network.py @@ -75,7 +75,7 @@ def create_network(self, name, driver=None, options=None, ipam=None, Example: A network using the bridge driver: - >>> client.create_network("network1", driver="bridge") + >>> client.api.create_network("network1", driver="bridge") You can also create more advanced networks with custom IPAM configurations. For example, setting the subnet to @@ -90,7 +90,7 @@ def create_network(self, name, driver=None, options=None, ipam=None, >>> ipam_config = docker.types.IPAMConfig( pool_configs=[ipam_pool] ) - >>> docker_client.create_network("network1", driver="bridge", + >>> client.api.create_network("network1", driver="bridge", ipam=ipam_config) """ if options is not None and not isinstance(options, dict): From f53e615e0fd4b7becf9d72c73b8a9e021d473f62 Mon Sep 17 00:00:00 2001 From: Felipe Ruhland Date: Wed, 7 Apr 2021 21:44:24 +0200 Subject: [PATCH 1114/1301] Update API and Engine versions The Makefile and `docker/constants.py` were with old versions, so I updated them to the current one Signed-off-by: Felipe Ruhland --- Makefile | 4 ++-- docker/constants.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 60d99842c3..78a0d334e2 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ -TEST_API_VERSION ?= 1.39 -TEST_ENGINE_VERSION ?= 19.03.13 +TEST_API_VERSION ?= 1.41 +TEST_ENGINE_VERSION ?= 20.10.05 .PHONY: all all: test diff --git a/docker/constants.py b/docker/constants.py index 43fce6138e..9cd58b6756 100644 --- a/docker/constants.py +++ b/docker/constants.py @@ -1,7 +1,7 @@ import sys from .version import version -DEFAULT_DOCKER_API_VERSION = '1.39' +DEFAULT_DOCKER_API_VERSION = '1.41' MINIMUM_DOCKER_API_VERSION = '1.21' DEFAULT_TIMEOUT_SECONDS = 60 STREAM_HEADER_SIZE_BYTES = 8 From 7ac8b56730c70e3b61ad628e7512082a4468e4f3 Mon Sep 17 00:00:00 2001 From: Felipe Ruhland Date: Wed, 7 Apr 2021 22:11:52 +0200 Subject: [PATCH 1115/1301] Add `CapabilityAdd` and `CapabilityDrop` to ContainerSpec Docker Engine v1.41 added `CapAdd` and `CapDrop` as part of the ContainerSpec, and `docker-py` should do the same. ``` GET /services now returns CapAdd and CapDrop as part of the ContainerSpec. GET /services/{id} now returns CapAdd and CapDrop as part of the ContainerSpec. POST /services/create now accepts CapAdd and CapDrop as part of the ContainerSpec. POST /services/{id}/update now accepts CapAdd and CapDrop as part of the ContainerSpec. GET /tasks now returns CapAdd and CapDrop as part of the ContainerSpec. GET /tasks/{id} now returns CapAdd and CapDrop as part of the ContainerSpec. ``` I added capabilities on docstrings, `service.create` init method and create tests for that. That change was mention in issue #2802. Signed-off-by: Felipe Ruhland --- docker/models/services.py | 6 ++++ docker/types/services.py | 19 +++++++++++- tests/integration/api_service_test.py | 30 ++++++++++++++++++ tests/integration/models_services_test.py | 38 +++++++++++++++++++++++ 4 files changed, 92 insertions(+), 1 deletion(-) diff --git a/docker/models/services.py b/docker/models/services.py index a29ff1326a..200dd333c7 100644 --- a/docker/models/services.py +++ b/docker/models/services.py @@ -213,6 +213,10 @@ def create(self, image, command=None, **kwargs): to the service. privileges (Privileges): Security options for the service's containers. + cap_add (:py:class:`list`): A list of kernel capabilities to add to + the default set for the container. + cap_drop (:py:class:`list`): A list of kernel capabilities to drop + from the default set for the container. Returns: :py:class:`Service`: The created service. @@ -277,6 +281,8 @@ def list(self, **kwargs): # kwargs to copy straight over to ContainerSpec CONTAINER_SPEC_KWARGS = [ 'args', + 'cap_add', + 'cap_drop', 'command', 'configs', 'dns_config', diff --git a/docker/types/services.py b/docker/types/services.py index 29498e9715..8e87c7b432 100644 --- a/docker/types/services.py +++ b/docker/types/services.py @@ -112,13 +112,18 @@ class ContainerSpec(dict): containers. Only used for Windows containers. init (boolean): Run an init inside the container that forwards signals and reaps processes. + cap_add (:py:class:`list`): A list of kernel capabilities to add to the + default set for the container. + cap_drop (:py:class:`list`): A list of kernel capabilities to drop from + the default set for the container. """ def __init__(self, image, command=None, args=None, hostname=None, env=None, workdir=None, user=None, labels=None, mounts=None, stop_grace_period=None, secrets=None, tty=None, groups=None, open_stdin=None, read_only=None, stop_signal=None, healthcheck=None, hosts=None, dns_config=None, configs=None, - privileges=None, isolation=None, init=None): + privileges=None, isolation=None, init=None, cap_add=None, + cap_drop=None): self['Image'] = image if isinstance(command, six.string_types): @@ -188,6 +193,18 @@ def __init__(self, image, command=None, args=None, hostname=None, env=None, if init is not None: self['Init'] = init + if cap_add is not None: + if not isinstance(cap_add, list): + raise TypeError('cap_add must be a list') + + self['CapabilityAdd'] = cap_add + + if cap_drop is not None: + if not isinstance(cap_drop, list): + raise TypeError('cap_drop must be a list') + + self['CapabilityDrop'] = cap_drop + class Mount(dict): """ diff --git a/tests/integration/api_service_test.py b/tests/integration/api_service_test.py index 1bee46e563..57077e6297 100644 --- a/tests/integration/api_service_test.py +++ b/tests/integration/api_service_test.py @@ -1358,3 +1358,33 @@ def _update_service(self, svc_id, *args, **kwargs): self.client.update_service(*args, **kwargs) else: raise + + @requires_api_version('1.41') + def test_create_service_cap_add(self): + name = self.get_service_name() + container_spec = docker.types.ContainerSpec( + TEST_IMG, ['echo', 'hello'], cap_add=['CAP_SYSLOG'] + ) + task_tmpl = docker.types.TaskTemplate(container_spec) + svc_id = self.client.create_service(task_tmpl, name=name) + assert self.client.inspect_service(svc_id) + services = self.client.services(filters={'name': name}) + assert len(services) == 1 + assert services[0]['ID'] == svc_id['ID'] + spec = services[0]['Spec']['TaskTemplate']['ContainerSpec'] + assert 'CAP_SYSLOG' in spec['CapabilityAdd'] + + @requires_api_version('1.41') + def test_create_service_cap_drop(self): + name = self.get_service_name() + container_spec = docker.types.ContainerSpec( + TEST_IMG, ['echo', 'hello'], cap_drop=['CAP_SYSLOG'] + ) + task_tmpl = docker.types.TaskTemplate(container_spec) + svc_id = self.client.create_service(task_tmpl, name=name) + assert self.client.inspect_service(svc_id) + services = self.client.services(filters={'name': name}) + assert len(services) == 1 + assert services[0]['ID'] == svc_id['ID'] + spec = services[0]['Spec']['TaskTemplate']['ContainerSpec'] + assert 'CAP_SYSLOG' in spec['CapabilityDrop'] diff --git a/tests/integration/models_services_test.py b/tests/integration/models_services_test.py index 36caa8513a..982842b326 100644 --- a/tests/integration/models_services_test.py +++ b/tests/integration/models_services_test.py @@ -333,3 +333,41 @@ def test_force_update_service_using_shorthand_method(self): assert service.force_update() service.reload() assert service.version > initial_version + + @helpers.requires_api_version('1.41') + def test_create_cap_add(self): + client = docker.from_env(version=TEST_API_VERSION) + name = helpers.random_name() + service = client.services.create( + name=name, + labels={'foo': 'bar'}, + image="alpine", + command="sleep 300", + container_labels={'container': 'label'}, + cap_add=["CAP_SYSLOG"] + ) + assert service.name == name + assert service.attrs['Spec']['Labels']['foo'] == 'bar' + container_spec = service.attrs['Spec']['TaskTemplate']['ContainerSpec'] + assert "alpine" in container_spec['Image'] + assert container_spec['Labels'] == {'container': 'label'} + assert "CAP_SYSLOG" in container_spec["CapabilityAdd"] + + @helpers.requires_api_version('1.41') + def test_create_cap_drop(self): + client = docker.from_env(version=TEST_API_VERSION) + name = helpers.random_name() + service = client.services.create( + name=name, + labels={'foo': 'bar'}, + image="alpine", + command="sleep 300", + container_labels={'container': 'label'}, + cap_drop=["CAP_SYSLOG"] + ) + assert service.name == name + assert service.attrs['Spec']['Labels']['foo'] == 'bar' + container_spec = service.attrs['Spec']['TaskTemplate']['ContainerSpec'] + assert "alpine" in container_spec['Image'] + assert container_spec['Labels'] == {'container': 'label'} + assert "CAP_SYSLOG" in container_spec["CapabilityDrop"] From 13c316de692fb21521df5e019c65f9241f7ab52a Mon Sep 17 00:00:00 2001 From: Felipe Ruhland Date: Wed, 7 Apr 2021 22:55:23 +0200 Subject: [PATCH 1116/1301] Fix swarm low-level documentation examples I realize that low-level documentation has outdated examples, so I created issue #2800 to fix that Signed-off-by: Felipe Ruhland --- docker/api/swarm.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docker/api/swarm.py b/docker/api/swarm.py index 897f08e423..420529ac05 100644 --- a/docker/api/swarm.py +++ b/docker/api/swarm.py @@ -58,10 +58,10 @@ def create_swarm_spec(self, *args, **kwargs): Example: - >>> spec = client.create_swarm_spec( + >>> spec = client.api.create_swarm_spec( snapshot_interval=5000, log_entries_for_slow_followers=1200 ) - >>> client.init_swarm( + >>> client.api.init_swarm( advertise_addr='eth0', listen_addr='0.0.0.0:5000', force_new_cluster=False, swarm_spec=spec ) @@ -354,8 +354,8 @@ def unlock_swarm(self, key): Example: - >>> key = client.get_unlock_key() - >>> client.unlock_node(key) + >>> key = client.api.get_unlock_key() + >>> client.unlock_swarm(key) """ if isinstance(key, dict): @@ -396,7 +396,7 @@ def update_node(self, node_id, version, node_spec=None): 'Role': 'manager', 'Labels': {'foo': 'bar'} } - >>> client.update_node(node_id='24ifsmvkjbyhk', version=8, + >>> client.api.update_node(node_id='24ifsmvkjbyhk', version=8, node_spec=node_spec) """ From d58ca9720725219fd25a4145b8b5adbe1ed2ebc5 Mon Sep 17 00:00:00 2001 From: Roger Camargo Date: Thu, 3 Jun 2021 09:33:24 -0300 Subject: [PATCH 1117/1301] [DOCS] Update the Image.save documentation with a working example. Issue #836 Signed-off-by: Roger Camargo --- docker/models/images.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/models/images.py b/docker/models/images.py index e63558859e..28cfc93ce9 100644 --- a/docker/models/images.py +++ b/docker/models/images.py @@ -84,9 +84,9 @@ def save(self, chunk_size=DEFAULT_DATA_CHUNK_SIZE, named=False): Example: - >>> image = cli.get_image("busybox:latest") + >>> image = cli.images.get("busybox:latest") >>> f = open('/tmp/busybox-latest.tar', 'wb') - >>> for chunk in image: + >>> for chunk in image.save(): >>> f.write(chunk) >>> f.close() """ From f42a81dca2aa7a152677ee1a7d5e14248e9a6e76 Mon Sep 17 00:00:00 2001 From: Sebastiano Mariani Date: Thu, 3 Jun 2021 15:51:52 -0700 Subject: [PATCH 1118/1301] Add the possibility to set a templating driver when creating a new Docker config Signed-off-by: Sebastiano Mariani --- docker/api/config.py | 9 +++++++-- tests/integration/api_config_test.py | 13 +++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/docker/api/config.py b/docker/api/config.py index 93e5168f64..303be682ae 100644 --- a/docker/api/config.py +++ b/docker/api/config.py @@ -6,8 +6,9 @@ class ConfigApiMixin(object): + # TODO: The templating field is only available starting from API v 1.37 @utils.minimum_version('1.30') - def create_config(self, name, data, labels=None): + def create_config(self, name, data, labels=None, templating=None): """ Create a config @@ -15,6 +16,9 @@ def create_config(self, name, data, labels=None): name (string): Name of the config data (bytes): Config data to be stored labels (dict): A mapping of labels to assign to the config + templating (dict): dictionary containing the name of the + templating driver to be used expressed as + { name: } Returns (dict): ID of the newly created config """ @@ -27,7 +31,8 @@ def create_config(self, name, data, labels=None): body = { 'Data': data, 'Name': name, - 'Labels': labels + 'Labels': labels, + 'Templating': templating } url = self._url('/configs/create') diff --git a/tests/integration/api_config_test.py b/tests/integration/api_config_test.py index 0ffd7675c8..7b7d9c1867 100644 --- a/tests/integration/api_config_test.py +++ b/tests/integration/api_config_test.py @@ -70,3 +70,16 @@ def test_list_configs(self): data = self.client.configs(filters={'name': ['favorite_character']}) assert len(data) == 1 assert data[0]['ID'] == config_id['ID'] + + @requires_api_version('1.37') + def test_create_config_with_templating(self): + config_id = self.client.create_config( + 'favorite_character', 'sakuya izayoi', + templating={ 'name': 'golang'} + ) + self.tmp_configs.append(config_id) + assert 'ID' in config_id + data = self.client.inspect_config(config_id) + assert data['Spec']['Name'] == 'favorite_character' + assert 'Templating' in data['Spec'] + assert data['Spec']['Templating']['Name'] == 'golang' From 5fcc293ba268a89ea1535114d36fbdcb73ec3d88 Mon Sep 17 00:00:00 2001 From: Anthony Sottile Date: Mon, 5 Jul 2021 18:24:23 -0400 Subject: [PATCH 1119/1301] use python3.6+ constructs Signed-off-by: Anthony Sottile --- docker/api/build.py | 12 +-- docker/api/client.py | 20 ++-- docker/api/config.py | 7 +- docker/api/container.py | 8 +- docker/api/daemon.py | 2 +- docker/api/exec_api.py | 6 +- docker/api/image.py | 10 +- docker/api/network.py | 2 +- docker/api/plugin.py | 6 +- docker/api/secret.py | 7 +- docker/api/service.py | 4 +- docker/api/swarm.py | 4 +- docker/api/volume.py | 2 +- docker/auth.py | 38 ++++--- docker/client.py | 4 +- docker/constants.py | 2 +- docker/context/api.py | 6 +- docker/context/config.py | 4 +- docker/context/context.py | 4 +- docker/credentials/store.py | 27 ++--- docker/errors.py | 26 ++--- docker/models/configs.py | 2 +- docker/models/images.py | 14 ++- docker/models/plugins.py | 5 +- docker/models/resource.py | 9 +- docker/models/secrets.py | 2 +- docker/models/swarm.py | 2 +- docker/tls.py | 2 +- docker/transport/basehttpadapter.py | 2 +- docker/transport/npipeconn.py | 17 ++- docker/transport/npipesocket.py | 8 +- docker/transport/sshconn.py | 27 +++-- docker/transport/ssladapter.py | 4 +- docker/transport/unixconn.py | 26 ++--- docker/types/base.py | 5 +- docker/types/containers.py | 42 ++++---- docker/types/daemon.py | 4 +- docker/types/healthcheck.py | 8 +- docker/types/services.py | 20 ++-- docker/utils/build.py | 25 ++--- docker/utils/config.py | 6 +- docker/utils/decorators.py | 2 +- docker/utils/fnmatch.py | 2 +- docker/utils/json_stream.py | 13 +-- docker/utils/ports.py | 2 +- docker/utils/socket.py | 14 ++- docker/utils/utils.py | 32 +++--- docker/version.py | 2 +- docs/conf.py | 19 ++-- scripts/versions.py | 4 +- setup.py | 1 - tests/helpers.py | 11 +- tests/integration/api_build_test.py | 19 ++-- tests/integration/api_client_test.py | 2 +- tests/integration/api_config_test.py | 4 +- tests/integration/api_container_test.py | 42 ++++---- tests/integration/api_exec_test.py | 2 +- tests/integration/api_image_test.py | 6 +- tests/integration/api_network_test.py | 2 +- tests/integration/api_secret_test.py | 4 +- tests/integration/api_service_test.py | 32 +++--- tests/integration/api_swarm_test.py | 4 +- tests/integration/base.py | 4 +- tests/integration/conftest.py | 6 +- tests/integration/credentials/store_test.py | 7 +- tests/integration/credentials/utils_test.py | 2 +- tests/integration/models_images_test.py | 22 ++-- tests/integration/regression_test.py | 10 +- tests/ssh/api_build_test.py | 19 ++-- tests/ssh/base.py | 2 +- tests/unit/api_container_test.py | 27 +++-- tests/unit/api_exec_test.py | 10 +- tests/unit/api_image_test.py | 2 +- tests/unit/api_network_test.py | 20 ++-- tests/unit/api_test.py | 44 ++++---- tests/unit/api_volume_test.py | 4 +- tests/unit/auth_test.py | 22 ++-- tests/unit/client_test.py | 2 +- tests/unit/dockertypes_test.py | 4 +- tests/unit/errors_test.py | 2 +- tests/unit/fake_api.py | 100 ++++++++--------- tests/unit/fake_api_client.py | 4 +- tests/unit/models_resources_test.py | 2 +- tests/unit/models_secrets_test.py | 2 +- tests/unit/models_services_test.py | 8 +- tests/unit/ssladapter_test.py | 38 +++---- tests/unit/swarm_test.py | 2 - tests/unit/utils_build_test.py | 112 ++++++++++---------- tests/unit/utils_config_test.py | 2 +- tests/unit/utils_json_stream_test.py | 12 +-- tests/unit/utils_proxy_test.py | 7 +- tests/unit/utils_test.py | 34 +++--- 92 files changed, 524 insertions(+), 658 deletions(-) diff --git a/docker/api/build.py b/docker/api/build.py index 365129a064..aac43c460a 100644 --- a/docker/api/build.py +++ b/docker/api/build.py @@ -12,7 +12,7 @@ log = logging.getLogger(__name__) -class BuildApiMixin(object): +class BuildApiMixin: def build(self, path=None, tag=None, quiet=False, fileobj=None, nocache=False, rm=False, timeout=None, custom_context=False, encoding=None, pull=False, @@ -132,7 +132,7 @@ def build(self, path=None, tag=None, quiet=False, fileobj=None, for key in container_limits.keys(): if key not in constants.CONTAINER_LIMITS_KEYS: raise errors.DockerException( - 'Invalid container_limits key {0}'.format(key) + f'Invalid container_limits key {key}' ) if custom_context: @@ -150,7 +150,7 @@ def build(self, path=None, tag=None, quiet=False, fileobj=None, dockerignore = os.path.join(path, '.dockerignore') exclude = None if os.path.exists(dockerignore): - with open(dockerignore, 'r') as f: + with open(dockerignore) as f: exclude = list(filter( lambda x: x != '' and x[0] != '#', [l.strip() for l in f.read().splitlines()] @@ -313,7 +313,7 @@ def _set_auth_headers(self, headers): auth_data[auth.INDEX_URL] = auth_data.get(auth.INDEX_NAME, {}) log.debug( - 'Sending auth config ({0})'.format( + 'Sending auth config ({})'.format( ', '.join(repr(k) for k in auth_data.keys()) ) ) @@ -344,9 +344,9 @@ def process_dockerfile(dockerfile, path): if (os.path.splitdrive(path)[0] != os.path.splitdrive(abs_dockerfile)[0] or os.path.relpath(abs_dockerfile, path).startswith('..')): # Dockerfile not in context - read data to insert into tar later - with open(abs_dockerfile, 'r') as df: + with open(abs_dockerfile) as df: return ( - '.dockerfile.{0:x}'.format(random.getrandbits(160)), + f'.dockerfile.{random.getrandbits(160):x}', df.read() ) diff --git a/docker/api/client.py b/docker/api/client.py index ee9ad9c3b4..f0cb39b864 100644 --- a/docker/api/client.py +++ b/docker/api/client.py @@ -107,7 +107,7 @@ def __init__(self, base_url=None, version=None, user_agent=DEFAULT_USER_AGENT, num_pools=None, credstore_env=None, use_ssh_client=False, max_pool_size=DEFAULT_MAX_POOL_SIZE): - super(APIClient, self).__init__() + super().__init__() if tls and not base_url: raise TLSParameterError( @@ -199,7 +199,7 @@ def __init__(self, base_url=None, version=None, self._version = version if not isinstance(self._version, str): raise DockerException( - 'Version parameter must be a string or None. Found {0}'.format( + 'Version parameter must be a string or None. Found {}'.format( type(version).__name__ ) ) @@ -219,7 +219,7 @@ def _retrieve_server_version(self): ) except Exception as e: raise DockerException( - 'Error while fetching server API version: {0}'.format(e) + f'Error while fetching server API version: {e}' ) def _set_request_timeout(self, kwargs): @@ -248,7 +248,7 @@ def _url(self, pathfmt, *args, **kwargs): for arg in args: if not isinstance(arg, str): raise ValueError( - 'Expected a string but found {0} ({1}) ' + 'Expected a string but found {} ({}) ' 'instead'.format(arg, type(arg)) ) @@ -256,11 +256,11 @@ def _url(self, pathfmt, *args, **kwargs): args = map(quote_f, args) if kwargs.get('versioned_api', True): - return '{0}/v{1}{2}'.format( + return '{}/v{}{}'.format( self.base_url, self._version, pathfmt.format(*args) ) else: - return '{0}{1}'.format(self.base_url, pathfmt.format(*args)) + return f'{self.base_url}{pathfmt.format(*args)}' def _raise_for_status(self, response): """Raises stored :class:`APIError`, if one occurred.""" @@ -341,8 +341,7 @@ def _stream_helper(self, response, decode=False): if response.raw._fp.chunked: if decode: - for chunk in json_stream(self._stream_helper(response, False)): - yield chunk + yield from json_stream(self._stream_helper(response, False)) else: reader = response.raw while not reader.closed: @@ -398,8 +397,7 @@ def _multiplexed_response_stream_helper(self, response): def _stream_raw_result(self, response, chunk_size=1, decode=True): ''' Stream result for TTY-enabled container and raw binary data''' self._raise_for_status(response) - for out in response.iter_content(chunk_size, decode): - yield out + yield from response.iter_content(chunk_size, decode) def _read_from_socket(self, response, stream, tty=True, demux=False): socket = self._get_raw_response_socket(response) @@ -477,7 +475,7 @@ def _unmount(self, *args): def get_adapter(self, url): try: - return super(APIClient, self).get_adapter(url) + return super().get_adapter(url) except requests.exceptions.InvalidSchema as e: if self._custom_adapter: return self._custom_adapter diff --git a/docker/api/config.py b/docker/api/config.py index 93e5168f64..8cf74e1a2e 100644 --- a/docker/api/config.py +++ b/docker/api/config.py @@ -1,11 +1,9 @@ import base64 -import six - from .. import utils -class ConfigApiMixin(object): +class ConfigApiMixin: @utils.minimum_version('1.30') def create_config(self, name, data, labels=None): """ @@ -22,8 +20,7 @@ def create_config(self, name, data, labels=None): data = data.encode('utf-8') data = base64.b64encode(data) - if six.PY3: - data = data.decode('ascii') + data = data.decode('ascii') body = { 'Data': data, 'Name': name, diff --git a/docker/api/container.py b/docker/api/container.py index 369eba952f..83fcd4f64a 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -1,7 +1,5 @@ from datetime import datetime -import six - from .. import errors from .. import utils from ..constants import DEFAULT_DATA_CHUNK_SIZE @@ -12,7 +10,7 @@ from ..types import NetworkingConfig -class ContainerApiMixin(object): +class ContainerApiMixin: @utils.check_resource('container') def attach(self, container, stdout=True, stderr=True, stream=False, logs=False, demux=False): @@ -408,7 +406,7 @@ def create_container(self, image, command=None, hostname=None, user=None, :py:class:`docker.errors.APIError` If the server returns an error. """ - if isinstance(volumes, six.string_types): + if isinstance(volumes, str): volumes = [volumes, ] if isinstance(environment, dict): @@ -790,7 +788,7 @@ def kill(self, container, signal=None): url = self._url("/containers/{0}/kill", container) params = {} if signal is not None: - if not isinstance(signal, six.string_types): + if not isinstance(signal, str): signal = int(signal) params['signal'] = signal res = self._post(url, params=params) diff --git a/docker/api/daemon.py b/docker/api/daemon.py index 6b719268ef..a857213265 100644 --- a/docker/api/daemon.py +++ b/docker/api/daemon.py @@ -4,7 +4,7 @@ from .. import auth, types, utils -class DaemonApiMixin(object): +class DaemonApiMixin: @utils.minimum_version('1.25') def df(self): """ diff --git a/docker/api/exec_api.py b/docker/api/exec_api.py index 4c49ac3380..496308a0f1 100644 --- a/docker/api/exec_api.py +++ b/docker/api/exec_api.py @@ -1,10 +1,8 @@ -import six - from .. import errors from .. import utils -class ExecApiMixin(object): +class ExecApiMixin: @utils.check_resource('container') def exec_create(self, container, cmd, stdout=True, stderr=True, stdin=False, tty=False, privileged=False, user='', @@ -45,7 +43,7 @@ def exec_create(self, container, cmd, stdout=True, stderr=True, 'Setting environment for exec is not supported in API < 1.25' ) - if isinstance(cmd, six.string_types): + if isinstance(cmd, str): cmd = utils.split_command(cmd) if isinstance(environment, dict): diff --git a/docker/api/image.py b/docker/api/image.py index 772101f4e9..772d88957c 100644 --- a/docker/api/image.py +++ b/docker/api/image.py @@ -1,15 +1,13 @@ import logging import os -import six - from .. import auth, errors, utils from ..constants import DEFAULT_DATA_CHUNK_SIZE log = logging.getLogger(__name__) -class ImageApiMixin(object): +class ImageApiMixin: @utils.check_resource('image') def get_image(self, image, chunk_size=DEFAULT_DATA_CHUNK_SIZE): @@ -130,7 +128,7 @@ def import_image(self, src=None, repository=None, tag=None, image=None, params = _import_image_params( repository, tag, image, - src=(src if isinstance(src, six.string_types) else None), + src=(src if isinstance(src, str) else None), changes=changes ) headers = {'Content-Type': 'application/tar'} @@ -139,7 +137,7 @@ def import_image(self, src=None, repository=None, tag=None, image=None, return self._result( self._post(u, data=None, params=params) ) - elif isinstance(src, six.string_types): # from file path + elif isinstance(src, str): # from file path with open(src, 'rb') as f: return self._result( self._post( @@ -571,7 +569,7 @@ def tag(self, image, repository, tag=None, force=False): def is_file(src): try: return ( - isinstance(src, six.string_types) and + isinstance(src, str) and os.path.isfile(src) ) except TypeError: # a data string will make isfile() raise a TypeError diff --git a/docker/api/network.py b/docker/api/network.py index 139c2d1a82..0b76bf3210 100644 --- a/docker/api/network.py +++ b/docker/api/network.py @@ -4,7 +4,7 @@ from .. import utils -class NetworkApiMixin(object): +class NetworkApiMixin: def networks(self, names=None, ids=None, filters=None): """ List networks. Similar to the ``docker network ls`` command. diff --git a/docker/api/plugin.py b/docker/api/plugin.py index f6c0b13387..57110f1131 100644 --- a/docker/api/plugin.py +++ b/docker/api/plugin.py @@ -1,9 +1,7 @@ -import six - from .. import auth, utils -class PluginApiMixin(object): +class PluginApiMixin: @utils.minimum_version('1.25') @utils.check_resource('name') def configure_plugin(self, name, options): @@ -21,7 +19,7 @@ def configure_plugin(self, name, options): url = self._url('/plugins/{0}/set', name) data = options if isinstance(data, dict): - data = ['{0}={1}'.format(k, v) for k, v in six.iteritems(data)] + data = [f'{k}={v}' for k, v in data.items()] res = self._post_json(url, data=data) self._raise_for_status(res) return True diff --git a/docker/api/secret.py b/docker/api/secret.py index e57952b53b..cd440b95f8 100644 --- a/docker/api/secret.py +++ b/docker/api/secret.py @@ -1,12 +1,10 @@ import base64 -import six - from .. import errors from .. import utils -class SecretApiMixin(object): +class SecretApiMixin: @utils.minimum_version('1.25') def create_secret(self, name, data, labels=None, driver=None): """ @@ -25,8 +23,7 @@ def create_secret(self, name, data, labels=None, driver=None): data = data.encode('utf-8') data = base64.b64encode(data) - if six.PY3: - data = data.decode('ascii') + data = data.decode('ascii') body = { 'Data': data, 'Name': name, diff --git a/docker/api/service.py b/docker/api/service.py index e9027bfa21..371f541e11 100644 --- a/docker/api/service.py +++ b/docker/api/service.py @@ -45,7 +45,7 @@ def raise_version_error(param, min_version): if task_template is not None: if 'ForceUpdate' in task_template and utils.version_lt( version, '1.25'): - raise_version_error('force_update', '1.25') + raise_version_error('force_update', '1.25') if task_template.get('Placement'): if utils.version_lt(version, '1.30'): @@ -113,7 +113,7 @@ def _merge_task_template(current, override): return merged -class ServiceApiMixin(object): +class ServiceApiMixin: @utils.minimum_version('1.24') def create_service( self, task_template, name=None, labels=None, mode=None, diff --git a/docker/api/swarm.py b/docker/api/swarm.py index 897f08e423..2ec1aea5e2 100644 --- a/docker/api/swarm.py +++ b/docker/api/swarm.py @@ -1,5 +1,5 @@ import logging -from six.moves import http_client +import http.client as http_client from ..constants import DEFAULT_SWARM_ADDR_POOL, DEFAULT_SWARM_SUBNET_SIZE from .. import errors from .. import types @@ -8,7 +8,7 @@ log = logging.getLogger(__name__) -class SwarmApiMixin(object): +class SwarmApiMixin: def create_swarm_spec(self, *args, **kwargs): """ diff --git a/docker/api/volume.py b/docker/api/volume.py index 900a6086b5..c6b26fe38c 100644 --- a/docker/api/volume.py +++ b/docker/api/volume.py @@ -2,7 +2,7 @@ from .. import utils -class VolumeApiMixin(object): +class VolumeApiMixin: def volumes(self, filters=None): """ List volumes currently registered by the docker daemon. Similar to the diff --git a/docker/auth.py b/docker/auth.py index 6a07ea2059..4fa798fcc0 100644 --- a/docker/auth.py +++ b/docker/auth.py @@ -2,14 +2,12 @@ import json import logging -import six - from . import credentials from . import errors from .utils import config INDEX_NAME = 'docker.io' -INDEX_URL = 'https://index.{0}/v1/'.format(INDEX_NAME) +INDEX_URL = f'https://index.{INDEX_NAME}/v1/' TOKEN_USERNAME = '' log = logging.getLogger(__name__) @@ -18,13 +16,13 @@ def resolve_repository_name(repo_name): if '://' in repo_name: raise errors.InvalidRepository( - 'Repository name cannot contain a scheme ({0})'.format(repo_name) + f'Repository name cannot contain a scheme ({repo_name})' ) index_name, remote_name = split_repo_name(repo_name) if index_name[0] == '-' or index_name[-1] == '-': raise errors.InvalidRepository( - 'Invalid index name ({0}). Cannot begin or end with a' + 'Invalid index name ({}). Cannot begin or end with a' ' hyphen.'.format(index_name) ) return resolve_index_name(index_name), remote_name @@ -98,10 +96,10 @@ def parse_auth(cls, entries, raise_on_error=False): """ conf = {} - for registry, entry in six.iteritems(entries): + for registry, entry in entries.items(): if not isinstance(entry, dict): log.debug( - 'Config entry for key {0} is not auth config'.format( + 'Config entry for key {} is not auth config'.format( registry ) ) @@ -111,14 +109,14 @@ def parse_auth(cls, entries, raise_on_error=False): # keys is not formatted properly. if raise_on_error: raise errors.InvalidConfigFile( - 'Invalid configuration for registry {0}'.format( + 'Invalid configuration for registry {}'.format( registry ) ) return {} if 'identitytoken' in entry: log.debug( - 'Found an IdentityToken entry for registry {0}'.format( + 'Found an IdentityToken entry for registry {}'.format( registry ) ) @@ -132,7 +130,7 @@ def parse_auth(cls, entries, raise_on_error=False): # a valid value in the auths config. # https://github.com/docker/compose/issues/3265 log.debug( - 'Auth data for {0} is absent. Client might be using a ' + 'Auth data for {} is absent. Client might be using a ' 'credentials store instead.'.format(registry) ) conf[registry] = {} @@ -140,7 +138,7 @@ def parse_auth(cls, entries, raise_on_error=False): username, password = decode_auth(entry['auth']) log.debug( - 'Found entry (registry={0}, username={1})' + 'Found entry (registry={}, username={})' .format(repr(registry), repr(username)) ) @@ -170,7 +168,7 @@ def load_config(cls, config_path, config_dict, credstore_env=None): try: with open(config_file) as f: config_dict = json.load(f) - except (IOError, KeyError, ValueError) as e: + except (OSError, KeyError, ValueError) as e: # Likely missing new Docker config file or it's in an # unknown format, continue to attempt to read old location # and format. @@ -230,7 +228,7 @@ def resolve_authconfig(self, registry=None): store_name = self.get_credential_store(registry) if store_name is not None: log.debug( - 'Using credentials store "{0}"'.format(store_name) + f'Using credentials store "{store_name}"' ) cfg = self._resolve_authconfig_credstore(registry, store_name) if cfg is not None: @@ -239,15 +237,15 @@ def resolve_authconfig(self, registry=None): # Default to the public index server registry = resolve_index_name(registry) if registry else INDEX_NAME - log.debug("Looking for auth entry for {0}".format(repr(registry))) + log.debug(f"Looking for auth entry for {repr(registry)}") if registry in self.auths: - log.debug("Found {0}".format(repr(registry))) + log.debug(f"Found {repr(registry)}") return self.auths[registry] - for key, conf in six.iteritems(self.auths): + for key, conf in self.auths.items(): if resolve_index_name(key) == registry: - log.debug("Found {0}".format(repr(key))) + log.debug(f"Found {repr(key)}") return conf log.debug("No entry found") @@ -258,7 +256,7 @@ def _resolve_authconfig_credstore(self, registry, credstore_name): # The ecosystem is a little schizophrenic with index.docker.io VS # docker.io - in that case, it seems the full URL is necessary. registry = INDEX_URL - log.debug("Looking for auth entry for {0}".format(repr(registry))) + log.debug(f"Looking for auth entry for {repr(registry)}") store = self._get_store_instance(credstore_name) try: data = store.get(registry) @@ -278,7 +276,7 @@ def _resolve_authconfig_credstore(self, registry, credstore_name): return None except credentials.StoreError as e: raise errors.DockerException( - 'Credentials store error: {0}'.format(repr(e)) + f'Credentials store error: {repr(e)}' ) def _get_store_instance(self, name): @@ -329,7 +327,7 @@ def convert_to_hostname(url): def decode_auth(auth): - if isinstance(auth, six.string_types): + if isinstance(auth, str): auth = auth.encode('ascii') s = base64.b64decode(auth) login, pwd = s.split(b':', 1) diff --git a/docker/client.py b/docker/client.py index 5add5d730e..4dbd846f1d 100644 --- a/docker/client.py +++ b/docker/client.py @@ -13,7 +13,7 @@ from .utils import kwargs_from_env -class DockerClient(object): +class DockerClient: """ A client for communicating with a Docker server. @@ -212,7 +212,7 @@ def close(self): close.__doc__ = APIClient.close.__doc__ def __getattr__(self, name): - s = ["'DockerClient' object has no attribute '{}'".format(name)] + s = [f"'DockerClient' object has no attribute '{name}'"] # If a user calls a method on APIClient, they if hasattr(APIClient, name): s.append("In Docker SDK for Python 2.0, this method is now on the " diff --git a/docker/constants.py b/docker/constants.py index 43fce6138e..218e491532 100644 --- a/docker/constants.py +++ b/docker/constants.py @@ -28,7 +28,7 @@ IS_WINDOWS_PLATFORM = (sys.platform == 'win32') WINDOWS_LONGPATH_PREFIX = '\\\\?\\' -DEFAULT_USER_AGENT = "docker-sdk-python/{0}".format(version) +DEFAULT_USER_AGENT = f"docker-sdk-python/{version}" DEFAULT_NUM_POOLS = 25 # The OpenSSH server default value for MaxSessions is 10 which means we can diff --git a/docker/context/api.py b/docker/context/api.py index c45115bce5..380e8c4c4f 100644 --- a/docker/context/api.py +++ b/docker/context/api.py @@ -9,7 +9,7 @@ from docker.context import Context -class ContextAPI(object): +class ContextAPI: """Context API. Contains methods for context management: create, list, remove, get, inspect. @@ -109,7 +109,7 @@ def contexts(cls): if filename == METAFILE: try: data = json.load( - open(os.path.join(dirname, filename), "r")) + open(os.path.join(dirname, filename))) names.append(data["Name"]) except Exception as e: raise errors.ContextException( @@ -138,7 +138,7 @@ def set_current_context(cls, name="default"): err = write_context_name_to_docker_config(name) if err: raise errors.ContextException( - 'Failed to set current context: {}'.format(err)) + f'Failed to set current context: {err}') @classmethod def remove_context(cls, name): diff --git a/docker/context/config.py b/docker/context/config.py index baf54f797e..d761aef13c 100644 --- a/docker/context/config.py +++ b/docker/context/config.py @@ -15,7 +15,7 @@ def get_current_context_name(): docker_cfg_path = find_config_file() if docker_cfg_path: try: - with open(docker_cfg_path, "r") as f: + with open(docker_cfg_path) as f: name = json.load(f).get("currentContext", "default") except Exception: return "default" @@ -29,7 +29,7 @@ def write_context_name_to_docker_config(name=None): config = {} if docker_cfg_path: try: - with open(docker_cfg_path, "r") as f: + with open(docker_cfg_path) as f: config = json.load(f) except Exception as e: return e diff --git a/docker/context/context.py b/docker/context/context.py index f4aff6b0d2..dbaa01cb5b 100644 --- a/docker/context/context.py +++ b/docker/context/context.py @@ -94,7 +94,7 @@ def _load_meta(cls, name): try: with open(meta_file) as f: metadata = json.load(f) - except (IOError, KeyError, ValueError) as e: + except (OSError, KeyError, ValueError) as e: # unknown format raise Exception("""Detected corrupted meta file for context {} : {}""".format(name, e)) @@ -171,7 +171,7 @@ def remove(self): rmtree(self.tls_path) def __repr__(self): - return "<%s: '%s'>" % (self.__class__.__name__, self.name) + return f"<{self.__class__.__name__}: '{self.name}'>" def __str__(self): return json.dumps(self.__call__(), indent=2) diff --git a/docker/credentials/store.py b/docker/credentials/store.py index 0017888978..e55976f189 100644 --- a/docker/credentials/store.py +++ b/docker/credentials/store.py @@ -2,15 +2,13 @@ import json import subprocess -import six - from . import constants from . import errors from .utils import create_environment_dict from .utils import find_executable -class Store(object): +class Store: def __init__(self, program, environment=None): """ Create a store object that acts as an interface to perform the basic operations for storing, retrieving @@ -30,7 +28,7 @@ def get(self, server): """ Retrieve credentials for `server`. If no credentials are found, a `StoreError` will be raised. """ - if not isinstance(server, six.binary_type): + if not isinstance(server, bytes): server = server.encode('utf-8') data = self._execute('get', server) result = json.loads(data.decode('utf-8')) @@ -41,7 +39,7 @@ def get(self, server): # raise CredentialsNotFound if result['Username'] == '' and result['Secret'] == '': raise errors.CredentialsNotFound( - 'No matching credentials in {}'.format(self.program) + f'No matching credentials in {self.program}' ) return result @@ -61,7 +59,7 @@ def erase(self, server): """ Erase credentials for `server`. Raises a `StoreError` if an error occurs. """ - if not isinstance(server, six.binary_type): + if not isinstance(server, bytes): server = server.encode('utf-8') self._execute('erase', server) @@ -75,20 +73,9 @@ def _execute(self, subcmd, data_input): output = None env = create_environment_dict(self.environment) try: - if six.PY3: - output = subprocess.check_output( - [self.exe, subcmd], input=data_input, env=env, - ) - else: - process = subprocess.Popen( - [self.exe, subcmd], stdin=subprocess.PIPE, - stdout=subprocess.PIPE, env=env, - ) - output, _ = process.communicate(data_input) - if process.returncode != 0: - raise subprocess.CalledProcessError( - returncode=process.returncode, cmd='', output=output - ) + output = subprocess.check_output( + [self.exe, subcmd], input=data_input, env=env, + ) except subprocess.CalledProcessError as e: raise errors.process_store_error(e, self.program) except OSError as e: diff --git a/docker/errors.py b/docker/errors.py index ab30a2908e..ba952562c6 100644 --- a/docker/errors.py +++ b/docker/errors.py @@ -38,25 +38,25 @@ class APIError(requests.exceptions.HTTPError, DockerException): def __init__(self, message, response=None, explanation=None): # requests 1.2 supports response as a keyword argument, but # requests 1.1 doesn't - super(APIError, self).__init__(message) + super().__init__(message) self.response = response self.explanation = explanation def __str__(self): - message = super(APIError, self).__str__() + message = super().__str__() if self.is_client_error(): - message = '{0} Client Error for {1}: {2}'.format( + message = '{} Client Error for {}: {}'.format( self.response.status_code, self.response.url, self.response.reason) elif self.is_server_error(): - message = '{0} Server Error for {1}: {2}'.format( + message = '{} Server Error for {}: {}'.format( self.response.status_code, self.response.url, self.response.reason) if self.explanation: - message = '{0} ("{1}")'.format(message, self.explanation) + message = f'{message} ("{self.explanation}")' return message @@ -133,11 +133,11 @@ def __init__(self, container, exit_status, command, image, stderr): self.image = image self.stderr = stderr - err = ": {}".format(stderr) if stderr is not None else "" + err = f": {stderr}" if stderr is not None else "" msg = ("Command '{}' in image '{}' returned non-zero exit " "status {}{}").format(command, image, exit_status, err) - super(ContainerError, self).__init__(msg) + super().__init__(msg) class StreamParseError(RuntimeError): @@ -147,7 +147,7 @@ def __init__(self, reason): class BuildError(DockerException): def __init__(self, reason, build_log): - super(BuildError, self).__init__(reason) + super().__init__(reason) self.msg = reason self.build_log = build_log @@ -157,8 +157,8 @@ class ImageLoadError(DockerException): def create_unexpected_kwargs_error(name, kwargs): - quoted_kwargs = ["'{}'".format(k) for k in sorted(kwargs)] - text = ["{}() ".format(name)] + quoted_kwargs = [f"'{k}'" for k in sorted(kwargs)] + text = [f"{name}() "] if len(quoted_kwargs) == 1: text.append("got an unexpected keyword argument ") else: @@ -172,7 +172,7 @@ def __init__(self, param): self.param = param def __str__(self): - return ("missing parameter: {}".format(self.param)) + return (f"missing parameter: {self.param}") class ContextAlreadyExists(DockerException): @@ -180,7 +180,7 @@ def __init__(self, name): self.name = name def __str__(self): - return ("context {} already exists".format(self.name)) + return (f"context {self.name} already exists") class ContextException(DockerException): @@ -196,4 +196,4 @@ def __init__(self, name): self.name = name def __str__(self): - return ("context '{}' not found".format(self.name)) + return (f"context '{self.name}' not found") diff --git a/docker/models/configs.py b/docker/models/configs.py index 7f23f65007..3588c8b5dc 100644 --- a/docker/models/configs.py +++ b/docker/models/configs.py @@ -7,7 +7,7 @@ class Config(Model): id_attribute = 'ID' def __repr__(self): - return "<%s: '%s'>" % (self.__class__.__name__, self.name) + return f"<{self.__class__.__name__}: '{self.name}'>" @property def name(self): diff --git a/docker/models/images.py b/docker/models/images.py index 28cfc93ce9..46f8efeed8 100644 --- a/docker/models/images.py +++ b/docker/models/images.py @@ -2,8 +2,6 @@ import re import warnings -import six - from ..api import APIClient from ..constants import DEFAULT_DATA_CHUNK_SIZE from ..errors import BuildError, ImageLoadError, InvalidArgument @@ -17,7 +15,7 @@ class Image(Model): An image on the server. """ def __repr__(self): - return "<%s: '%s'>" % (self.__class__.__name__, "', '".join(self.tags)) + return "<{}: '{}'>".format(self.__class__.__name__, "', '".join(self.tags)) @property def labels(self): @@ -93,10 +91,10 @@ def save(self, chunk_size=DEFAULT_DATA_CHUNK_SIZE, named=False): img = self.id if named: img = self.tags[0] if self.tags else img - if isinstance(named, six.string_types): + if isinstance(named, str): if named not in self.tags: raise InvalidArgument( - "{} is not a valid tag for this image".format(named) + f"{named} is not a valid tag for this image" ) img = named @@ -127,7 +125,7 @@ class RegistryData(Model): Image metadata stored on the registry, including available platforms. """ def __init__(self, image_name, *args, **kwargs): - super(RegistryData, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.image_name = image_name @property @@ -180,7 +178,7 @@ def has_platform(self, platform): parts = platform.split('/') if len(parts) > 3 or len(parts) < 1: raise InvalidArgument( - '"{0}" is not a valid platform descriptor'.format(platform) + f'"{platform}" is not a valid platform descriptor' ) platform = {'os': parts[0]} if len(parts) > 2: @@ -277,7 +275,7 @@ def build(self, **kwargs): If neither ``path`` nor ``fileobj`` is specified. """ resp = self.client.api.build(**kwargs) - if isinstance(resp, six.string_types): + if isinstance(resp, str): return self.get(resp) last_event = None image_id = None diff --git a/docker/models/plugins.py b/docker/models/plugins.py index ae5851c919..37ecefbe09 100644 --- a/docker/models/plugins.py +++ b/docker/models/plugins.py @@ -7,7 +7,7 @@ class Plugin(Model): A plugin on the server. """ def __repr__(self): - return "<%s: '%s'>" % (self.__class__.__name__, self.name) + return f"<{self.__class__.__name__}: '{self.name}'>" @property def name(self): @@ -117,8 +117,7 @@ def upgrade(self, remote=None): if remote is None: remote = self.name privileges = self.client.api.plugin_privileges(remote) - for d in self.client.api.upgrade_plugin(self.name, remote, privileges): - yield d + yield from self.client.api.upgrade_plugin(self.name, remote, privileges) self.reload() diff --git a/docker/models/resource.py b/docker/models/resource.py index ed3900af3a..dec2349f67 100644 --- a/docker/models/resource.py +++ b/docker/models/resource.py @@ -1,5 +1,4 @@ - -class Model(object): +class Model: """ A base class for representing a single object on the server. """ @@ -18,13 +17,13 @@ def __init__(self, attrs=None, client=None, collection=None): self.attrs = {} def __repr__(self): - return "<%s: %s>" % (self.__class__.__name__, self.short_id) + return f"<{self.__class__.__name__}: {self.short_id}>" def __eq__(self, other): return isinstance(other, self.__class__) and self.id == other.id def __hash__(self): - return hash("%s:%s" % (self.__class__.__name__, self.id)) + return hash(f"{self.__class__.__name__}:{self.id}") @property def id(self): @@ -49,7 +48,7 @@ def reload(self): self.attrs = new_model.attrs -class Collection(object): +class Collection: """ A base class for representing all objects of a particular type on the server. diff --git a/docker/models/secrets.py b/docker/models/secrets.py index e2ee88af02..da01d44c8f 100644 --- a/docker/models/secrets.py +++ b/docker/models/secrets.py @@ -7,7 +7,7 @@ class Secret(Model): id_attribute = 'ID' def __repr__(self): - return "<%s: '%s'>" % (self.__class__.__name__, self.name) + return f"<{self.__class__.__name__}: '{self.name}'>" @property def name(self): diff --git a/docker/models/swarm.py b/docker/models/swarm.py index 755c17db43..b0b1a2ef8a 100644 --- a/docker/models/swarm.py +++ b/docker/models/swarm.py @@ -11,7 +11,7 @@ class Swarm(Model): id_attribute = 'ID' def __init__(self, *args, **kwargs): - super(Swarm, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) if self.client: try: self.reload() diff --git a/docker/tls.py b/docker/tls.py index 1b297ab666..067d556300 100644 --- a/docker/tls.py +++ b/docker/tls.py @@ -5,7 +5,7 @@ from .transport import SSLHTTPAdapter -class TLSConfig(object): +class TLSConfig: """ TLS configuration. diff --git a/docker/transport/basehttpadapter.py b/docker/transport/basehttpadapter.py index 4d819b669c..dfbb193b9a 100644 --- a/docker/transport/basehttpadapter.py +++ b/docker/transport/basehttpadapter.py @@ -3,6 +3,6 @@ class BaseHTTPAdapter(requests.adapters.HTTPAdapter): def close(self): - super(BaseHTTPAdapter, self).close() + super().close() if hasattr(self, 'pools'): self.pools.clear() diff --git a/docker/transport/npipeconn.py b/docker/transport/npipeconn.py index 70d8519dc0..df67f21251 100644 --- a/docker/transport/npipeconn.py +++ b/docker/transport/npipeconn.py @@ -1,14 +1,11 @@ -import six +import queue import requests.adapters from docker.transport.basehttpadapter import BaseHTTPAdapter from .. import constants from .npipesocket import NpipeSocket -if six.PY3: - import http.client as httplib -else: - import httplib +import http.client as httplib try: import requests.packages.urllib3 as urllib3 @@ -18,9 +15,9 @@ RecentlyUsedContainer = urllib3._collections.RecentlyUsedContainer -class NpipeHTTPConnection(httplib.HTTPConnection, object): +class NpipeHTTPConnection(httplib.HTTPConnection): def __init__(self, npipe_path, timeout=60): - super(NpipeHTTPConnection, self).__init__( + super().__init__( 'localhost', timeout=timeout ) self.npipe_path = npipe_path @@ -35,7 +32,7 @@ def connect(self): class NpipeHTTPConnectionPool(urllib3.connectionpool.HTTPConnectionPool): def __init__(self, npipe_path, timeout=60, maxsize=10): - super(NpipeHTTPConnectionPool, self).__init__( + super().__init__( 'localhost', timeout=timeout, maxsize=maxsize ) self.npipe_path = npipe_path @@ -57,7 +54,7 @@ def _get_conn(self, timeout): except AttributeError: # self.pool is None raise urllib3.exceptions.ClosedPoolError(self, "Pool is closed.") - except six.moves.queue.Empty: + except queue.Empty: if self.block: raise urllib3.exceptions.EmptyPoolError( self, @@ -85,7 +82,7 @@ def __init__(self, base_url, timeout=60, self.pools = RecentlyUsedContainer( pool_connections, dispose_func=lambda p: p.close() ) - super(NpipeHTTPAdapter, self).__init__() + super().__init__() def get_connection(self, url, proxies=None): with self.pools.lock: diff --git a/docker/transport/npipesocket.py b/docker/transport/npipesocket.py index 176b5c87a9..766372aefd 100644 --- a/docker/transport/npipesocket.py +++ b/docker/transport/npipesocket.py @@ -2,7 +2,6 @@ import time import io -import six import win32file import win32pipe @@ -24,7 +23,7 @@ def wrapped(self, *args, **kwargs): return wrapped -class NpipeSocket(object): +class NpipeSocket: """ Partial implementation of the socket API over windows named pipes. This implementation is only designed to be used as a client socket, and server-specific methods (bind, listen, accept...) are not @@ -128,9 +127,6 @@ def recvfrom_into(self, buf, nbytes=0, flags=0): @check_closed def recv_into(self, buf, nbytes=0): - if six.PY2: - return self._recv_into_py2(buf, nbytes) - readbuf = buf if not isinstance(buf, memoryview): readbuf = memoryview(buf) @@ -195,7 +191,7 @@ def __init__(self, npipe_socket): self.sock = npipe_socket def close(self): - super(NpipeFileIOBase, self).close() + super().close() self.sock = None def fileno(self): diff --git a/docker/transport/sshconn.py b/docker/transport/sshconn.py index fb5c6bbe8a..3ca45c4c07 100644 --- a/docker/transport/sshconn.py +++ b/docker/transport/sshconn.py @@ -1,6 +1,7 @@ import paramiko +import queue +import urllib.parse import requests.adapters -import six import logging import os import signal @@ -10,10 +11,7 @@ from docker.transport.basehttpadapter import BaseHTTPAdapter from .. import constants -if six.PY3: - import http.client as httplib -else: - import httplib +import http.client as httplib try: import requests.packages.urllib3 as urllib3 @@ -25,7 +23,7 @@ class SSHSocket(socket.socket): def __init__(self, host): - super(SSHSocket, self).__init__( + super().__init__( socket.AF_INET, socket.SOCK_STREAM) self.host = host self.port = None @@ -90,8 +88,7 @@ def recv(self, n): def makefile(self, mode): if not self.proc: self.connect() - if six.PY3: - self.proc.stdout.channel = self + self.proc.stdout.channel = self return self.proc.stdout @@ -103,9 +100,9 @@ def close(self): self.proc.terminate() -class SSHConnection(httplib.HTTPConnection, object): +class SSHConnection(httplib.HTTPConnection): def __init__(self, ssh_transport=None, timeout=60, host=None): - super(SSHConnection, self).__init__( + super().__init__( 'localhost', timeout=timeout ) self.ssh_transport = ssh_transport @@ -129,7 +126,7 @@ class SSHConnectionPool(urllib3.connectionpool.HTTPConnectionPool): scheme = 'ssh' def __init__(self, ssh_client=None, timeout=60, maxsize=10, host=None): - super(SSHConnectionPool, self).__init__( + super().__init__( 'localhost', timeout=timeout, maxsize=maxsize ) self.ssh_transport = None @@ -152,7 +149,7 @@ def _get_conn(self, timeout): except AttributeError: # self.pool is None raise urllib3.exceptions.ClosedPoolError(self, "Pool is closed.") - except six.moves.queue.Empty: + except queue.Empty: if self.block: raise urllib3.exceptions.EmptyPoolError( self, @@ -188,12 +185,12 @@ def __init__(self, base_url, timeout=60, self.pools = RecentlyUsedContainer( pool_connections, dispose_func=lambda p: p.close() ) - super(SSHHTTPAdapter, self).__init__() + super().__init__() def _create_paramiko_client(self, base_url): logging.getLogger("paramiko").setLevel(logging.WARNING) self.ssh_client = paramiko.SSHClient() - base_url = six.moves.urllib_parse.urlparse(base_url) + base_url = urllib.parse.urlparse(base_url) self.ssh_params = { "hostname": base_url.hostname, "port": base_url.port, @@ -252,6 +249,6 @@ def get_connection(self, url, proxies=None): return pool def close(self): - super(SSHHTTPAdapter, self).close() + super().close() if self.ssh_client: self.ssh_client.close() diff --git a/docker/transport/ssladapter.py b/docker/transport/ssladapter.py index 12de76cdca..31e3014eab 100644 --- a/docker/transport/ssladapter.py +++ b/docker/transport/ssladapter.py @@ -36,7 +36,7 @@ def __init__(self, ssl_version=None, assert_hostname=None, self.ssl_version = ssl_version self.assert_hostname = assert_hostname self.assert_fingerprint = assert_fingerprint - super(SSLHTTPAdapter, self).__init__(**kwargs) + super().__init__(**kwargs) def init_poolmanager(self, connections, maxsize, block=False): kwargs = { @@ -59,7 +59,7 @@ def get_connection(self, *args, **kwargs): But we still need to take care of when there is a proxy poolmanager """ - conn = super(SSLHTTPAdapter, self).get_connection(*args, **kwargs) + conn = super().get_connection(*args, **kwargs) if conn.assert_hostname != self.assert_hostname: conn.assert_hostname = self.assert_hostname return conn diff --git a/docker/transport/unixconn.py b/docker/transport/unixconn.py index 3e040c5af8..adb6f18a1d 100644 --- a/docker/transport/unixconn.py +++ b/docker/transport/unixconn.py @@ -1,7 +1,6 @@ -import six import requests.adapters import socket -from six.moves import http_client as httplib +import http.client as httplib from docker.transport.basehttpadapter import BaseHTTPAdapter from .. import constants @@ -15,21 +14,10 @@ RecentlyUsedContainer = urllib3._collections.RecentlyUsedContainer -class UnixHTTPResponse(httplib.HTTPResponse, object): - def __init__(self, sock, *args, **kwargs): - disable_buffering = kwargs.pop('disable_buffering', False) - if six.PY2: - # FIXME: We may need to disable buffering on Py3 as well, - # but there's no clear way to do it at the moment. See: - # https://github.com/docker/docker-py/issues/1799 - kwargs['buffering'] = not disable_buffering - super(UnixHTTPResponse, self).__init__(sock, *args, **kwargs) - - -class UnixHTTPConnection(httplib.HTTPConnection, object): +class UnixHTTPConnection(httplib.HTTPConnection): def __init__(self, base_url, unix_socket, timeout=60): - super(UnixHTTPConnection, self).__init__( + super().__init__( 'localhost', timeout=timeout ) self.base_url = base_url @@ -44,7 +32,7 @@ def connect(self): self.sock = sock def putheader(self, header, *values): - super(UnixHTTPConnection, self).putheader(header, *values) + super().putheader(header, *values) if header == 'Connection' and 'Upgrade' in values: self.disable_buffering = True @@ -52,12 +40,12 @@ def response_class(self, sock, *args, **kwargs): if self.disable_buffering: kwargs['disable_buffering'] = True - return UnixHTTPResponse(sock, *args, **kwargs) + return httplib.HTTPResponse(sock, *args, **kwargs) class UnixHTTPConnectionPool(urllib3.connectionpool.HTTPConnectionPool): def __init__(self, base_url, socket_path, timeout=60, maxsize=10): - super(UnixHTTPConnectionPool, self).__init__( + super().__init__( 'localhost', timeout=timeout, maxsize=maxsize ) self.base_url = base_url @@ -89,7 +77,7 @@ def __init__(self, socket_url, timeout=60, self.pools = RecentlyUsedContainer( pool_connections, dispose_func=lambda p: p.close() ) - super(UnixHTTPAdapter, self).__init__() + super().__init__() def get_connection(self, url, proxies=None): with self.pools.lock: diff --git a/docker/types/base.py b/docker/types/base.py index 6891062313..8851f1e2cb 100644 --- a/docker/types/base.py +++ b/docker/types/base.py @@ -1,7 +1,4 @@ -import six - - class DictType(dict): def __init__(self, init): - for k, v in six.iteritems(init): + for k, v in init.items(): self[k] = v diff --git a/docker/types/containers.py b/docker/types/containers.py index 9fa4656ab8..f1b60b2d2f 100644 --- a/docker/types/containers.py +++ b/docker/types/containers.py @@ -1,5 +1,3 @@ -import six - from .. import errors from ..utils.utils import ( convert_port_bindings, convert_tmpfs_mounts, convert_volume_binds, @@ -10,7 +8,7 @@ from .healthcheck import Healthcheck -class LogConfigTypesEnum(object): +class LogConfigTypesEnum: _values = ( 'json-file', 'syslog', @@ -61,7 +59,7 @@ def __init__(self, **kwargs): if config and not isinstance(config, dict): raise ValueError("LogConfig.config must be a dictionary") - super(LogConfig, self).__init__({ + super().__init__({ 'Type': log_driver_type, 'Config': config }) @@ -117,13 +115,13 @@ def __init__(self, **kwargs): name = kwargs.get('name', kwargs.get('Name')) soft = kwargs.get('soft', kwargs.get('Soft')) hard = kwargs.get('hard', kwargs.get('Hard')) - if not isinstance(name, six.string_types): + if not isinstance(name, str): raise ValueError("Ulimit.name must be a string") if soft and not isinstance(soft, int): raise ValueError("Ulimit.soft must be an integer") if hard and not isinstance(hard, int): raise ValueError("Ulimit.hard must be an integer") - super(Ulimit, self).__init__({ + super().__init__({ 'Name': name, 'Soft': soft, 'Hard': hard @@ -184,7 +182,7 @@ def __init__(self, **kwargs): if driver is None: driver = '' - elif not isinstance(driver, six.string_types): + elif not isinstance(driver, str): raise ValueError('DeviceRequest.driver must be a string') if count is None: count = 0 @@ -203,7 +201,7 @@ def __init__(self, **kwargs): elif not isinstance(options, dict): raise ValueError('DeviceRequest.options must be a dict') - super(DeviceRequest, self).__init__({ + super().__init__({ 'Driver': driver, 'Count': count, 'DeviceIDs': device_ids, @@ -297,7 +295,7 @@ def __init__(self, version, binds=None, port_bindings=None, self['MemorySwappiness'] = mem_swappiness if shm_size is not None: - if isinstance(shm_size, six.string_types): + if isinstance(shm_size, str): shm_size = parse_bytes(shm_size) self['ShmSize'] = shm_size @@ -358,7 +356,7 @@ def __init__(self, version, binds=None, port_bindings=None, self['Devices'] = parse_devices(devices) if group_add: - self['GroupAdd'] = [six.text_type(grp) for grp in group_add] + self['GroupAdd'] = [str(grp) for grp in group_add] if dns is not None: self['Dns'] = dns @@ -378,11 +376,11 @@ def __init__(self, version, binds=None, port_bindings=None, if not isinstance(sysctls, dict): raise host_config_type_error('sysctls', sysctls, 'dict') self['Sysctls'] = {} - for k, v in six.iteritems(sysctls): - self['Sysctls'][k] = six.text_type(v) + for k, v in sysctls.items(): + self['Sysctls'][k] = str(v) if volumes_from is not None: - if isinstance(volumes_from, six.string_types): + if isinstance(volumes_from, str): volumes_from = volumes_from.split(',') self['VolumesFrom'] = volumes_from @@ -404,7 +402,7 @@ def __init__(self, version, binds=None, port_bindings=None, if isinstance(lxc_conf, dict): formatted = [] - for k, v in six.iteritems(lxc_conf): + for k, v in lxc_conf.items(): formatted.append({'Key': k, 'Value': str(v)}) lxc_conf = formatted @@ -559,7 +557,7 @@ def __init__(self, version, binds=None, port_bindings=None, self["PidsLimit"] = pids_limit if isolation: - if not isinstance(isolation, six.string_types): + if not isinstance(isolation, str): raise host_config_type_error('isolation', isolation, 'string') if version_lt(version, '1.24'): raise host_config_version_error('isolation', '1.24') @@ -609,7 +607,7 @@ def __init__(self, version, binds=None, port_bindings=None, self['CpuPercent'] = cpu_percent if nano_cpus: - if not isinstance(nano_cpus, six.integer_types): + if not isinstance(nano_cpus, int): raise host_config_type_error('nano_cpus', nano_cpus, 'int') if version_lt(version, '1.25'): raise host_config_version_error('nano_cpus', '1.25') @@ -699,17 +697,17 @@ def __init__( 'version 1.29' ) - if isinstance(command, six.string_types): + if isinstance(command, str): command = split_command(command) - if isinstance(entrypoint, six.string_types): + if isinstance(entrypoint, str): entrypoint = split_command(entrypoint) if isinstance(environment, dict): environment = format_environment(environment) if isinstance(labels, list): - labels = dict((lbl, six.text_type('')) for lbl in labels) + labels = {lbl: '' for lbl in labels} if isinstance(ports, list): exposed_ports = {} @@ -720,10 +718,10 @@ def __init__( if len(port_definition) == 2: proto = port_definition[1] port = port_definition[0] - exposed_ports['{0}/{1}'.format(port, proto)] = {} + exposed_ports[f'{port}/{proto}'] = {} ports = exposed_ports - if isinstance(volumes, six.string_types): + if isinstance(volumes, str): volumes = [volumes, ] if isinstance(volumes, list): @@ -752,7 +750,7 @@ def __init__( 'Hostname': hostname, 'Domainname': domainname, 'ExposedPorts': ports, - 'User': six.text_type(user) if user is not None else None, + 'User': str(user) if user is not None else None, 'Tty': tty, 'OpenStdin': stdin_open, 'StdinOnce': stdin_once, diff --git a/docker/types/daemon.py b/docker/types/daemon.py index af3e5bcb5e..10e8101447 100644 --- a/docker/types/daemon.py +++ b/docker/types/daemon.py @@ -8,7 +8,7 @@ from ..errors import DockerException -class CancellableStream(object): +class CancellableStream: """ Stream wrapper for real-time events, logs, etc. from the server. @@ -32,7 +32,7 @@ def __next__(self): return next(self._stream) except urllib3.exceptions.ProtocolError: raise StopIteration - except socket.error: + except OSError: raise StopIteration next = __next__ diff --git a/docker/types/healthcheck.py b/docker/types/healthcheck.py index 9815018db8..dfc88a9771 100644 --- a/docker/types/healthcheck.py +++ b/docker/types/healthcheck.py @@ -1,7 +1,5 @@ from .base import DictType -import six - class Healthcheck(DictType): """ @@ -31,7 +29,7 @@ class Healthcheck(DictType): """ def __init__(self, **kwargs): test = kwargs.get('test', kwargs.get('Test')) - if isinstance(test, six.string_types): + if isinstance(test, str): test = ["CMD-SHELL", test] interval = kwargs.get('interval', kwargs.get('Interval')) @@ -39,7 +37,7 @@ def __init__(self, **kwargs): retries = kwargs.get('retries', kwargs.get('Retries')) start_period = kwargs.get('start_period', kwargs.get('StartPeriod')) - super(Healthcheck, self).__init__({ + super().__init__({ 'Test': test, 'Interval': interval, 'Timeout': timeout, @@ -53,7 +51,7 @@ def test(self): @test.setter def test(self, value): - if isinstance(value, six.string_types): + if isinstance(value, str): value = ["CMD-SHELL", value] self['Test'] = value diff --git a/docker/types/services.py b/docker/types/services.py index 29498e9715..a6dd76e32e 100644 --- a/docker/types/services.py +++ b/docker/types/services.py @@ -1,5 +1,3 @@ -import six - from .. import errors from ..constants import IS_WINDOWS_PLATFORM from ..utils import ( @@ -121,7 +119,7 @@ def __init__(self, image, command=None, args=None, hostname=None, env=None, privileges=None, isolation=None, init=None): self['Image'] = image - if isinstance(command, six.string_types): + if isinstance(command, str): command = split_command(command) self['Command'] = command self['Args'] = args @@ -151,7 +149,7 @@ def __init__(self, image, command=None, args=None, hostname=None, env=None, if mounts is not None: parsed_mounts = [] for mount in mounts: - if isinstance(mount, six.string_types): + if isinstance(mount, str): parsed_mounts.append(Mount.parse_mount_string(mount)) else: # If mount already parsed @@ -224,7 +222,7 @@ def __init__(self, target, source, type='volume', read_only=False, self['Source'] = source if type not in ('bind', 'volume', 'tmpfs', 'npipe'): raise errors.InvalidArgument( - 'Unsupported mount type: "{}"'.format(type) + f'Unsupported mount type: "{type}"' ) self['Type'] = type self['ReadOnly'] = read_only @@ -260,7 +258,7 @@ def __init__(self, target, source, type='volume', read_only=False, elif type == 'tmpfs': tmpfs_opts = {} if tmpfs_mode: - if not isinstance(tmpfs_mode, six.integer_types): + if not isinstance(tmpfs_mode, int): raise errors.InvalidArgument( 'tmpfs_mode must be an integer' ) @@ -280,7 +278,7 @@ def parse_mount_string(cls, string): parts = string.split(':') if len(parts) > 3: raise errors.InvalidArgument( - 'Invalid mount format "{0}"'.format(string) + f'Invalid mount format "{string}"' ) if len(parts) == 1: return cls(target=parts[0], source=None) @@ -347,7 +345,7 @@ def _convert_generic_resources_dict(generic_resources): ' (found {})'.format(type(generic_resources)) ) resources = [] - for kind, value in six.iteritems(generic_resources): + for kind, value in generic_resources.items(): resource_type = None if isinstance(value, int): resource_type = 'DiscreteResourceSpec' @@ -443,7 +441,7 @@ class RollbackConfig(UpdateConfig): pass -class RestartConditionTypesEnum(object): +class RestartConditionTypesEnum: _values = ( 'none', 'on-failure', @@ -474,7 +472,7 @@ def __init__(self, condition=RestartConditionTypesEnum.NONE, delay=0, max_attempts=0, window=0): if condition not in self.condition_types._values: raise TypeError( - 'Invalid RestartPolicy condition {0}'.format(condition) + f'Invalid RestartPolicy condition {condition}' ) self['Condition'] = condition @@ -533,7 +531,7 @@ def convert_service_ports(ports): ) result = [] - for k, v in six.iteritems(ports): + for k, v in ports.items(): port_spec = { 'Protocol': 'tcp', 'PublishedPort': k diff --git a/docker/utils/build.py b/docker/utils/build.py index 5787cab0fd..ac060434de 100644 --- a/docker/utils/build.py +++ b/docker/utils/build.py @@ -4,8 +4,6 @@ import tarfile import tempfile -import six - from .fnmatch import fnmatch from ..constants import IS_WINDOWS_PLATFORM @@ -69,7 +67,7 @@ def create_archive(root, files=None, fileobj=None, gzip=False, t = tarfile.open(mode='w:gz' if gzip else 'w', fileobj=fileobj) if files is None: files = build_file_list(root) - extra_names = set(e[0] for e in extra_files) + extra_names = {e[0] for e in extra_files} for path in files: if path in extra_names: # Extra files override context files with the same name @@ -95,9 +93,9 @@ def create_archive(root, files=None, fileobj=None, gzip=False, try: with open(full_path, 'rb') as f: t.addfile(i, f) - except IOError: - raise IOError( - 'Can not read file in context: {}'.format(full_path) + except OSError: + raise OSError( + f'Can not read file in context: {full_path}' ) else: # Directories, FIFOs, symlinks... don't need to be read. @@ -119,12 +117,8 @@ def mkbuildcontext(dockerfile): t = tarfile.open(mode='w', fileobj=f) if isinstance(dockerfile, io.StringIO): dfinfo = tarfile.TarInfo('Dockerfile') - if six.PY3: - raise TypeError('Please use io.BytesIO to create in-memory ' - 'Dockerfiles with Python 3') - else: - dfinfo.size = len(dockerfile.getvalue()) - dockerfile.seek(0) + raise TypeError('Please use io.BytesIO to create in-memory ' + 'Dockerfiles with Python 3') elif isinstance(dockerfile, io.BytesIO): dfinfo = tarfile.TarInfo('Dockerfile') dfinfo.size = len(dockerfile.getvalue()) @@ -154,7 +148,7 @@ def walk(root, patterns, default=True): # Heavily based on # https://github.com/moby/moby/blob/master/pkg/fileutils/fileutils.go -class PatternMatcher(object): +class PatternMatcher: def __init__(self, patterns): self.patterns = list(filter( lambda p: p.dirs, [Pattern(p) for p in patterns] @@ -212,13 +206,12 @@ def rec_walk(current_dir): break if skip: continue - for sub in rec_walk(cur): - yield sub + yield from rec_walk(cur) return rec_walk(root) -class Pattern(object): +class Pattern: def __init__(self, pattern_str): self.exclusion = False if pattern_str.startswith('!'): diff --git a/docker/utils/config.py b/docker/utils/config.py index 82a0e2a5ec..8e24959a5d 100644 --- a/docker/utils/config.py +++ b/docker/utils/config.py @@ -18,11 +18,11 @@ def find_config_file(config_path=None): os.path.join(home_dir(), LEGACY_DOCKER_CONFIG_FILENAME), # 4 ])) - log.debug("Trying paths: {0}".format(repr(paths))) + log.debug(f"Trying paths: {repr(paths)}") for path in paths: if os.path.exists(path): - log.debug("Found file at path: {0}".format(path)) + log.debug(f"Found file at path: {path}") return path log.debug("No config file found") @@ -57,7 +57,7 @@ def load_general_config(config_path=None): try: with open(config_file) as f: return json.load(f) - except (IOError, ValueError) as e: + except (OSError, ValueError) as e: # In the case of a legacy `.dockercfg` file, we won't # be able to load any JSON data. log.debug(e) diff --git a/docker/utils/decorators.py b/docker/utils/decorators.py index c975d4b401..cf1baf496c 100644 --- a/docker/utils/decorators.py +++ b/docker/utils/decorators.py @@ -27,7 +27,7 @@ def decorator(f): def wrapper(self, *args, **kwargs): if utils.version_lt(self._version, version): raise errors.InvalidVersion( - '{0} is not available for version < {1}'.format( + '{} is not available for version < {}'.format( f.__name__, version ) ) diff --git a/docker/utils/fnmatch.py b/docker/utils/fnmatch.py index cc940a2e65..90e9f60f59 100644 --- a/docker/utils/fnmatch.py +++ b/docker/utils/fnmatch.py @@ -108,7 +108,7 @@ def translate(pat): stuff = '^' + stuff[1:] elif stuff[0] == '^': stuff = '\\' + stuff - res = '%s[%s]' % (res, stuff) + res = f'{res}[{stuff}]' else: res = res + re.escape(c) diff --git a/docker/utils/json_stream.py b/docker/utils/json_stream.py index addffdf2fb..f384175f75 100644 --- a/docker/utils/json_stream.py +++ b/docker/utils/json_stream.py @@ -1,11 +1,6 @@ -from __future__ import absolute_import -from __future__ import unicode_literals - import json import json.decoder -import six - from ..errors import StreamParseError @@ -20,7 +15,7 @@ def stream_as_text(stream): instead of byte streams. """ for data in stream: - if not isinstance(data, six.text_type): + if not isinstance(data, str): data = data.decode('utf-8', 'replace') yield data @@ -46,8 +41,8 @@ def json_stream(stream): return split_buffer(stream, json_splitter, json_decoder.decode) -def line_splitter(buffer, separator=u'\n'): - index = buffer.find(six.text_type(separator)) +def line_splitter(buffer, separator='\n'): + index = buffer.find(str(separator)) if index == -1: return None return buffer[:index + 1], buffer[index + 1:] @@ -61,7 +56,7 @@ def split_buffer(stream, splitter=None, decoder=lambda a: a): of the input. """ splitter = splitter or line_splitter - buffered = six.text_type('') + buffered = '' for data in stream_as_text(stream): buffered += data diff --git a/docker/utils/ports.py b/docker/utils/ports.py index 10b19d741c..e813936602 100644 --- a/docker/utils/ports.py +++ b/docker/utils/ports.py @@ -49,7 +49,7 @@ def port_range(start, end, proto, randomly_available_port=False): if not end: return [start + proto] if randomly_available_port: - return ['{}-{}'.format(start, end) + proto] + return [f'{start}-{end}' + proto] return [str(port) + proto for port in range(int(start), int(end) + 1)] diff --git a/docker/utils/socket.py b/docker/utils/socket.py index 7ba9505538..4a2076ec4a 100644 --- a/docker/utils/socket.py +++ b/docker/utils/socket.py @@ -4,8 +4,6 @@ import socket as pysocket import struct -import six - try: from ..transport import NpipeSocket except ImportError: @@ -27,16 +25,16 @@ def read(socket, n=4096): recoverable_errors = (errno.EINTR, errno.EDEADLK, errno.EWOULDBLOCK) - if six.PY3 and not isinstance(socket, NpipeSocket): + if not isinstance(socket, NpipeSocket): select.select([socket], [], []) try: if hasattr(socket, 'recv'): return socket.recv(n) - if six.PY3 and isinstance(socket, getattr(pysocket, 'SocketIO')): + if isinstance(socket, getattr(pysocket, 'SocketIO')): return socket.read(n) return os.read(socket.fileno(), n) - except EnvironmentError as e: + except OSError as e: if e.errno not in recoverable_errors: raise @@ -46,7 +44,7 @@ def read_exactly(socket, n): Reads exactly n bytes from socket Raises SocketError if there isn't enough data """ - data = six.binary_type() + data = bytes() while len(data) < n: next_data = read(socket, n - len(data)) if not next_data: @@ -134,7 +132,7 @@ def consume_socket_output(frames, demux=False): if demux is False: # If the streams are multiplexed, the generator returns strings, that # we just need to concatenate. - return six.binary_type().join(frames) + return bytes().join(frames) # If the streams are demultiplexed, the generator yields tuples # (stdout, stderr) @@ -166,4 +164,4 @@ def demux_adaptor(stream_id, data): elif stream_id == STDERR: return (None, data) else: - raise ValueError('{0} is not a valid stream'.format(stream_id)) + raise ValueError(f'{stream_id} is not a valid stream') diff --git a/docker/utils/utils.py b/docker/utils/utils.py index f703cbd342..f7c3dd7d82 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -136,13 +136,13 @@ def convert_volume_binds(binds): mode = 'rw' result.append( - str('{0}:{1}:{2}').format(k, bind, mode) + f'{k}:{bind}:{mode}' ) else: if isinstance(v, bytes): v = v.decode('utf-8') result.append( - str('{0}:{1}:rw').format(k, v) + f'{k}:{v}:rw' ) return result @@ -233,14 +233,14 @@ def parse_host(addr, is_win32=False, tls=False): if proto not in ('tcp', 'unix', 'npipe', 'ssh'): raise errors.DockerException( - "Invalid bind address protocol: {}".format(addr) + f"Invalid bind address protocol: {addr}" ) if proto == 'tcp' and not parsed_url.netloc: # "tcp://" is exceptionally disallowed by convention; # omitting a hostname for other protocols is fine raise errors.DockerException( - 'Invalid bind address format: {}'.format(addr) + f'Invalid bind address format: {addr}' ) if any([ @@ -248,7 +248,7 @@ def parse_host(addr, is_win32=False, tls=False): parsed_url.password ]): raise errors.DockerException( - 'Invalid bind address format: {}'.format(addr) + f'Invalid bind address format: {addr}' ) if parsed_url.path and proto == 'ssh': @@ -285,8 +285,8 @@ def parse_host(addr, is_win32=False, tls=False): proto = 'http+unix' if proto in ('http+unix', 'npipe'): - return "{}://{}".format(proto, path).rstrip('/') - return '{0}://{1}:{2}{3}'.format(proto, host, port, path).rstrip('/') + return f"{proto}://{path}".rstrip('/') + return f'{proto}://{host}:{port}{path}'.rstrip('/') def parse_devices(devices): @@ -297,7 +297,7 @@ def parse_devices(devices): continue if not isinstance(device, str): raise errors.DockerException( - 'Invalid device type {0}'.format(type(device)) + f'Invalid device type {type(device)}' ) device_mapping = device.split(':') if device_mapping: @@ -408,7 +408,7 @@ def parse_bytes(s): digits = float(digits_part) except ValueError: raise errors.DockerException( - 'Failed converting the string value for memory ({0}) to' + 'Failed converting the string value for memory ({}) to' ' an integer.'.format(digits_part) ) @@ -416,7 +416,7 @@ def parse_bytes(s): s = int(digits * units[suffix]) else: raise errors.DockerException( - 'The specified value for memory ({0}) should specify the' + 'The specified value for memory ({}) should specify the' ' units. The postfix should be one of the `b` `k` `m` `g`' ' characters'.format(s) ) @@ -428,7 +428,7 @@ def normalize_links(links): if isinstance(links, dict): links = iter(links.items()) - return ['{0}:{1}'.format(k, v) if v else k for k, v in sorted(links)] + return [f'{k}:{v}' if v else k for k, v in sorted(links)] def parse_env_file(env_file): @@ -438,7 +438,7 @@ def parse_env_file(env_file): """ environment = {} - with open(env_file, 'r') as f: + with open(env_file) as f: for line in f: if line[0] == '#': @@ -454,7 +454,7 @@ def parse_env_file(env_file): environment[k] = v else: raise errors.DockerException( - 'Invalid line in environment file {0}:\n{1}'.format( + 'Invalid line in environment file {}:\n{}'.format( env_file, line)) return environment @@ -471,7 +471,7 @@ def format_env(key, value): if isinstance(value, bytes): value = value.decode('utf-8') - return u'{key}={value}'.format(key=key, value=value) + return f'{key}={value}' return [format_env(*var) for var in iter(environment.items())] @@ -479,11 +479,11 @@ def format_extra_hosts(extra_hosts, task=False): # Use format dictated by Swarm API if container is part of a task if task: return [ - '{} {}'.format(v, k) for k, v in sorted(iter(extra_hosts.items())) + f'{v} {k}' for k, v in sorted(iter(extra_hosts.items())) ] return [ - '{}:{}'.format(k, v) for k, v in sorted(iter(extra_hosts.items())) + f'{k}:{v}' for k, v in sorted(iter(extra_hosts.items())) ] diff --git a/docker/version.py b/docker/version.py index bc09e63709..3554104128 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ version = "4.5.0-dev" -version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) +version_info = tuple(int(d) for d in version.split("-")[0].split(".")) diff --git a/docs/conf.py b/docs/conf.py index f46d1f76ea..2b0a719531 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # docker-sdk-python documentation build configuration file, created by # sphinx-quickstart on Wed Sep 14 15:48:58 2016. @@ -60,21 +59,21 @@ master_doc = 'index' # General information about the project. -project = u'Docker SDK for Python' +project = 'Docker SDK for Python' year = datetime.datetime.now().year -copyright = u'%d Docker Inc' % year -author = u'Docker Inc' +copyright = '%d Docker Inc' % year +author = 'Docker Inc' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # -with open('../docker/version.py', 'r') as vfile: +with open('../docker/version.py') as vfile: exec(vfile.read()) # The full version, including alpha/beta/rc tags. release = version # The short X.Y version. -version = '{}.{}'.format(version_info[0], version_info[1]) +version = f'{version_info[0]}.{version_info[1]}' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -283,8 +282,8 @@ # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - (master_doc, 'docker-sdk-python.tex', u'docker-sdk-python Documentation', - u'Docker Inc.', 'manual'), + (master_doc, 'docker-sdk-python.tex', 'docker-sdk-python Documentation', + 'Docker Inc.', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of @@ -325,7 +324,7 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ - (master_doc, 'docker-sdk-python', u'docker-sdk-python Documentation', + (master_doc, 'docker-sdk-python', 'docker-sdk-python Documentation', [author], 1) ] @@ -340,7 +339,7 @@ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - (master_doc, 'docker-sdk-python', u'docker-sdk-python Documentation', + (master_doc, 'docker-sdk-python', 'docker-sdk-python Documentation', author, 'docker-sdk-python', 'One line description of project.', 'Miscellaneous'), ] diff --git a/scripts/versions.py b/scripts/versions.py index 4bdcb74de2..75e5355ebf 100755 --- a/scripts/versions.py +++ b/scripts/versions.py @@ -52,8 +52,8 @@ def order(self): return (int(self.major), int(self.minor), int(self.patch)) + stage def __str__(self): - stage = '-{}'.format(self.stage) if self.stage else '' - edition = '-{}'.format(self.edition) if self.edition else '' + stage = f'-{self.stage}' if self.stage else '' + edition = f'-{self.edition}' if self.edition else '' return '.'.join(map(str, self[:3])) + edition + stage diff --git a/setup.py b/setup.py index ec1a51deb8..a966fea238 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -from __future__ import print_function import codecs import os diff --git a/tests/helpers.py b/tests/helpers.py index f344e1c333..63cbe2e63a 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -11,7 +11,6 @@ import docker import paramiko import pytest -import six def make_tree(dirs, files): @@ -54,7 +53,7 @@ def requires_api_version(version): return pytest.mark.skipif( docker.utils.version_lt(test_version, version), - reason="API version is too low (< {0})".format(version) + reason=f"API version is too low (< {version})" ) @@ -86,7 +85,7 @@ def wait_on_condition(condition, delay=0.1, timeout=40): def random_name(): - return u'dockerpytest_{0:x}'.format(random.getrandbits(64)) + return f'dockerpytest_{random.getrandbits(64):x}' def force_leave_swarm(client): @@ -105,11 +104,11 @@ def force_leave_swarm(client): def swarm_listen_addr(): - return '0.0.0.0:{0}'.format(random.randrange(10000, 25000)) + return f'0.0.0.0:{random.randrange(10000, 25000)}' def assert_cat_socket_detached_with_keys(sock, inputs): - if six.PY3 and hasattr(sock, '_sock'): + if hasattr(sock, '_sock'): sock = sock._sock for i in inputs: @@ -128,7 +127,7 @@ def assert_cat_socket_detached_with_keys(sock, inputs): # of the daemon no longer cause this to raise an error. try: sock.sendall(b'make sure the socket is closed\n') - except socket.error: + except OSError: return sock.sendall(b"make sure the socket is closed\n") diff --git a/tests/integration/api_build_test.py b/tests/integration/api_build_test.py index b830a106b9..ef48e12ed3 100644 --- a/tests/integration/api_build_test.py +++ b/tests/integration/api_build_test.py @@ -7,7 +7,6 @@ from docker.utils.proxy import ProxyConfig import pytest -import six from .base import BaseAPIIntegrationTest, TEST_IMG from ..helpers import random_name, requires_api_version, requires_experimental @@ -71,9 +70,8 @@ def test_build_streaming(self): assert len(logs) > 0 def test_build_from_stringio(self): - if six.PY3: - return - script = io.StringIO(six.text_type('\n').join([ + return + script = io.StringIO('\n'.join([ 'FROM busybox', 'RUN mkdir -p /tmp/test', 'EXPOSE 8080', @@ -83,8 +81,7 @@ def test_build_from_stringio(self): stream = self.client.build(fileobj=script) logs = '' for chunk in stream: - if six.PY3: - chunk = chunk.decode('utf-8') + chunk = chunk.decode('utf-8') logs += chunk assert logs != '' @@ -135,8 +132,7 @@ def test_build_with_dockerignore(self): self.client.wait(c) logs = self.client.logs(c) - if six.PY3: - logs = logs.decode('utf-8') + logs = logs.decode('utf-8') assert sorted(list(filter(None, logs.split('\n')))) == sorted([ '/test/#file.txt', @@ -340,8 +336,7 @@ def test_build_with_extra_hosts(self): assert self.client.inspect_image(img_name) ctnr = self.run_container(img_name, 'cat /hosts-file') logs = self.client.logs(ctnr) - if six.PY3: - logs = logs.decode('utf-8') + logs = logs.decode('utf-8') assert '127.0.0.1\textrahost.local.test' in logs assert '127.0.0.1\thello.world.test' in logs @@ -376,7 +371,7 @@ def test_build_stderr_data(self): snippet = 'Ancient Temple (Mystic Oriental Dream ~ Ancient Temple)' script = io.BytesIO(b'\n'.join([ b'FROM busybox', - 'RUN sh -c ">&2 echo \'{0}\'"'.format(snippet).encode('utf-8') + f'RUN sh -c ">&2 echo \'{snippet}\'"'.encode('utf-8') ])) stream = self.client.build( @@ -440,7 +435,7 @@ def test_build_gzip_custom_encoding(self): @requires_api_version('1.32') @requires_experimental(until=None) def test_build_invalid_platform(self): - script = io.BytesIO('FROM busybox\n'.encode('ascii')) + script = io.BytesIO(b'FROM busybox\n') with pytest.raises(errors.APIError) as excinfo: stream = self.client.build(fileobj=script, platform='foobar') diff --git a/tests/integration/api_client_test.py b/tests/integration/api_client_test.py index 9e348f3e3f..d1622fa88d 100644 --- a/tests/integration/api_client_test.py +++ b/tests/integration/api_client_test.py @@ -72,6 +72,6 @@ def test_resource_warnings(self): client.close() del client - assert len(w) == 0, "No warnings produced: {0}".format( + assert len(w) == 0, "No warnings produced: {}".format( w[0].message ) diff --git a/tests/integration/api_config_test.py b/tests/integration/api_config_test.py index 0ffd7675c8..72cbb431c9 100644 --- a/tests/integration/api_config_test.py +++ b/tests/integration/api_config_test.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - import docker import pytest @@ -31,7 +29,7 @@ def test_create_config(self): def test_create_config_unicode_data(self): config_id = self.client.create_config( - 'favorite_character', u'いざよいさくや' + 'favorite_character', 'いざよいさくや' ) self.tmp_configs.append(config_id) assert 'ID' in config_id diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index 3087045b20..9da2cfbf40 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -34,7 +34,7 @@ def test_list_containers(self): assert len(retrieved) == 1 retrieved = retrieved[0] assert 'Command' in retrieved - assert retrieved['Command'] == str('true') + assert retrieved['Command'] == 'true' assert 'Image' in retrieved assert re.search(r'alpine:.*', retrieved['Image']) assert 'Status' in retrieved @@ -104,10 +104,10 @@ def test_create_with_links(self): assert self.client.wait(container3_id)['StatusCode'] == 0 logs = self.client.logs(container3_id).decode('utf-8') - assert '{0}_NAME='.format(link_env_prefix1) in logs - assert '{0}_ENV_FOO=1'.format(link_env_prefix1) in logs - assert '{0}_NAME='.format(link_env_prefix2) in logs - assert '{0}_ENV_FOO=1'.format(link_env_prefix2) in logs + assert f'{link_env_prefix1}_NAME=' in logs + assert f'{link_env_prefix1}_ENV_FOO=1' in logs + assert f'{link_env_prefix2}_NAME=' in logs + assert f'{link_env_prefix2}_ENV_FOO=1' in logs def test_create_with_restart_policy(self): container = self.client.create_container( @@ -487,7 +487,7 @@ def test_create_with_uts_mode(self): ) class VolumeBindTest(BaseAPIIntegrationTest): def setUp(self): - super(VolumeBindTest, self).setUp() + super().setUp() self.mount_dest = '/mnt' @@ -618,7 +618,7 @@ class ArchiveTest(BaseAPIIntegrationTest): def test_get_file_archive_from_container(self): data = 'The Maid and the Pocket Watch of Blood' ctnr = self.client.create_container( - TEST_IMG, 'sh -c "echo {0} > /vol1/data.txt"'.format(data), + TEST_IMG, f'sh -c "echo {data} > /vol1/data.txt"', volumes=['/vol1'] ) self.tmp_containers.append(ctnr) @@ -636,7 +636,7 @@ def test_get_file_archive_from_container(self): def test_get_file_stat_from_container(self): data = 'The Maid and the Pocket Watch of Blood' ctnr = self.client.create_container( - TEST_IMG, 'sh -c "echo -n {0} > /vol1/data.txt"'.format(data), + TEST_IMG, f'sh -c "echo -n {data} > /vol1/data.txt"', volumes=['/vol1'] ) self.tmp_containers.append(ctnr) @@ -655,7 +655,7 @@ def test_copy_file_to_container(self): test_file.seek(0) ctnr = self.client.create_container( TEST_IMG, - 'cat {0}'.format( + 'cat {}'.format( os.path.join('/vol1/', os.path.basename(test_file.name)) ), volumes=['/vol1'] @@ -701,7 +701,7 @@ def test_rename_container(self): if version == '1.5.0': assert name == inspect['Name'] else: - assert '/{0}'.format(name) == inspect['Name'] + assert f'/{name}' == inspect['Name'] class StartContainerTest(BaseAPIIntegrationTest): @@ -807,7 +807,7 @@ class LogsTest(BaseAPIIntegrationTest): def test_logs(self): snippet = 'Flowering Nights (Sakuya Iyazoi)' container = self.client.create_container( - TEST_IMG, 'echo {0}'.format(snippet) + TEST_IMG, f'echo {snippet}' ) id = container['Id'] self.tmp_containers.append(id) @@ -821,7 +821,7 @@ def test_logs_tail_option(self): snippet = '''Line1 Line2''' container = self.client.create_container( - TEST_IMG, 'echo "{0}"'.format(snippet) + TEST_IMG, f'echo "{snippet}"' ) id = container['Id'] self.tmp_containers.append(id) @@ -834,7 +834,7 @@ def test_logs_tail_option(self): def test_logs_streaming_and_follow(self): snippet = 'Flowering Nights (Sakuya Iyazoi)' container = self.client.create_container( - TEST_IMG, 'echo {0}'.format(snippet) + TEST_IMG, f'echo {snippet}' ) id = container['Id'] self.tmp_containers.append(id) @@ -854,7 +854,7 @@ def test_logs_streaming_and_follow(self): def test_logs_streaming_and_follow_and_cancel(self): snippet = 'Flowering Nights (Sakuya Iyazoi)' container = self.client.create_container( - TEST_IMG, 'sh -c "echo \\"{0}\\" && sleep 3"'.format(snippet) + TEST_IMG, f'sh -c "echo \\"{snippet}\\" && sleep 3"' ) id = container['Id'] self.tmp_containers.append(id) @@ -872,7 +872,7 @@ def test_logs_streaming_and_follow_and_cancel(self): def test_logs_with_dict_instead_of_id(self): snippet = 'Flowering Nights (Sakuya Iyazoi)' container = self.client.create_container( - TEST_IMG, 'echo {0}'.format(snippet) + TEST_IMG, f'echo {snippet}' ) id = container['Id'] self.tmp_containers.append(id) @@ -885,7 +885,7 @@ def test_logs_with_dict_instead_of_id(self): def test_logs_with_tail_0(self): snippet = 'Flowering Nights (Sakuya Iyazoi)' container = self.client.create_container( - TEST_IMG, 'echo "{0}"'.format(snippet) + TEST_IMG, f'echo "{snippet}"' ) id = container['Id'] self.tmp_containers.append(id) @@ -899,7 +899,7 @@ def test_logs_with_tail_0(self): def test_logs_with_until(self): snippet = 'Shanghai Teahouse (Hong Meiling)' container = self.client.create_container( - TEST_IMG, 'echo "{0}"'.format(snippet) + TEST_IMG, f'echo "{snippet}"' ) self.tmp_containers.append(container) @@ -1095,7 +1095,7 @@ def test_top(self): self.client.start(container) res = self.client.top(container) if not IS_WINDOWS_PLATFORM: - assert res['Titles'] == [u'PID', u'USER', u'TIME', u'COMMAND'] + assert res['Titles'] == ['PID', 'USER', 'TIME', 'COMMAND'] assert len(res['Processes']) == 1 assert res['Processes'][0][-1] == 'sleep 60' self.client.kill(container) @@ -1113,7 +1113,7 @@ def test_top_with_psargs(self): self.client.start(container) res = self.client.top(container, '-eopid,user') - assert res['Titles'] == [u'PID', u'USER'] + assert res['Titles'] == ['PID', 'USER'] assert len(res['Processes']) == 1 assert res['Processes'][0][10] == 'sleep 60' @@ -1203,7 +1203,7 @@ def test_run_container_streaming(self): def test_run_container_reading_socket(self): line = 'hi there and stuff and things, words!' # `echo` appends CRLF, `printf` doesn't - command = "printf '{0}'".format(line) + command = f"printf '{line}'" container = self.client.create_container(TEST_IMG, command, detach=True, tty=False) self.tmp_containers.append(container) @@ -1487,7 +1487,7 @@ def test_remove_link(self): # Remove link linked_name = self.client.inspect_container(container2_id)['Name'][1:] - link_name = '%s/%s' % (linked_name, link_alias) + link_name = f'{linked_name}/{link_alias}' self.client.remove_container(link_name, link=True) # Link is gone diff --git a/tests/integration/api_exec_test.py b/tests/integration/api_exec_test.py index 554e8629e5..4d7748f5ee 100644 --- a/tests/integration/api_exec_test.py +++ b/tests/integration/api_exec_test.py @@ -239,7 +239,7 @@ class ExecDemuxTest(BaseAPIIntegrationTest): ) def setUp(self): - super(ExecDemuxTest, self).setUp() + super().setUp() self.container = self.client.create_container( TEST_IMG, 'cat', detach=True, stdin_open=True ) diff --git a/tests/integration/api_image_test.py b/tests/integration/api_image_test.py index d5f8989304..e30de46c04 100644 --- a/tests/integration/api_image_test.py +++ b/tests/integration/api_image_test.py @@ -265,7 +265,7 @@ def test_get_load_image(self): output = self.client.load_image(data) assert any([ line for line in output - if 'Loaded image: {}'.format(test_img) in line.get('stream', '') + if f'Loaded image: {test_img}' in line.get('stream', '') ]) @contextlib.contextmanager @@ -284,7 +284,7 @@ def do_GET(self): thread.setDaemon(True) thread.start() - yield 'http://%s:%s' % (socket.gethostname(), server.server_address[1]) + yield f'http://{socket.gethostname()}:{server.server_address[1]}' server.shutdown() @@ -350,7 +350,7 @@ def test_get_image_load_image(self): result = self.client.load_image(f.read()) success = False - result_line = 'Loaded image: {}\n'.format(TEST_IMG) + result_line = f'Loaded image: {TEST_IMG}\n' for data in result: print(data) if 'stream' in data: diff --git a/tests/integration/api_network_test.py b/tests/integration/api_network_test.py index af22da8d2d..2568138461 100644 --- a/tests/integration/api_network_test.py +++ b/tests/integration/api_network_test.py @@ -9,7 +9,7 @@ class TestNetworks(BaseAPIIntegrationTest): def tearDown(self): self.client.leave_swarm(force=True) - super(TestNetworks, self).tearDown() + super().tearDown() def create_network(self, *args, **kwargs): net_name = random_name() diff --git a/tests/integration/api_secret_test.py b/tests/integration/api_secret_test.py index b3d93b8fc1..fd98543414 100644 --- a/tests/integration/api_secret_test.py +++ b/tests/integration/api_secret_test.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - import docker import pytest @@ -31,7 +29,7 @@ def test_create_secret(self): def test_create_secret_unicode_data(self): secret_id = self.client.create_secret( - 'favorite_character', u'いざよいさくや' + 'favorite_character', 'いざよいさくや' ) self.tmp_secrets.append(secret_id) assert 'ID' in secret_id diff --git a/tests/integration/api_service_test.py b/tests/integration/api_service_test.py index 1bee46e563..19a6f15456 100644 --- a/tests/integration/api_service_test.py +++ b/tests/integration/api_service_test.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - import random import time @@ -30,10 +28,10 @@ def tearDown(self): self.client.remove_service(service['ID']) except docker.errors.APIError: pass - super(ServiceTest, self).tearDown() + super().tearDown() def get_service_name(self): - return 'dockerpytest_{0:x}'.format(random.getrandbits(64)) + return f'dockerpytest_{random.getrandbits(64):x}' def get_service_container(self, service_name, attempts=20, interval=0.5, include_stopped=False): @@ -54,7 +52,7 @@ def get_service_container(self, service_name, attempts=20, interval=0.5, def create_simple_service(self, name=None, labels=None): if name: - name = 'dockerpytest_{0}'.format(name) + name = f'dockerpytest_{name}' else: name = self.get_service_name() @@ -403,20 +401,20 @@ def test_create_service_with_placement(self): node_id = self.client.nodes()[0]['ID'] container_spec = docker.types.ContainerSpec(TEST_IMG, ['true']) task_tmpl = docker.types.TaskTemplate( - container_spec, placement=['node.id=={}'.format(node_id)] + container_spec, placement=[f'node.id=={node_id}'] ) name = self.get_service_name() svc_id = self.client.create_service(task_tmpl, name=name) svc_info = self.client.inspect_service(svc_id) assert 'Placement' in svc_info['Spec']['TaskTemplate'] assert (svc_info['Spec']['TaskTemplate']['Placement'] == - {'Constraints': ['node.id=={}'.format(node_id)]}) + {'Constraints': [f'node.id=={node_id}']}) def test_create_service_with_placement_object(self): node_id = self.client.nodes()[0]['ID'] container_spec = docker.types.ContainerSpec(TEST_IMG, ['true']) placemt = docker.types.Placement( - constraints=['node.id=={}'.format(node_id)] + constraints=[f'node.id=={node_id}'] ) task_tmpl = docker.types.TaskTemplate( container_spec, placement=placemt @@ -508,7 +506,7 @@ def test_create_service_with_endpoint_spec(self): assert port['TargetPort'] == 1990 assert port['Protocol'] == 'udp' else: - self.fail('Invalid port specification: {0}'.format(port)) + self.fail(f'Invalid port specification: {port}') assert len(ports) == 3 @@ -670,14 +668,14 @@ def test_create_service_with_secret(self): container = self.get_service_container(name) assert container is not None exec_id = self.client.exec_create( - container, 'cat /run/secrets/{0}'.format(secret_name) + container, f'cat /run/secrets/{secret_name}' ) assert self.client.exec_start(exec_id) == secret_data @requires_api_version('1.25') def test_create_service_with_unicode_secret(self): secret_name = 'favorite_touhou' - secret_data = u'東方花映塚' + secret_data = '東方花映塚' secret_id = self.client.create_secret(secret_name, secret_data) self.tmp_secrets.append(secret_id) secret_ref = docker.types.SecretReference(secret_id, secret_name) @@ -695,7 +693,7 @@ def test_create_service_with_unicode_secret(self): container = self.get_service_container(name) assert container is not None exec_id = self.client.exec_create( - container, 'cat /run/secrets/{0}'.format(secret_name) + container, f'cat /run/secrets/{secret_name}' ) container_secret = self.client.exec_start(exec_id) container_secret = container_secret.decode('utf-8') @@ -722,14 +720,14 @@ def test_create_service_with_config(self): container = self.get_service_container(name) assert container is not None exec_id = self.client.exec_create( - container, 'cat /{0}'.format(config_name) + container, f'cat /{config_name}' ) assert self.client.exec_start(exec_id) == config_data @requires_api_version('1.30') def test_create_service_with_unicode_config(self): config_name = 'favorite_touhou' - config_data = u'東方花映塚' + config_data = '東方花映塚' config_id = self.client.create_config(config_name, config_data) self.tmp_configs.append(config_id) config_ref = docker.types.ConfigReference(config_id, config_name) @@ -747,7 +745,7 @@ def test_create_service_with_unicode_config(self): container = self.get_service_container(name) assert container is not None exec_id = self.client.exec_create( - container, 'cat /{0}'.format(config_name) + container, f'cat /{config_name}' ) container_config = self.client.exec_start(exec_id) container_config = container_config.decode('utf-8') @@ -1136,7 +1134,7 @@ def test_update_service_with_defaults_endpoint_spec(self): assert port['TargetPort'] == 1990 assert port['Protocol'] == 'udp' else: - self.fail('Invalid port specification: {0}'.format(port)) + self.fail(f'Invalid port specification: {port}') assert len(ports) == 3 @@ -1163,7 +1161,7 @@ def test_update_service_with_defaults_endpoint_spec(self): assert port['TargetPort'] == 1990 assert port['Protocol'] == 'udp' else: - self.fail('Invalid port specification: {0}'.format(port)) + self.fail(f'Invalid port specification: {port}') assert len(ports) == 3 diff --git a/tests/integration/api_swarm_test.py b/tests/integration/api_swarm_test.py index f1cbc264e2..48c0592c62 100644 --- a/tests/integration/api_swarm_test.py +++ b/tests/integration/api_swarm_test.py @@ -8,7 +8,7 @@ class SwarmTest(BaseAPIIntegrationTest): def setUp(self): - super(SwarmTest, self).setUp() + super().setUp() force_leave_swarm(self.client) self._unlock_key = None @@ -19,7 +19,7 @@ def tearDown(self): except docker.errors.APIError: pass force_leave_swarm(self.client) - super(SwarmTest, self).tearDown() + super().tearDown() @requires_api_version('1.24') def test_init_swarm_simple(self): diff --git a/tests/integration/base.py b/tests/integration/base.py index a7613f6917..031079c917 100644 --- a/tests/integration/base.py +++ b/tests/integration/base.py @@ -75,11 +75,11 @@ class BaseAPIIntegrationTest(BaseIntegrationTest): """ def setUp(self): - super(BaseAPIIntegrationTest, self).setUp() + super().setUp() self.client = self.get_client_instance() def tearDown(self): - super(BaseAPIIntegrationTest, self).tearDown() + super().tearDown() self.client.close() @staticmethod diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index ec48835dcd..ae94595585 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -1,5 +1,3 @@ -from __future__ import print_function - import sys import warnings @@ -17,11 +15,11 @@ def setup_test_session(): try: c.inspect_image(TEST_IMG) except docker.errors.NotFound: - print("\npulling {0}".format(TEST_IMG), file=sys.stderr) + print(f"\npulling {TEST_IMG}", file=sys.stderr) for data in c.pull(TEST_IMG, stream=True, decode=True): status = data.get("status") progress = data.get("progress") - detail = "{0} - {1}".format(status, progress) + detail = f"{status} - {progress}" print(detail, file=sys.stderr) # Double make sure we now have busybox diff --git a/tests/integration/credentials/store_test.py b/tests/integration/credentials/store_test.py index dd543e24ad..d0cfd5417c 100644 --- a/tests/integration/credentials/store_test.py +++ b/tests/integration/credentials/store_test.py @@ -3,7 +3,6 @@ import sys import pytest -import six from distutils.spawn import find_executable from docker.credentials import ( @@ -12,7 +11,7 @@ ) -class TestStore(object): +class TestStore: def teardown_method(self): for server in self.tmp_keys: try: @@ -33,7 +32,7 @@ def setup_method(self): self.store = Store(DEFAULT_OSX_STORE) def get_random_servername(self): - res = 'pycreds_test_{:x}'.format(random.getrandbits(32)) + res = f'pycreds_test_{random.getrandbits(32):x}' self.tmp_keys.append(res) return res @@ -61,7 +60,7 @@ def test_store_and_erase(self): def test_unicode_strings(self): key = self.get_random_servername() - key = six.u(key) + key = key self.store.store(server=key, username='user', secret='pass') data = self.store.get(key) assert data diff --git a/tests/integration/credentials/utils_test.py b/tests/integration/credentials/utils_test.py index ad55f3216b..d7b2a1a4d5 100644 --- a/tests/integration/credentials/utils_test.py +++ b/tests/integration/credentials/utils_test.py @@ -5,7 +5,7 @@ try: from unittest import mock except ImportError: - import mock + from unittest import mock @mock.patch.dict(os.environ) diff --git a/tests/integration/models_images_test.py b/tests/integration/models_images_test.py index 0d60f37b08..94aa201004 100644 --- a/tests/integration/models_images_test.py +++ b/tests/integration/models_images_test.py @@ -13,8 +13,8 @@ class ImageCollectionTest(BaseIntegrationTest): def test_build(self): client = docker.from_env(version=TEST_API_VERSION) image, _ = client.images.build(fileobj=io.BytesIO( - "FROM alpine\n" - "CMD echo hello world".encode('ascii') + b"FROM alpine\n" + b"CMD echo hello world" )) self.tmp_imgs.append(image.id) assert client.containers.run(image) == b"hello world\n" @@ -24,8 +24,8 @@ def test_build_with_error(self): client = docker.from_env(version=TEST_API_VERSION) with pytest.raises(docker.errors.BuildError) as cm: client.images.build(fileobj=io.BytesIO( - "FROM alpine\n" - "RUN exit 1".encode('ascii') + b"FROM alpine\n" + b"RUN exit 1" )) assert ( "The command '/bin/sh -c exit 1' returned a non-zero code: 1" @@ -36,8 +36,8 @@ def test_build_with_multiple_success(self): client = docker.from_env(version=TEST_API_VERSION) image, _ = client.images.build( tag='some-tag', fileobj=io.BytesIO( - "FROM alpine\n" - "CMD echo hello world".encode('ascii') + b"FROM alpine\n" + b"CMD echo hello world" ) ) self.tmp_imgs.append(image.id) @@ -47,8 +47,8 @@ def test_build_with_success_build_output(self): client = docker.from_env(version=TEST_API_VERSION) image, _ = client.images.build( tag='dup-txt-tag', fileobj=io.BytesIO( - "FROM alpine\n" - "CMD echo Successfully built abcd1234".encode('ascii') + b"FROM alpine\n" + b"CMD echo Successfully built abcd1234" ) ) self.tmp_imgs.append(image.id) @@ -119,7 +119,7 @@ def test_save_and_load_repo_name(self): self.tmp_imgs.append(additional_tag) image.reload() with tempfile.TemporaryFile() as f: - stream = image.save(named='{}:latest'.format(additional_tag)) + stream = image.save(named=f'{additional_tag}:latest') for chunk in stream: f.write(chunk) @@ -129,7 +129,7 @@ def test_save_and_load_repo_name(self): assert len(result) == 1 assert result[0].id == image.id - assert '{}:latest'.format(additional_tag) in result[0].tags + assert f'{additional_tag}:latest' in result[0].tags def test_save_name_error(self): client = docker.from_env(version=TEST_API_VERSION) @@ -143,7 +143,7 @@ class ImageTest(BaseIntegrationTest): def test_tag_and_remove(self): repo = 'dockersdk.tests.images.test_tag' tag = 'some-tag' - identifier = '{}:{}'.format(repo, tag) + identifier = f'{repo}:{tag}' client = docker.from_env(version=TEST_API_VERSION) image = client.images.pull('alpine:latest') diff --git a/tests/integration/regression_test.py b/tests/integration/regression_test.py index a63883c4f5..deb9aff15a 100644 --- a/tests/integration/regression_test.py +++ b/tests/integration/regression_test.py @@ -2,7 +2,6 @@ import random import docker -import six from .base import BaseAPIIntegrationTest, TEST_IMG import pytest @@ -39,8 +38,7 @@ def test_715_handle_user_param_as_int_value(self): self.client.start(ctnr) self.client.wait(ctnr) logs = self.client.logs(ctnr) - if six.PY3: - logs = logs.decode('utf-8') + logs = logs.decode('utf-8') assert logs == '1000\n' def test_792_explicit_port_protocol(self): @@ -56,10 +54,10 @@ def test_792_explicit_port_protocol(self): self.client.start(ctnr) assert self.client.port( ctnr, 2000 - )[0]['HostPort'] == six.text_type(tcp_port) + )[0]['HostPort'] == str(tcp_port) assert self.client.port( ctnr, '2000/tcp' - )[0]['HostPort'] == six.text_type(tcp_port) + )[0]['HostPort'] == str(tcp_port) assert self.client.port( ctnr, '2000/udp' - )[0]['HostPort'] == six.text_type(udp_port) + )[0]['HostPort'] == str(udp_port) diff --git a/tests/ssh/api_build_test.py b/tests/ssh/api_build_test.py index b830a106b9..ef48e12ed3 100644 --- a/tests/ssh/api_build_test.py +++ b/tests/ssh/api_build_test.py @@ -7,7 +7,6 @@ from docker.utils.proxy import ProxyConfig import pytest -import six from .base import BaseAPIIntegrationTest, TEST_IMG from ..helpers import random_name, requires_api_version, requires_experimental @@ -71,9 +70,8 @@ def test_build_streaming(self): assert len(logs) > 0 def test_build_from_stringio(self): - if six.PY3: - return - script = io.StringIO(six.text_type('\n').join([ + return + script = io.StringIO('\n'.join([ 'FROM busybox', 'RUN mkdir -p /tmp/test', 'EXPOSE 8080', @@ -83,8 +81,7 @@ def test_build_from_stringio(self): stream = self.client.build(fileobj=script) logs = '' for chunk in stream: - if six.PY3: - chunk = chunk.decode('utf-8') + chunk = chunk.decode('utf-8') logs += chunk assert logs != '' @@ -135,8 +132,7 @@ def test_build_with_dockerignore(self): self.client.wait(c) logs = self.client.logs(c) - if six.PY3: - logs = logs.decode('utf-8') + logs = logs.decode('utf-8') assert sorted(list(filter(None, logs.split('\n')))) == sorted([ '/test/#file.txt', @@ -340,8 +336,7 @@ def test_build_with_extra_hosts(self): assert self.client.inspect_image(img_name) ctnr = self.run_container(img_name, 'cat /hosts-file') logs = self.client.logs(ctnr) - if six.PY3: - logs = logs.decode('utf-8') + logs = logs.decode('utf-8') assert '127.0.0.1\textrahost.local.test' in logs assert '127.0.0.1\thello.world.test' in logs @@ -376,7 +371,7 @@ def test_build_stderr_data(self): snippet = 'Ancient Temple (Mystic Oriental Dream ~ Ancient Temple)' script = io.BytesIO(b'\n'.join([ b'FROM busybox', - 'RUN sh -c ">&2 echo \'{0}\'"'.format(snippet).encode('utf-8') + f'RUN sh -c ">&2 echo \'{snippet}\'"'.encode('utf-8') ])) stream = self.client.build( @@ -440,7 +435,7 @@ def test_build_gzip_custom_encoding(self): @requires_api_version('1.32') @requires_experimental(until=None) def test_build_invalid_platform(self): - script = io.BytesIO('FROM busybox\n'.encode('ascii')) + script = io.BytesIO(b'FROM busybox\n') with pytest.raises(errors.APIError) as excinfo: stream = self.client.build(fileobj=script, platform='foobar') diff --git a/tests/ssh/base.py b/tests/ssh/base.py index c723d823bc..4825227f38 100644 --- a/tests/ssh/base.py +++ b/tests/ssh/base.py @@ -79,7 +79,7 @@ def setUpClass(cls): cls.client.pull(TEST_IMG) def tearDown(self): - super(BaseAPIIntegrationTest, self).tearDown() + super().tearDown() self.client.close() @staticmethod diff --git a/tests/unit/api_container_test.py b/tests/unit/api_container_test.py index 8a0577e78f..1ebd37df0a 100644 --- a/tests/unit/api_container_test.py +++ b/tests/unit/api_container_test.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - import datetime import json import signal @@ -7,7 +5,6 @@ import docker from docker.api import APIClient import pytest -import six from . import fake_api from ..helpers import requires_api_version @@ -19,7 +16,7 @@ try: from unittest import mock except ImportError: - import mock + from unittest import mock def fake_inspect_container_tty(self, container): @@ -771,7 +768,7 @@ def test_create_container_with_devices(self): def test_create_container_with_device_requests(self): client = APIClient(version='1.40') fake_api.fake_responses.setdefault( - '{0}/v1.40/containers/create'.format(fake_api.prefix), + f'{fake_api.prefix}/v1.40/containers/create', fake_api.post_fake_create_container, ) client.create_container( @@ -831,8 +828,8 @@ def test_create_container_with_device_requests(self): def test_create_container_with_labels_dict(self): labels_dict = { - six.text_type('foo'): six.text_type('1'), - six.text_type('bar'): six.text_type('2'), + 'foo': '1', + 'bar': '2', } self.client.create_container( @@ -848,12 +845,12 @@ def test_create_container_with_labels_dict(self): def test_create_container_with_labels_list(self): labels_list = [ - six.text_type('foo'), - six.text_type('bar'), + 'foo', + 'bar', ] labels_dict = { - six.text_type('foo'): six.text_type(), - six.text_type('bar'): six.text_type(), + 'foo': '', + 'bar': '', } self.client.create_container( @@ -1013,11 +1010,11 @@ def test_create_container_with_sysctl(self): def test_create_container_with_unicode_envvars(self): envvars_dict = { - 'foo': u'☃', + 'foo': '☃', } expected = [ - u'foo=☃' + 'foo=☃' ] self.client.create_container( @@ -1138,7 +1135,7 @@ def test_logs(self): stream=False ) - assert logs == 'Flowering Nights\n(Sakuya Iyazoi)\n'.encode('ascii') + assert logs == b'Flowering Nights\n(Sakuya Iyazoi)\n' def test_logs_with_dict_instead_of_id(self): with mock.patch('docker.api.client.APIClient.inspect_container', @@ -1154,7 +1151,7 @@ def test_logs_with_dict_instead_of_id(self): stream=False ) - assert logs == 'Flowering Nights\n(Sakuya Iyazoi)\n'.encode('ascii') + assert logs == b'Flowering Nights\n(Sakuya Iyazoi)\n' def test_log_streaming(self): with mock.patch('docker.api.client.APIClient.inspect_container', diff --git a/tests/unit/api_exec_test.py b/tests/unit/api_exec_test.py index a9d2dd5b65..4504250846 100644 --- a/tests/unit/api_exec_test.py +++ b/tests/unit/api_exec_test.py @@ -11,7 +11,7 @@ def test_exec_create(self): self.client.exec_create(fake_api.FAKE_CONTAINER_ID, ['ls', '-1']) args = fake_request.call_args - assert 'POST' == args[0][0], url_prefix + 'containers/{0}/exec'.format( + assert 'POST' == args[0][0], url_prefix + 'containers/{}/exec'.format( fake_api.FAKE_CONTAINER_ID ) @@ -32,7 +32,7 @@ def test_exec_start(self): self.client.exec_start(fake_api.FAKE_EXEC_ID) args = fake_request.call_args - assert args[0][1] == url_prefix + 'exec/{0}/start'.format( + assert args[0][1] == url_prefix + 'exec/{}/start'.format( fake_api.FAKE_EXEC_ID ) @@ -51,7 +51,7 @@ def test_exec_start_detached(self): self.client.exec_start(fake_api.FAKE_EXEC_ID, detach=True) args = fake_request.call_args - assert args[0][1] == url_prefix + 'exec/{0}/start'.format( + assert args[0][1] == url_prefix + 'exec/{}/start'.format( fake_api.FAKE_EXEC_ID ) @@ -68,7 +68,7 @@ def test_exec_inspect(self): self.client.exec_inspect(fake_api.FAKE_EXEC_ID) args = fake_request.call_args - assert args[0][1] == url_prefix + 'exec/{0}/json'.format( + assert args[0][1] == url_prefix + 'exec/{}/json'.format( fake_api.FAKE_EXEC_ID ) @@ -77,7 +77,7 @@ def test_exec_resize(self): fake_request.assert_called_with( 'POST', - url_prefix + 'exec/{0}/resize'.format(fake_api.FAKE_EXEC_ID), + url_prefix + f'exec/{fake_api.FAKE_EXEC_ID}/resize', params={'h': 20, 'w': 60}, timeout=DEFAULT_TIMEOUT_SECONDS ) diff --git a/tests/unit/api_image_test.py b/tests/unit/api_image_test.py index 0b60df43a7..843c11b841 100644 --- a/tests/unit/api_image_test.py +++ b/tests/unit/api_image_test.py @@ -11,7 +11,7 @@ try: from unittest import mock except ImportError: - import mock + from unittest import mock class ImageTest(BaseAPIClientTest): diff --git a/tests/unit/api_network_test.py b/tests/unit/api_network_test.py index 758f013230..84d6544969 100644 --- a/tests/unit/api_network_test.py +++ b/tests/unit/api_network_test.py @@ -1,14 +1,12 @@ import json -import six - from .api_test import BaseAPIClientTest, url_prefix, response from docker.types import IPAMConfig, IPAMPool try: from unittest import mock except ImportError: - import mock + from unittest import mock class NetworkTest(BaseAPIClientTest): @@ -103,16 +101,16 @@ def test_remove_network(self): self.client.remove_network(network_id) args = delete.call_args - assert args[0][0] == url_prefix + 'networks/{0}'.format(network_id) + assert args[0][0] == url_prefix + f'networks/{network_id}' def test_inspect_network(self): network_id = 'abc12345' network_name = 'foo' network_data = { - six.u('name'): network_name, - six.u('id'): network_id, - six.u('driver'): 'bridge', - six.u('containers'): {}, + 'name': network_name, + 'id': network_id, + 'driver': 'bridge', + 'containers': {}, } network_response = response(status_code=200, content=network_data) @@ -123,7 +121,7 @@ def test_inspect_network(self): assert result == network_data args = get.call_args - assert args[0][0] == url_prefix + 'networks/{0}'.format(network_id) + assert args[0][0] == url_prefix + f'networks/{network_id}' def test_connect_container_to_network(self): network_id = 'abc12345' @@ -141,7 +139,7 @@ def test_connect_container_to_network(self): ) assert post.call_args[0][0] == ( - url_prefix + 'networks/{0}/connect'.format(network_id) + url_prefix + f'networks/{network_id}/connect' ) assert json.loads(post.call_args[1]['data']) == { @@ -164,7 +162,7 @@ def test_disconnect_container_from_network(self): container={'Id': container_id}, net_id=network_id) assert post.call_args[0][0] == ( - url_prefix + 'networks/{0}/disconnect'.format(network_id) + url_prefix + f'networks/{network_id}/disconnect' ) assert json.loads(post.call_args[1]['data']) == { 'Container': container_id diff --git a/tests/unit/api_test.py b/tests/unit/api_test.py index cb14b74e11..dfc38164d4 100644 --- a/tests/unit/api_test.py +++ b/tests/unit/api_test.py @@ -10,11 +10,12 @@ import threading import time import unittest +import socketserver +import http.server import docker import pytest import requests -import six from docker.api import APIClient from docker.constants import DEFAULT_DOCKER_API_VERSION from requests.packages import urllib3 @@ -24,7 +25,7 @@ try: from unittest import mock except ImportError: - import mock + from unittest import mock DEFAULT_TIMEOUT_SECONDS = docker.constants.DEFAULT_TIMEOUT_SECONDS @@ -34,7 +35,7 @@ def response(status_code=200, content='', headers=None, reason=None, elapsed=0, request=None, raw=None): res = requests.Response() res.status_code = status_code - if not isinstance(content, six.binary_type): + if not isinstance(content, bytes): content = json.dumps(content).encode('ascii') res._content = content res.headers = requests.structures.CaseInsensitiveDict(headers or {}) @@ -60,7 +61,7 @@ def fake_resp(method, url, *args, **kwargs): elif (url, method) in fake_api.fake_responses: key = (url, method) if not key: - raise Exception('{0} {1}'.format(method, url)) + raise Exception(f'{method} {url}') status_code, content = fake_api.fake_responses[key]() return response(status_code=status_code, content=content) @@ -85,11 +86,11 @@ def fake_delete(self, url, *args, **kwargs): def fake_read_from_socket(self, response, stream, tty=False, demux=False): - return six.binary_type() + return bytes() -url_base = '{0}/'.format(fake_api.prefix) -url_prefix = '{0}v{1}/'.format( +url_base = f'{fake_api.prefix}/' +url_prefix = '{}v{}/'.format( url_base, docker.constants.DEFAULT_DOCKER_API_VERSION) @@ -133,20 +134,20 @@ def test_ctor(self): def test_url_valid_resource(self): url = self.client._url('/hello/{0}/world', 'somename') - assert url == '{0}{1}'.format(url_prefix, 'hello/somename/world') + assert url == '{}{}'.format(url_prefix, 'hello/somename/world') url = self.client._url( '/hello/{0}/world/{1}', 'somename', 'someothername' ) - assert url == '{0}{1}'.format( + assert url == '{}{}'.format( url_prefix, 'hello/somename/world/someothername' ) url = self.client._url('/hello/{0}/world', 'some?name') - assert url == '{0}{1}'.format(url_prefix, 'hello/some%3Fname/world') + assert url == '{}{}'.format(url_prefix, 'hello/some%3Fname/world') url = self.client._url("/images/{0}/push", "localhost:5000/image") - assert url == '{0}{1}'.format( + assert url == '{}{}'.format( url_prefix, 'images/localhost:5000/image/push' ) @@ -156,13 +157,13 @@ def test_url_invalid_resource(self): def test_url_no_resource(self): url = self.client._url('/simple') - assert url == '{0}{1}'.format(url_prefix, 'simple') + assert url == '{}{}'.format(url_prefix, 'simple') def test_url_unversioned_api(self): url = self.client._url( '/hello/{0}/world', 'somename', versioned_api=False ) - assert url == '{0}{1}'.format(url_base, 'hello/somename/world') + assert url == '{}{}'.format(url_base, 'hello/somename/world') def test_version(self): self.client.version() @@ -184,13 +185,13 @@ def test_version_no_api_version(self): def test_retrieve_server_version(self): client = APIClient(version="auto") - assert isinstance(client._version, six.string_types) + assert isinstance(client._version, str) assert not (client._version == "auto") client.close() def test_auto_retrieve_server_version(self): version = self.client._retrieve_server_version() - assert isinstance(version, six.string_types) + assert isinstance(version, str) def test_info(self): self.client.info() @@ -337,8 +338,7 @@ def test_create_host_config_secopt(self): def test_stream_helper_decoding(self): status_code, content = fake_api.fake_responses[url_prefix + 'events']() content_str = json.dumps(content) - if six.PY3: - content_str = content_str.encode('utf-8') + content_str = content_str.encode('utf-8') body = io.BytesIO(content_str) # mock a stream interface @@ -405,7 +405,7 @@ def run_server(self): while not self.stop_server: try: connection, client_address = self.server_socket.accept() - except socket.error: + except OSError: # Probably no connection to accept yet time.sleep(0.01) continue @@ -489,7 +489,7 @@ class TCPSocketStreamTest(unittest.TestCase): @classmethod def setup_class(cls): - cls.server = six.moves.socketserver.ThreadingTCPServer( + cls.server = socketserver.ThreadingTCPServer( ('', 0), cls.get_handler_class()) cls.thread = threading.Thread(target=cls.server.serve_forever) cls.thread.setDaemon(True) @@ -508,7 +508,7 @@ def get_handler_class(cls): stdout_data = cls.stdout_data stderr_data = cls.stderr_data - class Handler(six.moves.BaseHTTPServer.BaseHTTPRequestHandler, object): + class Handler(http.server.BaseHTTPRequestHandler): def do_POST(self): resp_data = self.get_resp_data() self.send_response(101) @@ -534,7 +534,7 @@ def get_resp_data(self): data += stderr_data return data else: - raise Exception('Unknown path {0}'.format(path)) + raise Exception(f'Unknown path {path}') @staticmethod def frame_header(stream, data): @@ -632,7 +632,7 @@ def test_custom_user_agent(self): class DisableSocketTest(unittest.TestCase): - class DummySocket(object): + class DummySocket: def __init__(self, timeout=60): self.timeout = timeout diff --git a/tests/unit/api_volume_test.py b/tests/unit/api_volume_test.py index 7850c224f2..a8d9193f75 100644 --- a/tests/unit/api_volume_test.py +++ b/tests/unit/api_volume_test.py @@ -104,7 +104,7 @@ def test_inspect_volume(self): args = fake_request.call_args assert args[0][0] == 'GET' - assert args[0][1] == '{0}volumes/{1}'.format(url_prefix, name) + assert args[0][1] == f'{url_prefix}volumes/{name}' def test_remove_volume(self): name = 'perfectcherryblossom' @@ -112,4 +112,4 @@ def test_remove_volume(self): args = fake_request.call_args assert args[0][0] == 'DELETE' - assert args[0][1] == '{0}volumes/{1}'.format(url_prefix, name) + assert args[0][1] == f'{url_prefix}volumes/{name}' diff --git a/tests/unit/auth_test.py b/tests/unit/auth_test.py index aac8910911..8bd2e1658b 100644 --- a/tests/unit/auth_test.py +++ b/tests/unit/auth_test.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - import base64 import json import os @@ -15,7 +13,7 @@ try: from unittest import mock except ImportError: - import mock + from unittest import mock class RegressionTest(unittest.TestCase): @@ -239,7 +237,7 @@ def test_load_legacy_config(self): cfg_path = os.path.join(folder, '.dockercfg') auth_ = base64.b64encode(b'sakuya:izayoi').decode('ascii') with open(cfg_path, 'w') as f: - f.write('auth = {0}\n'.format(auth_)) + f.write(f'auth = {auth_}\n') f.write('email = sakuya@scarlet.net') cfg = auth.load_config(cfg_path) @@ -297,13 +295,13 @@ def test_load_config_with_random_name(self): self.addCleanup(shutil.rmtree, folder) dockercfg_path = os.path.join(folder, - '.{0}.dockercfg'.format( + '.{}.dockercfg'.format( random.randrange(100000))) registry = 'https://your.private.registry.io' auth_ = base64.b64encode(b'sakuya:izayoi').decode('ascii') config = { registry: { - 'auth': '{0}'.format(auth_), + 'auth': f'{auth_}', 'email': 'sakuya@scarlet.net' } } @@ -329,7 +327,7 @@ def test_load_config_custom_config_env(self): auth_ = base64.b64encode(b'sakuya:izayoi').decode('ascii') config = { registry: { - 'auth': '{0}'.format(auth_), + 'auth': f'{auth_}', 'email': 'sakuya@scarlet.net' } } @@ -357,7 +355,7 @@ def test_load_config_custom_config_env_with_auths(self): config = { 'auths': { registry: { - 'auth': '{0}'.format(auth_), + 'auth': f'{auth_}', 'email': 'sakuya@scarlet.net' } } @@ -386,7 +384,7 @@ def test_load_config_custom_config_env_utf8(self): config = { 'auths': { registry: { - 'auth': '{0}'.format(auth_), + 'auth': f'{auth_}', 'email': 'sakuya@scarlet.net' } } @@ -794,9 +792,9 @@ def store(self, server, username, secret): } def list(self): - return dict( - [(k, v['Username']) for k, v in self.__store.items()] - ) + return { + k: v['Username'] for k, v in self.__store.items() + } def erase(self, server): del self.__store[server] diff --git a/tests/unit/client_test.py b/tests/unit/client_test.py index ad88e8456e..d647d3a1ae 100644 --- a/tests/unit/client_test.py +++ b/tests/unit/client_test.py @@ -15,7 +15,7 @@ try: from unittest import mock except ImportError: - import mock + from unittest import mock TEST_CERT_DIR = os.path.join(os.path.dirname(__file__), 'testdata/certs') POOL_SIZE = 20 diff --git a/tests/unit/dockertypes_test.py b/tests/unit/dockertypes_test.py index 0689d07b32..a0a171becd 100644 --- a/tests/unit/dockertypes_test.py +++ b/tests/unit/dockertypes_test.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - import unittest import pytest @@ -15,7 +13,7 @@ try: from unittest import mock except: # noqa: E722 - import mock + from unittest import mock def create_host_config(*args, **kwargs): diff --git a/tests/unit/errors_test.py b/tests/unit/errors_test.py index 54c2ba8f66..f8c3a6663d 100644 --- a/tests/unit/errors_test.py +++ b/tests/unit/errors_test.py @@ -126,7 +126,7 @@ def test_container_without_stderr(self): err = ContainerError(container, exit_status, command, image, stderr) msg = ("Command '{}' in image '{}' returned non-zero exit status {}" - ).format(command, image, exit_status, stderr) + ).format(command, image, exit_status) assert str(err) == msg def test_container_with_stderr(self): diff --git a/tests/unit/fake_api.py b/tests/unit/fake_api.py index 4fd4d11381..4c93329531 100644 --- a/tests/unit/fake_api.py +++ b/tests/unit/fake_api.py @@ -2,7 +2,7 @@ from . import fake_stat -CURRENT_VERSION = 'v{0}'.format(constants.DEFAULT_DOCKER_API_VERSION) +CURRENT_VERSION = f'v{constants.DEFAULT_DOCKER_API_VERSION}' FAKE_CONTAINER_ID = '3cc2351ab11b' FAKE_IMAGE_ID = 'e9aa60c60128' @@ -526,96 +526,96 @@ def post_fake_secret(): prefix = 'http+docker://localnpipe' fake_responses = { - '{0}/version'.format(prefix): + f'{prefix}/version': get_fake_version, - '{1}/{0}/version'.format(CURRENT_VERSION, prefix): + f'{prefix}/{CURRENT_VERSION}/version': get_fake_version, - '{1}/{0}/info'.format(CURRENT_VERSION, prefix): + f'{prefix}/{CURRENT_VERSION}/info': get_fake_info, - '{1}/{0}/auth'.format(CURRENT_VERSION, prefix): + f'{prefix}/{CURRENT_VERSION}/auth': post_fake_auth, - '{1}/{0}/_ping'.format(CURRENT_VERSION, prefix): + f'{prefix}/{CURRENT_VERSION}/_ping': get_fake_ping, - '{1}/{0}/images/search'.format(CURRENT_VERSION, prefix): + f'{prefix}/{CURRENT_VERSION}/images/search': get_fake_search, - '{1}/{0}/images/json'.format(CURRENT_VERSION, prefix): + f'{prefix}/{CURRENT_VERSION}/images/json': get_fake_images, - '{1}/{0}/images/test_image/history'.format(CURRENT_VERSION, prefix): + f'{prefix}/{CURRENT_VERSION}/images/test_image/history': get_fake_image_history, - '{1}/{0}/images/create'.format(CURRENT_VERSION, prefix): + f'{prefix}/{CURRENT_VERSION}/images/create': post_fake_import_image, - '{1}/{0}/containers/json'.format(CURRENT_VERSION, prefix): + f'{prefix}/{CURRENT_VERSION}/containers/json': get_fake_containers, - '{1}/{0}/containers/3cc2351ab11b/start'.format(CURRENT_VERSION, prefix): + f'{prefix}/{CURRENT_VERSION}/containers/3cc2351ab11b/start': post_fake_start_container, - '{1}/{0}/containers/3cc2351ab11b/resize'.format(CURRENT_VERSION, prefix): + f'{prefix}/{CURRENT_VERSION}/containers/3cc2351ab11b/resize': post_fake_resize_container, - '{1}/{0}/containers/3cc2351ab11b/json'.format(CURRENT_VERSION, prefix): + f'{prefix}/{CURRENT_VERSION}/containers/3cc2351ab11b/json': get_fake_inspect_container, - '{1}/{0}/containers/3cc2351ab11b/rename'.format(CURRENT_VERSION, prefix): + f'{prefix}/{CURRENT_VERSION}/containers/3cc2351ab11b/rename': post_fake_rename_container, - '{1}/{0}/images/e9aa60c60128/tag'.format(CURRENT_VERSION, prefix): + f'{prefix}/{CURRENT_VERSION}/images/e9aa60c60128/tag': post_fake_tag_image, - '{1}/{0}/containers/3cc2351ab11b/wait'.format(CURRENT_VERSION, prefix): + f'{prefix}/{CURRENT_VERSION}/containers/3cc2351ab11b/wait': get_fake_wait, - '{1}/{0}/containers/3cc2351ab11b/logs'.format(CURRENT_VERSION, prefix): + f'{prefix}/{CURRENT_VERSION}/containers/3cc2351ab11b/logs': get_fake_logs, - '{1}/{0}/containers/3cc2351ab11b/changes'.format(CURRENT_VERSION, prefix): + f'{prefix}/{CURRENT_VERSION}/containers/3cc2351ab11b/changes': get_fake_diff, - '{1}/{0}/containers/3cc2351ab11b/export'.format(CURRENT_VERSION, prefix): + f'{prefix}/{CURRENT_VERSION}/containers/3cc2351ab11b/export': get_fake_export, - '{1}/{0}/containers/3cc2351ab11b/update'.format(CURRENT_VERSION, prefix): + f'{prefix}/{CURRENT_VERSION}/containers/3cc2351ab11b/update': post_fake_update_container, - '{1}/{0}/containers/3cc2351ab11b/exec'.format(CURRENT_VERSION, prefix): + f'{prefix}/{CURRENT_VERSION}/containers/3cc2351ab11b/exec': post_fake_exec_create, - '{1}/{0}/exec/d5d177f121dc/start'.format(CURRENT_VERSION, prefix): + f'{prefix}/{CURRENT_VERSION}/exec/d5d177f121dc/start': post_fake_exec_start, - '{1}/{0}/exec/d5d177f121dc/json'.format(CURRENT_VERSION, prefix): + f'{prefix}/{CURRENT_VERSION}/exec/d5d177f121dc/json': get_fake_exec_inspect, - '{1}/{0}/exec/d5d177f121dc/resize'.format(CURRENT_VERSION, prefix): + f'{prefix}/{CURRENT_VERSION}/exec/d5d177f121dc/resize': post_fake_exec_resize, - '{1}/{0}/containers/3cc2351ab11b/stats'.format(CURRENT_VERSION, prefix): + f'{prefix}/{CURRENT_VERSION}/containers/3cc2351ab11b/stats': get_fake_stats, - '{1}/{0}/containers/3cc2351ab11b/top'.format(CURRENT_VERSION, prefix): + f'{prefix}/{CURRENT_VERSION}/containers/3cc2351ab11b/top': get_fake_top, - '{1}/{0}/containers/3cc2351ab11b/stop'.format(CURRENT_VERSION, prefix): + f'{prefix}/{CURRENT_VERSION}/containers/3cc2351ab11b/stop': post_fake_stop_container, - '{1}/{0}/containers/3cc2351ab11b/kill'.format(CURRENT_VERSION, prefix): + f'{prefix}/{CURRENT_VERSION}/containers/3cc2351ab11b/kill': post_fake_kill_container, - '{1}/{0}/containers/3cc2351ab11b/pause'.format(CURRENT_VERSION, prefix): + f'{prefix}/{CURRENT_VERSION}/containers/3cc2351ab11b/pause': post_fake_pause_container, - '{1}/{0}/containers/3cc2351ab11b/unpause'.format(CURRENT_VERSION, prefix): + f'{prefix}/{CURRENT_VERSION}/containers/3cc2351ab11b/unpause': post_fake_unpause_container, - '{1}/{0}/containers/3cc2351ab11b/restart'.format(CURRENT_VERSION, prefix): + f'{prefix}/{CURRENT_VERSION}/containers/3cc2351ab11b/restart': post_fake_restart_container, - '{1}/{0}/containers/3cc2351ab11b'.format(CURRENT_VERSION, prefix): + f'{prefix}/{CURRENT_VERSION}/containers/3cc2351ab11b': delete_fake_remove_container, - '{1}/{0}/images/create'.format(CURRENT_VERSION, prefix): + f'{prefix}/{CURRENT_VERSION}/images/create': post_fake_image_create, - '{1}/{0}/images/e9aa60c60128'.format(CURRENT_VERSION, prefix): + f'{prefix}/{CURRENT_VERSION}/images/e9aa60c60128': delete_fake_remove_image, - '{1}/{0}/images/e9aa60c60128/get'.format(CURRENT_VERSION, prefix): + f'{prefix}/{CURRENT_VERSION}/images/e9aa60c60128/get': get_fake_get_image, - '{1}/{0}/images/load'.format(CURRENT_VERSION, prefix): + f'{prefix}/{CURRENT_VERSION}/images/load': post_fake_load_image, - '{1}/{0}/images/test_image/json'.format(CURRENT_VERSION, prefix): + f'{prefix}/{CURRENT_VERSION}/images/test_image/json': get_fake_inspect_image, - '{1}/{0}/images/test_image/insert'.format(CURRENT_VERSION, prefix): + f'{prefix}/{CURRENT_VERSION}/images/test_image/insert': get_fake_insert_image, - '{1}/{0}/images/test_image/push'.format(CURRENT_VERSION, prefix): + f'{prefix}/{CURRENT_VERSION}/images/test_image/push': post_fake_push, - '{1}/{0}/commit'.format(CURRENT_VERSION, prefix): + f'{prefix}/{CURRENT_VERSION}/commit': post_fake_commit, - '{1}/{0}/containers/create'.format(CURRENT_VERSION, prefix): + f'{prefix}/{CURRENT_VERSION}/containers/create': post_fake_create_container, - '{1}/{0}/build'.format(CURRENT_VERSION, prefix): + f'{prefix}/{CURRENT_VERSION}/build': post_fake_build_container, - '{1}/{0}/events'.format(CURRENT_VERSION, prefix): + f'{prefix}/{CURRENT_VERSION}/events': get_fake_events, - ('{1}/{0}/volumes'.format(CURRENT_VERSION, prefix), 'GET'): + (f'{prefix}/{CURRENT_VERSION}/volumes', 'GET'): get_fake_volume_list, - ('{1}/{0}/volumes/create'.format(CURRENT_VERSION, prefix), 'POST'): + (f'{prefix}/{CURRENT_VERSION}/volumes/create', 'POST'): get_fake_volume, ('{1}/{0}/volumes/{2}'.format( CURRENT_VERSION, prefix, FAKE_VOLUME_NAME @@ -629,11 +629,11 @@ def post_fake_secret(): CURRENT_VERSION, prefix, FAKE_NODE_ID ), 'POST'): post_fake_update_node, - ('{1}/{0}/swarm/join'.format(CURRENT_VERSION, prefix), 'POST'): + (f'{prefix}/{CURRENT_VERSION}/swarm/join', 'POST'): post_fake_join_swarm, - ('{1}/{0}/networks'.format(CURRENT_VERSION, prefix), 'GET'): + (f'{prefix}/{CURRENT_VERSION}/networks', 'GET'): get_fake_network_list, - ('{1}/{0}/networks/create'.format(CURRENT_VERSION, prefix), 'POST'): + (f'{prefix}/{CURRENT_VERSION}/networks/create', 'POST'): post_fake_network, ('{1}/{0}/networks/{2}'.format( CURRENT_VERSION, prefix, FAKE_NETWORK_ID @@ -651,6 +651,6 @@ def post_fake_secret(): CURRENT_VERSION, prefix, FAKE_NETWORK_ID ), 'POST'): post_fake_network_disconnect, - '{1}/{0}/secrets/create'.format(CURRENT_VERSION, prefix): + f'{prefix}/{CURRENT_VERSION}/secrets/create': post_fake_secret, } diff --git a/tests/unit/fake_api_client.py b/tests/unit/fake_api_client.py index 5825b6ec00..1663ef1273 100644 --- a/tests/unit/fake_api_client.py +++ b/tests/unit/fake_api_client.py @@ -7,7 +7,7 @@ try: from unittest import mock except ImportError: - import mock + from unittest import mock class CopyReturnMagicMock(mock.MagicMock): @@ -15,7 +15,7 @@ class CopyReturnMagicMock(mock.MagicMock): A MagicMock which deep copies every return value. """ def _mock_call(self, *args, **kwargs): - ret = super(CopyReturnMagicMock, self)._mock_call(*args, **kwargs) + ret = super()._mock_call(*args, **kwargs) if isinstance(ret, (dict, list)): ret = copy.deepcopy(ret) return ret diff --git a/tests/unit/models_resources_test.py b/tests/unit/models_resources_test.py index 5af24ee69f..11dea29480 100644 --- a/tests/unit/models_resources_test.py +++ b/tests/unit/models_resources_test.py @@ -16,7 +16,7 @@ def test_reload(self): def test_hash(self): client = make_fake_client() container1 = client.containers.get(FAKE_CONTAINER_ID) - my_set = set([container1]) + my_set = {container1} assert len(my_set) == 1 container2 = client.containers.get(FAKE_CONTAINER_ID) diff --git a/tests/unit/models_secrets_test.py b/tests/unit/models_secrets_test.py index 4ccf4c6385..1c261a871f 100644 --- a/tests/unit/models_secrets_test.py +++ b/tests/unit/models_secrets_test.py @@ -8,4 +8,4 @@ class CreateServiceTest(unittest.TestCase): def test_secrets_repr(self): client = make_fake_client() secret = client.secrets.create(name="super_secret", data="secret") - assert secret.__repr__() == "".format(FAKE_SECRET_NAME) + assert secret.__repr__() == f"" diff --git a/tests/unit/models_services_test.py b/tests/unit/models_services_test.py index 07bb58970d..b9192e422b 100644 --- a/tests/unit/models_services_test.py +++ b/tests/unit/models_services_test.py @@ -40,10 +40,10 @@ def test_get_create_service_kwargs(self): 'update_config': {'update': 'config'}, 'endpoint_spec': {'blah': 'blah'}, } - assert set(task_template.keys()) == set([ + assert set(task_template.keys()) == { 'ContainerSpec', 'Resources', 'RestartPolicy', 'Placement', 'LogDriver', 'Networks' - ]) + } assert task_template['Placement'] == { 'Constraints': ['foo=bar'], 'Preferences': ['bar=baz'], @@ -55,7 +55,7 @@ def test_get_create_service_kwargs(self): 'Options': {'foo': 'bar'} } assert task_template['Networks'] == [{'Target': 'somenet'}] - assert set(task_template['ContainerSpec'].keys()) == set([ + assert set(task_template['ContainerSpec'].keys()) == { 'Image', 'Command', 'Args', 'Hostname', 'Env', 'Dir', 'User', 'Labels', 'Mounts', 'StopGracePeriod' - ]) + } diff --git a/tests/unit/ssladapter_test.py b/tests/unit/ssladapter_test.py index 73b73360c0..41a87f207e 100644 --- a/tests/unit/ssladapter_test.py +++ b/tests/unit/ssladapter_test.py @@ -32,30 +32,30 @@ def test_only_uses_tls(self): class MatchHostnameTest(unittest.TestCase): cert = { 'issuer': ( - (('countryName', u'US'),), - (('stateOrProvinceName', u'California'),), - (('localityName', u'San Francisco'),), - (('organizationName', u'Docker Inc'),), - (('organizationalUnitName', u'Docker-Python'),), - (('commonName', u'localhost'),), - (('emailAddress', u'info@docker.com'),) + (('countryName', 'US'),), + (('stateOrProvinceName', 'California'),), + (('localityName', 'San Francisco'),), + (('organizationName', 'Docker Inc'),), + (('organizationalUnitName', 'Docker-Python'),), + (('commonName', 'localhost'),), + (('emailAddress', 'info@docker.com'),) ), 'notAfter': 'Mar 25 23:08:23 2030 GMT', - 'notBefore': u'Mar 25 23:08:23 2016 GMT', - 'serialNumber': u'BD5F894C839C548F', + 'notBefore': 'Mar 25 23:08:23 2016 GMT', + 'serialNumber': 'BD5F894C839C548F', 'subject': ( - (('countryName', u'US'),), - (('stateOrProvinceName', u'California'),), - (('localityName', u'San Francisco'),), - (('organizationName', u'Docker Inc'),), - (('organizationalUnitName', u'Docker-Python'),), - (('commonName', u'localhost'),), - (('emailAddress', u'info@docker.com'),) + (('countryName', 'US'),), + (('stateOrProvinceName', 'California'),), + (('localityName', 'San Francisco'),), + (('organizationName', 'Docker Inc'),), + (('organizationalUnitName', 'Docker-Python'),), + (('commonName', 'localhost'),), + (('emailAddress', 'info@docker.com'),) ), 'subjectAltName': ( - ('DNS', u'localhost'), - ('DNS', u'*.gensokyo.jp'), - ('IP Address', u'127.0.0.1'), + ('DNS', 'localhost'), + ('DNS', '*.gensokyo.jp'), + ('IP Address', '127.0.0.1'), ), 'version': 3 } diff --git a/tests/unit/swarm_test.py b/tests/unit/swarm_test.py index 4385380281..aee1b9e802 100644 --- a/tests/unit/swarm_test.py +++ b/tests/unit/swarm_test.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - import json from . import fake_api diff --git a/tests/unit/utils_build_test.py b/tests/unit/utils_build_test.py index bc6fb5f409..9f183886b5 100644 --- a/tests/unit/utils_build_test.py +++ b/tests/unit/utils_build_test.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - import os import os.path import shutil @@ -82,7 +80,7 @@ def test_no_dupes(self): assert sorted(paths) == sorted(set(paths)) def test_wildcard_exclude(self): - assert self.exclude(['*']) == set(['Dockerfile', '.dockerignore']) + assert self.exclude(['*']) == {'Dockerfile', '.dockerignore'} def test_exclude_dockerfile_dockerignore(self): """ @@ -99,18 +97,18 @@ def test_exclude_custom_dockerfile(self): If we're using a custom Dockerfile, make sure that's not excluded. """ - assert self.exclude(['*'], dockerfile='Dockerfile.alt') == set( - ['Dockerfile.alt', '.dockerignore'] - ) + assert self.exclude(['*'], dockerfile='Dockerfile.alt') == { + 'Dockerfile.alt', '.dockerignore' + } assert self.exclude( ['*'], dockerfile='foo/Dockerfile3' - ) == convert_paths(set(['foo/Dockerfile3', '.dockerignore'])) + ) == convert_paths({'foo/Dockerfile3', '.dockerignore'}) # https://github.com/docker/docker-py/issues/1956 assert self.exclude( ['*'], dockerfile='./foo/Dockerfile3' - ) == convert_paths(set(['foo/Dockerfile3', '.dockerignore'])) + ) == convert_paths({'foo/Dockerfile3', '.dockerignore'}) def test_exclude_dockerfile_child(self): includes = self.exclude(['foo/'], dockerfile='foo/Dockerfile3') @@ -119,56 +117,56 @@ def test_exclude_dockerfile_child(self): def test_single_filename(self): assert self.exclude(['a.py']) == convert_paths( - self.all_paths - set(['a.py']) + self.all_paths - {'a.py'} ) def test_single_filename_leading_dot_slash(self): assert self.exclude(['./a.py']) == convert_paths( - self.all_paths - set(['a.py']) + self.all_paths - {'a.py'} ) # As odd as it sounds, a filename pattern with a trailing slash on the # end *will* result in that file being excluded. def test_single_filename_trailing_slash(self): assert self.exclude(['a.py/']) == convert_paths( - self.all_paths - set(['a.py']) + self.all_paths - {'a.py'} ) def test_wildcard_filename_start(self): assert self.exclude(['*.py']) == convert_paths( - self.all_paths - set(['a.py', 'b.py', 'cde.py']) + self.all_paths - {'a.py', 'b.py', 'cde.py'} ) def test_wildcard_with_exception(self): assert self.exclude(['*.py', '!b.py']) == convert_paths( - self.all_paths - set(['a.py', 'cde.py']) + self.all_paths - {'a.py', 'cde.py'} ) def test_wildcard_with_wildcard_exception(self): assert self.exclude(['*.*', '!*.go']) == convert_paths( - self.all_paths - set([ + self.all_paths - { 'a.py', 'b.py', 'cde.py', 'Dockerfile.alt', - ]) + } ) def test_wildcard_filename_end(self): assert self.exclude(['a.*']) == convert_paths( - self.all_paths - set(['a.py', 'a.go']) + self.all_paths - {'a.py', 'a.go'} ) def test_question_mark(self): assert self.exclude(['?.py']) == convert_paths( - self.all_paths - set(['a.py', 'b.py']) + self.all_paths - {'a.py', 'b.py'} ) def test_single_subdir_single_filename(self): assert self.exclude(['foo/a.py']) == convert_paths( - self.all_paths - set(['foo/a.py']) + self.all_paths - {'foo/a.py'} ) def test_single_subdir_single_filename_leading_slash(self): assert self.exclude(['/foo/a.py']) == convert_paths( - self.all_paths - set(['foo/a.py']) + self.all_paths - {'foo/a.py'} ) def test_exclude_include_absolute_path(self): @@ -176,57 +174,57 @@ def test_exclude_include_absolute_path(self): assert exclude_paths( base, ['/*', '!/*.py'] - ) == set(['a.py', 'b.py']) + ) == {'a.py', 'b.py'} def test_single_subdir_with_path_traversal(self): assert self.exclude(['foo/whoops/../a.py']) == convert_paths( - self.all_paths - set(['foo/a.py']) + self.all_paths - {'foo/a.py'} ) def test_single_subdir_wildcard_filename(self): assert self.exclude(['foo/*.py']) == convert_paths( - self.all_paths - set(['foo/a.py', 'foo/b.py']) + self.all_paths - {'foo/a.py', 'foo/b.py'} ) def test_wildcard_subdir_single_filename(self): assert self.exclude(['*/a.py']) == convert_paths( - self.all_paths - set(['foo/a.py', 'bar/a.py']) + self.all_paths - {'foo/a.py', 'bar/a.py'} ) def test_wildcard_subdir_wildcard_filename(self): assert self.exclude(['*/*.py']) == convert_paths( - self.all_paths - set(['foo/a.py', 'foo/b.py', 'bar/a.py']) + self.all_paths - {'foo/a.py', 'foo/b.py', 'bar/a.py'} ) def test_directory(self): assert self.exclude(['foo']) == convert_paths( - self.all_paths - set([ + self.all_paths - { 'foo', 'foo/a.py', 'foo/b.py', 'foo/bar', 'foo/bar/a.py', 'foo/Dockerfile3' - ]) + } ) def test_directory_with_trailing_slash(self): assert self.exclude(['foo']) == convert_paths( - self.all_paths - set([ + self.all_paths - { 'foo', 'foo/a.py', 'foo/b.py', 'foo/bar', 'foo/bar/a.py', 'foo/Dockerfile3' - ]) + } ) def test_directory_with_single_exception(self): assert self.exclude(['foo', '!foo/bar/a.py']) == convert_paths( - self.all_paths - set([ + self.all_paths - { 'foo/a.py', 'foo/b.py', 'foo', 'foo/bar', 'foo/Dockerfile3' - ]) + } ) def test_directory_with_subdir_exception(self): assert self.exclude(['foo', '!foo/bar']) == convert_paths( - self.all_paths - set([ + self.all_paths - { 'foo/a.py', 'foo/b.py', 'foo', 'foo/Dockerfile3' - ]) + } ) @pytest.mark.skipif( @@ -234,21 +232,21 @@ def test_directory_with_subdir_exception(self): ) def test_directory_with_subdir_exception_win32_pathsep(self): assert self.exclude(['foo', '!foo\\bar']) == convert_paths( - self.all_paths - set([ + self.all_paths - { 'foo/a.py', 'foo/b.py', 'foo', 'foo/Dockerfile3' - ]) + } ) def test_directory_with_wildcard_exception(self): assert self.exclude(['foo', '!foo/*.py']) == convert_paths( - self.all_paths - set([ + self.all_paths - { 'foo/bar', 'foo/bar/a.py', 'foo', 'foo/Dockerfile3' - ]) + } ) def test_subdirectory(self): assert self.exclude(['foo/bar']) == convert_paths( - self.all_paths - set(['foo/bar', 'foo/bar/a.py']) + self.all_paths - {'foo/bar', 'foo/bar/a.py'} ) @pytest.mark.skipif( @@ -256,33 +254,33 @@ def test_subdirectory(self): ) def test_subdirectory_win32_pathsep(self): assert self.exclude(['foo\\bar']) == convert_paths( - self.all_paths - set(['foo/bar', 'foo/bar/a.py']) + self.all_paths - {'foo/bar', 'foo/bar/a.py'} ) def test_double_wildcard(self): assert self.exclude(['**/a.py']) == convert_paths( - self.all_paths - set( - ['a.py', 'foo/a.py', 'foo/bar/a.py', 'bar/a.py'] - ) + self.all_paths - { + 'a.py', 'foo/a.py', 'foo/bar/a.py', 'bar/a.py' + } ) assert self.exclude(['foo/**/bar']) == convert_paths( - self.all_paths - set(['foo/bar', 'foo/bar/a.py']) + self.all_paths - {'foo/bar', 'foo/bar/a.py'} ) def test_single_and_double_wildcard(self): assert self.exclude(['**/target/*/*']) == convert_paths( - self.all_paths - set( - ['target/subdir/file.txt', + self.all_paths - { + 'target/subdir/file.txt', 'subdir/target/subdir/file.txt', - 'subdir/subdir2/target/subdir/file.txt'] - ) + 'subdir/subdir2/target/subdir/file.txt' + } ) def test_trailing_double_wildcard(self): assert self.exclude(['subdir/**']) == convert_paths( - self.all_paths - set( - ['subdir/file.txt', + self.all_paths - { + 'subdir/file.txt', 'subdir/target/file.txt', 'subdir/target/subdir/file.txt', 'subdir/subdir2/file.txt', @@ -292,16 +290,16 @@ def test_trailing_double_wildcard(self): 'subdir/target/subdir', 'subdir/subdir2', 'subdir/subdir2/target', - 'subdir/subdir2/target/subdir'] - ) + 'subdir/subdir2/target/subdir' + } ) def test_double_wildcard_with_exception(self): assert self.exclude(['**', '!bar', '!foo/bar']) == convert_paths( - set([ + { 'foo/bar', 'foo/bar/a.py', 'bar', 'bar/a.py', 'Dockerfile', '.dockerignore', - ]) + } ) def test_include_wildcard(self): @@ -324,7 +322,7 @@ def test_last_line_precedence(self): assert exclude_paths( base, ['*.md', '!README*.md', 'README-secret.md'] - ) == set(['README.md', 'README-bis.md']) + ) == {'README.md', 'README-bis.md'} def test_parent_directory(self): base = make_tree( @@ -340,7 +338,7 @@ def test_parent_directory(self): assert exclude_paths( base, ['../a.py', '/../b.py'] - ) == set(['c.py']) + ) == {'c.py'} class TarTest(unittest.TestCase): @@ -374,14 +372,14 @@ def test_tar_with_excludes(self): '.dockerignore', ] - expected_names = set([ + expected_names = { 'Dockerfile', '.dockerignore', 'a.go', 'b.py', 'bar', 'bar/a.py', - ]) + } base = make_tree(dirs, files) self.addCleanup(shutil.rmtree, base) @@ -413,7 +411,7 @@ def test_tar_with_inaccessible_file(self): with pytest.raises(IOError) as ei: tar(base) - assert 'Can not read file in context: {}'.format(full_path) in ( + assert f'Can not read file in context: {full_path}' in ( ei.exconly() ) diff --git a/tests/unit/utils_config_test.py b/tests/unit/utils_config_test.py index b0934f9568..83e04a146f 100644 --- a/tests/unit/utils_config_test.py +++ b/tests/unit/utils_config_test.py @@ -11,7 +11,7 @@ try: from unittest import mock except ImportError: - import mock + from unittest import mock class FindConfigFileTest(unittest.TestCase): diff --git a/tests/unit/utils_json_stream_test.py b/tests/unit/utils_json_stream_test.py index f7aefd0f18..821ebe42d4 100644 --- a/tests/unit/utils_json_stream_test.py +++ b/tests/unit/utils_json_stream_test.py @@ -1,11 +1,7 @@ -# encoding: utf-8 -from __future__ import absolute_import -from __future__ import unicode_literals - from docker.utils.json_stream import json_splitter, stream_as_text, json_stream -class TestJsonSplitter(object): +class TestJsonSplitter: def test_json_splitter_no_object(self): data = '{"foo": "bar' @@ -20,7 +16,7 @@ def test_json_splitter_leading_whitespace(self): assert json_splitter(data) == ({'foo': 'bar'}, '{"next": "obj"}') -class TestStreamAsText(object): +class TestStreamAsText: def test_stream_with_non_utf_unicode_character(self): stream = [b'\xed\xf3\xf3'] @@ -28,12 +24,12 @@ def test_stream_with_non_utf_unicode_character(self): assert output == '���' def test_stream_with_utf_character(self): - stream = ['ěĝ'.encode('utf-8')] + stream = ['ěĝ'.encode()] output, = stream_as_text(stream) assert output == 'ěĝ' -class TestJsonStream(object): +class TestJsonStream: def test_with_falsy_entries(self): stream = [ diff --git a/tests/unit/utils_proxy_test.py b/tests/unit/utils_proxy_test.py index ff0e14ba74..2da60401d6 100644 --- a/tests/unit/utils_proxy_test.py +++ b/tests/unit/utils_proxy_test.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- - import unittest -import six from docker.utils.proxy import ProxyConfig @@ -65,7 +62,7 @@ def test_inject_proxy_environment(self): # Proxy config is non null, env is None. self.assertSetEqual( set(CONFIG.inject_proxy_environment(None)), - set(['{}={}'.format(k, v) for k, v in six.iteritems(ENV)])) + {f'{k}={v}' for k, v in ENV.items()}) # Proxy config is null, env is None. self.assertIsNone(ProxyConfig().inject_proxy_environment(None), None) @@ -74,7 +71,7 @@ def test_inject_proxy_environment(self): # Proxy config is non null, env is non null actual = CONFIG.inject_proxy_environment(env) - expected = ['{}={}'.format(k, v) for k, v in six.iteritems(ENV)] + env + expected = [f'{k}={v}' for k, v in ENV.items()] + env # It's important that the first 8 variables are the ones from the proxy # config, and the last 2 are the ones from the input environment self.assertSetEqual(set(actual[:8]), set(expected[:8])) diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index 0d6ff22d7e..802d91962a 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - import base64 import json import os @@ -9,7 +7,6 @@ import unittest import pytest -import six from docker.api.client import APIClient from docker.constants import IS_WINDOWS_PLATFORM, DEFAULT_DOCKER_API_VERSION from docker.errors import DockerException @@ -195,22 +192,22 @@ def test_convert_volume_binds_no_mode(self): assert convert_volume_binds(data) == ['/mnt/vol1:/data:rw'] def test_convert_volume_binds_unicode_bytes_input(self): - expected = [u'/mnt/지연:/unicode/박:rw'] + expected = ['/mnt/지연:/unicode/박:rw'] data = { - u'/mnt/지연'.encode('utf-8'): { - 'bind': u'/unicode/박'.encode('utf-8'), + '/mnt/지연'.encode(): { + 'bind': '/unicode/박'.encode(), 'mode': 'rw' } } assert convert_volume_binds(data) == expected def test_convert_volume_binds_unicode_unicode_input(self): - expected = [u'/mnt/지연:/unicode/박:rw'] + expected = ['/mnt/지연:/unicode/박:rw'] data = { - u'/mnt/지연': { - 'bind': u'/unicode/박', + '/mnt/지연': { + 'bind': '/unicode/박', 'mode': 'rw' } } @@ -359,14 +356,14 @@ def test_private_reg_image_tag(self): ) def test_index_image_sha(self): - assert parse_repository_tag("root@sha256:{0}".format(self.sha)) == ( - "root", "sha256:{0}".format(self.sha) + assert parse_repository_tag(f"root@sha256:{self.sha}") == ( + "root", f"sha256:{self.sha}" ) def test_private_reg_image_sha(self): assert parse_repository_tag( - "url:5000/repo@sha256:{0}".format(self.sha) - ) == ("url:5000/repo", "sha256:{0}".format(self.sha)) + f"url:5000/repo@sha256:{self.sha}" + ) == ("url:5000/repo", f"sha256:{self.sha}") class ParseDeviceTest(unittest.TestCase): @@ -463,20 +460,13 @@ def test_convert_filters(self): def test_decode_json_header(self): obj = {'a': 'b', 'c': 1} data = None - if six.PY3: - data = base64.urlsafe_b64encode(bytes(json.dumps(obj), 'utf-8')) - else: - data = base64.urlsafe_b64encode(json.dumps(obj)) + data = base64.urlsafe_b64encode(bytes(json.dumps(obj), 'utf-8')) decoded_data = decode_json_header(data) assert obj == decoded_data class SplitCommandTest(unittest.TestCase): def test_split_command_with_unicode(self): - assert split_command(u'echo μμ') == ['echo', 'μμ'] - - @pytest.mark.skipif(six.PY3, reason="shlex doesn't support bytes in py3") - def test_split_command_with_bytes(self): assert split_command('echo μμ') == ['echo', 'μμ'] @@ -626,7 +616,7 @@ def test_format_env_binary_unicode_value(self): env_dict = { 'ARTIST_NAME': b'\xec\x86\xa1\xec\xa7\x80\xec\x9d\x80' } - assert format_environment(env_dict) == [u'ARTIST_NAME=송지은'] + assert format_environment(env_dict) == ['ARTIST_NAME=송지은'] def test_format_env_no_value(self): env_dict = { From 19d6cd8a015f1484e147a0bb9d0b4684c2a6aaac Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Fri, 6 Aug 2021 09:32:42 -0300 Subject: [PATCH 1120/1301] Bump requests => 2.26.0 Signed-off-by: Ulysses Souza --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 1d0be30a16..f6b17fd5f5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,6 +12,6 @@ pycparser==2.17 pyOpenSSL==18.0.0 pyparsing==2.2.0 pywin32==227; sys_platform == 'win32' -requests==2.20.0 +requests==2.26.0 urllib3==1.24.3 websocket-client==0.56.0 From 582f6277ce4dfe67b5be5a52b88bbdef3f349e11 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 6 Aug 2021 12:46:56 +0000 Subject: [PATCH 1121/1301] Bump urllib3 from 1.24.3 to 1.26.5 Bumps [urllib3](https://github.com/urllib3/urllib3) from 1.24.3 to 1.26.5. - [Release notes](https://github.com/urllib3/urllib3/releases) - [Changelog](https://github.com/urllib3/urllib3/blob/main/CHANGES.rst) - [Commits](https://github.com/urllib3/urllib3/compare/1.24.3...1.26.5) --- updated-dependencies: - dependency-name: urllib3 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f6b17fd5f5..42af699be1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,5 +13,5 @@ pyOpenSSL==18.0.0 pyparsing==2.2.0 pywin32==227; sys_platform == 'win32' requests==2.26.0 -urllib3==1.24.3 +urllib3==1.26.5 websocket-client==0.56.0 From e0d186d754693feb7d27c2352e455c5febb4a5cd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Aug 2021 20:57:04 +0000 Subject: [PATCH 1122/1301] Bump pywin32 from 227 to 301 Bumps [pywin32](https://github.com/mhammond/pywin32) from 227 to 301. - [Release notes](https://github.com/mhammond/pywin32/releases) - [Changelog](https://github.com/mhammond/pywin32/blob/master/CHANGES.txt) - [Commits](https://github.com/mhammond/pywin32/commits) --- updated-dependencies: - dependency-name: pywin32 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 42af699be1..26cbc6fb4b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,7 @@ paramiko==2.4.2 pycparser==2.17 pyOpenSSL==18.0.0 pyparsing==2.2.0 -pywin32==227; sys_platform == 'win32' +pywin32==301; sys_platform == 'win32' requests==2.26.0 urllib3==1.26.5 websocket-client==0.56.0 From 2fa56879a2f978387d230db087003d79eb2762d0 Mon Sep 17 00:00:00 2001 From: sinarostami Date: Mon, 16 Aug 2021 00:06:45 +0430 Subject: [PATCH 1123/1301] Improve containers documentation Signed-off-by: sinarostami --- docker/models/containers.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docker/models/containers.py b/docker/models/containers.py index 36cbbc41ad..957deed46d 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -761,6 +761,14 @@ def run(self, image, command=None, stdout=True, stderr=False, {'/home/user1/': {'bind': '/mnt/vol2', 'mode': 'rw'}, '/var/www': {'bind': '/mnt/vol1', 'mode': 'ro'}} + Or a list of strings which each one of its elements specifies a mount volume. + + For example: + + .. code-block:: python + + ['/home/user1/:/mnt/vol2','/var/www:/mnt/vol1'] + volumes_from (:py:class:`list`): List of container names or IDs to get volumes from. working_dir (str): Path to the working directory. From 8da03e01265f229a91aaffb7af2aa2057e08f1b9 Mon Sep 17 00:00:00 2001 From: Shehzaman Date: Thu, 27 May 2021 00:11:38 +0530 Subject: [PATCH 1124/1301] Put back identityfile parameter Signed-off-by: Shehzaman --- docker/transport/sshconn.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docker/transport/sshconn.py b/docker/transport/sshconn.py index fb5c6bbe8a..e08e3a8687 100644 --- a/docker/transport/sshconn.py +++ b/docker/transport/sshconn.py @@ -205,7 +205,6 @@ def _create_paramiko_client(self, base_url): with open(ssh_config_file) as f: conf.parse(f) host_config = conf.lookup(base_url.hostname) - self.ssh_conf = host_config if 'proxycommand' in host_config: self.ssh_params["sock"] = paramiko.ProxyCommand( self.ssh_conf['proxycommand'] @@ -213,9 +212,11 @@ def _create_paramiko_client(self, base_url): if 'hostname' in host_config: self.ssh_params['hostname'] = host_config['hostname'] if base_url.port is None and 'port' in host_config: - self.ssh_params['port'] = self.ssh_conf['port'] + self.ssh_params['port'] = host_config['port'] if base_url.username is None and 'user' in host_config: - self.ssh_params['username'] = self.ssh_conf['user'] + self.ssh_params['username'] = host_config['user'] + if 'identityfile' in host_config: + self.ssh_params['key_filename'] = host_config['identityfile'] self.ssh_client.load_system_host_keys() self.ssh_client.set_missing_host_key_policy(paramiko.WarningPolicy()) From 4a3cddf4bf926f3aa0d46d5f0318dbb212231377 Mon Sep 17 00:00:00 2001 From: Anca Iordache Date: Tue, 31 Aug 2021 15:57:32 +0200 Subject: [PATCH 1125/1301] Update changelog for v5.0.0 Signed-off-by: Anca Iordache --- docker/version.py | 2 +- docs/change-log.md | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index 3554104128..b95a1edef6 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "4.5.0-dev" +version = "5.0.0-dev" version_info = tuple(int(d) for d in version.split("-")[0].split(".")) diff --git a/docs/change-log.md b/docs/change-log.md index 8db3fc5821..63a029e135 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -1,6 +1,24 @@ Change log ========== +5.0.0 +----- + +[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/70?closed=1) + +### Breaking changes +- Remove support for Python 2.7 +- Make Python 3.6 the minimum version supported + +### Features +- Add `limit` parameter to image search endpoint + +### Bugfixes +- Fix `KeyError` exception on secret create +- Verify TLS keys loaded from docker contexts +- Update PORT_SPEC regex to allow square brackets for IPv6 addresses +- Fix containers and images documentation examples + 4.4.4 ----- From c5fc19385765b2724285689a94c408cfd486f210 Mon Sep 17 00:00:00 2001 From: Anca Iordache Date: Tue, 31 Aug 2021 16:39:50 +0200 Subject: [PATCH 1126/1301] Update changelog for 5.0.1 release Signed-off-by: Anca Iordache --- docker/version.py | 2 +- docs/change-log.md | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index b95a1edef6..5687086f16 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "5.0.0-dev" +version = "5.1.0-dev" version_info = tuple(int(d) for d in version.split("-")[0].split(".")) diff --git a/docs/change-log.md b/docs/change-log.md index 63a029e135..441e91def6 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -1,6 +1,20 @@ Change log ========== +5.0.1 +----- + +[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/74?closed=1) + +### Bugfixes +- Bring back support for ssh identity file +- Cleanup remaining python-2 dependencies +- Fix image save example in docs + +### Miscellaneous +- Bump urllib3 to 1.26.5 +- Bump requests to 2.26.0 + 5.0.0 ----- From f9b85586ca7244ada8b66a4dab1fd324caccbe24 Mon Sep 17 00:00:00 2001 From: Adam Aposhian Date: Tue, 31 Aug 2021 15:02:04 -0600 Subject: [PATCH 1127/1301] fix(transport): remove disable_buffering option Signed-off-by: Adam Aposhian --- docker/transport/unixconn.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/docker/transport/unixconn.py b/docker/transport/unixconn.py index adb6f18a1d..1b00762a60 100644 --- a/docker/transport/unixconn.py +++ b/docker/transport/unixconn.py @@ -23,7 +23,6 @@ def __init__(self, base_url, unix_socket, timeout=60): self.base_url = base_url self.unix_socket = unix_socket self.timeout = timeout - self.disable_buffering = False def connect(self): sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) @@ -33,13 +32,8 @@ def connect(self): def putheader(self, header, *values): super().putheader(header, *values) - if header == 'Connection' and 'Upgrade' in values: - self.disable_buffering = True def response_class(self, sock, *args, **kwargs): - if self.disable_buffering: - kwargs['disable_buffering'] = True - return httplib.HTTPResponse(sock, *args, **kwargs) From a9265197d262302d34846e26886347f68c83bb5d Mon Sep 17 00:00:00 2001 From: Anca Iordache Date: Wed, 1 Sep 2021 19:23:59 +0200 Subject: [PATCH 1128/1301] Post-release changelog update Signed-off-by: Anca Iordache --- docs/change-log.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/change-log.md b/docs/change-log.md index 441e91def6..2ff0774f4f 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -1,6 +1,14 @@ Change log ========== +5.0.2 +----- + +[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/75?closed=1) + +### Bugfixes +- Fix `disable_buffering` regression + 5.0.1 ----- From 63618b5e11e9326ed6e4cad6a0b012b9dc02593f Mon Sep 17 00:00:00 2001 From: Segev Finer Date: Thu, 15 Mar 2018 21:46:24 +0200 Subject: [PATCH 1129/1301] Fix getting a read timeout for logs/attach with a tty and slow output Fixes #931 Signed-off-by: Segev Finer --- docker/api/client.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docker/api/client.py b/docker/api/client.py index f0cb39b864..2667922d98 100644 --- a/docker/api/client.py +++ b/docker/api/client.py @@ -397,6 +397,12 @@ def _multiplexed_response_stream_helper(self, response): def _stream_raw_result(self, response, chunk_size=1, decode=True): ''' Stream result for TTY-enabled container and raw binary data''' self._raise_for_status(response) + + # Disable timeout on the underlying socket to prevent + # Read timed out(s) for long running processes + socket = self._get_raw_response_socket(response) + self._disable_socket_timeout(socket) + yield from response.iter_content(chunk_size, decode) def _read_from_socket(self, response, stream, tty=True, demux=False): From ecace769f5d81b5ea1a25befed8eebe2c723d33e Mon Sep 17 00:00:00 2001 From: Anca Iordache Date: Fri, 8 Oct 2021 00:58:26 +0200 Subject: [PATCH 1130/1301] Post-release changelog update Signed-off-by: Anca Iordache --- docs/change-log.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docs/change-log.md b/docs/change-log.md index 2ff0774f4f..91f3fe6f17 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -1,6 +1,21 @@ Change log ========== +5.0.3 +----- + +[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/76?closed=1) + +### Features +- Add `cap_add` and `cap_drop` parameters to service create and ContainerSpec +- Add `templating` parameter to config create + +### Bugfixes +- Fix getting a read timeout for logs/attach with a tty and slow output + +### Miscellaneous +- Fix documentation examples + 5.0.2 ----- From a9de3432103141c7519783ad4d8088797c892914 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Mon, 11 Oct 2021 22:30:36 +0300 Subject: [PATCH 1131/1301] Add support for Python 3.10 Signed-off-by: Hugo van Kemenade --- .github/workflows/ci.yml | 2 +- setup.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b692508220..a73bcbadea 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,7 +8,7 @@ jobs: strategy: max-parallel: 1 matrix: - python-version: [3.6, 3.7, 3.8, 3.9] + python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"] steps: - uses: actions/checkout@v2 diff --git a/setup.py b/setup.py index a966fea238..1e76fdb168 100644 --- a/setup.py +++ b/setup.py @@ -75,6 +75,7 @@ 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', 'Topic :: Software Development', 'Topic :: Utilities', 'License :: OSI Approved :: Apache Software License', From 4150fc4d9d3c9c68dea3a377410182aa33c02c2b Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Mon, 11 Oct 2021 22:30:42 +0300 Subject: [PATCH 1132/1301] Universal wheels are for code expected to work on both Python 2 and 3 Signed-off-by: Hugo van Kemenade --- setup.cfg | 3 --- 1 file changed, 3 deletions(-) diff --git a/setup.cfg b/setup.cfg index 907746f013..a37e5521d5 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,3 @@ -[bdist_wheel] -universal = 1 - [metadata] description_file = README.rst license = Apache License 2.0 From 72bcd1616da7c3d57fd90ec02b2fa7a9255dd08b Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Mon, 11 Oct 2021 22:38:56 +0300 Subject: [PATCH 1133/1301] Bump pytest (and other dependencies) for Python 3.10 Signed-off-by: Hugo van Kemenade --- test-requirements.txt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index 40161bb8ec..d135792b30 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,7 +1,7 @@ -setuptools==54.1.1 -coverage==4.5.2 -flake8==3.6.0 +setuptools==58.2.0 +coverage==6.0.1 +flake8==4.0.1 mock==1.0.1 -pytest==4.3.1 -pytest-cov==2.6.1 -pytest-timeout==1.3.3 +pytest==6.2.5 +pytest-cov==3.0.0 +pytest-timeout==2.0.1 From bbbc29191a8a430a7024cacd460b7e2d35e0dfb0 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Mon, 11 Oct 2021 22:43:25 +0300 Subject: [PATCH 1134/1301] Bump minimum paramiko to support Python 3.10 Signed-off-by: Hugo van Kemenade --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 26cbc6fb4b..d7c11aaa71 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,7 @@ enum34==1.1.6 idna==2.5 ipaddress==1.0.18 packaging==16.8 -paramiko==2.4.2 +paramiko==2.8.0 pycparser==2.17 pyOpenSSL==18.0.0 pyparsing==2.2.0 diff --git a/setup.py b/setup.py index 1e76fdb168..db2d6ebc41 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ 'tls': ['pyOpenSSL>=17.5.0', 'cryptography>=3.4.7', 'idna>=2.0.0'], # Only required when connecting using the ssh:// protocol - 'ssh': ['paramiko>=2.4.2'], + 'ssh': ['paramiko>=2.4.3'], } From 4bb99311e2911406dde543117438782a9524feea Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Mon, 11 Oct 2021 22:50:39 +0300 Subject: [PATCH 1135/1301] Don't install mock backport Signed-off-by: Hugo van Kemenade --- test-requirements.txt | 1 - tests/integration/credentials/utils_test.py | 6 +----- tests/unit/api_container_test.py | 6 +----- tests/unit/api_image_test.py | 6 +----- tests/unit/api_network_test.py | 6 +----- tests/unit/api_test.py | 6 +----- tests/unit/auth_test.py | 7 +------ tests/unit/client_test.py | 6 +----- tests/unit/dockertypes_test.py | 6 +----- tests/unit/fake_api_client.py | 6 +----- tests/unit/utils_config_test.py | 6 +----- 11 files changed, 10 insertions(+), 52 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index d135792b30..ccc97be46f 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,7 +1,6 @@ setuptools==58.2.0 coverage==6.0.1 flake8==4.0.1 -mock==1.0.1 pytest==6.2.5 pytest-cov==3.0.0 pytest-timeout==2.0.1 diff --git a/tests/integration/credentials/utils_test.py b/tests/integration/credentials/utils_test.py index d7b2a1a4d5..acf018d2ff 100644 --- a/tests/integration/credentials/utils_test.py +++ b/tests/integration/credentials/utils_test.py @@ -1,11 +1,7 @@ import os from docker.credentials.utils import create_environment_dict - -try: - from unittest import mock -except ImportError: - from unittest import mock +from unittest import mock @mock.patch.dict(os.environ) diff --git a/tests/unit/api_container_test.py b/tests/unit/api_container_test.py index 1ebd37df0a..a66aea047f 100644 --- a/tests/unit/api_container_test.py +++ b/tests/unit/api_container_test.py @@ -4,6 +4,7 @@ import docker from docker.api import APIClient +from unittest import mock import pytest from . import fake_api @@ -13,11 +14,6 @@ fake_inspect_container, url_base ) -try: - from unittest import mock -except ImportError: - from unittest import mock - def fake_inspect_container_tty(self, container): return fake_inspect_container(self, container, tty=True) diff --git a/tests/unit/api_image_test.py b/tests/unit/api_image_test.py index 843c11b841..8fb3e9d9f5 100644 --- a/tests/unit/api_image_test.py +++ b/tests/unit/api_image_test.py @@ -3,16 +3,12 @@ from . import fake_api from docker import auth +from unittest import mock from .api_test import ( BaseAPIClientTest, fake_request, DEFAULT_TIMEOUT_SECONDS, url_prefix, fake_resolve_authconfig ) -try: - from unittest import mock -except ImportError: - from unittest import mock - class ImageTest(BaseAPIClientTest): def test_image_viz(self): diff --git a/tests/unit/api_network_test.py b/tests/unit/api_network_test.py index 84d6544969..8afab7379d 100644 --- a/tests/unit/api_network_test.py +++ b/tests/unit/api_network_test.py @@ -2,11 +2,7 @@ from .api_test import BaseAPIClientTest, url_prefix, response from docker.types import IPAMConfig, IPAMPool - -try: - from unittest import mock -except ImportError: - from unittest import mock +from unittest import mock class NetworkTest(BaseAPIClientTest): diff --git a/tests/unit/api_test.py b/tests/unit/api_test.py index dfc38164d4..3234e55b11 100644 --- a/tests/unit/api_test.py +++ b/tests/unit/api_test.py @@ -19,14 +19,10 @@ from docker.api import APIClient from docker.constants import DEFAULT_DOCKER_API_VERSION from requests.packages import urllib3 +from unittest import mock from . import fake_api -try: - from unittest import mock -except ImportError: - from unittest import mock - DEFAULT_TIMEOUT_SECONDS = docker.constants.DEFAULT_TIMEOUT_SECONDS diff --git a/tests/unit/auth_test.py b/tests/unit/auth_test.py index 8bd2e1658b..ea953af0cb 100644 --- a/tests/unit/auth_test.py +++ b/tests/unit/auth_test.py @@ -8,14 +8,9 @@ import unittest from docker import auth, credentials, errors +from unittest import mock import pytest -try: - from unittest import mock -except ImportError: - from unittest import mock - - class RegressionTest(unittest.TestCase): def test_803_urlsafe_encode(self): auth_data = { diff --git a/tests/unit/client_test.py b/tests/unit/client_test.py index d647d3a1ae..e7c7eec827 100644 --- a/tests/unit/client_test.py +++ b/tests/unit/client_test.py @@ -9,14 +9,10 @@ DEFAULT_MAX_POOL_SIZE, IS_WINDOWS_PLATFORM ) from docker.utils import kwargs_from_env +from unittest import mock from . import fake_api -try: - from unittest import mock -except ImportError: - from unittest import mock - TEST_CERT_DIR = os.path.join(os.path.dirname(__file__), 'testdata/certs') POOL_SIZE = 20 diff --git a/tests/unit/dockertypes_test.py b/tests/unit/dockertypes_test.py index a0a171becd..76a99a627d 100644 --- a/tests/unit/dockertypes_test.py +++ b/tests/unit/dockertypes_test.py @@ -9,11 +9,7 @@ IPAMPool, LogConfig, Mount, ServiceMode, Ulimit, ) from docker.types.services import convert_service_ports - -try: - from unittest import mock -except: # noqa: E722 - from unittest import mock +from unittest import mock def create_host_config(*args, **kwargs): diff --git a/tests/unit/fake_api_client.py b/tests/unit/fake_api_client.py index 1663ef1273..95cf63b492 100644 --- a/tests/unit/fake_api_client.py +++ b/tests/unit/fake_api_client.py @@ -2,13 +2,9 @@ import docker from docker.constants import DEFAULT_DOCKER_API_VERSION +from unittest import mock from . import fake_api -try: - from unittest import mock -except ImportError: - from unittest import mock - class CopyReturnMagicMock(mock.MagicMock): """ diff --git a/tests/unit/utils_config_test.py b/tests/unit/utils_config_test.py index 83e04a146f..27d5a7cd43 100644 --- a/tests/unit/utils_config_test.py +++ b/tests/unit/utils_config_test.py @@ -5,14 +5,10 @@ import json from pytest import mark, fixture +from unittest import mock from docker.utils import config -try: - from unittest import mock -except ImportError: - from unittest import mock - class FindConfigFileTest(unittest.TestCase): From e0a3abfc3786800c8fce82e8efdd60c4383ebc80 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 29 Mar 2022 21:55:39 +0000 Subject: [PATCH 1136/1301] Bump paramiko from 2.8.0 to 2.10.1 Bumps [paramiko](https://github.com/paramiko/paramiko) from 2.8.0 to 2.10.1. - [Release notes](https://github.com/paramiko/paramiko/releases) - [Changelog](https://github.com/paramiko/paramiko/blob/main/NEWS) - [Commits](https://github.com/paramiko/paramiko/compare/2.8.0...2.10.1) --- updated-dependencies: - dependency-name: paramiko dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d7c11aaa71..a0eb531987 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,7 @@ enum34==1.1.6 idna==2.5 ipaddress==1.0.18 packaging==16.8 -paramiko==2.8.0 +paramiko==2.10.1 pycparser==2.17 pyOpenSSL==18.0.0 pyparsing==2.2.0 From a6db044bd4e0e0dae1d7d87f0c0fc85619757535 Mon Sep 17 00:00:00 2001 From: Milas Bowman Date: Tue, 26 Jul 2022 08:43:45 -0400 Subject: [PATCH 1137/1301] deps: upgrade pywin32 & relax version constraint (#3004) Upgrade to latest pywin32, which has support for Python 3.10 and resolves a CVE (related to ACL APIs, outside the scope of what `docker-py` relies on, which is npipe support, but still gets flagged by scanners). The version constraint has also been relaxed in `setup.py` to allow newer versions of pywin32. This is similar to how we handle the other packages there, and should be safe from a compatibility perspective. Fixes #2902. Closes #2972 and closes #2980. Signed-off-by: Milas Bowman --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index a0eb531987..c74d8cea25 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,7 @@ paramiko==2.10.1 pycparser==2.17 pyOpenSSL==18.0.0 pyparsing==2.2.0 -pywin32==301; sys_platform == 'win32' +pywin32==304; sys_platform == 'win32' requests==2.26.0 urllib3==1.26.5 websocket-client==0.56.0 diff --git a/setup.py b/setup.py index db2d6ebc41..3be63ba659 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ extras_require = { # win32 APIs if on Windows (required for npipe support) - ':sys_platform == "win32"': 'pywin32==227', + ':sys_platform == "win32"': 'pywin32>=304', # If using docker-py over TLS, highly recommend this option is # pip-installed or pinned. From 2933af2ca760cda128f1a48145170a56ba732abd Mon Sep 17 00:00:00 2001 From: Milas Bowman Date: Tue, 26 Jul 2022 08:51:52 -0400 Subject: [PATCH 1138/1301] ci: remove Python 3.6 and add 3.11 pre-releases (#3005) * Python 3.6 went EOL Dec 2021 * Python 3.11 is in beta and due for GA release in October 2022 Signed-off-by: Milas Bowman --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a73bcbadea..29e022a9ba 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,12 +8,12 @@ jobs: strategy: max-parallel: 1 matrix: - python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11.0-alpha - 3.11.0"] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - name: Install dependencies From f16c4e1147c81afd822fe72191f0f720cb0ba637 Mon Sep 17 00:00:00 2001 From: Milas Bowman Date: Tue, 26 Jul 2022 11:35:44 -0400 Subject: [PATCH 1139/1301] utils: fix IPv6 address w/ port parsing (#3006) This was using a deprecated function (`urllib.splitnport`), ostensibly to work around issues with brackets on IPv6 addresses. Ironically, its usage was broken, and would result in mangled IPv6 addresses if they had a port specified in some instances. Usage of the deprecated function has been eliminated and extra test cases added where missing. All existing cases pass as-is. (The only other change to the test was to improve assertion messages.) Signed-off-by: Milas Bowman --- docker/utils/utils.py | 38 ++++++++++++++++++++++++-------------- tests/unit/utils_test.py | 11 +++++++++-- 2 files changed, 33 insertions(+), 16 deletions(-) diff --git a/docker/utils/utils.py b/docker/utils/utils.py index f7c3dd7d82..7b2290991b 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -1,4 +1,5 @@ import base64 +import collections import json import os import os.path @@ -8,15 +9,20 @@ from distutils.version import StrictVersion from .. import errors -from .. import tls from ..constants import DEFAULT_HTTP_HOST from ..constants import DEFAULT_UNIX_SOCKET from ..constants import DEFAULT_NPIPE from ..constants import BYTE_UNITS +from ..tls import TLSConfig -from urllib.parse import splitnport, urlparse +from urllib.parse import urlparse, urlunparse +URLComponents = collections.namedtuple( + 'URLComponents', + 'scheme netloc url params query fragment', +) + def create_ipam_pool(*args, **kwargs): raise errors.DeprecatedMethod( 'utils.create_ipam_pool has been removed. Please use a ' @@ -201,10 +207,6 @@ def parse_repository_tag(repo_name): def parse_host(addr, is_win32=False, tls=False): - path = '' - port = None - host = None - # Sensible defaults if not addr and is_win32: return DEFAULT_NPIPE @@ -263,20 +265,20 @@ def parse_host(addr, is_win32=False, tls=False): # to be valid and equivalent to unix:///path path = '/'.join((parsed_url.hostname, path)) + netloc = parsed_url.netloc if proto in ('tcp', 'ssh'): - # parsed_url.hostname strips brackets from IPv6 addresses, - # which can be problematic hence our use of splitnport() instead. - host, port = splitnport(parsed_url.netloc) - if port is None or port < 0: + port = parsed_url.port or 0 + if port <= 0: if proto != 'ssh': raise errors.DockerException( 'Invalid bind address format: port is required:' ' {}'.format(addr) ) port = 22 + netloc = f'{parsed_url.netloc}:{port}' - if not host: - host = DEFAULT_HTTP_HOST + if not parsed_url.hostname: + netloc = f'{DEFAULT_HTTP_HOST}:{port}' # Rewrite schemes to fit library internals (requests adapters) if proto == 'tcp': @@ -286,7 +288,15 @@ def parse_host(addr, is_win32=False, tls=False): if proto in ('http+unix', 'npipe'): return f"{proto}://{path}".rstrip('/') - return f'{proto}://{host}:{port}{path}'.rstrip('/') + + return urlunparse(URLComponents( + scheme=proto, + netloc=netloc, + url=path, + params='', + query='', + fragment='', + )).rstrip('/') def parse_devices(devices): @@ -351,7 +361,7 @@ def kwargs_from_env(ssl_version=None, assert_hostname=None, environment=None): # so if it's not set already then set it to false. assert_hostname = False - params['tls'] = tls.TLSConfig( + params['tls'] = TLSConfig( client_cert=(os.path.join(cert_path, 'cert.pem'), os.path.join(cert_path, 'key.pem')), ca_cert=os.path.join(cert_path, 'ca.pem'), diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index 802d91962a..12cb7bd657 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -296,17 +296,24 @@ def test_parse_host(self): '[fd12::82d1]:2375/docker/engine': ( 'http://[fd12::82d1]:2375/docker/engine' ), + 'ssh://[fd12::82d1]': 'ssh://[fd12::82d1]:22', + 'ssh://user@[fd12::82d1]:8765': 'ssh://user@[fd12::82d1]:8765', 'ssh://': 'ssh://127.0.0.1:22', 'ssh://user@localhost:22': 'ssh://user@localhost:22', 'ssh://user@remote': 'ssh://user@remote:22', } for host in invalid_hosts: - with pytest.raises(DockerException): + msg = f'Should have failed to parse invalid host: {host}' + with self.assertRaises(DockerException, msg=msg): parse_host(host, None) for host, expected in valid_hosts.items(): - assert parse_host(host, None) == expected + self.assertEqual( + parse_host(host, None), + expected, + msg=f'Failed to parse valid host: {host}', + ) def test_parse_host_empty_value(self): unix_socket = 'http+unix:///var/run/docker.sock' From 7168e09b1628b85a09e95cf8bae6bfd94b61a6c4 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Tue, 26 Jul 2022 18:06:51 +0200 Subject: [PATCH 1140/1301] test: fix for cgroupv2 (#2940) This test was verifying that the container has the right options set (through `docker inspect`), but also checks if the cgroup-rules are set within the container by reading `/sys/fs/cgroup/devices/devices.list` Unlike cgroups v1, on cgroups v2, there is no file interface, and rules are handled through ebpf, which means that the test will fail because this file is not present. From the Linux documentation for cgroups v2: https://github.com/torvalds/linux/blob/v5.16/Documentation/admin-guide/cgroup-v2.rst#device-controller > (...) > Device controller manages access to device files. It includes both creation of > new device files (using mknod), and access to the existing device files. > > Cgroup v2 device controller has no interface files and is implemented on top > of cgroup BPF. To control access to device files, a user may create bpf programs > of type BPF_PROG_TYPE_CGROUP_DEVICE and attach them to cgroups with > BPF_CGROUP_DEVICE flag. (...) Given that setting the right cgroups is not really a responsibility of this SDK, it should be sufficient to verify that the right options were set in the container configuration, so this patch is removing the part that checks the cgroup, to allow this test to be run on a host with cgroups v2 enabled. Signed-off-by: Sebastiaan van Stijn --- tests/integration/api_container_test.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index 9da2cfbf40..062693ef0b 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -460,16 +460,13 @@ def test_create_with_cpu_rt_options(self): def test_create_with_device_cgroup_rules(self): rule = 'c 7:128 rwm' ctnr = self.client.create_container( - TEST_IMG, 'cat /sys/fs/cgroup/devices/devices.list', - host_config=self.client.create_host_config( + TEST_IMG, 'true', host_config=self.client.create_host_config( device_cgroup_rules=[rule] ) ) self.tmp_containers.append(ctnr) config = self.client.inspect_container(ctnr) assert config['HostConfig']['DeviceCgroupRules'] == [rule] - self.client.start(ctnr) - assert rule in self.client.logs(ctnr).decode('utf-8') def test_create_with_uts_mode(self): container = self.client.create_container( From 74e0c5eb8c38f0a219cc0120bc51de99c1c8159e Mon Sep 17 00:00:00 2001 From: Milas Bowman Date: Tue, 26 Jul 2022 12:55:14 -0400 Subject: [PATCH 1141/1301] test: fix flaky container log test Ensure the container has exited before attempting to grab the logs. Since we are not streaming them, it's possible to attach + grab logs before the output is processed, resulting in a test failure. If the container has exited, it's guaranteed to have logged :) Signed-off-by: Milas Bowman --- tests/integration/api_container_test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index 062693ef0b..0d6d9f96c5 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -1217,12 +1217,14 @@ def test_run_container_reading_socket(self): data = read_exactly(pty_stdout, next_size) assert data.decode('utf-8') == line + @pytest.mark.timeout(10) def test_attach_no_stream(self): container = self.client.create_container( TEST_IMG, 'echo hello' ) self.tmp_containers.append(container) self.client.start(container) + self.client.wait(container, condition='not-running') output = self.client.attach(container, stream=False, logs=True) assert output == 'hello\n'.encode(encoding='ascii') From 4765f624419c503012508f0fecbe4f63e492cde1 Mon Sep 17 00:00:00 2001 From: Milas Bowman Date: Tue, 26 Jul 2022 12:56:07 -0400 Subject: [PATCH 1142/1301] test: mark invalid test as xfail This test looks for some behavior on non-chunked HTTP requests. It now fails because it looks like recent versions of Docker Engine ALWAYS return chunked responses (or perhaps this specific response changed somehow to now trigger chunking whereas it did not previously). The actual logic it's trying to test is also unusual because it's trying to hackily propagate errors under the assumption that it'd get a non-chunked response on failure, which is...not reliable. Arguably, the chunked reader should be refactored somehow but that's a refactor we can't really commit to (and it's evidently been ok enough as is up until now). Signed-off-by: Milas Bowman --- tests/integration/regression_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/regression_test.py b/tests/integration/regression_test.py index deb9aff15a..10313a637c 100644 --- a/tests/integration/regression_test.py +++ b/tests/integration/regression_test.py @@ -8,6 +8,7 @@ class TestRegressions(BaseAPIIntegrationTest): + @pytest.mark.xfail(True, reason='Docker API always returns chunked resp') def test_443_handle_nonchunked_response_in_stream(self): dfile = io.BytesIO() with pytest.raises(docker.errors.APIError) as exc: From ce40d4bb34e9324e3ee640f0acc23604498db21d Mon Sep 17 00:00:00 2001 From: Milas Bowman Date: Tue, 26 Jul 2022 13:48:10 -0400 Subject: [PATCH 1143/1301] ci: add flake8 job Project is already configured for flake8 but it never gets run in CI. Signed-off-by: Milas Bowman --- .github/workflows/ci.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 29e022a9ba..0096ddd2f8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,6 +3,17 @@ name: Python package on: [push, pull_request] jobs: + flake8: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '3.x' + - run: pip install -U flake8 + - name: Run flake8 + run: flake8 docker/ tests/ + build: runs-on: ubuntu-latest strategy: From 3ffdd8a1c52cb7677d926feaf1a44d585a066dac Mon Sep 17 00:00:00 2001 From: Milas Bowman Date: Tue, 26 Jul 2022 13:48:47 -0400 Subject: [PATCH 1144/1301] lint: fix outstanding flake8 violations Since flake8 wasn't actually being run in CI, we'd accumulated some violations. Signed-off-by: Milas Bowman --- docker/api/build.py | 2 +- docker/api/container.py | 13 +++++++++---- docker/api/image.py | 10 ++++++++-- docker/api/volume.py | 19 +++++++++++-------- docker/models/containers.py | 3 ++- docker/models/images.py | 5 ++++- docker/models/plugins.py | 6 +++++- docker/utils/utils.py | 1 + tests/integration/api_config_test.py | 2 +- tests/unit/auth_test.py | 1 + tests/unit/utils_build_test.py | 24 ++++++++++++------------ 11 files changed, 55 insertions(+), 31 deletions(-) diff --git a/docker/api/build.py b/docker/api/build.py index aac43c460a..a48204a9fd 100644 --- a/docker/api/build.py +++ b/docker/api/build.py @@ -153,7 +153,7 @@ def build(self, path=None, tag=None, quiet=False, fileobj=None, with open(dockerignore) as f: exclude = list(filter( lambda x: x != '' and x[0] != '#', - [l.strip() for l in f.read().splitlines()] + [line.strip() for line in f.read().splitlines()] )) dockerfile = process_dockerfile(dockerfile, path) context = utils.tar( diff --git a/docker/api/container.py b/docker/api/container.py index 83fcd4f64a..17c09726b7 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -256,7 +256,9 @@ def create_container(self, image, command=None, hostname=None, user=None, .. code-block:: python - client.api.create_host_config(port_bindings={1111: ('127.0.0.1', 4567)}) + client.api.create_host_config( + port_bindings={1111: ('127.0.0.1', 4567)} + ) Or without host port assignment: @@ -579,10 +581,13 @@ def create_host_config(self, *args, **kwargs): Example: - >>> client.api.create_host_config(privileged=True, cap_drop=['MKNOD'], - volumes_from=['nostalgic_newton']) + >>> client.api.create_host_config( + ... privileged=True, + ... cap_drop=['MKNOD'], + ... volumes_from=['nostalgic_newton'], + ... ) {'CapDrop': ['MKNOD'], 'LxcConf': None, 'Privileged': True, - 'VolumesFrom': ['nostalgic_newton'], 'PublishAllPorts': False} + 'VolumesFrom': ['nostalgic_newton'], 'PublishAllPorts': False} """ if not kwargs: diff --git a/docker/api/image.py b/docker/api/image.py index 772d88957c..5e1466ec3d 100644 --- a/docker/api/image.py +++ b/docker/api/image.py @@ -377,7 +377,8 @@ def pull(self, repository, tag=None, stream=False, auth_config=None, Example: - >>> for line in client.api.pull('busybox', stream=True, decode=True): + >>> resp = client.api.pull('busybox', stream=True, decode=True) + ... for line in resp: ... print(json.dumps(line, indent=4)) { "status": "Pulling image (latest) from busybox", @@ -456,7 +457,12 @@ def push(self, repository, tag=None, stream=False, auth_config=None, If the server returns an error. Example: - >>> for line in client.api.push('yourname/app', stream=True, decode=True): + >>> resp = client.api.push( + ... 'yourname/app', + ... stream=True, + ... decode=True, + ... ) + ... for line in resp: ... print(line) {'status': 'Pushing repository yourname/app (1 tags)'} {'status': 'Pushing','progressDetail': {}, 'id': '511136ea3c5a'} diff --git a/docker/api/volume.py b/docker/api/volume.py index 86b0018769..98b42a124e 100644 --- a/docker/api/volume.py +++ b/docker/api/volume.py @@ -56,15 +56,18 @@ def create_volume(self, name=None, driver=None, driver_opts=None, Example: - >>> volume = client.api.create_volume(name='foobar', driver='local', - driver_opts={'foo': 'bar', 'baz': 'false'}, - labels={"key": "value"}) - >>> print(volume) + >>> volume = client.api.create_volume( + ... name='foobar', + ... driver='local', + ... driver_opts={'foo': 'bar', 'baz': 'false'}, + ... labels={"key": "value"}, + ... ) + ... print(volume) {u'Driver': u'local', - u'Labels': {u'key': u'value'}, - u'Mountpoint': u'/var/lib/docker/volumes/foobar/_data', - u'Name': u'foobar', - u'Scope': u'local'} + u'Labels': {u'key': u'value'}, + u'Mountpoint': u'/var/lib/docker/volumes/foobar/_data', + u'Name': u'foobar', + u'Scope': u'local'} """ url = self._url('/volumes/create') diff --git a/docker/models/containers.py b/docker/models/containers.py index 957deed46d..e34659cbee 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -761,7 +761,8 @@ def run(self, image, command=None, stdout=True, stderr=False, {'/home/user1/': {'bind': '/mnt/vol2', 'mode': 'rw'}, '/var/www': {'bind': '/mnt/vol1', 'mode': 'ro'}} - Or a list of strings which each one of its elements specifies a mount volume. + Or a list of strings which each one of its elements specifies a + mount volume. For example: diff --git a/docker/models/images.py b/docker/models/images.py index 46f8efeed8..ef668c7d4e 100644 --- a/docker/models/images.py +++ b/docker/models/images.py @@ -15,7 +15,10 @@ class Image(Model): An image on the server. """ def __repr__(self): - return "<{}: '{}'>".format(self.__class__.__name__, "', '".join(self.tags)) + return "<{}: '{}'>".format( + self.__class__.__name__, + "', '".join(self.tags), + ) @property def labels(self): diff --git a/docker/models/plugins.py b/docker/models/plugins.py index 37ecefbe09..69b94f3530 100644 --- a/docker/models/plugins.py +++ b/docker/models/plugins.py @@ -117,7 +117,11 @@ def upgrade(self, remote=None): if remote is None: remote = self.name privileges = self.client.api.plugin_privileges(remote) - yield from self.client.api.upgrade_plugin(self.name, remote, privileges) + yield from self.client.api.upgrade_plugin( + self.name, + remote, + privileges, + ) self.reload() diff --git a/docker/utils/utils.py b/docker/utils/utils.py index 7b2290991b..71e4014db9 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -23,6 +23,7 @@ 'scheme netloc url params query fragment', ) + def create_ipam_pool(*args, **kwargs): raise errors.DeprecatedMethod( 'utils.create_ipam_pool has been removed. Please use a ' diff --git a/tests/integration/api_config_test.py b/tests/integration/api_config_test.py index 82cb5161ac..982ec468a6 100644 --- a/tests/integration/api_config_test.py +++ b/tests/integration/api_config_test.py @@ -73,7 +73,7 @@ def test_list_configs(self): def test_create_config_with_templating(self): config_id = self.client.create_config( 'favorite_character', 'sakuya izayoi', - templating={ 'name': 'golang'} + templating={'name': 'golang'} ) self.tmp_configs.append(config_id) assert 'ID' in config_id diff --git a/tests/unit/auth_test.py b/tests/unit/auth_test.py index ea953af0cb..dd5b5f8b57 100644 --- a/tests/unit/auth_test.py +++ b/tests/unit/auth_test.py @@ -11,6 +11,7 @@ from unittest import mock import pytest + class RegressionTest(unittest.TestCase): def test_803_urlsafe_encode(self): auth_data = { diff --git a/tests/unit/utils_build_test.py b/tests/unit/utils_build_test.py index 9f183886b5..fa7d833de2 100644 --- a/tests/unit/utils_build_test.py +++ b/tests/unit/utils_build_test.py @@ -272,8 +272,8 @@ def test_single_and_double_wildcard(self): assert self.exclude(['**/target/*/*']) == convert_paths( self.all_paths - { 'target/subdir/file.txt', - 'subdir/target/subdir/file.txt', - 'subdir/subdir2/target/subdir/file.txt' + 'subdir/target/subdir/file.txt', + 'subdir/subdir2/target/subdir/file.txt' } ) @@ -281,16 +281,16 @@ def test_trailing_double_wildcard(self): assert self.exclude(['subdir/**']) == convert_paths( self.all_paths - { 'subdir/file.txt', - 'subdir/target/file.txt', - 'subdir/target/subdir/file.txt', - 'subdir/subdir2/file.txt', - 'subdir/subdir2/target/file.txt', - 'subdir/subdir2/target/subdir/file.txt', - 'subdir/target', - 'subdir/target/subdir', - 'subdir/subdir2', - 'subdir/subdir2/target', - 'subdir/subdir2/target/subdir' + 'subdir/target/file.txt', + 'subdir/target/subdir/file.txt', + 'subdir/subdir2/file.txt', + 'subdir/subdir2/target/file.txt', + 'subdir/subdir2/target/subdir/file.txt', + 'subdir/target', + 'subdir/target/subdir', + 'subdir/subdir2', + 'subdir/subdir2/target', + 'subdir/subdir2/target/subdir' } ) From bb11197ee3407798a53c50e43aa994fe8cd9c8e7 Mon Sep 17 00:00:00 2001 From: Maor Kleinberger Date: Tue, 26 Jul 2022 22:07:23 +0300 Subject: [PATCH 1145/1301] client: fix exception semantics in _raise_for_status (#2954) We want "The above exception was the direct cause of the following exception:" instead of "During handling of the above exception, another exception occurred:" Signed-off-by: Maor Kleinberger --- docker/api/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/api/client.py b/docker/api/client.py index 2667922d98..7733d33438 100644 --- a/docker/api/client.py +++ b/docker/api/client.py @@ -267,7 +267,7 @@ def _raise_for_status(self, response): try: response.raise_for_status() except requests.exceptions.HTTPError as e: - raise create_api_error_from_http_exception(e) + raise create_api_error_from_http_exception(e) from e def _result(self, response, json=False, binary=False): assert not (json and binary) From 56dd6de7dfad9bedc7c8af99308707ecc3fad78e Mon Sep 17 00:00:00 2001 From: Milas Bowman Date: Tue, 26 Jul 2022 15:12:03 -0400 Subject: [PATCH 1146/1301] tls: use auto-negotiated highest version (#3007) Specific TLS versions are deprecated in latest Python, which causes test failures due to treating deprecation errors as warnings. Luckily, the fix here is straightforward: we can eliminate some custom version selection logic by using `PROTOCOL_TLS_CLIENT`, which is the recommended method and will select the highest TLS version supported by both client and server. Signed-off-by: Milas Bowman --- docker/tls.py | 21 +-------------------- 1 file changed, 1 insertion(+), 20 deletions(-) diff --git a/docker/tls.py b/docker/tls.py index 067d556300..882a50eaf3 100644 --- a/docker/tls.py +++ b/docker/tls.py @@ -37,30 +37,11 @@ def __init__(self, client_cert=None, ca_cert=None, verify=None, self.assert_hostname = assert_hostname self.assert_fingerprint = assert_fingerprint - # TODO(dperny): according to the python docs, PROTOCOL_TLSvWhatever is - # depcreated, and it's recommended to use OPT_NO_TLSvWhatever instead - # to exclude versions. But I think that might require a bigger - # architectural change, so I've opted not to pursue it at this time - # If the user provides an SSL version, we should use their preference if ssl_version: self.ssl_version = ssl_version else: - # If the user provides no ssl version, we should default to - # TLSv1_2. This option is the most secure, and will work for the - # majority of users with reasonably up-to-date software. However, - # before doing so, detect openssl version to ensure we can support - # it. - if ssl.OPENSSL_VERSION_INFO[:3] >= (1, 0, 1) and hasattr( - ssl, 'PROTOCOL_TLSv1_2'): - # If the OpenSSL version is high enough to support TLSv1_2, - # then we should use it. - self.ssl_version = getattr(ssl, 'PROTOCOL_TLSv1_2') - else: - # Otherwise, TLS v1.0 seems to be the safest default; - # SSLv23 fails in mysterious ways: - # https://github.com/docker/docker-py/issues/963 - self.ssl_version = ssl.PROTOCOL_TLSv1 + self.ssl_version = ssl.PROTOCOL_TLS_CLIENT # "client_cert" must have both or neither cert/key files. In # either case, Alert the user when both are expected, but any are From 4e19cc48dfd88d0a9a8bdbbe4df4357322619d02 Mon Sep 17 00:00:00 2001 From: Guy Lichtman <1395797+glicht@users.noreply.github.com> Date: Tue, 26 Jul 2022 22:16:12 +0300 Subject: [PATCH 1147/1301] transport: fix ProxyCommand for SSH conn (#2993) Signed-off-by: Guy Lichtman --- docker/transport/sshconn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/transport/sshconn.py b/docker/transport/sshconn.py index 8e6beb2544..76d1fa4439 100644 --- a/docker/transport/sshconn.py +++ b/docker/transport/sshconn.py @@ -204,7 +204,7 @@ def _create_paramiko_client(self, base_url): host_config = conf.lookup(base_url.hostname) if 'proxycommand' in host_config: self.ssh_params["sock"] = paramiko.ProxyCommand( - self.ssh_conf['proxycommand'] + host_config['proxycommand'] ) if 'hostname' in host_config: self.ssh_params['hostname'] = host_config['hostname'] From 2e6dad798324a1d993314f39e9a844b705b61e0d Mon Sep 17 00:00:00 2001 From: Francesco Casalegno Date: Tue, 26 Jul 2022 21:45:51 +0200 Subject: [PATCH 1148/1301] deps: use `packaging` instead of deprecated `distutils` (#2931) Replace `distutils.Version` (deprecated) with `packaging.Version` Signed-off-by: Francesco Casalegno --- docker/transport/ssladapter.py | 4 ++-- docker/utils/utils.py | 6 +++--- requirements.txt | 2 +- setup.py | 1 + 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/docker/transport/ssladapter.py b/docker/transport/ssladapter.py index 31e3014eab..bdca1d0453 100644 --- a/docker/transport/ssladapter.py +++ b/docker/transport/ssladapter.py @@ -4,7 +4,7 @@ """ import sys -from distutils.version import StrictVersion +from packaging.version import Version from requests.adapters import HTTPAdapter from docker.transport.basehttpadapter import BaseHTTPAdapter @@ -70,4 +70,4 @@ def can_override_ssl_version(self): return False if urllib_ver == 'dev': return True - return StrictVersion(urllib_ver) > StrictVersion('1.5') + return Version(urllib_ver) > Version('1.5') diff --git a/docker/utils/utils.py b/docker/utils/utils.py index 7b2290991b..3683ac5462 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -6,7 +6,7 @@ import shlex import string from datetime import datetime -from distutils.version import StrictVersion +from packaging.version import Version from .. import errors from ..constants import DEFAULT_HTTP_HOST @@ -55,8 +55,8 @@ def compare_version(v1, v2): >>> compare_version(v2, v2) 0 """ - s1 = StrictVersion(v1) - s2 = StrictVersion(v2) + s1 = Version(v1) + s2 = Version(v2) if s1 == s2: return 0 elif s1 > s2: diff --git a/requirements.txt b/requirements.txt index c74d8cea25..7bcca763e5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ cryptography==3.4.7 enum34==1.1.6 idna==2.5 ipaddress==1.0.18 -packaging==16.8 +packaging==21.3 paramiko==2.10.1 pycparser==2.17 pyOpenSSL==18.0.0 diff --git a/setup.py b/setup.py index 3be63ba659..833de3aa71 100644 --- a/setup.py +++ b/setup.py @@ -10,6 +10,7 @@ SOURCE_DIR = os.path.join(ROOT_DIR) requirements = [ + 'packaging', 'websocket-client >= 0.32.0', 'requests >= 2.14.2, != 2.18.0', ] From 0ee9f260e48992d04d72c7bb8e4819f6b6a64717 Mon Sep 17 00:00:00 2001 From: Leonard Kinday Date: Tue, 26 Jul 2022 22:33:21 +0200 Subject: [PATCH 1149/1301] ci: run integration tests & fix race condition (#2947) * Fix integration tests race condition * Run integration tests on CI * Use existing DIND version Signed-off-by: Leonard Kinday Co-authored-by: Milas Bowman --- .github/workflows/ci.yml | 21 ++++++++-- Makefile | 84 +++++++++++++++++++++++++++++++++------- 2 files changed, 86 insertions(+), 19 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0096ddd2f8..e2987b49a7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,10 +14,9 @@ jobs: - name: Run flake8 run: flake8 docker/ tests/ - build: + unit-tests: runs-on: ubuntu-latest strategy: - max-parallel: 1 matrix: python-version: ["3.7", "3.8", "3.9", "3.10", "3.11.0-alpha - 3.11.0"] @@ -26,13 +25,27 @@ jobs: - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: - python-version: ${{ matrix.python-version }} + python-version: ${{ matrix.python-version }} - name: Install dependencies run: | python3 -m pip install --upgrade pip pip3 install -r test-requirements.txt -r requirements.txt - - name: Test with pytest + - name: Run unit tests run: | docker logout rm -rf ~/.docker py.test -v --cov=docker tests/unit + + integration-tests: + runs-on: ubuntu-latest + strategy: + matrix: + variant: [ "integration-dind", "integration-dind-ssl" ] + + steps: + - uses: actions/checkout@v3 + - name: make ${{ matrix.variant }} + run: | + docker logout + rm -rf ~/.docker + make ${{ matrix.variant }} diff --git a/Makefile b/Makefile index 78a0d334e2..b71479eee1 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ TEST_API_VERSION ?= 1.41 -TEST_ENGINE_VERSION ?= 20.10.05 +TEST_ENGINE_VERSION ?= 20.10 .PHONY: all all: test @@ -46,10 +46,32 @@ integration-dind: integration-dind-py3 .PHONY: integration-dind-py3 integration-dind-py3: build-py3 setup-network docker rm -vf dpy-dind-py3 || : - docker run -d --network dpy-tests --name dpy-dind-py3 --privileged\ - docker:${TEST_ENGINE_VERSION}-dind dockerd -H tcp://0.0.0.0:2375 --experimental - docker run -t --rm --env="DOCKER_HOST=tcp://dpy-dind-py3:2375" --env="DOCKER_TEST_API_VERSION=${TEST_API_VERSION}"\ - --network dpy-tests docker-sdk-python3 py.test tests/integration/${file} + + docker run \ + --detach \ + --name dpy-dind-py3 \ + --network dpy-tests \ + --privileged \ + docker:${TEST_ENGINE_VERSION}-dind \ + dockerd -H tcp://0.0.0.0:2375 --experimental + + # Wait for Docker-in-Docker to come to life + docker run \ + --network dpy-tests \ + --rm \ + --tty \ + busybox \ + sh -c 'while ! nc -z dpy-dind-py3 2375; do sleep 1; done' + + docker run \ + --env="DOCKER_HOST=tcp://dpy-dind-py3:2375" \ + --env="DOCKER_TEST_API_VERSION=${TEST_API_VERSION}" \ + --network dpy-tests \ + --rm \ + --tty \ + docker-sdk-python3 \ + py.test tests/integration/${file} + docker rm -vf dpy-dind-py3 @@ -66,18 +88,50 @@ integration-ssh-py3: build-dind-ssh build-py3 setup-network .PHONY: integration-dind-ssl -integration-dind-ssl: build-dind-certs build-py3 +integration-dind-ssl: build-dind-certs build-py3 setup-network docker rm -vf dpy-dind-certs dpy-dind-ssl || : docker run -d --name dpy-dind-certs dpy-dind-certs - docker run -d --env="DOCKER_HOST=tcp://localhost:2375" --env="DOCKER_TLS_VERIFY=1"\ - --env="DOCKER_CERT_PATH=/certs" --volumes-from dpy-dind-certs --name dpy-dind-ssl\ - --network dpy-tests --network-alias docker -v /tmp --privileged\ - docker:${TEST_ENGINE_VERSION}-dind\ - dockerd --tlsverify --tlscacert=/certs/ca.pem --tlscert=/certs/server-cert.pem\ - --tlskey=/certs/server-key.pem -H tcp://0.0.0.0:2375 --experimental - docker run -t --rm --volumes-from dpy-dind-ssl --env="DOCKER_HOST=tcp://docker:2375"\ - --env="DOCKER_TLS_VERIFY=1" --env="DOCKER_CERT_PATH=/certs" --env="DOCKER_TEST_API_VERSION=${TEST_API_VERSION}"\ - --network dpy-tests docker-sdk-python3 py.test tests/integration/${file} + + docker run \ + --detach \ + --env="DOCKER_CERT_PATH=/certs" \ + --env="DOCKER_HOST=tcp://localhost:2375" \ + --env="DOCKER_TLS_VERIFY=1" \ + --name dpy-dind-ssl \ + --network dpy-tests \ + --network-alias docker \ + --privileged \ + --volume /tmp \ + --volumes-from dpy-dind-certs \ + docker:${TEST_ENGINE_VERSION}-dind \ + dockerd \ + --tlsverify \ + --tlscacert=/certs/ca.pem \ + --tlscert=/certs/server-cert.pem \ + --tlskey=/certs/server-key.pem \ + -H tcp://0.0.0.0:2375 \ + --experimental + + # Wait for Docker-in-Docker to come to life + docker run \ + --network dpy-tests \ + --rm \ + --tty \ + busybox \ + sh -c 'while ! nc -z dpy-dind-ssl 2375; do sleep 1; done' + + docker run \ + --env="DOCKER_CERT_PATH=/certs" \ + --env="DOCKER_HOST=tcp://docker:2375" \ + --env="DOCKER_TEST_API_VERSION=${TEST_API_VERSION}" \ + --env="DOCKER_TLS_VERIFY=1" \ + --network dpy-tests \ + --rm \ + --volumes-from dpy-dind-ssl \ + --tty \ + docker-sdk-python3 \ + py.test tests/integration/${file} + docker rm -vf dpy-dind-ssl dpy-dind-certs .PHONY: flake8 From da62a2883715e15f8b83ab0e9a073b3655a2d456 Mon Sep 17 00:00:00 2001 From: Milas Bowman Date: Wed, 27 Jul 2022 14:44:36 -0400 Subject: [PATCH 1150/1301] deps: test on Python 3.10 by default (#3010) * Upgrade to latest Sphinx / recommonmark * Small CSS fix for issue in new version of Alabaster theme * Fix `Makefile` target for macOS Signed-off-by: Milas Bowman --- .readthedocs.yml | 6 +++++- Dockerfile | 2 +- Dockerfile-docs | 2 +- Jenkinsfile | 4 ++-- Makefile | 12 +++++++++++- docs-requirements.txt | 4 ++-- docs/_static/custom.css | 5 +++++ setup.py | 3 +-- tests/Dockerfile | 2 +- tests/Dockerfile-dind-certs | 2 +- tests/Dockerfile-ssh-dind | 4 ++-- 11 files changed, 32 insertions(+), 14 deletions(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index 32113fedb4..464c782604 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -3,8 +3,12 @@ version: 2 sphinx: configuration: docs/conf.py +build: + os: ubuntu-20.04 + tools: + python: '3.10' + python: - version: 3.6 install: - requirements: docs-requirements.txt - requirements: requirements.txt diff --git a/Dockerfile b/Dockerfile index 22732dec5c..8a0d32e430 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -ARG PYTHON_VERSION=3.7 +ARG PYTHON_VERSION=3.10 FROM python:${PYTHON_VERSION} diff --git a/Dockerfile-docs b/Dockerfile-docs index 9d11312fca..98901dfe6b 100644 --- a/Dockerfile-docs +++ b/Dockerfile-docs @@ -1,4 +1,4 @@ -ARG PYTHON_VERSION=3.7 +ARG PYTHON_VERSION=3.10 FROM python:${PYTHON_VERSION} diff --git a/Jenkinsfile b/Jenkinsfile index f524ae7a14..f9431eac06 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -25,7 +25,7 @@ def buildImages = { -> imageDindSSH = "${imageNameBase}:sshdind-${gitCommit()}" withDockerRegistry(credentialsId:'dockerbuildbot-index.docker.io') { buildImage(imageDindSSH, "-f tests/Dockerfile-ssh-dind .", "") - buildImage(imageNamePy3, "-f tests/Dockerfile --build-arg PYTHON_VERSION=3.7 .", "py3.7") + buildImage(imageNamePy3, "-f tests/Dockerfile --build-arg PYTHON_VERSION=3.10 .", "py3.10") } } } @@ -70,7 +70,7 @@ def runTests = { Map settings -> throw new Exception("Need Docker version to test, e.g.: `runTests(dockerVersion: '19.03.12')`") } if (!pythonVersion) { - throw new Exception("Need Python version being tested, e.g.: `runTests(pythonVersion: 'py3.7')`") + throw new Exception("Need Python version being tested, e.g.: `runTests(pythonVersion: 'py3.x')`") } { -> diff --git a/Makefile b/Makefile index b71479eee1..27144d4d8d 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,16 @@ TEST_API_VERSION ?= 1.41 TEST_ENGINE_VERSION ?= 20.10 +ifeq ($(OS),Windows_NT) + PLATFORM := Windows +else + PLATFORM := $(shell sh -c 'uname -s 2>/dev/null || echo Unknown') +endif + +ifeq ($(PLATFORM),Linux) + uid_args := "--build-arg uid=$(shell id -u) --build-arg gid=$(shell id -g)" +endif + .PHONY: all all: test @@ -19,7 +29,7 @@ build-py3: .PHONY: build-docs build-docs: - docker build -t docker-sdk-python-docs -f Dockerfile-docs --build-arg uid=$(shell id -u) --build-arg gid=$(shell id -g) . + docker build -t docker-sdk-python-docs -f Dockerfile-docs $(uid_args) . .PHONY: build-dind-certs build-dind-certs: diff --git a/docs-requirements.txt b/docs-requirements.txt index d69373d7c7..1f342fa272 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -1,2 +1,2 @@ -recommonmark==0.4.0 -Sphinx==1.4.6 +recommonmark==0.7.1 +Sphinx==5.1.1 diff --git a/docs/_static/custom.css b/docs/_static/custom.css index 5d711eeffb..b0b2e5d0b8 100644 --- a/docs/_static/custom.css +++ b/docs/_static/custom.css @@ -1,3 +1,8 @@ dl.hide-signature > dt { display: none; } + +dl.field-list > dt { + /* prevent code blocks from forcing wrapping on the "Parameters" header */ + word-break: initial; +} diff --git a/setup.py b/setup.py index 833de3aa71..0b113688fc 100644 --- a/setup.py +++ b/setup.py @@ -62,7 +62,7 @@ install_requires=requirements, tests_require=test_requirements, extras_require=extras_require, - python_requires='>=3.6', + python_requires='>=3.7', zip_safe=False, test_suite='tests', classifiers=[ @@ -72,7 +72,6 @@ 'Operating System :: OS Independent', 'Programming Language :: Python', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', diff --git a/tests/Dockerfile b/tests/Dockerfile index 3236f3875e..1d60cfe42d 100644 --- a/tests/Dockerfile +++ b/tests/Dockerfile @@ -1,4 +1,4 @@ -ARG PYTHON_VERSION=3.7 +ARG PYTHON_VERSION=3.10 FROM python:${PYTHON_VERSION} diff --git a/tests/Dockerfile-dind-certs b/tests/Dockerfile-dind-certs index 8829ff7946..6e711892ca 100644 --- a/tests/Dockerfile-dind-certs +++ b/tests/Dockerfile-dind-certs @@ -1,4 +1,4 @@ -ARG PYTHON_VERSION=3.6 +ARG PYTHON_VERSION=3.10 FROM python:${PYTHON_VERSION} RUN mkdir /tmp/certs diff --git a/tests/Dockerfile-ssh-dind b/tests/Dockerfile-ssh-dind index aba9bb34b2..6f080182d5 100644 --- a/tests/Dockerfile-ssh-dind +++ b/tests/Dockerfile-ssh-dind @@ -1,5 +1,5 @@ -ARG API_VERSION=1.39 -ARG ENGINE_VERSION=19.03.12 +ARG API_VERSION=1.41 +ARG ENGINE_VERSION=20.10.17 FROM docker:${ENGINE_VERSION}-dind From 52e29bd4463964a090e3425cf027a3a4a8c4473b Mon Sep 17 00:00:00 2001 From: Milas Bowman Date: Wed, 27 Jul 2022 14:44:50 -0400 Subject: [PATCH 1151/1301] deps: remove backports.ssl_match_hostname (#3011) This is no longer needed as it exists in every supported (non-EOL) version of Python that we target. Signed-off-by: Milas Bowman --- docker/transport/ssladapter.py | 8 -------- requirements.txt | 1 - tests/unit/ssladapter_test.py | 13 +++---------- 3 files changed, 3 insertions(+), 19 deletions(-) diff --git a/docker/transport/ssladapter.py b/docker/transport/ssladapter.py index bdca1d0453..6aa80037d7 100644 --- a/docker/transport/ssladapter.py +++ b/docker/transport/ssladapter.py @@ -2,8 +2,6 @@ https://lukasa.co.uk/2013/01/Choosing_SSL_Version_In_Requests/ https://github.com/kennethreitz/requests/pull/799 """ -import sys - from packaging.version import Version from requests.adapters import HTTPAdapter @@ -17,12 +15,6 @@ PoolManager = urllib3.poolmanager.PoolManager -# Monkey-patching match_hostname with a version that supports -# IP-address checking. Not necessary for Python 3.5 and above -if sys.version_info[0] < 3 or sys.version_info[1] < 5: - from backports.ssl_match_hostname import match_hostname - urllib3.connection.match_hostname = match_hostname - class SSLHTTPAdapter(BaseHTTPAdapter): '''An HTTPS Transport Adapter that uses an arbitrary SSL version.''' diff --git a/requirements.txt b/requirements.txt index 7bcca763e5..a74e69ea66 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,5 @@ appdirs==1.4.3 asn1crypto==0.22.0 -backports.ssl-match-hostname==3.5.0.1 cffi==1.14.4 cryptography==3.4.7 enum34==1.1.6 diff --git a/tests/unit/ssladapter_test.py b/tests/unit/ssladapter_test.py index 41a87f207e..d3f2407c39 100644 --- a/tests/unit/ssladapter_test.py +++ b/tests/unit/ssladapter_test.py @@ -1,15 +1,8 @@ import unittest -from docker.transport import ssladapter -import pytest +from ssl import match_hostname, CertificateError -try: - from backports.ssl_match_hostname import ( - match_hostname, CertificateError - ) -except ImportError: - from ssl import ( - match_hostname, CertificateError - ) +import pytest +from docker.transport import ssladapter try: from ssl import OP_NO_SSLv3, OP_NO_SSLv2, OP_NO_TLSv1 From bb40ba051fc67605d5c9e7fd1eb5f9aa3e0fb501 Mon Sep 17 00:00:00 2001 From: errorcode Date: Thu, 28 Jul 2022 02:57:26 +0800 Subject: [PATCH 1152/1301] ssh: do not create unnecessary subshell on exec (#2910) Signed-off-by: liubo --- docker/transport/sshconn.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docker/transport/sshconn.py b/docker/transport/sshconn.py index 76d1fa4439..ba8c11d1f5 100644 --- a/docker/transport/sshconn.py +++ b/docker/transport/sshconn.py @@ -58,9 +58,8 @@ def f(): env.pop('SSL_CERT_FILE', None) self.proc = subprocess.Popen( - ' '.join(args), + args, env=env, - shell=True, stdout=subprocess.PIPE, stdin=subprocess.PIPE, preexec_fn=None if constants.IS_WINDOWS_PLATFORM else preexec_func) From d9298647d91c52e1ee9ac448e43a7fea1c69bdbe Mon Sep 17 00:00:00 2001 From: "Audun V. Nes" Date: Wed, 27 Jul 2022 21:01:41 +0200 Subject: [PATCH 1153/1301] ssh: reject unknown host keys when using Python SSH impl (#2932) In the Secure Shell (SSH) protocol, host keys are used to verify the identity of remote hosts. Accepting unknown host keys may leave the connection open to man-in-the-middle attacks. Do not accept unknown host keys. In particular, do not set the default missing host key policy for the Paramiko library to either AutoAddPolicy or WarningPolicy. Both of these policies continue even when the host key is unknown. The default setting of RejectPolicy is secure because it throws an exception when it encounters an unknown host key. Reference: https://cwe.mitre.org/data/definitions/295.html NOTE: This only affects SSH connections using the native Python SSH implementation (Paramiko), when `use_ssh_client=False` (default). If using the system SSH client (`use_ssh_client=True`), the host configuration (e.g. `~/.ssh/config`) will apply. Signed-off-by: Audun Nes --- docker/transport/sshconn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/transport/sshconn.py b/docker/transport/sshconn.py index ba8c11d1f5..4f748f75ab 100644 --- a/docker/transport/sshconn.py +++ b/docker/transport/sshconn.py @@ -215,7 +215,7 @@ def _create_paramiko_client(self, base_url): self.ssh_params['key_filename'] = host_config['identityfile'] self.ssh_client.load_system_host_keys() - self.ssh_client.set_missing_host_key_policy(paramiko.WarningPolicy()) + self.ssh_client.set_missing_host_key_policy(paramiko.RejectPolicy()) def _connect(self): if self.ssh_client: From adf5a97b1203623ae47bf7aa1367b6bb7c261980 Mon Sep 17 00:00:00 2001 From: Karthikeyan Singaravelan Date: Thu, 28 Jul 2022 00:55:11 +0530 Subject: [PATCH 1154/1301] lint: fix deprecation warnings from threading package (#2823) Set `daemon` attribute instead of using `setDaemon` method that was deprecated in Python 3.10. Signed-off-by: Karthikeyan Singaravelan --- tests/integration/api_image_test.py | 2 +- tests/unit/api_test.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration/api_image_test.py b/tests/integration/api_image_test.py index e30de46c04..6a6686e377 100644 --- a/tests/integration/api_image_test.py +++ b/tests/integration/api_image_test.py @@ -281,7 +281,7 @@ def do_GET(self): server = socketserver.TCPServer(('', 0), Handler) thread = threading.Thread(target=server.serve_forever) - thread.setDaemon(True) + thread.daemon = True thread.start() yield f'http://{socket.gethostname()}:{server.server_address[1]}' diff --git a/tests/unit/api_test.py b/tests/unit/api_test.py index 3234e55b11..45d2e4c034 100644 --- a/tests/unit/api_test.py +++ b/tests/unit/api_test.py @@ -378,7 +378,7 @@ def setUp(self): self.server_socket = self._setup_socket() self.stop_server = False server_thread = threading.Thread(target=self.run_server) - server_thread.setDaemon(True) + server_thread.daemon = True server_thread.start() self.response = None self.request_handler = None @@ -488,7 +488,7 @@ def setup_class(cls): cls.server = socketserver.ThreadingTCPServer( ('', 0), cls.get_handler_class()) cls.thread = threading.Thread(target=cls.server.serve_forever) - cls.thread.setDaemon(True) + cls.thread.daemon = True cls.thread.start() cls.address = 'http://{}:{}'.format( socket.gethostname(), cls.server.server_address[1]) From ea4cefe4fd1e85ef94f477b8e969994117fcb076 Mon Sep 17 00:00:00 2001 From: Vilhelm Prytz Date: Wed, 27 Jul 2022 21:31:04 +0200 Subject: [PATCH 1155/1301] lint: remove unnecessary pass statements (#2541) Signed-off-by: Vilhelm Prytz --- docker/auth.py | 1 - docker/transport/npipeconn.py | 2 +- docker/transport/sshconn.py | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/docker/auth.py b/docker/auth.py index 4fa798fcc0..cb3885548f 100644 --- a/docker/auth.py +++ b/docker/auth.py @@ -383,7 +383,6 @@ def _load_legacy_config(config_file): }} except Exception as e: log.debug(e) - pass log.debug("All parsing attempts failed - returning empty config") return {} diff --git a/docker/transport/npipeconn.py b/docker/transport/npipeconn.py index df67f21251..87033cf2af 100644 --- a/docker/transport/npipeconn.py +++ b/docker/transport/npipeconn.py @@ -61,7 +61,7 @@ def _get_conn(self, timeout): "Pool reached maximum size and no more " "connections are allowed." ) - pass # Oh well, we'll create a new connection then + # Oh well, we'll create a new connection then return conn or self._new_conn() diff --git a/docker/transport/sshconn.py b/docker/transport/sshconn.py index 4f748f75ab..277640690a 100644 --- a/docker/transport/sshconn.py +++ b/docker/transport/sshconn.py @@ -155,7 +155,7 @@ def _get_conn(self, timeout): "Pool reached maximum size and no more " "connections are allowed." ) - pass # Oh well, we'll create a new connection then + # Oh well, we'll create a new connection then return conn or self._new_conn() From acdafbc116ac2348dcf41055402dbb5ecfad8be2 Mon Sep 17 00:00:00 2001 From: Milas Bowman Date: Wed, 27 Jul 2022 16:25:27 -0400 Subject: [PATCH 1156/1301] ci: run SSH integration tests (#3012) Fix & enable SSH integration test suite. This also adds a new test for connecting to unknown hosts when using the Python SSH implementation (Paramiko). See #2932 for more info. Because of the above, some of the config/static key files have been moved around and adjusted. Signed-off-by: Milas Bowman --- .github/workflows/ci.yml | 2 +- Makefile | 41 ++++++++++++++----- tests/Dockerfile | 4 +- tests/Dockerfile-ssh-dind | 19 ++++----- tests/ssh-keys/authorized_keys | 1 - tests/ssh-keys/config | 3 -- tests/ssh/base.py | 4 ++ tests/{ssh-keys => ssh/config/client}/id_rsa | 0 .../config/client}/id_rsa.pub | 0 tests/ssh/config/server/known_ed25519 | 7 ++++ tests/ssh/config/server/known_ed25519.pub | 1 + tests/ssh/config/server/sshd_config | 3 ++ tests/ssh/config/server/unknown_ed25519 | 7 ++++ tests/ssh/config/server/unknown_ed25519.pub | 1 + tests/ssh/connect_test.py | 22 ++++++++++ 15 files changed, 86 insertions(+), 29 deletions(-) delete mode 100755 tests/ssh-keys/authorized_keys delete mode 100644 tests/ssh-keys/config rename tests/{ssh-keys => ssh/config/client}/id_rsa (100%) rename tests/{ssh-keys => ssh/config/client}/id_rsa.pub (100%) create mode 100644 tests/ssh/config/server/known_ed25519 create mode 100644 tests/ssh/config/server/known_ed25519.pub create mode 100644 tests/ssh/config/server/sshd_config create mode 100644 tests/ssh/config/server/unknown_ed25519 create mode 100644 tests/ssh/config/server/unknown_ed25519.pub create mode 100644 tests/ssh/connect_test.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e2987b49a7..296bf0ddd6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,7 +40,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - variant: [ "integration-dind", "integration-dind-ssl" ] + variant: [ "integration-dind", "integration-dind-ssl", "integration-dind-ssh" ] steps: - uses: actions/checkout@v3 diff --git a/Makefile b/Makefile index 27144d4d8d..ae6ae34ef2 100644 --- a/Makefile +++ b/Makefile @@ -21,11 +21,21 @@ clean: .PHONY: build-dind-ssh build-dind-ssh: - docker build -t docker-dind-ssh -f tests/Dockerfile-ssh-dind --build-arg ENGINE_VERSION=${TEST_ENGINE_VERSION} --build-arg API_VERSION=${TEST_API_VERSION} --build-arg APT_MIRROR . + docker build \ + --pull \ + -t docker-dind-ssh \ + -f tests/Dockerfile-ssh-dind \ + --build-arg ENGINE_VERSION=${TEST_ENGINE_VERSION} \ + --build-arg API_VERSION=${TEST_API_VERSION} \ + --build-arg APT_MIRROR . .PHONY: build-py3 build-py3: - docker build -t docker-sdk-python3 -f tests/Dockerfile --build-arg APT_MIRROR . + docker build \ + --pull \ + -t docker-sdk-python3 \ + -f tests/Dockerfile \ + --build-arg APT_MIRROR . .PHONY: build-docs build-docs: @@ -61,6 +71,7 @@ integration-dind-py3: build-py3 setup-network --detach \ --name dpy-dind-py3 \ --network dpy-tests \ + --pull=always \ --privileged \ docker:${TEST_ENGINE_VERSION}-dind \ dockerd -H tcp://0.0.0.0:2375 --experimental @@ -85,16 +96,23 @@ integration-dind-py3: build-py3 setup-network docker rm -vf dpy-dind-py3 -.PHONY: integration-ssh-py3 -integration-ssh-py3: build-dind-ssh build-py3 setup-network - docker rm -vf dpy-dind-py3 || : - docker run -d --network dpy-tests --name dpy-dind-py3 --privileged\ +.PHONY: integration-dind-ssh +integration-dind-ssh: build-dind-ssh build-py3 setup-network + docker rm -vf dpy-dind-ssh || : + docker run -d --network dpy-tests --name dpy-dind-ssh --privileged \ docker-dind-ssh dockerd --experimental - # start SSH daemon - docker exec dpy-dind-py3 sh -c "/usr/sbin/sshd" - docker run -t --rm --env="DOCKER_HOST=ssh://dpy-dind-py3" --env="DOCKER_TEST_API_VERSION=${TEST_API_VERSION}"\ - --network dpy-tests docker-sdk-python3 py.test tests/ssh/${file} - docker rm -vf dpy-dind-py3 + # start SSH daemon for known key + docker exec dpy-dind-ssh sh -c "/usr/sbin/sshd -h /etc/ssh/known_ed25519 -p 22" + docker exec dpy-dind-ssh sh -c "/usr/sbin/sshd -h /etc/ssh/unknown_ed25519 -p 2222" + docker run \ + --tty \ + --rm \ + --env="DOCKER_HOST=ssh://dpy-dind-ssh" \ + --env="DOCKER_TEST_API_VERSION=${TEST_API_VERSION}" \ + --env="UNKNOWN_DOCKER_SSH_HOST=ssh://dpy-dind-ssh:2222" \ + --network dpy-tests \ + docker-sdk-python3 py.test tests/ssh/${file} + docker rm -vf dpy-dind-ssh .PHONY: integration-dind-ssl @@ -110,6 +128,7 @@ integration-dind-ssl: build-dind-certs build-py3 setup-network --name dpy-dind-ssl \ --network dpy-tests \ --network-alias docker \ + --pull=always \ --privileged \ --volume /tmp \ --volumes-from dpy-dind-certs \ diff --git a/tests/Dockerfile b/tests/Dockerfile index 1d60cfe42d..e24da47d46 100644 --- a/tests/Dockerfile +++ b/tests/Dockerfile @@ -11,7 +11,9 @@ RUN apt-get update && apt-get -y install --no-install-recommends \ pass # Add SSH keys and set permissions -COPY tests/ssh-keys /root/.ssh +COPY tests/ssh/config/client /root/.ssh +COPY tests/ssh/config/server/known_ed25519.pub /root/.ssh/known_hosts +RUN sed -i '1s;^;dpy-dind-ssh ;' /root/.ssh/known_hosts RUN chmod -R 600 /root/.ssh COPY ./tests/gpg-keys /gpg-keys diff --git a/tests/Dockerfile-ssh-dind b/tests/Dockerfile-ssh-dind index 6f080182d5..22c707a075 100644 --- a/tests/Dockerfile-ssh-dind +++ b/tests/Dockerfile-ssh-dind @@ -1,23 +1,18 @@ ARG API_VERSION=1.41 -ARG ENGINE_VERSION=20.10.17 +ARG ENGINE_VERSION=20.10 FROM docker:${ENGINE_VERSION}-dind -RUN apk add --no-cache \ +RUN apk add --no-cache --upgrade \ openssh -# Add the keys and set permissions -RUN ssh-keygen -A - -# copy the test SSH config -RUN echo "IgnoreUserKnownHosts yes" > /etc/ssh/sshd_config && \ - echo "PubkeyAuthentication yes" >> /etc/ssh/sshd_config && \ - echo "PermitRootLogin yes" >> /etc/ssh/sshd_config +COPY tests/ssh/config/server /etc/ssh/ +RUN chmod -R 600 /etc/ssh # set authorized keys for client paswordless connection -COPY tests/ssh-keys/authorized_keys /root/.ssh/authorized_keys -RUN chmod 600 /root/.ssh/authorized_keys +COPY tests/ssh/config/client/id_rsa.pub /root/.ssh/authorized_keys +RUN chmod -R 600 /root/.ssh -RUN echo "root:root" | chpasswd +# RUN echo "root:root" | chpasswd RUN ln -s /usr/local/bin/docker /usr/bin/docker EXPOSE 22 diff --git a/tests/ssh-keys/authorized_keys b/tests/ssh-keys/authorized_keys deleted file mode 100755 index 33252fe503..0000000000 --- a/tests/ssh-keys/authorized_keys +++ /dev/null @@ -1 +0,0 @@ -ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC/BiXkbL9oEbE3PJv1S2p12XK5BHW3qQT5Rf+CYG0ATYyMPIVM6+IXVyf3QNxpnvPXvbPBQJCs0qHeuPwZy2Gsbt35QnmlgrczFPiXXosCD2N+wrcOQPZGuLjQyUUP2yJRVSTLpp8zk2F8w3laGIB3Jk1hUcMUExemKxQYk/L40b5rXKkarLk5awBuicjRStMrchPRHZ2n715TG+zSvf8tB/UHRXKYPqai/Je5eiH3yGUzCY4zn+uEoqAFb4V8lpIj8Rw3EXmCYVwG0vg+44QIQ2gJnIhTlcmxwkynvZn97nug4NLlGJQ+sDCnIvMapycHfGkNlBz3fFtu/ORsxPpZbTNg/9noa3Zf8OpIwvE/FHNPqDctGltwxEgQxj5fE34x0fYnF08tejAUJJCZE3YsGgNabsS4pD+kRhI83eFZvgj3Q1AeTK0V9bRM7jujcc9Rz+V9Gb5zYEHN/l8PxEVlj0OlURf9ZlknNQK8xRh597jDXTfVQKCMO/nRaWH2bq0= diff --git a/tests/ssh-keys/config b/tests/ssh-keys/config deleted file mode 100644 index 8dd13540ff..0000000000 --- a/tests/ssh-keys/config +++ /dev/null @@ -1,3 +0,0 @@ -Host * - StrictHostKeyChecking no - UserKnownHostsFile=/dev/null diff --git a/tests/ssh/base.py b/tests/ssh/base.py index 4825227f38..4b91add4be 100644 --- a/tests/ssh/base.py +++ b/tests/ssh/base.py @@ -2,6 +2,8 @@ import shutil import unittest +import pytest + import docker from .. import helpers from docker.utils import kwargs_from_env @@ -68,6 +70,8 @@ def tearDown(self): client.close() +@pytest.mark.skipif(not os.environ.get('DOCKER_HOST', '').startswith('ssh://'), + reason='DOCKER_HOST is not an SSH target') class BaseAPIIntegrationTest(BaseIntegrationTest): """ A test case for `APIClient` integration tests. It sets up an `APIClient` diff --git a/tests/ssh-keys/id_rsa b/tests/ssh/config/client/id_rsa similarity index 100% rename from tests/ssh-keys/id_rsa rename to tests/ssh/config/client/id_rsa diff --git a/tests/ssh-keys/id_rsa.pub b/tests/ssh/config/client/id_rsa.pub similarity index 100% rename from tests/ssh-keys/id_rsa.pub rename to tests/ssh/config/client/id_rsa.pub diff --git a/tests/ssh/config/server/known_ed25519 b/tests/ssh/config/server/known_ed25519 new file mode 100644 index 0000000000..b79f217b88 --- /dev/null +++ b/tests/ssh/config/server/known_ed25519 @@ -0,0 +1,7 @@ +-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW +QyNTUxOQAAACCGsfNXVP18N7XC6IQGuuxXQRbTxlPGLj+5/CByj9eg4QAAAJgIMffcCDH3 +3AAAAAtzc2gtZWQyNTUxOQAAACCGsfNXVP18N7XC6IQGuuxXQRbTxlPGLj+5/CByj9eg4Q +AAAEDeXnt5AuNk4oTHjMU1vUsEwh64fuEPu4hXsG6wCVt/6Iax81dU/Xw3tcLohAa67FdB +FtPGU8YuP7n8IHKP16DhAAAAEXJvb3RAMGRkZmQyMWRkYjM3AQIDBA== +-----END OPENSSH PRIVATE KEY----- diff --git a/tests/ssh/config/server/known_ed25519.pub b/tests/ssh/config/server/known_ed25519.pub new file mode 100644 index 0000000000..ec0296e9d4 --- /dev/null +++ b/tests/ssh/config/server/known_ed25519.pub @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIax81dU/Xw3tcLohAa67FdBFtPGU8YuP7n8IHKP16Dh docker-py integration tests known diff --git a/tests/ssh/config/server/sshd_config b/tests/ssh/config/server/sshd_config new file mode 100644 index 0000000000..970dca337c --- /dev/null +++ b/tests/ssh/config/server/sshd_config @@ -0,0 +1,3 @@ +IgnoreUserKnownHosts yes +PubkeyAuthentication yes +PermitRootLogin yes diff --git a/tests/ssh/config/server/unknown_ed25519 b/tests/ssh/config/server/unknown_ed25519 new file mode 100644 index 0000000000..b79f217b88 --- /dev/null +++ b/tests/ssh/config/server/unknown_ed25519 @@ -0,0 +1,7 @@ +-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW +QyNTUxOQAAACCGsfNXVP18N7XC6IQGuuxXQRbTxlPGLj+5/CByj9eg4QAAAJgIMffcCDH3 +3AAAAAtzc2gtZWQyNTUxOQAAACCGsfNXVP18N7XC6IQGuuxXQRbTxlPGLj+5/CByj9eg4Q +AAAEDeXnt5AuNk4oTHjMU1vUsEwh64fuEPu4hXsG6wCVt/6Iax81dU/Xw3tcLohAa67FdB +FtPGU8YuP7n8IHKP16DhAAAAEXJvb3RAMGRkZmQyMWRkYjM3AQIDBA== +-----END OPENSSH PRIVATE KEY----- diff --git a/tests/ssh/config/server/unknown_ed25519.pub b/tests/ssh/config/server/unknown_ed25519.pub new file mode 100644 index 0000000000..a24403ed9b --- /dev/null +++ b/tests/ssh/config/server/unknown_ed25519.pub @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIax81dU/Xw3tcLohAa67FdBFtPGU8YuP7n8IHKP16Dh docker-py integration tests unknown diff --git a/tests/ssh/connect_test.py b/tests/ssh/connect_test.py new file mode 100644 index 0000000000..3d33a96db2 --- /dev/null +++ b/tests/ssh/connect_test.py @@ -0,0 +1,22 @@ +import os +import unittest + +import docker +import paramiko.ssh_exception +import pytest +from .base import TEST_API_VERSION + + +class SSHConnectionTest(unittest.TestCase): + @pytest.mark.skipif('UNKNOWN_DOCKER_SSH_HOST' not in os.environ, + reason='Unknown Docker SSH host not configured') + def test_ssh_unknown_host(self): + with self.assertRaises(paramiko.ssh_exception.SSHException) as cm: + docker.APIClient( + version=TEST_API_VERSION, + timeout=60, + # test only valid with Paramiko + use_ssh_client=False, + base_url=os.environ['UNKNOWN_DOCKER_SSH_HOST'], + ) + self.assertIn('not found in known_hosts', str(cm.exception)) From d2d097efbb1675393a1ac5b17754ba9090d2c52e Mon Sep 17 00:00:00 2001 From: Tim Gates Date: Thu, 28 Jul 2022 22:30:40 +1000 Subject: [PATCH 1157/1301] docs: fix simple typo, containe -> container (#3015) There is a small typo in docker/types/services.py. Should read `container` rather than `containe`. Signed-off-by: Tim Gates --- docker/types/services.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/types/services.py b/docker/types/services.py index fe7cc264cd..15cf511e8b 100644 --- a/docker/types/services.py +++ b/docker/types/services.py @@ -436,7 +436,7 @@ def __init__(self, parallelism=0, delay=None, failure_action='continue', class RollbackConfig(UpdateConfig): """ - Used to specify the way containe rollbacks should be performed by a service + Used to specify the way container rollbacks should be performed by a service Args: parallelism (int): Maximum number of tasks to be rolled back in one From bf026265e0adfd862373a601ed99e4f3ac8b3bd0 Mon Sep 17 00:00:00 2001 From: Milas Bowman Date: Thu, 28 Jul 2022 08:31:45 -0400 Subject: [PATCH 1158/1301] ci: bump version to 6.0.0-dev (#3013) It's been a long time without a release, and we've included a number of fixes as well as raised the minimum Python version, so a major release seems in order. Signed-off-by: Milas Bowman --- docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index 5687086f16..88ee8b0f3d 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "5.1.0-dev" +version = "6.0.0-dev" version_info = tuple(int(d) for d in version.split("-")[0].split(".")) From be942f83902fbd02e05270c39b6917880939c165 Mon Sep 17 00:00:00 2001 From: Milas Bowman Date: Thu, 28 Jul 2022 08:32:00 -0400 Subject: [PATCH 1159/1301] deps: upgrade & remove unnecessary dependencies (#3014) The `requirements.txt` and `setup.py` had a lot of extra transitive dependencies to try and address various SSL shortcomings from the Python ecosystem. Thankfully, between modern Python versions (3.6+) and corresponding `requests` versions (2.26+), this is all unnecessary now! As a result, a bunch of transitive dependencies have been removed from `requirements.txt`, the minimum version of `requests` increased, and the `tls` extra made into a no-op. Signed-off-by: Milas Bowman --- README.md | 5 ++--- appveyor.yml | 13 ------------- requirements.txt | 16 +++------------- setup.py | 19 +++++++------------ test-requirements.txt | 8 ++++---- 5 files changed, 16 insertions(+), 45 deletions(-) delete mode 100644 appveyor.yml diff --git a/README.md b/README.md index 4fc31f7d75..2db678dccc 100644 --- a/README.md +++ b/README.md @@ -10,9 +10,8 @@ The latest stable version [is available on PyPI](https://pypi.python.org/pypi/do pip install docker -If you are intending to connect to a docker host via TLS, add `docker[tls]` to your requirements instead, or install with pip: - - pip install docker[tls] +> Older versions (< 6.0) required installing `docker[tls]` for SSL/TLS support. +> This is no longer necessary and is a no-op, but is supported for backwards compatibility. ## Usage diff --git a/appveyor.yml b/appveyor.yml deleted file mode 100644 index 144ab35289..0000000000 --- a/appveyor.yml +++ /dev/null @@ -1,13 +0,0 @@ -version: '{branch}-{build}' - -install: - - "SET PATH=C:\\Python37-x64;C:\\Python37-x64\\Scripts;%PATH%" - - "python --version" - - "python -m pip install --upgrade pip" - - "pip install tox==2.9.1" - -# Build the binary after tests -build: false - -test_script: - - "tox" diff --git a/requirements.txt b/requirements.txt index a74e69ea66..52b5461e2e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,16 +1,6 @@ -appdirs==1.4.3 -asn1crypto==0.22.0 -cffi==1.14.4 -cryptography==3.4.7 -enum34==1.1.6 -idna==2.5 -ipaddress==1.0.18 packaging==21.3 -paramiko==2.10.1 -pycparser==2.17 -pyOpenSSL==18.0.0 -pyparsing==2.2.0 +paramiko==2.11.0 pywin32==304; sys_platform == 'win32' -requests==2.26.0 -urllib3==1.26.5 +requests==2.28.1 +urllib3==1.26.11 websocket-client==0.56.0 diff --git a/setup.py b/setup.py index 0b113688fc..c6346b0790 100644 --- a/setup.py +++ b/setup.py @@ -10,28 +10,23 @@ SOURCE_DIR = os.path.join(ROOT_DIR) requirements = [ - 'packaging', + 'packaging >= 14.0', + 'requests >= 2.26.0', + 'urllib3 >= 1.26.0', 'websocket-client >= 0.32.0', - 'requests >= 2.14.2, != 2.18.0', ] extras_require = { # win32 APIs if on Windows (required for npipe support) ':sys_platform == "win32"': 'pywin32>=304', - # If using docker-py over TLS, highly recommend this option is - # pip-installed or pinned. - - # TODO: if pip installing both "requests" and "requests[security]", the - # extra package from the "security" option are not installed (see - # https://github.com/pypa/pip/issues/4391). Once that's fixed, instead of - # installing the extra dependencies, install the following instead: - # 'requests[security] >= 2.5.2, != 2.11.0, != 2.12.2' - 'tls': ['pyOpenSSL>=17.5.0', 'cryptography>=3.4.7', 'idna>=2.0.0'], + # This is now a no-op, as similarly the requests[security] extra is + # a no-op as of requests 2.26.0, this is always available/by default now + # see https://github.com/psf/requests/pull/5867 + 'tls': [], # Only required when connecting using the ssh:// protocol 'ssh': ['paramiko>=2.4.3'], - } version = None diff --git a/test-requirements.txt b/test-requirements.txt index ccc97be46f..979b291cf7 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,6 +1,6 @@ -setuptools==58.2.0 -coverage==6.0.1 +setuptools==63.2.0 +coverage==6.4.2 flake8==4.0.1 -pytest==6.2.5 +pytest==7.1.2 pytest-cov==3.0.0 -pytest-timeout==2.0.1 +pytest-timeout==2.1.0 From 9bdb5ba2bab682a02bc3348e359822218dad7e96 Mon Sep 17 00:00:00 2001 From: Milas Bowman Date: Thu, 28 Jul 2022 11:25:17 -0400 Subject: [PATCH 1160/1301] lint: fix line length violation (#3017) Signed-off-by: Milas Bowman --- docker/types/services.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/types/services.py b/docker/types/services.py index 15cf511e8b..c2fce9f496 100644 --- a/docker/types/services.py +++ b/docker/types/services.py @@ -436,7 +436,8 @@ def __init__(self, parallelism=0, delay=None, failure_action='continue', class RollbackConfig(UpdateConfig): """ - Used to specify the way container rollbacks should be performed by a service + Used to specify the way container rollbacks should be performed by a + service Args: parallelism (int): Maximum number of tasks to be rolled back in one From ab43018b027e48c53f3cf6d71ce988358e3c204e Mon Sep 17 00:00:00 2001 From: Milas Bowman Date: Thu, 28 Jul 2022 16:38:57 -0400 Subject: [PATCH 1161/1301] docs: fix markdown rendering (#3020) Follow instructions at https://www.sphinx-doc.org/en/master/usage/markdown.html. This switches from `recommonmark` (deprecated) to `myst-parser` (recommended). Only impacts the changelog page, which was broken after recent upgrades to Sphinx for Python 3.10 compatibility. Signed-off-by: Milas Bowman --- docs-requirements.txt | 2 +- docs/conf.py | 17 ++++++----------- 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 1f342fa272..04d1aff268 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -1,2 +1,2 @@ -recommonmark==0.7.1 +myst-parser==0.18.0 Sphinx==5.1.1 diff --git a/docs/conf.py b/docs/conf.py index 2b0a719531..1258a42386 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -33,24 +33,19 @@ extensions = [ 'sphinx.ext.autodoc', 'sphinx.ext.napoleon', + 'myst_parser' ] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] -from recommonmark.parser import CommonMarkParser - -source_parsers = { - '.md': CommonMarkParser, +source_suffix = { + '.rst': 'restructuredtext', + '.txt': 'markdown', + '.md': 'markdown', } -# The suffix(es) of source filenames. -# You can specify multiple suffix as a list of string: -# -source_suffix = ['.rst', '.md'] -# source_suffix = '.md' - # The encoding of source files. # # source_encoding = 'utf-8-sig' @@ -80,7 +75,7 @@ # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +language = 'en' # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: From 23cf16f03a38e553d1e15493719cdb57e928fd95 Mon Sep 17 00:00:00 2001 From: Ben Fasoli Date: Fri, 29 Jul 2022 06:06:22 -0700 Subject: [PATCH 1162/1301] client: use 12 character short IDs (#2862) Use 12 characters for Docker resource IDs for consistency with the Docker CLI. Signed-off-by: Ben Fasoli --- docker/models/images.py | 10 ++-- docker/models/resource.py | 4 +- tests/unit/api_container_test.py | 87 ++++++++++++++++------------ tests/unit/api_image_test.py | 12 ++-- tests/unit/api_test.py | 2 +- tests/unit/fake_api.py | 56 +++++++++--------- tests/unit/models_containers_test.py | 5 ++ tests/unit/models_images_test.py | 4 +- 8 files changed, 99 insertions(+), 81 deletions(-) diff --git a/docker/models/images.py b/docker/models/images.py index ef668c7d4e..e247d351e1 100644 --- a/docker/models/images.py +++ b/docker/models/images.py @@ -31,12 +31,12 @@ def labels(self): @property def short_id(self): """ - The ID of the image truncated to 10 characters, plus the ``sha256:`` + The ID of the image truncated to 12 characters, plus the ``sha256:`` prefix. """ if self.id.startswith('sha256:'): - return self.id[:17] - return self.id[:10] + return self.id[:19] + return self.id[:12] @property def tags(self): @@ -141,10 +141,10 @@ def id(self): @property def short_id(self): """ - The ID of the image truncated to 10 characters, plus the ``sha256:`` + The ID of the image truncated to 12 characters, plus the ``sha256:`` prefix. """ - return self.id[:17] + return self.id[:19] def pull(self, platform=None): """ diff --git a/docker/models/resource.py b/docker/models/resource.py index dec2349f67..89030e592e 100644 --- a/docker/models/resource.py +++ b/docker/models/resource.py @@ -35,9 +35,9 @@ def id(self): @property def short_id(self): """ - The ID of the object, truncated to 10 characters. + The ID of the object, truncated to 12 characters. """ - return self.id[:10] + return self.id[:12] def reload(self): """ diff --git a/tests/unit/api_container_test.py b/tests/unit/api_container_test.py index a66aea047f..7030841682 100644 --- a/tests/unit/api_container_test.py +++ b/tests/unit/api_container_test.py @@ -24,7 +24,8 @@ def test_start_container(self): self.client.start(fake_api.FAKE_CONTAINER_ID) args = fake_request.call_args - assert args[0][1] == url_prefix + 'containers/3cc2351ab11b/start' + assert args[0][1] == (url_prefix + 'containers/' + + fake_api.FAKE_CONTAINER_ID + '/start') assert 'data' not in args[1] assert args[1]['timeout'] == DEFAULT_TIMEOUT_SECONDS @@ -117,7 +118,8 @@ def test_start_container_with_dict_instead_of_id(self): self.client.start({'Id': fake_api.FAKE_CONTAINER_ID}) args = fake_request.call_args - assert args[0][1] == url_prefix + 'containers/3cc2351ab11b/start' + assert args[0][1] == (url_prefix + 'containers/' + + fake_api.FAKE_CONTAINER_ID + '/start') assert 'data' not in args[1] assert args[1]['timeout'] == DEFAULT_TIMEOUT_SECONDS @@ -1079,7 +1081,8 @@ def test_resize_container(self): fake_request.assert_called_with( 'POST', - url_prefix + 'containers/3cc2351ab11b/resize', + (url_prefix + 'containers/' + + fake_api.FAKE_CONTAINER_ID + '/resize'), params={'h': 15, 'w': 120}, timeout=DEFAULT_TIMEOUT_SECONDS ) @@ -1092,7 +1095,8 @@ def test_rename_container(self): fake_request.assert_called_with( 'POST', - url_prefix + 'containers/3cc2351ab11b/rename', + (url_prefix + 'containers/' + + fake_api.FAKE_CONTAINER_ID + '/rename'), params={'name': 'foobar'}, timeout=DEFAULT_TIMEOUT_SECONDS ) @@ -1102,7 +1106,7 @@ def test_wait(self): fake_request.assert_called_with( 'POST', - url_prefix + 'containers/3cc2351ab11b/wait', + url_prefix + 'containers/' + fake_api.FAKE_CONTAINER_ID + '/wait', timeout=None, params={} ) @@ -1112,7 +1116,7 @@ def test_wait_with_dict_instead_of_id(self): fake_request.assert_called_with( 'POST', - url_prefix + 'containers/3cc2351ab11b/wait', + url_prefix + 'containers/' + fake_api.FAKE_CONTAINER_ID + '/wait', timeout=None, params={} ) @@ -1124,7 +1128,7 @@ def test_logs(self): fake_request.assert_called_with( 'GET', - url_prefix + 'containers/3cc2351ab11b/logs', + url_prefix + 'containers/' + fake_api.FAKE_CONTAINER_ID + '/logs', params={'timestamps': 0, 'follow': 0, 'stderr': 1, 'stdout': 1, 'tail': 'all'}, timeout=DEFAULT_TIMEOUT_SECONDS, @@ -1140,7 +1144,7 @@ def test_logs_with_dict_instead_of_id(self): fake_request.assert_called_with( 'GET', - url_prefix + 'containers/3cc2351ab11b/logs', + url_prefix + 'containers/' + fake_api.FAKE_CONTAINER_ID + '/logs', params={'timestamps': 0, 'follow': 0, 'stderr': 1, 'stdout': 1, 'tail': 'all'}, timeout=DEFAULT_TIMEOUT_SECONDS, @@ -1157,7 +1161,7 @@ def test_log_streaming(self): fake_request.assert_called_with( 'GET', - url_prefix + 'containers/3cc2351ab11b/logs', + url_prefix + 'containers/' + fake_api.FAKE_CONTAINER_ID + '/logs', params={'timestamps': 0, 'follow': 0, 'stderr': 1, 'stdout': 1, 'tail': 'all'}, timeout=DEFAULT_TIMEOUT_SECONDS, @@ -1172,7 +1176,7 @@ def test_log_following(self): fake_request.assert_called_with( 'GET', - url_prefix + 'containers/3cc2351ab11b/logs', + url_prefix + 'containers/' + fake_api.FAKE_CONTAINER_ID + '/logs', params={'timestamps': 0, 'follow': 1, 'stderr': 1, 'stdout': 1, 'tail': 'all'}, timeout=DEFAULT_TIMEOUT_SECONDS, @@ -1186,7 +1190,7 @@ def test_log_following_backwards(self): fake_request.assert_called_with( 'GET', - url_prefix + 'containers/3cc2351ab11b/logs', + url_prefix + 'containers/' + fake_api.FAKE_CONTAINER_ID + '/logs', params={'timestamps': 0, 'follow': 1, 'stderr': 1, 'stdout': 1, 'tail': 'all'}, timeout=DEFAULT_TIMEOUT_SECONDS, @@ -1201,7 +1205,7 @@ def test_log_streaming_and_following(self): fake_request.assert_called_with( 'GET', - url_prefix + 'containers/3cc2351ab11b/logs', + url_prefix + 'containers/' + fake_api.FAKE_CONTAINER_ID + '/logs', params={'timestamps': 0, 'follow': 1, 'stderr': 1, 'stdout': 1, 'tail': 'all'}, timeout=DEFAULT_TIMEOUT_SECONDS, @@ -1217,7 +1221,7 @@ def test_log_tail(self): fake_request.assert_called_with( 'GET', - url_prefix + 'containers/3cc2351ab11b/logs', + url_prefix + 'containers/' + fake_api.FAKE_CONTAINER_ID + '/logs', params={'timestamps': 0, 'follow': 0, 'stderr': 1, 'stdout': 1, 'tail': 10}, timeout=DEFAULT_TIMEOUT_SECONDS, @@ -1233,7 +1237,7 @@ def test_log_since(self): fake_request.assert_called_with( 'GET', - url_prefix + 'containers/3cc2351ab11b/logs', + url_prefix + 'containers/' + fake_api.FAKE_CONTAINER_ID + '/logs', params={'timestamps': 0, 'follow': 0, 'stderr': 1, 'stdout': 1, 'tail': 'all', 'since': ts}, timeout=DEFAULT_TIMEOUT_SECONDS, @@ -1250,7 +1254,7 @@ def test_log_since_with_datetime(self): fake_request.assert_called_with( 'GET', - url_prefix + 'containers/3cc2351ab11b/logs', + url_prefix + 'containers/' + fake_api.FAKE_CONTAINER_ID + '/logs', params={'timestamps': 0, 'follow': 0, 'stderr': 1, 'stdout': 1, 'tail': 'all', 'since': ts}, timeout=DEFAULT_TIMEOUT_SECONDS, @@ -1276,7 +1280,7 @@ def test_log_tty(self): assert m.called fake_request.assert_called_with( 'GET', - url_prefix + 'containers/3cc2351ab11b/logs', + url_prefix + 'containers/' + fake_api.FAKE_CONTAINER_ID + '/logs', params={'timestamps': 0, 'follow': 1, 'stderr': 1, 'stdout': 1, 'tail': 'all'}, timeout=DEFAULT_TIMEOUT_SECONDS, @@ -1288,7 +1292,8 @@ def test_diff(self): fake_request.assert_called_with( 'GET', - url_prefix + 'containers/3cc2351ab11b/changes', + (url_prefix + 'containers/' + + fake_api.FAKE_CONTAINER_ID + '/changes'), timeout=DEFAULT_TIMEOUT_SECONDS ) @@ -1297,7 +1302,8 @@ def test_diff_with_dict_instead_of_id(self): fake_request.assert_called_with( 'GET', - url_prefix + 'containers/3cc2351ab11b/changes', + (url_prefix + 'containers/' + + fake_api.FAKE_CONTAINER_ID + '/changes'), timeout=DEFAULT_TIMEOUT_SECONDS ) @@ -1306,7 +1312,7 @@ def test_port(self): fake_request.assert_called_with( 'GET', - url_prefix + 'containers/3cc2351ab11b/json', + url_prefix + 'containers/' + fake_api.FAKE_CONTAINER_ID + '/json', timeout=DEFAULT_TIMEOUT_SECONDS ) @@ -1317,7 +1323,7 @@ def test_stop_container(self): fake_request.assert_called_with( 'POST', - url_prefix + 'containers/3cc2351ab11b/stop', + url_prefix + 'containers/' + fake_api.FAKE_CONTAINER_ID + '/stop', params={'t': timeout}, timeout=(DEFAULT_TIMEOUT_SECONDS + timeout) ) @@ -1330,7 +1336,7 @@ def test_stop_container_with_dict_instead_of_id(self): fake_request.assert_called_with( 'POST', - url_prefix + 'containers/3cc2351ab11b/stop', + url_prefix + 'containers/' + fake_api.FAKE_CONTAINER_ID + '/stop', params={'t': timeout}, timeout=(DEFAULT_TIMEOUT_SECONDS + timeout) ) @@ -1340,7 +1346,8 @@ def test_pause_container(self): fake_request.assert_called_with( 'POST', - url_prefix + 'containers/3cc2351ab11b/pause', + (url_prefix + 'containers/' + + fake_api.FAKE_CONTAINER_ID + '/pause'), timeout=(DEFAULT_TIMEOUT_SECONDS) ) @@ -1349,7 +1356,8 @@ def test_unpause_container(self): fake_request.assert_called_with( 'POST', - url_prefix + 'containers/3cc2351ab11b/unpause', + (url_prefix + 'containers/' + + fake_api.FAKE_CONTAINER_ID + '/unpause'), timeout=(DEFAULT_TIMEOUT_SECONDS) ) @@ -1358,7 +1366,7 @@ def test_kill_container(self): fake_request.assert_called_with( 'POST', - url_prefix + 'containers/3cc2351ab11b/kill', + url_prefix + 'containers/' + fake_api.FAKE_CONTAINER_ID + '/kill', params={}, timeout=DEFAULT_TIMEOUT_SECONDS ) @@ -1368,7 +1376,7 @@ def test_kill_container_with_dict_instead_of_id(self): fake_request.assert_called_with( 'POST', - url_prefix + 'containers/3cc2351ab11b/kill', + url_prefix + 'containers/' + fake_api.FAKE_CONTAINER_ID + '/kill', params={}, timeout=DEFAULT_TIMEOUT_SECONDS ) @@ -1378,7 +1386,7 @@ def test_kill_container_with_signal(self): fake_request.assert_called_with( 'POST', - url_prefix + 'containers/3cc2351ab11b/kill', + url_prefix + 'containers/' + fake_api.FAKE_CONTAINER_ID + '/kill', params={'signal': signal.SIGTERM}, timeout=DEFAULT_TIMEOUT_SECONDS ) @@ -1388,7 +1396,8 @@ def test_restart_container(self): fake_request.assert_called_with( 'POST', - url_prefix + 'containers/3cc2351ab11b/restart', + (url_prefix + 'containers/' + + fake_api.FAKE_CONTAINER_ID + '/restart'), params={'t': 2}, timeout=(DEFAULT_TIMEOUT_SECONDS + 2) ) @@ -1398,7 +1407,8 @@ def test_restart_container_with_dict_instead_of_id(self): fake_request.assert_called_with( 'POST', - url_prefix + 'containers/3cc2351ab11b/restart', + (url_prefix + 'containers/' + + fake_api.FAKE_CONTAINER_ID + '/restart'), params={'t': 2}, timeout=(DEFAULT_TIMEOUT_SECONDS + 2) ) @@ -1408,7 +1418,7 @@ def test_remove_container(self): fake_request.assert_called_with( 'DELETE', - url_prefix + 'containers/3cc2351ab11b', + url_prefix + 'containers/' + fake_api.FAKE_CONTAINER_ID, params={'v': False, 'link': False, 'force': False}, timeout=DEFAULT_TIMEOUT_SECONDS ) @@ -1418,7 +1428,7 @@ def test_remove_container_with_dict_instead_of_id(self): fake_request.assert_called_with( 'DELETE', - url_prefix + 'containers/3cc2351ab11b', + url_prefix + 'containers/' + fake_api.FAKE_CONTAINER_ID, params={'v': False, 'link': False, 'force': False}, timeout=DEFAULT_TIMEOUT_SECONDS ) @@ -1428,7 +1438,8 @@ def test_export(self): fake_request.assert_called_with( 'GET', - url_prefix + 'containers/3cc2351ab11b/export', + (url_prefix + 'containers/' + + fake_api.FAKE_CONTAINER_ID + '/export'), stream=True, timeout=DEFAULT_TIMEOUT_SECONDS ) @@ -1438,7 +1449,8 @@ def test_export_with_dict_instead_of_id(self): fake_request.assert_called_with( 'GET', - url_prefix + 'containers/3cc2351ab11b/export', + (url_prefix + 'containers/' + + fake_api.FAKE_CONTAINER_ID + '/export'), stream=True, timeout=DEFAULT_TIMEOUT_SECONDS ) @@ -1448,7 +1460,7 @@ def test_inspect_container(self): fake_request.assert_called_with( 'GET', - url_prefix + 'containers/3cc2351ab11b/json', + url_prefix + 'containers/' + fake_api.FAKE_CONTAINER_ID + '/json', timeout=DEFAULT_TIMEOUT_SECONDS ) @@ -1464,7 +1476,7 @@ def test_container_stats(self): fake_request.assert_called_with( 'GET', - url_prefix + 'containers/3cc2351ab11b/stats', + url_prefix + 'containers/' + fake_api.FAKE_CONTAINER_ID + '/stats', timeout=60, stream=True ) @@ -1474,7 +1486,7 @@ def test_container_top(self): fake_request.assert_called_with( 'GET', - url_prefix + 'containers/3cc2351ab11b/top', + url_prefix + 'containers/' + fake_api.FAKE_CONTAINER_ID + '/top', params={}, timeout=DEFAULT_TIMEOUT_SECONDS ) @@ -1484,7 +1496,7 @@ def test_container_top_with_psargs(self): fake_request.assert_called_with( 'GET', - url_prefix + 'containers/3cc2351ab11b/top', + url_prefix + 'containers/' + fake_api.FAKE_CONTAINER_ID + '/top', params={'ps_args': 'waux'}, timeout=DEFAULT_TIMEOUT_SECONDS ) @@ -1496,7 +1508,8 @@ def test_container_update(self): blkio_weight=345 ) args = fake_request.call_args - assert args[0][1] == url_prefix + 'containers/3cc2351ab11b/update' + assert args[0][1] == (url_prefix + 'containers/' + + fake_api.FAKE_CONTAINER_ID + '/update') assert json.loads(args[1]['data']) == { 'Memory': 2 * 1024, 'CpuShares': 124, 'BlkioWeight': 345 } diff --git a/tests/unit/api_image_test.py b/tests/unit/api_image_test.py index 8fb3e9d9f5..e285932941 100644 --- a/tests/unit/api_image_test.py +++ b/tests/unit/api_image_test.py @@ -100,7 +100,7 @@ def test_commit(self): 'repo': None, 'comment': None, 'tag': None, - 'container': '3cc2351ab11b', + 'container': fake_api.FAKE_CONTAINER_ID, 'author': None, 'changes': None }, @@ -112,7 +112,7 @@ def test_remove_image(self): fake_request.assert_called_with( 'DELETE', - url_prefix + 'images/e9aa60c60128', + url_prefix + 'images/' + fake_api.FAKE_IMAGE_ID, params={'force': False, 'noprune': False}, timeout=DEFAULT_TIMEOUT_SECONDS ) @@ -287,7 +287,7 @@ def test_tag_image(self): fake_request.assert_called_with( 'POST', - url_prefix + 'images/e9aa60c60128/tag', + url_prefix + 'images/' + fake_api.FAKE_IMAGE_ID + '/tag', params={ 'tag': None, 'repo': 'repo', @@ -305,7 +305,7 @@ def test_tag_image_tag(self): fake_request.assert_called_with( 'POST', - url_prefix + 'images/e9aa60c60128/tag', + url_prefix + 'images/' + fake_api.FAKE_IMAGE_ID + '/tag', params={ 'tag': 'tag', 'repo': 'repo', @@ -320,7 +320,7 @@ def test_tag_image_force(self): fake_request.assert_called_with( 'POST', - url_prefix + 'images/e9aa60c60128/tag', + url_prefix + 'images/' + fake_api.FAKE_IMAGE_ID + '/tag', params={ 'tag': None, 'repo': 'repo', @@ -334,7 +334,7 @@ def test_get_image(self): fake_request.assert_called_with( 'GET', - url_prefix + 'images/e9aa60c60128/get', + url_prefix + 'images/' + fake_api.FAKE_IMAGE_ID + '/get', stream=True, timeout=DEFAULT_TIMEOUT_SECONDS ) diff --git a/tests/unit/api_test.py b/tests/unit/api_test.py index 45d2e4c034..a2348f08ba 100644 --- a/tests/unit/api_test.py +++ b/tests/unit/api_test.py @@ -318,7 +318,7 @@ def test_remove_link(self): fake_request.assert_called_with( 'DELETE', - url_prefix + 'containers/3cc2351ab11b', + url_prefix + 'containers/' + fake_api.FAKE_CONTAINER_ID, params={'v': False, 'link': True, 'force': False}, timeout=DEFAULT_TIMEOUT_SECONDS ) diff --git a/tests/unit/fake_api.py b/tests/unit/fake_api.py index 4c93329531..6acfb64b8c 100644 --- a/tests/unit/fake_api.py +++ b/tests/unit/fake_api.py @@ -4,10 +4,10 @@ CURRENT_VERSION = f'v{constants.DEFAULT_DOCKER_API_VERSION}' -FAKE_CONTAINER_ID = '3cc2351ab11b' -FAKE_IMAGE_ID = 'e9aa60c60128' -FAKE_EXEC_ID = 'd5d177f121dc' -FAKE_NETWORK_ID = '33fb6a3462b8' +FAKE_CONTAINER_ID = '81cf499cc928ce3fedc250a080d2b9b978df20e4517304c45211e8a68b33e254' # noqa: E501 +FAKE_IMAGE_ID = 'sha256:fe7a8fc91d3f17835cbb3b86a1c60287500ab01a53bc79c4497d09f07a3f0688' # noqa: E501 +FAKE_EXEC_ID = 'b098ec855f10434b5c7c973c78484208223a83f663ddaefb0f02a242840cb1c7' # noqa: E501 +FAKE_NETWORK_ID = '1999cfb42e414483841a125ade3c276c3cb80cb3269b14e339354ac63a31b02c' # noqa: E501 FAKE_IMAGE_NAME = 'test_image' FAKE_TARBALL_PATH = '/path/to/tarball' FAKE_REPO_NAME = 'repo' @@ -546,56 +546,56 @@ def post_fake_secret(): post_fake_import_image, f'{prefix}/{CURRENT_VERSION}/containers/json': get_fake_containers, - f'{prefix}/{CURRENT_VERSION}/containers/3cc2351ab11b/start': + f'{prefix}/{CURRENT_VERSION}/containers/{FAKE_CONTAINER_ID}/start': post_fake_start_container, - f'{prefix}/{CURRENT_VERSION}/containers/3cc2351ab11b/resize': + f'{prefix}/{CURRENT_VERSION}/containers/{FAKE_CONTAINER_ID}/resize': post_fake_resize_container, - f'{prefix}/{CURRENT_VERSION}/containers/3cc2351ab11b/json': + f'{prefix}/{CURRENT_VERSION}/containers/{FAKE_CONTAINER_ID}/json': get_fake_inspect_container, - f'{prefix}/{CURRENT_VERSION}/containers/3cc2351ab11b/rename': + f'{prefix}/{CURRENT_VERSION}/containers/{FAKE_CONTAINER_ID}/rename': post_fake_rename_container, - f'{prefix}/{CURRENT_VERSION}/images/e9aa60c60128/tag': + f'{prefix}/{CURRENT_VERSION}/images/{FAKE_IMAGE_ID}/tag': post_fake_tag_image, - f'{prefix}/{CURRENT_VERSION}/containers/3cc2351ab11b/wait': + f'{prefix}/{CURRENT_VERSION}/containers/{FAKE_CONTAINER_ID}/wait': get_fake_wait, - f'{prefix}/{CURRENT_VERSION}/containers/3cc2351ab11b/logs': + f'{prefix}/{CURRENT_VERSION}/containers/{FAKE_CONTAINER_ID}/logs': get_fake_logs, - f'{prefix}/{CURRENT_VERSION}/containers/3cc2351ab11b/changes': + f'{prefix}/{CURRENT_VERSION}/containers/{FAKE_CONTAINER_ID}/changes': get_fake_diff, - f'{prefix}/{CURRENT_VERSION}/containers/3cc2351ab11b/export': + f'{prefix}/{CURRENT_VERSION}/containers/{FAKE_CONTAINER_ID}/export': get_fake_export, - f'{prefix}/{CURRENT_VERSION}/containers/3cc2351ab11b/update': + f'{prefix}/{CURRENT_VERSION}/containers/{FAKE_CONTAINER_ID}/update': post_fake_update_container, - f'{prefix}/{CURRENT_VERSION}/containers/3cc2351ab11b/exec': + f'{prefix}/{CURRENT_VERSION}/containers/{FAKE_CONTAINER_ID}/exec': post_fake_exec_create, - f'{prefix}/{CURRENT_VERSION}/exec/d5d177f121dc/start': + f'{prefix}/{CURRENT_VERSION}/exec/{FAKE_EXEC_ID}/start': post_fake_exec_start, - f'{prefix}/{CURRENT_VERSION}/exec/d5d177f121dc/json': + f'{prefix}/{CURRENT_VERSION}/exec/{FAKE_EXEC_ID}/json': get_fake_exec_inspect, - f'{prefix}/{CURRENT_VERSION}/exec/d5d177f121dc/resize': + f'{prefix}/{CURRENT_VERSION}/exec/{FAKE_EXEC_ID}/resize': post_fake_exec_resize, - f'{prefix}/{CURRENT_VERSION}/containers/3cc2351ab11b/stats': + f'{prefix}/{CURRENT_VERSION}/containers/{FAKE_CONTAINER_ID}/stats': get_fake_stats, - f'{prefix}/{CURRENT_VERSION}/containers/3cc2351ab11b/top': + f'{prefix}/{CURRENT_VERSION}/containers/{FAKE_CONTAINER_ID}/top': get_fake_top, - f'{prefix}/{CURRENT_VERSION}/containers/3cc2351ab11b/stop': + f'{prefix}/{CURRENT_VERSION}/containers/{FAKE_CONTAINER_ID}/stop': post_fake_stop_container, - f'{prefix}/{CURRENT_VERSION}/containers/3cc2351ab11b/kill': + f'{prefix}/{CURRENT_VERSION}/containers/{FAKE_CONTAINER_ID}/kill': post_fake_kill_container, - f'{prefix}/{CURRENT_VERSION}/containers/3cc2351ab11b/pause': + f'{prefix}/{CURRENT_VERSION}/containers/{FAKE_CONTAINER_ID}/pause': post_fake_pause_container, - f'{prefix}/{CURRENT_VERSION}/containers/3cc2351ab11b/unpause': + f'{prefix}/{CURRENT_VERSION}/containers/{FAKE_CONTAINER_ID}/unpause': post_fake_unpause_container, - f'{prefix}/{CURRENT_VERSION}/containers/3cc2351ab11b/restart': + f'{prefix}/{CURRENT_VERSION}/containers/{FAKE_CONTAINER_ID}/restart': post_fake_restart_container, - f'{prefix}/{CURRENT_VERSION}/containers/3cc2351ab11b': + f'{prefix}/{CURRENT_VERSION}/containers/{FAKE_CONTAINER_ID}': delete_fake_remove_container, f'{prefix}/{CURRENT_VERSION}/images/create': post_fake_image_create, - f'{prefix}/{CURRENT_VERSION}/images/e9aa60c60128': + f'{prefix}/{CURRENT_VERSION}/images/{FAKE_IMAGE_ID}': delete_fake_remove_image, - f'{prefix}/{CURRENT_VERSION}/images/e9aa60c60128/get': + f'{prefix}/{CURRENT_VERSION}/images/{FAKE_IMAGE_ID}/get': get_fake_get_image, f'{prefix}/{CURRENT_VERSION}/images/load': post_fake_load_image, diff --git a/tests/unit/models_containers_test.py b/tests/unit/models_containers_test.py index c7aa46b2a0..785a849039 100644 --- a/tests/unit/models_containers_test.py +++ b/tests/unit/models_containers_test.py @@ -377,6 +377,11 @@ def side_effect(*args, **kwargs): class ContainerTest(unittest.TestCase): + def test_short_id(self): + container = Container(attrs={'Id': '8497fe9244dd45cac543eb3c37d8605077' + '6800eebef1f3ec2ee111e8ccf12db6'}) + assert container.short_id == '8497fe9244dd' + def test_name(self): client = make_fake_client() container = client.containers.get(FAKE_CONTAINER_ID) diff --git a/tests/unit/models_images_test.py b/tests/unit/models_images_test.py index f3ca0be4e7..436fd61f0e 100644 --- a/tests/unit/models_images_test.py +++ b/tests/unit/models_images_test.py @@ -122,11 +122,11 @@ class ImageTest(unittest.TestCase): def test_short_id(self): image = Image(attrs={'Id': 'sha256:b6846070672ce4e8f1f91564ea6782bd675' 'f69d65a6f73ef6262057ad0a15dcd'}) - assert image.short_id == 'sha256:b684607067' + assert image.short_id == 'sha256:b6846070672c' image = Image(attrs={'Id': 'b6846070672ce4e8f1f91564ea6782bd675' 'f69d65a6f73ef6262057ad0a15dcd'}) - assert image.short_id == 'b684607067' + assert image.short_id == 'b6846070672c' def test_tags(self): image = Image(attrs={ From 05e143429e892fb838bbff058391456ba3d0a19c Mon Sep 17 00:00:00 2001 From: Milas Bowman Date: Fri, 29 Jul 2022 11:08:00 -0400 Subject: [PATCH 1163/1301] api: preserve cause when re-raising error (#3023) Use `from e` to ensure that the error context is propagated correctly. Fixes #2702. Signed-off-by: Milas Bowman --- docker/errors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/errors.py b/docker/errors.py index ba952562c6..7725295f5c 100644 --- a/docker/errors.py +++ b/docker/errors.py @@ -28,7 +28,7 @@ def create_api_error_from_http_exception(e): cls = ImageNotFound else: cls = NotFound - raise cls(e, response=response, explanation=explanation) + raise cls(e, response=response, explanation=explanation) from e class APIError(requests.exceptions.HTTPError, DockerException): From 26064dd6b584ee14878157b4c8b001eefed70caf Mon Sep 17 00:00:00 2001 From: Milas Bowman Date: Fri, 29 Jul 2022 11:09:47 -0400 Subject: [PATCH 1164/1301] deps: upgrade websocket-client to latest (#3022) * Upgrade websocket-client to latest * Add basic integration test for streaming logs via websocket Signed-off-by: Milas Bowman --- requirements.txt | 2 +- tests/integration/api_container_test.py | 21 ++++++++++++++++++++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 52b5461e2e..36660b660c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,4 @@ paramiko==2.11.0 pywin32==304; sys_platform == 'win32' requests==2.28.1 urllib3==1.26.11 -websocket-client==0.56.0 +websocket-client==1.3.3 diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index 0d6d9f96c5..8f69e41ff0 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -1197,7 +1197,7 @@ def test_run_container_streaming(self): sock = self.client.attach_socket(container, ws=False) assert sock.fileno() > -1 - def test_run_container_reading_socket(self): + def test_run_container_reading_socket_http(self): line = 'hi there and stuff and things, words!' # `echo` appends CRLF, `printf` doesn't command = f"printf '{line}'" @@ -1217,6 +1217,25 @@ def test_run_container_reading_socket(self): data = read_exactly(pty_stdout, next_size) assert data.decode('utf-8') == line + @pytest.mark.xfail(condition=bool(os.environ.get('DOCKER_CERT_PATH', '')), + reason='DOCKER_CERT_PATH not respected for websockets') + def test_run_container_reading_socket_ws(self): + line = 'hi there and stuff and things, words!' + # `echo` appends CRLF, `printf` doesn't + command = f"printf '{line}'" + container = self.client.create_container(TEST_IMG, command, + detach=True, tty=False) + self.tmp_containers.append(container) + + opts = {"stdout": 1, "stream": 1, "logs": 1} + pty_stdout = self.client.attach_socket(container, opts, ws=True) + self.addCleanup(pty_stdout.close) + + self.client.start(container) + + data = pty_stdout.recv() + assert data.decode('utf-8') == line + @pytest.mark.timeout(10) def test_attach_no_stream(self): container = self.client.create_container( From 1a4cacdfb63f0fbf2299962732c75484c24ad8b0 Mon Sep 17 00:00:00 2001 From: Felix Fontein Date: Fri, 29 Jul 2022 19:57:30 +0200 Subject: [PATCH 1165/1301] api: add platform to container create (#2927) Add platform parameter for container creation/run Signed-off-by: Felix Fontein Signed-off-by: Milas Bowman Co-authored-by: Milas Bowman --- docker/api/container.py | 13 ++++++++++--- docker/errors.py | 16 +++++++++++---- docker/models/containers.py | 3 ++- tests/unit/api_container_test.py | 16 +++++++++++++++ tests/unit/models_containers_test.py | 29 ++++++++++++++++++++++++++++ 5 files changed, 69 insertions(+), 8 deletions(-) diff --git a/docker/api/container.py b/docker/api/container.py index 17c09726b7..f600be1811 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -223,7 +223,7 @@ def create_container(self, image, command=None, hostname=None, user=None, mac_address=None, labels=None, stop_signal=None, networking_config=None, healthcheck=None, stop_timeout=None, runtime=None, - use_config_proxy=True): + use_config_proxy=True, platform=None): """ Creates a container. Parameters are similar to those for the ``docker run`` command except it doesn't support the attach options (``-a``). @@ -398,6 +398,7 @@ def create_container(self, image, command=None, hostname=None, user=None, configuration file (``~/.docker/config.json`` by default) contains a proxy configuration, the corresponding environment variables will be set in the container being created. + platform (str): Platform in the format ``os[/arch[/variant]]``. Returns: A dictionary with an image 'Id' key and a 'Warnings' key. @@ -427,16 +428,22 @@ def create_container(self, image, command=None, hostname=None, user=None, stop_signal, networking_config, healthcheck, stop_timeout, runtime ) - return self.create_container_from_config(config, name) + return self.create_container_from_config(config, name, platform) def create_container_config(self, *args, **kwargs): return ContainerConfig(self._version, *args, **kwargs) - def create_container_from_config(self, config, name=None): + def create_container_from_config(self, config, name=None, platform=None): u = self._url("/containers/create") params = { 'name': name } + if platform: + if utils.version_lt(self._version, '1.41'): + raise errors.InvalidVersion( + 'platform is not supported for API version < 1.41' + ) + params['platform'] = platform res = self._post_json(u, data=config, params=params) return self._result(res, True) diff --git a/docker/errors.py b/docker/errors.py index 7725295f5c..8cf8670baf 100644 --- a/docker/errors.py +++ b/docker/errors.py @@ -1,5 +1,14 @@ import requests +_image_not_found_explanation_fragments = frozenset( + fragment.lower() for fragment in [ + 'no such image', + 'not found: does not exist or no pull access', + 'repository does not exist', + 'was found but does not match the specified platform', + ] +) + class DockerException(Exception): """ @@ -21,10 +30,9 @@ def create_api_error_from_http_exception(e): explanation = (response.content or '').strip() cls = APIError if response.status_code == 404: - if explanation and ('No such image' in str(explanation) or - 'not found: does not exist or no pull access' - in str(explanation) or - 'repository does not exist' in str(explanation)): + explanation_msg = (explanation or '').lower() + if any(fragment in explanation_msg + for fragment in _image_not_found_explanation_fragments): cls = ImageNotFound else: cls = NotFound diff --git a/docker/models/containers.py b/docker/models/containers.py index e34659cbee..7769ed0913 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -801,7 +801,7 @@ def run(self, image, command=None, stdout=True, stderr=False, image = image.id stream = kwargs.pop('stream', False) detach = kwargs.pop('detach', False) - platform = kwargs.pop('platform', None) + platform = kwargs.get('platform', None) if detach and remove: if version_gte(self.client.api._version, '1.25'): @@ -985,6 +985,7 @@ def prune(self, filters=None): 'mac_address', 'name', 'network_disabled', + 'platform', 'stdin_open', 'stop_signal', 'tty', diff --git a/tests/unit/api_container_test.py b/tests/unit/api_container_test.py index 7030841682..3a2fbde88e 100644 --- a/tests/unit/api_container_test.py +++ b/tests/unit/api_container_test.py @@ -348,6 +348,22 @@ def test_create_named_container(self): assert args[1]['headers'] == {'Content-Type': 'application/json'} assert args[1]['params'] == {'name': 'marisa-kirisame'} + def test_create_container_with_platform(self): + self.client.create_container('busybox', 'true', + platform='linux') + + args = fake_request.call_args + assert args[0][1] == url_prefix + 'containers/create' + assert json.loads(args[1]['data']) == json.loads(''' + {"Tty": false, "Image": "busybox", "Cmd": ["true"], + "AttachStdin": false, + "AttachStderr": true, "AttachStdout": true, + "StdinOnce": false, + "OpenStdin": false, "NetworkDisabled": false} + ''') + assert args[1]['headers'] == {'Content-Type': 'application/json'} + assert args[1]['params'] == {'name': None, 'platform': 'linux'} + def test_create_container_with_mem_limit_as_int(self): self.client.create_container( 'busybox', 'true', host_config=self.client.create_host_config( diff --git a/tests/unit/models_containers_test.py b/tests/unit/models_containers_test.py index 785a849039..e4ee074d87 100644 --- a/tests/unit/models_containers_test.py +++ b/tests/unit/models_containers_test.py @@ -77,6 +77,7 @@ def test_create_container_args(self): oom_score_adj=5, pid_mode='host', pids_limit=500, + platform='linux', ports={ 1111: 4567, 2222: None @@ -186,6 +187,7 @@ def test_create_container_args(self): name='somename', network_disabled=False, networking_config={'foo': None}, + platform='linux', ports=[('1111', 'tcp'), ('2222', 'tcp')], stdin_open=True, stop_signal=9, @@ -314,6 +316,33 @@ def test_run_remove(self): 'NetworkMode': 'default'} ) + def test_run_platform(self): + client = make_fake_client() + + # raise exception on first call, then return normal value + client.api.create_container.side_effect = [ + docker.errors.ImageNotFound(""), + client.api.create_container.return_value + ] + + client.containers.run(image='alpine', platform='linux/arm64') + + client.api.pull.assert_called_with( + 'alpine', + tag='latest', + all_tags=False, + stream=True, + platform='linux/arm64', + ) + + client.api.create_container.assert_called_with( + detach=False, + platform='linux/arm64', + image='alpine', + command=None, + host_config={'NetworkMode': 'default'}, + ) + def test_create(self): client = make_fake_client() container = client.containers.create( From d69de54d7ce967ecd48db50ceecf1a700e84d7eb Mon Sep 17 00:00:00 2001 From: David Date: Fri, 29 Jul 2022 20:04:47 +0200 Subject: [PATCH 1166/1301] api: add cgroupns option to container create (#2930) Signed-off-by: David Otto --- docker/models/containers.py | 6 ++++++ docker/types/containers.py | 6 +++++- tests/unit/models_containers_test.py | 2 ++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/docker/models/containers.py b/docker/models/containers.py index 7769ed0913..313d47d6ff 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -553,6 +553,11 @@ def run(self, image, command=None, stdout=True, stderr=False, ``["SYS_ADMIN", "MKNOD"]``. cap_drop (list of str): Drop kernel capabilities. cgroup_parent (str): Override the default parent cgroup. + cgroupns (str): Override the default cgroup namespace mode for the + container. One of: + - ``private`` the container runs in its own private cgroup + namespace. + - ``host`` use the host system's cgroup namespace. cpu_count (int): Number of usable CPUs (Windows only). cpu_percent (int): Usable percentage of the available CPUs (Windows only). @@ -1002,6 +1007,7 @@ def prune(self, filters=None): 'cap_add', 'cap_drop', 'cgroup_parent', + 'cgroupns', 'cpu_count', 'cpu_percent', 'cpu_period', diff --git a/docker/types/containers.py b/docker/types/containers.py index f1b60b2d2f..84df0f7e61 100644 --- a/docker/types/containers.py +++ b/docker/types/containers.py @@ -272,7 +272,8 @@ def __init__(self, version, binds=None, port_bindings=None, volume_driver=None, cpu_count=None, cpu_percent=None, nano_cpus=None, cpuset_mems=None, runtime=None, mounts=None, cpu_rt_period=None, cpu_rt_runtime=None, - device_cgroup_rules=None, device_requests=None): + device_cgroup_rules=None, device_requests=None, + cgroupns=None): if mem_limit is not None: self['Memory'] = parse_bytes(mem_limit) @@ -646,6 +647,9 @@ def __init__(self, version, binds=None, port_bindings=None, req = DeviceRequest(**req) self['DeviceRequests'].append(req) + if cgroupns: + self['CgroupnsMode'] = cgroupns + def host_config_type_error(param, param_value, expected): error_msg = 'Invalid type for {0} param: expected {1} but found {2}' diff --git a/tests/unit/models_containers_test.py b/tests/unit/models_containers_test.py index e4ee074d87..101708ebb7 100644 --- a/tests/unit/models_containers_test.py +++ b/tests/unit/models_containers_test.py @@ -39,6 +39,7 @@ def test_create_container_args(self): cap_add=['foo'], cap_drop=['bar'], cgroup_parent='foobar', + cgroupns='host', cpu_period=1, cpu_quota=2, cpu_shares=5, @@ -135,6 +136,7 @@ def test_create_container_args(self): 'BlkioWeight': 2, 'CapAdd': ['foo'], 'CapDrop': ['bar'], + 'CgroupnsMode': 'host', 'CgroupParent': 'foobar', 'CpuPeriod': 1, 'CpuQuota': 2, From b2a18d7209f827d83cc33acb80aa31bf404ffd4b Mon Sep 17 00:00:00 2001 From: Peter Dave Hello Date: Sat, 30 Jul 2022 02:09:06 +0800 Subject: [PATCH 1167/1301] build: disable pip cache in Dockerfile (#2828) Signed-off-by: Peter Dave Hello --- Dockerfile | 6 +++--- Dockerfile-docs | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8a0d32e430..c158a9d676 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,10 +6,10 @@ RUN mkdir /src WORKDIR /src COPY requirements.txt /src/requirements.txt -RUN pip install -r requirements.txt +RUN pip install --no-cache-dir -r requirements.txt COPY test-requirements.txt /src/test-requirements.txt -RUN pip install -r test-requirements.txt +RUN pip install --no-cache-dir -r test-requirements.txt COPY . /src -RUN pip install . +RUN pip install --no-cache-dir . diff --git a/Dockerfile-docs b/Dockerfile-docs index 98901dfe6b..e993822b85 100644 --- a/Dockerfile-docs +++ b/Dockerfile-docs @@ -10,6 +10,6 @@ RUN addgroup --gid $gid sphinx \ WORKDIR /src COPY requirements.txt docs-requirements.txt ./ -RUN pip install -r requirements.txt -r docs-requirements.txt +RUN pip install --no-cache-dir -r requirements.txt -r docs-requirements.txt USER sphinx From 0031ac2186406c9b48c6fc5253affd4b62fef0f5 Mon Sep 17 00:00:00 2001 From: Till! Date: Fri, 29 Jul 2022 20:51:43 +0200 Subject: [PATCH 1168/1301] api: add force to plugin disable (#2843) Signed-off-by: till --- docker/api/plugin.py | 5 +++-- docker/models/plugins.py | 7 +++++-- tests/integration/api_plugin_test.py | 4 ++-- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/docker/api/plugin.py b/docker/api/plugin.py index 57110f1131..10210c1a23 100644 --- a/docker/api/plugin.py +++ b/docker/api/plugin.py @@ -51,19 +51,20 @@ def create_plugin(self, name, plugin_data_dir, gzip=False): return True @utils.minimum_version('1.25') - def disable_plugin(self, name): + def disable_plugin(self, name, force=False): """ Disable an installed plugin. Args: name (string): The name of the plugin. The ``:latest`` tag is optional, and is the default if omitted. + force (bool): To enable the force query parameter. Returns: ``True`` if successful """ url = self._url('/plugins/{0}/disable', name) - res = self._post(url) + res = self._post(url, params={'force': force}) self._raise_for_status(res) return True diff --git a/docker/models/plugins.py b/docker/models/plugins.py index 69b94f3530..16f5245e9e 100644 --- a/docker/models/plugins.py +++ b/docker/models/plugins.py @@ -44,16 +44,19 @@ def configure(self, options): self.client.api.configure_plugin(self.name, options) self.reload() - def disable(self): + def disable(self, force=False): """ Disable the plugin. + Args: + force (bool): Force disable. Default: False + Raises: :py:class:`docker.errors.APIError` If the server returns an error. """ - self.client.api.disable_plugin(self.name) + self.client.api.disable_plugin(self.name, force) self.reload() def enable(self, timeout=0): diff --git a/tests/integration/api_plugin_test.py b/tests/integration/api_plugin_test.py index 38f9d12dad..3ecb028346 100644 --- a/tests/integration/api_plugin_test.py +++ b/tests/integration/api_plugin_test.py @@ -22,13 +22,13 @@ def teardown_class(cls): def teardown_method(self, method): client = self.get_client_instance() try: - client.disable_plugin(SSHFS) + client.disable_plugin(SSHFS, True) except docker.errors.APIError: pass for p in self.tmp_plugins: try: - client.remove_plugin(p, force=True) + client.remove_plugin(p) except docker.errors.APIError: pass From 26753c81defff28a1a38a34788e9653c8eb87c3d Mon Sep 17 00:00:00 2001 From: ercildoune <49232938+ercildoune@users.noreply.github.com> Date: Sat, 30 Jul 2022 02:54:55 +0800 Subject: [PATCH 1169/1301] api: add rollback_config to service create (#2917) `rollback_config` was not in the list of `CREATE_SERVICE_KWARGS` which prevented it from being an argument when creating services. It has now been added and the problem fixed, allowing services to have a rollback_config during creation and updating. Fixes #2832. Signed-off-by: Fraser Patten Signed-off-by: Milas Bowman Co-authored-by: Milas Bowman --- docker/models/services.py | 1 + tests/integration/models_services_test.py | 7 ++++++- tests/unit/models_services_test.py | 2 ++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/docker/models/services.py b/docker/models/services.py index 200dd333c7..9255068119 100644 --- a/docker/models/services.py +++ b/docker/models/services.py @@ -320,6 +320,7 @@ def list(self, **kwargs): 'labels', 'mode', 'update_config', + 'rollback_config', 'endpoint_spec', ] diff --git a/tests/integration/models_services_test.py b/tests/integration/models_services_test.py index 982842b326..f1439a418e 100644 --- a/tests/integration/models_services_test.py +++ b/tests/integration/models_services_test.py @@ -30,13 +30,18 @@ def test_create(self): # ContainerSpec arguments image="alpine", command="sleep 300", - container_labels={'container': 'label'} + container_labels={'container': 'label'}, + rollback_config={'order': 'start-first'} ) assert service.name == name assert service.attrs['Spec']['Labels']['foo'] == 'bar' container_spec = service.attrs['Spec']['TaskTemplate']['ContainerSpec'] assert "alpine" in container_spec['Image'] assert container_spec['Labels'] == {'container': 'label'} + spec_rollback = service.attrs['Spec'].get('RollbackConfig', None) + assert spec_rollback is not None + assert ('Order' in spec_rollback and + spec_rollback['Order'] == 'start-first') def test_create_with_network(self): client = docker.from_env(version=TEST_API_VERSION) diff --git a/tests/unit/models_services_test.py b/tests/unit/models_services_test.py index b9192e422b..94a27f0e5c 100644 --- a/tests/unit/models_services_test.py +++ b/tests/unit/models_services_test.py @@ -11,6 +11,7 @@ def test_get_create_service_kwargs(self): 'labels': {'key': 'value'}, 'hostname': 'test_host', 'mode': 'global', + 'rollback_config': {'rollback': 'config'}, 'update_config': {'update': 'config'}, 'networks': ['somenet'], 'endpoint_spec': {'blah': 'blah'}, @@ -37,6 +38,7 @@ def test_get_create_service_kwargs(self): 'name': 'somename', 'labels': {'key': 'value'}, 'mode': 'global', + 'rollback_config': {'rollback': 'config'}, 'update_config': {'update': 'config'}, 'endpoint_spec': {'blah': 'blah'}, } From 868e996269b6934420f0cd2104621b6f45f668e5 Mon Sep 17 00:00:00 2001 From: Milas Bowman Date: Fri, 29 Jul 2022 15:28:16 -0400 Subject: [PATCH 1170/1301] model: add remove() to Image (#3026) Allow an Image to be deleted by calling the remove() method on it, just like a Volume. Signed-off-by: Ahmon Dancy Signed-off-by: Milas Bowman Co-authored-by: Ahmon Dancy --- docker/models/images.py | 18 ++++++++++++++++++ tests/unit/models_images_test.py | 10 ++++++++++ 2 files changed, 28 insertions(+) diff --git a/docker/models/images.py b/docker/models/images.py index e247d351e1..79ccbe4095 100644 --- a/docker/models/images.py +++ b/docker/models/images.py @@ -61,6 +61,24 @@ def history(self): """ return self.client.api.history(self.id) + def remove(self, force=False, noprune=False): + """ + Remove this image. + + Args: + force (bool): Force removal of the image + noprune (bool): Do not delete untagged parents + + Raises: + :py:class:`docker.errors.APIError` + If the server returns an error. + """ + return self.client.api.remove_image( + self.id, + force=force, + noprune=noprune, + ) + def save(self, chunk_size=DEFAULT_DATA_CHUNK_SIZE, named=False): """ Get a tarball of an image. Similar to the ``docker save`` command. diff --git a/tests/unit/models_images_test.py b/tests/unit/models_images_test.py index 436fd61f0e..3478c3fedb 100644 --- a/tests/unit/models_images_test.py +++ b/tests/unit/models_images_test.py @@ -150,6 +150,16 @@ def test_history(self): image.history() client.api.history.assert_called_with(FAKE_IMAGE_ID) + def test_remove(self): + client = make_fake_client() + image = client.images.get(FAKE_IMAGE_ID) + image.remove() + client.api.remove_image.assert_called_with( + FAKE_IMAGE_ID, + force=False, + noprune=False, + ) + def test_save(self): client = make_fake_client() image = client.images.get(FAKE_IMAGE_ID) From 3ee3a2486fe75ed858f8a3defe0fc79b2743d5df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Loiselet?= Date: Fri, 29 Jul 2022 21:33:23 +0200 Subject: [PATCH 1171/1301] build: trim trailing whitespace from dockerignore entries (#2733) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fix(dockerignore): trim trailing whitespace Signed-off-by: Clément Loiselet --- docker/utils/build.py | 3 +++ tests/integration/api_build_test.py | 9 +++++++++ 2 files changed, 12 insertions(+) diff --git a/docker/utils/build.py b/docker/utils/build.py index ac060434de..59564c4cda 100644 --- a/docker/utils/build.py +++ b/docker/utils/build.py @@ -224,6 +224,9 @@ def __init__(self, pattern_str): @classmethod def normalize(cls, p): + # Remove trailing spaces + p = p.strip() + # Leading and trailing slashes are not relevant. Yes, # "foo.py/" must exclude the "foo.py" regular file. "." # components are not relevant either, even if the whole diff --git a/tests/integration/api_build_test.py b/tests/integration/api_build_test.py index ef48e12ed3..606c3b7e11 100644 --- a/tests/integration/api_build_test.py +++ b/tests/integration/api_build_test.py @@ -100,7 +100,9 @@ def test_build_with_dockerignore(self): 'ignored', 'Dockerfile', '.dockerignore', + ' ignored-with-spaces ', # check that spaces are trimmed '!ignored/subdir/excepted-file', + '! ignored/subdir/excepted-with-spaces ' '', # empty line, '#*', # comment line ])) @@ -111,6 +113,9 @@ def test_build_with_dockerignore(self): with open(os.path.join(base_dir, '#file.txt'), 'w') as f: f.write('this file should not be ignored') + with open(os.path.join(base_dir, 'ignored-with-spaces'), 'w') as f: + f.write("this file should be ignored") + subdir = os.path.join(base_dir, 'ignored', 'subdir') os.makedirs(subdir) with open(os.path.join(subdir, 'file'), 'w') as f: @@ -119,6 +124,9 @@ def test_build_with_dockerignore(self): with open(os.path.join(subdir, 'excepted-file'), 'w') as f: f.write("this file should not be ignored") + with open(os.path.join(subdir, 'excepted-with-spaces'), 'w') as f: + f.write("this file should not be ignored") + tag = 'docker-py-test-build-with-dockerignore' stream = self.client.build( path=base_dir, @@ -136,6 +144,7 @@ def test_build_with_dockerignore(self): assert sorted(list(filter(None, logs.split('\n')))) == sorted([ '/test/#file.txt', + '/test/ignored/subdir/excepted-with-spaces', '/test/ignored/subdir/excepted-file', '/test/not-ignored' ]) From 55f47299c45b0c12531a68e233ea98617b1f7928 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ville=20Skytt=C3=A4?= Date: Fri, 29 Jul 2022 22:54:27 +0300 Subject: [PATCH 1172/1301] docs: fix TLS server verify example (#2574) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Leaving out the verify parameter means verification will not be done. Signed-off-by: Ville Skyttä --- docs/tls.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tls.rst b/docs/tls.rst index 2e2f1ea94c..b95b468c5b 100644 --- a/docs/tls.rst +++ b/docs/tls.rst @@ -15,7 +15,7 @@ For example, to check the server against a specific CA certificate: .. code-block:: python - tls_config = docker.tls.TLSConfig(ca_cert='/path/to/ca.pem') + tls_config = docker.tls.TLSConfig(ca_cert='/path/to/ca.pem', verify=True) client = docker.DockerClient(base_url='', tls=tls_config) This is the equivalent of ``docker --tlsverify --tlscacert /path/to/ca.pem ...``. From 73421027be04c97fc6f50da0647ba47388ed60e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ville=20Skytt=C3=A4?= Date: Fri, 29 Jul 2022 22:55:14 +0300 Subject: [PATCH 1173/1301] docs: clarify TLSConfig verify parameter (#2573) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Ville Skyttä --- docker/tls.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docker/tls.py b/docker/tls.py index 882a50eaf3..f4dffb2e25 100644 --- a/docker/tls.py +++ b/docker/tls.py @@ -12,8 +12,9 @@ class TLSConfig: Args: client_cert (tuple of str): Path to client cert, path to client key. ca_cert (str): Path to CA cert file. - verify (bool or str): This can be ``False`` or a path to a CA cert - file. + verify (bool or str): This can be a bool or a path to a CA cert + file to verify against. If ``True``, verify using ca_cert; + if ``False`` or not specified, do not verify. ssl_version (int): A valid `SSL version`_. assert_hostname (bool): Verify the hostname of the server. From 003a16503a6a760ece2cffa549ebf25e0474108c Mon Sep 17 00:00:00 2001 From: Milas Bowman Date: Fri, 29 Jul 2022 16:01:29 -0400 Subject: [PATCH 1174/1301] docs: fix list formatting Signed-off-by: Milas Bowman --- docker/models/containers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/models/containers.py b/docker/models/containers.py index 3d01031c6b..c37df55e29 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -606,6 +606,7 @@ def run(self, image, command=None, stdout=True, stderr=False, IDs that the container process will run as. healthcheck (dict): Specify a test to perform to check that the container is healthy. The dict takes the following keys: + - test (:py:class:`list` or str): Test to perform to determine container health. Possible values: From 52fb27690c073638134b45cd462ab0c091f393a5 Mon Sep 17 00:00:00 2001 From: Hristo Georgiev Date: Fri, 29 Jul 2022 21:04:23 +0100 Subject: [PATCH 1175/1301] docs: fix image save example (#2570) Signed-off-by: Hristo Georgiev From dff849f6bb7d6805f52dd2112b8666c86d3f3235 Mon Sep 17 00:00:00 2001 From: Max Fan Date: Fri, 29 Jul 2022 16:15:58 -0400 Subject: [PATCH 1176/1301] docs: image build clarifications/grammar (#2489) I changed was build > was built and reorganized a few sentences to be more clear. Signed-off-by: InnovativeInventor --- docker/models/images.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docker/models/images.py b/docker/models/images.py index 79ccbe4095..ae4e294329 100644 --- a/docker/models/images.py +++ b/docker/models/images.py @@ -224,10 +224,10 @@ def build(self, **kwargs): Build an image and return it. Similar to the ``docker build`` command. Either ``path`` or ``fileobj`` must be set. - If you have a tar file for the Docker build context (including a - Dockerfile) already, pass a readable file-like object to ``fileobj`` - and also pass ``custom_context=True``. If the stream is compressed - also, set ``encoding`` to the correct value (e.g ``gzip``). + If you already have a tar file for the Docker build context (including a + Dockerfile), pass a readable file-like object to ``fileobj`` + and also pass ``custom_context=True``. If the stream is also compressed, + set ``encoding`` to the correct value (e.g ``gzip``). If you want to get the raw output of the build, use the :py:meth:`~docker.api.build.BuildApiMixin.build` method in the @@ -284,7 +284,7 @@ def build(self, **kwargs): Returns: (tuple): The first item is the :py:class:`Image` object for the - image that was build. The second item is a generator of the + image that was built. The second item is a generator of the build logs as JSON-decoded objects. Raises: From 828d06f5f5e2c8ecd9a8d53c1ef40f37d19a62f5 Mon Sep 17 00:00:00 2001 From: Milas Bowman Date: Sat, 30 Jul 2022 12:09:36 -0400 Subject: [PATCH 1177/1301] docs: fix RollbackConfig/Order values (#3027) Closes #2626. Signed-off-by: Milas Bowman --- docker/types/services.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/types/services.py b/docker/types/services.py index c2fce9f496..360aed06f3 100644 --- a/docker/types/services.py +++ b/docker/types/services.py @@ -399,7 +399,7 @@ class UpdateConfig(dict): an update before the failure action is invoked, specified as a floating point number between 0 and 1. Default: 0 order (string): Specifies the order of operations when rolling out an - updated task. Either ``start_first`` or ``stop_first`` are accepted. + updated task. Either ``start-first`` or ``stop-first`` are accepted. """ def __init__(self, parallelism=0, delay=None, failure_action='continue', monitor=None, max_failure_ratio=None, order=None): @@ -453,7 +453,7 @@ class RollbackConfig(UpdateConfig): a rollback before the failure action is invoked, specified as a floating point number between 0 and 1. Default: 0 order (string): Specifies the order of operations when rolling out a - rolled back task. Either ``start_first`` or ``stop_first`` are + rolled back task. Either ``start-first`` or ``stop-first`` are accepted. """ pass From cd2c35a9b699522b282cc4f024efa5699df24896 Mon Sep 17 00:00:00 2001 From: Milas Bowman Date: Sat, 30 Jul 2022 12:14:27 -0400 Subject: [PATCH 1178/1301] ci: add workflow for releases (#3018) GitHub Actions workflow to create a release: will upload to PyPI and create a GitHub release with the `sdist` and `bdist_wheel` as well. The version code is switched to `setuptools_scm` to work well with this flow (e.g. avoid needing to write a script that does a `sed` on the version file and commits as part of release). Signed-off-by: Milas Bowman --- .editorconfig | 3 +++ .github/workflows/ci.yml | 3 +++ .github/workflows/release.yml | 44 +++++++++++++++++++++++++++++++++++ .gitignore | 4 ++++ Dockerfile | 4 ++-- docker/__init__.py | 3 +-- docker/constants.py | 4 ++-- docker/version.py | 16 +++++++++++-- docs/conf.py | 11 ++++----- pyproject.toml | 5 ++++ setup.py | 8 +++---- tests/Dockerfile | 13 +++++++---- 12 files changed, 96 insertions(+), 22 deletions(-) create mode 100644 .github/workflows/release.yml create mode 100644 pyproject.toml diff --git a/.editorconfig b/.editorconfig index d7f2776ada..65d0c51972 100644 --- a/.editorconfig +++ b/.editorconfig @@ -9,3 +9,6 @@ max_line_length = 80 [*.md] trim_trailing_whitespace = false + +[*.{yaml,yml}] +indent_size = 2 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 296bf0ddd6..d1634125bc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,6 +2,9 @@ name: Python package on: [push, pull_request] +env: + DOCKER_BUILDKIT: '1' + jobs: flake8: runs-on: ubuntu-latest diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000000..50695b14dc --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,44 @@ +name: Release + +on: + workflow_dispatch: + inputs: + tag: + description: "Release Tag WITHOUT `v` Prefix (e.g. 6.0.0)" + required: true + dry-run: + description: 'Dry run' + required: false + type: boolean + default: true + +jobs: + publish: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: '3.10' + + - run: python setup.py sdist bdist_wheel + env: + SETUPTOOLS_SCM_PRETEND_VERSION_FOR_DOCKER: ${{ inputs.tag }} + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + if: ! inputs.dry-run + with: + password: ${{ secrets.PYPI_API_TOKEN }} + + - name: Create GitHub release + uses: ncipollo/release-action@v1 + if: ! inputs.dry-run + with: + artifacts: "dist/*" + generateReleaseNotes: true + draft: true + commit: ${{ github.sha }} + token: ${{ secrets.GITHUB_TOKEN }} + tag: ${{ inputs.tag }} diff --git a/.gitignore b/.gitignore index e626dc6cef..c88ccc1b3a 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,10 @@ html/* _build/ README.rst +# setuptools_scm +_version.py + env/ venv/ .idea/ +*.iml diff --git a/Dockerfile b/Dockerfile index c158a9d676..ef9b886cd4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,6 @@ ARG PYTHON_VERSION=3.10 FROM python:${PYTHON_VERSION} -RUN mkdir /src WORKDIR /src COPY requirements.txt /src/requirements.txt @@ -11,5 +10,6 @@ RUN pip install --no-cache-dir -r requirements.txt COPY test-requirements.txt /src/test-requirements.txt RUN pip install --no-cache-dir -r test-requirements.txt -COPY . /src +COPY . . +ARG SETUPTOOLS_SCM_PRETEND_VERSION_DOCKER RUN pip install --no-cache-dir . diff --git a/docker/__init__.py b/docker/__init__.py index e5c1a8f6e0..46beb532a7 100644 --- a/docker/__init__.py +++ b/docker/__init__.py @@ -4,7 +4,6 @@ from .context import Context from .context import ContextAPI from .tls import TLSConfig -from .version import version, version_info +from .version import __version__ -__version__ = version __title__ = 'docker' diff --git a/docker/constants.py b/docker/constants.py index d5bfc35dfb..ed341a9020 100644 --- a/docker/constants.py +++ b/docker/constants.py @@ -1,5 +1,5 @@ import sys -from .version import version +from .version import __version__ DEFAULT_DOCKER_API_VERSION = '1.41' MINIMUM_DOCKER_API_VERSION = '1.21' @@ -28,7 +28,7 @@ IS_WINDOWS_PLATFORM = (sys.platform == 'win32') WINDOWS_LONGPATH_PREFIX = '\\\\?\\' -DEFAULT_USER_AGENT = f"docker-sdk-python/{version}" +DEFAULT_USER_AGENT = f"docker-sdk-python/{__version__}" DEFAULT_NUM_POOLS = 25 # The OpenSSH server default value for MaxSessions is 10 which means we can diff --git a/docker/version.py b/docker/version.py index 88ee8b0f3d..44eac8c5dc 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,14 @@ -version = "6.0.0-dev" -version_info = tuple(int(d) for d in version.split("-")[0].split(".")) +try: + from ._version import __version__ +except ImportError: + try: + # importlib.metadata available in Python 3.8+, the fallback (0.0.0) + # is fine because release builds use _version (above) rather than + # this code path, so it only impacts developing w/ 3.7 + from importlib.metadata import version, PackageNotFoundError + try: + __version__ = version('docker') + except PackageNotFoundError: + __version__ = '0.0.0' + except ImportError: + __version__ = '0.0.0' diff --git a/docs/conf.py b/docs/conf.py index 1258a42386..dc3b37cc8a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -63,12 +63,11 @@ # |version| and |release|, also used in various other places throughout the # built documents. # -with open('../docker/version.py') as vfile: - exec(vfile.read()) -# The full version, including alpha/beta/rc tags. -release = version -# The short X.Y version. -version = f'{version_info[0]}.{version_info[1]}' +# see https://github.com/pypa/setuptools_scm#usage-from-sphinx +from importlib.metadata import version +release = version('docker') +# for example take major/minor +version = '.'.join(release.split('.')[:2]) # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000000..9554358e56 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,5 @@ +[build-system] +requires = ["setuptools>=45", "setuptools_scm[toml]>=6.2"] + +[tool.setuptools_scm] +write_to = 'docker/_version.py' diff --git a/setup.py b/setup.py index c6346b0790..68f7c27410 100644 --- a/setup.py +++ b/setup.py @@ -29,9 +29,6 @@ 'ssh': ['paramiko>=2.4.3'], } -version = None -exec(open('docker/version.py').read()) - with open('./test-requirements.txt') as test_reqs_txt: test_requirements = [line for line in test_reqs_txt] @@ -42,7 +39,9 @@ setup( name="docker", - version=version, + use_scm_version={ + 'write_to': 'docker/_version.py' + }, description="A Python library for the Docker Engine API.", long_description=long_description, long_description_content_type='text/markdown', @@ -54,6 +53,7 @@ 'Tracker': 'https://github.com/docker/docker-py/issues', }, packages=find_packages(exclude=["tests.*", "tests"]), + setup_requires=['setuptools_scm'], install_requires=requirements, tests_require=test_requirements, extras_require=extras_require, diff --git a/tests/Dockerfile b/tests/Dockerfile index e24da47d46..cf2cd67dfe 100644 --- a/tests/Dockerfile +++ b/tests/Dockerfile @@ -1,5 +1,5 @@ +# syntax = docker/dockerfile:1.4 ARG PYTHON_VERSION=3.10 - FROM python:${PYTHON_VERSION} ARG APT_MIRROR @@ -29,11 +29,16 @@ RUN curl -sSL -o /opt/docker-credential-pass.tar.gz \ chmod +x /usr/local/bin/docker-credential-pass WORKDIR /src + COPY requirements.txt /src/requirements.txt -RUN pip install -r requirements.txt +RUN --mount=type=cache,target=/root/.cache/pip \ + pip install -r requirements.txt COPY test-requirements.txt /src/test-requirements.txt -RUN pip install -r test-requirements.txt +RUN --mount=type=cache,target=/root/.cache/pip \ + pip install -r test-requirements.txt COPY . /src -RUN pip install . +ARG SETUPTOOLS_SCM_PRETEND_VERSION=99.0.0-docker +RUN --mount=type=cache,target=/root/.cache/pip \ + pip install -e . From 7f1bde162f8266800d336a97becc92aa92da13a9 Mon Sep 17 00:00:00 2001 From: Milas Bowman Date: Sat, 30 Jul 2022 12:20:50 -0400 Subject: [PATCH 1179/1301] ci: fix quoting in YAML Because apparently `!` is special Signed-off-by: Milas Bowman --- .github/workflows/release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 50695b14dc..a4b25652dc 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -28,13 +28,13 @@ jobs: - name: Publish to PyPI uses: pypa/gh-action-pypi-publish@release/v1 - if: ! inputs.dry-run + if: '! inputs.dry-run' with: password: ${{ secrets.PYPI_API_TOKEN }} - name: Create GitHub release uses: ncipollo/release-action@v1 - if: ! inputs.dry-run + if: '! inputs.dry-run' with: artifacts: "dist/*" generateReleaseNotes: true From 631b332cd917e07bc15a152b1066c70902b6cb92 Mon Sep 17 00:00:00 2001 From: Milas Bowman Date: Sat, 30 Jul 2022 12:23:53 -0400 Subject: [PATCH 1180/1301] ci: add missing wheel package Signed-off-by: Milas Bowman --- .github/workflows/release.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a4b25652dc..dde656c033 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -22,7 +22,10 @@ jobs: with: python-version: '3.10' - - run: python setup.py sdist bdist_wheel + - name: Generate Pacakge + run: | + pip3 install wheel + python setup.py sdist bdist_wheel env: SETUPTOOLS_SCM_PRETEND_VERSION_FOR_DOCKER: ${{ inputs.tag }} From 3f0095a7c1966c521652314e524ff362c24ff58c Mon Sep 17 00:00:00 2001 From: Thomas Gassmann Date: Sat, 30 Jul 2022 09:43:29 -0700 Subject: [PATCH 1181/1301] docs: remove duplicate 'on' in comment (#2370) Remove duplicate 'on' in comment Signed-off-by: Thomas Gassmann --- docker/models/containers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/models/containers.py b/docker/models/containers.py index c37df55e29..6661b213bf 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -670,7 +670,7 @@ def run(self, image, command=None, stdout=True, stderr=False, network_mode (str): One of: - ``bridge`` Create a new network stack for the container on - on the bridge network. + the bridge network. - ``none`` No networking for this container. - ``container:`` Reuse another container's network stack. From b7daa52feb8b897fb10fbe82c6e49a273746352a Mon Sep 17 00:00:00 2001 From: Saurav Maheshkar Date: Tue, 2 Aug 2022 19:38:24 +0530 Subject: [PATCH 1182/1301] docs: add `gzip` arg to `BuildApiMixin` (#2929) Signed-off-by: Saurav Maheshkar --- docker/api/build.py | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/api/build.py b/docker/api/build.py index a48204a9fd..3a1a3d9642 100644 --- a/docker/api/build.py +++ b/docker/api/build.py @@ -76,6 +76,7 @@ def build(self, path=None, tag=None, quiet=False, fileobj=None, forcerm (bool): Always remove intermediate containers, even after unsuccessful builds dockerfile (str): path within the build context to the Dockerfile + gzip (bool): If set to ``True``, gzip compression/encoding is used buildargs (dict): A dictionary of build arguments container_limits (dict): A dictionary of limits applied to each container created by the build process. Valid keys: From ab5e927300b0fd44b002e657eb371a6e7356c809 Mon Sep 17 00:00:00 2001 From: q0w <43147888+q0w@users.noreply.github.com> Date: Tue, 2 Aug 2022 17:11:07 +0300 Subject: [PATCH 1183/1301] lint: remove extraneous logic for `preexec_func` (#2920) `preexec_func` is still None if it is win32 Signed-off-by: q0w <43147888+q0w@users.noreply.github.com> --- docker/transport/sshconn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/transport/sshconn.py b/docker/transport/sshconn.py index 277640690a..7421f33bdc 100644 --- a/docker/transport/sshconn.py +++ b/docker/transport/sshconn.py @@ -62,7 +62,7 @@ def f(): env=env, stdout=subprocess.PIPE, stdin=subprocess.PIPE, - preexec_fn=None if constants.IS_WINDOWS_PLATFORM else preexec_func) + preexec_fn=preexec_func) def _write(self, data): if not self.proc or self.proc.stdin.closed: From 42789818bed5d86b487a030e2e60b02bf0cfa284 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ning=C3=BA?= <47453810+n1ngu@users.noreply.github.com> Date: Tue, 2 Aug 2022 16:19:50 +0200 Subject: [PATCH 1184/1301] credentials: eliminate distutils deprecation warnings (#3028) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit While removing any usage of the deprecated `distutils` package, ("The distutils package is deprecated and slated for removal in Python 3.12.") this internal utility can be removed straightaway because the `shutil.which` replacement for `distutils.spawn.find_executable` already honors the `PATHEXT` environment variable in windows systems. See https://docs.python.org/3/library/shutil.html#shutil.which Signed-off-by: Daniel Möller --- docker/credentials/store.py | 4 +-- docker/credentials/utils.py | 28 --------------------- tests/integration/credentials/store_test.py | 6 ++--- 3 files changed, 5 insertions(+), 33 deletions(-) diff --git a/docker/credentials/store.py b/docker/credentials/store.py index e55976f189..297f46841c 100644 --- a/docker/credentials/store.py +++ b/docker/credentials/store.py @@ -1,11 +1,11 @@ import errno import json +import shutil import subprocess from . import constants from . import errors from .utils import create_environment_dict -from .utils import find_executable class Store: @@ -15,7 +15,7 @@ def __init__(self, program, environment=None): and erasing credentials using `program`. """ self.program = constants.PROGRAM_PREFIX + program - self.exe = find_executable(self.program) + self.exe = shutil.which(self.program) self.environment = environment if self.exe is None: raise errors.InitializationError( diff --git a/docker/credentials/utils.py b/docker/credentials/utils.py index 3f720ef1a7..5c83d05cfb 100644 --- a/docker/credentials/utils.py +++ b/docker/credentials/utils.py @@ -1,32 +1,4 @@ -import distutils.spawn import os -import sys - - -def find_executable(executable, path=None): - """ - As distutils.spawn.find_executable, but on Windows, look up - every extension declared in PATHEXT instead of just `.exe` - """ - if sys.platform != 'win32': - return distutils.spawn.find_executable(executable, path) - - if path is None: - path = os.environ['PATH'] - - paths = path.split(os.pathsep) - extensions = os.environ.get('PATHEXT', '.exe').split(os.pathsep) - base, ext = os.path.splitext(executable) - - if not os.path.isfile(executable): - for p in paths: - for ext in extensions: - f = os.path.join(p, base + ext) - if os.path.isfile(f): - return f - return None - else: - return executable def create_environment_dict(overrides): diff --git a/tests/integration/credentials/store_test.py b/tests/integration/credentials/store_test.py index d0cfd5417c..213cf305e8 100644 --- a/tests/integration/credentials/store_test.py +++ b/tests/integration/credentials/store_test.py @@ -1,9 +1,9 @@ import os import random +import shutil import sys import pytest -from distutils.spawn import find_executable from docker.credentials import ( CredentialsNotFound, Store, StoreError, DEFAULT_LINUX_STORE, @@ -22,9 +22,9 @@ def teardown_method(self): def setup_method(self): self.tmp_keys = [] if sys.platform.startswith('linux'): - if find_executable('docker-credential-' + DEFAULT_LINUX_STORE): + if shutil.which('docker-credential-' + DEFAULT_LINUX_STORE): self.store = Store(DEFAULT_LINUX_STORE) - elif find_executable('docker-credential-pass'): + elif shutil.which('docker-credential-pass'): self.store = Store('pass') else: raise Exception('No supported docker-credential store in PATH') From 66402435d18d1ec6430217bba031abcf7776c549 Mon Sep 17 00:00:00 2001 From: Leonard Kinday Date: Thu, 11 Aug 2022 22:20:31 +0200 Subject: [PATCH 1185/1301] Support `global-job` and `replicated-job` modes in Docker Swarm (#3016) Add `global-job` and `replicated-job` modes Fixes #2829. Signed-off-by: Leonard Kinday --- docker/models/images.py | 8 +-- docker/types/services.py | 85 ++++++++++++++++++++------- tests/helpers.py | 2 +- tests/integration/api_service_test.py | 33 +++++++++++ tests/unit/dockertypes_test.py | 16 +++++ 5 files changed, 119 insertions(+), 25 deletions(-) diff --git a/docker/models/images.py b/docker/models/images.py index ae4e294329..e3ec39d28d 100644 --- a/docker/models/images.py +++ b/docker/models/images.py @@ -224,10 +224,10 @@ def build(self, **kwargs): Build an image and return it. Similar to the ``docker build`` command. Either ``path`` or ``fileobj`` must be set. - If you already have a tar file for the Docker build context (including a - Dockerfile), pass a readable file-like object to ``fileobj`` - and also pass ``custom_context=True``. If the stream is also compressed, - set ``encoding`` to the correct value (e.g ``gzip``). + If you already have a tar file for the Docker build context (including + a Dockerfile), pass a readable file-like object to ``fileobj`` + and also pass ``custom_context=True``. If the stream is also + compressed, set ``encoding`` to the correct value (e.g ``gzip``). If you want to get the raw output of the build, use the :py:meth:`~docker.api.build.BuildApiMixin.build` method in the diff --git a/docker/types/services.py b/docker/types/services.py index 360aed06f3..268684e0a9 100644 --- a/docker/types/services.py +++ b/docker/types/services.py @@ -29,6 +29,7 @@ class TaskTemplate(dict): force_update (int): A counter that triggers an update even if no relevant parameters have been changed. """ + def __init__(self, container_spec, resources=None, restart_policy=None, placement=None, log_driver=None, networks=None, force_update=None): @@ -115,6 +116,7 @@ class ContainerSpec(dict): cap_drop (:py:class:`list`): A list of kernel capabilities to drop from the default set for the container. """ + def __init__(self, image, command=None, args=None, hostname=None, env=None, workdir=None, user=None, labels=None, mounts=None, stop_grace_period=None, secrets=None, tty=None, groups=None, @@ -231,6 +233,7 @@ class Mount(dict): tmpfs_size (int or string): The size for the tmpfs mount in bytes. tmpfs_mode (int): The permission mode for the tmpfs mount. """ + def __init__(self, target, source, type='volume', read_only=False, consistency=None, propagation=None, no_copy=False, labels=None, driver_config=None, tmpfs_size=None, @@ -331,6 +334,7 @@ class Resources(dict): ``{ resource_name: resource_value }``. Alternatively, a list of of resource specifications as defined by the Engine API. """ + def __init__(self, cpu_limit=None, mem_limit=None, cpu_reservation=None, mem_reservation=None, generic_resources=None): limits = {} @@ -401,6 +405,7 @@ class UpdateConfig(dict): order (string): Specifies the order of operations when rolling out an updated task. Either ``start-first`` or ``stop-first`` are accepted. """ + def __init__(self, parallelism=0, delay=None, failure_action='continue', monitor=None, max_failure_ratio=None, order=None): self['Parallelism'] = parallelism @@ -512,6 +517,7 @@ class DriverConfig(dict): name (string): Name of the driver to use. options (dict): Driver-specific options. Default: ``None``. """ + def __init__(self, name, options=None): self['Name'] = name if options: @@ -533,6 +539,7 @@ class EndpointSpec(dict): is ``(target_port [, protocol [, publish_mode]])``. Ports can only be provided if the ``vip`` resolution mode is used. """ + def __init__(self, mode=None, ports=None): if ports: self['Ports'] = convert_service_ports(ports) @@ -575,37 +582,70 @@ def convert_service_ports(ports): class ServiceMode(dict): """ - Indicate whether a service should be deployed as a replicated or global - service, and associated parameters + Indicate whether a service or a job should be deployed as a replicated + or global service, and associated parameters Args: - mode (string): Can be either ``replicated`` or ``global`` + mode (string): Can be either ``replicated``, ``global``, + ``replicated-job`` or ``global-job`` replicas (int): Number of replicas. For replicated services only. + concurrency (int): Number of concurrent jobs. For replicated job + services only. """ - def __init__(self, mode, replicas=None): - if mode not in ('replicated', 'global'): - raise errors.InvalidArgument( - 'mode must be either "replicated" or "global"' - ) - if mode != 'replicated' and replicas is not None: + + def __init__(self, mode, replicas=None, concurrency=None): + replicated_modes = ('replicated', 'replicated-job') + supported_modes = replicated_modes + ('global', 'global-job') + + if mode not in supported_modes: raise errors.InvalidArgument( - 'replicas can only be used for replicated mode' + 'mode must be either "replicated", "global", "replicated-job"' + ' or "global-job"' ) - self[mode] = {} + + if mode not in replicated_modes: + if replicas is not None: + raise errors.InvalidArgument( + 'replicas can only be used for "replicated" or' + ' "replicated-job" mode' + ) + + if concurrency is not None: + raise errors.InvalidArgument( + 'concurrency can only be used for "replicated-job" mode' + ) + + service_mode = self._convert_mode(mode) + self.mode = service_mode + self[service_mode] = {} + if replicas is not None: - self[mode]['Replicas'] = replicas + if mode == 'replicated': + self[service_mode]['Replicas'] = replicas - @property - def mode(self): - if 'global' in self: - return 'global' - return 'replicated' + if mode == 'replicated-job': + self[service_mode]['MaxConcurrent'] = concurrency or 1 + self[service_mode]['TotalCompletions'] = replicas + + @staticmethod + def _convert_mode(original_mode): + if original_mode == 'global-job': + return 'GlobalJob' + + if original_mode == 'replicated-job': + return 'ReplicatedJob' + + return original_mode @property def replicas(self): - if self.mode != 'replicated': - return None - return self['replicated'].get('Replicas') + if 'replicated' in self: + return self['replicated'].get('Replicas') + + if 'ReplicatedJob' in self: + return self['ReplicatedJob'].get('TotalCompletions') + + return None class SecretReference(dict): @@ -679,6 +719,7 @@ class Placement(dict): platforms (:py:class:`list` of tuple): A list of platforms expressed as ``(arch, os)`` tuples """ + def __init__(self, constraints=None, preferences=None, platforms=None, maxreplicas=None): if constraints is not None: @@ -711,6 +752,7 @@ class PlacementPreference(dict): the scheduler will try to spread tasks evenly over groups of nodes identified by this label. """ + def __init__(self, strategy, descriptor): if strategy != 'spread': raise errors.InvalidArgument( @@ -732,6 +774,7 @@ class DNSConfig(dict): options (:py:class:`list`): A list of internal resolver variables to be modified (e.g., ``debug``, ``ndots:3``, etc.). """ + def __init__(self, nameservers=None, search=None, options=None): self['Nameservers'] = nameservers self['Search'] = search @@ -762,6 +805,7 @@ class Privileges(dict): selinux_type (string): SELinux type label selinux_level (string): SELinux level label """ + def __init__(self, credentialspec_file=None, credentialspec_registry=None, selinux_disable=None, selinux_user=None, selinux_role=None, selinux_type=None, selinux_level=None): @@ -804,6 +848,7 @@ class NetworkAttachmentConfig(dict): options (:py:class:`dict`): Driver attachment options for the network target. """ + def __init__(self, target, aliases=None, options=None): self['Target'] = target self['Aliases'] = aliases diff --git a/tests/helpers.py b/tests/helpers.py index 63cbe2e63a..bdb07f96b9 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -143,4 +143,4 @@ def ctrl_with(char): if re.match('[a-z]', char): return chr(ord(char) - ord('a') + 1).encode('ascii') else: - raise(Exception('char must be [a-z]')) + raise Exception('char must be [a-z]') diff --git a/tests/integration/api_service_test.py b/tests/integration/api_service_test.py index dcf195dec8..03770a03eb 100644 --- a/tests/integration/api_service_test.py +++ b/tests/integration/api_service_test.py @@ -626,6 +626,39 @@ def test_create_service_replicated_mode(self): assert 'Replicated' in svc_info['Spec']['Mode'] assert svc_info['Spec']['Mode']['Replicated'] == {'Replicas': 5} + @requires_api_version('1.41') + def test_create_service_global_job_mode(self): + container_spec = docker.types.ContainerSpec( + TEST_IMG, ['echo', 'hello'] + ) + task_tmpl = docker.types.TaskTemplate(container_spec) + name = self.get_service_name() + svc_id = self.client.create_service( + task_tmpl, name=name, mode='global-job' + ) + svc_info = self.client.inspect_service(svc_id) + assert 'Mode' in svc_info['Spec'] + assert 'GlobalJob' in svc_info['Spec']['Mode'] + + @requires_api_version('1.41') + def test_create_service_replicated_job_mode(self): + container_spec = docker.types.ContainerSpec( + TEST_IMG, ['echo', 'hello'] + ) + task_tmpl = docker.types.TaskTemplate(container_spec) + name = self.get_service_name() + svc_id = self.client.create_service( + task_tmpl, name=name, + mode=docker.types.ServiceMode('replicated-job', 5) + ) + svc_info = self.client.inspect_service(svc_id) + assert 'Mode' in svc_info['Spec'] + assert 'ReplicatedJob' in svc_info['Spec']['Mode'] + assert svc_info['Spec']['Mode']['ReplicatedJob'] == { + 'MaxConcurrent': 1, + 'TotalCompletions': 5 + } + @requires_api_version('1.25') def test_update_service_force_update(self): container_spec = docker.types.ContainerSpec( diff --git a/tests/unit/dockertypes_test.py b/tests/unit/dockertypes_test.py index 76a99a627d..f3d562e108 100644 --- a/tests/unit/dockertypes_test.py +++ b/tests/unit/dockertypes_test.py @@ -325,10 +325,26 @@ def test_global_simple(self): assert mode.mode == 'global' assert mode.replicas is None + def test_replicated_job_simple(self): + mode = ServiceMode('replicated-job') + assert mode == {'ReplicatedJob': {}} + assert mode.mode == 'ReplicatedJob' + assert mode.replicas is None + + def test_global_job_simple(self): + mode = ServiceMode('global-job') + assert mode == {'GlobalJob': {}} + assert mode.mode == 'GlobalJob' + assert mode.replicas is None + def test_global_replicas_error(self): with pytest.raises(InvalidArgument): ServiceMode('global', 21) + def test_global_job_replicas_simple(self): + with pytest.raises(InvalidArgument): + ServiceMode('global-job', 21) + def test_replicated_replicas(self): mode = ServiceMode('replicated', 21) assert mode == {'replicated': {'Replicas': 21}} From ff0b4ac60bdc61392c4b543c3be8ae97dc8cd191 Mon Sep 17 00:00:00 2001 From: Milas Bowman Date: Thu, 11 Aug 2022 17:20:13 -0400 Subject: [PATCH 1186/1301] docs: add changelog for 6.0.0 (#3019) Signed-off-by: Milas Bowman --- docs/change-log.md | 40 +++++++++++++++++++++++++++++++++++++++- tests/Dockerfile | 2 +- 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/docs/change-log.md b/docs/change-log.md index 91f3fe6f17..5927728b1a 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -1,6 +1,44 @@ -Change log +Changelog ========== +6.0.0 +----- + +### Upgrade Notes +- Minimum supported Python version is 3.7+ +- When installing with pip, the `docker[tls]` extra is deprecated and a no-op, + use `docker` for same functionality (TLS support is always available now) +- Native Python SSH client (used by default / `use_ssh_client=False`) will now + reject unknown host keys with `paramiko.ssh_exception.SSHException` +- Short IDs are now 12 characters instead of 10 characters (same as Docker CLI) + +### Features +- Python 3.10 support +- Automatically negotiate most secure TLS version +- Add `platform` (e.g. `linux/amd64`, `darwin/arm64`) to container create & run +- Add support for `GlobalJob` and `ReplicatedJobs` for Swarm +- Add `remove()` method on `Image` +- Add `force` param to `disable()` on `Plugin` + +### Bugfixes +- Fix install issues on Windows related to `pywin32` +- Do not accept unknown SSH host keys in native Python SSH mode +- Use 12 character short IDs for consistency with Docker CLI +- Ignore trailing whitespace in `.dockerignore` files +- Fix IPv6 host parsing when explicit port specified +- Fix `ProxyCommand` option for SSH connections +- Do not spawn extra subshell when launching external SSH client +- Improve exception semantics to preserve context +- Documentation improvements (formatting, examples, typos, missing params) + +### Miscellaneous +- Upgrade dependencies in `requirements.txt` to latest versions +- Remove extraneous transitive dependencies +- Eliminate usages of deprecated functions/methods +- Test suite reliability improvements +- GitHub Actions workflows for linting, unit tests, integration tests, and + publishing releases + 5.0.3 ----- diff --git a/tests/Dockerfile b/tests/Dockerfile index cf2cd67dfe..2cac785d9d 100644 --- a/tests/Dockerfile +++ b/tests/Dockerfile @@ -39,6 +39,6 @@ RUN --mount=type=cache,target=/root/.cache/pip \ pip install -r test-requirements.txt COPY . /src -ARG SETUPTOOLS_SCM_PRETEND_VERSION=99.0.0-docker +ARG SETUPTOOLS_SCM_PRETEND_VERSION=99.0.0+docker RUN --mount=type=cache,target=/root/.cache/pip \ pip install -e . From 58aa62bb154a2ccea433cf475aefbd695fb5abc8 Mon Sep 17 00:00:00 2001 From: Quentin Mathorel <110528861+Aadenei@users.noreply.github.com> Date: Fri, 12 Aug 2022 14:55:19 +0200 Subject: [PATCH 1187/1301] swarm: add sysctl support for services (#3029) Signed-off-by: Quentin Mathorel --- docker/models/services.py | 3 +++ docker/types/services.py | 10 +++++++++- tests/integration/api_service_test.py | 20 ++++++++++++++++++++ tests/unit/models_services_test.py | 5 +++-- 4 files changed, 35 insertions(+), 3 deletions(-) diff --git a/docker/models/services.py b/docker/models/services.py index 9255068119..06438748f3 100644 --- a/docker/models/services.py +++ b/docker/models/services.py @@ -217,6 +217,8 @@ def create(self, image, command=None, **kwargs): the default set for the container. cap_drop (:py:class:`list`): A list of kernel capabilities to drop from the default set for the container. + sysctls (:py:class:`dict`): A dict of sysctl values to add to the + container Returns: :py:class:`Service`: The created service. @@ -305,6 +307,7 @@ def list(self, **kwargs): 'tty', 'user', 'workdir', + 'sysctls', ] # kwargs to copy straight over to TaskTemplate diff --git a/docker/types/services.py b/docker/types/services.py index 268684e0a9..a3383ef75b 100644 --- a/docker/types/services.py +++ b/docker/types/services.py @@ -115,6 +115,8 @@ class ContainerSpec(dict): default set for the container. cap_drop (:py:class:`list`): A list of kernel capabilities to drop from the default set for the container. + sysctls (:py:class:`dict`): A dict of sysctl values to add to + the container """ def __init__(self, image, command=None, args=None, hostname=None, env=None, @@ -123,7 +125,7 @@ def __init__(self, image, command=None, args=None, hostname=None, env=None, open_stdin=None, read_only=None, stop_signal=None, healthcheck=None, hosts=None, dns_config=None, configs=None, privileges=None, isolation=None, init=None, cap_add=None, - cap_drop=None): + cap_drop=None, sysctls=None): self['Image'] = image if isinstance(command, str): @@ -205,6 +207,12 @@ def __init__(self, image, command=None, args=None, hostname=None, env=None, self['CapabilityDrop'] = cap_drop + if sysctls is not None: + if not isinstance(sysctls, dict): + raise TypeError('sysctls must be a dict') + + self['Sysctls'] = sysctls + class Mount(dict): """ diff --git a/tests/integration/api_service_test.py b/tests/integration/api_service_test.py index 03770a03eb..8ce7c9d57e 100644 --- a/tests/integration/api_service_test.py +++ b/tests/integration/api_service_test.py @@ -1419,3 +1419,23 @@ def test_create_service_cap_drop(self): assert services[0]['ID'] == svc_id['ID'] spec = services[0]['Spec']['TaskTemplate']['ContainerSpec'] assert 'CAP_SYSLOG' in spec['CapabilityDrop'] + + @requires_api_version('1.40') + def test_create_service_with_sysctl(self): + name = self.get_service_name() + sysctls = { + 'net.core.somaxconn': '1024', + 'net.ipv4.tcp_syncookies': '0', + } + container_spec = docker.types.ContainerSpec( + TEST_IMG, ['echo', 'hello'], sysctls=sysctls + ) + task_tmpl = docker.types.TaskTemplate(container_spec) + svc_id = self.client.create_service(task_tmpl, name=name) + assert self.client.inspect_service(svc_id) + services = self.client.services(filters={'name': name}) + assert len(services) == 1 + assert services[0]['ID'] == svc_id['ID'] + spec = services[0]['Spec']['TaskTemplate']['ContainerSpec'] + assert spec['Sysctls']['net.core.somaxconn'] == '1024' + assert spec['Sysctls']['net.ipv4.tcp_syncookies'] == '0' diff --git a/tests/unit/models_services_test.py b/tests/unit/models_services_test.py index 94a27f0e5c..45c63ac9e0 100644 --- a/tests/unit/models_services_test.py +++ b/tests/unit/models_services_test.py @@ -29,7 +29,8 @@ def test_get_create_service_kwargs(self): 'constraints': ['foo=bar'], 'preferences': ['bar=baz'], 'platforms': [('x86_64', 'linux')], - 'maxreplicas': 1 + 'maxreplicas': 1, + 'sysctls': {'foo': 'bar'} }) task_template = kwargs.pop('task_template') @@ -59,5 +60,5 @@ def test_get_create_service_kwargs(self): assert task_template['Networks'] == [{'Target': 'somenet'}] assert set(task_template['ContainerSpec'].keys()) == { 'Image', 'Command', 'Args', 'Hostname', 'Env', 'Dir', 'User', - 'Labels', 'Mounts', 'StopGracePeriod' + 'Labels', 'Mounts', 'StopGracePeriod', 'Sysctls' } From fc86ab0d8501b10dbe9be203625e9002cf3922ed Mon Sep 17 00:00:00 2001 From: Chris Hand Date: Fri, 12 Aug 2022 09:58:57 -0400 Subject: [PATCH 1188/1301] swarm: add support for DataPathPort on init (#2987) Adds support for setting the UDP port used for VXLAN traffic between swarm nodes Signed-off-by: Chris Hand --- docker/api/swarm.py | 13 ++++++++++++- docker/models/swarm.py | 7 ++++++- tests/integration/api_swarm_test.py | 5 +++++ 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/docker/api/swarm.py b/docker/api/swarm.py index db40fdd3d7..d09dd087b7 100644 --- a/docker/api/swarm.py +++ b/docker/api/swarm.py @@ -85,7 +85,7 @@ def get_unlock_key(self): def init_swarm(self, advertise_addr=None, listen_addr='0.0.0.0:2377', force_new_cluster=False, swarm_spec=None, default_addr_pool=None, subnet_size=None, - data_path_addr=None): + data_path_addr=None, data_path_port=None): """ Initialize a new Swarm using the current connected engine as the first node. @@ -118,6 +118,9 @@ def init_swarm(self, advertise_addr=None, listen_addr='0.0.0.0:2377', networks created from the default subnet pool. Default: None data_path_addr (string): Address or interface to use for data path traffic. For example, 192.168.1.1, or an interface, like eth0. + data_path_port (int): Port number to use for data path traffic. + Acceptable port range is 1024 to 49151. If set to ``None`` or + 0, the default port 4789 will be used. Default: None Returns: (str): The ID of the created node. @@ -166,6 +169,14 @@ def init_swarm(self, advertise_addr=None, listen_addr='0.0.0.0:2377', ) data['DataPathAddr'] = data_path_addr + if data_path_port is not None: + if utils.version_lt(self._version, '1.40'): + raise errors.InvalidVersion( + 'Data path port is only available for ' + 'API version >= 1.40' + ) + data['DataPathPort'] = data_path_port + response = self._post_json(url, data=data) return self._result(response, json=True) diff --git a/docker/models/swarm.py b/docker/models/swarm.py index b0b1a2ef8a..1e39f3fd2f 100644 --- a/docker/models/swarm.py +++ b/docker/models/swarm.py @@ -35,7 +35,8 @@ def get_unlock_key(self): def init(self, advertise_addr=None, listen_addr='0.0.0.0:2377', force_new_cluster=False, default_addr_pool=None, - subnet_size=None, data_path_addr=None, **kwargs): + subnet_size=None, data_path_addr=None, data_path_port=None, + **kwargs): """ Initialize a new swarm on this Engine. @@ -65,6 +66,9 @@ def init(self, advertise_addr=None, listen_addr='0.0.0.0:2377', networks created from the default subnet pool. Default: None data_path_addr (string): Address or interface to use for data path traffic. For example, 192.168.1.1, or an interface, like eth0. + data_path_port (int): Port number to use for data path traffic. + Acceptable port range is 1024 to 49151. If set to ``None`` or + 0, the default port 4789 will be used. Default: None task_history_retention_limit (int): Maximum number of tasks history stored. snapshot_interval (int): Number of logs entries between snapshot. @@ -121,6 +125,7 @@ def init(self, advertise_addr=None, listen_addr='0.0.0.0:2377', 'default_addr_pool': default_addr_pool, 'subnet_size': subnet_size, 'data_path_addr': data_path_addr, + 'data_path_port': data_path_port, } init_kwargs['swarm_spec'] = self.client.api.create_swarm_spec(**kwargs) node_id = self.client.api.init_swarm(**init_kwargs) diff --git a/tests/integration/api_swarm_test.py b/tests/integration/api_swarm_test.py index 48c0592c62..cffe12fc24 100644 --- a/tests/integration/api_swarm_test.py +++ b/tests/integration/api_swarm_test.py @@ -253,3 +253,8 @@ def test_rotate_manager_unlock_key(self): @pytest.mark.xfail(reason='Can fail if eth0 has multiple IP addresses') def test_init_swarm_data_path_addr(self): assert self.init_swarm(data_path_addr='eth0') + + @requires_api_version('1.40') + def test_init_swarm_data_path_port(self): + assert self.init_swarm(data_path_port=4242) + assert self.client.inspect_swarm()['DataPathPort'] == 4242 From e901eac7a8c5f29c7720eafb9f58c8356cca2324 Mon Sep 17 00:00:00 2001 From: Milas Bowman Date: Fri, 12 Aug 2022 14:27:53 -0400 Subject: [PATCH 1189/1301] test: add additional tests for cgroupns option (#3024) See #2930. Signed-off-by: Milas Bowman --- tests/integration/api_container_test.py | 14 ++++++++++++++ tests/unit/api_container_test.py | 19 +++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index 8f69e41ff0..0cb8fec68b 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -215,6 +215,20 @@ def test_create_with_mac_address(self): self.client.kill(id) + @requires_api_version('1.41') + def test_create_with_cgroupns(self): + host_config = self.client.create_host_config(cgroupns='private') + + container = self.client.create_container( + image=TEST_IMG, + command=['sleep', '60'], + host_config=host_config, + ) + self.tmp_containers.append(container) + + res = self.client.inspect_container(container) + assert 'private' == res['HostConfig']['CgroupnsMode'] + def test_group_id_ints(self): container = self.client.create_container( TEST_IMG, 'id -G', diff --git a/tests/unit/api_container_test.py b/tests/unit/api_container_test.py index 3a2fbde88e..8f120f4d49 100644 --- a/tests/unit/api_container_test.py +++ b/tests/unit/api_container_test.py @@ -1069,6 +1069,25 @@ def test_create_container_with_host_config_cpus(self): ''') assert args[1]['headers'] == {'Content-Type': 'application/json'} + @requires_api_version('1.41') + def test_create_container_with_cgroupns(self): + self.client.create_container( + image='busybox', + command='true', + host_config=self.client.create_host_config( + cgroupns='private', + ), + ) + + args = fake_request.call_args + assert args[0][1] == url_prefix + 'containers/create' + + expected_payload = self.base_create_payload() + expected_payload['HostConfig'] = self.client.create_host_config() + expected_payload['HostConfig']['CgroupnsMode'] = 'private' + assert json.loads(args[1]['data']) == expected_payload + assert args[1]['headers'] == {'Content-Type': 'application/json'} + class ContainerTest(BaseAPIClientTest): def test_list_containers(self): From 2494d63f36eba0e1811f05e7b2136f8b30f7cdb7 Mon Sep 17 00:00:00 2001 From: Milas Bowman Date: Thu, 18 Aug 2022 17:03:32 -0400 Subject: [PATCH 1190/1301] docs: install package in ReadTheDocs build (#3032) Need to install ourselves so that we can introspect on version using `setuptools_scm` in `docs/conf.py`. Signed-off-by: Milas Bowman --- .readthedocs.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index 464c782604..80000ee7f1 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -4,11 +4,14 @@ sphinx: configuration: docs/conf.py build: - os: ubuntu-20.04 - tools: - python: '3.10' + os: ubuntu-20.04 + tools: + python: '3.10' python: install: - requirements: docs-requirements.txt - - requirements: requirements.txt + - method: pip + path: . + extra_requirements: + - ssh From 1c27ec1f0c34f6b9510f5caadada5fd8ecc430d9 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Fri, 19 Aug 2022 21:09:12 +0200 Subject: [PATCH 1191/1301] ci: use latest stable syntax for Dockerfiles (#3035) I noticed one Dockerfile was pinned to 1.4; given that there's a backward compatibility guarantee on the stable syntax, the general recommendation is to use `dockerfile:1`, which makes sure that the latest stable release of the Dockerfile syntax is pulled before building. While changing, I also made some minor changes to some Dockerfiles to reduce some unneeded layers. Signed-off-by: Sebastiaan van Stijn --- Dockerfile | 2 ++ Dockerfile-docs | 2 ++ tests/Dockerfile | 4 +++- tests/Dockerfile-dind-certs | 2 ++ tests/Dockerfile-ssh-dind | 10 ++++++---- 5 files changed, 15 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index ef9b886cd4..3476c6d036 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,5 @@ +# syntax=docker/dockerfile:1 + ARG PYTHON_VERSION=3.10 FROM python:${PYTHON_VERSION} diff --git a/Dockerfile-docs b/Dockerfile-docs index e993822b85..11adbfe85d 100644 --- a/Dockerfile-docs +++ b/Dockerfile-docs @@ -1,3 +1,5 @@ +# syntax=docker/dockerfile:1 + ARG PYTHON_VERSION=3.10 FROM python:${PYTHON_VERSION} diff --git a/tests/Dockerfile b/tests/Dockerfile index 2cac785d9d..bf95cd6a3c 100644 --- a/tests/Dockerfile +++ b/tests/Dockerfile @@ -1,5 +1,7 @@ -# syntax = docker/dockerfile:1.4 +# syntax=docker/dockerfile:1 + ARG PYTHON_VERSION=3.10 + FROM python:${PYTHON_VERSION} ARG APT_MIRROR diff --git a/tests/Dockerfile-dind-certs b/tests/Dockerfile-dind-certs index 6e711892ca..288a340ab1 100644 --- a/tests/Dockerfile-dind-certs +++ b/tests/Dockerfile-dind-certs @@ -1,3 +1,5 @@ +# syntax=docker/dockerfile:1 + ARG PYTHON_VERSION=3.10 FROM python:${PYTHON_VERSION} diff --git a/tests/Dockerfile-ssh-dind b/tests/Dockerfile-ssh-dind index 22c707a075..0da15aa40f 100644 --- a/tests/Dockerfile-ssh-dind +++ b/tests/Dockerfile-ssh-dind @@ -1,18 +1,20 @@ +# syntax=docker/dockerfile:1 + ARG API_VERSION=1.41 ARG ENGINE_VERSION=20.10 FROM docker:${ENGINE_VERSION}-dind RUN apk add --no-cache --upgrade \ - openssh + openssh COPY tests/ssh/config/server /etc/ssh/ -RUN chmod -R 600 /etc/ssh # set authorized keys for client paswordless connection COPY tests/ssh/config/client/id_rsa.pub /root/.ssh/authorized_keys -RUN chmod -R 600 /root/.ssh # RUN echo "root:root" | chpasswd -RUN ln -s /usr/local/bin/docker /usr/bin/docker +RUN chmod -R 600 /etc/ssh \ + && chmod -R 600 /root/.ssh \ + && ln -s /usr/local/bin/docker /usr/bin/docker EXPOSE 22 From 923e067dddc3d4b86e4e620a99fcdcdafbd17a98 Mon Sep 17 00:00:00 2001 From: Rhiza <6900588+ArchiMoebius@users.noreply.github.com> Date: Fri, 19 Aug 2022 15:10:53 -0400 Subject: [PATCH 1192/1301] api: add support for floats to docker logs params since / until (#3031) Add support for floats to docker logs params `since` / `until` since the Docker Engine APIs support it. This allows using fractional seconds for greater precision. Signed-off-by: Archi Moebius --- docker/api/container.py | 17 +++++++++++------ docker/models/containers.py | 9 +++++---- tests/unit/api_container_test.py | 18 +++++++++++++++++- 3 files changed, 33 insertions(+), 11 deletions(-) diff --git a/docker/api/container.py b/docker/api/container.py index f600be1811..ce483710cb 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -826,11 +826,12 @@ def logs(self, container, stdout=True, stderr=True, stream=False, tail (str or int): Output specified number of lines at the end of logs. Either an integer of number of lines or the string ``all``. Default ``all`` - since (datetime or int): Show logs since a given datetime or - integer epoch (in seconds) + since (datetime, int, or float): Show logs since a given datetime, + integer epoch (in seconds) or float (in fractional seconds) follow (bool): Follow log output. Default ``False`` - until (datetime or int): Show logs that occurred before the given - datetime or integer epoch (in seconds) + until (datetime, int, or float): Show logs that occurred before + the given datetime, integer epoch (in seconds), or + float (in fractional seconds) Returns: (generator or str) @@ -855,9 +856,11 @@ def logs(self, container, stdout=True, stderr=True, stream=False, params['since'] = utils.datetime_to_timestamp(since) elif (isinstance(since, int) and since > 0): params['since'] = since + elif (isinstance(since, float) and since > 0.0): + params['since'] = since else: raise errors.InvalidArgument( - 'since value should be datetime or positive int, ' + 'since value should be datetime or positive int/float, ' 'not {}'.format(type(since)) ) @@ -870,9 +873,11 @@ def logs(self, container, stdout=True, stderr=True, stream=False, params['until'] = utils.datetime_to_timestamp(until) elif (isinstance(until, int) and until > 0): params['until'] = until + elif (isinstance(until, float) and until > 0.0): + params['until'] = until else: raise errors.InvalidArgument( - 'until value should be datetime or positive int, ' + 'until value should be datetime or positive int/float, ' 'not {}'.format(type(until)) ) diff --git a/docker/models/containers.py b/docker/models/containers.py index 6661b213bf..4508557d28 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -290,11 +290,12 @@ def logs(self, **kwargs): tail (str or int): Output specified number of lines at the end of logs. Either an integer of number of lines or the string ``all``. Default ``all`` - since (datetime or int): Show logs since a given datetime or - integer epoch (in seconds) + since (datetime, int, or float): Show logs since a given datetime, + integer epoch (in seconds) or float (in nanoseconds) follow (bool): Follow log output. Default ``False`` - until (datetime or int): Show logs that occurred before the given - datetime or integer epoch (in seconds) + until (datetime, int, or float): Show logs that occurred before + the given datetime, integer epoch (in seconds), or + float (in nanoseconds) Returns: (generator or str): Logs from the container. diff --git a/tests/unit/api_container_test.py b/tests/unit/api_container_test.py index 8f120f4d49..d7b356c444 100644 --- a/tests/unit/api_container_test.py +++ b/tests/unit/api_container_test.py @@ -1279,6 +1279,22 @@ def test_log_since(self): stream=False ) + def test_log_since_with_float(self): + ts = 809222400.000000 + with mock.patch('docker.api.client.APIClient.inspect_container', + fake_inspect_container): + self.client.logs(fake_api.FAKE_CONTAINER_ID, stream=False, + follow=False, since=ts) + + fake_request.assert_called_with( + 'GET', + url_prefix + 'containers/' + fake_api.FAKE_CONTAINER_ID + '/logs', + params={'timestamps': 0, 'follow': 0, 'stderr': 1, 'stdout': 1, + 'tail': 'all', 'since': ts}, + timeout=DEFAULT_TIMEOUT_SECONDS, + stream=False + ) + def test_log_since_with_datetime(self): ts = 809222400 time = datetime.datetime.utcfromtimestamp(ts) @@ -1301,7 +1317,7 @@ def test_log_since_with_invalid_value_raises_error(self): fake_inspect_container): with pytest.raises(docker.errors.InvalidArgument): self.client.logs(fake_api.FAKE_CONTAINER_ID, stream=False, - follow=False, since=42.42) + follow=False, since="42.42") def test_log_tty(self): m = mock.Mock() From bc0a5fbacd7617fd338d121adca61600fc70d221 Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Wed, 5 Oct 2022 10:54:45 -0700 Subject: [PATCH 1193/1301] test: use anonymous volume for prune (#3051) This is related to https://github.com/moby/moby/pull/44216 Prunes will, by default, no longer prune named volumes, only anonymous ones. Signed-off-by: Brian Goff --- tests/integration/api_volume_test.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/integration/api_volume_test.py b/tests/integration/api_volume_test.py index 8e7dd3afb1..2085e83113 100644 --- a/tests/integration/api_volume_test.py +++ b/tests/integration/api_volume_test.py @@ -57,11 +57,10 @@ def test_force_remove_volume(self): @requires_api_version('1.25') def test_prune_volumes(self): - name = 'hopelessmasquerade' - self.client.create_volume(name) - self.tmp_volumes.append(name) + v = self.client.create_volume() + self.tmp_volumes.append(v["Name"]) result = self.client.prune_volumes() - assert name in result['VolumesDeleted'] + assert v["Name"] in result['VolumesDeleted'] def test_remove_nonexistent_volume(self): name = 'shootthebullet' From 30022984f6445fbc322cbe97bb99aab1ddb1e4fd Mon Sep 17 00:00:00 2001 From: Nick Santos Date: Wed, 2 Nov 2022 15:31:00 -0400 Subject: [PATCH 1194/1301] socket: handle npipe close on Windows (#3056) Fixes https://github.com/docker/docker-py/issues/3045 Signed-off-by: Nick Santos --- docker/utils/socket.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docker/utils/socket.py b/docker/utils/socket.py index 4a2076ec4a..5aca30b17d 100644 --- a/docker/utils/socket.py +++ b/docker/utils/socket.py @@ -18,6 +18,11 @@ class SocketError(Exception): pass +# NpipeSockets have their own error types +# pywintypes.error: (109, 'ReadFile', 'The pipe has been ended.') +NPIPE_ENDED = 109 + + def read(socket, n=4096): """ Reads at most n bytes from socket @@ -37,6 +42,15 @@ def read(socket, n=4096): except OSError as e: if e.errno not in recoverable_errors: raise + except Exception as e: + is_pipe_ended = (isinstance(socket, NpipeSocket) and + len(e.args) > 0 and + e.args[0] == NPIPE_ENDED) + if is_pipe_ended: + # npipes don't support duplex sockets, so we interpret + # a PIPE_ENDED error as a close operation (0-length read). + return 0 + raise def read_exactly(socket, n): From 8590eaad3c4b1460606763332ab84b70033ad6a1 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Tue, 15 Nov 2022 15:10:56 +0200 Subject: [PATCH 1195/1301] ci: add support for Python 3.11 (#3064) Signed-off-by: Hugo van Kemenade --- .github/workflows/ci.yml | 2 +- .github/workflows/release.yml | 2 +- setup.py | 1 + tox.ini | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d1634125bc..f23873f0e8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,7 +21,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11.0-alpha - 3.11.0"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index dde656c033..7c6358a225 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -20,7 +20,7 @@ jobs: - uses: actions/setup-python@v4 with: - python-version: '3.10' + python-version: '3.x' - name: Generate Pacakge run: | diff --git a/setup.py b/setup.py index 68f7c27410..ff6da71419 100644 --- a/setup.py +++ b/setup.py @@ -71,6 +71,7 @@ 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', 'Topic :: Software Development', 'Topic :: Utilities', 'License :: OSI Approved :: Apache Software License', diff --git a/tox.ini b/tox.ini index d35d41ae89..9edc15c54e 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py36, py37, flake8 +envlist = py{37,38,39,310,311}, flake8 skipsdist=True [testenv] From 82cf559b5a641f53e9035b44b91f829f3b4cca80 Mon Sep 17 00:00:00 2001 From: loicleyendecker Date: Fri, 2 Dec 2022 19:48:04 +0000 Subject: [PATCH 1196/1301] volume: do not strip trailing characters from names (#3073) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Only remove `:ro` or `:rw` suffixes in their entirety; do not strip arbitrary `r` / `o` / `w` / `:` characters individually. Signed-off-by: Loïc Leyendecker --- docker/models/containers.py | 4 +++- tests/unit/models_containers_test.py | 6 +++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/docker/models/containers.py b/docker/models/containers.py index 4508557d28..61d048c4fe 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -1147,8 +1147,10 @@ def _host_volume_from_bind(bind): bits = rest.split(':', 1) if len(bits) == 1 or bits[1] in ('ro', 'rw'): return drive + bits[0] + elif bits[1].endswith(':ro') or bits[1].endswith(':rw'): + return bits[1][:-3] else: - return bits[1].rstrip(':ro').rstrip(':rw') + return bits[1] ExecResult = namedtuple('ExecResult', 'exit_code,output') diff --git a/tests/unit/models_containers_test.py b/tests/unit/models_containers_test.py index 101708ebb7..51f0018029 100644 --- a/tests/unit/models_containers_test.py +++ b/tests/unit/models_containers_test.py @@ -103,7 +103,7 @@ def test_create_container_args(self): volumes=[ '/home/user1/:/mnt/vol2', '/var/www:/mnt/vol1:ro', - 'volumename:/mnt/vol3', + 'volumename:/mnt/vol3r', '/volumewithnohostpath', '/anothervolumewithnohostpath:ro', 'C:\\windows\\path:D:\\hello\\world:rw' @@ -123,7 +123,7 @@ def test_create_container_args(self): 'Binds': [ '/home/user1/:/mnt/vol2', '/var/www:/mnt/vol1:ro', - 'volumename:/mnt/vol3', + 'volumename:/mnt/vol3r', '/volumewithnohostpath', '/anothervolumewithnohostpath:ro', 'C:\\windows\\path:D:\\hello\\world:rw' @@ -198,7 +198,7 @@ def test_create_container_args(self): volumes=[ '/mnt/vol2', '/mnt/vol1', - '/mnt/vol3', + '/mnt/vol3r', '/volumewithnohostpath', '/anothervolumewithnohostpath', 'D:\\hello\\world' From 3afb4b61c35678ae9b9986988922305ebb521018 Mon Sep 17 00:00:00 2001 From: Maxim Mironyuk Date: Wed, 11 Jan 2023 00:58:51 +0300 Subject: [PATCH 1197/1301] docs: fix wrong command syntax in code annotation (#3081) Signed-off-by: Maxim Mironyuk --- docker/models/networks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/models/networks.py b/docker/models/networks.py index 093deb7fe3..f502879070 100644 --- a/docker/models/networks.py +++ b/docker/models/networks.py @@ -184,7 +184,7 @@ def get(self, network_id, *args, **kwargs): def list(self, *args, **kwargs): """ - List networks. Similar to the ``docker networks ls`` command. + List networks. Similar to the ``docker network ls`` command. Args: names (:py:class:`list`): List of names to filter by. From d38b41a13c05530eca34691ffd4c70236f8f0d5c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 10 Jan 2023 17:25:12 -0500 Subject: [PATCH 1198/1301] build(deps): Bump setuptools from 63.2.0 to 65.5.1 (#3082) Bumps [setuptools](https://github.com/pypa/setuptools) from 63.2.0 to 65.5.1. - [Release notes](https://github.com/pypa/setuptools/releases) - [Changelog](https://github.com/pypa/setuptools/blob/main/CHANGES.rst) - [Commits](https://github.com/pypa/setuptools/compare/v63.2.0...v65.5.1) --- updated-dependencies: - dependency-name: setuptools dependency-type: direct:production ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 979b291cf7..b7457fa773 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,4 +1,4 @@ -setuptools==63.2.0 +setuptools==65.5.1 coverage==6.4.2 flake8==4.0.1 pytest==7.1.2 From 22718ba59a193263bed8c52cc1abd5ee52358440 Mon Sep 17 00:00:00 2001 From: Yanlong Wang Date: Wed, 11 Jan 2023 06:45:25 +0800 Subject: [PATCH 1199/1301] fix(store): warn on init instead of throw (#3080) Signed-off-by: yanlong.wang --- docker/credentials/store.py | 9 ++++++++- tests/integration/credentials/store_test.py | 7 +++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/docker/credentials/store.py b/docker/credentials/store.py index 297f46841c..b7ab53fbad 100644 --- a/docker/credentials/store.py +++ b/docker/credentials/store.py @@ -2,6 +2,7 @@ import json import shutil import subprocess +import warnings from . import constants from . import errors @@ -18,7 +19,7 @@ def __init__(self, program, environment=None): self.exe = shutil.which(self.program) self.environment = environment if self.exe is None: - raise errors.InitializationError( + warnings.warn( '{} not installed or not available in PATH'.format( self.program ) @@ -70,6 +71,12 @@ def list(self): return json.loads(data.decode('utf-8')) def _execute(self, subcmd, data_input): + if self.exe is None: + raise errors.StoreError( + '{} not installed or not available in PATH'.format( + self.program + ) + ) output = None env = create_environment_dict(self.environment) try: diff --git a/tests/integration/credentials/store_test.py b/tests/integration/credentials/store_test.py index 213cf305e8..16f4d60ab4 100644 --- a/tests/integration/credentials/store_test.py +++ b/tests/integration/credentials/store_test.py @@ -84,3 +84,10 @@ def test_execute_with_env_override(self): data = self.store._execute('--null', '') assert b'\0FOO=bar\0' in data assert 'FOO' not in os.environ + + def test_unavailable_store(self): + some_unavailable_store = None + with pytest.warns(UserWarning): + some_unavailable_store = Store('that-does-not-exist') + with pytest.raises(StoreError): + some_unavailable_store.get('anything-this-does-not-matter') From 34e6829dd40e99e9ba47ea02fcfabda09e08d36e Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Fri, 13 Jan 2023 21:41:01 +0100 Subject: [PATCH 1200/1301] exec: fix file handle leak with container.exec_* APIs (#2320) Requests with stream=True MUST be closed or else the connection will never be returned to the connection pool. Both ContainerApiMixin.attach and ExecApiMixin.exec_start were leaking in the stream=False case. exec_start was modified to follow attach for the stream=True case as that allows the caller to close the stream when done (untested). Tested with: # Test exec_run (stream=False) - observe one less leak make integration-test-py3 file=models_containers_test.py' -k test_exec_run_success -vs -W error::ResourceWarning' # Test exec_start (stream=True, fully reads from CancellableStream) make integration-test-py3 file=api_exec_test.py' -k test_execute_command -vs -W error::ResourceWarning' After this change, one resource leak is removed, the remaining resource leaks occur because none of the tests call client.close(). Fixes https://github.com/docker/docker-py/issues/1293 (Regression from https://github.com/docker/docker-py/pull/1130) Signed-off-by: Peter Wu Co-authored-by: Milas Bowman --- docker/api/client.py | 11 +++++++++-- docker/api/exec_api.py | 18 ++++++++++++++---- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/docker/api/client.py b/docker/api/client.py index 7733d33438..65b9d9d198 100644 --- a/docker/api/client.py +++ b/docker/api/client.py @@ -406,6 +406,10 @@ def _stream_raw_result(self, response, chunk_size=1, decode=True): yield from response.iter_content(chunk_size, decode) def _read_from_socket(self, response, stream, tty=True, demux=False): + """Consume all data from the socket, close the response and return the + data. If stream=True, then a generator is returned instead and the + caller is responsible for closing the response. + """ socket = self._get_raw_response_socket(response) gen = frames_iter(socket, tty) @@ -420,8 +424,11 @@ def _read_from_socket(self, response, stream, tty=True, demux=False): if stream: return gen else: - # Wait for all the frames, concatenate them, and return the result - return consume_socket_output(gen, demux=demux) + try: + # Wait for all frames, concatenate them, and return the result + return consume_socket_output(gen, demux=demux) + finally: + response.close() def _disable_socket_timeout(self, socket): """ Depending on the combination of python version and whether we're diff --git a/docker/api/exec_api.py b/docker/api/exec_api.py index 496308a0f1..63df9e6c6a 100644 --- a/docker/api/exec_api.py +++ b/docker/api/exec_api.py @@ -1,5 +1,6 @@ from .. import errors from .. import utils +from ..types import CancellableStream class ExecApiMixin: @@ -125,9 +126,10 @@ def exec_start(self, exec_id, detach=False, tty=False, stream=False, detach (bool): If true, detach from the exec command. Default: False tty (bool): Allocate a pseudo-TTY. Default: False - stream (bool): Stream response data. Default: False + stream (bool): Return response data progressively as an iterator + of strings, rather than a single string. socket (bool): Return the connection socket to allow custom - read/write operations. + read/write operations. Must be closed by the caller when done. demux (bool): Return stdout and stderr separately Returns: @@ -161,7 +163,15 @@ def exec_start(self, exec_id, detach=False, tty=False, stream=False, stream=True ) if detach: - return self._result(res) + try: + return self._result(res) + finally: + res.close() if socket: return self._get_raw_response_socket(res) - return self._read_from_socket(res, stream, tty=tty, demux=demux) + + output = self._read_from_socket(res, stream, tty=tty, demux=demux) + if stream: + return CancellableStream(output, res) + else: + return output From ee9151f33611f693958a29c38ab0fbf5b6741c1b Mon Sep 17 00:00:00 2001 From: Mariano Scazzariello Date: Fri, 27 Jan 2023 15:26:21 +0100 Subject: [PATCH 1201/1301] client: add `network_driver_opt` to container run and create (#3083) Signed-off-by: Mariano Scazzariello --- docker/models/containers.py | 16 +++++- tests/unit/models_containers_test.py | 84 +++++++++++++++++++++++++++- 2 files changed, 98 insertions(+), 2 deletions(-) diff --git a/docker/models/containers.py b/docker/models/containers.py index 6661b213bf..f1f6e44891 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -678,6 +678,10 @@ def run(self, image, command=None, stdout=True, stderr=False, This mode is incompatible with ``ports``. Incompatible with ``network``. + network_driver_opt (dict): A dictionary of options to provide + to the network driver. Defaults to ``None``. Used in + conjuction with ``network``. Incompatible + with ``network_mode``. oom_kill_disable (bool): Whether to disable OOM killer. oom_score_adj (int): An integer value containing the score given to the container in order to tune OOM killer preferences. @@ -842,6 +846,12 @@ def run(self, image, command=None, stdout=True, stderr=False, 'together.' ) + if kwargs.get('network_driver_opt') and not kwargs.get('network'): + raise RuntimeError( + 'The options "network_driver_opt" can not be used ' + 'without "network".' + ) + try: container = self.create(image=image, command=command, detach=detach, **kwargs) @@ -1112,8 +1122,12 @@ def _create_container_args(kwargs): host_config_kwargs['binds'] = volumes network = kwargs.pop('network', None) + network_driver_opt = kwargs.pop('network_driver_opt', None) if network: - create_kwargs['networking_config'] = {network: None} + network_configuration = {'driver_opt': network_driver_opt} \ + if network_driver_opt else None + + create_kwargs['networking_config'] = {network: network_configuration} host_config_kwargs['network_mode'] = network # All kwargs should have been consumed by this point, so raise diff --git a/tests/unit/models_containers_test.py b/tests/unit/models_containers_test.py index 101708ebb7..d2a7ea52cb 100644 --- a/tests/unit/models_containers_test.py +++ b/tests/unit/models_containers_test.py @@ -74,6 +74,7 @@ def test_create_container_args(self): name='somename', network_disabled=False, network='foo', + network_driver_opt={'key1': 'a'}, oom_kill_disable=True, oom_score_adj=5, pid_mode='host', @@ -188,7 +189,7 @@ def test_create_container_args(self): mac_address='abc123', name='somename', network_disabled=False, - networking_config={'foo': None}, + networking_config={'foo': {'driver_opt': {'key1': 'a'}}}, platform='linux', ports=[('1111', 'tcp'), ('2222', 'tcp')], stdin_open=True, @@ -345,6 +346,42 @@ def test_run_platform(self): host_config={'NetworkMode': 'default'}, ) + def test_run_network_driver_opts_without_network(self): + client = make_fake_client() + + with pytest.raises(RuntimeError): + client.containers.run( + image='alpine', + network_driver_opt={'key1': 'a'} + ) + + def test_run_network_driver_opts_with_network_mode(self): + client = make_fake_client() + + with pytest.raises(RuntimeError): + client.containers.run( + image='alpine', + network_mode='none', + network_driver_opt={'key1': 'a'} + ) + + def test_run_network_driver_opts(self): + client = make_fake_client() + + client.containers.run( + image='alpine', + network='foo', + network_driver_opt={'key1': 'a'} + ) + + client.api.create_container.assert_called_with( + detach=False, + image='alpine', + command=None, + networking_config={'foo': {'driver_opt': {'key1': 'a'}}}, + host_config={'NetworkMode': 'foo'} + ) + def test_create(self): client = make_fake_client() container = client.containers.create( @@ -372,6 +409,51 @@ def test_create_with_image_object(self): host_config={'NetworkMode': 'default'} ) + def test_create_network_driver_opts_without_network(self): + client = make_fake_client() + + client.containers.create( + image='alpine', + network_driver_opt={'key1': 'a'} + ) + + client.api.create_container.assert_called_with( + image='alpine', + command=None, + host_config={'NetworkMode': 'default'} + ) + + def test_create_network_driver_opts_with_network_mode(self): + client = make_fake_client() + + client.containers.create( + image='alpine', + network_mode='none', + network_driver_opt={'key1': 'a'} + ) + + client.api.create_container.assert_called_with( + image='alpine', + command=None, + host_config={'NetworkMode': 'none'} + ) + + def test_create_network_driver_opts(self): + client = make_fake_client() + + client.containers.create( + image='alpine', + network='foo', + network_driver_opt={'key1': 'a'} + ) + + client.api.create_container.assert_called_with( + image='alpine', + command=None, + networking_config={'foo': {'driver_opt': {'key1': 'a'}}}, + host_config={'NetworkMode': 'foo'} + ) + def test_get(self): client = make_fake_client() container = client.containers.get(FAKE_CONTAINER_ID) From e9d4ddfaece229fbd2df1a1bfa3f4f513f33d1ac Mon Sep 17 00:00:00 2001 From: Andy Roxby <107427605+aroxby-wayscript@users.noreply.github.com> Date: Thu, 16 Feb 2023 10:27:45 -0500 Subject: [PATCH 1202/1301] api: add `one-shot` option to container `stats` (#3089) Signed-off-by: Andy Roxby <107427605+aroxby-wayscript@users.noreply.github.com> --- docker/api/container.py | 24 ++++++++++++++++++++---- tests/unit/api_container_test.py | 13 ++++++++++++- 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/docker/api/container.py b/docker/api/container.py index ce483710cb..82a892288b 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -1126,7 +1126,7 @@ def start(self, container, *args, **kwargs): self._raise_for_status(res) @utils.check_resource('container') - def stats(self, container, decode=None, stream=True): + def stats(self, container, decode=None, stream=True, one_shot=None): """ Stream statistics for a specific container. Similar to the ``docker stats`` command. @@ -1138,6 +1138,9 @@ def stats(self, container, decode=None, stream=True): False by default. stream (bool): If set to false, only the current stats will be returned instead of a stream. True by default. + one_shot (bool): If set to true, Only get a single stat instead of + waiting for 2 cycles. Must be used with stream=false. False by + default. Raises: :py:class:`docker.errors.APIError` @@ -1145,16 +1148,29 @@ def stats(self, container, decode=None, stream=True): """ url = self._url("/containers/{0}/stats", container) + params = { + 'stream': stream + } + if one_shot is not None: + if utils.version_lt(self._version, '1.41'): + raise errors.InvalidVersion( + 'one_shot is not supported for API version < 1.41' + ) + params['one-shot'] = one_shot if stream: - return self._stream_helper(self._get(url, stream=True), + if one_shot: + raise errors.InvalidArgument( + 'one_shot is only available in conjunction with ' + 'stream=False' + ) + return self._stream_helper(self._get(url, params=params), decode=decode) else: if decode: raise errors.InvalidArgument( "decode is only available in conjunction with stream=True" ) - return self._result(self._get(url, params={'stream': False}), - json=True) + return self._result(self._get(url, params=params), json=True) @utils.check_resource('container') def stop(self, container, timeout=None): diff --git a/tests/unit/api_container_test.py b/tests/unit/api_container_test.py index d7b356c444..c605da371c 100644 --- a/tests/unit/api_container_test.py +++ b/tests/unit/api_container_test.py @@ -1529,7 +1529,18 @@ def test_container_stats(self): 'GET', url_prefix + 'containers/' + fake_api.FAKE_CONTAINER_ID + '/stats', timeout=60, - stream=True + params={'stream': True} + ) + + def test_container_stats_with_one_shot(self): + self.client.stats( + fake_api.FAKE_CONTAINER_ID, stream=False, one_shot=True) + + fake_request.assert_called_with( + 'GET', + url_prefix + 'containers/' + fake_api.FAKE_CONTAINER_ID + '/stats', + timeout=60, + params={'stream': False, 'one-shot': True} ) def test_container_top(self): From 7cd7458f2f41ef5af58589f28307a064936d7be1 Mon Sep 17 00:00:00 2001 From: Lorin Bucher Date: Thu, 16 Feb 2023 16:38:52 +0100 Subject: [PATCH 1203/1301] api: add `status` parameter to services list (#3093) Signed-off-by: Lorin Bucher --- docker/api/service.py | 10 +++++++++- docker/models/services.py | 2 ++ tests/integration/api_service_test.py | 14 ++++++++++++++ 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/docker/api/service.py b/docker/api/service.py index 371f541e11..652b7c2458 100644 --- a/docker/api/service.py +++ b/docker/api/service.py @@ -262,7 +262,7 @@ def remove_service(self, service): return True @utils.minimum_version('1.24') - def services(self, filters=None): + def services(self, filters=None, status=None): """ List services. @@ -270,6 +270,8 @@ def services(self, filters=None): filters (dict): Filters to process on the nodes list. Valid filters: ``id``, ``name`` , ``label`` and ``mode``. Default: ``None``. + status (bool): Include the service task count of running and + desired tasks. Default: ``None``. Returns: A list of dictionaries containing data about each service. @@ -281,6 +283,12 @@ def services(self, filters=None): params = { 'filters': utils.convert_filters(filters) if filters else None } + if status is not None: + if utils.version_lt(self._version, '1.41'): + raise errors.InvalidVersion( + 'status is not supported in API version < 1.41' + ) + params['status'] = status url = self._url('/services') return self._result(self._get(url, params=params), True) diff --git a/docker/models/services.py b/docker/models/services.py index 06438748f3..70037041a3 100644 --- a/docker/models/services.py +++ b/docker/models/services.py @@ -266,6 +266,8 @@ def list(self, **kwargs): filters (dict): Filters to process on the nodes list. Valid filters: ``id``, ``name`` , ``label`` and ``mode``. Default: ``None``. + status (bool): Include the service task count of running and + desired tasks. Default: ``None``. Returns: list of :py:class:`Service`: The services. diff --git a/tests/integration/api_service_test.py b/tests/integration/api_service_test.py index 8ce7c9d57e..dec3fa0071 100644 --- a/tests/integration/api_service_test.py +++ b/tests/integration/api_service_test.py @@ -85,6 +85,20 @@ def test_list_services_filter_by_label(self): assert len(test_services) == 1 assert test_services[0]['Spec']['Labels']['test_label'] == 'testing' + @requires_api_version('1.41') + def test_list_services_with_status(self): + test_services = self.client.services() + assert len(test_services) == 0 + self.create_simple_service() + test_services = self.client.services( + filters={'name': 'dockerpytest_'}, status=False + ) + assert 'ServiceStatus' not in test_services[0] + test_services = self.client.services( + filters={'name': 'dockerpytest_'}, status=True + ) + assert 'ServiceStatus' in test_services[0] + def test_inspect_service_by_id(self): svc_name, svc_id = self.create_simple_service() svc_info = self.client.inspect_service(svc_id) From f84623225e0227adde48ffd8f2e1cafb1f9aa38d Mon Sep 17 00:00:00 2001 From: Milas Bowman Date: Wed, 22 Feb 2023 12:00:47 -0500 Subject: [PATCH 1204/1301] socket: fix for errors on pipe close in Windows (#3099) Need to return data, not size. By returning an empty string, EOF will be detected properly since `len()` will be `0`. Fixes #3098. Signed-off-by: Milas Bowman --- docker/utils/socket.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/utils/socket.py b/docker/utils/socket.py index 5aca30b17d..47cb44f62f 100644 --- a/docker/utils/socket.py +++ b/docker/utils/socket.py @@ -49,7 +49,7 @@ def read(socket, n=4096): if is_pipe_ended: # npipes don't support duplex sockets, so we interpret # a PIPE_ENDED error as a close operation (0-length read). - return 0 + return '' raise From aaf68b7f98df7f886778395112267b9b0f6140bc Mon Sep 17 00:00:00 2001 From: Aarni Koskela Date: Wed, 22 Feb 2023 21:05:19 +0200 Subject: [PATCH 1205/1301] api: note the data arg may also be a stream in `put_archive` (#2478) Signed-off-by: Aarni Koskela --- docker/api/container.py | 2 +- docker/models/containers.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/api/container.py b/docker/api/container.py index 82a892288b..9a25b21489 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -966,7 +966,7 @@ def put_archive(self, container, path, data): container (str): The container where the file(s) will be extracted path (str): Path inside the container where the file(s) will be extracted. Must exist. - data (bytes): tar data to be extracted + data (bytes or stream): tar data to be extracted Returns: (bool): True if the call succeeds. diff --git a/docker/models/containers.py b/docker/models/containers.py index c718bbeac6..f451cf3fe7 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -324,7 +324,7 @@ def put_archive(self, path, data): Args: path (str): Path inside the container where the file(s) will be extracted. Must exist. - data (bytes): tar data to be extracted + data (bytes or stream): tar data to be extracted Returns: (bool): True if the call succeeds. From a02ba743338c27fd9348af2cf7767b140501734d Mon Sep 17 00:00:00 2001 From: I-question-this Date: Fri, 21 Apr 2023 16:53:58 -0500 Subject: [PATCH 1206/1301] socket: use poll() instead of select() except on Windows (#2865) Fixes #2278, which was originally addressed in #2279, but was not properly merged. Additionally it did not address the problem of poll not existing on Windows. This patch falls back on the more limited select method if host system is Windows. Signed-off-by: Tyler Westland --- docker/utils/socket.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docker/utils/socket.py b/docker/utils/socket.py index 47cb44f62f..3c31a98c5d 100644 --- a/docker/utils/socket.py +++ b/docker/utils/socket.py @@ -3,6 +3,7 @@ import select import socket as pysocket import struct +import sys try: from ..transport import NpipeSocket @@ -31,7 +32,13 @@ def read(socket, n=4096): recoverable_errors = (errno.EINTR, errno.EDEADLK, errno.EWOULDBLOCK) if not isinstance(socket, NpipeSocket): - select.select([socket], [], []) + if sys.platform == 'win32': + # Limited to 1024 + select.select([socket], [], []) + else: + poll = select.poll() + poll.register(socket) + poll.poll() try: if hasattr(socket, 'recv'): From 3178c8d48b7e7414d1f864ba1fe77139f4e923eb Mon Sep 17 00:00:00 2001 From: Felix Fontein Date: Fri, 5 May 2023 17:39:31 +0200 Subject: [PATCH 1207/1301] =?UTF-8?q?deps:=20compatiblity=20with=20request?= =?UTF-8?q?s=20=E2=89=A5=202.29.0=20and=20urllib3=202.x=20(#3116)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Requirements are the same, so it's still possible to use `urllib3 < 2` or `requests == 2.28.2` for example. Signed-off-by: Felix Fontein --- docker/transport/npipeconn.py | 10 +++------- docker/transport/sshconn.py | 10 +++------- docker/transport/ssladapter.py | 5 +---- docker/transport/unixconn.py | 15 +++------------ docker/types/daemon.py | 5 +---- tests/unit/api_test.py | 2 +- 6 files changed, 12 insertions(+), 35 deletions(-) diff --git a/docker/transport/npipeconn.py b/docker/transport/npipeconn.py index 87033cf2af..45988b2df1 100644 --- a/docker/transport/npipeconn.py +++ b/docker/transport/npipeconn.py @@ -5,17 +5,13 @@ from .. import constants from .npipesocket import NpipeSocket -import http.client as httplib - -try: - import requests.packages.urllib3 as urllib3 -except ImportError: - import urllib3 +import urllib3 +import urllib3.connection RecentlyUsedContainer = urllib3._collections.RecentlyUsedContainer -class NpipeHTTPConnection(httplib.HTTPConnection): +class NpipeHTTPConnection(urllib3.connection.HTTPConnection): def __init__(self, npipe_path, timeout=60): super().__init__( 'localhost', timeout=timeout diff --git a/docker/transport/sshconn.py b/docker/transport/sshconn.py index 7421f33bdc..a92beb621f 100644 --- a/docker/transport/sshconn.py +++ b/docker/transport/sshconn.py @@ -11,12 +11,8 @@ from docker.transport.basehttpadapter import BaseHTTPAdapter from .. import constants -import http.client as httplib - -try: - import requests.packages.urllib3 as urllib3 -except ImportError: - import urllib3 +import urllib3 +import urllib3.connection RecentlyUsedContainer = urllib3._collections.RecentlyUsedContainer @@ -99,7 +95,7 @@ def close(self): self.proc.terminate() -class SSHConnection(httplib.HTTPConnection): +class SSHConnection(urllib3.connection.HTTPConnection): def __init__(self, ssh_transport=None, timeout=60, host=None): super().__init__( 'localhost', timeout=timeout diff --git a/docker/transport/ssladapter.py b/docker/transport/ssladapter.py index 6aa80037d7..69274bd1dd 100644 --- a/docker/transport/ssladapter.py +++ b/docker/transport/ssladapter.py @@ -7,10 +7,7 @@ from docker.transport.basehttpadapter import BaseHTTPAdapter -try: - import requests.packages.urllib3 as urllib3 -except ImportError: - import urllib3 +import urllib3 PoolManager = urllib3.poolmanager.PoolManager diff --git a/docker/transport/unixconn.py b/docker/transport/unixconn.py index 1b00762a60..fae10f2664 100644 --- a/docker/transport/unixconn.py +++ b/docker/transport/unixconn.py @@ -1,20 +1,17 @@ import requests.adapters import socket -import http.client as httplib from docker.transport.basehttpadapter import BaseHTTPAdapter from .. import constants -try: - import requests.packages.urllib3 as urllib3 -except ImportError: - import urllib3 +import urllib3 +import urllib3.connection RecentlyUsedContainer = urllib3._collections.RecentlyUsedContainer -class UnixHTTPConnection(httplib.HTTPConnection): +class UnixHTTPConnection(urllib3.connection.HTTPConnection): def __init__(self, base_url, unix_socket, timeout=60): super().__init__( @@ -30,12 +27,6 @@ def connect(self): sock.connect(self.unix_socket) self.sock = sock - def putheader(self, header, *values): - super().putheader(header, *values) - - def response_class(self, sock, *args, **kwargs): - return httplib.HTTPResponse(sock, *args, **kwargs) - class UnixHTTPConnectionPool(urllib3.connectionpool.HTTPConnectionPool): def __init__(self, base_url, socket_path, timeout=60, maxsize=10): diff --git a/docker/types/daemon.py b/docker/types/daemon.py index 10e8101447..096b2cc169 100644 --- a/docker/types/daemon.py +++ b/docker/types/daemon.py @@ -1,9 +1,6 @@ import socket -try: - import requests.packages.urllib3 as urllib3 -except ImportError: - import urllib3 +import urllib3 from ..errors import DockerException diff --git a/tests/unit/api_test.py b/tests/unit/api_test.py index a2348f08ba..4b6099c904 100644 --- a/tests/unit/api_test.py +++ b/tests/unit/api_test.py @@ -16,9 +16,9 @@ import docker import pytest import requests +import urllib3 from docker.api import APIClient from docker.constants import DEFAULT_DOCKER_API_VERSION -from requests.packages import urllib3 from unittest import mock from . import fake_api From 576e47aaacf690a3fdd6cf98c345d48ecf834b7d Mon Sep 17 00:00:00 2001 From: John Yang Date: Fri, 5 May 2023 13:21:46 -0700 Subject: [PATCH 1208/1301] api: update return type of `diff` method (#3115) Signed-off-by: John Yang --- docker/api/container.py | 3 ++- docker/models/containers.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docker/api/container.py b/docker/api/container.py index 9a25b21489..fef760300e 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -678,7 +678,8 @@ def diff(self, container): container (str): The container to diff Returns: - (str) + (list) A list of dictionaries containing the attributes `Path` + and `Kind`. Raises: :py:class:`docker.errors.APIError` diff --git a/docker/models/containers.py b/docker/models/containers.py index f451cf3fe7..2eeefda1ee 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -141,7 +141,8 @@ def diff(self): Inspect changes on a container's filesystem. Returns: - (str) + (list) A list of dictionaries containing the attributes `Path` + and `Kind`. Raises: :py:class:`docker.errors.APIError` From 1d697680d2286153448fb6fe6085251aeb3fad1e Mon Sep 17 00:00:00 2001 From: Mariano Scazzariello Date: Sat, 6 May 2023 13:49:01 +0200 Subject: [PATCH 1209/1301] Full support to networking config during container creation Signed-off-by: Mariano Scazzariello --- docker/models/containers.py | 44 +++++++++++++++++++++++++++++-------- 1 file changed, 35 insertions(+), 9 deletions(-) diff --git a/docker/models/containers.py b/docker/models/containers.py index 2eeefda1ee..5ba6297bd5 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -8,7 +8,7 @@ ContainerError, DockerException, ImageNotFound, NotFound, create_unexpected_kwargs_error ) -from ..types import HostConfig +from ..types import EndpointConfig, HostConfig, NetworkingConfig from ..utils import version_gte from .images import Image from .resource import Collection, Model @@ -680,10 +680,32 @@ def run(self, image, command=None, stdout=True, stderr=False, This mode is incompatible with ``ports``. Incompatible with ``network``. - network_driver_opt (dict): A dictionary of options to provide - to the network driver. Defaults to ``None``. Used in - conjuction with ``network``. Incompatible - with ``network_mode``. + network_config (dict): A dictionary containing options that are + passed to the network driver during the connection. + Defaults to ``None``. + The dictionary contains the following keys: + + - ``aliases`` (:py:class:`list`): A list of aliases for + the network endpoint. + Names in that list can be used within the network to + reach this container. Defaults to ``None``. + - ``links`` (:py:class:`list`): A list of links for + the network endpoint endpoint. + Containers declared in this list will be linked to this + container. Defaults to ``None``. + - ``ipv4_address`` (str): The IP address to assign to + this container on the network, using the IPv4 protocol. + Defaults to ``None``. + - ``ipv6_address`` (str): The IP address to assign to + this container on the network, using the IPv6 protocol. + Defaults to ``None``. + - ``link_local_ips`` (:py:class:`list`): A list of link-local + (IPv4/IPv6) addresses. + - ``driver_opt`` (dict): A dictionary of options to provide to + the network driver. Defaults to ``None``. + + Used in conjuction with ``network``. + Incompatible with ``network_mode``. oom_kill_disable (bool): Whether to disable OOM killer. oom_score_adj (int): An integer value containing the score given to the container in order to tune OOM killer preferences. @@ -1124,12 +1146,16 @@ def _create_container_args(kwargs): host_config_kwargs['binds'] = volumes network = kwargs.pop('network', None) - network_driver_opt = kwargs.pop('network_driver_opt', None) + network_config = kwargs.pop('network_config', None) if network: - network_configuration = {'driver_opt': network_driver_opt} \ - if network_driver_opt else None + network_configuration = EndpointConfig( + host_config_kwargs['version'], + **network_config + ) if network_config else None - create_kwargs['networking_config'] = {network: network_configuration} + create_kwargs['networking_config'] = NetworkingConfig( + {network: network_configuration} + ) host_config_kwargs['network_mode'] = network # All kwargs should have been consumed by this point, so raise From a662d5a3051e49ac12caef967245d9e718eb1cb3 Mon Sep 17 00:00:00 2001 From: Mariano Scazzariello Date: Sun, 7 May 2023 11:42:23 +0200 Subject: [PATCH 1210/1301] Fix pytests Signed-off-by: Mariano Scazzariello --- docker/models/containers.py | 4 +- tests/unit/models_containers_test.py | 55 +++++++++++++++++----------- 2 files changed, 36 insertions(+), 23 deletions(-) diff --git a/docker/models/containers.py b/docker/models/containers.py index 5ba6297bd5..1d2e58c641 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -870,9 +870,9 @@ def run(self, image, command=None, stdout=True, stderr=False, 'together.' ) - if kwargs.get('network_driver_opt') and not kwargs.get('network'): + if kwargs.get('network_config') and not kwargs.get('network'): raise RuntimeError( - 'The options "network_driver_opt" can not be used ' + 'The option "network_config" can not be used ' 'without "network".' ) diff --git a/tests/unit/models_containers_test.py b/tests/unit/models_containers_test.py index 0592af5e04..240b592fb6 100644 --- a/tests/unit/models_containers_test.py +++ b/tests/unit/models_containers_test.py @@ -1,12 +1,12 @@ +import pytest +import unittest + import docker -from docker.constants import DEFAULT_DATA_CHUNK_SIZE +from docker.constants import DEFAULT_DATA_CHUNK_SIZE, DEFAULT_DOCKER_API_VERSION from docker.models.containers import Container, _create_container_args from docker.models.images import Image -import unittest - from .fake_api import FAKE_CONTAINER_ID, FAKE_IMAGE_ID, FAKE_EXEC_ID from .fake_api_client import make_fake_client -import pytest class ContainerCollectionTest(unittest.TestCase): @@ -74,7 +74,7 @@ def test_create_container_args(self): name='somename', network_disabled=False, network='foo', - network_driver_opt={'key1': 'a'}, + network_config={'aliases': ['test'], 'driver_opt': {'key1': 'a'}}, oom_kill_disable=True, oom_score_adj=5, pid_mode='host', @@ -99,7 +99,7 @@ def test_create_container_args(self): user='bob', userns_mode='host', uts_mode='host', - version='1.23', + version=DEFAULT_DOCKER_API_VERSION, volume_driver='some_driver', volumes=[ '/home/user1/:/mnt/vol2', @@ -189,7 +189,9 @@ def test_create_container_args(self): mac_address='abc123', name='somename', network_disabled=False, - networking_config={'foo': {'driver_opt': {'key1': 'a'}}}, + networking_config={'EndpointsConfig': { + 'foo': {'Aliases': ['test'], 'DriverOpts': {'key1': 'a'}}} + }, platform='linux', ports=[('1111', 'tcp'), ('2222', 'tcp')], stdin_open=True, @@ -346,39 +348,44 @@ def test_run_platform(self): host_config={'NetworkMode': 'default'}, ) - def test_run_network_driver_opts_without_network(self): + def test_run_network_config_without_network(self): client = make_fake_client() with pytest.raises(RuntimeError): client.containers.run( image='alpine', - network_driver_opt={'key1': 'a'} + network_config={'aliases': ['test'], + 'driver_opt': {'key1': 'a'}} ) - def test_run_network_driver_opts_with_network_mode(self): + def test_run_network_config_with_network_mode(self): client = make_fake_client() with pytest.raises(RuntimeError): client.containers.run( image='alpine', network_mode='none', - network_driver_opt={'key1': 'a'} + network_config={'aliases': ['test'], + 'driver_opt': {'key1': 'a'}} ) - def test_run_network_driver_opts(self): + def test_run_network_config(self): client = make_fake_client() client.containers.run( image='alpine', network='foo', - network_driver_opt={'key1': 'a'} + network_config={'aliases': ['test'], + 'driver_opt': {'key1': 'a'}} ) client.api.create_container.assert_called_with( detach=False, image='alpine', command=None, - networking_config={'foo': {'driver_opt': {'key1': 'a'}}}, + networking_config={'EndpointsConfig': { + 'foo': {'Aliases': ['test'], 'DriverOpts': {'key1': 'a'}}} + }, host_config={'NetworkMode': 'foo'} ) @@ -409,12 +416,13 @@ def test_create_with_image_object(self): host_config={'NetworkMode': 'default'} ) - def test_create_network_driver_opts_without_network(self): + def test_create_network_config_without_network(self): client = make_fake_client() client.containers.create( image='alpine', - network_driver_opt={'key1': 'a'} + network_config={'aliases': ['test'], + 'driver_opt': {'key1': 'a'}} ) client.api.create_container.assert_called_with( @@ -423,13 +431,14 @@ def test_create_network_driver_opts_without_network(self): host_config={'NetworkMode': 'default'} ) - def test_create_network_driver_opts_with_network_mode(self): + def test_create_network_config_with_network_mode(self): client = make_fake_client() client.containers.create( image='alpine', network_mode='none', - network_driver_opt={'key1': 'a'} + network_config={'aliases': ['test'], + 'driver_opt': {'key1': 'a'}} ) client.api.create_container.assert_called_with( @@ -438,19 +447,22 @@ def test_create_network_driver_opts_with_network_mode(self): host_config={'NetworkMode': 'none'} ) - def test_create_network_driver_opts(self): + def test_create_network_config(self): client = make_fake_client() client.containers.create( image='alpine', network='foo', - network_driver_opt={'key1': 'a'} + network_config={'aliases': ['test'], + 'driver_opt': {'key1': 'a'}} ) client.api.create_container.assert_called_with( image='alpine', command=None, - networking_config={'foo': {'driver_opt': {'key1': 'a'}}}, + networking_config={'EndpointsConfig': { + 'foo': {'Aliases': ['test'], 'DriverOpts': {'key1': 'a'}}} + }, host_config={'NetworkMode': 'foo'} ) @@ -479,6 +491,7 @@ def test_list(self): def test_list_ignore_removed(self): def side_effect(*args, **kwargs): raise docker.errors.NotFound('Container not found') + client = make_fake_client({ 'inspect_container.side_effect': side_effect }) From a18f91bf08b4dca8dcf7627c8477a12ff2c1ca6a Mon Sep 17 00:00:00 2001 From: Mariano Scazzariello Date: Sun, 7 May 2023 11:49:59 +0200 Subject: [PATCH 1211/1301] Fix long line Signed-off-by: Mariano Scazzariello --- tests/unit/models_containers_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/unit/models_containers_test.py b/tests/unit/models_containers_test.py index 240b592fb6..f721bedbe3 100644 --- a/tests/unit/models_containers_test.py +++ b/tests/unit/models_containers_test.py @@ -2,7 +2,8 @@ import unittest import docker -from docker.constants import DEFAULT_DATA_CHUNK_SIZE, DEFAULT_DOCKER_API_VERSION +from docker.constants import DEFAULT_DATA_CHUNK_SIZE, \ + DEFAULT_DOCKER_API_VERSION from docker.models.containers import Container, _create_container_args from docker.models.images import Image from .fake_api import FAKE_CONTAINER_ID, FAKE_IMAGE_ID, FAKE_EXEC_ID From 7870503c523a130a2c8731df292eb904cd1a7345 Mon Sep 17 00:00:00 2001 From: Mariano Scazzariello Date: Sun, 7 May 2023 12:15:32 +0200 Subject: [PATCH 1212/1301] Fix case when "network_config" is not passed Signed-off-by: Mariano Scazzariello --- docker/models/containers.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/models/containers.py b/docker/models/containers.py index 1d2e58c641..bc2ed011fc 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -1148,14 +1148,14 @@ def _create_container_args(kwargs): network = kwargs.pop('network', None) network_config = kwargs.pop('network_config', None) if network: - network_configuration = EndpointConfig( + endpoint_config = EndpointConfig( host_config_kwargs['version'], **network_config ) if network_config else None create_kwargs['networking_config'] = NetworkingConfig( - {network: network_configuration} - ) + {network: endpoint_config} + ) if endpoint_config else {network: None} host_config_kwargs['network_mode'] = network # All kwargs should have been consumed by this point, so raise From e011ff5be89f84f999847d73d73ff695b9c8c4d4 Mon Sep 17 00:00:00 2001 From: Mariano Scazzariello Date: Sun, 7 May 2023 12:40:08 +0200 Subject: [PATCH 1213/1301] More sanity checking of EndpointConfig params Signed-off-by: Mariano Scazzariello --- docker/models/containers.py | 29 ++++++-- tests/integration/models_containers_test.py | 57 ++++++++++++++++ tests/unit/models_containers_test.py | 75 +++++++++++++++++++++ 3 files changed, 157 insertions(+), 4 deletions(-) diff --git a/docker/models/containers.py b/docker/models/containers.py index bc2ed011fc..3312b0e2d8 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -703,6 +703,8 @@ def run(self, image, command=None, stdout=True, stderr=False, (IPv4/IPv6) addresses. - ``driver_opt`` (dict): A dictionary of options to provide to the network driver. Defaults to ``None``. + - ``mac_address`` (str): MAC Address to assign to the network + interface. Defaults to ``None``. Requires API >= 1.25. Used in conjuction with ``network``. Incompatible with ``network_mode``. @@ -1122,6 +1124,17 @@ def prune(self, filters=None): ] +NETWORKING_CONFIG_ARGS = [ + 'aliases', + 'links', + 'ipv4_address', + 'ipv6_address', + 'link_local_ips', + 'driver_opt', + 'mac_address' +] + + def _create_container_args(kwargs): """ Convert arguments to create() to arguments to create_container(). @@ -1148,10 +1161,18 @@ def _create_container_args(kwargs): network = kwargs.pop('network', None) network_config = kwargs.pop('network_config', None) if network: - endpoint_config = EndpointConfig( - host_config_kwargs['version'], - **network_config - ) if network_config else None + endpoint_config = None + + if network_config: + clean_endpoint_args = {} + for arg_name in NETWORKING_CONFIG_ARGS: + if arg_name in network_config: + clean_endpoint_args[arg_name] = network_config[arg_name] + + if clean_endpoint_args: + endpoint_config = EndpointConfig( + host_config_kwargs['version'], **clean_endpoint_args + ) create_kwargs['networking_config'] = NetworkingConfig( {network: endpoint_config} diff --git a/tests/integration/models_containers_test.py b/tests/integration/models_containers_test.py index eac4c97909..050efa01ca 100644 --- a/tests/integration/models_containers_test.py +++ b/tests/integration/models_containers_test.py @@ -104,6 +104,63 @@ def test_run_with_network(self): assert 'Networks' in attrs['NetworkSettings'] assert list(attrs['NetworkSettings']['Networks'].keys()) == [net_name] + def test_run_with_network_config(self): + net_name = random_name() + client = docker.from_env(version=TEST_API_VERSION) + client.networks.create(net_name) + self.tmp_networks.append(net_name) + + test_aliases = ['hello'] + test_driver_opt = {'key1': 'a'} + + container = client.containers.run( + 'alpine', 'echo hello world', network=net_name, + network_config={'aliases': test_aliases, + 'driver_opt': test_driver_opt}, + detach=True + ) + self.tmp_containers.append(container.id) + + attrs = container.attrs + + assert 'NetworkSettings' in attrs + assert 'Networks' in attrs['NetworkSettings'] + assert list(attrs['NetworkSettings']['Networks'].keys()) == [net_name] + assert attrs['NetworkSettings']['Networks'][net_name]['Aliases'] == \ + test_aliases + assert attrs['NetworkSettings']['Networks'][net_name]['DriverOpts'] \ + == test_driver_opt + + def test_run_with_network_config_undeclared_params(self): + net_name = random_name() + client = docker.from_env(version=TEST_API_VERSION) + client.networks.create(net_name) + self.tmp_networks.append(net_name) + + test_aliases = ['hello'] + test_driver_opt = {'key1': 'a'} + + container = client.containers.run( + 'alpine', 'echo hello world', network=net_name, + network_config={'aliases': test_aliases, + 'driver_opt': test_driver_opt, + 'undeclared_param': 'random_value'}, + detach=True + ) + self.tmp_containers.append(container.id) + + attrs = container.attrs + + assert 'NetworkSettings' in attrs + assert 'Networks' in attrs['NetworkSettings'] + assert list(attrs['NetworkSettings']['Networks'].keys()) == [net_name] + assert attrs['NetworkSettings']['Networks'][net_name]['Aliases'] == \ + test_aliases + assert attrs['NetworkSettings']['Networks'][net_name]['DriverOpts'] \ + == test_driver_opt + assert 'undeclared_param' not in \ + attrs['NetworkSettings']['Networks'][net_name] + def test_run_with_none_driver(self): client = docker.from_env(version=TEST_API_VERSION) diff --git a/tests/unit/models_containers_test.py b/tests/unit/models_containers_test.py index f721bedbe3..3425ea897b 100644 --- a/tests/unit/models_containers_test.py +++ b/tests/unit/models_containers_test.py @@ -390,6 +390,44 @@ def test_run_network_config(self): host_config={'NetworkMode': 'foo'} ) + def test_run_network_config_undeclared_params(self): + client = make_fake_client() + + client.containers.run( + image='alpine', + network='foo', + network_config={'aliases': ['test'], + 'driver_opt': {'key1': 'a'}, + 'undeclared_param': 'random_value'} + ) + + client.api.create_container.assert_called_with( + detach=False, + image='alpine', + command=None, + networking_config={'EndpointsConfig': { + 'foo': {'Aliases': ['test'], 'DriverOpts': {'key1': 'a'}}} + }, + host_config={'NetworkMode': 'foo'} + ) + + def test_run_network_config_only_undeclared_params(self): + client = make_fake_client() + + client.containers.run( + image='alpine', + network='foo', + network_config={'undeclared_param': 'random_value'} + ) + + client.api.create_container.assert_called_with( + detach=False, + image='alpine', + command=None, + networking_config={'foo': None}, + host_config={'NetworkMode': 'foo'} + ) + def test_create(self): client = make_fake_client() container = client.containers.create( @@ -467,6 +505,43 @@ def test_create_network_config(self): host_config={'NetworkMode': 'foo'} ) + def test_create_network_config_undeclared_params(self): + client = make_fake_client() + + client.containers.create( + image='alpine', + network='foo', + network_config={'aliases': ['test'], + 'driver_opt': {'key1': 'a'}, + 'undeclared_param': 'random_value'} + ) + + client.api.create_container.assert_called_with( + image='alpine', + command=None, + networking_config={'EndpointsConfig': { + 'foo': {'Aliases': ['test'], 'DriverOpts': {'key1': 'a'}}} + }, + host_config={'NetworkMode': 'foo'} + ) + + def test_create_network_config_only_undeclared_params(self): + client = make_fake_client() + + client.containers.create( + image='alpine', + network='foo', + network_config={'undeclared_param': 'random_value'} + ) + + client.api.create_container.assert_called_with( + image='alpine', + command=None, + networking_config={'foo': None}, + host_config={'NetworkMode': 'foo'} + ) + + def test_get(self): client = make_fake_client() container = client.containers.get(FAKE_CONTAINER_ID) From 443a35360fc426479d87c810ffca8e5a253408a1 Mon Sep 17 00:00:00 2001 From: RazCrimson <52282402+RazCrimson@users.noreply.github.com> Date: Mon, 8 May 2023 05:21:24 +0530 Subject: [PATCH 1214/1301] Fix container.stats infinite blocking on stream mode (#3120) fix: api - container.stats infinite blocking on stream mode Includes additional test for no streaming Signed-off-by: Bharath Vignesh J K <52282402+RazCrimson@users.noreply.github.com> --- docker/api/container.py | 5 +++-- tests/unit/api_container_test.py | 11 +++++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/docker/api/container.py b/docker/api/container.py index fef760300e..40607e79a3 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -1164,8 +1164,9 @@ def stats(self, container, decode=None, stream=True, one_shot=None): 'one_shot is only available in conjunction with ' 'stream=False' ) - return self._stream_helper(self._get(url, params=params), - decode=decode) + return self._stream_helper( + self._get(url, stream=True, params=params), decode=decode + ) else: if decode: raise errors.InvalidArgument( diff --git a/tests/unit/api_container_test.py b/tests/unit/api_container_test.py index c605da371c..c4e2250be0 100644 --- a/tests/unit/api_container_test.py +++ b/tests/unit/api_container_test.py @@ -1528,10 +1528,21 @@ def test_container_stats(self): fake_request.assert_called_with( 'GET', url_prefix + 'containers/' + fake_api.FAKE_CONTAINER_ID + '/stats', + stream=True, timeout=60, params={'stream': True} ) + def test_container_stats_without_streaming(self): + self.client.stats(fake_api.FAKE_CONTAINER_ID, stream=False) + + fake_request.assert_called_with( + 'GET', + url_prefix + 'containers/' + fake_api.FAKE_CONTAINER_ID + '/stats', + timeout=60, + params={'stream': False} + ) + def test_container_stats_with_one_shot(self): self.client.stats( fake_api.FAKE_CONTAINER_ID, stream=False, one_shot=True) From 9cadad009e6aa78e15d752e2011705d7d91b8d2e Mon Sep 17 00:00:00 2001 From: Imogen <59090860+ImogenBits@users.noreply.github.com> Date: Mon, 8 May 2023 19:01:19 +0200 Subject: [PATCH 1215/1301] api: respect timeouts on Windows named pipes (#3112) Signed-off-by: Imogen <59090860+ImogenBits@users.noreply.github.com> --- docker/transport/npipesocket.py | 55 +++++++++++++++++++++------------ 1 file changed, 36 insertions(+), 19 deletions(-) diff --git a/docker/transport/npipesocket.py b/docker/transport/npipesocket.py index 766372aefd..9cbe40cc7f 100644 --- a/docker/transport/npipesocket.py +++ b/docker/transport/npipesocket.py @@ -4,6 +4,9 @@ import win32file import win32pipe +import pywintypes +import win32event +import win32api cERROR_PIPE_BUSY = 0xe7 cSECURITY_SQOS_PRESENT = 0x100000 @@ -54,7 +57,9 @@ def connect(self, address, retry_count=0): 0, None, win32file.OPEN_EXISTING, - cSECURITY_ANONYMOUS | cSECURITY_SQOS_PRESENT, + (cSECURITY_ANONYMOUS + | cSECURITY_SQOS_PRESENT + | win32file.FILE_FLAG_OVERLAPPED), 0 ) except win32pipe.error as e: @@ -131,22 +136,37 @@ def recv_into(self, buf, nbytes=0): if not isinstance(buf, memoryview): readbuf = memoryview(buf) - err, data = win32file.ReadFile( - self._handle, - readbuf[:nbytes] if nbytes else readbuf - ) - return len(data) - - def _recv_into_py2(self, buf, nbytes): - err, data = win32file.ReadFile(self._handle, nbytes or len(buf)) - n = len(data) - buf[:n] = data - return n + event = win32event.CreateEvent(None, True, True, None) + try: + overlapped = pywintypes.OVERLAPPED() + overlapped.hEvent = event + err, data = win32file.ReadFile( + self._handle, + readbuf[:nbytes] if nbytes else readbuf, + overlapped + ) + wait_result = win32event.WaitForSingleObject(event, self._timeout) + if wait_result == win32event.WAIT_TIMEOUT: + win32file.CancelIo(self._handle) + raise TimeoutError + return win32file.GetOverlappedResult(self._handle, overlapped, 0) + finally: + win32api.CloseHandle(event) @check_closed def send(self, string, flags=0): - err, nbytes = win32file.WriteFile(self._handle, string) - return nbytes + event = win32event.CreateEvent(None, True, True, None) + try: + overlapped = pywintypes.OVERLAPPED() + overlapped.hEvent = event + win32file.WriteFile(self._handle, string, overlapped) + wait_result = win32event.WaitForSingleObject(event, self._timeout) + if wait_result == win32event.WAIT_TIMEOUT: + win32file.CancelIo(self._handle) + raise TimeoutError + return win32file.GetOverlappedResult(self._handle, overlapped, 0) + finally: + win32api.CloseHandle(event) @check_closed def sendall(self, string, flags=0): @@ -165,15 +185,12 @@ def setblocking(self, flag): def settimeout(self, value): if value is None: # Blocking mode - self._timeout = win32pipe.NMPWAIT_WAIT_FOREVER + self._timeout = win32event.INFINITE elif not isinstance(value, (float, int)) or value < 0: raise ValueError('Timeout value out of range') - elif value == 0: - # Non-blocking mode - self._timeout = win32pipe.NMPWAIT_NO_WAIT else: # Timeout mode - Value converted to milliseconds - self._timeout = value * 1000 + self._timeout = int(value * 1000) def gettimeout(self): return self._timeout From c5e582c413a4a3a9eff6ee8208d195f657ffda94 Mon Sep 17 00:00:00 2001 From: loicleyendecker Date: Thu, 11 May 2023 18:36:37 +0100 Subject: [PATCH 1216/1301] api: avoid socket timeouts when executing commands (#3125) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Only listen to read events when polling a socket in order to avoid incorrectly trying to read from a socket that is not actually ready. Signed-off-by: Loïc Leyendecker --- docker/utils/socket.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/utils/socket.py b/docker/utils/socket.py index 3c31a98c5d..efb7458ee8 100644 --- a/docker/utils/socket.py +++ b/docker/utils/socket.py @@ -37,7 +37,7 @@ def read(socket, n=4096): select.select([socket], [], []) else: poll = select.poll() - poll.register(socket) + poll.register(socket, select.POLLIN | select.POLLPRI) poll.poll() try: From 14e8d07d4515eb893c35926aca75ecd521781baf Mon Sep 17 00:00:00 2001 From: Milas Bowman Date: Thu, 11 May 2023 15:35:42 -0400 Subject: [PATCH 1217/1301] docs: update changelog (#3127) Signed-off-by: Milas Bowman --- docs/change-log.md | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/docs/change-log.md b/docs/change-log.md index 5927728b1a..0d60f882d6 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -1,6 +1,47 @@ Changelog ========== +6.1.2 +----- + +#### Bugfixes +- Fix for socket timeouts on long `docker exec` calls + +6.1.1 +----- + +#### Bugfixes +- Fix `containers.stats()` hanging with `stream=True` +- Correct return type in docs for `containers.diff()` method + + +6.1.0 +----- + +### Upgrade Notes +- Errors are no longer returned during client initialization if the credential helper cannot be found. A warning will be emitted instead, and an error is returned if the credential helper is used. + +### Features +- Python 3.11 support +- Use `poll()` instead of `select()` on non-Windows platforms +- New API fields + - `network_driver_opt` on container run / create + - `one-shot` on container stats + - `status` on services list + +### Bugfixes +- Support for requests 2.29.0+ and urllib3 2.x +- Do not strip characters from volume names +- Fix connection leak on container.exec_* operations +- Fix errors closing named pipes on Windows + +6.0.1 +----- + +### Bugfixes +- Fix for `The pipe has been ended errors` on Windows +- Support floats for container log filtering by timestamp (`since` / `until`) + 6.0.0 ----- From bc4c0d7cf4f6f794f9e92d93ddec02626eda739c Mon Sep 17 00:00:00 2001 From: Milas Bowman Date: Thu, 11 May 2023 16:05:16 -0400 Subject: [PATCH 1218/1301] ci: empty commit to trigger readthedocs Fixing integration Signed-off-by: Milas Bowman From 0318ad8e7ee67c9ef0fbffaaf70029f255963012 Mon Sep 17 00:00:00 2001 From: Mariano Scazzariello Date: Mon, 15 May 2023 14:49:55 +0200 Subject: [PATCH 1219/1301] Fix blank line Signed-off-by: Mariano Scazzariello --- tests/unit/models_containers_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/unit/models_containers_test.py b/tests/unit/models_containers_test.py index 3425ea897b..f6dccaaba1 100644 --- a/tests/unit/models_containers_test.py +++ b/tests/unit/models_containers_test.py @@ -541,7 +541,6 @@ def test_create_network_config_only_undeclared_params(self): host_config={'NetworkMode': 'foo'} ) - def test_get(self): client = make_fake_client() container = client.containers.get(FAKE_CONTAINER_ID) From 78439ebbe1aae77ff0fd9b666894d80807182e28 Mon Sep 17 00:00:00 2001 From: Mathieu Virbel Date: Thu, 1 Jun 2023 16:19:01 +0200 Subject: [PATCH 1220/1301] fix: eventlet compatibility (#3132) Check if poll attribute exists on select module instead of win32 platform check The implementation done in #2865 is breaking usage of docker-py library within eventlet. As per the Python `select.poll` documentation (https://docs.python.org/3/library/select.html#select.poll) and eventlet select removal advice (https://github.com/eventlet/eventlet/issues/608#issuecomment-612359458), it is preferable to use an implementation based on the availability of the `poll()` method that trying to check if the platform is `win32`. Fixes #3131 Signed-off-by: Mathieu Virbel --- docker/utils/socket.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docker/utils/socket.py b/docker/utils/socket.py index efb7458ee8..cdc485ea3a 100644 --- a/docker/utils/socket.py +++ b/docker/utils/socket.py @@ -3,7 +3,6 @@ import select import socket as pysocket import struct -import sys try: from ..transport import NpipeSocket @@ -32,7 +31,7 @@ def read(socket, n=4096): recoverable_errors = (errno.EINTR, errno.EDEADLK, errno.EWOULDBLOCK) if not isinstance(socket, NpipeSocket): - if sys.platform == 'win32': + if not hasattr(select, "poll"): # Limited to 1024 select.select([socket], [], []) else: From 84414e343e526cf93f285284dd2c2c40f703e4a9 Mon Sep 17 00:00:00 2001 From: Hao Yu <129033897+Longin-Yu@users.noreply.github.com> Date: Wed, 7 Jun 2023 02:28:15 +0800 Subject: [PATCH 1221/1301] fix user_guides/multiplex.rst (#3130) Signed-off-by: Longin-Yu --- docs/user_guides/multiplex.rst | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/docs/user_guides/multiplex.rst b/docs/user_guides/multiplex.rst index 78d7e3728d..7add69b121 100644 --- a/docs/user_guides/multiplex.rst +++ b/docs/user_guides/multiplex.rst @@ -16,10 +16,13 @@ Prepare the command we are going to use. It prints "hello stdout" in `stdout`, followed by "hello stderr" in `stderr`: >>> cmd = '/bin/sh -c "echo hello stdout ; echo hello stderr >&2"' + We'll run this command with all four the combinations of ``stream`` and ``demux``. + With ``stream=False`` and ``demux=False``, the output is a string that contains both the `stdout` and the `stderr` output: + >>> res = container.exec_run(cmd, stream=False, demux=False) >>> res.output b'hello stderr\nhello stdout\n' @@ -52,15 +55,8 @@ Traceback (most recent call last): File "", line 1, in StopIteration -Finally, with ``stream=False`` and ``demux=True``, the whole output -is returned, but the streams are still separated: +Finally, with ``stream=False`` and ``demux=True``, the output is a tuple ``(stdout, stderr)``: ->>> res = container.exec_run(cmd, stream=True, demux=True) ->>> next(res.output) -(b'hello stdout\n', None) ->>> next(res.output) -(None, b'hello stderr\n') ->>> next(res.output) -Traceback (most recent call last): - File "", line 1, in -StopIteration +>>> res = container.exec_run(cmd, stream=False, demux=True) +>>> res.output +(b'hello stdout\n', b'hello stderr\n') \ No newline at end of file From f0d38fb7f40e01904bc3f788b7c29545f0c9fab2 Mon Sep 17 00:00:00 2001 From: Jay Turner Date: Tue, 27 Jun 2023 12:51:40 +0100 Subject: [PATCH 1222/1301] Add health property to Containers model Signed-off-by: Jay Turner --- docker/models/containers.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docker/models/containers.py b/docker/models/containers.py index 2eeefda1ee..6febb19ebd 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -62,6 +62,13 @@ def status(self): return self.attrs['State']['Status'] return self.attrs['State'] + @property + def health(self): + """ + The healthcheck status of the container. For example, ``healthy`, or ``unhealthy`. + """ + return self.attrs.get('State', {}).get('Health', {}).get('Status', 'unknown') + @property def ports(self): """ From fb974de27a2ad4295807ab61e47a781c5518d384 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Mon, 14 Aug 2023 10:26:36 +0200 Subject: [PATCH 1223/1301] tests/integration: fix flake8 failures (E721 do not compare types) Run flake8 docker/ tests/ flake8 docker/ tests/ shell: /usr/bin/bash -e {0} env: DOCKER_BUILDKIT: 1 pythonLocation: /opt/hostedtoolcache/Python/3.11.4/x64 PKG_CONFIG_PATH: /opt/hostedtoolcache/Python/3.11.4/x64/lib/pkgconfig Python_ROOT_DIR: /opt/hostedtoolcache/Python/3.11.4/x64 Python2_ROOT_DIR: /opt/hostedtoolcache/Python/3.11.4/x64 Python3_ROOT_DIR: /opt/hostedtoolcache/Python/3.11.4/x64 LD_LIBRARY_PATH: /opt/hostedtoolcache/Python/3.11.4/x64/lib tests/integration/api_container_test.py:1395:16: E721 do not compare types, for exact checks use `is` / `is not`, for instance checks use `isinstance()` tests/integration/api_container_test.py:1408:24: E721 do not compare types, for exact checks use `is` / `is not`, for instance checks use `isinstance()` tests/integration/api_image_test.py:35:16: E721 do not compare types, for exact checks use `is` / `is not`, for instance checks use `isinstance()` tests/integration/api_image_test.py:46:16: E721 do not compare types, for exact checks use `is` / `is not`, for instance checks use `isinstance()` Error: Process completed with exit code 1. Signed-off-by: Sebastiaan van Stijn --- tests/integration/api_container_test.py | 4 ++-- tests/integration/api_image_test.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index 0cb8fec68b..0782b12cc8 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -1392,7 +1392,7 @@ def test_get_container_stats_no_stream(self): response = self.client.stats(container, stream=0) self.client.kill(container) - assert type(response) == dict + assert isinstance(response, dict) for key in ['read', 'networks', 'precpu_stats', 'cpu_stats', 'memory_stats', 'blkio_stats']: assert key in response @@ -1405,7 +1405,7 @@ def test_get_container_stats_stream(self): self.client.start(container) stream = self.client.stats(container) for chunk in stream: - assert type(chunk) == dict + assert isinstance(chunk, dict) for key in ['read', 'network', 'precpu_stats', 'cpu_stats', 'memory_stats', 'blkio_stats']: assert key in chunk diff --git a/tests/integration/api_image_test.py b/tests/integration/api_image_test.py index 6a6686e377..cb3d667112 100644 --- a/tests/integration/api_image_test.py +++ b/tests/integration/api_image_test.py @@ -32,7 +32,7 @@ def test_images(self): def test_images_quiet(self): res1 = self.client.images(quiet=True) - assert type(res1[0]) == str + assert isinstance(res1[0], str) class PullImageTest(BaseAPIIntegrationTest): @@ -43,7 +43,7 @@ def test_pull(self): pass res = self.client.pull('hello-world') self.tmp_imgs.append('hello-world') - assert type(res) == str + assert isinstance(res, str) assert len(self.client.images('hello-world')) >= 1 img_info = self.client.inspect_image('hello-world') assert 'Id' in img_info From 83e93228ea79d899c38cd7f75e688f634454318a Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Mon, 14 Aug 2023 10:32:04 +0200 Subject: [PATCH 1224/1301] tests/Dockerfile: fix Dockerfile for debian bookworm The Dockerfile failed to build due to the base-image having switched to "bookworm"; Dockerfile:8 -------------------- 7 | ARG APT_MIRROR 8 | >>> RUN sed -ri "s/(httpredir|deb).debian.org/${APT_MIRROR:-deb.debian.org}/g" /etc/apt/sources.list \ 9 | >>> && sed -ri "s/(security).debian.org/${APT_MIRROR:-security.debian.org}/g" /etc/apt/sources.list 10 | -------------------- ERROR: failed to solve: process "/bin/sh -c sed -ri \"s/(httpredir|deb).debian.org/${APT_MIRROR:-deb.debian.org}/g\" /etc/apt/sources.list && sed -ri \"s/(security).debian.org/${APT_MIRROR:-security.debian.org}/g\" /etc/apt/sources.list" did not complete successfully: exit code: 2 Signed-off-by: Sebastiaan van Stijn --- tests/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/Dockerfile b/tests/Dockerfile index bf95cd6a3c..4705acca54 100644 --- a/tests/Dockerfile +++ b/tests/Dockerfile @@ -5,8 +5,8 @@ ARG PYTHON_VERSION=3.10 FROM python:${PYTHON_VERSION} ARG APT_MIRROR -RUN sed -ri "s/(httpredir|deb).debian.org/${APT_MIRROR:-deb.debian.org}/g" /etc/apt/sources.list \ - && sed -ri "s/(security).debian.org/${APT_MIRROR:-security.debian.org}/g" /etc/apt/sources.list +RUN sed -ri "s/(httpredir|deb).debian.org/${APT_MIRROR:-deb.debian.org}/g" /etc/apt/sources.list.d/debian.sources \ + && sed -ri "s/(security).debian.org/${APT_MIRROR:-security.debian.org}/g" /etc/apt/sources.list.d/debian.sources RUN apt-get update && apt-get -y install --no-install-recommends \ gnupg2 \ From 5064995bc40768ad28af93d017d3c032312e41ec Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Mon, 14 Aug 2023 10:17:25 +0200 Subject: [PATCH 1225/1301] tests/integration: update some tests for updated error-messages I was in the process of cleaning up some error-messages, and it looks like the docker-py tests were depending on strings that will be removed; =================================== FAILURES =================================== _____________ CreateContainerTest.test_create_with_restart_policy ______________ tests/integration/api_container_test.py:126: in test_create_with_restart_policy assert 'You cannot remove ' in err E AssertionError: assert 'You cannot remove ' in 'cannot remove container d11580f6078108691096ec8a23404a6bda9ad1d1b2bafe88b17d127a67728833: container is restarting: stop the container before removing or force remove' ____________________ ErrorsTest.test_api_error_parses_json _____________________ tests/integration/errors_test.py:13: in test_api_error_parses_json assert 'You cannot remove a running container' in explanation E AssertionError: assert 'You cannot remove a running container' in 'cannot remove container 4b90ce2e907dd0f99d0f561619b803e7a2a31809ced366c537874dd13f8a47ec: container is running: stop the container before removing or force remove' This updates the tests to match on a string that will be present in both the old and new error-messages, but added a "lower()", so that matching will be done case-insensitive (Go errors generally should be lowercase). Signed-off-by: Sebastiaan van Stijn --- tests/integration/api_container_test.py | 4 ++-- tests/integration/errors_test.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index 0782b12cc8..b510979de0 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -122,8 +122,8 @@ def test_create_with_restart_policy(self): self.client.wait(id) with pytest.raises(docker.errors.APIError) as exc: self.client.remove_container(id) - err = exc.value.explanation - assert 'You cannot remove ' in err + err = exc.value.explanation.lower() + assert 'stop the container before' in err self.client.remove_container(id, force=True) def test_create_container_with_volumes_from(self): diff --git a/tests/integration/errors_test.py b/tests/integration/errors_test.py index 7bf156afb0..e2fce48b0f 100644 --- a/tests/integration/errors_test.py +++ b/tests/integration/errors_test.py @@ -9,7 +9,7 @@ def test_api_error_parses_json(self): self.client.start(container['Id']) with pytest.raises(APIError) as cm: self.client.remove_container(container['Id']) - explanation = cm.value.explanation - assert 'You cannot remove a running container' in explanation + explanation = cm.value.explanation.lower() + assert 'stop the container before' in explanation assert '{"message":' not in explanation self.client.remove_container(container['Id'], force=True) From 62b4bb8489001a8907046b8f4b68e6d68eca1d2b Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Mon, 14 Aug 2023 14:58:34 +0200 Subject: [PATCH 1226/1301] README: fix link for CI status badge The default branch was renamed from master to main, but the badge was still linking to the status for the master branch. Remove the branch-name so that the badge always refers to the "default" branch Signed-off-by: Sebastiaan van Stijn --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2db678dccc..921ffbcb88 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Docker SDK for Python -[![Build Status](https://github.com/docker/docker-py/actions/workflows/ci.yml/badge.svg?branch=master)](https://github.com/docker/docker-py/actions/workflows/ci.yml/) +[![Build Status](https://github.com/docker/docker-py/actions/workflows/ci.yml/badge.svg)](https://github.com/docker/docker-py/actions/workflows/ci.yml) A Python library for the Docker Engine API. It lets you do anything the `docker` command does, but from within Python apps – run containers, manage containers, manage Swarms, etc. From 0618951093bf3116af482c6dfb983219c5684343 Mon Sep 17 00:00:00 2001 From: Mehmet Nuri Deveci <5735811+mndeveci@users.noreply.github.com> Date: Mon, 14 Aug 2023 21:43:31 +0300 Subject: [PATCH 1227/1301] fix: use response.text to get string rather than bytes (#3156) Signed-off-by: Mehmet Nuri Deveci <5735811+mndeveci@users.noreply.github.com> Co-authored-by: Milas Bowman --- docker/errors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/errors.py b/docker/errors.py index 8cf8670baf..75e30a8c94 100644 --- a/docker/errors.py +++ b/docker/errors.py @@ -27,7 +27,7 @@ def create_api_error_from_http_exception(e): try: explanation = response.json()['message'] except ValueError: - explanation = (response.content or '').strip() + explanation = (response.text or '').strip() cls = APIError if response.status_code == 404: explanation_msg = (explanation or '').lower() From 4571f7f9b4044e87303bb2215f8a308b7ddc2e3d Mon Sep 17 00:00:00 2001 From: VincentLeeMax Date: Tue, 15 Aug 2023 02:52:38 +0800 Subject: [PATCH 1228/1301] feat: add pause option to commit api (#3159) add commit pause option Signed-off-by: VincentLeeMax Co-authored-by: Milas Bowman --- docker/api/container.py | 4 +++- docker/models/containers.py | 1 + tests/unit/api_image_test.py | 1 + 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docker/api/container.py b/docker/api/container.py index 40607e79a3..b8d5957b62 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -112,7 +112,7 @@ def attach_socket(self, container, params=None, ws=False): @utils.check_resource('container') def commit(self, container, repository=None, tag=None, message=None, - author=None, changes=None, conf=None): + author=None, pause=True, changes=None, conf=None): """ Commit a container to an image. Similar to the ``docker commit`` command. @@ -123,6 +123,7 @@ def commit(self, container, repository=None, tag=None, message=None, tag (str): The tag to push message (str): A commit message author (str): The name of the author + pause (bool): Whether to pause the container before committing changes (str): Dockerfile instructions to apply while committing conf (dict): The configuration for the container. See the `Engine API documentation @@ -139,6 +140,7 @@ def commit(self, container, repository=None, tag=None, message=None, 'tag': tag, 'comment': message, 'author': author, + 'pause': pause, 'changes': changes } u = self._url("/commit") diff --git a/docker/models/containers.py b/docker/models/containers.py index 2eeefda1ee..64838397a6 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -121,6 +121,7 @@ def commit(self, repository=None, tag=None, **kwargs): tag (str): The tag to push message (str): A commit message author (str): The name of the author + pause (bool): Whether to pause the container before committing changes (str): Dockerfile instructions to apply while committing conf (dict): The configuration for the container. See the `Engine API documentation diff --git a/tests/unit/api_image_test.py b/tests/unit/api_image_test.py index e285932941..b3428aa169 100644 --- a/tests/unit/api_image_test.py +++ b/tests/unit/api_image_test.py @@ -102,6 +102,7 @@ def test_commit(self): 'tag': None, 'container': fake_api.FAKE_CONTAINER_ID, 'author': None, + 'pause': True, 'changes': None }, timeout=DEFAULT_TIMEOUT_SECONDS From dbc061f4fab49a67994ca4de26cdab989c356247 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Aug 2023 15:08:38 -0400 Subject: [PATCH 1229/1301] build(deps): Bump requests from 2.28.1 to 2.31.0 (#3136) Bumps [requests](https://github.com/psf/requests) from 2.28.1 to 2.31.0. - [Release notes](https://github.com/psf/requests/releases) - [Changelog](https://github.com/psf/requests/blob/main/HISTORY.md) - [Commits](https://github.com/psf/requests/compare/v2.28.1...v2.31.0) --- updated-dependencies: - dependency-name: requests dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 36660b660c..897cdbd5ef 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ packaging==21.3 paramiko==2.11.0 pywin32==304; sys_platform == 'win32' -requests==2.28.1 +requests==2.31.0 urllib3==1.26.11 websocket-client==1.3.3 From ee2310595d1362428f2826afac2e17077231473a Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Mon, 14 Aug 2023 21:12:44 +0200 Subject: [PATCH 1230/1301] test: remove APT_MIRROR from Dockerfile (#3145) The official Python images on Docker Hub switched to debian bookworm, which is now the current stable version of Debian. However, the location of the apt repository config file changed, which causes the Dockerfile build to fail; Loaded image: emptyfs:latest Loaded image ID: sha256:0df1207206e5288f4a989a2f13d1f5b3c4e70467702c1d5d21dfc9f002b7bd43 INFO: Building docker-sdk-python3:5.0.3... tests/Dockerfile:6 -------------------- 5 | ARG APT_MIRROR 6 | >>> RUN sed -ri "s/(httpredir|deb).debian.org/${APT_MIRROR:-deb.debian.org}/g" /etc/apt/sources.list \ 7 | >>> && sed -ri "s/(security).debian.org/${APT_MIRROR:-security.debian.org}/g" /etc/apt/sources.list 8 | -------------------- ERROR: failed to solve: process "/bin/sh -c sed -ri \"s/(httpredir|deb).debian.org/${APT_MIRROR:-deb.debian.org}/g\" /etc/apt/sources.list && sed -ri \"s/(security).debian.org/${APT_MIRROR:-security.debian.org}/g\" /etc/apt/sources.list" did not complete successfully: exit code: 2 The APT_MIRROR build-arg was originally added when the Debian package repositories were known to be unreliable, but that hasn't been the case for quite a while, so let's remove this altogether. Signed-off-by: Sebastiaan van Stijn Signed-off-by: Milas Bowman Co-authored-by: Milas Bowman --- tests/Dockerfile | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/Dockerfile b/tests/Dockerfile index 4705acca54..366abe23bb 100644 --- a/tests/Dockerfile +++ b/tests/Dockerfile @@ -4,10 +4,6 @@ ARG PYTHON_VERSION=3.10 FROM python:${PYTHON_VERSION} -ARG APT_MIRROR -RUN sed -ri "s/(httpredir|deb).debian.org/${APT_MIRROR:-deb.debian.org}/g" /etc/apt/sources.list.d/debian.sources \ - && sed -ri "s/(security).debian.org/${APT_MIRROR:-security.debian.org}/g" /etc/apt/sources.list.d/debian.sources - RUN apt-get update && apt-get -y install --no-install-recommends \ gnupg2 \ pass From 8a3402c049437bf10066c0527fafa06de8bca73f Mon Sep 17 00:00:00 2001 From: Aarni Koskela Date: Thu, 11 May 2023 16:12:41 +0300 Subject: [PATCH 1231/1301] Replace string formatting with f-strings Signed-off-by: Aarni Koskela --- docker/api/build.py | 14 ++---- docker/api/client.py | 19 ++++---- docker/api/container.py | 10 ++-- docker/api/service.py | 4 +- docker/auth.py | 27 ++++------- docker/context/api.py | 4 +- docker/context/context.py | 10 ++-- docker/credentials/errors.py | 12 +---- docker/credentials/store.py | 18 ++------ docker/errors.py | 22 +++++---- docker/models/images.py | 11 ++--- docker/models/resource.py | 10 ++-- docker/transport/unixconn.py | 2 +- docker/types/containers.py | 18 ++++---- docker/types/services.py | 12 ++--- docker/utils/build.py | 4 +- docker/utils/decorators.py | 4 +- docker/utils/fnmatch.py | 16 +++---- docker/utils/ports.py | 2 +- docker/utils/proxy.py | 8 +++- docker/utils/utils.py | 37 +++++++-------- docs/conf.py | 2 +- tests/helpers.py | 2 +- tests/integration/api_client_test.py | 4 +- tests/integration/api_container_test.py | 16 +++---- tests/integration/base.py | 3 +- tests/integration/credentials/store_test.py | 2 +- tests/integration/models_containers_test.py | 4 +- tests/ssh/base.py | 3 +- tests/unit/api_build_test.py | 10 ++-- tests/unit/api_exec_test.py | 14 ++---- tests/unit/api_image_test.py | 48 +++++++++---------- tests/unit/api_network_test.py | 12 ++--- tests/unit/api_test.py | 51 +++++++++------------ tests/unit/api_volume_test.py | 8 ++-- tests/unit/auth_test.py | 7 +-- tests/unit/client_test.py | 20 ++------ tests/unit/fake_api.py | 28 +++-------- tests/unit/swarm_test.py | 6 +-- tests/unit/utils_test.py | 4 +- 40 files changed, 214 insertions(+), 294 deletions(-) diff --git a/docker/api/build.py b/docker/api/build.py index 3a1a3d9642..439f4dc351 100644 --- a/docker/api/build.py +++ b/docker/api/build.py @@ -314,9 +314,8 @@ def _set_auth_headers(self, headers): auth_data[auth.INDEX_URL] = auth_data.get(auth.INDEX_NAME, {}) log.debug( - 'Sending auth config ({})'.format( - ', '.join(repr(k) for k in auth_data.keys()) - ) + "Sending auth config (%s)", + ', '.join(repr(k) for k in auth_data), ) if auth_data: @@ -336,12 +335,9 @@ def process_dockerfile(dockerfile, path): abs_dockerfile = os.path.join(path, dockerfile) if constants.IS_WINDOWS_PLATFORM and path.startswith( constants.WINDOWS_LONGPATH_PREFIX): - abs_dockerfile = '{}{}'.format( - constants.WINDOWS_LONGPATH_PREFIX, - os.path.normpath( - abs_dockerfile[len(constants.WINDOWS_LONGPATH_PREFIX):] - ) - ) + normpath = os.path.normpath( + abs_dockerfile[len(constants.WINDOWS_LONGPATH_PREFIX):]) + abs_dockerfile = f'{constants.WINDOWS_LONGPATH_PREFIX}{normpath}' if (os.path.splitdrive(path)[0] != os.path.splitdrive(abs_dockerfile)[0] or os.path.relpath(abs_dockerfile, path).startswith('..')): # Dockerfile not in context - read data to insert into tar later diff --git a/docker/api/client.py b/docker/api/client.py index 65b9d9d198..8633025f3f 100644 --- a/docker/api/client.py +++ b/docker/api/client.py @@ -199,14 +199,13 @@ def __init__(self, base_url=None, version=None, self._version = version if not isinstance(self._version, str): raise DockerException( - 'Version parameter must be a string or None. Found {}'.format( - type(version).__name__ - ) + 'Version parameter must be a string or None. ' + f'Found {type(version).__name__}' ) if utils.version_lt(self._version, MINIMUM_DOCKER_API_VERSION): raise InvalidVersion( - 'API versions below {} are no longer supported by this ' - 'library.'.format(MINIMUM_DOCKER_API_VERSION) + f'API versions below {MINIMUM_DOCKER_API_VERSION} are ' + f'no longer supported by this library.' ) def _retrieve_server_version(self): @@ -248,19 +247,17 @@ def _url(self, pathfmt, *args, **kwargs): for arg in args: if not isinstance(arg, str): raise ValueError( - 'Expected a string but found {} ({}) ' - 'instead'.format(arg, type(arg)) + f'Expected a string but found {arg} ({type(arg)}) instead' ) quote_f = partial(urllib.parse.quote, safe="/:") args = map(quote_f, args) + formatted_path = pathfmt.format(*args) if kwargs.get('versioned_api', True): - return '{}/v{}{}'.format( - self.base_url, self._version, pathfmt.format(*args) - ) + return f'{self.base_url}/v{self._version}{formatted_path}' else: - return f'{self.base_url}{pathfmt.format(*args)}' + return f'{self.base_url}{formatted_path}' def _raise_for_status(self, response): """Raises stored :class:`APIError`, if one occurred.""" diff --git a/docker/api/container.py b/docker/api/container.py index b8d5957b62..ec28fd581b 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -863,8 +863,8 @@ def logs(self, container, stdout=True, stderr=True, stream=False, params['since'] = since else: raise errors.InvalidArgument( - 'since value should be datetime or positive int/float, ' - 'not {}'.format(type(since)) + 'since value should be datetime or positive int/float,' + f' not {type(since)}' ) if until is not None: @@ -880,8 +880,8 @@ def logs(self, container, stdout=True, stderr=True, stream=False, params['until'] = until else: raise errors.InvalidArgument( - 'until value should be datetime or positive int/float, ' - 'not {}'.format(type(until)) + f'until value should be datetime or positive int/float, ' + f'not {type(until)}' ) url = self._url("/containers/{0}/logs", container) @@ -953,7 +953,7 @@ def port(self, container, private_port): return port_settings.get(private_port) for protocol in ['tcp', 'udp', 'sctp']: - h_ports = port_settings.get(private_port + '/' + protocol) + h_ports = port_settings.get(f"{private_port}/{protocol}") if h_ports: break diff --git a/docker/api/service.py b/docker/api/service.py index 652b7c2458..3aed065175 100644 --- a/docker/api/service.py +++ b/docker/api/service.py @@ -7,9 +7,7 @@ def _check_api_features(version, task_template, update_config, endpoint_spec, def raise_version_error(param, min_version): raise errors.InvalidVersion( - '{} is not supported in API version < {}'.format( - param, min_version - ) + f'{param} is not supported in API version < {min_version}' ) if update_config is not None: diff --git a/docker/auth.py b/docker/auth.py index cb3885548f..4bce788701 100644 --- a/docker/auth.py +++ b/docker/auth.py @@ -22,15 +22,15 @@ def resolve_repository_name(repo_name): index_name, remote_name = split_repo_name(repo_name) if index_name[0] == '-' or index_name[-1] == '-': raise errors.InvalidRepository( - 'Invalid index name ({}). Cannot begin or end with a' - ' hyphen.'.format(index_name) + f'Invalid index name ({index_name}). ' + 'Cannot begin or end with a hyphen.' ) return resolve_index_name(index_name), remote_name def resolve_index_name(index_name): index_name = convert_to_hostname(index_name) - if index_name == 'index.' + INDEX_NAME: + if index_name == f"index.{INDEX_NAME}": index_name = INDEX_NAME return index_name @@ -99,9 +99,7 @@ def parse_auth(cls, entries, raise_on_error=False): for registry, entry in entries.items(): if not isinstance(entry, dict): log.debug( - 'Config entry for key {} is not auth config'.format( - registry - ) + f'Config entry for key {registry} is not auth config' ) # We sometimes fall back to parsing the whole config as if it # was the auth config by itself, for legacy purposes. In that @@ -109,17 +107,11 @@ def parse_auth(cls, entries, raise_on_error=False): # keys is not formatted properly. if raise_on_error: raise errors.InvalidConfigFile( - 'Invalid configuration for registry {}'.format( - registry - ) + f'Invalid configuration for registry {registry}' ) return {} if 'identitytoken' in entry: - log.debug( - 'Found an IdentityToken entry for registry {}'.format( - registry - ) - ) + log.debug(f'Found an IdentityToken entry for registry {registry}') conf[registry] = { 'IdentityToken': entry['identitytoken'] } @@ -130,16 +122,15 @@ def parse_auth(cls, entries, raise_on_error=False): # a valid value in the auths config. # https://github.com/docker/compose/issues/3265 log.debug( - 'Auth data for {} is absent. Client might be using a ' - 'credentials store instead.'.format(registry) + f'Auth data for {registry} is absent. ' + f'Client might be using a credentials store instead.' ) conf[registry] = {} continue username, password = decode_auth(entry['auth']) log.debug( - 'Found entry (registry={}, username={})' - .format(repr(registry), repr(username)) + f'Found entry (registry={registry!r}, username={username!r})' ) conf[registry] = { diff --git a/docker/context/api.py b/docker/context/api.py index 380e8c4c4f..e340fb6dd9 100644 --- a/docker/context/api.py +++ b/docker/context/api.py @@ -113,8 +113,8 @@ def contexts(cls): names.append(data["Name"]) except Exception as e: raise errors.ContextException( - "Failed to load metafile {}: {}".format( - filename, e)) + f"Failed to load metafile {filename}: {e}", + ) contexts = [cls.DEFAULT_CONTEXT] for name in names: diff --git a/docker/context/context.py b/docker/context/context.py index dbaa01cb5b..b607b77148 100644 --- a/docker/context/context.py +++ b/docker/context/context.py @@ -42,8 +42,9 @@ def __init__(self, name, orchestrator=None, host=None, endpoints=None, for k, v in endpoints.items(): if not isinstance(v, dict): # unknown format - raise ContextException("""Unknown endpoint format for - context {}: {}""".format(name, v)) + raise ContextException( + f"Unknown endpoint format for context {name}: {v}", + ) self.endpoints[k] = v if k != "docker": @@ -96,8 +97,9 @@ def _load_meta(cls, name): metadata = json.load(f) except (OSError, KeyError, ValueError) as e: # unknown format - raise Exception("""Detected corrupted meta file for - context {} : {}""".format(name, e)) + raise Exception( + f"Detected corrupted meta file for context {name} : {e}" + ) # for docker endpoints, set defaults for # Host and SkipTLSVerify fields diff --git a/docker/credentials/errors.py b/docker/credentials/errors.py index 42a1bc1a50..d059fd9fbb 100644 --- a/docker/credentials/errors.py +++ b/docker/credentials/errors.py @@ -13,13 +13,5 @@ class InitializationError(StoreError): def process_store_error(cpe, program): message = cpe.output.decode('utf-8') if 'credentials not found in native keychain' in message: - return CredentialsNotFound( - 'No matching credentials in {}'.format( - program - ) - ) - return StoreError( - 'Credentials store {} exited with "{}".'.format( - program, cpe.output.decode('utf-8').strip() - ) - ) + return CredentialsNotFound(f'No matching credentials in {program}') + return StoreError(f'Credentials store {program} exited with "{message}".') diff --git a/docker/credentials/store.py b/docker/credentials/store.py index b7ab53fbad..37c703e78c 100644 --- a/docker/credentials/store.py +++ b/docker/credentials/store.py @@ -20,9 +20,7 @@ def __init__(self, program, environment=None): self.environment = environment if self.exe is None: warnings.warn( - '{} not installed or not available in PATH'.format( - self.program - ) + f'{self.program} not installed or not available in PATH' ) def get(self, server): @@ -73,10 +71,8 @@ def list(self): def _execute(self, subcmd, data_input): if self.exe is None: raise errors.StoreError( - '{} not installed or not available in PATH'.format( - self.program - ) - ) + f'{self.program} not installed or not available in PATH' + ) output = None env = create_environment_dict(self.environment) try: @@ -88,14 +84,10 @@ def _execute(self, subcmd, data_input): except OSError as e: if e.errno == errno.ENOENT: raise errors.StoreError( - '{} not installed or not available in PATH'.format( - self.program - ) + f'{self.program} not installed or not available in PATH' ) else: raise errors.StoreError( - 'Unexpected OS error "{}", errno={}'.format( - e.strerror, e.errno - ) + f'Unexpected OS error "{e.strerror}", errno={e.errno}' ) return output diff --git a/docker/errors.py b/docker/errors.py index 75e30a8c94..d03e10f693 100644 --- a/docker/errors.py +++ b/docker/errors.py @@ -54,14 +54,16 @@ def __str__(self): message = super().__str__() if self.is_client_error(): - message = '{} Client Error for {}: {}'.format( - self.response.status_code, self.response.url, - self.response.reason) + message = ( + f'{self.response.status_code} Client Error for ' + f'{self.response.url}: {self.response.reason}' + ) elif self.is_server_error(): - message = '{} Server Error for {}: {}'.format( - self.response.status_code, self.response.url, - self.response.reason) + message = ( + f'{self.response.status_code} Server Error for ' + f'{self.response.url}: {self.response.reason}' + ) if self.explanation: message = f'{message} ("{self.explanation}")' @@ -142,10 +144,10 @@ def __init__(self, container, exit_status, command, image, stderr): self.stderr = stderr err = f": {stderr}" if stderr is not None else "" - msg = ("Command '{}' in image '{}' returned non-zero exit " - "status {}{}").format(command, image, exit_status, err) - - super().__init__(msg) + super().__init__( + f"Command '{command}' in image '{image}' " + f"returned non-zero exit status {exit_status}{err}" + ) class StreamParseError(RuntimeError): diff --git a/docker/models/images.py b/docker/models/images.py index e3ec39d28d..abb4b12b50 100644 --- a/docker/models/images.py +++ b/docker/models/images.py @@ -15,10 +15,8 @@ class Image(Model): An image on the server. """ def __repr__(self): - return "<{}: '{}'>".format( - self.__class__.__name__, - "', '".join(self.tags), - ) + tag_str = "', '".join(self.tags) + return f"<{self.__class__.__name__}: '{tag_str}'>" @property def labels(self): @@ -471,9 +469,8 @@ def pull(self, repository, tag=None, all_tags=False, **kwargs): # to be pulled. pass if not all_tags: - return self.get('{0}{2}{1}'.format( - repository, tag, '@' if tag.startswith('sha256:') else ':' - )) + sep = '@' if tag.startswith('sha256:') else ':' + return self.get(f'{repository}{sep}{tag}') return self.list(repository) def push(self, repository, tag=None, **kwargs): diff --git a/docker/models/resource.py b/docker/models/resource.py index 89030e592e..d3a35e84be 100644 --- a/docker/models/resource.py +++ b/docker/models/resource.py @@ -64,9 +64,10 @@ def __init__(self, client=None): def __call__(self, *args, **kwargs): raise TypeError( - "'{}' object is not callable. You might be trying to use the old " - "(pre-2.0) API - use docker.APIClient if so." - .format(self.__class__.__name__)) + f"'{self.__class__.__name__}' object is not callable. " + "You might be trying to use the old (pre-2.0) API - " + "use docker.APIClient if so." + ) def list(self): raise NotImplementedError @@ -88,5 +89,4 @@ def prepare_model(self, attrs): elif isinstance(attrs, dict): return self.model(attrs=attrs, client=self.client, collection=self) else: - raise Exception("Can't create %s from %s" % - (self.model.__name__, attrs)) + raise Exception(f"Can't create {self.model.__name__} from {attrs}") diff --git a/docker/transport/unixconn.py b/docker/transport/unixconn.py index fae10f2664..09d373dd6d 100644 --- a/docker/transport/unixconn.py +++ b/docker/transport/unixconn.py @@ -55,7 +55,7 @@ def __init__(self, socket_url, timeout=60, max_pool_size=constants.DEFAULT_MAX_POOL_SIZE): socket_path = socket_url.replace('http+unix://', '') if not socket_path.startswith('/'): - socket_path = '/' + socket_path + socket_path = f"/{socket_path}" self.socket_path = socket_path self.timeout = timeout self.max_pool_size = max_pool_size diff --git a/docker/types/containers.py b/docker/types/containers.py index 84df0f7e61..6d54aa65cc 100644 --- a/docker/types/containers.py +++ b/docker/types/containers.py @@ -652,25 +652,25 @@ def __init__(self, version, binds=None, port_bindings=None, def host_config_type_error(param, param_value, expected): - error_msg = 'Invalid type for {0} param: expected {1} but found {2}' - return TypeError(error_msg.format(param, expected, type(param_value))) + return TypeError( + f'Invalid type for {param} param: expected {expected} ' + f'but found {type(param_value)}' + ) def host_config_version_error(param, version, less_than=True): operator = '<' if less_than else '>' - error_msg = '{0} param is not supported in API versions {1} {2}' - return errors.InvalidVersion(error_msg.format(param, operator, version)) - + return errors.InvalidVersion( + f'{param} param is not supported in API versions {operator} {version}', + ) def host_config_value_error(param, param_value): - error_msg = 'Invalid value for {0} param: {1}' - return ValueError(error_msg.format(param, param_value)) + return ValueError(f'Invalid value for {param} param: {param_value}') def host_config_incompatible_error(param, param_value, incompatible_param): - error_msg = '\"{1}\" {0} is incompatible with {2}' return errors.InvalidArgument( - error_msg.format(param, param_value, incompatible_param) + f'\"{param_value}\" {param} is incompatible with {incompatible_param}' ) diff --git a/docker/types/services.py b/docker/types/services.py index a3383ef75b..0b07c350ee 100644 --- a/docker/types/services.py +++ b/docker/types/services.py @@ -370,8 +370,8 @@ def _convert_generic_resources_dict(generic_resources): return generic_resources if not isinstance(generic_resources, dict): raise errors.InvalidArgument( - 'generic_resources must be a dict or a list' - ' (found {})'.format(type(generic_resources)) + 'generic_resources must be a dict or a list ' + f'(found {type(generic_resources)})' ) resources = [] for kind, value in generic_resources.items(): @@ -381,9 +381,9 @@ def _convert_generic_resources_dict(generic_resources): elif isinstance(value, str): resource_type = 'NamedResourceSpec' else: + kv = {kind: value} raise errors.InvalidArgument( - 'Unsupported generic resource reservation ' - 'type: {}'.format({kind: value}) + f'Unsupported generic resource reservation type: {kv}' ) resources.append({ resource_type: {'Kind': kind, 'Value': value} @@ -764,8 +764,8 @@ class PlacementPreference(dict): def __init__(self, strategy, descriptor): if strategy != 'spread': raise errors.InvalidArgument( - 'PlacementPreference strategy value is invalid ({}):' - ' must be "spread".'.format(strategy) + f'PlacementPreference strategy value is invalid ({strategy}): ' + 'must be "spread".' ) self['Spread'] = {'SpreadDescriptor': descriptor} diff --git a/docker/utils/build.py b/docker/utils/build.py index 59564c4cda..6b38eacdb2 100644 --- a/docker/utils/build.py +++ b/docker/utils/build.py @@ -42,7 +42,7 @@ def exclude_paths(root, patterns, dockerfile=None): if dockerfile is None: dockerfile = 'Dockerfile' - patterns.append('!' + dockerfile) + patterns.append(f"!{dockerfile}") pm = PatternMatcher(patterns) return set(pm.walk(root)) @@ -180,7 +180,7 @@ def rec_walk(current_dir): fpath = os.path.join( os.path.relpath(current_dir, root), f ) - if fpath.startswith('.' + os.path.sep): + if fpath.startswith(f".{os.path.sep}"): fpath = fpath[2:] match = self.matches(fpath) if not match: diff --git a/docker/utils/decorators.py b/docker/utils/decorators.py index cf1baf496c..5aab98cd46 100644 --- a/docker/utils/decorators.py +++ b/docker/utils/decorators.py @@ -27,9 +27,7 @@ def decorator(f): def wrapper(self, *args, **kwargs): if utils.version_lt(self._version, version): raise errors.InvalidVersion( - '{} is not available for version < {}'.format( - f.__name__, version - ) + f'{f.__name__} is not available for version < {version}', ) return f(self, *args, **kwargs) return wrapper diff --git a/docker/utils/fnmatch.py b/docker/utils/fnmatch.py index 90e9f60f59..be745381e4 100644 --- a/docker/utils/fnmatch.py +++ b/docker/utils/fnmatch.py @@ -79,18 +79,18 @@ def translate(pat): i = i + 1 if i >= n: # is "**EOF" - to align with .gitignore just accept all - res = res + '.*' + res = f"{res}.*" else: # is "**" # Note that this allows for any # of /'s (even 0) because # the .* will eat everything, even /'s - res = res + '(.*/)?' + res = f"{res}(.*/)?" else: # is "*" so map it to anything but "/" - res = res + '[^/]*' + res = f"{res}[^/]*" elif c == '?': # "?" is any char except "/" - res = res + '[^/]' + res = f"{res}[^/]" elif c == '[': j = i if j < n and pat[j] == '!': @@ -100,16 +100,16 @@ def translate(pat): while j < n and pat[j] != ']': j = j + 1 if j >= n: - res = res + '\\[' + res = f"{res}\\[" else: stuff = pat[i:j].replace('\\', '\\\\') i = j + 1 if stuff[0] == '!': - stuff = '^' + stuff[1:] + stuff = f"^{stuff[1:]}" elif stuff[0] == '^': - stuff = '\\' + stuff + stuff = f"\\{stuff}" res = f'{res}[{stuff}]' else: res = res + re.escape(c) - return res + '$' + return f"{res}$" diff --git a/docker/utils/ports.py b/docker/utils/ports.py index e813936602..9fd6e8f6b8 100644 --- a/docker/utils/ports.py +++ b/docker/utils/ports.py @@ -49,7 +49,7 @@ def port_range(start, end, proto, randomly_available_port=False): if not end: return [start + proto] if randomly_available_port: - return [f'{start}-{end}' + proto] + return [f"{start}-{end}{proto}"] return [str(port) + proto for port in range(int(start), int(end) + 1)] diff --git a/docker/utils/proxy.py b/docker/utils/proxy.py index 49e98ed912..e7164b6cea 100644 --- a/docker/utils/proxy.py +++ b/docker/utils/proxy.py @@ -69,5 +69,9 @@ def inject_proxy_environment(self, environment): return proxy_env + environment def __str__(self): - return 'ProxyConfig(http={}, https={}, ftp={}, no_proxy={})'.format( - self.http, self.https, self.ftp, self.no_proxy) + return ( + 'ProxyConfig(' + f'http={self.http}, https={self.https}, ' + f'ftp={self.ftp}, no_proxy={self.no_proxy}' + ')' + ) diff --git a/docker/utils/utils.py b/docker/utils/utils.py index 7b2bbf4ba1..15e3869000 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -127,8 +127,7 @@ def convert_volume_binds(binds): if isinstance(v, dict): if 'ro' in v and 'mode' in v: raise ValueError( - 'Binding cannot contain both "ro" and "mode": {}' - .format(repr(v)) + f'Binding cannot contain both "ro" and "mode": {v!r}' ) bind = v['bind'] @@ -160,8 +159,8 @@ def convert_tmpfs_mounts(tmpfs): if not isinstance(tmpfs, list): raise ValueError( - 'Expected tmpfs value to be either a list or a dict, found: {}' - .format(type(tmpfs).__name__) + 'Expected tmpfs value to be either a list or a dict, ' + f'found: {type(tmpfs).__name__}' ) result = {} @@ -175,8 +174,8 @@ def convert_tmpfs_mounts(tmpfs): else: raise ValueError( - "Expected item in tmpfs list to be a string, found: {}" - .format(type(mount).__name__) + "Expected item in tmpfs list to be a string, " + f"found: {type(mount).__name__}" ) result[name] = options @@ -218,9 +217,9 @@ def parse_host(addr, is_win32=False, tls=False): parsed_url = urlparse(addr) proto = parsed_url.scheme - if not proto or any([x not in string.ascii_letters + '+' for x in proto]): + if not proto or any([x not in f"{string.ascii_letters}+" for x in proto]): # https://bugs.python.org/issue754016 - parsed_url = urlparse('//' + addr, 'tcp') + parsed_url = urlparse(f"//{addr}", 'tcp') proto = 'tcp' if proto == 'fd': @@ -256,15 +255,14 @@ def parse_host(addr, is_win32=False, tls=False): if parsed_url.path and proto == 'ssh': raise errors.DockerException( - 'Invalid bind address format: no path allowed for this protocol:' - ' {}'.format(addr) + f'Invalid bind address format: no path allowed for this protocol: {addr}' ) else: path = parsed_url.path if proto == 'unix' and parsed_url.hostname is not None: # For legacy reasons, we consider unix://path # to be valid and equivalent to unix:///path - path = '/'.join((parsed_url.hostname, path)) + path = f"{parsed_url.hostname}/{path}" netloc = parsed_url.netloc if proto in ('tcp', 'ssh'): @@ -272,8 +270,7 @@ def parse_host(addr, is_win32=False, tls=False): if port <= 0: if proto != 'ssh': raise errors.DockerException( - 'Invalid bind address format: port is required:' - ' {}'.format(addr) + f'Invalid bind address format: port is required: {addr}' ) port = 22 netloc = f'{parsed_url.netloc}:{port}' @@ -283,7 +280,7 @@ def parse_host(addr, is_win32=False, tls=False): # Rewrite schemes to fit library internals (requests adapters) if proto == 'tcp': - proto = 'http{}'.format('s' if tls else '') + proto = f"http{'s' if tls else ''}" elif proto == 'unix': proto = 'http+unix' @@ -419,17 +416,16 @@ def parse_bytes(s): digits = float(digits_part) except ValueError: raise errors.DockerException( - 'Failed converting the string value for memory ({}) to' - ' an integer.'.format(digits_part) + 'Failed converting the string value for memory ' + f'({digits_part}) to an integer.' ) # Reconvert to long for the final result s = int(digits * units[suffix]) else: raise errors.DockerException( - 'The specified value for memory ({}) should specify the' - ' units. The postfix should be one of the `b` `k` `m` `g`' - ' characters'.format(s) + f'The specified value for memory ({s}) should specify the units. ' + 'The postfix should be one of the `b` `k` `m` `g` characters' ) return s @@ -465,8 +461,7 @@ def parse_env_file(env_file): environment[k] = v else: raise errors.DockerException( - 'Invalid line in environment file {}:\n{}'.format( - env_file, line)) + f'Invalid line in environment file {env_file}:\n{line}') return environment diff --git a/docs/conf.py b/docs/conf.py index dc3b37cc8a..e9971e0d2e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -56,7 +56,7 @@ # General information about the project. project = 'Docker SDK for Python' year = datetime.datetime.now().year -copyright = '%d Docker Inc' % year +copyright = f'{year} Docker Inc' author = 'Docker Inc' # The version info for the project you're documenting, acts as replacement for diff --git a/tests/helpers.py b/tests/helpers.py index bdb07f96b9..e0785774b3 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -80,7 +80,7 @@ def wait_on_condition(condition, delay=0.1, timeout=40): start_time = time.time() while not condition(): if time.time() - start_time > timeout: - raise AssertionError("Timeout: %s" % condition) + raise AssertionError(f"Timeout: {condition}") time.sleep(delay) diff --git a/tests/integration/api_client_test.py b/tests/integration/api_client_test.py index d1622fa88d..d7a22a04af 100644 --- a/tests/integration/api_client_test.py +++ b/tests/integration/api_client_test.py @@ -72,6 +72,4 @@ def test_resource_warnings(self): client.close() del client - assert len(w) == 0, "No warnings produced: {}".format( - w[0].message - ) + assert len(w) == 0, f"No warnings produced: {w[0].message}" diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index b510979de0..590c4fa0ce 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -666,9 +666,7 @@ def test_copy_file_to_container(self): test_file.seek(0) ctnr = self.client.create_container( TEST_IMG, - 'cat {}'.format( - os.path.join('/vol1/', os.path.basename(test_file.name)) - ), + f"cat {os.path.join('/vol1/', os.path.basename(test_file.name))}", volumes=['/vol1'] ) self.tmp_containers.append(ctnr) @@ -826,7 +824,7 @@ def test_logs(self): exitcode = self.client.wait(id)['StatusCode'] assert exitcode == 0 logs = self.client.logs(id) - assert logs == (snippet + '\n').encode(encoding='ascii') + assert logs == f"{snippet}\n".encode(encoding='ascii') def test_logs_tail_option(self): snippet = '''Line1 @@ -857,7 +855,7 @@ def test_logs_streaming_and_follow(self): exitcode = self.client.wait(id)['StatusCode'] assert exitcode == 0 - assert logs == (snippet + '\n').encode(encoding='ascii') + assert logs == f"{snippet}\n".encode(encoding='ascii') @pytest.mark.timeout(5) @pytest.mark.skipif(os.environ.get('DOCKER_HOST', '').startswith('ssh://'), @@ -878,7 +876,7 @@ def test_logs_streaming_and_follow_and_cancel(self): for chunk in generator: logs += chunk - assert logs == (snippet + '\n').encode(encoding='ascii') + assert logs == f"{snippet}\n".encode(encoding='ascii') def test_logs_with_dict_instead_of_id(self): snippet = 'Flowering Nights (Sakuya Iyazoi)' @@ -891,7 +889,7 @@ def test_logs_with_dict_instead_of_id(self): exitcode = self.client.wait(id)['StatusCode'] assert exitcode == 0 logs = self.client.logs(container) - assert logs == (snippet + '\n').encode(encoding='ascii') + assert logs == f"{snippet}\n".encode(encoding='ascii') def test_logs_with_tail_0(self): snippet = 'Flowering Nights (Sakuya Iyazoi)' @@ -920,7 +918,7 @@ def test_logs_with_until(self): logs_until_1 = self.client.logs(container, until=1) assert logs_until_1 == b'' logs_until_now = self.client.logs(container, datetime.now()) - assert logs_until_now == (snippet + '\n').encode(encoding='ascii') + assert logs_until_now == f"{snippet}\n".encode(encoding='ascii') class DiffTest(BaseAPIIntegrationTest): @@ -1086,7 +1084,7 @@ def test_port(self): ip, host_port = port_binding['HostIp'], port_binding['HostPort'] - port_binding = port if not protocol else port + "/" + protocol + port_binding = port if not protocol else f"{port}/{protocol}" assert ip == port_bindings[port_binding][0] assert host_port == port_bindings[port_binding][1] diff --git a/tests/integration/base.py b/tests/integration/base.py index 031079c917..e4073757ee 100644 --- a/tests/integration/base.py +++ b/tests/integration/base.py @@ -103,8 +103,7 @@ def run_container(self, *args, **kwargs): if exitcode != 0: output = self.client.logs(container) raise Exception( - "Container exited with code {}:\n{}" - .format(exitcode, output)) + f"Container exited with code {exitcode}:\n{output}") return container diff --git a/tests/integration/credentials/store_test.py b/tests/integration/credentials/store_test.py index 16f4d60ab4..82ea84741d 100644 --- a/tests/integration/credentials/store_test.py +++ b/tests/integration/credentials/store_test.py @@ -22,7 +22,7 @@ def teardown_method(self): def setup_method(self): self.tmp_keys = [] if sys.platform.startswith('linux'): - if shutil.which('docker-credential-' + DEFAULT_LINUX_STORE): + if shutil.which(f"docker-credential-{DEFAULT_LINUX_STORE}"): self.store = Store(DEFAULT_LINUX_STORE) elif shutil.which('docker-credential-pass'): self.store = Store('pass') diff --git a/tests/integration/models_containers_test.py b/tests/integration/models_containers_test.py index eac4c97909..5b0470b937 100644 --- a/tests/integration/models_containers_test.py +++ b/tests/integration/models_containers_test.py @@ -49,7 +49,7 @@ def test_run_with_volume(self): container = client.containers.run( "alpine", "sh -c 'echo \"hello\" > /insidecontainer/test'", - volumes=["%s:/insidecontainer" % path], + volumes=[f"{path}:/insidecontainer"], detach=True ) self.tmp_containers.append(container.id) @@ -58,7 +58,7 @@ def test_run_with_volume(self): name = "container_volume_test" out = client.containers.run( "alpine", "cat /insidecontainer/test", - volumes=["%s:/insidecontainer" % path], + volumes=[f"{path}:/insidecontainer"], name=name ) self.tmp_containers.append(name) diff --git a/tests/ssh/base.py b/tests/ssh/base.py index 4b91add4be..d6ff130a1d 100644 --- a/tests/ssh/base.py +++ b/tests/ssh/base.py @@ -110,8 +110,7 @@ def run_container(self, *args, **kwargs): if exitcode != 0: output = self.client.logs(container) raise Exception( - "Container exited with code {}:\n{}" - .format(exitcode, output)) + f"Container exited with code {exitcode}:\n{output}") return container diff --git a/tests/unit/api_build_test.py b/tests/unit/api_build_test.py index 7e07a2695e..cbecd1e544 100644 --- a/tests/unit/api_build_test.py +++ b/tests/unit/api_build_test.py @@ -89,7 +89,7 @@ def test_build_remote_with_registry_auth(self): fake_request.assert_called_with( 'POST', - url_prefix + 'build', + f"{url_prefix}build", stream=True, data=None, headers=expected_headers, @@ -193,10 +193,10 @@ def pre(path): 'foo/Dockerfile.foo', None ) assert process_dockerfile( - '../Dockerfile', pre(base + '\\foo') + '../Dockerfile', pre(f"{base}\\foo") )[1] is not None assert process_dockerfile( - '../baz/Dockerfile.baz', pre(base + '/baz') + '../baz/Dockerfile.baz', pre(f"{base}/baz") ) == ('../baz/Dockerfile.baz', None) def test_process_dockerfile(self): @@ -218,8 +218,8 @@ def test_process_dockerfile(self): 'foo/Dockerfile.foo', None ) assert process_dockerfile( - '../Dockerfile', base + '/foo' + '../Dockerfile', f"{base}/foo" )[1] is not None - assert process_dockerfile('../baz/Dockerfile.baz', base + '/baz') == ( + assert process_dockerfile('../baz/Dockerfile.baz', f"{base}/baz") == ( '../baz/Dockerfile.baz', None ) diff --git a/tests/unit/api_exec_test.py b/tests/unit/api_exec_test.py index 4504250846..1760239fd6 100644 --- a/tests/unit/api_exec_test.py +++ b/tests/unit/api_exec_test.py @@ -32,9 +32,7 @@ def test_exec_start(self): self.client.exec_start(fake_api.FAKE_EXEC_ID) args = fake_request.call_args - assert args[0][1] == url_prefix + 'exec/{}/start'.format( - fake_api.FAKE_EXEC_ID - ) + assert args[0][1] == f"{url_prefix}exec/{fake_api.FAKE_EXEC_ID}/start" assert json.loads(args[1]['data']) == { 'Tty': False, @@ -51,9 +49,7 @@ def test_exec_start_detached(self): self.client.exec_start(fake_api.FAKE_EXEC_ID, detach=True) args = fake_request.call_args - assert args[0][1] == url_prefix + 'exec/{}/start'.format( - fake_api.FAKE_EXEC_ID - ) + assert args[0][1] == f"{url_prefix}exec/{fake_api.FAKE_EXEC_ID}/start" assert json.loads(args[1]['data']) == { 'Tty': False, @@ -68,16 +64,14 @@ def test_exec_inspect(self): self.client.exec_inspect(fake_api.FAKE_EXEC_ID) args = fake_request.call_args - assert args[0][1] == url_prefix + 'exec/{}/json'.format( - fake_api.FAKE_EXEC_ID - ) + assert args[0][1] == f"{url_prefix}exec/{fake_api.FAKE_EXEC_ID}/json" def test_exec_resize(self): self.client.exec_resize(fake_api.FAKE_EXEC_ID, height=20, width=60) fake_request.assert_called_with( 'POST', - url_prefix + f'exec/{fake_api.FAKE_EXEC_ID}/resize', + f"{url_prefix}exec/{fake_api.FAKE_EXEC_ID}/resize", params={'h': 20, 'w': 60}, timeout=DEFAULT_TIMEOUT_SECONDS ) diff --git a/tests/unit/api_image_test.py b/tests/unit/api_image_test.py index b3428aa169..aea3a0e136 100644 --- a/tests/unit/api_image_test.py +++ b/tests/unit/api_image_test.py @@ -21,7 +21,7 @@ def test_images(self): fake_request.assert_called_with( 'GET', - url_prefix + 'images/json', + f"{url_prefix}images/json", params={'only_ids': 0, 'all': 1}, timeout=DEFAULT_TIMEOUT_SECONDS ) @@ -31,7 +31,7 @@ def test_images_name(self): fake_request.assert_called_with( 'GET', - url_prefix + 'images/json', + f"{url_prefix}images/json", params={'only_ids': 0, 'all': 0, 'filters': '{"reference": ["foo:bar"]}'}, timeout=DEFAULT_TIMEOUT_SECONDS @@ -42,7 +42,7 @@ def test_images_quiet(self): fake_request.assert_called_with( 'GET', - url_prefix + 'images/json', + f"{url_prefix}images/json", params={'only_ids': 1, 'all': 1}, timeout=DEFAULT_TIMEOUT_SECONDS ) @@ -52,7 +52,7 @@ def test_image_ids(self): fake_request.assert_called_with( 'GET', - url_prefix + 'images/json', + f"{url_prefix}images/json", params={'only_ids': 1, 'all': 0}, timeout=DEFAULT_TIMEOUT_SECONDS ) @@ -62,7 +62,7 @@ def test_images_filters(self): fake_request.assert_called_with( 'GET', - url_prefix + 'images/json', + f"{url_prefix}images/json", params={'only_ids': 0, 'all': 0, 'filters': '{"dangling": ["true"]}'}, timeout=DEFAULT_TIMEOUT_SECONDS @@ -72,7 +72,7 @@ def test_pull(self): self.client.pull('joffrey/test001') args = fake_request.call_args - assert args[0][1] == url_prefix + 'images/create' + assert args[0][1] == f"{url_prefix}images/create" assert args[1]['params'] == { 'tag': 'latest', 'fromImage': 'joffrey/test001' } @@ -82,7 +82,7 @@ def test_pull_stream(self): self.client.pull('joffrey/test001', stream=True) args = fake_request.call_args - assert args[0][1] == url_prefix + 'images/create' + assert args[0][1] == f"{url_prefix}images/create" assert args[1]['params'] == { 'tag': 'latest', 'fromImage': 'joffrey/test001' } @@ -93,7 +93,7 @@ def test_commit(self): fake_request.assert_called_with( 'POST', - url_prefix + 'commit', + f"{url_prefix}commit", data='{}', headers={'Content-Type': 'application/json'}, params={ @@ -113,7 +113,7 @@ def test_remove_image(self): fake_request.assert_called_with( 'DELETE', - url_prefix + 'images/' + fake_api.FAKE_IMAGE_ID, + f"{url_prefix}images/{fake_api.FAKE_IMAGE_ID}", params={'force': False, 'noprune': False}, timeout=DEFAULT_TIMEOUT_SECONDS ) @@ -123,7 +123,7 @@ def test_image_history(self): fake_request.assert_called_with( 'GET', - url_prefix + 'images/test_image/history', + f"{url_prefix}images/test_image/history", timeout=DEFAULT_TIMEOUT_SECONDS ) @@ -136,7 +136,7 @@ def test_import_image(self): fake_request.assert_called_with( 'POST', - url_prefix + 'images/create', + f"{url_prefix}images/create", params={ 'repo': fake_api.FAKE_REPO_NAME, 'tag': fake_api.FAKE_TAG_NAME, @@ -157,7 +157,7 @@ def test_import_image_from_bytes(self): fake_request.assert_called_with( 'POST', - url_prefix + 'images/create', + f"{url_prefix}images/create", params={ 'repo': fake_api.FAKE_REPO_NAME, 'tag': fake_api.FAKE_TAG_NAME, @@ -179,7 +179,7 @@ def test_import_image_from_image(self): fake_request.assert_called_with( 'POST', - url_prefix + 'images/create', + f"{url_prefix}images/create", params={ 'repo': fake_api.FAKE_REPO_NAME, 'tag': fake_api.FAKE_TAG_NAME, @@ -194,7 +194,7 @@ def test_inspect_image(self): fake_request.assert_called_with( 'GET', - url_prefix + 'images/test_image/json', + f"{url_prefix}images/test_image/json", timeout=DEFAULT_TIMEOUT_SECONDS ) @@ -212,7 +212,7 @@ def test_push_image(self): fake_request.assert_called_with( 'POST', - url_prefix + 'images/test_image/push', + f"{url_prefix}images/test_image/push", params={ 'tag': None }, @@ -231,7 +231,7 @@ def test_push_image_with_tag(self): fake_request.assert_called_with( 'POST', - url_prefix + 'images/test_image/push', + f"{url_prefix}images/test_image/push", params={ 'tag': fake_api.FAKE_TAG_NAME, }, @@ -255,7 +255,7 @@ def test_push_image_with_auth(self): fake_request.assert_called_with( 'POST', - url_prefix + 'images/test_image/push', + f"{url_prefix}images/test_image/push", params={ 'tag': fake_api.FAKE_TAG_NAME, }, @@ -273,7 +273,7 @@ def test_push_image_stream(self): fake_request.assert_called_with( 'POST', - url_prefix + 'images/test_image/push', + f"{url_prefix}images/test_image/push", params={ 'tag': None }, @@ -288,7 +288,7 @@ def test_tag_image(self): fake_request.assert_called_with( 'POST', - url_prefix + 'images/' + fake_api.FAKE_IMAGE_ID + '/tag', + f"{url_prefix}images/{fake_api.FAKE_IMAGE_ID}/tag", params={ 'tag': None, 'repo': 'repo', @@ -306,7 +306,7 @@ def test_tag_image_tag(self): fake_request.assert_called_with( 'POST', - url_prefix + 'images/' + fake_api.FAKE_IMAGE_ID + '/tag', + f"{url_prefix}images/{fake_api.FAKE_IMAGE_ID}/tag", params={ 'tag': 'tag', 'repo': 'repo', @@ -321,7 +321,7 @@ def test_tag_image_force(self): fake_request.assert_called_with( 'POST', - url_prefix + 'images/' + fake_api.FAKE_IMAGE_ID + '/tag', + f"{url_prefix}images/{fake_api.FAKE_IMAGE_ID}/tag", params={ 'tag': None, 'repo': 'repo', @@ -335,7 +335,7 @@ def test_get_image(self): fake_request.assert_called_with( 'GET', - url_prefix + 'images/' + fake_api.FAKE_IMAGE_ID + '/get', + f"{url_prefix}images/{fake_api.FAKE_IMAGE_ID}/get", stream=True, timeout=DEFAULT_TIMEOUT_SECONDS ) @@ -345,7 +345,7 @@ def test_load_image(self): fake_request.assert_called_with( 'POST', - url_prefix + 'images/load', + f"{url_prefix}images/load", data='Byte Stream....', stream=True, params={}, @@ -357,7 +357,7 @@ def test_load_image_quiet(self): fake_request.assert_called_with( 'POST', - url_prefix + 'images/load', + f"{url_prefix}images/load", data='Byte Stream....', stream=True, params={'quiet': True}, diff --git a/tests/unit/api_network_test.py b/tests/unit/api_network_test.py index 8afab7379d..d3daa44c41 100644 --- a/tests/unit/api_network_test.py +++ b/tests/unit/api_network_test.py @@ -28,7 +28,7 @@ def test_list_networks(self): with mock.patch('docker.api.client.APIClient.get', get): assert self.client.networks() == networks - assert get.call_args[0][0] == url_prefix + 'networks' + assert get.call_args[0][0] == f"{url_prefix}networks" filters = json.loads(get.call_args[1]['params']['filters']) assert not filters @@ -54,7 +54,7 @@ def test_create_network(self): result = self.client.create_network('foo') assert result == network_data - assert post.call_args[0][0] == url_prefix + 'networks/create' + assert post.call_args[0][0] == f"{url_prefix}networks/create" assert json.loads(post.call_args[1]['data']) == {"Name": "foo"} @@ -97,7 +97,7 @@ def test_remove_network(self): self.client.remove_network(network_id) args = delete.call_args - assert args[0][0] == url_prefix + f'networks/{network_id}' + assert args[0][0] == f"{url_prefix}networks/{network_id}" def test_inspect_network(self): network_id = 'abc12345' @@ -117,7 +117,7 @@ def test_inspect_network(self): assert result == network_data args = get.call_args - assert args[0][0] == url_prefix + f'networks/{network_id}' + assert args[0][0] == f"{url_prefix}networks/{network_id}" def test_connect_container_to_network(self): network_id = 'abc12345' @@ -135,7 +135,7 @@ def test_connect_container_to_network(self): ) assert post.call_args[0][0] == ( - url_prefix + f'networks/{network_id}/connect' + f"{url_prefix}networks/{network_id}/connect" ) assert json.loads(post.call_args[1]['data']) == { @@ -158,7 +158,7 @@ def test_disconnect_container_from_network(self): container={'Id': container_id}, net_id=network_id) assert post.call_args[0][0] == ( - url_prefix + f'networks/{network_id}/disconnect' + f"{url_prefix}networks/{network_id}/disconnect" ) assert json.loads(post.call_args[1]['data']) == { 'Container': container_id diff --git a/tests/unit/api_test.py b/tests/unit/api_test.py index 4b6099c904..78c0bab12e 100644 --- a/tests/unit/api_test.py +++ b/tests/unit/api_test.py @@ -86,9 +86,7 @@ def fake_read_from_socket(self, response, stream, tty=False, demux=False): url_base = f'{fake_api.prefix}/' -url_prefix = '{}v{}/'.format( - url_base, - docker.constants.DEFAULT_DOCKER_API_VERSION) +url_prefix = f'{url_base}v{docker.constants.DEFAULT_DOCKER_API_VERSION}/' class BaseAPIClientTest(unittest.TestCase): @@ -130,22 +128,18 @@ def test_ctor(self): def test_url_valid_resource(self): url = self.client._url('/hello/{0}/world', 'somename') - assert url == '{}{}'.format(url_prefix, 'hello/somename/world') + assert url == f"{url_prefix}hello/somename/world" url = self.client._url( '/hello/{0}/world/{1}', 'somename', 'someothername' ) - assert url == '{}{}'.format( - url_prefix, 'hello/somename/world/someothername' - ) + assert url == f"{url_prefix}hello/somename/world/someothername" url = self.client._url('/hello/{0}/world', 'some?name') - assert url == '{}{}'.format(url_prefix, 'hello/some%3Fname/world') + assert url == f"{url_prefix}hello/some%3Fname/world" url = self.client._url("/images/{0}/push", "localhost:5000/image") - assert url == '{}{}'.format( - url_prefix, 'images/localhost:5000/image/push' - ) + assert url == f"{url_prefix}images/localhost:5000/image/push" def test_url_invalid_resource(self): with pytest.raises(ValueError): @@ -153,20 +147,20 @@ def test_url_invalid_resource(self): def test_url_no_resource(self): url = self.client._url('/simple') - assert url == '{}{}'.format(url_prefix, 'simple') + assert url == f"{url_prefix}simple" def test_url_unversioned_api(self): url = self.client._url( '/hello/{0}/world', 'somename', versioned_api=False ) - assert url == '{}{}'.format(url_base, 'hello/somename/world') + assert url == f"{url_base}hello/somename/world" def test_version(self): self.client.version() fake_request.assert_called_with( 'GET', - url_prefix + 'version', + f"{url_prefix}version", timeout=DEFAULT_TIMEOUT_SECONDS ) @@ -175,7 +169,7 @@ def test_version_no_api_version(self): fake_request.assert_called_with( 'GET', - url_base + 'version', + f"{url_base}version", timeout=DEFAULT_TIMEOUT_SECONDS ) @@ -194,7 +188,7 @@ def test_info(self): fake_request.assert_called_with( 'GET', - url_prefix + 'info', + f"{url_prefix}info", timeout=DEFAULT_TIMEOUT_SECONDS ) @@ -203,7 +197,7 @@ def test_search(self): fake_request.assert_called_with( 'GET', - url_prefix + 'images/search', + f"{url_prefix}images/search", params={'term': 'busybox'}, timeout=DEFAULT_TIMEOUT_SECONDS ) @@ -212,7 +206,7 @@ def test_login(self): self.client.login('sakuya', 'izayoi') args = fake_request.call_args assert args[0][0] == 'POST' - assert args[0][1] == url_prefix + 'auth' + assert args[0][1] == f"{url_prefix}auth" assert json.loads(args[1]['data']) == { 'username': 'sakuya', 'password': 'izayoi' } @@ -229,7 +223,7 @@ def test_events(self): fake_request.assert_called_with( 'GET', - url_prefix + 'events', + f"{url_prefix}events", params={'since': None, 'until': None, 'filters': None}, stream=True, timeout=None @@ -245,7 +239,7 @@ def test_events_with_since_until(self): fake_request.assert_called_with( 'GET', - url_prefix + 'events', + f"{url_prefix}events", params={ 'since': ts - 10, 'until': ts + 10, @@ -264,7 +258,7 @@ def test_events_with_filters(self): expected_filters = docker.utils.convert_filters(filters) fake_request.assert_called_with( 'GET', - url_prefix + 'events', + f"{url_prefix}events", params={ 'since': None, 'until': None, @@ -318,7 +312,7 @@ def test_remove_link(self): fake_request.assert_called_with( 'DELETE', - url_prefix + 'containers/' + fake_api.FAKE_CONTAINER_ID, + f"{url_prefix}containers/{fake_api.FAKE_CONTAINER_ID}", params={'v': False, 'link': True, 'force': False}, timeout=DEFAULT_TIMEOUT_SECONDS ) @@ -332,7 +326,7 @@ def test_create_host_config_secopt(self): self.client.create_host_config(security_opt='wrong') def test_stream_helper_decoding(self): - status_code, content = fake_api.fake_responses[url_prefix + 'events']() + status_code, content = fake_api.fake_responses[f"{url_prefix}events"]() content_str = json.dumps(content) content_str = content_str.encode('utf-8') body = io.BytesIO(content_str) @@ -443,7 +437,7 @@ def test_early_stream_response(self): lines = [] for i in range(0, 50): line = str(i).encode() - lines += [('%x' % len(line)).encode(), line] + lines += [f'{len(line):x}'.encode(), line] lines.append(b'0') lines.append(b'') @@ -454,7 +448,7 @@ def test_early_stream_response(self): ) + b'\r\n'.join(lines) with APIClient( - base_url="http+unix://" + self.socket_file, + base_url=f"http+unix://{self.socket_file}", version=DEFAULT_DOCKER_API_VERSION) as client: for i in range(5): try: @@ -490,8 +484,7 @@ def setup_class(cls): cls.thread = threading.Thread(target=cls.server.serve_forever) cls.thread.daemon = True cls.thread.start() - cls.address = 'http://{}:{}'.format( - socket.gethostname(), cls.server.server_address[1]) + cls.address = f'http://{socket.gethostname()}:{cls.server.server_address[1]}' @classmethod def teardown_class(cls): @@ -600,7 +593,7 @@ def setUp(self): self.patcher = mock.patch.object( APIClient, 'send', - return_value=fake_resp("GET", "%s/version" % fake_api.prefix) + return_value=fake_resp("GET", f"{fake_api.prefix}/version") ) self.mock_send = self.patcher.start() @@ -613,7 +606,7 @@ def test_default_user_agent(self): assert self.mock_send.call_count == 1 headers = self.mock_send.call_args[0][0].headers - expected = 'docker-sdk-python/%s' % docker.__version__ + expected = f'docker-sdk-python/{docker.__version__}' assert headers['User-Agent'] == expected def test_custom_user_agent(self): diff --git a/tests/unit/api_volume_test.py b/tests/unit/api_volume_test.py index a8d9193f75..0a97ca5150 100644 --- a/tests/unit/api_volume_test.py +++ b/tests/unit/api_volume_test.py @@ -14,7 +14,7 @@ def test_list_volumes(self): args = fake_request.call_args assert args[0][0] == 'GET' - assert args[0][1] == url_prefix + 'volumes' + assert args[0][1] == f"{url_prefix}volumes" def test_list_volumes_and_filters(self): volumes = self.client.volumes(filters={'dangling': True}) @@ -23,7 +23,7 @@ def test_list_volumes_and_filters(self): args = fake_request.call_args assert args[0][0] == 'GET' - assert args[0][1] == url_prefix + 'volumes' + assert args[0][1] == f"{url_prefix}volumes" assert args[1] == {'params': {'filters': '{"dangling": ["true"]}'}, 'timeout': 60} @@ -37,7 +37,7 @@ def test_create_volume(self): args = fake_request.call_args assert args[0][0] == 'POST' - assert args[0][1] == url_prefix + 'volumes/create' + assert args[0][1] == f"{url_prefix}volumes/create" assert json.loads(args[1]['data']) == {'Name': name} @requires_api_version('1.23') @@ -63,7 +63,7 @@ def test_create_volume_with_driver(self): args = fake_request.call_args assert args[0][0] == 'POST' - assert args[0][1] == url_prefix + 'volumes/create' + assert args[0][1] == f"{url_prefix}volumes/create" data = json.loads(args[1]['data']) assert 'Driver' in data assert data['Driver'] == driver_name diff --git a/tests/unit/auth_test.py b/tests/unit/auth_test.py index dd5b5f8b57..26254fadde 100644 --- a/tests/unit/auth_test.py +++ b/tests/unit/auth_test.py @@ -290,9 +290,10 @@ def test_load_config_with_random_name(self): folder = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, folder) - dockercfg_path = os.path.join(folder, - '.{}.dockercfg'.format( - random.randrange(100000))) + dockercfg_path = os.path.join( + folder, + f'.{random.randrange(100000)}.dockercfg', + ) registry = 'https://your.private.registry.io' auth_ = base64.b64encode(b'sakuya:izayoi').decode('ascii') config = { diff --git a/tests/unit/client_test.py b/tests/unit/client_test.py index e7c7eec827..1148d7ac1c 100644 --- a/tests/unit/client_test.py +++ b/tests/unit/client_test.py @@ -85,10 +85,7 @@ def test_default_pool_size_unix(self, mock_obj): mock_obj.return_value.urlopen.return_value.status = 200 client.ping() - base_url = "{base_url}/v{version}/_ping".format( - base_url=client.api.base_url, - version=client.api._version - ) + base_url = f"{client.api.base_url}/v{client.api._version}/_ping" mock_obj.assert_called_once_with(base_url, "/var/run/docker.sock", @@ -124,10 +121,7 @@ def test_pool_size_unix(self, mock_obj): mock_obj.return_value.urlopen.return_value.status = 200 client.ping() - base_url = "{base_url}/v{version}/_ping".format( - base_url=client.api.base_url, - version=client.api._version - ) + base_url = f"{client.api.base_url}/v{client.api._version}/_ping" mock_obj.assert_called_once_with(base_url, "/var/run/docker.sock", @@ -198,10 +192,7 @@ def test_default_pool_size_from_env_unix(self, mock_obj): mock_obj.return_value.urlopen.return_value.status = 200 client.ping() - base_url = "{base_url}/v{version}/_ping".format( - base_url=client.api.base_url, - version=client.api._version - ) + base_url = f"{client.api.base_url}/v{client.api._version}/_ping" mock_obj.assert_called_once_with(base_url, "/var/run/docker.sock", @@ -235,10 +226,7 @@ def test_pool_size_from_env_unix(self, mock_obj): mock_obj.return_value.urlopen.return_value.status = 200 client.ping() - base_url = "{base_url}/v{version}/_ping".format( - base_url=client.api.base_url, - version=client.api._version - ) + base_url = f"{client.api.base_url}/v{client.api._version}/_ping" mock_obj.assert_called_once_with(base_url, "/var/run/docker.sock", diff --git a/tests/unit/fake_api.py b/tests/unit/fake_api.py index 6acfb64b8c..133a99f80e 100644 --- a/tests/unit/fake_api.py +++ b/tests/unit/fake_api.py @@ -617,17 +617,11 @@ def post_fake_secret(): get_fake_volume_list, (f'{prefix}/{CURRENT_VERSION}/volumes/create', 'POST'): get_fake_volume, - ('{1}/{0}/volumes/{2}'.format( - CURRENT_VERSION, prefix, FAKE_VOLUME_NAME - ), 'GET'): + (f'{prefix}/{CURRENT_VERSION}/volumes/{FAKE_VOLUME_NAME}', 'GET'): get_fake_volume, - ('{1}/{0}/volumes/{2}'.format( - CURRENT_VERSION, prefix, FAKE_VOLUME_NAME - ), 'DELETE'): + (f'{prefix}/{CURRENT_VERSION}/volumes/{FAKE_VOLUME_NAME}', 'DELETE'): fake_remove_volume, - ('{1}/{0}/nodes/{2}/update?version=1'.format( - CURRENT_VERSION, prefix, FAKE_NODE_ID - ), 'POST'): + (f'{prefix}/{CURRENT_VERSION}/nodes/{FAKE_NODE_ID}/update?version=1', 'POST'): post_fake_update_node, (f'{prefix}/{CURRENT_VERSION}/swarm/join', 'POST'): post_fake_join_swarm, @@ -635,21 +629,13 @@ def post_fake_secret(): get_fake_network_list, (f'{prefix}/{CURRENT_VERSION}/networks/create', 'POST'): post_fake_network, - ('{1}/{0}/networks/{2}'.format( - CURRENT_VERSION, prefix, FAKE_NETWORK_ID - ), 'GET'): + (f'{prefix}/{CURRENT_VERSION}/networks/{FAKE_NETWORK_ID}', 'GET'): get_fake_network, - ('{1}/{0}/networks/{2}'.format( - CURRENT_VERSION, prefix, FAKE_NETWORK_ID - ), 'DELETE'): + (f'{prefix}/{CURRENT_VERSION}/networks/{FAKE_NETWORK_ID}', 'DELETE'): delete_fake_network, - ('{1}/{0}/networks/{2}/connect'.format( - CURRENT_VERSION, prefix, FAKE_NETWORK_ID - ), 'POST'): + (f'{prefix}/{CURRENT_VERSION}/networks/{FAKE_NETWORK_ID}/connect', 'POST'): post_fake_network_connect, - ('{1}/{0}/networks/{2}/disconnect'.format( - CURRENT_VERSION, prefix, FAKE_NETWORK_ID - ), 'POST'): + (f'{prefix}/{CURRENT_VERSION}/networks/{FAKE_NETWORK_ID}/disconnect', 'POST'): post_fake_network_disconnect, f'{prefix}/{CURRENT_VERSION}/secrets/create': post_fake_secret, diff --git a/tests/unit/swarm_test.py b/tests/unit/swarm_test.py index aee1b9e802..3fc7c68cd5 100644 --- a/tests/unit/swarm_test.py +++ b/tests/unit/swarm_test.py @@ -20,7 +20,7 @@ def test_node_update(self): ) args = fake_request.call_args assert args[0][1] == ( - url_prefix + 'nodes/24ifsmvkjbyhk/update?version=1' + f"{url_prefix}nodes/24ifsmvkjbyhk/update?version=1" ) assert json.loads(args[1]['data']) == node_spec assert args[1]['headers']['Content-Type'] == 'application/json' @@ -45,7 +45,7 @@ def test_join_swarm(self): args = fake_request.call_args - assert (args[0][1] == url_prefix + 'swarm/join') + assert (args[0][1] == f"{url_prefix}swarm/join") assert (json.loads(args[1]['data']) == data) assert (args[1]['headers']['Content-Type'] == 'application/json') @@ -64,6 +64,6 @@ def test_join_swarm_no_listen_address_takes_default(self): args = fake_request.call_args - assert (args[0][1] == url_prefix + 'swarm/join') + assert (args[0][1] == f"{url_prefix}swarm/join") assert (json.loads(args[1]['data']) == data) assert (args[1]['headers']['Content-Type'] == 'application/json') diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index 12cb7bd657..9c8a55bd52 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -486,9 +486,9 @@ def test_split_port_with_host_ip(self): def test_split_port_with_protocol(self): for protocol in ['tcp', 'udp', 'sctp']: internal_port, external_port = split_port( - "127.0.0.1:1000:2000/" + protocol + f"127.0.0.1:1000:2000/{protocol}" ) - assert internal_port == ["2000/" + protocol] + assert internal_port == [f"2000/{protocol}"] assert external_port == [("127.0.0.1", "1000")] def test_split_port_with_host_ip_no_port(self): From 9313536601064f8e7654af655604e4de24483b09 Mon Sep 17 00:00:00 2001 From: Aarni Koskela Date: Thu, 11 May 2023 15:45:07 +0300 Subject: [PATCH 1232/1301] Switch linting from flake8 to ruff Signed-off-by: Aarni Koskela --- .github/workflows/ci.yml | 10 +++++----- CONTRIBUTING.md | 2 +- Makefile | 8 ++++---- docker/__init__.py | 1 - docker/api/__init__.py | 1 - docker/context/__init__.py | 1 - docker/credentials/__init__.py | 4 ++-- docker/transport/__init__.py | 1 - docker/types/__init__.py | 1 - docker/utils/__init__.py | 2 +- pyproject.toml | 3 +++ test-requirements.txt | 2 +- tox.ini | 6 +++--- 13 files changed, 20 insertions(+), 22 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f23873f0e8..dfbcc701eb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,16 +6,16 @@ env: DOCKER_BUILDKIT: '1' jobs: - flake8: + lint: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 with: - python-version: '3.x' - - run: pip install -U flake8 - - name: Run flake8 - run: flake8 docker/ tests/ + python-version: '3.11' + - run: pip install -U ruff==0.0.284 + - name: Run ruff + run: ruff docker tests unit-tests: runs-on: ubuntu-latest diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 861731188c..acf22ef7ca 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -44,7 +44,7 @@ paragraph in the Docker contribution guidelines. Before we can review your pull request, please ensure that nothing has been broken by your changes by running the test suite. You can do so simply by running `make test` in the project root. This also includes coding style using -`flake8` +`ruff` ### 3. Write clear, self-contained commits diff --git a/Makefile b/Makefile index ae6ae34ef2..79486e3ec2 100644 --- a/Makefile +++ b/Makefile @@ -46,7 +46,7 @@ build-dind-certs: docker build -t dpy-dind-certs -f tests/Dockerfile-dind-certs . .PHONY: test -test: flake8 unit-test-py3 integration-dind integration-dind-ssl +test: ruff unit-test-py3 integration-dind integration-dind-ssl .PHONY: unit-test-py3 unit-test-py3: build-py3 @@ -163,9 +163,9 @@ integration-dind-ssl: build-dind-certs build-py3 setup-network docker rm -vf dpy-dind-ssl dpy-dind-certs -.PHONY: flake8 -flake8: build-py3 - docker run -t --rm docker-sdk-python3 flake8 docker tests +.PHONY: ruff +ruff: build-py3 + docker run -t --rm docker-sdk-python3 ruff docker tests .PHONY: docs docs: build-docs diff --git a/docker/__init__.py b/docker/__init__.py index 46beb532a7..c1c518c56d 100644 --- a/docker/__init__.py +++ b/docker/__init__.py @@ -1,4 +1,3 @@ -# flake8: noqa from .api import APIClient from .client import DockerClient, from_env from .context import Context diff --git a/docker/api/__init__.py b/docker/api/__init__.py index ff5184414b..7260e9537e 100644 --- a/docker/api/__init__.py +++ b/docker/api/__init__.py @@ -1,2 +1 @@ -# flake8: noqa from .client import APIClient diff --git a/docker/context/__init__.py b/docker/context/__init__.py index 0a6707f997..dbf172fdac 100644 --- a/docker/context/__init__.py +++ b/docker/context/__init__.py @@ -1,3 +1,2 @@ -# flake8: noqa from .context import Context from .api import ContextAPI diff --git a/docker/credentials/__init__.py b/docker/credentials/__init__.py index 31ad28e34d..db3e1fbfd0 100644 --- a/docker/credentials/__init__.py +++ b/docker/credentials/__init__.py @@ -1,4 +1,4 @@ -# flake8: noqa + from .store import Store from .errors import StoreError, CredentialsNotFound -from .constants import * +from .constants import * # noqa: F403 diff --git a/docker/transport/__init__.py b/docker/transport/__init__.py index e37fc3ba21..54492c11ac 100644 --- a/docker/transport/__init__.py +++ b/docker/transport/__init__.py @@ -1,4 +1,3 @@ -# flake8: noqa from .unixconn import UnixHTTPAdapter from .ssladapter import SSLHTTPAdapter try: diff --git a/docker/types/__init__.py b/docker/types/__init__.py index b425746e78..89f2238934 100644 --- a/docker/types/__init__.py +++ b/docker/types/__init__.py @@ -1,4 +1,3 @@ -# flake8: noqa from .containers import ( ContainerConfig, HostConfig, LogConfig, Ulimit, DeviceRequest ) diff --git a/docker/utils/__init__.py b/docker/utils/__init__.py index 81c8186c84..944c6e65e0 100644 --- a/docker/utils/__init__.py +++ b/docker/utils/__init__.py @@ -1,4 +1,4 @@ -# flake8: noqa + from .build import create_archive, exclude_paths, mkbuildcontext, tar from .decorators import check_resource, minimum_version, update_headers from .utils import ( diff --git a/pyproject.toml b/pyproject.toml index 9554358e56..eb87cefbbf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,3 +3,6 @@ requires = ["setuptools>=45", "setuptools_scm[toml]>=6.2"] [tool.setuptools_scm] write_to = 'docker/_version.py' + +[tool.ruff.per-file-ignores] +"**/__init__.py" = ["F401"] diff --git a/test-requirements.txt b/test-requirements.txt index b7457fa773..951b3be9fc 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,6 +1,6 @@ setuptools==65.5.1 coverage==6.4.2 -flake8==4.0.1 +ruff==0.0.284 pytest==7.1.2 pytest-cov==3.0.0 pytest-timeout==2.1.0 diff --git a/tox.ini b/tox.ini index 9edc15c54e..2028dd3957 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py{37,38,39,310,311}, flake8 +envlist = py{37,38,39,310,311}, ruff skipsdist=True [testenv] @@ -10,7 +10,7 @@ deps = -r{toxinidir}/test-requirements.txt -r{toxinidir}/requirements.txt -[testenv:flake8] -commands = flake8 docker tests setup.py +[testenv:ruff] +commands = ruff docker tests setup.py deps = -r{toxinidir}/test-requirements.txt From fad792bfc76852a17fb7d717b23122aeb7b0bbd6 Mon Sep 17 00:00:00 2001 From: Aarni Koskela Date: Thu, 11 May 2023 15:50:13 +0300 Subject: [PATCH 1233/1301] Get rid of star import Signed-off-by: Aarni Koskela --- docker/credentials/__init__.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docker/credentials/__init__.py b/docker/credentials/__init__.py index db3e1fbfd0..a1247700d3 100644 --- a/docker/credentials/__init__.py +++ b/docker/credentials/__init__.py @@ -1,4 +1,8 @@ - from .store import Store from .errors import StoreError, CredentialsNotFound -from .constants import * # noqa: F403 +from .constants import ( + DEFAULT_LINUX_STORE, + DEFAULT_OSX_STORE, + DEFAULT_WIN32_STORE, + PROGRAM_PREFIX, +) From ec58856ee3145eef2b0d02e72d3ee4548b241b0e Mon Sep 17 00:00:00 2001 From: Aarni Koskela Date: Thu, 11 May 2023 15:52:37 +0300 Subject: [PATCH 1234/1301] Clean up unnecessary noqa comments Signed-off-by: Aarni Koskela --- docker/types/containers.py | 7 +++++-- docs/conf.py | 2 +- setup.py | 2 +- tests/integration/api_client_test.py | 2 +- tests/unit/fake_api.py | 6 +++--- 5 files changed, 11 insertions(+), 8 deletions(-) diff --git a/docker/types/containers.py b/docker/types/containers.py index 6d54aa65cc..a28061383d 100644 --- a/docker/types/containers.py +++ b/docker/types/containers.py @@ -48,8 +48,11 @@ class LogConfig(DictType): >>> container = client.create_container('busybox', 'true', ... host_config=hc) >>> client.inspect_container(container)['HostConfig']['LogConfig'] - {'Type': 'json-file', 'Config': {'labels': 'production_status,geo', 'max-size': '1g'}} - """ # noqa: E501 + { + 'Type': 'json-file', + 'Config': {'labels': 'production_status,geo', 'max-size': '1g'} + } + """ types = LogConfigTypesEnum def __init__(self, **kwargs): diff --git a/docs/conf.py b/docs/conf.py index e9971e0d2e..a529f8be82 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -18,6 +18,7 @@ import datetime import os import sys +from importlib.metadata import version sys.path.insert(0, os.path.abspath('..')) @@ -64,7 +65,6 @@ # built documents. # # see https://github.com/pypa/setuptools_scm#usage-from-sphinx -from importlib.metadata import version release = version('docker') # for example take major/minor version = '.'.join(release.split('.')[:2]) diff --git a/setup.py b/setup.py index ff6da71419..d0145d235e 100644 --- a/setup.py +++ b/setup.py @@ -48,7 +48,7 @@ url='https://github.com/docker/docker-py', project_urls={ 'Documentation': 'https://docker-py.readthedocs.io', - 'Changelog': 'https://docker-py.readthedocs.io/en/stable/change-log.html', # noqa: E501 + 'Changelog': 'https://docker-py.readthedocs.io/en/stable/change-log.html', 'Source': 'https://github.com/docker/docker-py', 'Tracker': 'https://github.com/docker/docker-py/issues', }, diff --git a/tests/integration/api_client_test.py b/tests/integration/api_client_test.py index d7a22a04af..ae71a57bf9 100644 --- a/tests/integration/api_client_test.py +++ b/tests/integration/api_client_test.py @@ -47,7 +47,7 @@ def test_timeout(self): # This call isn't supposed to complete, and it should fail fast. try: res = self.client.inspect_container('id') - except: # noqa: E722 + except Exception: pass end = time.time() assert res is None diff --git a/tests/unit/fake_api.py b/tests/unit/fake_api.py index 133a99f80e..87d8927578 100644 --- a/tests/unit/fake_api.py +++ b/tests/unit/fake_api.py @@ -4,10 +4,10 @@ CURRENT_VERSION = f'v{constants.DEFAULT_DOCKER_API_VERSION}' -FAKE_CONTAINER_ID = '81cf499cc928ce3fedc250a080d2b9b978df20e4517304c45211e8a68b33e254' # noqa: E501 +FAKE_CONTAINER_ID = '81cf499cc928ce3fedc250a080d2b9b978df20e4517304c45211e8a68b33e254' FAKE_IMAGE_ID = 'sha256:fe7a8fc91d3f17835cbb3b86a1c60287500ab01a53bc79c4497d09f07a3f0688' # noqa: E501 -FAKE_EXEC_ID = 'b098ec855f10434b5c7c973c78484208223a83f663ddaefb0f02a242840cb1c7' # noqa: E501 -FAKE_NETWORK_ID = '1999cfb42e414483841a125ade3c276c3cb80cb3269b14e339354ac63a31b02c' # noqa: E501 +FAKE_EXEC_ID = 'b098ec855f10434b5c7c973c78484208223a83f663ddaefb0f02a242840cb1c7' +FAKE_NETWORK_ID = '1999cfb42e414483841a125ade3c276c3cb80cb3269b14e339354ac63a31b02c' FAKE_IMAGE_NAME = 'test_image' FAKE_TARBALL_PATH = '/path/to/tarball' FAKE_REPO_NAME = 'repo' From 601476733c26aeff303b27c9da59701c86d49742 Mon Sep 17 00:00:00 2001 From: Aarni Koskela Date: Thu, 11 May 2023 15:55:09 +0300 Subject: [PATCH 1235/1301] Enable Ruff C rules and autofix Signed-off-by: Aarni Koskela --- docker/api/client.py | 2 +- docker/utils/utils.py | 2 +- pyproject.toml | 11 ++ setup.py | 2 +- tests/integration/api_build_test.py | 6 +- tests/integration/api_healthcheck_test.py | 40 ++--- tests/integration/api_image_test.py | 6 +- tests/integration/api_plugin_test.py | 4 +- tests/integration/context_api_test.py | 2 +- tests/integration/models_containers_test.py | 8 +- tests/integration/models_images_test.py | 4 +- tests/integration/models_networks_test.py | 4 +- tests/ssh/api_build_test.py | 6 +- tests/unit/models_containers_test.py | 190 ++++++++++---------- 14 files changed, 147 insertions(+), 140 deletions(-) diff --git a/docker/api/client.py b/docker/api/client.py index 8633025f3f..ce1b3a307b 100644 --- a/docker/api/client.py +++ b/docker/api/client.py @@ -476,7 +476,7 @@ def _get_result_tty(self, stream, res, is_tty): return self._multiplexed_response_stream_helper(res) else: return sep.join( - [x for x in self._multiplexed_buffer_helper(res)] + list(self._multiplexed_buffer_helper(res)) ) def _unmount(self, *args): diff --git a/docker/utils/utils.py b/docker/utils/utils.py index 15e3869000..234be32076 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -217,7 +217,7 @@ def parse_host(addr, is_win32=False, tls=False): parsed_url = urlparse(addr) proto = parsed_url.scheme - if not proto or any([x not in f"{string.ascii_letters}+" for x in proto]): + if not proto or any(x not in f"{string.ascii_letters}+" for x in proto): # https://bugs.python.org/issue754016 parsed_url = urlparse(f"//{addr}", 'tcp') proto = 'tcp' diff --git a/pyproject.toml b/pyproject.toml index eb87cefbbf..82d4869006 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,5 +4,16 @@ requires = ["setuptools>=45", "setuptools_scm[toml]>=6.2"] [tool.setuptools_scm] write_to = 'docker/_version.py' +[tool.ruff] +target-version = "py37" +extend-select = [ + "C", + "F", + "W", +] +ignore = [ + "C901", # too complex (there's a whole bunch of these) +] + [tool.ruff.per-file-ignores] "**/__init__.py" = ["F401"] diff --git a/setup.py b/setup.py index d0145d235e..866aa23c8d 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ } with open('./test-requirements.txt') as test_reqs_txt: - test_requirements = [line for line in test_reqs_txt] + test_requirements = list(test_reqs_txt) long_description = '' diff --git a/tests/integration/api_build_test.py b/tests/integration/api_build_test.py index 606c3b7e11..e5e7904d8f 100644 --- a/tests/integration/api_build_test.py +++ b/tests/integration/api_build_test.py @@ -142,7 +142,7 @@ def test_build_with_dockerignore(self): logs = logs.decode('utf-8') - assert sorted(list(filter(None, logs.split('\n')))) == sorted([ + assert sorted(filter(None, logs.split('\n'))) == sorted([ '/test/#file.txt', '/test/ignored/subdir/excepted-with-spaces', '/test/ignored/subdir/excepted-file', @@ -312,7 +312,7 @@ def test_build_with_network_mode(self): ) self.tmp_imgs.append('dockerpytest_nonebuild') - logs = [chunk for chunk in stream] + logs = list(stream) assert 'errorDetail' in logs[-1] assert logs[-1]['errorDetail']['code'] == 1 @@ -392,7 +392,7 @@ def test_build_stderr_data(self): expected = '{0}{2}\n{1}'.format( control_chars[0], control_chars[1], snippet ) - assert any([line == expected for line in lines]) + assert any(line == expected for line in lines) def test_build_gzip_encoding(self): base_dir = tempfile.mkdtemp() diff --git a/tests/integration/api_healthcheck_test.py b/tests/integration/api_healthcheck_test.py index c54583b0be..9ecdcd86a6 100644 --- a/tests/integration/api_healthcheck_test.py +++ b/tests/integration/api_healthcheck_test.py @@ -16,7 +16,7 @@ class HealthcheckTest(BaseAPIIntegrationTest): @helpers.requires_api_version('1.24') def test_healthcheck_shell_command(self): container = self.client.create_container( - TEST_IMG, 'top', healthcheck=dict(test='echo "hello world"')) + TEST_IMG, 'top', healthcheck={'test': 'echo "hello world"'}) self.tmp_containers.append(container) res = self.client.inspect_container(container) @@ -27,12 +27,12 @@ def test_healthcheck_shell_command(self): @helpers.requires_api_version('1.24') def test_healthcheck_passes(self): container = self.client.create_container( - TEST_IMG, 'top', healthcheck=dict( - test="true", - interval=1 * SECOND, - timeout=1 * SECOND, - retries=1, - )) + TEST_IMG, 'top', healthcheck={ + 'test': "true", + 'interval': 1 * SECOND, + 'timeout': 1 * SECOND, + 'retries': 1, + }) self.tmp_containers.append(container) self.client.start(container) wait_on_health_status(self.client, container, "healthy") @@ -40,12 +40,12 @@ def test_healthcheck_passes(self): @helpers.requires_api_version('1.24') def test_healthcheck_fails(self): container = self.client.create_container( - TEST_IMG, 'top', healthcheck=dict( - test="false", - interval=1 * SECOND, - timeout=1 * SECOND, - retries=1, - )) + TEST_IMG, 'top', healthcheck={ + 'test': "false", + 'interval': 1 * SECOND, + 'timeout': 1 * SECOND, + 'retries': 1, + }) self.tmp_containers.append(container) self.client.start(container) wait_on_health_status(self.client, container, "unhealthy") @@ -53,14 +53,14 @@ def test_healthcheck_fails(self): @helpers.requires_api_version('1.29') def test_healthcheck_start_period(self): container = self.client.create_container( - TEST_IMG, 'top', healthcheck=dict( - test="echo 'x' >> /counter.txt && " + TEST_IMG, 'top', healthcheck={ + 'test': "echo 'x' >> /counter.txt && " "test `cat /counter.txt | wc -l` -ge 3", - interval=1 * SECOND, - timeout=1 * SECOND, - retries=1, - start_period=3 * SECOND - ) + 'interval': 1 * SECOND, + 'timeout': 1 * SECOND, + 'retries': 1, + 'start_period': 3 * SECOND + } ) self.tmp_containers.append(container) diff --git a/tests/integration/api_image_test.py b/tests/integration/api_image_test.py index cb3d667112..7081b53b8f 100644 --- a/tests/integration/api_image_test.py +++ b/tests/integration/api_image_test.py @@ -263,10 +263,8 @@ def test_get_load_image(self): data = self.client.get_image(test_img) assert data output = self.client.load_image(data) - assert any([ - line for line in output - if f'Loaded image: {test_img}' in line.get('stream', '') - ]) + assert any(line for line in output + if f'Loaded image: {test_img}' in line.get('stream', '')) @contextlib.contextmanager def temporary_http_file_server(self, stream): diff --git a/tests/integration/api_plugin_test.py b/tests/integration/api_plugin_test.py index 3ecb028346..a35c30d3e9 100644 --- a/tests/integration/api_plugin_test.py +++ b/tests/integration/api_plugin_test.py @@ -118,7 +118,7 @@ def test_install_plugin(self): pass prv = self.client.plugin_privileges(SSHFS) - logs = [d for d in self.client.pull_plugin(SSHFS, prv)] + logs = list(self.client.pull_plugin(SSHFS, prv)) assert filter(lambda x: x['status'] == 'Download complete', logs) assert self.client.inspect_plugin(SSHFS) assert self.client.enable_plugin(SSHFS) @@ -128,7 +128,7 @@ def test_upgrade_plugin(self): pl_data = self.ensure_plugin_installed(SSHFS) assert pl_data['Enabled'] is False prv = self.client.plugin_privileges(SSHFS) - logs = [d for d in self.client.upgrade_plugin(SSHFS, SSHFS, prv)] + logs = list(self.client.upgrade_plugin(SSHFS, SSHFS, prv)) assert filter(lambda x: x['status'] == 'Download complete', logs) assert self.client.inspect_plugin(SSHFS) assert self.client.enable_plugin(SSHFS) diff --git a/tests/integration/context_api_test.py b/tests/integration/context_api_test.py index a2a12a5cb0..1a13f2817e 100644 --- a/tests/integration/context_api_test.py +++ b/tests/integration/context_api_test.py @@ -29,7 +29,7 @@ def test_lifecycle(self): "test", tls_cfg=docker_tls) # check for a context 'test' in the context store - assert any([ctx.Name == "test" for ctx in ContextAPI.contexts()]) + assert any(ctx.Name == "test" for ctx in ContextAPI.contexts()) # retrieve a context object for 'test' assert ContextAPI.get_context("test") # remove context diff --git a/tests/integration/models_containers_test.py b/tests/integration/models_containers_test.py index 5b0470b937..3cf74cbc59 100644 --- a/tests/integration/models_containers_test.py +++ b/tests/integration/models_containers_test.py @@ -109,7 +109,7 @@ def test_run_with_none_driver(self): out = client.containers.run( "alpine", "echo hello", - log_config=dict(type='none') + log_config={"type": 'none'} ) assert out is None @@ -118,7 +118,7 @@ def test_run_with_json_file_driver(self): out = client.containers.run( "alpine", "echo hello", - log_config=dict(type='json-file') + log_config={"type": 'json-file'} ) assert out == b'hello\n' @@ -150,7 +150,7 @@ def test_run_with_streamed_logs(self): out = client.containers.run( 'alpine', 'sh -c "echo hello && echo world"', stream=True ) - logs = [line for line in out] + logs = list(out) assert logs[0] == b'hello\n' assert logs[1] == b'world\n' @@ -165,7 +165,7 @@ def test_run_with_streamed_logs_and_cancel(self): threading.Timer(1, out.close).start() - logs = [line for line in out] + logs = list(out) assert len(logs) == 2 assert logs[0] == b'hello\n' diff --git a/tests/integration/models_images_test.py b/tests/integration/models_images_test.py index 94aa201004..d335da4a71 100644 --- a/tests/integration/models_images_test.py +++ b/tests/integration/models_images_test.py @@ -88,9 +88,7 @@ def test_pull_multiple(self): client = docker.from_env(version=TEST_API_VERSION) images = client.images.pull('hello-world', all_tags=True) assert len(images) >= 1 - assert any([ - 'hello-world:latest' in img.attrs['RepoTags'] for img in images - ]) + assert any('hello-world:latest' in img.attrs['RepoTags'] for img in images) def test_load_error(self): client = docker.from_env(version=TEST_API_VERSION) diff --git a/tests/integration/models_networks_test.py b/tests/integration/models_networks_test.py index 08d7ad2955..f4052e4ba1 100644 --- a/tests/integration/models_networks_test.py +++ b/tests/integration/models_networks_test.py @@ -59,11 +59,11 @@ def test_connect_disconnect(self): network.connect(container) container.start() assert client.networks.get(network.id).containers == [container] - network_containers = list( + network_containers = [ c for net in client.networks.list(ids=[network.id], greedy=True) for c in net.containers - ) + ] assert network_containers == [container] network.disconnect(container) assert network.containers == [] diff --git a/tests/ssh/api_build_test.py b/tests/ssh/api_build_test.py index ef48e12ed3..160d53f1e5 100644 --- a/tests/ssh/api_build_test.py +++ b/tests/ssh/api_build_test.py @@ -134,7 +134,7 @@ def test_build_with_dockerignore(self): logs = logs.decode('utf-8') - assert sorted(list(filter(None, logs.split('\n')))) == sorted([ + assert sorted(filter(None, logs.split('\n'))) == sorted([ '/test/#file.txt', '/test/ignored/subdir/excepted-file', '/test/not-ignored' @@ -303,7 +303,7 @@ def test_build_with_network_mode(self): ) self.tmp_imgs.append('dockerpytest_nonebuild') - logs = [chunk for chunk in stream] + logs = list(stream) assert 'errorDetail' in logs[-1] assert logs[-1]['errorDetail']['code'] == 1 @@ -383,7 +383,7 @@ def test_build_stderr_data(self): expected = '{0}{2}\n{1}'.format( control_chars[0], control_chars[1], snippet ) - assert any([line == expected for line in lines]) + assert any(line == expected for line in lines) def test_build_gzip_encoding(self): base_dir = tempfile.mkdtemp() diff --git a/tests/unit/models_containers_test.py b/tests/unit/models_containers_test.py index 0592af5e04..2eabd26859 100644 --- a/tests/unit/models_containers_test.py +++ b/tests/unit/models_containers_test.py @@ -31,77 +31,77 @@ def test_run(self): ) def test_create_container_args(self): - create_kwargs = _create_container_args(dict( - image='alpine', - command='echo hello world', - blkio_weight_device=[{'Path': 'foo', 'Weight': 3}], - blkio_weight=2, - cap_add=['foo'], - cap_drop=['bar'], - cgroup_parent='foobar', - cgroupns='host', - cpu_period=1, - cpu_quota=2, - cpu_shares=5, - cpuset_cpus='0-3', - detach=False, - device_read_bps=[{'Path': 'foo', 'Rate': 3}], - device_read_iops=[{'Path': 'foo', 'Rate': 3}], - device_write_bps=[{'Path': 'foo', 'Rate': 3}], - device_write_iops=[{'Path': 'foo', 'Rate': 3}], - devices=['/dev/sda:/dev/xvda:rwm'], - dns=['8.8.8.8'], - domainname='example.com', - dns_opt=['foo'], - dns_search=['example.com'], - entrypoint='/bin/sh', - environment={'FOO': 'BAR'}, - extra_hosts={'foo': '1.2.3.4'}, - group_add=['blah'], - ipc_mode='foo', - kernel_memory=123, - labels={'key': 'value'}, - links={'foo': 'bar'}, - log_config={'Type': 'json-file', 'Config': {}}, - lxc_conf={'foo': 'bar'}, - healthcheck={'test': 'true'}, - hostname='somehost', - mac_address='abc123', - mem_limit=123, - mem_reservation=123, - mem_swappiness=2, - memswap_limit=456, - name='somename', - network_disabled=False, - network='foo', - network_driver_opt={'key1': 'a'}, - oom_kill_disable=True, - oom_score_adj=5, - pid_mode='host', - pids_limit=500, - platform='linux', - ports={ + create_kwargs = _create_container_args({ + "image": 'alpine', + "command": 'echo hello world', + "blkio_weight_device": [{'Path': 'foo', 'Weight': 3}], + "blkio_weight": 2, + "cap_add": ['foo'], + "cap_drop": ['bar'], + "cgroup_parent": 'foobar', + "cgroupns": 'host', + "cpu_period": 1, + "cpu_quota": 2, + "cpu_shares": 5, + "cpuset_cpus": '0-3', + "detach": False, + "device_read_bps": [{'Path': 'foo', 'Rate': 3}], + "device_read_iops": [{'Path': 'foo', 'Rate': 3}], + "device_write_bps": [{'Path': 'foo', 'Rate': 3}], + "device_write_iops": [{'Path': 'foo', 'Rate': 3}], + "devices": ['/dev/sda:/dev/xvda:rwm'], + "dns": ['8.8.8.8'], + "domainname": 'example.com', + "dns_opt": ['foo'], + "dns_search": ['example.com'], + "entrypoint": '/bin/sh', + "environment": {'FOO': 'BAR'}, + "extra_hosts": {'foo': '1.2.3.4'}, + "group_add": ['blah'], + "ipc_mode": 'foo', + "kernel_memory": 123, + "labels": {'key': 'value'}, + "links": {'foo': 'bar'}, + "log_config": {'Type': 'json-file', 'Config': {}}, + "lxc_conf": {'foo': 'bar'}, + "healthcheck": {'test': 'true'}, + "hostname": 'somehost', + "mac_address": 'abc123', + "mem_limit": 123, + "mem_reservation": 123, + "mem_swappiness": 2, + "memswap_limit": 456, + "name": 'somename', + "network_disabled": False, + "network": 'foo', + "network_driver_opt": {'key1': 'a'}, + "oom_kill_disable": True, + "oom_score_adj": 5, + "pid_mode": 'host', + "pids_limit": 500, + "platform": 'linux', + "ports": { 1111: 4567, 2222: None }, - privileged=True, - publish_all_ports=True, - read_only=True, - restart_policy={'Name': 'always'}, - security_opt=['blah'], - shm_size=123, - stdin_open=True, - stop_signal=9, - sysctls={'foo': 'bar'}, - tmpfs={'/blah': ''}, - tty=True, - ulimits=[{"Name": "nofile", "Soft": 1024, "Hard": 2048}], - user='bob', - userns_mode='host', - uts_mode='host', - version='1.23', - volume_driver='some_driver', - volumes=[ + "privileged": True, + "publish_all_ports": True, + "read_only": True, + "restart_policy": {'Name': 'always'}, + "security_opt": ['blah'], + "shm_size": 123, + "stdin_open": True, + "stop_signal": 9, + "sysctls": {'foo': 'bar'}, + "tmpfs": {'/blah': ''}, + "tty": True, + "ulimits": [{"Name": "nofile", "Soft": 1024, "Hard": 2048}], + "user": 'bob', + "userns_mode": 'host', + "uts_mode": 'host', + "version": '1.23', + "volume_driver": 'some_driver', + "volumes": [ '/home/user1/:/mnt/vol2', '/var/www:/mnt/vol1:ro', 'volumename:/mnt/vol3r', @@ -109,18 +109,18 @@ def test_create_container_args(self): '/anothervolumewithnohostpath:ro', 'C:\\windows\\path:D:\\hello\\world:rw' ], - volumes_from=['container'], - working_dir='/code' - )) + "volumes_from": ['container'], + "working_dir": '/code' + }) - expected = dict( - image='alpine', - command='echo hello world', - domainname='example.com', - detach=False, - entrypoint='/bin/sh', - environment={'FOO': 'BAR'}, - host_config={ + expected = { + "image": 'alpine', + "command": 'echo hello world', + "domainname": 'example.com', + "detach": False, + "entrypoint": '/bin/sh', + "environment": {'FOO': 'BAR'}, + "host_config": { 'Binds': [ '/home/user1/:/mnt/vol2', '/var/www:/mnt/vol1:ro', @@ -183,20 +183,20 @@ def test_create_container_args(self): 'VolumeDriver': 'some_driver', 'VolumesFrom': ['container'], }, - healthcheck={'test': 'true'}, - hostname='somehost', - labels={'key': 'value'}, - mac_address='abc123', - name='somename', - network_disabled=False, - networking_config={'foo': {'driver_opt': {'key1': 'a'}}}, - platform='linux', - ports=[('1111', 'tcp'), ('2222', 'tcp')], - stdin_open=True, - stop_signal=9, - tty=True, - user='bob', - volumes=[ + "healthcheck": {'test': 'true'}, + "hostname": 'somehost', + "labels": {'key': 'value'}, + "mac_address": 'abc123', + "name": 'somename', + "network_disabled": False, + "networking_config": {'foo': {'driver_opt': {'key1': 'a'}}}, + "platform": 'linux', + "ports": [('1111', 'tcp'), ('2222', 'tcp')], + "stdin_open": True, + "stop_signal": 9, + "tty": True, + "user": 'bob', + "volumes": [ '/mnt/vol2', '/mnt/vol1', '/mnt/vol3r', @@ -204,8 +204,8 @@ def test_create_container_args(self): '/anothervolumewithnohostpath', 'D:\\hello\\world' ], - working_dir='/code' - ) + "working_dir": '/code' + } assert create_kwargs == expected From 8447f7b0f02d10399916b285f2fea4284f5a3005 Mon Sep 17 00:00:00 2001 From: Aarni Koskela Date: Thu, 11 May 2023 15:57:01 +0300 Subject: [PATCH 1236/1301] Enable Ruff B rules and autofix Signed-off-by: Aarni Koskela --- docker/models/plugins.py | 2 +- docker/utils/socket.py | 2 +- pyproject.toml | 1 + tests/integration/api_build_test.py | 18 +++++++++--------- tests/integration/api_plugin_test.py | 2 +- tests/integration/regression_test.py | 2 +- tests/ssh/api_build_test.py | 18 +++++++++--------- tests/unit/api_test.py | 6 +++--- 8 files changed, 26 insertions(+), 25 deletions(-) diff --git a/docker/models/plugins.py b/docker/models/plugins.py index 16f5245e9e..85d768c935 100644 --- a/docker/models/plugins.py +++ b/docker/models/plugins.py @@ -187,7 +187,7 @@ def install(self, remote_name, local_name=None): """ privileges = self.client.api.plugin_privileges(remote_name) it = self.client.api.pull_plugin(remote_name, privileges, local_name) - for data in it: + for _data in it: pass return self.get(local_name or remote_name) diff --git a/docker/utils/socket.py b/docker/utils/socket.py index cdc485ea3a..2306ed0736 100644 --- a/docker/utils/socket.py +++ b/docker/utils/socket.py @@ -42,7 +42,7 @@ def read(socket, n=4096): try: if hasattr(socket, 'recv'): return socket.recv(n) - if isinstance(socket, getattr(pysocket, 'SocketIO')): + if isinstance(socket, pysocket.SocketIO): return socket.read(n) return os.read(socket.fileno(), n) except OSError as e: diff --git a/pyproject.toml b/pyproject.toml index 82d4869006..0a67279661 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,6 +7,7 @@ write_to = 'docker/_version.py' [tool.ruff] target-version = "py37" extend-select = [ + "B", "C", "F", "W", diff --git a/tests/integration/api_build_test.py b/tests/integration/api_build_test.py index e5e7904d8f..2add2d87af 100644 --- a/tests/integration/api_build_test.py +++ b/tests/integration/api_build_test.py @@ -132,7 +132,7 @@ def test_build_with_dockerignore(self): path=base_dir, tag=tag, ) - for chunk in stream: + for _chunk in stream: pass c = self.client.create_container(tag, ['find', '/test', '-type', 'f']) @@ -160,7 +160,7 @@ def test_build_with_buildargs(self): fileobj=script, tag='buildargs', buildargs={'test': 'OK'} ) self.tmp_imgs.append('buildargs') - for chunk in stream: + for _chunk in stream: pass info = self.client.inspect_image('buildargs') @@ -180,7 +180,7 @@ def test_build_shmsize(self): fileobj=script, tag=tag, shmsize=shmsize ) self.tmp_imgs.append(tag) - for chunk in stream: + for _chunk in stream: pass # There is currently no way to get the shmsize @@ -198,7 +198,7 @@ def test_build_isolation(self): isolation='default' ) - for chunk in stream: + for _chunk in stream: pass @requires_api_version('1.23') @@ -213,7 +213,7 @@ def test_build_labels(self): fileobj=script, tag='labels', labels=labels ) self.tmp_imgs.append('labels') - for chunk in stream: + for _chunk in stream: pass info = self.client.inspect_image('labels') @@ -230,7 +230,7 @@ def test_build_with_cache_from(self): stream = self.client.build(fileobj=script, tag='build1') self.tmp_imgs.append('build1') - for chunk in stream: + for _chunk in stream: pass stream = self.client.build( @@ -271,7 +271,7 @@ def test_build_container_with_target(self): fileobj=script, target='first', tag='build1' ) self.tmp_imgs.append('build1') - for chunk in stream: + for _chunk in stream: pass info = self.client.inspect_image('build1') @@ -300,7 +300,7 @@ def test_build_with_network_mode(self): ) self.tmp_imgs.append('dockerpytest_customnetbuild') - for chunk in stream: + for _chunk in stream: pass assert self.client.inspect_image('dockerpytest_customnetbuild') @@ -365,7 +365,7 @@ def build_squashed(squash): fileobj=script, tag=tag, squash=squash ) self.tmp_imgs.append(tag) - for chunk in stream: + for _chunk in stream: pass return self.client.inspect_image(tag) diff --git a/tests/integration/api_plugin_test.py b/tests/integration/api_plugin_test.py index a35c30d3e9..3f1633900d 100644 --- a/tests/integration/api_plugin_test.py +++ b/tests/integration/api_plugin_test.py @@ -39,7 +39,7 @@ def ensure_plugin_installed(self, plugin_name): return self.client.inspect_plugin(plugin_name) except docker.errors.NotFound: prv = self.client.plugin_privileges(plugin_name) - for d in self.client.pull_plugin(plugin_name, prv): + for _d in self.client.pull_plugin(plugin_name, prv): pass return self.client.inspect_plugin(plugin_name) diff --git a/tests/integration/regression_test.py b/tests/integration/regression_test.py index 10313a637c..7d2b228cc9 100644 --- a/tests/integration/regression_test.py +++ b/tests/integration/regression_test.py @@ -12,7 +12,7 @@ class TestRegressions(BaseAPIIntegrationTest): def test_443_handle_nonchunked_response_in_stream(self): dfile = io.BytesIO() with pytest.raises(docker.errors.APIError) as exc: - for line in self.client.build(fileobj=dfile, tag="a/b/c"): + for _line in self.client.build(fileobj=dfile, tag="a/b/c"): pass assert exc.value.is_error() dfile.close() diff --git a/tests/ssh/api_build_test.py b/tests/ssh/api_build_test.py index 160d53f1e5..d060f465f2 100644 --- a/tests/ssh/api_build_test.py +++ b/tests/ssh/api_build_test.py @@ -124,7 +124,7 @@ def test_build_with_dockerignore(self): path=base_dir, tag=tag, ) - for chunk in stream: + for _chunk in stream: pass c = self.client.create_container(tag, ['find', '/test', '-type', 'f']) @@ -151,7 +151,7 @@ def test_build_with_buildargs(self): fileobj=script, tag='buildargs', buildargs={'test': 'OK'} ) self.tmp_imgs.append('buildargs') - for chunk in stream: + for _chunk in stream: pass info = self.client.inspect_image('buildargs') @@ -171,7 +171,7 @@ def test_build_shmsize(self): fileobj=script, tag=tag, shmsize=shmsize ) self.tmp_imgs.append(tag) - for chunk in stream: + for _chunk in stream: pass # There is currently no way to get the shmsize @@ -189,7 +189,7 @@ def test_build_isolation(self): isolation='default' ) - for chunk in stream: + for _chunk in stream: pass @requires_api_version('1.23') @@ -204,7 +204,7 @@ def test_build_labels(self): fileobj=script, tag='labels', labels=labels ) self.tmp_imgs.append('labels') - for chunk in stream: + for _chunk in stream: pass info = self.client.inspect_image('labels') @@ -221,7 +221,7 @@ def test_build_with_cache_from(self): stream = self.client.build(fileobj=script, tag='build1') self.tmp_imgs.append('build1') - for chunk in stream: + for _chunk in stream: pass stream = self.client.build( @@ -262,7 +262,7 @@ def test_build_container_with_target(self): fileobj=script, target='first', tag='build1' ) self.tmp_imgs.append('build1') - for chunk in stream: + for _chunk in stream: pass info = self.client.inspect_image('build1') @@ -291,7 +291,7 @@ def test_build_with_network_mode(self): ) self.tmp_imgs.append('dockerpytest_customnetbuild') - for chunk in stream: + for _chunk in stream: pass assert self.client.inspect_image('dockerpytest_customnetbuild') @@ -356,7 +356,7 @@ def build_squashed(squash): fileobj=script, tag=tag, squash=squash ) self.tmp_imgs.append(tag) - for chunk in stream: + for _chunk in stream: pass return self.client.inspect_image(tag) diff --git a/tests/unit/api_test.py b/tests/unit/api_test.py index 78c0bab12e..99aa23bc20 100644 --- a/tests/unit/api_test.py +++ b/tests/unit/api_test.py @@ -333,8 +333,8 @@ def test_stream_helper_decoding(self): # mock a stream interface raw_resp = urllib3.HTTPResponse(body=body) - setattr(raw_resp._fp, 'chunked', True) - setattr(raw_resp._fp, 'chunk_left', len(body.getvalue()) - 1) + raw_resp._fp.chunked = True + raw_resp._fp.chunk_left = len(body.getvalue()) - 1 # pass `decode=False` to the helper raw_resp._fp.seek(0) @@ -349,7 +349,7 @@ def test_stream_helper_decoding(self): assert result == content # non-chunked response, pass `decode=False` to the helper - setattr(raw_resp._fp, 'chunked', False) + raw_resp._fp.chunked = False raw_resp._fp.seek(0) resp = response(status_code=status_code, content=content, raw=raw_resp) result = next(self.client._stream_helper(resp)) From 6aec90a41bc07e1757f28304d3c9e068245afdb9 Mon Sep 17 00:00:00 2001 From: Aarni Koskela Date: Thu, 11 May 2023 15:58:14 +0300 Subject: [PATCH 1237/1301] Fix Ruff B904s (be explicit about exception causes) Signed-off-by: Aarni Koskela --- docker/api/client.py | 14 +++++++------- docker/auth.py | 2 +- docker/context/api.py | 2 +- docker/context/context.py | 2 +- docker/credentials/store.py | 6 +++--- docker/models/containers.py | 4 ++-- docker/tls.py | 2 +- docker/transport/npipeconn.py | 7 +++---- docker/transport/sshconn.py | 6 +++--- docker/types/daemon.py | 4 ++-- docker/utils/build.py | 4 ++-- docker/utils/json_stream.py | 2 +- docker/utils/utils.py | 4 ++-- tests/unit/auth_test.py | 4 ++-- 14 files changed, 31 insertions(+), 32 deletions(-) diff --git a/docker/api/client.py b/docker/api/client.py index ce1b3a307b..a2cb459de0 100644 --- a/docker/api/client.py +++ b/docker/api/client.py @@ -160,10 +160,10 @@ def __init__(self, base_url=None, version=None, base_url, timeout, pool_connections=num_pools, max_pool_size=max_pool_size ) - except NameError: + except NameError as err: raise DockerException( 'Install pypiwin32 package to enable npipe:// support' - ) + ) from err self.mount('http+docker://', self._custom_adapter) self.base_url = 'http+docker://localnpipe' elif base_url.startswith('ssh://'): @@ -172,10 +172,10 @@ def __init__(self, base_url=None, version=None, base_url, timeout, pool_connections=num_pools, max_pool_size=max_pool_size, shell_out=use_ssh_client ) - except NameError: + except NameError as err: raise DockerException( 'Install paramiko package to enable ssh:// support' - ) + ) from err self.mount('http+docker://ssh', self._custom_adapter) self._unmount('http://', 'https://') self.base_url = 'http+docker://ssh' @@ -211,15 +211,15 @@ def __init__(self, base_url=None, version=None, def _retrieve_server_version(self): try: return self.version(api_version=False)["ApiVersion"] - except KeyError: + except KeyError as ke: raise DockerException( 'Invalid response from docker daemon: key "ApiVersion"' ' is missing.' - ) + ) from ke except Exception as e: raise DockerException( f'Error while fetching server API version: {e}' - ) + ) from e def _set_request_timeout(self, kwargs): """Prepare the kwargs for an HTTP request by inserting the timeout diff --git a/docker/auth.py b/docker/auth.py index 4bce788701..7a301ba407 100644 --- a/docker/auth.py +++ b/docker/auth.py @@ -268,7 +268,7 @@ def _resolve_authconfig_credstore(self, registry, credstore_name): except credentials.StoreError as e: raise errors.DockerException( f'Credentials store error: {repr(e)}' - ) + ) from e def _get_store_instance(self, name): if name not in self._stores: diff --git a/docker/context/api.py b/docker/context/api.py index e340fb6dd9..493f470e5d 100644 --- a/docker/context/api.py +++ b/docker/context/api.py @@ -114,7 +114,7 @@ def contexts(cls): except Exception as e: raise errors.ContextException( f"Failed to load metafile {filename}: {e}", - ) + ) from e contexts = [cls.DEFAULT_CONTEXT] for name in names: diff --git a/docker/context/context.py b/docker/context/context.py index b607b77148..4faf8e7017 100644 --- a/docker/context/context.py +++ b/docker/context/context.py @@ -99,7 +99,7 @@ def _load_meta(cls, name): # unknown format raise Exception( f"Detected corrupted meta file for context {name} : {e}" - ) + ) from e # for docker endpoints, set defaults for # Host and SkipTLSVerify fields diff --git a/docker/credentials/store.py b/docker/credentials/store.py index 37c703e78c..5edeaa7f63 100644 --- a/docker/credentials/store.py +++ b/docker/credentials/store.py @@ -80,14 +80,14 @@ def _execute(self, subcmd, data_input): [self.exe, subcmd], input=data_input, env=env, ) except subprocess.CalledProcessError as e: - raise errors.process_store_error(e, self.program) + raise errors.process_store_error(e, self.program) from e except OSError as e: if e.errno == errno.ENOENT: raise errors.StoreError( f'{self.program} not installed or not available in PATH' - ) + ) from e else: raise errors.StoreError( f'Unexpected OS error "{e.strerror}", errno={e.errno}' - ) + ) from e return output diff --git a/docker/models/containers.py b/docker/models/containers.py index 64838397a6..44bb92a0fc 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -47,11 +47,11 @@ def labels(self): try: result = self.attrs['Config'].get('Labels') return result or {} - except KeyError: + except KeyError as ke: raise DockerException( 'Label data is not available for sparse objects. Call reload()' ' to retrieve all information' - ) + ) from ke @property def status(self): diff --git a/docker/tls.py b/docker/tls.py index f4dffb2e25..a4dd002091 100644 --- a/docker/tls.py +++ b/docker/tls.py @@ -55,7 +55,7 @@ def __init__(self, client_cert=None, ca_cert=None, verify=None, raise errors.TLSParameterError( 'client_cert must be a tuple of' ' (client certificate, key file)' - ) + ) from None if not (tls_cert and tls_key) or (not os.path.isfile(tls_cert) or not os.path.isfile(tls_key)): diff --git a/docker/transport/npipeconn.py b/docker/transport/npipeconn.py index 45988b2df1..d335d8718f 100644 --- a/docker/transport/npipeconn.py +++ b/docker/transport/npipeconn.py @@ -46,9 +46,8 @@ def _get_conn(self, timeout): conn = None try: conn = self.pool.get(block=self.block, timeout=timeout) - - except AttributeError: # self.pool is None - raise urllib3.exceptions.ClosedPoolError(self, "Pool is closed.") + except AttributeError as ae: # self.pool is None + raise urllib3.exceptions.ClosedPoolError(self, "Pool is closed.") from ae except queue.Empty: if self.block: @@ -56,7 +55,7 @@ def _get_conn(self, timeout): self, "Pool reached maximum size and no more " "connections are allowed." - ) + ) from None # Oh well, we'll create a new connection then return conn or self._new_conn() diff --git a/docker/transport/sshconn.py b/docker/transport/sshconn.py index a92beb621f..6e1d0ee723 100644 --- a/docker/transport/sshconn.py +++ b/docker/transport/sshconn.py @@ -141,8 +141,8 @@ def _get_conn(self, timeout): try: conn = self.pool.get(block=self.block, timeout=timeout) - except AttributeError: # self.pool is None - raise urllib3.exceptions.ClosedPoolError(self, "Pool is closed.") + except AttributeError as ae: # self.pool is None + raise urllib3.exceptions.ClosedPoolError(self, "Pool is closed.") from ae except queue.Empty: if self.block: @@ -150,7 +150,7 @@ def _get_conn(self, timeout): self, "Pool reached maximum size and no more " "connections are allowed." - ) + ) from None # Oh well, we'll create a new connection then return conn or self._new_conn() diff --git a/docker/types/daemon.py b/docker/types/daemon.py index 096b2cc169..04e6ccb2d7 100644 --- a/docker/types/daemon.py +++ b/docker/types/daemon.py @@ -28,9 +28,9 @@ def __next__(self): try: return next(self._stream) except urllib3.exceptions.ProtocolError: - raise StopIteration + raise StopIteration from None except OSError: - raise StopIteration + raise StopIteration from None next = __next__ diff --git a/docker/utils/build.py b/docker/utils/build.py index 6b38eacdb2..8d18c2be71 100644 --- a/docker/utils/build.py +++ b/docker/utils/build.py @@ -93,10 +93,10 @@ def create_archive(root, files=None, fileobj=None, gzip=False, try: with open(full_path, 'rb') as f: t.addfile(i, f) - except OSError: + except OSError as oe: raise OSError( f'Can not read file in context: {full_path}' - ) + ) from oe else: # Directories, FIFOs, symlinks... don't need to be read. t.addfile(i, None) diff --git a/docker/utils/json_stream.py b/docker/utils/json_stream.py index f384175f75..266193e567 100644 --- a/docker/utils/json_stream.py +++ b/docker/utils/json_stream.py @@ -72,4 +72,4 @@ def split_buffer(stream, splitter=None, decoder=lambda a: a): try: yield decoder(buffered) except Exception as e: - raise StreamParseError(e) + raise StreamParseError(e) from e diff --git a/docker/utils/utils.py b/docker/utils/utils.py index 234be32076..4affeb3396 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -414,11 +414,11 @@ def parse_bytes(s): if suffix in units.keys() or suffix.isdigit(): try: digits = float(digits_part) - except ValueError: + except ValueError as ve: raise errors.DockerException( 'Failed converting the string value for memory ' f'({digits_part}) to an integer.' - ) + ) from ve # Reconvert to long for the final result s = int(digits * units[suffix]) diff --git a/tests/unit/auth_test.py b/tests/unit/auth_test.py index 26254fadde..0ed890fdf3 100644 --- a/tests/unit/auth_test.py +++ b/tests/unit/auth_test.py @@ -778,8 +778,8 @@ def __init__(self, *args, **kwargs): def get(self, server): try: return self.__store[server] - except KeyError: - raise credentials.errors.CredentialsNotFound() + except KeyError as ke: + raise credentials.errors.CredentialsNotFound() from ke def store(self, server, username, secret): self.__store[server] = { From 09f12f20460e5d0d063cda68c825f926953f388c Mon Sep 17 00:00:00 2001 From: Aarni Koskela Date: Thu, 11 May 2023 16:18:13 +0300 Subject: [PATCH 1238/1301] Fix B005 (probably an actual bug too) Signed-off-by: Aarni Koskela --- docker/context/config.py | 3 ++- tests/unit/context_test.py | 8 +++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/docker/context/config.py b/docker/context/config.py index d761aef13c..8c3fe25007 100644 --- a/docker/context/config.py +++ b/docker/context/config.py @@ -77,5 +77,6 @@ def get_context_host(path=None, tls=False): host = utils.parse_host(path, IS_WINDOWS_PLATFORM, tls) if host == DEFAULT_UNIX_SOCKET: # remove http+ from default docker socket url - return host.strip("http+") + if host.startswith("http+"): + host = host[5:] return host diff --git a/tests/unit/context_test.py b/tests/unit/context_test.py index 6d6d6726bc..25f0d8c6ba 100644 --- a/tests/unit/context_test.py +++ b/tests/unit/context_test.py @@ -13,7 +13,7 @@ class BaseContextTest(unittest.TestCase): ) def test_url_compatibility_on_linux(self): c = Context("test") - assert c.Host == DEFAULT_UNIX_SOCKET.strip("http+") + assert c.Host == DEFAULT_UNIX_SOCKET[5:] @pytest.mark.skipif( not IS_WINDOWS_PLATFORM, reason='Windows specific path check' @@ -45,5 +45,7 @@ def test_context_inspect_without_params(self): ctx = ContextAPI.inspect_context() assert ctx["Name"] == "default" assert ctx["Metadata"]["StackOrchestrator"] == "swarm" - assert ctx["Endpoints"]["docker"]["Host"] in [ - DEFAULT_NPIPE, DEFAULT_UNIX_SOCKET.strip("http+")] + assert ctx["Endpoints"]["docker"]["Host"] in ( + DEFAULT_NPIPE, + DEFAULT_UNIX_SOCKET[5:], + ) From cc76c9c20d2af71f759abe02b02d5c96f14e6fdf Mon Sep 17 00:00:00 2001 From: Aarni Koskela Date: Thu, 11 May 2023 16:20:27 +0300 Subject: [PATCH 1239/1301] Fix B082 (no explicit stacklevel for warnings) Signed-off-by: Aarni Koskela --- docker/credentials/store.py | 3 ++- docker/models/images.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docker/credentials/store.py b/docker/credentials/store.py index 5edeaa7f63..4e63a5ba60 100644 --- a/docker/credentials/store.py +++ b/docker/credentials/store.py @@ -20,7 +20,8 @@ def __init__(self, program, environment=None): self.environment = environment if self.exe is None: warnings.warn( - f'{self.program} not installed or not available in PATH' + f'{self.program} not installed or not available in PATH', + stacklevel=1, ) def get(self, server): diff --git a/docker/models/images.py b/docker/models/images.py index abb4b12b50..b4777d8da9 100644 --- a/docker/models/images.py +++ b/docker/models/images.py @@ -456,7 +456,8 @@ def pull(self, repository, tag=None, all_tags=False, **kwargs): if 'stream' in kwargs: warnings.warn( '`stream` is not a valid parameter for this method' - ' and will be overridden' + ' and will be overridden', + stacklevel=1, ) del kwargs['stream'] From 0566f1260cd6f89588df2128ec4e86e8266e5d74 Mon Sep 17 00:00:00 2001 From: Aarni Koskela Date: Thu, 11 May 2023 16:21:05 +0300 Subject: [PATCH 1240/1301] Fix missing asserts or assignments Signed-off-by: Aarni Koskela --- tests/integration/api_swarm_test.py | 4 ++-- tests/integration/models_containers_test.py | 2 +- tests/unit/api_test.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/integration/api_swarm_test.py b/tests/integration/api_swarm_test.py index cffe12fc24..d6aab9665f 100644 --- a/tests/integration/api_swarm_test.py +++ b/tests/integration/api_swarm_test.py @@ -127,11 +127,11 @@ def test_leave_swarm(self): assert self.init_swarm() with pytest.raises(docker.errors.APIError) as exc_info: self.client.leave_swarm() - exc_info.value.response.status_code == 500 + assert exc_info.value.response.status_code == 500 assert self.client.leave_swarm(force=True) with pytest.raises(docker.errors.APIError) as exc_info: self.client.inspect_swarm() - exc_info.value.response.status_code == 406 + assert exc_info.value.response.status_code == 406 assert self.client.leave_swarm(force=True) @requires_api_version('1.24') diff --git a/tests/integration/models_containers_test.py b/tests/integration/models_containers_test.py index 3cf74cbc59..4d33e622e8 100644 --- a/tests/integration/models_containers_test.py +++ b/tests/integration/models_containers_test.py @@ -221,7 +221,7 @@ def test_list_sparse(self): assert container.status == 'running' assert container.image == client.images.get('alpine') with pytest.raises(docker.errors.DockerException): - container.labels + _ = container.labels container.kill() container.remove() diff --git a/tests/unit/api_test.py b/tests/unit/api_test.py index 99aa23bc20..7bc2ea8cda 100644 --- a/tests/unit/api_test.py +++ b/tests/unit/api_test.py @@ -581,7 +581,7 @@ def test_read_from_socket_no_stream_tty_demux(self): def test_read_from_socket_no_stream_no_tty(self): res = self.request(stream=False, tty=False, demux=False) - res == self.stdout_data + self.stderr_data + assert res == self.stdout_data + self.stderr_data def test_read_from_socket_no_stream_no_tty_demux(self): res = self.request(stream=False, tty=False, demux=True) From 3948540c89fc3a58508580bf550e39a883745700 Mon Sep 17 00:00:00 2001 From: Aarni Koskela Date: Thu, 11 May 2023 16:22:43 +0300 Subject: [PATCH 1241/1301] Fix or noqa B003 (assigning to os.environ doesn't do what you expect) Signed-off-by: Aarni Koskela --- tests/integration/credentials/utils_test.py | 2 +- tests/unit/client_test.py | 3 ++- tests/unit/utils_test.py | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/integration/credentials/utils_test.py b/tests/integration/credentials/utils_test.py index acf018d2ff..4644039793 100644 --- a/tests/integration/credentials/utils_test.py +++ b/tests/integration/credentials/utils_test.py @@ -7,7 +7,7 @@ @mock.patch.dict(os.environ) def test_create_environment_dict(): base = {'FOO': 'bar', 'BAZ': 'foobar'} - os.environ = base + os.environ = base # noqa: B003 assert create_environment_dict({'FOO': 'baz'}) == { 'FOO': 'baz', 'BAZ': 'foobar', } diff --git a/tests/unit/client_test.py b/tests/unit/client_test.py index 1148d7ac1c..7012b21236 100644 --- a/tests/unit/client_test.py +++ b/tests/unit/client_test.py @@ -153,7 +153,8 @@ def setUp(self): self.os_environ = os.environ.copy() def tearDown(self): - os.environ = self.os_environ + os.environ.clear() + os.environ.update(self.os_environ) def test_from_env(self): """Test that environment variables are passed through to diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index 9c8a55bd52..b47cb0c62f 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -59,7 +59,8 @@ def setUp(self): self.os_environ = os.environ.copy() def tearDown(self): - os.environ = self.os_environ + os.environ.clear() + os.environ.update(self.os_environ) def test_kwargs_from_env_empty(self): os.environ.update(DOCKER_HOST='', From a9a3775b15e7557b9a7f3db6e27d70b400e91d7e Mon Sep 17 00:00:00 2001 From: Aarni Koskela Date: Thu, 11 May 2023 16:25:31 +0300 Subject: [PATCH 1242/1301] Noqa pytest.raises(Exception) Signed-off-by: Aarni Koskela --- tests/unit/api_image_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/api_image_test.py b/tests/unit/api_image_test.py index aea3a0e136..22b27fe0da 100644 --- a/tests/unit/api_image_test.py +++ b/tests/unit/api_image_test.py @@ -12,7 +12,7 @@ class ImageTest(BaseAPIClientTest): def test_image_viz(self): - with pytest.raises(Exception): + with pytest.raises(Exception): # noqa: B017 self.client.images('busybox', viz=True) self.fail('Viz output should not be supported!') From c68d532f540906b366aa9ec657208bbb47bc51ae Mon Sep 17 00:00:00 2001 From: Aarni Koskela Date: Tue, 15 Aug 2023 13:31:10 +0300 Subject: [PATCH 1243/1301] Fix duplicate dict key literal (ruff F601) Signed-off-by: Aarni Koskela --- tests/unit/fake_api.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/tests/unit/fake_api.py b/tests/unit/fake_api.py index 87d8927578..0524becdc7 100644 --- a/tests/unit/fake_api.py +++ b/tests/unit/fake_api.py @@ -111,13 +111,6 @@ def get_fake_image_history(): return status_code, response -def post_fake_import_image(): - status_code = 200 - response = 'Import messages...' - - return status_code, response - - def get_fake_containers(): status_code = 200 response = [{ @@ -542,8 +535,6 @@ def post_fake_secret(): get_fake_images, f'{prefix}/{CURRENT_VERSION}/images/test_image/history': get_fake_image_history, - f'{prefix}/{CURRENT_VERSION}/images/create': - post_fake_import_image, f'{prefix}/{CURRENT_VERSION}/containers/json': get_fake_containers, f'{prefix}/{CURRENT_VERSION}/containers/{FAKE_CONTAINER_ID}/start': From bea63224e028226085c85caecace6480fe0aa6b0 Mon Sep 17 00:00:00 2001 From: Janne Jakob Fleischer Date: Wed, 9 Aug 2023 10:03:52 +0200 Subject: [PATCH 1244/1301] volume: added support for bind propagation https://docs.docker.com/storage/bind-mounts/#configure-bind-propagation Signed-off-by: Janne Jakob Fleischer Signed-off-by: Milas Bowman --- docker/api/container.py | 8 +++++- docker/utils/utils.py | 17 ++++++++++- tests/integration/api_container_test.py | 38 ++++++++++++++++++++++++- 3 files changed, 60 insertions(+), 3 deletions(-) diff --git a/docker/api/container.py b/docker/api/container.py index ec28fd581b..5a267d13f1 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -319,6 +319,11 @@ def create_container(self, image, command=None, hostname=None, user=None, '/var/www': { 'bind': '/mnt/vol1', 'mode': 'ro', + }, + '/autofs/user1': { + 'bind': '/mnt/vol3', + 'mode': 'rw', + 'propagation': 'shared' } }) ) @@ -329,10 +334,11 @@ def create_container(self, image, command=None, hostname=None, user=None, .. code-block:: python container_id = client.api.create_container( - 'busybox', 'ls', volumes=['/mnt/vol1', '/mnt/vol2'], + 'busybox', 'ls', volumes=['/mnt/vol1', '/mnt/vol2', '/mnt/vol3'], host_config=client.api.create_host_config(binds=[ '/home/user1/:/mnt/vol2', '/var/www:/mnt/vol1:ro', + '/autofs/user1:/mnt/vol3:rw,shared', ]) ) diff --git a/docker/utils/utils.py b/docker/utils/utils.py index 4affeb3396..0f28afb116 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -17,7 +17,6 @@ from urllib.parse import urlparse, urlunparse - URLComponents = collections.namedtuple( 'URLComponents', 'scheme netloc url params query fragment', @@ -141,6 +140,22 @@ def convert_volume_binds(binds): else: mode = 'rw' + # NOTE: this is only relevant for Linux hosts + # (doesn't apply in Docker Desktop) + propagation_modes = [ + 'rshared', + 'shared', + 'rslave', + 'slave', + 'rprivate', + 'private', + ] + if 'propagation' in v and v['propagation'] in propagation_modes: + if mode: + mode = ','.join([mode, v['propagation']]) + else: + mode = v['propagation'] + result.append( f'{k}:{bind}:{mode}' ) diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index 590c4fa0ce..ecda1d65cb 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -542,6 +542,24 @@ def test_create_with_binds_ro(self): inspect_data = self.client.inspect_container(container) self.check_container_data(inspect_data, False) + def test_create_with_binds_rw_rshared(self): + self.run_with_volume_propagation( + False, + 'rshared', + TEST_IMG, + ['touch', os.path.join(self.mount_dest, self.filename)], + ) + container = self.run_with_volume_propagation( + True, + 'rshared', + TEST_IMG, + ['ls', self.mount_dest], + ) + logs = self.client.logs(container).decode('utf-8') + assert self.filename in logs + inspect_data = self.client.inspect_container(container) + self.check_container_data(inspect_data, True, 'rshared') + @requires_api_version('1.30') def test_create_with_mounts(self): mount = docker.types.Mount( @@ -597,7 +615,7 @@ def test_create_with_volume_mount(self): assert mount['Source'] == mount_data['Name'] assert mount_data['RW'] is True - def check_container_data(self, inspect_data, rw): + def check_container_data(self, inspect_data, rw, propagation='rprivate'): assert 'Mounts' in inspect_data filtered = list(filter( lambda x: x['Destination'] == self.mount_dest, @@ -607,6 +625,7 @@ def check_container_data(self, inspect_data, rw): mount_data = filtered[0] assert mount_data['Source'] == self.mount_origin assert mount_data['RW'] == rw + assert mount_data['Propagation'] == propagation def run_with_volume(self, ro, *args, **kwargs): return self.run_container( @@ -624,6 +643,23 @@ def run_with_volume(self, ro, *args, **kwargs): **kwargs ) + def run_with_volume_propagation(self, ro, propagation, *args, **kwargs): + return self.run_container( + *args, + volumes={self.mount_dest: {}}, + host_config=self.client.create_host_config( + binds={ + self.mount_origin: { + 'bind': self.mount_dest, + 'ro': ro, + 'propagation': propagation + }, + }, + network_mode='none' + ), + **kwargs + ) + class ArchiveTest(BaseAPIIntegrationTest): def test_get_file_archive_from_container(self): From 378325363eb01edf60efb3a6d352b6d4047c985a Mon Sep 17 00:00:00 2001 From: Albin Kerouanton <557933+akerouanton@users.noreply.github.com> Date: Mon, 21 Aug 2023 15:30:21 +0200 Subject: [PATCH 1245/1301] integration: Fix bad subnet declaration (#3169) Some network integration tests are creating networks with subnet `2001:389::1/64`. This is an invalid subnet as the host fragment is non-zero (ie. it should be `2001:389::/64`). PR moby/moby#45759 is adding strict validation of network configuration. Docker API will now return an error whenever a bad subnet is passed. Signed-off-by: Albin Kerouanton --- tests/integration/api_network_test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration/api_network_test.py b/tests/integration/api_network_test.py index 78d54e282b..74dad60053 100644 --- a/tests/integration/api_network_test.py +++ b/tests/integration/api_network_test.py @@ -233,7 +233,7 @@ def test_create_with_ipv6_address(self): net_name, net_id = self.create_network( ipam=IPAMConfig( driver='default', - pool_configs=[IPAMPool(subnet="2001:389::1/64")], + pool_configs=[IPAMPool(subnet="2001:389::/64")], ), ) container = self.client.create_container( @@ -389,7 +389,7 @@ def test_connect_with_ipv6_address(self): driver='default', pool_configs=[ IPAMPool( - subnet="2001:389::1/64", iprange="2001:389::0/96", + subnet="2001:389::/64", iprange="2001:389::0/96", gateway="2001:389::ffff" ) ] @@ -455,7 +455,7 @@ def test_create_network_ipv6_enabled(self): driver='default', pool_configs=[ IPAMPool( - subnet="2001:389::1/64", iprange="2001:389::0/96", + subnet="2001:389::/64", iprange="2001:389::0/96", gateway="2001:389::ffff" ) ] From c38656dc7894363f32317affecc3e4279e1163f8 Mon Sep 17 00:00:00 2001 From: Albin Kerouanton <557933+akerouanton@users.noreply.github.com> Date: Mon, 21 Aug 2023 15:31:57 +0200 Subject: [PATCH 1246/1301] integration: Remove test_create_check_duplicate (#3170) integration: check_duplicate is now the default behavior moby/moby#46251 marks CheckDuplicate as deprecated. Any NetworkCreate request with a conflicting network name will now return an error. Signed-off-by: Albin Kerouanton --- tests/integration/api_network_test.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/integration/api_network_test.py b/tests/integration/api_network_test.py index 74dad60053..6689044b68 100644 --- a/tests/integration/api_network_test.py +++ b/tests/integration/api_network_test.py @@ -327,8 +327,6 @@ def test_create_check_duplicate(self): net_name, net_id = self.create_network() with pytest.raises(docker.errors.APIError): self.client.create_network(net_name, check_duplicate=True) - net_id = self.client.create_network(net_name, check_duplicate=False) - self.tmp_networks.append(net_id['Id']) @requires_api_version('1.22') def test_connect_with_links(self): From 7752996f783bf56084902eb931836edd0b368a90 Mon Sep 17 00:00:00 2001 From: Mariano Scazzariello Date: Sat, 30 Sep 2023 00:20:44 +0200 Subject: [PATCH 1247/1301] Replace `network_config` with a dict of EndpointConfig - Renamed parameter from `network_config` to `networking_config` to be more semantically correct with the rest of the API. --- docker/models/containers.py | 74 +++--------- tests/integration/models_containers_test.py | 65 +++++++--- tests/unit/models_containers_test.py | 127 ++++++++++++++------ 3 files changed, 159 insertions(+), 107 deletions(-) diff --git a/docker/models/containers.py b/docker/models/containers.py index 3312b0e2d8..87e64ed48d 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -2,16 +2,16 @@ import ntpath from collections import namedtuple +from .images import Image +from .resource import Collection, Model from ..api import APIClient from ..constants import DEFAULT_DATA_CHUNK_SIZE from ..errors import ( ContainerError, DockerException, ImageNotFound, NotFound, create_unexpected_kwargs_error ) -from ..types import EndpointConfig, HostConfig, NetworkingConfig +from ..types import HostConfig, NetworkingConfig from ..utils import version_gte -from .images import Image -from .resource import Collection, Model class Container(Model): @@ -21,6 +21,7 @@ class Container(Model): query the Docker daemon for the current properties, causing :py:attr:`attrs` to be refreshed. """ + @property def name(self): """ @@ -680,33 +681,13 @@ def run(self, image, command=None, stdout=True, stderr=False, This mode is incompatible with ``ports``. Incompatible with ``network``. - network_config (dict): A dictionary containing options that are - passed to the network driver during the connection. + networking_config (Dict[str, EndpointConfig]): + Dictionary of EndpointConfig objects for each container network. + The key is the name of the network. Defaults to ``None``. - The dictionary contains the following keys: - - - ``aliases`` (:py:class:`list`): A list of aliases for - the network endpoint. - Names in that list can be used within the network to - reach this container. Defaults to ``None``. - - ``links`` (:py:class:`list`): A list of links for - the network endpoint endpoint. - Containers declared in this list will be linked to this - container. Defaults to ``None``. - - ``ipv4_address`` (str): The IP address to assign to - this container on the network, using the IPv4 protocol. - Defaults to ``None``. - - ``ipv6_address`` (str): The IP address to assign to - this container on the network, using the IPv6 protocol. - Defaults to ``None``. - - ``link_local_ips`` (:py:class:`list`): A list of link-local - (IPv4/IPv6) addresses. - - ``driver_opt`` (dict): A dictionary of options to provide to - the network driver. Defaults to ``None``. - - ``mac_address`` (str): MAC Address to assign to the network - interface. Defaults to ``None``. Requires API >= 1.25. Used in conjuction with ``network``. + Incompatible with ``network_mode``. oom_kill_disable (bool): Whether to disable OOM killer. oom_score_adj (int): An integer value containing the score given @@ -872,9 +853,9 @@ def run(self, image, command=None, stdout=True, stderr=False, 'together.' ) - if kwargs.get('network_config') and not kwargs.get('network'): + if kwargs.get('networking_config') and not kwargs.get('network'): raise RuntimeError( - 'The option "network_config" can not be used ' + 'The option "networking_config" can not be used ' 'without "network".' ) @@ -1030,6 +1011,7 @@ def list(self, all=False, before=None, filters=None, limit=-1, since=None, def prune(self, filters=None): return self.client.api.prune_containers(filters=filters) + prune.__doc__ = APIClient.prune_containers.__doc__ @@ -1124,17 +1106,6 @@ def prune(self, filters=None): ] -NETWORKING_CONFIG_ARGS = [ - 'aliases', - 'links', - 'ipv4_address', - 'ipv6_address', - 'link_local_ips', - 'driver_opt', - 'mac_address' -] - - def _create_container_args(kwargs): """ Convert arguments to create() to arguments to create_container(). @@ -1159,24 +1130,17 @@ def _create_container_args(kwargs): host_config_kwargs['binds'] = volumes network = kwargs.pop('network', None) - network_config = kwargs.pop('network_config', None) + networking_config = kwargs.pop('networking_config', None) if network: - endpoint_config = None - - if network_config: - clean_endpoint_args = {} - for arg_name in NETWORKING_CONFIG_ARGS: - if arg_name in network_config: - clean_endpoint_args[arg_name] = network_config[arg_name] - - if clean_endpoint_args: - endpoint_config = EndpointConfig( - host_config_kwargs['version'], **clean_endpoint_args - ) + if networking_config: + # Sanity check: check if the network is defined in the + # networking config dict, otherwise switch to None + if network not in networking_config: + networking_config = None create_kwargs['networking_config'] = NetworkingConfig( - {network: endpoint_config} - ) if endpoint_config else {network: None} + networking_config + ) if networking_config else {network: None} host_config_kwargs['network_mode'] = network # All kwargs should have been consumed by this point, so raise diff --git a/tests/integration/models_containers_test.py b/tests/integration/models_containers_test.py index 050efa01ca..330c658e1a 100644 --- a/tests/integration/models_containers_test.py +++ b/tests/integration/models_containers_test.py @@ -5,10 +5,10 @@ import pytest import docker -from ..helpers import random_name -from ..helpers import requires_api_version from .base import BaseIntegrationTest from .base import TEST_API_VERSION +from ..helpers import random_name +from ..helpers import requires_api_version class ContainerCollectionTest(BaseIntegrationTest): @@ -104,7 +104,7 @@ def test_run_with_network(self): assert 'Networks' in attrs['NetworkSettings'] assert list(attrs['NetworkSettings']['Networks'].keys()) == [net_name] - def test_run_with_network_config(self): + def test_run_with_networking_config(self): net_name = random_name() client = docker.from_env(version=TEST_API_VERSION) client.networks.create(net_name) @@ -113,10 +113,16 @@ def test_run_with_network_config(self): test_aliases = ['hello'] test_driver_opt = {'key1': 'a'} + networking_config = { + net_name: client.api.create_endpoint_config( + aliases=test_aliases, + driver_opt=test_driver_opt + ) + } + container = client.containers.run( 'alpine', 'echo hello world', network=net_name, - network_config={'aliases': test_aliases, - 'driver_opt': test_driver_opt}, + networking_config=networking_config, detach=True ) self.tmp_containers.append(container.id) @@ -131,7 +137,7 @@ def test_run_with_network_config(self): assert attrs['NetworkSettings']['Networks'][net_name]['DriverOpts'] \ == test_driver_opt - def test_run_with_network_config_undeclared_params(self): + def test_run_with_networking_config_with_undeclared_network(self): net_name = random_name() client = docker.from_env(version=TEST_API_VERSION) client.networks.create(net_name) @@ -140,11 +146,41 @@ def test_run_with_network_config_undeclared_params(self): test_aliases = ['hello'] test_driver_opt = {'key1': 'a'} + networking_config = { + net_name: client.api.create_endpoint_config( + aliases=test_aliases, + driver_opt=test_driver_opt + ), + 'bar': client.api.create_endpoint_config( + aliases=['test'], + driver_opt={'key2': 'b'} + ), + } + + with pytest.raises(docker.errors.APIError) as e: + container = client.containers.run( + 'alpine', 'echo hello world', network=net_name, + networking_config=networking_config, + detach=True + ) + self.tmp_containers.append(container.id) + + def test_run_with_networking_config_only_undeclared_network(self): + net_name = random_name() + client = docker.from_env(version=TEST_API_VERSION) + client.networks.create(net_name) + self.tmp_networks.append(net_name) + + networking_config = { + 'bar': client.api.create_endpoint_config( + aliases=['hello'], + driver_opt={'key1': 'a'} + ), + } + container = client.containers.run( 'alpine', 'echo hello world', network=net_name, - network_config={'aliases': test_aliases, - 'driver_opt': test_driver_opt, - 'undeclared_param': 'random_value'}, + networking_config=networking_config, detach=True ) self.tmp_containers.append(container.id) @@ -154,12 +190,9 @@ def test_run_with_network_config_undeclared_params(self): assert 'NetworkSettings' in attrs assert 'Networks' in attrs['NetworkSettings'] assert list(attrs['NetworkSettings']['Networks'].keys()) == [net_name] - assert attrs['NetworkSettings']['Networks'][net_name]['Aliases'] == \ - test_aliases - assert attrs['NetworkSettings']['Networks'][net_name]['DriverOpts'] \ - == test_driver_opt - assert 'undeclared_param' not in \ - attrs['NetworkSettings']['Networks'][net_name] + assert attrs['NetworkSettings']['Networks'][net_name]['Aliases'] is None + assert (attrs['NetworkSettings']['Networks'][net_name]['DriverOpts'] + is None) def test_run_with_none_driver(self): client = docker.from_env(version=TEST_API_VERSION) @@ -244,7 +277,7 @@ def test_get(self): container = client.containers.run("alpine", "sleep 300", detach=True) self.tmp_containers.append(container.id) assert client.containers.get(container.id).attrs[ - 'Config']['Image'] == "alpine" + 'Config']['Image'] == "alpine" def test_list(self): client = docker.from_env(version=TEST_API_VERSION) diff --git a/tests/unit/models_containers_test.py b/tests/unit/models_containers_test.py index f6dccaaba1..bd3092b678 100644 --- a/tests/unit/models_containers_test.py +++ b/tests/unit/models_containers_test.py @@ -1,11 +1,13 @@ -import pytest import unittest +import pytest + import docker from docker.constants import DEFAULT_DATA_CHUNK_SIZE, \ DEFAULT_DOCKER_API_VERSION from docker.models.containers import Container, _create_container_args from docker.models.images import Image +from docker.types import EndpointConfig from .fake_api import FAKE_CONTAINER_ID, FAKE_IMAGE_ID, FAKE_EXEC_ID from .fake_api_client import make_fake_client @@ -32,6 +34,13 @@ def test_run(self): ) def test_create_container_args(self): + networking_config = { + 'foo': EndpointConfig( + DEFAULT_DOCKER_API_VERSION, aliases=['test'], + driver_opt={'key1': 'a'} + ) + } + create_kwargs = _create_container_args(dict( image='alpine', command='echo hello world', @@ -75,7 +84,7 @@ def test_create_container_args(self): name='somename', network_disabled=False, network='foo', - network_config={'aliases': ['test'], 'driver_opt': {'key1': 'a'}}, + networking_config=networking_config, oom_kill_disable=True, oom_score_adj=5, pid_mode='host', @@ -349,35 +358,41 @@ def test_run_platform(self): host_config={'NetworkMode': 'default'}, ) - def test_run_network_config_without_network(self): + def test_run_networking_config_without_network(self): client = make_fake_client() with pytest.raises(RuntimeError): client.containers.run( image='alpine', - network_config={'aliases': ['test'], - 'driver_opt': {'key1': 'a'}} + networking_config={'aliases': ['test'], + 'driver_opt': {'key1': 'a'}} ) - def test_run_network_config_with_network_mode(self): + def test_run_networking_config_with_network_mode(self): client = make_fake_client() with pytest.raises(RuntimeError): client.containers.run( image='alpine', network_mode='none', - network_config={'aliases': ['test'], - 'driver_opt': {'key1': 'a'}} + networking_config={'aliases': ['test'], + 'driver_opt': {'key1': 'a'}} ) - def test_run_network_config(self): + def test_run_networking_config(self): client = make_fake_client() + networking_config = { + 'foo': EndpointConfig( + DEFAULT_DOCKER_API_VERSION, aliases=['test'], + driver_opt={'key1': 'a'} + ) + } + client.containers.run( image='alpine', network='foo', - network_config={'aliases': ['test'], - 'driver_opt': {'key1': 'a'}} + networking_config=networking_config ) client.api.create_container.assert_called_with( @@ -390,15 +405,24 @@ def test_run_network_config(self): host_config={'NetworkMode': 'foo'} ) - def test_run_network_config_undeclared_params(self): + def test_run_networking_config_with_undeclared_network(self): client = make_fake_client() + networking_config = { + 'foo': EndpointConfig( + DEFAULT_DOCKER_API_VERSION, aliases=['test_foo'], + driver_opt={'key2': 'b'} + ), + 'bar': EndpointConfig( + DEFAULT_DOCKER_API_VERSION, aliases=['test'], + driver_opt={'key1': 'a'} + ) + } + client.containers.run( image='alpine', network='foo', - network_config={'aliases': ['test'], - 'driver_opt': {'key1': 'a'}, - 'undeclared_param': 'random_value'} + networking_config=networking_config ) client.api.create_container.assert_called_with( @@ -406,18 +430,26 @@ def test_run_network_config_undeclared_params(self): image='alpine', command=None, networking_config={'EndpointsConfig': { - 'foo': {'Aliases': ['test'], 'DriverOpts': {'key1': 'a'}}} - }, + 'foo': {'Aliases': ['test_foo'], 'DriverOpts': {'key2': 'b'}}, + 'bar': {'Aliases': ['test'], 'DriverOpts': {'key1': 'a'}}, + }}, host_config={'NetworkMode': 'foo'} ) - def test_run_network_config_only_undeclared_params(self): + def test_run_networking_config_only_undeclared_network(self): client = make_fake_client() + networking_config = { + 'bar': EndpointConfig( + DEFAULT_DOCKER_API_VERSION, aliases=['test'], + driver_opt={'key1': 'a'} + ) + } + client.containers.run( image='alpine', network='foo', - network_config={'undeclared_param': 'random_value'} + networking_config=networking_config ) client.api.create_container.assert_called_with( @@ -455,13 +487,13 @@ def test_create_with_image_object(self): host_config={'NetworkMode': 'default'} ) - def test_create_network_config_without_network(self): + def test_create_networking_config_without_network(self): client = make_fake_client() client.containers.create( image='alpine', - network_config={'aliases': ['test'], - 'driver_opt': {'key1': 'a'}} + networking_config={'aliases': ['test'], + 'driver_opt': {'key1': 'a'}} ) client.api.create_container.assert_called_with( @@ -470,14 +502,14 @@ def test_create_network_config_without_network(self): host_config={'NetworkMode': 'default'} ) - def test_create_network_config_with_network_mode(self): + def test_create_networking_config_with_network_mode(self): client = make_fake_client() client.containers.create( image='alpine', network_mode='none', - network_config={'aliases': ['test'], - 'driver_opt': {'key1': 'a'}} + networking_config={'aliases': ['test'], + 'driver_opt': {'key1': 'a'}} ) client.api.create_container.assert_called_with( @@ -486,14 +518,20 @@ def test_create_network_config_with_network_mode(self): host_config={'NetworkMode': 'none'} ) - def test_create_network_config(self): + def test_create_networking_config(self): client = make_fake_client() + networking_config = { + 'foo': EndpointConfig( + DEFAULT_DOCKER_API_VERSION, aliases=['test'], + driver_opt={'key1': 'a'} + ) + } + client.containers.create( image='alpine', network='foo', - network_config={'aliases': ['test'], - 'driver_opt': {'key1': 'a'}} + networking_config=networking_config ) client.api.create_container.assert_called_with( @@ -505,33 +543,50 @@ def test_create_network_config(self): host_config={'NetworkMode': 'foo'} ) - def test_create_network_config_undeclared_params(self): + def test_create_networking_config_with_undeclared_network(self): client = make_fake_client() + networking_config = { + 'foo': EndpointConfig( + DEFAULT_DOCKER_API_VERSION, aliases=['test_foo'], + driver_opt={'key2': 'b'} + ), + 'bar': EndpointConfig( + DEFAULT_DOCKER_API_VERSION, aliases=['test'], + driver_opt={'key1': 'a'} + ) + } + client.containers.create( image='alpine', network='foo', - network_config={'aliases': ['test'], - 'driver_opt': {'key1': 'a'}, - 'undeclared_param': 'random_value'} + networking_config=networking_config ) client.api.create_container.assert_called_with( image='alpine', command=None, networking_config={'EndpointsConfig': { - 'foo': {'Aliases': ['test'], 'DriverOpts': {'key1': 'a'}}} - }, + 'foo': {'Aliases': ['test_foo'], 'DriverOpts': {'key2': 'b'}}, + 'bar': {'Aliases': ['test'], 'DriverOpts': {'key1': 'a'}}, + }}, host_config={'NetworkMode': 'foo'} ) - def test_create_network_config_only_undeclared_params(self): + def test_create_networking_config_only_undeclared_network(self): client = make_fake_client() + networking_config = { + 'bar': EndpointConfig( + DEFAULT_DOCKER_API_VERSION, aliases=['test'], + driver_opt={'key1': 'a'} + ) + } + client.containers.create( image='alpine', network='foo', - network_config={'undeclared_param': 'random_value'} + networking_config=networking_config ) client.api.create_container.assert_called_with( From c9e3efddb86d244e01303106faac72d9ec76a876 Mon Sep 17 00:00:00 2001 From: Aarni Koskela Date: Mon, 20 Nov 2023 22:55:28 +0200 Subject: [PATCH 1248/1301] feat: move websocket-client to extra dependency (#3123) Also bump minimum version to that prescribed by #3022 Signed-off-by: Aarni Koskela --- docker/api/client.py | 12 ++++++++++-- setup.py | 4 +++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/docker/api/client.py b/docker/api/client.py index a2cb459de0..20f8a2af7c 100644 --- a/docker/api/client.py +++ b/docker/api/client.py @@ -5,7 +5,6 @@ import requests import requests.exceptions -import websocket from .. import auth from ..constants import (DEFAULT_NUM_POOLS, DEFAULT_NUM_POOLS_SSH, @@ -309,7 +308,16 @@ def _attach_websocket(self, container, params=None): return self._create_websocket_connection(full_url) def _create_websocket_connection(self, url): - return websocket.create_connection(url) + try: + import websocket + return websocket.create_connection(url) + except ImportError as ie: + raise DockerException( + 'The `websocket-client` library is required ' + 'for using websocket connections. ' + 'You can install the `docker` library ' + 'with the [websocket] extra to install it.' + ) from ie def _get_raw_response_socket(self, response): self._raise_for_status(response) diff --git a/setup.py b/setup.py index 866aa23c8d..79bf3bdb68 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,6 @@ 'packaging >= 14.0', 'requests >= 2.26.0', 'urllib3 >= 1.26.0', - 'websocket-client >= 0.32.0', ] extras_require = { @@ -27,6 +26,9 @@ # Only required when connecting using the ssh:// protocol 'ssh': ['paramiko>=2.4.3'], + + # Only required when using websockets + 'websockets': ['websocket-client >= 1.3.0'], } with open('./test-requirements.txt') as test_reqs_txt: From 26e07251d42d3edc320ea967a68e855739a2a749 Mon Sep 17 00:00:00 2001 From: Milas Bowman Date: Mon, 20 Nov 2023 16:10:38 -0500 Subject: [PATCH 1249/1301] chore: fix lint issues ruff ruff ruff! Signed-off-by: Milas Bowman --- tests/integration/models_containers_test.py | 2 +- tests/unit/models_containers_test.py | 213 ++++++++++---------- 2 files changed, 111 insertions(+), 104 deletions(-) diff --git a/tests/integration/models_containers_test.py b/tests/integration/models_containers_test.py index 8fde851a63..219b9a4cb1 100644 --- a/tests/integration/models_containers_test.py +++ b/tests/integration/models_containers_test.py @@ -157,7 +157,7 @@ def test_run_with_networking_config_with_undeclared_network(self): ), } - with pytest.raises(docker.errors.APIError) as e: + with pytest.raises(docker.errors.APIError): container = client.containers.run( 'alpine', 'echo hello world', network=net_name, networking_config=networking_config, diff --git a/tests/unit/models_containers_test.py b/tests/unit/models_containers_test.py index bd3092b678..05005815f3 100644 --- a/tests/unit/models_containers_test.py +++ b/tests/unit/models_containers_test.py @@ -41,77 +41,73 @@ def test_create_container_args(self): ) } - create_kwargs = _create_container_args(dict( - image='alpine', - command='echo hello world', - blkio_weight_device=[{'Path': 'foo', 'Weight': 3}], - blkio_weight=2, - cap_add=['foo'], - cap_drop=['bar'], - cgroup_parent='foobar', - cgroupns='host', - cpu_period=1, - cpu_quota=2, - cpu_shares=5, - cpuset_cpus='0-3', - detach=False, - device_read_bps=[{'Path': 'foo', 'Rate': 3}], - device_read_iops=[{'Path': 'foo', 'Rate': 3}], - device_write_bps=[{'Path': 'foo', 'Rate': 3}], - device_write_iops=[{'Path': 'foo', 'Rate': 3}], - devices=['/dev/sda:/dev/xvda:rwm'], - dns=['8.8.8.8'], - domainname='example.com', - dns_opt=['foo'], - dns_search=['example.com'], - entrypoint='/bin/sh', - environment={'FOO': 'BAR'}, - extra_hosts={'foo': '1.2.3.4'}, - group_add=['blah'], - ipc_mode='foo', - kernel_memory=123, - labels={'key': 'value'}, - links={'foo': 'bar'}, - log_config={'Type': 'json-file', 'Config': {}}, - lxc_conf={'foo': 'bar'}, - healthcheck={'test': 'true'}, - hostname='somehost', - mac_address='abc123', - mem_limit=123, - mem_reservation=123, - mem_swappiness=2, - memswap_limit=456, - name='somename', - network_disabled=False, - network='foo', - networking_config=networking_config, - oom_kill_disable=True, - oom_score_adj=5, - pid_mode='host', - pids_limit=500, - platform='linux', - ports={ - 1111: 4567, - 2222: None - }, - privileged=True, - publish_all_ports=True, - read_only=True, - restart_policy={'Name': 'always'}, - security_opt=['blah'], - shm_size=123, - stdin_open=True, - stop_signal=9, - sysctls={'foo': 'bar'}, - tmpfs={'/blah': ''}, - tty=True, - ulimits=[{"Name": "nofile", "Soft": 1024, "Hard": 2048}], - user='bob', - userns_mode='host', - uts_mode='host', - version=DEFAULT_DOCKER_API_VERSION, - volume_driver='some_driver', - volumes=[ + create_kwargs = _create_container_args({ + 'image': 'alpine', + 'command': 'echo hello world', + 'blkio_weight_device': [{'Path': 'foo', 'Weight': 3}], + 'blkio_weight': 2, + 'cap_add': ['foo'], + 'cap_drop': ['bar'], + 'cgroup_parent': 'foobar', + 'cgroupns': 'host', + 'cpu_period': 1, + 'cpu_quota': 2, + 'cpu_shares': 5, + 'cpuset_cpus': '0-3', + 'detach': False, + 'device_read_bps': [{'Path': 'foo', 'Rate': 3}], + 'device_read_iops': [{'Path': 'foo', 'Rate': 3}], + 'device_write_bps': [{'Path': 'foo', 'Rate': 3}], + 'device_write_iops': [{'Path': 'foo', 'Rate': 3}], + 'devices': ['/dev/sda:/dev/xvda:rwm'], + 'dns': ['8.8.8.8'], + 'domainname': 'example.com', + 'dns_opt': ['foo'], + 'dns_search': ['example.com'], + 'entrypoint': '/bin/sh', + 'environment': {'FOO': 'BAR'}, + 'extra_hosts': {'foo': '1.2.3.4'}, + 'group_add': ['blah'], + 'ipc_mode': 'foo', + 'kernel_memory': 123, + 'labels': {'key': 'value'}, + 'links': {'foo': 'bar'}, + 'log_config': {'Type': 'json-file', 'Config': {}}, + 'lxc_conf': {'foo': 'bar'}, + 'healthcheck': {'test': 'true'}, + 'hostname': 'somehost', + 'mac_address': 'abc123', + 'mem_limit': 123, + 'mem_reservation': 123, + 'mem_swappiness': 2, + 'memswap_limit': 456, + 'name': 'somename', + 'network_disabled': False, + 'network': 'foo', + 'networking_config': networking_config, + 'oom_kill_disable': True, + 'oom_score_adj': 5, + 'pid_mode': 'host', + 'pids_limit': 500, + 'platform': 'linux', + 'ports': {1111: 4567, 2222: None}, + 'privileged': True, + 'publish_all_ports': True, + 'read_only': True, + 'restart_policy': {'Name': 'always'}, + 'security_opt': ['blah'], + 'shm_size': 123, + 'stdin_open': True, + 'stop_signal': 9, + 'sysctls': {'foo': 'bar'}, + 'tmpfs': {'/blah': ''}, + 'tty': True, + 'ulimits': [{"Name": "nofile", "Soft": 1024, "Hard": 2048}], + 'user': 'bob', + 'userns_mode': 'host', + 'uts_mode': 'host', + 'version': DEFAULT_DOCKER_API_VERSION, + 'volume_driver': 'some_driver', 'volumes': [ '/home/user1/:/mnt/vol2', '/var/www:/mnt/vol1:ro', 'volumename:/mnt/vol3r', @@ -119,18 +115,18 @@ def test_create_container_args(self): '/anothervolumewithnohostpath:ro', 'C:\\windows\\path:D:\\hello\\world:rw' ], - volumes_from=['container'], - working_dir='/code' - )) + 'volumes_from': ['container'], + 'working_dir': '/code', + }) - expected = dict( - image='alpine', - command='echo hello world', - domainname='example.com', - detach=False, - entrypoint='/bin/sh', - environment={'FOO': 'BAR'}, - host_config={ + expected = { + 'image': 'alpine', + 'command': 'echo hello world', + 'domainname': 'example.com', + 'detach': False, + 'entrypoint': '/bin/sh', + 'environment': {'FOO': 'BAR'}, + 'host_config': { 'Binds': [ '/home/user1/:/mnt/vol2', '/var/www:/mnt/vol1:ro', @@ -153,9 +149,13 @@ def test_create_container_args(self): 'CpuQuota': 2, 'CpuShares': 5, 'CpusetCpus': '0-3', - 'Devices': [{'PathOnHost': '/dev/sda', - 'CgroupPermissions': 'rwm', - 'PathInContainer': '/dev/xvda'}], + 'Devices': [ + { + 'PathOnHost': '/dev/sda', + 'CgroupPermissions': 'rwm', + 'PathInContainer': '/dev/xvda', + }, + ], 'Dns': ['8.8.8.8'], 'DnsOptions': ['foo'], 'DnsSearch': ['example.com'], @@ -187,28 +187,35 @@ def test_create_container_args(self): 'ShmSize': 123, 'Sysctls': {'foo': 'bar'}, 'Tmpfs': {'/blah': ''}, - 'Ulimits': [{"Name": "nofile", "Soft": 1024, "Hard": 2048}], + 'Ulimits': [ + {"Name": "nofile", "Soft": 1024, "Hard": 2048}, + ], 'UsernsMode': 'host', 'UTSMode': 'host', 'VolumeDriver': 'some_driver', 'VolumesFrom': ['container'], }, - healthcheck={'test': 'true'}, - hostname='somehost', - labels={'key': 'value'}, - mac_address='abc123', - name='somename', - network_disabled=False, - networking_config={'EndpointsConfig': { - 'foo': {'Aliases': ['test'], 'DriverOpts': {'key1': 'a'}}} + 'healthcheck': {'test': 'true'}, + 'hostname': 'somehost', + 'labels': {'key': 'value'}, + 'mac_address': 'abc123', + 'name': 'somename', + 'network_disabled': False, + 'networking_config': { + 'EndpointsConfig': { + 'foo': { + 'Aliases': ['test'], + 'DriverOpts': {'key1': 'a'}, + }, + } }, - platform='linux', - ports=[('1111', 'tcp'), ('2222', 'tcp')], - stdin_open=True, - stop_signal=9, - tty=True, - user='bob', - volumes=[ + 'platform': 'linux', + 'ports': [('1111', 'tcp'), ('2222', 'tcp')], + 'stdin_open': True, + 'stop_signal': 9, + 'tty': True, + 'user': 'bob', + 'volumes': [ '/mnt/vol2', '/mnt/vol1', '/mnt/vol3r', @@ -216,8 +223,8 @@ def test_create_container_args(self): '/anothervolumewithnohostpath', 'D:\\hello\\world' ], - working_dir='/code' - ) + 'working_dir': '/code', + } assert create_kwargs == expected From b2378db7f174fee78f67748308b3c98855454d68 Mon Sep 17 00:00:00 2001 From: Milas Bowman Date: Mon, 20 Nov 2023 16:18:08 -0500 Subject: [PATCH 1250/1301] chore: fix lint issue Signed-off-by: Milas Bowman --- docker/models/containers.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker/models/containers.py b/docker/models/containers.py index f8aeb39b13..4725d6f6fb 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -66,7 +66,9 @@ def status(self): @property def health(self): """ - The healthcheck status of the container. For example, ``healthy`, or ``unhealthy`. + The healthcheck status of the container. + + For example, ``healthy`, or ``unhealthy`. """ return self.attrs.get('State', {}).get('Health', {}).get('Status', 'unknown') From 976c84c481dce82d129980b87e88a102badb4109 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Nov 2023 16:56:50 -0500 Subject: [PATCH 1251/1301] build(deps): Bump urllib3 from 1.26.11 to 1.26.18 (#3183) Bumps [urllib3](https://github.com/urllib3/urllib3) from 1.26.11 to 1.26.18. - [Release notes](https://github.com/urllib3/urllib3/releases) - [Changelog](https://github.com/urllib3/urllib3/blob/main/CHANGES.rst) - [Commits](https://github.com/urllib3/urllib3/compare/1.26.11...1.26.18) --- updated-dependencies: - dependency-name: urllib3 dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 897cdbd5ef..6d932eb371 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,5 +2,5 @@ packaging==21.3 paramiko==2.11.0 pywin32==304; sys_platform == 'win32' requests==2.31.0 -urllib3==1.26.11 +urllib3==1.26.18 websocket-client==1.3.3 From db4878118b02124100c669b39249f0bdeed2aad0 Mon Sep 17 00:00:00 2001 From: Milas Bowman Date: Tue, 21 Nov 2023 10:42:53 -0500 Subject: [PATCH 1252/1301] breaking: Python 3.12 compatibility & remove custom SSL adapter (#3185) Add support for Python 3.12. `match_hostname` is gone in Python 3.12 and has been unused by Python since 3.7. The custom SSL adapter allows passing a specific SSL version; this was first introduced a looong time ago to handle some SSL issues at the time. Closes #3176. --------- Signed-off-by: Hugo van Kemenade Signed-off-by: Milas Bowman Co-authored-by: Hugo van Kemenade --- .github/workflows/ci.yml | 12 ++- .github/workflows/release.yml | 6 +- Dockerfile | 2 +- Dockerfile-docs | 2 +- Jenkinsfile | 147 ----------------------------- docker/api/client.py | 5 +- docker/client.py | 2 - docker/tls.py | 29 +----- docker/transport/__init__.py | 1 - docker/transport/ssladapter.py | 62 ------------ docker/utils/utils.py | 9 +- setup.py | 1 + test-requirements.txt | 6 +- tests/Dockerfile | 2 +- tests/Dockerfile-dind-certs | 2 +- tests/unit/models_networks_test.py | 18 ++-- tests/unit/ssladapter_test.py | 71 -------------- tests/unit/utils_test.py | 15 ++- tox.ini | 2 +- 19 files changed, 41 insertions(+), 353 deletions(-) delete mode 100644 Jenkinsfile delete mode 100644 docker/transport/ssladapter.py delete mode 100644 tests/unit/ssladapter_test.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dfbcc701eb..977199cebd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,15 +4,16 @@ on: [push, pull_request] env: DOCKER_BUILDKIT: '1' + FORCE_COLOR: 1 jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-python@v4 with: - python-version: '3.11' + python-version: '3.x' - run: pip install -U ruff==0.0.284 - name: Run ruff run: ruff docker tests @@ -21,14 +22,15 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} + allow-prereleases: true - name: Install dependencies run: | python3 -m pip install --upgrade pip @@ -46,7 +48,7 @@ jobs: variant: [ "integration-dind", "integration-dind-ssl", "integration-dind-ssh" ] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: make ${{ matrix.variant }} run: | docker logout diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7c6358a225..b8b1f57d1f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -12,11 +12,15 @@ on: type: boolean default: true +env: + DOCKER_BUILDKIT: '1' + FORCE_COLOR: 1 + jobs: publish: runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-python@v4 with: diff --git a/Dockerfile b/Dockerfile index 3476c6d036..293888d725 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ # syntax=docker/dockerfile:1 -ARG PYTHON_VERSION=3.10 +ARG PYTHON_VERSION=3.12 FROM python:${PYTHON_VERSION} diff --git a/Dockerfile-docs b/Dockerfile-docs index 11adbfe85d..266b2099e9 100644 --- a/Dockerfile-docs +++ b/Dockerfile-docs @@ -1,6 +1,6 @@ # syntax=docker/dockerfile:1 -ARG PYTHON_VERSION=3.10 +ARG PYTHON_VERSION=3.12 FROM python:${PYTHON_VERSION} diff --git a/Jenkinsfile b/Jenkinsfile deleted file mode 100644 index f9431eac06..0000000000 --- a/Jenkinsfile +++ /dev/null @@ -1,147 +0,0 @@ -#!groovy - -def imageNameBase = "dockerpinata/docker-py" -def imageNamePy3 -def imageDindSSH -def images = [:] - -def buildImage = { name, buildargs, pyTag -> - img = docker.image(name) - try { - img.pull() - } catch (Exception exc) { - img = docker.build(name, buildargs) - img.push() - } - if (pyTag?.trim()) images[pyTag] = img.id -} - -def buildImages = { -> - wrappedNode(label: "amd64 && ubuntu-2004 && overlay2", cleanWorkspace: true) { - stage("build image") { - checkout(scm) - - imageNamePy3 = "${imageNameBase}:py3-${gitCommit()}" - imageDindSSH = "${imageNameBase}:sshdind-${gitCommit()}" - withDockerRegistry(credentialsId:'dockerbuildbot-index.docker.io') { - buildImage(imageDindSSH, "-f tests/Dockerfile-ssh-dind .", "") - buildImage(imageNamePy3, "-f tests/Dockerfile --build-arg PYTHON_VERSION=3.10 .", "py3.10") - } - } - } -} - -def getDockerVersions = { -> - def dockerVersions = ["19.03.12"] - wrappedNode(label: "amd64 && ubuntu-2004 && overlay2") { - def result = sh(script: """docker run --rm \\ - --entrypoint=python \\ - ${imageNamePy3} \\ - /src/scripts/versions.py - """, returnStdout: true - ) - dockerVersions = dockerVersions + result.trim().tokenize(' ') - } - return dockerVersions -} - -def getAPIVersion = { engineVersion -> - def versionMap = [ - '18.09': '1.39', - '19.03': '1.40' - ] - def result = versionMap[engineVersion.substring(0, 5)] - if (!result) { - return '1.40' - } - return result -} - -def runTests = { Map settings -> - def dockerVersion = settings.get("dockerVersion", null) - def pythonVersion = settings.get("pythonVersion", null) - def testImage = settings.get("testImage", null) - def apiVersion = getAPIVersion(dockerVersion) - - if (!testImage) { - throw new Exception("Need test image object, e.g.: `runTests(testImage: img)`") - } - if (!dockerVersion) { - throw new Exception("Need Docker version to test, e.g.: `runTests(dockerVersion: '19.03.12')`") - } - if (!pythonVersion) { - throw new Exception("Need Python version being tested, e.g.: `runTests(pythonVersion: 'py3.x')`") - } - - { -> - wrappedNode(label: "amd64 && ubuntu-2004 && overlay2", cleanWorkspace: true) { - stage("test python=${pythonVersion} / docker=${dockerVersion}") { - checkout(scm) - def dindContainerName = "dpy-dind-\$BUILD_NUMBER-\$EXECUTOR_NUMBER-${pythonVersion}-${dockerVersion}" - def testContainerName = "dpy-tests-\$BUILD_NUMBER-\$EXECUTOR_NUMBER-${pythonVersion}-${dockerVersion}" - def testNetwork = "dpy-testnet-\$BUILD_NUMBER-\$EXECUTOR_NUMBER-${pythonVersion}-${dockerVersion}" - withDockerRegistry(credentialsId:'dockerbuildbot-index.docker.io') { - try { - // unit tests - sh """docker run --rm \\ - -e 'DOCKER_TEST_API_VERSION=${apiVersion}' \\ - ${testImage} \\ - py.test -v -rxs --cov=docker tests/unit - """ - // integration tests - sh """docker network create ${testNetwork}""" - sh """docker run --rm -d --name ${dindContainerName} -v /tmp --privileged --network ${testNetwork} \\ - ${imageDindSSH} dockerd -H tcp://0.0.0.0:2375 - """ - sh """docker run --rm \\ - --name ${testContainerName} \\ - -e "DOCKER_HOST=tcp://${dindContainerName}:2375" \\ - -e 'DOCKER_TEST_API_VERSION=${apiVersion}' \\ - --network ${testNetwork} \\ - --volumes-from ${dindContainerName} \\ - -v $DOCKER_CONFIG/config.json:/root/.docker/config.json \\ - ${testImage} \\ - py.test -v -rxs --cov=docker tests/integration - """ - sh """docker stop ${dindContainerName}""" - // start DIND container with SSH - sh """docker run --rm -d --name ${dindContainerName} -v /tmp --privileged --network ${testNetwork} \\ - ${imageDindSSH} dockerd --experimental""" - sh """docker exec ${dindContainerName} sh -c /usr/sbin/sshd """ - // run SSH tests only - sh """docker run --rm \\ - --name ${testContainerName} \\ - -e "DOCKER_HOST=ssh://${dindContainerName}:22" \\ - -e 'DOCKER_TEST_API_VERSION=${apiVersion}' \\ - --network ${testNetwork} \\ - --volumes-from ${dindContainerName} \\ - -v $DOCKER_CONFIG/config.json:/root/.docker/config.json \\ - ${testImage} \\ - py.test -v -rxs --cov=docker tests/ssh - """ - } finally { - sh """ - docker stop ${dindContainerName} - docker network rm ${testNetwork} - """ - } - } - } - } - } -} - - -buildImages() - -def dockerVersions = getDockerVersions() - -def testMatrix = [failFast: false] - -for (imgKey in new ArrayList(images.keySet())) { - for (version in dockerVersions) { - testMatrix["${imgKey}_${version}"] = runTests([testImage: images[imgKey], dockerVersion: version, pythonVersion: imgKey]) - } -} - -parallel(testMatrix) diff --git a/docker/api/client.py b/docker/api/client.py index 20f8a2af7c..499a7c785e 100644 --- a/docker/api/client.py +++ b/docker/api/client.py @@ -4,6 +4,7 @@ from functools import partial import requests +import requests.adapters import requests.exceptions from .. import auth @@ -14,7 +15,7 @@ from ..errors import (DockerException, InvalidVersion, TLSParameterError, create_api_error_from_http_exception) from ..tls import TLSConfig -from ..transport import SSLHTTPAdapter, UnixHTTPAdapter +from ..transport import UnixHTTPAdapter from ..utils import check_resource, config, update_headers, utils from ..utils.json_stream import json_stream from ..utils.proxy import ProxyConfig @@ -183,7 +184,7 @@ def __init__(self, base_url=None, version=None, if isinstance(tls, TLSConfig): tls.configure_client(self) elif tls: - self._custom_adapter = SSLHTTPAdapter( + self._custom_adapter = requests.adapters.HTTPAdapter( pool_connections=num_pools) self.mount('https://', self._custom_adapter) self.base_url = base_url diff --git a/docker/client.py b/docker/client.py index 4dbd846f1d..2910c12596 100644 --- a/docker/client.py +++ b/docker/client.py @@ -71,8 +71,6 @@ def from_env(cls, **kwargs): timeout (int): Default timeout for API calls, in seconds. max_pool_size (int): The maximum number of connections to save in the pool. - ssl_version (int): A valid `SSL version`_. - assert_hostname (bool): Verify the hostname of the server. environment (dict): The environment to read environment variables from. Default: the value of ``os.environ`` credstore_env (dict): Override environment variables when calling diff --git a/docker/tls.py b/docker/tls.py index a4dd002091..ad4966c903 100644 --- a/docker/tls.py +++ b/docker/tls.py @@ -1,8 +1,6 @@ import os -import ssl from . import errors -from .transport import SSLHTTPAdapter class TLSConfig: @@ -15,35 +13,18 @@ class TLSConfig: verify (bool or str): This can be a bool or a path to a CA cert file to verify against. If ``True``, verify using ca_cert; if ``False`` or not specified, do not verify. - ssl_version (int): A valid `SSL version`_. - assert_hostname (bool): Verify the hostname of the server. - - .. _`SSL version`: - https://docs.python.org/3.5/library/ssl.html#ssl.PROTOCOL_TLSv1 """ cert = None ca_cert = None verify = None - ssl_version = None - def __init__(self, client_cert=None, ca_cert=None, verify=None, - ssl_version=None, assert_hostname=None, - assert_fingerprint=None): + def __init__(self, client_cert=None, ca_cert=None, verify=None): # Argument compatibility/mapping with # https://docs.docker.com/engine/articles/https/ # This diverges from the Docker CLI in that users can specify 'tls' # here, but also disable any public/default CA pool verification by # leaving verify=False - self.assert_hostname = assert_hostname - self.assert_fingerprint = assert_fingerprint - - # If the user provides an SSL version, we should use their preference - if ssl_version: - self.ssl_version = ssl_version - else: - self.ssl_version = ssl.PROTOCOL_TLS_CLIENT - # "client_cert" must have both or neither cert/key files. In # either case, Alert the user when both are expected, but any are # missing. @@ -77,8 +58,6 @@ def configure_client(self, client): """ Configure a client with these TLS options. """ - client.ssl_version = self.ssl_version - if self.verify and self.ca_cert: client.verify = self.ca_cert else: @@ -86,9 +65,3 @@ def configure_client(self, client): if self.cert: client.cert = self.cert - - client.mount('https://', SSLHTTPAdapter( - ssl_version=self.ssl_version, - assert_hostname=self.assert_hostname, - assert_fingerprint=self.assert_fingerprint, - )) diff --git a/docker/transport/__init__.py b/docker/transport/__init__.py index 54492c11ac..07bc7fd582 100644 --- a/docker/transport/__init__.py +++ b/docker/transport/__init__.py @@ -1,5 +1,4 @@ from .unixconn import UnixHTTPAdapter -from .ssladapter import SSLHTTPAdapter try: from .npipeconn import NpipeHTTPAdapter from .npipesocket import NpipeSocket diff --git a/docker/transport/ssladapter.py b/docker/transport/ssladapter.py deleted file mode 100644 index 69274bd1dd..0000000000 --- a/docker/transport/ssladapter.py +++ /dev/null @@ -1,62 +0,0 @@ -""" Resolves OpenSSL issues in some servers: - https://lukasa.co.uk/2013/01/Choosing_SSL_Version_In_Requests/ - https://github.com/kennethreitz/requests/pull/799 -""" -from packaging.version import Version -from requests.adapters import HTTPAdapter - -from docker.transport.basehttpadapter import BaseHTTPAdapter - -import urllib3 - - -PoolManager = urllib3.poolmanager.PoolManager - - -class SSLHTTPAdapter(BaseHTTPAdapter): - '''An HTTPS Transport Adapter that uses an arbitrary SSL version.''' - - __attrs__ = HTTPAdapter.__attrs__ + ['assert_fingerprint', - 'assert_hostname', - 'ssl_version'] - - def __init__(self, ssl_version=None, assert_hostname=None, - assert_fingerprint=None, **kwargs): - self.ssl_version = ssl_version - self.assert_hostname = assert_hostname - self.assert_fingerprint = assert_fingerprint - super().__init__(**kwargs) - - def init_poolmanager(self, connections, maxsize, block=False): - kwargs = { - 'num_pools': connections, - 'maxsize': maxsize, - 'block': block, - 'assert_hostname': self.assert_hostname, - 'assert_fingerprint': self.assert_fingerprint, - } - if self.ssl_version and self.can_override_ssl_version(): - kwargs['ssl_version'] = self.ssl_version - - self.poolmanager = PoolManager(**kwargs) - - def get_connection(self, *args, **kwargs): - """ - Ensure assert_hostname is set correctly on our pool - - We already take care of a normal poolmanager via init_poolmanager - - But we still need to take care of when there is a proxy poolmanager - """ - conn = super().get_connection(*args, **kwargs) - if conn.assert_hostname != self.assert_hostname: - conn.assert_hostname = self.assert_hostname - return conn - - def can_override_ssl_version(self): - urllib_ver = urllib3.__version__.split('-')[0] - if urllib_ver is None: - return False - if urllib_ver == 'dev': - return True - return Version(urllib_ver) > Version('1.5') diff --git a/docker/utils/utils.py b/docker/utils/utils.py index 0f28afb116..759ddd2f1a 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -341,7 +341,7 @@ def parse_devices(devices): return device_list -def kwargs_from_env(ssl_version=None, assert_hostname=None, environment=None): +def kwargs_from_env(environment=None): if not environment: environment = os.environ host = environment.get('DOCKER_HOST') @@ -369,18 +369,11 @@ def kwargs_from_env(ssl_version=None, assert_hostname=None, environment=None): if not cert_path: cert_path = os.path.join(os.path.expanduser('~'), '.docker') - if not tls_verify and assert_hostname is None: - # assert_hostname is a subset of TLS verification, - # so if it's not set already then set it to false. - assert_hostname = False - params['tls'] = TLSConfig( client_cert=(os.path.join(cert_path, 'cert.pem'), os.path.join(cert_path, 'key.pem')), ca_cert=os.path.join(cert_path, 'ca.pem'), verify=tls_verify, - ssl_version=ssl_version, - assert_hostname=assert_hostname, ) return params diff --git a/setup.py b/setup.py index 79bf3bdb68..d63cbe0a1c 100644 --- a/setup.py +++ b/setup.py @@ -74,6 +74,7 @@ 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', 'Topic :: Software Development', 'Topic :: Utilities', 'License :: OSI Approved :: Apache Software License', diff --git a/test-requirements.txt b/test-requirements.txt index 951b3be9fc..031d0acf0a 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,6 +1,6 @@ setuptools==65.5.1 -coverage==6.4.2 +coverage==7.2.7 ruff==0.0.284 -pytest==7.1.2 -pytest-cov==3.0.0 +pytest==7.4.2 +pytest-cov==4.1.0 pytest-timeout==2.1.0 diff --git a/tests/Dockerfile b/tests/Dockerfile index 366abe23bb..d7c14b6cca 100644 --- a/tests/Dockerfile +++ b/tests/Dockerfile @@ -1,6 +1,6 @@ # syntax=docker/dockerfile:1 -ARG PYTHON_VERSION=3.10 +ARG PYTHON_VERSION=3.12 FROM python:${PYTHON_VERSION} diff --git a/tests/Dockerfile-dind-certs b/tests/Dockerfile-dind-certs index 288a340ab1..7b819eb154 100644 --- a/tests/Dockerfile-dind-certs +++ b/tests/Dockerfile-dind-certs @@ -1,6 +1,6 @@ # syntax=docker/dockerfile:1 -ARG PYTHON_VERSION=3.10 +ARG PYTHON_VERSION=3.12 FROM python:${PYTHON_VERSION} RUN mkdir /tmp/certs diff --git a/tests/unit/models_networks_test.py b/tests/unit/models_networks_test.py index 58c9fce669..f10e1e3e33 100644 --- a/tests/unit/models_networks_test.py +++ b/tests/unit/models_networks_test.py @@ -10,8 +10,8 @@ def test_create(self): client = make_fake_client() network = client.networks.create("foobar", labels={'foo': 'bar'}) assert network.id == FAKE_NETWORK_ID - assert client.api.inspect_network.called_once_with(FAKE_NETWORK_ID) - assert client.api.create_network.called_once_with( + client.api.inspect_network.assert_called_once_with(FAKE_NETWORK_ID) + client.api.create_network.assert_called_once_with( "foobar", labels={'foo': 'bar'} ) @@ -20,21 +20,21 @@ def test_get(self): client = make_fake_client() network = client.networks.get(FAKE_NETWORK_ID) assert network.id == FAKE_NETWORK_ID - assert client.api.inspect_network.called_once_with(FAKE_NETWORK_ID) + client.api.inspect_network.assert_called_once_with(FAKE_NETWORK_ID) def test_list(self): client = make_fake_client() networks = client.networks.list() assert networks[0].id == FAKE_NETWORK_ID - assert client.api.networks.called_once_with() + client.api.networks.assert_called_once_with() client = make_fake_client() client.networks.list(ids=["abc"]) - assert client.api.networks.called_once_with(ids=["abc"]) + client.api.networks.assert_called_once_with(ids=["abc"]) client = make_fake_client() client.networks.list(names=["foobar"]) - assert client.api.networks.called_once_with(names=["foobar"]) + client.api.networks.assert_called_once_with(names=["foobar"]) class NetworkTest(unittest.TestCase): @@ -43,7 +43,7 @@ def test_connect(self): client = make_fake_client() network = client.networks.get(FAKE_NETWORK_ID) network.connect(FAKE_CONTAINER_ID) - assert client.api.connect_container_to_network.called_once_with( + client.api.connect_container_to_network.assert_called_once_with( FAKE_CONTAINER_ID, FAKE_NETWORK_ID ) @@ -52,7 +52,7 @@ def test_disconnect(self): client = make_fake_client() network = client.networks.get(FAKE_NETWORK_ID) network.disconnect(FAKE_CONTAINER_ID) - assert client.api.disconnect_container_from_network.called_once_with( + client.api.disconnect_container_from_network.assert_called_once_with( FAKE_CONTAINER_ID, FAKE_NETWORK_ID ) @@ -61,4 +61,4 @@ def test_remove(self): client = make_fake_client() network = client.networks.get(FAKE_NETWORK_ID) network.remove() - assert client.api.remove_network.called_once_with(FAKE_NETWORK_ID) + client.api.remove_network.assert_called_once_with(FAKE_NETWORK_ID) diff --git a/tests/unit/ssladapter_test.py b/tests/unit/ssladapter_test.py deleted file mode 100644 index d3f2407c39..0000000000 --- a/tests/unit/ssladapter_test.py +++ /dev/null @@ -1,71 +0,0 @@ -import unittest -from ssl import match_hostname, CertificateError - -import pytest -from docker.transport import ssladapter - -try: - from ssl import OP_NO_SSLv3, OP_NO_SSLv2, OP_NO_TLSv1 -except ImportError: - OP_NO_SSLv2 = 0x1000000 - OP_NO_SSLv3 = 0x2000000 - OP_NO_TLSv1 = 0x4000000 - - -class SSLAdapterTest(unittest.TestCase): - def test_only_uses_tls(self): - ssl_context = ssladapter.urllib3.util.ssl_.create_urllib3_context() - - assert ssl_context.options & OP_NO_SSLv3 - # if OpenSSL is compiled without SSL2 support, OP_NO_SSLv2 will be 0 - assert not bool(OP_NO_SSLv2) or ssl_context.options & OP_NO_SSLv2 - assert not ssl_context.options & OP_NO_TLSv1 - - -class MatchHostnameTest(unittest.TestCase): - cert = { - 'issuer': ( - (('countryName', 'US'),), - (('stateOrProvinceName', 'California'),), - (('localityName', 'San Francisco'),), - (('organizationName', 'Docker Inc'),), - (('organizationalUnitName', 'Docker-Python'),), - (('commonName', 'localhost'),), - (('emailAddress', 'info@docker.com'),) - ), - 'notAfter': 'Mar 25 23:08:23 2030 GMT', - 'notBefore': 'Mar 25 23:08:23 2016 GMT', - 'serialNumber': 'BD5F894C839C548F', - 'subject': ( - (('countryName', 'US'),), - (('stateOrProvinceName', 'California'),), - (('localityName', 'San Francisco'),), - (('organizationName', 'Docker Inc'),), - (('organizationalUnitName', 'Docker-Python'),), - (('commonName', 'localhost'),), - (('emailAddress', 'info@docker.com'),) - ), - 'subjectAltName': ( - ('DNS', 'localhost'), - ('DNS', '*.gensokyo.jp'), - ('IP Address', '127.0.0.1'), - ), - 'version': 3 - } - - def test_match_ip_address_success(self): - assert match_hostname(self.cert, '127.0.0.1') is None - - def test_match_localhost_success(self): - assert match_hostname(self.cert, 'localhost') is None - - def test_match_dns_success(self): - assert match_hostname(self.cert, 'touhou.gensokyo.jp') is None - - def test_match_ip_address_failure(self): - with pytest.raises(CertificateError): - match_hostname(self.cert, '192.168.0.25') - - def test_match_dns_failure(self): - with pytest.raises(CertificateError): - match_hostname(self.cert, 'foobar.co.uk') diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index b47cb0c62f..de79e3037d 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -75,13 +75,12 @@ def test_kwargs_from_env_tls(self): os.environ.update(DOCKER_HOST='tcp://192.168.59.103:2376', DOCKER_CERT_PATH=TEST_CERT_DIR, DOCKER_TLS_VERIFY='1') - kwargs = kwargs_from_env(assert_hostname=False) + kwargs = kwargs_from_env() assert 'tcp://192.168.59.103:2376' == kwargs['base_url'] assert 'ca.pem' in kwargs['tls'].ca_cert assert 'cert.pem' in kwargs['tls'].cert[0] assert 'key.pem' in kwargs['tls'].cert[1] - assert kwargs['tls'].assert_hostname is False - assert kwargs['tls'].verify + assert kwargs['tls'].verify is True parsed_host = parse_host(kwargs['base_url'], IS_WINDOWS_PLATFORM, True) kwargs['version'] = DEFAULT_DOCKER_API_VERSION @@ -97,12 +96,11 @@ def test_kwargs_from_env_tls_verify_false(self): os.environ.update(DOCKER_HOST='tcp://192.168.59.103:2376', DOCKER_CERT_PATH=TEST_CERT_DIR, DOCKER_TLS_VERIFY='') - kwargs = kwargs_from_env(assert_hostname=True) + kwargs = kwargs_from_env() assert 'tcp://192.168.59.103:2376' == kwargs['base_url'] assert 'ca.pem' in kwargs['tls'].ca_cert assert 'cert.pem' in kwargs['tls'].cert[0] assert 'key.pem' in kwargs['tls'].cert[1] - assert kwargs['tls'].assert_hostname is True assert kwargs['tls'].verify is False parsed_host = parse_host(kwargs['base_url'], IS_WINDOWS_PLATFORM, True) kwargs['version'] = DEFAULT_DOCKER_API_VERSION @@ -123,12 +121,12 @@ def test_kwargs_from_env_tls_verify_false_no_cert(self): HOME=temp_dir, DOCKER_TLS_VERIFY='') os.environ.pop('DOCKER_CERT_PATH', None) - kwargs = kwargs_from_env(assert_hostname=True) + kwargs = kwargs_from_env() assert 'tcp://192.168.59.103:2376' == kwargs['base_url'] def test_kwargs_from_env_no_cert_path(self): + temp_dir = tempfile.mkdtemp() try: - temp_dir = tempfile.mkdtemp() cert_dir = os.path.join(temp_dir, '.docker') shutil.copytree(TEST_CERT_DIR, cert_dir) @@ -142,8 +140,7 @@ def test_kwargs_from_env_no_cert_path(self): assert cert_dir in kwargs['tls'].cert[0] assert cert_dir in kwargs['tls'].cert[1] finally: - if temp_dir: - shutil.rmtree(temp_dir) + shutil.rmtree(temp_dir) def test_kwargs_from_env_alternate_env(self): # Values in os.environ are entirely ignored if an alternate is diff --git a/tox.ini b/tox.ini index 2028dd3957..03467aea26 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py{37,38,39,310,311}, ruff +envlist = py{37,38,39,310,311,312}, ruff skipsdist=True [testenv] From fd2f5029f0b9c1b67addfa8956900db018efdf25 Mon Sep 17 00:00:00 2001 From: Milas Bowman Date: Tue, 21 Nov 2023 12:08:25 -0500 Subject: [PATCH 1253/1301] chore: add changelog for 7.0.0 (#3186) Signed-off-by: Milas Bowman --- docs/change-log.md | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/docs/change-log.md b/docs/change-log.md index 0d60f882d6..7719bbfb64 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -1,6 +1,40 @@ Changelog ========== +7.0.0 +----- +### Upgrade Notes +- Removed SSL version (`ssl_version`) and explicit hostname check (`assert_hostname`) options + - `assert_hostname` has not been used since Python 3.6 and was removed in 3.12 + - Python 3.7+ supports TLSv1.3 by default +- Websocket support is no longer included by default + - Use `pip install docker[websockets]` to include `websocket-client` dependency + - By default, `docker-py` hijacks the TCP connection and does not use Websockets + - Websocket client is only required to use `attach_socket(container, ws=True)` +- Python 3.7 no longer officially supported (reached end-of-life June 2023) + +### Features +- Python 3.12 support +- Full `networking_config` support for `containers.create()` + - Replaces `network_driver_opt` (added in 6.1.0) +- Add `health()` property to container that returns status (e.g. `unhealthy`) +- Add `pause` option to `container.commit()` +- Add support for bind mount propagation (e.g. `rshared`, `private`) + +### Bugfixes +- Consistently return `docker.errors.NotFound` on 404 responses + +### Miscellaneous +- Upgraded urllib3 version in `requirements.txt` (used for development/tests) +- Documentation typo fixes & formatting improvements +- Fixed integration test compatibility for newer Moby engine versions +- Switch to [ruff](https://github.com/astral-sh/ruff) for linting + +6.1.3 +----- +#### Bugfixes +- Fix compatibility with [`eventlet/eventlet`](https://github.com/eventlet/eventlet) + 6.1.2 ----- From 586988ce2d942a7cc093a9c162fa0401ceefc225 Mon Sep 17 00:00:00 2001 From: Milas Bowman Date: Tue, 21 Nov 2023 12:14:23 -0500 Subject: [PATCH 1254/1301] chore: remove support for Python 3.7 (#3187) Python 3.7 reached EOL in June 2023: https://endoflife.date/python Signed-off-by: Milas Bowman --- .github/workflows/ci.yml | 2 +- docker/version.py | 12 +++--------- setup.py | 3 +-- 3 files changed, 5 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 977199cebd..127d5b6822 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,7 +22,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v4 diff --git a/docker/version.py b/docker/version.py index 44eac8c5dc..dca45bf047 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,14 +1,8 @@ try: from ._version import __version__ except ImportError: + from importlib.metadata import version, PackageNotFoundError try: - # importlib.metadata available in Python 3.8+, the fallback (0.0.0) - # is fine because release builds use _version (above) rather than - # this code path, so it only impacts developing w/ 3.7 - from importlib.metadata import version, PackageNotFoundError - try: - __version__ = version('docker') - except PackageNotFoundError: - __version__ = '0.0.0' - except ImportError: + __version__ = version('docker') + except PackageNotFoundError: __version__ = '0.0.0' diff --git a/setup.py b/setup.py index d63cbe0a1c..98736247d6 100644 --- a/setup.py +++ b/setup.py @@ -59,7 +59,7 @@ install_requires=requirements, tests_require=test_requirements, extras_require=extras_require, - python_requires='>=3.7', + python_requires='>=3.8', zip_safe=False, test_suite='tests', classifiers=[ @@ -69,7 +69,6 @@ 'Operating System :: OS Independent', 'Programming Language :: Python', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', From 714096923918183f3ab4e11973156551dc5559f7 Mon Sep 17 00:00:00 2001 From: Milas Bowman Date: Tue, 21 Nov 2023 12:17:12 -0500 Subject: [PATCH 1255/1301] chore: update MAINTAINERS and remove CODEOWNERS (#3188) Update `MAINTAINERS` with the current folks, adn remove the `CODEOWNERS` file entirely -- it's not really helpful here, as this project isn't big enough to have multiple subsections with different maintainers/owners. Signed-off-by: Milas Bowman --- .github/CODEOWNERS | 6 ------ MAINTAINERS | 16 ++++++++++++++-- 2 files changed, 14 insertions(+), 8 deletions(-) delete mode 100644 .github/CODEOWNERS diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS deleted file mode 100644 index 5df3014937..0000000000 --- a/.github/CODEOWNERS +++ /dev/null @@ -1,6 +0,0 @@ -# GitHub code owners -# See https://help.github.com/articles/about-codeowners/ -# -# KEEP THIS FILE SORTED. Order is important. Last match takes precedence. - -* @aiordache @ulyssessouza diff --git a/MAINTAINERS b/MAINTAINERS index b74cb28fd3..96ba4752e8 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -11,17 +11,19 @@ [Org] [Org."Core maintainers"] people = [ - "aiordache", - "ulyssessouza", + "glours", + "milas", ] [Org.Alumni] people = [ + "aiordache", "aanand", "bfirsh", "dnephin", "mnowster", "mpetazzoni", "shin-", + "ulyssessouza", ] [people] @@ -52,6 +54,16 @@ Email = "dnephin@gmail.com" GitHub = "dnephin" + [people.glours] + Name = "Guillaume Lours" + Email = "705411+glours@users.noreply.github.com" + GitHub = "glours" + + [people.milas] + Name = "Milas Bowman" + Email = "devnull@milas.dev" + GitHub = "milas" + [people.mnowster] Name = "Mazz Mosley" Email = "mazz@houseofmnowster.com" From cb8f2c6630584d6d1b2d9296a0c780af0f5e5549 Mon Sep 17 00:00:00 2001 From: Milas Bowman Date: Mon, 27 Nov 2023 09:17:47 -0500 Subject: [PATCH 1256/1301] chore: fix missing setuptools in CI (#3189) Install `setuptools` in addition to `wheel` before trying to run `python setup.py` manually. Note that `setuptools` is already correctly listed in the `pyproject.toml` file for consumers installing via `pip` etc, but in CI the file is run directly to generate `sdist` and `bdist_wheel` artifacts for PyPI. Signed-off-by: Milas Bowman --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b8b1f57d1f..721020ac33 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -28,7 +28,7 @@ jobs: - name: Generate Pacakge run: | - pip3 install wheel + pip3 install setuptools wheel python setup.py sdist bdist_wheel env: SETUPTOOLS_SCM_PRETEND_VERSION_FOR_DOCKER: ${{ inputs.tag }} From a9b5494fd0574e520c735e1d0c4a303528d48063 Mon Sep 17 00:00:00 2001 From: Daniel Lombardi Date: Tue, 5 Dec 2023 02:03:13 -0300 Subject: [PATCH 1257/1301] fix: validate tag before build using OCI regex (#3191) Sources: * https://github.com/opencontainers/distribution-spec * https://docs.docker.com/engine/reference/commandline/tag/ Closes #3153. --------- Signed-off-by: Daniel Lombardi --- docker/api/build.py | 9 +- docker/utils/__init__.py | 2 +- docker/utils/build.py | 8 ++ tests/unit/api_build_test.py | 257 +++++++++++++++++++---------------- 4 files changed, 157 insertions(+), 119 deletions(-) diff --git a/docker/api/build.py b/docker/api/build.py index 439f4dc351..9c8b4e6ae9 100644 --- a/docker/api/build.py +++ b/docker/api/build.py @@ -129,13 +129,16 @@ def build(self, path=None, tag=None, quiet=False, fileobj=None, raise errors.DockerException( 'Can not use custom encoding if gzip is enabled' ) - + if tag is not None: + if not utils.match_tag(tag): + raise errors.DockerException( + f"invalid tag '{tag}': invalid reference format" + ) for key in container_limits.keys(): if key not in constants.CONTAINER_LIMITS_KEYS: raise errors.DockerException( - f'Invalid container_limits key {key}' + f"invalid tag '{tag}': invalid reference format" ) - if custom_context: if not fileobj: raise TypeError("You must specify fileobj with custom_context") diff --git a/docker/utils/__init__.py b/docker/utils/__init__.py index 944c6e65e0..b4bef7d47c 100644 --- a/docker/utils/__init__.py +++ b/docker/utils/__init__.py @@ -1,5 +1,5 @@ -from .build import create_archive, exclude_paths, mkbuildcontext, tar +from .build import match_tag, create_archive, exclude_paths, mkbuildcontext, tar from .decorators import check_resource, minimum_version, update_headers from .utils import ( compare_version, convert_port_bindings, convert_volume_binds, diff --git a/docker/utils/build.py b/docker/utils/build.py index 8d18c2be71..a5c4b0c2d9 100644 --- a/docker/utils/build.py +++ b/docker/utils/build.py @@ -9,6 +9,14 @@ _SEP = re.compile('/|\\\\') if IS_WINDOWS_PLATFORM else re.compile('/') +_TAG = re.compile( + r"^[a-z0-9]+((\.|_|__|-+)[a-z0-9]+)*(\/[a-z0-9]+((\.|_|__|-+)[a-z0-9]+)*)*" \ + + "(:[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127})?$" +) + + +def match_tag(tag: str) -> bool: + return bool(_TAG.match(tag)) def tar(path, exclude=None, dockerfile=None, fileobj=None, gzip=False): diff --git a/tests/unit/api_build_test.py b/tests/unit/api_build_test.py index cbecd1e544..01958c3e1f 100644 --- a/tests/unit/api_build_test.py +++ b/tests/unit/api_build_test.py @@ -2,181 +2,206 @@ import io import shutil +import pytest + import docker -from docker import auth +from docker import auth, errors from docker.api.build import process_dockerfile -import pytest - from ..helpers import make_tree from .api_test import BaseAPIClientTest, fake_request, url_prefix class BuildTest(BaseAPIClientTest): def test_build_container(self): - script = io.BytesIO('\n'.join([ - 'FROM busybox', - 'RUN mkdir -p /tmp/test', - 'EXPOSE 8080', - 'ADD https://dl.dropboxusercontent.com/u/20637798/silence.tar.gz' - ' /tmp/silence.tar.gz' - ]).encode('ascii')) + script = io.BytesIO( + "\n".join( + [ + "FROM busybox", + "RUN mkdir -p /tmp/test", + "EXPOSE 8080", + "ADD https://dl.dropboxusercontent.com/u/20637798/silence.tar.gz" + " /tmp/silence.tar.gz", + ] + ).encode("ascii") + ) self.client.build(fileobj=script) def test_build_container_pull(self): - script = io.BytesIO('\n'.join([ - 'FROM busybox', - 'RUN mkdir -p /tmp/test', - 'EXPOSE 8080', - 'ADD https://dl.dropboxusercontent.com/u/20637798/silence.tar.gz' - ' /tmp/silence.tar.gz' - ]).encode('ascii')) + script = io.BytesIO( + "\n".join( + [ + "FROM busybox", + "RUN mkdir -p /tmp/test", + "EXPOSE 8080", + "ADD https://dl.dropboxusercontent.com/u/20637798/silence.tar.gz" + " /tmp/silence.tar.gz", + ] + ).encode("ascii") + ) self.client.build(fileobj=script, pull=True) def test_build_container_custom_context(self): - script = io.BytesIO('\n'.join([ - 'FROM busybox', - 'RUN mkdir -p /tmp/test', - 'EXPOSE 8080', - 'ADD https://dl.dropboxusercontent.com/u/20637798/silence.tar.gz' - ' /tmp/silence.tar.gz' - ]).encode('ascii')) + script = io.BytesIO( + "\n".join( + [ + "FROM busybox", + "RUN mkdir -p /tmp/test", + "EXPOSE 8080", + "ADD https://dl.dropboxusercontent.com/u/20637798/silence.tar.gz" + " /tmp/silence.tar.gz", + ] + ).encode("ascii") + ) context = docker.utils.mkbuildcontext(script) self.client.build(fileobj=context, custom_context=True) def test_build_container_custom_context_gzip(self): - script = io.BytesIO('\n'.join([ - 'FROM busybox', - 'RUN mkdir -p /tmp/test', - 'EXPOSE 8080', - 'ADD https://dl.dropboxusercontent.com/u/20637798/silence.tar.gz' - ' /tmp/silence.tar.gz' - ]).encode('ascii')) + script = io.BytesIO( + "\n".join( + [ + "FROM busybox", + "RUN mkdir -p /tmp/test", + "EXPOSE 8080", + "ADD https://dl.dropboxusercontent.com/u/20637798/silence.tar.gz" + " /tmp/silence.tar.gz", + ] + ).encode("ascii") + ) context = docker.utils.mkbuildcontext(script) gz_context = gzip.GzipFile(fileobj=context) - self.client.build( - fileobj=gz_context, - custom_context=True, - encoding="gzip" - ) + self.client.build(fileobj=gz_context, custom_context=True, encoding="gzip") def test_build_remote_with_registry_auth(self): - self.client._auth_configs = auth.AuthConfig({ - 'auths': { - 'https://example.com': { - 'user': 'example', - 'password': 'example', - 'email': 'example@example.com' + self.client._auth_configs = auth.AuthConfig( + { + "auths": { + "https://example.com": { + "user": "example", + "password": "example", + "email": "example@example.com", + } } } - }) + ) - expected_params = {'t': None, 'q': False, 'dockerfile': None, - 'rm': False, 'nocache': False, 'pull': False, - 'forcerm': False, - 'remote': 'https://github.com/docker-library/mongo'} + expected_params = { + "t": None, + "q": False, + "dockerfile": None, + "rm": False, + "nocache": False, + "pull": False, + "forcerm": False, + "remote": "https://github.com/docker-library/mongo", + } expected_headers = { - 'X-Registry-Config': auth.encode_header( - self.client._auth_configs.auths - ) + "X-Registry-Config": auth.encode_header(self.client._auth_configs.auths) } - self.client.build(path='https://github.com/docker-library/mongo') + self.client.build(path="https://github.com/docker-library/mongo") fake_request.assert_called_with( - 'POST', + "POST", f"{url_prefix}build", stream=True, data=None, headers=expected_headers, params=expected_params, - timeout=None + timeout=None, ) def test_build_container_with_named_dockerfile(self): - self.client.build('.', dockerfile='nameddockerfile') + self.client.build(".", dockerfile="nameddockerfile") + + def test_build_with_invalid_tag(self): + with pytest.raises(errors.DockerException): + self.client.build(".", tag="https://example.com") def test_build_container_with_container_limits(self): - self.client.build('.', container_limits={ - 'memory': 1024 * 1024, - 'cpusetcpus': 1, - 'cpushares': 1000, - 'memswap': 1024 * 1024 * 8 - }) + self.client.build( + ".", + container_limits={ + "memory": 1024 * 1024, + "cpusetcpus": 1, + "cpushares": 1000, + "memswap": 1024 * 1024 * 8, + }, + ) def test_build_container_invalid_container_limits(self): with pytest.raises(docker.errors.DockerException): - self.client.build('.', container_limits={ - 'foo': 'bar' - }) + self.client.build(".", container_limits={"foo": "bar"}) def test_set_auth_headers_with_empty_dict_and_auth_configs(self): - self.client._auth_configs = auth.AuthConfig({ - 'auths': { - 'https://example.com': { - 'user': 'example', - 'password': 'example', - 'email': 'example@example.com' + self.client._auth_configs = auth.AuthConfig( + { + "auths": { + "https://example.com": { + "user": "example", + "password": "example", + "email": "example@example.com", + } } } - }) + ) headers = {} expected_headers = { - 'X-Registry-Config': auth.encode_header( - self.client._auth_configs.auths - ) + "X-Registry-Config": auth.encode_header(self.client._auth_configs.auths) } self.client._set_auth_headers(headers) assert headers == expected_headers def test_set_auth_headers_with_dict_and_auth_configs(self): - self.client._auth_configs = auth.AuthConfig({ - 'auths': { - 'https://example.com': { - 'user': 'example', - 'password': 'example', - 'email': 'example@example.com' + self.client._auth_configs = auth.AuthConfig( + { + "auths": { + "https://example.com": { + "user": "example", + "password": "example", + "email": "example@example.com", + } } } - }) + ) - headers = {'foo': 'bar'} + headers = {"foo": "bar"} expected_headers = { - 'X-Registry-Config': auth.encode_header( - self.client._auth_configs.auths - ), - 'foo': 'bar' + "X-Registry-Config": auth.encode_header(self.client._auth_configs.auths), + "foo": "bar", } self.client._set_auth_headers(headers) assert headers == expected_headers def test_set_auth_headers_with_dict_and_no_auth_configs(self): - headers = {'foo': 'bar'} - expected_headers = { - 'foo': 'bar' - } + headers = {"foo": "bar"} + expected_headers = {"foo": "bar"} self.client._set_auth_headers(headers) assert headers == expected_headers @pytest.mark.skipif( - not docker.constants.IS_WINDOWS_PLATFORM, - reason='Windows-specific syntax') + not docker.constants.IS_WINDOWS_PLATFORM, reason="Windows-specific syntax" + ) def test_process_dockerfile_win_longpath_prefix(self): dirs = [ - 'foo', 'foo/bar', 'baz', + "foo", + "foo/bar", + "baz", ] files = [ - 'Dockerfile', 'foo/Dockerfile.foo', 'foo/bar/Dockerfile.bar', - 'baz/Dockerfile.baz', + "Dockerfile", + "foo/Dockerfile.foo", + "foo/bar/Dockerfile.bar", + "baz/Dockerfile.baz", ] base = make_tree(dirs, files) @@ -186,40 +211,42 @@ def pre(path): return docker.constants.WINDOWS_LONGPATH_PREFIX + path assert process_dockerfile(None, pre(base)) == (None, None) - assert process_dockerfile('Dockerfile', pre(base)) == ( - 'Dockerfile', None + assert process_dockerfile("Dockerfile", pre(base)) == ("Dockerfile", None) + assert process_dockerfile("foo/Dockerfile.foo", pre(base)) == ( + "foo/Dockerfile.foo", + None, ) - assert process_dockerfile('foo/Dockerfile.foo', pre(base)) == ( - 'foo/Dockerfile.foo', None + assert process_dockerfile("../Dockerfile", pre(f"{base}\\foo"))[1] is not None + assert process_dockerfile("../baz/Dockerfile.baz", pre(f"{base}/baz")) == ( + "../baz/Dockerfile.baz", + None, ) - assert process_dockerfile( - '../Dockerfile', pre(f"{base}\\foo") - )[1] is not None - assert process_dockerfile( - '../baz/Dockerfile.baz', pre(f"{base}/baz") - ) == ('../baz/Dockerfile.baz', None) def test_process_dockerfile(self): dirs = [ - 'foo', 'foo/bar', 'baz', + "foo", + "foo/bar", + "baz", ] files = [ - 'Dockerfile', 'foo/Dockerfile.foo', 'foo/bar/Dockerfile.bar', - 'baz/Dockerfile.baz', + "Dockerfile", + "foo/Dockerfile.foo", + "foo/bar/Dockerfile.bar", + "baz/Dockerfile.baz", ] base = make_tree(dirs, files) self.addCleanup(shutil.rmtree, base) assert process_dockerfile(None, base) == (None, None) - assert process_dockerfile('Dockerfile', base) == ('Dockerfile', None) - assert process_dockerfile('foo/Dockerfile.foo', base) == ( - 'foo/Dockerfile.foo', None + assert process_dockerfile("Dockerfile", base) == ("Dockerfile", None) + assert process_dockerfile("foo/Dockerfile.foo", base) == ( + "foo/Dockerfile.foo", + None, ) - assert process_dockerfile( - '../Dockerfile', f"{base}/foo" - )[1] is not None - assert process_dockerfile('../baz/Dockerfile.baz', f"{base}/baz") == ( - '../baz/Dockerfile.baz', None + assert process_dockerfile("../Dockerfile", f"{base}/foo")[1] is not None + assert process_dockerfile("../baz/Dockerfile.baz", f"{base}/baz") == ( + "../baz/Dockerfile.baz", + None, ) From 3d0a3f1d77878a7197b5b65ba0abefd3b72c6f72 Mon Sep 17 00:00:00 2001 From: Emran Batmanghelich Date: Tue, 5 Dec 2023 18:35:44 +0330 Subject: [PATCH 1258/1301] feat: accept all / filters / keep_storage in prune_builds (#3192) Added in API v1.39. --------- Signed-off-by: Emran Batmanghelich --- docker/api/build.py | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/docker/api/build.py b/docker/api/build.py index 9c8b4e6ae9..abd5ab52a8 100644 --- a/docker/api/build.py +++ b/docker/api/build.py @@ -279,10 +279,24 @@ def build(self, path=None, tag=None, quiet=False, fileobj=None, return self._stream_helper(response, decode=decode) @utils.minimum_version('1.31') - def prune_builds(self): + def prune_builds(self, filters=None, keep_storage=None, all=None): """ Delete the builder cache + Args: + filters (dict): Filters to process on the prune list. + Needs Docker API v1.39+ + Available filters: + - dangling (bool): When set to true (or 1), prune only + unused and untagged images. + - until (str): Can be Unix timestamps, date formatted + timestamps, or Go duration strings (e.g. 10m, 1h30m) computed + relative to the daemon's local time. + keep_storage (int): Amount of disk space in bytes to keep for cache. + Needs Docker API v1.39+ + all (bool): Remove all types of build cache. + Needs Docker API v1.39+ + Returns: (dict): A dictionary containing information about the operation's result. The ``SpaceReclaimed`` key indicates the amount of @@ -293,7 +307,20 @@ def prune_builds(self): If the server returns an error. """ url = self._url("/build/prune") - return self._result(self._post(url), True) + if (filters, keep_storage, all) != (None, None, None) \ + and utils.version_lt(self._version, '1.39'): + raise errors.InvalidVersion( + '`filters`, `keep_storage`, and `all` args are only available ' + 'for API version > 1.38' + ) + params = {} + if filters is not None: + params['filters'] = utils.convert_filters(filters) + if keep_storage is not None: + params['keep-storage'] = keep_storage + if all is not None: + params['all'] = all + return self._result(self._post(url, params=params), True) def _set_auth_headers(self, headers): log.debug('Looking for auth config') From 5388413dde6894c41b90945507cd81c343b9aeee Mon Sep 17 00:00:00 2001 From: Milas Bowman Date: Thu, 7 Dec 2023 15:41:29 -0500 Subject: [PATCH 1259/1301] chore: update changelog and maintainer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Preparing for the 7.0.0 final release 🎉 Added a couple more changelog items that came in as part of `7.0.0b2` and updated the maintainer to be generically Docker, Inc. instead of an individual. Signed-off-by: Milas Bowman --- docs/change-log.md | 2 ++ setup.py | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/change-log.md b/docs/change-log.md index 7719bbfb64..faf868ff8c 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -20,9 +20,11 @@ Changelog - Add `health()` property to container that returns status (e.g. `unhealthy`) - Add `pause` option to `container.commit()` - Add support for bind mount propagation (e.g. `rshared`, `private`) +- Add `filters`, `keep_storage`, and `all` parameters to `prune_builds()` (requires API v1.39+) ### Bugfixes - Consistently return `docker.errors.NotFound` on 404 responses +- Validate tag format before image push ### Miscellaneous - Upgraded urllib3 version in `requirements.txt` (used for development/tests) diff --git a/setup.py b/setup.py index 98736247d6..b6a024f81a 100644 --- a/setup.py +++ b/setup.py @@ -78,6 +78,6 @@ 'Topic :: Utilities', 'License :: OSI Approved :: Apache Software License', ], - maintainer='Ulysses Souza', - maintainer_email='ulysses.souza@docker.com', + maintainer='Docker, Inc.', + maintainer_email='no-reply@docker.com', ) From 2a5f354b502fe0624239f8b2607cc624857027cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Fri, 15 Dec 2023 10:40:27 +0100 Subject: [PATCH 1260/1301] Bump default API version to 1.43 (Moby 24.0) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Paweł Gronowski --- Makefile | 4 ++-- docker/constants.py | 2 +- tests/Dockerfile-ssh-dind | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 79486e3ec2..00ebca05ce 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ -TEST_API_VERSION ?= 1.41 -TEST_ENGINE_VERSION ?= 20.10 +TEST_API_VERSION ?= 1.43 +TEST_ENGINE_VERSION ?= 24.0 ifeq ($(OS),Windows_NT) PLATFORM := Windows diff --git a/docker/constants.py b/docker/constants.py index ed341a9020..71e543e530 100644 --- a/docker/constants.py +++ b/docker/constants.py @@ -1,7 +1,7 @@ import sys from .version import __version__ -DEFAULT_DOCKER_API_VERSION = '1.41' +DEFAULT_DOCKER_API_VERSION = '1.43' MINIMUM_DOCKER_API_VERSION = '1.21' DEFAULT_TIMEOUT_SECONDS = 60 STREAM_HEADER_SIZE_BYTES = 8 diff --git a/tests/Dockerfile-ssh-dind b/tests/Dockerfile-ssh-dind index 0da15aa40f..2b7332b8b6 100644 --- a/tests/Dockerfile-ssh-dind +++ b/tests/Dockerfile-ssh-dind @@ -1,7 +1,7 @@ # syntax=docker/dockerfile:1 -ARG API_VERSION=1.41 -ARG ENGINE_VERSION=20.10 +ARG API_VERSION=1.43 +ARG ENGINE_VERSION=24.0 FROM docker:${ENGINE_VERSION}-dind From 0fad869cc62829b6e51898dc8e2ca0832aec6d43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Tue, 19 Dec 2023 10:25:34 +0100 Subject: [PATCH 1261/1301] integration/commit: Don't check for deprecated fields MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Container related Image fields (`Container` and `ContainerConfig`) will be deprecated in API v1.44 and will be removed in v1.45. Signed-off-by: Paweł Gronowski --- tests/integration/api_image_test.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/tests/integration/api_image_test.py b/tests/integration/api_image_test.py index 7081b53b8f..5c37219581 100644 --- a/tests/integration/api_image_test.py +++ b/tests/integration/api_image_test.py @@ -85,13 +85,8 @@ def test_commit(self): img_id = res['Id'] self.tmp_imgs.append(img_id) img = self.client.inspect_image(img_id) - assert 'Container' in img - assert img['Container'].startswith(id) - assert 'ContainerConfig' in img - assert 'Image' in img['ContainerConfig'] - assert TEST_IMG == img['ContainerConfig']['Image'] - busybox_id = self.client.inspect_image(TEST_IMG)['Id'] assert 'Parent' in img + busybox_id = self.client.inspect_image(TEST_IMG)['Id'] assert img['Parent'] == busybox_id def test_commit_with_changes(self): @@ -103,8 +98,6 @@ def test_commit_with_changes(self): ) self.tmp_imgs.append(img_id) img = self.client.inspect_image(img_id) - assert 'Container' in img - assert img['Container'].startswith(cid['Id']) assert '8000/tcp' in img['Config']['ExposedPorts'] assert img['Config']['Cmd'] == ['bash'] From 1784cc2962529dac8ea9aa8f3ecc5798f600b6e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Fri, 22 Dec 2023 10:20:12 +0100 Subject: [PATCH 1262/1301] utils: Fix datetime_to_timestamp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace usage of deprecated function `datetime.utcfromtimestamp` and make sure the input date is UTC before subtracting. Signed-off-by: Paweł Gronowski --- docker/utils/utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/utils/utils.py b/docker/utils/utils.py index 759ddd2f1a..dbd5130301 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -5,7 +5,7 @@ import os.path import shlex import string -from datetime import datetime +from datetime import datetime, timezone from packaging.version import Version from .. import errors @@ -394,8 +394,8 @@ def convert_filters(filters): def datetime_to_timestamp(dt): - """Convert a UTC datetime to a Unix timestamp""" - delta = dt - datetime.utcfromtimestamp(0) + """Convert a datetime to a Unix timestamp""" + delta = dt.astimezone(timezone.utc) - datetime(1970, 1, 1, tzinfo=timezone.utc) return delta.seconds + delta.days * 24 * 3600 From 3ec5a6849a6cad4c5f5f3bafb5f74b5827fec14c Mon Sep 17 00:00:00 2001 From: Sven Date: Wed, 3 Jan 2024 16:48:45 +0100 Subject: [PATCH 1263/1301] fix(build): tag regex should allow ports (#3196) Update the regex and add test cases. (There are some xfails here for cases that the regex is not currently handling. It's too strict for IPv6 domains at the moment.) Closes: https://github.com/docker/docker-py/issues/3195 Related: https://github.com/opencontainers/distribution-spec/pull/498 Signed-off-by: Sven Kieske Signed-off-by: Milas Bowman Co-authored-by: Milas Bowman --- docker/utils/build.py | 5 ++-- tests/unit/utils_build_test.py | 53 ++++++++++++++++++++++++++++++++-- 2 files changed, 53 insertions(+), 5 deletions(-) diff --git a/docker/utils/build.py b/docker/utils/build.py index a5c4b0c2d9..86a4423f0b 100644 --- a/docker/utils/build.py +++ b/docker/utils/build.py @@ -10,8 +10,9 @@ _SEP = re.compile('/|\\\\') if IS_WINDOWS_PLATFORM else re.compile('/') _TAG = re.compile( - r"^[a-z0-9]+((\.|_|__|-+)[a-z0-9]+)*(\/[a-z0-9]+((\.|_|__|-+)[a-z0-9]+)*)*" \ - + "(:[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127})?$" + r"^[a-z0-9]+((\.|_|__|-+)[a-z0-9]+)*" + r"(?::[0-9]+)?(/[a-z0-9]+((\.|_|__|-+)[a-z0-9]+)*)*" + r"(:[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127})?$" ) diff --git a/tests/unit/utils_build_test.py b/tests/unit/utils_build_test.py index fa7d833de2..5f1bb1ec0c 100644 --- a/tests/unit/utils_build_test.py +++ b/tests/unit/utils_build_test.py @@ -6,11 +6,10 @@ import tempfile import unittest +import pytest from docker.constants import IS_WINDOWS_PLATFORM -from docker.utils import exclude_paths, tar - -import pytest +from docker.utils import exclude_paths, tar, match_tag from ..helpers import make_tree @@ -489,3 +488,51 @@ def test_tar_directory_link(self): assert member in names assert 'a/c/b' in names assert 'a/c/b/utils.py' not in names + + +# selected test cases from https://github.com/distribution/reference/blob/8507c7fcf0da9f570540c958ea7b972c30eeaeca/reference_test.go#L13-L328 +@pytest.mark.parametrize("tag,expected", [ + ("test_com", True), + ("test.com:tag", True), + # N.B. this implicitly means "docker.io/library/test.com:5000" + # i.e. the `5000` is a tag, not a port here! + ("test.com:5000", True), + ("test.com/repo:tag", True), + ("test:5000/repo", True), + ("test:5000/repo:tag", True), + ("test:5000/repo", True), + ("", False), + (":justtag", False), + ("Uppercase:tag", False), + ("test:5000/Uppercase/lowercase:tag", False), + ("lowercase:Uppercase", True), + # length limits not enforced + pytest.param("a/"*128 + "a:tag", False, marks=pytest.mark.xfail), + ("a/"*127 + "a:tag-puts-this-over-max", True), + ("aa/asdf$$^/aa", False), + ("sub-dom1.foo.com/bar/baz/quux", True), + ("sub-dom1.foo.com/bar/baz/quux:some-long-tag", True), + ("b.gcr.io/test.example.com/my-app:test.example.com", True), + ("xn--n3h.com/myimage:xn--n3h.com", True), + ("foo_bar.com:8080", True), + ("foo/foo_bar.com:8080", True), + ("192.168.1.1", True), + ("192.168.1.1:tag", True), + ("192.168.1.1:5000", True), + ("192.168.1.1/repo", True), + ("192.168.1.1:5000/repo", True), + ("192.168.1.1:5000/repo:5050", True), + # regex does not properly handle ipv6 + pytest.param("[2001:db8::1]", False, marks=pytest.mark.xfail), + ("[2001:db8::1]:5000", False), + pytest.param("[2001:db8::1]/repo", True, marks=pytest.mark.xfail), + pytest.param("[2001:db8:1:2:3:4:5:6]/repo:tag", True, marks=pytest.mark.xfail), + pytest.param("[2001:db8::1]:5000/repo", True, marks=pytest.mark.xfail), + pytest.param("[2001:db8::1]:5000/repo:tag", True, marks=pytest.mark.xfail), + pytest.param("[2001:db8::]:5000/repo", True, marks=pytest.mark.xfail), + pytest.param("[::1]:5000/repo", True, marks=pytest.mark.xfail), + ("[fe80::1%eth0]:5000/repo", False), + ("[fe80::1%@invalidzone]:5000/repo", False), +]) +def test_match_tag(tag: str, expected: bool): + assert match_tag(tag) == expected From b8a6987cd5fc254a71db4505398479ceef2d15c8 Mon Sep 17 00:00:00 2001 From: Khushiyant Date: Thu, 4 Jan 2024 00:14:53 +0530 Subject: [PATCH 1264/1301] fix: keyerror when creating new config (#3200) Closes #3110. --------- Signed-off-by: Khushiyant --- docker/models/configs.py | 1 + tests/unit/fake_api.py | 9 +++++++++ tests/unit/fake_api_client.py | 1 + tests/unit/models_configs_test.py | 10 ++++++++++ 4 files changed, 21 insertions(+) create mode 100644 tests/unit/models_configs_test.py diff --git a/docker/models/configs.py b/docker/models/configs.py index 3588c8b5dc..5ef1377844 100644 --- a/docker/models/configs.py +++ b/docker/models/configs.py @@ -30,6 +30,7 @@ class ConfigCollection(Collection): def create(self, **kwargs): obj = self.client.api.create_config(**kwargs) + obj.setdefault("Spec", {})["Name"] = kwargs.get("name") return self.prepare_model(obj) create.__doc__ = APIClient.create_config.__doc__ diff --git a/tests/unit/fake_api.py b/tests/unit/fake_api.py index 0524becdc7..03e53cc648 100644 --- a/tests/unit/fake_api.py +++ b/tests/unit/fake_api.py @@ -19,6 +19,8 @@ FAKE_NODE_ID = '24ifsmvkjbyhk' FAKE_SECRET_ID = 'epdyrw4tsi03xy3deu8g8ly6o' FAKE_SECRET_NAME = 'super_secret' +FAKE_CONFIG_ID = 'sekvs771242jfdjnvfuds8232' +FAKE_CONFIG_NAME = 'super_config' # Each method is prefixed with HTTP method (get, post...) # for clarity and readability @@ -512,6 +514,11 @@ def post_fake_secret(): response = {'ID': FAKE_SECRET_ID} return status_code, response +def post_fake_config(): + status_code = 200 + response = {'ID': FAKE_CONFIG_ID} + return status_code, response + # Maps real api url to fake response callback prefix = 'http+docker://localhost' @@ -630,4 +637,6 @@ def post_fake_secret(): post_fake_network_disconnect, f'{prefix}/{CURRENT_VERSION}/secrets/create': post_fake_secret, + f'{prefix}/{CURRENT_VERSION}/configs/create': + post_fake_config, } diff --git a/tests/unit/fake_api_client.py b/tests/unit/fake_api_client.py index 95cf63b492..7979942167 100644 --- a/tests/unit/fake_api_client.py +++ b/tests/unit/fake_api_client.py @@ -37,6 +37,7 @@ def make_fake_api_client(overrides=None): 'create_host_config.side_effect': api_client.create_host_config, 'create_network.return_value': fake_api.post_fake_network()[1], 'create_secret.return_value': fake_api.post_fake_secret()[1], + 'create_config.return_value': fake_api.post_fake_config()[1], 'exec_create.return_value': fake_api.post_fake_exec_create()[1], 'exec_start.return_value': fake_api.post_fake_exec_start()[1], 'images.return_value': fake_api.get_fake_images()[1], diff --git a/tests/unit/models_configs_test.py b/tests/unit/models_configs_test.py new file mode 100644 index 0000000000..6960397ff6 --- /dev/null +++ b/tests/unit/models_configs_test.py @@ -0,0 +1,10 @@ +import unittest + +from .fake_api_client import make_fake_client +from .fake_api import FAKE_CONFIG_NAME + +class CreateConfigsTest(unittest.TestCase): + def test_create_config(self): + client = make_fake_client() + config = client.configs.create(name="super_config", data="config") + assert config.__repr__() == "".format(FAKE_CONFIG_NAME) From 08956b5fbc1adc76681fea308526a9c77065c22b Mon Sep 17 00:00:00 2001 From: Aarni Koskela Date: Wed, 3 Jan 2024 20:49:07 +0200 Subject: [PATCH 1265/1301] ci: update Ruff & fix some minor issues (#3206) Signed-off-by: Aarni Koskela --- .github/workflows/ci.yml | 2 +- docker/models/containers.py | 6 +++--- docker/utils/socket.py | 4 ++-- docker/utils/utils.py | 2 +- pyproject.toml | 4 +++- test-requirements.txt | 2 +- tests/integration/api_build_test.py | 4 +--- tests/ssh/api_build_test.py | 4 +--- tests/unit/api_test.py | 2 +- 9 files changed, 14 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 127d5b6822..628c53504a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,7 +14,7 @@ jobs: - uses: actions/setup-python@v4 with: python-version: '3.x' - - run: pip install -U ruff==0.0.284 + - run: pip install -U ruff==0.1.8 - name: Run ruff run: ruff docker tests diff --git a/docker/models/containers.py b/docker/models/containers.py index 4725d6f6fb..32676bb852 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -903,9 +903,9 @@ def run(self, image, command=None, stdout=True, stderr=False, container, exit_status, command, image, out ) - return out if stream or out is None else b''.join( - [line for line in out] - ) + if stream or out is None: + return out + return b''.join(out) def create(self, image, command=None, **kwargs): """ diff --git a/docker/utils/socket.py b/docker/utils/socket.py index 2306ed0736..c7cb584d4f 100644 --- a/docker/utils/socket.py +++ b/docker/utils/socket.py @@ -64,7 +64,7 @@ def read_exactly(socket, n): Reads exactly n bytes from socket Raises SocketError if there isn't enough data """ - data = bytes() + data = b"" while len(data) < n: next_data = read(socket, n - len(data)) if not next_data: @@ -152,7 +152,7 @@ def consume_socket_output(frames, demux=False): if demux is False: # If the streams are multiplexed, the generator returns strings, that # we just need to concatenate. - return bytes().join(frames) + return b"".join(frames) # If the streams are demultiplexed, the generator yields tuples # (stdout, stderr) diff --git a/docker/utils/utils.py b/docker/utils/utils.py index dbd5130301..efe9f9a3cd 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -152,7 +152,7 @@ def convert_volume_binds(binds): ] if 'propagation' in v and v['propagation'] in propagation_modes: if mode: - mode = ','.join([mode, v['propagation']]) + mode = f"{mode},{v['propagation']}" else: mode = v['propagation'] diff --git a/pyproject.toml b/pyproject.toml index 0a67279661..a64e120eee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,14 +5,16 @@ requires = ["setuptools>=45", "setuptools_scm[toml]>=6.2"] write_to = 'docker/_version.py' [tool.ruff] -target-version = "py37" +target-version = "py38" extend-select = [ "B", "C", "F", + "UP", "W", ] ignore = [ + "UP012", # unnecessary `UTF-8` argument (we want to be explicit) "C901", # too complex (there's a whole bunch of these) ] diff --git a/test-requirements.txt b/test-requirements.txt index 031d0acf0a..2c0e3622c6 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,6 +1,6 @@ setuptools==65.5.1 coverage==7.2.7 -ruff==0.0.284 +ruff==0.1.8 pytest==7.4.2 pytest-cov==4.1.0 pytest-timeout==2.1.0 diff --git a/tests/integration/api_build_test.py b/tests/integration/api_build_test.py index 2add2d87af..540ef2b0fb 100644 --- a/tests/integration/api_build_test.py +++ b/tests/integration/api_build_test.py @@ -389,9 +389,7 @@ def test_build_stderr_data(self): lines = [] for chunk in stream: lines.append(chunk.get('stream')) - expected = '{0}{2}\n{1}'.format( - control_chars[0], control_chars[1], snippet - ) + expected = f'{control_chars[0]}{snippet}\n{control_chars[1]}' assert any(line == expected for line in lines) def test_build_gzip_encoding(self): diff --git a/tests/ssh/api_build_test.py b/tests/ssh/api_build_test.py index d060f465f2..3b542994fc 100644 --- a/tests/ssh/api_build_test.py +++ b/tests/ssh/api_build_test.py @@ -380,9 +380,7 @@ def test_build_stderr_data(self): lines = [] for chunk in stream: lines.append(chunk.get('stream')) - expected = '{0}{2}\n{1}'.format( - control_chars[0], control_chars[1], snippet - ) + expected = f'{control_chars[0]}{snippet}\n{control_chars[1]}' assert any(line == expected for line in lines) def test_build_gzip_encoding(self): diff --git a/tests/unit/api_test.py b/tests/unit/api_test.py index 7bc2ea8cda..0ca9bbb950 100644 --- a/tests/unit/api_test.py +++ b/tests/unit/api_test.py @@ -82,7 +82,7 @@ def fake_delete(self, url, *args, **kwargs): def fake_read_from_socket(self, response, stream, tty=False, demux=False): - return bytes() + return b'' url_base = f'{fake_api.prefix}/' From eeb9ea1937ba5e67087551bdabed858fab5e0ef3 Mon Sep 17 00:00:00 2001 From: Khushiyant Date: Thu, 4 Jan 2024 00:26:10 +0530 Subject: [PATCH 1266/1301] docs: change image.history() return type to list (#3202) Fixes #3076. Signed-off-by: Khushiyant --- docker/api/image.py | 2 +- docker/models/images.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/api/image.py b/docker/api/image.py index 5e1466ec3d..85109473bc 100644 --- a/docker/api/image.py +++ b/docker/api/image.py @@ -47,7 +47,7 @@ def history(self, image): image (str): The image to show history for Returns: - (str): The history of the image + (list): The history of the image Raises: :py:class:`docker.errors.APIError` diff --git a/docker/models/images.py b/docker/models/images.py index b4777d8da9..4f058d24d9 100644 --- a/docker/models/images.py +++ b/docker/models/images.py @@ -51,7 +51,7 @@ def history(self): Show the history of an image. Returns: - (str): The history of the image. + (list): The history of the image. Raises: :py:class:`docker.errors.APIError` From 694d9792e6c5f89b9fc6fd4d32a80216d5129ef7 Mon Sep 17 00:00:00 2001 From: Milas Bowman Date: Wed, 3 Jan 2024 14:01:42 -0500 Subject: [PATCH 1267/1301] lint: fix string formatting (#3211) Merged a linter upgrade along with an older PR, so this was immediately in violation Signed-off-by: Milas Bowman --- tests/unit/models_configs_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/models_configs_test.py b/tests/unit/models_configs_test.py index 6960397ff6..5d52daf76f 100644 --- a/tests/unit/models_configs_test.py +++ b/tests/unit/models_configs_test.py @@ -7,4 +7,4 @@ class CreateConfigsTest(unittest.TestCase): def test_create_config(self): client = make_fake_client() config = client.configs.create(name="super_config", data="config") - assert config.__repr__() == "".format(FAKE_CONFIG_NAME) + assert config.__repr__() == f"" From 249654d4d9c260c84350deeca3e77c7c76669663 Mon Sep 17 00:00:00 2001 From: Aarni Koskela Date: Thu, 21 Dec 2023 10:20:32 +0200 Subject: [PATCH 1268/1301] Drop `packaging` dependency Compare versions like Moby (api/types/versions/compare.go) Signed-off-by: Aarni Koskela --- docker/utils/utils.py | 24 ++++++++++++++++-------- setup.py | 1 - tests/unit/utils_test.py | 27 ++++++++++++++++++++++----- 3 files changed, 38 insertions(+), 14 deletions(-) diff --git a/docker/utils/utils.py b/docker/utils/utils.py index efe9f9a3cd..e1e064385f 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -6,7 +6,8 @@ import shlex import string from datetime import datetime, timezone -from packaging.version import Version +from functools import lru_cache +from itertools import zip_longest from .. import errors from ..constants import DEFAULT_HTTP_HOST @@ -43,6 +44,7 @@ def decode_json_header(header): return json.loads(data) +@lru_cache(maxsize=None) def compare_version(v1, v2): """Compare docker versions @@ -55,14 +57,20 @@ def compare_version(v1, v2): >>> compare_version(v2, v2) 0 """ - s1 = Version(v1) - s2 = Version(v2) - if s1 == s2: + if v1 == v2: return 0 - elif s1 > s2: - return -1 - else: - return 1 + # Split into `sys.version_info` like tuples. + s1 = tuple(int(p) for p in v1.split('.')) + s2 = tuple(int(p) for p in v2.split('.')) + # Compare each component, padding with 0 if necessary. + for c1, c2 in zip_longest(s1, s2, fillvalue=0): + if c1 == c2: + continue + elif c1 > c2: + return -1 + else: + return 1 + return 0 def version_lt(v1, v2): diff --git a/setup.py b/setup.py index b6a024f81a..3d33139240 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,6 @@ SOURCE_DIR = os.path.join(ROOT_DIR) requirements = [ - 'packaging >= 14.0', 'requests >= 2.26.0', 'urllib3 >= 1.26.0', ] diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index de79e3037d..c9434e11bb 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -10,12 +10,13 @@ from docker.api.client import APIClient from docker.constants import IS_WINDOWS_PLATFORM, DEFAULT_DOCKER_API_VERSION from docker.errors import DockerException -from docker.utils import (convert_filters, convert_volume_binds, - decode_json_header, kwargs_from_env, parse_bytes, - parse_devices, parse_env_file, parse_host, - parse_repository_tag, split_command, update_headers) +from docker.utils import ( + compare_version, convert_filters, convert_volume_binds, decode_json_header, + format_environment, kwargs_from_env, parse_bytes, parse_devices, + parse_env_file, parse_host, parse_repository_tag, split_command, + update_headers, version_gte, version_lt +) from docker.utils.ports import build_port_bindings, split_port -from docker.utils.utils import format_environment TEST_CERT_DIR = os.path.join( os.path.dirname(__file__), @@ -629,3 +630,19 @@ def test_format_env_no_value(self): 'BAR': '', } assert sorted(format_environment(env_dict)) == ['BAR=', 'FOO'] + + +def test_compare_versions(): + assert compare_version('1.0', '1.1') == 1 + assert compare_version('1.10', '1.1') == -1 + assert compare_version('1.10', '1.10') == 0 + assert compare_version('1.10.0', '1.10.1') == 1 + assert compare_version('1.9', '1.10') == 1 + assert compare_version('1.9.1', '1.10') == 1 + # Test comparison helpers + assert version_lt('1.0', '1.27') + assert version_gte('1.27', '1.20') + # Test zero-padding + assert compare_version('1', '1.0') == 0 + assert compare_version('1.10', '1.10.1') == 1 + assert compare_version('1.10.0', '1.10') == 0 From f128956034b2df1f8741e4c0246f79e1e2598146 Mon Sep 17 00:00:00 2001 From: Aarni Koskela Date: Thu, 21 Dec 2023 10:35:28 +0200 Subject: [PATCH 1269/1301] Use `build` instead of calling setup.py Signed-off-by: Aarni Koskela --- .github/workflows/release.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 721020ac33..6987fce0d4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -26,10 +26,10 @@ jobs: with: python-version: '3.x' - - name: Generate Pacakge + - name: Generate Package run: | - pip3 install setuptools wheel - python setup.py sdist bdist_wheel + pip3 install build + python -m build . env: SETUPTOOLS_SCM_PRETEND_VERSION_FOR_DOCKER: ${{ inputs.tag }} From ae45d477c44b880cdc7155f8829e0cdea84d0033 Mon Sep 17 00:00:00 2001 From: Aarni Koskela Date: Thu, 21 Dec 2023 10:37:23 +0200 Subject: [PATCH 1270/1301] Use `hatch` for packaging Signed-off-by: Aarni Koskela --- .github/workflows/release.yml | 2 + pyproject.toml | 64 +++++++++++++++++++++++++-- setup.cfg | 3 -- setup.py | 82 ----------------------------------- 4 files changed, 63 insertions(+), 88 deletions(-) delete mode 100644 setup.cfg delete mode 100644 setup.py diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6987fce0d4..953b59bf7a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -31,6 +31,8 @@ jobs: pip3 install build python -m build . env: + # This is also supported by Hatch; see + # https://github.com/ofek/hatch-vcs#version-source-environment-variables SETUPTOOLS_SCM_PRETEND_VERSION_FOR_DOCKER: ${{ inputs.tag }} - name: Publish to PyPI diff --git a/pyproject.toml b/pyproject.toml index a64e120eee..83d22b8019 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,8 +1,66 @@ [build-system] -requires = ["setuptools>=45", "setuptools_scm[toml]>=6.2"] +requires = ["hatchling", "hatch-vcs"] +build-backend = "hatchling.build" -[tool.setuptools_scm] -write_to = 'docker/_version.py' +[project] +name = "docker" +dynamic = ["version"] +description = "A Python library for the Docker Engine API." +readme = "README.md" +license = "Apache-2.0" +requires-python = ">=3.8" +maintainers = [ + { name = "Docker Inc.", email = "no-reply@docker.com" }, +] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Environment :: Other Environment", + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Topic :: Software Development", + "Topic :: Utilities", +] + +dependencies = [ + "requests >= 2.26.0", + "urllib3 >= 1.26.0", + "pywin32>=304; sys_platform == \"win32\"", +] + +[project.optional-dependencies] +ssh = [ + "paramiko>=2.4.3", +] +tls = [] # kept for backwards compatibility +websockets = [ + "websocket-client >= 1.3.0", +] + +[project.urls] +Changelog = "https://docker-py.readthedocs.io/en/stable/change-log.html" +Documentation = "https://docker-py.readthedocs.io" +Homepage = "https://github.com/docker/docker-py" +Source = "https://github.com/docker/docker-py" +Tracker = "https://github.com/docker/docker-py/issues" + +[tool.hatch.version] +source = "vcs" + +[tool.hatch.build.hooks.vcs] +version-file = "docker/_version.py" + +[tool.hatch.build.targets.sdist] +include = [ + "/docker", +] [tool.ruff] target-version = "py38" diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index a37e5521d5..0000000000 --- a/setup.cfg +++ /dev/null @@ -1,3 +0,0 @@ -[metadata] -description_file = README.rst -license = Apache License 2.0 diff --git a/setup.py b/setup.py deleted file mode 100644 index 3d33139240..0000000000 --- a/setup.py +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/env python - -import codecs -import os - -from setuptools import find_packages -from setuptools import setup - -ROOT_DIR = os.path.dirname(__file__) -SOURCE_DIR = os.path.join(ROOT_DIR) - -requirements = [ - 'requests >= 2.26.0', - 'urllib3 >= 1.26.0', -] - -extras_require = { - # win32 APIs if on Windows (required for npipe support) - ':sys_platform == "win32"': 'pywin32>=304', - - # This is now a no-op, as similarly the requests[security] extra is - # a no-op as of requests 2.26.0, this is always available/by default now - # see https://github.com/psf/requests/pull/5867 - 'tls': [], - - # Only required when connecting using the ssh:// protocol - 'ssh': ['paramiko>=2.4.3'], - - # Only required when using websockets - 'websockets': ['websocket-client >= 1.3.0'], -} - -with open('./test-requirements.txt') as test_reqs_txt: - test_requirements = list(test_reqs_txt) - - -long_description = '' -with codecs.open('./README.md', encoding='utf-8') as readme_md: - long_description = readme_md.read() - -setup( - name="docker", - use_scm_version={ - 'write_to': 'docker/_version.py' - }, - description="A Python library for the Docker Engine API.", - long_description=long_description, - long_description_content_type='text/markdown', - url='https://github.com/docker/docker-py', - project_urls={ - 'Documentation': 'https://docker-py.readthedocs.io', - 'Changelog': 'https://docker-py.readthedocs.io/en/stable/change-log.html', - 'Source': 'https://github.com/docker/docker-py', - 'Tracker': 'https://github.com/docker/docker-py/issues', - }, - packages=find_packages(exclude=["tests.*", "tests"]), - setup_requires=['setuptools_scm'], - install_requires=requirements, - tests_require=test_requirements, - extras_require=extras_require, - python_requires='>=3.8', - zip_safe=False, - test_suite='tests', - classifiers=[ - 'Development Status :: 5 - Production/Stable', - 'Environment :: Other Environment', - 'Intended Audience :: Developers', - 'Operating System :: OS Independent', - 'Programming Language :: Python', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - 'Programming Language :: Python :: 3.10', - 'Programming Language :: Python :: 3.11', - 'Programming Language :: Python :: 3.12', - 'Topic :: Software Development', - 'Topic :: Utilities', - 'License :: OSI Approved :: Apache Software License', - ], - maintainer='Docker, Inc.', - maintainer_email='no-reply@docker.com', -) From 047df6b0d31250d5d15ce25d1252ea7a04c7fcd4 Mon Sep 17 00:00:00 2001 From: Aarni Koskela Date: Thu, 21 Dec 2023 10:51:05 +0200 Subject: [PATCH 1271/1301] Build wheel in CI, upload artifact for perusal Signed-off-by: Aarni Koskela --- .github/workflows/ci.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 628c53504a..9eb450a6ae 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,6 +18,19 @@ jobs: - name: Run ruff run: ruff docker tests + build: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v4 + with: + python-version: '3.x' + - run: pip3 install build && python -m build . + - uses: actions/upload-artifact@v4 + with: + name: dist + path: dist + unit-tests: runs-on: ubuntu-latest strategy: From d50cc429c2adc61c0af5275972bd740f846d55fe Mon Sep 17 00:00:00 2001 From: Aarni Koskela Date: Wed, 3 Jan 2024 21:23:04 +0200 Subject: [PATCH 1272/1301] Enable Ruff I (import sort), autofix Signed-off-by: Aarni Koskela --- docker/__init__.py | 3 +- docker/api/build.py | 6 +--- docker/api/client.py | 22 +++++++++++---- docker/api/container.py | 15 +++++----- docker/api/exec_api.py | 3 +- docker/api/network.py | 5 ++-- docker/api/secret.py | 3 +- docker/api/swarm.py | 7 ++--- docker/api/volume.py | 3 +- docker/auth.py | 3 +- docker/client.py | 2 +- docker/constants.py | 1 + docker/context/__init__.py | 2 +- docker/context/api.py | 10 ++++--- docker/context/config.py | 7 ++--- docker/context/context.py | 15 ++++++---- docker/credentials/__init__.py | 4 +-- docker/credentials/store.py | 3 +- docker/models/configs.py | 2 +- docker/models/containers.py | 11 +++++--- docker/models/networks.py | 2 +- docker/models/nodes.py | 2 +- docker/models/secrets.py | 2 +- docker/models/services.py | 8 ++++-- docker/models/swarm.py | 1 + docker/models/volumes.py | 2 +- docker/transport/__init__.py | 1 + docker/transport/npipeconn.py | 7 +++-- docker/transport/npipesocket.py | 8 +++--- docker/transport/sshconn.py | 15 +++++----- docker/transport/unixconn.py | 8 +++--- docker/types/__init__.py | 27 ++++++++++++------ docker/types/containers.py | 14 ++++++++-- docker/types/services.py | 8 ++++-- docker/utils/__init__.py | 31 +++++++++++++++------ docker/utils/build.py | 3 +- docker/utils/json_stream.py | 1 - docker/utils/utils.py | 13 +++++---- docker/version.py | 2 +- docs/conf.py | 1 + pyproject.toml | 1 + setup.py | 3 +- tests/helpers.py | 3 +- tests/integration/api_build_test.py | 6 ++-- tests/integration/api_config_test.py | 3 +- tests/integration/api_container_test.py | 18 ++++++------ tests/integration/api_exec_test.py | 15 +++++----- tests/integration/api_healthcheck_test.py | 2 +- tests/integration/api_image_test.py | 7 ++--- tests/integration/api_network_test.py | 5 ++-- tests/integration/api_plugin_test.py | 5 ++-- tests/integration/api_secret_test.py | 3 +- tests/integration/api_service_test.py | 9 +++--- tests/integration/api_swarm_test.py | 4 ++- tests/integration/api_volume_test.py | 3 +- tests/integration/base.py | 3 +- tests/integration/client_test.py | 3 +- tests/integration/conftest.py | 3 +- tests/integration/context_api_test.py | 3 ++ tests/integration/credentials/store_test.py | 7 +++-- tests/integration/credentials/utils_test.py | 2 +- tests/integration/errors_test.py | 6 ++-- tests/integration/models_containers_test.py | 7 ++--- tests/integration/models_images_test.py | 5 ++-- tests/integration/models_networks_test.py | 3 +- tests/integration/models_resources_test.py | 3 +- tests/integration/models_services_test.py | 7 +++-- tests/integration/models_swarm_test.py | 3 +- tests/integration/models_volumes_test.py | 3 +- tests/integration/regression_test.py | 5 ++-- tests/ssh/api_build_test.py | 6 ++-- tests/ssh/base.py | 3 +- tests/ssh/connect_test.py | 4 ++- tests/unit/api_container_test.py | 15 ++++++---- tests/unit/api_exec_test.py | 5 +++- tests/unit/api_image_test.py | 15 ++++++---- tests/unit/api_network_test.py | 5 ++-- tests/unit/api_test.py | 10 +++---- tests/unit/api_volume_test.py | 2 +- tests/unit/auth_test.py | 5 ++-- tests/unit/client_test.py | 11 +++++--- tests/unit/context_test.py | 10 +++---- tests/unit/dockertypes_test.py | 13 +++++++-- tests/unit/errors_test.py | 11 ++++++-- tests/unit/fake_api_client.py | 3 +- tests/unit/models_configs_test.py | 3 +- tests/unit/models_containers_test.py | 6 ++-- tests/unit/models_networks_test.py | 2 +- tests/unit/models_secrets_test.py | 2 +- tests/unit/models_services_test.py | 1 + tests/unit/sshadapter_test.py | 1 + tests/unit/swarm_test.py | 4 +-- tests/unit/utils_build_test.py | 2 +- tests/unit/utils_config_test.py | 8 +++--- tests/unit/utils_json_stream_test.py | 2 +- tests/unit/utils_test.py | 22 +++++++++++---- 96 files changed, 364 insertions(+), 240 deletions(-) diff --git a/docker/__init__.py b/docker/__init__.py index c1c518c56d..fb7a5e921a 100644 --- a/docker/__init__.py +++ b/docker/__init__.py @@ -1,7 +1,6 @@ from .api import APIClient from .client import DockerClient, from_env -from .context import Context -from .context import ContextAPI +from .context import Context, ContextAPI from .tls import TLSConfig from .version import __version__ diff --git a/docker/api/build.py b/docker/api/build.py index abd5ab52a8..47216a58fd 100644 --- a/docker/api/build.py +++ b/docker/api/build.py @@ -3,11 +3,7 @@ import os import random -from .. import auth -from .. import constants -from .. import errors -from .. import utils - +from .. import auth, constants, errors, utils log = logging.getLogger(__name__) diff --git a/docker/api/client.py b/docker/api/client.py index 499a7c785e..394ceb1f56 100644 --- a/docker/api/client.py +++ b/docker/api/client.py @@ -8,12 +8,22 @@ import requests.exceptions from .. import auth -from ..constants import (DEFAULT_NUM_POOLS, DEFAULT_NUM_POOLS_SSH, - DEFAULT_MAX_POOL_SIZE, DEFAULT_TIMEOUT_SECONDS, - DEFAULT_USER_AGENT, IS_WINDOWS_PLATFORM, - MINIMUM_DOCKER_API_VERSION, STREAM_HEADER_SIZE_BYTES) -from ..errors import (DockerException, InvalidVersion, TLSParameterError, - create_api_error_from_http_exception) +from ..constants import ( + DEFAULT_MAX_POOL_SIZE, + DEFAULT_NUM_POOLS, + DEFAULT_NUM_POOLS_SSH, + DEFAULT_TIMEOUT_SECONDS, + DEFAULT_USER_AGENT, + IS_WINDOWS_PLATFORM, + MINIMUM_DOCKER_API_VERSION, + STREAM_HEADER_SIZE_BYTES, +) +from ..errors import ( + DockerException, + InvalidVersion, + TLSParameterError, + create_api_error_from_http_exception, +) from ..tls import TLSConfig from ..transport import UnixHTTPAdapter from ..utils import check_resource, config, update_headers, utils diff --git a/docker/api/container.py b/docker/api/container.py index 5a267d13f1..1f153eeb79 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -1,13 +1,14 @@ from datetime import datetime -from .. import errors -from .. import utils +from .. import errors, utils from ..constants import DEFAULT_DATA_CHUNK_SIZE -from ..types import CancellableStream -from ..types import ContainerConfig -from ..types import EndpointConfig -from ..types import HostConfig -from ..types import NetworkingConfig +from ..types import ( + CancellableStream, + ContainerConfig, + EndpointConfig, + HostConfig, + NetworkingConfig, +) class ContainerApiMixin: diff --git a/docker/api/exec_api.py b/docker/api/exec_api.py index 63df9e6c6a..d8fc50dd3d 100644 --- a/docker/api/exec_api.py +++ b/docker/api/exec_api.py @@ -1,5 +1,4 @@ -from .. import errors -from .. import utils +from .. import errors, utils from ..types import CancellableStream diff --git a/docker/api/network.py b/docker/api/network.py index dd4e3761ac..2b1925710e 100644 --- a/docker/api/network.py +++ b/docker/api/network.py @@ -1,7 +1,6 @@ -from ..errors import InvalidVersion -from ..utils import check_resource, minimum_version -from ..utils import version_lt from .. import utils +from ..errors import InvalidVersion +from ..utils import check_resource, minimum_version, version_lt class NetworkApiMixin: diff --git a/docker/api/secret.py b/docker/api/secret.py index cd440b95f8..db1701bdc0 100644 --- a/docker/api/secret.py +++ b/docker/api/secret.py @@ -1,7 +1,6 @@ import base64 -from .. import errors -from .. import utils +from .. import errors, utils class SecretApiMixin: diff --git a/docker/api/swarm.py b/docker/api/swarm.py index d09dd087b7..d60d18b619 100644 --- a/docker/api/swarm.py +++ b/docker/api/swarm.py @@ -1,9 +1,8 @@ -import logging import http.client as http_client +import logging + +from .. import errors, types, utils from ..constants import DEFAULT_SWARM_ADDR_POOL, DEFAULT_SWARM_SUBNET_SIZE -from .. import errors -from .. import types -from .. import utils log = logging.getLogger(__name__) diff --git a/docker/api/volume.py b/docker/api/volume.py index 98b42a124e..c6c036fad0 100644 --- a/docker/api/volume.py +++ b/docker/api/volume.py @@ -1,5 +1,4 @@ -from .. import errors -from .. import utils +from .. import errors, utils class VolumeApiMixin: diff --git a/docker/auth.py b/docker/auth.py index 7a301ba407..96a6e3a656 100644 --- a/docker/auth.py +++ b/docker/auth.py @@ -2,8 +2,7 @@ import json import logging -from . import credentials -from . import errors +from . import credentials, errors from .utils import config INDEX_NAME = 'docker.io' diff --git a/docker/client.py b/docker/client.py index 2910c12596..9012d24c9c 100644 --- a/docker/client.py +++ b/docker/client.py @@ -1,5 +1,5 @@ from .api.client import APIClient -from .constants import (DEFAULT_TIMEOUT_SECONDS, DEFAULT_MAX_POOL_SIZE) +from .constants import DEFAULT_MAX_POOL_SIZE, DEFAULT_TIMEOUT_SECONDS from .models.configs import ConfigCollection from .models.containers import ContainerCollection from .models.images import ImageCollection diff --git a/docker/constants.py b/docker/constants.py index 71e543e530..433e6c4e2f 100644 --- a/docker/constants.py +++ b/docker/constants.py @@ -1,4 +1,5 @@ import sys + from .version import __version__ DEFAULT_DOCKER_API_VERSION = '1.43' diff --git a/docker/context/__init__.py b/docker/context/__init__.py index dbf172fdac..46d462b0cf 100644 --- a/docker/context/__init__.py +++ b/docker/context/__init__.py @@ -1,2 +1,2 @@ -from .context import Context from .api import ContextAPI +from .context import Context diff --git a/docker/context/api.py b/docker/context/api.py index 493f470e5d..ae5d67bb25 100644 --- a/docker/context/api.py +++ b/docker/context/api.py @@ -2,11 +2,13 @@ import os from docker import errors -from docker.context.config import get_meta_dir -from docker.context.config import METAFILE -from docker.context.config import get_current_context_name -from docker.context.config import write_context_name_to_docker_config from docker.context import Context +from docker.context.config import ( + METAFILE, + get_current_context_name, + get_meta_dir, + write_context_name_to_docker_config, +) class ContextAPI: diff --git a/docker/context/config.py b/docker/context/config.py index 8c3fe25007..5a6373aa4e 100644 --- a/docker/context/config.py +++ b/docker/context/config.py @@ -1,10 +1,9 @@ -import os -import json import hashlib +import json +import os from docker import utils -from docker.constants import IS_WINDOWS_PLATFORM -from docker.constants import DEFAULT_UNIX_SOCKET +from docker.constants import DEFAULT_UNIX_SOCKET, IS_WINDOWS_PLATFORM from docker.utils.config import find_config_file METAFILE = "meta.json" diff --git a/docker/context/context.py b/docker/context/context.py index 4faf8e7017..317bcf61df 100644 --- a/docker/context/context.py +++ b/docker/context/context.py @@ -1,12 +1,15 @@ -import os import json +import os from shutil import copyfile, rmtree -from docker.tls import TLSConfig + +from docker.context.config import ( + get_context_host, + get_meta_dir, + get_meta_file, + get_tls_dir, +) from docker.errors import ContextException -from docker.context.config import get_meta_dir -from docker.context.config import get_meta_file -from docker.context.config import get_tls_dir -from docker.context.config import get_context_host +from docker.tls import TLSConfig class Context: diff --git a/docker/credentials/__init__.py b/docker/credentials/__init__.py index a1247700d3..80d19e7986 100644 --- a/docker/credentials/__init__.py +++ b/docker/credentials/__init__.py @@ -1,8 +1,8 @@ -from .store import Store -from .errors import StoreError, CredentialsNotFound from .constants import ( DEFAULT_LINUX_STORE, DEFAULT_OSX_STORE, DEFAULT_WIN32_STORE, PROGRAM_PREFIX, ) +from .errors import CredentialsNotFound, StoreError +from .store import Store diff --git a/docker/credentials/store.py b/docker/credentials/store.py index 4e63a5ba60..00d693a4be 100644 --- a/docker/credentials/store.py +++ b/docker/credentials/store.py @@ -4,8 +4,7 @@ import subprocess import warnings -from . import constants -from . import errors +from . import constants, errors from .utils import create_environment_dict diff --git a/docker/models/configs.py b/docker/models/configs.py index 5ef1377844..4eba87f4e3 100644 --- a/docker/models/configs.py +++ b/docker/models/configs.py @@ -1,5 +1,5 @@ from ..api import APIClient -from .resource import Model, Collection +from .resource import Collection, Model class Config(Model): diff --git a/docker/models/containers.py b/docker/models/containers.py index 32676bb852..35d8b31418 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -2,16 +2,19 @@ import ntpath from collections import namedtuple -from .images import Image -from .resource import Collection, Model from ..api import APIClient from ..constants import DEFAULT_DATA_CHUNK_SIZE from ..errors import ( - ContainerError, DockerException, ImageNotFound, - NotFound, create_unexpected_kwargs_error + ContainerError, + DockerException, + ImageNotFound, + NotFound, + create_unexpected_kwargs_error, ) from ..types import HostConfig, NetworkingConfig from ..utils import version_gte +from .images import Image +from .resource import Collection, Model class Container(Model): diff --git a/docker/models/networks.py b/docker/models/networks.py index f502879070..9b3ed7829c 100644 --- a/docker/models/networks.py +++ b/docker/models/networks.py @@ -1,7 +1,7 @@ from ..api import APIClient from ..utils import version_gte from .containers import Container -from .resource import Model, Collection +from .resource import Collection, Model class Network(Model): diff --git a/docker/models/nodes.py b/docker/models/nodes.py index 8dd9350c02..2fa480c544 100644 --- a/docker/models/nodes.py +++ b/docker/models/nodes.py @@ -1,4 +1,4 @@ -from .resource import Model, Collection +from .resource import Collection, Model class Node(Model): diff --git a/docker/models/secrets.py b/docker/models/secrets.py index da01d44c8f..38c48dc7eb 100644 --- a/docker/models/secrets.py +++ b/docker/models/secrets.py @@ -1,5 +1,5 @@ from ..api import APIClient -from .resource import Model, Collection +from .resource import Collection, Model class Secret(Model): diff --git a/docker/models/services.py b/docker/models/services.py index 70037041a3..09502633e5 100644 --- a/docker/models/services.py +++ b/docker/models/services.py @@ -1,7 +1,9 @@ import copy -from docker.errors import create_unexpected_kwargs_error, InvalidArgument -from docker.types import TaskTemplate, ContainerSpec, Placement, ServiceMode -from .resource import Model, Collection + +from docker.errors import InvalidArgument, create_unexpected_kwargs_error +from docker.types import ContainerSpec, Placement, ServiceMode, TaskTemplate + +from .resource import Collection, Model class Service(Model): diff --git a/docker/models/swarm.py b/docker/models/swarm.py index 1e39f3fd2f..271cc5dcb1 100644 --- a/docker/models/swarm.py +++ b/docker/models/swarm.py @@ -1,5 +1,6 @@ from docker.api import APIClient from docker.errors import APIError + from .resource import Model diff --git a/docker/models/volumes.py b/docker/models/volumes.py index 3c2e837805..12c9f14b27 100644 --- a/docker/models/volumes.py +++ b/docker/models/volumes.py @@ -1,5 +1,5 @@ from ..api import APIClient -from .resource import Model, Collection +from .resource import Collection, Model class Volume(Model): diff --git a/docker/transport/__init__.py b/docker/transport/__init__.py index 07bc7fd582..8c68b1f6e2 100644 --- a/docker/transport/__init__.py +++ b/docker/transport/__init__.py @@ -1,4 +1,5 @@ from .unixconn import UnixHTTPAdapter + try: from .npipeconn import NpipeHTTPAdapter from .npipesocket import NpipeSocket diff --git a/docker/transport/npipeconn.py b/docker/transport/npipeconn.py index d335d8718f..fe740a5f82 100644 --- a/docker/transport/npipeconn.py +++ b/docker/transport/npipeconn.py @@ -1,13 +1,14 @@ import queue + import requests.adapters +import urllib3 +import urllib3.connection from docker.transport.basehttpadapter import BaseHTTPAdapter + from .. import constants from .npipesocket import NpipeSocket -import urllib3 -import urllib3.connection - RecentlyUsedContainer = urllib3._collections.RecentlyUsedContainer diff --git a/docker/transport/npipesocket.py b/docker/transport/npipesocket.py index 9cbe40cc7f..d91938e766 100644 --- a/docker/transport/npipesocket.py +++ b/docker/transport/npipesocket.py @@ -1,12 +1,12 @@ import functools -import time import io +import time -import win32file -import win32pipe import pywintypes -import win32event import win32api +import win32event +import win32file +import win32pipe cERROR_PIPE_BUSY = 0xe7 cSECURITY_SQOS_PRESENT = 0x100000 diff --git a/docker/transport/sshconn.py b/docker/transport/sshconn.py index 6e1d0ee723..91671e9200 100644 --- a/docker/transport/sshconn.py +++ b/docker/transport/sshconn.py @@ -1,19 +1,20 @@ -import paramiko -import queue -import urllib.parse -import requests.adapters import logging import os +import queue import signal import socket import subprocess +import urllib.parse -from docker.transport.basehttpadapter import BaseHTTPAdapter -from .. import constants - +import paramiko +import requests.adapters import urllib3 import urllib3.connection +from docker.transport.basehttpadapter import BaseHTTPAdapter + +from .. import constants + RecentlyUsedContainer = urllib3._collections.RecentlyUsedContainer diff --git a/docker/transport/unixconn.py b/docker/transport/unixconn.py index 09d373dd6d..f88d29ebfb 100644 --- a/docker/transport/unixconn.py +++ b/docker/transport/unixconn.py @@ -1,12 +1,12 @@ -import requests.adapters import socket -from docker.transport.basehttpadapter import BaseHTTPAdapter -from .. import constants - +import requests.adapters import urllib3 import urllib3.connection +from docker.transport.basehttpadapter import BaseHTTPAdapter + +from .. import constants RecentlyUsedContainer = urllib3._collections.RecentlyUsedContainer diff --git a/docker/types/__init__.py b/docker/types/__init__.py index 89f2238934..fbe247210b 100644 --- a/docker/types/__init__.py +++ b/docker/types/__init__.py @@ -1,13 +1,24 @@ -from .containers import ( - ContainerConfig, HostConfig, LogConfig, Ulimit, DeviceRequest -) +from .containers import ContainerConfig, DeviceRequest, HostConfig, LogConfig, Ulimit from .daemon import CancellableStream from .healthcheck import Healthcheck from .networks import EndpointConfig, IPAMConfig, IPAMPool, NetworkingConfig from .services import ( - ConfigReference, ContainerSpec, DNSConfig, DriverConfig, EndpointSpec, - Mount, Placement, PlacementPreference, Privileges, Resources, - RestartPolicy, RollbackConfig, SecretReference, ServiceMode, TaskTemplate, - UpdateConfig, NetworkAttachmentConfig + ConfigReference, + ContainerSpec, + DNSConfig, + DriverConfig, + EndpointSpec, + Mount, + NetworkAttachmentConfig, + Placement, + PlacementPreference, + Privileges, + Resources, + RestartPolicy, + RollbackConfig, + SecretReference, + ServiceMode, + TaskTemplate, + UpdateConfig, ) -from .swarm import SwarmSpec, SwarmExternalCA +from .swarm import SwarmExternalCA, SwarmSpec diff --git a/docker/types/containers.py b/docker/types/containers.py index a28061383d..598188a25e 100644 --- a/docker/types/containers.py +++ b/docker/types/containers.py @@ -1,8 +1,16 @@ from .. import errors from ..utils.utils import ( - convert_port_bindings, convert_tmpfs_mounts, convert_volume_binds, - format_environment, format_extra_hosts, normalize_links, parse_bytes, - parse_devices, split_command, version_gte, version_lt, + convert_port_bindings, + convert_tmpfs_mounts, + convert_volume_binds, + format_environment, + format_extra_hosts, + normalize_links, + parse_bytes, + parse_devices, + split_command, + version_gte, + version_lt, ) from .base import DictType from .healthcheck import Healthcheck diff --git a/docker/types/services.py b/docker/types/services.py index 0b07c350ee..821115411c 100644 --- a/docker/types/services.py +++ b/docker/types/services.py @@ -1,8 +1,12 @@ from .. import errors from ..constants import IS_WINDOWS_PLATFORM from ..utils import ( - check_resource, format_environment, format_extra_hosts, parse_bytes, - split_command, convert_service_networks, + check_resource, + convert_service_networks, + format_environment, + format_extra_hosts, + parse_bytes, + split_command, ) diff --git a/docker/utils/__init__.py b/docker/utils/__init__.py index b4bef7d47c..c086a9f073 100644 --- a/docker/utils/__init__.py +++ b/docker/utils/__init__.py @@ -1,13 +1,28 @@ -from .build import match_tag, create_archive, exclude_paths, mkbuildcontext, tar +from .build import create_archive, exclude_paths, match_tag, mkbuildcontext, tar from .decorators import check_resource, minimum_version, update_headers from .utils import ( - compare_version, convert_port_bindings, convert_volume_binds, - parse_repository_tag, parse_host, - kwargs_from_env, convert_filters, datetime_to_timestamp, - create_host_config, parse_bytes, parse_env_file, version_lt, - version_gte, decode_json_header, split_command, create_ipam_config, - create_ipam_pool, parse_devices, normalize_links, convert_service_networks, - format_environment, format_extra_hosts + compare_version, + convert_filters, + convert_port_bindings, + convert_service_networks, + convert_volume_binds, + create_host_config, + create_ipam_config, + create_ipam_pool, + datetime_to_timestamp, + decode_json_header, + format_environment, + format_extra_hosts, + kwargs_from_env, + normalize_links, + parse_bytes, + parse_devices, + parse_env_file, + parse_host, + parse_repository_tag, + split_command, + version_gte, + version_lt, ) diff --git a/docker/utils/build.py b/docker/utils/build.py index 86a4423f0b..b841391044 100644 --- a/docker/utils/build.py +++ b/docker/utils/build.py @@ -4,9 +4,8 @@ import tarfile import tempfile -from .fnmatch import fnmatch from ..constants import IS_WINDOWS_PLATFORM - +from .fnmatch import fnmatch _SEP = re.compile('/|\\\\') if IS_WINDOWS_PLATFORM else re.compile('/') _TAG = re.compile( diff --git a/docker/utils/json_stream.py b/docker/utils/json_stream.py index 266193e567..41d25920ce 100644 --- a/docker/utils/json_stream.py +++ b/docker/utils/json_stream.py @@ -3,7 +3,6 @@ from ..errors import StreamParseError - json_decoder = json.JSONDecoder() diff --git a/docker/utils/utils.py b/docker/utils/utils.py index e1e064385f..f36a3afb89 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -8,16 +8,17 @@ from datetime import datetime, timezone from functools import lru_cache from itertools import zip_longest +from urllib.parse import urlparse, urlunparse from .. import errors -from ..constants import DEFAULT_HTTP_HOST -from ..constants import DEFAULT_UNIX_SOCKET -from ..constants import DEFAULT_NPIPE -from ..constants import BYTE_UNITS +from ..constants import ( + BYTE_UNITS, + DEFAULT_HTTP_HOST, + DEFAULT_NPIPE, + DEFAULT_UNIX_SOCKET, +) from ..tls import TLSConfig -from urllib.parse import urlparse, urlunparse - URLComponents = collections.namedtuple( 'URLComponents', 'scheme netloc url params query fragment', diff --git a/docker/version.py b/docker/version.py index dca45bf047..72b12b84df 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,7 +1,7 @@ try: from ._version import __version__ except ImportError: - from importlib.metadata import version, PackageNotFoundError + from importlib.metadata import PackageNotFoundError, version try: __version__ = version('docker') except PackageNotFoundError: diff --git a/docs/conf.py b/docs/conf.py index a529f8be82..02694d3cdf 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -19,6 +19,7 @@ import os import sys from importlib.metadata import version + sys.path.insert(0, os.path.abspath('..')) diff --git a/pyproject.toml b/pyproject.toml index a64e120eee..96fb272288 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,6 +10,7 @@ extend-select = [ "B", "C", "F", + "I", "UP", "W", ] diff --git a/setup.py b/setup.py index 3d33139240..b939ec58d4 100644 --- a/setup.py +++ b/setup.py @@ -3,8 +3,7 @@ import codecs import os -from setuptools import find_packages -from setuptools import setup +from setuptools import find_packages, setup ROOT_DIR = os.path.dirname(__file__) SOURCE_DIR = os.path.join(ROOT_DIR) diff --git a/tests/helpers.py b/tests/helpers.py index 748ee70a74..3d60a3faf9 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -8,10 +8,11 @@ import tempfile import time -import docker import paramiko import pytest +import docker + def make_tree(dirs, files): base = tempfile.mkdtemp() diff --git a/tests/integration/api_build_test.py b/tests/integration/api_build_test.py index 540ef2b0fb..62e93a7384 100644 --- a/tests/integration/api_build_test.py +++ b/tests/integration/api_build_test.py @@ -3,13 +3,13 @@ import shutil import tempfile +import pytest + from docker import errors from docker.utils.proxy import ProxyConfig -import pytest - -from .base import BaseAPIIntegrationTest, TEST_IMG from ..helpers import random_name, requires_api_version, requires_experimental +from .base import TEST_IMG, BaseAPIIntegrationTest class BuildTest(BaseAPIIntegrationTest): diff --git a/tests/integration/api_config_test.py b/tests/integration/api_config_test.py index 982ec468a6..4261599d84 100644 --- a/tests/integration/api_config_test.py +++ b/tests/integration/api_config_test.py @@ -1,6 +1,7 @@ -import docker import pytest +import docker + from ..helpers import force_leave_swarm, requires_api_version from .base import BaseAPIIntegrationTest diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index aa27fbfd7a..0215e14c25 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -9,15 +9,17 @@ import requests import docker -from .. import helpers -from ..helpers import assert_cat_socket_detached_with_keys -from ..helpers import ctrl_with -from ..helpers import requires_api_version, skip_if_desktop -from .base import BaseAPIIntegrationTest -from .base import TEST_IMG from docker.constants import IS_WINDOWS_PLATFORM -from docker.utils.socket import next_frame_header -from docker.utils.socket import read_exactly +from docker.utils.socket import next_frame_header, read_exactly + +from .. import helpers +from ..helpers import ( + assert_cat_socket_detached_with_keys, + ctrl_with, + requires_api_version, + skip_if_desktop, +) +from .base import TEST_IMG, BaseAPIIntegrationTest class ListContainersTest(BaseAPIIntegrationTest): diff --git a/tests/integration/api_exec_test.py b/tests/integration/api_exec_test.py index 4d7748f5ee..5b829e2875 100644 --- a/tests/integration/api_exec_test.py +++ b/tests/integration/api_exec_test.py @@ -1,11 +1,12 @@ -from ..helpers import assert_cat_socket_detached_with_keys -from ..helpers import ctrl_with -from ..helpers import requires_api_version -from .base import BaseAPIIntegrationTest -from .base import TEST_IMG from docker.utils.proxy import ProxyConfig -from docker.utils.socket import next_frame_header -from docker.utils.socket import read_exactly +from docker.utils.socket import next_frame_header, read_exactly + +from ..helpers import ( + assert_cat_socket_detached_with_keys, + ctrl_with, + requires_api_version, +) +from .base import TEST_IMG, BaseAPIIntegrationTest class ExecTest(BaseAPIIntegrationTest): diff --git a/tests/integration/api_healthcheck_test.py b/tests/integration/api_healthcheck_test.py index 9ecdcd86a6..f00d804b44 100644 --- a/tests/integration/api_healthcheck_test.py +++ b/tests/integration/api_healthcheck_test.py @@ -1,5 +1,5 @@ -from .base import BaseAPIIntegrationTest, TEST_IMG from .. import helpers +from .base import TEST_IMG, BaseAPIIntegrationTest SECOND = 1000000000 diff --git a/tests/integration/api_image_test.py b/tests/integration/api_image_test.py index 5c37219581..d3915c9b51 100644 --- a/tests/integration/api_image_test.py +++ b/tests/integration/api_image_test.py @@ -2,19 +2,18 @@ import json import shutil import socket +import socketserver import tarfile import tempfile import threading - -import pytest from http.server import SimpleHTTPRequestHandler -import socketserver +import pytest import docker from ..helpers import requires_api_version, requires_experimental -from .base import BaseAPIIntegrationTest, TEST_IMG +from .base import TEST_IMG, BaseAPIIntegrationTest class ListImagesTest(BaseAPIIntegrationTest): diff --git a/tests/integration/api_network_test.py b/tests/integration/api_network_test.py index 6689044b68..ce2e8ea4c3 100644 --- a/tests/integration/api_network_test.py +++ b/tests/integration/api_network_test.py @@ -1,9 +1,10 @@ +import pytest + import docker from docker.types import IPAMConfig, IPAMPool -import pytest from ..helpers import random_name, requires_api_version -from .base import BaseAPIIntegrationTest, TEST_IMG +from .base import TEST_IMG, BaseAPIIntegrationTest class TestNetworks(BaseAPIIntegrationTest): diff --git a/tests/integration/api_plugin_test.py b/tests/integration/api_plugin_test.py index 3f1633900d..168c81b231 100644 --- a/tests/integration/api_plugin_test.py +++ b/tests/integration/api_plugin_test.py @@ -1,10 +1,11 @@ import os -import docker import pytest -from .base import BaseAPIIntegrationTest +import docker + from ..helpers import requires_api_version +from .base import BaseAPIIntegrationTest SSHFS = 'vieux/sshfs:latest' diff --git a/tests/integration/api_secret_test.py b/tests/integration/api_secret_test.py index fd98543414..588aaeb99d 100644 --- a/tests/integration/api_secret_test.py +++ b/tests/integration/api_secret_test.py @@ -1,6 +1,7 @@ -import docker import pytest +import docker + from ..helpers import force_leave_swarm, requires_api_version from .base import BaseAPIIntegrationTest diff --git a/tests/integration/api_service_test.py b/tests/integration/api_service_test.py index dec3fa0071..7a7ae4ea41 100644 --- a/tests/integration/api_service_test.py +++ b/tests/integration/api_service_test.py @@ -1,13 +1,12 @@ import random import time -import docker import pytest -from ..helpers import ( - force_leave_swarm, requires_api_version, requires_experimental -) -from .base import BaseAPIIntegrationTest, TEST_IMG +import docker + +from ..helpers import force_leave_swarm, requires_api_version, requires_experimental +from .base import TEST_IMG, BaseAPIIntegrationTest class ServiceTest(BaseAPIIntegrationTest): diff --git a/tests/integration/api_swarm_test.py b/tests/integration/api_swarm_test.py index b4125d24d9..00477e1036 100644 --- a/tests/integration/api_swarm_test.py +++ b/tests/integration/api_swarm_test.py @@ -1,7 +1,9 @@ import copy -import docker + import pytest +import docker + from ..helpers import force_leave_swarm, requires_api_version from .base import BaseAPIIntegrationTest diff --git a/tests/integration/api_volume_test.py b/tests/integration/api_volume_test.py index 2085e83113..ecd19da2d5 100644 --- a/tests/integration/api_volume_test.py +++ b/tests/integration/api_volume_test.py @@ -1,6 +1,7 @@ -import docker import pytest +import docker + from ..helpers import requires_api_version from .base import BaseAPIIntegrationTest diff --git a/tests/integration/base.py b/tests/integration/base.py index e4073757ee..51ee05daa5 100644 --- a/tests/integration/base.py +++ b/tests/integration/base.py @@ -3,9 +3,10 @@ import unittest import docker -from .. import helpers from docker.utils import kwargs_from_env +from .. import helpers + TEST_IMG = 'alpine:3.10' TEST_API_VERSION = os.environ.get('DOCKER_TEST_API_VERSION') diff --git a/tests/integration/client_test.py b/tests/integration/client_test.py index 7df172c885..1d1be077e0 100644 --- a/tests/integration/client_test.py +++ b/tests/integration/client_test.py @@ -1,10 +1,9 @@ import threading import unittest +from datetime import datetime, timedelta import docker -from datetime import datetime, timedelta - from ..helpers import requires_api_version from .base import TEST_API_VERSION diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index ae94595585..443c5b7950 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -1,9 +1,10 @@ import sys import warnings +import pytest + import docker.errors from docker.utils import kwargs_from_env -import pytest from .base import TEST_IMG diff --git a/tests/integration/context_api_test.py b/tests/integration/context_api_test.py index 1a13f2817e..2131ebe749 100644 --- a/tests/integration/context_api_test.py +++ b/tests/integration/context_api_test.py @@ -1,9 +1,12 @@ import os import tempfile + import pytest + from docker import errors from docker.context import ContextAPI from docker.tls import TLSConfig + from .base import BaseAPIIntegrationTest diff --git a/tests/integration/credentials/store_test.py b/tests/integration/credentials/store_test.py index 82ea84741d..e1eba33a18 100644 --- a/tests/integration/credentials/store_test.py +++ b/tests/integration/credentials/store_test.py @@ -6,8 +6,11 @@ import pytest from docker.credentials import ( - CredentialsNotFound, Store, StoreError, DEFAULT_LINUX_STORE, - DEFAULT_OSX_STORE + DEFAULT_LINUX_STORE, + DEFAULT_OSX_STORE, + CredentialsNotFound, + Store, + StoreError, ) diff --git a/tests/integration/credentials/utils_test.py b/tests/integration/credentials/utils_test.py index 4644039793..75bdea1009 100644 --- a/tests/integration/credentials/utils_test.py +++ b/tests/integration/credentials/utils_test.py @@ -1,7 +1,7 @@ import os +from unittest import mock from docker.credentials.utils import create_environment_dict -from unittest import mock @mock.patch.dict(os.environ) diff --git a/tests/integration/errors_test.py b/tests/integration/errors_test.py index e2fce48b0f..438caacbc4 100644 --- a/tests/integration/errors_test.py +++ b/tests/integration/errors_test.py @@ -1,7 +1,9 @@ -from docker.errors import APIError -from .base import BaseAPIIntegrationTest, TEST_IMG import pytest +from docker.errors import APIError + +from .base import TEST_IMG, BaseAPIIntegrationTest + class ErrorsTest(BaseAPIIntegrationTest): def test_api_error_parses_json(self): diff --git a/tests/integration/models_containers_test.py b/tests/integration/models_containers_test.py index 219b9a4cb1..f2813e74e9 100644 --- a/tests/integration/models_containers_test.py +++ b/tests/integration/models_containers_test.py @@ -5,10 +5,9 @@ import pytest import docker -from .base import BaseIntegrationTest -from .base import TEST_API_VERSION -from ..helpers import random_name -from ..helpers import requires_api_version + +from ..helpers import random_name, requires_api_version +from .base import TEST_API_VERSION, BaseIntegrationTest class ContainerCollectionTest(BaseIntegrationTest): diff --git a/tests/integration/models_images_test.py b/tests/integration/models_images_test.py index d335da4a71..9d42cc48f2 100644 --- a/tests/integration/models_images_test.py +++ b/tests/integration/models_images_test.py @@ -1,11 +1,12 @@ import io import tempfile -import docker import pytest -from .base import BaseIntegrationTest, TEST_IMG, TEST_API_VERSION +import docker + from ..helpers import random_name +from .base import TEST_API_VERSION, TEST_IMG, BaseIntegrationTest class ImageCollectionTest(BaseIntegrationTest): diff --git a/tests/integration/models_networks_test.py b/tests/integration/models_networks_test.py index f4052e4ba1..f5e6fcf573 100644 --- a/tests/integration/models_networks_test.py +++ b/tests/integration/models_networks_test.py @@ -1,6 +1,7 @@ import docker + from .. import helpers -from .base import BaseIntegrationTest, TEST_API_VERSION +from .base import TEST_API_VERSION, BaseIntegrationTest class NetworkCollectionTest(BaseIntegrationTest): diff --git a/tests/integration/models_resources_test.py b/tests/integration/models_resources_test.py index 4aafe0cc74..7d9762702f 100644 --- a/tests/integration/models_resources_test.py +++ b/tests/integration/models_resources_test.py @@ -1,5 +1,6 @@ import docker -from .base import BaseIntegrationTest, TEST_API_VERSION + +from .base import TEST_API_VERSION, BaseIntegrationTest class ModelTest(BaseIntegrationTest): diff --git a/tests/integration/models_services_test.py b/tests/integration/models_services_test.py index f1439a418e..947ba46d27 100644 --- a/tests/integration/models_services_test.py +++ b/tests/integration/models_services_test.py @@ -1,13 +1,14 @@ import unittest -import docker import pytest -from .. import helpers -from .base import TEST_API_VERSION +import docker from docker.errors import InvalidArgument from docker.types.services import ServiceMode +from .. import helpers +from .base import TEST_API_VERSION + class ServiceTest(unittest.TestCase): @classmethod diff --git a/tests/integration/models_swarm_test.py b/tests/integration/models_swarm_test.py index 6c1836dc60..f43824c75f 100644 --- a/tests/integration/models_swarm_test.py +++ b/tests/integration/models_swarm_test.py @@ -1,10 +1,11 @@ import unittest +import pytest + import docker from .. import helpers from .base import TEST_API_VERSION -import pytest class SwarmTest(unittest.TestCase): diff --git a/tests/integration/models_volumes_test.py b/tests/integration/models_volumes_test.py index 47b4a4550f..7d3ffda99d 100644 --- a/tests/integration/models_volumes_test.py +++ b/tests/integration/models_volumes_test.py @@ -1,5 +1,6 @@ import docker -from .base import BaseIntegrationTest, TEST_API_VERSION + +from .base import TEST_API_VERSION, BaseIntegrationTest class VolumesTest(BaseIntegrationTest): diff --git a/tests/integration/regression_test.py b/tests/integration/regression_test.py index 7d2b228cc9..5df9d31210 100644 --- a/tests/integration/regression_test.py +++ b/tests/integration/regression_test.py @@ -1,10 +1,11 @@ import io import random +import pytest + import docker -from .base import BaseAPIIntegrationTest, TEST_IMG -import pytest +from .base import TEST_IMG, BaseAPIIntegrationTest class TestRegressions(BaseAPIIntegrationTest): diff --git a/tests/ssh/api_build_test.py b/tests/ssh/api_build_test.py index 3b542994fc..20476fc74d 100644 --- a/tests/ssh/api_build_test.py +++ b/tests/ssh/api_build_test.py @@ -3,13 +3,13 @@ import shutil import tempfile +import pytest + from docker import errors from docker.utils.proxy import ProxyConfig -import pytest - -from .base import BaseAPIIntegrationTest, TEST_IMG from ..helpers import random_name, requires_api_version, requires_experimental +from .base import TEST_IMG, BaseAPIIntegrationTest class BuildTest(BaseAPIIntegrationTest): diff --git a/tests/ssh/base.py b/tests/ssh/base.py index d6ff130a1d..bf3c11d7a7 100644 --- a/tests/ssh/base.py +++ b/tests/ssh/base.py @@ -5,9 +5,10 @@ import pytest import docker -from .. import helpers from docker.utils import kwargs_from_env +from .. import helpers + TEST_IMG = 'alpine:3.10' TEST_API_VERSION = os.environ.get('DOCKER_TEST_API_VERSION') diff --git a/tests/ssh/connect_test.py b/tests/ssh/connect_test.py index 3d33a96db2..8780e3b8bd 100644 --- a/tests/ssh/connect_test.py +++ b/tests/ssh/connect_test.py @@ -1,9 +1,11 @@ import os import unittest -import docker import paramiko.ssh_exception import pytest + +import docker + from .base import TEST_API_VERSION diff --git a/tests/unit/api_container_test.py b/tests/unit/api_container_test.py index c4e2250be0..b2e5237a2a 100644 --- a/tests/unit/api_container_test.py +++ b/tests/unit/api_container_test.py @@ -1,17 +1,22 @@ import datetime import json import signal +from unittest import mock + +import pytest import docker from docker.api import APIClient -from unittest import mock -import pytest -from . import fake_api from ..helpers import requires_api_version +from . import fake_api from .api_test import ( - BaseAPIClientTest, url_prefix, fake_request, DEFAULT_TIMEOUT_SECONDS, - fake_inspect_container, url_base + DEFAULT_TIMEOUT_SECONDS, + BaseAPIClientTest, + fake_inspect_container, + fake_request, + url_base, + url_prefix, ) diff --git a/tests/unit/api_exec_test.py b/tests/unit/api_exec_test.py index 1760239fd6..9d789723a0 100644 --- a/tests/unit/api_exec_test.py +++ b/tests/unit/api_exec_test.py @@ -2,7 +2,10 @@ from . import fake_api from .api_test import ( - BaseAPIClientTest, url_prefix, fake_request, DEFAULT_TIMEOUT_SECONDS, + DEFAULT_TIMEOUT_SECONDS, + BaseAPIClientTest, + fake_request, + url_prefix, ) diff --git a/tests/unit/api_image_test.py b/tests/unit/api_image_test.py index 22b27fe0da..148109d37e 100644 --- a/tests/unit/api_image_test.py +++ b/tests/unit/api_image_test.py @@ -1,12 +1,17 @@ -import docker +from unittest import mock + import pytest -from . import fake_api +import docker from docker import auth -from unittest import mock + +from . import fake_api from .api_test import ( - BaseAPIClientTest, fake_request, DEFAULT_TIMEOUT_SECONDS, url_prefix, - fake_resolve_authconfig + DEFAULT_TIMEOUT_SECONDS, + BaseAPIClientTest, + fake_request, + fake_resolve_authconfig, + url_prefix, ) diff --git a/tests/unit/api_network_test.py b/tests/unit/api_network_test.py index d3daa44c41..1f9e596655 100644 --- a/tests/unit/api_network_test.py +++ b/tests/unit/api_network_test.py @@ -1,8 +1,9 @@ import json +from unittest import mock -from .api_test import BaseAPIClientTest, url_prefix, response from docker.types import IPAMConfig, IPAMPool -from unittest import mock + +from .api_test import BaseAPIClientTest, response, url_prefix class NetworkTest(BaseAPIClientTest): diff --git a/tests/unit/api_test.py b/tests/unit/api_test.py index 0ca9bbb950..3ce127b346 100644 --- a/tests/unit/api_test.py +++ b/tests/unit/api_test.py @@ -1,29 +1,29 @@ import datetime +import http.server import io import json import os import re import shutil import socket +import socketserver import struct import tempfile import threading import time import unittest -import socketserver -import http.server +from unittest import mock -import docker import pytest import requests import urllib3 + +import docker from docker.api import APIClient from docker.constants import DEFAULT_DOCKER_API_VERSION -from unittest import mock from . import fake_api - DEFAULT_TIMEOUT_SECONDS = docker.constants.DEFAULT_TIMEOUT_SECONDS diff --git a/tests/unit/api_volume_test.py b/tests/unit/api_volume_test.py index 0a97ca5150..fd32063966 100644 --- a/tests/unit/api_volume_test.py +++ b/tests/unit/api_volume_test.py @@ -3,7 +3,7 @@ import pytest from ..helpers import requires_api_version -from .api_test import BaseAPIClientTest, url_prefix, fake_request +from .api_test import BaseAPIClientTest, fake_request, url_prefix class VolumeTest(BaseAPIClientTest): diff --git a/tests/unit/auth_test.py b/tests/unit/auth_test.py index 0ed890fdf3..b2fedb32e4 100644 --- a/tests/unit/auth_test.py +++ b/tests/unit/auth_test.py @@ -6,11 +6,12 @@ import shutil import tempfile import unittest - -from docker import auth, credentials, errors from unittest import mock + import pytest +from docker import auth, credentials, errors + class RegressionTest(unittest.TestCase): def test_803_urlsafe_encode(self): diff --git a/tests/unit/client_test.py b/tests/unit/client_test.py index 7012b21236..60a6d5c0f5 100644 --- a/tests/unit/client_test.py +++ b/tests/unit/client_test.py @@ -1,15 +1,18 @@ import datetime import os import unittest +from unittest import mock -import docker import pytest + +import docker from docker.constants import ( - DEFAULT_DOCKER_API_VERSION, DEFAULT_TIMEOUT_SECONDS, - DEFAULT_MAX_POOL_SIZE, IS_WINDOWS_PLATFORM + DEFAULT_DOCKER_API_VERSION, + DEFAULT_MAX_POOL_SIZE, + DEFAULT_TIMEOUT_SECONDS, + IS_WINDOWS_PLATFORM, ) from docker.utils import kwargs_from_env -from unittest import mock from . import fake_api diff --git a/tests/unit/context_test.py b/tests/unit/context_test.py index 25f0d8c6ba..9e9fc9ba13 100644 --- a/tests/unit/context_test.py +++ b/tests/unit/context_test.py @@ -1,10 +1,10 @@ import unittest -import docker + import pytest -from docker.constants import DEFAULT_UNIX_SOCKET -from docker.constants import DEFAULT_NPIPE -from docker.constants import IS_WINDOWS_PLATFORM -from docker.context import ContextAPI, Context + +import docker +from docker.constants import DEFAULT_NPIPE, DEFAULT_UNIX_SOCKET, IS_WINDOWS_PLATFORM +from docker.context import Context, ContextAPI class BaseContextTest(unittest.TestCase): diff --git a/tests/unit/dockertypes_test.py b/tests/unit/dockertypes_test.py index f3d562e108..03e7d2eda0 100644 --- a/tests/unit/dockertypes_test.py +++ b/tests/unit/dockertypes_test.py @@ -1,15 +1,22 @@ import unittest +from unittest import mock import pytest from docker.constants import DEFAULT_DOCKER_API_VERSION from docker.errors import InvalidArgument, InvalidVersion from docker.types import ( - ContainerSpec, EndpointConfig, HostConfig, IPAMConfig, - IPAMPool, LogConfig, Mount, ServiceMode, Ulimit, + ContainerSpec, + EndpointConfig, + HostConfig, + IPAMConfig, + IPAMPool, + LogConfig, + Mount, + ServiceMode, + Ulimit, ) from docker.types.services import convert_service_ports -from unittest import mock def create_host_config(*args, **kwargs): diff --git a/tests/unit/errors_test.py b/tests/unit/errors_test.py index f8c3a6663d..5ccc40474f 100644 --- a/tests/unit/errors_test.py +++ b/tests/unit/errors_test.py @@ -2,9 +2,14 @@ import requests -from docker.errors import (APIError, ContainerError, DockerException, - create_unexpected_kwargs_error, - create_api_error_from_http_exception) +from docker.errors import ( + APIError, + ContainerError, + DockerException, + create_api_error_from_http_exception, + create_unexpected_kwargs_error, +) + from .fake_api import FAKE_CONTAINER_ID, FAKE_IMAGE_ID from .fake_api_client import make_fake_client diff --git a/tests/unit/fake_api_client.py b/tests/unit/fake_api_client.py index 7979942167..017e99d0c8 100644 --- a/tests/unit/fake_api_client.py +++ b/tests/unit/fake_api_client.py @@ -1,8 +1,9 @@ import copy +from unittest import mock import docker from docker.constants import DEFAULT_DOCKER_API_VERSION -from unittest import mock + from . import fake_api diff --git a/tests/unit/models_configs_test.py b/tests/unit/models_configs_test.py index 5d52daf76f..9f10830687 100644 --- a/tests/unit/models_configs_test.py +++ b/tests/unit/models_configs_test.py @@ -1,7 +1,8 @@ import unittest -from .fake_api_client import make_fake_client from .fake_api import FAKE_CONFIG_NAME +from .fake_api_client import make_fake_client + class CreateConfigsTest(unittest.TestCase): def test_create_config(self): diff --git a/tests/unit/models_containers_test.py b/tests/unit/models_containers_test.py index 05005815f3..0e2ae341a9 100644 --- a/tests/unit/models_containers_test.py +++ b/tests/unit/models_containers_test.py @@ -3,12 +3,12 @@ import pytest import docker -from docker.constants import DEFAULT_DATA_CHUNK_SIZE, \ - DEFAULT_DOCKER_API_VERSION +from docker.constants import DEFAULT_DATA_CHUNK_SIZE, DEFAULT_DOCKER_API_VERSION from docker.models.containers import Container, _create_container_args from docker.models.images import Image from docker.types import EndpointConfig -from .fake_api import FAKE_CONTAINER_ID, FAKE_IMAGE_ID, FAKE_EXEC_ID + +from .fake_api import FAKE_CONTAINER_ID, FAKE_EXEC_ID, FAKE_IMAGE_ID from .fake_api_client import make_fake_client diff --git a/tests/unit/models_networks_test.py b/tests/unit/models_networks_test.py index f10e1e3e33..099fb21936 100644 --- a/tests/unit/models_networks_test.py +++ b/tests/unit/models_networks_test.py @@ -1,6 +1,6 @@ import unittest -from .fake_api import FAKE_NETWORK_ID, FAKE_CONTAINER_ID +from .fake_api import FAKE_CONTAINER_ID, FAKE_NETWORK_ID from .fake_api_client import make_fake_client diff --git a/tests/unit/models_secrets_test.py b/tests/unit/models_secrets_test.py index 1c261a871f..1f5aaace2a 100644 --- a/tests/unit/models_secrets_test.py +++ b/tests/unit/models_secrets_test.py @@ -1,7 +1,7 @@ import unittest -from .fake_api_client import make_fake_client from .fake_api import FAKE_SECRET_NAME +from .fake_api_client import make_fake_client class CreateServiceTest(unittest.TestCase): diff --git a/tests/unit/models_services_test.py b/tests/unit/models_services_test.py index 45c63ac9e0..0277563435 100644 --- a/tests/unit/models_services_test.py +++ b/tests/unit/models_services_test.py @@ -1,4 +1,5 @@ import unittest + from docker.models.services import _get_create_service_kwargs diff --git a/tests/unit/sshadapter_test.py b/tests/unit/sshadapter_test.py index 874239ac8d..8736662101 100644 --- a/tests/unit/sshadapter_test.py +++ b/tests/unit/sshadapter_test.py @@ -1,4 +1,5 @@ import unittest + import docker from docker.transport.sshconn import SSHSocket diff --git a/tests/unit/swarm_test.py b/tests/unit/swarm_test.py index 3fc7c68cd5..4c0f2fd00c 100644 --- a/tests/unit/swarm_test.py +++ b/tests/unit/swarm_test.py @@ -1,8 +1,8 @@ import json -from . import fake_api from ..helpers import requires_api_version -from .api_test import BaseAPIClientTest, url_prefix, fake_request +from . import fake_api +from .api_test import BaseAPIClientTest, fake_request, url_prefix class SwarmTest(BaseAPIClientTest): diff --git a/tests/unit/utils_build_test.py b/tests/unit/utils_build_test.py index 5f1bb1ec0c..2089afb49d 100644 --- a/tests/unit/utils_build_test.py +++ b/tests/unit/utils_build_test.py @@ -9,7 +9,7 @@ import pytest from docker.constants import IS_WINDOWS_PLATFORM -from docker.utils import exclude_paths, tar, match_tag +from docker.utils import exclude_paths, match_tag, tar from ..helpers import make_tree diff --git a/tests/unit/utils_config_test.py b/tests/unit/utils_config_test.py index 27d5a7cd43..c87231a99f 100644 --- a/tests/unit/utils_config_test.py +++ b/tests/unit/utils_config_test.py @@ -1,12 +1,12 @@ +import json import os -import unittest import shutil import tempfile -import json - -from pytest import mark, fixture +import unittest from unittest import mock +from pytest import fixture, mark + from docker.utils import config diff --git a/tests/unit/utils_json_stream_test.py b/tests/unit/utils_json_stream_test.py index 821ebe42d4..5a8310cb52 100644 --- a/tests/unit/utils_json_stream_test.py +++ b/tests/unit/utils_json_stream_test.py @@ -1,4 +1,4 @@ -from docker.utils.json_stream import json_splitter, stream_as_text, json_stream +from docker.utils.json_stream import json_splitter, json_stream, stream_as_text class TestJsonSplitter: diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index c9434e11bb..21da0b58e8 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -7,14 +7,26 @@ import unittest import pytest + from docker.api.client import APIClient -from docker.constants import IS_WINDOWS_PLATFORM, DEFAULT_DOCKER_API_VERSION +from docker.constants import DEFAULT_DOCKER_API_VERSION, IS_WINDOWS_PLATFORM from docker.errors import DockerException from docker.utils import ( - compare_version, convert_filters, convert_volume_binds, decode_json_header, - format_environment, kwargs_from_env, parse_bytes, parse_devices, - parse_env_file, parse_host, parse_repository_tag, split_command, - update_headers, version_gte, version_lt + compare_version, + convert_filters, + convert_volume_binds, + decode_json_header, + format_environment, + kwargs_from_env, + parse_bytes, + parse_devices, + parse_env_file, + parse_host, + parse_repository_tag, + split_command, + update_headers, + version_gte, + version_lt, ) from docker.utils.ports import build_port_bindings, split_port From 1818712b8c16c3eaefef41a5e35bc049cc6fbf4f Mon Sep 17 00:00:00 2001 From: Aarni Koskela Date: Wed, 3 Jan 2024 21:38:53 +0200 Subject: [PATCH 1273/1301] Untangle circular imports Signed-off-by: Aarni Koskela --- docker/context/api.py | 5 +++-- docker/context/context.py | 7 ++++--- docker/transport/npipeconn.py | 3 +-- docker/transport/sshconn.py | 3 +-- docker/transport/unixconn.py | 3 +-- 5 files changed, 10 insertions(+), 11 deletions(-) diff --git a/docker/context/api.py b/docker/context/api.py index ae5d67bb25..9ac4ff470a 100644 --- a/docker/context/api.py +++ b/docker/context/api.py @@ -2,13 +2,14 @@ import os from docker import errors -from docker.context import Context -from docker.context.config import ( + +from .config import ( METAFILE, get_current_context_name, get_meta_dir, write_context_name_to_docker_config, ) +from .context import Context class ContextAPI: diff --git a/docker/context/context.py b/docker/context/context.py index 317bcf61df..da17d94781 100644 --- a/docker/context/context.py +++ b/docker/context/context.py @@ -2,14 +2,15 @@ import os from shutil import copyfile, rmtree -from docker.context.config import ( +from docker.errors import ContextException +from docker.tls import TLSConfig + +from .config import ( get_context_host, get_meta_dir, get_meta_file, get_tls_dir, ) -from docker.errors import ContextException -from docker.tls import TLSConfig class Context: diff --git a/docker/transport/npipeconn.py b/docker/transport/npipeconn.py index fe740a5f82..44d6921c2c 100644 --- a/docker/transport/npipeconn.py +++ b/docker/transport/npipeconn.py @@ -4,9 +4,8 @@ import urllib3 import urllib3.connection -from docker.transport.basehttpadapter import BaseHTTPAdapter - from .. import constants +from .basehttpadapter import BaseHTTPAdapter from .npipesocket import NpipeSocket RecentlyUsedContainer = urllib3._collections.RecentlyUsedContainer diff --git a/docker/transport/sshconn.py b/docker/transport/sshconn.py index 91671e9200..1870668010 100644 --- a/docker/transport/sshconn.py +++ b/docker/transport/sshconn.py @@ -11,9 +11,8 @@ import urllib3 import urllib3.connection -from docker.transport.basehttpadapter import BaseHTTPAdapter - from .. import constants +from .basehttpadapter import BaseHTTPAdapter RecentlyUsedContainer = urllib3._collections.RecentlyUsedContainer diff --git a/docker/transport/unixconn.py b/docker/transport/unixconn.py index f88d29ebfb..d571833f04 100644 --- a/docker/transport/unixconn.py +++ b/docker/transport/unixconn.py @@ -4,9 +4,8 @@ import urllib3 import urllib3.connection -from docker.transport.basehttpadapter import BaseHTTPAdapter - from .. import constants +from .basehttpadapter import BaseHTTPAdapter RecentlyUsedContainer = urllib3._collections.RecentlyUsedContainer From cb21af7f69fc44572c9f9a326c84275ae2be9c63 Mon Sep 17 00:00:00 2001 From: Rob Murray Date: Wed, 13 Mar 2024 14:54:25 +0000 Subject: [PATCH 1274/1301] Fix tests that look at 'Aliases' Inspect output for 'NetworkSettings.Networks..Aliases' includes the container's short-id (although it will be removed in API v1.45, in moby 26.0). Signed-off-by: Rob Murray --- tests/integration/models_containers_test.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tests/integration/models_containers_test.py b/tests/integration/models_containers_test.py index 219b9a4cb1..87ba89b1de 100644 --- a/tests/integration/models_containers_test.py +++ b/tests/integration/models_containers_test.py @@ -110,12 +110,12 @@ def test_run_with_networking_config(self): client.networks.create(net_name) self.tmp_networks.append(net_name) - test_aliases = ['hello'] + test_alias = 'hello' test_driver_opt = {'key1': 'a'} networking_config = { net_name: client.api.create_endpoint_config( - aliases=test_aliases, + aliases=[test_alias], driver_opt=test_driver_opt ) } @@ -132,8 +132,10 @@ def test_run_with_networking_config(self): assert 'NetworkSettings' in attrs assert 'Networks' in attrs['NetworkSettings'] assert list(attrs['NetworkSettings']['Networks'].keys()) == [net_name] - assert attrs['NetworkSettings']['Networks'][net_name]['Aliases'] == \ - test_aliases + # Expect Aliases to list 'test_alias' and the container's short-id. + # In API version 1.45, the short-id will be removed. + assert attrs['NetworkSettings']['Networks'][net_name]['Aliases'] \ + == [test_alias, attrs['Id'][:12]] assert attrs['NetworkSettings']['Networks'][net_name]['DriverOpts'] \ == test_driver_opt @@ -190,7 +192,9 @@ def test_run_with_networking_config_only_undeclared_network(self): assert 'NetworkSettings' in attrs assert 'Networks' in attrs['NetworkSettings'] assert list(attrs['NetworkSettings']['Networks'].keys()) == [net_name] - assert attrs['NetworkSettings']['Networks'][net_name]['Aliases'] is None + # Aliases should include the container's short-id (but it will be removed + # in API v1.45). + assert attrs['NetworkSettings']['Networks'][net_name]['Aliases'] == [attrs["Id"][:12]] assert (attrs['NetworkSettings']['Networks'][net_name]['DriverOpts'] is None) From e91b280074784026135573ab1726a565c090cd82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Thu, 7 Mar 2024 13:12:32 +0100 Subject: [PATCH 1275/1301] Bump default API version to 1.44 (Moby 25.0) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Paweł Gronowski --- Makefile | 4 ++-- docker/constants.py | 2 +- tests/Dockerfile-ssh-dind | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 00ebca05ce..25a83205b6 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ -TEST_API_VERSION ?= 1.43 -TEST_ENGINE_VERSION ?= 24.0 +TEST_API_VERSION ?= 1.44 +TEST_ENGINE_VERSION ?= 25.0 ifeq ($(OS),Windows_NT) PLATFORM := Windows diff --git a/docker/constants.py b/docker/constants.py index 71e543e530..8e7350d3d9 100644 --- a/docker/constants.py +++ b/docker/constants.py @@ -1,7 +1,7 @@ import sys from .version import __version__ -DEFAULT_DOCKER_API_VERSION = '1.43' +DEFAULT_DOCKER_API_VERSION = '1.44' MINIMUM_DOCKER_API_VERSION = '1.21' DEFAULT_TIMEOUT_SECONDS = 60 STREAM_HEADER_SIZE_BYTES = 8 diff --git a/tests/Dockerfile-ssh-dind b/tests/Dockerfile-ssh-dind index 2b7332b8b6..250c20f2c8 100644 --- a/tests/Dockerfile-ssh-dind +++ b/tests/Dockerfile-ssh-dind @@ -1,7 +1,7 @@ # syntax=docker/dockerfile:1 -ARG API_VERSION=1.43 -ARG ENGINE_VERSION=24.0 +ARG API_VERSION=1.44 +ARG ENGINE_VERSION=25.0 FROM docker:${ENGINE_VERSION}-dind From dd82f9ae8e601d3c82ef8d4f2dbab0c16109152f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Thu, 7 Mar 2024 13:12:45 +0100 Subject: [PATCH 1276/1301] Bump minimum API version to 1.24 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 25.0 raised the minimum supported API verison: https://github.com/moby/moby/pull/46887 Signed-off-by: Paweł Gronowski --- docker/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/constants.py b/docker/constants.py index 8e7350d3d9..213ff61ed1 100644 --- a/docker/constants.py +++ b/docker/constants.py @@ -2,7 +2,7 @@ from .version import __version__ DEFAULT_DOCKER_API_VERSION = '1.44' -MINIMUM_DOCKER_API_VERSION = '1.21' +MINIMUM_DOCKER_API_VERSION = '1.24' DEFAULT_TIMEOUT_SECONDS = 60 STREAM_HEADER_SIZE_BYTES = 8 CONTAINER_LIMITS_KEYS = [ From 9ad4bddc9ee23f3646f256280a21ef86274e39bc Mon Sep 17 00:00:00 2001 From: Milas Bowman Date: Wed, 3 Apr 2024 08:44:29 -0400 Subject: [PATCH 1277/1301] chore(ci): fix-ups across Make / Docker / GitHub Actions (#3241) --- .github/workflows/ci.yml | 2 +- .readthedocs.yml | 6 ++-- Dockerfile | 16 ++++------- Dockerfile-docs | 9 ++++-- MANIFEST.in | 9 ------ Makefile | 60 +++++++++++++++++++++++++--------------- README.md | 2 +- docs-requirements.txt | 2 -- pyproject.toml | 23 ++++++++++++++- requirements.txt | 6 ---- test-requirements.txt | 6 ---- tests/Dockerfile | 19 ++++--------- tox.ini | 7 ++--- 13 files changed, 86 insertions(+), 81 deletions(-) delete mode 100644 MANIFEST.in delete mode 100644 docs-requirements.txt delete mode 100644 requirements.txt delete mode 100644 test-requirements.txt diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9eb450a6ae..2cac5b1321 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,7 +47,7 @@ jobs: - name: Install dependencies run: | python3 -m pip install --upgrade pip - pip3 install -r test-requirements.txt -r requirements.txt + pip3 install '.[ssh,dev]' - name: Run unit tests run: | docker logout diff --git a/.readthedocs.yml b/.readthedocs.yml index 80000ee7f1..907454ea92 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -4,14 +4,14 @@ sphinx: configuration: docs/conf.py build: - os: ubuntu-20.04 + os: ubuntu-22.04 tools: - python: '3.10' + python: '3.12' python: install: - - requirements: docs-requirements.txt - method: pip path: . extra_requirements: - ssh + - docs diff --git a/Dockerfile b/Dockerfile index 293888d725..0189f46c8f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,17 +1,13 @@ # syntax=docker/dockerfile:1 ARG PYTHON_VERSION=3.12 - FROM python:${PYTHON_VERSION} WORKDIR /src - -COPY requirements.txt /src/requirements.txt -RUN pip install --no-cache-dir -r requirements.txt - -COPY test-requirements.txt /src/test-requirements.txt -RUN pip install --no-cache-dir -r test-requirements.txt - COPY . . -ARG SETUPTOOLS_SCM_PRETEND_VERSION_DOCKER -RUN pip install --no-cache-dir . + +ARG VERSION +RUN --mount=type=cache,target=/cache/pip \ + PIP_CACHE_DIR=/cache/pip \ + SETUPTOOLS_SCM_PRETEND_VERSION=${VERSION} \ + pip install .[ssh] diff --git a/Dockerfile-docs b/Dockerfile-docs index 266b2099e9..14d615c43c 100644 --- a/Dockerfile-docs +++ b/Dockerfile-docs @@ -11,7 +11,12 @@ RUN addgroup --gid $gid sphinx \ && useradd --uid $uid --gid $gid -M sphinx WORKDIR /src -COPY requirements.txt docs-requirements.txt ./ -RUN pip install --no-cache-dir -r requirements.txt -r docs-requirements.txt +COPY . . + +ARG VERSION +RUN --mount=type=cache,target=/cache/pip \ + PIP_CACHE_DIR=/cache/pip \ + SETUPTOOLS_SCM_PRETEND_VERSION=${VERSION} \ + pip install .[ssh,docs] USER sphinx diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 2ba6e0274c..0000000000 --- a/MANIFEST.in +++ /dev/null @@ -1,9 +0,0 @@ -include test-requirements.txt -include requirements.txt -include README.md -include README.rst -include LICENSE -recursive-include tests *.py -recursive-include tests/unit/testdata * -recursive-include tests/integration/testdata * -recursive-include tests/gpg-keys * diff --git a/Makefile b/Makefile index 25a83205b6..13a00f5e20 100644 --- a/Makefile +++ b/Makefile @@ -11,12 +11,17 @@ ifeq ($(PLATFORM),Linux) uid_args := "--build-arg uid=$(shell id -u) --build-arg gid=$(shell id -g)" endif +SETUPTOOLS_SCM_PRETEND_VERSION_DOCKER ?= $(shell git describe --match '[0-9]*' --dirty='.m' --always --tags 2>/dev/null | sed -r 's/-([0-9]+)/.dev\1/' | sed 's/-/+/') +ifeq ($(SETUPTOOLS_SCM_PRETEND_VERSION_DOCKER),) + SETUPTOOLS_SCM_PRETEND_VERSION_DOCKER = "dev" +endif + .PHONY: all all: test .PHONY: clean clean: - -docker rm -f dpy-dind-py3 dpy-dind-certs dpy-dind-ssl + -docker rm -f dpy-dind dpy-dind-certs dpy-dind-ssl find -name "__pycache__" | xargs rm -rf .PHONY: build-dind-ssh @@ -25,35 +30,46 @@ build-dind-ssh: --pull \ -t docker-dind-ssh \ -f tests/Dockerfile-ssh-dind \ + --build-arg VERSION=${SETUPTOOLS_SCM_PRETEND_VERSION_DOCKER} \ --build-arg ENGINE_VERSION=${TEST_ENGINE_VERSION} \ --build-arg API_VERSION=${TEST_API_VERSION} \ --build-arg APT_MIRROR . -.PHONY: build-py3 -build-py3: +.PHONY: build +build: docker build \ --pull \ -t docker-sdk-python3 \ -f tests/Dockerfile \ + --build-arg VERSION=${SETUPTOOLS_SCM_PRETEND_VERSION_DOCKER} \ --build-arg APT_MIRROR . .PHONY: build-docs build-docs: - docker build -t docker-sdk-python-docs -f Dockerfile-docs $(uid_args) . + docker build \ + -t docker-sdk-python-docs \ + -f Dockerfile-docs \ + --build-arg VERSION=${SETUPTOOLS_SCM_PRETEND_VERSION_DOCKER} \ + $(uid_args) \ + . .PHONY: build-dind-certs build-dind-certs: - docker build -t dpy-dind-certs -f tests/Dockerfile-dind-certs . + docker build \ + -t dpy-dind-certs \ + -f tests/Dockerfile-dind-certs \ + --build-arg VERSION=${SETUPTOOLS_SCM_PRETEND_VERSION_DOCKER} \ + . .PHONY: test -test: ruff unit-test-py3 integration-dind integration-dind-ssl +test: ruff unit-test integration-dind integration-dind-ssl -.PHONY: unit-test-py3 -unit-test-py3: build-py3 +.PHONY: unit-test +unit-test: build docker run -t --rm docker-sdk-python3 py.test tests/unit -.PHONY: integration-test-py3 -integration-test-py3: build-py3 +.PHONY: integration-test +integration-test: build docker run -t --rm -v /var/run/docker.sock:/var/run/docker.sock docker-sdk-python3 py.test -v tests/integration/${file} .PHONY: setup-network @@ -61,15 +77,15 @@ setup-network: docker network inspect dpy-tests || docker network create dpy-tests .PHONY: integration-dind -integration-dind: integration-dind-py3 +integration-dind: integration-dind -.PHONY: integration-dind-py3 -integration-dind-py3: build-py3 setup-network - docker rm -vf dpy-dind-py3 || : +.PHONY: integration-dind +integration-dind: build setup-network + docker rm -vf dpy-dind || : docker run \ --detach \ - --name dpy-dind-py3 \ + --name dpy-dind \ --network dpy-tests \ --pull=always \ --privileged \ @@ -82,10 +98,10 @@ integration-dind-py3: build-py3 setup-network --rm \ --tty \ busybox \ - sh -c 'while ! nc -z dpy-dind-py3 2375; do sleep 1; done' + sh -c 'while ! nc -z dpy-dind 2375; do sleep 1; done' docker run \ - --env="DOCKER_HOST=tcp://dpy-dind-py3:2375" \ + --env="DOCKER_HOST=tcp://dpy-dind:2375" \ --env="DOCKER_TEST_API_VERSION=${TEST_API_VERSION}" \ --network dpy-tests \ --rm \ @@ -93,11 +109,11 @@ integration-dind-py3: build-py3 setup-network docker-sdk-python3 \ py.test tests/integration/${file} - docker rm -vf dpy-dind-py3 + docker rm -vf dpy-dind .PHONY: integration-dind-ssh -integration-dind-ssh: build-dind-ssh build-py3 setup-network +integration-dind-ssh: build-dind-ssh build setup-network docker rm -vf dpy-dind-ssh || : docker run -d --network dpy-tests --name dpy-dind-ssh --privileged \ docker-dind-ssh dockerd --experimental @@ -116,7 +132,7 @@ integration-dind-ssh: build-dind-ssh build-py3 setup-network .PHONY: integration-dind-ssl -integration-dind-ssl: build-dind-certs build-py3 setup-network +integration-dind-ssl: build-dind-certs build setup-network docker rm -vf dpy-dind-certs dpy-dind-ssl || : docker run -d --name dpy-dind-certs dpy-dind-certs @@ -164,7 +180,7 @@ integration-dind-ssl: build-dind-certs build-py3 setup-network docker rm -vf dpy-dind-ssl dpy-dind-certs .PHONY: ruff -ruff: build-py3 +ruff: build docker run -t --rm docker-sdk-python3 ruff docker tests .PHONY: docs @@ -172,5 +188,5 @@ docs: build-docs docker run --rm -t -v `pwd`:/src docker-sdk-python-docs sphinx-build docs docs/_build .PHONY: shell -shell: build-py3 +shell: build docker run -it -v /var/run/docker.sock:/var/run/docker.sock docker-sdk-python3 python diff --git a/README.md b/README.md index 921ffbcb88..a6e06a229f 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ A Python library for the Docker Engine API. It lets you do anything the `docker` ## Installation -The latest stable version [is available on PyPI](https://pypi.python.org/pypi/docker/). Either add `docker` to your `requirements.txt` file or install with pip: +The latest stable version [is available on PyPI](https://pypi.python.org/pypi/docker/). Install with pip: pip install docker diff --git a/docs-requirements.txt b/docs-requirements.txt deleted file mode 100644 index 04d1aff268..0000000000 --- a/docs-requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -myst-parser==0.18.0 -Sphinx==5.1.1 diff --git a/pyproject.toml b/pyproject.toml index 73f5ddad63..525a9b81a8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,13 +36,34 @@ dependencies = [ ] [project.optional-dependencies] +# ssh feature allows DOCKER_HOST=ssh://... style connections ssh = [ "paramiko>=2.4.3", ] -tls = [] # kept for backwards compatibility +# tls is always supported, the feature is a no-op for backwards compatibility +tls = [] +# websockets can be used as an alternate container attach mechanism but +# by default docker-py hijacks the TCP connection and does not use Websockets +# unless attach_socket(container, ws=True) is called websockets = [ "websocket-client >= 1.3.0", ] +# docs are dependencies required to build the ReadTheDocs site +# this is only needed for CI / working on the docs! +docs = [ + "myst-parser==0.18.0", + "Sphinx==5.1.1", + +] +# dev are dependencies required to test & lint this project +# this is only needed if you are making code changes to docker-py! +dev = [ + "coverage==7.2.7", + "pytest==7.4.2", + "pytest-cov==4.1.0", + "pytest-timeout==2.1.0", + "ruff==0.1.8", +] [project.urls] Changelog = "https://docker-py.readthedocs.io/en/stable/change-log.html" diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 6d932eb371..0000000000 --- a/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -packaging==21.3 -paramiko==2.11.0 -pywin32==304; sys_platform == 'win32' -requests==2.31.0 -urllib3==1.26.18 -websocket-client==1.3.3 diff --git a/test-requirements.txt b/test-requirements.txt deleted file mode 100644 index 2c0e3622c6..0000000000 --- a/test-requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -setuptools==65.5.1 -coverage==7.2.7 -ruff==0.1.8 -pytest==7.4.2 -pytest-cov==4.1.0 -pytest-timeout==2.1.0 diff --git a/tests/Dockerfile b/tests/Dockerfile index d7c14b6cca..c0af8e85fa 100644 --- a/tests/Dockerfile +++ b/tests/Dockerfile @@ -1,7 +1,6 @@ # syntax=docker/dockerfile:1 ARG PYTHON_VERSION=3.12 - FROM python:${PYTHON_VERSION} RUN apt-get update && apt-get -y install --no-install-recommends \ @@ -27,16 +26,10 @@ RUN curl -sSL -o /opt/docker-credential-pass.tar.gz \ chmod +x /usr/local/bin/docker-credential-pass WORKDIR /src +COPY . . -COPY requirements.txt /src/requirements.txt -RUN --mount=type=cache,target=/root/.cache/pip \ - pip install -r requirements.txt - -COPY test-requirements.txt /src/test-requirements.txt -RUN --mount=type=cache,target=/root/.cache/pip \ - pip install -r test-requirements.txt - -COPY . /src -ARG SETUPTOOLS_SCM_PRETEND_VERSION=99.0.0+docker -RUN --mount=type=cache,target=/root/.cache/pip \ - pip install -e . +ARG VERSION +RUN --mount=type=cache,target=/cache/pip \ + PIP_CACHE_DIR=/cache/pip \ + SETUPTOOLS_SCM_PRETEND_VERSION=${VERSION} \ + pip install .[dev,ssh,websockets] diff --git a/tox.ini b/tox.ini index 03467aea26..19689b9645 100644 --- a/tox.ini +++ b/tox.ini @@ -6,11 +6,8 @@ skipsdist=True usedevelop=True commands = py.test -v --cov=docker {posargs:tests/unit} -deps = - -r{toxinidir}/test-requirements.txt - -r{toxinidir}/requirements.txt +extras = dev [testenv:ruff] commands = ruff docker tests setup.py -deps = - -r{toxinidir}/test-requirements.txt +extras = dev From b6464dbed92b14b2c61d5ee49805fce041a3e083 Mon Sep 17 00:00:00 2001 From: Bob Du Date: Wed, 10 Apr 2024 04:13:21 +0800 Subject: [PATCH 1278/1301] chore: fix return type docs for `container.logs()` (#2240) --- docker/api/container.py | 2 +- docker/models/containers.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/api/container.py b/docker/api/container.py index 1f153eeb79..d1b870f9c2 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -844,7 +844,7 @@ def logs(self, container, stdout=True, stderr=True, stream=False, float (in fractional seconds) Returns: - (generator or str) + (generator of bytes or bytes) Raises: :py:class:`docker.errors.APIError` diff --git a/docker/models/containers.py b/docker/models/containers.py index 35d8b31418..4795523a15 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -313,7 +313,7 @@ def logs(self, **kwargs): float (in nanoseconds) Returns: - (generator or str): Logs from the container. + (generator of bytes or bytes): Logs from the container. Raises: :py:class:`docker.errors.APIError` From 205d2f2bd00640e83c917f07e50c684173d2e2f6 Mon Sep 17 00:00:00 2001 From: Christopher Petito <47751006+krissetto@users.noreply.github.com> Date: Wed, 22 May 2024 10:58:13 +0000 Subject: [PATCH 1279/1301] Fix to get our CI working again since we rely on parsing tags. See https://github.com/docker/docker-py/pull/3259 attempts for more details Signed-off-by: Christopher Petito <47751006+krissetto@users.noreply.github.com> --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2cac5b1321..e31b2bd5c5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,6 +62,9 @@ jobs: steps: - uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true - name: make ${{ matrix.variant }} run: | docker logout From e34bcf20d9d0dc7156aaba72c635a4aa3bb0658e Mon Sep 17 00:00:00 2001 From: Christopher Petito <47751006+krissetto@users.noreply.github.com> Date: Wed, 22 May 2024 11:10:22 +0000 Subject: [PATCH 1280/1301] Update setup-python gh action Signed-off-by: Christopher Petito <47751006+krissetto@users.noreply.github.com> --- .github/workflows/ci.yml | 6 +++--- .github/workflows/release.yml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e31b2bd5c5..9b43a27bc8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: '3.x' - run: pip install -U ruff==0.1.8 @@ -22,7 +22,7 @@ jobs: runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: '3.x' - run: pip3 install build && python -m build . @@ -40,7 +40,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} allow-prereleases: true diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 953b59bf7a..f4d0919ac7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -22,7 +22,7 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: '3.x' From e33e0a437ecd895158c8cb4322a0cdad79312636 Mon Sep 17 00:00:00 2001 From: Felix Fontein Date: Mon, 20 May 2024 21:13:41 +0200 Subject: [PATCH 1281/1301] Hotfix for requests 2.32.0. Signed-off-by: Felix Fontein --- docker/transport/basehttpadapter.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docker/transport/basehttpadapter.py b/docker/transport/basehttpadapter.py index dfbb193b9a..281897a27e 100644 --- a/docker/transport/basehttpadapter.py +++ b/docker/transport/basehttpadapter.py @@ -6,3 +6,10 @@ def close(self): super().close() if hasattr(self, 'pools'): self.pools.clear() + + # Hotfix for requests 2.32.0: its commit + # https://github.com/psf/requests/commit/c0813a2d910ea6b4f8438b91d315b8d181302356 + # changes requests.adapters.HTTPAdapter to no longer call get_connection() from + # send(), but instead call _get_connection(). + def _get_connection(self, request, *args, proxies=None, **kwargs): + return self.get_connection(request.url, proxies) From 2a059a9f19c7b37c6c71c233754c6845e325d1ec Mon Sep 17 00:00:00 2001 From: Felix Fontein Date: Tue, 21 May 2024 18:44:08 +0200 Subject: [PATCH 1282/1301] Extend fix to requests 2.32.2+. Signed-off-by: Felix Fontein --- docker/transport/basehttpadapter.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docker/transport/basehttpadapter.py b/docker/transport/basehttpadapter.py index 281897a27e..c5996bb3e3 100644 --- a/docker/transport/basehttpadapter.py +++ b/docker/transport/basehttpadapter.py @@ -7,9 +7,14 @@ def close(self): if hasattr(self, 'pools'): self.pools.clear() - # Hotfix for requests 2.32.0: its commit + # Hotfix for requests 2.32.0 and 2.32.1: its commit # https://github.com/psf/requests/commit/c0813a2d910ea6b4f8438b91d315b8d181302356 # changes requests.adapters.HTTPAdapter to no longer call get_connection() from # send(), but instead call _get_connection(). def _get_connection(self, request, *args, proxies=None, **kwargs): return self.get_connection(request.url, proxies) + + # Fix for requests 2.32.2+: + # https://github.com/psf/requests/commit/c98e4d133ef29c46a9b68cd783087218a8075e05 + def get_connection_with_tls_context(self, request, verify, proxies=None, cert=None): + return self.get_connection(request.url, proxies) From d8e9bcb2780607faf388f8832bff3865eb24dce0 Mon Sep 17 00:00:00 2001 From: Felix Fontein Date: Tue, 21 May 2024 21:05:36 +0200 Subject: [PATCH 1283/1301] requests 2.32.0 and 2.32.1 have been yanked. Signed-off-by: Felix Fontein --- docker/transport/basehttpadapter.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/docker/transport/basehttpadapter.py b/docker/transport/basehttpadapter.py index c5996bb3e3..2301b6b07a 100644 --- a/docker/transport/basehttpadapter.py +++ b/docker/transport/basehttpadapter.py @@ -7,13 +7,6 @@ def close(self): if hasattr(self, 'pools'): self.pools.clear() - # Hotfix for requests 2.32.0 and 2.32.1: its commit - # https://github.com/psf/requests/commit/c0813a2d910ea6b4f8438b91d315b8d181302356 - # changes requests.adapters.HTTPAdapter to no longer call get_connection() from - # send(), but instead call _get_connection(). - def _get_connection(self, request, *args, proxies=None, **kwargs): - return self.get_connection(request.url, proxies) - # Fix for requests 2.32.2+: # https://github.com/psf/requests/commit/c98e4d133ef29c46a9b68cd783087218a8075e05 def get_connection_with_tls_context(self, request, verify, proxies=None, cert=None): From 4f2a26d21e81774cbb8e025b01de001a3ac3a545 Mon Sep 17 00:00:00 2001 From: Christopher Petito <47751006+krissetto@users.noreply.github.com> Date: Thu, 23 May 2024 09:27:07 +0000 Subject: [PATCH 1284/1301] Added 7.1.0 changelog Signed-off-by: Christopher Petito <47751006+krissetto@users.noreply.github.com> --- docs/change-log.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/docs/change-log.md b/docs/change-log.md index faf868ff8c..ebbdb71301 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -1,6 +1,27 @@ Changelog ========== +7.1.0 +----- +### Upgrade Notes +- Bumped minimum engine API version to 1.24 +- Bumped default engine API version to 1.44 (Moby 25.0) + +### Bugfixes +- Fixed issue with tag parsing when the registry address includes ports that resulted in `invalid tag format` errors +- Fixed issue preventing creating new configs (`ConfigCollection`), which failed with a `KeyError` due to the `name` field +- Fixed an issue due to an update in the [requests](https://github.com/psf/requests) package breaking `docker-py` by applying the [suggested fix](https://github.com/psf/requests/pull/6710) + +### Miscellaneous +- Documentation improvements +- Updated Ruff (linter) and fixed minor linting issues +- Packaging/CI updates + - Started using hatch for packaging (https://github.com/pypa/hatch) + - Updated `setup-python` github action +- Updated tests + - Stopped checking for deprecated container and image related fields (`Container` and `ContainerConfig`) + - Updated tests that check `NetworkSettings.Networks..Aliases` due to engine changes + 7.0.0 ----- ### Upgrade Notes From 45488acfc1851c5b5358ec7d8030a754c5f23783 Mon Sep 17 00:00:00 2001 From: Christopher Petito <47751006+krissetto@users.noreply.github.com> Date: Thu, 23 May 2024 10:14:18 +0000 Subject: [PATCH 1285/1301] Fix env var name in release pipeline Signed-off-by: Christopher Petito <47751006+krissetto@users.noreply.github.com> --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f4d0919ac7..3717e096b3 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -33,7 +33,7 @@ jobs: env: # This is also supported by Hatch; see # https://github.com/ofek/hatch-vcs#version-source-environment-variables - SETUPTOOLS_SCM_PRETEND_VERSION_FOR_DOCKER: ${{ inputs.tag }} + SETUPTOOLS_SCM_PRETEND_VERSION_DOCKER: ${{ inputs.tag }} - name: Publish to PyPI uses: pypa/gh-action-pypi-publish@release/v1 From 1ab40c8e926c0b892b3ef47ae8acc274fc13f250 Mon Sep 17 00:00:00 2001 From: Christopher Petito <47751006+krissetto@users.noreply.github.com> Date: Thu, 23 May 2024 10:49:23 +0000 Subject: [PATCH 1286/1301] Fix env var name in release pipeline to match hatch expectations Signed-off-by: Christopher Petito <47751006+krissetto@users.noreply.github.com> --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3717e096b3..17be00163d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -33,7 +33,7 @@ jobs: env: # This is also supported by Hatch; see # https://github.com/ofek/hatch-vcs#version-source-environment-variables - SETUPTOOLS_SCM_PRETEND_VERSION_DOCKER: ${{ inputs.tag }} + SETUPTOOLS_SCM_PRETEND_VERSION: ${{ inputs.tag }} - name: Publish to PyPI uses: pypa/gh-action-pypi-publish@release/v1 From 96ef4d3bee53ed17bf62670837b51c73911d7455 Mon Sep 17 00:00:00 2001 From: Laura Brehm Date: Fri, 27 Sep 2024 13:36:48 +0100 Subject: [PATCH 1287/1301] tests/exec: expect 127 exit code for missing executable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Docker Engine has always returned `126` when starting an exec fails due to a missing binary, but this was due to a bug in the daemon causing the correct exit code to be overwritten in some cases – see: https://github.com/moby/moby/issues/45795 Change tests to expect correct exit code (`127`). Signed-off-by: Laura Brehm --- tests/integration/models_containers_test.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/integration/models_containers_test.py b/tests/integration/models_containers_test.py index 476263ae23..c65f3d7f88 100644 --- a/tests/integration/models_containers_test.py +++ b/tests/integration/models_containers_test.py @@ -359,8 +359,11 @@ def test_exec_run_failed(self): "alpine", "sh -c 'sleep 60'", detach=True ) self.tmp_containers.append(container.id) - exec_output = container.exec_run("docker ps") - assert exec_output[0] == 126 + exec_output = container.exec_run("non-existent") + # older versions of docker return `126` in the case that an exec cannot + # be started due to a missing executable. We're fixing this for the + # future, so accept both for now. + assert exec_output[0] == 127 or exec_output == 126 def test_kill(self): client = docker.from_env(version=TEST_API_VERSION) From 6bbf741c8c74a684dcc1ab8da8133e99c021bfff Mon Sep 17 00:00:00 2001 From: yasonk Date: Sun, 29 Sep 2024 18:58:38 -0700 Subject: [PATCH 1288/1301] fixing doc for stream param in exec_run Signed-off-by: yasonk --- docker/models/containers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/models/containers.py b/docker/models/containers.py index 4795523a15..9c9e92c90f 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -181,7 +181,8 @@ def exec_run(self, cmd, stdout=True, stderr=True, stdin=False, tty=False, user (str): User to execute command as. Default: root detach (bool): If true, detach from the exec command. Default: False - stream (bool): Stream response data. Default: False + stream (bool): Stream response data. Ignored if ``detach`` is true. + Default: False socket (bool): Return the connection socket to allow custom read/write operations. Default: False environment (dict or list): A dictionary or a list of strings in From b1265470e64c29d8b270e43387c5abb442e084c7 Mon Sep 17 00:00:00 2001 From: Laura Brehm Date: Fri, 27 Sep 2024 15:05:08 +0100 Subject: [PATCH 1289/1301] tests/exec: add test for exit code from exec Execs should return the exit code of the exec'd process, if it started. Signed-off-by: Laura Brehm --- tests/integration/models_containers_test.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/integration/models_containers_test.py b/tests/integration/models_containers_test.py index c65f3d7f88..2cd713ec8a 100644 --- a/tests/integration/models_containers_test.py +++ b/tests/integration/models_containers_test.py @@ -353,6 +353,15 @@ def test_exec_run_success(self): assert exec_output[0] == 0 assert exec_output[1] == b"hello\n" + def test_exec_run_error_code_from_exec(self): + client = docker.from_env(version=TEST_API_VERSION) + container = client.containers.run( + "alpine", "sh -c 'sleep 20'", detach=True + ) + self.tmp_containers.append(container.id) + exec_output = container.exec_run("sh -c 'exit 42'") + assert exec_output[0] == 42 + def test_exec_run_failed(self): client = docker.from_env(version=TEST_API_VERSION) container = client.containers.run( @@ -363,7 +372,7 @@ def test_exec_run_failed(self): # older versions of docker return `126` in the case that an exec cannot # be started due to a missing executable. We're fixing this for the # future, so accept both for now. - assert exec_output[0] == 127 or exec_output == 126 + assert exec_output[0] == 127 or exec_output[0] == 126 def test_kill(self): client = docker.from_env(version=TEST_API_VERSION) From e47e966e944deb637a16f445abd5fc92f6dba38a Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Wed, 22 May 2024 15:12:42 +0200 Subject: [PATCH 1290/1301] Bump default API version to 1.45 (Moby 26.0/26.1) - Update API version to the latest maintained release. 0 Adjust tests for API 1.45 Signed-off-by: Sebastiaan van Stijn --- Makefile | 4 ++-- docker/constants.py | 2 +- tests/Dockerfile-ssh-dind | 4 ++-- tests/integration/models_containers_test.py | 11 +++++------ 4 files changed, 10 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index 13a00f5e20..d1c07ac739 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ -TEST_API_VERSION ?= 1.44 -TEST_ENGINE_VERSION ?= 25.0 +TEST_API_VERSION ?= 1.45 +TEST_ENGINE_VERSION ?= 26.1 ifeq ($(OS),Windows_NT) PLATFORM := Windows diff --git a/docker/constants.py b/docker/constants.py index 3c527b47e3..0e39dc2917 100644 --- a/docker/constants.py +++ b/docker/constants.py @@ -2,7 +2,7 @@ from .version import __version__ -DEFAULT_DOCKER_API_VERSION = '1.44' +DEFAULT_DOCKER_API_VERSION = '1.45' MINIMUM_DOCKER_API_VERSION = '1.24' DEFAULT_TIMEOUT_SECONDS = 60 STREAM_HEADER_SIZE_BYTES = 8 diff --git a/tests/Dockerfile-ssh-dind b/tests/Dockerfile-ssh-dind index 250c20f2c8..49529f84b7 100644 --- a/tests/Dockerfile-ssh-dind +++ b/tests/Dockerfile-ssh-dind @@ -1,7 +1,7 @@ # syntax=docker/dockerfile:1 -ARG API_VERSION=1.44 -ARG ENGINE_VERSION=25.0 +ARG API_VERSION=1.45 +ARG ENGINE_VERSION=26.1 FROM docker:${ENGINE_VERSION}-dind diff --git a/tests/integration/models_containers_test.py b/tests/integration/models_containers_test.py index 2cd713ec8a..8727455932 100644 --- a/tests/integration/models_containers_test.py +++ b/tests/integration/models_containers_test.py @@ -131,10 +131,9 @@ def test_run_with_networking_config(self): assert 'NetworkSettings' in attrs assert 'Networks' in attrs['NetworkSettings'] assert list(attrs['NetworkSettings']['Networks'].keys()) == [net_name] - # Expect Aliases to list 'test_alias' and the container's short-id. - # In API version 1.45, the short-id will be removed. + # Aliases no longer include the container's short-id in API v1.45. assert attrs['NetworkSettings']['Networks'][net_name]['Aliases'] \ - == [test_alias, attrs['Id'][:12]] + == [test_alias] assert attrs['NetworkSettings']['Networks'][net_name]['DriverOpts'] \ == test_driver_opt @@ -191,9 +190,9 @@ def test_run_with_networking_config_only_undeclared_network(self): assert 'NetworkSettings' in attrs assert 'Networks' in attrs['NetworkSettings'] assert list(attrs['NetworkSettings']['Networks'].keys()) == [net_name] - # Aliases should include the container's short-id (but it will be removed - # in API v1.45). - assert attrs['NetworkSettings']['Networks'][net_name]['Aliases'] == [attrs["Id"][:12]] + # Aliases no longer include the container's short-id in API v1.45. + assert (attrs['NetworkSettings']['Networks'][net_name]['Aliases'] + is None) assert (attrs['NetworkSettings']['Networks'][net_name]['DriverOpts'] is None) From 504ce6193cb945ee3b06f14bf42687bdc7c8c18f Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Tue, 28 May 2024 11:31:20 +0200 Subject: [PATCH 1291/1301] Set a dummy-version if none set Make sure the Dockerfiles can be built even if no VERSION build-arg is passed. Signed-off-by: Sebastiaan van Stijn --- Dockerfile | 2 +- Dockerfile-docs | 2 +- Makefile | 2 +- tests/Dockerfile | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0189f46c8f..e77e713738 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ FROM python:${PYTHON_VERSION} WORKDIR /src COPY . . -ARG VERSION +ARG VERSION=0.0.0.dev0 RUN --mount=type=cache,target=/cache/pip \ PIP_CACHE_DIR=/cache/pip \ SETUPTOOLS_SCM_PRETEND_VERSION=${VERSION} \ diff --git a/Dockerfile-docs b/Dockerfile-docs index 14d615c43c..4671d2c492 100644 --- a/Dockerfile-docs +++ b/Dockerfile-docs @@ -13,7 +13,7 @@ RUN addgroup --gid $gid sphinx \ WORKDIR /src COPY . . -ARG VERSION +ARG VERSION=0.0.0.dev0 RUN --mount=type=cache,target=/cache/pip \ PIP_CACHE_DIR=/cache/pip \ SETUPTOOLS_SCM_PRETEND_VERSION=${VERSION} \ diff --git a/Makefile b/Makefile index d1c07ac739..3230d20bb0 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,7 @@ endif SETUPTOOLS_SCM_PRETEND_VERSION_DOCKER ?= $(shell git describe --match '[0-9]*' --dirty='.m' --always --tags 2>/dev/null | sed -r 's/-([0-9]+)/.dev\1/' | sed 's/-/+/') ifeq ($(SETUPTOOLS_SCM_PRETEND_VERSION_DOCKER),) - SETUPTOOLS_SCM_PRETEND_VERSION_DOCKER = "dev" + SETUPTOOLS_SCM_PRETEND_VERSION_DOCKER = "0.0.0.dev0" endif .PHONY: all diff --git a/tests/Dockerfile b/tests/Dockerfile index c0af8e85fa..1d967e563b 100644 --- a/tests/Dockerfile +++ b/tests/Dockerfile @@ -28,7 +28,7 @@ RUN curl -sSL -o /opt/docker-credential-pass.tar.gz \ WORKDIR /src COPY . . -ARG VERSION +ARG VERSION=0.0.0.dev0 RUN --mount=type=cache,target=/cache/pip \ PIP_CACHE_DIR=/cache/pip \ SETUPTOOLS_SCM_PRETEND_VERSION=${VERSION} \ From 99ce2e6d565d4e449b55d5746be7f212193854be Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Tue, 28 May 2024 11:34:28 +0200 Subject: [PATCH 1292/1301] Makefile: remove unused APT_MIRROR build-arg The APT_MIRROR build-arg was removed from the Dockerfile in commit ee2310595d1362428f2826afac2e17077231473a, but wasn't removed from the Makefile. Signed-off-by: Sebastiaan van Stijn --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 3230d20bb0..7af4e7d6b5 100644 --- a/Makefile +++ b/Makefile @@ -33,7 +33,7 @@ build-dind-ssh: --build-arg VERSION=${SETUPTOOLS_SCM_PRETEND_VERSION_DOCKER} \ --build-arg ENGINE_VERSION=${TEST_ENGINE_VERSION} \ --build-arg API_VERSION=${TEST_API_VERSION} \ - --build-arg APT_MIRROR . + . .PHONY: build build: @@ -42,7 +42,7 @@ build: -t docker-sdk-python3 \ -f tests/Dockerfile \ --build-arg VERSION=${SETUPTOOLS_SCM_PRETEND_VERSION_DOCKER} \ - --build-arg APT_MIRROR . + . .PHONY: build-docs build-docs: From d9f9b965b2f72a8925f97f7efc9a02348aa0c39f Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Sun, 27 Oct 2024 16:59:28 +0100 Subject: [PATCH 1293/1301] Makefile: fix circular reference for integration-dind Noticed this warning; make: Circular integration-dind <- integration-dind dependency dropped. Signed-off-by: Sebastiaan van Stijn --- Makefile | 3 --- 1 file changed, 3 deletions(-) diff --git a/Makefile b/Makefile index 7af4e7d6b5..79aa15e2e9 100644 --- a/Makefile +++ b/Makefile @@ -76,9 +76,6 @@ integration-test: build setup-network: docker network inspect dpy-tests || docker network create dpy-tests -.PHONY: integration-dind -integration-dind: integration-dind - .PHONY: integration-dind integration-dind: build setup-network docker rm -vf dpy-dind || : From 8ee28517c758cb069543a4e9c125e2dc46b1c393 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Thu, 3 Oct 2019 01:49:26 +0200 Subject: [PATCH 1294/1301] test_service_logs: stop testing experimental versions Service logs are no longer experimental, so updating the tests to only test against "stable" implementations, and no longer test the experimental ones. Signed-off-by: Sebastiaan van Stijn --- tests/integration/api_service_test.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/integration/api_service_test.py b/tests/integration/api_service_test.py index 7a7ae4ea41..d670968786 100644 --- a/tests/integration/api_service_test.py +++ b/tests/integration/api_service_test.py @@ -5,7 +5,7 @@ import docker -from ..helpers import force_leave_swarm, requires_api_version, requires_experimental +from ..helpers import force_leave_swarm, requires_api_version from .base import TEST_IMG, BaseAPIIntegrationTest @@ -140,8 +140,7 @@ def test_create_service_simple(self): assert len(services) == 1 assert services[0]['ID'] == svc_id['ID'] - @requires_api_version('1.25') - @requires_experimental(until='1.29') + @requires_api_version('1.29') def test_service_logs(self): name, svc_id = self.create_simple_service() assert self.get_service_container(name, include_stopped=True) From 5a8a42466e3ffdc89c4a7b80018ee60ae77e32bc Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Tue, 14 Jan 2025 13:23:38 +0100 Subject: [PATCH 1295/1301] image load: don't depend on deprecated JSONMessage.error field The error field was deprecated in favor of the errorDetail struct in [moby@3043c26], but the API continued to return both. This patch updates docker-py to not depend on the deprecated field. [moby@3043c26]: https://github.com/moby/moby/commit/3043c2641990d94298c6377b7ef14709263a4709 Signed-off-by: Sebastiaan van Stijn --- docker/models/images.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/models/images.py b/docker/models/images.py index 4f058d24d9..0e8cce3f82 100644 --- a/docker/models/images.py +++ b/docker/models/images.py @@ -407,8 +407,8 @@ def load(self, data): if match: image_id = match.group(2) images.append(image_id) - if 'error' in chunk: - raise ImageLoadError(chunk['error']) + if 'errorDetail' in chunk: + raise ImageLoadError(chunk['errorDetail']['message']) return [self.get(i) for i in images] From fad84c371a874210ca92dfc56cba588c4d65fc2a Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Sun, 27 Oct 2024 16:55:41 +0100 Subject: [PATCH 1296/1301] integration: test_create_volume_invalid_driver allow either 400 or 404 The API currently returns a 404 error when trying to create a volume with an invalid (non-existing) driver. We are considering changing this status code to be a 400 (invalid parameter), as even though the _reason_ of the error may be that the plugin / driver is not found, the _cause_ of the error is that the user provided a plugin / driver that's invalid for the engine they're connected to. This patch updates the test to pass for either case. Signed-off-by: Sebastiaan van Stijn --- tests/integration/api_volume_test.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/integration/api_volume_test.py b/tests/integration/api_volume_test.py index ecd19da2d5..413b1d9bc6 100644 --- a/tests/integration/api_volume_test.py +++ b/tests/integration/api_volume_test.py @@ -17,10 +17,16 @@ def test_create_volume(self): assert result['Driver'] == 'local' def test_create_volume_invalid_driver(self): - driver_name = 'invalid.driver' + # special name to avoid exponential timeout loop + # https://github.com/moby/moby/blob/9e00a63d65434cdedc444e79a2b33a7c202b10d8/pkg/plugins/client.go#L253-L254 + driver_name = 'this-plugin-does-not-exist' - with pytest.raises(docker.errors.NotFound): + with pytest.raises(docker.errors.APIError) as cm: self.client.create_volume('perfectcherryblossom', driver_name) + assert ( + cm.value.response.status_code == 404 or + cm.value.response.status_code == 400 + ) def test_list_volumes(self): name = 'imperishablenight' From 820769e23cc4b822fcf53a3d17b5f544444c9b26 Mon Sep 17 00:00:00 2001 From: Khushiyant Date: Tue, 18 Mar 2025 22:56:15 +0530 Subject: [PATCH 1297/1301] feat(docker/api/container): add support for subpath in volume_opts TESTED: Yes, added unit tests to verify subpath functionality Signed-off-by: Khushiyant --- docker/types/services.py | 7 +++- tests/integration/api_container_test.py | 50 +++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 2 deletions(-) diff --git a/docker/types/services.py b/docker/types/services.py index 821115411c..69c0c498ea 100644 --- a/docker/types/services.py +++ b/docker/types/services.py @@ -242,6 +242,7 @@ class Mount(dict): for the ``volume`` type. driver_config (DriverConfig): Volume driver configuration. Only valid for the ``volume`` type. + subpath (str): Path inside a volume to mount instead of the volume root. tmpfs_size (int or string): The size for the tmpfs mount in bytes. tmpfs_mode (int): The permission mode for the tmpfs mount. """ @@ -249,7 +250,7 @@ class Mount(dict): def __init__(self, target, source, type='volume', read_only=False, consistency=None, propagation=None, no_copy=False, labels=None, driver_config=None, tmpfs_size=None, - tmpfs_mode=None): + tmpfs_mode=None, subpath=None): self['Target'] = target self['Source'] = source if type not in ('bind', 'volume', 'tmpfs', 'npipe'): @@ -267,7 +268,7 @@ def __init__(self, target, source, type='volume', read_only=False, self['BindOptions'] = { 'Propagation': propagation } - if any([labels, driver_config, no_copy, tmpfs_size, tmpfs_mode]): + if any([labels, driver_config, no_copy, tmpfs_size, tmpfs_mode, subpath]): raise errors.InvalidArgument( 'Incompatible options have been provided for the bind ' 'type mount.' @@ -280,6 +281,8 @@ def __init__(self, target, source, type='volume', read_only=False, volume_opts['Labels'] = labels if driver_config: volume_opts['DriverConfig'] = driver_config + if subpath: + volume_opts['Subpath'] = subpath if volume_opts: self['VolumeOptions'] = volume_opts if any([propagation, tmpfs_size, tmpfs_mode]): diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index 0215e14c25..21c2f35797 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -620,6 +620,56 @@ def test_create_with_volume_mount(self): assert mount['Source'] == mount_data['Name'] assert mount_data['RW'] is True + @requires_api_version('1.45') + def test_create_with_subpath_volume_mount(self): + source_volume = helpers.random_name() + self.client.create_volume(name=source_volume) + + setup_container = None + test_container = None + + + # Create a file structure in the volume to test with + setup_container = self.client.create_container( + TEST_IMG, + [ + "sh", + "-c", + 'mkdir -p /vol/subdir && echo "test content" > /vol/subdir/testfile.txt', + ], + host_config=self.client.create_host_config( + binds=[f"{source_volume}:/vol"] + ), + ) + self.client.start(setup_container) + self.client.wait(setup_container) + + # Now test with subpath + mount = docker.types.Mount( + type="volume", + source=source_volume, + target=self.mount_dest, + read_only=True, + subpath="subdir", + ) + + + host_config = self.client.create_host_config(mounts=[mount]) + test_container = self.client.create_container( + TEST_IMG, + ["cat", os.path.join(self.mount_dest, "testfile.txt")], + host_config=host_config, + ) + + self.client.start(test_container) + self.client.wait(test_container) # Wait for container to finish + output = self.client.logs(test_container).decode("utf-8").strip() + + # If the subpath feature is working, we should be able to see the content + # of the file in the subdir + assert output == "test content" + + def check_container_data(self, inspect_data, rw, propagation='rprivate'): assert 'Mounts' in inspect_data filtered = list(filter( From e5c3eb18b6faa5982d967d17ccb18acf44d7a9dc Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Thu, 22 May 2025 01:46:53 +0200 Subject: [PATCH 1298/1301] integration: adjust tests for omitted "OnBuild" The Docker API may either return an empty "OnBuild" or omit the field altogether if it's not set. Adjust the tests to make either satisfy the test. Signed-off-by: Sebastiaan van Stijn --- tests/integration/api_build_test.py | 2 +- tests/ssh/api_build_test.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/api_build_test.py b/tests/integration/api_build_test.py index 62e93a7384..0f560159b3 100644 --- a/tests/integration/api_build_test.py +++ b/tests/integration/api_build_test.py @@ -275,7 +275,7 @@ def test_build_container_with_target(self): pass info = self.client.inspect_image('build1') - assert not info['Config']['OnBuild'] + assert 'OnBuild' not in info['Config'] or not info['Config']['OnBuild'] @requires_api_version('1.25') def test_build_with_network_mode(self): diff --git a/tests/ssh/api_build_test.py b/tests/ssh/api_build_test.py index 20476fc74d..f17c75630f 100644 --- a/tests/ssh/api_build_test.py +++ b/tests/ssh/api_build_test.py @@ -266,7 +266,7 @@ def test_build_container_with_target(self): pass info = self.client.inspect_image('build1') - assert not info['Config']['OnBuild'] + assert 'OnBuild' not in info['Config'] or not info['Config']['OnBuild'] @requires_api_version('1.25') def test_build_with_network_mode(self): From 49479baa9f08018571ae5d3ae88ffdeee1d38d44 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Tue, 14 Oct 2025 23:01:01 +0200 Subject: [PATCH 1299/1301] tests: fix ssl generation the "integration-dind-ssl" tests were failing due to an issue with the test-certs created; ==================================== ERRORS ==================================== _________ ERROR at setup of BuildTest.test_build_container_with_target _________ /usr/local/lib/python3.12/site-packages/urllib3/connectionpool.py:464: in _make_request self._validate_conn(conn) /usr/local/lib/python3.12/site-packages/urllib3/connectionpool.py:1093: in _validate_conn conn.connect() /usr/local/lib/python3.12/site-packages/urllib3/connection.py:790: in connect sock_and_verified = _ssl_wrap_socket_and_match_hostname( /usr/local/lib/python3.12/site-packages/urllib3/connection.py:969: in _ssl_wrap_socket_and_match_hostname ssl_sock = ssl_wrap_socket( /usr/local/lib/python3.12/site-packages/urllib3/util/ssl_.py:480: in ssl_wrap_socket ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls, server_hostname) /usr/local/lib/python3.12/site-packages/urllib3/util/ssl_.py:524: in _ssl_wrap_socket_impl return ssl_context.wrap_socket(sock, server_hostname=server_hostname) /usr/local/lib/python3.12/ssl.py:455: in wrap_socket return self.sslsocket_class._create( /usr/local/lib/python3.12/ssl.py:1041: in _create self.do_handshake() /usr/local/lib/python3.12/ssl.py:1319: in do_handshake self._sslobj.do_handshake() E ssl.SSLCertVerificationError: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: invalid CA certificate (_ssl.c:1010) During handling of the above exception, another exception occurred: /usr/local/lib/python3.12/site-packages/urllib3/connectionpool.py:787: in urlopen response = self._make_request( /usr/local/lib/python3.12/site-packages/urllib3/connectionpool.py:488: in _make_request raise new_e E urllib3.exceptions.SSLError: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: invalid CA certificate (_ssl.c:1010) Signed-off-by: Sebastiaan van Stijn --- tests/Dockerfile-dind-certs | 43 +++++++++++++++++++++++++++++++------ 1 file changed, 36 insertions(+), 7 deletions(-) diff --git a/tests/Dockerfile-dind-certs b/tests/Dockerfile-dind-certs index 7b819eb154..9d5c58fb2d 100644 --- a/tests/Dockerfile-dind-certs +++ b/tests/Dockerfile-dind-certs @@ -7,17 +7,46 @@ RUN mkdir /tmp/certs VOLUME /certs WORKDIR /tmp/certs + +# ---- CA (with proper v3_ca) ---- RUN openssl genrsa -aes256 -passout pass:foobar -out ca-key.pem 4096 -RUN echo "[req]\nprompt=no\ndistinguished_name = req_distinguished_name\n[req_distinguished_name]\ncountryName=AU" > /tmp/config -RUN openssl req -new -x509 -passin pass:foobar -config /tmp/config -days 365 -key ca-key.pem -sha256 -out ca.pem -RUN openssl genrsa -out server-key.pem -passout pass:foobar 4096 +COPY <<'EOF' /tmp/ca.cnf +[req] +prompt = no +distinguished_name = req_distinguished_name +x509_extensions = v3_ca + +[req_distinguished_name] +countryName = AU + +[v3_ca] +basicConstraints = critical, CA:TRUE +keyUsage = critical, keyCertSign, cRLSign +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid:always,issuer +EOF +RUN openssl req -new -x509 -passin pass:foobar -config /tmp/ca.cnf -days 365 -key ca-key.pem -sha256 -out ca.pem + +# ---- Server cert (SAN + KU/EKU) ---- +RUN openssl genrsa -out server-key.pem 4096 RUN openssl req -subj "/CN=docker" -sha256 -new -key server-key.pem -out server.csr -RUN echo subjectAltName = DNS:docker,DNS:localhost > extfile.cnf -RUN openssl x509 -req -days 365 -passin pass:foobar -sha256 -in server.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out server-cert.pem -extfile extfile.cnf +COPY <<'EOF' /tmp/server-ext.cnf +basicConstraints = CA:FALSE +keyUsage = critical, digitalSignature, keyEncipherment +extendedKeyUsage = serverAuth +subjectAltName = DNS:docker, DNS:localhost +EOF +RUN openssl x509 -req -days 365 -passin pass:foobar -sha256 -in server.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out server-cert.pem -extfile /tmp/server-ext.cnf + +# ---- Client cert (KU/EKU) ---- RUN openssl genrsa -out key.pem 4096 RUN openssl req -passin pass:foobar -subj '/CN=client' -new -key key.pem -out client.csr -RUN echo extendedKeyUsage = clientAuth > extfile.cnf -RUN openssl x509 -req -passin pass:foobar -days 365 -sha256 -in client.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out cert.pem -extfile extfile.cnf +COPY <<'EOF' /tmp/client-ext.cnf +basicConstraints = CA:FALSE +keyUsage = critical, digitalSignature +extendedKeyUsage = clientAuth +EOF +RUN openssl x509 -req -passin pass:foobar -days 365 -sha256 -in client.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out cert.pem -extfile /tmp/client-ext.cnf RUN chmod -v 0400 ca-key.pem key.pem server-key.pem RUN chmod -v 0444 ca.pem server-cert.pem cert.pem From b520fb89514ce9e1b8cfd7527485ece588a90f6b Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Tue, 14 Oct 2025 18:36:34 +0200 Subject: [PATCH 1300/1301] test/integration: don't check for deprecated Networks field These tests depended on the deprecated Spec.Networks field, which is no longer part of current API versions, causing the test to fail; =================================== FAILURES =================================== _____________ ServiceTest.test_create_service_with_custom_networks _____________ tests/integration/api_service_test.py:379: in test_create_service_with_custom_networks assert 'Networks' in svc_info['Spec'] E AssertionError: assert 'Networks' in {'Labels': {}, 'Mode': {'Replicated': {'Replicas': 1}}, 'Name': 'dockerpytest_a538894175d07404', 'TaskTemplate': {'Con...pec': {'Command': ['true'], 'Image': 'alpine:3.10', 'Isolation': 'default'}, 'ForceUpdate': 0, 'Runtime': 'container'}} ____________ ServiceTest.test_update_service_with_defaults_networks ____________ tests/integration/api_service_test.py:1128: in test_update_service_with_defaults_networks assert 'Networks' in svc_info['Spec'] E AssertionError: assert 'Networks' in {'Labels': {}, 'Mode': {'Replicated': {'Replicas': 1}}, 'Name': 'dockerpytest_6d8e30f359c0f5e', 'TaskTemplate': {'Cont...pec': {'Command': ['true'], 'Image': 'alpine:3.10', 'Isolation': 'default'}, 'ForceUpdate': 0, 'Runtime': 'container'}} _____________ ServiceTest.test_update_service_with_network_change ______________ tests/integration/api_service_test.py:1333: in test_update_service_with_network_change assert 'Networks' in svc_info['Spec'] E AssertionError: assert 'Networks' in {'Labels': {}, 'Mode': {'Replicated': {'Replicas': 1}}, 'Name': 'dockerpytest_d4e23667cdbaf159', 'TaskTemplate': {'Con... {'Command': ['echo', 'hello'], 'Image': 'busybox', 'Isolation': 'default'}, 'ForceUpdate': 0, 'Runtime': 'container'}} ------- generated xml file: /src/bundles/test-docker-py/junit-report.xml ------- =========================== short test summary info ============================ Signed-off-by: Sebastiaan van Stijn --- tests/integration/api_service_test.py | 56 +++++++++++++++++---------- 1 file changed, 35 insertions(+), 21 deletions(-) diff --git a/tests/integration/api_service_test.py b/tests/integration/api_service_test.py index d670968786..ba67c6d538 100644 --- a/tests/integration/api_service_test.py +++ b/tests/integration/api_service_test.py @@ -367,16 +367,18 @@ def test_create_service_with_custom_networks(self): ) self.tmp_networks.append(net2['Id']) container_spec = docker.types.ContainerSpec(TEST_IMG, ['true']) - task_tmpl = docker.types.TaskTemplate(container_spec) - name = self.get_service_name() - svc_id = self.client.create_service( - task_tmpl, name=name, networks=[ + task_tmpl = docker.types.TaskTemplate( + container_spec, networks=[ 'dockerpytest_1', {'Target': 'dockerpytest_2'} ] ) + name = self.get_service_name() + svc_id = self.client.create_service( + task_tmpl, name=name + ) svc_info = self.client.inspect_service(svc_id) - assert 'Networks' in svc_info['Spec'] - assert svc_info['Spec']['Networks'] == [ + assert 'Networks' in svc_info['Spec']['TaskTemplate'] + assert svc_info['Spec']['TaskTemplate']['Networks'] == [ {'Target': net1['Id']}, {'Target': net2['Id']} ] @@ -1116,16 +1118,18 @@ def test_update_service_with_defaults_networks(self): ) self.tmp_networks.append(net2['Id']) container_spec = docker.types.ContainerSpec(TEST_IMG, ['true']) - task_tmpl = docker.types.TaskTemplate(container_spec) - name = self.get_service_name() - svc_id = self.client.create_service( - task_tmpl, name=name, networks=[ + task_tmpl = docker.types.TaskTemplate( + container_spec, networks=[ 'dockerpytest_1', {'Target': 'dockerpytest_2'} ] ) + name = self.get_service_name() + svc_id = self.client.create_service( + task_tmpl, name=name + ) svc_info = self.client.inspect_service(svc_id) - assert 'Networks' in svc_info['Spec'] - assert svc_info['Spec']['Networks'] == [ + assert 'Networks' in svc_info['Spec']['TaskTemplate'] + assert svc_info['Spec']['TaskTemplate']['Networks'] == [ {'Target': net1['Id']}, {'Target': net2['Id']} ] @@ -1143,8 +1147,11 @@ def test_update_service_with_defaults_networks(self): {'Target': net1['Id']}, {'Target': net2['Id']} ] + task_tmpl = docker.types.TaskTemplate( + container_spec, networks=[net1['Id']] + ) self._update_service( - svc_id, name, new_index, networks=[net1['Id']], + svc_id, name, new_index, task_tmpl, fetch_current_spec=True ) svc_info = self.client.inspect_service(svc_id) @@ -1313,7 +1320,6 @@ def test_update_service_with_network_change(self): container_spec = docker.types.ContainerSpec( 'busybox', ['echo', 'hello'] ) - task_tmpl = docker.types.TaskTemplate(container_spec) net1 = self.client.create_network( self.get_service_name(), driver='overlay', ipam={'Driver': 'default'} @@ -1324,22 +1330,27 @@ def test_update_service_with_network_change(self): ipam={'Driver': 'default'} ) self.tmp_networks.append(net2['Id']) + task_tmpl = docker.types.TaskTemplate( + container_spec, networks=[net1['Id']] + ) name = self.get_service_name() svc_id = self.client.create_service( - task_tmpl, name=name, networks=[net1['Id']] + task_tmpl, name=name ) svc_info = self.client.inspect_service(svc_id) - assert 'Networks' in svc_info['Spec'] - assert len(svc_info['Spec']['Networks']) > 0 - assert svc_info['Spec']['Networks'][0]['Target'] == net1['Id'] + assert 'Networks' in svc_info['Spec']['TaskTemplate'] + assert len(svc_info['Spec']['TaskTemplate']['Networks']) > 0 + assert svc_info['Spec']['TaskTemplate']['Networks'][0]['Target'] == net1['Id'] svc_info = self.client.inspect_service(svc_id) version_index = svc_info['Version']['Index'] - task_tmpl = docker.types.TaskTemplate(container_spec) + task_tmpl = docker.types.TaskTemplate( + container_spec, networks=[net2['Id']] + ) self._update_service( svc_id, name, version_index, task_tmpl, name=name, - networks=[net2['Id']], fetch_current_spec=True + fetch_current_spec=True ) svc_info = self.client.inspect_service(svc_id) task_template = svc_info['Spec']['TaskTemplate'] @@ -1351,8 +1362,11 @@ def test_update_service_with_network_change(self): new_index = svc_info['Version']['Index'] assert new_index > version_index + task_tmpl = docker.types.TaskTemplate( + container_spec, networks=[net1['Id']] + ) self._update_service( - svc_id, name, new_index, name=name, networks=[net1['Id']], + svc_id, name, new_index, task_tmpl, name=name, fetch_current_spec=True ) svc_info = self.client.inspect_service(svc_id) From cf97720bd35c718552e60c8c4f085bc94744e054 Mon Sep 17 00:00:00 2001 From: Ricardo Branco Date: Sat, 18 Oct 2025 11:32:43 +0200 Subject: [PATCH 1301/1301] test: Skip from_env_unix tests if DOCKER_HOST is network socket Signed-off-by: Ricardo Branco --- tests/unit/client_test.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/unit/client_test.py b/tests/unit/client_test.py index 60a6d5c0f5..5ba712d240 100644 --- a/tests/unit/client_test.py +++ b/tests/unit/client_test.py @@ -188,7 +188,8 @@ def test_from_env_without_timeout_uses_default(self): assert client.api.timeout == DEFAULT_TIMEOUT_SECONDS @pytest.mark.skipif( - IS_WINDOWS_PLATFORM, reason='Unix Connection Pool only on Linux' + os.environ.get('DOCKER_HOST', '').startswith('tcp://') or IS_WINDOWS_PLATFORM, + reason='Requires a Unix socket' ) @mock.patch("docker.transport.unixconn.UnixHTTPConnectionPool") def test_default_pool_size_from_env_unix(self, mock_obj): @@ -219,7 +220,8 @@ def test_default_pool_size_from_env_win(self, mock_obj): ) @pytest.mark.skipif( - IS_WINDOWS_PLATFORM, reason='Unix Connection Pool only on Linux' + os.environ.get('DOCKER_HOST', '').startswith('tcp://') or IS_WINDOWS_PLATFORM, + reason='Requires a Unix socket' ) @mock.patch("docker.transport.unixconn.UnixHTTPConnectionPool") def test_pool_size_from_env_unix(self, mock_obj):