Merge branch 'release/1.1.0'

Author: mblaschke

CHANGELOG.md

@@ -2,6 +2,15 @@
 All notable changes to this project will be documented in this file.
 This project adheres to [WebDevOps.io Dockerfile](https://github.com/webdevops/Dockerfile).
 
+## [1.1.0] - 2016-12-14
+- Fixed dnsmasq startup
+- Removed all logfiles inside containers (using stdout)
+- Fixed syslog-ng setup (was complaining about version)
+- Fixed some php/hhvm tests
+- Improve bin/console
+- Add cleanup after container installation
+- Add multiple vhost support for dns lookup (VIRTUAL_HOST)
+
 ## [1.0.0] - 2016-11-28
 - Introduced python based processing script
 - Introduced testinfra test suite

bin/command/docker_build_command.py

@@ -28,6 +28,7 @@ class DockerBuildCommand(DoitCommand):
     Build images
 
     docker:build
+        {docker images?*         : Docker images (whitelist)}
         {--dry-run               : show only which images will be build}
         {--no-cache              : build without caching}
         {--t|threads=0           : threads}

bin/command/docker_exec_command.py

@@ -0,0 +1,93 @@
+#!/usr/bin/env/python
+# -*- coding: utf-8 -*-
+#
+# (c) 2016 WebDevOps.io
+#
+# This file is part of Dockerfile Repository.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+# documentation files (the "Software"), to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and
+# to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions
+# of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO
+# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS
+# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+from cleo import Output
+from termcolor import colored
+from webdevops.command import DoitCommand
+from webdevops.taskloader import DockerPushTaskLoader
+from webdevops import Command, DockerfileUtility
+
+class DockerExecCommand(DoitCommand):
+    """
+    Push images to registry/hub
+
+    docker:exec
+        {docker command*         : Command}
+        {--dry-run               : show only which images will be build}
+        {--whitelist=?*          : image/tag whitelist }
+        {--blacklist=?*          : image/tag blacklist }
+    """
+
+    def run_task(self, configuration):
+        dockerfile_list = DockerfileUtility.find_dockerfiles_in_path(
+            base_path=configuration.get('dockerPath'),
+            path_regex=configuration.get('docker.pathRegex'),
+            image_prefix=configuration.get('docker.imagePrefix'),
+            whitelist=configuration.get('whitelist'),
+            blacklist=configuration.get('blacklist'),
+        )
+
+        image_fail_list = []
+
+        docker_command = self.argument('docker command')
+
+        for dockerfile in dockerfile_list:
+            title = dockerfile['image']['fullname']
+
+            print title
+            print '~' * len(title)
+
+            if configuration['dryRun']:
+                print '  exec: %s' % (docker_command)
+            else:
+
+                cmd = [
+                    'docker',
+                    'run',
+                    '-t',
+                    '--rm',
+                    dockerfile['image']['fullname'],
+                    'sh -c "%s"' % (' '.join(docker_command))
+                ]
+
+                status = Command.execute(cmd)
+
+                if status:
+                    print colored(' -> successfull', 'green')
+                else:
+                    print colored(' -> failed', 'red')
+                    image_fail_list.append(dockerfile['image']['fullname'])
+            print ''
+            print ''
+
+        if len(image_fail_list) >= 1:
+            print ''
+            print colored(' => failed images (%s):' % (str(len(image_fail_list))), 'red')
+            for image in image_fail_list:
+                print '   - %s ' % image
+            print ''
+
+            return False
+        else:
+            return True
+
+
+
+

bin/command/docker_pull_command.py

@@ -27,6 +27,7 @@ class DockerPullCommand(DoitCommand):
     Pull all built images from registry/hub
 
     docker:pull
+        {docker images?*         : Docker images (whitelist)}
         {--dry-run               : show only which images will be build}
         {--t|threads=0           : threads}
         {--r|retry=0             : retry}

bin/command/docker_push_command.py

@@ -27,6 +27,7 @@ class DockerPushCommand(DoitCommand):
     Push images to registry/hub
 
     docker:push
+        {docker images?*         : Docker images (whitelist)}
         {--dry-run               : show only which images will be build}
         {--t|threads=0           : threads}
         {--r|retry=0             : retry}

bin/command/generate_dockerfile_command.py

@@ -29,6 +29,7 @@ class GenerateDockerfileCommand(BaseCommand):
     Build Dockerfile containers
 
     generate:dockerfile
+        {docker images?*         : Docker images (whitelist)}
         {--whitelist=?*          : image/tag whitelist }
         {--blacklist=?*          : image/tag blacklist }
     """

bin/command/test_serverspec_command.py

@@ -29,6 +29,7 @@ class TestServerspecCommand(DoitCommand):
     Test docker images with Serverspec
 
     test:serverspec
+        {docker images?*         : Docker images (whitelist)}
         {--dry-run               : show only which images will be build}
         {--t|threads=0           : threads}
         {--r|retry=0             : retry}

bin/command/test_testinfra_command.py

@@ -27,6 +27,7 @@ class TestTestinfraCommand(DoitCommand):
     Test docker images with Testinfra
 
     test:testinfra
+        {docker images?*         : Docker images (whitelist)}
         {--dry-run               : show only which images will be build}
         {--t|threads=0           : threads}
         {--whitelist=?*          : image/tag whitelist }

bin/console

@@ -20,6 +20,7 @@ from webdevops.docker.DockerCliClient import DockerCliClient
 from command.docker_build_command import DockerBuildCommand
 from command.docker_push_command import DockerPushCommand
 from command.docker_pull_command import DockerPullCommand
+from command.docker_exec_command import DockerExecCommand
 from command.test_testinfra_command import TestTestinfraCommand
 from command.test_serverspec_command import TestServerspecCommand
 from command.generate_dockerfile_command import GenerateDockerfileCommand
@@ -88,6 +89,7 @@ if __name__ == '__main__':
     application.add(DockerBuildCommand(configuration=configuration))
     application.add(DockerPushCommand(configuration=configuration))
     application.add(DockerPullCommand(configuration=configuration))
+    application.add(DockerExecCommand(configuration=configuration))
     application.add(TestTestinfraCommand(configuration=configuration))
     application.add(TestServerspecCommand(configuration=configuration))
     application.add(GenerateDockerfileCommand(configuration=configuration))

bin/webdevops/DockerfileUtility.py

@@ -53,8 +53,13 @@ def find_file_in_path(dockerfile_path, filename="Dockerfile", whitelist=False, b
 
     # filter by whitelist
     if whitelist:
-        for term in whitelist:
-            file_list = filter(lambda x: term in x, file_list)
+        tmp = []
+        for file in file_list:
+            for term in whitelist:
+                if term in file:
+                    tmp.append(file)
+                    break
+        file_list = tmp
 
     if blacklist:
         for term in blacklist:
@@ -108,12 +113,11 @@ def parse_docker_info_from_path(path):
             }
             ret.append(dockerfile)
 
-    if whitelist or blacklist:
-        ret = filter_dockerfile(
-            dockerfile_list=ret,
-            whitelist=whitelist,
-            blacklist = blacklist
-        )
+    ret = filter_dockerfile(
+        dockerfile_list=ret,
+        whitelist=whitelist,
+        blacklist = blacklist
+    )
 
     return ret
 
@@ -123,8 +127,13 @@ def filter_dockerfile(dockerfile_list, whitelist=False, blacklist=False):
     Filter Dockerfiles by white- and blacklist
     """
     if whitelist:
-        for term in whitelist:
-            dockerfile_list = filter(lambda x: term in x['image']['fullname'], dockerfile_list)
+        tmp = []
+        for dockerfile in dockerfile_list:
+            for term in whitelist:
+                if term in dockerfile['image']['fullname']:
+                    tmp.append(dockerfile)
+                    break
+        dockerfile_list = tmp
 
     if blacklist:
         for term in blacklist:

bin/webdevops/command/BaseCommand.py

@@ -183,7 +183,17 @@ def get_whitelist(self):
         """
         Get whitelist
         """
-        return list(self.option('whitelist'))
+
+        # add whitelist from --whitelist
+        ret = list(self.option('whitelist'))
+
+        # add whitelist from argument list
+        try:
+            ret.extend(self.argument('docker images'))
+        except (Exception):
+            pass
+
+        return ret
 
     def get_blacklist(self):
         """
@@ -211,13 +221,13 @@ def get_threads(self):
             # use configuration value
             threads = self.configuration.get('threads')
 
-        match = re.match('auto(([-*+/])([0-9]+))?', str(threads))
+        match = re.match('auto(([-*+/])([0-9]+\.?[0-9]?))?', str(threads))
         if match is not None:
             ret = multiprocessing.cpu_count()
 
             if match.group(2) and match.group(3):
                 math_sign = match.group(2)
-                math_value = int(match.group(3))
+                math_value = float(match.group(3))
 
                 if math_sign == "*":
                     ret = int(ret * math_value)

bin/webdevops/docker/DockerCliClient.py

@@ -18,13 +18,10 @@
 # OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
 # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
-import subprocess
-import os
-import tempfile
+import os, subprocess, tempfile
 from .DockerBaseClient import DockerBaseClient
 from webdevops import Command
 
-
 class DockerCliClient(DockerBaseClient):
 
     def pull_image(self, name, tag):

bin/webdevops/docker/DockerPyClient.py

@@ -18,7 +18,6 @@
 # OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
 # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
-
 import os
 from .DockerBaseClient import DockerBaseClient
 

bin/webdevops/doit/DoitReporter.py

@@ -18,13 +18,10 @@
 # OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
 # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
-import sys
-import os
-import time
-import datetime
-import StringIO
+import os, sys, time, datetime, StringIO
 import termcolor
 from termcolor import colored
+from ..taskloader.BaseTaskLoader import BaseTaskLoader
 
 class TaskResult(object):
     """
@@ -141,7 +138,7 @@ def add_failure(self, task, exception):
 
         if task.actions and (task.name[0] != '_'):
             duration = self.duration(self.t_results[task.name].elapsed)
-            self.write(colored('.  %s FAILED (%s)\n' % (task.title(), duration), 'red'))
+            self.writeln(colored('.  %s FAILED (%s)' % (BaseTaskLoader.human_task_name(task.title()), duration), 'red'))
         self.failures.append({'task': task, 'exception': exception})
 
     def add_success(self, task):
@@ -152,7 +149,7 @@ def add_success(self, task):
 
         if task.actions and (task.name[0] != '_'):
             duration = self.duration(self.t_results[task.name].elapsed)
-            self.write(colored('.  %s finished (%s)\n' % (task.title(), duration), 'green'))
+            self.writeln(colored('.  %s finished (%s)' % (BaseTaskLoader.human_task_name(task.title()), duration), 'green'))
 
     def skip_uptodate(self, task):
         """
@@ -252,7 +249,7 @@ def task_stdout(self, title, duration=False, stdout=False, stderr=False, error=F
         if duration:
             text_duration = ' (%s)' % self.duration(duration)
 
-        title_full = 'Task %s%s:' % (title, text_duration)
+        title_full = 'Task %s%s:' % (BaseTaskLoader.human_task_name(title), text_duration)
 
         self.writeln(title_full)
         self.writeln('~' * len(title_full))
@@ -277,7 +274,7 @@ def task_stdout(self, title, duration=False, stdout=False, stderr=False, error=F
             self.write('%s' % exception.get_msg())
 
         self.writeln()
-        self.writeln(':: end of output "%s"' % title)
+        self.writeln(':: end of output "%s"' % (BaseTaskLoader.human_task_name(title)))
         self.writeln()
 
     def duration(self, duration):

bin/webdevops/taskloader/BaseDockerTaskLoader.py

@@ -18,10 +18,7 @@
 # OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
 # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
-import os
-import sys
-import re
-import copy
+import os, sys, re, copy
 from .BaseTaskLoader import BaseTaskLoader
 from webdevops import DockerfileUtility
 from doit.task import dict_to_task

bin/webdevops/taskloader/BaseTaskLoader.py

@@ -18,10 +18,7 @@
 # OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
 # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
-import sys
-import re
-import time
-import StringIO
+import sys, re, time, StringIO
 from webdevops import DockerfileUtility
 from doit.cmd_base import TaskLoader
 from doit.task import dict_to_task
@@ -44,6 +41,9 @@ def process_tasklist(self, tasklist):
         ret = []
         for task in tasklist:
             ret.append(dict_to_task(task))
+
+        print 'Starting execution of %s tasks...' % (len(ret))
+
         return ret
 
 
@@ -52,7 +52,12 @@ def human_task_name(name):
         """
         Translate internal task name to human readable name
         """
-        return re.search('^.*\|(.*)', name).group(1)
+        res = re.search('^.*\|(.*)', name)
+
+        if res:
+            return re.search('^.*\|(.*)', name).group(1)
+        else:
+            return name
 
 
     @staticmethod

bin/webdevops/taskloader/DockerBuildTaskLoader.py

@@ -18,11 +18,7 @@
 # OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
 # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
-import os
-import sys
-import re
-import copy
-import time
+import os, sys, re, copy, time
 from .BaseTaskLoader import BaseTaskLoader
 from .BaseDockerTaskLoader import BaseDockerTaskLoader
 from webdevops import DockerfileUtility