mx_gate.py

#
# ----------------------------------------------------------------------------------------------------
#
# Copyright (c) 2007, 2015, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License version 2 only, as
# published by the Free Software Foundation.
#
# This code is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
# version 2 for more details (a copy is included in the LICENSE file that
# accompanied this code).
#
# You should have received a copy of the GNU General Public License version
# 2 along with this work; if not, write to the Free Software Foundation,
# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
# or visit www.oracle.com if you need additional information or have any
# questions.
#
# ----------------------------------------------------------------------------------------------------

from __future__ import print_function

import os, re, time, datetime, json
import tempfile
import pipes
import zipfile
from os.path import join, exists
from argparse import ArgumentParser

import mx
import mx_javacompliance
import sys
from mx_urlrewrites import rewriteurl


"""
Predefined Task tags.
"""
class Tags:
    always = 'always'       # special tag that is always implicitly selected
    style = 'style'         # code style checks (without build)
    build = 'build'         # build
    ecjbuild = 'ecjbuild'   # build with ecj only
    fullbuild = 'fullbuild' # full build (including warnings, spotbugs and ide init)

"""
Context manager for a single gate task that can prevent the
task from executing or time and log its execution.
"""
class Task:
    # A list of strings. If not empty, only tasks whose title
    # matches at least one of the substrings in this list will return
    # a non-None value from __enter__. The body of a 'with Task(...) as t'
    # statement should check 't' and exit immediately if it is None.
    filters = []
    log = True # whether to log task messages
    dryRun = False
    startAtFilter = None
    filtersExclude = False

    tags = None
    tagsExclude = False
    # map from tag to a pair [from(inclusive), to(exclusive)]
    tags_range = dict()
    # map from tag to count
    tags_count = dict()

    verbose = False
    startTime = None

    def tag_matches(self, _tags):
        for t in _tags:
            assert isinstance(t, str), '{} is not a string and thus not a valid tag'.format(t)
            if Task.tags is not None and t in Task.tags: # pylint: disable=unsupported-membership-test
                if t not in Task.tags_range:
                    # no range restriction
                    return True
                else:
                    frm, to = Task.tags_range[t]
                    cnt = Task.tags_count[t]
                    # increment counter
                    Task.tags_count[t] += 1
                    if frm <= cnt < to:
                        return True
        return False

    def _timestamp(self, suffix):
        stamp = time.strftime('gate: %d %b %Y %H:%M:%S')
        if Task.startTime:
            duration = datetime.timedelta(seconds=time.time() - Task.startTime)
            # Strip microseconds and convert to a string
            duration = str(duration - datetime.timedelta(microseconds=duration.microseconds))
            # Strip hours if 0
            if duration.startswith('0:'):
                duration = duration[2:]
            stamp += '(+{})'.format(duration)
        return stamp + suffix

    def __init__(self, title, tasks=None, disableJacoco=False, tags=None, legacyTitles=None):
        self.tasks = tasks
        self.title = title
        self.legacyTitles = legacyTitles or []
        self.skipped = False
        self.tags = tags
        if tasks is not None:
            for t in tasks:
                if t.title == title:
                    mx.abort('Gate task with title "' + title + '" is already defined')

            if Task.startAtFilter:
                assert not Task.filters
                if Task.startAtFilter in title:
                    self.skipped = False
                    Task.startAtFilter = None
                else:
                    self.skipped = True
            elif len(Task.filters) > 0:
                titles = [self.title] + self.legacyTitles
                if Task.filtersExclude:
                    self.skipped = any([f in t for t in titles for f in Task.filters])
                else:
                    self.skipped = not any([f in t for t in titles for f in Task.filters])
            if Task.tags is not None:
                if Task.tagsExclude:
                    self.skipped = all([t in Task.tags for t in self.tags]) if tags else False # pylint: disable=unsupported-membership-test
                else:
                    _tags = self.tags if self.tags else []
                    self.skipped = not self.tag_matches(_tags)
        if not self.skipped:
            self.start = time.time()
            self.end = None
            self.duration = None
            self.disableJacoco = disableJacoco
            if Task.log:
                mx.log(self._timestamp(' BEGIN: ') + title)
    def __enter__(self):
        assert self.tasks is not None, "using Task with 'with' statement requires to pass the tasks list in the constructor"
        if self.skipped:
            return None
        if self.disableJacoco:
            self.jacacoSave = _jacoco
        if Task.dryRun:
            return None
        return self
    def __exit__(self, exc_type, exc_value, traceback):
        if not self.skipped:
            self.tasks.append(self.stop())
            if self.disableJacoco:
                global _jacoco
                _jacoco = self.jacacoSave

    @staticmethod
    def _human_fmt(num):
        for unit in ['', 'K', 'M', 'G']:
            if abs(num) < 1024.0:
                return "%3.1f%sB" % (num, unit)
            num /= 1024.0
        return "%.1fTB" % (num)

    @staticmethod
    def _diskstats():
        if hasattr(os, 'statvfs'):
            _, f_frsize, f_blocks, _, f_bavail, _, _, _, _, _ = os.statvfs(os.getcwd())
            total = f_frsize * f_blocks
            free = f_frsize * f_bavail
            return ' [disk (free/total): {}/{}]'.format(Task._human_fmt(free), Task._human_fmt(total))
        return ''

    def stop(self):
        if Task.log:
            self.end = time.time()
            self.duration = datetime.timedelta(seconds=self.end - self.start)
            mx.log(self._timestamp(' END:   ') + self.title + ' [' + str(self.duration) + ']' + Task._diskstats())
        return self
    def abort(self, codeOrMessage):
        if Task.log:
            self.end = time.time()
            self.duration = datetime.timedelta(seconds=self.end - self.start)
            mx.log(self._timestamp(' ABORT: ') + self.title + ' [' + str(self.duration) + ']' + Task._diskstats())
            mx.abort(codeOrMessage)
        return self

    def __repr__(self):
        return "Task: " + self.title

_gate_runners = []
_pre_gate_runners = []
_extra_gate_arguments = []
_mx_args = []
_mx_command_and_args = []

def add_gate_argument(*args, **kwargs):
    """
    Adds an argument declaration to the ArgumentParser used by the gate method.
    """
    _extra_gate_arguments.append((args, kwargs))

def add_gate_runner(suite, runner):
    """
    Adds a gate runner function for a given suite to be called by the gate once common gate tasks
    have been executed. The 'runner' function is called with these arguments:
      args: the argparse.Namespace object containing result of parsing gate command line
      tasks: list of Task to which extra Tasks should be added
    """
    suiteRunner = (suite, runner)
    _gate_runners.append(suiteRunner)

def prepend_gate_runner(suite, runner):
    """
    Prepends a gate runner function for a given suite to be called by the gate before common gate tasks
    are executed. The 'runner' function is called with these arguments:
      args: the argparse.Namespace object containing result of parsing gate command line
      tasks: list of Task to which extra Tasks should be added
    """
    suiteRunner = (suite, runner)
    _pre_gate_runners.append(suiteRunner)

def add_omit_clean_args(parser):
    parser.add_argument('-j', '--omit-java-clean', action='store_false', dest='cleanJava', help='omit cleaning Java native code')
    parser.add_argument('-n', '--omit-native-clean', action='store_false', dest='cleanNative', help='omit cleaning and building native code')
    parser.add_argument('-e', '--omit-ide-clean', action='store_false', dest='cleanIDE', help='omit ideclean/ideinit')
    parser.add_argument('-d', '--omit-dist-clean', action='store_false', dest='cleanDist', help='omit cleaning distributions')
    parser.add_argument('--omit-clean-all', action='store_false', dest='cleanAll', help='omit cleaning non-default build targets')
    parser.add_argument('-o', '--omit-clean', action='store_true', dest='noClean', help='equivalent to -j -n -e')

def gate_clean(cleanArgs, tasks, name='Clean', tags=None):
    with Task(name, tasks, tags=tags) as t:
        if t:
            mx.command_function('clean')(cleanArgs)

def check_gate_noclean_arg(args):
    '''
    Checks the -o option (noClean) and sets the sub-options in args appropriately
    and returns the relevant args for the clean command (N.B. IDE currently ignored).
    '''
    if args.noClean:
        args.cleanIDE = False
        args.cleanJava = False
        args.cleanNative = False
        args.cleanDist = False
        args.cleanAll = False
    cleanArgs = []
    if not args.cleanNative:
        cleanArgs.append('--no-native')
    if not args.cleanJava:
        cleanArgs.append('--no-java')
    if not args.cleanDist:
        cleanArgs.append('--no-dist')
    if args.cleanAll:
        cleanArgs.append('--all')
    return cleanArgs

def _warn_or_abort(msg, strict_mode):
    reporter = mx.abort if strict_mode else mx.warn
    reporter(msg)


def parse_tags_argument(tags_arg, exclude):
    pattern = re.compile(r"^(?P<tag>[^:]*)(?::(?P<from>\d+):(?P<to>\d+)?)?$")
    tags = tags_arg.split(',')
    Task.tags = []
    for tag_spec in tags:
        m = pattern.match(tag_spec)
        if not m:
            mx.abort('--tags option requires the format `name[:from:[to]]`: {0}'.format(tag_spec))
        (tag, t_from, t_to) = m.groups()
        if t_from:
            if exclude:
                mx.abort('-x option cannot be used tag ranges: {0}'.format(tag_spec))
            frm = int(t_from)
            to = int(t_to) if t_to else sys.maxsize
            # insert range tuple
            Task.tags_range[tag] = (frm, to)
            # sanity check
            if to <= frm:
                mx.abort('`from` must be less than `to` for tag ranges: {0}'.format(tag_spec))
            # init counter
            Task.tags_count[tag] = 0
        Task.tags.append(tag)
_command_level = 0
def gate(args):
    """run the tests used to validate a push

    If this command exits with a 0 exit code, then the gate passed."""

    parser = ArgumentParser(prog='mx gate')
    add_omit_clean_args(parser)
    parser.add_argument('--all-suites', action='store_true', help='run gate tasks for all suites, not just the primary suite')
    parser.add_argument('--dry-run', action='store_true', help='just show the tasks that will be run without running them')
    parser.add_argument('-x', action='store_true', help='makes --task-filter or --tags an exclusion instead of inclusion filter')
    jacoco = parser.add_mutually_exclusive_group()
    jacoco.add_argument('--jacocout', help='specify the output directory for jacoco report')
    jacoco.add_argument('--jacoco-zip', help='specify the output zip file for jacoco report')
    parser.add_argument('--jacoco-omit-excluded', action='store_true', help='omit excluded files from jacoco report')
    parser.add_argument('--strict-mode', action='store_true', help='abort if a task cannot be executed due to missing tool configuration')
    parser.add_argument('--no-warning-as-error', action='store_true', help='compile warnings are not treated as errors')
    parser.add_argument('-B', dest='extra_build_args', action='append', metavar='<build_args>', help='append additional arguments to mx build commands used in the gate')
    parser.add_argument('-p', '--partial', help='run only a subset of the tasks in the gate (index/total). Eg. "--partial 2/5" runs the second fifth of the tasks in the gate. Tasks with tag build are repeated for each run.')
    filtering = parser.add_mutually_exclusive_group()
    filtering.add_argument('-t', '--task-filter', help='comma separated list of substrings to select subset of tasks to be run')
    filtering.add_argument('-s', '--start-at', help='substring to select starting task')
    filtering.add_argument('--tags', help='comma separated list of tags to select subset of tasks to be run. Tags can have a range specifier `name[:from:[to]]`.'
                           'If present only the [from,to) tasks are executed. If `to` is omitted all tasks starting with `from` are executed.')

    for a, k in _extra_gate_arguments:
        parser.add_argument(*a, **k)
    args = parser.parse_args(args)
    cleanArgs = check_gate_noclean_arg(args)

    if args.dry_run:
        Task.dryRun = True
    if args.start_at:
        Task.startAtFilter = args.start_at
    elif args.task_filter:
        Task.filters = args.task_filter.split(',')
        Task.filtersExclude = args.x
    elif args.tags:
        parse_tags_argument(args.tags, args.x)
        Task.tagsExclude = args.x
        if not Task.tagsExclude:
            # implicitly include 'always'
            Task.tags += [Tags.always]
    elif args.x:
        mx.abort('-x option cannot be used without --task-filter or the --tags option')

    if args.jacoco_zip:
        args.jacocout = 'html'

    if not args.extra_build_args:
        args.extra_build_args = []

    if args.partial:
        partialArgs = args.partial.split('/')
        if len(partialArgs) != 2:
            mx.abort('invalid partial argument specified')

        selected = int(partialArgs[0]) - 1
        total = int(partialArgs[1])
        if selected < 0 or selected >= total:
            mx.abort('out of bounds partial argument specified')

        tasks = _collect_tasks(cleanArgs, args)

        # build and always tags must be run by every partial gate run
        alwaysTags = [Tags.always, Tags.build]
        buildTasks = [task for task in tasks if not task.skipped and any([f in t for t in alwaysTags for f in task.tags])]
        nonBuildTasks = [task for task in tasks if not task.skipped and not any([f in t for t in alwaysTags for f in task.tags])]

        partialTasks = nonBuildTasks[selected::total]
        runTaskNames = [task.title for task in buildTasks + partialTasks]

        # we have already ran the filters in the dry run when collecting
        # so we can safely overwrite other filter settings.
        Task.filters = runTaskNames
        Task.filtersExclude = False

        mx.log('Running gate with partial tasks ' + args.partial + ". " + str(len(partialTasks)) + " out of " + str(len(nonBuildTasks)) + " non-build tasks selected.")
        if len(partialTasks) == 0:
            mx.log('No partial tasks left to run. Finishing gate early.')
            return

    Task.startTime = time.time()
    tasks = []
    total = Task('Gate')
    all_commands = []

    def shell_quoted_args(args):
        args_string = ' '.join([pipes.quote(str(arg)) for arg in args])
        if args_string != '':
            args_string = ' ' + args_string
        return args_string

    def mx_command_entered(command, *args, **kwargs):
        global _command_level
        if _command_level == 0:
            all_commands.append((command.command, args, kwargs))
            mx.log(mx.colorize('Running: ' + command_in_gate_message(command.command, args, kwargs), color='blue'))
        _command_level = _command_level + 1

    def mx_command_left(_, *__, **___):
        global _command_level
        assert _command_level >= 0
        _command_level = _command_level - 1

    def print_commands_on_failure():
        sys.stdout.flush()
        sys.stderr.flush()

        mx.log_error('\nThe sequence of mx commands that were executed until the failure follows:\n')
        for command, command_args, kwargs in all_commands:
            mx.log_error(command_in_gate_message(command, command_args, kwargs))

        mx.log_error('\nIf the previous sequence is incomplete or some commands were executed programmatically use:\n')
        mx.log_error('mx' + shell_quoted_args(_mx_args + _mx_command_and_args) + '\n')

        sys.stderr.flush()

    def command_in_gate_message(command, command_args, kwargs):
        one_list = len(command_args) == 1 and isinstance(command_args[0], list)
        kwargs_absent = len(kwargs) == 0
        if one_list and kwargs_absent:  # gate command reproducible on the command line
            quoted_args = (' '.join([pipes.quote(str(arg)) for arg in command_args[0]]))
            message = 'mx' + shell_quoted_args(_mx_args) + ' ' + command + ' ' + quoted_args
        else:
            args_message = '(Programmatically executed. '
            if not one_list:
                args_message += 'Args: ' + str(command_args)
            if not kwargs_absent:
                args_message += 'Kwargs: ' + str(kwargs)
            args_message += ')'
            message = 'mx' + shell_quoted_args(_mx_args) + ' ' + command + ' ' + args_message
        return message

    try:
        mx._mx_commands.add_command_callback(mx_command_entered, mx_command_left)
        _run_gate(cleanArgs, args, tasks)
    except KeyboardInterrupt:
        total.abort(1)
    except BaseException as e:
        import traceback
        traceback.print_exc()
        print_commands_on_failure()
        total.abort(str(e))
    finally:
        mx._mx_commands.remove_command_callback(mx_command_entered, mx_command_left)

    total.stop()

    mx.log('Gate task times:')
    for t in tasks:
        mx.log('  ' + str(t.duration) + '\t' + t.title + ("" if not (Task.verbose and t.tags) else (' [' + ','.join(t.tags) + ']')))
    mx.log('  =======')
    mx.log('  ' + str(total.duration))

    if args.task_filter:
        Task.filters = []

def _collect_tasks(cleanArgs, args):
    prevDryRun = Task.dryRun
    prevLog = Task.log
    Task.dryRun = True
    Task.log = False
    tasks = []
    try:
        _run_gate(cleanArgs, args, tasks)
    finally:
        Task.dryRun = prevDryRun
        Task.log = prevLog
    return tasks

def _run_mx_suite_tests():
    """
    Mx suite specific tests.
    """
    mx_javacompliance._test()

def _run_gate(cleanArgs, args, tasks):
    global _jacoco
    with Task('Versions', tasks, tags=[Tags.always]) as t:
        if t:
            mx.command_function('version')(['--oneline'])
            mx.command_function('sversions')([])
            mx.log("Python version: {}".format(sys.version_info))

    with Task('JDKReleaseInfo', tasks, tags=[Tags.always]) as t:
        if t:
            jdkDirs = os.pathsep.join([mx.get_env('JAVA_HOME', ''), mx.get_env('EXTRA_JAVA_HOMES', '')])
            for jdkDir in jdkDirs.split(os.pathsep):
                release = join(jdkDir, 'release')
                if exists(release):
                    mx.log('==== ' + jdkDir + ' ====')
                    with open(release) as fp:
                        mx.log(fp.read().strip())

    if mx.primary_suite() is mx._mx_suite:
        _run_mx_suite_tests()

    with Task('VerifyMultiReleaseProjects', tasks, tags=[Tags.always]) as t:
        if t:
            mx.command_function('verifymultireleaseprojects')([])

    for suiteRunner in _pre_gate_runners:
        suite, runner = suiteRunner
        if args.all_suites or suite is mx.primary_suite():
            runner(args, tasks)

    with Task('Pylint', tasks, tags=[Tags.style]) as t:
        if t:
            if mx.command_function('pylint')(['--primary']) != 0:
                _warn_or_abort('Pylint not configured correctly. Cannot execute Pylint task.', args.strict_mode)

    gate_clean(cleanArgs, tasks, tags=[Tags.build, Tags.fullbuild, Tags.ecjbuild])

    with Task('Distribution Overlap Check', tasks, tags=[Tags.style]) as t:
        if t:
            if mx.command_function('checkoverlap')([]) != 0:
                t.abort('Found overlapping distributions.')

    with Task('Canonicalization Check', tasks, tags=[Tags.style]) as t:
        if t:
            mx.log(time.strftime('%d %b %Y %H:%M:%S - Ensuring mx/projects files are canonicalized...'))
            if mx.command_function('canonicalizeprojects')([]) != 0:
                t.abort('Rerun "mx canonicalizeprojects" and modify the suite.py files as suggested.')

    with Task('Verify Java Sources in Project', tasks, tags=[Tags.style]) as t:
        if t:
            mx.log(time.strftime('%d %b %Y %H:%M:%S - Ensuring all Java sources are in a Java project directory...'))
            if mx.command_function('verifysourceinproject')([]) != 0:
                t.abort('Move or delete the Java sources that are not in a Java project directory.')

    if mx._is_supported_by_jdt(mx.DEFAULT_JDK_TAG):
        with Task('BuildWithEcj', tasks, tags=[Tags.fullbuild, Tags.ecjbuild], legacyTitles=['BuildJavaWithEcj']) as t:
            if t:
                defaultBuildArgs = ['-p']
                fullbuild = True if Task.tags is None else Tags.fullbuild in Task.tags # pylint: disable=unsupported-membership-test
                # Using ecj alone is not compatible with --warning-as-error (see GR-3969)
                if not args.no_warning_as_error and fullbuild:
                    defaultBuildArgs += ['--warning-as-error']
                if mx.get_env('JDT'):
                    mx.command_function('build')(defaultBuildArgs + args.extra_build_args)
                    if fullbuild:
                        gate_clean(cleanArgs, tasks, name='CleanAfterEcjBuild', tags=[Tags.fullbuild])
                else:
                    _warn_or_abort('JDT environment variable not set. Cannot execute BuildWithEcj task.', args.strict_mode)

    with Task('BuildWithJavac', tasks, tags=[Tags.build, Tags.fullbuild], legacyTitles=['BuildJavaWithJavac']) as t:
        if t:
            defaultBuildArgs = ['-p']
            if not args.no_warning_as_error:
                defaultBuildArgs += ['--warning-as-error']
            mx.command_function('build')(defaultBuildArgs + ['--force-javac'] + args.extra_build_args)

    with Task('IDEConfigCheck', tasks, tags=[Tags.fullbuild]) as t:
        if t:
            if args.cleanIDE:
                mx.command_function('ideclean')([])
                mx.command_function('ideinit')([])

    with Task('CodeFormatCheck', tasks, tags=[Tags.style]) as t:
        if t:
            eclipse_exe = mx.get_env('ECLIPSE_EXE')
            if eclipse_exe is not None:
                if mx.command_function('eclipseformat')(['-e', eclipse_exe, '--primary']) != 0:
                    t.abort('Formatter modified files - run "mx eclipseformat", check in changes and repush')
            else:
                _warn_or_abort('ECLIPSE_EXE environment variable not set. Cannot execute CodeFormatCheck task.', args.strict_mode)

    with Task('Checkstyle', tasks, tags=[Tags.style]) as t:
        if t and mx.command_function('checkstyle')(['--primary']) != 0:
            t.abort('Checkstyle warnings were found')

    with Task('SpotBugs', tasks, tags=[Tags.fullbuild]) as t:
        if t and mx.command_function('spotbugs')([]) != 0:
            t.abort('FindBugs warnings were found')

    with Task('VerifyLibraryURLs', tasks, tags=[Tags.fullbuild]) as t:
        if t:
            mx.command_function('verifylibraryurls')([])

    jacoco_exec = JACOCO_EXEC
    if exists(jacoco_exec):
        os.unlink(jacoco_exec)

    if args.jacocout is not None:
        _jacoco = 'append'
    else:
        _jacoco = 'off'

    for suiteRunner in _gate_runners:
        suite, runner = suiteRunner
        if args.all_suites or suite is mx.primary_suite():
            runner(args, tasks)

    if args.jacocout is not None:
        jacoco_args = [args.jacocout]
        if args.jacoco_omit_excluded:
            jacoco_args = ['--omit-excluded'] + jacoco_args
        mx.command_function('jacocoreport')(jacoco_args)
        _jacoco = 'off'
    if args.jacoco_zip is not None:
        mx.log('Creating JaCoCo report archive: {}'.format(args.jacoco_zip))
        with zipfile.ZipFile(args.jacoco_zip, 'w', compression=zipfile.ZIP_DEFLATED) as zf:
            zf.write(jacoco_exec, join(args.jacocout, jacoco_exec))
            for root, _, files in os.walk(args.jacocout):
                for f in files:
                    zf.write(os.path.join(root, f))
        mx.log('Archiving done.')

def checkheaders(args):
    """check Java source headers against any required pattern"""
    mx.log('The checkheaders command is obsolete.  The checkstyle or checkcopyrights command performs\n'\
           'the required checks depending on the mx configuration.')
    return 0


JACOCO_EXEC = 'jacoco.exec'

_jacoco = 'off'

_jacoco_includes = []

def add_jacoco_includes(patterns):
    """
    Adds to the list of JaCoCo includes.
    """
    _jacoco_includes.extend(patterns)

_jacoco_excluded_annotations = ['@Test']

def add_jacoco_excluded_annotations(annotations):
    """
    Adds to the list of annotations which if present denote a class that should
    be excluded from JaCoCo analysis.
    """
    _jacoco_excluded_annotations.extend(annotations)

_jacoco_whitelisted_packages = []

def _jacoco_is_package_whitelisted(package):
    if not _jacoco_whitelisted_packages:
        return True
    return any(package.startswith(w) for w in _jacoco_whitelisted_packages)

def _jacoco_excludes_includes():
    includes = list(_jacoco_includes)
    baseExcludes = []
    for p in mx.projects():
        if p.isJavaProject():
            projsetting = getattr(p, 'jacoco', '')
            if not _jacoco_is_package_whitelisted(p.name):
                pass
            elif projsetting == 'exclude':
                baseExcludes.append(p.name)
            elif projsetting == 'include':
                includes.append(p.name + '.*')
    if _jacoco_whitelisted_packages:
        includes.extend((x + '.*' for x in _jacoco_whitelisted_packages))

    def _filter(l):
        # filter out specific classes which are already covered by a baseExclude package
        return [clazz for clazz in l if not any([clazz.startswith(package) for package in baseExcludes])]

    excludes = []
    for p in mx.projects():
        if p.isJavaProject() and p.name not in baseExcludes and _jacoco_is_package_whitelisted(p.name):
            excludes += _filter(
                p.find_classes_with_annotations(None, _jacoco_excluded_annotations, includeInnerClasses=True,
                                                includeGenSrc=True).keys())
            excludes += _filter(p.find_classes_with_matching_source_line(None, lambda line: 'JaCoCo Exclude' in line,
                                                                         includeInnerClasses=True,
                                                                         includeGenSrc=True).keys())
    excludes += [package + '.*' for package in baseExcludes]
    return excludes, includes

def get_jacoco_agent_path(resolve):
    return mx.library('JACOCOAGENT_0.8.4', True).get_path(resolve)

def get_jacoco_agent_args():
    '''
    Gets the args to be added to a VM command line for injecting the JaCoCo agent
    if use of JaCoCo has been requested otherwise returns None.
    '''
    if _jacoco in ('on', 'append'):
        excludes, includes = _jacoco_excludes_includes()
        agentOptions = {
                        'append' : 'true' if _jacoco == 'append' else 'false',
                        'inclbootstrapclasses' : 'true',
                        'includes' : ':'.join(includes),
                        'excludes' : ':'.join(excludes),
                        'destfile' : JACOCO_EXEC,
        }
        return ['-javaagent:' + get_jacoco_agent_path(True) + '=' + ','.join([k + '=' + v for k, v in agentOptions.items()])]
    return None

def jacocoreport(args):
    _jacocoreport(args)

def _jacocoreport(args):
    """create a JaCoCo coverage report

    Creates the report from the 'jacoco.exec' file in the current directory.
    Default output directory is 'coverage', but an alternative can be provided as an argument.

    This function returns the included projects and excludes used for this report"""

    dist_name = "MX_JACOCO_REPORT"
    mx.command_function("build")(['--dependencies', dist_name])
    dist = mx.distribution(dist_name)
    jdk = mx.get_jdk(dist.javaCompliance)

    parser = ArgumentParser(prog='mx jacocoreport')
    parser.add_argument('--format', help='Export format (HTML or XML)', default='html', choices=['html', 'xml'])
    parser.add_argument('--omit-excluded', action='store_true', help='omit excluded files from report')
    parser.add_argument('output_directory', help='Output directory', default='coverage', nargs='?')
    args = parser.parse_args(args)

    # list of strings of the form "project-dir:binary-dir"
    includedirs = []
    includedprojects = []
    for p in mx.projects():
        projsetting = getattr(p, 'jacoco', '')
        if projsetting in ('include', '') and _jacoco_is_package_whitelisted(p.name):
            if isinstance(p, mx.ClasspathDependency):
                if args.omit_excluded and p.is_test_project(): # skip test projects when omit-excluded
                    continue
                source_dirs = []
                if p.isJavaProject():
                    source_dirs += p.source_dirs() + [p.source_gen_dir()]
                includedirs.append(":".join([p.dir, p.classpath_repr(jdk)] + source_dirs))
                includedprojects.append(p.name)

    def _run_reporter(extra_args=None):
        mx.run_java(['-cp', mx.classpath([dist_name], jdk=jdk), '-jar', dist.path, '--in', JACOCO_EXEC, '--out',
                     args.output_directory, '--format', args.format] +
                    (extra_args or []) +
                    sorted(includedirs),
                    jdk=jdk, addDefaultArgs=False)

    if not args.omit_excluded:
        _run_reporter()
        excludes = []
    else:
        with tempfile.NamedTemporaryFile(suffix="jacoco-report-exclude", mode="w") as fp:
            excludes, _ = _jacoco_excludes_includes()
            fp.writelines((e + "\n" for e in excludes))
            fp.flush()
            _run_reporter(['--exclude-file', fp.name])
    return includedprojects, excludes


def _parse_java_properties(args):
    prop_re = re.compile('-D(?P<key>[^=]+)=(?P<value>.*)')
    remainder = []
    java_properties = {}
    for arg in args:
        m = prop_re.match(arg)
        if m:
            java_properties[m.group('key')] = m.group('value')
        else:
            remainder.append(arg)
    return java_properties, remainder


def _jacoco_excludes_includes_projects(limit_to_primary=False):
    includes = []
    excludes = []

    projects = mx.projects(limit_to_primary=limit_to_primary)
    for p in projects:
        if p.isJavaProject():
            projsetting = getattr(p, 'jacoco', '')
            if not _jacoco_is_package_whitelisted(p.name):
                excludes.append(p)
            elif projsetting == 'exclude':
                excludes.append(p)
            else:
                includes.append(p)
    return excludes, includes

def _jacoco_exclude_classes(projects):
    excludeClasses = {}

    for p in projects:
        r = p.find_classes_with_annotations(None, _jacoco_excluded_annotations, includeGenSrc=True)
        excludeClasses.update(r)
        r = p.find_classes_with_matching_source_line(None, lambda line: 'JaCoCo Exclude' in line, includeGenSrc=True)
        excludeClasses.update(r)
    return excludeClasses

def coverage_upload(args):
    parser = ArgumentParser(prog='mx coverage-upload')
    parser.add_argument('--upload-url', required=False, default=mx.get_env('COVERAGE_UPLOAD_URL'), help='Format is like rsync: user@host:/directory')
    parser.add_argument('--build-name', required=False, default=mx.get_env('BUILD_NAME'))
    parser.add_argument('--build-url', required=False, default=mx.get_env('BUILD_URL'))
    parser.add_argument('--build-number', required=False, default=mx.get_env('BUILD_NUMBER'))
    args, other_args = parser.parse_known_args(args)
    if not args.upload_url:
        parser.print_help()
        return
    remote_host, remote_basedir = args.upload_url.split(':')
    if not remote_host:
        mx.abort('Cannot determine remote host from {}'.format(args.upload_url))

    primary = mx.primary_suite()
    info = primary.vc.parent_info(primary.dir)
    rev = primary.vc.parent(primary.dir)
    if len(remote_basedir) > 0 and not remote_basedir.endswith('/'):
        remote_basedir += '/'
    remote_dir = '{}_{}_{}'.format(primary.name, datetime.datetime.fromtimestamp(info['author-ts']).strftime('%Y-%m-%d_%H_%M'), rev[:7])
    if args.build_name:
        remote_dir += '_' + args.build_name
    if args.build_number:
        remote_dir += '_' + args.build_number
    upload_dir = remote_basedir + remote_dir
    includes, excludes = _jacocoreport(['--omit-excluded'] + other_args)

    # Upload jar+sources
    coverage_sources = 'java_sources.tar.gz'
    coverage_binaries = 'java_binaries.tar.gz'

    with mx.Archiver(os.path.realpath(coverage_sources), kind='tgz') as sources, mx.Archiver(os.path.realpath(coverage_binaries), kind='tgz') as binaries:
        def _visit_deps(dep, edge):
            if dep.isJavaProject() and not dep.is_test_project():
                binaries.zf.add(dep.output_dir(), dep.name)
                for d in dep.source_dirs():
                    sources.zf.add(d, dep.name)
                if os.path.exists(dep.source_gen_dir()):
                    sources.zf.add(dep.source_gen_dir(), dep.name)
        mx.walk_deps(mx.projects(), visit=_visit_deps)

    files = [JACOCO_EXEC, 'coverage', coverage_sources, coverage_binaries]
    print("Syncing {} to {}:{}".format(" ".join(files), remote_host, upload_dir))
    mx.run([
        'bash',
        '-c',
        r'tar -czf - {files} | ssh {remote} bash -c \'"mkdir -p {remotedir} && cd {remotedir} && cat | tar -x{verbose}z && chmod -R 755 ."\''
            .format(
                files=" ".join(files),
                remote=remote_host,
                remotedir=upload_dir,
                verbose='v' if mx._opts.verbose else '')
    ])
    def upload_string(content, path):
        mx.run(['ssh', remote_host, 'bash', '-c', 'cat > "' + path + '"'], stdin=content)

    upload_string(json.dumps({
        'timestamp': time.time(),
        'suite': primary.name,
        'revision': rev,
        'directory': remote_dir,
        'build_name': args.build_name,
        'build_url': args.build_url,
        'jdk_version': str(mx.get_jdk().version),
        'build_number': args.build_number,
        'primary_info': info,
        'excludes': [str(e) for e in excludes],
        'includes': [str(i) for i in includes]}), upload_dir + '/description.json')
    mx.run(['ssh', remote_host, 'bash', '-c', r'"(echo \[; for i in {remote_basedir}/*/description.json; do if \[ -s \$i \];then cat \$i; echo ,; fi done; echo null\]) > {remote_basedir}/index.json"'.format(remote_basedir=remote_basedir)])
    upload_string("""<html>
<script language="javascript">
  function urlChange(url) {
    if (url.pathname !== "blank") {
      window.history.replaceState(null, null, url.pathname.replace("/coverage_upload/", "/coverage_upload/#"))
    }
  }
</script>
<frameset rows="40,*">
  <frame id="navigation" src="navigation.html"/>
  <frame id="content" src="" onload="urlChange(this.contentWindow.location);" />
</frameset>
</html>""", remote_basedir + '/index.html')
    js_library_url = rewriteurl("https://ajax.googleapis.com/ajax/libs/angularjs/1.7.7/angular.js")
    upload_string(r"""<html>
    <head>
        <script src="%js_library_url"></script>
        <script language="javascript">
        var App = angular.module('myApp', [])
            .controller('IndexCtrl', function IndexCtrl($scope, $http) {
                var hash = parent.window.location.hash;
                if(hash) {
                    hash = hash.substring(1, hash.length); // remove leading hash
                }
                $http.get('index.json').then(function(response, status) {
                    var data = response.data.filter(x => x != null);
                    /*
                        #GR-17399
                        Filter build that are unique per suite with revision as key and merge builds.
                    */
                    data = data
                        .filter(x => !x.hasOwnProperty('merge'))
                        .filter( // filter builds that are unique per suite with revision as key
                            x => !data
                                .filter(z => x != z && x.suite == z.suite) // exclude self build and build for other suites.
                                .map(z => z.revision) // map from array of build to array of revision
                                .includes(x.revision) // check if revision of x is index data.
                        ).concat(data.filter(x => x.hasOwnProperty('merge'))); // concat unique build with merged build.

                    data.sort((l,r) => r.timestamp - l.timestamp);
                    if(data.length > 0) {
                        var startdir;
                        if(hash) {
                            startdir = data.find(build => hash.includes(build.directory));
                            startdir.hash = hash;
                        }
                        if(!startdir) {
                            startdir = data[0];
                        }
                        $scope.directory = startdir;
                    }
                    $scope.data = data;
                });
                $scope.$watch('directory', (dir, olddir) => {
                    if(dir) {
                        var content = parent.document.getElementById("content");
                        var contentDocument = content.contentDocument || content.contentWindow.document;
                        var newpath;
                        if(olddir && olddir.suite === dir.suite) {
                            newpath = contentDocument.location.href.replace(olddir.directory, dir.directory);
                        } else {
                            newpath = dir.hasOwnProperty('hash') ? hash : dir.directory + "/coverage/";
                        }
                        contentDocument.location.href = newpath;
                        parent.window.history.replaceState(undefined, undefined, "#" + newpath.replace(/^.+coverage_upload\//, ""));
                    }
                });
                $scope.step = (i) => $scope.directory = $scope.data[$scope.data.indexOf($scope.directory)+i];
            });
        function copy(url) {
            var content = parent.document.getElementById("content");
            var contentDocument = content.contentDocument || content.contentWindow.document;
            var copyText = document.getElementById("copy");
            copyText.value = contentDocument.location.href.replace("coverage_upload/", "coverage_upload/#");
            copyText.select();
            document.execCommand("copy");
        }
        </script>
    </head>
    <body ng-app="myApp" ng-controller="IndexCtrl">
       <button ng-click="step(1)" ng-disabled="data.indexOf(directory) >= data.length-1">&lt;&lt;</button>
       <button ng-click="step(-1)" ng-disabled="data.indexOf(directory) <= 0">&gt;&gt;</button>
       <select ng-model="directory" ng-options="(i.primary_info['author-ts']*1000|date:'yy-MM-dd hh:mm') + ' ' + i.build_name + ' ' + i.revision.substr(0,8) group by i.suite for i in data"></select>
       <a href="{{directory.build_url}}" ng-if="directory.build_url" target="_blank">Build</a> Commit: {{directory.revision.substr(0,5)}}: {{directory.primary_info.description}}
       <input type="text" style="opacity: 0;width: 20;" id="copy" />
       <button style="float: right;" onclick="copy(window.location);">Share url</button>
    </body>
</html>""".replace("%js_library_url", js_library_url), remote_basedir + '/navigation.html')

def sonarqube_upload(args):
    """run SonarQube scanner and upload JaCoCo results"""

    sonarqube_cli = mx.library("SONARSCANNER_CLI_4_2_0_1873", True)

    parser = ArgumentParser(prog='mx sonarqube-upload')
    parser.add_argument('--exclude-generated', action='store_true', help='Exclude generated source files')
    parser.add_argument('--skip-coverage', action='store_true', default=False, help='Do not upload coverage reports')
    args, sonar_args = mx.extract_VM_args(args, useDoubleDash=True, defaultAllVMArgs=True)
    args, other_args = parser.parse_known_args(args)
    java_props, other_args = _parse_java_properties(other_args)

    def _check_required_prop(prop):
        if prop not in java_props:
            mx.abort("Required property '{prop}' not present. (Format is '-D{prop}=<value>')".format(prop=prop))

    _check_required_prop('sonar.projectKey')
    _check_required_prop('sonar.host.url')

    basedir = mx.primary_suite().dir

    # collect excluded projects
    excludes, includes = _jacoco_excludes_includes_projects(limit_to_primary=True)
    # collect excluded classes
    exclude_classes = _jacoco_exclude_classes(includes)
    java_bin = []
    java_src = []
    java_libs = []

    def _visit_deps(dep, edge):
        if dep.isJARDistribution() or dep.isLibrary():
            java_libs.append(dep.classpath_repr())

    mx.walk_deps(includes, visit=_visit_deps)

    # collect all sources and binaries -- do exclusion later
    for p in includes:
        java_src.extend(p.source_dirs())
        if not args.exclude_generated:
            gen_dir = p.source_gen_dir()
            if os.path.exists(gen_dir):
                java_src.append(gen_dir)
        java_bin.append(p.output_dir())

    java_src = [os.path.relpath(s, basedir) for s in java_src]
    java_bin = [os.path.relpath(b, basedir) for b in java_bin]

    # Overlayed sources and classes must be excluded
    jdk_compliance = mx.get_jdk().javaCompliance
    overlayed_sources = []
    overlayed_classfiles = {}
    for p in includes:
        if hasattr(p, "multiReleaseJarVersion") and jdk_compliance not in p.javaCompliance: # JDK9+ overlays
            for srcDir in p.source_dirs():
                for root, _, files in os.walk(srcDir):
                    for name in files:
                        if name.endswith('.java') and name != 'package-info.java':
                            overlayed_sources.append(join(os.path.relpath(root, basedir), name))
        elif hasattr(p, "overlayTarget"): # JDK8 overlays
            target = mx.project(p.overlayTarget)
            overlay_sources = []
            for srcDir in p.source_dirs():
                for root, _, files in os.walk(srcDir):
                    for name in files:
                        if name.endswith('.java') and name != 'package-info.java':
                            overlay_sources.append(join(os.path.relpath(root, srcDir), name))
            print(p, target, overlay_sources)
            for srcDir in target.source_dirs():
                for root, _, files in os.walk(srcDir):
                    for name in files:
                        if name.endswith('.java') and name != 'package-info.java':
                            s = join(os.path.relpath(root, srcDir), name)
                            if s in overlay_sources:
                                overlayed = join(os.path.relpath(root, basedir), name)
                                overlayed_sources.append(overlayed)
            for s in overlay_sources:
                classfile = join(os.path.relpath(target.output_dir(), basedir), s[:-len('java')] + 'class')
                with open(classfile, 'rb') as fp:
                    overlayed_classfiles[classfile] = fp.read()

    exclude_dirs = []
    for p in excludes:
        exclude_dirs.extend(p.source_dirs())
        exclude_dirs.append(p.source_gen_dir())

    javaCompliance = max([p.javaCompliance for p in includes]) if includes else mx.JavaCompliance('1.7')

    jacoco_exec = JACOCO_EXEC
    if not os.path.exists(jacoco_exec) and not args.skip_coverage:
        mx.abort('No JaCoCo report file found: ' + jacoco_exec)

    def _add_default_prop(key, value):
        if key not in java_props:
            java_props[key] = value

    # default properties
    _add_default_prop('sonar.java.source', str(javaCompliance))
    _add_default_prop('sonar.projectBaseDir', basedir)
    if not args.skip_coverage:
        _add_default_prop('sonar.jacoco.reportPaths', jacoco_exec)
    _add_default_prop('sonar.sources', ','.join(java_src))
    _add_default_prop('sonar.java.binaries', ','.join(java_bin))
    _add_default_prop('sonar.java.libraries', ','.join(java_libs))
    exclude_patterns = [os.path.relpath(e, basedir) + '**' for e in exclude_dirs] + \
                       overlayed_sources + \
                       list(set([os.path.relpath(match[0], basedir) for _, match in exclude_classes.items()]))
    if exclude_patterns:
        _add_default_prop('sonar.exclusions', ','.join(exclude_patterns))
        _add_default_prop('sonar.coverage.exclusions', ','.join(exclude_patterns))
    _add_default_prop('sonar.verbose', 'true' if mx._opts.verbose else 'false')

    with tempfile.NamedTemporaryFile(suffix="-sonarqube.properties", mode="w+") as fp:
        # prepare properties file
        fp.writelines(('{}={}\n'.format(k, v) for k, v in java_props.items()))
        fp.flush()

        # Since there's no options to exclude individual classes,
        # we temporarily delete the overlayed class files instead.
        for classfile in overlayed_classfiles:
            os.remove(classfile)

        try:
            # run sonarqube cli
            java_args = other_args + ['-Dproject.settings=' + fp.name, '-jar', sonarqube_cli.get_path(True)] + sonar_args
            exit_code = mx.run_java(java_args, nonZeroIsFatal=False)
        finally:
            # Restore temporarily deleted class files
            for classfile, data in overlayed_classfiles.items():
                with open(classfile, 'wb') as cf:
                    cf.write(data)

        if exit_code != 0:
            fp.seek(0)
            mx.abort('SonarQube scanner terminated with non-zero exit code: {}\n  Properties file:\n{}'.format(
                exit_code, ''.join(('    ' + l for l in fp.readlines()))))