diff --git a/MANIFEST.in b/MANIFEST.in index adf391a6..5641fa44 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,3 @@ recursive-include bloom/generators/debian/templates * recursive-include bloom/generators/rpm/templates * +recursive-include bloom/generators/vcpkg/templates * diff --git a/bloom/generators/common.py b/bloom/generators/common.py index c2bfccaa..cb541f83 100644 --- a/bloom/generators/common.py +++ b/bloom/generators/common.py @@ -32,31 +32,82 @@ from __future__ import print_function +import collections +import json +import io +import os import pkg_resources +import shutil import sys import traceback +from bloom.commands.git.patch.common import get_patch_config +from bloom.commands.git.patch.common import set_patch_config + +from bloom.git import inbranch +from bloom.git import get_branches +from bloom.git import get_commit_hash +from bloom.git import get_current_branch +from bloom.git import has_changes +from bloom.git import show +from bloom.git import tag_exists + +from bloom.logging import ansi from bloom.logging import debug from bloom.logging import error +from bloom.logging import fmt from bloom.logging import info +from bloom.logging import warning + +from bloom.packages import get_package_data from bloom.rosdistro_api import get_distribution_type from bloom.util import code +from bloom.util import execute_command from bloom.util import maybe_continue from bloom.util import print_exc +try: + from configparser import SafeConfigParser +except ImportError: + from ConfigParser import SafeConfigParser + try: from rosdep2 import create_default_installer_context from rosdep2.catkin_support import get_catkin_view from rosdep2.lookup import ResolutionError import rosdep2.catkin_support -except ImportError as err: +except ImportError: debug(traceback.format_exc()) error("rosdep was not detected, please install it.", exit=True) +try: + import rosdistro +except ImportError: + debug(traceback.format_exc()) + error("rosdistro was not detected, please install it.", exit=True) + +try: + import em +except ImportError: + debug(traceback.format_exc()) + error("empy was not detected, please install it.", exit=True) + +# Fix unicode bug in empy +# This should be removed once upstream empy is fixed +# See: https://github.com/ros-infrastructure/bloom/issues/196 +try: + em.str = unicode + em.Stream.write_old = em.Stream.write + em.Stream.write = lambda self, data: em.Stream.write_old(self, data.encode('utf8')) +except NameError: + pass +# End fix + BLOOM_GROUP = 'bloom.generators' DEFAULT_ROS_DISTRO = 'indigo' +TEMPLATE_EXTENSION = '.em' def list_generators(): @@ -203,11 +254,284 @@ def resolve_dependencies( # Do not compare the installer key here since this is a general purpose function # They installer is verified in the OS specific generator, when the keys are pre-checked. if resolved_key is None: - resolved_key = fallback_resolver(key, peer_packages) + resolved_key = fallback_resolver(key, peer_packages, os_name, os_version, ros_distro) resolved_keys[key] = resolved_key return resolved_keys +def generate_substitutions_from_package( + package, + os_name, + os_version, + ros_distro, + format_description, + format_depends, + installation_prefix='/usr', + inc=0, + peer_packages=None, + fallback_resolver=None, + native=False +): + peer_packages = peer_packages or [] + data = {} + # Name, Version, Description + data['Name'] = package.name + data['Version'] = package.version + data['Description'] = format_description(package.description) + # Websites + websites = [str(url) for url in package.urls if url.type == 'website'] + homepage = websites[0] if websites else '' + if homepage == '': + warning("No homepage set, defaulting to ''") + data['Homepage'] = homepage + # Increment Number + data['Inc'] = '' if native else '{0}'.format(inc) + # Package name + data['Package'] = sanitize_package_name(package.name) + # Installation prefix + data['InstallationPrefix'] = installation_prefix + # Resolve dependencies + package.evaluate_conditions(package_conditional_context(ros_distro)) + depends = [ + dep for dep in (package.run_depends + package.buildtool_export_depends) + if dep.evaluated_condition] + build_depends = [ + dep for dep in (package.build_depends + package.buildtool_depends + package.test_depends) + if dep.evaluated_condition] + + unresolved_keys = [ + dep for dep in (depends + build_depends + package.replaces + package.conflicts) + if dep.evaluated_condition] + # The installer key is not considered here, but it is checked when the keys are checked before this + resolved_deps = resolve_dependencies(unresolved_keys, os_name, + os_version, ros_distro, + peer_packages + [d.name for d in package.replaces + package.conflicts], + fallback_resolver) + data['Depends'] = sorted( + set(format_depends(depends, resolved_deps)) + ) + data['BuildDepends'] = sorted( + set(format_depends(build_depends, resolved_deps)) + ) + data['Replaces'] = sorted( + set(format_depends(package.replaces, resolved_deps)) + ) + data['Conflicts'] = sorted( + set(format_depends(package.conflicts, resolved_deps)) + ) + + # Build-type specific substitutions. + build_type = package.get_build_type() + if build_type == 'catkin': + pass + elif build_type == 'cmake': + pass + elif build_type == 'ament_cmake': + pass + elif build_type == 'ament_python': + # Don't set the install-scripts flag if it's already set in setup.cfg. + package_path = os.path.abspath(os.path.dirname(package.filename)) + setup_cfg_path = os.path.join(package_path, 'setup.cfg') + data['pass_install_scripts'] = True + if os.path.isfile(setup_cfg_path): + setup_cfg = SafeConfigParser() + setup_cfg.read([setup_cfg_path]) + if ( + setup_cfg.has_option('install', 'install-scripts') or + setup_cfg.has_option('install', 'install_scripts') + ): + data['pass_install_scripts'] = False + else: + error("Build type '{}' is not supported by this version of bloom.". + format(build_type), exit=True) + + # Set the distribution + data['Distribution'] = os_version + # Maintainers + maintainers = [] + for m in package.maintainers: + maintainers.append(str(m)) + data['Maintainer'] = maintainers[0] + data['Maintainers'] = ', '.join(maintainers) + + # Summarize dependencies + summarize_dependency_mapping(data, depends, build_depends, resolved_deps) + + return data + + +def __place_template_folder(group, src, dst, gbp=False, overwrite=False): + template_files = pkg_resources.resource_listdir(group, src) + # For each template, place + for template_file in template_files: + if not gbp and os.path.basename(template_file) == 'gbp.conf.em': + debug("Skipping template '{0}'".format(template_file)) + continue + template_path = os.path.join(src, template_file) + template_dst = os.path.join(dst, template_file) + if pkg_resources.resource_isdir(group, template_path): + debug("Recursing on folder '{0}'".format(template_path)) + __place_template_folder(group, template_path, template_dst, gbp) + else: + try: + debug("Placing template '{0}'".format(template_path)) + template = pkg_resources.resource_string(group, template_path) + template_abs_path = pkg_resources.resource_filename(group, template_path) + except IOError as err: + error("Failed to load template " + "'{0}': {1}".format(template_file, str(err)), exit=True) + if not os.path.exists(dst): + os.makedirs(dst) + if os.path.exists(template_dst) and not overwrite: + debug("Not overwriting existing file '{0}'".format(template_dst)) + else: + with io.open(template_dst, 'w', encoding='utf-8') as f: + if not isinstance(template, str): + template = template.decode('utf-8') + # Python 2 API needs a `unicode` not a utf-8 string. + elif sys.version_info.major == 2: + template = template.decode('utf-8') + f.write(template) + shutil.copystat(template_abs_path, template_dst) + + +def convertToUnicode(obj): + if sys.version_info.major == 2: + if isinstance(obj, str): + return unicode(obj.decode('utf8')) + elif isinstance(obj, unicode): + return obj + else: + if isinstance(obj, bytes): + return str(obj.decode('utf8')) + elif isinstance(obj, str): + return obj + if isinstance(obj, list): + for i, val in enumerate(obj): + obj[i] = convertToUnicode(val) + return obj + elif isinstance(obj, type(None)): + return None + elif isinstance(obj, tuple): + obj_tmp = list(obj) + for i, val in enumerate(obj_tmp): + obj_tmp[i] = convertToUnicode(obj_tmp[i]) + return tuple(obj_tmp) + elif isinstance(obj, int): + return obj + raise RuntimeError('need to deal with type %s' % (str(type(obj)))) + + +def place_template_files(path, build_type, package_manager, gbp=False, overwrite=False): + info(fmt("@!@{bf}==>@| Placing templates files in the '" + package_manager + "' folder.")) + dir_path = os.path.join(path, package_manager) + if not os.path.exists(dir_path): + os.makedirs(dir_path) + # Place template files + group = 'bloom.generators.' + package_manager + templates = os.path.join('templates', build_type) + __place_template_folder(group, templates, dir_path, gbp, overwrite=overwrite) + + +def summarize_dependency_mapping(data, deps, build_deps, resolved_deps): + if len(deps) == 0 and len(build_deps) == 0: + return + info("Package '" + data['Package'] + "' has dependencies:") + header = " " + ansi('boldoff') + ansi('ulon') + \ + "rosdep key => " + data['Distribution'] + \ + " key" + ansi('reset') + template = " " + ansi('cyanf') + "{0:<20} " + ansi('purplef') + \ + "=> " + ansi('cyanf') + "{1}" + ansi('reset') + if len(deps) != 0: + info(ansi('purplef') + "Run Dependencies:" + + ansi('reset')) + info(header) + for key in [d.name for d in deps]: + info(template.format(key, resolved_deps[key])) + if len(build_deps) != 0: + info(ansi('purplef') + + "Build and Build Tool Dependencies:" + ansi('reset')) + info(header) + for key in [d.name for d in build_deps]: + info(template.format(key, resolved_deps[key])) + + +def __process_template_folder(path, subs): + items = os.listdir(path) + processed_items = [] + for item in list(items): + item = os.path.abspath(os.path.join(path, item)) + if os.path.basename(item) in ['.', '..', '.git', '.svn']: + continue + if os.path.isdir(item): + sub_items = __process_template_folder(item, subs) + processed_items.extend([os.path.join(item, s) for s in sub_items]) + if not item.endswith(TEMPLATE_EXTENSION): + continue + with open(item, 'r') as f: + template = f.read() + # Remove extension + template_path = item[:-len(TEMPLATE_EXTENSION)] + # Expand template + info("Expanding '{0}' -> '{1}'".format( + os.path.relpath(item), + os.path.relpath(template_path))) + result = em.expand(template, **subs) + # Don't write an empty file + if len(result) == 0 and \ + os.path.basename(template_path) in ['copyright']: + processed_items.append(item) + continue + # Write the result + with io.open(template_path, 'w', encoding='utf-8') as f: + if sys.version_info.major == 2: + result = result.decode('utf-8') + f.write(result) + # Copy the permissions + shutil.copymode(item, template_path) + processed_items.append(item) + return processed_items + + +def process_template_files(path, subs, pacakge_manager): + info(fmt("@!@{bf}==>@| In place processing templates files in '" + pacakge_manager + "' folder.")) + dir_path = os.path.join(path, pacakge_manager) + if not os.path.exists(dir_path): + sys.exit("No {0} directory found at '{1}', cannot process templates." + .format(pacakge_manager, dir_path)) + return __process_template_folder(dir_path, subs) + + +def match_branches_with_prefix(prefix, get_branches, prune=False): + debug("match_branches_with_prefix(" + str(prefix) + ", " + + str(get_branches()) + ")") + branches = [] + # Match branches + existing_branches = get_branches() + for branch in existing_branches: + if branch.startswith('remotes/origin/'): + branch = branch.split('/', 2)[-1] + if branch.startswith(prefix): + branches.append(branch) + branches = list(set(branches)) + if prune: + # Prune listed branches by packages in latest upstream + with inbranch('upstream'): + pkg_names, version, pkgs_dict = get_package_data('upstream') + for branch in branches: + if branch.split(prefix)[-1].strip('/') not in pkg_names: + branches.remove(branch) + return branches + + +def rosify_package_name(name, rosdistro): + return 'ros-{0}-{1}'.format(rosdistro, name) + + +def sanitize_package_name(name): + return name.replace('_', '-') + + class GeneratorError(Exception): def __init__(self, msg, returncode=code.UNKNOWN): super(GeneratorError, self).__init__("Error running generator: " + msg) @@ -363,3 +687,436 @@ def post_patch(self, branch_name): :returns: return code, return 0 or None for OK, anythign else on error """ return 0 + + +class PackageManagerGenerator(BloomGenerator): + package_manager = 'none' + has_run_rosdep = False + default_install_prefix = '/usr' + rosdistro = os.environ.get('ROS_DISTRO', 'indigo') + + def prepare_arguments(self, parser): + # The common command line arguments for every package system + add = parser.add_argument + add('-i', '--inc', help="increment number", default='0') + add('-p', '--prefix', required=True, + help="branch prefix to match, and from which create packages" + " hint: if you want to match 'release/foo' use 'release'") + add('-a', '--match-all', default=False, action="store_true", + help="match all branches with the given prefix, " + "even if not in current upstream") + add('--distros', nargs='+', required=False, default=[], + help='A list of os distros to generate for certain package system') + add('--install-prefix', default=None, + help="overrides the default installation prefix (/usr)") + + def get_package_from_branch(self, branch): + with inbranch(branch): + try: + package_data = get_package_data(branch) + except SystemExit: + return None + if type(package_data) not in [list, tuple]: + # It is a ret code + self.exit(package_data) + names, version, packages = package_data + if type(names) is list and len(names) > 1: + self.exit( + "{0} generator does not support generating " + "from branches with multiple packages in them, use " + "the release generator first to split packages into " + "individual branches." + .format(self.package_manager)) + if type(packages) is dict: + return list(packages.values())[0] + + def set_default_distros(self): + index = rosdistro.get_index(rosdistro.get_index_url()) + distribution_file = rosdistro.get_distribution_file(index, self.rosdistro) + if self.os_name not in distribution_file.release_platforms: + if hasattr(self, "os_not_required") and self.os_not_required: + warning("No platforms defined for os '{0}' in release file for the " + "'{1}' distro. This os was not required; continuing without error." + .format(self.os_name, self.rosdistro)) + sys.exit(0) + error("No platforms defined for os '{0}' in release file for the '{1}' distro." + .format(self.os_name, self.rosdistro), exit=True) + self.distros = distribution_file.release_platforms[self.os_name] + + def handle_arguments(self, args): + self.interactive = args.interactive + self.inc = args.inc + self.distros = args.distros + if self.distros in [None, []]: + self.set_default_distros() + self.install_prefix = args.install_prefix + if args.install_prefix is None: + self.install_prefix = self.default_install_prefix + self.prefix = args.prefix + self.branches = match_branches_with_prefix(self.prefix, get_branches, prune=not args.match_all) + if len(self.branches) == 0: + error( + "No packages found, check your --prefix or --src arguments.", + exit=True + ) + self.packages = {} + self.tag_names = {} + self.names = [] + self.branch_args = [] + self.package_manager_branches = [] + for branch in self.branches: + package = self.get_package_from_branch(branch) + if package is None: + # This is an ignored package + continue + self.packages[package.name] = package + self.names.append(package.name) + args = self.generate_branching_arguments(package, branch) + # First branch is package_manager/[/] + self.package_manager_branches.append(args[0][0]) + self.branch_args.extend(args) + + def summarize(self): + info("Generating {0} source for the packages: {1}".format(self.os_name, str(self.names))) + info("Incremental Version: " + str(self.inc)) + info("Distributions: " + str(self.distros)) + + def get_branching_arguments(self): + return self.branch_args + + def update_rosdep(self): + update_rosdep() + self.has_run_rosdep = True + + def _check_all_keys_are_valid(self, peer_packages, rosdistro): + keys_to_resolve = [] + key_to_packages_which_depends_on = collections.defaultdict(list) + keys_to_ignore = set() + for package in self.packages.values(): + package.evaluate_conditions(package_conditional_context(rosdistro)) + depends = [ + dep for dep in (package.run_depends + package.buildtool_export_depends) + if dep.evaluated_condition] + build_depends = [ + dep for dep in (package.build_depends + package.buildtool_depends + package.test_depends) + if dep.evaluated_condition] + unresolved_keys = [ + dep for dep in (depends + build_depends + package.replaces + package.conflicts) + if dep.evaluated_condition] + keys_to_ignore = { + dep for dep in keys_to_ignore.union(package.replaces + package.conflicts) + if dep.evaluated_condition} + keys = [d.name for d in unresolved_keys] + keys_to_resolve.extend(keys) + for key in keys: + key_to_packages_which_depends_on[key].append(package.name) + + os_name = self.os_name + rosdistro = self.rosdistro + all_keys_valid = True + for key in sorted(set(keys_to_resolve)): + for os_version in self.distros: + try: + extended_peer_packages = peer_packages + [d.name for d in keys_to_ignore] + rule, installer_key, default_installer_key = \ + resolve_rosdep_key(key, os_name, os_version, rosdistro, extended_peer_packages, + retry=False) + if rule is None: + continue + if installer_key != default_installer_key: + error("Key '{0}' resolved to '{1}' with installer '{2}', " + "which does not match the default installer '{3}'." + .format(key, rule, installer_key, default_installer_key)) + self.exit( + "The {0} generator does not support dependencies " + "which are installed with the '{1}' installer." + .format(self.package_manager, installer_key), + returncode=code.GENERATOR_INVALID_INSTALLER_KEY) + except (GeneratorError, RuntimeError) as e: + print(fmt("Failed to resolve @{cf}@!{key}@| on @{bf}{os_name}@|:@{cf}@!{os_version}@| with: {e}") + .format(**locals())) + print(fmt("@{cf}@!{0}@| is depended on by these packages: ").format(key) + + str(list(set(key_to_packages_which_depends_on[key])))) + print(fmt("@{kf}@!<== @{rf}@!Failed@|")) + all_keys_valid = False + return all_keys_valid + + def check_all_keys_are_valid(self, key_invalid_error_msg): + info("\nPre-verifying {0} dependency keys...".format(self.package_manager)) + # Run rosdep update is needed + if not self.has_run_rosdep: + self.update_rosdep() + + peer_packages = [p.name for p in self.packages.values()] + + while not self._check_all_keys_are_valid(peer_packages, self.rosdistro): + error(key_invalid_error_msg) + try: + if not maybe_continue(msg="Would you like to try again?"): + error("User aborted after rosdep keys were not resolved.") + sys.exit(code.GENERATOR_NO_ROSDEP_KEY_FOR_DISTRO) + except (KeyboardInterrupt, EOFError): + error("\nUser quit.", exit=True) + update_rosdep() + invalidate_view_cache() + + info("All keys are " + ansi('greenf') + "OK" + ansi('reset') + "\n") + + def pre_branch(self, destination, source): + if destination in self.package_manager_branches: + return + # Run rosdep update is needed + if not self.has_run_rosdep: + self.update_rosdep() + # Determine the current package being generated + name = destination.split('/')[-1] + distro = destination.split('/')[-2] + # Retrieve the package + package = self.packages[name] + # Report on this package + self.summarize_package(package, distro) + + def pre_rebase(self, destination): + # Get the stored configs is any + patches_branch = 'patches/' + destination + config = self.load_original_config(patches_branch) + if config is not None: + curr_config = get_patch_config(patches_branch) + if curr_config['parent'] == config['parent']: + set_patch_config(patches_branch, config) + + def post_rebase(self, destination): + name = destination.split('/')[-1] + # Retrieve the package + package = self.packages[name] + # Handle differently if this is a package system vs distro branch + if destination in self.package_manager_branches: + info("Placing {0} template files into '{1}' branch." + .format(self.package_manager, destination)) + # Then this is a package system branch + # Place the raw template files + self.place_template_files(package.get_build_type()) + else: + # This is a distro specific package system branch + # Determine the current package being generated + distro = destination.split('/')[-2] + # Create package for each distro + with inbranch(destination): + data = self.generate_package(package, distro) + # Create the tag name for later + self.tag_names[destination] = self.generate_tag_name(data) + # Update the patch configs + patches_branch = 'patches/' + destination + config = get_patch_config(patches_branch) + # Store it + self.store_original_config(config, patches_branch) + # Modify the base so import/export patch works + current_branch = get_current_branch() + if current_branch is None: + error("Could not determine current branch.", exit=True) + config['base'] = get_commit_hash(current_branch) + # Set it + set_patch_config(patches_branch, config) + + def post_patch(self, destination, color='bluef'): + if destination in self.package_manager_branches: + return + # Tag after patches have been applied + with inbranch(destination): + # Tag + tag_name = self.tag_names[destination] + if tag_exists(tag_name): + if self.interactive: + warning("Tag exists: " + tag_name) + warning("Do you wish to overwrite it?") + if not maybe_continue('y'): + error("Answered no to continue, aborting.", exit=True) + else: + warning("Overwriting tag: " + tag_name) + else: + info("Creating tag: " + tag_name) + execute_command('git tag -f ' + tag_name) + # Report of success + name = destination.split('/')[-1] + package = self.packages[name] + distro = destination.split('/')[-2] + info(ansi(color) + "####" + ansi('reset'), use_prefix=False) + info( + ansi(color) + "#### " + ansi('greenf') + "Successfully" + + ansi(color) + " generated '" + ansi('boldon') + distro + + ansi('boldoff') + "' {0} for package".format(self.package_manager) + + " '" + ansi('boldon') + package.name + ansi('boldoff') + "'" + + " at version '" + ansi('boldon') + package.version + + "-" + str(self.inc) + ansi('boldoff') + "'" + + ansi('reset'), + use_prefix=False + ) + info(ansi(color) + "####\n" + ansi('reset'), use_prefix=False) + + def store_original_config(self, config, patches_branch): + with inbranch(patches_branch): + with open('{0}.store'.format(self.package_manager), 'w+') as f: + f.write(json.dumps(config)) + execute_command('git add {0}.store'.format(self.package_manager)) + if has_changes(): + execute_command('git commit -m "Store original patch config"') + + def load_original_config(self, patches_branch): + config_store = show(patches_branch, '{0}.store'.format(self.package_manager)) + if config_store is None: + return config_store + return json.loads(config_store) + + def get_releaser_history(self): + # Assumes that this is called in the target branch + patches_branch = 'patches/' + get_current_branch() + raw = show(patches_branch, 'releaser_history.json') + return None if raw is None else json.loads(raw) + + def set_releaser_history(self, history): + # Assumes that this is called in the target branch + patches_branch = 'patches/' + get_current_branch() + debug("Writing release history to '{0}' branch".format(patches_branch)) + with inbranch(patches_branch): + with open('releaser_history.json', 'w') as f: + f.write(json.dumps(history)) + execute_command('git add releaser_history.json') + if has_changes(): + execute_command('git commit -m "Store releaser history"') + + @staticmethod + def missing_dep_resolver(key, peer_packages, os_name, os_version, ros_distro): + """ + This should be a staticmethod since we will use it when we call + `generate_substitutions_from_package` in `get_subs` + Notice that os_name, os_version, ros_distro maybe useful when we + want to add new resolver in the future + """ + if key in peer_packages: + return [sanitize_package_name(key)] + return default_fallback_resolver(key, peer_packages) + + def place_template_files(self, build_type, dir_path=None): + # Create/Clean the package system folder + overwrite = False + if dir_path is None: + dir_path = os.path.join(".", self.package_manager) + if os.path.exists(dir_path): + if self.interactive: + warning("{0} directory exists: {1}".format(self.package_manager, dir_path)) + warning("Do you wish to overwrite it?") + if not maybe_continue('y'): + error("Answered no to continue, aborting.", exit=True) + overwrite = True + elif 'BLOOM_CLEAR_TEMPLATE_ON_GENERATION' in os.environ: + warning("Overwriting {0} directory: {1}".format(self.package_manager, dir_path)) + execute_command('git rm -rf ' + dir_path) + execute_command('git commit -m "Clearing previous {0} folder"' + .format(self.package_manager)) + if os.path.exists(dir_path): + shutil.rmtree(dir_path) + else: + warning("Not overwriting {0} directory.".format(self.package_manager)) + # Use generic place template files command + place_template_files('.', build_type, self.package_manager, gbp=True, overwrite=overwrite) + # Commit results + execute_command('git add ' + dir_path) + _, has_files, _ = execute_command('git diff --cached --name-only', return_io=True) + if has_files: + execute_command('git commit -m "Placing {0} template files"'.format(self.package_manager)) + + def get_subs(self, package, os_version, format_description, format_depends): + # This is the common part for generate templacte substitute, then successor of + # the generator will add its specic content via define its get_subs_hook function + subs = generate_substitutions_from_package( + package, + self.os_name, + os_version, + self.rosdistro, + format_description, + format_depends, + self.install_prefix, + self.inc, + [p.name for p in self.packages.values()], + fallback_resolver=self.missing_dep_resolver + ) + # Try to retrieve the releaser_history + releaser_history = self.get_releaser_history() + subs = self.get_subs_hook(subs, package, self.rosdistro, releaser_history=releaser_history) + for item in subs.items(): + subs[item[0]] = convertToUnicode(item[1]) + return subs + + def summarize_package(self, package, distro, color='bluef'): + info(ansi(color) + "\n####" + ansi('reset'), use_prefix=False) + info( + ansi(color) + "#### Generating '" + ansi('boldon') + distro + + ansi('boldoff') + "' {0} for package".format(self.package_manager) + + " '" + ansi('boldon') + package.name + ansi('boldoff') + "'" + + " at version '" + ansi('boldon') + package.version + + "-" + str(self.inc) + ansi('boldoff') + "'" + + ansi('reset'), + use_prefix=False + ) + info(ansi(color) + "####" + ansi('reset'), use_prefix=False) + + def generate_branching_arguments(self, package, branch): + """ + The default branch for placing package system release data + + :param package: the package metadata extract from package.xml + :param branch: every branch match the prefix in command line input + + :return: list of (destination, source, interactive) + """ + n = package.name + # package branch + package_branch = self.package_manager + '/' + n + # Branch first to the package branch + args = [[package_branch, branch, False]] + # Then for each os distro, branch from the base package branch + args.extend([ + [self.package_manager + '/' + d + '/' + n, package_branch, False] + for d in self.distros + ]) + return args + + def generate_package(self, package, os_version): + """ + Assume we have the templactes file in directory + The overriten function should generate the package, including + 1. use the result of get_subs to replace template content + 2. set the newest release history + 3. some git commit operation + + :param package: the substitute for in place of the templacte content + :param os_version: the specific operate system version + + :returns: substitutes for other use + """ + raise NotImplemented + + @staticmethod + def get_subs_hook(subs, package, rosdistro, releaser_history=None): + """ + The specific package system related substitute operation + + :param subs: the substitute for in place of the template content + :param package: the substitute for in place of the templacte content + :param rosdistro: the ros version + :param releaser_history: the substitute for in place of the templacte content + + :returns: the improved subs + """ + return subs + + @staticmethod + def generate_tag_name(subs): + """ + Generate tag name based on the substitute, this method need be overwriten + + :param subs: the substitute for in place of the templacte content + + :returns: tag name + """ + raise NotImplemented diff --git a/bloom/generators/debian/__init__.py b/bloom/generators/debian/__init__.py index 385f4c99..a1f3f8c7 100755 --- a/bloom/generators/debian/__init__.py +++ b/bloom/generators/debian/__init__.py @@ -1,4 +1,5 @@ from .generator import DebianGenerator -from .generator import sanitize_package_name +from .generator import format_depends +from .generator import format_description -__all__ = ['DebianGenerator', 'sanitize_package_name'] +__all__ = ['DebianGenerator', 'format_description', 'format_depends'] diff --git a/bloom/generators/debian/generate_cmd.py b/bloom/generators/debian/generate_cmd.py index dba1a8f1..ee25ffad 100644 --- a/bloom/generators/debian/generate_cmd.py +++ b/bloom/generators/debian/generate_cmd.py @@ -42,9 +42,13 @@ from bloom.logging import fmt from bloom.logging import info -from bloom.generators.debian.generator import generate_substitutions_from_package -from bloom.generators.debian.generator import place_template_files -from bloom.generators.debian.generator import process_template_files +from bloom.generators.common import generate_substitutions_from_package +from bloom.generators.common import place_template_files +from bloom.generators.common import process_template_files + +from bloom.generators.debian.generator import DebianGenerator +from bloom.generators.debian.generator import format_description +from bloom.generators.debian.generator import format_depends from bloom.util import get_distro_list_prompt @@ -80,13 +84,18 @@ def prepare_arguments(parser): def get_subs(pkg, os_name, os_version, ros_distro, native=False): - return generate_substitutions_from_package( + subs = generate_substitutions_from_package( pkg, os_name, os_version, ros_distro, - native=native + format_description, + format_depends, ) + subs = DebianGenerator.get_subs_hook(subs, pkg, ros_distro) + # Debian Package Format + subs['format'] = 'native' if native else 'quilt' + return subs def main(args=None, get_subs_fn=None): @@ -120,20 +129,21 @@ def main(args=None, get_subs_fn=None): fmt("Generating debs for @{cf}%s:%s@| for package(s) %s" % (os_name, os_version, [p.name for p in pkgs_dict.values()]))) + package_manager = DebianGenerator.package_manager for path, pkg in pkgs_dict.items(): template_files = None try: subs = get_subs_fn(pkg, os_name, os_version, ros_distro, args.native) if _place_template_files: # Place template files - place_template_files(path, pkg.get_build_type()) + place_template_files(path, pkg.get_build_type(), package_manager) if _process_template_files: # Just process existing template files - template_files = process_template_files(path, subs) + template_files = process_template_files(path, subs, package_manager) if not _place_template_files and not _process_template_files: # If neither, do both - place_template_files(path, pkg.get_build_type()) - template_files = process_template_files(path, subs) + place_template_files(path, pkg.get_build_type(), package_manager) + template_files = process_template_files(path, subs, package_manager) if template_files is not None: for template_file in template_files: os.remove(os.path.normpath(template_file)) diff --git a/bloom/generators/debian/generator.py b/bloom/generators/debian/generator.py index 0f9e24f9..c8796ce9 100644 --- a/bloom/generators/debian/generator.py +++ b/bloom/generators/debian/generator.py @@ -32,58 +32,25 @@ from __future__ import print_function -import collections import datetime -import io -import json import os -import pkg_resources import re -import shutil import sys import traceback -# Python 2/3 support. -try: - from configparser import SafeConfigParser -except ImportError: - from ConfigParser import SafeConfigParser from dateutil import tz from pkg_resources import parse_version -from bloom.generators import BloomGenerator -from bloom.generators import GeneratorError -from bloom.generators import resolve_dependencies -from bloom.generators import update_rosdep - -from bloom.generators.common import default_fallback_resolver -from bloom.generators.common import invalidate_view_cache -from bloom.generators.common import package_conditional_context -from bloom.generators.common import resolve_rosdep_key - -from bloom.git import inbranch -from bloom.git import get_branches -from bloom.git import get_commit_hash -from bloom.git import get_current_branch -from bloom.git import has_changes -from bloom.git import show -from bloom.git import tag_exists - -from bloom.logging import ansi +from bloom.generators.common import PackageManagerGenerator +from bloom.generators.common import process_template_files + from bloom.logging import debug from bloom.logging import enable_drop_first_log_prefix from bloom.logging import error -from bloom.logging import fmt from bloom.logging import info from bloom.logging import is_debug from bloom.logging import warning -from bloom.commands.git.patch.common import get_patch_config -from bloom.commands.git.patch.common import set_patch_config - -from bloom.packages import get_package_data - -from bloom.util import code from bloom.util import to_unicode from bloom.util import execute_command from bloom.util import get_rfc_2822_date @@ -96,104 +63,9 @@ debug(traceback.format_exc()) error("catkin_pkg was not detected, please install it.", exit=True) -try: - import rosdistro -except ImportError as err: - debug(traceback.format_exc()) - error("rosdistro was not detected, please install it.", exit=True) - -try: - import em -except ImportError: - debug(traceback.format_exc()) - error("empy was not detected, please install it.", exit=True) - -# Fix unicode bug in empy -# This should be removed once upstream empy is fixed -# See: https://github.com/ros-infrastructure/bloom/issues/196 -try: - em.str = unicode - em.Stream.write_old = em.Stream.write - em.Stream.write = lambda self, data: em.Stream.write_old(self, data.encode('utf8')) -except NameError: - pass -# End fix - # Drop the first log prefix for this command enable_drop_first_log_prefix(True) -TEMPLATE_EXTENSION = '.em' - - -def __place_template_folder(group, src, dst, gbp=False): - template_files = pkg_resources.resource_listdir(group, src) - # For each template, place - for template_file in template_files: - if not gbp and os.path.basename(template_file) == 'gbp.conf.em': - debug("Skipping template '{0}'".format(template_file)) - continue - template_path = os.path.join(src, template_file) - template_dst = os.path.join(dst, template_file) - if pkg_resources.resource_isdir(group, template_path): - debug("Recursing on folder '{0}'".format(template_path)) - __place_template_folder(group, template_path, template_dst, gbp) - else: - try: - debug("Placing template '{0}'".format(template_path)) - template = pkg_resources.resource_string(group, template_path) - template_abs_path = pkg_resources.resource_filename(group, template_path) - except IOError as err: - error("Failed to load template " - "'{0}': {1}".format(template_file, str(err)), exit=True) - if not os.path.exists(dst): - os.makedirs(dst) - if os.path.exists(template_dst): - debug("Not overwriting existing file '{0}'".format(template_dst)) - else: - with io.open(template_dst, 'w', encoding='utf-8') as f: - if not isinstance(template, str): - template = template.decode('utf-8') - # Python 2 API needs a `unicode` not a utf-8 string. - elif sys.version_info.major == 2: - template = template.decode('utf-8') - f.write(template) - shutil.copystat(template_abs_path, template_dst) - - -def place_template_files(path, build_type, gbp=False): - info(fmt("@!@{bf}==>@| Placing templates files in the 'debian' folder.")) - debian_path = os.path.join(path, 'debian') - # Create/Clean the debian folder - if not os.path.exists(debian_path): - os.makedirs(debian_path) - # Place template files - group = 'bloom.generators.debian' - templates = os.path.join('templates', build_type) - __place_template_folder(group, templates, debian_path, gbp) - - -def summarize_dependency_mapping(data, deps, build_deps, resolved_deps): - if len(deps) == 0 and len(build_deps) == 0: - return - info("Package '" + data['Package'] + "' has dependencies:") - header = " " + ansi('boldoff') + ansi('ulon') + \ - "rosdep key => " + data['Distribution'] + \ - " key" + ansi('reset') - template = " " + ansi('cyanf') + "{0:<20} " + ansi('purplef') + \ - "=> " + ansi('cyanf') + "{1}" + ansi('reset') - if len(deps) != 0: - info(ansi('purplef') + "Run Dependencies:" + - ansi('reset')) - info(header) - for key in [d.name for d in deps]: - info(template.format(key, resolved_deps[key])) - if len(build_deps) != 0: - info(ansi('purplef') + - "Build and Build Tool Dependencies:" + ansi('reset')) - info(header) - for key in [d.name for d in build_deps]: - info(template.format(key, resolved_deps[key])) - def format_depends(depends, resolved_deps): versions = { @@ -218,6 +90,14 @@ def format_depends(depends, resolved_deps): return formatted +def debianize_string(value): + markup_remover = re.compile(r'<.*?>') + value = markup_remover.sub('', value) + value = re.sub('\s+', ' ', value) + value = value.strip() + return value + + def format_description(value): """ Format proper string following Debian control file @@ -274,674 +154,130 @@ def get_changelogs(package, releaser_history=None): return [] -def missing_dep_resolver(key, peer_packages): - if key in peer_packages: - return [sanitize_package_name(key)] - return default_fallback_resolver(key, peer_packages) - - -def generate_substitutions_from_package( - package, - os_name, - os_version, - ros_distro, - installation_prefix='/usr', - deb_inc=0, - peer_packages=None, - releaser_history=None, - fallback_resolver=None, - native=False -): - peer_packages = peer_packages or [] - data = {} - # Name, Version, Description - data['Name'] = package.name - data['Version'] = package.version - data['Description'] = format_description(package.description) - # Websites - websites = [str(url) for url in package.urls if url.type == 'website'] - homepage = websites[0] if websites else '' - if homepage == '': - warning("No homepage set, defaulting to ''") - data['Homepage'] = homepage - # Debian Increment Number - data['DebianInc'] = '' if native else '-{0}'.format(deb_inc) - # Debian Package Format - data['format'] = 'native' if native else 'quilt' - # Package name - data['Package'] = sanitize_package_name(package.name) - # Installation prefix - data['InstallationPrefix'] = installation_prefix - # Resolve dependencies - package.evaluate_conditions(package_conditional_context(ros_distro)) - depends = [ - dep for dep in (package.run_depends + package.buildtool_export_depends) - if dep.evaluated_condition] - build_depends = [ - dep for dep in (package.build_depends + package.buildtool_depends + package.test_depends) - if dep.evaluated_condition] - - unresolved_keys = [ - dep for dep in (depends + build_depends + package.replaces + package.conflicts) - if dep.evaluated_condition] - # The installer key is not considered here, but it is checked when the keys are checked before this - resolved_deps = resolve_dependencies(unresolved_keys, os_name, - os_version, ros_distro, - peer_packages + [d.name for d in package.replaces + package.conflicts], - fallback_resolver) - data['Depends'] = sorted( - set(format_depends(depends, resolved_deps)) - ) - data['BuildDepends'] = sorted( - set(format_depends(build_depends, resolved_deps)) - ) - data['Replaces'] = sorted( - set(format_depends(package.replaces, resolved_deps)) - ) - data['Conflicts'] = sorted( - set(format_depends(package.conflicts, resolved_deps)) - ) - - # Build-type specific substitutions. - build_type = package.get_build_type() - if build_type == 'catkin': - pass - elif build_type == 'cmake': - pass - elif build_type == 'ament_cmake': - pass - elif build_type == 'ament_python': - # Don't set the install-scripts flag if it's already set in setup.cfg. - package_path = os.path.abspath(os.path.dirname(package.filename)) - setup_cfg_path = os.path.join(package_path, 'setup.cfg') - data['pass_install_scripts'] = True - if os.path.isfile(setup_cfg_path): - setup_cfg = SafeConfigParser() - setup_cfg.read([setup_cfg_path]) - if ( - setup_cfg.has_option('install', 'install-scripts') or - setup_cfg.has_option('install', 'install_scripts') - ): - data['pass_install_scripts'] = False - else: - error( - "Build type '{}' is not supported by this version of bloom.". - format(build_type), exit=True) - - # Set the distribution - data['Distribution'] = os_version - # Use the time stamp to set the date strings - stamp = datetime.datetime.now(tz.tzlocal()) - data['Date'] = stamp.strftime('%a, %d %b %Y %T %z') - data['YYYY'] = stamp.strftime('%Y') - # Maintainers - maintainers = [] - for m in package.maintainers: - maintainers.append(str(m)) - data['Maintainer'] = maintainers[0] - data['Maintainers'] = ', '.join(maintainers) - # Changelog - changelogs = get_changelogs(package, releaser_history) - if changelogs and package.version not in [x[0] for x in changelogs]: - warning("") - warning("A CHANGELOG.rst was found, but no changelog for this version was found.") - warning("You REALLY should have a entry (even a blank one) for each version of your package.") - warning("") - if not changelogs: - # Ensure at least a minimal changelog - changelogs = [] - if package.version not in [x[0] for x in changelogs]: - changelogs.insert(0, ( - package.version, - get_rfc_2822_date(datetime.datetime.now()), - ' * Autogenerated, no changelog for this version found in CHANGELOG.rst.', - package.maintainers[0].name, - package.maintainers[0].email - )) - bad_changelog = False - # Make sure that the first change log is the version being released - if package.version != changelogs[0][0]: - error("") - error("The version of the first changelog entry '{0}' is not the " - "same as the version being currently released '{1}'." - .format(package.version, changelogs[0][0])) - bad_changelog = True - # Make sure that the current version is the latest in the changelog - for changelog in changelogs: - if parse_version(package.version) < parse_version(changelog[0]): - error("") - error("There is at least one changelog entry, '{0}', which has a " - "newer version than the version of package '{1}' being released, '{2}'." - .format(changelog[0], package.name, package.version)) - bad_changelog = True - if bad_changelog: - error("This is almost certainly by mistake, you should really take a " - "look at the changelogs for the package you are releasing.") - error("") - if not maybe_continue('n', 'Continue anyways'): - sys.exit("User quit.") - data['changelogs'] = changelogs - # Use debhelper version 7 for oneric, otherwise 9 - data['debhelper_version'] = 7 if os_version in ['oneiric'] else 9 - # Summarize dependencies - summarize_dependency_mapping(data, depends, build_depends, resolved_deps) - # Copyright - licenses = [] - separator = '\n' + '=' * 80 + '\n\n' - for l in package.licenses: - if hasattr(l, 'file') and l.file is not None: - license_file = os.path.join(os.path.dirname(package.filename), l.file) - if not os.path.exists(license_file): - error("License file '{}' is not found.". - format(license_file), exit=True) - license_text = open(license_file, 'r').read() - if not license_text.endswith('\n'): - license_text += '\n' - licenses.append(license_text) - data['Copyright'] = separator.join(licenses) - - def convertToUnicode(obj): - if sys.version_info.major == 2: - if isinstance(obj, str): - return unicode(obj.decode('utf8')) - elif isinstance(obj, unicode): - return obj - else: - if isinstance(obj, bytes): - return str(obj.decode('utf8')) - elif isinstance(obj, str): - return obj - if isinstance(obj, list): - for i, val in enumerate(obj): - obj[i] = convertToUnicode(val) - return obj - elif isinstance(obj, type(None)): - return None - elif isinstance(obj, tuple): - obj_tmp = list(obj) - for i, val in enumerate(obj_tmp): - obj_tmp[i] = convertToUnicode(obj_tmp[i]) - return tuple(obj_tmp) - elif isinstance(obj, int): - return obj - raise RuntimeError('need to deal with type %s' % (str(type(obj)))) - - for item in data.items(): - data[item[0]] = convertToUnicode(item[1]) - - return data - - -def __process_template_folder(path, subs): - items = os.listdir(path) - processed_items = [] - for item in list(items): - item = os.path.abspath(os.path.join(path, item)) - if os.path.basename(item) in ['.', '..', '.git', '.svn']: - continue - if os.path.isdir(item): - sub_items = __process_template_folder(item, subs) - processed_items.extend([os.path.join(item, s) for s in sub_items]) - if not item.endswith(TEMPLATE_EXTENSION): - continue - with open(item, 'r') as f: - template = f.read() - # Remove extension - template_path = item[:-len(TEMPLATE_EXTENSION)] - # Expand template - info("Expanding '{0}' -> '{1}'".format( - os.path.relpath(item), - os.path.relpath(template_path))) - result = em.expand(template, **subs) - # Don't write an empty file - if len(result) == 0 and \ - os.path.basename(template_path) in ['copyright']: - processed_items.append(item) - continue - # Write the result - with io.open(template_path, 'w', encoding='utf-8') as f: - if sys.version_info.major == 2: - result = result.decode('utf-8') - f.write(result) - # Copy the permissions - shutil.copymode(item, template_path) - processed_items.append(item) - return processed_items - - -def process_template_files(path, subs): - info(fmt("@!@{bf}==>@| In place processing templates in 'debian' folder.")) - debian_dir = os.path.join(path, 'debian') - if not os.path.exists(debian_dir): - sys.exit("No debian directory found at '{0}', cannot process templates." - .format(debian_dir)) - return __process_template_folder(debian_dir, subs) - - -def match_branches_with_prefix(prefix, get_branches, prune=False): - debug("match_branches_with_prefix(" + str(prefix) + ", " + - str(get_branches()) + ")") - branches = [] - # Match branches - existing_branches = get_branches() - for branch in existing_branches: - if branch.startswith('remotes/origin/'): - branch = branch.split('/', 2)[-1] - if branch.startswith(prefix): - branches.append(branch) - branches = list(set(branches)) - if prune: - # Prune listed branches by packages in latest upstream - with inbranch('upstream'): - pkg_names, version, pkgs_dict = get_package_data('upstream') - for branch in branches: - if branch.split(prefix)[-1].strip('/') not in pkg_names: - branches.remove(branch) - return branches - - -def get_package_from_branch(branch): - with inbranch(branch): - try: - package_data = get_package_data(branch) - except SystemExit: - return None - if type(package_data) not in [list, tuple]: - # It is a ret code - DebianGenerator.exit(package_data) - names, version, packages = package_data - if type(names) is list and len(names) > 1: - DebianGenerator.exit( - "Debian generator does not support generating " - "from branches with multiple packages in them, use " - "the release generator first to split packages into " - "individual branches.") - if type(packages) is dict: - return list(packages.values())[0] - - -def debianize_string(value): - markup_remover = re.compile(r'<.*?>') - value = markup_remover.sub('', value) - value = re.sub('\s+', ' ', value) - value = value.strip() - return value - - -def sanitize_package_name(name): - return name.replace('_', '-') - - -class DebianGenerator(BloomGenerator): +class DebianGenerator(PackageManagerGenerator): title = 'debian' + package_manager = 'debian' description = "Generates debians from the catkin meta data" - has_run_rosdep = False - default_install_prefix = '/usr' - rosdistro = os.environ.get('ROS_DISTRO', 'indigo') def prepare_arguments(self, parser): - # Add command line arguments for this generator add = parser.add_argument - add('-i', '--debian-inc', help="debian increment number", default='0') - add('-p', '--prefix', required=True, - help="branch prefix to match, and from which create debians" - " hint: if you want to match 'release/foo' use 'release'") - add('-a', '--match-all', default=False, action="store_true", - help="match all branches with the given prefix, " - "even if not in current upstream") - add('--distros', nargs='+', required=False, default=[], - help='A list of debian (ubuntu) distros to generate for') - add('--install-prefix', default=None, - help="overrides the default installation prefix (/usr)") add('--os-name', default='ubuntu', help="overrides os_name, set to 'ubuntu' by default") add('--os-not-required', default=False, action="store_true", help="Do not error if this os is not in the platforms " "list for rosdistro") + return PackageManagerGenerator.prepare_arguments(self, parser) def handle_arguments(self, args): - self.interactive = args.interactive - self.debian_inc = args.debian_inc self.os_name = args.os_name - self.distros = args.distros - if self.distros in [None, []]: - index = rosdistro.get_index(rosdistro.get_index_url()) - distribution_file = rosdistro.get_distribution_file(index, self.rosdistro) - if self.os_name not in distribution_file.release_platforms: - if args.os_not_required: - warning("No platforms defined for os '{0}' in release file for the " - "'{1}' distro. This os was not required; continuing without error." - .format(self.os_name, self.rosdistro)) - sys.exit(0) - error("No platforms defined for os '{0}' in release file for the '{1}' distro." - .format(self.os_name, self.rosdistro), exit=True) - self.distros = distribution_file.release_platforms[self.os_name] - self.install_prefix = args.install_prefix - if args.install_prefix is None: - self.install_prefix = self.default_install_prefix - self.prefix = args.prefix - self.branches = match_branches_with_prefix(self.prefix, get_branches, prune=not args.match_all) - if len(self.branches) == 0: - error( - "No packages found, check your --prefix or --src arguments.", - exit=True - ) - self.packages = {} - self.tag_names = {} - self.names = [] - self.branch_args = [] - self.debian_branches = [] - for branch in self.branches: - package = get_package_from_branch(branch) - if package is None: - # This is an ignored package - continue - self.packages[package.name] = package - self.names.append(package.name) - args = self.generate_branching_arguments(package, branch) - # First branch is debian/[/] - self.debian_branches.append(args[0][0]) - self.branch_args.extend(args) - - def summarize(self): - info("Generating source debs for the packages: " + str(self.names)) - info("Debian Incremental Version: " + str(self.debian_inc)) - info("Debian Distributions: " + str(self.distros)) - - def get_branching_arguments(self): - return self.branch_args - - def update_rosdep(self): - update_rosdep() - self.has_run_rosdep = True - - def _check_all_keys_are_valid(self, peer_packages, ros_distro): - keys_to_resolve = [] - key_to_packages_which_depends_on = collections.defaultdict(list) - keys_to_ignore = set() - for package in self.packages.values(): - package.evaluate_conditions(package_conditional_context(ros_distro)) - depends = [ - dep for dep in (package.run_depends + package.buildtool_export_depends) - if dep.evaluated_condition] - build_depends = [ - dep for dep in (package.build_depends + package.buildtool_depends + package.test_depends) - if dep.evaluated_condition] - unresolved_keys = [ - dep for dep in (depends + build_depends + package.replaces + package.conflicts) - if dep.evaluated_condition] - keys_to_ignore = { - dep for dep in keys_to_ignore.union(package.replaces + package.conflicts) - if dep.evaluated_condition} - keys = [d.name for d in unresolved_keys] - keys_to_resolve.extend(keys) - for key in keys: - key_to_packages_which_depends_on[key].append(package.name) - - os_name = self.os_name - rosdistro = self.rosdistro - all_keys_valid = True - for key in sorted(set(keys_to_resolve)): - for os_version in self.distros: - try: - extended_peer_packages = peer_packages + [d.name for d in keys_to_ignore] - rule, installer_key, default_installer_key = \ - resolve_rosdep_key(key, os_name, os_version, rosdistro, extended_peer_packages, - retry=False) - if rule is None: - continue - if installer_key != default_installer_key: - error("Key '{0}' resolved to '{1}' with installer '{2}', " - "which does not match the default installer '{3}'." - .format(key, rule, installer_key, default_installer_key)) - BloomGenerator.exit( - "The Debian generator does not support dependencies " - "which are installed with the '{0}' installer." - .format(installer_key), - returncode=code.GENERATOR_INVALID_INSTALLER_KEY) - except (GeneratorError, RuntimeError) as e: - print(fmt("Failed to resolve @{cf}@!{key}@| on @{bf}{os_name}@|:@{cf}@!{os_version}@| with: {e}") - .format(**locals())) - print(fmt("@{cf}@!{0}@| is depended on by these packages: ").format(key) + - str(list(set(key_to_packages_which_depends_on[key])))) - print(fmt("@{kf}@!<== @{rf}@!Failed@|")) - all_keys_valid = False - return all_keys_valid + self.os_not_required = args.os_not_required + ret = PackageManagerGenerator.handle_arguments(self, args) + return ret def pre_modify(self): - info("\nPre-verifying Debian dependency keys...") - # Run rosdep update is needed - if not self.has_run_rosdep: - self.update_rosdep() - - peer_packages = [p.name for p in self.packages.values()] - - while not self._check_all_keys_are_valid(peer_packages, self.rosdistro): - error("Some of the dependencies for packages in this repository could not be resolved by rosdep.") - error("You can try to address the issues which appear above and try again if you wish.") - try: - if not maybe_continue(msg="Would you like to try again?"): - error("User aborted after rosdep keys were not resolved.") - sys.exit(code.GENERATOR_NO_ROSDEP_KEY_FOR_DISTRO) - except (KeyboardInterrupt, EOFError): - error("\nUser quit.", exit=True) - update_rosdep() - invalidate_view_cache() - - info("All keys are " + ansi('greenf') + "OK" + ansi('reset') + "\n") - - def pre_branch(self, destination, source): - if destination in self.debian_branches: - return - # Run rosdep update is needed - if not self.has_run_rosdep: - self.update_rosdep() - # Determine the current package being generated - name = destination.split('/')[-1] - distro = destination.split('/')[-2] - # Retrieve the package - package = self.packages[name] - # Report on this package - self.summarize_package(package, distro) - - def pre_rebase(self, destination): - # Get the stored configs is any - patches_branch = 'patches/' + destination - config = self.load_original_config(patches_branch) - if config is not None: - curr_config = get_patch_config(patches_branch) - if curr_config['parent'] == config['parent']: - set_patch_config(patches_branch, config) - - def post_rebase(self, destination): - name = destination.split('/')[-1] - # Retrieve the package - package = self.packages[name] - # Handle differently if this is a debian vs distro branch - if destination in self.debian_branches: - info("Placing debian template files into '{0}' branch." - .format(destination)) - # Then this is a debian branch - # Place the raw template files - self.place_template_files(package.get_build_type()) - else: - # This is a distro specific debian branch - # Determine the current package being generated - distro = destination.split('/')[-2] - # Create debians for each distro - with inbranch(destination): - data = self.generate_debian(package, distro) - # Create the tag name for later - self.tag_names[destination] = self.generate_tag_name(data) - # Update the patch configs - patches_branch = 'patches/' + destination - config = get_patch_config(patches_branch) - # Store it - self.store_original_config(config, patches_branch) - # Modify the base so import/export patch works - current_branch = get_current_branch() - if current_branch is None: - error("Could not determine current branch.", exit=True) - config['base'] = get_commit_hash(current_branch) - # Set it - set_patch_config(patches_branch, config) - - def post_patch(self, destination, color='bluef'): - if destination in self.debian_branches: - return - # Tag after patches have been applied - with inbranch(destination): - # Tag - tag_name = self.tag_names[destination] - if tag_exists(tag_name): - if self.interactive: - warning("Tag exists: " + tag_name) - warning("Do you wish to overwrite it?") - if not maybe_continue('y'): - error("Answered no to continue, aborting.", exit=True) - else: - warning("Overwriting tag: " + tag_name) - else: - info("Creating tag: " + tag_name) - execute_command('git tag -f ' + tag_name) - # Report of success - name = destination.split('/')[-1] - package = self.packages[name] - distro = destination.split('/')[-2] - info(ansi(color) + "####" + ansi('reset'), use_prefix=False) - info( - ansi(color) + "#### " + ansi('greenf') + "Successfully" + - ansi(color) + " generated '" + ansi('boldon') + distro + - ansi('boldoff') + "' debian for package" - " '" + ansi('boldon') + package.name + ansi('boldoff') + "'" + - " at version '" + ansi('boldon') + package.version + - "-" + str(self.debian_inc) + ansi('boldoff') + "'" + - ansi('reset'), - use_prefix=False - ) - info(ansi(color) + "####\n" + ansi('reset'), use_prefix=False) - - def store_original_config(self, config, patches_branch): - with inbranch(patches_branch): - with open('debian.store', 'w+') as f: - f.write(json.dumps(config)) - execute_command('git add debian.store') - if has_changes(): - execute_command('git commit -m "Store original patch config"') - - def load_original_config(self, patches_branch): - config_store = show(patches_branch, 'debian.store') - if config_store is None: - return config_store - return json.loads(config_store) - - def place_template_files(self, build_type, debian_dir='debian'): - # Create/Clean the debian folder - if os.path.exists(debian_dir): - if self.interactive: - warning("debian directory exists: " + debian_dir) - warning("Do you wish to overwrite it?") - if not maybe_continue('y'): - error("Answered no to continue, aborting.", exit=True) - elif 'BLOOM_CLEAR_DEBIAN_ON_GENERATION' in os.environ: - warning("Overwriting debian directory: " + debian_dir) - execute_command('git rm -rf ' + debian_dir) - execute_command('git commit -m "Clearing previous debian folder"') - if os.path.exists(debian_dir): - shutil.rmtree(debian_dir) - else: - warning("Not overwriting debian directory.") - # Use generic place template files command - place_template_files('.', build_type, gbp=True) - # Commit results - execute_command('git add ' + debian_dir) - _, has_files, _ = execute_command('git diff --cached --name-only', return_io=True) - if has_files: - execute_command('git commit -m "Placing debian template files"') - - def get_releaser_history(self): - # Assumes that this is called in the target branch - patches_branch = 'patches/' + get_current_branch() - raw = show(patches_branch, 'releaser_history.json') - return None if raw is None else json.loads(raw) - - def set_releaser_history(self, history): - # Assumes that this is called in the target branch - patches_branch = 'patches/' + get_current_branch() - debug("Writing release history to '{0}' branch".format(patches_branch)) - with inbranch(patches_branch): - with open('releaser_history.json', 'w') as f: - f.write(json.dumps(history)) - execute_command('git add releaser_history.json') - if has_changes(): - execute_command('git commit -m "Store releaser history"') - - def get_subs(self, package, debian_distro, releaser_history=None): - return generate_substitutions_from_package( - package, - self.os_name, - debian_distro, - self.rosdistro, - self.install_prefix, - self.debian_inc, - [p.name for p in self.packages.values()], - releaser_history=releaser_history, - fallback_resolver=missing_dep_resolver - ) - - def generate_debian(self, package, debian_distro): - info("Generating debian for {0}...".format(debian_distro)) - # Try to retrieve the releaser_history - releaser_history = self.get_releaser_history() + error_msg = ''.join([ + "Some of the dependencies for packages in this repository could not be resolved by rosdep.\n", + "You can try to address the issues which appear above and try again if you wish." + ]) + PackageManagerGenerator.check_all_keys_are_valid(self, error_msg) + + def generate_package(self, package, os_version): + info("Generating {0} for {1}...".format(self.package_manager, os_version)) # Generate substitution values - subs = self.get_subs(package, debian_distro, releaser_history) + subs = self.get_subs(package, os_version, format_description, format_depends) # Use subs to create and store releaser history releaser_history = [(v, (n, e)) for v, _, _, n, e in subs['changelogs']] self.set_releaser_history(dict(releaser_history)) - # Handle gbp.conf - subs['release_tag'] = self.get_release_tag(subs) # Template files - template_files = process_template_files('.', subs) + template_files = process_template_files(".", subs, self.package_manager) # Remove any residual template files execute_command('git rm -rf ' + ' '.join("'{}'".format(t) for t in template_files)) - # Add changes to the debian folder - execute_command('git add debian') + # Add changes to the package system folder + execute_command('git add {0}'.format(self.package_manager)) # Commit changes - execute_command('git commit -m "Generated debian files for ' + - debian_distro + '"') + execute_command('git commit -m "Generated {0} files for {1}"' + .format(self.package_manager, os_version)) # Return the subs for other use return subs - def get_release_tag(self, data): - return 'release/{0}/{1}-{2}'.format(data['Name'], data['Version'], - self.debian_inc) + @staticmethod + def get_subs_hook(subs, package, rosdistro, releaser_history=None): + # Use the time stamp to set the date strings + stamp = datetime.datetime.now(tz.tzlocal()) + subs['Date'] = stamp.strftime('%a, %d %b %Y %T %z') + subs['YYYY'] = stamp.strftime('%Y') + + # Changelog + changelogs = get_changelogs(package, releaser_history) + if changelogs and package.version not in [x[0] for x in changelogs]: + warning("") + warning("A CHANGELOG.rst was found, but no changelog for this version was found.") + warning("You REALLY should have a entry (even a blank one) for each version of your package.") + warning("") + if not changelogs: + # Ensure at least a minimal changelog + changelogs = [] + if package.version not in [x[0] for x in changelogs]: + changelogs.insert(0, ( + package.version, + get_rfc_2822_date(datetime.datetime.now()), + ' * Autogenerated, no changelog for this version found in CHANGELOG.rst.', + package.maintainers[0].name, + package.maintainers[0].email + )) + bad_changelog = False + # Make sure that the first change log is the version being released + if package.version != changelogs[0][0]: + error("") + error("The version of the first changelog entry '{0}' is not the " + "same as the version being currently released '{1}'." + .format(package.version, changelogs[0][0])) + bad_changelog = True + # Make sure that the current version is the latest in the changelog + for changelog in changelogs: + if parse_version(package.version) < parse_version(changelog[0]): + error("") + error("There is at least one changelog entry, '{0}', which has a " + "newer version than the version of package '{1}' being released, '{2}'." + .format(changelog[0], package.name, package.version)) + bad_changelog = True + if bad_changelog: + error("This is almost certainly by mistake, you should really take a " + "look at the changelogs for the package you are releasing.") + error("") + if not maybe_continue('n', 'Continue anyways'): + sys.exit("User quit.") + subs['changelogs'] = changelogs + + # Use debhelper version 7 for oneric, otherwise 9 + subs['debhelper_version'] = 7 if subs['Distribution'] in ['oneiric'] else 9 + + # Copyright + licenses = [] + separator = '\n' + '=' * 80 + '\n\n' + for l in package.licenses: + if hasattr(l, 'file') and l.file is not None: + license_file = os.path.join(os.path.dirname(package.filename), l.file) + if not os.path.exists(license_file): + error("License file '{}' is not found.". + format(license_file), exit=True) + license_text = open(license_file, 'r').read() + if not license_text.endswith('\n'): + license_text += '\n' + licenses.append(license_text) + subs['Copyright'] = separator.join(licenses) - def generate_tag_name(self, data): - tag_name = '{Package}_{Version}{DebianInc}_{Distribution}' - tag_name = 'debian/' + tag_name.format(**data) - return tag_name + # Handle gbp.conf + release_tag = 'release/{0}/{1}-{2}'.format(subs['Name'], subs['Version'], subs['Inc']) + subs['release_tag'] = release_tag - def generate_branching_arguments(self, package, branch): - n = package.name - # Debian branch - deb_branch = 'debian/' + n - # Branch first to the debian branch - args = [[deb_branch, branch, False]] - # Then for each debian distro, branch from the base debian branch - args.extend([ - ['debian/' + d + '/' + n, deb_branch, False] for d in self.distros - ]) - return args - - def summarize_package(self, package, distro, color='bluef'): - info(ansi(color) + "\n####" + ansi('reset'), use_prefix=False) - info( - ansi(color) + "#### Generating '" + ansi('boldon') + distro + - ansi('boldoff') + "' debian for package" - " '" + ansi('boldon') + package.name + ansi('boldoff') + "'" + - " at version '" + ansi('boldon') + package.version + - "-" + str(self.debian_inc) + ansi('boldoff') + "'" + - ansi('reset'), - use_prefix=False - ) - info(ansi(color) + "####" + ansi('reset'), use_prefix=False) + # Debian Package Format + subs['format'] = 'quilt' + + return subs + + @staticmethod + def generate_tag_name(subs): + tag_name = '{Package}_{Version}-{Inc}_{Distribution}' + tag_name = DebianGenerator.package_manager + '/' + tag_name.format(**subs) + return tag_name diff --git a/bloom/generators/debian/templates/ament_cmake/changelog.em b/bloom/generators/debian/templates/ament_cmake/changelog.em index 35859090..9c4ed62c 100644 --- a/bloom/generators/debian/templates/ament_cmake/changelog.em +++ b/bloom/generators/debian/templates/ament_cmake/changelog.em @@ -1,4 +1,4 @@ -@[for change_version, change_date, changelog, main_name, main_email in changelogs]@(Package) (@(change_version)@(DebianInc)@(Distribution)) @(Distribution); urgency=high +@[for change_version, change_date, changelog, main_name, main_email in changelogs]@(Package) (@(change_version)@(Inc)@(Distribution)) @(Distribution); urgency=high @(changelog) diff --git a/bloom/generators/debian/templates/ament_python/changelog.em b/bloom/generators/debian/templates/ament_python/changelog.em index 35859090..9c4ed62c 100644 --- a/bloom/generators/debian/templates/ament_python/changelog.em +++ b/bloom/generators/debian/templates/ament_python/changelog.em @@ -1,4 +1,4 @@ -@[for change_version, change_date, changelog, main_name, main_email in changelogs]@(Package) (@(change_version)@(DebianInc)@(Distribution)) @(Distribution); urgency=high +@[for change_version, change_date, changelog, main_name, main_email in changelogs]@(Package) (@(change_version)@(Inc)@(Distribution)) @(Distribution); urgency=high @(changelog) diff --git a/bloom/generators/debian/templates/catkin/changelog.em b/bloom/generators/debian/templates/catkin/changelog.em index 35859090..9c4ed62c 100644 --- a/bloom/generators/debian/templates/catkin/changelog.em +++ b/bloom/generators/debian/templates/catkin/changelog.em @@ -1,4 +1,4 @@ -@[for change_version, change_date, changelog, main_name, main_email in changelogs]@(Package) (@(change_version)@(DebianInc)@(Distribution)) @(Distribution); urgency=high +@[for change_version, change_date, changelog, main_name, main_email in changelogs]@(Package) (@(change_version)@(Inc)@(Distribution)) @(Distribution); urgency=high @(changelog) diff --git a/bloom/generators/debian/templates/cmake/changelog.em b/bloom/generators/debian/templates/cmake/changelog.em index 35859090..9c4ed62c 100644 --- a/bloom/generators/debian/templates/cmake/changelog.em +++ b/bloom/generators/debian/templates/cmake/changelog.em @@ -1,4 +1,4 @@ -@[for change_version, change_date, changelog, main_name, main_email in changelogs]@(Package) (@(change_version)@(DebianInc)@(Distribution)) @(Distribution); urgency=high +@[for change_version, change_date, changelog, main_name, main_email in changelogs]@(Package) (@(change_version)@(Inc)@(Distribution)) @(Distribution); urgency=high @(changelog) diff --git a/bloom/generators/release.py b/bloom/generators/release.py index 067edfcb..86f5e8d5 100755 --- a/bloom/generators/release.py +++ b/bloom/generators/release.py @@ -97,11 +97,9 @@ def handle_arguments(self, args): self.src = args.src self.name = args.name self.release_inc = args.release_increment + self.branch_list = self.detect_branches() def summarize(self): - self.branch_list = self.detect_branches() - if type(self.branch_list) not in [list, tuple]: - self.exit(self.branch_list if self.branch_list is not None else 1) info("Releasing package" + ('' if len(self.branch_list) == 1 else 's') + ": " + str(self.branch_list)) @@ -135,14 +133,14 @@ def post_patch(self, destination): Cannot automatically tag the release because this is not a catkin project.""") warning("""\ Please checkout the release branch and then create a tag manually with:""") - warning(" git checkout release/" + str(self.name)) - warning(" git tag -f release/" + str(self.name) + "/") + warning(" git checkout " + destination) + warning(" git tag -f " + destination + "/") return with inbranch(destination): name, version, packages = get_package_data(destination) # Execute git tag - release_tag = destination + '/' + version + '-' + self.release_inc - execute_command('git tag ' + release_tag) + execute_command('git tag -f ' + destination + '/' + version + + '-' + str(self.release_inc)) def metapackage_check(self, path, pkg): if pkg.is_metapackage(): diff --git a/bloom/generators/rosdebian.py b/bloom/generators/rosdebian.py index 12e217c4..af03b161 100644 --- a/bloom/generators/rosdebian.py +++ b/bloom/generators/rosdebian.py @@ -34,11 +34,12 @@ from __future__ import print_function from bloom.generators.common import default_fallback_resolver - -from bloom.generators.debian.generator import sanitize_package_name +from bloom.generators.common import generate_substitutions_from_package +from bloom.generators.common import sanitize_package_name from bloom.generators.debian import DebianGenerator -from bloom.generators.debian.generator import generate_substitutions_from_package +from bloom.generators.debian import format_description +from bloom.generators.debian import format_depends from bloom.generators.debian.generate_cmd import main as debian_main from bloom.generators.debian.generate_cmd import prepare_arguments @@ -71,40 +72,33 @@ def summarize(self): info("Releasing for rosdistro: " + self.rosdistro) return ret - def get_subs(self, package, debian_distro, releaser_history, native=False): - def fallback_resolver(key, peer_packages, rosdistro=self.rosdistro): - if key in peer_packages: - return [sanitize_package_name(rosify_package_name(key, rosdistro))] - return default_fallback_resolver(key, peer_packages) - subs = generate_substitutions_from_package( - package, - self.os_name, - debian_distro, - self.rosdistro, - self.install_prefix, - self.debian_inc, - [p.name for p in self.packages.values()], - releaser_history=releaser_history, - fallback_resolver=fallback_resolver - ) - subs['Package'] = rosify_package_name(subs['Package'], self.rosdistro) + @staticmethod + def missing_dep_resolver(key, peer_packages, os_name, os_version, ros_distro): + if key in peer_packages: + return [sanitize_package_name(rosify_package_name(key, ros_distro))] + return default_fallback_resolver(key, peer_packages) + + @staticmethod + def get_subs_hook(subs, package, rosdistro, releaser_history=None): + subs = DebianGenerator.get_subs_hook(subs, package, rosdistro, releaser_history=releaser_history) + subs['Package'] = rosify_package_name(subs['Package'], rosdistro) # ROS 2 specific bloom extensions. ros2_distros = [ name for name, values in get_index().distributions.items() if values.get('distribution_type') == 'ros2'] - if self.rosdistro in ros2_distros: + if rosdistro in ros2_distros: # Add ros-workspace package as a dependency to any package other # than ros_workspace and its dependencies. if package.name not in ['ament_cmake_core', 'ament_package', 'ros_workspace']: - workspace_pkg_name = rosify_package_name('ros-workspace', self.rosdistro) + workspace_pkg_name = rosify_package_name('ros-workspace', rosdistro) subs['BuildDepends'].append(workspace_pkg_name) subs['Depends'].append(workspace_pkg_name) # Add packages necessary to build vendor typesupport for rosidl_interface_packages to their # build dependencies. - if self.rosdistro in ros2_distros and \ - self.rosdistro not in ('r2b2', 'r2b3', 'ardent') and \ + if rosdistro in ros2_distros and \ + rosdistro not in ('r2b2', 'r2b3', 'ardent') and \ 'rosidl_interface_packages' in [p.name for p in package.member_of_groups]: ROS2_VENDOR_TYPESUPPORT_DEPENDENCIES = [ 'rmw-connext-cpp', @@ -118,21 +112,21 @@ def fallback_resolver(key, peer_packages, rosdistro=self.rosdistro): ] subs['BuildDepends'] += [ - rosify_package_name(name, self.rosdistro) for name in ROS2_VENDOR_TYPESUPPORT_DEPENDENCIES] + rosify_package_name(name, rosdistro) for name in ROS2_VENDOR_TYPESUPPORT_DEPENDENCIES] + return subs def generate_branching_arguments(self, package, branch): - deb_branch = 'debian/' + self.rosdistro + '/' + package.name - args = [[deb_branch, branch, False]] - n, r, b, ds = package.name, self.rosdistro, deb_branch, self.distros + package_branch = self.package_manager + '/' + self.rosdistro + '/' + package.name + args = [[package_branch, branch, False]] + n, r, b, ds = package.name, self.rosdistro, package_branch, self.distros args.extend([ - ['debian/' + r + '/' + d + '/' + n, b, False] for d in ds + [self.package_manager + '/' + r + '/' + d + '/' + n, b, False] for d in ds ]) return args def get_release_tag(self, data): - return 'release/{0}/{1}/{2}-{3}'\ - .format(self.rosdistro, data['Name'], data['Version'], self.debian_inc) + return 'release/{0}/{1}/{2}-{3}'.format(self.rosdistro, data['Name'], data['Version'], self.inc) def rosify_package_name(name, rosdistro): @@ -146,10 +140,13 @@ def get_subs(pkg, os_name, os_version, ros_distro, native): os_name, os_version, ros_distro, + format_description, + format_depends, RosDebianGenerator.default_install_prefix + ros_distro, - native=native ) - subs['Package'] = rosify_package_name(subs['Package'], ros_distro) + subs = RosDebianGenerator.get_subs_hook(subs, pkg, ros_distro) + # Debian Package Format + subs['format'] = 'native' if native else 'quilt' return subs diff --git a/bloom/generators/rosrelease.py b/bloom/generators/rosrelease.py index f7f09acd..71bde0eb 100644 --- a/bloom/generators/rosrelease.py +++ b/bloom/generators/rosrelease.py @@ -47,37 +47,3 @@ def pre_rebase(self, destination): name, self.rosdistro, destination ) ) - - def post_patch(self, destination): - # Figure out the version of the given package - if self.name is not None: - warning("""\ -Cannot automatically tag the release because this is not a catkin project.""") - warning("""\ -Please checkout the release branch and then create a tag manually with:""") - warning(" git checkout " + destination) - warning(" git tag -f " + destination + "/") - return - with inbranch(destination): - name, version, packages = get_package_data(destination) - # Execute git tag - execute_command('git tag -f ' + destination + '/' + version + - '-' + str(self.release_inc)) - - def detect_branches(self): - self.packages = None - with inbranch(self.src): - if self.name is not None: - self.packages = [self.name] - return [self.name] - package_data = get_package_data(self.src) - if type(package_data) not in [list, tuple]: - return package_data - name, version, packages = package_data - self.packages = packages - # Check meta packages for valid CMakeLists.txt - if isinstance(self.packages, dict): - for path, pkg in self.packages.items(): - # Check for valid CMakeLists.txt if a metapackage - self.metapackage_check(path, pkg) - return name if type(name) is list else [name] diff --git a/bloom/generators/rosrpm.py b/bloom/generators/rosrpm.py index ad8ae70f..3dc3b4fc 100644 --- a/bloom/generators/rosrpm.py +++ b/bloom/generators/rosrpm.py @@ -34,11 +34,13 @@ from __future__ import print_function from bloom.generators.common import default_fallback_resolver - -from bloom.generators.rpm.generator import sanitize_package_name +from bloom.generators.common import generate_substitutions_from_package +from bloom.generators.common import sanitize_package_name from bloom.generators.rpm import RpmGenerator -from bloom.generators.rpm.generator import generate_substitutions_from_package +from bloom.generators.rpm import format_depends +from bloom.generators.rpm import format_description + from bloom.generators.rpm.generate_cmd import main as rpm_main from bloom.generators.rpm.generate_cmd import prepare_arguments @@ -69,37 +71,29 @@ def summarize(self): info("Releasing for rosdistro: " + self.rosdistro) return ret - def get_subs(self, package, rpm_distro, releaser_history): - def fallback_resolver(key, peer_packages, rosdistro=self.rosdistro): - if key in peer_packages: - return [sanitize_package_name(rosify_package_name(key, rosdistro))] - return default_fallback_resolver(key, peer_packages) - subs = generate_substitutions_from_package( - package, - self.os_name, - rpm_distro, - self.rosdistro, - self.install_prefix, - self.rpm_inc, - [p.name for p in self.packages.values()], - releaser_history=releaser_history, - fallback_resolver=fallback_resolver - ) - subs['Package'] = rosify_package_name(subs['Package'], self.rosdistro) + @staticmethod + def missing_dep_resolver(key, peer_packages, os_name, os_version, ros_distro): + if key in peer_packages: + return [sanitize_package_name(rosify_package_name(key, ros_distro))] + return default_fallback_resolver(key, peer_packages) + + @staticmethod + def get_subs_hook(subs, package, rosdistro, releaser_history=None): + subs = RpmGenerator.get_subs_hook(subs, package, rosdistro, releaser_history=releaser_history) + subs['Package'] = rosify_package_name(subs['Package'], rosdistro) return subs def generate_branching_arguments(self, package, branch): - rpm_branch = 'rpm/' + self.rosdistro + '/' + package.name - args = [[rpm_branch, branch, False]] - n, r, b, ds = package.name, self.rosdistro, rpm_branch, self.distros + package_branch = self.package_manager + '/' + self.rosdistro + '/' + package.name + args = [[package_branch, branch, False]] + n, r, b, ds = package.name, self.rosdistro, package_branch, self.distros args.extend([ - ['rpm/' + r + '/' + d + '/' + n, b, False] for d in ds + [self.package_manager + '/' + r + '/' + d + '/' + n, b, False] for d in ds ]) return args def get_release_tag(self, data): - return 'release/{0}/{1}/{2}-{3}'\ - .format(self.rosdistro, data['Name'], data['Version'], self.rpm_inc) + return 'release/{0}/{1}/{2}-{3}'.format(self.rosdistro, data['Name'], data['Version'], self.inc) def rosify_package_name(name, rosdistro): @@ -112,9 +106,12 @@ def get_subs(pkg, os_name, os_version, ros_distro): pkg, os_name, os_version, - ros_distro + ros_distro, + format_description, + format_depends, + RosRpmGenerator.default_install_prefix + ros_distro, ) - subs['Package'] = rosify_package_name(subs['Package'], ros_distro) + subs = RosRpmGenerator.get_subs_hook(subs, pkg, ros_distro) return subs diff --git a/bloom/generators/rosvcpkg.py b/bloom/generators/rosvcpkg.py new file mode 100644 index 00000000..91c5f08b --- /dev/null +++ b/bloom/generators/rosvcpkg.py @@ -0,0 +1,159 @@ +# Software License Agreement (BSD License) +# +# Copyright (c) 2013, Open Source Robotics Foundation, Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Open Source Robotics Foundation, Inc. nor +# the names of its contributors may be used to endorse or promote +# products derived from this software without specific prior +# written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN +# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +from __future__ import print_function + +from bloom.generators.common import default_fallback_resolver +from bloom.generators.common import generate_substitutions_from_package +from bloom.generators.common import sanitize_package_name + +from bloom.generators.vcpkg import format_description +from bloom.generators.vcpkg import format_depend +from bloom.generators.vcpkg import format_depends +from bloom.generators.vcpkg import VcpkgGenerator + +from bloom.generators.vcpkg.generate_cmd import main as vcpkg_main +from bloom.generators.vcpkg.generate_cmd import prepare_arguments + +from bloom.logging import info + +from bloom.rosdistro_api import get_index + +from bloom.util import get_distro_list_prompt + + +class RosVcpkgGenerator(VcpkgGenerator): + title = 'rosvcpkg' + description = "Generates vcpkg tailored for the given rosdistro" + default_install_prefix = "c:\\opt\\ros" + + def prepare_arguments(self, parser): + # Add command line arguments for this generator + add = parser.add_argument + add('rosdistro', help="ROS distro to target (%s, etc.)" % get_distro_list_prompt()) + return VcpkgGenerator.prepare_arguments(self, parser) + + def handle_arguments(self, args): + self.rosdistro = args.rosdistro + self.default_install_prefix += self.rosdistro + ret = VcpkgGenerator.handle_arguments(self, args) + return ret + + def summarize(self): + ret = VcpkgGenerator.summarize(self) + info("Releasing for rosdistro: " + self.rosdistro) + return ret + + @staticmethod + def missing_dep_resolver(key, peer_packages, os_name, os_version, ros_distro): + if key in peer_packages: + return [sanitize_package_name(rosify_package_name(key, ros_distro))] + return default_fallback_resolver(key, peer_packages, ros_distro) + + @staticmethod + def get_subs_hook(subs, package, rosdistro, releaser_history=None): + subs['Package'] = rosify_package_name(subs['Package'], rosdistro) + + # ROS 2 specific bloom extensions. + ros2_distros = [ + name for name, values in get_index().distributions.items() + if values.get('distribution_type') == 'ros2'] + if rosdistro in ros2_distros: + if package.name not in ['ament_cmake_core', 'ament_package', 'ros_workspace']: + workspace_formatted = format_depend(rosify_package_name("ros-workspace", rosdistro)) + subs['BuildDepends'].append(workspace_formatted) + subs['Depends'].append(workspace_formatted) + + # Add packages necessary to build vendor typesupport for rosidl_interface_packages to their + # build dependencies. + if rosdistro in ros2_distros and \ + rosdistro not in ('r2b2', 'r2b3', 'ardent') and \ + 'rosidl_interface_packages' in [p.name for p in package.member_of_groups]: + ROS2_VENDOR_TYPESUPPORT_DEPENDENCIES = [ + 'rmw-connext-cpp', + 'rmw-fastrtps-cpp', + 'rmw-implementation', + 'rmw-opensplice-cpp', + 'rosidl-typesupport-connext-c', + 'rosidl-typesupport-connext-cpp', + 'rosidl-typesupport-opensplice-c', + 'rosidl-typesupport-opensplice-cpp', + ] + + subs['BuildDepends'] += [ + format_depend(rosify_package_name(name, rosdistro)) + for name in ROS2_VENDOR_TYPESUPPORT_DEPENDENCIES] + + subs = VcpkgGenerator.get_subs_hook(subs, package, rosdistro, releaser_history=releaser_history) + return subs + + def generate_branching_arguments(self, package, branch): + package_branch = self.package_manager + '/' + self.rosdistro + '/' + package.name + args = [[package_branch, branch, False]] + n, r, b, ds = package.name, self.rosdistro, package_branch, self.distros + args.extend([ + [self.package_manager + '/' + r + '/' + d + '/' + n, b, False] for d in ds + ]) + return args + + +def rosify_package_name(name, rosdistro): + return 'ros-{0}-{1}'.format(rosdistro, name) + + +def get_subs(pkg, os_name, os_version, ros_distro): + # No fallback_resolver provided because peer packages not considered. + subs = generate_substitutions_from_package( + pkg, + os_name, + os_version, + ros_distro, + format_description, + format_depends, + RosVcpkgGenerator.default_install_prefix + ros_distro, + ) + subs = RosVcpkgGenerator.get_subs_hook(subs, pkg, ros_distro) + return subs + + +def main(args=None): + vcpkg_main(args, get_subs) + + +# This describes this command to the loader +description = dict( + title='rosvcpkg', + description="Generates ROS style vcpkg packaging files for a catkin package", + main=main, + prepare_arguments=prepare_arguments +) diff --git a/bloom/generators/rpm/__init__.py b/bloom/generators/rpm/__init__.py index ffa3421a..0632bc9f 100755 --- a/bloom/generators/rpm/__init__.py +++ b/bloom/generators/rpm/__init__.py @@ -1,4 +1,5 @@ from .generator import RpmGenerator -from .generator import sanitize_package_name +from .generator import format_depends +from .generator import format_description -__all__ = ['RpmGenerator', 'sanitize_package_name'] +__all__ = ['RpmGenerator', 'format_description', 'format_depends'] diff --git a/bloom/generators/rpm/generate_cmd.py b/bloom/generators/rpm/generate_cmd.py index 4d91e88b..cf01501d 100644 --- a/bloom/generators/rpm/generate_cmd.py +++ b/bloom/generators/rpm/generate_cmd.py @@ -42,9 +42,13 @@ from bloom.logging import fmt from bloom.logging import info -from bloom.generators.rpm.generator import generate_substitutions_from_package -from bloom.generators.rpm.generator import place_template_files -from bloom.generators.rpm.generator import process_template_files +from bloom.generators.common import generate_substitutions_from_package +from bloom.generators.common import place_template_files +from bloom.generators.common import process_template_files + +from bloom.generators.rpm.generator import RpmGenerator +from bloom.generators.rpm.generator import format_description +from bloom.generators.rpm.generator import format_depends from bloom.util import get_distro_list_prompt @@ -79,12 +83,18 @@ def prepare_arguments(parser): def get_subs(pkg, os_name, os_version, ros_distro): - return generate_substitutions_from_package( + # No fallback_resolver provided because peer packages not considered. + subs = generate_substitutions_from_package( pkg, os_name, os_version, - ros_distro - ) + ros_distro, + format_description, + format_depends, + RpmGenerator.default_install_prefix + ) + subs = RpmGenerator.get_subs_hook(subs, pkg, ros_distro) + return subs def main(args=None, get_subs_fn=None): @@ -118,20 +128,21 @@ def main(args=None, get_subs_fn=None): fmt("Generating RPMs for @{cf}%s:%s@| for package(s) %s" % (os_name, os_version, [p.name for p in pkgs_dict.values()]))) + package_manager = RpmGenerator.package_manager for path, pkg in pkgs_dict.items(): template_files = None try: subs = get_subs_fn(pkg, os_name, os_version, ros_distro) if _place_template_files: # Place template files - place_template_files(path, pkg.get_build_type()) + place_template_files(path, pkg.get_build_type(), package_manager) if _process_template_files: # Just process existing template files - template_files = process_template_files(path, subs) + template_files = process_template_files(path, subs, package_manager) if not _place_template_files and not _process_template_files: # If neither, do both - place_template_files(path, pkg.get_build_type()) - template_files = process_template_files(path, subs) + place_template_files(path, pkg.get_build_type(), package_manager) + template_files = process_template_files(path, subs, package_manager) if template_files is not None: for template_file in template_files: os.remove(os.path.normpath(template_file)) diff --git a/bloom/generators/rpm/generator.py b/bloom/generators/rpm/generator.py index ddb676d5..3883dec4 100644 --- a/bloom/generators/rpm/generator.py +++ b/bloom/generators/rpm/generator.py @@ -32,143 +32,26 @@ from __future__ import print_function -import collections import datetime -import io -import json -import os -import pkg_resources import re -import shutil -import sys -import traceback import textwrap -# Python 2/3 support. -try: - from configparser import SafeConfigParser -except ImportError: - from ConfigParser import SafeConfigParser from dateutil import tz from distutils.version import LooseVersion from time import strptime -from bloom.generators import BloomGenerator -from bloom.generators import GeneratorError -from bloom.generators import resolve_dependencies -from bloom.generators import update_rosdep +from bloom.generators.common import PackageManagerGenerator +from bloom.generators.common import process_template_files -from bloom.generators.common import default_fallback_resolver -from bloom.generators.common import invalidate_view_cache -from bloom.generators.common import package_conditional_context -from bloom.generators.common import resolve_rosdep_key - -from bloom.git import inbranch -from bloom.git import get_branches -from bloom.git import get_commit_hash -from bloom.git import get_current_branch -from bloom.git import has_changes -from bloom.git import show -from bloom.git import tag_exists - -from bloom.logging import ansi -from bloom.logging import debug from bloom.logging import enable_drop_first_log_prefix from bloom.logging import error -from bloom.logging import fmt from bloom.logging import info -from bloom.logging import warning - -from bloom.commands.git.patch.common import get_patch_config -from bloom.commands.git.patch.common import set_patch_config - -from bloom.packages import get_package_data -from bloom.util import code from bloom.util import execute_command -from bloom.util import maybe_continue - -try: - import rosdistro -except ImportError as err: - debug(traceback.format_exc()) - error("rosdistro was not detected, please install it.", exit=True) - -try: - import em -except ImportError: - debug(traceback.format_exc()) - error("empy was not detected, please install it.", exit=True) # Drop the first log prefix for this command enable_drop_first_log_prefix(True) -TEMPLATE_EXTENSION = '.em' - - -def __place_template_folder(group, src, dst, gbp=False): - template_files = pkg_resources.resource_listdir(group, src) - # For each template, place - for template_file in template_files: - template_path = os.path.join(src, template_file) - template_dst = os.path.join(dst, template_file) - if pkg_resources.resource_isdir(group, template_path): - debug("Recursing on folder '{0}'".format(template_path)) - __place_template_folder(group, template_path, template_dst, gbp) - else: - try: - debug("Placing template '{0}'".format(template_path)) - template = pkg_resources.resource_string(group, template_path) - template_abs_path = pkg_resources.resource_filename(group, template_path) - except IOError as err: - error("Failed to load template " - "'{0}': {1}".format(template_file, str(err)), exit=True) - if not os.path.exists(dst): - os.makedirs(dst) - if os.path.exists(template_dst): - debug("Removing existing file '{0}'".format(template_dst)) - os.remove(template_dst) - with open(template_dst, 'w') as f: - if not isinstance(template, str): - template = template.decode('utf-8') - f.write(template) - shutil.copystat(template_abs_path, template_dst) - - -def place_template_files(path, build_type, gbp=False): - info(fmt("@!@{bf}==>@| Placing templates files in the 'rpm' folder.")) - rpm_path = os.path.join(path, 'rpm') - # Create/Clean the rpm folder - if not os.path.exists(rpm_path): - os.makedirs(rpm_path) - # Place template files - group = 'bloom.generators.rpm' - templates = os.path.join('templates', build_type) - __place_template_folder(group, templates, rpm_path, gbp) - - -def summarize_dependency_mapping(data, deps, build_deps, resolved_deps): - if len(deps) == 0 and len(build_deps) == 0: - return - info("Package '" + data['Package'] + "' has dependencies:") - header = " " + ansi('boldoff') + ansi('ulon') + \ - "rosdep key => " + data['Distribution'] + \ - " key" + ansi('reset') - template = " " + ansi('cyanf') + "{0:<20} " + ansi('purplef') + \ - "=> " + ansi('cyanf') + "{1}" + ansi('reset') - if len(deps) != 0: - info(ansi('purplef') + "Run Dependencies:" + - ansi('reset')) - info(header) - for key in [d.name for d in deps]: - info(template.format(key, resolved_deps[key])) - if len(build_deps) != 0: - info(ansi('purplef') + - "Build and Build Tool Dependencies:" + ansi('reset')) - info(header) - for key in [d.name for d in build_deps]: - info(template.format(key, resolved_deps[key])) - def format_depends(depends, resolved_deps): versions = { @@ -193,258 +76,7 @@ def format_depends(depends, resolved_deps): return formatted -def missing_dep_resolver(key, peer_packages): - if key in peer_packages: - return [sanitize_package_name(key)] - return default_fallback_resolver(key, peer_packages) - - -def generate_substitutions_from_package( - package, - os_name, - os_version, - ros_distro, - installation_prefix='/usr', - rpm_inc=0, - peer_packages=None, - releaser_history=None, - fallback_resolver=None -): - peer_packages = peer_packages or [] - data = {} - # Name, Version, Description - data['Name'] = package.name - data['Version'] = package.version - data['Description'] = rpmify_string(package.description) - # License - if not package.licenses or not package.licenses[0]: - error("No license set for package '{0}', aborting.".format(package.name), exit=True) - data['License'] = package.licenses[0] - # Websites - websites = [str(url) for url in package.urls if url.type == 'website'] - data['Homepage'] = websites[0] if websites else '' - if data['Homepage'] == '': - warning("No homepage set") - # RPM Increment Number - data['RPMInc'] = rpm_inc - # Package name - data['Package'] = sanitize_package_name(package.name) - # Installation prefix - data['InstallationPrefix'] = installation_prefix - # Resolve dependencies - package.evaluate_conditions(package_conditional_context(ros_distro)) - depends = [ - dep for dep in (package.run_depends + package.buildtool_export_depends) - if dep.evaluated_condition] - build_depends = [ - dep for dep in (package.build_depends + package.buildtool_depends + package.test_depends) - if dep.evaluated_condition] - unresolved_keys = [ - dep for dep in (depends + build_depends + package.replaces + package.conflicts) - if dep.evaluated_condition] - # The installer key is not considered here, but it is checked when the keys are checked before this - resolved_deps = resolve_dependencies(unresolved_keys, os_name, - os_version, ros_distro, - peer_packages + [d.name for d in package.replaces + package.conflicts], - fallback_resolver) - data['Depends'] = sorted( - set(format_depends(depends, resolved_deps)) - ) - data['BuildDepends'] = sorted( - set(format_depends(build_depends, resolved_deps)) - ) - data['Replaces'] = sorted( - set(format_depends(package.replaces, resolved_deps)) - ) - data['Conflicts'] = sorted( - set(format_depends(package.conflicts, resolved_deps)) - ) - - # Build-type specific substitutions. - build_type = package.get_build_type() - if build_type == 'catkin': - pass - elif build_type == 'cmake': - pass - elif build_type == 'ament_cmake': - error( - "Build type '{}' is not supported by this version of bloom.". - format(build_type), exit=True) - elif build_type == 'ament_python': - error( - "Build type '{}' is not supported by this version of bloom.". - format(build_type), exit=True) - # Don't set the install-scripts flag if it's already set in setup.cfg. - package_path = os.path.abspath(os.path.dirname(package.filename)) - setup_cfg_path = os.path.join(package_path, 'setup.cfg') - data['pass_install_scripts'] = True - if os.path.isfile(setup_cfg_path): - setup_cfg = SafeConfigParser() - setup_cfg.read([setup_cfg_path]) - if ( - setup_cfg.has_option('install', 'install-scripts') or - setup_cfg.has_option('install', 'install_scripts') - ): - data['pass_install_scripts'] = False - else: - error( - "Build type '{}' is not supported by this version of bloom.". - format(build_type), exit=True) - - # Set the distribution - data['Distribution'] = os_version - # Use the time stamp to set the date strings - stamp = datetime.datetime.now(tz.tzlocal()) - data['Date'] = stamp.strftime('%a %b %d %Y') - # Maintainers - maintainers = [] - for m in package.maintainers: - maintainers.append(str(m)) - data['Maintainer'] = maintainers[0] - data['Maintainers'] = ', '.join(maintainers) - # Changelog - if releaser_history: - sorted_releaser_history = sorted(releaser_history, - key=lambda k: LooseVersion(k), reverse=True) - sorted_releaser_history = sorted(sorted_releaser_history, - key=lambda k: strptime(releaser_history.get(k)[0], '%a %b %d %Y'), - reverse=True) - changelogs = [(v, releaser_history[v]) for v in sorted_releaser_history] - else: - # Ensure at least a minimal changelog - changelogs = [] - if package.version + '-' + str(rpm_inc) not in [x[0] for x in changelogs]: - changelogs.insert(0, ( - package.version + '-' + str(rpm_inc), ( - data['Date'], - package.maintainers[0].name, - package.maintainers[0].email - ) - )) - exported_tags = [e.tagname for e in package.exports] - data['NoArch'] = 'metapackage' in exported_tags or 'architecture_independent' in exported_tags - data['changelogs'] = changelogs - # Summarize dependencies - summarize_dependency_mapping(data, depends, build_depends, resolved_deps) - - def convertToUnicode(obj): - if sys.version_info.major == 2: - if isinstance(obj, str): - return unicode(obj.decode('utf8')) - elif isinstance(obj, unicode): - return obj - else: - if isinstance(obj, bytes): - return str(obj.decode('utf8')) - elif isinstance(obj, str): - return obj - if isinstance(obj, list): - for i, val in enumerate(obj): - obj[i] = convertToUnicode(val) - return obj - elif isinstance(obj, type(None)): - return None - elif isinstance(obj, tuple): - obj_tmp = list(obj) - for i, val in enumerate(obj_tmp): - obj_tmp[i] = convertToUnicode(obj_tmp[i]) - return tuple(obj_tmp) - elif isinstance(obj, int): - return obj - elif isinstance(obj, int): - return obj - raise RuntimeError('need to deal with type %s' % (str(type(obj)))) - - for item in data.items(): - data[item[0]] = convertToUnicode(item[1]) - - return data - - -def __process_template_folder(path, subs): - items = os.listdir(path) - processed_items = [] - for item in list(items): - item = os.path.abspath(os.path.join(path, item)) - if os.path.basename(item) in ['.', '..', '.git', '.svn']: - continue - if os.path.isdir(item): - sub_items = __process_template_folder(item, subs) - processed_items.extend([os.path.join(item, s) for s in sub_items]) - if not item.endswith(TEMPLATE_EXTENSION): - continue - with open(item, 'r') as f: - template = f.read() - # Remove extension - template_path = item[:-len(TEMPLATE_EXTENSION)] - # Expand template - info("Expanding '{0}' -> '{1}'".format( - os.path.relpath(item), - os.path.relpath(template_path))) - result = em.expand(template, **subs) - # Write the result - with io.open(template_path, 'w', encoding='utf-8') as f: - if sys.version_info.major == 2: - result = result.decode('utf-8') - f.write(result) - # Copy the permissions - shutil.copymode(item, template_path) - processed_items.append(item) - return processed_items - - -def process_template_files(path, subs): - info(fmt("@!@{bf}==>@| In place processing templates in 'rpm' folder.")) - rpm_dir = os.path.join(path, 'rpm') - if not os.path.exists(rpm_dir): - sys.exit("No rpm directory found at '{0}', cannot process templates." - .format(rpm_dir)) - return __process_template_folder(rpm_dir, subs) - - -def match_branches_with_prefix(prefix, get_branches, prune=False): - debug("match_branches_with_prefix(" + str(prefix) + ", " + - str(get_branches()) + ")") - branches = [] - # Match branches - existing_branches = get_branches() - for branch in existing_branches: - if branch.startswith('remotes/origin/'): - branch = branch.split('/', 2)[-1] - if branch.startswith(prefix): - branches.append(branch) - branches = list(set(branches)) - if prune: - # Prune listed branches by packages in latest upstream - with inbranch('upstream'): - pkg_names, version, pkgs_dict = get_package_data('upstream') - for branch in branches: - if branch.split(prefix)[-1].strip('/') not in pkg_names: - branches.remove(branch) - return branches - - -def get_package_from_branch(branch): - with inbranch(branch): - try: - package_data = get_package_data(branch) - except SystemExit: - return None - if type(package_data) not in [list, tuple]: - # It is a ret code - RpmGenerator.exit(package_data) - names, version, packages = package_data - if type(names) is list and len(names) > 1: - RpmGenerator.exit( - "RPM generator does not support generating " - "from branches with multiple packages in them, use " - "the release generator first to split packages into " - "individual branches.") - if type(packages) is dict: - return list(packages.values())[0] - - -def rpmify_string(value): +def format_description(value): markup_remover = re.compile(r'<.*?>') value = markup_remover.sub('', value) value = re.sub('\s+', ' ', value) @@ -453,375 +85,102 @@ def rpmify_string(value): return value -def sanitize_package_name(name): - return name.replace('_', '-') - - -class RpmGenerator(BloomGenerator): +class RpmGenerator(PackageManagerGenerator): title = 'rpm' + package_manager = 'rpm' description = "Generates RPMs from the catkin meta data" - has_run_rosdep = False - default_install_prefix = '/usr' - rosdistro = os.environ.get('ROS_DISTRO', 'indigo') def prepare_arguments(self, parser): # Add command line arguments for this generator add = parser.add_argument - add('-i', '--rpm-inc', help="RPM increment number", default='0') - add('-p', '--prefix', required=True, - help="branch prefix to match, and from which create RPMs" - " hint: if you want to match 'release/foo' use 'release'") - add('-a', '--match-all', default=False, action="store_true", - help="match all branches with the given prefix, " - "even if not in current upstream") - add('--distros', nargs='+', required=False, default=[], - help='A list of RPM (fedora) distros to generate for') - add('--install-prefix', default=None, - help="overrides the default installation prefix (/usr)") add('--os-name', default='fedora', help="overrides os_name, set to 'fedora' by default") + return PackageManagerGenerator.prepare_arguments(self, parser) def handle_arguments(self, args): - self.interactive = args.interactive - self.rpm_inc = args.rpm_inc self.os_name = args.os_name - self.distros = args.distros - if self.distros in [None, []]: - index = rosdistro.get_index(rosdistro.get_index_url()) - distribution_file = rosdistro.get_distribution_file(index, self.rosdistro) - if self.os_name not in distribution_file.release_platforms: - warning("No platforms defined for os '{0}' in release file for the '{1}' distro." - "\nNot performing RPM generation." - .format(self.os_name, self.rosdistro)) - sys.exit(0) - self.distros = distribution_file.release_platforms[self.os_name] - self.install_prefix = args.install_prefix - if args.install_prefix is None: - self.install_prefix = self.default_install_prefix - self.prefix = args.prefix - self.branches = match_branches_with_prefix(self.prefix, get_branches, prune=not args.match_all) - if len(self.branches) == 0: - error( - "No packages found, check your --prefix or --src arguments.", - exit=True - ) - self.packages = {} - self.tag_names = {} - self.names = [] - self.branch_args = [] - self.rpm_branches = [] - for branch in self.branches: - package = get_package_from_branch(branch) - if package is None: - # This is an ignored package - continue - self.packages[package.name] = package - self.names.append(package.name) - args = self.generate_branching_arguments(package, branch) - # First branch is rpm/[/] - self.rpm_branches.append(args[0][0]) - self.branch_args.extend(args) - - def summarize(self): - info("Generating source RPMs for the packages: " + str(self.names)) - info("RPM Incremental Version: " + str(self.rpm_inc)) - info("RPM Distributions: " + str(self.distros)) - - def get_branching_arguments(self): - return self.branch_args - - def update_rosdep(self): - update_rosdep() - self.has_run_rosdep = True - - def _check_all_keys_are_valid(self, peer_packages, rosdistro): - keys_to_resolve = [] - key_to_packages_which_depends_on = collections.defaultdict(list) - keys_to_ignore = set() - for package in self.packages.values(): - package.evaluate_conditions(package_conditional_context(rosdistro)) - depends = [ - dep for dep in (package.run_depends + package.buildtool_export_depends) - if dep.evaluated_condition] - build_depends = [ - dep for dep in (package.build_depends + package.buildtool_depends + package.test_depends) - if dep.evaluated_condition] - unresolved_keys = [ - dep for dep in (depends + build_depends + package.replaces + package.conflicts) - if dep.evaluated_condition] - keys_to_ignore = { - dep for dep in keys_to_ignore.union(package.replaces + package.conflicts) - if dep.evaluated_condition} - keys = [d.name for d in unresolved_keys] - keys_to_resolve.extend(keys) - for key in keys: - key_to_packages_which_depends_on[key].append(package.name) - - os_name = self.os_name - rosdistro = self.rosdistro - all_keys_valid = True - for key in sorted(set(keys_to_resolve)): - for os_version in self.distros: - try: - extended_peer_packages = peer_packages + [d.name for d in keys_to_ignore] - rule, installer_key, default_installer_key = \ - resolve_rosdep_key(key, os_name, os_version, rosdistro, extended_peer_packages, - retry=False) - if rule is None: - continue - if installer_key != default_installer_key: - error("Key '{0}' resolved to '{1}' with installer '{2}', " - "which does not match the default installer '{3}'." - .format(key, rule, installer_key, default_installer_key)) - BloomGenerator.exit( - "The RPM generator does not support dependencies " - "which are installed with the '{0}' installer." - .format(installer_key), - returncode=code.GENERATOR_INVALID_INSTALLER_KEY) - except (GeneratorError, RuntimeError) as e: - print(fmt("Failed to resolve @{cf}@!{key}@| on @{bf}{os_name}@|:@{cf}@!{os_version}@| with: {e}") - .format(**locals())) - print(fmt("@{cf}@!{0}@| is depended on by these packages: ").format(key) + - str(list(set(key_to_packages_which_depends_on[key])))) - print(fmt("@{kf}@!<== @{rf}@!Failed@|")) - all_keys_valid = False - return all_keys_valid + ret = PackageManagerGenerator.handle_arguments(self, args) + return ret def pre_modify(self): - info("\nPre-verifying RPM dependency keys...") - # Run rosdep update is needed - if not self.has_run_rosdep: - self.update_rosdep() - - peer_packages = [p.name for p in self.packages.values()] - - while not self._check_all_keys_are_valid(peer_packages, self.rosdistro): - error("Some of the dependencies for packages in this repository could not be resolved by rosdep.") - error("You can try to address the issues which appear above and try again if you wish, " - "or continue without releasing into RPM-based distributions (e.g. Fedora 24).") - try: - if not maybe_continue(msg="Would you like to try again?"): - error("User aborted after rosdep keys were not resolved.") - sys.exit(code.GENERATOR_NO_ROSDEP_KEY_FOR_DISTRO) - except (KeyboardInterrupt, EOFError): - error("\nUser quit.", exit=True) - update_rosdep() - invalidate_view_cache() - - info("All keys are " + ansi('greenf') + "OK" + ansi('reset') + "\n") + error_msg = ''.join([ + "Some of the dependencies for packages in this repository could not be resolved by rosdep.\n", + "You can try to address the issues which appear above and try again if you wish, ", + "or continue without releasing into RPM-based distributions (e.g. Fedora 24)." + ]) + PackageManagerGenerator.check_all_keys_are_valid(self, error_msg) for package in self.packages.values(): if not package.licenses or not package.licenses[0]: error("No license set for package '{0}', aborting.".format(package.name), exit=True) - def pre_branch(self, destination, source): - if destination in self.rpm_branches: - return - # Run rosdep update is needed - if not self.has_run_rosdep: - self.update_rosdep() - # Determine the current package being generated - name = destination.split('/')[-1] - distro = destination.split('/')[-2] - # Retrieve the package - package = self.packages[name] - # Report on this package - self.summarize_package(package, distro) - - def pre_rebase(self, destination): - # Get the stored configs is any - patches_branch = 'patches/' + destination - config = self.load_original_config(patches_branch) - if config is not None: - curr_config = get_patch_config(patches_branch) - if curr_config['parent'] == config['parent']: - set_patch_config(patches_branch, config) - - def post_rebase(self, destination): - name = destination.split('/')[-1] - # Retrieve the package - package = self.packages[name] - # Handle differently if this is an rpm vs distro branch - if destination in self.rpm_branches: - info("Placing RPM template files into '{0}' branch." - .format(destination)) - # Then this is an rpm branch - # Place the raw template files - self.place_template_files(package.get_build_type()) - else: - # This is a distro specific rpm branch - # Determine the current package being generated - distro = destination.split('/')[-2] - # Create RPMs for each distro - with inbranch(destination): - data = self.generate_rpm(package, distro) - # Create the tag name for later - self.tag_names[destination] = self.generate_tag_name(data) - # Update the patch configs - patches_branch = 'patches/' + destination - config = get_patch_config(patches_branch) - # Store it - self.store_original_config(config, patches_branch) - # Modify the base so import/export patch works - current_branch = get_current_branch() - if current_branch is None: - error("Could not determine current branch.", exit=True) - config['base'] = get_commit_hash(current_branch) - # Set it - set_patch_config(patches_branch, config) - - def post_patch(self, destination, color='bluef'): - if destination in self.rpm_branches: - return - # Tag after patches have been applied - with inbranch(destination): - # Tag - tag_name = self.tag_names[destination] - if tag_exists(tag_name): - if self.interactive: - warning("Tag exists: " + tag_name) - warning("Do you wish to overwrite it?") - if not maybe_continue('y'): - error("Answered no to continue, aborting.", exit=True) - else: - warning("Overwriting tag: " + tag_name) - else: - info("Creating tag: " + tag_name) - execute_command('git tag -f ' + tag_name) - # Report of success - name = destination.split('/')[-1] - package = self.packages[name] - distro = destination.split('/')[-2] - info(ansi(color) + "####" + ansi('reset'), use_prefix=False) - info( - ansi(color) + "#### " + ansi('greenf') + "Successfully" + - ansi(color) + " generated '" + ansi('boldon') + distro + - ansi('boldoff') + "' RPM for package" - " '" + ansi('boldon') + package.name + ansi('boldoff') + "'" + - " at version '" + ansi('boldon') + package.version + - "-" + str(self.rpm_inc) + ansi('boldoff') + "'" + - ansi('reset'), - use_prefix=False - ) - info(ansi(color) + "####\n" + ansi('reset'), use_prefix=False) - - def store_original_config(self, config, patches_branch): - with inbranch(patches_branch): - with open('rpm.store', 'w+') as f: - f.write(json.dumps(config)) - execute_command('git add rpm.store') - if has_changes(): - execute_command('git commit -m "Store original patch config"') - - def load_original_config(self, patches_branch): - config_store = show(patches_branch, 'rpm.store') - if config_store is None: - return config_store - return json.loads(config_store) - - def place_template_files(self, build_type, rpm_dir='rpm'): - # Create/Clean the rpm folder - if os.path.exists(rpm_dir): - if self.interactive: - warning("rpm directory exists: " + rpm_dir) - warning("Do you wish to overwrite it?") - if not maybe_continue('y'): - error("Answered no to continue, aborting.", exit=True) - else: - warning("Overwriting rpm directory: " + rpm_dir) - execute_command('git rm -rf ' + rpm_dir) - execute_command('git commit -m "Clearing previous rpm folder"') - if os.path.exists(rpm_dir): - shutil.rmtree(rpm_dir) - # Use generic place template files command - place_template_files('.', build_type, gbp=True) - # Commit results - execute_command('git add ' + rpm_dir) - execute_command('git commit -m "Placing rpm template files"') - - def get_releaser_history(self): - # Assumes that this is called in the target branch - patches_branch = 'patches/' + get_current_branch() - raw = show(patches_branch, 'releaser_history.json') - return None if raw is None else json.loads(raw) - - def set_releaser_history(self, history): - # Assumes that this is called in the target branch - patches_branch = 'patches/' + get_current_branch() - debug("Writing release history to '{0}' branch".format(patches_branch)) - with inbranch(patches_branch): - with open('releaser_history.json', 'w') as f: - f.write(json.dumps(history)) - execute_command('git add releaser_history.json') - if has_changes(): - execute_command('git commit -m "Store releaser history"') - - def get_subs(self, package, rpm_distro, releaser_history=None): - return generate_substitutions_from_package( - package, - self.os_name, - rpm_distro, - self.rosdistro, - self.install_prefix, - self.rpm_inc, - [p.name for p in self.packages.values()], - releaser_history=releaser_history, - fallback_resolver=missing_dep_resolver - ) - - def generate_rpm(self, package, rpm_distro, rpm_dir='rpm'): - info("Generating RPM for {0}...".format(rpm_distro)) - # Try to retrieve the releaser_history - releaser_history = self.get_releaser_history() + def generate_package(self, package, os_version): + info("Generating {0} for {1}...".format(self.package_manager, os_version)) # Generate substitution values - subs = self.get_subs(package, rpm_distro, releaser_history) + subs = self.get_subs(package, os_version, format_description, format_depends) # Use subs to create and store releaser history self.set_releaser_history(dict(subs['changelogs'])) # Template files - template_files = process_template_files('.', subs) + template_files = process_template_files('.', subs, self.package_manager) # Remove any residual template files execute_command('git rm -rf ' + ' '.join("'{}'".format(t) for t in template_files)) # Add marker file to tell mock to archive the sources open('.write_tar', 'a').close() # Add marker file changes to the rpm folder - execute_command('git add .write_tar ' + rpm_dir) + execute_command('git add .write_tar ' + self.package_manager) # Commit changes - execute_command('git commit -m "Generated RPM files for ' + - rpm_distro + '"') + execute_command('git commit -m "Generated {0} files for {1}"' + .format(self.package_manager, os_version)) # Rename the template spec file - execute_command('git mv ' + rpm_dir + '/template.spec ' + rpm_dir + '/' + subs['Package'] + '.spec') + execute_command('git mv {0}/template.spec {1}/{2}.spec' + .format(self.package_manager, self.package_manager, subs['Package'])) # Commit changes - execute_command('git commit -m "Renamed RPM spec file for ' + - rpm_distro + '"') + execute_command('git commit -m "Renamed {0} spec files for {1}"' + .format(self.package_manager, os_version)) # Return the subs for other use return subs - def generate_tag_name(self, data): - tag_name = '{Package}-{Version}-{RPMInc}_{Distribution}' - tag_name = 'rpm/' + tag_name.format(**data) - return tag_name + @staticmethod + def get_subs_hook(subs, package, rosdistro, releaser_history=None): + # Use the time stamp to set the date strings + stamp = datetime.datetime.now(tz.tzlocal()) + subs['Date'] = stamp.strftime('%a %b %d %Y') + # Maintainers + maintainers = [] + for m in package.maintainers: + maintainers.append(str(m)) + # Changelog + if releaser_history: + sorted_releaser_history = sorted(releaser_history, + key=lambda k: LooseVersion(k), reverse=True) + sorted_releaser_history = sorted(sorted_releaser_history, + key=lambda k: strptime(releaser_history.get(k)[0], '%a %b %d %Y'), + reverse=True) + changelogs = [(v, releaser_history[v]) for v in sorted_releaser_history] + else: + # Ensure at least a minimal changelog + changelogs = [] + if package.version + '-' + str(subs['Inc']) not in [x[0] for x in changelogs]: + changelogs.insert(0, ( + package.version + '-' + str(subs['Inc']), ( + subs['Date'], + package.maintainers[0].name, + package.maintainers[0].email + ) + )) + exported_tags = [e.tagname for e in package.exports] + subs['NoArch'] = 'metapackage' in exported_tags or 'architecture_independent' in exported_tags + subs['changelogs'] = changelogs + + # License + if not package.licenses or not package.licenses[0]: + error("No license set for package '{0}', aborting.".format(package.name), exit=True) + subs['License'] = package.licenses[0] - def generate_branching_arguments(self, package, branch): - n = package.name - # rpm branch - rpm_branch = 'rpm/' + n - # Branch first to the rpm branch - args = [[rpm_branch, branch, False]] - # Then for each RPM distro, branch from the base rpm branch - args.extend([ - ['rpm/' + d + '/' + n, rpm_branch, False] for d in self.distros - ]) - return args + return subs - def summarize_package(self, package, distro, color='bluef'): - info(ansi(color) + "\n####" + ansi('reset'), use_prefix=False) - info( - ansi(color) + "#### Generating '" + ansi('boldon') + distro + - ansi('boldoff') + "' RPM for package" - " '" + ansi('boldon') + package.name + ansi('boldoff') + "'" + - " at version '" + ansi('boldon') + package.version + - "-" + str(self.rpm_inc) + ansi('boldoff') + "'" + - ansi('reset'), - use_prefix=False - ) - info(ansi(color) + "####" + ansi('reset'), use_prefix=False) + @staticmethod + def generate_tag_name(subs): + tag_name = '{Package}-{Version}-{Inc}_{Distribution}' + tag_name = RpmGenerator.package_manager + '/' + tag_name.format(**subs) + return tag_name diff --git a/bloom/generators/rpm/templates/catkin/template.spec.em b/bloom/generators/rpm/templates/catkin/template.spec.em index f669229c..816f949f 100644 --- a/bloom/generators/rpm/templates/catkin/template.spec.em +++ b/bloom/generators/rpm/templates/catkin/template.spec.em @@ -4,7 +4,7 @@ Name: @(Package) Version: @(Version) -Release: @(RPMInc)%{?dist} +Release: @(Inc)%{?dist} Summary: ROS @(Name) package License: @(License) diff --git a/bloom/generators/rpm/templates/cmake/template.spec.em b/bloom/generators/rpm/templates/cmake/template.spec.em index f669229c..816f949f 100644 --- a/bloom/generators/rpm/templates/cmake/template.spec.em +++ b/bloom/generators/rpm/templates/cmake/template.spec.em @@ -4,7 +4,7 @@ Name: @(Package) Version: @(Version) -Release: @(RPMInc)%{?dist} +Release: @(Inc)%{?dist} Summary: ROS @(Name) package License: @(License) diff --git a/bloom/generators/vcpkg/__init__.py b/bloom/generators/vcpkg/__init__.py new file mode 100644 index 00000000..c4c47623 --- /dev/null +++ b/bloom/generators/vcpkg/__init__.py @@ -0,0 +1,6 @@ +from .generator import VcpkgGenerator +from .generator import format_depend +from .generator import format_depends +from .generator import format_description + +__all__ = ['VcpkgGenerator', 'format_description', 'format_depend', 'format_depends'] diff --git a/bloom/generators/vcpkg/generate_cmd.py b/bloom/generators/vcpkg/generate_cmd.py new file mode 100644 index 00000000..1b1fd9ac --- /dev/null +++ b/bloom/generators/vcpkg/generate_cmd.py @@ -0,0 +1,159 @@ +# Software License Agreement (BSD License) +# +# Copyright (c) 2013, Open Source Robotics Foundation, Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Open Source Robotics Foundation, Inc. nor +# the names of its contributors may be used to endorse or promote +# products derived from this software without specific prior +# written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN +# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +from __future__ import print_function + +import os +import sys +import traceback + +from bloom.logging import debug +from bloom.logging import error +from bloom.logging import fmt +from bloom.logging import info + +from bloom.generators.common import generate_substitutions_from_package +from bloom.generators.common import place_template_files +from bloom.generators.common import process_template_files + +from bloom.generators.vcpkg.generator import VcpkgGenerator +from bloom.generators.vcpkg.generator import format_description +from bloom.generators.vcpkg.generator import format_depends + +from bloom.util import get_distro_list_prompt + +try: + from rosdep2 import create_default_installer_context +except ImportError: + debug(traceback.format_exc()) + error("rosdep was not detected, please install it.", exit=True) + +try: + from catkin_pkg.packages import find_packages +except ImportError: + debug(traceback.format_exc()) + error("catkin_pkg was not detected, please install it.", exit=True) + + +def prepare_arguments(parser): + add = parser.add_argument + add('package_path', nargs='?', + help="path to or containing the package.xml of a package") + action = parser.add_mutually_exclusive_group(required=False) + add = action.add_argument + add('--place-template-files', action='store_true', + help="places vcpkg/* template files only") + add('--process-template-files', action='store_true', + help="processes templates in vcpkg/* only") + add = parser.add_argument + add('--os-name', help='OS name, e.g. vcpkg') + add('--os-version', help='OS version or codename, e.g. precise, wheezy') + add('--ros-distro', help="ROS distro, e.g. %s (used for rosdep)" % get_distro_list_prompt()) + return parser + + +def get_subs(pkg, os_name, os_version, ros_distro): + subs = generate_substitutions_from_package( + pkg, + os_name, + os_version, + ros_distro, + format_description, + format_depends, + ) + subs = VcpkgGenerator.get_subs_hook(subs, pkg, ros_distro) + return subs + + +def main(args=None, get_subs_fn=None): + get_subs_fn = get_subs_fn or get_subs + _place_template_files = True + _process_template_files = True + package_path = os.getcwd() + if args is not None: + package_path = args.package_path or os.getcwd() + _place_template_files = args.place_template_files + _process_template_files = args.process_template_files + + pkgs_dict = find_packages(package_path) + if len(pkgs_dict) == 0: + sys.exit("No packages found in path: '{0}'".format(package_path)) + if len(pkgs_dict) > 1: + sys.exit("Multiple packages found, " + "this tool only supports one package at a time.") + + os_data = create_default_installer_context().get_os_name_and_version() + os_name, os_version = os_data + ros_distro = os.environ.get('ROS_DISTRO', 'indigo') + + # Allow args overrides + os_name = args.os_name or os_name + os_version = args.os_version or os_version + ros_distro = args.ros_distro or ros_distro + + # Summarize + info(fmt("@!@{gf}==> @|") + + fmt("Generating ports for @{cf}%s:%s@| for package(s) %s" % + (os_name, os_version, [p.name for p in pkgs_dict.values()]))) + + package_manager = VcpkgGenerator.package_manager + for path, pkg in pkgs_dict.items(): + template_files = None + try: + subs = get_subs_fn(pkg, os_name, os_version, ros_distro) + if _place_template_files: + # Place template files + place_template_files(path, pkg.get_build_type(), package_manager) + if _process_template_files: + # Just process existing template files + template_files = process_template_files(path, subs, package_manager) + if not _place_template_files and not _process_template_files: + # If neither, do both + place_template_files(path, pkg.get_build_type(), package_manager) + template_files = process_template_files(path, subs, package_manager) + if template_files is not None: + for template_file in template_files: + os.remove(os.path.normpath(template_file)) + except Exception as exc: + debug(traceback.format_exc()) + error(type(exc).__name__ + ": " + str(exc), exit=True) + except (KeyboardInterrupt, EOFError): + sys.exit(1) + +# This describes this command to the loader +description = dict( + title='vcpkg', + description="Generates vcpkg packaging files for a catkin package", + main=main, + prepare_arguments=prepare_arguments +) diff --git a/bloom/generators/vcpkg/generator.py b/bloom/generators/vcpkg/generator.py new file mode 100644 index 00000000..aae95371 --- /dev/null +++ b/bloom/generators/vcpkg/generator.py @@ -0,0 +1,201 @@ +# Software License Agreement (BSD License) +# +# Copyright (c) 2013, Willow Garage, Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Willow Garage, Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN +# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +from __future__ import print_function + +import re +import traceback + +from bloom.generators.common import PackageManagerGenerator +from bloom.generators.common import process_template_files + +from bloom.logging import debug +from bloom.logging import enable_drop_first_log_prefix +from bloom.logging import error +from bloom.logging import info + +from bloom.util import execute_command + +try: + from catkin_pkg.changelog import get_changelog_from_path + from catkin_pkg.changelog import CHANGELOG_FILENAME +except ImportError: + debug(traceback.format_exc()) + error("catkin_pkg was not detected, please install it.", exit=True) + +try: + import rosdistro +except ImportError: + debug(traceback.format_exc()) + error("rosdistro was not detected, please install it.", exit=True) + +# Drop the first log prefix for this command +enable_drop_first_log_prefix(True) + +NORMAL_DEPENDENCY_TEMPLATE = '\t\t\n' +SEMANTIC_VERSION_DEPENDENCY_TEMPLATE = '\t\t\n' + + +def format_depend(id, version=None): + if version is None: + return NORMAL_DEPENDENCY_TEMPLATE.format(id) + else: + return SEMANTIC_VERSION_DEPENDENCY_TEMPLATE.format(id, version) + + +def format_depends(depends, resolved_deps): + # you can see NuGet's semantic version documents via + # https://docs.microsoft.com/en-us/nuget/reference/package-versioning#version-ranges-and-wildcards + versions = { + 'version_lt': '(,{0})', + 'version_lte': '(,{0}]', + 'version_eq': '[{0}]', + 'version_gte': '{0}', + 'version_gt': '({0},)' + } + + formatted = [] + for d in depends: + for resolved_dep in resolved_deps[d.name]: + version_depends = [k + for k in versions.keys() + if getattr(d, k, None) is not None] + if not version_depends: + formatted.append(format_depend(resolved_dep)) + else: + for v in version_depends: + formatted.append(format_depend(resolved_dep, versions[v].format(getattr(d, v)))) + return formatted + + +def format_description(value): + markup_remover = re.compile(r'<.*?>') + value = markup_remover.sub('', value) + value = re.sub('\s+', ' ', value) + + parts = value.split('. ', 1) + if len(parts) == 1 or len(parts[1]) == 0: + # most likely single line description + return value + # format according to rules in linked field documentation + return u"{0}.\n {1}".format(parts[0], parts[1].strip()) + + +class VcpkgGenerator(PackageManagerGenerator): + title = 'vcpkg' + package_manager = 'vcpkg' + description = "Generates vcpkg from the catkin meta data" + + def prepare_arguments(self, parser): + add = parser.add_argument + add('--os-name', default='windows', + help="overrides os_name, set to 'windows' by default") + return PackageManagerGenerator.prepare_arguments(self, parser) + + def handle_arguments(self, args): + self.os_name = args.os_name + ret = PackageManagerGenerator.handle_arguments(self, args) + return ret + + def generate_package(self, package, os_version): + info("Generating {0} for {1}...".format(self.package_manager, os_version)) + # Generate substitution values + subs = self.get_subs(package, os_version, format_description, format_depends) + # Template files + template_files = process_template_files(".", subs, self.package_manager) + # Remove any residual template files + execute_command('git rm -rf ' + ' '.join("'{}'".format(t) for t in template_files)) + # Add changes to the package system folder + execute_command('git add {0}'.format(self.package_manager)) + # Commit changes + execute_command('git commit -m "Generated {0} files for {1}"' + .format(self.package_manager, os_version)) + # Rename the template nuspec file + execute_command('git mv {0}/template.nuspec {1}/{2}.nuspec'. + format(self.package_manager, self.package_manager, subs['Package'])) + # Commit changes + execute_command('git commit -m "Renamed {0} nuspec files for {1}"' + .format(self.package_manager, subs['Package'])) + # Return the subs for other use + return subs + + @staticmethod + def get_subs_hook(subs, package, ros_distro, releaser_history=None): + # Get pacakge's release url from rosdistro repository + index = rosdistro.get_index(rosdistro.get_index_url()) + distribution_file = rosdistro.get_distribution_file(index, ros_distro) + try: + repo = distribution_file.repositories[package.name] + except KeyError as e: + # The current package is exist in a meta-package, + # Notice that the type of distribution_file.repositories is dict + for meta_repo in distribution_file.repositories.values(): + if package.name in meta_repo.release_repository.package_names: + repo = meta_repo + break + else: + raise e + release_url = repo.release_repository.url + + vcpkg_support_git_sources = ["github", "gitlab", "bitbucket"] + for git_source in vcpkg_support_git_sources: + if git_source in release_url: + subs['GitSource'] = git_source + break + else: + error("Currently Bloom don't support release url: {0} currently" + .format(release_url), exit=True) + # release url format: https://github.com// + subs["UserName"] = release_url.split('/')[-2] + subs["RepoName"] = release_url.split('/')[-1] + if ".git" in subs["RepoName"]: + subs["RepoName"] = subs["RepoName"][:-4] + + subs['TagName'] = VcpkgGenerator.generate_tag_name(subs) + subs['RosDistro'] = ros_distro + + # We only use fist maintainer's name, because full string of maintainer contains + # '@' in email part, which will cause NuGet parse error + subs['Authors'] = package.maintainers[0].name + + subs['BuildDependenciesConfig'] = [ + d.replace(" + +@[for p in BuildDependenciesConfig]@(p)@[end for] + \ No newline at end of file diff --git a/bloom/generators/vcpkg/templates/ament_cmake/portfile.cmake.em b/bloom/generators/vcpkg/templates/ament_cmake/portfile.cmake.em new file mode 100644 index 00000000..31378779 --- /dev/null +++ b/bloom/generators/vcpkg/templates/ament_cmake/portfile.cmake.em @@ -0,0 +1,26 @@ +include(vcpkg_common_functions) + +set(VCPKG_BUILD_TYPE release) + +@[if GitSource == 'gitlab']@ +vcpkg_from_gitlab( +@[elif GitSource == 'github']@ +vcpkg_from_github( +@[elif GitSource == 'bitbucket']@ +vcpkg_from_bitbucket( +@[end if]@ + OUT_SOURCE_PATH SOURCE_PATH + REPO @(UserName)/@(RepoName) + REF @(TagName) +) + +set(ROS_BASE_PATH "C:/opt/ros/@(RosDistro)") +file(TO_NATIVE_PATH "${ROS_BASE_PATH}" ROS_BASE_PATH) + +vcpkg_configure_cmake( + SOURCE_PATH ${SOURCE_PATH} + OPTIONS + -DAMENT_PREFIX_PATH=${ROS_BASE_PATH} +) + +vcpkg_install_cmake() diff --git a/bloom/generators/vcpkg/templates/ament_cmake/template.nuspec.em b/bloom/generators/vcpkg/templates/ament_cmake/template.nuspec.em new file mode 100644 index 00000000..31bcea08 --- /dev/null +++ b/bloom/generators/vcpkg/templates/ament_cmake/template.nuspec.em @@ -0,0 +1,20 @@ + + + + @(Package) + @(Version) + @(Package) + @(Authors) + OSRF + false + @(Description) + +@[for d in Depends]@(d)@[end for] + + + + + + + + diff --git a/bloom/generators/vcpkg/templates/ament_python/CONTROL.em b/bloom/generators/vcpkg/templates/ament_python/CONTROL.em new file mode 100644 index 00000000..6894a2ab --- /dev/null +++ b/bloom/generators/vcpkg/templates/ament_python/CONTROL.em @@ -0,0 +1,4 @@ +Source: @(Package) +Version: @(Version) +Description: @(Description) +Build-Depends: \ No newline at end of file diff --git a/bloom/generators/vcpkg/templates/ament_python/build_dependencies.config.em b/bloom/generators/vcpkg/templates/ament_python/build_dependencies.config.em new file mode 100644 index 00000000..3a8f12ea --- /dev/null +++ b/bloom/generators/vcpkg/templates/ament_python/build_dependencies.config.em @@ -0,0 +1,4 @@ + + +@[for p in BuildDependenciesConfig]@(p)@[end for] + \ No newline at end of file diff --git a/bloom/generators/vcpkg/templates/ament_python/portfile.cmake.em b/bloom/generators/vcpkg/templates/ament_python/portfile.cmake.em new file mode 100644 index 00000000..52b718b5 --- /dev/null +++ b/bloom/generators/vcpkg/templates/ament_python/portfile.cmake.em @@ -0,0 +1,44 @@ +include(vcpkg_common_functions) + +set(VCPKG_BUILD_TYPE release) + +@[if GitSource == 'gitlab']@ +vcpkg_from_gitlab( +@[elif GitSource == 'github']@ +vcpkg_from_github( +@[elif GitSource == 'bitbucket']@ +vcpkg_from_bitbucket( +@[end if]@ + OUT_SOURCE_PATH SOURCE_PATH + REPO @(UserName)/@(RepoName) + REF @(TagName) +) + +find_package(PythonInterp 3) + +if (${PYTHONINTERP_FOUND}) + set(SETUP_INSTALL_PREFIX "${CURRENT_PACKAGES_DIR}") + set(SETUP_INSTALL_PYTHONPATH "${SETUP_INSTALL_PREFIX}/Lib/site-packages") + file(TO_NATIVE_PATH "${SETUP_INSTALL_PREFIX}" SETUP_INSTALL_PREFIX) + file(TO_NATIVE_PATH "${SETUP_INSTALL_PYTHONPATH}" SETUP_INSTALL_PYTHONPATH) + + file(MAKE_DIRECTORY ${SETUP_INSTALL_PYTHONPATH}) + set(INSTALL_CMD + # if we want to use install --prefix, we must use following line to set PYTHONPATH + ${CMAKE_COMMAND} -E env PYTHONPATH=${SETUP_INSTALL_PYTHONPATH} + ${PYTHON_EXECUTABLE} + setup.py + egg_info --egg-base . + build --build-base build + install --prefix ${SETUP_INSTALL_PREFIX} + --record install.log + --single-version-externally-managed + ) + + execute_process( + COMMAND ${INSTALL_CMD} + WORKING_DIRECTORY ${SOURCE_PATH} + ) +else() + message(FATAL_ERROR "Python executable not fould, stop building") +endif() \ No newline at end of file diff --git a/bloom/generators/vcpkg/templates/ament_python/template.nuspec.em b/bloom/generators/vcpkg/templates/ament_python/template.nuspec.em new file mode 100644 index 00000000..31bcea08 --- /dev/null +++ b/bloom/generators/vcpkg/templates/ament_python/template.nuspec.em @@ -0,0 +1,20 @@ + + + + @(Package) + @(Version) + @(Package) + @(Authors) + OSRF + false + @(Description) + +@[for d in Depends]@(d)@[end for] + + + + + + + + diff --git a/bloom/generators/vcpkg/templates/catkin/CONTROL.em b/bloom/generators/vcpkg/templates/catkin/CONTROL.em new file mode 100644 index 00000000..6894a2ab --- /dev/null +++ b/bloom/generators/vcpkg/templates/catkin/CONTROL.em @@ -0,0 +1,4 @@ +Source: @(Package) +Version: @(Version) +Description: @(Description) +Build-Depends: \ No newline at end of file diff --git a/bloom/generators/vcpkg/templates/catkin/build_dependencies.config.em b/bloom/generators/vcpkg/templates/catkin/build_dependencies.config.em new file mode 100644 index 00000000..3a8f12ea --- /dev/null +++ b/bloom/generators/vcpkg/templates/catkin/build_dependencies.config.em @@ -0,0 +1,4 @@ + + +@[for p in BuildDependenciesConfig]@(p)@[end for] + \ No newline at end of file diff --git a/bloom/generators/vcpkg/templates/catkin/portfile.cmake.em b/bloom/generators/vcpkg/templates/catkin/portfile.cmake.em new file mode 100644 index 00000000..ffb33ea9 --- /dev/null +++ b/bloom/generators/vcpkg/templates/catkin/portfile.cmake.em @@ -0,0 +1,28 @@ +include(vcpkg_common_functions) + +set(VCPKG_BUILD_TYPE release) + +@[if GitSource == 'gitlab']@ +vcpkg_from_gitlab( +@[elif GitSource == 'github']@ +vcpkg_from_github( +@[elif GitSource == 'bitbucket']@ +vcpkg_from_bitbucket( +@[end if]@ + OUT_SOURCE_PATH SOURCE_PATH + REPO @(UserName)/@(RepoName) + REF @(TagName) +) + +set(ROS_BASE_PATH "C:/opt/ros/@(RosDistro)") +file(TO_NATIVE_PATH "${ROS_BASE_PATH}" ROS_BASE_PATH) + +vcpkg_configure_cmake( + SOURCE_PATH ${SOURCE_PATH} + OPTIONS + -DCATKIN_BUILD_BINARY_PACKAGE="1" + -DCMAKE_PREFIX_PATH=${ROS_BASE_PATH} + -DAMENT_PREFIX_PATH=${ROS_BASE_PATH} +) + +vcpkg_install_cmake() diff --git a/bloom/generators/vcpkg/templates/catkin/template.nuspec.em b/bloom/generators/vcpkg/templates/catkin/template.nuspec.em new file mode 100644 index 00000000..31bcea08 --- /dev/null +++ b/bloom/generators/vcpkg/templates/catkin/template.nuspec.em @@ -0,0 +1,20 @@ + + + + @(Package) + @(Version) + @(Package) + @(Authors) + OSRF + false + @(Description) + +@[for d in Depends]@(d)@[end for] + + + + + + + + diff --git a/bloom/generators/vcpkg/templates/cmake/CONTROL.em b/bloom/generators/vcpkg/templates/cmake/CONTROL.em new file mode 100644 index 00000000..6894a2ab --- /dev/null +++ b/bloom/generators/vcpkg/templates/cmake/CONTROL.em @@ -0,0 +1,4 @@ +Source: @(Package) +Version: @(Version) +Description: @(Description) +Build-Depends: \ No newline at end of file diff --git a/bloom/generators/vcpkg/templates/cmake/build_dependencies.config.em b/bloom/generators/vcpkg/templates/cmake/build_dependencies.config.em new file mode 100644 index 00000000..3a8f12ea --- /dev/null +++ b/bloom/generators/vcpkg/templates/cmake/build_dependencies.config.em @@ -0,0 +1,4 @@ + + +@[for p in BuildDependenciesConfig]@(p)@[end for] + \ No newline at end of file diff --git a/bloom/generators/vcpkg/templates/cmake/portfile.cmake.em b/bloom/generators/vcpkg/templates/cmake/portfile.cmake.em new file mode 100644 index 00000000..fb455557 --- /dev/null +++ b/bloom/generators/vcpkg/templates/cmake/portfile.cmake.em @@ -0,0 +1,26 @@ +include(vcpkg_common_functions) + +set(VCPKG_BUILD_TYPE release) + +@[if GitSource == 'gitlab']@ +vcpkg_from_gitlab( +@[elif GitSource == 'github']@ +vcpkg_from_github( +@[elif GitSource == 'bitbucket']@ +vcpkg_from_bitbucket( +@[end if]@ + OUT_SOURCE_PATH SOURCE_PATH + REPO @(UserName)/@(RepoName) + REF @(TagName) +) + +set(ROS_BASE_PATH "C:/opt/ros/@(RosDistro)") +file(TO_NATIVE_PATH "${ROS_BASE_PATH}" ROS_BASE_PATH) + +vcpkg_configure_cmake( + SOURCE_PATH ${SOURCE_PATH} + OPTIONS + -DAMENT_PREFIX_PATH=${ROS_BASE_PATH} +) + +vcpkg_install_cmake() diff --git a/bloom/generators/vcpkg/templates/cmake/template.nuspec.em b/bloom/generators/vcpkg/templates/cmake/template.nuspec.em new file mode 100644 index 00000000..31bcea08 --- /dev/null +++ b/bloom/generators/vcpkg/templates/cmake/template.nuspec.em @@ -0,0 +1,20 @@ + + + + @(Package) + @(Version) + @(Package) + @(Authors) + OSRF + false + @(Description) + +@[for d in Depends]@(d)@[end for] + + + + + + + + diff --git a/setup.py b/setup.py index 0ce619c6..b5500bbf 100755 --- a/setup.py +++ b/setup.py @@ -31,6 +31,9 @@ ], 'bloom.generators.rpm': [ 'bloom/generators/rpm/templates/*' + ], + 'bloom.generators.vcpkg': [ + 'bloom/generators/vcpkg/templates/*' ] }, include_package_data=True, @@ -72,13 +75,17 @@ 'debian = bloom.generators.debian:DebianGenerator', 'rosdebian = bloom.generators.rosdebian:RosDebianGenerator', 'rpm = bloom.generators.rpm:RpmGenerator', - 'rosrpm = bloom.generators.rosrpm:RosRpmGenerator' + 'rosrpm = bloom.generators.rosrpm:RosRpmGenerator', + 'vcpkg = bloom.generators.vcpkg:VcpkgGenerator', + 'rosvcpkg = bloom.generators.rosvcpkg:RosVcpkgGenerator' ], 'bloom.generate_cmds': [ 'debian = bloom.generators.debian.generate_cmd:description', 'rosdebian = bloom.generators.rosdebian:description', 'rpm = bloom.generators.rpm.generate_cmd:description', - 'rosrpm = bloom.generators.rosrpm:description' + 'rosrpm = bloom.generators.rosrpm:description', + 'vcpk = bloom.generators.vcpkg.generate_cmd:description', + 'rosvcpkg = bloom.generators.rosvcpkg:description' ] } ) diff --git a/test/system_tests/test_catkin_release.py b/test/system_tests/test_catkin_release.py index d30921e6..3c6caced 100644 --- a/test/system_tests/test_catkin_release.py +++ b/test/system_tests/test_catkin_release.py @@ -31,7 +31,7 @@ from bloom.commands.git.patch import import_cmd from bloom.commands.git.patch import remove_cmd -from bloom.generators.debian.generator import sanitize_package_name +from bloom.generators.common import sanitize_package_name def create_upstream_repository(packages, directory=None, format_versions=None): diff --git a/test/unit_tests/test_generators/test_common/__init__.py b/test/unit_tests/test_generators/test_common/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/unit_tests/test_generators/test_common/test_generator.py b/test/unit_tests/test_generators/test_common/test_generator.py new file mode 100644 index 00000000..92fc3b65 --- /dev/null +++ b/test/unit_tests/test_generators/test_common/test_generator.py @@ -0,0 +1,236 @@ +import os + +from collections import namedtuple +from distutils.dir_util import copy_tree + +from ....utils.common import AssertRaisesContext +from ....utils.common import bloom_answer +from ....utils.common import change_directory +from ....utils.common import redirected_stdio +from ....utils.common import temporary_directory +from ....utils.common import user + +from bloom.generators.common import PackageManagerGenerator + +from bloom.util import code + +from catkin_pkg.packages import find_packages + +GENERATE_DATA_PATH = 'test_generator_data' +test_data_dir = os.path.join(os.path.dirname(__file__), GENERATE_DATA_PATH) + + +def get_package(pkg_name): + packages = dict([(pkg.name, pkg) for path, pkg in find_packages(test_data_dir).items()]) + return packages[pkg_name] + + +def get_generator(): + gen = PackageManagerGenerator() + gen.package_manager = 'debian' + return gen + + +def test_set_default_distros(): + gen = get_generator() + + gen.rosdistro = 'dashing' + gen.os_name = 'ubuntu' + gen.set_default_distros() + assert gen.distros == ['bionic'] + + gen.distros = None + gen.os_name = "debian" + gen.os_not_required = True + with AssertRaisesContext(SystemExit, ""): + with redirected_stdio(): + gen.set_default_distros() + gen.os_not_required = False + with AssertRaisesContext(SystemExit, "No platforms defined"): + with redirected_stdio(): + gen.set_default_distros() + + +def listdir(dirname): + # list files in one direcotory, including its subdirectory's file + files = [] + for f in os.listdir(dirname): + if os.path.isfile(os.path.join(dirname, f)): + files.append(os.path.join(dirname, f)) + else: + for sub_f in os.listdir(os.path.join(dirname, f)): + files.append(os.path.join(dirname, f, sub_f)) + return files + + +def test_place_template_files(): + pkg_name = 'test_pkg' + pkg = get_package(pkg_name) + gen = get_generator() + gen.interactive = False + build_type = pkg.get_build_type() + + # Test normal place template files + with redirected_stdio(): + with temporary_directory(): + user('git init .') + gen.place_template_files(build_type) + placed_files = listdir('debian') + template_file_list = [ + 'debian/source/format.em', + 'debian/source/options.em', + 'debian/changelog.em', + 'debian/compat.em', + 'debian/control.em', + 'debian/copyright.em', + 'debian/gbp.conf.em', + 'debian/rules.em', + ] + for f in template_file_list: + assert f in placed_files, "{0} not placed".format(f) + + # Test if package system directory exists + test_dir_exist_func_list = [ + dir_exist_with_interactive, + dir_exist_with_clean, + dir_exist_default, + ] + TestPlaceTemplateFileStruct = namedtuple( + 'TestPlaceTemplateFileStruct', + 'generator build_type old_dir, directory,' + 'original_template_files_dict,' + 'original_normal_files' + ) + dir_target = os.path.join(os.path.dirname(__file__), GENERATE_DATA_PATH, pkg_name) + with change_directory(dir_target): + old_dir = os.getcwd() + original_files = listdir('debian') + original_template_files_dict = dict([(f, open(f).read()) + for f in original_files if f.endswith('.em')]) + original_normal_files = [f for f in original_files if not f.endswith('.em')] + for f in test_dir_exist_func_list: + with temporary_directory() as directory: + user('git init .') + copy_tree(old_dir, directory) + user('git add .') + user('git commit --allow-empty -m "Initial commit"') + data = TestPlaceTemplateFileStruct(gen, build_type, old_dir,directory, + original_template_files_dict, + original_normal_files) + f(data) + + +def dir_exist_with_interactive(data): + gen = data.generator + gen.interactive = True + + with AssertRaisesContext(SystemExit, "Answered no to continue"): + with redirected_stdio(): + with bloom_answer(['n']): + gen.place_template_files(data.build_type) + + with redirected_stdio(): + with bloom_answer(['y']): + gen.place_template_files(data.build_type) + placed_files_dict = dict([(f, open(f).read()) for f in listdir('debian')]) + # overwrite the template files should not remove the origianl debian files + for f in data.original_normal_files: + assert f in placed_files_dict.keys() + for f, content in data.original_template_files_dict.items(): + assert f in placed_files_dict.keys() + # Your gbp.conf.em will be changed if you answer yes to overwrite + if f == 'gbp.conf.em': + assert content != placed_files_dict[f] + + gen.interactive = False + + +def dir_exist_with_clean(data): + gen = data.generator + CLEAR_TEMPLATE_ENV_PARAMETER = 'BLOOM_CLEAR_TEMPLATE_ON_GENERATION' + os.environ.setdefault(CLEAR_TEMPLATE_ENV_PARAMETER, "1") + + with redirected_stdio(): + gen.place_template_files(data.build_type) + placed_files = listdir('debian') + for f in data.original_normal_files: + assert f not in placed_files + + os.environ.pop(CLEAR_TEMPLATE_ENV_PARAMETER) + + +def dir_exist_default(data): + gen = data.generator + with redirected_stdio(): + gen.place_template_files(data.build_type) + placed_files_dict = dict([(f, open(f).read()) for f in listdir('debian')]) + # default doesn't influence your file under debian directory + for f, content in data.original_template_files_dict.items(): + assert f in placed_files_dict.keys() + assert content == placed_files_dict[f] + for f in data.original_normal_files: + assert f in placed_files_dict.keys() + + +def test_bad_dependency(): + bad_pkg_name = 'bad_dependency_pkg' + bad_pkg = get_package(bad_pkg_name) + pkg_bad_dict = {bad_pkg_name: bad_pkg} + + gen = get_generator() + gen.packages = pkg_bad_dict + gen.rosdistro = 'kinetic' + gen.os_name = 'ubuntu' + gen.distros = ['xenial'] + with bloom_answer(['n']): + with AssertRaisesContext(SystemExit, str(code.GENERATOR_NO_ROSDEP_KEY_FOR_DISTRO)): + with redirected_stdio(): + gen.check_all_keys_are_valid("") + + +def format_depends(depends, resolved_deps): + formatted = [] + for d in depends: + formatted.append("{0}".format(d)) + return formatted + + +def format_description(value): + return value + + +class OneGenerator(PackageManagerGenerator): + package_manager = "test" + @staticmethod + def get_subs_hook(subs, package, rosdistro, releaser_history=None): + subs['Name'] = 'test_pkg improved' + subs['specific_part'] = 'none' + return subs + + +def test_get_substitute(): + pkg_name = 'test_pkg' + pkg = get_package(pkg_name) + pkg_dict = {pkg_name: pkg} + + gen = OneGenerator() + gen.packages = pkg_dict + gen.os_name = "ubuntu" + gen.os_version = "xenial" + gen.rosdistro = "kinetic" + gen.install_prefix = gen.default_install_prefix + gen.inc = "1" + + with redirected_stdio(): + subs = gen.get_subs(pkg, gen.os_version, format_description, format_depends) + + assert 'test_pkg improved' == subs['Name'] + assert '0.1.0' == subs['Version'] + assert 'The test_pkg package' == subs['Description'] + assert '1' == subs['Inc'] + assert 'test-pkg' == subs['Package'] + assert ['roscpp', 'rospy'] == subs['Depends'] + assert gen.os_version == subs['Distribution'] + assert "nobody1 " == subs['Maintainer'] + assert "nobody1 , nobody2 " == subs['Maintainers'] + assert 'none' == subs['specific_part'] diff --git a/test/unit_tests/test_generators/test_common/test_generator_data/bad_dependency_pkg/CHANGELOG.rst b/test/unit_tests/test_generators/test_common/test_generator_data/bad_dependency_pkg/CHANGELOG.rst new file mode 100644 index 00000000..f8eaf716 --- /dev/null +++ b/test/unit_tests/test_generators/test_common/test_generator_data/bad_dependency_pkg/CHANGELOG.rst @@ -0,0 +1,7 @@ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Changelog for package bad_denpendency_pkg +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +0.1.0 (2013-10-09) +------------------- +* Initial test package diff --git a/test/unit_tests/test_generators/test_common/test_generator_data/bad_dependency_pkg/package.xml b/test/unit_tests/test_generators/test_common/test_generator_data/bad_dependency_pkg/package.xml new file mode 100644 index 00000000..6d6d5715 --- /dev/null +++ b/test/unit_tests/test_generators/test_common/test_generator_data/bad_dependency_pkg/package.xml @@ -0,0 +1,19 @@ + + + bad_dependency_pkg + 0.1.0 + The bad_dependency_pkg package + http://wiki.ros.org/test_common_pkg + + nobody1 + nobody2 + + TODO + + roscpp + rospy + bad_dependency + + catkin + + diff --git a/test/unit_tests/test_generators/test_common/test_generator_data/test_pkg/CHANGELOG.rst b/test/unit_tests/test_generators/test_common/test_generator_data/test_pkg/CHANGELOG.rst new file mode 100644 index 00000000..731c4bb6 --- /dev/null +++ b/test/unit_tests/test_generators/test_common/test_generator_data/test_pkg/CHANGELOG.rst @@ -0,0 +1,7 @@ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Changelog for package test_pkg +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +0.1.0 (2013-10-09) +------------------- +* Initial test package diff --git a/test/unit_tests/test_generators/test_common/test_generator_data/test_pkg/debian/gbp.conf.em b/test/unit_tests/test_generators/test_common/test_generator_data/test_pkg/debian/gbp.conf.em new file mode 100644 index 00000000..20bcdd92 --- /dev/null +++ b/test/unit_tests/test_generators/test_common/test_generator_data/test_pkg/debian/gbp.conf.em @@ -0,0 +1,8 @@ +[DEFAULT] +# This section is for global settings. Affects all commands. +# Options set here have the lowest priority. +key = value + +[git-buildpackage] +upstream-tag=@(release_tag) +upstream-tree=tag diff --git a/test/unit_tests/test_generators/test_common/test_generator_data/test_pkg/debian/source/local_options.em b/test/unit_tests/test_generators/test_common/test_generator_data/test_pkg/debian/source/local_options.em new file mode 100644 index 00000000..f7812e91 --- /dev/null +++ b/test/unit_tests/test_generators/test_common/test_generator_data/test_pkg/debian/source/local_options.em @@ -0,0 +1,2 @@ +unapply-patches +abort-on-upstream-changes \ No newline at end of file diff --git a/test/unit_tests/test_generators/test_common/test_generator_data/test_pkg/debian/test_pkg.service b/test/unit_tests/test_generators/test_common/test_generator_data/test_pkg/debian/test_pkg.service new file mode 100644 index 00000000..3a4e12f0 --- /dev/null +++ b/test/unit_tests/test_generators/test_common/test_generator_data/test_pkg/debian/test_pkg.service @@ -0,0 +1,2 @@ +[Unit] +Description=Test pkg for bloom diff --git a/test/unit_tests/test_generators/test_common/test_generator_data/test_pkg/package.xml b/test/unit_tests/test_generators/test_common/test_generator_data/test_pkg/package.xml new file mode 100644 index 00000000..c11af778 --- /dev/null +++ b/test/unit_tests/test_generators/test_common/test_generator_data/test_pkg/package.xml @@ -0,0 +1,18 @@ + + + test_pkg + 0.1.0 + The test_pkg package + http://wiki.ros.org/test_common_pkg + + nobody1 + nobody2 + + TODO + + roscpp + rospy + + catkin + + diff --git a/test/unit_tests/test_generators/test_debian/test_generator.py b/test/unit_tests/test_generators/test_debian/test_generator.py index d0eb2723..317699bf 100644 --- a/test/unit_tests/test_generators/test_debian/test_generator.py +++ b/test/unit_tests/test_generators/test_debian/test_generator.py @@ -2,7 +2,7 @@ from ....utils.common import redirected_stdio -from bloom.generators.debian.generator import em +from bloom.generators.common import em from bloom.generators.debian.generator import get_changelogs from bloom.generators.debian.generator import format_description