From 348e9927dfbd7d4046d95e660e1041b64ef32ee8 Mon Sep 17 00:00:00 2001 From: Guido Günther Date: Thu, 3 Aug 2017 17:54:33 -0300 Subject: builpackage: Move tarball creation functions to export_orig --- gbp/scripts/buildpackage.py | 246 +--------------------------------------- gbp/scripts/export_orig.py | 270 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 272 insertions(+), 244 deletions(-) create mode 100755 gbp/scripts/export_orig.py (limited to 'gbp/scripts') diff --git a/gbp/scripts/buildpackage.py b/gbp/scripts/buildpackage.py index e0ae07c7..36a614c0 100755 --- a/gbp/scripts/buildpackage.py +++ b/gbp/scripts/buildpackage.py @@ -39,84 +39,12 @@ from gbp.scripts.common.pq import is_pq_branch, pq_branch_base import gbp.log import gbp.notifications from gbp.scripts.common.buildpackage import (index_name, wc_name, - git_archive_submodules, - git_archive_single, dump_tree, + dump_tree, write_wc, drop_index) from gbp.scripts.common import ExitCodes from gbp.scripts.common.hook import Hook -from gbp.pkg import compressor_opts, compressor_aliases, parse_archive_filename - -# upstream tarball preparation -def git_archive(repo, source, output_dir, treeish, comp_type, comp_level, with_submodules, component=None): - "create a compressed orig tarball in output_dir using git_archive" - try: - comp_opts = compressor_opts[comp_type][0] - except KeyError: - raise GbpError("Unsupported compression type '%s'" % comp_type) - - output = os.path.join(output_dir, du.orig_file(source, comp_type, component=component)) - prefix = "%s-%s" % (source.name, source.upstream_version) - - try: - if repo.has_submodules() and with_submodules: - repo.update_submodules() - git_archive_submodules(repo, treeish, output, prefix, - comp_type, comp_level, comp_opts) - - else: - git_archive_single(treeish, output, prefix, - comp_type, comp_level, comp_opts) - except (GitRepositoryError, CommandExecFailed): - gbp.log.err("Error generating submodules' archives") - return False - except OSError as e: - gbp.log.err("Error creating %s: %s" % (output, str(e))) - return False - except GbpError: - raise - except Exception as e: - gbp.log.err("Error creating %s: %s" % (output, str(e))) - return False - return True - - -def prepare_upstream_tarball(repo, source, options, tarball_dir, output_dir): - """ - Make sure we have an upstream tarball. This involves looking in - tarball_dir, symlinking or building it. - """ - if not source.is_native() and not source.upstream_version: - raise GbpError("Non-native package '%s' " - "has invalid version '%s'" % (source.name, source.version)) - - options.comp_type = guess_comp_type(repo, - options.comp_type, - source, - options.tarball_dir) - - orig_files = [du.orig_file(source, options.comp_type)] - if options.components: - orig_files += [du.orig_file(source, options.comp_type, c) for c in options.components] - - # look in tarball_dir first, if found force a symlink to it - if options.tarball_dir: - gbp.log.debug("Looking for orig tarballs '%s' at '%s'" % (", ".join(orig_files), tarball_dir)) - missing = du.DebianPkgPolicy.symlink_origs(orig_files, tarball_dir, output_dir, force=True) - if missing: - msg = "Tarballs '%s' not found at '%s'" % (", ".join(missing), tarball_dir) - else: - msg = "All Orig tarballs '%s' found at '%s'" % (", ".join(orig_files), tarball_dir) - gbp.log.info(msg) - if options.no_create_orig: - return - # Create tarball if missing or forced - if not du.DebianPkgPolicy.has_origs(orig_files, output_dir) or options.force_create: - if not pristine_tar_build_orig(repo, source, output_dir, options): - upstream_tree = git_archive_build_orig(repo, source, output_dir, options) - if options.pristine_tar_commit: - pristine_tar_commit(repo, source, options, output_dir, orig_files[0], upstream_tree) - pristine_tar_verify_orig(repo, source, options, output_dir, orig_files) +from gbp.scripts.export_orig import prepare_upstream_tarball def pristine_tar_commit(repo, source, options, output_dir, orig_file, upstream_tree): @@ -260,176 +188,6 @@ def clean_working_tree(options, repo): raise GbpError("Use --git-ignore-new to ignore.") -def pristine_tar_prepare_orig_tree(repo, source, options): - """Make sure the upstream tree exists - In case of component tarballs we need to recreate a tree for the - main tarball without the component subdirs. - """ - if options.components: - try: - upstream_tag = repo.version_to_tag(options.upstream_tag, - source.upstream_version) - tree_name = "%s^{tree}" % upstream_tag - repo.tree_drop_dirs(tree_name, options.components) - except GitRepositoryError: - raise GbpError("Couldn't find upstream tree '%s' to create " - "orig tarball via pristine-tar" % tree_name) - - -def pristine_tar_build_orig(repo, source, output_dir, options): - """ - Build orig tarball using pristine-tar - - @returns: C{True} if tarball was build, C{False} otherwise - """ - if options.pristine_tar: - if not repo.has_branch(repo.pristine_tar_branch): - gbp.log.warn('Pristine-tar branch "%s" not found' % - repo.pristine_tar.branch) - - pristine_tar_prepare_orig_tree(repo, source, options) - try: - repo.pristine_tar.checkout(source.name, - source.upstream_version, - options.comp_type, - output_dir) - for component in options.components: - repo.pristine_tar.checkout(source.name, - source.upstream_version, - options.comp_type, - output_dir, - component=component) - return True - except CommandExecFailed: - if options.pristine_tar_commit: - gbp.log.debug("pristine-tar checkout failed, " - "will commit tarball due to " - "'--pristine-tar-commit'") - else: - raise - return False - - -def pristine_tar_verify_orig(repo, source, options, output_dir, orig_files): - """ - Verify orig tarballs using prstine tar - - @returns: C{True} if tarball was build, C{False} otherwise - """ - if not options.pristine_tar: - return True - - pristine_tar_prepare_orig_tree(repo, source, options) - for f in orig_files: - repo.pristine_tar.verify(os.path.join(output_dir, f)) - return True - - -def get_upstream_tree(repo, source, options): - """Determine the upstream tree from the given options""" - if options.upstream_tree.upper() == 'TAG': - if source.upstream_version is None: - raise GitRepositoryError("Can't determine upstream version from changelog") - upstream_tree = repo.version_to_tag(options.upstream_tag, - source.upstream_version) - elif options.upstream_tree.upper() == 'BRANCH': - if not repo.has_branch(options.upstream_branch): - raise GbpError("%s is not a valid branch" % options.upstream_branch) - upstream_tree = options.upstream_branch - elif options.upstream_tree.upper() == 'SLOPPY': - tree_name = "%s^{tree}" % options.debian_branch - upstream_tree = repo.tree_drop_dirs(tree_name, ["debian"]) - else: - upstream_tree = options.upstream_tree - if not repo.has_treeish(upstream_tree): - raise GbpError("%s is not a valid treeish" % upstream_tree) - return upstream_tree - - -def git_archive_build_orig(repo, source, output_dir, options): - """ - Build orig tarball(s) using git-archive - - @param source: the changelog of the package we're acting on - @type source: L{DebianSource} - @param output_dir: where to put the tarball - @type output_dir: C{Str} - @param options: the parsed options - @type options: C{dict} of options - @return: the tree we built the tarball from - @rtype: C{str} - """ - upstream_tree = get_upstream_tree(repo, source, options) - gbp.log.info("Creating %s from '%s'" % (du.orig_file(source, - options.comp_type), - upstream_tree)) - comp_level = int(options.comp_level) if options.comp_level != '' else None - gbp.log.debug("Building upstream tarball with compression '%s'%s" % - (options.comp_type, - "' -%s'" % comp_level if comp_level is not None else '')) - main_tree = repo.tree_drop_dirs(upstream_tree, options.components) if options.components else upstream_tree - if not git_archive(repo, source, output_dir, main_tree, - options.comp_type, - comp_level, - options.with_submodules): - raise GbpError("Cannot create upstream tarball at '%s'" % output_dir) - for component in options.components: - subtree = repo.tree_get_dir(upstream_tree, component) - if not subtree: - raise GbpError("No tree for '%s' found in '%s' to create additional tarball from" - % (component, upstream_tree)) - gbp.log.info("Creating additional tarball '%s' from '%s'" - % (du.orig_file(source, options.comp_type, component=component), - subtree)) - if not git_archive(repo, source, output_dir, subtree, - options.comp_type, - comp_level, - options.with_submodules, - component=component): - raise GbpError("Cannot create additional tarball %s at '%s'" - % (component, output_dir)) - return upstream_tree - - -def guess_comp_type(repo, comp_type, source, tarball_dir): - """Guess compression type""" - if comp_type != 'auto': - comp_type = compressor_aliases.get(comp_type, comp_type) - if comp_type not in compressor_opts: - gbp.log.warn("Unknown compression type - guessing.") - comp_type = 'auto' - - if comp_type == 'auto': - if not repo.has_pristine_tar_branch(): - if not tarball_dir: - tarball_dir = '..' - detected = None - for comp in compressor_opts.keys(): - if du.DebianPkgPolicy.has_orig(du.orig_file(source, comp), tarball_dir): - if detected is not None: - raise GbpError("Multiple orig tarballs found.") - detected = comp - if detected is not None: - comp_type = detected - else: - comp_type = 'gzip' - else: - regex = 'pristine-tar .* %s_%s\.orig.tar\.' % (source.name, source.upstream_version) - commits = repo.grep_log(regex, repo.pristine_tar_branch) - if commits: - commit = commits[-1] - gbp.log.debug("Found pristine-tar commit at '%s'" % commit) - else: - commit = repo.pristine_tar_branch - tarball = repo.get_commit_info(commit)['subject'] - (base_name, archive_fmt, comp_type) = parse_archive_filename(tarball) - gbp.log.debug("Determined compression type '%s'" % comp_type) - if not comp_type: - comp_type = 'gzip' - gbp.log.warn("Unknown compression type of %s, assuming %s" % (tarball, comp_type)) - return comp_type - - def check_tag(options, repo, source): """Perform specified consistency checks on git history""" tag = repo.version_to_tag(options.debian_tag, source.version) diff --git a/gbp/scripts/export_orig.py b/gbp/scripts/export_orig.py new file mode 100755 index 00000000..3627a5a8 --- /dev/null +++ b/gbp/scripts/export_orig.py @@ -0,0 +1,270 @@ +# vim: set fileencoding=utf-8 : +# +# (C) 2017 Guido Günther +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, please see +# +# +"""Create orig tarballs from git""" + +import os +import gbp.deb as du +from gbp.command_wrappers import CommandExecFailed +from gbp.deb.git import GitRepositoryError +from gbp.errors import GbpError +import gbp.log +import gbp.notifications +from gbp.scripts.common.buildpackage import (git_archive_submodules, + git_archive_single) +from gbp.pkg import compressor_opts, compressor_aliases, parse_archive_filename + + +# upstream tarball preparation +def git_archive(repo, source, output_dir, treeish, comp_type, comp_level, with_submodules, component=None): + "create a compressed orig tarball in output_dir using git_archive" + try: + comp_opts = compressor_opts[comp_type][0] + except KeyError: + raise GbpError("Unsupported compression type '%s'" % comp_type) + + output = os.path.join(output_dir, du.orig_file(source, comp_type, component=component)) + prefix = "%s-%s" % (source.name, source.upstream_version) + + try: + if repo.has_submodules() and with_submodules: + repo.update_submodules() + git_archive_submodules(repo, treeish, output, prefix, + comp_type, comp_level, comp_opts) + + else: + git_archive_single(treeish, output, prefix, + comp_type, comp_level, comp_opts) + except (GitRepositoryError, CommandExecFailed): + gbp.log.err("Error generating submodules' archives") + return False + except OSError as e: + gbp.log.err("Error creating %s: %s" % (output, str(e))) + return False + except GbpError: + raise + except Exception as e: + gbp.log.err("Error creating %s: %s" % (output, str(e))) + return False + return True + + +def prepare_upstream_tarball(repo, source, options, tarball_dir, output_dir): + """ + Make sure we have an upstream tarball. This involves looking in + tarball_dir, symlinking or building it. + """ + if not source.is_native() and not source.upstream_version: + raise GbpError("Non-native package '%s' " + "has invalid version '%s'" % (source.name, source.version)) + + options.comp_type = guess_comp_type(repo, + options.comp_type, + source, + options.tarball_dir) + + orig_files = [du.orig_file(source, options.comp_type)] + if options.components: + orig_files += [du.orig_file(source, options.comp_type, c) for c in options.components] + + # look in tarball_dir first, if found force a symlink to it + if options.tarball_dir: + gbp.log.debug("Looking for orig tarballs '%s' at '%s'" % (", ".join(orig_files), tarball_dir)) + missing = du.DebianPkgPolicy.symlink_origs(orig_files, tarball_dir, output_dir, force=True) + if missing: + msg = "Tarballs '%s' not found at '%s'" % (", ".join(missing), tarball_dir) + else: + msg = "All Orig tarballs '%s' found at '%s'" % (", ".join(orig_files), tarball_dir) + gbp.log.info(msg) + # FIXME: trouble with "gbp buildpackage" + # if options.no_create_orig: + # return + # Create tarball if missing or forced + if not du.DebianPkgPolicy.has_origs(orig_files, output_dir) or options.force_create: + if not pristine_tar_build_orig(repo, source, output_dir, options): + git_archive_build_orig(repo, source, output_dir, options) + pristine_tar_verify_orig(repo, source, options, output_dir, orig_files) + + +def pristine_tar_prepare_orig_tree(repo, source, options): + """Make sure the upstream tree exists + In case of component tarballs we need to recreate a tree for the + main tarball without the component subdirs. + """ + if options.components: + try: + upstream_tag = repo.version_to_tag(options.upstream_tag, + source.upstream_version) + tree_name = "%s^{tree}" % upstream_tag + repo.tree_drop_dirs(tree_name, options.components) + except GitRepositoryError: + raise GbpError("Couldn't find upstream tree '%s' to create " + "orig tarball via pristine-tar" % tree_name) + + +def pristine_tar_build_orig(repo, source, output_dir, options): + """ + Build orig tarball using pristine-tar + + @returns: C{True} if tarball was build, C{False} otherwise + """ + if options.pristine_tar: + if not repo.has_branch(repo.pristine_tar_branch): + gbp.log.warn('Pristine-tar branch "%s" not found' % + repo.pristine_tar.branch) + + pristine_tar_prepare_orig_tree(repo, source, options) + try: + repo.pristine_tar.checkout(source.name, + source.upstream_version, + options.comp_type, + output_dir) + for component in options.components: + repo.pristine_tar.checkout(source.name, + source.upstream_version, + options.comp_type, + output_dir, + component=component) + return True + except CommandExecFailed: + if options.pristine_tar_commit: + gbp.log.debug("pristine-tar checkout failed, " + "will commit tarball due to " + "'--pristine-tar-commit'") + else: + raise + return False + + +def pristine_tar_verify_orig(repo, source, options, output_dir, orig_files): + """ + Verify orig tarballs using prstine tar + + @returns: C{True} if tarball was build, C{False} otherwise + """ + if not options.pristine_tar: + return True + + pristine_tar_prepare_orig_tree(repo, source, options) + for f in orig_files: + repo.pristine_tar.verify(os.path.join(output_dir, f)) + return True + + +def get_upstream_tree(repo, source, options): + """Determine the upstream tree from the given options""" + if options.upstream_tree.upper() == 'TAG': + if source.upstream_version is None: + raise GitRepositoryError("Can't determine upstream version from changelog") + upstream_tree = repo.version_to_tag(options.upstream_tag, + source.upstream_version) + elif options.upstream_tree.upper() == 'BRANCH': + if not repo.has_branch(options.upstream_branch): + raise GbpError("%s is not a valid branch" % options.upstream_branch) + upstream_tree = options.upstream_branch + elif options.upstream_tree.upper() == 'SLOPPY': + tree_name = "%s^{tree}" % options.debian_branch + upstream_tree = repo.tree_drop_dirs(tree_name, ["debian"]) + else: + upstream_tree = options.upstream_tree + if not repo.has_treeish(upstream_tree): + raise GbpError("%s is not a valid treeish" % upstream_tree) + return upstream_tree + + +def git_archive_build_orig(repo, source, output_dir, options): + """ + Build orig tarball(s) using git-archive + + @param source: the changelog of the package we're acting on + @type source: L{DebianSource} + @param output_dir: where to put the tarball + @type output_dir: C{Str} + @param options: the parsed options + @type options: C{dict} of options + @return: the tree we built the tarball from + @rtype: C{str} + """ + upstream_tree = get_upstream_tree(repo, source, options) + gbp.log.info("Creating %s from '%s'" % (du.orig_file(source, + options.comp_type), + upstream_tree)) + comp_level = int(options.comp_level) if options.comp_level != '' else None + gbp.log.debug("Building upstream tarball with compression '%s'%s" % + (options.comp_type, + "' -%s'" % comp_level if comp_level is not None else '')) + main_tree = repo.tree_drop_dirs(upstream_tree, options.components) if options.components else upstream_tree + if not git_archive(repo, source, output_dir, main_tree, + options.comp_type, + comp_level, + options.with_submodules): + raise GbpError("Cannot create upstream tarball at '%s'" % output_dir) + for component in options.components: + subtree = repo.tree_get_dir(upstream_tree, component) + if not subtree: + raise GbpError("No tree for '%s' found in '%s' to create additional tarball from" + % (component, upstream_tree)) + gbp.log.info("Creating additional tarball '%s' from '%s'" + % (du.orig_file(source, options.comp_type, component=component), + subtree)) + if not git_archive(repo, source, output_dir, subtree, + options.comp_type, + comp_level, + options.with_submodules, + component=component): + raise GbpError("Cannot create additional tarball %s at '%s'" + % (component, output_dir)) + return upstream_tree + + +def guess_comp_type(repo, comp_type, source, tarball_dir): + """Guess compression type""" + if comp_type != 'auto': + comp_type = compressor_aliases.get(comp_type, comp_type) + if comp_type not in compressor_opts: + gbp.log.warn("Unknown compression type - guessing.") + comp_type = 'auto' + + if comp_type == 'auto': + if not repo.has_pristine_tar_branch(): + if not tarball_dir: + tarball_dir = '..' + detected = None + for comp in compressor_opts.keys(): + if du.DebianPkgPolicy.has_orig(du.orig_file(source, comp), tarball_dir): + if detected is not None: + raise GbpError("Multiple orig tarballs found.") + detected = comp + if detected is not None: + comp_type = detected + else: + comp_type = 'gzip' + else: + regex = 'pristine-tar .* %s_%s\.orig.tar\.' % (source.name, source.upstream_version) + commits = repo.grep_log(regex, repo.pristine_tar_branch) + if commits: + commit = commits[-1] + gbp.log.debug("Found pristine-tar commit at '%s'" % commit) + else: + commit = repo.pristine_tar_branch + tarball = repo.get_commit_info(commit)['subject'] + (base_name, archive_fmt, comp_type) = parse_archive_filename(tarball) + gbp.log.debug("Determined compression type '%s'" % comp_type) + if not comp_type: + comp_type = 'gzip' + gbp.log.warn("Unknown compression type of %s, assuming %s" % (tarball, comp_type)) + return comp_type -- cgit v1.2.3