From ff9ff7e2b54785acc733da2ca78d0da15bb5d681 Mon Sep 17 00:00:00 2001 From: Guido Günther Date: Sun, 27 Aug 2017 14:35:31 +0200 Subject: Use UpstreamSource for additional tarballs as well This will help on upstream tarball signatures as well as improving additional tarball handling. --- gbp/deb/git.py | 22 +++++++------- gbp/deb/upstreamsource.py | 64 ++++++++++++++++++++++++--------------- gbp/pkg/upstreamsource.py | 2 +- gbp/scripts/buildpackage.py | 13 +++++--- gbp/scripts/common/__init__.py | 4 ++- gbp/scripts/import_dsc.py | 26 ++++++++-------- gbp/scripts/import_orig.py | 69 ++++++++++++++++++++++-------------------- gbp/scripts/pristine_tar.py | 23 +++++++------- 8 files changed, 123 insertions(+), 100 deletions(-) (limited to 'gbp') diff --git a/gbp/deb/git.py b/gbp/deb/git.py index 12ad34d2..5f9003c4 100644 --- a/gbp/deb/git.py +++ b/gbp/deb/git.py @@ -282,28 +282,28 @@ class DebianGitRepository(PkgGitRepository): """ return True if self.has_branch(self.pristine_tar_branch) else False - def create_pristine_tar_commits(self, upstream_tree, tarball, component_tarballs): + def create_pristine_tar_commits(self, upstream_tree, sources): """ Create pristine-tar commits for a package with main tarball and (optional) component tarballs based on upstream_tree - @param tarball: path to main tarball - @param component_tarballs: C{list} of C{tuple}s of component - name and path to additional tarball @param upstream_tree: the treeish in the git repo to create the commits against + @param soures: C{list} of tarball as I{UpstreamSource}. First one being the main + tarball the other ones additional tarballs. """ - components = [c for (c, t) in component_tarballs] + components = [t.component for t in sources[1:]] main_tree = self.tree_drop_dirs(upstream_tree, components) try: - for component, name in component_tarballs: - subtree = self.tree_get_dir(upstream_tree, component) + for source in sources[1:]: + subtree = self.tree_get_dir(upstream_tree, source.component) if not subtree: raise GitRepositoryError("No tree for '%s' found in '%s' to create " - "pristine tar commit from" % (component, upstream_tree)) - gbp.log.debug("Creating pristine tar commit '%s' from '%s'" % (component, subtree)) - self.pristine_tar.commit(name, subtree, quiet=True) - self.pristine_tar.commit(tarball, main_tree, quiet=True) + "pristine tar commit from" % (source.component, + upstream_tree)) + gbp.log.debug("Creating pristine tar commit '%s' from '%s'" % (source.path, subtree)) + self.pristine_tar.commit(source.path, subtree, quiet=True) + self.pristine_tar.commit(sources[0].path, main_tree, quiet=True) except CommandExecFailed as e: raise GitRepositoryError(str(e)) diff --git a/gbp/deb/upstreamsource.py b/gbp/deb/upstreamsource.py index 04c37747..cead62c9 100644 --- a/gbp/deb/upstreamsource.py +++ b/gbp/deb/upstreamsource.py @@ -28,29 +28,43 @@ import tempfile class DebianUpstreamSource(UpstreamSource): """Upstream source class for Debian""" - def __init__(self, name, unpacked=None): + def __init__(self, name, unpacked=None, sig=None): super(DebianUpstreamSource, self).__init__(name, - unpacked, - DebianPkgPolicy) - - -def unpack_component_tarball(dest, component, tarball, filters): - """ - Unpack the tarball I{tarball} into dest naming it I{component}. - Apply filters during unpack. - """ - olddir = os.path.abspath(os.path.curdir) - tmpdir = None - try: - tmpdir = os.path.abspath(tempfile.mkdtemp(dir=os.path.join(dest, '..'))) - source = DebianUpstreamSource(tarball) - source.unpack(tmpdir, filters) - - newdest = os.path.join(dest, component) - if os.path.exists(newdest): - shutil.rmtree(newdest) - shutil.move(source.unpacked, newdest) - finally: - os.chdir(olddir) - if tmpdir is not None: - gbp.command_wrappers.RemoveTree(tmpdir)() + unpacked=unpacked, + sig=sig, + pkg_policy=DebianPkgPolicy) + + +class DebianAdditionalTarball(DebianUpstreamSource): + """Upstream source class for additional tarballs""" + def __init__(self, name, component, unpacked=None, sig=None): + self.component = component + super(DebianAdditionalTarball, self).__init__(name, + unpacked=unpacked, + sig=sig) + + def unpack(self, dest, filters): + """ + Unpack the additional tarball into {dir} naming it + I{component}. Apply filters during unpack. + + @param dir: the main tarball dir + @param filters: filters to apply + + We can't simply use unpack since we need to remove any preexisting dirs and + name the target directory after the component name. + """ + olddir = os.path.abspath(os.path.curdir) + tmpdir = None + try: + tmpdir = os.path.abspath(tempfile.mkdtemp(dir=os.path.join(dest, '..'))) + super(DebianAdditionalTarball, self).unpack(tmpdir, filters) + + newdest = os.path.join(dest, self.component) + if os.path.exists(newdest): + shutil.rmtree(newdest) + shutil.move(self.unpacked, newdest) + finally: + os.chdir(olddir) + if tmpdir is not None: + gbp.command_wrappers.RemoveTree(tmpdir)() diff --git a/gbp/pkg/upstreamsource.py b/gbp/pkg/upstreamsource.py index ab9e61de..f04d5675 100644 --- a/gbp/pkg/upstreamsource.py +++ b/gbp/pkg/upstreamsource.py @@ -39,7 +39,7 @@ class UpstreamSource(object): @cvar _unpacked: path to the unpacked source tree @type _unpacked: string """ - def __init__(self, name, unpacked=None, pkg_policy=PkgPolicy): + def __init__(self, name, unpacked=None, pkg_policy=PkgPolicy, sig=None): self._orig = False self._pkg_policy = pkg_policy self._path = name diff --git a/gbp/scripts/buildpackage.py b/gbp/scripts/buildpackage.py index 2114ed05..2aca53df 100755 --- a/gbp/scripts/buildpackage.py +++ b/gbp/scripts/buildpackage.py @@ -33,7 +33,7 @@ from gbp.deb.git import (GitRepositoryError, DebianGitRepository) from gbp.deb.source import DebianSource, DebianSourceError, FileVfs from gbp.deb.format import DebianSourceFormat from gbp.git.vfs import GitVfs -from gbp.deb.upstreamsource import DebianUpstreamSource, unpack_component_tarball +from gbp.deb.upstreamsource import DebianUpstreamSource, DebianAdditionalTarball from gbp.errors import GbpError import gbp.log import gbp.notifications @@ -139,10 +139,13 @@ def overlay_extract_origs(source, tarball_dir, dest_dir, options): # Unpack additional tarballs for c in options.components: - tarball = os.path.join(tarball_dir, source.upstream_tarball_name( - comp_type, component=c)) - gbp.log.info("Extracting '%s' to '%s/%s'" % (os.path.basename(tarball), dest_dir, c)) - unpack_component_tarball(dest_dir, c, tarball, []) + tarball = DebianAdditionalTarball(os.path.join(tarball_dir, + source.upstream_tarball_name( + comp_type, component=c)), + component=c) + gbp.log.info("Extracting '%s' to '%s/%s'" % (os.path.basename(tarball.path), + dest_dir, c)) + tarball.unpack(dest_dir, []) def source_vfs(repo, options, tree): diff --git a/gbp/scripts/common/__init__.py b/gbp/scripts/common/__init__.py index 023d2b84..b67e44c7 100644 --- a/gbp/scripts/common/__init__.py +++ b/gbp/scripts/common/__init__.py @@ -22,6 +22,7 @@ import traceback from gbp.errors import GbpError from gbp.deb import DebianPkgPolicy from gbp.pkg import Archive +from gbp.deb.upstreamsource import DebianAdditionalTarball class ExitCodes(object): @@ -51,6 +52,7 @@ def is_download(args): return False +# FIXME: this could become a method of DebianUpstreamSource def get_component_tarballs(name, version, tarball, components): """ Figure out the paths to the component tarballs based on the main @@ -64,7 +66,7 @@ def get_component_tarballs(name, version, tarball, components): comp_type, os.path.dirname(tarball), component) - tarballs.append((component, cname)) + tarballs.append(DebianAdditionalTarball(cname, component)) if not os.path.exists(cname): raise GbpError("Can not find component tarball %s" % cname) return tarballs diff --git a/gbp/scripts/import_dsc.py b/gbp/scripts/import_dsc.py index 7d6e4919..aa3308d6 100644 --- a/gbp/scripts/import_dsc.py +++ b/gbp/scripts/import_dsc.py @@ -26,7 +26,8 @@ import pipes import time import gbp.command_wrappers as gbpc from gbp.deb.dscfile import DscFile -from gbp.deb.upstreamsource import DebianUpstreamSource, unpack_component_tarball +from gbp.deb.upstreamsource import (DebianUpstreamSource, + DebianAdditionalTarball) from gbp.deb.git import (DebianGitRepository, GitRepositoryError) from gbp.deb.changelog import ChangeLog from gbp.git import rfc822_date_to_git @@ -487,11 +488,13 @@ def main(argv): # unpack dirs['tmp'] = os.path.abspath(tempfile.mkdtemp(dir='..')) - source = DebianUpstreamSource(dsc.tgz) - source.unpack(dirs['tmp'], options.filters) - for (component, tarball) in dsc.additional_tarballs.items(): - gbp.log.info("Found component tarball '%s'" % os.path.basename(tarball)) - unpack_component_tarball(source.unpacked, component, tarball, options.filters) + # FIXME: need to add signatures to DebianUpstreamSource later here + sources = [DebianUpstreamSource(dsc.tgz)] + sources += [DebianAdditionalTarball(t[1], t[0]) for t in dsc.additional_tarballs.items()] + sources[0].unpack(dirs['tmp'], options.filters) + for tarball in sources[1:]: + gbp.log.info("Found component tarball '%s'" % os.path.basename(tarball.path)) + tarball.unpack(sources[0].unpacked, options.filters) if repo.find_version(options.debian_tag, dsc.version): gbp.log.warn("Version %s already imported." % dsc.version) @@ -503,12 +506,12 @@ def main(argv): # import if dsc.native: - import_native(repo, source, dsc, options) + import_native(repo, sources[0], dsc, options) else: imported = False commit = repo.find_version(options.upstream_tag, dsc.upstream_version) if not repo.find_version(options.upstream_tag, dsc.upstream_version): - commit = import_upstream(repo, source, dsc, options) + commit = import_upstream(repo, sources[0], dsc, options) imported = True if not repo.has_branch(options.debian_branch): @@ -519,15 +522,12 @@ def main(argv): options.debian_branch) if dsc.diff or dsc.deb_tgz: - apply_debian_patch(repo, source, dsc, commit, options) + apply_debian_patch(repo, sources[0], dsc, commit, options) else: gbp.log.warn("Didn't find a diff to apply.") if imported and options.pristine_tar: - repo.create_pristine_tar_commits(commit, - dsc.tgz, - dsc.additional_tarballs.items()) - + repo.create_pristine_tar_commits(commit, sources) if repo.get_branch() == options.debian_branch or repo.empty: # Update HEAD if we modified the checked out branch repo.force_head(options.debian_branch, hard=True) diff --git a/gbp/scripts/import_orig.py b/gbp/scripts/import_orig.py index 6d5c147b..26eb3c79 100644 --- a/gbp/scripts/import_orig.py +++ b/gbp/scripts/import_orig.py @@ -1,6 +1,6 @@ # vim: set fileencoding=utf-8 : # -# (C) 2006, 2007, 2009, 2011, 2015, 2016 Guido Günther +# (C) 2006, 2007, 2009, 2011, 2015, 2016, 2019 Guido Günther # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or @@ -25,7 +25,8 @@ import time import gbp.command_wrappers as gbpc from gbp.deb import (DebianPkgPolicy, parse_changelog_repo) from gbp.deb.format import DebianSourceFormat -from gbp.deb.upstreamsource import DebianUpstreamSource, unpack_component_tarball +from gbp.deb.upstreamsource import (DebianUpstreamSource, + DebianAdditionalTarball) from gbp.deb.uscan import (Uscan, UscanError) from gbp.deb.changelog import ChangeLog, NoChangeLogError from gbp.deb.git import GitRepositoryError @@ -275,20 +276,24 @@ def debian_branch_merge_by_merge(repo, tag, version, options): repo.set_branch(branch) -def unpack_tarballs(sourcepackage, source, version, component_tarballs, options): +def unpack_tarballs(name, sources, version, options): tmpdir = tempfile.mkdtemp(dir='../') - if not source.is_dir(): # Unpack main tarball - source.unpack(tmpdir, options.filters) - gbp.log.debug("Unpacked '%s' to '%s'" % (source.path, source.unpacked)) + if not sources[0].is_dir(): # Unpack main tarball + sources[0].unpack(tmpdir, options.filters) + gbp.log.debug("Unpacked '%s' to '%s'" % (sources[0].path, sources[0].unpacked)) - if orig_needs_repack(source, options): - gbp.log.debug("Filter pristine-tar: repacking '%s' from '%s'" % (source.path, source.unpacked)) - (source, tmpdir) = repack_upstream(source, sourcepackage, version, tmpdir, options.filters) + if orig_needs_repack(sources[0], options): + gbp.log.debug("Filter pristine-tar: repacking '%s' from '%s'" % (sources[0].path, + sources[0].unpacked)) + # FIXME: we should repack the other tarballs here too (See #860457) + # for that we better move around sources instead of source[0] + (source, tmpdir) = repack_upstream(sources[0], name, version, tmpdir, options.filters) + sources[0] = source - if not source.is_dir(): # Unpack component tarballs - for (component, tarball) in component_tarballs: - unpack_component_tarball(source.unpacked, component, tarball, options.filters) - return (source, tmpdir) + if not sources[0].is_dir(): # Unpack component tarballs + for s in sources[1:]: + s.unpack(sources[0].unpacked, options.filters) + return (sources, tmpdir) def set_bare_repo_options(options): @@ -430,19 +435,16 @@ def main(argv): # Download the main tarball if options.download: - upstream = download_orig(args[0]) + sources = [download_orig(args[0])] else: - upstream = find_upstream(options.uscan, args, options.version) - if not upstream: + sources = [find_upstream(options.uscan, args, options.version)] + if not sources[0]: return ExitCodes.uscan_up_to_date # The main tarball - (name, version) = detect_name_and_version(repo, upstream, options) + (name, version) = detect_name_and_version(repo, sources[0], options) # Additional tarballs we expect to exist - component_tarballs = get_component_tarballs(name, - version, - upstream.path, - options.components) + sources += get_component_tarballs(name, version, sources[0].path, options.components) tag = repo.version_to_tag(options.upstream_tag, version) if repo.has_tag(tag): @@ -451,19 +453,19 @@ def main(argv): if repo.bare: set_bare_repo_options(options) - upstream, tmpdir = unpack_tarballs(name, upstream, version, component_tarballs, options) + sources, tmpdir = unpack_tarballs(name, sources, version, options) try: postunpack_hook(repo, tmpdir, options) except gbpc.CommandExecFailed: raise GbpError() # The hook already printed an error message - (pristine_orig, linked) = prepare_pristine_tar(upstream.path, + (pristine_orig, linked) = prepare_pristine_tar(sources[0].path, name, version) # Don't mess up our repo with git metadata from an upstream tarball try: - if os.path.isdir(os.path.join(upstream.unpacked, '.git/')): + if os.path.isdir(os.path.join(sources[0].unpacked, '.git/')): raise GbpError("The orig tarball contains .git metadata - giving up.") except OSError: pass @@ -472,7 +474,7 @@ def main(argv): import_branch = options.upstream_branch filter_msg = ["", " (filtering out %s)" % options.filters][len(options.filters) > 0] - gbp.log.info("Importing '%s' to branch '%s'%s..." % (upstream.path, + gbp.log.info("Importing '%s' to branch '%s'%s..." % (sources[0].path, import_branch, filter_msg)) gbp.log.info("Source package is %s" % name) @@ -480,7 +482,7 @@ def main(argv): msg = upstream_import_commit_msg(options, version) - commit = repo.commit_dir(upstream.unpacked, + commit = repo.commit_dir(sources[0].unpacked, msg=msg, branch=import_branch, other_parents=repo.vcs_tag_parent(options.vcs_tag, version), @@ -490,11 +492,12 @@ def main(argv): if options.pristine_tar: if pristine_orig: repo.rrr_branch('pristine-tar') - repo.create_pristine_tar_commits(import_branch, - pristine_orig, - component_tarballs) + # For all practical purposes we're interested in pristine_orig's path + if pristine_orig != sources[0].path: + sources[0]._path = pristine_orig + repo.create_pristine_tar_commits(import_branch, sources) else: - gbp.log.warn("'%s' not an archive, skipping pristine-tar" % upstream.path) + gbp.log.warn("'%s' not an archive, skipping pristine-tar" % sources[0].path) repo.create_tag(name=tag, msg="Upstream version %s" % version, @@ -526,9 +529,9 @@ def main(argv): postimport_hook(repo, tag, version, options) except (gbpc.CommandExecFailed, GitRepositoryError) as err: msg = str(err) or 'Unknown error, please report a bug' - raise GbpError("Import of %s failed: %s" % (upstream.path, msg)) + raise GbpError("Import of %s failed: %s" % (sources[0].path, msg)) except KeyboardInterrupt: - raise GbpError("Import of %s failed: aborted by user" % (upstream.path)) + raise GbpError("Import of %s failed: aborted by user" % (sources[0].path)) except GbpError as err: if str(err): gbp.log.err(err) @@ -542,7 +545,7 @@ def main(argv): cleanup_tmp_tree(tmpdir) if not ret: - gbp.log.info("Successfully imported version %s of %s" % (version, upstream.path)) + gbp.log.info("Successfully imported version %s of %s" % (version, sources[0].path)) return ret diff --git a/gbp/scripts/pristine_tar.py b/gbp/scripts/pristine_tar.py index 792bcab6..e9a556a8 100644 --- a/gbp/scripts/pristine_tar.py +++ b/gbp/scripts/pristine_tar.py @@ -24,6 +24,7 @@ from gbp.command_wrappers import CommandExecFailed from gbp.config import GbpOptionParserDebian from gbp.deb.git import (GitRepositoryError, DebianGitRepository) from gbp.deb.source import DebianSource +from gbp.deb.upstreamsource import DebianUpstreamSource from gbp.errors import GbpError from gbp.scripts.common import ExitCodes, get_component_tarballs @@ -90,16 +91,16 @@ def main(argv): except GitRepositoryError: raise GbpError("%s is not a git repository" % (os.path.abspath('.'))) - source = DebianSource('.') - component_tarballs = get_component_tarballs(source.sourcepkg, - source.upstream_version, - tarball, - options.components) + debsource = DebianSource('.') + # FIXME: this should be a single call + sources = [DebianUpstreamSource(tarball)] + sources += get_component_tarballs(debsource.sourcepkg, + debsource.upstream_version, + sources[0].path, + options.components) upstream_tag = repo.version_to_tag(options.upstream_tag, - source.upstream_version) - repo.create_pristine_tar_commits(upstream_tag, - tarball, - component_tarballs) + debsource.upstream_version) + repo.create_pristine_tar_commits(upstream_tag, sources) ret = 0 except (GitRepositoryError, GbpError, CommandExecFailed) as err: if str(err): @@ -109,8 +110,8 @@ def main(argv): if not ret: comp_msg = (' with additional tarballs for %s' - % ", ".join([os.path.basename(t[1]) for t in component_tarballs])) if component_tarballs else '' - gbp.log.info("Successfully committed pristine-tar data for version %s of %s%s" % (source.upstream_version, + % ", ".join([os.path.basename(t.path) for t in sources[1:]])) if sources[1:] else '' + gbp.log.info("Successfully committed pristine-tar data for version %s of %s%s" % (debsource.version, tarball, comp_msg)) return ret -- cgit v1.2.3