Use UpstreamSource for additional tarballs as well

This will help on upstream tarball signatures as well as
improving additional tarball handling.

8 files changed, 123 insertions, 100 deletions

diff --git a/gbp/deb/git.py b/gbp/deb/git.py
index 12ad34d2..5f9003c4 100644
--- a/gbp/deb/git.py
+++ b/gbp/deb/git.py
@@ -282,28 +282,28 @@ class DebianGitRepository(PkgGitRepository):
         """
         return True if self.has_branch(self.pristine_tar_branch) else False
 
-    def create_pristine_tar_commits(self, upstream_tree, tarball, component_tarballs):
+    def create_pristine_tar_commits(self, upstream_tree, sources):
         """
         Create pristine-tar commits for a package with main tarball
         and (optional) component tarballs based on upstream_tree
 
-        @param tarball: path to main tarball
-        @param component_tarballs: C{list} of C{tuple}s of component
-            name and path to additional tarball
         @param upstream_tree: the treeish in the git repo to create the commits against
+        @param soures: C{list} of tarball as I{UpstreamSource}. First one being the main
+                       tarball the other ones additional tarballs.
         """
-        components = [c for (c, t) in component_tarballs]
+        components = [t.component for t in sources[1:]]
         main_tree = self.tree_drop_dirs(upstream_tree, components)
 
         try:
-            for component, name in component_tarballs:
-                subtree = self.tree_get_dir(upstream_tree, component)
+            for source in sources[1:]:
+                subtree = self.tree_get_dir(upstream_tree, source.component)
                 if not subtree:
                     raise GitRepositoryError("No tree for '%s' found in '%s' to create "
-                                             "pristine tar commit from" % (component, upstream_tree))
-                gbp.log.debug("Creating pristine tar commit '%s' from '%s'" % (component, subtree))
-                self.pristine_tar.commit(name, subtree, quiet=True)
-            self.pristine_tar.commit(tarball, main_tree, quiet=True)
+                                             "pristine tar commit from" % (source.component,
+                                                                           upstream_tree))
+                gbp.log.debug("Creating pristine tar commit '%s' from '%s'" % (source.path, subtree))
+                self.pristine_tar.commit(source.path, subtree, quiet=True)
+            self.pristine_tar.commit(sources[0].path, main_tree, quiet=True)
         except CommandExecFailed as e:
             raise GitRepositoryError(str(e))
 
diff --git a/gbp/deb/upstreamsource.py b/gbp/deb/upstreamsource.py
index 04c37747..cead62c9 100644
--- a/gbp/deb/upstreamsource.py
+++ b/gbp/deb/upstreamsource.py
@@ -28,29 +28,43 @@ import tempfile
 
 class DebianUpstreamSource(UpstreamSource):
     """Upstream source class for Debian"""
-    def __init__(self, name, unpacked=None):
+    def __init__(self, name, unpacked=None, sig=None):
         super(DebianUpstreamSource, self).__init__(name,
-                                                   unpacked,
-                                                   DebianPkgPolicy)
-
-
-def unpack_component_tarball(dest, component, tarball, filters):
-    """
-    Unpack the tarball I{tarball} into dest naming it I{component}.
-    Apply filters during unpack.
-    """
-    olddir = os.path.abspath(os.path.curdir)
-    tmpdir = None
-    try:
-        tmpdir = os.path.abspath(tempfile.mkdtemp(dir=os.path.join(dest, '..')))
-        source = DebianUpstreamSource(tarball)
-        source.unpack(tmpdir, filters)
-
-        newdest = os.path.join(dest, component)
-        if os.path.exists(newdest):
-            shutil.rmtree(newdest)
-        shutil.move(source.unpacked, newdest)
-    finally:
-        os.chdir(olddir)
-        if tmpdir is not None:
-            gbp.command_wrappers.RemoveTree(tmpdir)()
+                                                   unpacked=unpacked,
+                                                   sig=sig,
+                                                   pkg_policy=DebianPkgPolicy)
+
+
+class DebianAdditionalTarball(DebianUpstreamSource):
+    """Upstream source class for additional tarballs"""
+    def __init__(self, name, component, unpacked=None, sig=None):
+        self.component = component
+        super(DebianAdditionalTarball, self).__init__(name,
+                                                      unpacked=unpacked,
+                                                      sig=sig)
+
+    def unpack(self, dest, filters):
+        """
+        Unpack the additional tarball into {dir} naming it
+        I{component}.  Apply filters during unpack.
+
+        @param dir: the main tarball dir
+        @param filters: filters to apply
+
+        We can't simply use unpack since we need to remove any preexisting dirs and
+        name the target directory after the component name.
+        """
+        olddir = os.path.abspath(os.path.curdir)
+        tmpdir = None
+        try:
+            tmpdir = os.path.abspath(tempfile.mkdtemp(dir=os.path.join(dest, '..')))
+            super(DebianAdditionalTarball, self).unpack(tmpdir, filters)
+
+            newdest = os.path.join(dest, self.component)
+            if os.path.exists(newdest):
+                shutil.rmtree(newdest)
+            shutil.move(self.unpacked, newdest)
+        finally:
+            os.chdir(olddir)
+            if tmpdir is not None:
+                gbp.command_wrappers.RemoveTree(tmpdir)()
diff --git a/gbp/pkg/upstreamsource.py b/gbp/pkg/upstreamsource.py
index ab9e61de..f04d5675 100644
--- a/gbp/pkg/upstreamsource.py
+++ b/gbp/pkg/upstreamsource.py
@@ -39,7 +39,7 @@ class UpstreamSource(object):
     @cvar _unpacked: path to the unpacked source tree
     @type _unpacked: string
     """
-    def __init__(self, name, unpacked=None, pkg_policy=PkgPolicy):
+    def __init__(self, name, unpacked=None, pkg_policy=PkgPolicy, sig=None):
         self._orig = False
         self._pkg_policy = pkg_policy
         self._path = name
diff --git a/gbp/scripts/buildpackage.py b/gbp/scripts/buildpackage.py
index 2114ed05..2aca53df 100755
--- a/gbp/scripts/buildpackage.py
+++ b/gbp/scripts/buildpackage.py
@@ -33,7 +33,7 @@ from gbp.deb.git import (GitRepositoryError, DebianGitRepository)
 from gbp.deb.source import DebianSource, DebianSourceError, FileVfs
 from gbp.deb.format import DebianSourceFormat
 from gbp.git.vfs import GitVfs
-from gbp.deb.upstreamsource import DebianUpstreamSource, unpack_component_tarball
+from gbp.deb.upstreamsource import DebianUpstreamSource, DebianAdditionalTarball
 from gbp.errors import GbpError
 import gbp.log
 import gbp.notifications
@@ -139,10 +139,13 @@ def overlay_extract_origs(source, tarball_dir, dest_dir, options):
 
     # Unpack additional tarballs
     for c in options.components:
-        tarball = os.path.join(tarball_dir, source.upstream_tarball_name(
-            comp_type, component=c))
-        gbp.log.info("Extracting '%s' to '%s/%s'" % (os.path.basename(tarball), dest_dir, c))
-        unpack_component_tarball(dest_dir, c, tarball, [])
+        tarball = DebianAdditionalTarball(os.path.join(tarball_dir,
+                                                       source.upstream_tarball_name(
+                                                           comp_type, component=c)),
+                                          component=c)
+        gbp.log.info("Extracting '%s' to '%s/%s'" % (os.path.basename(tarball.path),
+                                                     dest_dir, c))
+        tarball.unpack(dest_dir, [])
 
 
 def source_vfs(repo, options, tree):
diff --git a/gbp/scripts/common/__init__.py b/gbp/scripts/common/__init__.py
index 023d2b84..b67e44c7 100644
--- a/gbp/scripts/common/__init__.py
+++ b/gbp/scripts/common/__init__.py
@@ -22,6 +22,7 @@ import traceback
 from gbp.errors import GbpError
 from gbp.deb import DebianPkgPolicy
 from gbp.pkg import Archive
+from gbp.deb.upstreamsource import DebianAdditionalTarball
 
 
 class ExitCodes(object):
@@ -51,6 +52,7 @@ def is_download(args):
     return False
 
 
+# FIXME: this could become a method of DebianUpstreamSource
 def get_component_tarballs(name, version, tarball, components):
     """
     Figure out the paths to the component tarballs based on the main
@@ -64,7 +66,7 @@ def get_component_tarballs(name, version, tarball, components):
                                                    comp_type,
                                                    os.path.dirname(tarball),
                                                    component)
-        tarballs.append((component, cname))
+        tarballs.append(DebianAdditionalTarball(cname, component))
         if not os.path.exists(cname):
             raise GbpError("Can not find component tarball %s" % cname)
     return tarballs
diff --git a/gbp/scripts/import_dsc.py b/gbp/scripts/import_dsc.py
index 7d6e4919..aa3308d6 100644
--- a/gbp/scripts/import_dsc.py
+++ b/gbp/scripts/import_dsc.py
@@ -26,7 +26,8 @@ import pipes
 import time
 import gbp.command_wrappers as gbpc
 from gbp.deb.dscfile import DscFile
-from gbp.deb.upstreamsource import DebianUpstreamSource, unpack_component_tarball
+from gbp.deb.upstreamsource import (DebianUpstreamSource,
+                                    DebianAdditionalTarball)
 from gbp.deb.git import (DebianGitRepository, GitRepositoryError)
 from gbp.deb.changelog import ChangeLog
 from gbp.git import rfc822_date_to_git
@@ -487,11 +488,13 @@ def main(argv):
 
         # unpack
         dirs['tmp'] = os.path.abspath(tempfile.mkdtemp(dir='..'))
-        source = DebianUpstreamSource(dsc.tgz)
-        source.unpack(dirs['tmp'], options.filters)
-        for (component, tarball) in dsc.additional_tarballs.items():
-            gbp.log.info("Found component tarball '%s'" % os.path.basename(tarball))
-            unpack_component_tarball(source.unpacked, component, tarball, options.filters)
+        # FIXME: need to add signatures to DebianUpstreamSource later here
+        sources = [DebianUpstreamSource(dsc.tgz)]
+        sources += [DebianAdditionalTarball(t[1], t[0]) for t in dsc.additional_tarballs.items()]
+        sources[0].unpack(dirs['tmp'], options.filters)
+        for tarball in sources[1:]:
+            gbp.log.info("Found component tarball '%s'" % os.path.basename(tarball.path))
+            tarball.unpack(sources[0].unpacked, options.filters)
 
         if repo.find_version(options.debian_tag, dsc.version):
             gbp.log.warn("Version %s already imported." % dsc.version)
@@ -503,12 +506,12 @@ def main(argv):
 
         # import
         if dsc.native:
-            import_native(repo, source, dsc, options)
+            import_native(repo, sources[0], dsc, options)
         else:
             imported = False
             commit = repo.find_version(options.upstream_tag, dsc.upstream_version)
             if not repo.find_version(options.upstream_tag, dsc.upstream_version):
-                commit = import_upstream(repo, source, dsc, options)
+                commit = import_upstream(repo, sources[0], dsc, options)
                 imported = True
 
             if not repo.has_branch(options.debian_branch):
@@ -519,15 +522,12 @@ def main(argv):
                                    options.debian_branch)
 
             if dsc.diff or dsc.deb_tgz:
-                apply_debian_patch(repo, source, dsc, commit, options)
+                apply_debian_patch(repo, sources[0], dsc, commit, options)
             else:
                 gbp.log.warn("Didn't find a diff to apply.")
 
             if imported and options.pristine_tar:
-                repo.create_pristine_tar_commits(commit,
-                                                 dsc.tgz,
-                                                 dsc.additional_tarballs.items())
-
+                repo.create_pristine_tar_commits(commit, sources)
         if repo.get_branch() == options.debian_branch or repo.empty:
             # Update HEAD if we modified the checked out branch
             repo.force_head(options.debian_branch, hard=True)
diff --git a/gbp/scripts/import_orig.py b/gbp/scripts/import_orig.py
index 6d5c147b..26eb3c79 100644
--- a/gbp/scripts/import_orig.py
+++ b/gbp/scripts/import_orig.py
@@ -1,6 +1,6 @@
 # vim: set fileencoding=utf-8 :
 #
-# (C) 2006, 2007, 2009, 2011, 2015, 2016 Guido Günther <agx@sigxcpu.org>
+# (C) 2006, 2007, 2009, 2011, 2015, 2016, 2019 Guido Günther <agx@sigxcpu.org>
 #    This program is free software; you can redistribute it and/or modify
 #    it under the terms of the GNU General Public License as published by
 #    the Free Software Foundation; either version 2 of the License, or
@@ -25,7 +25,8 @@ import time
 import gbp.command_wrappers as gbpc
 from gbp.deb import (DebianPkgPolicy, parse_changelog_repo)
 from gbp.deb.format import DebianSourceFormat
-from gbp.deb.upstreamsource import DebianUpstreamSource, unpack_component_tarball
+from gbp.deb.upstreamsource import (DebianUpstreamSource,
+                                    DebianAdditionalTarball)
 from gbp.deb.uscan import (Uscan, UscanError)
 from gbp.deb.changelog import ChangeLog, NoChangeLogError
 from gbp.deb.git import GitRepositoryError
@@ -275,20 +276,24 @@ def debian_branch_merge_by_merge(repo, tag, version, options):
     repo.set_branch(branch)
 
 
-def unpack_tarballs(sourcepackage, source, version, component_tarballs, options):
+def unpack_tarballs(name, sources, version, options):
     tmpdir = tempfile.mkdtemp(dir='../')
-    if not source.is_dir():  # Unpack main tarball
-        source.unpack(tmpdir, options.filters)
-        gbp.log.debug("Unpacked '%s' to '%s'" % (source.path, source.unpacked))
+    if not sources[0].is_dir():  # Unpack main tarball
+        sources[0].unpack(tmpdir, options.filters)
+        gbp.log.debug("Unpacked '%s' to '%s'" % (sources[0].path, sources[0].unpacked))
 
-    if orig_needs_repack(source, options):
-        gbp.log.debug("Filter pristine-tar: repacking '%s' from '%s'" % (source.path, source.unpacked))
-        (source, tmpdir) = repack_upstream(source, sourcepackage, version, tmpdir, options.filters)
+    if orig_needs_repack(sources[0], options):
+        gbp.log.debug("Filter pristine-tar: repacking '%s' from '%s'" % (sources[0].path,
+                                                                         sources[0].unpacked))
+        # FIXME: we should repack the other tarballs here too (See #860457)
+        # for that we better move around sources instead of source[0]
+        (source, tmpdir) = repack_upstream(sources[0], name, version, tmpdir, options.filters)
+        sources[0] = source
 
-    if not source.is_dir():  # Unpack component tarballs
-        for (component, tarball) in component_tarballs:
-            unpack_component_tarball(source.unpacked, component, tarball, options.filters)
-    return (source, tmpdir)
+    if not sources[0].is_dir():  # Unpack component tarballs
+        for s in sources[1:]:
+            s.unpack(sources[0].unpacked, options.filters)
+    return (sources, tmpdir)
 
 
 def set_bare_repo_options(options):
@@ -430,19 +435,16 @@ def main(argv):
 
         # Download the main tarball
         if options.download:
-            upstream = download_orig(args[0])
+            sources = [download_orig(args[0])]
         else:
-            upstream = find_upstream(options.uscan, args, options.version)
-            if not upstream:
+            sources = [find_upstream(options.uscan, args, options.version)]
+            if not sources[0]:
                 return ExitCodes.uscan_up_to_date
 
         # The main tarball
-        (name, version) = detect_name_and_version(repo, upstream, options)
+        (name, version) = detect_name_and_version(repo, sources[0], options)
         # Additional tarballs we expect to exist
-        component_tarballs = get_component_tarballs(name,
-                                                    version,
-                                                    upstream.path,
-                                                    options.components)
+        sources += get_component_tarballs(name, version, sources[0].path, options.components)
 
         tag = repo.version_to_tag(options.upstream_tag, version)
         if repo.has_tag(tag):
@@ -451,19 +453,19 @@ def main(argv):
         if repo.bare:
             set_bare_repo_options(options)
 
-        upstream, tmpdir = unpack_tarballs(name, upstream, version, component_tarballs, options)
+        sources, tmpdir = unpack_tarballs(name, sources, version, options)
         try:
             postunpack_hook(repo, tmpdir, options)
         except gbpc.CommandExecFailed:
             raise GbpError()  # The hook already printed an error message
 
-        (pristine_orig, linked) = prepare_pristine_tar(upstream.path,
+        (pristine_orig, linked) = prepare_pristine_tar(sources[0].path,
                                                        name,
                                                        version)
 
         # Don't mess up our repo with git metadata from an upstream tarball
         try:
-            if os.path.isdir(os.path.join(upstream.unpacked, '.git/')):
+            if os.path.isdir(os.path.join(sources[0].unpacked, '.git/')):
                 raise GbpError("The orig tarball contains .git metadata - giving up.")
         except OSError:
             pass
@@ -472,7 +474,7 @@ def main(argv):
             import_branch = options.upstream_branch
             filter_msg = ["", " (filtering out %s)"
                               % options.filters][len(options.filters) > 0]
-            gbp.log.info("Importing '%s' to branch '%s'%s..." % (upstream.path,
+            gbp.log.info("Importing '%s' to branch '%s'%s..." % (sources[0].path,
                                                                  import_branch,
                                                                  filter_msg))
             gbp.log.info("Source package is %s" % name)
@@ -480,7 +482,7 @@ def main(argv):
 
             msg = upstream_import_commit_msg(options, version)
 
-            commit = repo.commit_dir(upstream.unpacked,
+            commit = repo.commit_dir(sources[0].unpacked,
                                      msg=msg,
                                      branch=import_branch,
                                      other_parents=repo.vcs_tag_parent(options.vcs_tag, version),
@@ -490,11 +492,12 @@ def main(argv):
             if options.pristine_tar:
                 if pristine_orig:
                     repo.rrr_branch('pristine-tar')
-                    repo.create_pristine_tar_commits(import_branch,
-                                                     pristine_orig,
-                                                     component_tarballs)
+                    # For all practical purposes we're interested in pristine_orig's path
+                    if pristine_orig != sources[0].path:
+                        sources[0]._path = pristine_orig
+                    repo.create_pristine_tar_commits(import_branch, sources)
                 else:
-                    gbp.log.warn("'%s' not an archive, skipping pristine-tar" % upstream.path)
+                    gbp.log.warn("'%s' not an archive, skipping pristine-tar" % sources[0].path)
 
             repo.create_tag(name=tag,
                             msg="Upstream version %s" % version,
@@ -526,9 +529,9 @@ def main(argv):
             postimport_hook(repo, tag, version, options)
         except (gbpc.CommandExecFailed, GitRepositoryError) as err:
             msg = str(err) or 'Unknown error, please report a bug'
-            raise GbpError("Import of %s failed: %s" % (upstream.path, msg))
+            raise GbpError("Import of %s failed: %s" % (sources[0].path, msg))
         except KeyboardInterrupt:
-            raise GbpError("Import of %s failed: aborted by user" % (upstream.path))
+            raise GbpError("Import of %s failed: aborted by user" % (sources[0].path))
     except GbpError as err:
         if str(err):
             gbp.log.err(err)
@@ -542,7 +545,7 @@ def main(argv):
         cleanup_tmp_tree(tmpdir)
 
     if not ret:
-        gbp.log.info("Successfully imported version %s of %s" % (version, upstream.path))
+        gbp.log.info("Successfully imported version %s of %s" % (version, sources[0].path))
     return ret
 
 
diff --git a/gbp/scripts/pristine_tar.py b/gbp/scripts/pristine_tar.py
index 792bcab6..e9a556a8 100644
--- a/gbp/scripts/pristine_tar.py
+++ b/gbp/scripts/pristine_tar.py
@@ -24,6 +24,7 @@ from gbp.command_wrappers import CommandExecFailed
 from gbp.config import GbpOptionParserDebian
 from gbp.deb.git import (GitRepositoryError, DebianGitRepository)
 from gbp.deb.source import DebianSource
+from gbp.deb.upstreamsource import DebianUpstreamSource
 from gbp.errors import GbpError
 from gbp.scripts.common import ExitCodes, get_component_tarballs
 
@@ -90,16 +91,16 @@ def main(argv):
         except GitRepositoryError:
             raise GbpError("%s is not a git repository" % (os.path.abspath('.')))
 
-        source = DebianSource('.')
-        component_tarballs = get_component_tarballs(source.sourcepkg,
-                                                    source.upstream_version,
-                                                    tarball,
-                                                    options.components)
+        debsource = DebianSource('.')
+        # FIXME: this should be a single call
+        sources = [DebianUpstreamSource(tarball)]
+        sources += get_component_tarballs(debsource.sourcepkg,
+                                          debsource.upstream_version,
+                                          sources[0].path,
+                                          options.components)
         upstream_tag = repo.version_to_tag(options.upstream_tag,
-                                           source.upstream_version)
-        repo.create_pristine_tar_commits(upstream_tag,
-                                         tarball,
-                                         component_tarballs)
+                                           debsource.upstream_version)
+        repo.create_pristine_tar_commits(upstream_tag, sources)
         ret = 0
     except (GitRepositoryError, GbpError, CommandExecFailed) as err:
         if str(err):
@@ -109,8 +110,8 @@ def main(argv):
 
     if not ret:
         comp_msg = (' with additional tarballs for %s'
-                    % ", ".join([os.path.basename(t[1]) for t in component_tarballs])) if component_tarballs else ''
-        gbp.log.info("Successfully committed pristine-tar data for version %s of %s%s" % (source.upstream_version,
+                    % ", ".join([os.path.basename(t.path) for t in sources[1:]])) if sources[1:] else ''
+        gbp.log.info("Successfully committed pristine-tar data for version %s of %s%s" % (debsource.version,
                                                                                           tarball,
                                                                                           comp_msg))
     return ret