deb.git: helper for upstream tarball creation

once we have sorted out all the parameters let DebianGitRepository
handle the details.

5 files changed, 110 insertions, 85 deletions

diff --git a/gbp/deb/__init__.py b/gbp/deb/__init__.py
index 7f75b8fe..533b71bd 100644
--- a/gbp/deb/__init__.py
+++ b/gbp/deb/__init__.py
@@ -25,7 +25,7 @@ from gbp.git import GitRepositoryError
 
 # Make sure these are available with 'import gbp.deb'
 from gbp.deb.changelog import ChangeLog, NoChangeLogError
-from gbp.deb.policy import DebianPkgPolicy
+from gbp.deb.policy import DebianPkgPolicy                # noqa: F401
 
 
 class DpkgCompareVersions(gbpc.Command):
diff --git a/gbp/deb/git.py b/gbp/deb/git.py
index ddea38b5..3edd4546 100644
--- a/gbp/deb/git.py
+++ b/gbp/deb/git.py
@@ -16,7 +16,9 @@
 #    <http://www.gnu.org/licenses/>
 """A Git Repository that keeps a Debian Package"""
 
+import os
 import re
+
 from gbp.command_wrappers import CommandExecFailed
 from gbp.git import GitRepositoryError
 from gbp.deb.pristinetar import DebianPristineTar
@@ -301,11 +303,55 @@ class DebianGitRepository(PkgGitRepository):
             for component, name in component_tarballs:
                 subtree = self.tree_get_dir(upstream_tree, component)
                 if not subtree:
-                    raise GitRepositoryError("No tree for '%s' found in '%s' to create pristine tar commit from" % (component, upstream_tree))
+                    raise GitRepositoryError("No tree for '%s' found in '%s' to create "
+                                             "pristine tar commit from" % (component, upstream_tree))
                 gbp.log.debug("Creating pristine tar commit '%s' from '%s'" % (component, subtree))
                 self.pristine_tar.commit(name, subtree, quiet=True)
             self.pristine_tar.commit(tarball, main_tree, quiet=True)
         except CommandExecFailed as e:
             raise GitRepositoryError(str(e))
 
+    def create_upstream_tarball_via_pristine_tar(self, source, output_dir, comp, component=None):
+        output = source.upstream_tarball_name(comp.type, component=component)
+        try:
+            self.pristine_tar.checkout(source.name, source.upstream_version, comp.type, output_dir,
+                                       component=component)
+        except Exception as e:
+            raise GitRepositoryError("Error creating %s: %s" % (output, e))
+        return True
+
+    def create_upstream_tarball_via_git_archive(self, source, output_dir, treeish,
+                                                comp, with_submodules, component=None):
+        """
+        Create a compressed orig tarball in output_dir using git archive
+
+        @param source: debian source package
+        @type source: L{DebianSource}
+        @param output_dir: output directory
+        @param type: C{str}
+        @param treeish: git treeish
+        @type treeish: C{str}
+        @param comp: compressor
+        @type comp: L{Compressor}
+        @param with_submodules: wether to add submodules
+        @type with_submodules: C{bool}
+        @param component: component to add to tarball name
+        @type component: C{str}
+
+        Raises GitRepositoryError in case of an error
+        """
+        submodules = False
+        output = os.path.join(output_dir,
+                              source.upstream_tarball_name(comp.type, component=component))
+        prefix = "%s-%s" % (source.name, source.upstream_version)
+
+        try:
+            if self.has_submodules() and with_submodules:
+                submodules = True
+                self.update_submodules()
+            self.archive_comp(treeish, output, prefix, comp, submodules=submodules)
+        except Exception as e:
+            raise GitRepositoryError("Error creating %s: %s" % (output, e))
+        return True
+
 # vim:et:ts=4:sw=4:et:sts=4:ai:set list listchars=tab\:»·,trail\:·:
diff --git a/gbp/scripts/export_orig.py b/gbp/scripts/export_orig.py
index f4fc4a12..75b3e0df 100755
--- a/gbp/scripts/export_orig.py
+++ b/gbp/scripts/export_orig.py
@@ -27,34 +27,15 @@ import gbp.notifications
 from gbp.pkg import Compressor, Archive
 
 
-# upstream tarball preparation
-def git_archive(repo, source, output_dir, treeish, comp, with_submodules, component=None):
-    """create a compressed orig tarball in output_dir using git_archive"""
-    submodules = False
-    output = os.path.join(output_dir,
-                          source.upstream_tarball_name(comp.type, component=component))
-    prefix = "%s-%s" % (source.name, source.upstream_version)
-
-    try:
-        if repo.has_submodules() and with_submodules:
-            submodules = True
-            repo.update_submodules()
-        repo.archive_comp(treeish, output, prefix, comp, submodules=submodules)
-    except (GbpError, GitRepositoryError, CommandExecFailed) as e:
-        gbp.log.err("%s" % e)
-        return False
-    except Exception as e:
-        gbp.log.err("Error creating %s: %s" % (output, e))
-        return False
-    return True
-
-
 def prepare_upstream_tarballs(repo, source, options, tarball_dir, output_dir):
     """
     Make sure we have the needed upstream tarballs. This involves
     looking in tarball_dir and symlinking them or building them from either
     pristine-tar or plain git.
     """
+    if hasattr(options, 'no_create_orig') and options.no_create_orig:
+        return
+
     if not source.is_native() and not source.upstream_version:
         raise GbpError("Non-native package '%s' "
                        "has invalid version '%s'" % (source.name, source.version))
@@ -64,6 +45,7 @@ def prepare_upstream_tarballs(repo, source, options, tarball_dir, output_dir):
                                         source,
                                         options.tarball_dir)
     orig_files = source.upstream_tarball_names(options.comp_type, options.components)
+
     # look in tarball_dir first, if found force a symlink to it
     if options.tarball_dir:
         gbp.log.debug("Looking for orig tarballs '%s' at '%s'" % (", ".join(orig_files), tarball_dir))
@@ -73,18 +55,18 @@ def prepare_upstream_tarballs(repo, source, options, tarball_dir, output_dir):
         else:
             msg = "All Orig tarballs '%s' found at '%s'" % (", ".join(orig_files), tarball_dir)
         gbp.log.info(msg)
-    # FIXME: trouble with "gbp buildpackage"
-    # if options.no_create_orig:
-    #     return
+
     # Create tarball if missing or forced
     if not du.DebianPkgPolicy.has_origs(orig_files, output_dir) or options.force_create:
-        if not pristine_tar_build_orig(repo, source, output_dir, options):
-            git_archive_build_orig(repo, source, output_dir, options)
-    pristine_tar_verify_orig(repo, source, options, output_dir, orig_files)
+        if not pristine_tar_build_origs(repo, source, output_dir, options):
+            git_archive_build_origs(repo, source, output_dir, options)
+    pristine_tar_verify_origs(repo, source, options, output_dir, orig_files)
 
 
 def pristine_tar_prepare_orig_tree(repo, source, options):
-    """Make sure the upstream tree exists
+    """
+    Make sure the upstream tree exists
+
     In case of component tarballs we need to recreate a tree for the
     main tarball without the component subdirs.
     """
@@ -99,41 +81,41 @@ def pristine_tar_prepare_orig_tree(repo, source, options):
                            "orig tarball via pristine-tar" % tree_name)
 
 
-def pristine_tar_build_orig(repo, source, output_dir, options):
+def pristine_tar_build_origs(repo, source, output_dir, options):
     """
     Build orig tarball using pristine-tar
 
     @returns: C{True} if tarball was build, C{False} otherwise
     """
-    if options.pristine_tar:
-        if not repo.has_branch(repo.pristine_tar_branch):
-            gbp.log.warn('Pristine-tar branch "%s" not found' %
-                         repo.pristine_tar.branch)
+    if not options.pristine_tar:
+        return False
 
-        pristine_tar_prepare_orig_tree(repo, source, options)
-        try:
-            repo.pristine_tar.checkout(source.name,
-                                       source.upstream_version,
-                                       options.comp_type,
-                                       output_dir)
-            for component in options.components:
-                repo.pristine_tar.checkout(source.name,
-                                           source.upstream_version,
-                                           options.comp_type,
-                                           output_dir,
-                                           component=component)
-            return True
-        except CommandExecFailed:
-            if options.pristine_tar_commit:
-                gbp.log.debug("pristine-tar checkout failed, "
-                              "will commit tarball due to "
-                              "'--pristine-tar-commit'")
-            else:
-                raise
+    if not repo.has_branch(repo.pristine_tar_branch):
+        gbp.log.warn('Pristine-tar branch "%s" not found' %
+                     repo.pristine_tar.branch)
+
+    comp = Compressor(options.comp_type)
+    pristine_tar_prepare_orig_tree(repo, source, options)
+    try:
+        repo.create_upstream_tarball_via_pristine_tar(source,
+                                                      output_dir,
+                                                      comp)
+        for component in options.components:
+            repo.create_upstream_tarball_via_pristine_tar(source,
+                                                          output_dir,
+                                                          comp,
+                                                          component=component)
+        return True
+    except GitRepositoryError:
+        if options.pristine_tar_commit:
+            gbp.log.debug("pristine-tar checkout failed, will commit tarball "
+                          "due to '--pristine-tar-commit'")
+        else:
+            raise
     return False
 
 
-def pristine_tar_verify_orig(repo, source, options, output_dir, orig_files):
+def pristine_tar_verify_origs(repo, source, options, output_dir, orig_files):
     """
     Verify orig tarballs using prstine tar
 
@@ -148,8 +130,12 @@ def pristine_tar_verify_orig(repo, source, options, output_dir, orig_files):
     return True
 
 
-def get_upstream_tree(repo, source, options):
-    """Determine the upstream tree from the given options"""
+def git_archive_get_upstream_tree(repo, source, options):
+    """
+    Determine the upstream tree from the given options
+
+    for a git archive export
+    """
     if options.upstream_tree.upper() == 'TAG':
         if source.upstream_version is None:
             raise GitRepositoryError("Can't determine upstream version from changelog")
@@ -169,7 +155,7 @@ def get_upstream_tree(repo, source, options):
     return upstream_tree
 
 
-def git_archive_build_orig(repo, source, output_dir, options):
+def git_archive_build_origs(repo, source, output_dir, options):
     """
     Build orig tarball(s) using git-archive
 
@@ -179,17 +165,14 @@ def git_archive_build_orig(repo, source, output_dir, options):
     @type output_dir: C{Str}
     @param options: the parsed options
     @type options: C{dict} of options
-    @return: the tree we built the tarball from
-    @rtype: C{str}
     """
     comp = Compressor(options.comp_type, options.comp_level)
-    upstream_tree = get_upstream_tree(repo, source, options)
+    upstream_tree = git_archive_get_upstream_tree(repo, source, options)
     gbp.log.info("Creating %s from '%s'" % (source.upstream_tarball_name(comp.type),
                                             upstream_tree))
     gbp.log.debug("Building upstream tarball with compression %s" % comp)
-    main_tree = repo.tree_drop_dirs(upstream_tree, options.components) if options.components else upstream_tree
-    if not git_archive(repo, source, output_dir, main_tree, comp, options.with_submodules):
-        raise GbpError("Cannot create upstream tarball at '%s'" % output_dir)
+    tree = repo.tree_drop_dirs(upstream_tree, options.components) if options.components else upstream_tree
+    repo.create_upstream_tarball_via_git_archive(source, output_dir, tree, comp, options.with_submodules)
     for component in options.components:
         subtree = repo.tree_get_dir(upstream_tree, component)
         if not subtree:
@@ -198,11 +181,8 @@ def git_archive_build_orig(repo, source, output_dir, options):
         gbp.log.info("Creating additional tarball '%s' from '%s'"
                      % (source.upstream_tarball_name(options.comp_type, component=component),
                         subtree))
-        if not git_archive(repo, source, output_dir, subtree, comp, options.with_submodules,
-                           component=component):
-            raise GbpError("Cannot create additional tarball %s at '%s'"
-                           % (component, output_dir))
-    return upstream_tree
+        repo.create_upstream_tarball_via_git_archive(source, output_dir, subtree, comp,
+                                                     options.with_submodules, component=component)
 
 
 def guess_comp_type(repo, comp_type, source, tarball_dir):
diff --git a/tests/04_test_submodules.py b/tests/04_test_submodules.py
index 4cff46d0..52e15ddb 100644
--- a/tests/04_test_submodules.py
+++ b/tests/04_test_submodules.py
@@ -14,11 +14,10 @@ import gbp.git
 import gbp.command_wrappers
 
 from gbp.deb.policy import DebianPkgPolicy as Policy
-from gbp.pkg.git import PkgGitRepository
+from gbp.deb.git import DebianGitRepository
 from gbp.pkg import Compressor
 
 from gbp.scripts import buildpackage
-from gbp.scripts import export_orig
 from tests.testutils import ls_zip
 
 REPO = None
@@ -45,7 +44,7 @@ def setup():
 
     TMPDIR = context.new_tmpdir(__name__)
     REPODIR = TMPDIR.join('test_repo')
-    REPO = PkgGitRepository.create(REPODIR)
+    REPO = DebianGitRepository.create(REPODIR)
 
     for name in SUBMODULE_NAMES:
         SUBMODULES.append(Submodule(name, str(TMPDIR)))
@@ -140,12 +139,12 @@ def test_create_tarballs():
     comp = Compressor('bzip2')
     # Tarball with submodules
     s = MockedSource('0.1')
-    ok_(export_orig.git_archive(REPO, s, str(TMPDIR), "HEAD", comp,
-                                with_submodules=True))
+    ok_(REPO.create_upstream_tarball_via_git_archive(s, str(TMPDIR), "HEAD", comp,
+                                                     with_submodules=True))
     # Tarball without submodules
     s = MockedSource('0.2')
-    ok_(export_orig.git_archive(REPO, s, str(TMPDIR), "HEAD", comp,
-                                with_submodules=False))
+    ok_(REPO.create_upstream_tarball_via_git_archive(s, str(TMPDIR), "HEAD", comp,
+                                                     with_submodules=False))
 
 
 def test_create_zip_archives():
diff --git a/tests/10_test_get_upstream_tree.py b/tests/10_test_get_upstream_tree.py
index 5f90276e..8ee762b5 100644
--- a/tests/10_test_get_upstream_tree.py
+++ b/tests/10_test_get_upstream_tree.py
@@ -1,6 +1,6 @@
 # vim: set fileencoding=utf-8 :
 
-"""Test  L{buildpackage}'s get_upstream_tree method"""
+"""Test  L{export_orig}'s git_archive_get_upstream_tree method"""
 
 from . import context  # noqa: 401
 from . import testutils
@@ -29,7 +29,7 @@ class TestGetUpstreamTree(testutils.DebianGitTestRepo):
         self.repo.create_branch('upstream')
         options = MockOptions(upstream_tree='BRANCH',
                               upstream_branch='upstream')
-        t = export_orig.get_upstream_tree(self.repo, None, options)
+        t = export_orig.git_archive_get_upstream_tree(self.repo, None, options)
         self.assertEqual(t, 'upstream')
 
     def test_invalid_upstream_branch(self):
@@ -38,7 +38,7 @@ class TestGetUpstreamTree(testutils.DebianGitTestRepo):
         options = MockOptions(upstream_tree='BRANCH',
                               upstream_branch='upstream')
         self.assertRaises(gbp.errors.GbpError,
-                          export_orig.get_upstream_tree,
+                          export_orig.git_archive_get_upstream_tree,
                           self.repo,
                           None,
                           options)
@@ -48,7 +48,7 @@ class TestGetUpstreamTree(testutils.DebianGitTestRepo):
         self.add_file('foo')
         tree = self.repo.rev_parse('master')
         options = MockOptions(upstream_tree=tree)
-        t = export_orig.get_upstream_tree(self.repo, None, options)
+        t = export_orig.git_archive_get_upstream_tree(self.repo, None, options)
         self.assertEqual(t, tree)
 
     def test_invalid_tree(self):
@@ -56,7 +56,7 @@ class TestGetUpstreamTree(testutils.DebianGitTestRepo):
         self.add_file('foo')
         options = MockOptions(upstream_tree='doesnotexist')
         self.assertRaises(gbp.errors.GbpError,
-                          export_orig.get_upstream_tree,
+                          export_orig.git_archive_get_upstream_tree,
                           self.repo,
                           None,
                           options)
@@ -68,7 +68,7 @@ class TestGetUpstreamTree(testutils.DebianGitTestRepo):
         self.repo.create_tag('upstream/1.0_rc3')
         options = MockOptions(upstream_tree="TAG",
                               upstream_tag="upstream/%(version)s")
-        tag = export_orig.get_upstream_tree(self.repo, self.source, options)
+        tag = export_orig.git_archive_get_upstream_tree(self.repo, self.source, options)
         self.assertEqual(tag, "upstream/1.0_rc3")
 
     def test_invalid_tag(self):
@@ -77,7 +77,7 @@ class TestGetUpstreamTree(testutils.DebianGitTestRepo):
         options = MockOptions(upstream_tree="TAG",
                               upstream_tag="upstream/%(version)s")
         self.assertRaises(gbp.errors.GbpError,
-                          export_orig.get_upstream_tree,
+                          export_orig.git_archive_get_upstream_tree,
                           self.repo,
                           self.source,
                           options)