From c49a0381f496fae4f94a1fbeed278f014ae556b7 Mon Sep 17 00:00:00 2001 From: Guido Günther Date: Thu, 3 Aug 2017 17:54:33 -0300 Subject: deb.git: helper for upstream tarball creation once we have sorted out all the parameters let DebianGitRepository handle the details. --- gbp/deb/__init__.py | 2 +- gbp/deb/git.py | 48 ++++++++++++++- gbp/scripts/export_orig.py | 118 +++++++++++++++---------------------- tests/04_test_submodules.py | 13 ++-- tests/10_test_get_upstream_tree.py | 14 ++--- 5 files changed, 110 insertions(+), 85 deletions(-) diff --git a/gbp/deb/__init__.py b/gbp/deb/__init__.py index 7f75b8fe..533b71bd 100644 --- a/gbp/deb/__init__.py +++ b/gbp/deb/__init__.py @@ -25,7 +25,7 @@ from gbp.git import GitRepositoryError # Make sure these are available with 'import gbp.deb' from gbp.deb.changelog import ChangeLog, NoChangeLogError -from gbp.deb.policy import DebianPkgPolicy +from gbp.deb.policy import DebianPkgPolicy # noqa: F401 class DpkgCompareVersions(gbpc.Command): diff --git a/gbp/deb/git.py b/gbp/deb/git.py index ddea38b5..3edd4546 100644 --- a/gbp/deb/git.py +++ b/gbp/deb/git.py @@ -16,7 +16,9 @@ # """A Git Repository that keeps a Debian Package""" +import os import re + from gbp.command_wrappers import CommandExecFailed from gbp.git import GitRepositoryError from gbp.deb.pristinetar import DebianPristineTar @@ -301,11 +303,55 @@ class DebianGitRepository(PkgGitRepository): for component, name in component_tarballs: subtree = self.tree_get_dir(upstream_tree, component) if not subtree: - raise GitRepositoryError("No tree for '%s' found in '%s' to create pristine tar commit from" % (component, upstream_tree)) + raise GitRepositoryError("No tree for '%s' found in '%s' to create " + "pristine tar commit from" % (component, upstream_tree)) gbp.log.debug("Creating pristine tar commit '%s' from '%s'" % (component, subtree)) self.pristine_tar.commit(name, subtree, quiet=True) self.pristine_tar.commit(tarball, main_tree, quiet=True) except CommandExecFailed as e: raise GitRepositoryError(str(e)) + def create_upstream_tarball_via_pristine_tar(self, source, output_dir, comp, component=None): + output = source.upstream_tarball_name(comp.type, component=component) + try: + self.pristine_tar.checkout(source.name, source.upstream_version, comp.type, output_dir, + component=component) + except Exception as e: + raise GitRepositoryError("Error creating %s: %s" % (output, e)) + return True + + def create_upstream_tarball_via_git_archive(self, source, output_dir, treeish, + comp, with_submodules, component=None): + """ + Create a compressed orig tarball in output_dir using git archive + + @param source: debian source package + @type source: L{DebianSource} + @param output_dir: output directory + @param type: C{str} + @param treeish: git treeish + @type treeish: C{str} + @param comp: compressor + @type comp: L{Compressor} + @param with_submodules: wether to add submodules + @type with_submodules: C{bool} + @param component: component to add to tarball name + @type component: C{str} + + Raises GitRepositoryError in case of an error + """ + submodules = False + output = os.path.join(output_dir, + source.upstream_tarball_name(comp.type, component=component)) + prefix = "%s-%s" % (source.name, source.upstream_version) + + try: + if self.has_submodules() and with_submodules: + submodules = True + self.update_submodules() + self.archive_comp(treeish, output, prefix, comp, submodules=submodules) + except Exception as e: + raise GitRepositoryError("Error creating %s: %s" % (output, e)) + return True + # vim:et:ts=4:sw=4:et:sts=4:ai:set list listchars=tab\:»·,trail\:·: diff --git a/gbp/scripts/export_orig.py b/gbp/scripts/export_orig.py index f4fc4a12..75b3e0df 100755 --- a/gbp/scripts/export_orig.py +++ b/gbp/scripts/export_orig.py @@ -27,34 +27,15 @@ import gbp.notifications from gbp.pkg import Compressor, Archive -# upstream tarball preparation -def git_archive(repo, source, output_dir, treeish, comp, with_submodules, component=None): - """create a compressed orig tarball in output_dir using git_archive""" - submodules = False - output = os.path.join(output_dir, - source.upstream_tarball_name(comp.type, component=component)) - prefix = "%s-%s" % (source.name, source.upstream_version) - - try: - if repo.has_submodules() and with_submodules: - submodules = True - repo.update_submodules() - repo.archive_comp(treeish, output, prefix, comp, submodules=submodules) - except (GbpError, GitRepositoryError, CommandExecFailed) as e: - gbp.log.err("%s" % e) - return False - except Exception as e: - gbp.log.err("Error creating %s: %s" % (output, e)) - return False - return True - - def prepare_upstream_tarballs(repo, source, options, tarball_dir, output_dir): """ Make sure we have the needed upstream tarballs. This involves looking in tarball_dir and symlinking them or building them from either pristine-tar or plain git. """ + if hasattr(options, 'no_create_orig') and options.no_create_orig: + return + if not source.is_native() and not source.upstream_version: raise GbpError("Non-native package '%s' " "has invalid version '%s'" % (source.name, source.version)) @@ -64,6 +45,7 @@ def prepare_upstream_tarballs(repo, source, options, tarball_dir, output_dir): source, options.tarball_dir) orig_files = source.upstream_tarball_names(options.comp_type, options.components) + # look in tarball_dir first, if found force a symlink to it if options.tarball_dir: gbp.log.debug("Looking for orig tarballs '%s' at '%s'" % (", ".join(orig_files), tarball_dir)) @@ -73,18 +55,18 @@ def prepare_upstream_tarballs(repo, source, options, tarball_dir, output_dir): else: msg = "All Orig tarballs '%s' found at '%s'" % (", ".join(orig_files), tarball_dir) gbp.log.info(msg) - # FIXME: trouble with "gbp buildpackage" - # if options.no_create_orig: - # return + # Create tarball if missing or forced if not du.DebianPkgPolicy.has_origs(orig_files, output_dir) or options.force_create: - if not pristine_tar_build_orig(repo, source, output_dir, options): - git_archive_build_orig(repo, source, output_dir, options) - pristine_tar_verify_orig(repo, source, options, output_dir, orig_files) + if not pristine_tar_build_origs(repo, source, output_dir, options): + git_archive_build_origs(repo, source, output_dir, options) + pristine_tar_verify_origs(repo, source, options, output_dir, orig_files) def pristine_tar_prepare_orig_tree(repo, source, options): - """Make sure the upstream tree exists + """ + Make sure the upstream tree exists + In case of component tarballs we need to recreate a tree for the main tarball without the component subdirs. """ @@ -99,41 +81,41 @@ def pristine_tar_prepare_orig_tree(repo, source, options): "orig tarball via pristine-tar" % tree_name) -def pristine_tar_build_orig(repo, source, output_dir, options): +def pristine_tar_build_origs(repo, source, output_dir, options): """ Build orig tarball using pristine-tar @returns: C{True} if tarball was build, C{False} otherwise """ - if options.pristine_tar: - if not repo.has_branch(repo.pristine_tar_branch): - gbp.log.warn('Pristine-tar branch "%s" not found' % - repo.pristine_tar.branch) + if not options.pristine_tar: + return False - pristine_tar_prepare_orig_tree(repo, source, options) - try: - repo.pristine_tar.checkout(source.name, - source.upstream_version, - options.comp_type, - output_dir) - for component in options.components: - repo.pristine_tar.checkout(source.name, - source.upstream_version, - options.comp_type, - output_dir, - component=component) - return True - except CommandExecFailed: - if options.pristine_tar_commit: - gbp.log.debug("pristine-tar checkout failed, " - "will commit tarball due to " - "'--pristine-tar-commit'") - else: - raise + if not repo.has_branch(repo.pristine_tar_branch): + gbp.log.warn('Pristine-tar branch "%s" not found' % + repo.pristine_tar.branch) + + comp = Compressor(options.comp_type) + pristine_tar_prepare_orig_tree(repo, source, options) + try: + repo.create_upstream_tarball_via_pristine_tar(source, + output_dir, + comp) + for component in options.components: + repo.create_upstream_tarball_via_pristine_tar(source, + output_dir, + comp, + component=component) + return True + except GitRepositoryError: + if options.pristine_tar_commit: + gbp.log.debug("pristine-tar checkout failed, will commit tarball " + "due to '--pristine-tar-commit'") + else: + raise return False -def pristine_tar_verify_orig(repo, source, options, output_dir, orig_files): +def pristine_tar_verify_origs(repo, source, options, output_dir, orig_files): """ Verify orig tarballs using prstine tar @@ -148,8 +130,12 @@ def pristine_tar_verify_orig(repo, source, options, output_dir, orig_files): return True -def get_upstream_tree(repo, source, options): - """Determine the upstream tree from the given options""" +def git_archive_get_upstream_tree(repo, source, options): + """ + Determine the upstream tree from the given options + + for a git archive export + """ if options.upstream_tree.upper() == 'TAG': if source.upstream_version is None: raise GitRepositoryError("Can't determine upstream version from changelog") @@ -169,7 +155,7 @@ def get_upstream_tree(repo, source, options): return upstream_tree -def git_archive_build_orig(repo, source, output_dir, options): +def git_archive_build_origs(repo, source, output_dir, options): """ Build orig tarball(s) using git-archive @@ -179,17 +165,14 @@ def git_archive_build_orig(repo, source, output_dir, options): @type output_dir: C{Str} @param options: the parsed options @type options: C{dict} of options - @return: the tree we built the tarball from - @rtype: C{str} """ comp = Compressor(options.comp_type, options.comp_level) - upstream_tree = get_upstream_tree(repo, source, options) + upstream_tree = git_archive_get_upstream_tree(repo, source, options) gbp.log.info("Creating %s from '%s'" % (source.upstream_tarball_name(comp.type), upstream_tree)) gbp.log.debug("Building upstream tarball with compression %s" % comp) - main_tree = repo.tree_drop_dirs(upstream_tree, options.components) if options.components else upstream_tree - if not git_archive(repo, source, output_dir, main_tree, comp, options.with_submodules): - raise GbpError("Cannot create upstream tarball at '%s'" % output_dir) + tree = repo.tree_drop_dirs(upstream_tree, options.components) if options.components else upstream_tree + repo.create_upstream_tarball_via_git_archive(source, output_dir, tree, comp, options.with_submodules) for component in options.components: subtree = repo.tree_get_dir(upstream_tree, component) if not subtree: @@ -198,11 +181,8 @@ def git_archive_build_orig(repo, source, output_dir, options): gbp.log.info("Creating additional tarball '%s' from '%s'" % (source.upstream_tarball_name(options.comp_type, component=component), subtree)) - if not git_archive(repo, source, output_dir, subtree, comp, options.with_submodules, - component=component): - raise GbpError("Cannot create additional tarball %s at '%s'" - % (component, output_dir)) - return upstream_tree + repo.create_upstream_tarball_via_git_archive(source, output_dir, subtree, comp, + options.with_submodules, component=component) def guess_comp_type(repo, comp_type, source, tarball_dir): diff --git a/tests/04_test_submodules.py b/tests/04_test_submodules.py index 4cff46d0..52e15ddb 100644 --- a/tests/04_test_submodules.py +++ b/tests/04_test_submodules.py @@ -14,11 +14,10 @@ import gbp.git import gbp.command_wrappers from gbp.deb.policy import DebianPkgPolicy as Policy -from gbp.pkg.git import PkgGitRepository +from gbp.deb.git import DebianGitRepository from gbp.pkg import Compressor from gbp.scripts import buildpackage -from gbp.scripts import export_orig from tests.testutils import ls_zip REPO = None @@ -45,7 +44,7 @@ def setup(): TMPDIR = context.new_tmpdir(__name__) REPODIR = TMPDIR.join('test_repo') - REPO = PkgGitRepository.create(REPODIR) + REPO = DebianGitRepository.create(REPODIR) for name in SUBMODULE_NAMES: SUBMODULES.append(Submodule(name, str(TMPDIR))) @@ -140,12 +139,12 @@ def test_create_tarballs(): comp = Compressor('bzip2') # Tarball with submodules s = MockedSource('0.1') - ok_(export_orig.git_archive(REPO, s, str(TMPDIR), "HEAD", comp, - with_submodules=True)) + ok_(REPO.create_upstream_tarball_via_git_archive(s, str(TMPDIR), "HEAD", comp, + with_submodules=True)) # Tarball without submodules s = MockedSource('0.2') - ok_(export_orig.git_archive(REPO, s, str(TMPDIR), "HEAD", comp, - with_submodules=False)) + ok_(REPO.create_upstream_tarball_via_git_archive(s, str(TMPDIR), "HEAD", comp, + with_submodules=False)) def test_create_zip_archives(): diff --git a/tests/10_test_get_upstream_tree.py b/tests/10_test_get_upstream_tree.py index 5f90276e..8ee762b5 100644 --- a/tests/10_test_get_upstream_tree.py +++ b/tests/10_test_get_upstream_tree.py @@ -1,6 +1,6 @@ # vim: set fileencoding=utf-8 : -"""Test L{buildpackage}'s get_upstream_tree method""" +"""Test L{export_orig}'s git_archive_get_upstream_tree method""" from . import context # noqa: 401 from . import testutils @@ -29,7 +29,7 @@ class TestGetUpstreamTree(testutils.DebianGitTestRepo): self.repo.create_branch('upstream') options = MockOptions(upstream_tree='BRANCH', upstream_branch='upstream') - t = export_orig.get_upstream_tree(self.repo, None, options) + t = export_orig.git_archive_get_upstream_tree(self.repo, None, options) self.assertEqual(t, 'upstream') def test_invalid_upstream_branch(self): @@ -38,7 +38,7 @@ class TestGetUpstreamTree(testutils.DebianGitTestRepo): options = MockOptions(upstream_tree='BRANCH', upstream_branch='upstream') self.assertRaises(gbp.errors.GbpError, - export_orig.get_upstream_tree, + export_orig.git_archive_get_upstream_tree, self.repo, None, options) @@ -48,7 +48,7 @@ class TestGetUpstreamTree(testutils.DebianGitTestRepo): self.add_file('foo') tree = self.repo.rev_parse('master') options = MockOptions(upstream_tree=tree) - t = export_orig.get_upstream_tree(self.repo, None, options) + t = export_orig.git_archive_get_upstream_tree(self.repo, None, options) self.assertEqual(t, tree) def test_invalid_tree(self): @@ -56,7 +56,7 @@ class TestGetUpstreamTree(testutils.DebianGitTestRepo): self.add_file('foo') options = MockOptions(upstream_tree='doesnotexist') self.assertRaises(gbp.errors.GbpError, - export_orig.get_upstream_tree, + export_orig.git_archive_get_upstream_tree, self.repo, None, options) @@ -68,7 +68,7 @@ class TestGetUpstreamTree(testutils.DebianGitTestRepo): self.repo.create_tag('upstream/1.0_rc3') options = MockOptions(upstream_tree="TAG", upstream_tag="upstream/%(version)s") - tag = export_orig.get_upstream_tree(self.repo, self.source, options) + tag = export_orig.git_archive_get_upstream_tree(self.repo, self.source, options) self.assertEqual(tag, "upstream/1.0_rc3") def test_invalid_tag(self): @@ -77,7 +77,7 @@ class TestGetUpstreamTree(testutils.DebianGitTestRepo): options = MockOptions(upstream_tree="TAG", upstream_tag="upstream/%(version)s") self.assertRaises(gbp.errors.GbpError, - export_orig.get_upstream_tree, + export_orig.git_archive_get_upstream_tree, self.repo, self.source, options) -- cgit v1.2.3