From bba75a1b8f1a46da8da36444001d257533bbf03a Mon Sep 17 00:00:00 2001 From: Guido Günther Date: Fri, 15 Apr 2016 16:37:40 +0200 Subject: import_orig: add support for importing additional tarballs We expect the additional tarballs to be located next to the orig tarball and to be already named properly. Closes: #561071 --- docs/manpages/gbp-import-orig.sgml | 25 ++++++++++- gbp/scripts/import_orig.py | 47 +++++++++++++++++---- tests/component/deb/data | 2 +- tests/component/deb/test_import_orig.py | 73 ++++++++++++++++++++++++++++++--- 4 files changed, 133 insertions(+), 14 deletions(-) diff --git a/docs/manpages/gbp-import-orig.sgml b/docs/manpages/gbp-import-orig.sgml index f72a741..c0a858a 100644 --- a/docs/manpages/gbp-import-orig.sgml +++ b/docs/manpages/gbp-import-orig.sgml @@ -30,7 +30,8 @@ gpg-keyid tag-format - pattern + pattern + component @@ -192,6 +193,28 @@ + + COMPONENT + + + + When importing the upstream tarball also look for an additional tarball + with component name COMPONENT. E.g. in + hello-debhelper_1.0.orig-foo.tar.gz + the component would be foo. The additional + tarball is expected to be in the same directory than the upstream tarball + and to use the same compression type. + + + Using additional original tarballs is a feature of the 3.0 + (quilt) source format. See + the dpkg-source manpage for + details. Note that this option can't be set via &gbp.conf; + since it's considered an experimental feature and might + change incompatibly. + + + diff --git a/gbp/scripts/import_orig.py b/gbp/scripts/import_orig.py index ba4fc28..c2d484a 100644 --- a/gbp/scripts/import_orig.py +++ b/gbp/scripts/import_orig.py @@ -23,12 +23,13 @@ import sys import tempfile import gbp.command_wrappers as gbpc from gbp.deb import (DebianPkgPolicy, parse_changelog_repo) -from gbp.deb.upstreamsource import DebianUpstreamSource +from gbp.deb.upstreamsource import DebianUpstreamSource, unpack_component_tarball from gbp.deb.uscan import (Uscan, UscanError) from gbp.deb.changelog import ChangeLog, NoChangeLogError from gbp.deb.git import (GitRepositoryError, DebianGitRepository) from gbp.config import GbpOptionParserDebian, GbpOptionGroup, no_upstream_branch_msg from gbp.errors import GbpError +from gbp.pkg import parse_archive_filename from gbp.format import format_str import gbp.log from gbp.scripts.common.import_orig import (orig_needs_repack, cleanup_tmp_tree, @@ -232,12 +233,12 @@ def detect_name_and_version(repo, source, options): def find_source(use_uscan, args): - """Find the tarball to import - either via uscan or via command line argument + """Find the main tarball to import - either via uscan or via command line argument @return: upstream source filename or None if nothing to import @rtype: string @raise GbpError: raised on all detected errors - >>> find_source(False, ['too', 'much']) + >>> find_source(False, ['too', 'many']) Traceback (most recent call last): ... GbpError: More than one archive specified. Try --help. @@ -277,8 +278,7 @@ def find_source(use_uscan, args): elif len(args) == 0: raise GbpError("No archive to import specified. Try --help.") else: - archive = DebianUpstreamSource(args[0]) - return archive + return DebianUpstreamSource(args[0]) def debian_branch_merge(repo, tag, version, options): @@ -326,6 +326,25 @@ def debian_branch_merge_by_replace(repo, tag, version, options): repo.force_head(commit, hard=True) +def get_component_tarballs(name, version, tarball, components): + """ + Figure out the paths to the component tarballs based on the main + tarball. + """ + tarballs = [] + for component in components: + (_, _, comp_type) = parse_archive_filename(tarball) + cname = DebianPkgPolicy.build_tarball_name(name, + version, + comp_type, + os.path.dirname(tarball), + component) + tarballs.append((component, cname)) + if not os.path.exists(cname): + raise GbpError("Can not find component tarball %s" % cname) + return tarballs + + def debian_branch_merge_by_merge(repo, tag, version, options): gbp.log.info("Merging to '%s'" % options.debian_branch) try: @@ -391,8 +410,11 @@ def build_parser(name): dest="import_msg") import_group.add_boolean_config_file_option(option_name="symlink-orig", dest="symlink_orig") + import_group.add_option("--component", action="append", metavar='COMPONENT', + dest="components", help="additional component to import, can be given multiple times", default=[]) cmd_group.add_config_file_option(option_name="postimport", dest="postimport") + parser.add_boolean_config_file_option(option_name="interactive", dest='interactive') parser.add_boolean_config_file_option(option_name="rollback", @@ -477,7 +499,7 @@ def main(argv): gbp.log.err("Repository has uncommitted changes, commit these first: ") raise GbpError(out) - # Download the source + # Download the main tarball if options.download: source = download_orig(args[0]) else: @@ -485,7 +507,13 @@ def main(argv): if not source: return ret + # The main tarball (sourcepackage, version) = detect_name_and_version(repo, source, options) + # Additionl tarballs we expect to exist + component_tarballs = get_component_tarballs(sourcepackage, + version, + source.path, + options.components) tag = repo.version_to_tag(options.upstream_tag, version) if repo.has_tag(tag): @@ -498,6 +526,8 @@ def main(argv): tmpdir = tempfile.mkdtemp(dir='../') source.unpack(tmpdir, options.filters) gbp.log.debug("Unpacked '%s' to '%s'" % (source.path, source.unpacked)) + for (component, tarball) in component_tarballs: + unpack_component_tarball(source.unpacked, component, tarball, options.filters) if orig_needs_repack(source, options): gbp.log.debug("Filter pristine-tar: repacking '%s' from '%s'" % (source.path, source.unpacked)) @@ -525,6 +555,7 @@ def main(argv): gbp.log.info("Upstream version is %s" % version) msg = upstream_import_commit_msg(options, version) + commit = repo.commit_dir(source.unpacked, msg=msg, branch=import_branch, @@ -535,7 +566,9 @@ def main(argv): if options.pristine_tar: if pristine_orig: repo.rrr_branch('pristine-tar') - repo.pristine_tar.commit(pristine_orig, import_branch) + repo.create_pristinetar_commits(import_branch, + pristine_orig, + component_tarballs) else: gbp.log.warn("'%s' not an archive, skipping pristine-tar" % source.path) diff --git a/tests/component/deb/data b/tests/component/deb/data index 748f13d..5f4d41e 160000 --- a/tests/component/deb/data +++ b/tests/component/deb/data @@ -1 +1 @@ -Subproject commit 748f13d484359942aa0d7b7ca631325d5e8fe0e0 +Subproject commit 5f4d41e5a593215f3fc61648bfd05502cfb123b1 diff --git a/tests/component/deb/test_import_orig.py b/tests/component/deb/test_import_orig.py index 91564a6..f66b746 100644 --- a/tests/component/deb/test_import_orig.py +++ b/tests/component/deb/test_import_orig.py @@ -26,9 +26,11 @@ from tests.component.deb import DEB_TEST_DATA_DIR from gbp.scripts.import_dsc import main as import_dsc from gbp.scripts.import_orig import main as import_orig +from gbp.deb.pristinetar import DebianPristineTar +from gbp.deb.dscfile import DscFile from gbp.git.repository import GitRepository, GitRepositoryError -from nose.tools import ok_ +from nose.tools import ok_, eq_ def raise_if_tag_match(match): @@ -45,14 +47,14 @@ class TestImportOrig(ComponentTestBase): pkg = "hello-debhelper" def_branches = ['master', 'upstream', 'pristine-tar'] - def _orig(self, version): + def _orig(self, version, dir='dsc-3.0'): return os.path.join(DEB_TEST_DATA_DIR, - 'dsc-3.0', + dir, '%s_%s.orig.tar.gz' % (self.pkg, version)) - def _dsc(self, version): + def _dsc(self, version, dir='dsc-3.0'): return os.path.join(DEB_TEST_DATA_DIR, - 'dsc-3.0', + dir, '%s_%s.dsc' % (self.pkg, version)) def test_initial_import(self): @@ -77,6 +79,67 @@ class TestImportOrig(ComponentTestBase): self._check_repo_state(repo, 'master', ['master', 'upstream', 'pristine-tar'], tags=['debian/2.6-2', 'upstream/2.6', 'upstream/2.8']) + def test_update_component_tarballs(self): + dsc = self._dsc('2.6-2') + ok_(import_dsc(['arg0', '--pristine-tar', dsc]) == 0) + repo = ComponentTestGitRepository(self.pkg) + os.chdir(self.pkg) + self._check_repo_state(repo, 'master', ['master', 'upstream', 'pristine-tar']) + + # Import 2.8 + orig = self._orig('2.8', dir='dsc-3.0-additional-tarballs') + ok_(import_orig(['arg0', '--component=foo', '--no-interactive', '--pristine-tar', orig]) == 0) + self._check_repo_state(repo, 'master', ['master', 'upstream', 'pristine-tar'], + tags=['debian/2.6-2', 'upstream/2.6', 'upstream/2.8']) + for file in ['foo/test1', 'foo/test2']: + ok_(file in repo.ls_tree('HEAD'), + "Could not find component tarball file %s in %s" % (file, repo.ls_tree('HEAD'))) + ok_(file in repo.ls_tree('upstream'), + "Could not find component tarball file %s in %s" % (file, repo.ls_tree('HEAD'))) + + dsc = DscFile.parse(self._dsc('2.8-1', dir='dsc-3.0-additional-tarballs')) + # Check if we can rebuild the upstream tarball and additional tarball + ptars = [('hello-debhelper_2.8.orig.tar.gz', 'pristine-tar', '', dsc.tgz), + ('hello-debhelper_2.8.orig-foo.tar.gz', 'pristine-tar^', 'foo', dsc.additional_tarballs['foo'])] + + p = DebianPristineTar(repo) + outdir = os.path.abspath('.') + for f, w, s, o in ptars: + eq_(repo.get_subject(w), 'pristine-tar data for %s' % f) + old = self.hash_file(o) + p.checkout('hello-debhelper', '2.8', 'gzip', outdir, component=s) + out = os.path.join(outdir, f) + new = self.hash_file(out) + eq_(old, new, "Checksum %s of regenerated tarball %s does not match original %s" % + (f, old, new)) + os.unlink(out) + + # Import 2.9 + orig = self._orig('2.9', dir='dsc-3.0-additional-tarballs') + ok_(import_orig(['arg0', '--component=foo', '--no-interactive', '--pristine-tar', orig]) == 0) + self._check_repo_state(repo, 'master', ['master', 'upstream', 'pristine-tar'], + tags=['debian/2.6-2', 'upstream/2.6', 'upstream/2.8', 'upstream/2.9']) + for file in ['foo/test1', 'foo/test2', 'foo/test3']: + ok_(file in repo.ls_tree('HEAD'), + "Could not find component tarball file %s in %s" % (file, repo.ls_tree('HEAD'))) + ok_(file in repo.ls_tree('upstream'), + "Could not find component tarball file %s in %s" % (file, repo.ls_tree('HEAD'))) + + dsc = DscFile.parse(self._dsc('2.9-1', dir='dsc-3.0-additional-tarballs')) + # Check if we can rebuild the upstream tarball and additional tarball + ptars = [('hello-debhelper_2.9.orig.tar.gz', 'pristine-tar', '', dsc.tgz), + ('hello-debhelper_2.9.orig-foo.tar.gz', 'pristine-tar^', 'foo', dsc.additional_tarballs['foo'])] + + p = DebianPristineTar(repo) + outdir = os.path.abspath('.') + for f, w, s, o in ptars: + eq_(repo.get_subject(w), 'pristine-tar data for %s' % f) + old = self.hash_file(o) + p.checkout('hello-debhelper', '2.9', 'gzip', outdir, component=s) + new = self.hash_file(os.path.join(outdir, f)) + eq_(old, new, "Checksum %s of regenerated tarball %s does not match original %s" % + (f, old, new)) + def test_tag_exists(self): """Test that importing an already imported version fails""" repo = GitRepository.create(self.pkg) -- cgit v1.2.3