import-orig: Allow the post-unpack hook to filter tarball files

With post-unpack scripts is currently possible to filter out files that are
not needed from the upstream branch, however it is not possible to use it to
filter files that will end up in the orig file, and in some scenarios this
is not easily doable just using a filter list.

So, run the post-unpack hook just after unpacking and before repacking the
tar.

Add tests to verify this both when using with regular --filter option
and by itself.

Closes: #812721

3 files changed, 166 insertions, 9 deletions

diff --git a/gbp/scripts/common/import_orig.py b/gbp/scripts/common/import_orig.py
index 59093683..8b573095 100644
--- a/gbp/scripts/common/import_orig.py
+++ b/gbp/scripts/common/import_orig.py
@@ -40,20 +40,26 @@ def orig_needs_repack(upstream_source, options):
     Determine if the upstream sources needs to be repacked
 
     We repack if
-     1. we want to filter out files and use pristine tar since we want
-        to make a filtered tarball available to pristine-tar
+     1. we want to filter out files via filters or post-unpack script and use
+        pristine tar since we want to make a filtered tarball available to
+        pristine-tar
      2. we don't have a suitable upstream tarball (e.g. zip archive or unpacked dir)
         and want to use filters
      3. we don't have a suitable upstream tarball (e.g. zip archive or unpacked dir)
         and want to use pristine-tar
+     4. we don't have a suitable upstream tarball (e.g. zip archive or unpacked dir)
+        and want to use a post-unpack script
     """
-    if ((options.pristine_tar and options.filter_pristine_tar and len(options.filters) > 0)):
+    if ((options.pristine_tar and options.filter_pristine_tar and
+       (options.filters or options.postunpack))):
         return True
     elif not upstream_source.is_orig():
         if len(options.filters):
             return True
         elif options.pristine_tar:
             return True
+        elif options.postunpack:
+            return True
     return False
 
 
diff --git a/gbp/scripts/import_orig.py b/gbp/scripts/import_orig.py
index 2947235a..215bc8ad 100644
--- a/gbp/scripts/import_orig.py
+++ b/gbp/scripts/import_orig.py
@@ -293,12 +293,17 @@ def debian_branch_merge_by_merge(repo, tag, version, options):
     repo.set_branch(branch)
 
 
-def unpack_tarballs(name, sources, version, options):
+def unpack_tarballs(repo, name, sources, version, options):
     tmpdir = tempfile.mkdtemp(dir='../')
     if not sources[0].is_dir():  # Unpack main tarball
         sources[0].unpack(tmpdir, options.filters)
         gbp.log.debug("Unpacked '%s' to '%s'" % (sources[0].path, sources[0].unpacked))
 
+    try:
+        postunpack_hook(repo, tmpdir, sources, options)
+    except gbpc.CommandExecFailed:
+        raise GbpError()  # The hook already printed an error message
+
     if orig_needs_repack(sources[0], options):
         gbp.log.debug("Filter pristine-tar: repacking '%s' from '%s'" % (sources[0].path,
                                                                          sources[0].unpacked))
@@ -473,11 +478,7 @@ def main(argv):
         if repo.bare:
             set_bare_repo_options(options)
 
-        sources, tmpdir = unpack_tarballs(name, sources, version, options)
-        try:
-            postunpack_hook(repo, tmpdir, sources, options)
-        except gbpc.CommandExecFailed:
-            raise GbpError()  # The hook already printed an error message
+        sources, tmpdir = unpack_tarballs(repo, name, sources, version, options)
 
         if options.verbose:
             for source in sources:
diff --git a/tests/component/deb/test_import_orig.py b/tests/component/deb/test_import_orig.py
index ef75919e..cadda636 100644
--- a/tests/component/deb/test_import_orig.py
+++ b/tests/component/deb/test_import_orig.py
@@ -274,6 +274,49 @@ class TestImportOrig(ComponentTestBase):
         t.close()
 
     @RepoFixtures.quilt30(DEFAULT_DSC, opts=['--pristine-tar'])
+    def test_filter_with_component_tarballs_and_postunpack_changes(self, repo):
+        """
+        Test that using a filter works with component tarballs (#840602) and
+        that the postunpack hook can be used to do more sophisticated changes
+        to the orig (#951534).
+        """
+        # copy data since we don't want the repacked tarball to end up in DEB_TEST_DATA_DIR
+        os.mkdir('../tarballs')
+        for f in ['hello-debhelper_2.8.orig-foo.tar.gz', 'hello-debhelper_2.8.orig.tar.gz']:
+            src = os.path.join(DEB_TEST_DATA_DIR, 'dsc-3.0-additional-tarballs', f)
+            shutil.copy(src, '../tarballs')
+
+        ok_(import_orig(['arg0',
+                         '--component=foo',
+                         '--no-interactive',
+                         '--pristine-tar',
+                         '--filter-pristine-tar',
+                         '--filter=README*',
+                         '--postunpack=printenv > $GBP_SOURCES_DIR/postunpack.out;' +
+                         'rm $GBP_SOURCES_DIR/TODO',
+                         '../tarballs/hello-debhelper_2.8.orig.tar.gz']) == 0)
+        self._check_repo_state(repo, 'master', ['master', 'upstream', 'pristine-tar'],
+                               tags=['debian/2.6-2', 'upstream/2.6', 'upstream/2.8'])
+        self._check_component_tarballs(repo, ['foo/test1', 'foo/test2'])
+
+        ok_('README' not in repo.ls_tree('HEAD'),
+            "README not filtered out of %s" % repo.ls_tree('HEAD'))
+        ok_('TODO' not in repo.ls_tree('HEAD'),
+            "TODO not filtered out of %s" % repo.ls_tree('HEAD'))
+        tar = '../hello-debhelper_2.8.orig.tar.gz'
+
+        # Check if tar got filtered properly
+        ok_(os.path.exists(tar))
+        t = tarfile.open(name=tar, mode="r:gz")
+        for f in ['hello-2.8/configure', 'hello-2.8/postunpack.out']:
+            i = t.getmember(f)
+            eq_(type(i), tarfile.TarInfo)
+        for f in ['hello-2.8/README', 'hello-2.8/TODO']:
+            with assert_raises(KeyError):
+                t.getmember(f)
+        t.close()
+
+    @RepoFixtures.quilt30(DEFAULT_DSC, opts=['--pristine-tar'])
     def test_filter_with_orig_tarball(self, repo):
         """
         Test that using a filter works with an upstream tarball that has
@@ -307,6 +350,76 @@ class TestImportOrig(ComponentTestBase):
         t.close()
 
     @RepoFixtures.quilt30(DEFAULT_DSC, opts=['--pristine-tar'])
+    def test_filter_with_orig_tarball_and_postunpack_changes(self, repo):
+        """
+        Test that using a filter works with an upstream tarball that has
+        already the correct name (#558777) and that the postunpack hook can
+        be used to do more sophisticated changes to the orig (#951534).
+        """
+        f = 'hello-debhelper_2.8.orig.tar.gz'
+        src = os.path.join(DEB_TEST_DATA_DIR, 'dsc-3.0', f)
+        shutil.copy(src, '..')
+
+        ok_(import_orig(['arg0',
+                         '--no-interactive',
+                         '--pristine-tar',
+                         '--filter-pristine-tar',
+                         '--filter=README*',
+                         '--postunpack=printenv > $GBP_SOURCES_DIR/postunpack.out;' +
+                         'rm $GBP_SOURCES_DIR/TODO',
+                         '../hello-debhelper_2.8.orig.tar.gz']) == 0)
+        self._check_repo_state(repo, 'master', ['master', 'upstream', 'pristine-tar'],
+                               tags=['debian/2.6-2', 'upstream/2.6', 'upstream/2.8'])
+
+        filtered = os.path.join('..', f)
+        ok_(os.path.exists(filtered))
+        eq_(os.readlink(filtered).split('/')[-1],
+            'hello-debhelper_2.8.orig.gbp.tar.gz')
+        # Check if tar got filtered properly
+        t = tarfile.open(name=filtered, mode="r:gz")
+        for f in ['hello-2.8/configure', 'hello-2.8/postunpack.out']:
+            i = t.getmember(f)
+            eq_(type(i), tarfile.TarInfo)
+        for f in ['hello-2.8/README', 'hello-2.8/TODO']:
+            with assert_raises(KeyError):
+                t.getmember(f)
+        t.close()
+
+    @RepoFixtures.quilt30(DEFAULT_DSC, opts=['--pristine-tar'])
+    def test_postunpack_changes_with_orig_tarball(self, repo):
+        """
+        Test that using a postupack script to apply changes works with an
+        upstream tarball that has already the correct name (#951534).
+        """
+        f = 'hello-debhelper_2.8.orig.tar.gz'
+        src = os.path.join(DEB_TEST_DATA_DIR, 'dsc-3.0', f)
+        shutil.copy(src, '..')
+
+        ok_(import_orig(['arg0',
+                         '--no-interactive',
+                         '--pristine-tar',
+                         '--filter-pristine-tar',
+                         '--postunpack=printenv > $GBP_SOURCES_DIR/postunpack.out;' +
+                         'rm $GBP_SOURCES_DIR/TODO; rm $GBP_SOURCES_DIR/README',
+                         '../hello-debhelper_2.8.orig.tar.gz']) == 0)
+        self._check_repo_state(repo, 'master', ['master', 'upstream', 'pristine-tar'],
+                               tags=['debian/2.6-2', 'upstream/2.6', 'upstream/2.8'])
+
+        filtered = os.path.join('..', f)
+        ok_(os.path.exists(filtered))
+        eq_(os.readlink(filtered).split('/')[-1],
+            'hello-debhelper_2.8.orig.gbp.tar.gz')
+        # Check if tar got filtered properly
+        t = tarfile.open(name=filtered, mode="r:gz")
+        for f in ['hello-2.8/configure', 'hello-2.8/postunpack.out']:
+            i = t.getmember(f)
+            eq_(type(i), tarfile.TarInfo)
+        for f in ['hello-2.8/README', 'hello-2.8/TODO']:
+            with assert_raises(KeyError):
+                t.getmember(f)
+        t.close()
+
+    @RepoFixtures.quilt30(DEFAULT_DSC, opts=['--pristine-tar'])
     def test_filter_unpacked_dir(self, repo):
         """
         Test that importing and filtering unpacked upstream source works.
@@ -340,6 +453,43 @@ class TestImportOrig(ComponentTestBase):
         t.close()
 
     @RepoFixtures.quilt30(DEFAULT_DSC, opts=['--pristine-tar'])
+    def test_filter_unpacked_dir_with_postunpack_changes(self, repo):
+        """
+        Test that importing and filtering unpacked upstream source works and
+        that the postunpack hook can be used to do more sophisticated changes
+        to the orig (#951534).
+        """
+        f = 'hello-debhelper_2.8.orig.tar.gz'
+        src = os.path.join(DEB_TEST_DATA_DIR, 'dsc-3.0', f)
+
+        # Create an unpacked tarball we can import
+        UnpackTarArchive(src, '..')()
+        ok_(os.path.exists('../hello-2.8'))
+
+        ok_(import_orig(['arg0',
+                         '--no-interactive',
+                         '--pristine-tar',
+                         '--filter-pristine-tar',
+                         '--filter=README*',
+                         '--postunpack=printenv > $GBP_SOURCES_DIR/postunpack.out;' +
+                         'rm $GBP_SOURCES_DIR/TODO',
+                         '../hello-2.8']) == 0)
+        self._check_repo_state(repo, 'master', ['master', 'upstream', 'pristine-tar'],
+                               tags=['debian/2.6-2', 'upstream/2.6', 'upstream/2.8'])
+
+        filtered = os.path.join('..', f)
+        ok_(os.path.exists(filtered))
+        # Check if tar got filtered properly
+        t = tarfile.open(name=filtered, mode="r:gz")
+        for f in ['hello-2.8/configure', 'hello-2.8/postunpack.out']:
+            i = t.getmember(f)
+            eq_(type(i), tarfile.TarInfo)
+        for f in ['hello-2.8/README', 'hello-2.8/TODO']:
+            with assert_raises(KeyError):
+                t.getmember(f)
+        t.close()
+
+    @RepoFixtures.quilt30(DEFAULT_DSC, opts=['--pristine-tar'])
     def test_import_in_submodule(self, repo):
         """
         Test that importing works if repo is a git submodule (#674015)