pq: Parse DEP3 headers

Currently the patch headers in DEP3 format are partially
supported, as git's mailinfo only reads the From and Subject fields from
the first paragraph. But the default in dep3 patches is Description and
Author, that are ignored by git. Even worse, when this fields are in the
first paragraph (again the default) git mailinfo drops all the contained
information.

This patch parses the dep3 headers if git's mailinfo couldn't obtain any
useful information, any header other than Subject|Description and
Author|From is appended to the patch message.

The description field is splitted in first line for the short
description and the rest is prepended to the patch message.

Closes: #785274

3 files changed, 276 insertions, 2 deletions

diff --git a/gbp/patch_series.py b/gbp/patch_series.py
index ccb12fe8..ec606379 100644
--- a/gbp/patch_series.py
+++ b/gbp/patch_series.py
@@ -16,12 +16,15 @@
 #    <http://www.gnu.org/licenses/>
 """Handle Patches and Patch Series"""
 
+import collections
 import os
 import re
 import subprocess
 import tempfile
 from gbp.errors import GbpError
 
+VALID_DEP3_ENDS = re.compile(r'(?:---|\*\*\*|Index:)[ \t][^ \t]|^diff -|^---')
+
 
 class Patch(object):
     """
@@ -56,6 +59,9 @@ class Patch(object):
         return repr
 
     def _read_info(self):
+        self._read_git_mailinfo()
+
+    def _read_git_mailinfo(self):
         """
         Read patch information into a structured form
 
@@ -127,7 +133,7 @@ class Patch(object):
         @param key: key to fetch
         @type key: C{str}
         @param get_val: alternate value if key is not in info dict
-        @type get_val: C{str}
+        @type get_val: C{()->str}
         """
         if self.info is None:
             self._read_info()
@@ -160,6 +166,150 @@ class Patch(object):
         return self._get_info_field('date')
 
 
+class Dep3Patch(Patch):
+    def _read_info(self):
+        self._read_git_mailinfo()
+        if not self.info:
+            self._check_dep3()
+
+    def _dep3_get_value(self, lines):
+        value = []
+        for line in lines:
+            if line.startswith(' '):
+                line = line[1:]
+                if line == '.\n':
+                    line = line[1:]
+            else:
+                line = line.split(':', 1)[1].lstrip()
+            value.append(line)
+        return ''.join(value)
+
+    def _dep3_to_info(self, headers):
+        """
+        Process the ordered dict generated by check_dep3 and add the
+        information to self.info
+        """
+
+        def add_author(lines):
+            value = self._dep3_get_value(lines).strip()
+            m = re.match('(.*)<([^<>]+)>', value)
+            if m:
+                value = m.group(1).strip()
+                self.info['email'] = m.group(2)
+            self.info['author'] = value
+            return 1
+
+        def add_subject(lines, long_desc, changes):
+            value = self._dep3_get_value(lines).lstrip()
+            if '\n' in value:
+                value, description = value.split('\n', 1)
+                # prepend the continuation lines
+                long_desc = description + long_desc
+            self.info['subject'] = value
+            return long_desc, changes + 1
+
+        changes = 0
+        long_desc = self._dep3_get_value(headers.get('long_desc', list()))
+
+        for k, v in headers.items():
+            if k in ('author', 'from'):
+                changes += add_author(v)
+            elif k in ('subject', 'description'):
+                long_desc, changes = add_subject(v, long_desc, changes)
+            elif k == 'long_desc':
+                pass
+            else:
+                long_desc += ''.join(v)
+                changes += 1
+        if changes:
+            self.long_desc = long_desc + self.long_desc
+
+    def _check_dep3(self):
+        """
+        Read DEP3 patch information into a structured form
+        """
+        if not os.path.exists(self.path):
+            return
+
+        # patch_header logic from quilt plus any line starting with ---
+        # which is the dep3 stop processing and the git separation between the
+        # header and diff stat
+        headers = collections.OrderedDict()
+        current = 'long_desc'
+        with open(self.path) as file:
+            for line in file:
+                if VALID_DEP3_ENDS.search(line):
+                    break
+
+                if line.startswith(' '):
+                    # continuation
+                    headers.setdefault(current, list()).append(line)
+                elif ':' in line:
+                    current = line.split(':', 1)[0].lower()
+                    headers.setdefault(current, list()).append(line)
+                else:
+                    # end of paragraph or not a header, read_info already left
+                    # everything else in the long_desc, nothing else to do
+                    break
+        self._dep3_to_info(headers)
+
+    def _get_subject_from_filename(self):
+        """
+        Determine the patch's subject based on the its filename
+
+        >>> p = Patch('debian/patches/foo.patch')
+        >>> p._get_subject_from_filename()
+        'foo'
+        >>> Patch('foo.patch')._get_subject_from_filename()
+        'foo'
+        >>> Patch('debian/patches/foo.bar')._get_subject_from_filename()
+        'foo.bar'
+        >>> p = Patch('debian/patches/foo')
+        >>> p._get_subject_from_filename()
+        'foo'
+        >>> Patch('0123-foo.patch')._get_subject_from_filename()
+        'foo'
+        >>> Patch('0123.patch')._get_subject_from_filename()
+        '0123'
+        >>> Patch('0123-foo-0123.patch')._get_subject_from_filename()
+        'foo-0123'
+
+        @return: the patch's subject
+        @rtype: C{str}
+        """
+        subject = os.path.basename(self.path)
+        # Strip of .diff or .patch from patch name
+        try:
+            base, ext = subject.rsplit('.', 1)
+            if ext in self.patch_exts:
+                subject = base
+        except ValueError:
+                pass  # No ext so keep subject as is
+        return subject.lstrip('0123456789-') or subject
+
+    def _get_info_field(self, key, get_val=None):
+        """
+        Return the key I{key} from the info C{dict}
+        or use val if I{key} is not a valid key.
+
+        Fill self.info if not already done.
+
+        @param key: key to fetch
+        @type key: C{str}
+        @param get_val: alternate value if key is not in info dict
+        @type get_val: C{str}
+        """
+        if self.info is None:
+            self._read_info()
+        if not self.info:
+            self._check_dep3()
+
+        if key in self.info:
+            return self.info[key]
+        else:
+            return get_val() if get_val else None
+
+
 class PatchSeries(list):
     """
     A series of L{Patch}es as read from a quilt series file).
@@ -281,4 +431,4 @@ class PatchSeries(list):
         line = cls._strip_comment(line.rstrip())
         topic = cls._get_topic(line)
         (patch, split) = cls._split_strip(line)
-        return Patch(os.path.join(patch_dir, patch), topic, split)
+        return Dep3Patch(os.path.join(patch_dir, patch), topic, split)
diff --git a/tests/component/__init__.py b/tests/component/__init__.py
index 84acf6a6..c670851f 100644
--- a/tests/component/__init__.py
+++ b/tests/component/__init__.py
@@ -86,6 +86,25 @@ class ComponentTestGitRepository(GitRepository):
         blobs = [obj[3] for obj in objs if obj[1] == 'blob']
         return set(blobs)
 
+    def get_head_author_subject(self):
+        out, err, ret = self._git_inout('format-patch', ['-1', '--stdout', '--subject-prefix='],
+                                        capture_stderr=True)
+        if ret:
+            raise GitRepositoryError("Cannot get head author/subject: %s" %
+                                     err.strip())
+
+        output = out.decode('utf-8')
+        for line in output.split('\n'):
+            line = line.strip()
+            if not line:
+                # end of headers
+                break
+            if line.startswith('From:'):
+                author = line.replace('From:', '').strip()
+            elif line.startswith('Subject:'):
+                subject = line.replace('Subject:', '').strip()
+        return author, subject
+
 
 class ComponentTestBase(unittest.TestCase, GbpLogTester):
     """Base class for testing cmdline tools of git-buildpackage"""
diff --git a/tests/component/deb/test_pq.py b/tests/component/deb/test_pq.py
index 976e4154..9dd985a4 100644
--- a/tests/component/deb/test_pq.py
+++ b/tests/component/deb/test_pq.py
@@ -19,11 +19,14 @@
 import os
 
 from tests.component import (ComponentTestBase)
+
+from tests.component.deb import DEB_TEST_DATA_DIR
 from tests.component.deb.fixtures import RepoFixtures
 
 from nose.tools import ok_, eq_
 
 from gbp.scripts.pq import main as pq
+from gbp.scripts.import_dsc import main as import_dsc
 
 
 class TestPq(ComponentTestBase):
@@ -81,3 +84,105 @@ class TestPq(ComponentTestBase):
         with open(patch) as f:
             self.assertTrue('rename from' not in f.read())
             self.assertTrue('rename to' not in f.read())
+
+    @staticmethod
+    def _dsc_name(pkg, version, dir):
+        return os.path.join(DEB_TEST_DATA_DIR,
+                            dir,
+                            '%s_%s.dsc' % (pkg, version))
+
+    @staticmethod
+    def _append_patch(repo, name, contents):
+        with open(os.path.join(repo.path, 'debian/patches/series'), 'a') as series_file:
+            series_file.write('{}.patch\n'.format(name))
+
+        with open(os.path.join(repo.path, 'debian/patches/{}.patch'.format(name)), 'w') as patch:
+            patch.write(contents)
+
+        repo.add_files('debian/patches/{}.patch'.format(name))
+        repo.commit_files(msg='Add patch: {}.patch'.format(name),
+                          files=['debian/patches/series',
+                                 'debian/patches/{}.patch'.format(name)])
+
+    @RepoFixtures.quilt30()
+    def test_import(self, repo):
+        pkg = 'hello-debhelper'
+        dsc = self._dsc_name(pkg, '2.6-2', 'dsc-3.0')
+        eq_(import_dsc(['arg0', dsc]), 0)
+        self._test_pq(repo, 'import')
+
+        author, subject = repo.get_head_author_subject()
+        eq_(author, 'Santiago Vila <sanvila@debian.org>')
+        eq_(subject, 'Modified doc/Makefile.in to avoid '
+                     '/usr/share/info/dir.gz')
+
+        self._test_pq(repo, 'switch')
+
+        self._append_patch(repo, 'foo', '''\
+Author: Mr. T. St <t@example.com>
+Description: Short DEP3 description
+ Long DEP3 description
+ .
+ Continued
+--- /dev/null
++++ b/foo
+@@ -0,0 +1 @@
++foo
+''')
+        self._test_pq(repo, 'import', ['--force'])
+
+        author, subject = repo.get_head_author_subject()
+        eq_(subject, 'Short DEP3 description')
+        eq_(author, '"Mr. T. St" <t@example.com>')
+
+    @RepoFixtures.quilt30()
+    def test_import_poor_dep3_behaviour(self, repo):
+        """Demonstrate the issues with the current DEP3 support"""
+
+        pkg = 'hello-debhelper'
+        dsc = self._dsc_name(pkg, '2.6-2', 'dsc-3.0')
+        eq_(import_dsc(['arg0', dsc]), 0)
+
+        self._append_patch(repo, 'foo', '''\
+Author: Mr. T. St <t@example.com>
+Description: A very long description with wrapp-
+  ing to increase readability in the file, which
+  is currently split into a short and long description.
+Origin: https://twitter.com/MrT/status/941789967361097728
+Forwarded: not-needed
+--- /dev/null
++++ b/foo
+@@ -0,0 +1 @@
++foo
+''')
+        self._test_pq(repo, 'import', ['--force'])
+
+        _, subject = repo.get_head_author_subject()
+        eq_(subject, 'A very long description with wrapp-')
+
+        self._test_pq(repo, 'export')
+
+        relevant_parts_of_patch = ''
+        with open('debian/patches/foo.patch') as patch_file:
+            for line in patch_file:
+                # skip the date as it's currently set to now(),
+                # not a deterministic value
+                if line.startswith('Date: '):
+                    continue
+
+                # stop reading after the main part of the description;
+                # i.e. ignore the bit that git(1) fully controls.
+                if line.startswith('---'):
+                    break
+
+                relevant_parts_of_patch += line
+
+        eq_(relevant_parts_of_patch, '''\
+From: "Mr. T. St" <t@example.com>
+Subject: A very long description with wrapp-
+
+ ing to increase readability in the file, which
+ is currently split into a short and long description.
+Origin: https://twitter.com/MrT/status/941789967361097728
+Forwarded: not-needed
+''')