import-orig: Add --download option

to download tarballs via HTTP

Closes: #747101

4 files changed, 78 insertions, 4 deletions

diff --git a/debian/control b/debian/control
index 58d9a35a..3e423264 100644
--- a/debian/control
+++ b/debian/control
@@ -44,7 +44,7 @@ Depends: ${python:Depends},
  man-db,
  python-dateutil,
  python-pkg-resources,
-Recommends: pristine-tar (>= 0.5), cowbuilder
+Recommends: pristine-tar (>= 0.5), cowbuilder, python-requests
 Suggests: python-notify, unzip
 Description: Suite to help with Debian packages in Git repositories
  This package contains the following tools:
diff --git a/docs/manpages/gbp-import-orig.sgml b/docs/manpages/gbp-import-orig.sgml
index 31f433b4..d6fb2693 100644
--- a/docs/manpages/gbp-import-orig.sgml
+++ b/docs/manpages/gbp-import-orig.sgml
@@ -36,6 +36,7 @@
       <arg><option>--[no-]symlink-orig</option></arg>
       <arg><option>--postimport=cmd</option></arg>
       <arg><option>--[no-]interactive</option></arg>
+      <arg><option>--download</option></arg>
       <group choice="plain">
         <arg choice="plain"><replaceable>upstream-source</replaceable></arg>
         <arg><option>--uscan</option></arg>
@@ -214,6 +215,15 @@
         </listitem>
       </varlistentry>
       <varlistentry>
+        <term><option>--uscan</option></term>
+        <listitem>
+          <para>
+            Download the tarball from the given HTTP URL. This needs
+            python-request installed.
+          </para>
+        </listitem>
+      </varlistentry>
+      <varlistentry>
         <term><option>--[no-]interactive</option></term>
         <listitem>
           <para>
diff --git a/gbp/scripts/common/import_orig.py b/gbp/scripts/common/import_orig.py
index 8e18e978..54ada528 100644
--- a/gbp/scripts/common/import_orig.py
+++ b/gbp/scripts/common/import_orig.py
@@ -17,12 +17,16 @@
 #    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 #
 """Common functionality for import-orig scripts"""
+import contextlib
 import os
 import tempfile
 import gbp.command_wrappers as gbpc
-from gbp.pkg import UpstreamSource
 import gbp.log
 
+from gbp.pkg import UpstreamSource
+from gbp.errors import GbpError
+from gbp.deb.upstreamsource import DebianUpstreamSource
+
 # Try to import readline, since that will cause raw_input to get fancy
 # line editing and history capabilities. However, if readline is not
 # available, raw_input will still work.
@@ -31,6 +35,10 @@ try:
 except ImportError:
     pass
 
+try:
+    import requests
+except ImportError:
+    requests = None
 
 def orig_needs_repack(upstream_source, options):
     """
@@ -136,3 +144,35 @@ def repack_source(source, name, version, tmpdir, filters):
         repacked.unpack(tmpdir, filters)
     return (repacked, tmpdir)
 
+
+def download_orig(url):
+    """
+    Download orig tarball from given URL
+    @param url: the download URL
+    @type url: C{str}
+    @returns: The upstream source tarball
+    @rtype: DebianUpstreamSource
+    @raises GbpError: on all errors
+    """
+    CHUNK_SIZE=4096
+
+    if requests is None:
+        raise GbpError("python-requests not installed")
+
+    tarball = os.path.basename(url)
+    target = os.path.join('..', tarball)
+
+    if os.path.exists(target):
+        raise GbpError("Failed to download %s: %s already exists" % (url, target))
+
+    try:
+        with contextlib.closing(requests.get(url, verify=True, stream=True)) as r:
+            with contextlib.closing(open(target, 'w', CHUNK_SIZE)) as target_fd:
+                for d in r.iter_content(CHUNK_SIZE):
+                    target_fd.write(d)
+    except Exception as e:
+        raise GbpError("Failed to download %s: %s" % (url, e))
+        if os.path.exists(target):
+            os.unlink(target)
+
+    return DebianUpstreamSource(target)
diff --git a/gbp/scripts/import_orig.py b/gbp/scripts/import_orig.py
index 6256431c..71841161 100644
--- a/gbp/scripts/import_orig.py
+++ b/gbp/scripts/import_orig.py
@@ -33,7 +33,7 @@ from gbp.format import format_msg
 import gbp.log
 from gbp.scripts.common.import_orig import (orig_needs_repack, cleanup_tmp_tree,
                                             ask_package_name, ask_package_version,
-                                            repack_source, is_link_target)
+                                            repack_source, is_link_target, download_orig)
 
 
 def prepare_pristine_tar(archive, pkg, version):
@@ -243,10 +243,23 @@ def build_parser(name):
                       default=False, help="deprecated - don't use.")
     parser.add_option("--uscan", dest='uscan', action="store_true",
                       default=False, help="use uscan(1) to download the new tarball.")
+    parser.add_option("--download", dest='download', action="store_true",
+                      default=False, help="Download from URL via http(s).")
     return parser
 
 
 def parse_args(argv):
+    """Parse the command line arguments
+    @return: options and arguments
+
+    # Silence error output
+    >>> gbp.log.error = lambda x: None
+    >>> parse_args(['arg0', '--download', '--uscan'])
+    (None, None)
+    >>> parse_args(['arg0', '--download', 'first', 'second'])
+    (None, None)
+    """
+
     parser = build_parser(argv[0])
     if not parser:
         return None, None
@@ -257,6 +270,14 @@ def parse_args(argv):
     if options.no_dch:
         gbp.log.warn("'--no-dch' passed. This is now the default, please remove this option.")
 
+    if options.uscan and options.download:
+        gbp.log.error("Either uscan or --download can be used, not both.")
+        return None, None
+
+    if options.download and len(args) != 1:
+        gbp.log.error("Need exactly one URL to download not %s" % args)
+        return None, None
+
     return options, args
 
 
@@ -271,7 +292,10 @@ def main(argv):
         return 1
 
     try:
-        source = find_source(options.uscan, args)
+        if options.download:
+            source = download_orig(args[0])
+        else:
+            source = find_source(options.uscan, args)
         if not source:
             return ret