aboutsummaryrefslogtreecommitdiffhomepage
path: root/git-import-orig
blob: dc8d7e2c27c0e65a9c31357b1c19a41a60b570e7 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
#!/usr/bin/python -u
# vim: set fileencoding=utf-8 :
#
# (C) 2006, 2007, 2009 Guido Guenther <agx@sigxcpu.org>
#    This program is free software; you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation; either version 2 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program; if not, write to the Free Software
#    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
#
"""Import a new upstream version into a git repository"""

import ConfigParser
import glob
import os
import sys
import re
import subprocess
import tarfile
import time
import tempfile
import gbp.command_wrappers as gbpc
from gbp.deb import (parse_changelog, unpack_orig, repack_orig,
                     NoChangelogError, has_epoch, tar_toplevel,
                     guess_upstream_version)
from gbp.git import (GitRepositoryError, GitRepository, build_tag)
from gbp.config import GbpOptionParser, GbpOptionGroup
from gbp.errors import (GbpError, GbpNothingImported)

class FastImport(object):
    """Invoke git-fast-import"""
    _bufsize = 1024

    m_regular = 644
    m_exec    = 755
    m_symlink = 120000

    def __init__(self):
        try:
            self._fi = subprocess.Popen([ 'git', 'fast-import', '--quiet'], stdin=subprocess.PIPE)
            self._out = self._fi.stdin
        except OSError, err:
            raise GbpError, "Error spawning git fast-import: %s", err
        except ValueError, err:
            raise GbpError, "Invalid argument when spawning git fast-import: %s", err

    def _do_data(self, fd, size):
        self._out.write("data %s\n" % size)
        while True:
            data = fd.read(self._bufsize)
            self._out.write(data)
            if len(data) != self._bufsize:
                break
        self._out.write("\n")

    def _do_file(self, filename, mode, fd, size):
        name = "/".join(filename.split('/')[1:])
        self._out.write("M %d inline %s\n" % (mode, name))
        self._do_data(fd, size)

    def add_file(self, filename, fd, size):
        self._do_file(filename, self.m_regular, fd, size)

    def add_executable(self, filename, fd, size):
        self._do_file(filename, self.m_exec, fd, size)

    def add_symlink(self, filename, linkname):
        name = "/".join(filename.split('/')[1:])
        self._out.write("M %d inline %s\n" % (self.m_symlink, name))
        self._out.write("data %s\n" % len(linkname))
        self._out.write("%s\n" % linkname)

    def start_commit(self, branch, committer, email, time, msg):
        length = len(msg)
        self._out.write("""commit refs/heads/%(branch)s
committer %(committer)s <%(email)s> %(time)s
data %(length)s
%(msg)s
from refs/heads/%(branch)s^0
""" % locals())

    def do_deleteall(self):
        self._out.write("deleteall\n")

    def close(self):
        if self._out:
            self._out.close()
        if self._fi:
            self._fi.wait()

    def __del__(self):
        self.close()


def cleanup_tmp_tree(tree):
    """remove a tree of temporary files"""
    try:
        gbpc.RemoveTree(tree)()
    except gbpc.CommandExecFailed:
        print >>sys.stderr, "Removal of tmptree %s failed." % tree


def is_link_target(target, link):
    """does symlink link already point to target?"""
    if os.path.exists(link):
            if os.path.samefile(target, link):
                return True
    return False


def symlink_orig(archive, pkg, version):
    """
    create a symlink <pkg>_<version>.orig.tar.gz so pristine-tar will see the
    correct basename
    @return: archive path to be used by pristine tar
    """
    if os.path.isdir(archive):
        return None
    ext = os.path.splitext(archive)[1]
    link = "../%s_%s.orig.tar%s" % (pkg, version, ext)
    if os.path.basename(archive) != os.path.basename(link):
        try:
            if not is_link_target(archive, link):
                os.symlink(os.path.abspath(archive), link)
        except OSError, err:
                raise GbpError, "Cannot symlink '%s' to '%s': %s" % (archive, link, err[1])
        return link
    else:
        return archive


def upstream_import_commit_msg(version):
    return "Imported Upstream version %s" % version


def import_upstream_tree(repo, src_dir, version, filters, verbose):
    """import the upstream tree to the current branch"""
    try:
        if repo.replace_tree(src_dir, filters, verbose=True):
            gbpc.GitCommitAll(verbose=verbose)(msg=upstream_import_commit_msg(version))
        else:
            raise GbpNothingImported
    except gbpc.CommandExecFailed:
        raise GbpError, "Import of upstream version %s failed." % version


def fast_import_upstream_tree(repo, tarball, version, options):
    """import the upstream tree to the current branch using git fast-import"""

    try:
        compr = tarball.split('.')[-1]

        if not tarfile.is_tarfile(tarball):
            raise GbpError, "'%s' not a tarball" % tarball 

        tar = tarfile.open(tarball, "r:%s" % compr)

        now = "%d %s" % (time.time(), time.strftime("%z"))
        fastimport = FastImport()
        name, email = repo.get_author_info()
        if options.verbose:
            print "Starting fastimport of %s" % tarball
        fastimport.start_commit(options.upstream_branch, name, email, now,
                                upstream_import_commit_msg(version))
        fastimport.do_deleteall()

        for item in tar:
            if item.isfile():
                if item.mode & 0100:
                    fastimport.add_executable(item.name, tar.extractfile(item.name), item.size)
                else:
                    fastimport.add_file(item.name, tar.extractfile(item.name), item.size)
            elif item.isdir():
                continue # handled by git transparently
            elif item.issym():
                fastimport.add_symlink(item.name, item.linkname)
            # if tarinfo.isextended() not implemented:
            elif item.type in ( "x", "g", "X" ):
                if options.verbose:
                    print "Skipping %s of type '%s'" % (item.name, item.type)
                continue
            else:
                raise GbpError, "'%s' is not a regular file (%s) - don't use fastimport." % (item.name, item.type)
    except gbpc.CommandExecFailed:
        raise GbpError, "Fastimport of upstream version %s failed." % version
    finally:
        tar.close()
        fastimport.close()

    if options.verbose:
        print "FastImport done."


def turn_off_fastimport(options, msg):
    if options.fast_import:
        print >>sys.stderr, msg
        print >>sys.stderr, "Turning off fastimport."
        options.fast_import = False


def main(argv):
    ret = 0
    tmpdir = ''
    pristine_orig = None

    try:
        parser = GbpOptionParser(command=os.path.basename(argv[0]), prefix='',
                                 usage='%prog [-u version] /path/to/upstream-version.tar.gz')
    except ConfigParser.ParsingError, err:
        print >>sys.stderr, err
        return 1

    cl_group = GbpOptionGroup(parser, "changelog mangling",
                      "options for mangling the changelog after the import")
    import_group = GbpOptionGroup(parser, "import options",
                      "pristine-tar and filtering")
    tag_group = GbpOptionGroup(parser, "tag options",
                      "options related to git tag creation")
    branch_group = GbpOptionGroup(parser, "version and branch naming options",
                      "version number and branch layout options")
    cmd_group = GbpOptionGroup(parser, "external command options", "how and when to invoke external commands and hooks")

    for group in [import_group, branch_group, cl_group, tag_group, cmd_group ]:
        parser.add_option_group(group)

    branch_group.add_option("-u", "--upstream-version", dest="version",
                      help="Upstream Version")
    branch_group.add_config_file_option(option_name="debian-branch",
                      dest="debian_branch")
    branch_group.add_config_file_option(option_name="upstream-branch",
                      dest="upstream_branch")
    branch_group.add_option("--no-merge", dest='merge', action="store_false",
                      default=True,
                      help="after import dont do any merging to another branch")

    tag_group.add_boolean_config_file_option(option_name="sign-tags",
                      dest="sign_tags")
    tag_group.add_config_file_option(option_name="keyid",
                      dest="keyid")
    tag_group.add_config_file_option(option_name="upstream-tag",
                      dest="upstream_tag")

    import_group.add_option("--fast-import", action="store_true",
                      dest="fast_import", default=False,
                      help="use 'git fastimport' (experimental)")
    import_group.add_config_file_option(option_name="filter",
                      dest="filters", action="append")
    import_group.add_boolean_config_file_option(option_name="pristine-tar",
                      dest="pristine_tar")
    import_group.add_boolean_config_file_option(option_name="filter-pristine-tar",
                      dest="filter_pristine_tar")
    cmd_group.add_config_file_option(option_name="postimport", dest="postimport",
                      help="hook run after a successful import, default is '%(postimport)s'")

    parser.add_option("-v", "--verbose", action="store_true", dest="verbose", default=False,
                      help="verbose command execution")

    # Accepted for compatibility
    parser.add_option("--no-dch", dest='no_dch', action="store_true",
                      default=False, help="deprecated - don't use.")

    (options, args) = parser.parse_args(argv[1:])

    if options.verbose:
        gbpc.Command.verbose = True

    if options.no_dch:
        print >>sys.stderr, "'--no-dch' passed. This is now the default, please remove this option."

    if options.filters:
        turn_off_fastimport(options, "Import filters currently not supported with fastimport.")

    try:
        if len(args) != 1:
            parser.print_help()
            raise GbpError
        else:
            archive = args[0]

        try:
            repo = GitRepository('.')
        except GitRepositoryError:
            raise GbpError, "%s is not a git repository" % (os.path.abspath('.'))

        # an empty repo has now branches:
        initial_branch = repo.get_branch()
        if initial_branch:
            is_empty = False
        else:
            is_empty = True
            turn_off_fastimport(options, "Fast importing into empty archives not yet supported.")

        if not repo.has_branch(options.upstream_branch) and not is_empty:
            print >>sys.stderr, """
Repository does not have branch '%s' for upstream sources. If there is none see
file:///usr/share/doc/git-buildpackage/manual-html/gbp.import.html#GBP.IMPORT.CONVERT
on howto create it otherwise use --upstream-branch to specify it.
"""  % options.upstream_branch
            raise GbpError

        if options.version:
            version = options.version
        else:
            version = guess_upstream_version(archive)

        if version:
            print "Upstream version is %s" % version
        else:
            print >>sys.stderr, "Cannot determine upstream version from %s - use -u <version>" % archive
            parser.print_help()
            raise GbpError

        (clean, out) = repo.is_clean()
        if not clean and not is_empty:
            print >>sys.stderr, "Repository has uncommitted changes, commit these first: "
            raise GbpError, out

        if os.path.isdir(archive):
            orig_dir = archive
            turn_off_fastimport(options, "Fastimport only supported for tar achives.")

        else:
            if not options.fast_import:
                tmpdir = tempfile.mkdtemp(dir='../')
                unpack_orig(archive, tmpdir, options.filters)
                if options.verbose:
                    print "Unpacked %s to '%s'" % (archive , tmpdir)
                orig_dir = tar_toplevel(tmpdir)
                if options.pristine_tar and options.filter_pristine_tar and len(options.filters) > 0:
                    if options.verbose:
                        print "Filter pristine-tar: repacking %s from '%s'" % (archive, tmpdir)
                    archive = os.path.join(
                        os.path.dirname(archive),
                        os.path.basename(archive).replace(".tar", ".gbp.tar")
                        )
                    repack_orig(archive, tmpdir, os.path.basename(orig_dir))
            try:
                cp = parse_changelog('debian/changelog')
                pristine_orig = symlink_orig(archive, cp['Source'], version)
            except NoChangelogError:
                print "Warning: Can't symlink orig.tar.gz due to missing debian/changelog"
                pristine_orig = archive

        try:
            filter_msg = ["", " (filtering out %s)" % options.filters][len(options.filters) > 0]
            if is_empty:
                print "Initial import of '%s' %s..." % (archive, filter_msg)
            else:
                print "Importing '%s' to branch '%s'%s..." % (archive, options.upstream_branch, filter_msg)
                if not options.fast_import:
                    repo.set_branch(options.upstream_branch)

            if options.fast_import:
                fast_import_upstream_tree(repo, pristine_orig, version, options)
            else:
                import_upstream_tree(repo, orig_dir, version, options.filters, verbose=not is_empty)

            if options.pristine_tar:
                upstream_branch = [ options.upstream_branch, 'master' ][is_empty]
                if pristine_orig:
                    gbpc.PristineTar().commit(pristine_orig, 'refs/heads/%s' % upstream_branch)
                else:
                    print >>sys.stderr, "Warning: '%s' not an archive, skipping pristine-tar" % archive
            tag = build_tag(options.upstream_tag, version)
            gbpc.GitTag(options.sign_tags, options.keyid)(tag,
                                                          msg="Upstream version %s" % version,
                                                          commit=[None, options.upstream_branch][options.fast_import])

            if is_empty:
                gbpc.GitBranch()(options.upstream_branch)
            elif options.merge:
                print "Merging to '%s'" % options.debian_branch
                repo.set_branch(options.debian_branch)
                try:
                    gbpc.GitMerge(tag)()
                except gbpc.CommandExecFailed:
                    raise GbpError, """Merge failed, please resolve."""
                if options.postimport:
                    epoch = ''
                    if os.access('debian/changelog', os.R_OK):
                        cp = parse_changelog('debian/changelog')
                        if has_epoch(cp):
                            epoch = '%s:' % cp['Epoch']
                    info = { 'version': "%s%s-1" % (epoch, version) }
                    env = { 'GBP_BRANCH': options.debian_branch }
                    cmd = gbpc.Command(options.postimport % info, extra_env=env, shell=True)()
        except gbpc.CommandExecFailed:
            raise GbpError, "Import of %s failed" % archive
    except GbpNothingImported, err:
        print >>sys.stderr, err
        repo.set_branch(initial_branch)
        ret = 1
    except GbpError, err:
        if len(err.__str__()):
            print >>sys.stderr, err
        ret = 1

    if tmpdir:
        cleanup_tmp_tree(tmpdir)

    if not ret:
        print "Succesfully merged version %s of %s into ." % (version, archive)
    return ret

if __name__ == "__main__":
    sys.exit(main(sys.argv))

# vim:et:ts=4:sw=4:et:sts=4:ai:set list listchars=tab\:»·,trail\:·: