path: root/stagepkg.py

#!/usr/bin/python
#
# (c) 2006,2014 Guido Guenther <agx@sigxcpu.org>

"""
Move a package between repositories
@license: GPLv2+
"""

import errno
import re
import sys
import os
import shutil
import stat
import glob
import optparse 

# move to conf file
archive_root = "/home/debian-packages"
default_arch = "i386"
incoming = "/mini-dinstall/incoming"
conf_dir = "/etc/stagepkg/"
list_dir = os.path.join(conf_dir, "lists")

simulate = False

class ListNotFound(Exception):
    """List file not found"""
    pass


class OptionParser (optparse.OptionParser):
    """Extend the standard option parser to check for required arguments"""
    def check_required (self, opt):
        option = self.get_option(opt)
        if getattr(self.values, option.dest) is None:
            self.error("%s option not supplied, use --help for help" % option)


def get_files(changes_file):
    """Get all the files referenced in a changes file"""
    files = []
    try:
        cf = file(changes_file)
    except:
        return []
    fre=re.compile(" [a-f0-9]{32}\s+[0-9]{2,}\s+[a-z0-9/-]+\s+[a-z-]+\s+(?P<fname>.*_(?P<arch>(all|i386|.*))\.(?P<ext>(dsc|gz|bz2|deb|udeb)))")
    for line in cf.readlines():
        r = fre.match(line)
        if r:
            if r.group('ext') in [ 'gz', 'bz2', 'dsc' ]:
                files.append("source/%s" % (r.group('fname'), ))
            elif r.group('arch'):
                files.append("%s/%s" % (r.group('arch'),r.group('fname')))
    cf.close()
    return files
       

def get_subarchive(repo):
    """in case we have a subarchive like foo-component return this
       otherwise an empty string"""
    subarchive = repo.split('/',1)[0]
    return [ '', subarchive+'/' ][subarchive != repo]


def get_repo_path(repo):
    """This is a real (on disk) repository path"""
    return "%s/%s" % (archive_root, repo)


def get_incoming_path(archive):
    """Find the upload queue for this archive"""
    return "%s/%s/%s" % (archive_root, archive, incoming)


def guess_version(repo, pkg, arch):
    res = glob.glob("%s/%s_*_%s.changes" % (get_repo_path(repo), pkg, arch))
    if res:
        (dummy, version, dummy) = res[-1].split("_", 2)
    else:
        version = None
    return version


def get_changes_file_path(repo, pkg, version, arch):
    return "%s/%s_%s_%s.changes" % (get_repo_path(repo), pkg, version, arch)


def copy_files(files, repo_path, incoming_path):
    """copy files to the incoming queue"""
    dst = incoming_path
    for f in files:
        src = "%s/%s" % (repo_path, f)
        if simulate:
            print "Copy: %s to %s" % (src, dst)
        else:
            shutil.copy(src, dst)


def link_files(files, repo_path, incoming_path):
    """link files to the incoming queue"""
    dst = incoming_path
    for f in files:
        src = "%s/%s" % (repo_path, f)
        if simulate:
            print "Linking: %s to %s" % (src, dst)
        else:
            os.link(src, dst)


def mangle_changes_file(changes_file, pkg, version, from_repo, to_repo, arch):
    def get_dist(dist):
        """we might have things like unstable/foo, but the
        distribution (as noted in the changelog/changes file) is only foo."""
        return dist.split('/')[-1]
    cf = file(changes_file)
    distline = "Distribution: %s"
    distre = re.compile(distline % (get_dist(from_repo), ))
    chline = r" %s \(%s\) %s; urgency="
    chre = re.compile(chline % ( pkg, version.replace('+', '\+'), get_dist(from_repo)) )
    pgpre = re.compile(r"(Hash: SHA1|-{5,}BEGIN PGP SIG(?P<sig>NATURE)?.*-{5,})$")
    emptyre = re.compile(r"(^$)")
    matched = False
    changes = []
    for line in cf.readlines():
        (l, n) = distre.subn(distline % (get_dist(to_repo), ), line)
        if n: 
            matched = True
            changes += [ l ]
            continue
        (l, n) = chre.subn(" %s (%s) %s; urgency=" % (pkg, version, get_dist(to_repo)), line)
        if n:
            changes += [ l ]
            continue
        n = emptyre.match(line)
        if n:
            continue
        n = pgpre.match(line)
        if n:
            if n.group('sig'):
                break
            else:
                continue
        changes += [ line ]
    changes = "".join(changes)
    
    filename = get_changes_file_path(get_subarchive(to_repo)+"mini-dinstall/incoming", pkg, version, arch)
    if simulate:
        print
        print "Changes file %s" % (filename, )
        print changes
        print
    return matched, changes, filename


def sign_changes_file(changes, signas, dest):
    """sign a changes file"""
    cmd = "gpg --yes --local-user %s --clearsign \
           --verify-options no-show-policy-urls  \
           --armor --textmode --output %s" % (signas, dest)
    if not simulate:
        p = os.popen(cmd, "w")
        p.write(changes)
        p.close()
    else:
        print cmd
    

def dump_changes_file(changes, dest):
    """writeout unsigned changes file"""
    if not simulate:
        f = open(dest, "w+")
        f.write(changes)
        f.close()
    else:
        print "writing changes file to '%s'" % dest


def remove_files(files, repo_path):
    for tounlink in [ "%s/%s" % (repo_path, f) for f in files ]:
        if simulate:
            print "Unlink: %s" % (tounlink, )
        else:
            os.unlink(tounlink)


def parse_pkg_list(list):
    """parse a pkg list file and return the packages therein"""
    pkgs = []

    if not list.startswith("/"):
        list = os.path.join(list_dir, list)

    if not os.access(list, os.R_OK):
        raise ListNotFound, "List '%s' not found" % list

    lf = file(list)
    for line in lf:
        pkg = line.strip()
        if pkg.startswith("#"):
            continue
        else:
           pkgs.append(pkg)
    return pkgs


def parachute(from_repo, to_repo):
    """Check if we're following our release rules"""
    order = [ "unstable", "testing", "stable" ]
    try:
        try:
            from_suite, from_comp = from_repo.split("/")
        except ValueError:
            raise ValueError, "Source repository has no '/'"
        try:
            to_suite, to_comp = to_repo.split("/")
        except ValueError:
            raise ValueError, "Target repository has no '/'"
        if from_suite != to_suite:
            raise ValueError, "Source suite '%s' is different from destination suite '%s'" % (from_suite, to_suite)
        from_level = from_comp.split("-")[-1]
        to_level = to_comp.split("-")[-1]
        if to_level not in order:
            to_level = "stable"
        if from_level not in order:
            from_level = "stable"
        if from_level == "stable":
            raise ValueError, "Refusing to remove out of a stable archive"
        index = order.index(from_level)+1
        if order[index] != to_level:
            raise ValueError, "'%s' is not the next step after '%s'" % (to_level, from_level)
    except ValueError, msg:
        print >>sys.stderr, "%s - use --force to override" % msg
        return False
    return True


def check_repo(path): 
    try:
        if not stat.S_ISDIR(os.stat(path).st_mode):
            print >>sys.stderr, "Target '%s' is not a directory" % path
            return False
    except OSError, (errnum, errmsg):
        if errnum == errno.ENOENT:
            msg = "Target archive '%s' does not exist" % path
        else:
            msg = errmsg
        print >>sys.stderr, msg
        return False
    if not os.access (path, os.R_OK):
        print >>sys.stderr, "Target archive '%s' not readable" % path
        return False
    if not os.access (path, os.W_OK):
        print >>sys.stderr, "Target archive '%s' not writeable" % path
        return False
    return True


def main():
    pkgs =  []
    global simulate
    retval = 0

    parser = OptionParser(usage="""usage: %prog [options] pkg1[=version] [pkg2[=version]] ...
    Move a package between stages""")
    parser.add_option("-f", "--from", dest="from_repo",
                  help="move from repository")
    parser.add_option("-t", "--to",  dest="to_repo",
                  help="move to repository")
    parser.add_option("-m", "--move",  action="store_true", dest="move",
                  default=False, help="move instead of copying")
    parser.add_option("-s", "--simulate",  action="store_true",
                  dest="simulate", default=False, help="show what would be done")
    parser.add_option("-H", "--hard-links",  action="store_true",
                  dest="link", default=False, help="hard link instead of copying")
    parser.add_option("-M", "--maintainer",
                  dest="maintainer", help="maintainer name used for signing")
    parser.add_option("-k", "--keyid",
                  dest="keyid", help="keyid name used for signing")
    parser.add_option("--force", action="store_true", default=False,
                  dest="force", help="override any mismatching arguments - use with care")
    parser.add_option("-a", "--arch", default=default_arch,
                  dest="arch", help="architecture we're acting on")

    (options, args) = parser.parse_args()
    parser.check_required('-f')
    parser.check_required('-t')
    parser.check_required('-k')

    if options.keyid:
        signas = options.keyid
    elif options.maintainer:
        signas = options.maintainer
    else:
        signas = None

    simulate = options.simulate

    from_repo = options.from_repo.strip('/')
    to_repo = options.to_repo.strip('/')

    if not options.force and not parachute(from_repo, to_repo):
        return 1

    if options.link:
        dropoff = link_files
    else:
        dropoff = copy_files

    if not len(args):
        parser.print_help()
        sys.exit(1)

    try:
        for pkg in args:
            if pkg.endswith(".list"):
                pkgs += parse_pkg_list(pkg)
            else:
                pkgs.append(pkg)
    except ListNotFound as err:
        print >>sys.stderr, "\n%s - doing nothing\n" % err
        parser.print_help()
        return 1

    to_path = get_repo_path(to_repo)
   
    if not check_repo(to_path):
        return 2

    for pkg in pkgs:
        pkglist = pkg.split('=', 1)
        if(len(pkglist) != 2):   
            version = guess_version(from_repo, pkglist[0], options.arch)
            if not version:
                print >>sys.stderr, "No changes file found in '%s' for package '%s' - skipping package '%s'." % (from_repo, pkglist[0], pkg)
                retval = 1
                continue
        else:
            version = pkglist[1]
        changes_file = get_changes_file_path(from_repo, pkglist[0], version, options.arch)
        files = get_files(changes_file)
        if not files:
            print >>sys.stderr, "Couldn't parse list of files from %s - skipping package '%s'" % (changes_file, pkg)
            retval = 1
            continue

        try:
            dropoff(files, get_repo_path(from_repo), get_incoming_path(get_subarchive(to_repo)))
        except IOError, (num, msg):
            if num == errno.ENOENT: 
                print >>sys.stderr, "%s, skipping package %s." % (msg, pkg)
                retval = 1
                continue
            else:
                raise IOError, (num, msg)
        (ret, changes, dest) = mangle_changes_file(changes_file, pkglist[0], version, from_repo, to_repo, options.arch)
        if not ret:
            print "Couldn't find %s in changes file %s - bad changes file - skipping package '%s'." % (from_repo, changes_file, pkg)
            retval = 1
            continue
        try:
            if signas:
                sign_changes_file(changes, signas, dest)
            else:
                dump_changes_file(changes, dest)
        except OSError as err:
            print >>sys.stderr, "Error writing changes file %s - skipping package '%s'." % (err, pkg)
            retval = 1
            continue
        if options.move:
            remove_files( files, get_repo_path(from_repo))
    return retval

if __name__ == "__main__":
    sys.exit(main())

# vim:et:ts=4:sw=4: