From 17c2ee87ee338b2634d51bde18f4eeb216e562e5 Mon Sep 17 00:00:00 2001 From: Guido Günther Date: Tue, 17 Jun 2014 19:30:27 +0200 Subject: Use entry point --- setup.py | 29 ++++ stagepkg.py | 378 --------------------------------------------------- stagepkg/__init__.py | 0 stagepkg/command.py | 377 ++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 406 insertions(+), 378 deletions(-) create mode 100644 setup.py delete mode 100755 stagepkg.py create mode 100644 stagepkg/__init__.py create mode 100644 stagepkg/command.py diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..73a4f2d --- /dev/null +++ b/setup.py @@ -0,0 +1,29 @@ +#!/usr/bin/python +# vim: set fileencoding=utf-8 : +# +# (C) 2014 Guido Günther +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from setuptools import setup + +setup(name = "stagepkg", + author = 'Guido Günther', + author_email = 'agx@sigxcpu.org', + packages = ['stagepkg'], + entry_points = { + 'console_scripts': [ 'stagepkg = stagepkg.command:run' ], + }, +) + +# vim:et:ts=4:sw=4:et:sts=4:ai:set list listchars=tab\:»·,trail\:·: diff --git a/stagepkg.py b/stagepkg.py deleted file mode 100755 index 732c31d..0000000 --- a/stagepkg.py +++ /dev/null @@ -1,378 +0,0 @@ -#!/usr/bin/python -# -# (c) 2006,2014 Guido Guenther - -""" -Move a package between repositories -@license: GPLv2+ -""" - -import errno -import re -import sys -import os -import shutil -import stat -import glob -import optparse - -# move to conf file -archive_root = "/home/debian-packages" -default_arch = "i386" -incoming = "/mini-dinstall/incoming" -conf_dir = "/etc/stagepkg/" -list_dir = os.path.join(conf_dir, "lists") - -simulate = False - -class ListNotFound(Exception): - """List file not found""" - pass - - -class OptionParser (optparse.OptionParser): - """Extend the standard option parser to check for required arguments""" - def check_required (self, opt): - option = self.get_option(opt) - if getattr(self.values, option.dest) is None: - self.error("%s option not supplied, use --help for help" % option) - - -def get_files(changes_file): - """Get all the files referenced in a changes file""" - files = [] - try: - cf = file(changes_file) - except: - return [] - fre=re.compile(" [a-f0-9]{32}\s+[0-9]{2,}\s+[a-z0-9/-]+\s+[a-z-]+\s+(?P.*_(?P(all|i386|.*))\.(?P(dsc|gz|bz2|deb|udeb)))") - for line in cf.readlines(): - r = fre.match(line) - if r: - if r.group('ext') in [ 'gz', 'bz2', 'dsc' ]: - files.append("source/%s" % (r.group('fname'), )) - elif r.group('arch'): - files.append("%s/%s" % (r.group('arch'),r.group('fname'))) - cf.close() - return files - - -def get_subarchive(repo): - """in case we have a subarchive like foo-component return this - otherwise an empty string""" - subarchive = repo.split('/',1)[0] - return [ '', subarchive+'/' ][subarchive != repo] - - -def get_repo_path(repo): - """This is a real (on disk) repository path""" - return "%s/%s" % (archive_root, repo) - - -def get_incoming_path(archive): - """Find the upload queue for this archive""" - return "%s/%s/%s" % (archive_root, archive, incoming) - - -def guess_version(repo, pkg, arch): - res = glob.glob("%s/%s_*_%s.changes" % (get_repo_path(repo), pkg, arch)) - if res: - (dummy, version, dummy) = res[-1].split("_", 2) - else: - version = None - return version - - -def get_changes_file_path(repo, pkg, version, arch): - return "%s/%s_%s_%s.changes" % (get_repo_path(repo), pkg, version, arch) - - -def copy_files(files, repo_path, incoming_path): - """copy files to the incoming queue""" - dst = incoming_path - for f in files: - src = "%s/%s" % (repo_path, f) - if simulate: - print "Copy: %s to %s" % (src, dst) - else: - shutil.copy(src, dst) - - -def link_files(files, repo_path, incoming_path): - """link files to the incoming queue""" - dst = incoming_path - for f in files: - src = "%s/%s" % (repo_path, f) - if simulate: - print "Linking: %s to %s" % (src, dst) - else: - os.link(src, dst) - - -def mangle_changes_file(changes_file, pkg, version, from_repo, to_repo, arch): - def get_dist(dist): - """we might have things like unstable/foo, but the - distribution (as noted in the changelog/changes file) is only foo.""" - return dist.split('/')[-1] - cf = file(changes_file) - distline = "Distribution: %s" - distre = re.compile(distline % (get_dist(from_repo), )) - chline = r" %s \(%s\) %s; urgency=" - chre = re.compile(chline % ( pkg, version.replace('+', '\+'), get_dist(from_repo)) ) - pgpre = re.compile(r"(Hash: SHA1|-{5,}BEGIN PGP SIG(?PNATURE)?.*-{5,})$") - emptyre = re.compile(r"(^$)") - matched = False - changes = [] - for line in cf.readlines(): - (l, n) = distre.subn(distline % (get_dist(to_repo), ), line) - if n: - matched = True - changes += [ l ] - continue - (l, n) = chre.subn(" %s (%s) %s; urgency=" % (pkg, version, get_dist(to_repo)), line) - if n: - changes += [ l ] - continue - n = emptyre.match(line) - if n: - continue - n = pgpre.match(line) - if n: - if n.group('sig'): - break - else: - continue - changes += [ line ] - changes = "".join(changes) - - filename = get_changes_file_path(get_subarchive(to_repo)+"mini-dinstall/incoming", pkg, version, arch) - if simulate: - print - print "Changes file %s" % (filename, ) - print changes - print - return matched, changes, filename - - -def sign_changes_file(changes, signas, dest): - """sign a changes file""" - cmd = "gpg --yes --local-user %s --clearsign \ - --verify-options no-show-policy-urls \ - --armor --textmode --output %s" % (signas, dest) - if not simulate: - p = os.popen(cmd, "w") - p.write(changes) - p.close() - else: - print cmd - - -def dump_changes_file(changes, dest): - """writeout unsigned changes file""" - if not simulate: - f = open(dest, "w+") - f.write(changes) - f.close() - else: - print "writing changes file to '%s'" % dest - - -def remove_files(files, repo_path): - for tounlink in [ "%s/%s" % (repo_path, f) for f in files ]: - if simulate: - print "Unlink: %s" % (tounlink, ) - else: - os.unlink(tounlink) - - -def parse_pkg_list(list): - """parse a pkg list file and return the packages therein""" - pkgs = [] - - if not list.startswith("/"): - list = os.path.join(list_dir, list) - - if not os.access(list, os.R_OK): - raise ListNotFound, "List '%s' not found" % list - - lf = file(list) - for line in lf: - pkg = line.strip() - if pkg.startswith("#"): - continue - else: - pkgs.append(pkg) - return pkgs - - -def parachute(from_repo, to_repo): - """Check if we're following our release rules""" - order = [ "unstable", "testing", "stable" ] - try: - try: - from_suite, from_comp = from_repo.split("/") - except ValueError: - raise ValueError, "Source repository has no '/'" - try: - to_suite, to_comp = to_repo.split("/") - except ValueError: - raise ValueError, "Target repository has no '/'" - if from_suite != to_suite: - raise ValueError, "Source suite '%s' is different from destination suite '%s'" % (from_suite, to_suite) - from_level = from_comp.split("-")[-1] - to_level = to_comp.split("-")[-1] - if to_level not in order: - to_level = "stable" - if from_level not in order: - from_level = "stable" - if from_level == "stable": - raise ValueError, "Refusing to remove out of a stable archive" - index = order.index(from_level)+1 - if order[index] != to_level: - raise ValueError, "'%s' is not the next step after '%s'" % (to_level, from_level) - except ValueError, msg: - print >>sys.stderr, "%s - use --force to override" % msg - return False - return True - - -def check_repo(path): - try: - if not stat.S_ISDIR(os.stat(path).st_mode): - print >>sys.stderr, "Target '%s' is not a directory" % path - return False - except OSError, (errnum, errmsg): - if errnum == errno.ENOENT: - msg = "Target archive '%s' does not exist" % path - else: - msg = errmsg - print >>sys.stderr, msg - return False - if not os.access (path, os.R_OK): - print >>sys.stderr, "Target archive '%s' not readable" % path - return False - if not os.access (path, os.W_OK): - print >>sys.stderr, "Target archive '%s' not writeable" % path - return False - return True - - -def main(): - pkgs = [] - global simulate - retval = 0 - - parser = OptionParser(usage="""usage: %prog [options] pkg1[=version] [pkg2[=version]] ... - Move a package between stages""") - parser.add_option("-f", "--from", dest="from_repo", - help="move from repository") - parser.add_option("-t", "--to", dest="to_repo", - help="move to repository") - parser.add_option("-m", "--move", action="store_true", dest="move", - default=False, help="move instead of copying") - parser.add_option("-s", "--simulate", action="store_true", - dest="simulate", default=False, help="show what would be done") - parser.add_option("-H", "--hard-links", action="store_true", - dest="link", default=False, help="hard link instead of copying") - parser.add_option("-M", "--maintainer", - dest="maintainer", help="maintainer name used for signing") - parser.add_option("-k", "--keyid", - dest="keyid", help="keyid name used for signing") - parser.add_option("--force", action="store_true", default=False, - dest="force", help="override any mismatching arguments - use with care") - parser.add_option("-a", "--arch", default=default_arch, - dest="arch", help="architecture we're acting on") - - (options, args) = parser.parse_args() - parser.check_required('-f') - parser.check_required('-t') - parser.check_required('-k') - - if options.keyid: - signas = options.keyid - elif options.maintainer: - signas = options.maintainer - else: - signas = None - - simulate = options.simulate - - from_repo = options.from_repo.strip('/') - to_repo = options.to_repo.strip('/') - - if not options.force and not parachute(from_repo, to_repo): - return 1 - - if options.link: - dropoff = link_files - else: - dropoff = copy_files - - if not len(args): - parser.print_help() - sys.exit(1) - - try: - for pkg in args: - if pkg.endswith(".list"): - pkgs += parse_pkg_list(pkg) - else: - pkgs.append(pkg) - except ListNotFound as err: - print >>sys.stderr, "\n%s - doing nothing\n" % err - parser.print_help() - return 1 - - to_path = get_repo_path(to_repo) - - if not check_repo(to_path): - return 2 - - for pkg in pkgs: - pkglist = pkg.split('=', 1) - if(len(pkglist) != 2): - version = guess_version(from_repo, pkglist[0], options.arch) - if not version: - print >>sys.stderr, "No changes file found in '%s' for package '%s' - skipping package '%s'." % (from_repo, pkglist[0], pkg) - retval = 1 - continue - else: - version = pkglist[1] - changes_file = get_changes_file_path(from_repo, pkglist[0], version, options.arch) - files = get_files(changes_file) - if not files: - print >>sys.stderr, "Couldn't parse list of files from %s - skipping package '%s'" % (changes_file, pkg) - retval = 1 - continue - - try: - dropoff(files, get_repo_path(from_repo), get_incoming_path(get_subarchive(to_repo))) - except IOError, (num, msg): - if num == errno.ENOENT: - print >>sys.stderr, "%s, skipping package %s." % (msg, pkg) - retval = 1 - continue - else: - raise IOError, (num, msg) - (ret, changes, dest) = mangle_changes_file(changes_file, pkglist[0], version, from_repo, to_repo, options.arch) - if not ret: - print "Couldn't find %s in changes file %s - bad changes file - skipping package '%s'." % (from_repo, changes_file, pkg) - retval = 1 - continue - try: - if signas: - sign_changes_file(changes, signas, dest) - else: - dump_changes_file(changes, dest) - except OSError as err: - print >>sys.stderr, "Error writing changes file %s - skipping package '%s'." % (err, pkg) - retval = 1 - continue - if options.move: - remove_files( files, get_repo_path(from_repo)) - return retval - -if __name__ == "__main__": - sys.exit(main()) - -# vim:et:ts=4:sw=4: diff --git a/stagepkg/__init__.py b/stagepkg/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/stagepkg/command.py b/stagepkg/command.py new file mode 100644 index 0000000..1d71f81 --- /dev/null +++ b/stagepkg/command.py @@ -0,0 +1,377 @@ +#!/usr/bin/python +# +# (c) 2006,2014 Guido Guenther + +""" +Move a package between repositories +@license: GPLv3+ +""" +import errno +import re +import sys +import os +import shutil +import stat +import glob +import optparse + +# move to conf file +archive_root = "/home/debian-packages" +default_arch = "i386" +incoming = "/mini-dinstall/incoming" +conf_dir = "/etc/stagepkg/" +list_dir = os.path.join(conf_dir, "lists") + +simulate = False + +class ListNotFound(Exception): + """List file not found""" + pass + + +class OptionParser (optparse.OptionParser): + """Extend the standard option parser to check for required arguments""" + def check_required (self, opt): + option = self.get_option(opt) + if getattr(self.values, option.dest) is None: + self.error("%s option not supplied, use --help for help" % option) + + +def get_files(changes_file): + """Get all the files referenced in a changes file""" + files = [] + try: + cf = file(changes_file) + except: + return [] + fre=re.compile(" [a-f0-9]{32}\s+[0-9]{2,}\s+[a-z0-9/-]+\s+[a-z-]+\s+(?P.*_(?P(all|i386|.*))\.(?P(dsc|gz|bz2|deb|udeb)))") + for line in cf.readlines(): + r = fre.match(line) + if r: + if r.group('ext') in [ 'gz', 'bz2', 'dsc' ]: + files.append("source/%s" % (r.group('fname'), )) + elif r.group('arch'): + files.append("%s/%s" % (r.group('arch'),r.group('fname'))) + cf.close() + return files + + +def get_subarchive(repo): + """in case we have a subarchive like foo-component return this + otherwise an empty string""" + subarchive = repo.split('/',1)[0] + return [ '', subarchive+'/' ][subarchive != repo] + + +def get_repo_path(repo): + """This is a real (on disk) repository path""" + return "%s/%s" % (archive_root, repo) + + +def get_incoming_path(archive): + """Find the upload queue for this archive""" + return "%s/%s/%s" % (archive_root, archive, incoming) + + +def guess_version(repo, pkg, arch): + res = glob.glob("%s/%s_*_%s.changes" % (get_repo_path(repo), pkg, arch)) + if res: + (dummy, version, dummy) = res[-1].split("_", 2) + else: + version = None + return version + + +def get_changes_file_path(repo, pkg, version, arch): + return "%s/%s_%s_%s.changes" % (get_repo_path(repo), pkg, version, arch) + + +def copy_files(files, repo_path, incoming_path): + """copy files to the incoming queue""" + dst = incoming_path + for f in files: + src = "%s/%s" % (repo_path, f) + if simulate: + print "Copy: %s to %s" % (src, dst) + else: + shutil.copy(src, dst) + + +def link_files(files, repo_path, incoming_path): + """link files to the incoming queue""" + dst = incoming_path + for f in files: + src = "%s/%s" % (repo_path, f) + if simulate: + print "Linking: %s to %s" % (src, dst) + else: + os.link(src, dst) + + +def mangle_changes_file(changes_file, pkg, version, from_repo, to_repo, arch): + def get_dist(dist): + """we might have things like unstable/foo, but the + distribution (as noted in the changelog/changes file) is only foo.""" + return dist.split('/')[-1] + cf = file(changes_file) + distline = "Distribution: %s" + distre = re.compile(distline % (get_dist(from_repo), )) + chline = r" %s \(%s\) %s; urgency=" + chre = re.compile(chline % ( pkg, version.replace('+', '\+'), get_dist(from_repo)) ) + pgpre = re.compile(r"(Hash: SHA1|-{5,}BEGIN PGP SIG(?PNATURE)?.*-{5,})$") + emptyre = re.compile(r"(^$)") + matched = False + changes = [] + for line in cf.readlines(): + (l, n) = distre.subn(distline % (get_dist(to_repo), ), line) + if n: + matched = True + changes += [ l ] + continue + (l, n) = chre.subn(" %s (%s) %s; urgency=" % (pkg, version, get_dist(to_repo)), line) + if n: + changes += [ l ] + continue + n = emptyre.match(line) + if n: + continue + n = pgpre.match(line) + if n: + if n.group('sig'): + break + else: + continue + changes += [ line ] + changes = "".join(changes) + + filename = get_changes_file_path(get_subarchive(to_repo)+"mini-dinstall/incoming", pkg, version, arch) + if simulate: + print + print "Changes file %s" % (filename, ) + print changes + print + return matched, changes, filename + + +def sign_changes_file(changes, signas, dest): + """sign a changes file""" + cmd = "gpg --yes --local-user %s --clearsign \ + --verify-options no-show-policy-urls \ + --armor --textmode --output %s" % (signas, dest) + if not simulate: + p = os.popen(cmd, "w") + p.write(changes) + p.close() + else: + print cmd + + +def dump_changes_file(changes, dest): + """writeout unsigned changes file""" + if not simulate: + f = open(dest, "w+") + f.write(changes) + f.close() + else: + print "writing changes file to '%s'" % dest + + +def remove_files(files, repo_path): + for tounlink in [ "%s/%s" % (repo_path, f) for f in files ]: + if simulate: + print "Unlink: %s" % (tounlink, ) + else: + os.unlink(tounlink) + + +def parse_pkg_list(list): + """parse a pkg list file and return the packages therein""" + pkgs = [] + + if not list.startswith("/"): + list = os.path.join(list_dir, list) + + if not os.access(list, os.R_OK): + raise ListNotFound, "List '%s' not found" % list + + lf = file(list) + for line in lf: + pkg = line.strip() + if pkg.startswith("#"): + continue + else: + pkgs.append(pkg) + return pkgs + + +def parachute(from_repo, to_repo): + """Check if we're following our release rules""" + order = [ "unstable", "testing", "stable" ] + try: + try: + from_suite, from_comp = from_repo.split("/") + except ValueError: + raise ValueError, "Source repository has no '/'" + try: + to_suite, to_comp = to_repo.split("/") + except ValueError: + raise ValueError, "Target repository has no '/'" + if from_suite != to_suite: + raise ValueError, "Source suite '%s' is different from destination suite '%s'" % (from_suite, to_suite) + from_level = from_comp.split("-")[-1] + to_level = to_comp.split("-")[-1] + if to_level not in order: + to_level = "stable" + if from_level not in order: + from_level = "stable" + if from_level == "stable": + raise ValueError, "Refusing to remove out of a stable archive" + index = order.index(from_level)+1 + if order[index] != to_level: + raise ValueError, "'%s' is not the next step after '%s'" % (to_level, from_level) + except ValueError, msg: + print >>sys.stderr, "%s - use --force to override" % msg + return False + return True + + +def check_repo(path): + try: + if not stat.S_ISDIR(os.stat(path).st_mode): + print >>sys.stderr, "Target '%s' is not a directory" % path + return False + except OSError, (errnum, errmsg): + if errnum == errno.ENOENT: + msg = "Target archive '%s' does not exist" % path + else: + msg = errmsg + print >>sys.stderr, msg + return False + if not os.access (path, os.R_OK): + print >>sys.stderr, "Target archive '%s' not readable" % path + return False + if not os.access (path, os.W_OK): + print >>sys.stderr, "Target archive '%s' not writeable" % path + return False + return True + + +def run(): + pkgs = [] + global simulate + retval = 0 + + parser = OptionParser(usage="""usage: %prog [options] pkg1[=version] [pkg2[=version]] ... + Move a package between stages""") + parser.add_option("-f", "--from", dest="from_repo", + help="move from repository") + parser.add_option("-t", "--to", dest="to_repo", + help="move to repository") + parser.add_option("-m", "--move", action="store_true", dest="move", + default=False, help="move instead of copying") + parser.add_option("-s", "--simulate", action="store_true", + dest="simulate", default=False, help="show what would be done") + parser.add_option("-H", "--hard-links", action="store_true", + dest="link", default=False, help="hard link instead of copying") + parser.add_option("-M", "--maintainer", + dest="maintainer", help="maintainer name used for signing") + parser.add_option("-k", "--keyid", + dest="keyid", help="keyid name used for signing") + parser.add_option("--force", action="store_true", default=False, + dest="force", help="override any mismatching arguments - use with care") + parser.add_option("-a", "--arch", default=default_arch, + dest="arch", help="architecture we're acting on") + + (options, args) = parser.parse_args() + parser.check_required('-f') + parser.check_required('-t') + parser.check_required('-k') + + if options.keyid: + signas = options.keyid + elif options.maintainer: + signas = options.maintainer + else: + signas = None + + simulate = options.simulate + + from_repo = options.from_repo.strip('/') + to_repo = options.to_repo.strip('/') + + if not options.force and not parachute(from_repo, to_repo): + return 1 + + if options.link: + dropoff = link_files + else: + dropoff = copy_files + + if not len(args): + parser.print_help() + sys.exit(1) + + try: + for pkg in args: + if pkg.endswith(".list"): + pkgs += parse_pkg_list(pkg) + else: + pkgs.append(pkg) + except ListNotFound as err: + print >>sys.stderr, "\n%s - doing nothing\n" % err + parser.print_help() + return 1 + + to_path = get_repo_path(to_repo) + + if not check_repo(to_path): + return 2 + + for pkg in pkgs: + pkglist = pkg.split('=', 1) + if(len(pkglist) != 2): + version = guess_version(from_repo, pkglist[0], options.arch) + if not version: + print >>sys.stderr, "No changes file found in '%s' for package '%s' - skipping package '%s'." % (from_repo, pkglist[0], pkg) + retval = 1 + continue + else: + version = pkglist[1] + changes_file = get_changes_file_path(from_repo, pkglist[0], version, options.arch) + files = get_files(changes_file) + if not files: + print >>sys.stderr, "Couldn't parse list of files from %s - skipping package '%s'" % (changes_file, pkg) + retval = 1 + continue + + try: + dropoff(files, get_repo_path(from_repo), get_incoming_path(get_subarchive(to_repo))) + except IOError, (num, msg): + if num == errno.ENOENT: + print >>sys.stderr, "%s, skipping package %s." % (msg, pkg) + retval = 1 + continue + else: + raise IOError, (num, msg) + (ret, changes, dest) = mangle_changes_file(changes_file, pkglist[0], version, from_repo, to_repo, options.arch) + if not ret: + print "Couldn't find %s in changes file %s - bad changes file - skipping package '%s'." % (from_repo, changes_file, pkg) + retval = 1 + continue + try: + if signas: + sign_changes_file(changes, signas, dest) + else: + dump_changes_file(changes, dest) + except OSError as err: + print >>sys.stderr, "Error writing changes file %s - skipping package '%s'." % (err, pkg) + retval = 1 + continue + if options.move: + remove_files( files, get_repo_path(from_repo)) + return retval + +if __name__ == "__main__": + sys.exit(main()) + +# vim:et:ts=4:sw=4: -- cgit v1.2.3