From 68b257d055679c5c6dcb66a7e8158a80f11611a8 Mon Sep 17 00:00:00 2001 From: Priit Laes Date: Mon, 28 Jun 2010 22:22:26 +0300 Subject: Add package sync utility based on mtime-checks --- utils/grumpy_sync.py | 200 ++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 152 insertions(+), 48 deletions(-) (limited to 'utils/grumpy_sync.py') diff --git a/utils/grumpy_sync.py b/utils/grumpy_sync.py index f170dd9..aa54ace 100755 --- a/utils/grumpy_sync.py +++ b/utils/grumpy_sync.py @@ -1,6 +1,9 @@ #!/usr/bin/env python import os, sys +from datetime import datetime +from fnmatch import fnmatch + from pkgcore.config import load_config from pkgcore.cache import metadata from pkgcore.ebuild import repository @@ -12,7 +15,7 @@ from sqlalchemy.orm.exc import NoResultFound path = os.path.join(os.path.dirname(__file__), os.path.pardir) sys.path.insert(0, path) -from grumpy.models import Base, Developer, Ebuild, Herd, Package +from grumpy.models import Base, Category, Developer, Ebuild, Herd, Package def main(path): engine = create_engine('postgresql://grumpy:grumpy@localhost/grumpy') @@ -21,64 +24,165 @@ def main(path): bind=engine)) Base.query = session.query_property() - Base.metadata.drop_all(bind=engine) - Base.metadata.create_all(bind=engine) - # pkgcore part to fetch all the ebuild information conf = load_config() eclass_cache = conf.eclass_cache['eclass stack'] cache = metadata.database(readonly=True, location=path) - overlay_repo = repository.UnconfiguredTree(path, cache=cache, \ - eclass_cache=eclass_cache) + repo = repository.UnconfiguredTree(path, cache=cache, \ + eclass_cache=eclass_cache) - for pkg in overlay_repo: - pkg_id = "%s/%s" % (pkg.package, pkg.category) + def extract_version(pkg, cpv): + """...""" + return cpv[len(pkg)+1:-7] - package = Package.query.filter_by(cp=pkg_id).first() - if not package: + def package_update(cat, pkg, files, mtime): + """Update package information in database.""" + + # Fetch package from database + package = Package.query.filter_by(cat=cat).filter_by(pkg=pkg).first() + + # Check whether package in database is up-to-date + if package and package.mtime == datetime.fromtimestamp(mtime): + assert len(package.ebuilds) == len(files) + return - package = Package(pkg.category, pkg.package, pkg.description, \ - pkg.longdescription, pkg.homepage) - # Fetch devs and herds from db - devs = [] - for dev in pkg.maintainers: - try: - devs.append(Developer.query.filter_by(email=dev.email).one()) - except NoResultFound: - devs.append(Developer(dev.email)) - session.add_all(devs) - herds = [] - for herd in pkg.herds: - # Workaround for empty tags - if herd is None: - herd = 'fix-me' - herd = herd.strip() - try: - herds.append(Herd.query.filter_by(name=herd).one()) - except NoResultFound: - herds.append(Herd(herd)) - session.add_all(herds) - - package.devs = devs - package.herds = herds + print "DEBUG: updating package %s/%s" % (cat, pkg) + # No ebuilds for package? + if len(files) == 0: + # TODO, need to check whether there's been pkgmove + raise NotImplementedError + + pack = repo[(cat, pkg, extract_version(pkg, files[0]))] + + # Update or create new package + if not package: + package = Package(pack.category, pack.package, pack.description, \ + pack.longdescription, pack.homepage, mtime) session.add(package) - session.commit() - - # Parse IUSE (iuse contains all flags, fiuse contains only '+use') - iuse = list() - fiuse = list() - for u in pkg.iuse: - if u[0] == '+': - iuse.append(u[1:]) - fiuse.append(u[1:]) - else: - iuse.append(u) - - session.add(Ebuild(package, pkg.fullver, pkg.eapi, \ - pkg.slot, pkg.keywords, iuse, fiuse)) + else: + # Update package fields + package.cat = pack.category + package.pkg = pack.package + package.desc = pack.description + package.ldesc = pack.longdescription + package.homepage = pack.homepage + package.mtime = datetime.fromtimestamp(mtime) + + # Add/Update devs and herds + new = [d.email for d in pack.maintainers] + old = [d.email for d in package.devs] + # Remove links to removed developers + for dev in [item for item in old if item not in new]: + print "DEBUG: removing developer reference:", dev + package.devs.remove(Developer.query.filter_by(email=dev).one()) + # Add/update new developers + for dev in new: + if dev in old: + continue + print "DEBUG: adding developer reference:", dev + d = Developer.query.filter_by(email=dev).first() + if not d: + print "DEBUG: adding new developer to database:", dev + d = Developer(dev) + package.devs.append(d) + + # Handle herds + new = [] + for herd in pack.herds: + if herd is None: + herd = 'fix-me' + new.append(herd.strip()) + old = [h.name for h in package.herds] + for herd in [item for item in old if item not in new]: + print "DEBUG: removing herd reference", herd + package.herds.remove(Herd.query.filter_by(name=herd).one()) + for herd in new: + if herd in old: + continue + print "DEBUG: adding herd reference:", herd + h = Herd.query.filter_by(name=herd).first() + if not h: + print "DEBUG: adding new herd to database:", herd + h = Herd(herd) + package.herds.append(h) + + # Handle ebuilds + new = [extract_version(pkg, file) for file in files] + old = [e.version for e in package.ebuilds] + print "DEBUG: old: ", old + print "DEBUG: new: ", new + print "DEBUG: diff", [item for item in new if item not in old] + for ver in [item for item in old if item not in new]: + # Delete old ebuilds + ebuild = Ebuild.query.filter_by(cpv="%s/%s-%s" % (pkg, cat, ver)).first() + if not ebuild: + print "Corruption detected: ebuild not found in database" + raise RuntimeError + session.delete(ebuild) + + # Updates/add new ebuilds + for ver in new: + ebuild = repo[(cat, pkg, ver)] + if ver not in old: + iuse = list() + fiuse = list() + for u in ebuild.iuse: + if u[0] == '+': + iuse.append(u[1:]) + fiuse.append(u[1:]) + else: + iuse.append(u) + package.ebuilds.append(Ebuild(package, ebuild.fullver, \ + ebuild.eapi, ebuild.slot, \ + ebuild.keywords, iuse, fiuse)) + continue + # FIXME: use package.ebuilds for lookup + oeb = Ebuild.query.filter_by(cpv="%s/%s-%s" % (pkg, cat, ver)).first() + if not oeb: + print "Ebuild must be in database, corruption detected" + raise RuntimeError + + iuse = list() + fiuse = list() + for u in ebuild.iuse: + if u[0] == '+': + iuse.append(u[1:]) + fiuse.append(u[1:]) + else: + iuse.append(u) + + oeb.iuse = iuse + oeb.fiuse = fiuse + oeb.eapi = ebuild.eapi + oeb.slot = ebuild.slot + oeb.keywords = list(ebuild.keywords) + session.commit() + # Compare list of categories in portage vs database + cat_sql = [c.cat for c in Category.query.all()] + cats = repo.categories.keys() + + # Store for later + cat_diff = list(set(cat_sql) | set(cats)) + + # TODO + # save/del categories from database + + # Traverse portage + for cat in cats: + catdir = os.path.join(path, cat) + pkgs = os.listdir(catdir) + # TODO + # handle package moves (when one isn't in tree and when one is..) + for pkg in pkgs: + dir = os.path.join(catdir, pkg) + if not os.path.isdir(dir): + continue + files = [f for f in os.listdir(dir) if fnmatch(f, '*.ebuild')] + package_update(cat, pkg, files, int(os.stat(dir).st_mtime)) + if __name__ == '__main__': if len(sys.argv) != 2: print "Please provide path to portage directory as argument" -- cgit v1.2.3-65-gdbad