#!/usr/bin/env python import os, sys from datetime import datetime from fnmatch import fnmatch from pkgcore.config import load_config from pkgcore.cache import metadata from pkgcore.ebuild import repository from sqlalchemy import create_engine from sqlalchemy.orm import scoped_session, sessionmaker from sqlalchemy.orm.exc import NoResultFound from snakeoil.fileutils import iter_read_bash path = os.path.join(os.path.dirname(__file__), os.path.pardir) sys.path.insert(0, path) del path from grumpy.models import Base, Category, Developer, Ebuild, Herd, Package def main(path): engine = create_engine('postgresql://grumpy:grumpy@localhost/grumpy') session = scoped_session(sessionmaker(autocommit=False, autoflush=False, bind=engine)) Base.query = session.query_property() # pkgcore part to fetch all the ebuild information conf = load_config() eclass_cache = conf.eclass_cache['eclass stack'] cache = metadata.database(readonly=True, location=path) repo = repository.UnconfiguredTree(path, cache=cache, \ eclass_cache=eclass_cache) def extract_version(pkg, cpv): """...""" return cpv[len(pkg)+1:-7] def package_update(cat, pkg, files, mtime): """Update package information in database.""" # Fetch package from database package = Package.query.filter_by(cat=cat).filter_by(pkg=pkg).first() # Check whether package in database is up-to-date if package and package.mtime == datetime.fromtimestamp(mtime): assert len(package.ebuilds) == len(files) return print "DEBUG: updating package %s/%s" % (cat, pkg) pack = repo[(cat, pkg, extract_version(pkg, files[0]))] # Update or create new package if not package: package = Package(pack.category, pack.package, pack.description, \ pack.longdescription, pack.homepage, mtime) session.add(package) else: # Update package fields package.cat = pack.category package.pkg = pack.package package.desc = pack.description package.ldesc = pack.longdescription package.homepage = pack.homepage package.mtime = datetime.fromtimestamp(mtime) # Add/Update devs and herds new = [d.email for d in pack.maintainers] old = [d.email for d in package.devs] # Remove links to removed developers for dev in [item for item in old if item not in new]: print "DEBUG: removing developer reference:", dev package.devs.remove(Developer.query.filter_by(email=dev).one()) # Add/update new developers for dev in new: if dev in old: continue print "DEBUG: adding developer reference:", dev d = Developer.query.filter_by(email=dev).first() if not d: print "DEBUG: adding new developer to database:", dev d = Developer(dev) package.devs.append(d) # Handle herds new = [] for herd in pack.herds: if herd is None: herd = 'fix-me' new.append(herd.strip()) old = [h.name for h in package.herds] for herd in [item for item in old if item not in new]: print "DEBUG: removing herd reference", herd package.herds.remove(Herd.query.filter_by(name=herd).one()) for herd in new: if herd in old: continue print "DEBUG: adding herd reference:", herd h = Herd.query.filter_by(name=herd).first() if not h: print "DEBUG: adding new herd to database:", herd h = Herd(herd) package.herds.append(h) # Handle ebuilds new = [extract_version(pkg, file) for file in files] old = [e.version for e in package.ebuilds] print "DEBUG: old: ", old print "DEBUG: new: ", new print "DEBUG: diff", [item for item in new if item not in old] for ver in [item for item in old if item not in new]: # Delete old ebuilds ebuild = Ebuild.query.filter_by(cpv="%s/%s-%s" % (pkg, cat, ver)).first() if not ebuild: print "Corruption detected: ebuild not found in database" raise RuntimeError session.delete(ebuild) # Updates/add new ebuilds for ver in new: ebuild = repo[(cat, pkg, ver)] if ver not in old: iuse = list() fiuse = list() for u in ebuild.iuse: if u[0] == '+': iuse.append(u[1:]) fiuse.append(u[1:]) else: iuse.append(u) package.ebuilds.append(Ebuild(package, ebuild.fullver, \ ebuild.eapi, ebuild.slot, \ ebuild.keywords, iuse, fiuse)) continue # FIXME: use package.ebuilds for lookup oeb = Ebuild.query.filter_by(cpv="%s/%s-%s" % (pkg, cat, ver)).first() if not oeb: print "Ebuild must be in database, corruption detected" raise RuntimeError iuse = list() fiuse = list() for u in ebuild.iuse: if u[0] == '+': iuse.append(u[1:]) fiuse.append(u[1:]) else: iuse.append(u) oeb.iuse = iuse oeb.fiuse = fiuse oeb.eapi = ebuild.eapi oeb.slot = ebuild.slot oeb.keywords = list(ebuild.keywords) session.commit() # Handle package moves dir = os.path.join(path, 'profiles', 'updates') moves = {} if os.path.isdir(dir): def get_key(fname): return tuple(reversed(fname.split('-'))) for update_file in sorted(os.listdir(dir), key=get_key): for line in iter_read_bash(os.path.join(dir, update_file)): line = line.split() if line[0] != 'move': continue moves[line[1]] = line[2] # Compare list of categories in portage vs database cat_sql = [c.cat for c in Category.query.all()] cats = repo.categories.keys() # Store for later cat_diff = list(set(cat_sql) | set(cats)) # TODO # save/del categories from database # Traverse portage for cat in cats: catdir = os.path.join(path, cat) # Get list of existing packages in this category old = [p.pkg for p in Package.query.filter_by(cat=cat).all()] new = [d for d in os.listdir(catdir) \ if os.path.isdir(os.path.join(catdir, d))] for pkg in [i for i in old if i not in new]: # Handle package move or deletion if "%s/%s" % (cat, pkg) in moves.keys(): print "DEBUG: package has been moved:", pkg raise NotImplementedError else: print "DEBUG: package has been (re)moved:", pkg package = Package.query.filter_by(cat=cat).filter_by(pkg=pkg).one() session.delete(package) session.commit() for pkg in new: dir = os.path.join(catdir, pkg) files = [f for f in os.listdir(dir) if fnmatch(f, '*.ebuild')] package_update(cat, pkg, files, int(os.stat(dir).st_mtime)) if __name__ == '__main__': if len(sys.argv) != 2: print "Please provide path to portage directory as argument" sys.exit() main(sys.argv[1])