aboutsummaryrefslogtreecommitdiff
path: root/utils
diff options
context:
space:
mode:
authorPriit Laes <plaes@plaes.org>2010-06-28 22:22:26 +0300
committerPriit Laes <plaes@plaes.org>2010-06-30 12:23:40 +0300
commit68b257d055679c5c6dcb66a7e8158a80f11611a8 (patch)
treeb0d87e4ccda685384d8e2d38b771df851fa61be6 /utils
parentAdd mtime column for packages (diff)
downloadgsoc2010-grumpy-68b257d055679c5c6dcb66a7e8158a80f11611a8.tar.gz
gsoc2010-grumpy-68b257d055679c5c6dcb66a7e8158a80f11611a8.tar.bz2
gsoc2010-grumpy-68b257d055679c5c6dcb66a7e8158a80f11611a8.zip
Add package sync utility based on mtime-checks
Diffstat (limited to 'utils')
-rwxr-xr-xutils/grumpy_sync.py200
1 files changed, 152 insertions, 48 deletions
diff --git a/utils/grumpy_sync.py b/utils/grumpy_sync.py
index f170dd9..aa54ace 100755
--- a/utils/grumpy_sync.py
+++ b/utils/grumpy_sync.py
@@ -1,6 +1,9 @@
#!/usr/bin/env python
import os, sys
+from datetime import datetime
+from fnmatch import fnmatch
+
from pkgcore.config import load_config
from pkgcore.cache import metadata
from pkgcore.ebuild import repository
@@ -12,7 +15,7 @@ from sqlalchemy.orm.exc import NoResultFound
path = os.path.join(os.path.dirname(__file__), os.path.pardir)
sys.path.insert(0, path)
-from grumpy.models import Base, Developer, Ebuild, Herd, Package
+from grumpy.models import Base, Category, Developer, Ebuild, Herd, Package
def main(path):
engine = create_engine('postgresql://grumpy:grumpy@localhost/grumpy')
@@ -21,64 +24,165 @@ def main(path):
bind=engine))
Base.query = session.query_property()
- Base.metadata.drop_all(bind=engine)
- Base.metadata.create_all(bind=engine)
-
# pkgcore part to fetch all the ebuild information
conf = load_config()
eclass_cache = conf.eclass_cache['eclass stack']
cache = metadata.database(readonly=True, location=path)
- overlay_repo = repository.UnconfiguredTree(path, cache=cache, \
- eclass_cache=eclass_cache)
+ repo = repository.UnconfiguredTree(path, cache=cache, \
+ eclass_cache=eclass_cache)
- for pkg in overlay_repo:
- pkg_id = "%s/%s" % (pkg.package, pkg.category)
+ def extract_version(pkg, cpv):
+ """..."""
+ return cpv[len(pkg)+1:-7]
- package = Package.query.filter_by(cp=pkg_id).first()
- if not package:
+ def package_update(cat, pkg, files, mtime):
+ """Update package information in database."""
+
+ # Fetch package from database
+ package = Package.query.filter_by(cat=cat).filter_by(pkg=pkg).first()
+
+ # Check whether package in database is up-to-date
+ if package and package.mtime == datetime.fromtimestamp(mtime):
+ assert len(package.ebuilds) == len(files)
+ return
- package = Package(pkg.category, pkg.package, pkg.description, \
- pkg.longdescription, pkg.homepage)
- # Fetch devs and herds from db
- devs = []
- for dev in pkg.maintainers:
- try:
- devs.append(Developer.query.filter_by(email=dev.email).one())
- except NoResultFound:
- devs.append(Developer(dev.email))
- session.add_all(devs)
- herds = []
- for herd in pkg.herds:
- # Workaround for empty <herd></herd> tags
- if herd is None:
- herd = 'fix-me'
- herd = herd.strip()
- try:
- herds.append(Herd.query.filter_by(name=herd).one())
- except NoResultFound:
- herds.append(Herd(herd))
- session.add_all(herds)
-
- package.devs = devs
- package.herds = herds
+ print "DEBUG: updating package %s/%s" % (cat, pkg)
+ # No ebuilds for package?
+ if len(files) == 0:
+ # TODO, need to check whether there's been pkgmove
+ raise NotImplementedError
+
+ pack = repo[(cat, pkg, extract_version(pkg, files[0]))]
+
+ # Update or create new package
+ if not package:
+ package = Package(pack.category, pack.package, pack.description, \
+ pack.longdescription, pack.homepage, mtime)
session.add(package)
- session.commit()
-
- # Parse IUSE (iuse contains all flags, fiuse contains only '+use')
- iuse = list()
- fiuse = list()
- for u in pkg.iuse:
- if u[0] == '+':
- iuse.append(u[1:])
- fiuse.append(u[1:])
- else:
- iuse.append(u)
-
- session.add(Ebuild(package, pkg.fullver, pkg.eapi, \
- pkg.slot, pkg.keywords, iuse, fiuse))
+ else:
+ # Update package fields
+ package.cat = pack.category
+ package.pkg = pack.package
+ package.desc = pack.description
+ package.ldesc = pack.longdescription
+ package.homepage = pack.homepage
+ package.mtime = datetime.fromtimestamp(mtime)
+
+ # Add/Update devs and herds
+ new = [d.email for d in pack.maintainers]
+ old = [d.email for d in package.devs]
+ # Remove links to removed developers
+ for dev in [item for item in old if item not in new]:
+ print "DEBUG: removing developer reference:", dev
+ package.devs.remove(Developer.query.filter_by(email=dev).one())
+ # Add/update new developers
+ for dev in new:
+ if dev in old:
+ continue
+ print "DEBUG: adding developer reference:", dev
+ d = Developer.query.filter_by(email=dev).first()
+ if not d:
+ print "DEBUG: adding new developer to database:", dev
+ d = Developer(dev)
+ package.devs.append(d)
+
+ # Handle herds
+ new = []
+ for herd in pack.herds:
+ if herd is None:
+ herd = 'fix-me'
+ new.append(herd.strip())
+ old = [h.name for h in package.herds]
+ for herd in [item for item in old if item not in new]:
+ print "DEBUG: removing herd reference", herd
+ package.herds.remove(Herd.query.filter_by(name=herd).one())
+ for herd in new:
+ if herd in old:
+ continue
+ print "DEBUG: adding herd reference:", herd
+ h = Herd.query.filter_by(name=herd).first()
+ if not h:
+ print "DEBUG: adding new herd to database:", herd
+ h = Herd(herd)
+ package.herds.append(h)
+
+ # Handle ebuilds
+ new = [extract_version(pkg, file) for file in files]
+ old = [e.version for e in package.ebuilds]
+ print "DEBUG: old: ", old
+ print "DEBUG: new: ", new
+ print "DEBUG: diff", [item for item in new if item not in old]
+ for ver in [item for item in old if item not in new]:
+ # Delete old ebuilds
+ ebuild = Ebuild.query.filter_by(cpv="%s/%s-%s" % (pkg, cat, ver)).first()
+ if not ebuild:
+ print "Corruption detected: ebuild not found in database"
+ raise RuntimeError
+ session.delete(ebuild)
+
+ # Updates/add new ebuilds
+ for ver in new:
+ ebuild = repo[(cat, pkg, ver)]
+ if ver not in old:
+ iuse = list()
+ fiuse = list()
+ for u in ebuild.iuse:
+ if u[0] == '+':
+ iuse.append(u[1:])
+ fiuse.append(u[1:])
+ else:
+ iuse.append(u)
+ package.ebuilds.append(Ebuild(package, ebuild.fullver, \
+ ebuild.eapi, ebuild.slot, \
+ ebuild.keywords, iuse, fiuse))
+ continue
+ # FIXME: use package.ebuilds for lookup
+ oeb = Ebuild.query.filter_by(cpv="%s/%s-%s" % (pkg, cat, ver)).first()
+ if not oeb:
+ print "Ebuild must be in database, corruption detected"
+ raise RuntimeError
+
+ iuse = list()
+ fiuse = list()
+ for u in ebuild.iuse:
+ if u[0] == '+':
+ iuse.append(u[1:])
+ fiuse.append(u[1:])
+ else:
+ iuse.append(u)
+
+ oeb.iuse = iuse
+ oeb.fiuse = fiuse
+ oeb.eapi = ebuild.eapi
+ oeb.slot = ebuild.slot
+ oeb.keywords = list(ebuild.keywords)
+
session.commit()
+ # Compare list of categories in portage vs database
+ cat_sql = [c.cat for c in Category.query.all()]
+ cats = repo.categories.keys()
+
+ # Store for later
+ cat_diff = list(set(cat_sql) | set(cats))
+
+ # TODO
+ # save/del categories from database
+
+ # Traverse portage
+ for cat in cats:
+ catdir = os.path.join(path, cat)
+ pkgs = os.listdir(catdir)
+ # TODO
+ # handle package moves (when one isn't in tree and when one is..)
+ for pkg in pkgs:
+ dir = os.path.join(catdir, pkg)
+ if not os.path.isdir(dir):
+ continue
+ files = [f for f in os.listdir(dir) if fnmatch(f, '*.ebuild')]
+ package_update(cat, pkg, files, int(os.stat(dir).st_mtime))
+
if __name__ == '__main__':
if len(sys.argv) != 2:
print "Please provide path to portage directory as argument"