Add package sync utility based on mtime-checks

1 files changed, 152 insertions, 48 deletions

diff --git a/utils/grumpy_sync.py b/utils/grumpy_sync.py
index f170dd9..aa54ace 100755
--- a/utils/grumpy_sync.py
+++ b/utils/grumpy_sync.py
@@ -1,6 +1,9 @@
 #!/usr/bin/env python
 import os, sys
 
+from datetime import datetime
+from fnmatch import fnmatch
+
 from pkgcore.config import load_config
 from pkgcore.cache import metadata
 from pkgcore.ebuild import repository
@@ -12,7 +15,7 @@ from sqlalchemy.orm.exc import NoResultFound
 path = os.path.join(os.path.dirname(__file__), os.path.pardir)
 sys.path.insert(0, path)
 
-from grumpy.models import Base, Developer, Ebuild, Herd, Package
+from grumpy.models import Base, Category, Developer, Ebuild, Herd, Package
 
 def main(path):
     engine = create_engine('postgresql://grumpy:grumpy@localhost/grumpy')
@@ -21,64 +24,165 @@ def main(path):
                                           bind=engine))
     Base.query = session.query_property()
 
-    Base.metadata.drop_all(bind=engine)
-    Base.metadata.create_all(bind=engine)
-
     # pkgcore part to fetch all the ebuild information
     conf = load_config()
     eclass_cache = conf.eclass_cache['eclass stack']
     cache = metadata.database(readonly=True, location=path)
-    overlay_repo = repository.UnconfiguredTree(path, cache=cache, \
-                                               eclass_cache=eclass_cache)
+    repo = repository.UnconfiguredTree(path, cache=cache, \
+                                       eclass_cache=eclass_cache)
 
-    for pkg in overlay_repo:
-        pkg_id = "%s/%s" % (pkg.package, pkg.category)
+    def extract_version(pkg, cpv):
+        """..."""
+        return cpv[len(pkg)+1:-7]
 
-        package = Package.query.filter_by(cp=pkg_id).first()
-        if not package:
+    def package_update(cat, pkg, files, mtime):
+        """Update package information in database."""
+
+        # Fetch package from database
+        package = Package.query.filter_by(cat=cat).filter_by(pkg=pkg).first()
+
+        # Check whether package in database is up-to-date
+        if package and package.mtime == datetime.fromtimestamp(mtime):
+            assert len(package.ebuilds) == len(files)
+            return
 
-            package = Package(pkg.category, pkg.package, pkg.description, \
-                              pkg.longdescription, pkg.homepage)
-            # Fetch devs and herds from db
-            devs = []
-            for dev in pkg.maintainers:
-                try:
-                    devs.append(Developer.query.filter_by(email=dev.email).one())
-                except NoResultFound:
-                    devs.append(Developer(dev.email))
-            session.add_all(devs)
-            herds = []
-            for herd in pkg.herds:
-                # Workaround for empty <herd></herd> tags
-                if herd is None:
-                    herd = 'fix-me'
-                herd = herd.strip()
-                try:
-                    herds.append(Herd.query.filter_by(name=herd).one())
-                except NoResultFound:
-                    herds.append(Herd(herd))
-            session.add_all(herds)
-
-            package.devs = devs
-            package.herds = herds
+        print "DEBUG: updating package %s/%s" % (cat, pkg)
 
+        # No ebuilds for package?
+        if len(files) == 0:
+            # TODO, need to check whether there's been pkgmove
+            raise NotImplementedError
+
+        pack = repo[(cat, pkg, extract_version(pkg, files[0]))]
+
+        # Update or create new package
+        if not package:
+            package = Package(pack.category, pack.package, pack.description, \
+                              pack.longdescription, pack.homepage, mtime)
             session.add(package)
-            session.commit()
-
-        # Parse IUSE (iuse contains all flags, fiuse contains only '+use')
-        iuse = list()
-        fiuse = list()
-        for u in pkg.iuse:
-            if u[0] == '+':
-                iuse.append(u[1:])
-                fiuse.append(u[1:])
-            else:
-                iuse.append(u)
-
-        session.add(Ebuild(package, pkg.fullver, pkg.eapi, \
-                           pkg.slot, pkg.keywords, iuse, fiuse))
+        else:
+            # Update package fields
+            package.cat = pack.category
+            package.pkg = pack.package
+            package.desc = pack.description
+            package.ldesc = pack.longdescription
+            package.homepage = pack.homepage
+            package.mtime = datetime.fromtimestamp(mtime)
+
+        # Add/Update devs and herds
+        new = [d.email for d in pack.maintainers]
+        old = [d.email for d in package.devs]
+        # Remove links to removed developers
+        for dev in [item for item in old if item not in new]:
+            print "DEBUG: removing developer reference:", dev
+            package.devs.remove(Developer.query.filter_by(email=dev).one())
+        # Add/update new developers
+        for dev in new:
+            if dev in old:
+                continue
+            print "DEBUG: adding developer reference:", dev
+            d = Developer.query.filter_by(email=dev).first()
+            if not d:
+                print "DEBUG: adding new developer to database:", dev
+                d = Developer(dev)
+            package.devs.append(d)
+
+        # Handle herds
+        new = []
+        for herd in pack.herds:
+            if herd is None:
+                herd = 'fix-me'
+            new.append(herd.strip())
+        old = [h.name for h in package.herds]
+        for herd in [item for item in old if item not in new]:
+            print "DEBUG: removing herd reference", herd
+            package.herds.remove(Herd.query.filter_by(name=herd).one())
+        for herd in new:
+            if herd in old:
+                continue
+            print "DEBUG: adding herd reference:", herd
+            h = Herd.query.filter_by(name=herd).first()
+            if not h:
+                print "DEBUG: adding new herd to database:", herd
+                h = Herd(herd)
+            package.herds.append(h)
+
+        # Handle ebuilds
+        new = [extract_version(pkg, file) for file in files]
+        old = [e.version for e in package.ebuilds]
+        print "DEBUG: old: ", old
+        print "DEBUG: new: ", new
+        print "DEBUG: diff", [item for item in new if item not in old]
+        for ver in [item for item in old if item not in new]:
+            # Delete old ebuilds
+            ebuild = Ebuild.query.filter_by(cpv="%s/%s-%s" % (pkg, cat, ver)).first()
+            if not ebuild:
+                print "Corruption detected: ebuild not found in database"
+                raise RuntimeError
+            session.delete(ebuild)
+
+        # Updates/add new ebuilds
+        for ver in new:
+            ebuild = repo[(cat, pkg, ver)]
+            if ver not in old:
+                iuse = list()
+                fiuse = list()
+                for u in ebuild.iuse:
+                    if u[0] == '+':
+                        iuse.append(u[1:])
+                        fiuse.append(u[1:])
+                    else:
+                        iuse.append(u)
+                package.ebuilds.append(Ebuild(package, ebuild.fullver, \
+                                       ebuild.eapi, ebuild.slot, \
+                                       ebuild.keywords, iuse, fiuse))
+                continue
+            # FIXME: use package.ebuilds for lookup
+            oeb = Ebuild.query.filter_by(cpv="%s/%s-%s" % (pkg, cat, ver)).first()
+            if not oeb:
+                print "Ebuild must be in database, corruption detected"
+                raise RuntimeError
+
+            iuse = list()
+            fiuse = list()
+            for u in ebuild.iuse:
+                if u[0] == '+':
+                    iuse.append(u[1:])
+                    fiuse.append(u[1:])
+                else:
+                    iuse.append(u)
+
+            oeb.iuse = iuse
+            oeb.fiuse = fiuse
+            oeb.eapi = ebuild.eapi
+            oeb.slot = ebuild.slot
+            oeb.keywords = list(ebuild.keywords)
+
         session.commit()
 
+    # Compare list of categories in portage vs database
+    cat_sql = [c.cat for c in Category.query.all()]
+    cats = repo.categories.keys()
+
+    # Store for later
+    cat_diff = list(set(cat_sql) | set(cats))
+
+    # TODO
+    # save/del categories from database
+
+    # Traverse portage
+    for cat in cats:
+        catdir = os.path.join(path, cat)
+        pkgs = os.listdir(catdir)
+        # TODO
+        # handle package moves (when one isn't in tree and when one is..)
+        for pkg in pkgs:
+            dir = os.path.join(catdir, pkg)
+            if not os.path.isdir(dir):
+                continue
+            files = [f for f in os.listdir(dir) if fnmatch(f, '*.ebuild')]
+            package_update(cat, pkg, files, int(os.stat(dir).st_mtime))
+
 if __name__ == '__main__':
     if len(sys.argv) != 2:
         print "Please provide path to portage directory as argument"