diff options
Diffstat (limited to 'utils')
| -rw-r--r-- | utils/qa_check_developer.py | 79 | ||||
| -rw-r--r-- | utils/qa_check_herd.py | 26 |
2 files changed, 95 insertions, 10 deletions
diff --git a/utils/qa_check_developer.py b/utils/qa_check_developer.py new file mode 100644 index 0000000..4d353f3 --- /dev/null +++ b/utils/qa_check_developer.py @@ -0,0 +1,79 @@ +#! /usr/bin/env python +import os, sys +import urllib2 + +from datetime import datetime +from lxml.html import fromstring +from optparse import OptionParser + +path = os.path.join(os.path.dirname(__file__), os.path.pardir) +sys.path.insert(0, path) +del path + +from grumpy import app +from grumpy.models import db, Developer, PkgIssue, Setting + +DEVS_URL = 'http://www.gentoo.org/proj/en/devrel/roll-call/userinfo.xml' +PLUGIN_NAME='qa::valid_developer' + +def gc_collect(timestamp): + """Remove old QA issues from database returning number of rows deleted.""" + db.session.expire_all() + print ("DEBUG: Deleted %d old issues." % PkgIssue.query \ + .filter_by(plugin=PLUGIN_NAME) \ + .filter(PkgIssue.created_on < timestamp).delete(False)) + +def insert_issues(invalid): + """Insert QA issues into db.""" + if 'maintainer-needed@gentoo.org' in invalid: + h = Developer.query.filter_by(email='maintainer-needed@gentoo.org').first() + for pkg in h.packages: + pkg.qaissues.append(PkgIssue(pkg, PLUGIN_NAME, 'maintainer-needed')) + invalid.remove('maintainer-needed@gentoo.org') + for dev in invalid: + d = Developer.query.filter_by(email=dev).first() + for pkg in d.packages: + pkg.qaissues.append(PkgIssue(pkg, PLUGIN_NAME, 'unknown-dev', \ + 'User %s is not as active Gentoo developer.' % dev)) + db.session.commit() + +def download_and_parse_devs(): + """Return list of herd names from 'userinfo.xml'""" + data = urllib2.urlopen(DEVS_URL).read() + devs = [] + # Select <table class='ntable'> from page and iterate over rows + for tr in fromstring(data).cssselect('.ntable')[0].getchildren(): + # Next we have tr > td > b + devs.append('%s@gentoo.org' % tr.getchildren()[0].getchildren()[0].text) + # Skip first (this is table header + del devs[0] + return devs + +if __name__ == '__main__': + parser = OptionParser(usage="usage: %prog [options] CONFFILE") + (opts, args) = parser.parse_args() + if len(args) != 1: + parser.error("provide path to configuration file as first argument") + sys.exit(1) + # Fetch data and parse it + devs = download_and_parse_devs() + b0rks = [] + timestamp = datetime.now() + with app.test_request_context(): + # Load configuration + app.config.from_pyfile(args[0]) + # Fetch list of developers from db + dbdevs = [d.email for d in Developer.query.filter( \ + Developer.email.like('%@gentoo.org')).all()] + for dev in dbdevs: + if dev in devs: + devs.remove(dev) + else: + b0rks.append(dev) + insert_issues(b0rks) + # Clean up issues < timestamp + gc_collect(timestamp) + # Update settings and add info about last run.. + Setting.query.filter_by(name=PLUGIN_NAME).delete(False) + db.session.add(Setting(PLUGIN_NAME, str(timestamp))) + db.session.commit() diff --git a/utils/qa_check_herd.py b/utils/qa_check_herd.py index 21c5ead..7088d5a 100644 --- a/utils/qa_check_herd.py +++ b/utils/qa_check_herd.py @@ -1,8 +1,10 @@ #! /usr/bin/env python import os, sys +import urllib2 from datetime import datetime -from lxml.etree import parse +from lxml.etree import fromstring +from optparse import OptionParser path = os.path.join(os.path.dirname(__file__), os.path.pardir) sys.path.insert(0, path) @@ -11,6 +13,7 @@ del path from grumpy import app from grumpy.models import db, Herd, PkgIssue, Setting +HERDS_URL = 'http://sources.gentoo.org/cgi-bin/viewvc.cgi/gentoo/xml/htdocs/proj/en/metastructure/herds/herds.xml' PLUGIN_NAME='qa::valid_herd' def gc_collect(timestamp): @@ -45,26 +48,29 @@ def insert_issues(invalid): 'Herd %s is not listed in official herd list.' % herd)) db.session.commit() -def parse_herds_xml(file): +def download_and_parse_herds(): """Return list of herd names from 'herds.xml'""" + data = urllib2.urlopen(HERDS_URL) herds = [] - if not os.path.isfile(file): - print ("File '%s' does not exist" % file) - raise RuntimeError - for child in parse(file).getroot().getchildren(): + for child in fromstring(data.read()).getchildren(): for value in child.getchildren(): if value.tag == 'name': herds.append(value.text) return herds if __name__ == '__main__': - # TODO: Download latest herds file - # Parse list of herds from file - herds = parse_herds_xml('herds.xml') + parser = OptionParser(usage="usage: %prog [options] CONFFILE") + (opts, args) = parser.parse_args() + if len(args) != 1: + parser.error("provide path to configuration file as first argument") + sys.exit(1) + # Fetch data and parse it + herds = download_and_parse_herds() b0rks = [] timestamp = datetime.now() - # Setup database for application with app.test_request_context(): + # Load configuration + app.config.from_pyfile(args[0]) # Fetch list of herds from db for herd in Herd.query.all(): if herd.name in herds: |