diff options
author | Robin H. Johnson <robbat2@gentoo.org> | 2015-11-29 15:13:11 -0800 |
---|---|---|
committer | Robin H. Johnson <robbat2@gentoo.org> | 2015-11-29 15:13:11 -0800 |
commit | a88756a60a04f469139077268338c10af29e85df (patch) | |
tree | f7462e6ee33af2f9fa7ac0191f1a21190e309469 /gen-report-xml.py | |
download | mastermirror-scripts-a88756a60a04f469139077268338c10af29e85df.tar.gz mastermirror-scripts-a88756a60a04f469139077268338c10af29e85df.tar.bz2 mastermirror-scripts-a88756a60a04f469139077268338c10af29e85df.zip |
Initial commit.
Copied from git+ssh://git@git.gentoo.org/infra/cfengine.git repo as:
timestamp 2015-11-29T21:57:00Z
commit 3b63da8fbbb848d5a1f7e7cd7c6989638ed0d817
No passwords, passphrases or key material is contained herein, but it
may reference the pathes to such.
Reviewed-by: Robin H. Johnson <robbat2@gentoo.org>
Signed-off-by: Robin H. Johnson <robbat2@gentoo.org>
Diffstat (limited to 'gen-report-xml.py')
-rwxr-xr-x | gen-report-xml.py | 245 |
1 files changed, 245 insertions, 0 deletions
diff --git a/gen-report-xml.py b/gen-report-xml.py new file mode 100755 index 0000000..a1abe72 --- /dev/null +++ b/gen-report-xml.py @@ -0,0 +1,245 @@ +#!/usr/bin/env python + +from __future__ import unicode_literals + +import io +import os +import sys +import time +from xml.sax.saxutils import escape as escape_xml + +def usage(): + sys.stderr.write(("usage: %s <failure log> <success log> " + "<scheduled deletion log> <log date format> <distfiles dir> " + "<output xml file> [whitelist dir]\n") % \ + os.path.basename(sys.argv[0])) + sys.stderr.flush() + +if len(sys.argv) < 7 or len(sys.argv) > 8: + usage() + sys.exit(1) + + +if not os.access(sys.argv[1], os.R_OK): + sys.stderr.write("couldn't read %s\n" % sys.argv[1]) + sys.exit(2) + + +fail_f=file(sys.argv[1], "r") +suc_f=file(sys.argv[2],"r") + +if not os.path.isfile(sys.argv[3]): + sys.stderr.write("%s isn't a file.\n" % sys.argv[3]) + sys.exit(3) +scheduled_deletions_log = sys.argv[3] +log_date_format = sys.argv[4] + +try: + outf = io.open(sys.argv[6], mode='w', encoding="utf_8") +except OSError as e: + sys.stderr.write("failed opening out file; lack write " + "perms, dir exist? %s\n" % (e,)) + sys.exit(4) + +def grab_whitelists(whitelists_dir): + whitelists = {} + for x in os.listdir(whitelists_dir): + if x[:1] == ".": + continue + x = os.path.join(whitelists_dir, x) + if not os.path.isfile(x): + continue + whitelists[x] = [] + with io.open(x, mode='r', encoding='utf_8') as f: + for entry in f: + entry = entry.lstrip().rstrip() + if len(entry) == 0 or entry.startswith("#"): + continue + whitelists[x].append(entry.lstrip().rstrip()) + + if not whitelists[x]: + del whitelists[x] + return whitelists + +if len(sys.argv) >= 8: + whitelist = grab_whitelists(sys.argv[7]) +else: + whitelist = {} + +failed_ebuilds={} +for l in fail_f: + try: + cpv, uri, reason = l.split("\t") + except ValueError: + continue + cpv = cpv.lstrip().rstrip() + uri = uri.lstrip().rstrip() + reason = reason.lstrip().rstrip() + #cpv, file, reason + failed_ebuilds.setdefault(cpv, []).append((uri, reason)) + +fail_f.close() + +removed_files = {} +added_files = {} +for l in suc_f: + cpv, file, comment = l.split("\t") + if comment.rstrip().lstrip() == "removed": + removed_files.setdefault(cpv,[]).append(file) + elif comment.lstrip().startswith("added"): + added_files.setdefault(cpv,[]).append(file) +suc_f.close() + +if not os.path.isdir(sys.argv[5]): + sys.stderr.write("%s isn't a dir, thus isn't distdir. die.\n" % sys.argv[5]) + sys.exit(6) + +dist_files={} +for f in os.listdir(sys.argv[5]): + dist_files[f] = os.stat(sys.argv[5]+os.path.sep+f).st_size + +del_list={} +del_totals={} +del_count=0 +t = None +with io.open(scheduled_deletions_log, mode='r', encoding='utf_8') as f: + for line in f: + split = line.rstrip().split('\t') + if len(split) == 1: + try: + t = time.strftime("%s", + time.strptime(split[0], log_date_format)) + except ValueError: + continue + # epoch time. + del_totals[t] = 0 + l = {} + del_list[t] = l + + elif len(split) == 3 and not split[0] and t is not None: + filename = split[1] + cpv = split[2] + l.setdefault(cpv, []).append(filename) + try: + del_totals[t] += dist_files[filename] + del_count += 1 + except KeyError: + pass + +cur_ep = time.strftime("%s", time.strptime(time.strftime("%Y-%m-%d",time.gmtime()), "%Y-%m-%d")) +l=del_list.keys() +l.sort() +while len(l) and cur_ep > l[0]: + del del_list[l[0]] + l.pop(0) +slated_for_death_total = 0 +for x in l: slated_for_death_total += del_totals[x] + +f_len=max(len(str(len(dist_files.keys()))), len(str(del_count))) +dist_sum=sum(dist_files.values()) +d_len = max(len(str(dist_sum)), len(str(slated_for_death_total))) + +outf.write("""<?xml version='1.0'?> +<?xml-stylesheet href="/xsl/guide.xsl" type="text/xsl"?> +<guide link="failure.xml"> +<title>Distfiles Mirroring Changes Report</title> +<version>1.0</version> +<date>"""+time.asctime(time.localtime())+"""</date> +<chapter><title>Master Stats</title><section><body> +<p>Report was generated on %s</p> +<impo>Remember the tree could've changed since then.</impo> +<p>%s bytes: %s files: Mirrored<br/> +%s bytes: %s files: Slated for deletion<br/> +Added %i files since beginning of day (UTC)<br/> +Removed %i files in last run</p></body></section></chapter>""" % \ +(time.strftime("%A %b %d %Y, %H:%M:%S UTC", time.gmtime()), str(dist_sum).rjust(d_len), str(len(dist_files.keys())).rjust(f_len), +str(slated_for_death_total).rjust(d_len), str(del_count).rjust(f_len), +sum([len(x) for x in added_files.values()]), sum([len(x) for x in removed_files.values()]))) +dist_files.clear() + +outf.write(""" +<chapter><title>Failed Fetches</title> +<section><body> +<impo>Why has a file failed to be fetched?</impo> +<p>Upstream re-releases, a large failure in upstream mirrors/host, or flat out invalid URL's are typically the cause. Broken mirror +specifications also (mirror:/gentoo is not valid for example, must be mirror://gentoo). Odd ports for +the host can cause issues also (guidelines for ebuilds suggest standard ports for URI hosts). +</p><p>In other words, there <b>is</b> something wrong with the ebuild referenced, so please check the ebuild. It's +more likely then not broke in some fashion even if the maintainer isn't aware of it.</p> + +<table><tr><th>Ebuild</th><th>File</th><th>Reason</th></tr>\n""") + +#quote stripping is needed. + +esort = failed_ebuilds.keys() +esort.sort() +for cpv in esort: + for f,r in failed_ebuilds[cpv]: + outf.write("<tr><ti>%s</ti><ti>%s</ti><ti>%s</ti></tr>\n" % (escape_xml(cpv), escape_xml(f), escape_xml(r))) +outf.write("</table>\n") + + +outf.write("""</body></section></chapter> +<chapter><title>Files Added</title><section><body><table><tr><th>Ebuild</th><th>File</th></tr>\n""") + +added_s = added_files.keys() +added_s.sort() +for cpv in added_s: + added_files[cpv].sort() + for f in added_files[cpv]: + outf.write("<tr><ti>%s</ti><ti>%s</ti></tr>\n" % (escape_xml(cpv), escape_xml(f))) + +outf.write("""</table></body></section></chapter> +<chapter><title>Files Removed</title><section><body><table><tr><th>Ebuild</th><th>File</th></tr>\n\n""") + +rem_s = removed_files.keys() +rem_s.sort() +for cpv in rem_s: + removed_files[cpv].sort() + count = 0 + for f in removed_files[cpv]: + if count == 0: + outf.write(""" <tr><ti>%s</ti>""" % escape_xml(cpv)) + count+=1 + else: + outf.write(""" <tr><ti></ti>""") + outf.write("""<ti>%s</ti></tr>\n""" % escape_xml(f)) + +outf.write("""</table></body></section></chapter> +<chapter><title>Scheduled Deletions</title><section><body>\n<ul>""") + +l=del_list.keys() +l.sort() +for date in l: + outf.write(""" <li><uri link="#%s">%s</uri>, %i files for %i bytes</li>\n""" % (date, time.strftime("%B %d %Y", + time.gmtime(float(date))), sum([len(x) for x in del_list[date].values()]), del_totals[date])) +outf.write("</ul></body></section>\n") + +for date in l: + outf.write("""<section id="%s"><title>Deletions for %s</title> +<body><table><tr><th>Ebuild (If known)</th><th>File</th></tr>\n""" % (date, time.strftime("%A %B %d %Y", time.gmtime(float(date))))) + cpvs = del_list[date].keys() + cpvs.sort() + for cpv in cpvs: + l=del_list[date][cpv] + l.sort() + outf.write(" <tr><ti>%s</ti><ti>%s</ti></tr>\n" % (escape_xml(cpv), escape_xml(l[0]))) + l.pop(0) + for f in l: + outf.write(" <tr><ti></ti><ti>%s</ti></tr>\n" % escape_xml(f)) + outf.write("</table>\n</body>\n</section>\n") +outf.write("</chapter>") +outf.write("<chapter><title>White Lists</title>") +if not whitelist: + outf.write("<section><body><p><uri link=\"whitelists.xml\">Whitelist Report</uri></p></body></section>") + #outf.write("<section><body><p>No whitelists.</p></body></section>") +l=whitelist.keys() +l.sort() +for x in l: + outf.write("<section><title>%s</title><body><ul>\n" % os.path.basename(x)) + whitelist[x].sort() + for y in whitelist[x]: + outf.write(" <li>%s</li>\n" % escape_xml(y)) + outf.write("</ul></body></section>\n") +outf.write("</chapter></guide>") +outf.close() |