Initial commit.

Copied from git+ssh://git@git.gentoo.org/infra/cfengine.git repo as:
timestamp 2015-11-29T21:57:00Z
commit 3b63da8fbbb848d5a1f7e7cd7c6989638ed0d817

No passwords, passphrases or key material is contained herein, but it
may reference the pathes to such.

Reviewed-by: Robin H. Johnson <robbat2@gentoo.org>
Signed-off-by: Robin H. Johnson <robbat2@gentoo.org>

1 files changed, 245 insertions, 0 deletions

diff --git a/gen-report-xml.py b/gen-report-xml.py
new file mode 100755
index 0000000..a1abe72
--- /dev/null
+++ b/gen-report-xml.py
@@ -0,0 +1,245 @@
+#!/usr/bin/env python
+
+from __future__ import unicode_literals
+
+import io
+import os
+import sys
+import time
+from xml.sax.saxutils import escape as escape_xml
+
+def usage():
+	sys.stderr.write(("usage: %s <failure log> <success log> "
+		"<scheduled deletion log> <log date format> <distfiles dir> "
+		"<output xml file> [whitelist dir]\n") % \
+		os.path.basename(sys.argv[0]))
+	sys.stderr.flush()
+
+if len(sys.argv) < 7 or len(sys.argv) > 8:
+	usage()
+	sys.exit(1)
+
+
+if not os.access(sys.argv[1], os.R_OK):
+	sys.stderr.write("couldn't read %s\n" % sys.argv[1])
+	sys.exit(2)
+
+
+fail_f=file(sys.argv[1], "r")
+suc_f=file(sys.argv[2],"r")
+
+if not os.path.isfile(sys.argv[3]):
+	sys.stderr.write("%s isn't a file.\n" % sys.argv[3])
+	sys.exit(3)
+scheduled_deletions_log = sys.argv[3]
+log_date_format = sys.argv[4]
+
+try:
+	outf = io.open(sys.argv[6], mode='w', encoding="utf_8")
+except OSError as e:
+	sys.stderr.write("failed opening out file; lack write "
+		"perms, dir exist? %s\n" % (e,))
+	sys.exit(4)
+
+def grab_whitelists(whitelists_dir):
+	whitelists = {}
+	for x in os.listdir(whitelists_dir):
+		if x[:1] == ".":
+			continue
+		x = os.path.join(whitelists_dir, x)
+		if not os.path.isfile(x):
+			continue
+		whitelists[x] = []
+		with io.open(x, mode='r', encoding='utf_8') as f:
+			for entry in f:
+				entry = entry.lstrip().rstrip()
+				if len(entry) == 0 or entry.startswith("#"):
+					continue
+				whitelists[x].append(entry.lstrip().rstrip())
+
+		if not whitelists[x]:
+			del whitelists[x]
+	return whitelists
+
+if len(sys.argv) >= 8:
+	whitelist = grab_whitelists(sys.argv[7])
+else:
+	whitelist = {}
+
+failed_ebuilds={}
+for l in fail_f:
+	try:
+		cpv, uri, reason = l.split("\t")
+	except ValueError:
+		continue
+	cpv = cpv.lstrip().rstrip()
+	uri = uri.lstrip().rstrip()
+	reason = reason.lstrip().rstrip()
+	#cpv, file, reason
+	failed_ebuilds.setdefault(cpv, []).append((uri, reason))
+
+fail_f.close()
+
+removed_files = {}
+added_files = {}
+for l in suc_f:
+	cpv, file, comment = l.split("\t")
+	if comment.rstrip().lstrip() == "removed":
+		removed_files.setdefault(cpv,[]).append(file)
+	elif comment.lstrip().startswith("added"):
+		added_files.setdefault(cpv,[]).append(file)
+suc_f.close()
+
+if not os.path.isdir(sys.argv[5]):
+	sys.stderr.write("%s isn't a dir, thus isn't distdir.  die.\n" % sys.argv[5])
+	sys.exit(6)
+
+dist_files={}
+for f in os.listdir(sys.argv[5]):
+	dist_files[f] = os.stat(sys.argv[5]+os.path.sep+f).st_size
+
+del_list={}
+del_totals={}
+del_count=0
+t = None
+with io.open(scheduled_deletions_log, mode='r', encoding='utf_8') as f:
+	for line in f:
+		split = line.rstrip().split('\t')
+		if len(split) == 1:
+			try:
+				t = time.strftime("%s",
+					time.strptime(split[0], log_date_format))
+			except ValueError:
+				continue
+			# epoch time.
+			del_totals[t] = 0
+			l = {}
+			del_list[t] = l
+
+		elif len(split) == 3 and not split[0] and t is not None:
+			filename = split[1]
+			cpv = split[2]
+			l.setdefault(cpv, []).append(filename)
+			try:	
+				del_totals[t] += dist_files[filename]
+				del_count += 1
+			except KeyError:
+				pass
+
+cur_ep = time.strftime("%s", time.strptime(time.strftime("%Y-%m-%d",time.gmtime()), "%Y-%m-%d"))
+l=del_list.keys()
+l.sort()
+while len(l) and cur_ep > l[0]:
+	del del_list[l[0]]
+	l.pop(0)
+slated_for_death_total = 0
+for x in l:	slated_for_death_total += del_totals[x]
+
+f_len=max(len(str(len(dist_files.keys()))), len(str(del_count)))
+dist_sum=sum(dist_files.values())
+d_len = max(len(str(dist_sum)), len(str(slated_for_death_total)))
+
+outf.write("""<?xml version='1.0'?>
+<?xml-stylesheet href="/xsl/guide.xsl" type="text/xsl"?>
+<guide link="failure.xml">
+<title>Distfiles Mirroring Changes Report</title>
+<version>1.0</version>
+<date>"""+time.asctime(time.localtime())+"""</date>
+<chapter><title>Master Stats</title><section><body>
+<p>Report was generated on %s</p>
+<impo>Remember the tree could've changed since then.</impo>
+<p>%s bytes: %s files: Mirrored<br/>
+%s bytes: %s files: Slated for deletion<br/>
+Added %i files since beginning of day (UTC)<br/>
+Removed %i files in last run</p></body></section></chapter>""" % \
+(time.strftime("%A %b %d %Y, %H:%M:%S UTC", time.gmtime()), str(dist_sum).rjust(d_len), str(len(dist_files.keys())).rjust(f_len),
+str(slated_for_death_total).rjust(d_len), str(del_count).rjust(f_len), 
+sum([len(x) for x in added_files.values()]), sum([len(x) for x in removed_files.values()])))
+dist_files.clear()
+
+outf.write("""
+<chapter><title>Failed Fetches</title>
+<section><body>
+<impo>Why has a file failed to be fetched?</impo>
+<p>Upstream re-releases, a large failure in upstream mirrors/host, or flat out invalid URL's are typically the cause.  Broken mirror 
+specifications also (mirror:/gentoo is not valid for example, must be mirror://gentoo).  Odd ports for 
+the host can cause issues also (guidelines for ebuilds suggest standard ports for URI hosts).
+</p><p>In other words, there <b>is</b> something wrong with the ebuild referenced, so please check the ebuild.  It's 
+more likely then not broke in some fashion even if the maintainer isn't aware of it.</p>
+
+<table><tr><th>Ebuild</th><th>File</th><th>Reason</th></tr>\n""")
+
+#quote stripping is needed.
+
+esort = failed_ebuilds.keys()
+esort.sort()
+for cpv in esort:
+	for f,r in failed_ebuilds[cpv]:
+		outf.write("<tr><ti>%s</ti><ti>%s</ti><ti>%s</ti></tr>\n" % (escape_xml(cpv), escape_xml(f), escape_xml(r)))
+outf.write("</table>\n")
+
+
+outf.write("""</body></section></chapter>
+<chapter><title>Files Added</title><section><body><table><tr><th>Ebuild</th><th>File</th></tr>\n""")
+
+added_s = added_files.keys()
+added_s.sort()
+for cpv in added_s:
+	added_files[cpv].sort()
+	for f in added_files[cpv]:
+		outf.write("<tr><ti>%s</ti><ti>%s</ti></tr>\n" % (escape_xml(cpv), escape_xml(f)))
+
+outf.write("""</table></body></section></chapter>
+<chapter><title>Files Removed</title><section><body><table><tr><th>Ebuild</th><th>File</th></tr>\n\n""")
+
+rem_s = removed_files.keys()
+rem_s.sort()
+for cpv in rem_s:
+	removed_files[cpv].sort()
+	count = 0
+	for f in removed_files[cpv]:
+		if count == 0:
+			outf.write(""" <tr><ti>%s</ti>""" % escape_xml(cpv))
+			count+=1
+		else:
+			outf.write(""" <tr><ti></ti>""")
+		outf.write("""<ti>%s</ti></tr>\n""" % escape_xml(f))
+
+outf.write("""</table></body></section></chapter>  
+<chapter><title>Scheduled Deletions</title><section><body>\n<ul>""")
+
+l=del_list.keys()
+l.sort()
+for date in l:
+	outf.write("""  <li><uri link="#%s">%s</uri>, %i files for %i bytes</li>\n""" % (date, time.strftime("%B %d %Y", 
+	time.gmtime(float(date))), sum([len(x) for x in del_list[date].values()]), del_totals[date]))
+outf.write("</ul></body></section>\n")
+
+for date in l:
+	outf.write("""<section id="%s"><title>Deletions for %s</title>
+<body><table><tr><th>Ebuild (If known)</th><th>File</th></tr>\n""" %  (date, time.strftime("%A %B %d %Y", time.gmtime(float(date)))))
+	cpvs = del_list[date].keys()
+	cpvs.sort()
+	for cpv in cpvs:
+		l=del_list[date][cpv]
+		l.sort()
+		outf.write("  <tr><ti>%s</ti><ti>%s</ti></tr>\n" % (escape_xml(cpv), escape_xml(l[0])))
+		l.pop(0)
+		for f in l:
+			outf.write("  <tr><ti></ti><ti>%s</ti></tr>\n" % escape_xml(f))
+	outf.write("</table>\n</body>\n</section>\n")
+outf.write("</chapter>")
+outf.write("<chapter><title>White Lists</title>")
+if not whitelist:
+	outf.write("<section><body><p><uri link=\"whitelists.xml\">Whitelist Report</uri></p></body></section>")
+	#outf.write("<section><body><p>No whitelists.</p></body></section>")
+l=whitelist.keys()
+l.sort()
+for x in l:
+	outf.write("<section><title>%s</title><body><ul>\n" % os.path.basename(x))
+	whitelist[x].sort()
+	for y in whitelist[x]:
+		outf.write("  <li>%s</li>\n" % escape_xml(y))
+	outf.write("</ul></body></section>\n")
+outf.write("</chapter></guide>")
+outf.close()