diff options
| -rw-r--r-- | README | 3 | ||||
| -rwxr-xr-x | clean-old-deltas.py | 49 | ||||
| -rwxr-xr-x | datestamp.sh | 18 | ||||
| -rwxr-xr-x | gen-report-whitelist-xml.py | 69 | ||||
| -rwxr-xr-x | gen-report-xml.py | 245 | ||||
| -rwxr-xr-x | generate-distfiles-reports.sh | 13 | ||||
| -rwxr-xr-x | rsync-gen.sh | 258 | ||||
| -rw-r--r-- | rsync-gen.vars | 23 | ||||
| -rwxr-xr-x | sign-autobuilds.sh | 62 | ||||
| -rwxr-xr-x | snapshots-create.sh | 158 | ||||
| -rwxr-xr-x | sync-autobuilds.sh | 35 | ||||
| -rwxr-xr-x | sync-distfiles.sh | 53 | ||||
| -rwxr-xr-x | sync-experimental.sh | 14 | ||||
| -rwxr-xr-x | sync-masterdistfiles-mirror.sh | 10 | ||||
| -rw-r--r-- | thicken-manifests.py | 150 | ||||
| -rwxr-xr-x | timestamp-releases.sh | 16 | ||||
| -rwxr-xr-x | timestamp-rsync.sh | 7 |
17 files changed, 1183 insertions, 0 deletions
@@ -0,0 +1,3 @@ +Scripts for master{rsync,releases,snap,distfiles,portage} go in this directory. +It is not harmful to install the scripts on hosts that don't exactly need them +and it is nicer to have all the mirroring scripts in the same place. diff --git a/clean-old-deltas.py b/clean-old-deltas.py new file mode 100755 index 0000000..5f7e6b6 --- /dev/null +++ b/clean-old-deltas.py @@ -0,0 +1,49 @@ +#!/usr/bin/python + +import os,sys +if len(sys.argv) != 4: + print "need 3 args, first being directory to scan, second being latest, third being it's size" + sys.exit(1) + +path=sys.argv[1] +maxsize=long(sys.argv[3]) +if len(sys.argv[2]) != 8: + print "date arg must be YYYYMMDD" + sys.exit(1) +root = sys.argv[2] + +if not hasattr(__builtins__, "set"): + from sets import Set as set +file_list = [x for x in os.listdir(path) if os.path.isfile(os.path.join(path, x))] +pref_name="snapshot-" +post_name=".patch.bz2" +deltas = [x for x in file_list if x.startswith(pref_name) and x.endswith(post_name)] +chains={} +for x in deltas: + try: + start, end = x[len(pref_name):-len(post_name)].split("-") + except ValueError: + continue + if end in chains: + print "warning, end %s already exists; this is a bit weird" % end + continue + chains[end] = start + +save_files = set() +node = root +count = 0 +while node in chains: + s="%s%s-%s%s" % (pref_name, chains[node], node, post_name) + count += os.stat(os.path.join(path, s)).st_size + if count > maxsize: + break + save_files.update([s, s+".md5sum", s+".gpgsig"]) + node = chains[node] + +for x in file_list: + if x not in save_files: + print "nuking %s" % x + try: + os.unlink(os.path.join(path,x)) + except OSError, oe: + print "failed removing %s cause of %s" % (os.path.join(path, x), str(oe)) diff --git a/datestamp.sh b/datestamp.sh new file mode 100755 index 0000000..de6ef9a --- /dev/null +++ b/datestamp.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# distributed by cfengine + +umask 022 # ensure globally readable perms on new files for rsyncd + +source /usr/local/bin/mastermirror/rsync-gen.vars + +# The only time ${STAGEDIR}/metadata should not exist is when rsync-gen.sh +# hasn't ran even once yet +if [[ -d ${STAGEDIR}/metadata ]]; then + /bin/date -u '+%s %c' > ${STAGEDIR}/metadata/timestamp.x.tmp + /bin/mv ${STAGEDIR}/metadata/timestamp.x.tmp ${STAGEDIR}/metadata/timestamp.x +fi + +if [[ -d ${FINALDIR}/metadata ]]; then + /bin/date -R -u > ${FINALDIR}/metadata/timestamp.chk.tmp + /bin/mv ${FINALDIR}/metadata/timestamp.chk.tmp ${FINALDIR}/metadata/timestamp.chk +fi diff --git a/gen-report-whitelist-xml.py b/gen-report-whitelist-xml.py new file mode 100755 index 0000000..8997e53 --- /dev/null +++ b/gen-report-whitelist-xml.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python + +from __future__ import unicode_literals + +import io +import os +import sys +import time +from xml.sax.saxutils import escape as escape_xml + +def grab_whitelists(whitelists_dir): + whitelists = {} + for x in os.listdir(whitelists_dir): + if x[:1] == ".": + continue + x = os.path.join(whitelists_dir, x) + if not os.path.isfile(x): + continue + whitelists[x] = [] + with io.open(x, mode='r', encoding='utf_8') as f: + for entry in f: + entry = entry.lstrip().rstrip() + if len(entry) == 0 or entry.startswith("#"): + continue + whitelists[x].append(entry.lstrip().rstrip()) + + if not whitelists[x]: + del whitelists[x] + return whitelists + +def write_report(whitelists, outf): + outf.write("""<?xml version='1.0'?> + <?xml-stylesheet href="/xsl/guide.xsl" type="text/xsl"?> + <guide link="failure.xml"> + <title>Distfiles Mirroring Whitelist Report</title> + <version>1.0</version> + <date>"""+time.asctime(time.localtime())+"""</date> + """) + outf.write("<chapter><title>White Lists</title>") + if not whitelists: + outf.write("<section><body><p>No whitelists.</p></body></section>") + for x in sorted(whitelists): + outf.write("<section><title>%s</title><body><ul>\n" % os.path.basename(x)) + whitelists[x].sort() + for y in whitelists[x]: + outf.write(" <li>%s</li>\n" % escape_xml(y)) + outf.write("</ul></body></section>\n") + outf.write("</chapter></guide>") + outf.close() + +def usage(): + sys.stderr.write("usage: %s <whitelists dir> <output file>\n" % \ + os.path.basename(sys.argv[0])) + sys.stderr.flush() + +def main(argv): + if len(argv) != 3 or \ + not os.path.isdir(argv[1]) or \ + not os.path.isdir(os.path.dirname(argv[2])) or \ + not os.access(os.path.dirname(argv[2]), os.W_OK): + usage() + return 1 + with io.open(argv[2], mode='w', encoding="utf_8") as outf: + whitelists = grab_whitelists(argv[1]) + write_report(whitelists, outf) + return os.EX_OK + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/gen-report-xml.py b/gen-report-xml.py new file mode 100755 index 0000000..a1abe72 --- /dev/null +++ b/gen-report-xml.py @@ -0,0 +1,245 @@ +#!/usr/bin/env python + +from __future__ import unicode_literals + +import io +import os +import sys +import time +from xml.sax.saxutils import escape as escape_xml + +def usage(): + sys.stderr.write(("usage: %s <failure log> <success log> " + "<scheduled deletion log> <log date format> <distfiles dir> " + "<output xml file> [whitelist dir]\n") % \ + os.path.basename(sys.argv[0])) + sys.stderr.flush() + +if len(sys.argv) < 7 or len(sys.argv) > 8: + usage() + sys.exit(1) + + +if not os.access(sys.argv[1], os.R_OK): + sys.stderr.write("couldn't read %s\n" % sys.argv[1]) + sys.exit(2) + + +fail_f=file(sys.argv[1], "r") +suc_f=file(sys.argv[2],"r") + +if not os.path.isfile(sys.argv[3]): + sys.stderr.write("%s isn't a file.\n" % sys.argv[3]) + sys.exit(3) +scheduled_deletions_log = sys.argv[3] +log_date_format = sys.argv[4] + +try: + outf = io.open(sys.argv[6], mode='w', encoding="utf_8") +except OSError as e: + sys.stderr.write("failed opening out file; lack write " + "perms, dir exist? %s\n" % (e,)) + sys.exit(4) + +def grab_whitelists(whitelists_dir): + whitelists = {} + for x in os.listdir(whitelists_dir): + if x[:1] == ".": + continue + x = os.path.join(whitelists_dir, x) + if not os.path.isfile(x): + continue + whitelists[x] = [] + with io.open(x, mode='r', encoding='utf_8') as f: + for entry in f: + entry = entry.lstrip().rstrip() + if len(entry) == 0 or entry.startswith("#"): + continue + whitelists[x].append(entry.lstrip().rstrip()) + + if not whitelists[x]: + del whitelists[x] + return whitelists + +if len(sys.argv) >= 8: + whitelist = grab_whitelists(sys.argv[7]) +else: + whitelist = {} + +failed_ebuilds={} +for l in fail_f: + try: + cpv, uri, reason = l.split("\t") + except ValueError: + continue + cpv = cpv.lstrip().rstrip() + uri = uri.lstrip().rstrip() + reason = reason.lstrip().rstrip() + #cpv, file, reason + failed_ebuilds.setdefault(cpv, []).append((uri, reason)) + +fail_f.close() + +removed_files = {} +added_files = {} +for l in suc_f: + cpv, file, comment = l.split("\t") + if comment.rstrip().lstrip() == "removed": + removed_files.setdefault(cpv,[]).append(file) + elif comment.lstrip().startswith("added"): + added_files.setdefault(cpv,[]).append(file) +suc_f.close() + +if not os.path.isdir(sys.argv[5]): + sys.stderr.write("%s isn't a dir, thus isn't distdir. die.\n" % sys.argv[5]) + sys.exit(6) + +dist_files={} +for f in os.listdir(sys.argv[5]): + dist_files[f] = os.stat(sys.argv[5]+os.path.sep+f).st_size + +del_list={} +del_totals={} +del_count=0 +t = None +with io.open(scheduled_deletions_log, mode='r', encoding='utf_8') as f: + for line in f: + split = line.rstrip().split('\t') + if len(split) == 1: + try: + t = time.strftime("%s", + time.strptime(split[0], log_date_format)) + except ValueError: + continue + # epoch time. + del_totals[t] = 0 + l = {} + del_list[t] = l + + elif len(split) == 3 and not split[0] and t is not None: + filename = split[1] + cpv = split[2] + l.setdefault(cpv, []).append(filename) + try: + del_totals[t] += dist_files[filename] + del_count += 1 + except KeyError: + pass + +cur_ep = time.strftime("%s", time.strptime(time.strftime("%Y-%m-%d",time.gmtime()), "%Y-%m-%d")) +l=del_list.keys() +l.sort() +while len(l) and cur_ep > l[0]: + del del_list[l[0]] + l.pop(0) +slated_for_death_total = 0 +for x in l: slated_for_death_total += del_totals[x] + +f_len=max(len(str(len(dist_files.keys()))), len(str(del_count))) +dist_sum=sum(dist_files.values()) +d_len = max(len(str(dist_sum)), len(str(slated_for_death_total))) + +outf.write("""<?xml version='1.0'?> +<?xml-stylesheet href="/xsl/guide.xsl" type="text/xsl"?> +<guide link="failure.xml"> +<title>Distfiles Mirroring Changes Report</title> +<version>1.0</version> +<date>"""+time.asctime(time.localtime())+"""</date> +<chapter><title>Master Stats</title><section><body> +<p>Report was generated on %s</p> +<impo>Remember the tree could've changed since then.</impo> +<p>%s bytes: %s files: Mirrored<br/> +%s bytes: %s files: Slated for deletion<br/> +Added %i files since beginning of day (UTC)<br/> +Removed %i files in last run</p></body></section></chapter>""" % \ +(time.strftime("%A %b %d %Y, %H:%M:%S UTC", time.gmtime()), str(dist_sum).rjust(d_len), str(len(dist_files.keys())).rjust(f_len), +str(slated_for_death_total).rjust(d_len), str(del_count).rjust(f_len), +sum([len(x) for x in added_files.values()]), sum([len(x) for x in removed_files.values()]))) +dist_files.clear() + +outf.write(""" +<chapter><title>Failed Fetches</title> +<section><body> +<impo>Why has a file failed to be fetched?</impo> +<p>Upstream re-releases, a large failure in upstream mirrors/host, or flat out invalid URL's are typically the cause. Broken mirror +specifications also (mirror:/gentoo is not valid for example, must be mirror://gentoo). Odd ports for +the host can cause issues also (guidelines for ebuilds suggest standard ports for URI hosts). +</p><p>In other words, there <b>is</b> something wrong with the ebuild referenced, so please check the ebuild. It's +more likely then not broke in some fashion even if the maintainer isn't aware of it.</p> + +<table><tr><th>Ebuild</th><th>File</th><th>Reason</th></tr>\n""") + +#quote stripping is needed. + +esort = failed_ebuilds.keys() +esort.sort() +for cpv in esort: + for f,r in failed_ebuilds[cpv]: + outf.write("<tr><ti>%s</ti><ti>%s</ti><ti>%s</ti></tr>\n" % (escape_xml(cpv), escape_xml(f), escape_xml(r))) +outf.write("</table>\n") + + +outf.write("""</body></section></chapter> +<chapter><title>Files Added</title><section><body><table><tr><th>Ebuild</th><th>File</th></tr>\n""") + +added_s = added_files.keys() +added_s.sort() +for cpv in added_s: + added_files[cpv].sort() + for f in added_files[cpv]: + outf.write("<tr><ti>%s</ti><ti>%s</ti></tr>\n" % (escape_xml(cpv), escape_xml(f))) + +outf.write("""</table></body></section></chapter> +<chapter><title>Files Removed</title><section><body><table><tr><th>Ebuild</th><th>File</th></tr>\n\n""") + +rem_s = removed_files.keys() +rem_s.sort() +for cpv in rem_s: + removed_files[cpv].sort() + count = 0 + for f in removed_files[cpv]: + if count == 0: + outf.write(""" <tr><ti>%s</ti>""" % escape_xml(cpv)) + count+=1 + else: + outf.write(""" <tr><ti></ti>""") + outf.write("""<ti>%s</ti></tr>\n""" % escape_xml(f)) + +outf.write("""</table></body></section></chapter> +<chapter><title>Scheduled Deletions</title><section><body>\n<ul>""") + +l=del_list.keys() +l.sort() +for date in l: + outf.write(""" <li><uri link="#%s">%s</uri>, %i files for %i bytes</li>\n""" % (date, time.strftime("%B %d %Y", + time.gmtime(float(date))), sum([len(x) for x in del_list[date].values()]), del_totals[date])) +outf.write("</ul></body></section>\n") + +for date in l: + outf.write("""<section id="%s"><title>Deletions for %s</title> +<body><table><tr><th>Ebuild (If known)</th><th>File</th></tr>\n""" % (date, time.strftime("%A %B %d %Y", time.gmtime(float(date))))) + cpvs = del_list[date].keys() + cpvs.sort() + for cpv in cpvs: + l=del_list[date][cpv] + l.sort() + outf.write(" <tr><ti>%s</ti><ti>%s</ti></tr>\n" % (escape_xml(cpv), escape_xml(l[0]))) + l.pop(0) + for f in l: + outf.write(" <tr><ti></ti><ti>%s</ti></tr>\n" % escape_xml(f)) + outf.write("</table>\n</body>\n</section>\n") +outf.write("</chapter>") +outf.write("<chapter><title>White Lists</title>") +if not whitelist: + outf.write("<section><body><p><uri link=\"whitelists.xml\">Whitelist Report</uri></p></body></section>") + #outf.write("<section><body><p>No whitelists.</p></body></section>") +l=whitelist.keys() +l.sort() +for x in l: + outf.write("<section><title>%s</title><body><ul>\n" % os.path.basename(x)) + whitelist[x].sort() + for y in whitelist[x]: + outf.write(" <li>%s</li>\n" % escape_xml(y)) + outf.write("</ul></body></section>\n") +outf.write("</chapter></guide>") +outf.close() diff --git a/generate-distfiles-reports.sh b/generate-distfiles-reports.sh new file mode 100755 index 0000000..a205b65 --- /dev/null +++ b/generate-distfiles-reports.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +DATADIR="/data/gmirror-distfiles" +SCRIPTDIR="/usr/local/bin/mastermirror/" +LOGDIR="${DATADIR}/log" +REPORTS="${DATADIR}/reports" +[[ -d ${REPORTS} ]] || mkdir $REPORTS + +${SCRIPTDIR}/gen-report-xml.py ${LOGDIR}/failure.log \ + ${LOGDIR}/success.log ${LOGDIR}/deletion.log '%Y-%m-%d' \ + ${DATADIR}/distfiles ${REPORTS}/failure.xml +${SCRIPTDIR}/gen-report-whitelist-xml.py ${DATADIR}/distfiles-whitelist \ + ${REPORTS}/whitelists.xml diff --git a/rsync-gen.sh b/rsync-gen.sh new file mode 100755 index 0000000..ad7750e --- /dev/null +++ b/rsync-gen.sh @@ -0,0 +1,258 @@ +#!/bin/bash +# distributed by cfengine + +# 0) Make a backup incase there is something wrong with server side +# 1) Update rsync checkouts or exports from cvs.g.o::mastermirror-staging +# 1b) rsync EXPORTS/gentoo-x86 to STAGEDIR +# 1c) source (bash -n) eclasses to check for syntax errors +# 2) generate metadata (egencache) +# 3) place dtd info in STAGEDIR +# 4) place glsa's in STAGEDIR +# 5) place news in STAGEDIR +# 6) place herds.xml in STAGEDIR +# 7) rsync from STAGEDIR to FINALDIR + +umask 022 +source /usr/local/bin/mastermirror/rsync-gen.vars + +# We want to exclude many RCS files, but 'core' is actually a valid package +# name that we do use for dev-ml/core. +RSYNC_GIT_EXCLUDE="--include core/ --cvs-exclude --exclude=.gitignore" +#--filter='dir-merge /.cvsignore' --filter='exclude .cvsignore' +#--filter='dir-merge /.gitignore' --filter='exclude .gitignore' + +# sanity checking +[[ -d ${EXPORTS} ]] || mkdir ${EXPORTS} +[[ -d ${STAGEDIR} ]] || mkdir ${STAGEDIR} +[[ -d ${FINALDIR} ]] || mkdir ${FINALDIR} +[[ -d ${LOGDIR} ]] || mkdir ${LOGDIR} +[[ -d ${BASE}/tmp ]] || mkdir ${BASE}/tmp +TIMESLOG="${LOGDIR}/rsync-gen-times.log" + +echo "---------------------------------------------------------" >> ${TIMESLOG} +echo "START ENTIRE SCRIPT $(date -u)" >> ${TIMESLOG} + +# 0) Make a backup incase there is something wrong with server side +#echo "START BACKUP $(date -u)" >> ${TIMESLOG} +#cp -al ${FINALDIR}/ ${FINALDIR}.bak/ +#echo "END BACKUP $(date -u)" >> ${TIMESLOG} + +# 1) Update rsync checkouts or exports from cvs.g.o::mastermirror-staging +PASSWD_FILE="/etc/mastermirror-fetch/gcvsd-rsync.rsync.passwd" +RSYNC="/usr/bin/rsync" +RSYNC_ARGS="--no-motd --recursive --times --links --port=60024 --password-file ${PASSWD_FILE} --ignore-errors --timeout=300 --checksum" +RSYNC_ARGS="${RSYNC_ARGS} --quiet" +RSYNC_ARGS_DELETE="--delete --delete-after --delete-excluded" + +echo "START STAGING RSYNC $(date -u)" >> ${TIMESLOG} +${RSYNC} ${RSYNC_ARGS} ${RSYNC_ARGS_DELETE} git.gentoo.org::mastermirror-staging/ ${EXPORTS}/ +echo "END STAGING RSYNC $(date -u)" >> ${TIMESLOG} +# end 1) + +# 1b) rsync EXPORTS/gentoo-x86 to STAGEDIR + +# timestamp.chk is generated every 5 mins in datestamp.sh script in the FINALDIR +# timestamp.x is generated every 5 mins in datestamp.sh script in the STAGEDIR +# We want to track the progress of STAGEDIR->FINALDIR so exclude .x file here +# (so it isn't deleted) +# Keep /metadata/cache around so the --rsync switch of egencache will work as +# designed +echo "START STAGEDIR RSYNC $(date -u)" >> ${TIMESLOG} +rsync -Wqa \ + $RSYNC_GIT_EXCLUDE \ + --exclude=/metadata/timestamp.x \ + --exclude=/metadata/md5-cache \ + --exclude=ChangeLog \ + --delete \ + ${EXPORTS}/gentoo-x86/ ${STAGEDIR}/ || exit +echo "END STAGEDIR RSYNC $(date -u)" >> ${TIMESLOG} +# end 1b) + +# 1c) source (bash -n) eclasses to check for syntax errors +echo "START ECLASS CHECK $(date -u)" >> ${TIMESLOG} +for i in ${STAGEDIR}/eclass/*.eclass; do + bash -n $i || { echo "failed to source $i, exiting"; exit 1; } +done +echo "END ECLASS CHECK $(date -u)" >> ${TIMESLOG} +# end 1c) + +# start 1d) +# Temporary: import old CVS changelogs +echo "START CHANGELOG-IMPORT $(date -u)" >> ${TIMESLOG} +for file in $EXPORTS/changelogs/*/*/ChangeLog*; do + # remove the prefix so we can copy + file=${file#$EXPORTS/changelogs/} + dir=$(dirname "$file") + # do the copy, the stderr redirect is for changelogs that no longer have an ebuild + # (which is why rsync is not used, as it would create a directory/package when none should exist) + [ -d "${STAGEDIR}/${dir}" ] && cp -a $EXPORTS/changelogs/${file} ${STAGEDIR}/${file} +done +echo "END CHANGELOG-IMPORT $(date -u)" >> ${TIMESLOG} +# end 1d) + +# 2) generate metadata (egencache) +# ${STAGEDIR}/metadata/cache is created automatically +export PORTAGE_USERNAME=gmirror PORTAGE_GRPNAME=gmirror +LOG_TIMESTAMP=$(date -u '+%Y%m%d-%H%M') +REGEN_LOG_FILE=regen-run-${LOG_TIMESTAMP}.log +REGEN_LOG_DIR=${LOGDIR}/regen +[[ -d ${REGEN_LOG_DIR} ]] || mkdir ${REGEN_LOG_DIR} + +function parallel_repoman_manifest_THICKEN() { + /usr/local/bin/mastermirror/thicken-manifests.py \ + ${PARALLEL_PARAMS_noloadavg} \ + "${STAGEDIR}" +} + +function parallel_repoman_manifest_REPOMAN() { + find ${STAGEDIR} \ + -maxdepth 1 -mindepth 1 \ + -type d \ + ! -name eclass \ + ! -name scripts \ + ! -name licenses \ + ! -name metadata \ + ! -name profiles \ + | parallel --no-notice \ + ${PARALLEL_PARAMS/--load-average/--load} \ + 'cd {} && repoman manifest' \ + >>${REGEN_LOG_DIR}/${REGEN_LOG_FILE} +} + +# TODO: Implement a monitoring system that guarantees low-latency human +# intervention when necessary, and stop using --tolerant (bug #239266). + +echo "START REGEN $(date -u)" >> ${TIMESLOG} + +# Force Manifests to thick, unsigned +# also disable commit signing for now. +# TODO: add infra signing of Manifests +sed -i \ + -e '/^thin-manifests/s,true,false,g' \ + -e '/^sign-manifests/s,true,false,g' \ + -e '/^sign-commits/s,true,false,g' \ + ${STAGEDIR}/metadata/layout.conf +parallel_repoman_manifest_REPOMAN + +# Only update the changelogs every 6 hours +# because right now it takes a very long time to do it (exceeding 1 hour at times). +HOURS=$(date +%H) +EGENCACHE_CHANGELOG="" +case $HOURS in + 3|9|15|21) EGENCACHE_CHANGELOG="--update-changelogs --changelog-reversed --changelog-output ChangeLog" ;; +esac + +# for egencache, set user/group or make sure the user is in the portage group + #--update-changelogs \ +GIT_DIR=${EXPORTS}/gentoo-x86/.git/ \ +egencache --update --rsync $PARALLEL_PARAMS \ + --tolerant --cache-dir=${BASE}/tmp/ \ + --portdir=${STAGEDIR} \ + --update-use-local-desc \ + --update-manifests --thin-manifests=n \ + --repo=gentoo \ + $EGENCACHE_CHANGELOG \ + >> ${REGEN_LOG_DIR}/${REGEN_LOG_FILE} 2>&1 +rval=$? +echo "END REGEN $(date -u)" >> ${TIMESLOG} + +if [[ ${rval} != 0 ]]; then + echo "$0: something happened with egencache, cowardly refusing to continue" + echo "${REGEN_LOG_DIR}/${REGEN_LOG_FILE}:" + cat "${REGEN_LOG_DIR}/${REGEN_LOG_FILE}" + exit 5 +fi + +# Redo the Manifests, because the ChangeLog is not added to the Manifest for +# some reason. +parallel_repoman_manifest_REPOMAN + +# don't save empty files +if [[ ! -s ${REGEN_LOG_DIR}/${REGEN_LOG_FILE} ]]; then + rm ${REGEN_LOG_DIR}/${REGEN_LOG_FILE} +fi + +# Keep 30 days of logs +find ${REGEN_LOG_DIR} -type f -mtime +30 -print0 | xargs -0r rm + +# Mark that metadata is done +date -u > ${STAGEDIR}/metadata/timestamp + +echo "START MANIFEST-VALIDATE $(date -u)" >> ${TIMESLOG} +cd ${STAGEDIR} +PORTDIR=${STAGEDIR} repoman manifest-check 2>&1 >${REGEN_LOG_DIR}/${REGEN_LOG_FILE}.validate +rval=$? +if [[ ${rval} != 0 ]]; then + echo "$0: A Manifest has a failure!" + echo "${REGEN_LOG_DIR}/${REGEN_LOG_FILE}.validate:" + cat "${REGEN_LOG_DIR}/${REGEN_LOG_FILE}.validate" + exit 5 +fi +echo "END MANIFEST-VALIDATE $(date -u)" >> ${TIMESLOG} +# end 2) + +# 3) place dtd info in STAGEDIR +echo "START DTD $(date -u)" >> ${TIMESLOG} +rsync -Wqa --exclude=CVS ${EXPORTS}/dtd ${STAGEDIR}/metadata/ +date -R -u > ${STAGEDIR}/metadata/dtd/timestamp.chk +echo "END DTD $(date -u)" >> ${TIMESLOG} +# end 3) + +# 4) place glsa's in STAGEDIR +echo "START GLSA $(date -u)" >> ${TIMESLOG} +rsync -Wqa --exclude=CVS --exclude=.git ${EXPORTS}/glsa ${STAGEDIR}/metadata/ +date -R -u > ${STAGEDIR}/metadata/glsa/timestamp.chk +echo "END GLSA $(date -u)" >> ${TIMESLOG} +# end 4) + +# 5) place news in STAGEDIR +echo "START NEWS $(date -u)" >> ${TIMESLOG} +rsync -Wqa --exclude=CVS --exclude=.git ${EXPORTS}/gentoo-news/. ${STAGEDIR}/metadata/news +date -R -u > ${STAGEDIR}/metadata/news/timestamp.chk +echo "END NEWS $(date -u)" >> ${TIMESLOG} +# end 5) + +# 6) place herds.xml in STAGEDIR +echo "START HERDS $(date -u)" >> ${TIMESLOG} +rsync -Wqa ${EXPORTS}/herds/herds.xml ${STAGEDIR}/metadata/herds.xml +echo "END HERDS $(date -u)" >> ${TIMESLOG} +# end 6) + +# 7) rsync from STAGEDIR to FINALDIR +# note, call exit above if it is not desired to update the FINALDIR. This is +# where all the rsync nodes pull from. + +# timestamp.chk is generated every 5 mins in datestamp.sh script in the FINALDIR +# timestamp.x is generated every 5 mins in datestamp.sh script in the STAGEDIR +# We want to track the progress of STAGEDIR->FINALDIR so exclude .chk file here +echo "START FINAL RSYNC $(date -u)" >> ${TIMESLOG} +rsync -Wqa --exclude=/metadata/timestamp.chk --delete ${STAGEDIR}/ ${FINALDIR}/ +chmod -R u-s,g-s ${FINALDIR}/metadata + +# idea: "max-delete" - require infra approval if wc -l of bak - finaldir is > +# 500? +# The previous method of "detecting" some failure was --max-delete=100, this +# exposed breakage to users though. +#if [[ XXXX ]]; then +# echo "Something went wrong, putting backup in place from last sync" +# mv ${FINALDIR} ${FINALDIR}.broken +# mv ${FINALDIR}.bak ${FINALDIR} +#fi + +echo "END FINAL RSYNC $(date -u)" >> ${TIMESLOG} +# end 7) + +# 8) cache generated data: +# md5-cache +# ChangeLog +echo "START CACHE RSYNC $(date -u)" >> ${TIMESLOG} +CACHEDIR_md5cache=${CACHEDIR}/metadata_md5-cache +CACHEDIR_changelogs=${CACHEDIR}/changelogs +mkdir -p $CACHEDIR_md5cache $CACHEDIR_changelogs +# MD5 cache +rsync -Wqatm --delete ${STAGEDIR}/metadata/md5-cache/ $CACHEDIR_md5cache/ +# ChangeLogs: we do NOT use --delete, as we want to keep old ChangeLogs +rsync -Wqatm --include 'ChangeLog' --include='*/' --exclude='*' ${STAGEDIR}/ $CACHEDIR_changelogs/ +echo "END CACHE RSYNC $(date -u)" >> ${TIMESLOG} + +echo "END ENTIRE SCRIPT $(date -u)" >> ${TIMESLOG} diff --git a/rsync-gen.vars b/rsync-gen.vars new file mode 100644 index 0000000..1c343e0 --- /dev/null +++ b/rsync-gen.vars @@ -0,0 +1,23 @@ +#!/bin/bash + +BASE="/var/tmp/gmirror-rsync" +[[ $HOSTNAME == kookaburra ]] && BASE="/var/tmp/gmirror" +EXPORTS="${BASE}/exports" +STAGEDIR="${BASE}/gentoo-x86-stage" +CACHEDIR="${BASE}/cache" +LOGDIR="${BASE}/logs" +TIMESLOG="${LOGDIR}/$(basename $0)-times.log" +PARALLEL_PARAMS="--jobs 10 --load-average 6" +PARALLEL_PARAMS_noloadavg="--jobs 8" +# Use Python3.3 for all portage processing +export EPYTHON=python3.3 + +# XXX: If these change, then you MUST change the rsyncd modules +FINALDIR="${BASE}/gentoo-x86-final" +# snapshots-create.sh +UPLOAD="${BASE}/snapshots-final/" + +# common function +write_time_log() { + echo "$1" >> ${TIMESLOG} +} diff --git a/sign-autobuilds.sh b/sign-autobuilds.sh new file mode 100755 index 0000000..3d859d9 --- /dev/null +++ b/sign-autobuilds.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +ARCHES="alpha amd64 arm hppa ia64 ppc s390 sh sparc x86" + #alpha amd64 arm hppa ia64 mips ppc s390 sh sparc x86 + +RELEASES=/var/tmp/gmirror-releases/releases/ +[[ $HOSTNAME == TODO ]] && RELEASES=/var/tmp/gmirror/releases/ + +DEBUG='' +VERBOSE='' + +# Do not change +DEBUGP= +VERBOSEP=false +[ -n "$DEBUG" ] && DEBUGP=echo +[ -n "$VERBOSE" ] && VERBOSEP= +[ -n "$DEBUG" ] && RSYNC_OPTS="${RSYNC_OPTS} -n" +[ -n "$VERBOSE" ] && RSYNC_OPTS="${RSYNC_OPTS} -v" + +signone() { + f="$1" + $DEBUGP gpg --homedir /home/gmirror/.gnupg-releng/ --armor --clearsign "${f}" +} + +for a in $ARCHES ; do +pushd $RELEASES/$a >/dev/null || continue + +#echo "ISOS:" +[[ -d autobuilds ]] || exit +digests="$(find autobuilds -name '*.DIGESTS' )" +sigs="$(find autobuilds -name '*.DIGESTS.asc' )" +unsigned="$(comm -23 <(echo "$digests" |sort) <(echo "$sigs" | sed -e 's,.asc$,,g' |sort))" + +#$VERBOSEP echo "=== ARCH: $a" + +for dgst in $unsigned ; do + if [ ! -f ${dgst}.asc ]; then + $VERBOSEP echo "Signing $dgst" + signone $dgst + fi +done + +for dgst in $digests ; do + if [ -f ${dgst}.asc -a ${dgst} -nt ${dgst}.asc ]; then + $VERBOSEP echo "Resigning $dgst" + rm -f ${dgst}.asc + signone $dgst + fi +done + +#echo "$isos" +#latest="$(echo "$isos" | tail -n1)" +#if [[ -n $latest ]]; then +# #echo "latest is $latest" +# latest=$(echo "$latest" | awk '{print $2}') +# #pwd +# rm current && ln -s $latest current +#fi + +popd >/dev/null + +done diff --git a/snapshots-create.sh b/snapshots-create.sh new file mode 100755 index 0000000..fd13241 --- /dev/null +++ b/snapshots-create.sh @@ -0,0 +1,158 @@ +#!/bin/bash +# distributed by cfengine + +# 1) Create the tarball +# 2) Sanity check the tarball size and bail out if it appears abnormal. +# 3) create xz tarball +# 4) sign +# 5) delta generation +# 6) create some symlinks +# 7) tidy up +# 8) clean up old deltas + +umask 022 +source /usr/local/bin/mastermirror/rsync-gen.vars + +# locations (if used in multiple files, please move to rsync-gen.vars file) +MASTER="${FINALDIR}" # the master gentoo-x86 copy +TEMP="${BASE}/snapshots-tmp/" # working directory +HISTORY="7" #number in dsays of previous snapshots to keep +DELTA_UPLOAD="${UPLOAD}/deltas/" + +[[ -d ${TEMP} ]] || mkdir ${TEMP} +[[ -d ${UPLOAD} ]] || mkdir ${UPLOAD} +[[ -d ${DELTA_UPLOAD} ]] || mkdir ${DELTA_UPLOAD} + +write_time_log "---------------------------------------------------------" +write_time_log "START ENTIRE SCRIPT $(date -u)" + +# used to name the file +DELTA_BASE=`/bin/date -d '-2 day' +%Y%m%d` +DELTA_FILENAME="portage-${DELTA_BASE}.tar.bz2" +YESTERDAY=`/bin/date -d yesterday +%Y%m%d` +FILENAME="portage-${YESTERDAY}.tar.bz2" + +# GPG info +#SIGNKEYID="D8BA32AA" +#SIGNKEYID="7DDAD20D" +#SIGNKEYID="239C75C4" +SIGNKEYID="96D8BF6D" + +if [[ ! $(gpg -k | grep ${SIGNKEYID}) ]]; then + echo "${SIGNKEYID} not imported! exiting" + exit 1 +fi + +if [[ ! -e "${UPLOAD}/${DELTA_FILENAME}" ]]; then + echo "Previous snapshot does not exist: '${UPLOAD}/${DELTA_FILENAME}'" + exit 1 +fi + +# working dir +cd ${TEMP} + +# 1) Create the tarball +# create the tarball and move it to the right location +write_time_log "START TARBALL $(date -u)" +if [ ! -f "${FILENAME%.bz2}" ]; then + rm -rf portage + rsync -Wqa "$MASTER"/ portage/ + if [ $? -ne 0 ]; then + echo "rsync run failed!" + exit 1 + fi + tar --owner=portage --group=portage -cf ${FILENAME%.bz2} portage + rc=$? + rm -rf portage + if [ $rc -ne 0 ]; then + echo "Tar run failed!" + exit 1 + fi +fi +[ ! -f " ${FILENAME}.umd5sum" ] && md5sum ${FILENAME%.bz2} > ${FILENAME}.umd5sum +[ ! -f "${FILENAME%.bz2}.bz2" ] && bzip2 -k9 ${FILENAME%.bz2} +write_time_log "END TARBALL $(date -u)" +# end 1) + +# 2) Sanity check the tarball size and bail out if it appears abnormal. +write_time_log "START SIZE SANITY $(date -u)" +current_size=$(stat -c '%s' "${FILENAME}") +previous_size=$(stat -c '%s' "${UPLOAD}/${DELTA_FILENAME}") +if [ ${current_size} -lt ${previous_size} ]; then + size_difference=$(expr ${previous_size} - ${current_size}) + difference_ratio=$(expr ${previous_size} / ${size_difference}) + if [ ${difference_ratio} -lt 5 ]; then + echo "Snapshot size has decreased by more than 20% in one day!!!" + echo "${FILENAME} ${current_size} bytes" + echo "${DELTA_FILENAME} ${previous_size} bytes" + exit 1 + fi +fi +write_time_log "END SIZE SANITY $(date -u)" +# end 2) + +# 3) create xz tarball +write_time_log "START XZ $(date -u)" +if [ ! -f "${FILENAME%.*}.xz" ] ; then + xz -k -9 -e "${FILENAME%.*}" || exit $? +fi +write_time_log "END XZ $(date -u)" +# end 3) + +# 4) sign +write_time_log "START SIGN $(date -u)" +for f in "${FILENAME}" "${FILENAME%.*}".xz ; do + if [ ! -f "${UPLOAD}${f}".umd5sum ]; then + cp "${FILENAME}".umd5sum "${UPLOAD}${f}".umd5sum || exit $? + md5sum "$f" > "$f".md5sum || exit $? + fi + if [ ! -f "$f".gpgsig ]; then + gpg --batch -u "${SIGNKEYID}" --armor --detach-sign \ + --output "$f".gpgsig "$f" || exit $? + fi + mv "$f" "$f".md5sum "$f".gpgsig "${UPLOAD}"/ || exit $? +done +write_time_log "END SIGN $(date -u)" +# end 4) + +# 5) delta generation +write_time_log "START DELTA $(date -u)" +PATCH=snapshot-${DELTA_BASE}-${YESTERDAY}.patch.bz2 +if [ ! -f "${PATCH}" ]; then + bzip2 -dkc ${UPLOAD}/${DELTA_FILENAME} > orig + /usr/bin/differ -f bdelta orig ${FILENAME%.bz2} ${PATCH%.bz2} + bzip2 -9 ${PATCH%.bz2} + md5sum ${PATCH} > ${PATCH}.md5sum + chmod 644 ${PATCH}{,.md5sum} + mv ${PATCH}{,.md5sum} ${DELTA_UPLOAD} + rm orig ${FILENAME%.bz2} "${FILENAME}".umd5sum || exit $? +fi +write_time_log "END DELTA $(date -u)" +# end 5) + +# 6) create some symlinks +write_time_log "START SYMLINK $(date -u)" +cd ${UPLOAD} +for f in "${FILENAME}" "${FILENAME%.*}".xz ; do + ext=${f##*.} + ln -sf "$f" "${UPLOAD}"portage-latest.tar.${ext} || exit $? + rm -f "${UPLOAD}"portage-latest.tar.${ext}.md5sum || exit $? + sed "s/${f}\$/portage-latest.tar.${ext}/" "${UPLOAD}"${f}.md5sum > \ + "${UPLOAD}"portage-latest.tar.${ext}.md5sum || exit $? + ln -sf "${f}".gpgsig "${UPLOAD}"portage-latest.tar.${ext}.gpgsig || exit $? +done +write_time_log "END SYMLINK $(date -u)" +# end 6) + +# 7) tidy up +write_time_log "START CLEANUP $(date -u)" +/usr/bin/find ${UPLOAD} -maxdepth 1 -mtime +${HISTORY} -type f | /usr/bin/xargs /bin/rm -f +write_time_log "END CLEANUP $(date -u)" +# end 7) + +# 8) clean up old deltas +write_time_log "START CLEANUP DELTA $(date -u)" +/usr/local/bin/mastermirror/clean-old-deltas.py "${DELTA_UPLOAD}" "${YESTERDAY}" $(stat -c '%s' "${UPLOAD}/${FILENAME}") > /dev/null +write_time_log "END CLEANUP DELTA $(date -u)" +# end 8) +write_time_log "END ENTIRE SCRIPT $(date -u)" diff --git a/sync-autobuilds.sh b/sync-autobuilds.sh new file mode 100755 index 0000000..a2b5597 --- /dev/null +++ b/sync-autobuilds.sh @@ -0,0 +1,35 @@ +#!/bin/sh +ARCHES="alpha amd64 arm hppa ia64 ppc s390 sh sparc x86" + #alpha amd64 arm hppa ia64 mips ppc s390 sh sparc x86 + +# The -rp_*asc filter rules causes rsync to only delete .asc files that are in +# directories that would otherwise be deleted. +RSYNC_OPTS="-aO --password-file=/etc/mastermirror-fetch/poseidon-rsync.passwd --delete --delete-delay --timeout=600 --exclude='.*' --filter=-rp_*.asc" + +# Do NOT expand the following yet +#_SRC='masterdistfiles@poseidon.amd64.dev.gentoo.org::weekly/${ARCH}/' +#_SRC='masterdistfiles@skimmer.gentoo.org::weekly/${ARCH}/' +_SRC='masterdistfiles@nightheron.gentoo.org::weekly/${ARCH}/' +_DST_BASE='/var/tmp/gmirror-releases/releases/' +# compat +[[ $HOSTNAME == TODO ]] && _DST_BASE='/var/tmp/gmirror/releases/' +[[ -d $_DST_BASE ]] || mkdir $_DST_BASE +# No expansion is IMPORTANT +_DST=${_DST_BASE}/'${ARCH}'/autobuilds + +DEBUG='' +VERBOSE='' +# Nothing to edit beyond this point +DEBUGP= +VERBOSEP= +[ -n "$DEBUG" ] && DEBUGP=echo +[ -n "$DEBUG" ] && RSYNC_OPTS="${RSYNC_OPTS} -n" +[ -n "$VERBOSE" ] && RSYNC_OPTS="${RSYNC_OPTS} -v" + +for ARCH in $ARCHES ; do + src="$(eval echo $_SRC)" + dst="$(eval echo $_DST)" + prg="$(eval echo $_PRG)" + $DEBUGP mkdir -pv "$dst" + rsync ${RSYNC_OPTS} "$src" "$dst" +done diff --git a/sync-distfiles.sh b/sync-distfiles.sh new file mode 100755 index 0000000..1abaf2d --- /dev/null +++ b/sync-distfiles.sh @@ -0,0 +1,53 @@ +#!/bin/bash + +DATADIR="/data/gmirror-distfiles" +LOGDIR="${DATADIR}/log" +RSYNC="/usr/bin/rsync" +RSYNC_ARGS="--no-motd --recursive --times --links --ignore-errors --timeout=300 --quiet" +RSYNC_ARGS_DELETE="--delete --delete-after" +[[ -d ${LOGDIR} ]] || mkdir ${LOGDIR} + +# get the helper files +${RSYNC} ${RSYNC_ARGS} ${RSYNC_ARGS_DELETE} \ + woodpecker.gentoo.org::distfiles-local/ \ + ${DATADIR}/distfiles-local +# Do NOT delete old whitelist entries; they must explicitly be set to empty if +# you want to remove them. +${RSYNC} ${RSYNC_ARGS} \ + woodpecker.gentoo.org::distfiles-whitelist/ \ + ${DATADIR}/distfiles-whitelist +# Copy distfiles-local files to distfiles dir so that emirrordist can process +# them +# Do NOT --delete, let emirrordist handle the deletion logic +# TODO: we could make this a hardlink for speed? +${RSYNC} ${RSYNC_ARGS} \ + ${DATADIR}/distfiles{-local/*,} + +# build one master file due to bug 500030 +cat $(/usr/bin/find ${DATADIR}/distfiles-whitelist -type f ) > \ + ${DATADIR}/tmp/whitelist-master.txt + +# Clear out the success/fail.log so that we can report on it +# (emirrordist appends) +> ${LOGDIR}/failure.log +if [[ $(/bin/date '+%k') -eq 0 ]]; then + > ${LOGDIR}/success.log +fi + +# human readable time format, 1814400 seconds equals 3 weeks +DELAY=1814400 +/usr/bin/emirrordist \ + --distfiles=${DATADIR}/distfiles/ \ + --delete --jobs=10 --repo=gentoo \ + --deletion-delay=${DELAY} \ + --failure-log=${LOGDIR}/failure.log \ + --success-log=${LOGDIR}/success.log \ + --scheduled-deletion-log=${LOGDIR}/deletion.log \ + --deletion-db=${LOGDIR}/deletion-db.bdb \ + --distfiles-db=${LOGDIR}/distfile-db.bdb \ + --temp-dir=${DATADIR}/tmp/ \ + --whitelist-from=${DATADIR}/tmp/whitelist-master.txt \ + --distfiles-local=${DATADIR}/distfiles-local \ + --mirror + +/bin/date -u '+%s' > ${DATADIR}/distfiles/timestamp.mirmon diff --git a/sync-experimental.sh b/sync-experimental.sh new file mode 100755 index 0000000..60ee45c --- /dev/null +++ b/sync-experimental.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +FINALDIR="/var/tmp/gmirror-releases/experimental" +RSYNC="/usr/bin/rsync" +RSYNC_ARGS="--no-motd --recursive --times --links --ignore-errors --delete --delete-after --timeout=300 --exclude=timestamp*" +RSYNC_ARGS="${RSYNC_ARGS} --quiet" + +[[ -d $FINALDIR ]] || mkdir $FINALDIR +${RSYNC} ${RSYNC_ARGS} woodpecker.gentoo.org::experimental-local/ ${FINALDIR}/ + + +/bin/date -u '+%s %c' > ${FINALDIR}/timestamp.x +/bin/date -R -u > ${FINALDIR}/timestamp.chk +/bin/date -u '+%s' > ${FINALDIR}/timestamp.mirmon diff --git a/sync-masterdistfiles-mirror.sh b/sync-masterdistfiles-mirror.sh new file mode 100755 index 0000000..9ff516a --- /dev/null +++ b/sync-masterdistfiles-mirror.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +FINALDIR="/data/mirror" +PASSWD_FILE="/etc/mastermirror-fetch/osuosl-rsync.passwd" +RSYNC="/usr/bin/rsync" +RSYNC_ARGS="--recursive --links --perms --times --delete --hard-links --no-motd --timeout=300 --password-file ${PASSWD_FILE}" +RSYNC_ARGS="${RSYNC_ARGS} --quiet" + +# Reminder: "masterdistfiles.g.o" is OSUOSL, hence the special user +${RSYNC} ${RSYNC_ARGS} gentoo@masterdistfiles.gentoo.org::gentoo/ ${FINALDIR}/ diff --git a/thicken-manifests.py b/thicken-manifests.py new file mode 100644 index 0000000..3b04fa4 --- /dev/null +++ b/thicken-manifests.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python3 +# Copyright 2015 Doug Freed (dwfreed) +# +# - Acquire local copy of repo/gentoo (.git directory not required) +# - Ensure mtime of files matches last time they were committed to, including +# merge commits (for optimal behavior, this should be the last time they were +# actually modified, but the last time a commit affected them in any way is +# alright too) +# - Run egencache against the checkout (this step is NOT optional; manifest generation is torturously slow without md5-cache) +# - Run thicken-manifests.py +# +# Takes an optional --jobs parameter, defaults to the number of CPUs on the host; also takes an optional directory to run against, defaults to the current working directory. +# For GPG signing, add --sign parameter. +# It behaves exactly like repoman manifest, so you can override GPG behavior by setting PORTAGE_GPG_SIGNING_COMMAND, PORTAGE_GPG_KEY, and PORTAGE_GPG_DIR in make.conf or the environment. +# You do NOT need to modify layout.conf for this script to work. Stick with GPG 2.0 for now, because 2.1 moves key operations into the agent, which makes them not parallel. + +import os +import multiprocessing +import subprocess +import logging +import argparse +import errno +import math + +import portage +import portage.exception +import portage.manifest +import portage.util + +portage_settings = None +portage_portdbapi = None +logger = None +args = None + + +def gpg_sign(filename): + gpgcmd = portage_settings.get("PORTAGE_GPG_SIGNING_COMMAND") + if gpgcmd in [None, '']: + raise portage.exception.MissingParameter( + "PORTAGE_GPG_SIGNING_COMMAND is unset! Is make.globals missing?") + if "${PORTAGE_GPG_KEY}" in gpgcmd and "PORTAGE_GPG_KEY" not in portage_settings: + raise portage.exception.MissingParameter("PORTAGE_GPG_KEY is unset!") + if "${PORTAGE_GPG_DIR}" in gpgcmd: + if "PORTAGE_GPG_DIR" not in portage_settings: + portage_settings["PORTAGE_GPG_DIR"] = os.path.expanduser("~/.gnupg") + else: + portage_settings["PORTAGE_GPG_DIR"] = os.path.expanduser(portage_settings["PORTAGE_GPG_DIR"]) + if not os.access(portage_settings["PORTAGE_GPG_DIR"], os.X_OK): + raise portage.exception.InvalidLocation( + "Unable to access directory: PORTAGE_GPG_DIR='%s'" % + portage_settings["PORTAGE_GPG_DIR"]) + gpgvars = {"FILE": filename} + for k in ("PORTAGE_GPG_DIR", "PORTAGE_GPG_KEY"): + v = portage_settings.get(k) + if v is not None: + gpgvars[k] = v + gpgcmd = portage.util.varexpand(gpgcmd, mydict=gpgvars) + gpgcmd = portage.util.shlex_split(gpgcmd) + gpgcmd = [portage._unicode_encode(arg, encoding=portage._encodings['fs'], errors='strict') for arg in gpgcmd] + + return_code = subprocess.call(gpgcmd) + if return_code == os.EX_OK: + os.rename(filename + ".asc", filename) + else: + raise portage.exception.PortageException("!!! gpg exited with '" + str(return_code) + "' status") + + +def worker_init(): + global portage_settings, portage_portdbapi + portage_settings = portage.config(clone=portage.settings) + portage_portdbapi = portage.portdbapi(portage_settings) + + +def maybe_thicken_manifest(pkg_dir): + try: + manifest_mtime = math.floor(os.stat(os.path.join(pkg_dir, 'Manifest')).st_mtime) + except OSError as e: + if e.errno != errno.ENOENT: + logger.exception("%s OSError thrown trying to stat Manifest", pkg_dir) + manifest_mtime = 0 + newest_mtime = manifest_mtime + for root, subdirectories, files in os.walk(pkg_dir): + for file in files: + if file == 'Manifest': + continue + + file_mtime = math.floor(os.stat(os.path.join(root, file)).st_mtime) + if file_mtime > newest_mtime: + newest_mtime = file_mtime + + if newest_mtime == manifest_mtime: + try: + with open(os.path.join(pkg_dir, 'Manifest'), 'r') as f: + if f.readline().startswith('-'): + return + except: + pass + + fetchlistdict = portage.FetchlistDict(pkg_dir, portage_settings, portage_portdbapi) + manifest = portage.manifest.Manifest(pkg_dir, fetchlist_dict=fetchlistdict, distdir=portage_settings['DISTDIR']) + try: + manifest.create(assumeDistHashesAlways=True) + except portage.exception.FileNotFound: + logger.exception("%s is missing DIST entries!", pkg_dir) + if manifest.write(): + if args.sign: + try: + gpg_sign(manifest.getFullname()) + except: + logger.exception("%s Exception thrown during GPG signing", pkg_dir) + os.utime(manifest.getFullname(), (newest_mtime, newest_mtime)) + + +def main(): + global logger, args + parser = argparse.ArgumentParser(description='Thicken ebuild manifests as needed') + parser.add_argument('-j', '--jobs', default=multiprocessing.cpu_count(), type=int, + help='Number of parallel jobs to run; default: number of CPUs') + parser.add_argument('-s', '--sign', action='store_true') + parser.add_argument('location', nargs='?', default='.', help='The location to thicken manifests in; default: .') + args = parser.parse_args() + + logger = multiprocessing.log_to_stderr() + logger.setLevel(logging.WARN) + + os.environ['PORTAGE_REPOSITORIES'] = ''' +[DEFAULT] +main-repo = gentoo + +[gentoo] +location = %s +''' % os.path.realpath(args.location) + + os.chdir(args.location) + pkg_dirs = [] + for root, subdirectories, files in os.walk('.'): + for dir in ('eclass', 'metadata', 'profiles', '.git', 'files'): + if dir in subdirectories: + subdirectories.remove(dir) + + if not subdirectories and 'metadata.xml' in files: + pkg_dirs.append(root) + + pool = multiprocessing.Pool(args.jobs, worker_init) + pool.map(maybe_thicken_manifest, pkg_dirs) + pool.close() + + +if __name__ == '__main__': + main() diff --git a/timestamp-releases.sh b/timestamp-releases.sh new file mode 100755 index 0000000..8935acd --- /dev/null +++ b/timestamp-releases.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +# this is ugly and I don't like it + +if [[ $HOSTNAME == TODO ]]; then + /bin/date -u +'%s %a, %d %b %Y %H:%M:%S %z' > \ + /var/tmp/gmirror/experimental/.timestamp-experimental.x + /bin/date -u +'%s %a, %d %b %Y %H:%M:%S %z' > \ + /var/tmp/gmirror/releases/.timestamp-releases.x +fi +if [[ $HOSTNAME == dipper ]]; then + /bin/date -u +'%s %a, %d %b %Y %H:%M:%S %z' > \ + /var/tmp/gmirror-releases/experimental/.timestamp-experimental.x + /bin/date -u +'%s %a, %d %b %Y %H:%M:%S %z' > \ + /var/tmp/gmirror-releases/releases/.timestamp-releases.x +fi diff --git a/timestamp-rsync.sh b/timestamp-rsync.sh new file mode 100755 index 0000000..1d8cf2f --- /dev/null +++ b/timestamp-rsync.sh @@ -0,0 +1,7 @@ +#!/bin/sh +case $HOSTNAME in + kookaburra) TARGET=/var/tmp/gmirror/snapshots-final/.timestamp-snapshots.x ;; + dipper) TARGET=/var/tmp/gmirror-rsync/snapshots-final/.timestamp-snapshots.x ;; + *) echo "Unknown host in timestamp-rsync.sh!" 1>&2 ; exit 1 ;; +esac +/bin/date -u +'%s %a, %d %b %Y %H:%M:%S %z' >"${TARGET}" |