diff options
| -rwxr-xr-x | rsync-gen.sh | 76 | ||||
| -rwxr-xr-x | thicken-manifests.py | 71 |
2 files changed, 98 insertions, 49 deletions
diff --git a/rsync-gen.sh b/rsync-gen.sh index 048e196..3023351 100755 --- a/rsync-gen.sh +++ b/rsync-gen.sh @@ -14,6 +14,13 @@ # 6) place herds.xml and projects.xml in STAGEDIR # 7) rsync from STAGEDIR to FINALDIR +################################################################################ +# NOTE: You MUST run git-restore-mtime -m on the git checkouts hosted in +# git.gentoo.org::mastermirror-staging/ or this WILL NOT WORK. +# You can get git-restore-mtime here: +# https://github.com/MestreLion/git-tools/blob/master/git-restore-mtime +################################################################################ + umask 022 source /usr/local/bin/mastermirror/rsync-gen.vars @@ -73,6 +80,18 @@ ${RSYNC} ${RSYNC_ARGS} ${RSYNC_ARGS_DELETE} git.gentoo.org::mastermirror-staging timelog___end "STAGING RSYNC" | timelogger # end 1) +# start 1a) +# Temporary: import old CVS changelogs +# This will cause some churn, because it'll copy changelogs for packages which +# don't exist anymore, and then the next rsync will delete them. You can reduce +# churn by occasionally culling dead packages from the repo +timelog_start "CHANGELOG-IMPORT" | timelogger +rsync -Wqa \ + $RSYNC_GIT_EXCLUDE \ + ${EXPORTS}/changelogs/ ${STAGEDIR} +timelog___end "CHANGELOG-IMPORT" | timelogger +# end 1a) + # 1b) rsync EXPORTS/gentoo-x86 to STAGEDIR # timestamp.chk is generated every 5 mins in datestamp.sh script in the FINALDIR @@ -82,12 +101,12 @@ timelog___end "STAGING RSYNC" | timelogger # Keep /metadata/cache around so the --rsync switch of egencache will work as # designed timelog_start "STAGEDIR RSYNC" | timelogger -rsync -Wqa \ +rsync -Wqau \ $RSYNC_GIT_EXCLUDE \ - --exclude=/metadata/timestamp.x \ - --exclude=/metadata/md5-cache \ - --exclude=ChangeLog \ - --delete \ + --filter 'P /metadata/***' \ + --filter 'Pp Manifest' \ + --filter 'Pp ChangeLog*' \ + --delete --delete-excluded \ ${EXPORTS}/gentoo-x86/ ${STAGEDIR}/ || exit timelog___end "STAGEDIR RSYNC" | timelogger # end 1b) @@ -100,20 +119,6 @@ done timelog___end "ECLASS CHECK" | timelogger # end 1c) -# start 1d) -# Temporary: import old CVS changelogs -timelog_start "CHANGELOG-IMPORT" | timelogger -for file in $EXPORTS/changelogs/*/*/ChangeLog*; do - # remove the prefix so we can copy - file=${file#$EXPORTS/changelogs/} - dir=$(dirname "$file") - # do the copy, the stderr redirect is for changelogs that no longer have an ebuild - # (which is why rsync is not used, as it would create a directory/package when none should exist) - [ -d "${STAGEDIR}/${dir}" ] && cp -a $EXPORTS/changelogs/${file} ${STAGEDIR}/${file} -done -timelog___end "CHANGELOG-IMPORT" | timelogger -# end 1d) - # 2) generate metadata (egencache) # ${STAGEDIR}/metadata/cache is created automatically export PORTAGE_USERNAME=gmirror PORTAGE_GRPNAME=gmirror @@ -148,15 +153,12 @@ function parallel_repoman_manifest_REPOMAN() { timelog_start "REGEN" | timelogger -# Force Manifests to thick, unsigned -# also disable commit signing for now. -# TODO: add infra signing of Manifests +# Ensure layout.conf is set to thin-manifests so egencache doesn't error sed -i \ - -e '/^thin-manifests/s,true,false,g' \ + -e '/^thin-manifests/s,false,true,g' \ -e '/^sign-manifests/s,true,false,g' \ - -e '/^sign-commits/s,true,false,g' \ + -e '/^sign-commits/s,false,true,g' \ ${STAGEDIR}/metadata/layout.conf -parallel_repoman_manifest_REPOMAN # Only update the changelogs every 6 hours # because right now it takes a very long time to do it (exceeding 1 hour at times). @@ -171,9 +173,11 @@ esac GIT_DIR=${EXPORTS}/gentoo-x86/.git/ \ egencache --update --rsync $PARALLEL_PARAMS \ --tolerant --cache-dir=${BASE}/tmp/ \ - --portdir=${STAGEDIR} \ + --repositories-configuration=" +[gentoo] +location = ${STAGEDIR} +" \ --update-use-local-desc \ - --update-manifests --thin-manifests=n \ --repo=gentoo \ $EGENCACHE_CHANGELOG \ >> ${REGEN_LOG_DIR}/${REGEN_LOG_FILE} 2>&1 @@ -187,10 +191,6 @@ if [[ ${rval} != 0 ]]; then exit 5 fi -# Redo the Manifests, because the ChangeLog is not added to the Manifest for -# some reason. -parallel_repoman_manifest_REPOMAN - # don't save empty files if [[ ! -s ${REGEN_LOG_DIR}/${REGEN_LOG_FILE} ]]; then rm ${REGEN_LOG_DIR}/${REGEN_LOG_FILE} @@ -199,6 +199,20 @@ fi # Keep 30 days of logs find ${REGEN_LOG_DIR} -type f -mtime +30 -print0 | xargs -0r rm +# Force Manifests to thick, unsigned +# also disable commit signing for now. +# TODO: add infra signing of Manifests +sed -i \ + -e '/^thin-manifests/s,true,false,g' \ + -e '/^sign-manifests/s,true,false,g' \ + -e '/^sign-commits/s,true,false,g' \ + ${STAGEDIR}/metadata/layout.conf + +# Thicken manifests +timelog_start "THICKEN" | timelogger +parallel_repoman_manifest_THICKEN +timelog___end "THICKEN" | timelogger + # Mark that metadata is done date -u > ${STAGEDIR}/metadata/timestamp diff --git a/thicken-manifests.py b/thicken-manifests.py index 3b04fa4..db52bdc 100755 --- a/thicken-manifests.py +++ b/thicken-manifests.py @@ -8,10 +8,10 @@ # alright too) # - Run egencache against the checkout (this step is NOT optional; manifest generation is torturously slow without md5-cache) # - Run thicken-manifests.py -# -# Takes an optional --jobs parameter, defaults to the number of CPUs on the host; also takes an optional directory to run against, defaults to the current working directory. -# For GPG signing, add --sign parameter. -# It behaves exactly like repoman manifest, so you can override GPG behavior by setting PORTAGE_GPG_SIGNING_COMMAND, PORTAGE_GPG_KEY, and PORTAGE_GPG_DIR in make.conf or the environment. +# +# Takes an optional --jobs parameter, defaults to the number of CPUs on the host; also takes an optional directory to run against, defaults to the current working directory +# For GPG signing, add --sign parameter. +# It behaves exactly like repoman manifest, so you can override GPG behavior by setting PORTAGE_GPG_SIGNING_COMMAND, PORTAGE_GPG_KEY, and PORTAGE_GPG_DIR in make.conf or the environment. # You do NOT need to modify layout.conf for this script to work. Stick with GPG 2.0 for now, because 2.1 moves key operations into the agent, which makes them not parallel. import os @@ -21,6 +21,7 @@ import logging import argparse import errno import math +import collections import portage import portage.exception @@ -31,6 +32,7 @@ portage_settings = None portage_portdbapi = None logger = None args = None +dir_contents = collections.defaultdict(list) def gpg_sign(filename): @@ -79,8 +81,9 @@ def maybe_thicken_manifest(pkg_dir): logger.exception("%s OSError thrown trying to stat Manifest", pkg_dir) manifest_mtime = 0 newest_mtime = manifest_mtime - for root, subdirectories, files in os.walk(pkg_dir): - for file in files: + manifest_entries = [] + for root in pkg_dir, os.path.join(pkg_dir, 'files'): + for file in dir_contents[root]: if file == 'Manifest': continue @@ -88,11 +91,29 @@ def maybe_thicken_manifest(pkg_dir): if file_mtime > newest_mtime: newest_mtime = file_mtime + if os.path.split(root)[1] == 'files': + manifest_entries.append('AUX ' + file) + elif file.endswith('.ebuild'): + manifest_entries.append('EBUILD ' + file) + else: + manifest_entries.append('MISC ' + file) + if newest_mtime == manifest_mtime: try: with open(os.path.join(pkg_dir, 'Manifest'), 'r') as f: - if f.readline().startswith('-'): - return + for line in f: + if not line.startswith(('AUX', 'EBUILD', 'MISC')): + continue + + manifest_entry = ' '.join(line.split(' ')[0:2]) + if manifest_entry in manifest_entries: + manifest_entries.remove(manifest_entry) + else: + newest_mtime += 1 + break + else: + if not manifest_entries: + return except: pass @@ -112,13 +133,14 @@ def maybe_thicken_manifest(pkg_dir): def main(): - global logger, args + global logger, args, dir_contents parser = argparse.ArgumentParser(description='Thicken ebuild manifests as needed') parser.add_argument('-j', '--jobs', default=multiprocessing.cpu_count(), type=int, - help='Number of parallel jobs to run; default: number of CPUs') - parser.add_argument('-s', '--sign', action='store_true') + help='Number of parallel jobs to run; default: number of CPUs') + parser.add_argument('-s', '--sign', action='store_true', help='Sign manifests with GPG') parser.add_argument('location', nargs='?', default='.', help='The location to thicken manifests in; default: .') args = parser.parse_args() + args.location = os.path.realpath(args.location) logger = multiprocessing.log_to_stderr() logger.setLevel(logging.WARN) @@ -129,18 +151,31 @@ main-repo = gentoo [gentoo] location = %s -''' % os.path.realpath(args.location) +''' % args.location os.chdir(args.location) pkg_dirs = [] for root, subdirectories, files in os.walk('.'): - for dir in ('eclass', 'metadata', 'profiles', '.git', 'files'): - if dir in subdirectories: + if root == '.': + for dir in ('eclass', 'licenses', 'metadata', 'profiles', 'scripts', '.git'): + if dir in subdirectories: + subdirectories.remove(dir) + + if (not subdirectories or subdirectories == ['files']) and 'metadata.xml' in files: + pkg_dirs.append(root[2:]) + dir_contents[root[2:]] = files + elif os.path.split(root)[1] == 'files' and 'metadata.xml' not in files: + dir_contents[root[2:]] = files + for dir in subdirectories[:]: subdirectories.remove(dir) - - if not subdirectories and 'metadata.xml' in files: - pkg_dirs.append(root) - + os.chdir(root) + for root2, subdirectories2, files2 in os.walk('.'): + if root2 != '.': + for file in files2: + dir_contents[root[2:]].append(os.path.join(root2[2:], file)) + os.chdir(args.location) + + pkg_dirs.sort() pool = multiprocessing.Pool(args.jobs, worker_init) pool.map(maybe_thicken_manifest, pkg_dirs) pool.close() |