Merge changes from dwfreed.

Signed-off-by: Robin H. Johnson <robbat2@gentoo.org>

1 files changed, 45 insertions, 31 deletions

diff --git a/rsync-gen.sh b/rsync-gen.sh
index 048e196..3023351 100755
--- a/rsync-gen.sh
+++ b/rsync-gen.sh
@@ -14,6 +14,13 @@
 # 6) place herds.xml and projects.xml in STAGEDIR
 # 7) rsync from STAGEDIR to FINALDIR
 
+################################################################################
+# NOTE: You MUST run git-restore-mtime -m on the git checkouts hosted in
+# git.gentoo.org::mastermirror-staging/ or this WILL NOT WORK.
+# You can get git-restore-mtime here:
+# https://github.com/MestreLion/git-tools/blob/master/git-restore-mtime
+################################################################################
+
 umask 022
 source /usr/local/bin/mastermirror/rsync-gen.vars
 
@@ -73,6 +80,18 @@ ${RSYNC} ${RSYNC_ARGS} ${RSYNC_ARGS_DELETE} git.gentoo.org::mastermirror-staging
 timelog___end "STAGING RSYNC" | timelogger
 # end 1)
 
+# start 1a)
+# Temporary: import old CVS changelogs
+# This will cause some churn, because it'll copy changelogs for packages which
+# don't exist anymore, and then the next rsync will delete them.  You can reduce
+# churn by occasionally culling dead packages from the repo
+timelog_start "CHANGELOG-IMPORT" | timelogger
+rsync -Wqa \
+	$RSYNC_GIT_EXCLUDE \
+	${EXPORTS}/changelogs/ ${STAGEDIR}
+timelog___end "CHANGELOG-IMPORT" | timelogger
+# end 1a)
+
 # 1b) rsync EXPORTS/gentoo-x86 to STAGEDIR
 
 # timestamp.chk is generated every 5 mins in datestamp.sh script in the FINALDIR
@@ -82,12 +101,12 @@ timelog___end "STAGING RSYNC" | timelogger
 # Keep /metadata/cache around so the --rsync switch of egencache will work as
 # designed
 timelog_start "STAGEDIR RSYNC" | timelogger
-rsync -Wqa \
+rsync -Wqau \
 	$RSYNC_GIT_EXCLUDE \
-	--exclude=/metadata/timestamp.x \
-	--exclude=/metadata/md5-cache \
-	--exclude=ChangeLog \
-	--delete \
+	--filter 'P /metadata/***' \
+	--filter 'Pp Manifest' \
+	--filter 'Pp ChangeLog*' \
+	--delete --delete-excluded \
 	${EXPORTS}/gentoo-x86/ ${STAGEDIR}/ || exit
 timelog___end "STAGEDIR RSYNC" | timelogger
 # end 1b)
@@ -100,20 +119,6 @@ done
 timelog___end "ECLASS CHECK" | timelogger
 # end 1c)
 
-# start 1d)
-# Temporary: import old CVS changelogs
-timelog_start "CHANGELOG-IMPORT" | timelogger
-for file in $EXPORTS/changelogs/*/*/ChangeLog*; do
-  # remove the prefix so we can copy
-  file=${file#$EXPORTS/changelogs/}
-  dir=$(dirname "$file")
-  # do the copy, the stderr redirect is for changelogs that no longer have an ebuild
-  # (which is why rsync is not used, as it would create a directory/package when none should exist)
-  [ -d "${STAGEDIR}/${dir}" ] && cp -a $EXPORTS/changelogs/${file} ${STAGEDIR}/${file}
-done
-timelog___end "CHANGELOG-IMPORT" | timelogger
-# end 1d)
-
 # 2) generate metadata (egencache)
 # ${STAGEDIR}/metadata/cache is created automatically
 export PORTAGE_USERNAME=gmirror PORTAGE_GRPNAME=gmirror
@@ -148,15 +153,12 @@ function parallel_repoman_manifest_REPOMAN() {
 
 timelog_start "REGEN" | timelogger
 
-# Force Manifests to thick, unsigned
-# also disable commit signing for now.
-# TODO: add infra signing of Manifests
+# Ensure layout.conf is set to thin-manifests so egencache doesn't error
 sed -i \
-	-e '/^thin-manifests/s,true,false,g' \
+	-e '/^thin-manifests/s,false,true,g' \
 	-e '/^sign-manifests/s,true,false,g' \
-	-e '/^sign-commits/s,true,false,g' \
+	-e '/^sign-commits/s,false,true,g' \
 	${STAGEDIR}/metadata/layout.conf
-parallel_repoman_manifest_REPOMAN
 
 # Only update the changelogs every 6 hours
 # because right now it takes a very long time to do it (exceeding 1 hour at times).
@@ -171,9 +173,11 @@ esac
 GIT_DIR=${EXPORTS}/gentoo-x86/.git/ \
 egencache --update --rsync $PARALLEL_PARAMS \
 	--tolerant --cache-dir=${BASE}/tmp/ \
-	--portdir=${STAGEDIR} \
+	--repositories-configuration="
+[gentoo]
+location = ${STAGEDIR}
+" \
 	--update-use-local-desc \
-	--update-manifests --thin-manifests=n \
 	--repo=gentoo \
 	$EGENCACHE_CHANGELOG \
 	>> ${REGEN_LOG_DIR}/${REGEN_LOG_FILE} 2>&1
@@ -187,10 +191,6 @@ if [[ ${rval} != 0 ]]; then
 	exit 5
 fi
 
-# Redo the Manifests, because the ChangeLog is not added to the Manifest for
-# some reason.
-parallel_repoman_manifest_REPOMAN
-
 # don't save empty files
 if [[ ! -s ${REGEN_LOG_DIR}/${REGEN_LOG_FILE} ]]; then
 	rm ${REGEN_LOG_DIR}/${REGEN_LOG_FILE}
@@ -199,6 +199,20 @@ fi
 # Keep 30 days of logs
 find ${REGEN_LOG_DIR} -type f -mtime +30 -print0 | xargs -0r rm
 
+# Force Manifests to thick, unsigned
+# also disable commit signing for now.
+# TODO: add infra signing of Manifests
+sed -i \
+	-e '/^thin-manifests/s,true,false,g' \
+	-e '/^sign-manifests/s,true,false,g' \
+	-e '/^sign-commits/s,true,false,g' \
+	${STAGEDIR}/metadata/layout.conf
+
+# Thicken manifests
+timelog_start "THICKEN" | timelogger
+parallel_repoman_manifest_THICKEN
+timelog___end "THICKEN" | timelogger
+
 # Mark that metadata is done
 date -u > ${STAGEDIR}/metadata/timestamp