aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorRobin H. Johnson <robbat2@gentoo.org>2016-12-18 16:28:19 -0800
committerRobin H. Johnson <robbat2@gentoo.org>2016-12-18 16:28:19 -0800
commit931a49ce1da106310b8b76537ce0acaef0bb2a40 (patch)
tree550a708b9178a9947901ab75696f60ea595d3336
parentrsync-gen: start to drop old changelog code. (diff)
downloadmastermirror-scripts-931a49ce1da106310b8b76537ce0acaef0bb2a40.tar.gz
mastermirror-scripts-931a49ce1da106310b8b76537ce0acaef0bb2a40.tar.bz2
mastermirror-scripts-931a49ce1da106310b8b76537ce0acaef0bb2a40.zip
rsync-gen: be clear about variable purposes.
Signed-off-by: Robin H. Johnson <robbat2@gentoo.org>
-rwxr-xr-xrsync-gen.sh118
-rw-r--r--rsync-gen.vars4
2 files changed, 61 insertions, 61 deletions
diff --git a/rsync-gen.sh b/rsync-gen.sh
index 7816c5a..0981600 100755
--- a/rsync-gen.sh
+++ b/rsync-gen.sh
@@ -5,16 +5,16 @@
# 0) Make a backup incase there is something wrong with server side
# 1) Update rsync checkouts or exports from cvs.g.o::mastermirror-staging
# - cvs.g.o::mastermirror-staging is populated by script mastermirror-staging.sh
-# 1b) rsync EXPORTS/gentoo-x86 to STAGEDIR
+# 1b) rsync EXPORTS/gentoo-x86 to STAGEDIR_repo_gentoo
# 1c) source (bash -n) eclasses to check for syntax errors
# 2) generate metadata (egencache)
-# 3) place dtd info & xml schemas in STAGEDIR
-# 4) place glsa's in STAGEDIR
-# 5) place news in STAGEDIR
-# 6) place projects.xml in STAGEDIR
-# 7) rsync from STAGEDIR to FINALDIR
+# 3) place dtd info & xml schemas in STAGEDIR_repo_gentoo
+# 4) place glsa's in STAGEDIR_repo_gentoo
+# 5) place news in STAGEDIR_repo_gentoo
+# 6) place projects.xml in STAGEDIR_repo_gentoo
+# 7) rsync from STAGEDIR_repo_gentoo to FINALDIR_repo_gentoo
# 8) cache generated data
-# 9) rsync from STAGEDIR to REPODIR ### testing
+# 9) rsync from STAGEDIR_repo_gentoo to REPODIR ### testing
################################################################################
# NOTE: You MUST run git-restore-mtime -m on the git checkouts hosted in
@@ -52,8 +52,8 @@ RSYNC_GIT_EXCLUDE="--include core/ --include '*.old' --cvs-exclude --exclude=.gi
# sanity checking
[[ -d ${EXPORTS} ]] || mkdir ${EXPORTS}
-[[ -d ${STAGEDIR} ]] || mkdir ${STAGEDIR}
-[[ -d ${FINALDIR} ]] || mkdir ${FINALDIR}
+[[ -d ${STAGEDIR_repo_gentoo} ]] || mkdir ${STAGEDIR_repo_gentoo}
+[[ -d ${FINALDIR_repo_gentoo} ]] || mkdir ${FINALDIR_repo_gentoo}
[[ -d ${REPODIR} ]] || mkdir ${REPODIR}
[[ -d ${LOGDIR} ]] || mkdir ${LOGDIR}
[[ -d ${BASE}/tmp ]] || mkdir ${BASE}/tmp
@@ -76,7 +76,7 @@ timelog_start "ENTIRE SCRIPT" | timelogger
# 0) Make a backup incase there is something wrong with server side
#echo "START BACKUP $(date -u)" | timelogger
-#cp -al ${FINALDIR}/ ${FINALDIR}.bak/
+#cp -al ${FINALDIR_repo_gentoo}/ ${FINALDIR_repo_gentoo}.bak/
#echo "END BACKUP $(date -u)" | timelogger
# 1) Update rsync checkouts or exports from cvs.g.o::mastermirror-staging
@@ -99,39 +99,39 @@ timelog___end "STAGING RSYNC" | timelogger
timelog_start "CHANGELOG-IMPORT" | timelogger
rsync -Wqa \
$RSYNC_GIT_EXCLUDE \
- ${EXPORTS}/changelogs/ ${STAGEDIR}
+ ${EXPORTS}/changelogs/ ${STAGEDIR_repo_gentoo}
timelog___end "CHANGELOG-IMPORT" | timelogger
# end 1a)
-# 1b) rsync EXPORTS/gentoo-x86 to STAGEDIR
+# 1b) rsync EXPORTS/gentoo-x86 to STAGEDIR_repo_gentoo
-# timestamp.chk is generated every 5 mins in datestamp.sh script in the FINALDIR
-# timestamp.x is generated every 5 mins in datestamp.sh script in the STAGEDIR
-# We want to track the progress of STAGEDIR->FINALDIR so exclude .x file here
+# timestamp.chk is generated every 5 mins in datestamp.sh script in the FINALDIR_repo_gentoo
+# timestamp.x is generated every 5 mins in datestamp.sh script in the STAGEDIR_repo_gentoo
+# We want to track the progress of STAGEDIR_repo_gentoo->FINALDIR_repo_gentoo so exclude .x file here
# (so it isn't deleted)
# Keep /metadata/cache around so the --rsync switch of egencache will work as
# designed
-timelog_start "STAGEDIR RSYNC" | timelogger
+timelog_start "STAGEDIR_repo_gentoo RSYNC" | timelogger
rsync -Wqau \
$RSYNC_GIT_EXCLUDE \
--filter 'P /metadata/***' \
--filter 'Pp Manifest' \
--filter 'Pp ChangeLog*' \
--delete --delete-excluded \
- ${EXPORTS}/gentoo-x86/ ${STAGEDIR}/ || exit
-timelog___end "STAGEDIR RSYNC" | timelogger
+ ${EXPORTS}/gentoo-x86/ ${STAGEDIR_repo_gentoo}/ || exit
+timelog___end "STAGEDIR_repo_gentoo RSYNC" | timelogger
# end 1b)
# 1c) source (bash -n) eclasses to check for syntax errors
timelog_start "ECLASS CHECK" | timelogger
-for i in ${STAGEDIR}/eclass/*.eclass; do
+for i in ${STAGEDIR_repo_gentoo}/eclass/*.eclass; do
bash -n $i || { echo "failed to source $i, exiting"; exit 1; }
done
timelog___end "ECLASS CHECK" | timelogger
# end 1c)
# 2) generate metadata (egencache)
-# ${STAGEDIR}/metadata/cache is created automatically
+# ${STAGEDIR_repo_gentoo}/metadata/cache is created automatically
export PORTAGE_USERNAME=gmirror PORTAGE_GRPNAME=gmirror
LOG_TIMESTAMP=$(date -u '+%Y%m%d-%H%M')
REGEN_LOG_FILE=regen-run-${LOG_TIMESTAMP}.log
@@ -141,11 +141,11 @@ REGEN_LOG_DIR=${LOGDIR}/regen
function parallel_repoman_manifest_THICKEN() {
/usr/local/bin/mastermirror/thicken-manifests.py \
${PARALLEL_PARAMS_noloadavg} \
- "${STAGEDIR}"
+ "${STAGEDIR_repo_gentoo}"
}
function parallel_repoman_manifest_REPOMAN() {
- find ${STAGEDIR} \
+ find ${STAGEDIR_repo_gentoo} \
-maxdepth 1 -mindepth 1 \
-type d \
! -name eclass \
@@ -169,7 +169,7 @@ sed -i \
-e '/^thin-manifests/s,false,true,g' \
-e '/^sign-manifests/s,true,false,g' \
-e '/^sign-commits/s,false,true,g' \
- ${STAGEDIR}/metadata/layout.conf
+ ${STAGEDIR_repo_gentoo}/metadata/layout.conf
# Only update the changelogs every 6 hours
# because right now it takes a very long time to do it (exceeding 1 hour at times).
@@ -181,7 +181,7 @@ egencache --update --rsync $PARALLEL_PARAMS \
--tolerant --cache-dir=${BASE}/tmp/ \
--repositories-configuration="
[gentoo]
-location = ${STAGEDIR}
+location = ${STAGEDIR_repo_gentoo}
" \
--update-use-local-desc \
--repo=gentoo \
@@ -211,7 +211,7 @@ sed -i \
-e '/^thin-manifests/s,true,false,g' \
-e '/^sign-manifests/s,true,false,g' \
-e '/^sign-commits/s,true,false,g' \
- ${STAGEDIR}/metadata/layout.conf
+ ${STAGEDIR_repo_gentoo}/metadata/layout.conf
# Thicken manifests
timelog_start "THICKEN" | timelogger
@@ -219,11 +219,11 @@ parallel_repoman_manifest_THICKEN
timelog___end "THICKEN" | timelogger
# Mark that metadata is done
-date -u > ${STAGEDIR}/metadata/timestamp
+date -u > ${STAGEDIR_repo_gentoo}/metadata/timestamp
timelog_start "MANIFEST-VALIDATE" | timelogger
-cd ${STAGEDIR}
-PORTDIR=${STAGEDIR} repoman manifest-check 2>&1 >${REGEN_LOG_DIR}/${REGEN_LOG_FILE}.validate
+cd ${STAGEDIR_repo_gentoo}
+PORTDIR=${STAGEDIR_repo_gentoo} repoman manifest-check 2>&1 >${REGEN_LOG_DIR}/${REGEN_LOG_FILE}.validate
rval=$?
if [[ ${rval} != 0 ]]; then
echo "$0: A Manifest has a failure!"
@@ -234,66 +234,66 @@ fi
timelog___end "MANIFEST-VALIDATE" | timelogger
# end 2)
-# 3) place dtd info in STAGEDIR
+# 3) place dtd info in STAGEDIR_repo_gentoo
timelog_start "DTD" | timelogger
-rsync -Wqa --checksum --exclude=CVS --exclude=.git --delete ${EXPORTS}/dtd ${STAGEDIR}/metadata/
-date -R -u > ${STAGEDIR}/metadata/dtd/timestamp.chk
+rsync -Wqa --checksum --exclude=CVS --exclude=.git --delete ${EXPORTS}/dtd ${STAGEDIR_repo_gentoo}/metadata/
+date -R -u > ${STAGEDIR_repo_gentoo}/metadata/dtd/timestamp.chk
timelog___end "DTD" | timelogger
# end 3)
-# 3b) place xml schemas in STAGEDIR
+# 3b) place xml schemas in STAGEDIR_repo_gentoo
timelog_start "XML-SCHEMA" | timelogger
-rsync -Wqa --checksum --exclude=CVS --exclude=.git --delete ${EXPORTS}/xml-schema ${STAGEDIR}/metadata/
-date -R -u > ${STAGEDIR}/metadata/xml-schema/timestamp.chk
+rsync -Wqa --checksum --exclude=CVS --exclude=.git --delete ${EXPORTS}/xml-schema ${STAGEDIR_repo_gentoo}/metadata/
+date -R -u > ${STAGEDIR_repo_gentoo}/metadata/xml-schema/timestamp.chk
timelog___end "XML-SCHEMA" | timelogger
# end 3b)
-# 4) place glsa's in STAGEDIR
+# 4) place glsa's in STAGEDIR_repo_gentoo
timelog_start "GLSA" | timelogger
-rsync -Wqa --checksum --exclude=CVS --exclude=.git --delete ${EXPORTS}/glsa ${STAGEDIR}/metadata/
-date -R -u > ${STAGEDIR}/metadata/glsa/timestamp.chk
+rsync -Wqa --checksum --exclude=CVS --exclude=.git --delete ${EXPORTS}/glsa ${STAGEDIR_repo_gentoo}/metadata/
+date -R -u > ${STAGEDIR_repo_gentoo}/metadata/glsa/timestamp.chk
timelog___end "GLSA" | timelogger
# end 4)
-# 5) place news in STAGEDIR
+# 5) place news in STAGEDIR_repo_gentoo
timelog_start "NEWS" | timelogger
-rsync -Wqa --checksum --exclude=CVS --exclude=.git --delete ${EXPORTS}/gentoo-news/. ${STAGEDIR}/metadata/news
-date -R -u > ${STAGEDIR}/metadata/news/timestamp.chk
+rsync -Wqa --checksum --exclude=CVS --exclude=.git --delete ${EXPORTS}/gentoo-news/. ${STAGEDIR_repo_gentoo}/metadata/news
+date -R -u > ${STAGEDIR_repo_gentoo}/metadata/news/timestamp.chk
timelog___end "NEWS" | timelogger
# end 5)
-# 6) place projects.xml in STAGEDIR
+# 6) place projects.xml in STAGEDIR_repo_gentoo
timelog_start "HERDS-AND-PROJ" | timelogger
-rsync -Wqa --checksum ${EXPORTS}/projects/projects.xml ${STAGEDIR}/metadata/projects.xml
+rsync -Wqa --checksum ${EXPORTS}/projects/projects.xml ${STAGEDIR_repo_gentoo}/metadata/projects.xml
timelog___end "HERDS-AND-PROJ" | timelogger
# end 6)
-# 7) rsync from STAGEDIR to FINALDIR
-# note, call exit above if it is not desired to update the FINALDIR. This is
+# 7) rsync from STAGEDIR_repo_gentoo to FINALDIR_repo_gentoo
+# note, call exit above if it is not desired to update the FINALDIR_repo_gentoo. This is
# where all the rsync nodes pull from.
-# timestamp.chk is generated every 5 mins in datestamp.sh script in the FINALDIR
-# timestamp.x is generated every 5 mins in datestamp.sh script in the STAGEDIR
-# We want to track the progress of STAGEDIR->FINALDIR so exclude .chk file here
+# timestamp.chk is generated every 5 mins in datestamp.sh script in the FINALDIR_repo_gentoo
+# timestamp.x is generated every 5 mins in datestamp.sh script in the STAGEDIR_repo_gentoo
+# We want to track the progress of STAGEDIR_repo_gentoo->FINALDIR_repo_gentoo so exclude .chk file here
# Checksum is needed because we explicitly set the timestamp of Manifests/ChangeLogs.
# exclude of .git/CVS is just in case we still accidently created them.
timelog_start "FINAL RSYNC" | timelogger
-FINALDIR_tmp=${FINALDIR%/}
-mkdir -p "${FINALDIR_tmp}-1" "${FINALDIR_tmp}-2"
+FINALDIR_repo_gentoo_tmp=${FINALDIR_repo_gentoo%/}
+mkdir -p "${FINALDIR_repo_gentoo_tmp}-1" "${FINALDIR_repo_gentoo_tmp}-2"
$atomic_rsync -Wqa --exclude=/metadata/timestamp.chk --delete --checksum \
--exclude=.git --exclude=CVS --exclude=.gitignore \
--chmod=u-s,g-s \
- ${STAGEDIR}/ \
- ${FINALDIR}/
+ ${STAGEDIR_repo_gentoo}/ \
+ ${FINALDIR_repo_gentoo}/
-# idea: "max-delete" - require infra approval if wc -l of bak - FINALDIR is >
+# idea: "max-delete" - require infra approval if wc -l of bak - FINALDIR_repo_gentoo is >
# 500?
# The previous method of "detecting" some failure was --max-delete=100, this
# exposed breakage to users though.
#if [[ XXXX ]]; then
# echo "Something went wrong, putting backup in place from last sync"
-# mv ${FINALDIR} ${FINALDIR}.broken
-# mv ${FINALDIR}.bak ${FINALDIR}
+# mv ${FINALDIR_repo_gentoo} ${FINALDIR_repo_gentoo}.broken
+# mv ${FINALDIR_repo_gentoo}.bak ${FINALDIR_repo_gentoo}
#fi
# Testcase for mirrors without --checksum:
@@ -301,7 +301,7 @@ $atomic_rsync -Wqa --exclude=/metadata/timestamp.chk --delete --checksum \
# They will get the file the first time around, but not any later versions.
# The date is deliberately set for just before this experiment started, at a
# very round timestamp, works out to 'Sun Jan 17 03:06:40 UTC 2016'.
-ts=1453000000 f=$FINALDIR/metadata/.checksum-test-marker
+ts=1453000000 f=$FINALDIR_repo_gentoo/metadata/.checksum-test-marker
cat >$f <<EOF
$(date +%s) $(date -u)
# Testcase for mirror network:
@@ -323,18 +323,18 @@ CACHEDIR_md5cache=${CACHEDIR}/metadata_md5-cache
CACHEDIR_changelogs=${CACHEDIR}/changelogs
mkdir -p $CACHEDIR_md5cache $CACHEDIR_changelogs
# MD5 cache
-rsync -Wqatm --delete ${STAGEDIR}/metadata/md5-cache/ $CACHEDIR_md5cache/
+rsync -Wqatm --delete ${STAGEDIR_repo_gentoo}/metadata/md5-cache/ $CACHEDIR_md5cache/
# ChangeLogs: we do NOT use --delete, as we want to keep old ChangeLogs
-rsync -Wqatm --include 'ChangeLog' --include='*/' --exclude='*' ${STAGEDIR}/ $CACHEDIR_changelogs/
+rsync -Wqatm --include 'ChangeLog' --include='*/' --exclude='*' ${STAGEDIR_repo_gentoo}/ $CACHEDIR_changelogs/
timelog___end "CACHE RSYNC" | timelogger
-# 9) rsync from STAGEDIR to REPODIR ### testing
+# 9) rsync from STAGEDIR_repo_gentoo to REPODIR ### testing
timelog_start "GIT RSYNC" | timelogger
rsync -Wqa --exclude=/metadata/timestamp.chk --delete --checksum \
--exclude=.git --exclude=CVS --exclude=.gitignore \
--chmod=u-s,g-s \
--exclude ChangeLog* \
- ${STAGEDIR}/ \
+ ${STAGEDIR_repo_gentoo}/ \
${REPODIR}/ && \
( cd $REPODIR && \
git add -A && \
diff --git a/rsync-gen.vars b/rsync-gen.vars
index 944d53c..58b5ee0 100644
--- a/rsync-gen.vars
+++ b/rsync-gen.vars
@@ -5,7 +5,7 @@
BASE="/var/tmp/gmirror-rsync"
[[ $HOSTNAME == kookaburra ]] && BASE="/var/tmp/gmirror"
EXPORTS="${BASE}/exports"
-STAGEDIR="${BASE}/gentoo-x86-stage"
+STAGEDIR_repo_gentoo="${BASE}/gentoo-x86-stage" # Legacy path
CACHEDIR="${BASE}/cache"
LOGDIR="${BASE}/logs"
TIMESLOG="${LOGDIR}/$(basename $0)-times.log"
@@ -15,7 +15,7 @@ PARALLEL_PARAMS_noloadavg="--jobs 8"
export EPYTHON=python3.4
# XXX: If these change, then you MUST change the rsyncd modules
-FINALDIR="${BASE}/gentoo-x86-final"
+FINALDIR_repo_gentoo="${BASE}/gentoo-x86-final" # Legacy path
# snapshots-create.sh
UPLOAD="${BASE}/snapshots-final/"
REPODIR="${BASE}/gentoo-repo-final" # GITDIR is a magic variable for Git