diff options
| -rwxr-xr-x | rsync-gen.sh | 118 | ||||
| -rw-r--r-- | rsync-gen.vars | 4 |
2 files changed, 61 insertions, 61 deletions
diff --git a/rsync-gen.sh b/rsync-gen.sh index 7816c5a..0981600 100755 --- a/rsync-gen.sh +++ b/rsync-gen.sh @@ -5,16 +5,16 @@ # 0) Make a backup incase there is something wrong with server side # 1) Update rsync checkouts or exports from cvs.g.o::mastermirror-staging # - cvs.g.o::mastermirror-staging is populated by script mastermirror-staging.sh -# 1b) rsync EXPORTS/gentoo-x86 to STAGEDIR +# 1b) rsync EXPORTS/gentoo-x86 to STAGEDIR_repo_gentoo # 1c) source (bash -n) eclasses to check for syntax errors # 2) generate metadata (egencache) -# 3) place dtd info & xml schemas in STAGEDIR -# 4) place glsa's in STAGEDIR -# 5) place news in STAGEDIR -# 6) place projects.xml in STAGEDIR -# 7) rsync from STAGEDIR to FINALDIR +# 3) place dtd info & xml schemas in STAGEDIR_repo_gentoo +# 4) place glsa's in STAGEDIR_repo_gentoo +# 5) place news in STAGEDIR_repo_gentoo +# 6) place projects.xml in STAGEDIR_repo_gentoo +# 7) rsync from STAGEDIR_repo_gentoo to FINALDIR_repo_gentoo # 8) cache generated data -# 9) rsync from STAGEDIR to REPODIR ### testing +# 9) rsync from STAGEDIR_repo_gentoo to REPODIR ### testing ################################################################################ # NOTE: You MUST run git-restore-mtime -m on the git checkouts hosted in @@ -52,8 +52,8 @@ RSYNC_GIT_EXCLUDE="--include core/ --include '*.old' --cvs-exclude --exclude=.gi # sanity checking [[ -d ${EXPORTS} ]] || mkdir ${EXPORTS} -[[ -d ${STAGEDIR} ]] || mkdir ${STAGEDIR} -[[ -d ${FINALDIR} ]] || mkdir ${FINALDIR} +[[ -d ${STAGEDIR_repo_gentoo} ]] || mkdir ${STAGEDIR_repo_gentoo} +[[ -d ${FINALDIR_repo_gentoo} ]] || mkdir ${FINALDIR_repo_gentoo} [[ -d ${REPODIR} ]] || mkdir ${REPODIR} [[ -d ${LOGDIR} ]] || mkdir ${LOGDIR} [[ -d ${BASE}/tmp ]] || mkdir ${BASE}/tmp @@ -76,7 +76,7 @@ timelog_start "ENTIRE SCRIPT" | timelogger # 0) Make a backup incase there is something wrong with server side #echo "START BACKUP $(date -u)" | timelogger -#cp -al ${FINALDIR}/ ${FINALDIR}.bak/ +#cp -al ${FINALDIR_repo_gentoo}/ ${FINALDIR_repo_gentoo}.bak/ #echo "END BACKUP $(date -u)" | timelogger # 1) Update rsync checkouts or exports from cvs.g.o::mastermirror-staging @@ -99,39 +99,39 @@ timelog___end "STAGING RSYNC" | timelogger timelog_start "CHANGELOG-IMPORT" | timelogger rsync -Wqa \ $RSYNC_GIT_EXCLUDE \ - ${EXPORTS}/changelogs/ ${STAGEDIR} + ${EXPORTS}/changelogs/ ${STAGEDIR_repo_gentoo} timelog___end "CHANGELOG-IMPORT" | timelogger # end 1a) -# 1b) rsync EXPORTS/gentoo-x86 to STAGEDIR +# 1b) rsync EXPORTS/gentoo-x86 to STAGEDIR_repo_gentoo -# timestamp.chk is generated every 5 mins in datestamp.sh script in the FINALDIR -# timestamp.x is generated every 5 mins in datestamp.sh script in the STAGEDIR -# We want to track the progress of STAGEDIR->FINALDIR so exclude .x file here +# timestamp.chk is generated every 5 mins in datestamp.sh script in the FINALDIR_repo_gentoo +# timestamp.x is generated every 5 mins in datestamp.sh script in the STAGEDIR_repo_gentoo +# We want to track the progress of STAGEDIR_repo_gentoo->FINALDIR_repo_gentoo so exclude .x file here # (so it isn't deleted) # Keep /metadata/cache around so the --rsync switch of egencache will work as # designed -timelog_start "STAGEDIR RSYNC" | timelogger +timelog_start "STAGEDIR_repo_gentoo RSYNC" | timelogger rsync -Wqau \ $RSYNC_GIT_EXCLUDE \ --filter 'P /metadata/***' \ --filter 'Pp Manifest' \ --filter 'Pp ChangeLog*' \ --delete --delete-excluded \ - ${EXPORTS}/gentoo-x86/ ${STAGEDIR}/ || exit -timelog___end "STAGEDIR RSYNC" | timelogger + ${EXPORTS}/gentoo-x86/ ${STAGEDIR_repo_gentoo}/ || exit +timelog___end "STAGEDIR_repo_gentoo RSYNC" | timelogger # end 1b) # 1c) source (bash -n) eclasses to check for syntax errors timelog_start "ECLASS CHECK" | timelogger -for i in ${STAGEDIR}/eclass/*.eclass; do +for i in ${STAGEDIR_repo_gentoo}/eclass/*.eclass; do bash -n $i || { echo "failed to source $i, exiting"; exit 1; } done timelog___end "ECLASS CHECK" | timelogger # end 1c) # 2) generate metadata (egencache) -# ${STAGEDIR}/metadata/cache is created automatically +# ${STAGEDIR_repo_gentoo}/metadata/cache is created automatically export PORTAGE_USERNAME=gmirror PORTAGE_GRPNAME=gmirror LOG_TIMESTAMP=$(date -u '+%Y%m%d-%H%M') REGEN_LOG_FILE=regen-run-${LOG_TIMESTAMP}.log @@ -141,11 +141,11 @@ REGEN_LOG_DIR=${LOGDIR}/regen function parallel_repoman_manifest_THICKEN() { /usr/local/bin/mastermirror/thicken-manifests.py \ ${PARALLEL_PARAMS_noloadavg} \ - "${STAGEDIR}" + "${STAGEDIR_repo_gentoo}" } function parallel_repoman_manifest_REPOMAN() { - find ${STAGEDIR} \ + find ${STAGEDIR_repo_gentoo} \ -maxdepth 1 -mindepth 1 \ -type d \ ! -name eclass \ @@ -169,7 +169,7 @@ sed -i \ -e '/^thin-manifests/s,false,true,g' \ -e '/^sign-manifests/s,true,false,g' \ -e '/^sign-commits/s,false,true,g' \ - ${STAGEDIR}/metadata/layout.conf + ${STAGEDIR_repo_gentoo}/metadata/layout.conf # Only update the changelogs every 6 hours # because right now it takes a very long time to do it (exceeding 1 hour at times). @@ -181,7 +181,7 @@ egencache --update --rsync $PARALLEL_PARAMS \ --tolerant --cache-dir=${BASE}/tmp/ \ --repositories-configuration=" [gentoo] -location = ${STAGEDIR} +location = ${STAGEDIR_repo_gentoo} " \ --update-use-local-desc \ --repo=gentoo \ @@ -211,7 +211,7 @@ sed -i \ -e '/^thin-manifests/s,true,false,g' \ -e '/^sign-manifests/s,true,false,g' \ -e '/^sign-commits/s,true,false,g' \ - ${STAGEDIR}/metadata/layout.conf + ${STAGEDIR_repo_gentoo}/metadata/layout.conf # Thicken manifests timelog_start "THICKEN" | timelogger @@ -219,11 +219,11 @@ parallel_repoman_manifest_THICKEN timelog___end "THICKEN" | timelogger # Mark that metadata is done -date -u > ${STAGEDIR}/metadata/timestamp +date -u > ${STAGEDIR_repo_gentoo}/metadata/timestamp timelog_start "MANIFEST-VALIDATE" | timelogger -cd ${STAGEDIR} -PORTDIR=${STAGEDIR} repoman manifest-check 2>&1 >${REGEN_LOG_DIR}/${REGEN_LOG_FILE}.validate +cd ${STAGEDIR_repo_gentoo} +PORTDIR=${STAGEDIR_repo_gentoo} repoman manifest-check 2>&1 >${REGEN_LOG_DIR}/${REGEN_LOG_FILE}.validate rval=$? if [[ ${rval} != 0 ]]; then echo "$0: A Manifest has a failure!" @@ -234,66 +234,66 @@ fi timelog___end "MANIFEST-VALIDATE" | timelogger # end 2) -# 3) place dtd info in STAGEDIR +# 3) place dtd info in STAGEDIR_repo_gentoo timelog_start "DTD" | timelogger -rsync -Wqa --checksum --exclude=CVS --exclude=.git --delete ${EXPORTS}/dtd ${STAGEDIR}/metadata/ -date -R -u > ${STAGEDIR}/metadata/dtd/timestamp.chk +rsync -Wqa --checksum --exclude=CVS --exclude=.git --delete ${EXPORTS}/dtd ${STAGEDIR_repo_gentoo}/metadata/ +date -R -u > ${STAGEDIR_repo_gentoo}/metadata/dtd/timestamp.chk timelog___end "DTD" | timelogger # end 3) -# 3b) place xml schemas in STAGEDIR +# 3b) place xml schemas in STAGEDIR_repo_gentoo timelog_start "XML-SCHEMA" | timelogger -rsync -Wqa --checksum --exclude=CVS --exclude=.git --delete ${EXPORTS}/xml-schema ${STAGEDIR}/metadata/ -date -R -u > ${STAGEDIR}/metadata/xml-schema/timestamp.chk +rsync -Wqa --checksum --exclude=CVS --exclude=.git --delete ${EXPORTS}/xml-schema ${STAGEDIR_repo_gentoo}/metadata/ +date -R -u > ${STAGEDIR_repo_gentoo}/metadata/xml-schema/timestamp.chk timelog___end "XML-SCHEMA" | timelogger # end 3b) -# 4) place glsa's in STAGEDIR +# 4) place glsa's in STAGEDIR_repo_gentoo timelog_start "GLSA" | timelogger -rsync -Wqa --checksum --exclude=CVS --exclude=.git --delete ${EXPORTS}/glsa ${STAGEDIR}/metadata/ -date -R -u > ${STAGEDIR}/metadata/glsa/timestamp.chk +rsync -Wqa --checksum --exclude=CVS --exclude=.git --delete ${EXPORTS}/glsa ${STAGEDIR_repo_gentoo}/metadata/ +date -R -u > ${STAGEDIR_repo_gentoo}/metadata/glsa/timestamp.chk timelog___end "GLSA" | timelogger # end 4) -# 5) place news in STAGEDIR +# 5) place news in STAGEDIR_repo_gentoo timelog_start "NEWS" | timelogger -rsync -Wqa --checksum --exclude=CVS --exclude=.git --delete ${EXPORTS}/gentoo-news/. ${STAGEDIR}/metadata/news -date -R -u > ${STAGEDIR}/metadata/news/timestamp.chk +rsync -Wqa --checksum --exclude=CVS --exclude=.git --delete ${EXPORTS}/gentoo-news/. ${STAGEDIR_repo_gentoo}/metadata/news +date -R -u > ${STAGEDIR_repo_gentoo}/metadata/news/timestamp.chk timelog___end "NEWS" | timelogger # end 5) -# 6) place projects.xml in STAGEDIR +# 6) place projects.xml in STAGEDIR_repo_gentoo timelog_start "HERDS-AND-PROJ" | timelogger -rsync -Wqa --checksum ${EXPORTS}/projects/projects.xml ${STAGEDIR}/metadata/projects.xml +rsync -Wqa --checksum ${EXPORTS}/projects/projects.xml ${STAGEDIR_repo_gentoo}/metadata/projects.xml timelog___end "HERDS-AND-PROJ" | timelogger # end 6) -# 7) rsync from STAGEDIR to FINALDIR -# note, call exit above if it is not desired to update the FINALDIR. This is +# 7) rsync from STAGEDIR_repo_gentoo to FINALDIR_repo_gentoo +# note, call exit above if it is not desired to update the FINALDIR_repo_gentoo. This is # where all the rsync nodes pull from. -# timestamp.chk is generated every 5 mins in datestamp.sh script in the FINALDIR -# timestamp.x is generated every 5 mins in datestamp.sh script in the STAGEDIR -# We want to track the progress of STAGEDIR->FINALDIR so exclude .chk file here +# timestamp.chk is generated every 5 mins in datestamp.sh script in the FINALDIR_repo_gentoo +# timestamp.x is generated every 5 mins in datestamp.sh script in the STAGEDIR_repo_gentoo +# We want to track the progress of STAGEDIR_repo_gentoo->FINALDIR_repo_gentoo so exclude .chk file here # Checksum is needed because we explicitly set the timestamp of Manifests/ChangeLogs. # exclude of .git/CVS is just in case we still accidently created them. timelog_start "FINAL RSYNC" | timelogger -FINALDIR_tmp=${FINALDIR%/} -mkdir -p "${FINALDIR_tmp}-1" "${FINALDIR_tmp}-2" +FINALDIR_repo_gentoo_tmp=${FINALDIR_repo_gentoo%/} +mkdir -p "${FINALDIR_repo_gentoo_tmp}-1" "${FINALDIR_repo_gentoo_tmp}-2" $atomic_rsync -Wqa --exclude=/metadata/timestamp.chk --delete --checksum \ --exclude=.git --exclude=CVS --exclude=.gitignore \ --chmod=u-s,g-s \ - ${STAGEDIR}/ \ - ${FINALDIR}/ + ${STAGEDIR_repo_gentoo}/ \ + ${FINALDIR_repo_gentoo}/ -# idea: "max-delete" - require infra approval if wc -l of bak - FINALDIR is > +# idea: "max-delete" - require infra approval if wc -l of bak - FINALDIR_repo_gentoo is > # 500? # The previous method of "detecting" some failure was --max-delete=100, this # exposed breakage to users though. #if [[ XXXX ]]; then # echo "Something went wrong, putting backup in place from last sync" -# mv ${FINALDIR} ${FINALDIR}.broken -# mv ${FINALDIR}.bak ${FINALDIR} +# mv ${FINALDIR_repo_gentoo} ${FINALDIR_repo_gentoo}.broken +# mv ${FINALDIR_repo_gentoo}.bak ${FINALDIR_repo_gentoo} #fi # Testcase for mirrors without --checksum: @@ -301,7 +301,7 @@ $atomic_rsync -Wqa --exclude=/metadata/timestamp.chk --delete --checksum \ # They will get the file the first time around, but not any later versions. # The date is deliberately set for just before this experiment started, at a # very round timestamp, works out to 'Sun Jan 17 03:06:40 UTC 2016'. -ts=1453000000 f=$FINALDIR/metadata/.checksum-test-marker +ts=1453000000 f=$FINALDIR_repo_gentoo/metadata/.checksum-test-marker cat >$f <<EOF $(date +%s) $(date -u) # Testcase for mirror network: @@ -323,18 +323,18 @@ CACHEDIR_md5cache=${CACHEDIR}/metadata_md5-cache CACHEDIR_changelogs=${CACHEDIR}/changelogs mkdir -p $CACHEDIR_md5cache $CACHEDIR_changelogs # MD5 cache -rsync -Wqatm --delete ${STAGEDIR}/metadata/md5-cache/ $CACHEDIR_md5cache/ +rsync -Wqatm --delete ${STAGEDIR_repo_gentoo}/metadata/md5-cache/ $CACHEDIR_md5cache/ # ChangeLogs: we do NOT use --delete, as we want to keep old ChangeLogs -rsync -Wqatm --include 'ChangeLog' --include='*/' --exclude='*' ${STAGEDIR}/ $CACHEDIR_changelogs/ +rsync -Wqatm --include 'ChangeLog' --include='*/' --exclude='*' ${STAGEDIR_repo_gentoo}/ $CACHEDIR_changelogs/ timelog___end "CACHE RSYNC" | timelogger -# 9) rsync from STAGEDIR to REPODIR ### testing +# 9) rsync from STAGEDIR_repo_gentoo to REPODIR ### testing timelog_start "GIT RSYNC" | timelogger rsync -Wqa --exclude=/metadata/timestamp.chk --delete --checksum \ --exclude=.git --exclude=CVS --exclude=.gitignore \ --chmod=u-s,g-s \ --exclude ChangeLog* \ - ${STAGEDIR}/ \ + ${STAGEDIR_repo_gentoo}/ \ ${REPODIR}/ && \ ( cd $REPODIR && \ git add -A && \ diff --git a/rsync-gen.vars b/rsync-gen.vars index 944d53c..58b5ee0 100644 --- a/rsync-gen.vars +++ b/rsync-gen.vars @@ -5,7 +5,7 @@ BASE="/var/tmp/gmirror-rsync" [[ $HOSTNAME == kookaburra ]] && BASE="/var/tmp/gmirror" EXPORTS="${BASE}/exports" -STAGEDIR="${BASE}/gentoo-x86-stage" +STAGEDIR_repo_gentoo="${BASE}/gentoo-x86-stage" # Legacy path CACHEDIR="${BASE}/cache" LOGDIR="${BASE}/logs" TIMESLOG="${LOGDIR}/$(basename $0)-times.log" @@ -15,7 +15,7 @@ PARALLEL_PARAMS_noloadavg="--jobs 8" export EPYTHON=python3.4 # XXX: If these change, then you MUST change the rsyncd modules -FINALDIR="${BASE}/gentoo-x86-final" +FINALDIR_repo_gentoo="${BASE}/gentoo-x86-final" # Legacy path # snapshots-create.sh UPLOAD="${BASE}/snapshots-final/" REPODIR="${BASE}/gentoo-repo-final" # GITDIR is a magic variable for Git |