#!/bin/bash # Git export script # distributed by puppet # this takes about 1.5mins to run the first time. <45s thereafter # if the lv is really small (2G), the default inode ratio is too large. # eg. mkfs.ext2 -i 4096 /dev/vg/var_mastermirror-staging # Notes 2015/12/14 by robbat2 # This entire script was run on the CVS server directly, as it was MUCH faster # than remote checkouts. In the era of Git, this script can probably be merged # with rsync-gen.sh instead. GITROOT_file="file:///var/gitroot" GITROOT_http="https://anongit.gentoo.org/git/" case $HOSTNAME in oystercatcher|swan|manakin) export GITROOT=${GITROOT_file} ;; *) export GITROOT=${GITROOT_http} ;; esac GIT="/usr/bin/git" GIT_CL="${GIT} clone -q" GIT_PL="${GIT} pull -q --ff-only --no-stat --no-rebase" WGET="/usr/bin/wget --timeout=2 --quiet --timestamping" GIT_RESTORE_MTIME="/usr/lib/python-exec/python2.7/git-restore-mtime -q -m -c --first-parent" rc=0 for cmd in \ "${GIT_RESTORE_MTIME}" \ "${WGET}" \ "${GIT}" \ ; do cmd=${cmd/ *} if [ ! -x $cmd ]; then echo "Missing tool: $cmd" 1>&2 rc=1 fi done [[ $rc -ne 0 ]] && exit 1 STAGING_DIR="${1:-/var/tmp/mastermirror-staging}" STAGING_DIR_changelogs="${STAGING_DIR}/changelogs" STAGING_DIR_glsa="${STAGING_DIR}/glsa" STAGING_DIR_dtd="${STAGING_DIR}/dtd" STAGING_DIR_xmlschema="${STAGING_DIR}/xml-schema" STAGING_DIR_news="${STAGING_DIR}/gentoo-news" STAGING_DIR_gentoo="${STAGING_DIR}/gentoo-x86" STAGING_DIR_projects="${STAGING_DIR}/projects" REPO_gentoo=${GITROOT}/repo/gentoo.git REPO_news=${GITROOT}/data/gentoo-news.git REPO_dtd=${GITROOT}/data/dtd.git REPO_xmlschema=${GITROOT}/data/xml-schema.git REPO_glsa=${GITROOT}/data/glsa.git REPO_changelogs=${GITROOT}/data/gentoo-changelogs.git PROJECTS_XML_URI=https://api.gentoo.org/metastructure/projects.xml # Ensure all files are readable umask 0022 fetch_git() { targetdir=$1 repo=$2 if [[ ! -d ${targetdir} ]] ; then # not checked out yet, run initial co ${GIT_CL} $repo --separate-git-dir=${targetdir}.git $targetdir || return 1 else cd ${targetdir} ${GIT_PL} || return 1 fi cd $targetdir && $GIT_RESTORE_MTIME || return 1 } fetch_uri() { targetfile=$1 uri=$2 targetdir=$(dirname $(readlink -m $targetfile)) [[ ! -d ${targetdir} ]] && mkdir -p ${targetdir} tmpfile=$(mktemp -p $targetdir) ${WGET} "${uri}" -O $tmpfile && chmod a+r $tmpfile && mv -f "$tmpfile" "$targetfile" rc=$? rm -f "$tmpfile" return $rc } # TODO: we should probably be checking out to a directory and only swapping on # success of ALL checkouts and fetches. rc_sum=0 # repo/gentoo (formerly CVS gentoo-x86) fetch_git ${STAGING_DIR_gentoo} ${REPO_gentoo} rc=$? rc_sum=$((rc_sum + $rc)) # gentoo-news fetch_git ${STAGING_DIR_news} ${REPO_news} rc=$? rc_sum=$((rc_sum + $rc)) # projects.xml fetch_uri ${STAGING_DIR_projects}/projects.xml "${PROJECTS_XML_URI}" rc=$? rc_sum=$((rc_sum + $rc)) # dtd fetch_git ${STAGING_DIR_dtd} ${REPO_dtd} rc=$? rc_sum=$((rc_sum + $rc)) # xml-schema fetch_git ${STAGING_DIR_xmlschema} ${REPO_xmlschema} rc=$? rc_sum=$((rc_sum + $rc)) # glsa fetch_git ${STAGING_DIR_glsa} ${REPO_glsa} rc=$? rc_sum=$((rc_sum + $rc)) # changelogs fetch_git ${STAGING_DIR_changelogs} ${REPO_changelogs} rc=$? rc_sum=$((rc_sum + $rc)) exit $rc_sum