path: root/snapshots-create.sh

#!/bin/bash
# Copyright 2011-2015 Gentoo Foundation; Distributed under the GPL v2
# might be earlier copyright, no history available

# 1) Create the tarball
# 2) Sanity check the tarball size and bail out if it appears abnormal.
# 3) create xz tarball
# 4) sign
# 5) delta generation
# 6) create some symlinks
# 7) tidy up
# 8) clean up old deltas

umask 022
source /usr/local/bin/mastermirror/rsync-gen.vars

# locations (if used in multiple files, please move to rsync-gen.vars file)
MASTER="${FINALDIR_repo_gentoo}" # the master gentoo-x86 copy
TEMP="${BASE}/snapshots-tmp/" # working directory
HISTORY="7" #number in dsays of previous snapshots to keep
DELTA_UPLOAD="${UPLOAD}/deltas/"

[[ -d ${TEMP} ]]			|| mkdir ${TEMP}
[[ -d ${UPLOAD} ]]			|| mkdir ${UPLOAD}
[[ -d ${DELTA_UPLOAD} ]]		|| mkdir ${DELTA_UPLOAD}

write_time_log "---------------------------------------------------------"
write_time_log "START ENTIRE SCRIPT	$(date -u)"

# used to name the file
DELTA_BASE=`/bin/date -d '-2 day' +%Y%m%d`
DELTA_FILENAME="portage-${DELTA_BASE}.tar.bz2"
YESTERDAY=`/bin/date -d yesterday +%Y%m%d`
FILENAME="portage-${YESTERDAY}.tar.bz2"

# Parallel compressors can use a LOT of cpu, be nice about it
NICE="nice -n 10"

# GPG info
#SIGNKEYID="D8BA32AA"
#SIGNKEYID="7DDAD20D"
#SIGNKEYID="239C75C4"
SIGNKEYID="96D8BF6D"

if [[ ! $(gpg -k | grep ${SIGNKEYID}) ]]; then
	echo "${SIGNKEYID} not imported! exiting"
	exit 1
fi

if [[ ! -e "${UPLOAD}/${DELTA_FILENAME}" ]]; then
	echo "Previous snapshot does not exist: '${UPLOAD}/${DELTA_FILENAME}'"
	exit 1
fi

# Find fastest BZIP2
# lbzip2,pbzip2 default to multiple threads
# emerge-delta-webrsync relies on verifying signature of tarball after
# re-compressing it with bzip2, therefore failing if the signature
# was done on lbzip2 or pbzip2 compressed tarball, #573908
for BZIP2_PROG in bzip2 FAIL ; do
	[ -n "$(type $BZIP2_PROG 2>/dev/null)" ] && break
done
if [ $BZIP2_PROG == FAIL ];  then
	echo "Could not find any BZIP2" 1>&2
	exit 1
fi
# Find fastest XZ
# pixz appends some data and leads to SIGPIPE, #573642
for XZ_PROG in xz FAIL ; do
	[ -n "$(type $XZ_PROG 2>/dev/null)" ] && break
done
if [ $XZ_PROG == FAIL ];  then
	echo "Could not find any xz" 1>&2
	exit 1
fi
# Newer 'xz' supports threads as well, but defaults to single-threaded
if $XZ_PROG --help |grep -sq threads=NUM ; then
	XZ_PROG="${XZ_PROG} -T 0"
fi

# working dir
cd ${TEMP}

# 1) Create the tarball
# create the tarball and move it to the right location
write_time_log "START TARBALL		$(date -u)"
if [ ! -f "${FILENAME%.bz2}" ]; then
	# Build exclusion list
	# app-bar/ChangeLog is a valid package name, so we have to be careful about it!
	EXCLUSION_LIST="$(mktemp -p ${TEMP} snapshot-exclude.XXXXXXXXXX)"
	find "${MASTER}" \
		\( \
			-type f \
			-regextype posix-egrep \
			\( \
				-path "${MASTER}/eclass/ChangeLog*" -o \
				-path "${MASTER}/profiles/ChangeLog*" -o \
				-path "${MASTER}/profiles/*/ChangeLog*" -o \
				-regex "${MASTER}/[^/]+/[^/]+/ChangeLog(-[0-9]+)?$" \
			\) \
		\) \
		-o \
		-name '.checksum-test-marker' \
		\
		| sed "s,${MASTER}/*,,g" \
		\
		| sort \
		\
		>"${EXCLUSION_LIST}"

	# Build a directory of stuff to tar up.
	rm -rf portage
	rsync -Wqa \
		"$MASTER"/ \
		portage/ \
		--exclude-from "${EXCLUSION_LIST}"
	if [ $? -ne 0 ]; then
		echo "rsync run failed!"
		exit 1
	fi

	TAR_OPTIONS=(
	# GNU tar format saves approximately 1K per file in the tarball over POSIX
	# format. Multiply ~170k files, and the savings are large.
		--format=gnu
	# Sorting by name produces consistent ordering and helps compression of
	# related content. Custom ordering might further improve ordering in future
	# (eg all metadata.xml first)
		--sort=name
	# Force ownership of content:
		--owner=portage
		--group=portage
	# Excluded content:
	# This is handled further up, by not rsyncing them at all!
	#	--no-wildcards
	#	--exclude-from "${EXCLUSION_LIST}"
	# Do not capture any xattr/acl info at all.
		--no-acls
		--no-xattrs
		--no-selinux
	)
	tar "${TAR_OPTIONS[@]}" \
		-cf ${FILENAME%.bz2} \
		portage
	rc=$?
	rm -rf portage
	if [ $rc -ne 0 ]; then
		echo "Tar run failed!"
		exit 1
	fi
fi
[ ! -f " ${FILENAME}.umd5sum" ] && md5sum ${FILENAME%.bz2} > ${FILENAME}.umd5sum
[ ! -f "${FILENAME%.bz2}.bz2" ] && ${NICE} $BZIP2_PROG -k9 ${FILENAME%.bz2}
write_time_log "END TARBALL		$(date -u)"
# end 1)

# 2) Sanity check the tarball size and bail out if it appears abnormal.
write_time_log "START SIZE SANITY	$(date -u)"
current_size=$(stat -c '%s' "${FILENAME}")
previous_size=$(stat -c '%s' "${UPLOAD}/${DELTA_FILENAME}")
if [ ${current_size} -lt ${previous_size} ]; then
	size_difference=$(expr ${previous_size} - ${current_size})
	difference_ratio=$(expr ${previous_size} / ${size_difference})
	if [ ${difference_ratio} -lt 5 ]; then
		echo "Snapshot size has decreased by more than 20% in one day!!!"
		echo "${FILENAME} ${current_size} bytes"
		echo "${DELTA_FILENAME} ${previous_size} bytes"
		exit 1
	fi
fi
write_time_log "END SIZE SANITY		$(date -u)"
# end 2)

# 3) create xz tarball
write_time_log "START XZ		$(date -u)"
if [ ! -f "${FILENAME%.*}.xz" ] ; then
	# pixz, pxz, xz all differ in filename generation
	# xz: .tar -> .tar.xz
	# pixz: .tar -> .tpxz
	# pxz: .tar -> .txz
	#
	# To avoid this, be explicit by using IO.
	${NICE} ${XZ_PROG} -9 -e <"${FILENAME%.*}" >"${FILENAME%.*}.xz" || exit $?
fi
write_time_log "END XZ			$(date -u)"
# end 3)

# 4) sign
write_time_log "START SIGN		$(date -u)"
for f in "${FILENAME}" "${FILENAME%.*}".xz ; do
	if [ ! -f "${UPLOAD}${f}".umd5sum ]; then
		cp "${FILENAME}".umd5sum "${UPLOAD}${f}".umd5sum || exit $?
		md5sum "$f" > "$f".md5sum || exit $?
	fi
	if [ ! -f "$f".gpgsig ]; then
		gpg --batch -u "${SIGNKEYID}" --armor --detach-sign \
			--output "$f".gpgsig "$f" || exit $?
	fi
	mv "$f" "$f".md5sum "$f".gpgsig "${UPLOAD}"/ || exit $?
done
write_time_log "END SIGN		$(date -u)"
# end 4)

# 5) delta generation
write_time_log "START DELTA		$(date -u)"
PATCH=snapshot-${DELTA_BASE}-${YESTERDAY}.patch.bz2
if [ ! -f "${PATCH}" ]; then
	${NICE} ${BZIP2_PROG} -dkc ${UPLOAD}/${DELTA_FILENAME} > orig
	/usr/bin/differ -f bdelta orig ${FILENAME%.bz2} ${PATCH%.bz2}
	${NICE} ${BZIP2_PROG} -9 ${PATCH%.bz2}
	md5sum ${PATCH} > ${PATCH}.md5sum
	chmod 644 ${PATCH}{,.md5sum}
	mv ${PATCH}{,.md5sum} ${DELTA_UPLOAD}
	rm orig ${FILENAME%.bz2} "${FILENAME}".umd5sum || exit $?
fi
write_time_log "END DELTA		$(date -u)"
# end 5)

# 6) create some symlinks
write_time_log "START SYMLINK		$(date -u)"
cd ${UPLOAD}
for f in "${FILENAME}" "${FILENAME%.*}".xz ; do
	ext=${f##*.}
	ln -sf "$f" "${UPLOAD}"portage-latest.tar.${ext} || exit $?
	rm -f "${UPLOAD}"portage-latest.tar.${ext}.md5sum || exit $?
	sed "s/${f}\$/portage-latest.tar.${ext}/" "${UPLOAD}"${f}.md5sum > \
		"${UPLOAD}"portage-latest.tar.${ext}.md5sum || exit $?
	ln -sf "${f}".gpgsig "${UPLOAD}"portage-latest.tar.${ext}.gpgsig || exit $?
done
write_time_log "END SYMLINK		$(date -u)"
# end 6)

# 7) tidy up
write_time_log "START CLEANUP		$(date -u)"
/usr/bin/find ${UPLOAD} -maxdepth 1 -mtime +${HISTORY} -type f | /usr/bin/xargs /bin/rm -f
write_time_log "END CLEANUP		$(date -u)"
# end 7)

# 8) clean up old deltas
write_time_log "START CLEANUP DELTA	$(date -u)"
/usr/local/bin/mastermirror/clean-old-deltas.py "${DELTA_UPLOAD}" "${YESTERDAY}" $(stat -c '%s' "${UPLOAD}/${FILENAME}") > /dev/null
write_time_log "END CLEANUP DELTA	$(date -u)"
# end 8)
write_time_log "END ENTIRE SCRIPT	$(date -u)"