snapshots-create: implement new output file for bug #57475220191207T075744Z

Write a new output file gentoo-YYYYMMDD.tar.xz
with the initial path element being gentoo-YYYYMMDD

Do it by transforming the main tarball, rather than recreating an
entirely new run of tar (which requires seeks).

Fixes: https://bugs.gentoo.org/574752
Signed-off-by: Robin H. Johnson <robbat2@gentoo.org>

1 files changed, 58 insertions, 3 deletions

diff --git a/snapshots-create.sh b/snapshots-create.sh
index e0ee097..1721bbd 100755
--- a/snapshots-create.sh
+++ b/snapshots-create.sh
@@ -4,7 +4,7 @@
 
 # 1) Create the tarball
 # 2) Sanity check the tarball size and bail out if it appears abnormal.
-# 3) create xz tarball
+# 3) Compress tarball(bzip2,xz)
 # 4) sign
 # 5) delta generation
 # 6) create some symlinks
@@ -33,6 +33,8 @@ DELTA_FILENAME="portage-${DELTA_BASE}.tar.bz2"
 YESTERDAY=`/bin/date -d yesterday +%Y%m%d`
 FILENAME="portage-${YESTERDAY}.tar.bz2"
 
+FILENAME_NEW="gentoo-${YESTERDAY}.tar.xz"
+
 # Parallel compressors can use a LOT of cpu, be nice about it
 NICE="nice -n 10"
 
@@ -153,8 +155,39 @@ write_time_log "END TARBALL(tar,old)		$(date -u)"
 write_time_log "START TARBALL(umd5,old)		$(date -u)"
 [ ! -f " ${FILENAME}.umd5sum" ] && md5sum ${FILENAME%.bz2} > ${FILENAME}.umd5sum
 write_time_log "END TARBALL(umd5,old)		$(date -u)"
-write_time_log "END TARBALL		$(date -u)"
+# end 1b)
+
+# 1c) Also create new-style tarball,
+# but do it via transformation of old tarball
+write_time_log "START TARBALL(transform,new) $(date -u)"
+if [ ! -f "${FILENAME_NEW%.xz}" ]; then
+
+	# The newer tarball differs in the following ways:
+	# Filename 'portage-YYYYMMDD' -> 'gentoo-YYYYMMDD'
+	# Path prefex 'portage' -> 'gentoo-YYYYMMDD'
+	#
+	# Instead of rebuilding the entire tarball, it's MUCH faster to consider it
+	# a single IO stream and process it as such.
+	# tar-transform-names.pl is a small wrapper around Perl Archive::Tar::Stream
+	# that just changes filenames
+	/usr/local/bin/mastermirror//tar-transform-names.pl \
+		--input-file "${FILENAME%.bz2}" \
+		--output-file "${FILENAME_NEW%.xz}" \
+		--regex "s/^portage/${FILENAME_NEW%%.*}/"
+	rc=$?
+	if [ $rc -ne 0 ]; then
+		echo "Tar transform run!"
+		exit 1
+	fi
+fi
+write_time_log "END TARBALL(transform,new) $(date -u)"
+write_time_log "START TARBALL(umd5,new)		$(date -u)"
+[ ! -f "${FILENAME_NEW}.umd5sum" ] && md5sum ${FILENAME_NEW%.xz} > ${FILENAME_NEW}.umd5sum
+write_time_log "END TARBALL(umd5,new)		$(date -u)"
+# end 1c)
+
 # end 1)
+write_time_log "END TARBALL		$(date -u)"
 
 # 2) Sanity check the tarball size and bail out if it appears abnormal.
 write_time_log "START SIZE SANITY	$(date -u)"
@@ -198,12 +231,24 @@ if [ ! -f "${FILENAME%.*}.xz" ] ; then
 	${NICE} ${XZ_PROG} -9 -e <"${FILENAME%.*}" >"${FILENAME%.*}.xz" || exit $?
 fi
 write_time_log "END COMPRESS(xz,old)			$(date -u)"
+# 3c) xz of new tarball
+write_time_log "START COMPRESS(xz,new) $(date -u)"
+if [ ! -f "${FILENAME_NEW%.*}.xz" ] ; then
+	# pixz, pxz, xz all differ in filename generation
+	# xz: .tar -> .tar.xz
+	# pixz: .tar -> .tpxz
+	# pxz: .tar -> .txz
+	#
+	# To avoid this, be explicit by using IO.
+	${NICE} ${XZ_PROG} -9 -e <"${FILENAME_NEW%.*}" >"${FILENAME_NEW%.*}.xz" || exit $?
+fi
+write_time_log "END COMPRESS(xz,new)			$(date -u)"
 write_time_log "END COMPRESS		$(date -u)"
 # end 3)
 
 # 4) sign
 write_time_log "START SIGN		$(date -u)"
-for f in "${FILENAME}" "${FILENAME%.*}".xz ; do
+for f in "${FILENAME}" "${FILENAME%.*}".xz "${FILENAME_NEW}"; do
 	if [ ! -f "${UPLOAD}${f}".umd5sum ]; then
 		cp "${FILENAME}".umd5sum "${UPLOAD}${f}".umd5sum || exit $?
 		md5sum "$f" > "$f".md5sum || exit $?
@@ -218,6 +263,7 @@ write_time_log "END SIGN		$(date -u)"
 # end 4)
 
 # 5) delta generation
+# Delta is not generated for new filename at this time
 write_time_log "START DELTA		$(date -u)"
 PATCH=snapshot-${DELTA_BASE}-${YESTERDAY}.patch.bz2
 if [ ! -f "${PATCH}" ]; then
@@ -243,6 +289,14 @@ for f in "${FILENAME}" "${FILENAME%.*}".xz ; do
 		"${UPLOAD}"portage-latest.tar.${ext}.md5sum || exit $?
 	ln -sf "${f}".gpgsig "${UPLOAD}"portage-latest.tar.${ext}.gpgsig || exit $?
 done
+for f in "${FILENAME_NEW}" ; do
+	ext=${f##*.}
+	ln -sf "$f" "${UPLOAD}"gentoo-latest.tar.${ext} || exit $?
+	rm -f "${UPLOAD}"gentoo-latest.tar.${ext}.md5sum || exit $?
+	sed "s/${f}\$/gentoo-latest.tar.${ext}/" "${UPLOAD}"${f}.md5sum > \
+		"${UPLOAD}"gentoo-latest.tar.${ext}.md5sum || exit $?
+	ln -sf "${f}".gpgsig "${UPLOAD}"gentoo-latest.tar.${ext}.gpgsig || exit $?
+done
 write_time_log "END SYMLINK		$(date -u)"
 # end 6)
 
@@ -253,6 +307,7 @@ write_time_log "END CLEANUP		$(date -u)"
 # end 7)
 
 # 8) clean up old deltas
+# Delta is not generated for new filename at this time
 write_time_log "START CLEANUP DELTA	$(date -u)"
 /usr/local/bin/mastermirror/clean-old-deltas.py "${DELTA_UPLOAD}" "${YESTERDAY}" $(stat -c '%s' "${UPLOAD}/${FILENAME}") > /dev/null
 write_time_log "END CLEANUP DELTA	$(date -u)"