aboutsummaryrefslogtreecommitdiff
blob: 5255a8510f2350ce0a3394e929b721f83a1a473a (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
#!/bin/bash
# Copyright 2011-2017 Gentoo Foundation; Distributed under the GPL v2
# might be earlier copyright, no history available

# 1) Create the tarball
# 2) Sanity check the tarball size and bail out if it appears abnormal.
# 3) create xz tarball
# 4) sign
# 5) delta generation
# 6) create some symlinks
# 7) tidy up
# 8) clean up old deltas

umask 022
source /usr/local/bin/mastermirror/rsync-gen.vars

# locations (if used in multiple files, please move to rsync-gen.vars file)
MASTER="${FINALDIR_repo_gentoo}" # the master gentoo-x86 copy
TEMP="${BASE}/snapshots-tmp/" # working directory
HISTORY="7" #number in dsays of previous snapshots to keep
DELTA_UPLOAD="${UPLOAD}/deltas/"

[[ -d ${TEMP} ]]			|| mkdir ${TEMP}
[[ -d ${UPLOAD} ]]			|| mkdir ${UPLOAD}
[[ -d ${DELTA_UPLOAD} ]]		|| mkdir ${DELTA_UPLOAD}

write_time_log "---------------------------------------------------------"
write_time_log "START ENTIRE SCRIPT	$(date -u)"

# used to name the file
DELTA_BASE=`/bin/date -d '-2 day' +%Y%m%d`
DELTA_FILENAME="portage-${DELTA_BASE}.tar.bz2"
YESTERDAY=`/bin/date -d yesterday +%Y%m%d`
FILENAME="portage-${YESTERDAY}.tar.bz2"

# Parallel compressors can use a LOT of cpu, be nice about it
NICE="nice -n 10"

SIGNKEYID=${SIGNKEYID_snapshot}

if [[ ! $(gpg -k | grep ${SIGNKEYID}) ]]; then
	echo "${SIGNKEYID} not imported! exiting"
	exit 1
fi

if [[ ! -e "${UPLOAD}/${DELTA_FILENAME}" ]]; then
	echo "Previous snapshot does not exist: '${UPLOAD}/${DELTA_FILENAME}'"
	exit 1
fi

# Find fastest BZIP2
# lbzip2,pbzip2 default to multiple threads
# emerge-delta-webrsync relies on verifying signature of tarball after
# re-compressing it with bzip2, therefore failing if the signature
# was done on lbzip2 or pbzip2 compressed tarball, #573908
for BZIP2_PROG in bzip2 FAIL ; do
	[ -n "$(type $BZIP2_PROG 2>/dev/null)" ] && break
done
if [ $BZIP2_PROG == FAIL ];  then
	echo "Could not find any BZIP2" 1>&2
	exit 1
fi
# Find fastest XZ
# pixz appends some data and leads to SIGPIPE, #573642
for XZ_PROG in xz FAIL ; do
	[ -n "$(type $XZ_PROG 2>/dev/null)" ] && break
done
if [ $XZ_PROG == FAIL ];  then
	echo "Could not find any xz" 1>&2
	exit 1
fi
# Newer 'xz' supports threads as well, but defaults to single-threaded
if $XZ_PROG --help |grep -sq threads=NUM ; then
	XZ_PROG="${XZ_PROG} -T 0"
fi

# working dir
cd ${TEMP}

# 1) Create the tarball
# create the tarball and move it to the right location
write_time_log "START TARBALL		$(date -u)"
if [ ! -f "${FILENAME%.bz2}" ]; then
	# Build exclusion list
	EXCLUSION_LIST="$(mktemp -p ${TEMP} snapshot-exclude.XXXXXXXXXX)"
	"$(dirname $0)"/print-exclusion-list.sh "${MASTER}" >"${EXCLUSION_LIST}"

	TAR_OPTIONS=(
	# Force a small block size
		--blocking-factor=1
		--record-size=512
	# GNU tar format saves approximately 1K per file in the tarball over POSIX
	# format. Multiply ~170k files, and the savings are large.
		--format=gnu
	# Sorting by name produces consistent ordering and helps compression of
	# related content. Custom ordering might further improve ordering in future
	# (eg all metadata.xml first)
		--sort=name
	# Force ownership of content:
		--owner=portage
		--group=portage
	# Excluded content:
		--no-wildcards
		--exclude-from "${EXCLUSION_LIST}"
	# Do not capture any xattr/acl info at all.
		--no-acls
		--no-xattrs
		--no-selinux
	# Include a volume ID for tracing
	# volume header is not supported by:
	# - Docker https://bugs.gentoo.org/631644
	# - tarsync https://bugs.gentoo.org/631616
	#	-V "${FILENAME%.bz2}"
	# do everything relative to the destination
		-C "${MASTER}"
		# The . needs to match the file argument
		--transform='s,^\.,portage,g'
	# The operation, destination, source arguments
		--create
		--file ${FILENAME%.bz2}
		.
	)

	tar "${TAR_OPTIONS[@]}"
	rc=$?
	if [ $rc -ne 0 ]; then
		echo "Tar run failed!"
		exit 1
	fi
	rm -f "${EXCLUSION_LIST}"
fi
[ ! -f " ${FILENAME}.umd5sum" ] && md5sum ${FILENAME%.bz2} > ${FILENAME}.umd5sum
[ ! -f "${FILENAME%.bz2}.bz2" ] && ${NICE} $BZIP2_PROG -k9 ${FILENAME%.bz2}
write_time_log "END TARBALL		$(date -u)"
# end 1)

# 2) Sanity check the tarball size and bail out if it appears abnormal.
write_time_log "START SIZE SANITY	$(date -u)"
current_size=$(stat -c '%s' "${FILENAME}")
previous_size=$(stat -c '%s' "${UPLOAD}/${DELTA_FILENAME}")
if [ ${current_size} -lt ${previous_size} ]; then
	size_difference=$(expr ${previous_size} - ${current_size})
	difference_ratio=$(expr ${previous_size} / ${size_difference})
	if [ ${difference_ratio} -lt 2 ]; then
		echo "Snapshot size has decreased by more than 50% in one day!!!"
		echo "${FILENAME} ${current_size} bytes"
		echo "${DELTA_FILENAME} ${previous_size} bytes"
		exit 1
	elif [ ${difference_ratio} -lt 5 ]; then
		echo "Snapshot size has decreased by more than 20% in one day!!!"
		echo "${FILENAME} ${current_size} bytes"
		echo "${DELTA_FILENAME} ${previous_size} bytes"
		# Make this non-fatal while we recover more space.
		#exit 1
	fi
fi
write_time_log "END SIZE SANITY		$(date -u)"
# end 2)

# 3) create xz tarball
write_time_log "START XZ		$(date -u)"
if [ ! -f "${FILENAME%.*}.xz" ] ; then
	# pixz, pxz, xz all differ in filename generation
	# xz: .tar -> .tar.xz
	# pixz: .tar -> .tpxz
	# pxz: .tar -> .txz
	#
	# To avoid this, be explicit by using IO.
	${NICE} ${XZ_PROG} -9 -e <"${FILENAME%.*}" >"${FILENAME%.*}.xz" || exit $?
fi
write_time_log "END XZ			$(date -u)"
# end 3)

# 4) sign
write_time_log "START SIGN		$(date -u)"
for f in "${FILENAME}" "${FILENAME%.*}".xz ; do
	if [ ! -f "${UPLOAD}${f}".umd5sum ]; then
		cp "${FILENAME}".umd5sum "${UPLOAD}${f}".umd5sum || exit $?
		md5sum "$f" > "$f".md5sum || exit $?
	fi
	if [ ! -f "$f".gpgsig ]; then
		gpg --batch -u "${SIGNKEYID}" --armor --detach-sign \
			--output "$f".gpgsig "$f" || exit $?
	fi
	mv "$f" "$f".md5sum "$f".gpgsig "${UPLOAD}"/ || exit $?
done
write_time_log "END SIGN		$(date -u)"
# end 4)

# 5) delta generation
write_time_log "START DELTA		$(date -u)"
PATCH=snapshot-${DELTA_BASE}-${YESTERDAY}.patch.bz2
if [ ! -f "${PATCH}" ]; then
	${NICE} ${BZIP2_PROG} -dkc ${UPLOAD}/${DELTA_FILENAME} > orig
	/usr/bin/differ -f bdelta orig ${FILENAME%.bz2} ${PATCH%.bz2}
	${NICE} ${BZIP2_PROG} -9 ${PATCH%.bz2}
	md5sum ${PATCH} > ${PATCH}.md5sum
	chmod 644 ${PATCH}{,.md5sum}
	mv ${PATCH}{,.md5sum} ${DELTA_UPLOAD}
	rm orig ${FILENAME%.bz2} "${FILENAME}".umd5sum || exit $?
fi
write_time_log "END DELTA		$(date -u)"
# end 5)

# 6) create some symlinks
write_time_log "START SYMLINK		$(date -u)"
cd ${UPLOAD}
for f in "${FILENAME}" "${FILENAME%.*}".xz ; do
	ext=${f##*.}
	ln -sf "$f" "${UPLOAD}"portage-latest.tar.${ext} || exit $?
	rm -f "${UPLOAD}"portage-latest.tar.${ext}.md5sum || exit $?
	sed "s/${f}\$/portage-latest.tar.${ext}/" "${UPLOAD}"${f}.md5sum > \
		"${UPLOAD}"portage-latest.tar.${ext}.md5sum || exit $?
	ln -sf "${f}".gpgsig "${UPLOAD}"portage-latest.tar.${ext}.gpgsig || exit $?
done
write_time_log "END SYMLINK		$(date -u)"
# end 6)

# 7) tidy up
write_time_log "START CLEANUP		$(date -u)"
/usr/bin/find ${UPLOAD} -maxdepth 1 -mtime +${HISTORY} -type f | /usr/bin/xargs /bin/rm -f
write_time_log "END CLEANUP		$(date -u)"
# end 7)

# 8) clean up old deltas
write_time_log "START CLEANUP DELTA	$(date -u)"
/usr/local/bin/mastermirror/clean-old-deltas.py "${DELTA_UPLOAD}" "${YESTERDAY}" $(stat -c '%s' "${UPLOAD}/${FILENAME}") > /dev/null
write_time_log "END CLEANUP DELTA	$(date -u)"
# end 8)
write_time_log "END ENTIRE SCRIPT	$(date -u)"