aboutsummaryrefslogtreecommitdiff
blob: 499ace96fb51401e6c800e39b98e5451a0df2220 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
#!/bin/bash
# Git export script
# distributed by puppet

# this takes about 1.5mins to run the first time. <45s thereafter

# if the lv is really small (2G), the default inode ratio is too large.
# eg. mkfs.ext2 -i 4096 /dev/vg/var_mastermirror-staging

# Notes 2015/12/14 by robbat2
# This entire script was run on the CVS server directly, as it was MUCH faster
# than remote checkouts. In the era of Git, this script can probably be merged
# with rsync-gen.sh instead.

GITROOT_file="file:///var/gitroot"
GITROOT_http="https://anongit.gentoo.org/git/"
case $HOSTNAME in
	oystercatcher|swan|manakin) export GITROOT=${GITROOT_file} ;;
	*) export GITROOT=${GITROOT_http} ;;
esac

GIT="/usr/bin/git"
GIT_CL="${GIT} clone -q"
GIT_PL="${GIT} pull -q --ff-only --no-stat --no-rebase"

WGET="/usr/bin/wget --timeout=2 --quiet --timestamping"

GIT_RESTORE_MTIME="$(find /usr/lib/python-exec/python*/ -maxdepth 1 -name git-restore-mtime |sort -r |head -n1)"
GIT_RESTORE_MTIME_ARGS="-q -m -c --first-parent"

rc=0
for cmdname in \
	GIT_RESTORE_MTIME \
	WGET \
	GIT \
	; do
	cmd=${!cmdname}
	cmd=${cmd/ *}
	if [[ -z "$cmd" ]] || [[ ! -x "$cmd" ]]; then
		echo "Missing tool ($cmdname): $cmd" 1>&2
		rc=1
	fi
done
[[ $rc -ne 0 ]] && exit 1

STAGING_DIR="${1:-/var/tmp/mastermirror-staging}"
STAGING_DIR_glsa="${STAGING_DIR}/glsa"
STAGING_DIR_dtd="${STAGING_DIR}/dtd"
STAGING_DIR_xmlschema="${STAGING_DIR}/xml-schema"
STAGING_DIR_news="${STAGING_DIR}/gentoo-news"
STAGING_DIR_gentoo="${STAGING_DIR}/gentoo-x86"
STAGING_DIR_projects="${STAGING_DIR}/projects"
REPO_gentoo=${GITROOT}/repo/gentoo.git
REPO_news=${GITROOT}/data/gentoo-news.git
REPO_dtd=${GITROOT}/data/dtd.git
REPO_xmlschema=${GITROOT}/data/xml-schema.git
REPO_glsa=${GITROOT}/data/glsa.git

PROJECTS_XML_URI=https://api.gentoo.org/metastructure/projects.xml

# gitattributes
ATTRIB_gentoo=(
  '* -ident'
)

# Ensure all files are readable
umask 0022

fetch_git() {
	targetdir=$1
	repo=$2
	shift 2
	local timestampfile=${timestampfile:-timestamp.commit}
	gitdir=${targetdir}.git
	timestampfile=${targetdir}/$timestampfile
	[ -f "$timestampfile" ] && rm -f "$timestampfile"
	if [[ ! -d "${targetdir}" ]] ; then
		# not checked out yet, run initial co
		mkdir -p "${targetdir}.git" "$gitdir" && \
			$GIT init -q \
				--separate-git-dir "$gitdir"
				"${targetdir}" && \
			GIT_DIR=${gitdir} $GIT remote add origin "$repo"
	fi
	attribfile=${gitdir}/info/attributes
	for line in "$@" ; do
		fgrep -sq -e "$line" "$attribfile" ||
			echo "$line" >>"$attribfile"
	done
	excludefile=${gitdir}/info/exclude
	for line in timestamp.commit timestamp.chk timestamp.x timestamp ; do
		fgrep -sq -e "$line" "$excludefile" ||
			echo "$line" >>"$excludefile"
	done
	cd "${targetdir}" || return 1
	$GIT fetch -q --force origin master || return 1
	$GIT reset -q --hard origin/master || return 1
	$GIT_RESTORE_MTIME $GIT_RESTORE_MTIME_ARGS || return 1
	TZ=UTC $GIT log -1 --date=iso-strict-local --format='%H %ct %cd' >${timestampfile}
}

fetch_uri() {
	targetfile=$1
	uri=$2
	targetdir=$(dirname $(readlink -m $targetfile))
	[[ ! -d ${targetdir} ]] &&  mkdir -p ${targetdir}
	tmpfile=$(mktemp -p $targetdir)
	${WGET} "${uri}" -O $tmpfile && chmod a+r $tmpfile && mv -f "$tmpfile" "$targetfile"
	rc=$?
	rm -f "$tmpfile"
	return $rc
}

# TODO: we should probably be checking out to a directory and only swapping on
# success of ALL checkouts and fetches.
rc_sum=0

# repo/gentoo (formerly CVS gentoo-x86)
timestampfile=metadata/timestamp.commit fetch_git ${STAGING_DIR_gentoo} ${REPO_gentoo} "${ATTRIB_gentoo[@]}"
rc=$?
rc_sum=$((rc_sum + $rc))

# gentoo-news
fetch_git ${STAGING_DIR_news} ${REPO_news}
rc=$?
rc_sum=$((rc_sum + $rc))

# projects.xml
fetch_uri ${STAGING_DIR_projects}/projects.xml "${PROJECTS_XML_URI}"
rc=$?
rc_sum=$((rc_sum + $rc))

# dtd
fetch_git ${STAGING_DIR_dtd} ${REPO_dtd}
rc=$?
rc_sum=$((rc_sum + $rc))

# xml-schema
fetch_git ${STAGING_DIR_xmlschema} ${REPO_xmlschema}
rc=$?
rc_sum=$((rc_sum + $rc))

# glsa
fetch_git ${STAGING_DIR_glsa} ${REPO_glsa}
rc=$?
rc_sum=$((rc_sum + $rc))

exit $rc_sum