blob: af45fc0a2c1b9d3737ce4d27de414f2a3e047e38 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
|
#!/bin/sh
# download current package indexes to data/<series>{,-proposed}/ for running
# britney against a PPA. The PPA will play the role of "-proposed" (i. e.
# "unstable" in britney terms, containing the updated packages to test), the
# Ubuntu archive has the "-release" part (i. e. "testing" in britney terms, in
# which the -proposed packages are being landed).
#
# Author: Martin Pitt <martin.pitt@ubuntu.com>
# Author: Robert Bruce Park <robert.park@canonical.com>
set -u
export ETC="/etc/bileto"
export OVERLAY="stable-phone-overlay"
export ARCHES="i386 amd64"
export PORTS_ARCHES="armhf arm64 ppc64el"
AMQP_URI=$(cat $ETC/amqp_uri)
TEAM=$(cat $ETC/ppa_team)
export AMQP_URI TEAM
BRITNEY=/var/lib/britney/britney.py
CODEDIR=$(dirname "$(readlink -f "$0")")
ROOTDIR=$(dirname "$CODEDIR")
DATADIR=/tmp/britney_data
OUTDIR=/tmp/britney_output
HINTDIR=/tmp/britney_hints
CACHE=$DATADIR/CACHE
ARCHIVE=http://archive.ubuntu.com/ubuntu/dists
PORTS=http://ports.ubuntu.com/dists
. $ROOTDIR/scripts/funcs.sh
usage() {
cat <<EOF
Usage: $0 SERIES [PPA TICKETID]
$1
SERIES:
Ubuntu release codename, eg trusty, xenial, etc.
PPA (optional):
Name of PPA to fetch; defaults to stable-phone-overlay.
TICKETID (optional, but must be specified with PPA):
Bileto Ticket # that we are acting on.
EOF
exit 1
}
# Download files in parallel in background, only if there is an update
refresh() {
DIR=$CACHE/$pocket/$(echo $1 | rev | cut --delimiter=/ --fields=2,3 | rev)
mkdir --parents $DIR
touch --no-create $CACHE $CACHE/$pocket "$(dirname $DIR)" $DIR # Timestamp thwarts expire.sh
wget --directory-prefix $DIR --timestamping $1 --append-output $DIR/$$-wget-log --no-verbose &
}
[ $# -gt 0 ] && export RAW_SERIES="$1" && shift || usage "Missing series."
SERIES="$(echo $RAW_SERIES | sed 's/-overlay//')"
export SERIES
case "$#" in
2)
export PPA="$1"
export TICKETID="$2"
;;
0)
export PPA="$OVERLAY"
;;
*)
usage "Wrong number of arguments (must be 1 or 3)."
;;
esac
PPA_URL=http://ppa.launchpad.net/$TEAM/$PPA/ubuntu/dists/$SERIES/main
# Enable s390x only for xenial or newer.
case "$SERIES" in
precise|trusty)
export NEW_ARCHES=" "
export PORTS_ARCHES="$PORTS_ARCHES powerpc"
;;
xenial|yakkety)
export NEW_ARCHES="s390x"
export PORTS_ARCHES="$PORTS_ARCHES powerpc s390x"
;;
*)
export NEW_ARCHES=" "
export PORTS_ARCHES="$PORTS_ARCHES s390x"
;;
esac
log 'Refreshing package indexes...'
# Get archive bits
if [ "$PPA" = "$OVERLAY" ]; then
for pocket in $SERIES $SERIES-updates $SERIES-proposed; do
for component in main restricted universe multiverse; do
for arch in $ARCHES; do
refresh $ARCHIVE/$pocket/$component/binary-$arch/Packages.gz
done
for arch in $PORTS_ARCHES; do
refresh $PORTS/$pocket/$component/binary-$arch/Packages.gz
done
refresh $ARCHIVE/$pocket/$component/source/Sources.gz
done
done
fi
# Get ppa bits
pocket=$PPA-$SERIES
for arch in $ARCHES $PORTS_ARCHES; do
refresh $PPA_URL/binary-$arch/Packages.gz
done
refresh $PPA_URL/source/Sources.gz
# Get autopkgtest history so that we can correctly identify regressions
DIR=$DATADIR/$SERIES
wget --directory-prefix $DIR --timestamping http://people.canonical.com/~ubuntu-archive/proposed-migration/$SERIES/results.cache --append-output $DIR/$$-wget-log --no-verbose &
wait # for wgets to finish
find $DATADIR -name "$$-wget-log*" -exec cat '{}' \; -delete 1>&2
log 'Building britney indexes...'
if [ "$PPA" = "$OVERLAY" ]; then
# There's one "testing" with overlay PPA...
DEST=$DATADIR/$SERIES-overlay
mkdir --parents $DEST
touch --no-create $DEST
zcat $CACHE/$SERIES*/*/source/Sources.gz \
$CACHE/$OVERLAY-$SERIES/*/source/Sources.gz \
> $DEST/Sources
for arch in $ARCHES $PORTS_ARCHES; do
zcat $CACHE/$SERIES*/*/binary-$arch/Packages.gz \
$CACHE/$OVERLAY-$SERIES/*/binary-$arch/Packages.gz \
> $DEST/Packages_${arch}
done
# ... and a second "testing" without overlay PPA.
DEST=$DATADIR/$SERIES
mkdir --parents $DEST
touch --no-create $DEST
zcat $CACHE/$SERIES*/*/source/Sources.gz > $DEST/Sources
for arch in $ARCHES $PORTS_ARCHES; do
zcat $CACHE/$SERIES*/*/binary-$arch/Packages.gz > $DEST/Packages_${arch}
done
# Delete *directories* leftover from previous run.
rm --recursive --force ${OUTDIR:?}/*/
else
set_swift_creds
# "Unstable" is just silo PPA
DEST=$DATADIR/$PPA-$SERIES
mkdir --parents $DEST "$OUTDIR/$BRITNEY_TIMESTAMP/"
touch --no-create $DEST
loudly ln --verbose --symbolic --force --no-dereference $HINTDIR $DEST/Hints
zcat $CACHE/$PPA-$SERIES/*/source/Sources.gz > $DEST/Sources
for arch in $ARCHES $PORTS_ARCHES; do
zcat $CACHE/$PPA-$SERIES/*/binary-$arch/Packages.gz > $DEST/Packages_${arch}
done
touch $DEST/Blocks
touch "$DATADIR/$SERIES/Dates"
# Create config file atomically.
CONFIG="$DEST.conf"
envsubst "$CONFIG" < "$CODEDIR/britney.conf.in"
log 'Running britney...'
loudly $BRITNEY -v --config "$CONFIG" --series $SERIES | sed "s#$AMQP_URI#AMQP_URI#"
log 'Uploading to swift...'
files=""
container="bileto-$TICKETID-excuses"
for filename in $OUTDIR/$BRITNEY_TIMESTAMP/${PPA}_${SERIES}_*; do
base="$(echo $filename | sed "s#$OUTDIR/##")"
confirmed="$(stdout swift upload "$container" "$filename" --object-name="$base")"
files="$files\n$SWIFT_BASE/$container/$confirmed"
done
set_swift_readable "$container" || true
# shellcheck disable=SC2059
printf "$files\n" >> $OUTDIR/$BRITNEY_TIMESTAMP/$TICKETID.manifest
fi
log "$0 done."
|
