| ... |
... |
@@ -80,6 +80,29 @@ PROCESSDIR="${base}/morgue" |
|
80
|
80
|
FARMBASE="/srv/snapshot.debian.org/farm"
|
|
81
|
81
|
FARMURL="http://snapshot.debian.org/file/"
|
|
82
|
82
|
PROGRAM="link_morgue"
|
|
|
83
|
+DBHOST="lw08.debian.org"
|
|
|
84
|
+HASHFILE="${dbdir}/hashes"
|
|
|
85
|
+NOW=$(date -Is)
|
|
|
86
|
+
|
|
|
87
|
+# We have to prepare our file with list of hashes. We get it directly
|
|
|
88
|
+# from the snapshot db. Thats a costly operation taking some 15 or so
|
|
|
89
|
+# minutes, but still better than the rate limiting we run into when
|
|
|
90
|
+# using the web api.
|
|
|
91
|
+#
|
|
|
92
|
+# The preparehashes is an otion the ssh forced command on the remote
|
|
|
93
|
+# host uses to generate a nice file with hashes, one per line. It does
|
|
|
94
|
+# so by running something like "psql service=snapshot-guest -c "select
|
|
|
95
|
+# hash from file" > somefile", then packs the file. To not stress the
|
|
|
96
|
+# db host too much with that query, it only refreshes the file if its
|
|
|
97
|
+# older than 24 hours.
|
|
|
98
|
+ssh ${DBHOST} preparehashes
|
|
|
99
|
+
|
|
|
100
|
+# And now we get us the file here, so we can easily lookup hashes.
|
|
|
101
|
+# (the rsync uses the same ssh key and runs into the forced command.
|
|
|
102
|
+# That just knows to send the file for rsync instead of preparing it.)
|
|
|
103
|
+cd "${dbdir}"
|
|
|
104
|
+rsync ${DBHOST}:/srv/ftp-master.debian.org/home/hashes.gz ${HASHFILE}.gz
|
|
|
105
|
+gunzip --keep ${HASHFILE}.gz
|
|
83
|
106
|
|
|
84
|
107
|
cd "${PROCESSDIR}"
|
|
85
|
108
|
log "Processing ${PROCESSDIR}"
|
| ... |
... |
@@ -111,14 +134,17 @@ while read mfile; do |
|
111
|
134
|
ln -sf "${FARMBASE}/${LVL1}/${LVL2}/${mshasum}" "${mfile}"
|
|
112
|
135
|
fi
|
|
113
|
136
|
else
|
|
114
|
|
- # If we run wherever, use curl and the http interface
|
|
115
|
|
- if out=$(curl --fail --show-error --silent --max-time 120 --head ${FARMURL}/${mshasum} 2>&1); then
|
|
|
137
|
+ # Now lookup the hash. stop after first hit, its shasums, it
|
|
|
138
|
+ # *shouldnt* list multiple. Also, even if it does, we don*t
|
|
|
139
|
+ # care. It shows us snapshot has it, which is all we care
|
|
|
140
|
+ # about.
|
|
|
141
|
+ if grep --max-count=1 ${mshasum} ${HASHFILE}; then
|
|
116
|
142
|
# Yes, lets symlink it
|
|
117
|
143
|
# Yay for tons of dangling symlinks, but when this is done a rsync
|
|
118
|
144
|
# will run and transfer the whole shitload of links over to the morgue host.
|
|
119
|
145
|
ln -sf "${FARMBASE}/${LVL1}/${LVL2}/${mshasum}" "${mfile}"
|
|
120
|
146
|
else
|
|
121
|
|
- echo $out > "${mfile}.nosnapshot" || true
|
|
|
147
|
+ echo "No shasum found for ${mfile} at ${NOW}" > "${mfile}.nosnapshot" || true
|
|
122
|
148
|
fi
|
|
123
|
149
|
fi
|
|
124
|
150
|
done # for mfile in...
|