Skip to content

Commit bc00036

Browse files
yvanzoyvanzo
authored
Merge pull request #61 from metabrainz/solrcloud9backups
Add helper scripts to handle Solr backup tar archives
2 parents bdd8d7e + cd6d832 commit bc00036

7 files changed

Lines changed: 454 additions & 18 deletions

File tree

‎Dockerfile‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,8 @@ USER root
3939
ARG DEBIAN_FRONTEND=noninteractive
4040
RUN apt-get update && \
4141
apt-get install -y --no-install-recommends \
42+
# Needed to parse API JSON output in scripts
43+
jq \
4244
# Needed to prepare uploading configsets
4345
zip \
4446
# Needed to decompress search index dumps
@@ -64,7 +66,9 @@ RUN cd /usr/lib/mbsssss && \
6466
cd /usr/lib/mbsssss/"$conf_dir" && \
6567
zip -r /usr/lib/mbsssss/"$core_name".zip * && \
6668
chown solr:solr /usr/lib/mbsssss/"$core_name".zip * || exit 1; \
67-
done
69+
done && \
70+
mkdir -p -m0770 /var/cache/musicbrainz/solr-backups && \
71+
chown -R "$SOLR_USER:0" /var/cache/musicbrainz/solr-backups
6872

6973
COPY --chmod=0755 \
7074
./docker/entrypoint-initdb.d/* \
@@ -90,5 +94,7 @@ LABEL org.label-schema.build-date="${BUILD_DATE}" \
9094
org.metabrainz.builder-image="maven:${MAVEN_TAG}" \
9195
org.metabrainz.mb-solr.version="${MB_SOLR_VERSION}"
9296

97+
VOLUME /var/cache/musicbrainz/solr-backups
98+
9399
# Restoring value set in the parent image
94100
USER solr

‎HACKING.md‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -73,8 +73,8 @@ Solr Admin
7373
To debug how other components (MBS, SIR) interact with the search server,
7474
the Solr Admin web interface (browsable from <http://localhost:8983>) is your friend.
7575

76-
See “[Using the Solr Administration User Interface](https://solr.apache.org/guide/7_7/using-the-solr-administration-user-interface.html)”
77-
in the Apache Solr Reference Guide 7.7 for more information.
76+
See “[Getting Started / Solr Admin UI](https://solr.apache.org/guide/solr/latest/getting-started/solr-admin-ui.html)"
77+
in the Apache Solr Reference Guide 9.7 for more information.
7878

7979
### Query screen
8080

@@ -115,8 +115,8 @@ These formats are incompatible with the “fl” field and the
115115
`debugQuery` and `explainOther` options, so setting any of these
116116
will not change the output.
117117

118-
See “[Query Screen](https://solr.apache.org/guide/7_7/query-screen.html#query-screen)”
119-
in Apache Solr Reference Guide 7.7 for more information.
118+
See “[Query Screen](https://solr.apache.org/guide/solr/9_7/query-guide/query-screen.html)"
119+
in Apache Solr Reference Guide 9.7 for more information.
120120

121121
#### Parameter priority
122122

‎docker/scripts/create-musicbrainz-collections‎

Lines changed: 25 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -77,24 +77,36 @@ for SUBDIR in $SUBDIRS; do
7777

7878
# Make the curl PUT request
7979
echo "Uploading $ZIP_FILE to $UPLOAD_URL/$SUBDIR..."
80-
if curl -sS -X PUT --header "Content-Type: application/octet-stream" --data-binary @"$ZIP_FILE" "$UPLOAD_URL/$SUBDIR"; then
81-
echo
80+
upload_output="$(curl -sS -X PUT --header "Content-Type: application/octet-stream" --data-binary @"$ZIP_FILE" "$UPLOAD_URL/$SUBDIR")"
81+
if [[ "$(echo "$upload_output" | jq .responseHeader.status)" =~ 0 ]]; then
8282
echo "Upload successful."
8383

84+
case "$SUBDIR" in
85+
'recording') SHARDS_COUNT=4;;
86+
*) SHARDS_COUNT=1;;
87+
esac
8488
echo "Creating collection $SUBDIR..."
85-
curl -sS -X POST "$SOLR_BASE_URL/api/collections" \
86-
-H 'Content-Type: application/json' \
87-
--data-binary "{
88-
\"name\": \"${SUBDIR}\",
89-
\"config\": \"${SUBDIR}\",
90-
\"numShards\": 1,
91-
\"waitForFinalState\": true
92-
}"
93-
echo
94-
echo "Collection creation successful."
89+
creation_output="$(
90+
curl -sS -X POST "$SOLR_BASE_URL/api/collections" \
91+
-H 'Content-Type: application/json' \
92+
--data-binary "{
93+
\"name\": \"${SUBDIR}\",
94+
\"config\": \"${SUBDIR}\",
95+
\"numShards\": $SHARDS_COUNT,
96+
\"waitForFinalState\": true
97+
}"
98+
)"
99+
if [[ "$(echo "$creation_output" | jq .responseHeader.status)" =~ 0 ]]; then
100+
echo "$creation_output" | jq .success
101+
echo "Collection creation successful."
102+
else
103+
echo >&2 "Collection creation failed for $SUBDIR. Leaving as is for debugging."
104+
echo "$creation_output" | jq >&2
105+
exit 70 # EX_SOFTWARE
106+
fi
95107
else
96-
echo
97108
echo >&2 "Upload failed for $ZIP_FILE. Keeping the file for debugging."
109+
echo "$upload_output" | jq >&2
98110
exit 70 # EX_SOFTWARE
99111
fi
100112
echo

‎docker/scripts/delete-indexed-documents‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
#!/usr/bin/env bash
2+
3+
set -e -o pipefail -u
4+
5+
SCRIPT_NAME=$(basename "$0")
6+
HELP=$(cat <<EOH
7+
Usage: $SCRIPT_NAME --all
8+
or: $SCRIPT_NAME COLLECTION... (for example: $SCRIPT_NAME artist release)
9+
or: $SCRIPT_NAME --help
10+
11+
For all or each of the given MusicBrainz Solr collections,
12+
delete the indexed documents.
13+
14+
Pre-requisites:
15+
Solr must be running (in SolrCloud mode).
16+
EOH
17+
)
18+
19+
# Parse arguments
20+
21+
if [ $# -eq 0 ]
22+
then
23+
echo >&2 "$SCRIPT_NAME: missing argument"
24+
echo >&2 "Try '$SCRIPT_NAME --help' for usage."
25+
exit 64 # EX_USAGE
26+
elif [[ "$*" =~ ^-*h(elp)?$ ]]
27+
then
28+
echo "$HELP"
29+
exit 0 # EX_OK
30+
elif [[ $# -eq 1 && $1 =~ ^-*all$ ]]
31+
then
32+
collections=()
33+
else
34+
collections=("$@")
35+
fi
36+
37+
# Check pre-requisites
38+
39+
max_try=${MAX_TRY:-12}
40+
wait_seconds=${WAIT_SECONDS:-5}
41+
local_tries=1
42+
while [[ -f $SOLR_HOME/musicbrainz-solrcloud-first-run && $local_tries -lt $max_try ]]
43+
do
44+
local_tries=$((local_tries + 1))
45+
sleep "$wait_seconds"
46+
done
47+
if [[ -f $SOLR_HOME/musicbrainz-solrcloud-first-run ]] # $local_tries -eq $max_try
48+
then
49+
echo >&2 "$SCRIPT_NAME: Solr first run is not complete (after waiting for $((max_try * wait_seconds)) seconds)"
50+
echo >&2 "Try '$SCRIPT_NAME help' for pre-requisites."
51+
exit 69 # EX_UNAVAILABLE
52+
fi
53+
if ! wait-for-solr.sh --max-attempts "$max_try" --wait-seconds "$wait_seconds"
54+
then
55+
echo >&2 "$SCRIPT_NAME: Solr is not running (after waiting for $((max_try * wait_seconds)) seconds)"
56+
echo >&2 "Try '$SCRIPT_NAME help' for pre-requisites."
57+
exit 69 # EX_UNAVAILABLE
58+
fi
59+
60+
# List all the collections if needed
61+
62+
if [[ ${#collections[@]} -eq 0 ]]
63+
then
64+
readarray -t collections < <(find '/usr/lib/mbsssss' \
65+
-maxdepth 1 -mindepth 1 -type d \
66+
-not -name 'lib' -not -name 'common' -not -name '_template' \
67+
-print0 | xargs -0 -n 1 basename | sort)
68+
fi
69+
70+
# For each collection, delete all the indexed documents
71+
72+
COLLECTIONS_TOTAL_COUNT=${#collections[@]}
73+
collection_index=0
74+
for collection in "${collections[@]}"
75+
do
76+
collection_index=$((collection_index + 1))
77+
echo -n "Posting deletion query for the collection ($collection_index/$COLLECTIONS_TOTAL_COUNT) '$collection'... "
78+
solr post \
79+
--name "$collection" \
80+
--mode args \
81+
--type application/xml \
82+
'<delete><query>*:*</query></delete>' \
83+
2> >(grep -v deprecated >&2)
84+
echo Done
85+
done
86+
87+
echo "Successfully posted all the deletion queries."
88+
89+
# vi: set noexpandtab softtabstop=0:

‎docker/scripts/fetch-backup-archives‎

Lines changed: 163 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,163 @@
1+
#!/usr/bin/env bash
2+
3+
set -e -o pipefail -u
4+
5+
DB_DUMP_DIR=/media/dbdump
6+
SEARCH_DUMP_DIR=/var/cache/musicbrainz/solr-backups
7+
BASE_DOWNLOAD_URL="${MUSICBRAINZ_BASE_DOWNLOAD_URL:-https://data.metabrainz.org/pub/musicbrainz}"
8+
WGET_CMD=(wget)
9+
10+
SCRIPT_NAME=$(basename "$0")
11+
HELP=$(cat <<EOH
12+
Usage: $SCRIPT_NAME [<options>]
13+
14+
Fetch backup archives of the MusicBrainz SolrCloud collections.
15+
16+
Options:
17+
--base-download-url <url> Specify URL of a MetaBrainz/MusicBrainz download server.
18+
(Default: '$BASE_DOWNLOAD_URL')
19+
--wget-options <wget options> Specify additional options to be passed to wget,
20+
these should be separated with whitespace,
21+
the list should be a single argument
22+
(escape whitespaces if needed).
23+
-h, --help Print this help message and exit.
24+
25+
Environment:
26+
MUSICBRAINZ_BASE_DOWNLOAD_URL Default URL for MetaBrainz/MusicBrainz download server.
27+
EOH
28+
)
29+
30+
# Parse arguments
31+
32+
while [[ $# -gt 0 ]]
33+
do
34+
case "$1" in
35+
--base-download-url )
36+
shift
37+
BASE_DOWNLOAD_URL="${1%/data/fullexport/}"
38+
if ! [[ $BASE_DOWNLOAD_URL =~ ^(ftp|https?):// ]]
39+
then
40+
echo >&2 "$SCRIPT_NAME: --base-download-url must begin with ftp://, http:// or https://"
41+
exit 64 # EX_USAGE
42+
fi
43+
;;
44+
--wget-options )
45+
shift
46+
IFS=' ' read -r -a WGET_OPTIONS <<< "$1"
47+
WGET_CMD+=("${WGET_OPTIONS[@]}")
48+
unset WGET_OPTIONS
49+
;;
50+
-h | --help )
51+
echo "$HELP"
52+
exit 0 # EX_OK
53+
;;
54+
-* )
55+
echo >&2 "$SCRIPT_NAME: unrecognized option '$1'"
56+
echo >&2 "Try '$SCRIPT_NAME --help' for usage."
57+
exit 64 # EX_USAGE
58+
;;
59+
* )
60+
echo >&2 "$SCRIPT_NAME: unrecognized argument '$1'"
61+
echo >&2 "Try '$SCRIPT_NAME --help' for usage."
62+
exit 64 # EX_USAGE
63+
;;
64+
esac
65+
shift
66+
done
67+
68+
# Show information about signing up for data use
69+
70+
if [[ ! -a "$DB_DUMP_DIR/.for-commercial-use" &&
71+
! -a "$DB_DUMP_DIR/.for-non-commercial-use" &&
72+
! -a "$SEARCH_DUMP_DIR/.for-commercial-use" &&
73+
! -a "$SEARCH_DUMP_DIR/.for-non-commercial-use" ]]
74+
then
75+
prompt=$(cat <<-EOQ
76+
The data you are about to download is provided by the MetaBrainz Foundation.
77+
Are you planning to use this data for commercial or business purposes?
78+
(y/n)
79+
EOQ
80+
)
81+
read -e -p "$prompt " -r
82+
while [[ ! ${REPLY:0:1} =~ [YNyn] ]]
83+
do
84+
read -e -p "Invalid reply. Yes or no? " -r
85+
done
86+
echo
87+
if [[ ${REPLY:0:1} =~ [Yy] ]]
88+
then
89+
prompt=$(cat <<-EOQ
90+
The MetaBrainz Foundation is supported by commercial users of our data and
91+
through end-user donations. If you are using our data in a commercial context,
92+
we require you to support MetaBrainz financially in order for us ensure the
93+
availability of these datasets in the future.
94+
95+
Please sign up at https://metabrainz.org/supporters/account-type
96+
97+
[Press any key when OK]
98+
EOQ
99+
)
100+
read -e -N 1 -p "$prompt" -r -s
101+
echo OK
102+
touch "$SEARCH_DUMP_DIR/.for-commercial-use"
103+
else
104+
prompt=$(cat <<-EOQ
105+
Could you please sign up at https://metabrainz.org/supporters/account-type
106+
(for free!) so that we may better understand how our data is being used?
107+
108+
We also encourage our non-commercial users who can afford it to make a donation
109+
to the MetaBrainz Foundation so that we may continue our mission:
110+
https://metabrainz.org/donate
111+
112+
[Press any key when OK]
113+
EOQ
114+
)
115+
read -e -N 1 -p "$prompt" -r -s
116+
echo OK
117+
touch "$SEARCH_DUMP_DIR/.for-non-commercial-use"
118+
fi
119+
fi
120+
121+
# Check timestamp for previously downloaded archives
122+
123+
echo "$(date): 1/3: Checking timestamps of available archives..."
124+
125+
PREVIOUS_DUMP_TIMESTAMP=''
126+
if [[ -a "$SEARCH_DUMP_DIR/LATEST" ]]
127+
then
128+
PREVIOUS_DUMP_TIMESTAMP=$(<"$SEARCH_DUMP_DIR/LATEST")
129+
rm -f "$SEARCH_DUMP_DIR/LATEST"
130+
fi
131+
132+
# Check timestamp for remotely available archives
133+
134+
"${WGET_CMD[@]}" -nd -nH -P "$SEARCH_DUMP_DIR" \
135+
"${BASE_DOWNLOAD_URL}/data/solr-backups/LATEST"
136+
DUMP_TIMESTAMP=$(<"$SEARCH_DUMP_DIR/LATEST")
137+
138+
# Remove previously downloaded files if new archives are available
139+
140+
if [[ $PREVIOUS_DUMP_TIMESTAMP != "$DUMP_TIMESTAMP" ]]
141+
then
142+
find "$SEARCH_DUMP_DIR" \
143+
! -path "$SEARCH_DUMP_DIR" \
144+
! -path "$SEARCH_DUMP_DIR/.for-commercial-use" \
145+
! -path "$SEARCH_DUMP_DIR/.for-non-commercial-use" \
146+
! -path "$SEARCH_DUMP_DIR/LATEST" \
147+
-delete
148+
fi
149+
150+
# Actually fetch latest MusicBrainz Solr backup archives
151+
152+
echo "$(date): 2/3: Fetching MusicBrainz Solr backup archives..."
153+
"${WGET_CMD[@]}" -nd -nH -c -r -P "$SEARCH_DUMP_DIR" \
154+
--accept 'MD5SUMS,*.tar.zst' --no-parent --relative \
155+
"${BASE_DOWNLOAD_URL}/data/solr-backups/$DUMP_TIMESTAMP/"
156+
157+
# Check archives integrity
158+
159+
echo "$(date): 3/3: Checking MD5 sums..."
160+
cd "$SEARCH_DUMP_DIR" && md5sum -c MD5SUMS && cd - >/dev/null
161+
162+
echo "$(date): Done fetching MusicBrainz Solr backup archives."
163+
# vi: set noexpandtab softtabstop=0:

0 commit comments

Comments
 (0)