|
| 1 | +#!/bin/bash |
| 2 | + |
| 3 | +set -e |
| 4 | +source ./scripts/solr_builder/utils.sh |
| 5 | + |
| 6 | +OL_HOME="ol-home0.us.archive.org" |
| 7 | +SOLR_SERVER=$1 |
| 8 | +SOLR_HOST="$(ol_server $SOLR_SERVER)" |
| 9 | +DUMP_FILE="${SOLR_SERVER}_dump.tar.gz" |
| 10 | +DUMP_OFFSET_FILE="${SOLR_SERVER}_dump.offset" |
| 11 | + |
| 12 | +if [ "$SOLR_SERVER" == "ol-solr0" ]; then |
| 13 | + solr_container="openlibrary-solr-1" |
| 14 | + solr_updater_container="openlibrary-solr-updater-1" |
| 15 | + offset_file="solr-update.offset" |
| 16 | +elif [ "$SOLR_SERVER" == "ol-solr1" ]; then |
| 17 | + solr_container="solr_builder-solr_prod-1" |
| 18 | + solr_updater_container="openlibrary-solr-next-updater-1" |
| 19 | + offset_file="solr-next-update.offset" |
| 20 | +else |
| 21 | + echo "Invalid SOLR_SERVER: $SOLR_SERVER" |
| 22 | + exit 1 |
| 23 | +fi |
| 24 | + |
| 25 | +# Make sure it has ubuntu xenial and openlibrary/olbase:latest |
| 26 | +echo "Pulling down images on $SOLR_SERVER ... " |
| 27 | +ssh -t $SOLR_SERVER " |
| 28 | + set -e |
| 29 | + docker pull ubuntu:xenial |
| 30 | + docker pull openlibrary/olbase:latest |
| 31 | +" |
| 32 | + |
| 33 | +echo "Check if either solr-dump container is running" |
| 34 | +echo -n " solr-dump ... " |
| 35 | +solr_dump_running=$(ssh $SOLR_SERVER "docker ps --quiet --filter name=solr-dump") |
| 36 | +if [ -n "$solr_dump_running" ]; then |
| 37 | + echo "✗ ERROR: $SOLR_SERVER has a solr-dump container running" |
| 38 | + exit 1 |
| 39 | +else |
| 40 | + echo "✓" |
| 41 | +fi |
| 42 | +echo -n " solr-dump-upload ... " |
| 43 | +solr_dump_upload_running=$(ssh $SOLR_SERVER "docker ps --quiet --filter name=solr-dump-upload") |
| 44 | +if [ -n "$solr_dump_upload_running" ]; then |
| 45 | + echo "✗ ERROR: $SOLR_SERVER has a solr-dump-upload container running" |
| 46 | + exit 1 |
| 47 | +else |
| 48 | + echo "✓" |
| 49 | +fi |
| 50 | + |
| 51 | + |
| 52 | +# Clean up old dumps |
| 53 | +echo "Cleaning up old dumps on $SOLR_SERVER ... " |
| 54 | +ssh $SOLR_SERVER " |
| 55 | + set -e |
| 56 | + rm -f /tmp/solr/$DUMP_FILE || true |
| 57 | + rm -f /tmp/solr/$DUMP_OFFSET_FILE || true |
| 58 | +" |
| 59 | + |
| 60 | + |
| 61 | +# Check if enough disk space |
| 62 | +echo -n "Checking disk space on $SOLR_SERVER ... " |
| 63 | +var_solr_size=$(ssh $SOLR_SERVER " |
| 64 | + set -e |
| 65 | + docker run --rm \ |
| 66 | + --volumes-from $solr_container \ |
| 67 | + ubuntu:xenial \ |
| 68 | + du -s /var/solr | awk '{print \$1}' |
| 69 | +") |
| 70 | +empty_space=$(ssh $SOLR_SERVER " |
| 71 | + set -e |
| 72 | + df /tmp | tail -n 1 | awk '{print \$4}' |
| 73 | +") |
| 74 | +var_solr_size_str=$(numfmt --to=iec --suffix=B $((var_solr_size * 1000))) |
| 75 | +empty_space_str=$(numfmt --to=iec --suffix=B $((empty_space * 1000))) |
| 76 | + |
| 77 | +if [ $var_solr_size -gt $empty_space ]; then |
| 78 | + echo "✗ ERROR: Not enough space on $SOLR_SERVER" |
| 79 | + echo "(/var/solr is $var_solr_size_str, $SOLR_SERVER has $empty_space_str free)" |
| 80 | + # exit 1 |
| 81 | +fi |
| 82 | +echo "✓" |
| 83 | +echo "(/var/solr is $var_solr_size_str, $SOLR_SERVER has $empty_space_str free)" |
| 84 | + |
| 85 | +echo "Check if $solr_updater_container is running" |
| 86 | +solr_updater_running=$(ssh $OL_HOME "docker ps --quiet --filter name=$solr_updater_container") |
| 87 | + |
| 88 | +# If running, stop it |
| 89 | +if [ -n "$solr_updater_running" ]; then |
| 90 | + echo -n "Stopping $solr_updater_container ..." |
| 91 | + ssh $OL_HOME " |
| 92 | + set -e |
| 93 | + docker stop $solr_updater_container |
| 94 | + " > /dev/null |
| 95 | + echo "✓" |
| 96 | +else |
| 97 | + echo "$solr_updater_container not running" |
| 98 | +fi |
| 99 | + |
| 100 | +# Note the offset |
| 101 | +offset=$(ssh $OL_HOME " |
| 102 | + set -e |
| 103 | + docker run --rm \ |
| 104 | + --volumes-from $solr_updater_container \ |
| 105 | + ubuntu:xenial \ |
| 106 | + cat solr-updater-data/$offset_file |
| 107 | +") |
| 108 | +echo "solr-updater offset: $offset" |
| 109 | +# Save the offset to /tmp/solr/ol_solr_dump_$(date +%Y-%m-%d).offset |
| 110 | +ssh $SOLR_SERVER " |
| 111 | + set -e |
| 112 | + mkdir -p /tmp/solr |
| 113 | + echo '$offset' > /tmp/solr/$DUMP_OFFSET_FILE |
| 114 | +" |
| 115 | + |
| 116 | +echo "Committing any transient changes into solr before dumping... (takes ~30s)" |
| 117 | +time ssh $SOLR_SERVER " |
| 118 | + set -e |
| 119 | + curl -s 'http://localhost:8984/solr/openlibrary/update?commit=true' |
| 120 | + sleep 15 # Just in case |
| 121 | +" |
| 122 | + |
| 123 | +echo "Pause solr" |
| 124 | +ssh $SOLR_SERVER "docker pause $solr_container" |
| 125 | + |
| 126 | +echo "Beginning dump (took 1h30 minutes 2024-08)" |
| 127 | +echo "Dumping solr data to $SOLR_SERVER:/tmp/solr/$DUMP_FILE" |
| 128 | +# Run detached so it persists if the ssh connection is lost |
| 129 | +time ssh -t $SOLR_SERVER " |
| 130 | + set -e |
| 131 | + mkdir -p /tmp/solr |
| 132 | + docker run --rm -d \ |
| 133 | + --name solr-dump \ |
| 134 | + --volumes-from $solr_container \ |
| 135 | + -v /tmp/solr:/tmp/solr \ |
| 136 | + ubuntu:xenial \ |
| 137 | + tar czf /tmp/solr/$DUMP_FILE /var/solr |
| 138 | + docker logs -f solr-dump |
| 139 | +" |
| 140 | + |
| 141 | +# Unpause solr |
| 142 | +ssh $SOLR_SERVER "docker unpause $solr_container" |
| 143 | + |
| 144 | +# Should be ~39G |
| 145 | +ssh $SOLR_SERVER "du -sh /tmp/solr/$DUMP_FILE" |
| 146 | + |
| 147 | +# Restart solr_updater_container if it was running |
| 148 | +if [ -n "$solr_updater_running" ]; then |
| 149 | + echo -n "Starting $solr_updater_container ... " |
| 150 | + ssh $OL_HOME " |
| 151 | + set -e |
| 152 | + docker start $solr_updater_container |
| 153 | + " > /dev/null |
| 154 | + echo "✓" |
| 155 | +fi |
| 156 | + |
| 157 | +echo "Uploading dump to IA" |
| 158 | +time ssh -t $SOLR_SERVER " |
| 159 | + set -e |
| 160 | + docker run --rm -d \ |
| 161 | + --name solr-dump-upload \ |
| 162 | + -v /tmp/solr:/tmp/solr \ |
| 163 | + -v /opt/olsystem/etc/ia.ini:/olsystem/etc/ia.ini:ro \ |
| 164 | + openlibrary/olbase:latest \ |
| 165 | + bash -c ' |
| 166 | + ia \ |
| 167 | + --config-file=/olsystem/etc/ia.ini \ |
| 168 | + upload ol_solr_dump \ |
| 169 | + --retries 300 \ |
| 170 | + -H x-archive-keep-old-version:0 \ |
| 171 | + /tmp/solr/$DUMP_FILE \ |
| 172 | + /tmp/solr/$DUMP_OFFSET_FILE \ |
| 173 | + --metadata 'collection:ol_exports' \ |
| 174 | + --metadata 'year:$(date +%Y)' \ |
| 175 | + --metadata 'format:Data' |
| 176 | + ' |
| 177 | + docker logs -f solr-dump-upload |
| 178 | +" |
| 179 | + |
| 180 | +echo "Dump complete!" |
0 commit comments