|
| 1 | +# Job pooling for bash shell scripts |
| 2 | +# This script provides a job pooling functionality where you can keep up to n |
| 3 | +# processes/functions running in parallel so that you don't saturate a system |
| 4 | +# with concurrent processes. |
| 5 | +# |
| 6 | +# Got inspiration from http://stackoverflow.com/questions/6441509/how-to-write-a-process-pool-bash-shell |
| 7 | +# |
| 8 | +# Copyright (c) 2012 Vince Tse |
| 9 | +# with changes by Geoff Clements (c) 2014 |
| 10 | +# |
| 11 | +# Permission is hereby granted, free of charge, to any person obtaining a copy |
| 12 | +# of this software and associated documentation files (the "Software"), to deal |
| 13 | +# in the Software without restriction, including without limitation the rights |
| 14 | +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell |
| 15 | +# copies of the Software, and to permit persons to whom the Software is |
| 16 | +# furnished to do so, subject to the following conditions: |
| 17 | +# |
| 18 | +# The above copyright notice and this permission notice shall be included in |
| 19 | +# all copies or substantial portions of the Software. |
| 20 | +# |
| 21 | +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| 22 | +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| 23 | +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
| 24 | +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
| 25 | +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
| 26 | +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN |
| 27 | +# THE SOFTWARE. |
| 28 | + |
| 29 | +# end-of-jobs marker |
| 30 | +job_pool_end_of_jobs="JOBPOOL_END_OF_JOBS" |
| 31 | + |
| 32 | +# job queue used to send jobs to the workers |
| 33 | +job_pool_job_queue=/tmp/job_pool_job_queue_$$ |
| 34 | + |
| 35 | +# where to run results to |
| 36 | +job_pool_result_log=/tmp/job_pool_result_log_$$ |
| 37 | + |
| 38 | +# toggle command echoing |
| 39 | +job_pool_echo_command=0 |
| 40 | + |
| 41 | +# number of parallel jobs allowed. also used to determine if job_pool_init |
| 42 | +# has been called when jobs are queued. |
| 43 | +job_pool_pool_size=-1 |
| 44 | + |
| 45 | +# \brief variable to check for number of non-zero exits |
| 46 | +job_pool_nerrors=0 |
| 47 | + |
| 48 | +################################################################################ |
| 49 | +# private functions |
| 50 | +################################################################################ |
| 51 | + |
| 52 | +# \brief debug output |
| 53 | +function _job_pool_echo() |
| 54 | +{ |
| 55 | + if [[ "${job_pool_echo_command}" == "1" ]]; then |
| 56 | + echo $@ |
| 57 | + fi |
| 58 | +} |
| 59 | + |
| 60 | +# \brief cleans up |
| 61 | +function _job_pool_cleanup() |
| 62 | +{ |
| 63 | + rm -f ${job_pool_job_queue} ${job_pool_result_log} |
| 64 | +} |
| 65 | + |
| 66 | +# \brief signal handler |
| 67 | +function _job_pool_exit_handler() |
| 68 | +{ |
| 69 | + _job_pool_stop_workers |
| 70 | + _job_pool_cleanup |
| 71 | +} |
| 72 | + |
| 73 | +# \brief print the exit codes for each command |
| 74 | +# \param[in] result_log the file where the exit codes are written to |
| 75 | +function _job_pool_print_result_log() |
| 76 | +{ |
| 77 | + job_pool_nerrors=$(grep ^ERROR "${job_pool_result_log}" | wc -l) |
| 78 | + cat "${job_pool_result_log}" | sed -e 's/^ERROR//' |
| 79 | +} |
| 80 | + |
| 81 | +# \brief the worker function that is called when we fork off worker processes |
| 82 | +# \param[in] id the worker ID |
| 83 | +# \param[in] job_queue the fifo to read jobs from |
| 84 | +# \param[in] result_log the temporary log file to write exit codes to |
| 85 | +function _job_pool_worker() |
| 86 | +{ |
| 87 | + local id=$1 |
| 88 | + local job_queue=$2 |
| 89 | + local result_log=$3 |
| 90 | + local cmd= |
| 91 | + local args= |
| 92 | + |
| 93 | + exec 7<> ${job_queue} |
| 94 | + while [[ "${cmd}" != "${job_pool_end_of_jobs}" && -e "${job_queue}" ]]; do |
| 95 | + # workers block on the exclusive lock to read the job queue |
| 96 | + flock --exclusive 7 |
| 97 | + IFS=$'\v' |
| 98 | + read cmd args <${job_queue} |
| 99 | + set -- ${args} |
| 100 | + unset IFS |
| 101 | + flock --unlock 7 |
| 102 | + # the worker should exit if it sees the end-of-job marker or run the |
| 103 | + # job otherwise and save its exit code to the result log. |
| 104 | + if [[ "${cmd}" == "${job_pool_end_of_jobs}" ]]; then |
| 105 | + # write it one more time for the next sibling so that everyone |
| 106 | + # will know we are exiting. |
| 107 | + echo "${cmd}" >&7 |
| 108 | + else |
| 109 | + _job_pool_echo "### _job_pool_worker-${id}: ${cmd}" |
| 110 | + # run the job |
| 111 | + { ${cmd} "$@" ; } |
| 112 | + # now check the exit code and prepend "ERROR" to the result log entry |
| 113 | + # which we will use to count errors and then strip out later. |
| 114 | + local result=$? |
| 115 | + local status= |
| 116 | + if [[ "${result}" != "0" ]]; then |
| 117 | + status=ERROR |
| 118 | + fi |
| 119 | + # now write the error to the log, making sure multiple processes |
| 120 | + # don't trample over each other. |
| 121 | + exec 8<> ${result_log} |
| 122 | + flock --exclusive 8 |
| 123 | + _job_pool_echo "${status}job_pool: exited ${result}: ${cmd} $@" >> ${result_log} |
| 124 | + flock --unlock 8 |
| 125 | + exec 8>&- |
| 126 | + _job_pool_echo "### _job_pool_worker-${id}: exited ${result}: ${cmd} $@" |
| 127 | + fi |
| 128 | + done |
| 129 | + exec 7>&- |
| 130 | +} |
| 131 | + |
| 132 | +# \brief sends message to worker processes to stop |
| 133 | +function _job_pool_stop_workers() |
| 134 | +{ |
| 135 | + # send message to workers to exit, and wait for them to stop before |
| 136 | + # doing cleanup. |
| 137 | + echo ${job_pool_end_of_jobs} >> ${job_pool_job_queue} |
| 138 | + wait |
| 139 | +} |
| 140 | + |
| 141 | +# \brief fork off the workers |
| 142 | +# \param[in] job_queue the fifo used to send jobs to the workers |
| 143 | +# \param[in] result_log the temporary log file to write exit codes to |
| 144 | +function _job_pool_start_workers() |
| 145 | +{ |
| 146 | + local job_queue=$1 |
| 147 | + local result_log=$2 |
| 148 | + for ((i=0; i<${job_pool_pool_size}; i++)); do |
| 149 | + _job_pool_worker ${i} ${job_queue} ${result_log} & |
| 150 | + done |
| 151 | +} |
| 152 | + |
| 153 | +################################################################################ |
| 154 | +# public functions |
| 155 | +################################################################################ |
| 156 | + |
| 157 | +# \brief initializes the job pool |
| 158 | +# \param[in] pool_size number of parallel jobs allowed |
| 159 | +# \param[in] echo_command 1 to turn on echo, 0 to turn off |
| 160 | +function job_pool_init() |
| 161 | +{ |
| 162 | + local pool_size=$1 |
| 163 | + local echo_command=$2 |
| 164 | + |
| 165 | + # set the global attibutes |
| 166 | + job_pool_pool_size=${pool_size:=1} |
| 167 | + job_pool_echo_command=${echo_command:=0} |
| 168 | + |
| 169 | + # create the fifo job queue and create the exit code log |
| 170 | + rm -rf ${job_pool_job_queue} ${job_pool_result_log} |
| 171 | + mkfifo ${job_pool_job_queue} |
| 172 | + touch ${job_pool_result_log} |
| 173 | + |
| 174 | + # fork off the workers |
| 175 | + _job_pool_start_workers ${job_pool_job_queue} ${job_pool_result_log} |
| 176 | +} |
| 177 | + |
| 178 | +# \brief waits for all queued up jobs to complete and shuts down the job pool |
| 179 | +function job_pool_shutdown() |
| 180 | +{ |
| 181 | + _job_pool_stop_workers |
| 182 | + _job_pool_print_result_log |
| 183 | + _job_pool_cleanup |
| 184 | +} |
| 185 | + |
| 186 | +# \brief run a job in the job pool |
| 187 | +function job_pool_run() |
| 188 | +{ |
| 189 | + if [[ "${job_pool_pool_size}" == "-1" ]]; then |
| 190 | + job_pool_init |
| 191 | + fi |
| 192 | + printf "%s\v" "$@" >> ${job_pool_job_queue} |
| 193 | + echo >> ${job_pool_job_queue} |
| 194 | +} |
| 195 | + |
| 196 | +# \brief waits for all queued up jobs to complete before starting new jobs |
| 197 | +# This function actually fakes a wait by telling the workers to exit |
| 198 | +# when done with the jobs and then restarting them. |
| 199 | +function job_pool_wait() |
| 200 | +{ |
| 201 | + _job_pool_stop_workers |
| 202 | + _job_pool_start_workers ${job_pool_job_queue} ${job_pool_result_log} |
| 203 | +} |
| 204 | +######################################### |
| 205 | +# End of Job Pool |
| 206 | +######################################### |
0 commit comments