2020-09-05 15:46:05 +00:00
|
|
|
#!/usr/bin/env bash
|
|
|
|
set -e;
|
|
|
|
|
2020-09-05 18:24:18 +00:00
|
|
|
cd "$(dirname "$(readlink -f "$0")")" || { echo "Error: Failed to cd to script directory" >&2; exit 1; };
|
|
|
|
|
2020-09-05 19:55:02 +00:00
|
|
|
imagename="${1}";
|
2020-09-05 15:46:05 +00:00
|
|
|
|
2020-09-05 18:24:18 +00:00
|
|
|
if [[ -z "${NO_COLOR}" ]]; then
|
|
|
|
RS="\033[0m" # reset
|
|
|
|
HC="\033[1m" # hicolor
|
|
|
|
|
|
|
|
FRED="\033[31m" # foreground red
|
|
|
|
FGRN="\033[32m" # foreground green
|
|
|
|
FYEL="\033[33m" # foreground yellow
|
|
|
|
FBLE="\033[34m" # foreground blue
|
|
|
|
fi
|
|
|
|
|
2020-09-05 15:46:05 +00:00
|
|
|
help() {
|
2020-09-05 19:55:02 +00:00
|
|
|
echo -e "${FGRN}${HC}nomad_smartrestart${RS}, ${HC}v0.1${RS}
|
2020-09-05 15:46:05 +00:00
|
|
|
By Starbeamrainbowlabs
|
|
|
|
|
2020-09-05 18:24:18 +00:00
|
|
|
${FBLE}${HC}USAGE${RS}
|
|
|
|
path/to/nomad_smartrestart.sh {IMAGE_NAME}
|
|
|
|
|
|
|
|
${FBLE}${HC}ENVIRONMENT VARIABLES${RS}
|
|
|
|
${FYEL}URL_NOMAD${RS} The URL of the nomad server to talk to (using the Consul domain instead of a direct IP is recommended)"
|
|
|
|
}
|
2020-09-05 15:46:05 +00:00
|
|
|
|
2020-09-05 18:24:18 +00:00
|
|
|
log_msg() {
|
2020-09-05 18:25:35 +00:00
|
|
|
echo -e "${HC}>>> nomad_smartrestart: ${RS}$*" >&2;
|
2020-09-05 18:24:18 +00:00
|
|
|
}
|
|
|
|
log_error() {
|
2020-09-05 18:25:35 +00:00
|
|
|
echo -e "${HC}>>> nomad_smartrestart: ${FRED}$*${RS}";
|
2020-09-05 15:46:05 +00:00
|
|
|
}
|
|
|
|
|
2020-09-05 18:24:18 +00:00
|
|
|
if [[ -z "${imagename}" ]]; then
|
|
|
|
log_error "Error: No image name specified." >&2;
|
2020-09-05 18:25:35 +00:00
|
|
|
help;
|
2020-09-05 15:46:05 +00:00
|
|
|
exit 1;
|
|
|
|
fi
|
2020-09-05 18:24:18 +00:00
|
|
|
|
2020-09-05 19:55:02 +00:00
|
|
|
loc_nomad_txt="./images/${imagename}/nomad.txt";
|
|
|
|
|
|
|
|
if [[ ! -f "${loc_nomad_txt}" ]]; then
|
2020-09-05 18:24:18 +00:00
|
|
|
log_msg "No nomad.txt present for ${HC}${FGRN}${imagename}${RS}, nothing to do";
|
2020-09-05 18:26:30 +00:00
|
|
|
help;
|
2020-09-05 18:24:18 +00:00
|
|
|
exit 0;
|
2020-09-05 15:46:05 +00:00
|
|
|
fi
|
|
|
|
|
2020-09-05 18:24:18 +00:00
|
|
|
|
2020-09-05 15:46:05 +00:00
|
|
|
URL_NOMAD="${URL_NOMAD:-http://nomad.service.mooncarrot.space:4646}";
|
|
|
|
|
|
|
|
###############################################################################
|
|
|
|
|
|
|
|
|
|
|
|
# Finds allocations for a given job / task name combo.
|
|
|
|
# $1 The job name
|
|
|
|
# $2 The task name
|
|
|
|
nomad_allocs_find() {
|
|
|
|
local jobname="${1}"
|
|
|
|
local taskname="${2}"
|
|
|
|
|
|
|
|
if [[ -z "${taskname}" ]]; then
|
|
|
|
curl -sS "${URL_NOMAD}/v1/allocations" | JOB_NAME="${jobname}" jq --raw-output '.[] | select(.JobID == env.JOB_NAME) | .ID'
|
|
|
|
else
|
|
|
|
curl -sS "${URL_NOMAD}/v1/allocations" | JOB_NAME="${jobname}" TASK_NAME="${taskname}" jq --raw-output '.[] | select(.JobID == env.JOB_NAME and (.TaskStates|keys|.[]|. == env.TASK_NAME)) | .ID'
|
|
|
|
fi
|
|
|
|
}
|
|
|
|
|
|
|
|
# Restarts a Nomad allocation in-place.
|
|
|
|
# $1 The allocation id to restart.
|
|
|
|
nomad_alloc_restart() {
|
|
|
|
local alloc_id="${1}"
|
|
|
|
|
|
|
|
curl -sS "${URL_NOMAD}/v1/client/allocation/${alloc_id}/restart";
|
2020-09-05 19:55:02 +00:00
|
|
|
echo
|
2020-09-05 15:46:05 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
###############################################################################
|
|
|
|
|
2020-09-05 18:24:18 +00:00
|
|
|
|
|
|
|
count_tasks=0;
|
2020-09-05 15:46:05 +00:00
|
|
|
count_restarted=0;
|
2020-09-05 18:24:18 +00:00
|
|
|
while read -r jobname taskname; do
|
|
|
|
if [[ -z "${jobname}" ]]; then
|
|
|
|
log_error "Error: No jobname specified." >&2;
|
|
|
|
continue;
|
|
|
|
fi
|
|
|
|
if [[ -z "${taskname}" ]]; then
|
|
|
|
log_error "Error: No taskname specified." >&2;
|
|
|
|
continue;
|
|
|
|
fi
|
2020-09-05 15:46:05 +00:00
|
|
|
|
2020-09-05 18:24:18 +00:00
|
|
|
log_msg "Checking ${FYEL}${HC}${jobname}:${taskname}${RS}";
|
2020-09-05 15:46:05 +00:00
|
|
|
|
2020-09-05 18:24:18 +00:00
|
|
|
while read -r alloc_id jobname taskname; do
|
|
|
|
alloc_id_short="$(echo "${alloc_id}" | head -c7)";
|
2020-09-05 19:55:02 +00:00
|
|
|
log_msg "Restarting allocation ${HC}${FBLE}${alloc_id_short}${RS} in-place";
|
2020-09-05 18:24:18 +00:00
|
|
|
|
|
|
|
nomad_alloc_restart "${alloc_id}";
|
|
|
|
count_restarted="$((count_restarted+1))";
|
|
|
|
done < <(nomad_allocs_find "${jobname}" "${taskname}");
|
|
|
|
count_tasks="$((count_tasks+1))";
|
2020-09-05 19:55:02 +00:00
|
|
|
done <"${loc_nomad_txt}"
|
2020-09-05 15:46:05 +00:00
|
|
|
|
2020-09-05 18:24:18 +00:00
|
|
|
log_msg "Restarted ${HC}${count_restarted}${RS} allocations for ${HC}${count_tasks}${RS} tasks in-place";
|