start-all.sh

#!/usr/bin/env bash

. versions.sh
if [ -f env.sh ]; then
	. env.sh
fi

if [ "$LOG_FILE" = "" ]; then
	LOG_FILE="./run.log"
fi

log() {
    local level="$1"
    shift
    local msg="$*"
    local ts

    ts="$(date '+%F %T')"

    # full format to file
    printf '%s [%s] %s\n' "$ts" "$level" "$msg" >> "$LOG_FILE"

    # only message to stdout
    printf '%s\n' "$msg"
}

run_script() {
    local script="$1"
    shift
    local start end rc

    start=$(date +%s)

    log INFO "Starting $script $*"
    "$script" "$@" >>"$LOG_FILE" 2>&1
    rc=$?

    end=$(date +%s)

    if [ "$rc" -eq 0 ]; then
        log INFO "Finished $script successfully in $((end-start))s"
    else
        log ERROR "Failed $script rc=$rc after $((end-start))s"
    fi

    return "$rc"
}

CURRENT_VERSION="master"
if [ -f CURRENT_VERSION.sh ]; then
	CURRENT_VERSION=$(cat CURRENT_VERSION.sh)
fi

if [ -z "$BRANCH_VERSION" ]; then
	BRANCH_VERSION=$CURRENT_VERSION
fi
if [ -z ${DEFAULT_VERSION[$CURRENT_VERSION]} ]; then
	BRANCH_VERSION=$(echo $CURRENT_VERSION | cut -d'.' -f1,2)
fi
if [ "$1" = "-e" ]; then
	DEFAULT_VERSION=${DEFAULT_ENTERPRISE_VERSION[$BRANCH_VERSION]}
fi

if [ -z "$MANAGER_VERSION" ]; then
	MANAGER_VERSION=${MANAGER_DEFAULT_VERSION[$BRANCH_VERSION]}
fi
if [[ ! -v GRAFANA_ENV_ARRAY ]]; then
    GRAFANA_ENV_ARRAY=()
fi

if [ "$1" = "--version" ]; then
	echo "Scylla-Monitoring Stack version: $CURRENT_VERSION"
	echo "Supported versions:" ${SUPPORTED_VERSIONS[$BRANCH_VERSION]}
	echo "Manager supported versions:" ${MANAGER_SUPPORTED_VERSIONS[$BRANCH_VERSION]}
	exit
fi

if [ "$(id -u)" -eq 0 ]; then
	log WARNING "Running as root is not advised, please check the documentation on how to run as non-root user"
	USER_PERMISSIONS="-u 0:0"
else
	GROUPID=$(id -g)
	USER_PERMISSIONS="-u $UID:$GROUPID"
fi

group_args=()
is_podman="$(docker --help | grep -o podman)"
if [ ! -z "$is_podman" ]; then
	group_args+=(--userns=keep-id)
fi

if [[ $(uname) == "Linux" ]]; then
	readlink_command="readlink -f"
elif [[ $(uname) == "Darwin" ]]; then
	readlink_command="realpath "
fi

function usage {
	__usage="Usage: $(basename $0) [-h] [--version] [-e] [-d Prometheus data-dir] [-L resolve the servers from the manager running on the given address] [-G path to grafana data-dir] [-s scylla-target-file] [-n node-target-file] [-l] [-v comma separated versions] [-j additional dashboard to load to Grafana, multiple params are supported] [-c grafana environment variable, multiple params are supported] [-b Prometheus command line options] [-g grafana port ] [ -p prometheus port ] [-a admin password] [-m alertmanager port] [ -M scylla-manager version ] [-D encapsulate docker param] [-r alert-manager-config] [-R prometheus-alert-file] [-N manager target file] [-A bind-to-ip-address] [-C alertmanager commands] [-Q Grafana anonymous role (Admin/Editor/Viewer)] [-S start with a system specific dashboard set] [-T additional-prometheus-targets] [--no-loki] [--no-alertmanager] [--loki-port port] [--promtail-port port] [--auto-restart] [--no-renderer] [-f alertmanager-dir]

Options:
  -h print this help and exit
  --version print the current monitoring version and the supported versions and exit.
  -e
  -d path/to/Prometheus/data/dir - Set an external data directory for the Prometheus data
  -L ip                          - Resolve the servers from a Scylla Manager running on the given address.
  -G path/to/Grafana/data-dir    - Set an external data directory for the Grafana data.
  -s path/to/scylla-target-file  - Read Scylla's target from the given file.
  -n path/to/node-target-file    - Override scylla target file for node_exporter.
  -l                             - If Set use the local host network, especially useful when a container needs
                                   to access the local host.
  -v comma separated versions    - Specify one or more Scylla versions, check --version for the supported versions.
  -j additional dashboard        - List additional dashboards to load to Grafana, multiple params are supported.
  -c grafana_environment         - Grafana environment variable, multiple params are supported.
  -b Prometheus command          - Prometheus command line options.
  -g grafana port                - Override the default Grafana port.
  -p prometheus port             - Override the default Prometheus port.
  -a admin password              - Set Grafana's Admin password.
  -m alertmanager port           - Override the default Prometheus port.
  -M scylla-manager version      - Override the default Scylla Manager version to use.
  -D docker param                - Encapsulate docker param, the parameter will be used by all containers.
  -r alert-manager-config        - Override the default alert-manager configuration file.
  -f path/to/alertmanager/data   - If set, the alertmanager would store its data in the given directory.
  -R prometheus-alert-file       - Override the default Prometheus alerts configuration file.
  -N path/to/manager/target file - Set the location of the target file for Scylla Manager.
  -A bind-to-ip-address          - Bind to a specific interface.
  -C alertmanager commands       - Pass the command to the alertmanager.
  -Q Grafana anonymous role      - Set the Grafana anonymous role to one of Admin/Editor/Viewer.
  -S dashbards-list              - Override the default set of dashboards with the spcefied one.
  -T path/to/prometheus-targets  - Adds additional Prometheus target files.
  -k path/to/loki/storage        - When set, will use the given directory for Loki's data
  --no-loki                      - If set, do not run Loki and promtail.
  --no-alertmanager              - If set, do not run the Alertmanager.
  --loki-port port               - If set, loki would use the given port number
  --promtail-port port           - If set, promtail would use the given port number
  --promtail-binary-port port    - If set, promtail would use the given port number for the binary protocol
  --no-cas                       - If set, Prometheus will drop all cas related metrics while scrapping
  --no-cdc                       - If set, Prometheus will drop all cdc related metrics while scrapping
  --auto-restart                 - If set, auto restarts the containers on failure.
  --no-renderer                  - If set, do not run the Grafana renderer container.
  --thanos-sc                    - If set, run thanos sidecar with the Prometheus server.
  --thanos                       - If set, run thanos query as a Grafana datasource.
  --local-thanos                 - If set, run thanos query as a front end to the local thanos sidecar.
  --enable-protobuf              - [Deprecated]If set, enable the experimental Prometheus Protobuf with Native histograms support.
  --native-histogram             - If set, enable the Prometheus Protobuf with Native histograms support.
  --scrap [scrap duration]       - Change the default Prometheus scrap duration. Duration is in seconds.
  --vector-store path/to/file.yml - [depricated] see vector-search
  --vector-search path/to/file.yml- Specifiy a vector search target file, the file should contain a list of vector search servers.
  --target-directory             - If set, prometheus/targets/ directory will be set as a root directory for the target files
                                   the file names should be scylla_servers.yml, node_exporter_servers.yml, scylla_manager_agents.yml, and scylla_manager_servers.yml
  --stack id                     - Use this option when running a secondary stack, id could be 1-4
  --limit container,param        - Allow to set a specific Docker parameter for a container, where container can be:
                                   prometheus, grafana, alertmanager, loki, sidecar, grafanarender
  --archive  data-directory      - Treat data directory as an archive. This disables Prometheus time-to-live (infinite retention), and would run a minimal mode
  --quick-startup                - If set, the script will not validate that each of the processes start correctly.
The script starts Scylla Monitoring stack.
"
	echo "$__usage"
}

is_local() {
	for var in "$@"; do
		if grep -q '\s127.' $1; then
			log WARNING "Local host found in $1"
			grep '\s127.' $1
			return 0
		fi
		if grep -q 'localhost' $1; then
			return 0
		fi
	done
	return 1
}

# Resolve the address of a Docker container.
# Usage: container_address <container_name> <port>
# Prints <ip>:<port>. Falls back to BIND_ADDRESS or host IP if the container has no IP.
container_address() {
	local name=$1
	local port=$2
	local ip
	ip=$(docker inspect --format '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$name")
	if [ "$ip" = "invalid IP" ] || [ -z "$ip" ]; then
		ip=""
	fi
	if [ -z "$ip" ]; then
		if [ ! -z "$BIND_ADDRESS" ]; then
			ip=$(echo $BIND_ADDRESS | sed 's/:$//')
		elif [[ $(uname) == "Linux" ]]; then
			ip=$(hostname -I | awk '{print $1}')
		elif [[ $(uname) == "Darwin" ]]; then
			ip=$(ifconfig en0 | awk '/inet / {print $2}')
		fi
	fi
	echo "$ip:$port"
}

if [ -z "$PROMETHEUS_RULES" ]; then
	PROMETHEUS_RULES="$PWD/prometheus/prom_rules/:/etc/prometheus/prom_rules/"
fi

if [ -z "$VERSIONS" ]; then
	VERSIONS=${DEFAULT_VERSION[$BRANCH_VERSION]}
fi

if [ -z "$SCYLLA_TARGET_FILES" ]; then
	SCYLLA_TARGET_FILES=($PWD/prometheus/scylla_servers.yml $PWD/scylla_servers.yml)
fi
if [ -z "$SCYLLA_MANGER_TARGET_FILES" ]; then
	SCYLLA_MANGER_TARGET_FILES=($PWD/prometheus/scylla_manager_servers.yml $PWD/scylla_manager_servers.yml $PWD/prometheus/scylla_manager_servers.example.yml)
fi
if [ -z "$GRAFANA_ADMIN_PASSWORD" ]; then
	GRAFANA_ADMIN_PASSWORD=""
fi

if [ -z "$ALERTMANAGER_PORT_CMD" ]; then
	ALERTMANAGER_PORT_CMD=""
fi

if [ -z "$DOCKER_PARAM" ]; then
	DOCKER_PARAM=""
fi
if [ -z "$DATA_DIR" ]; then
	DATA_DIR=""
fi
if [ -z "$DATA_DIR_CMD" ]; then
	DATA_DIR_CMD=""
fi
if [ -z "$CONSUL_ADDRESS" ]; then
	CONSUL_ADDRESS=""
fi
if [ -z "$PROMETHEUS_TARGETS" ]; then
	PROMETHEUS_TARGETS=""
fi
if [ -z "$BIND_ADDRESS" ]; then
	BIND_ADDRESS=""
fi
if [ -z "$BIND_ADDRESS_CONFIG" ]; then
	BIND_ADDRESS_CONFIG=""
fi
if [ -z "$GRAFNA_ANONYMOUS_ROLE" ]; then
	GRAFNA_ANONYMOUS_ROLE=""
fi
if [ -z "$SPECIFIC_SOLUTION" ]; then
	SPECIFIC_SOLUTION=""
fi
if [ -z "$LDAP_FILE" ]; then
	LDAP_FILE=""
fi
if [ -z "$RUN_RENDERER" ]; then
	RUN_RENDERER="-E"
fi
if [ -z "$RUN_LOKI" ]; then
	RUN_LOKI=1
fi
if [ -z "$RUN_THANOS_SC" ]; then
	RUN_THANOS_SC=0
fi
if [ -z "$RUN_THANOS" ]; then
	RUN_THANOS=0
fi
if [ -z "$ALERT_MANAGER_DIR" ]; then
	ALERT_MANAGER_DIR=""
fi
if [ -z "$LOKI_DIR" ]; then
	LOKI_DIR=""
fi
if [ -z "$LOKI_PORT_CMD" ]; then
	LOKI_PORT_CMD=""
fi
LIMITS=""
VOLUMES=""
PARAMS=""
for arg; do
	if [ "$arg" = "--compose" ]; then
		log INFO "Using compose"
		exec ./make-compose.sh "$@"
	fi
done

for arg; do
	shift
	if [ -z "$LIMIT" ]; then
		case $arg in
		--no-loki)
			RUN_LOKI=0
			;;
		--no-alertmanager)
			SKIP_ALERTMANAGER=1
			;;
        --quick-startup)
            QUICK_STARTUP=1
            ;;
		--loki-port)
			LIMIT="1"
			PARAM="loki-port"
			;;
		--promtail-port)
			LIMIT="1"
			PARAM="promtail-port"
			;;
		--promtail-binary-port)
			LIMIT="1"
			PARAM="promtail-binary-port"
			;;
		--no-renderer)
			RUN_RENDERER=""
			;;
		--thanos-sc)
			RUN_THANOS_SC=1
			;;
		--thanos)
			RUN_THANOS=1
			;;
		--local-thanos)
		    RUN_LOCAL_THANOS=1
		    ;;
        --no-thanos-datasource)
            NO_THANOS_DATASOURCE="1"
            ;;
		--auto-restart)
			DOCKER_PARAM="--restart=unless-stopped"
			;;
		--victoria-metrics)
			VICTORIA_METRICS="1"
			;;
		--auth)
			GRAFANA_ENV_ARRAY+=(--auth)
			;;
        --support-dashboard)
            SUPPORT_DASHBOARD="1"
            ;;
        --clear)
            GRAFANA_ENV_ARRAY+=(--clear)
            ;;
		--disable-anonymous)
			GRAFANA_ENV_ARRAY+=(--disable-anonymous)
			;;
		--enable-protobuf)
			NATIVE_HISTOGRAM="1"
			;;
		--native-histogram)
			NATIVE_HISTOGRAM="1"
			;;
		--alternator)
			RUN_ALTERNATOR="1"
			;;
		--limit)
			LIMIT="1"
			;;
		--volume)
			LIMIT="1"
			VOLUME="1"
			;;
		--param)
			LIMIT="1"
			PARAM="param"
			;;
		--evaluation-interval)
			LIMIT="1"
			PARAM="evaluation-interval"
			;;
		--manager-agents)
			LIMIT="1"
			PARAM="manager-agents"
			;;
		--datadog-api-keys)
			LIMIT="1"
			PARAM="datadog-api-keys"
			;;
		--datadog-hostname)
			LIMIT="1"
			PARAM="datadog-hostname"
			;;
		--stack)
			LIMIT="1"
			PARAM="stack"
			;;
		--scrap)
			LIMIT="1"
			PARAM="scrap"
			;;
		--vector-store)
			LIMIT="1"
			PARAM="vector-search"
			VECTOR_SEARCH_CMD="--vector-search"
			;;
        --vector-search)
            LIMIT="1"
            PARAM="vector-search"
            VECTOR_SEARCH_CMD="--vector-search"
            ;;
		--no-cas-cdc)
			PROMETHEUS_TARGETS="$PROMETHEUS_TARGETS --no-cas-cdc"
			;;
		--no-cas)
			PROMETHEUS_TARGETS="$PROMETHEUS_TARGETS --no-cas"
			;;
		--no-cdc)
			PROMETHEUS_TARGETS="$PROMETHEUS_TARGETS --no-cdc"
			;;
		--target-directory)
			LIMIT="1"
			PARAM="target-directory"
			;;
		--help)
			usage
			;;
		--archive)
			ARCHIVE="1"
			LIMIT="1"
			PARAM="archive"
			;;
		*)
			set -- "$@" "$arg"
			;;
		esac
	else
		DOCR=$(echo $arg | cut -d',' -f1)
		VALUE=$(echo $arg | cut -d',' -f2- | sed 's/#/ /g')
		NOSPACE=$(echo $arg | sed 's/ /#/g')
		if [[ $NOSPACE == --* ]]; then
			log ERROR "Error: No value given to --$PARAM"
			echo
			usage
			exit 1
		fi
		if [ "$PARAM" = "param" ]; then
			if [ -z "${DOCKER_PARAMS[$DOCR]}" ]; then
				DOCKER_PARAMS[$DOCR]=""
			fi
			DOCKER_PARAMS[$DOCR]="${DOCKER_PARAMS[$DOCR]} $VALUE"
			PARAMS="$PARAMS --param $NOSPACE"
			unset PARAM
		elif [ "$PARAM" = "evaluation-interval" ]; then
			PROMETHEUS_TARGETS="$PROMETHEUS_TARGETS -E $NOSPACE"
			unset PARAM
		elif [ "$PARAM" = "manager-agents" ]; then
			SCYLLA_MANGER_AGENT_TARGET_FILE="$NOSPACE"
			unset PARAM
		elif [ "$PARAM" = "target-directory" ]; then
			TARGET_DIRECTORY="$NOSPACE"
			unset PARAM
		elif [ "$PARAM" = "datadog-api-keys" ]; then
			DATDOGPARAM="$DATDOGPARAM -A $NOSPACE"
			unset PARAM
		elif [ "$PARAM" = "datadog-hostname" ]; then
			DATDOGPARAM="$DATDOGPARAM -H $NOSPACE"
			unset PARAM
		elif [ "$PARAM" = "loki-port" ]; then
			LOKI_PORT = "$NOSPACE"
			LOKI_PORT_CMD="$LOKI_PORT_CMD -p $NOSPACE"
			unset PARAM
		elif [ "$PARAM" = "promtail-port" ]; then
			LOKI_PORT_CMD="$LOKI_PORT_CMD -t $NOSPACE"
			unset PARAM
		elif [ "$PARAM" = "promtail-binary-port" ]; then
			LOKI_PORT_CMD="$LOKI_PORT_CMD -T $NOSPACE"
			unset PARAM
		elif [ "$PARAM" = "stack" ]; then
			STACK_ID="$NOSPACE"
			STACK_CMD="-s $NOSPACE"
			STACK="/stack/$NOSPACE"
			unset PARAM
		elif [ "$PARAM" = "scrap" ]; then
			SCRAP_CMD="--scrap $NOSPACE"
			unset PARAM
		elif [ "$PARAM" = "vector-search" ]; then
			VECTOR_SEARCH="$NOSPACE"
			unset PARAM
		elif [ "$PARAM" = "archive" ]; then
			DATA_DIR="$NOSPACE"
			unset PARAM
		else
			if [ -z "${DOCKER_LIMITS[$DOCR]}" ]; then
				DOCKER_LIMITS[$DOCR]=""
			fi
			if [ "$VOLUME" = "1" ]; then
				SRC=$(echo $VALUE | cut -d':' -f1)
				DST=$(echo $VALUE | cut -d':' -f2-)
				SRC=$($readlink_command "$SRC")
				DOCKER_LIMITS[$DOCR]="${DOCKER_LIMITS[$DOCR]} -v $SRC:$DST"
				VOLUMES="$VOLUMES --volume $NOSPACE"
				unset VOLUME
			else
				DOCKER_LIMITS[$DOCR]="${DOCKER_LIMITS[$DOCR]} $VALUE"
				LIMITS="$LIMITS --limit $NOSPACE"
			fi
		fi
		unset LIMIT
	fi
done

if [ ! -z $LIMIT ]; then
	log ERROR "Error: No value given to --$PARAM"
	echo
	usage
	exit -1
fi
if [ "$DOCKER_PARAM" != "" ]; then
	DOCKER_PARAM_FROM_FILE="1"
fi
while getopts ':hleEd:g:p:v:s:n:a:c:j:b:m:r:R:M:G:D:L:N:C:Q:A:f:P:S:T:k:' option; do
	case "$option" in
	h)
		usage
		exit
		;;
	v)
		VERSIONS=$OPTARG
		;;
	M)
		MANAGER_VERSION=$OPTARG
		;;
	d)
		DATA_DIR=$OPTARG
		;;
	G)
		EXTERNAL_VOLUME="-G $OPTARG"
		;;
	A)
		BIND_ADDRESS="$OPTARG:"
		BIND_ADDRESS_CONFIG="-A $OPTARG"
		;;
	r)
		ALERT_MANAGER_RULE_CONFIG="-r $OPTARG"
		;;
	R)
		if [[ -d "$OPTARG" ]]; then
			PROMETHEUS_RULES=$($readlink_command $OPTARG)":/etc/prometheus/prom_rules/"
		else
			PROMETHEUS_RULES=$($readlink_command $OPTARG)":/etc/prometheus/prometheus.rules.yml"
		fi
		;;
	g)
		GRAFANA_PORT="-g $OPTARG"
		;;
	m)
		ALERTMANAGER_PORT="$OPTARG"
		;;
	T)
		PROMETHEUS_TARGETS="$PROMETHEUS_TARGETS -T $OPTARG"
		;;
	Q)
		GRAFNA_ANONYMOUS_ROLE="-Q $OPTARG"
		;;
	p)
		PROMETHEUS_PORT=$OPTARG
		;;
	s)
		SCYLLA_TARGET_FILES=("$OPTARG")
		;;
	n)
		NODE_TARGET_FILE=$OPTARG
		;;
	l)
		if [[ "$DOCKER_PARAM" != *"--net=host"* ]]; then
			DOCKER_PARAM="$DOCKER_PARAM --net=host"
		fi
		;;
	L)
		CONSUL_ADDRESS="-L $OPTARG"
		;;
	P)
		LDAP_FILE="-P $OPTARG"
		;;
	a)
		GRAFANA_ADMIN_PASSWORD="-a $OPTARG"
		;;
	j)
		GRAFANA_DASHBOARD_ARRAY+=("$OPTARG")
		;;
	c)
		GRAFANA_ENV_ARRAY+=(-c "$OPTARG")
		;;
	C)
		ALERTMANAGER_COMMANDS+=("$OPTARG")
		;;
	D)
		if [ "$DOCKER_PARAM_FROM_FILE" = "1" ]; then
			DOCKER_PARAM=""
			DOCKER_PARAM_FROM_FILE=""
		fi
		DOCKER_PARAM="$DOCKER_PARAM $OPTARG"
		;;
	b)
		PROMETHEUS_COMMAND_LINE_OPTIONS_ARRAY+=("$OPTARG")
		;;
	N)
		SCYLLA_MANGER_TARGET_FILES=($OPTARG)
		;;
	S)
		SPECIFIC_SOLUTION="-S $OPTARG"
		;;
	E)
		RUN_RENDERER="-E"
		;;
	f)
		ALERT_MANAGER_DIR="-f $OPTARG"
		;;
	k)
		LOKI_DIR="-k $OPTARG"
		;;
	:)
		printf "missing argument for -%s\n" "$OPTARG" >&2
		echo "$usage" >&2
		exit 1
		;;
	\?)
		printf "illegal option: -%s\n" "$OPTARG" >&2
		echo "$usage" >&2
		exit 1
		;;
	esac
done
if [ "$ARCHIVE" == "1" ]; then
	PROMETHEUS_COMMAND_LINE_OPTIONS_ARRAY+=(--storage.tsdb.retention.time=100y)
	RUN_LOKI=0
	SKIP_ALERTMANAGER="1"
	RUN_RENDERER=""
	CONSUL_ADDRESS="-L 127.0.0.1:0"
	if [ ! -d $DATA_DIR/ ]; then
		log ERROR "The giving data directory $DATA_DIR does not exist"
		exit 1
	fi
	if [ -f $DATA_DIR/scylla.txt ]; then
		. $DATA_DIR/scylla.txt
		log INFO "Taking version from $DATA_DIR/scylla.txt"
		log INFO "Version set to $VERSIONS"
	else
		log WARNING "scylla.txt not found in $DATA_DIR/. You can use it to start the monitoring stack with a given version"
		log WARNING "For example, to start the monitoring stack with version 2014.1 and manager 3.3"
		log WARNING 'echo VERSIONS="2024.1">'$DATA_DIR/scylla.txt
		log WARNING 'echo MANAGER_VERSION="3.3">>'$DATA_DIR/scylla.txt
	fi
fi

if [ -z "$VERSIONS" ]; then
	log ERROR "Scylla-version was not not found, add the -v command-line with a specific version (i.e. -v 2021.1)"
	exit 1
fi
if [ "$QUICK_STARTUP" = "1" ]; then
    QUICK_STARTUP_CMD="--quick-startup"
fi
if [ ! "$SUPPORT_DASHBOARD" = "" ]; then
    SUPPORT_DASHBOARD="--support-dashboard"
    GRAFANA_ENV_ARRAY+=(--support-dashboard)
fi
if [ "$CURRENT_VERSION" = "master" ]; then
	if [ "$ARCHIVE" = "" ]; then
		echo ""
		echo "*****************************************************"
		echo "* WARNING: You are using the unstable master branch *"
		echo "* Check the README.md file for the stable releases  *"
		echo "*****************************************************"
		./generate-dashboards.sh -v $VERSIONS -m $MANAGER_VERSION $STACK_CMD "$SUPPORT_DASHBOARD" $VECTOR_SEARCH_CMD
	else
		./generate-dashboards.sh -v $VERSIONS -F -R 0 -m $MANAGER_VERSION $STACK_CMD $VECTOR_SEARCH_CMD
	fi
	log INFO "Generating the dashboards"

fi

if [ -z "$ALERTMANAGER_PORT" ]; then
	ALERTMANAGER_PORT="9093"
else
	if [  $ALERTMANAGER_PORT != "9093" ]; then
		ALERTMANAGER_PORT_CMD="-p $ALERTMANAGER_PORT"
	fi
fi

if [[ $DOCKER_PARAM = *"--net=host"* ]]; then
	if [ ! -z "$ALERTMANAGER_PORT_CMD" ] || [ ! -z "$GRAFANA_PORT" ] || [ ! -z $PROMETHEUS_PORT ]; then
		log ERROR "Port mapping is not supported with host network, remove the -l flag from the command line"
		exit 1
	fi
	HOST_NETWORK=host
	GRAFANA_ADDRESS="-G localhost:3000"
elif [[ ! $DOCKER_PARAM = *"--net"* ]]; then
    HOST_NETWORK="monitor-net"
    if [ "$STACK_ID" != "" ]; then
        HOST_NETWORK="$HOST_NETWORK$STACK_ID"
    fi
    if ! docker network inspect $HOST_NETWORK >/dev/null 2>&1; then
      docker network create $HOST_NETWORK
    fi
    DOCKER_PARAM="$DOCKER_PARAM --net=$HOST_NETWORK"
fi

if [ -z "$TARGET_DIRECTORY" ] && [ -z "$CONSUL_ADDRESS" ]; then
	for f in ${SCYLLA_TARGET_FILES[@]}; do
		if [ -f $f ]; then
			SCYLLA_TARGET_FILE=$f
			break
		fi
	done

	if [ -z $SCYLLA_TARGET_FILE ]; then
		log ERROR "Scylla target file '${SCYLLA_TARGET_FILES}' does not exist, you can use prometheus/scylla_servers.example.yml as an example."
		exit 1
	fi

	if [ -z $NODE_TARGET_FILE ]; then
		PROMETHEUS_TARGETS="$PROMETHEUS_TARGETS --no-node-exporter-file"
		NODE_TARGET_FILE=$SCYLLA_TARGET_FILE
	fi

	if [ -z $SCYLLA_MANGER_AGENT_TARGET_FILE ]; then
		PROMETHEUS_TARGETS="$PROMETHEUS_TARGETS --no-manager-agent-file"
		SCYLLA_MANGER_AGENT_TARGET_FILE=$SCYLLA_TARGET_FILE
	fi
	if [ ! -f $NODE_TARGET_FILE ]; then
		log ERROR "Node target file '${NODE_TARGET_FILE}' does not exist"
		exit 1
	fi

	for f in ${SCYLLA_MANGER_TARGET_FILES[@]}; do
		if [ -f $f ]; then
			SCYLLA_MANGER_TARGET_FILE=$f
			break
		fi
	done
	if [ -z $SCYLLA_MANGER_TARGET_FILE ]; then
		log ERROR "Scylla-Manager target file '${SCYLLA_MANGER_TARGET_FILES}' does not exist, you can use prometheus/scylla_manager_servers.example.yml as an example."
		exit 1
	fi
	if [ -z "$HOST_NETWORK" ]; then
		if is_local $SCYLLA_TARGET_FILE $SCYLLA_MANGER_TARGET_FILE $NODE_TARGET_FILE; then
			echo "Warning: It seems that you are trying to connect to localhost (either localhost or IP on the 127.x.x.x range)."
			echo "  For example, maybe Scylla Manager is running on the localhost."
			echo "If that is the case, you should set your Docker to use the host network. You can do that with the -l flag."
			echo ""
		fi
	fi
	if [ -z "$VECTOR_SEARCH" ]; then
		VECTOR_SEARCH=""
	else
		VECTOR_SEARCH="-v "$($readlink_command $VECTOR_SEARCH)":/etc/scylla.d/prometheus/targets/vector_search_servers.yml"
	fi
	SCYLLA_TARGET_FILE="-v "$($readlink_command $SCYLLA_TARGET_FILE)":/etc/scylla.d/prometheus/targets/scylla_servers.yml"
	SCYLLA_MANGER_TARGET_FILE="-v "$($readlink_command $SCYLLA_MANGER_TARGET_FILE)":/etc/scylla.d/prometheus/targets/scylla_manager_servers.yml"
	NODE_TARGET_FILE="-v "$($readlink_command $NODE_TARGET_FILE)":/etc/scylla.d/prometheus/targets/node_exporter_servers.yml"
	SCYLLA_MANGER_AGENT_TARGET_FILE="-v "$($readlink_command $SCYLLA_MANGER_AGENT_TARGET_FILE)":/etc/scylla.d/prometheus/targets/scylla_manager_agents.yml"
else
    if [ ! -z "$VECTOR_SEARCH" ]; then
        if [ "$VECTOR_SEARCH" != "vector_search_servers.yml" ]; then
            log WARNING "When using target directory the vector-search file is called vector_search_servers.yml and should be inside the target directory"
        fi
    fi
    VECTOR_SEARCH=""
	SCYLLA_TARGET_FILE=""
	SCYLLA_MANGER_TARGET_FILE=""
	SCYLLA_MANGER_AGENT_TARGET_FILE=""
	NODE_TARGET_FILE=""
fi

if [ "$TARGET_DIRECTORY" != "" ]; then
	SCYLLA_TARGET_FILE="-v "$($readlink_command $TARGET_DIRECTORY)":/etc/scylla.d/prometheus/targets/:z"
	if [ ! -f $TARGET_DIRECTORY/scylla_servers.yml ]; then
		log WARNING "WARNING: Using $TARGET_DIRECTORY for Prometheus target directory, scylla_servers.yml is missing, make sure to create it, or ScyllaDB targets will be missing"
	fi
	if [ ! -f $TARGET_DIRECTORY/node_exporter_servers.yml ]; then
		log WARNING "WARNING: Using $TARGET_DIRECTORY for Prometheus target directory, node_exporter_servers.yml is missing, make sure to create it, or node-exporter targets will be missing"
	fi
	if [ ! -f $TARGET_DIRECTORY/scylla_manager_agents.yml ]; then
		log WARNING "WARNING: Using $TARGET_DIRECTORY for Prometheus target directory, scylla_manager_agents.yml is missing, make sure to create it, or ScyllaDB manager-agent targets will be missing"
	fi
	if [ ! -f $TARGET_DIRECTORY/scylla_manager_servers.yml ]; then
		log WARNING "WARNING: Using $TARGET_DIRECTORY for Prometheus target directory, scylla_manager_servers.yml is missing, make sure to create it, or ScyllaDB manager target will be missing"
	fi
fi
if [ -z $DATA_DIR ]; then
	USER_PERMISSIONS=""
	log WARNING "WARNING: Without an external Prometheus directory, Prometheus data will be deleted on shutdown, use the -d command line flag for data persistence."
else
	if [ -d $DATA_DIR ]; then
		log INFO "Loading prometheus data from $DATA_DIR"
	else
		log INFO "Creating data directory $DATA_DIR"
		mkdir -p $DATA_DIR
	fi
	if [[ "$VICTORIA_METRICS" = "1" ]]; then
		DATA_DIR_CMD="-v "$($readlink_command $DATA_DIR)":/victoria-metrics-data"
	else
		DATA_DIR_CMD="-v "$($readlink_command $DATA_DIR)":/prometheus/data:Z"
	fi
fi

if [ "$VERSIONS" = "latest" ]; then
	if [ -z "$BRANCH_VERSION" ] || [ "$BRANCH_VERSION" = "master" ]; then
		log ERROR "Default versions (-v latest) is not supported on the master branch, use specific version instead"
		exit 1
	fi
	VERSIONS=${DEFAULT_VERSION[$BRANCH_VERSION]}
	log WARNING "The use of -v latest is deprecated. Use a specific version instead."
else
	if [ "$VERSIONS" = "all" ]; then
		VERSIONS=$ALL
	fi
fi
if [ "$STACK_ID" != "" ]; then
	log INFO "Running a seconddary stack $STACK_ID"
	echo "Note that the following containers will not run: loki, promtail, grafana renderer"
	echo "to stop it use ./kill-all.sh --stack $STACK_ID"
	RUN_LOKI=0
	RUN_RENDERER=""
	PROMETHEUS_PORT=${STACK_PROMETHEUS["$STACK_ID"]}
	GRAFANA_PORT="-g"${STACK_GRAFANA["$STACK_ID"]}
	ALERTMANAGER_PORT_CMD="-p "${STACK_ALERTMANAGER["$STACK_ID"]}
fi

ALERTMANAGER_COMMAND=""
for val in "${ALERTMANAGER_COMMANDS[@]}"; do
	ALERTMANAGER_COMMAND="$ALERTMANAGER_COMMAND -C $val"
done

if [ "$SKIP_ALERTMANAGER" = "1" ]; then
	AM_ADDRESS="127.0.0.1:9093"
else
	run_script ./start-alertmanager.sh $ALERTMANAGER_PORT_CMD $QUICK_STARTUP_CMD $ALERT_MANAGER_DIR -D "$DOCKER_PARAM" $LIMITS $VOLUMES $PARAMS $ALERTMANAGER_COMMAND $BIND_ADDRESS_CONFIG $ALERT_MANAGER_RULE_CONFIG
	if [ $? -ne 0 ]; then
		exit 1
	fi
	if [ $ALERTMANAGER_PORT = "9093" ]; then
		ALERTMANAGER_NAME=aalert
	else
		ALERTMANAGER_NAME=aalert-$ALERTMANAGER_PORT
	fi
	AM_ADDRESS=$(container_address $ALERTMANAGER_NAME $ALERTMANAGER_PORT)
fi

LOKI_ADDRESS=""
if [ $RUN_LOKI -eq 1 ]; then
	run_script ./start-loki.sh $BIND_ADDRESS_CONFIG $LOKI_DIR $LOKI_PORT_CMD $QUICK_STARTUP_CMD -D "$DOCKER_PARAM" $LIMITS $VOLUMES $PARAMS -m $AM_ADDRESS
	if [ $? -ne 0 ]; then
		exit 1
	fi
	if [ -z "$LOKI_PORT" ]; then
		LOKI_PORT=3100
	fi
	if [ $LOKI_PORT -eq 3100 ]; then
		LOKI_NAME=aloki
	else
		LOKI_NAME=aloki-$LOKI_PORT
	fi
	LOKI_ADDRESS=$(container_address $LOKI_NAME $LOKI_PORT)
	LOKI_ADDRESS="-L $LOKI_ADDRESS"
fi

if [ -z $PROMETHEUS_PORT ]; then
	PROMETHEUS_PORT=9090
	PROMETHEUS_NAME=aprom
else
	PROMETHEUS_NAME=aprom-$PROMETHEUS_PORT
fi

docker container inspect $PROMETHEUS_NAME >/dev/null 2>&1
if [ $? -eq 0 ]; then
	printf "\nSome of the monitoring docker instances ($PROMETHEUS_NAME) exist. Make sure all containers are killed and removed. You can use kill-all.sh for that\n"
	exit 1
fi

# Exit if Docker engine is not running
if [ ! "$(docker ps)" ]; then
	echo "Error: Docker engine is not running"
	exit 1
fi

for val in "${PROMETHEUS_COMMAND_LINE_OPTIONS_ARRAY[@]}"; do
	if [[ $val = "--"* ]]; then
		PROMETHEUS_COMMAND_LINE_OPTIONS+=" $val"
	else
		echo "Using single hyphen is deprecated and will be removed in future version use -$val instead"
		PROMETHEUS_COMMAND_LINE_OPTIONS+=" -$val"
	fi
done
if [ "$NATIVE_HISTOGRAM" = "1" ]; then
	NATIVE_HISTOGRAM="--native-histogram"
else
	NATIVE_HISTOGRAM=""
fi
./prometheus-config.sh -m $AM_ADDRESS $STACK_CMD $GRAFANA_ADDRESS $NATIVE_HISTOGRAM $SCRAP_CMD $CONSUL_ADDRESS $PROMETHEUS_TARGETS $VECTOR_SEARCH_CMD
if [ "$DATA_DIR" != "" ] && [ "$ARCHIVE" != "1" ]; then
	DATE=$(date +"%Y-%m-%d_%H_%M_%S")
	if [ -f $DATA_DIR/scylla.txt ]; then
		mv $DATA_DIR/scylla.txt $DATA_DIR/scylla.$DATE.txt
	fi
	echo LAST_COMMAND_LINE='"'"$@"'"' >$DATA_DIR/scylla.txt
	echo VERSIONS='"'"$VERSIONS"'"' >>$DATA_DIR/scylla.txt
	echo MANAGER_VERSION='"'"$MANAGER_VERSION"'"' >>$DATA_DIR/scylla.txt
	echo MONITORING_VERSION='"'"$CURRENT_VERSION"'"' >>$DATA_DIR/scylla.txt
	echo PROMETHEUS_VERSION='"'"$PROMETHEUS_VERSION"'"' >>$DATA_DIR/scylla.txt
	echo LAST_RUN='"'"$DATE"'"' >>$DATA_DIR/scylla.txt
	if [ "$RUN_ALTERNATOR" = "1" ]; then
		echo RUN_ALTERNATOR=1 >>$DATA_DIR/scylla.txt
	fi
fi
if [  "$HOST_NETWORK" != "host" ]; then
	PORT_MAPPING="-p $BIND_ADDRESS$PROMETHEUS_PORT:9090"
fi
if [[ "$VICTORIA_METRICS" = "1" ]]; then
	echo "Using victoria metrics"

	docker run -d --rm $DATA_DIR_CMD $PORT_MAPPING --name $PROMETHEUS_NAME \
		-v $PWD/prometheus/build/prometheus.yml:/etc/promscrape.config.yml:z \
		$SCYLLA_TARGET_FILE \
		$SCYLLA_MANGER_TARGET_FILE \
		$NODE_TARGET_FILE \
		$SCYLLA_MANGER_AGENT_TARGET_FILE \
		victoriametrics/victoria-metrics:$VICTORIA_METRICS_VERSION $PROMETHEUS_COMMAND_LINE_OPTIONS \
		${DOCKER_PARAMS["prometheus"]} -promscrape.config=/etc/promscrape.config.yml -promscrape.config.strictParse=false -httpListenAddr=:9090
else
	docker run -d $DOCKER_PARAM ${DOCKER_LIMITS["prometheus"]} $USER_PERMISSIONS \
		$DATA_DIR_CMD \
		"${group_args[@]}" \
		-v $PWD/prometheus/build$STACK/prometheus.yml:/etc/prometheus/prometheus.yml:z \
		-v $PROMETHEUS_RULES:z \
		$SCYLLA_TARGET_FILE \
		$SCYLLA_MANGER_TARGET_FILE \
		$NODE_TARGET_FILE \
		$VECTOR_SEARCH \
		$SCYLLA_MANGER_AGENT_TARGET_FILE \
		$PORT_MAPPING --name $PROMETHEUS_NAME docker.io/prom/prometheus:$PROMETHEUS_VERSION \
		--web.enable-lifecycle --enable-feature=promql-experimental-functions --config.file=/etc/prometheus/prometheus.yml $PROMETHEUS_COMMAND_LINE_OPTIONS \
		${DOCKER_PARAMS["prometheus"]}
fi

if [ $? -ne 0 ]; then
	echo "Error: Prometheus container failed to start"
	echo "For more information use: docker logs $PROMETHEUS_NAME"
	exit 1
fi

# Number of retries waiting for a Docker container to start
RETRIES=35

# Wait till Prometheus is available
printf "Wait for Prometheus container to start."
TRIES=0
if [ ! "$QUICK_STARTUP" = "1" ]; then
    until $(curl --output /dev/null -f --silent http://localhost:$PROMETHEUS_PORT) || [ $TRIES -eq $RETRIES ]; do
    	printf '.'
    	((TRIES = TRIES + 1))
    	sleep 1
    done
fi
echo

if [ ! "$(docker ps -q -f name=$PROMETHEUS_NAME)" ]; then
	echo "Error: Prometheus container failed to start"
	echo "For more information use: docker logs $PROMETHEUS_NAME"
	exit 1
fi

# Can't use localhost here, because the monitoring may be running remotely.
# Also note that the port to which we need to connect is 9090, regardless of which port we bind to at localhost.
DB_ADDRESS=$(container_address $PROMETHEUS_NAME 9090)

if [[ "$VICTORIA_METRICS" = "1" ]]; then
	echo "running vmalert"

	docker run -d \
		--name vmalert \
		-v $PROMETHEUS_RULES:z \
		victoriametrics/vmalert:$VICTORIA_METRICS_VERSION -rule=/etc/prometheus/prom_rules/*yml \
		-datasource.url=http://$DB_ADDRESS \
		-notifier.url=http://$AM_ADDRESS \
		-notifier.url=http://$AM_ADDRESS \
		-remoteWrite.url=http://$DB_ADDRESS \
		-remoteRead.url=http://$DB_ADDRESS
fi
if [ $RUN_THANOS_SC -eq 1 ]; then
	if [ -z $DATA_DIR ]; then
		log WARNING "You must use external prometheus directory to use the thanos side cart"
	else
		run_script ./start-thanos-sc.sh -d $DATA_DIR -D "$DOCKER_PARAM" -a $DB_ADDRESS $LIMITS $VOLUMES $PARAMS $BIND_ADDRESS_CONFIG
	fi
fi

if [ ! -z "$NO_THANOS_DATASOURCE" ]; then
    NO_THANOS_DATASOURCE="--no-thanos-datasource"
fi
if [ $RUN_THANOS -eq 1 ]; then
	run_script ./start-thanos.sh $NO_THANOS_DATASOURCE -D "$DOCKER_PARAM" $BIND_ADDRESS_CONFIG
elif [ "$RUN_LOCAL_THANOS" = "1" ]; then
    IP=$(docker inspect --format '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $PROMETHEUS_NAME)
    if [ "$IP" = "invalid IP" ] || [ -z "$IP" ]; then
       IP=""
    fi
    STORE_ADDRESS="$(docker inspect --format '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' sidecar1):10911"
    run_script ./start-thanos.sh $NO_THANOS_DATASOURCE -S $STORE_ADDRESS
fi

for val in "${GRAFANA_DASHBOARD_ARRAY[@]}"; do
	GRAFANA_DASHBOARD_COMMAND="$GRAFANA_DASHBOARD_COMMAND -j $val"
done
if [ ! -z "$DATDOGPARAM" ]; then
	run_script ./start-datadog.sh $DATDOGPARAM -p $DB_ADDRESS
fi
if [ "$RUN_ALTERNATOR" = 1 ]; then
	GRAFANA_ENV_ARRAY+=(--alternator)
fi
run_script ./start-grafana.sh $QUICK_STARTUP_CMD $SCRAP_CMD $LDAP_FILE $LOKI_ADDRESS $LIMITS $VOLUMES $PARAMS $BIND_ADDRESS_CONFIG $RUN_RENDERER $SPECIFIC_SOLUTION -p $DB_ADDRESS $GRAFNA_ANONYMOUS_ROLE -D "$DOCKER_PARAM" $GRAFANA_PORT $EXTERNAL_VOLUME -m $AM_ADDRESS -M $MANAGER_VERSION -v $VERSIONS "${GRAFANA_ENV_ARRAY[@]}" $GRAFANA_DASHBOARD_COMMAND $GRAFANA_ADMIN_PASSWORD $STACK_CMD $VECTOR_SEARCH_CMD