diff --git a/.gitignore b/.gitignore index e8bf9f9..b0549b5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ sshkeys/ backups/ +.dccache diff --git a/Dockerfile b/Dockerfile index 4bd5914..62ad1ca 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,6 +6,7 @@ ENV GID=1000 ENV MAINTENANCE_ENABLE="false" ENV INTERACTIVE_MODE="false" ENV RUN_INSTALL_SCRIPT="false" +ENV RUN_PROMETHEUS_EXPORTER="false" ENV TZ="" # Add Folders and Shell Scripts @@ -21,10 +22,13 @@ COPY variables.sh / COPY .bash_profile /root/ COPY .bashrc /root/ +COPY prometheus-borg-exporter/borg_exporter.sh /usr/local/bin/ +COPY prometheus-borg-exporter/borg_exporter.rc /etc/ + # Install packages RUN apk update ; apk upgrade -RUN apk add --no-cache sudo bash bash-completion tzdata openssh-server openrc neofetch \ - borgbackup +RUN apk add --no-cache sudo bash bash-completion tzdata openssh openrc neofetch \ + borgbackup dateutils prometheus-node-exporter curl RUN rm -rf /var/cache/apk/* # Setup SSH-Server diff --git a/README.md b/README.md index 140419d..60bafa2 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,18 @@ Create New Volumen into /logs if you like to log anything or get the logs ## Set Timezone - not into 1.1.17 Use ENV TZ="Your time zone" if not set will use UTC +## Export Data to Grafana and Prometheus - not into 1.1.17 +Borg exporter for Prometheus from https://github.com/mad-ady/prometheus-borg-exporter + +Use ENV RUN_PROMETHEUS_EXPORTER and set it to any CRONJOB TASK like ```0 * * * *``` to update the /logs/borg_exporter.prom every hour +Config is into ```/etc/borg_exporter.rc``` sample config: +``` +BORG_PASSPHRASE="" +REPOSITORY="" +PUSHGATEWAY_URL="" +BASEREPODIR="/backups" +``` + ## borgbackup Version into Tags | TAG | Borg Backup Version | Alpine Version | diff --git a/build.sh b/build.sh index 958efd5..b174c0d 100755 --- a/build.sh +++ b/build.sh @@ -7,11 +7,13 @@ GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD) run_docker_container() { echo "Running..." docker run -dp 3000:22 \ + -p 9100:9100 \ -e UID=$(id -u) \ -e GID=$(id -g) \ -e MAINTENANCE_ENABLE="true" \ -e INTERACTIVE_MODE="true" \ -e TZ="Europe/Vienna" \ + -e RUN_PROMETHEUS_EXPORTER="0 * * * *" \ -v "$PWD"/Testing/crontab.txt:/crontab.txt \ -v "$PWD"/Testing/test_script.sh:/test_script.sh \ -v "$PWD"/sshkeys/clients:/sshkeys/clients \ diff --git a/entrypoint.sh b/entrypoint.sh index 3ec0acc..57814bb 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -144,6 +144,24 @@ function run_install_script { fi fi } + +function run_prometheus_exporter() { + if [ "$RUN_PROMETHEUS_EXPORTER" != "false" ]; then + echo "* STARTING Prometheus Exporter for Borg Backup" + + crontab -l > /tmp/cron_bkp + echo "" >> /tmp/cron_bkp + + echo "* Add Cronjob to Crontab" + echo "$RUN_PROMETHEUS_EXPORTER /usr/local/bin/borg_exporter.sh 2>&1" >> /tmp/cron_bkp + crontab /tmp/cron_bkp + rm /tmp/cron_bkp + + echo "* STARTING Node Exporter" + node_exporter --collector.textfile.directory="$NODE_EXPORTER_DIR" & + sepurator + fi +} ############################################################################################################################## # Main Code ############################################################################################################################## @@ -159,6 +177,7 @@ sepurator maintenance_enable set_timezone +run_prometheus_exporter run_install_script echo "* Init done! - Starting SSH-Daemon..." diff --git a/prometheus-borg-exporter/borg_exporter.rc b/prometheus-borg-exporter/borg_exporter.rc new file mode 100644 index 0000000..aae2ee4 --- /dev/null +++ b/prometheus-borg-exporter/borg_exporter.rc @@ -0,0 +1,4 @@ +BORG_PASSPHRASE="" +REPOSITORY="" +PUSHGATEWAY_URL="" +BASEREPODIR="/backups" diff --git a/prometheus-borg-exporter/borg_exporter.sh b/prometheus-borg-exporter/borg_exporter.sh new file mode 100755 index 0000000..06f8fd9 --- /dev/null +++ b/prometheus-borg-exporter/borg_exporter.sh @@ -0,0 +1,218 @@ +#!/bin/bash +source /etc/borg_exporter.rc +source /variables.sh + +TMP_FILE="/tmp/prometheus-borg" +DATEDIFF=`which datediff` +if [ -z "$DATEDIFF" ]; then + #ubuntu packages have a different executable name + DATEDIFF=`which dateutils.ddiff` +fi + +[ -e $TMP_FILE ] && rm -f $TMP_FILE + +#prevent "Attempting to access a previously unknown unencrypted repository" prompt +export BORG_RELOCATED_REPO_ACCESS_IS_OK=yes +export BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK=yes +HOSTNAME=$(hostname) +############################################################################################################################## +# Funktionen +############################################################################################################################## +function writeToFile() { + msg="$1" + local -n arr=$2 + + for key in "${!arr[@]}"; do + echo "$key$msg ${arr[$key]}" >> $TMP_FILE + done +} + +function calc_bytes { + NUM=$1 + UNIT=$2 + + case "$UNIT" in + kB) + echo $NUM | awk '{ print $1 * 1024 }' + ;; + MB) + echo $NUM | awk '{ print $1 * 1024 * 1024 }' + ;; + GB) + echo $NUM | awk '{ print $1 * 1024 * 1024 * 1024 }' + ;; + TB) + echo $NUM | awk '{ print $1 * 1024 * 1024 * 1024 * 1024 }' + ;; + B) + echo $NUM | awk '{ print $1 }' + ;; + esac +} + +function writeDefinitionsToMetrics() { + #print the definition of the metrics + echo "# HELP borg_hours_from_last_archive How many hours have passed since the last archive was added to the repo (counted by borg_exporter.sh)" >> $TMP_FILE + echo "# TYPE borg_hours_from_last_archive gauge" >> $TMP_FILE + echo "# HELP borg_last_archive_timestamp The timestamp of the last archive (unixtimestamp)" >> $TMP_FILE + echo "# TYPE borg_last_archive_timestamp gauge" >> $TMP_FILE + echo "# HELP borg_total_size The total size of all archives in the repo" >> $TMP_FILE + echo "# TYPE borg_total_size gauge" >> $TMP_FILE + echo "# HELP borg_total_size_compressed The total compressed size of all archives in the repo" >> $TMP_FILE + echo "# TYPE borg_total_size_compressed gauge" >> $TMP_FILE + echo "# HELP borg_total_size_dedup The total deduplicated size of all archives in the repo (size on disk)" >> $TMP_FILE + echo "# TYPE borg_total_size_dedup gauge" >> $TMP_FILE + echo "# HELP borg_archives_count The total number of archives in the repo" >> $TMP_FILE + echo "# TYPE borg_archives_count gauge" >> $TMP_FILE + echo "# HELP borg_archives_count_today The total number of archives created today in the repo" >> $TMP_FILE + echo "# TYPE borg_archives_count_today gauge" >> $TMP_FILE + echo "# HELP borg_files_count The number of files contained in the archive (today)" >> $TMP_FILE + echo "# TYPE borg_files_count gauge" >> $TMP_FILE + echo "# HELP borg_chunks_unique The number of unique chunks in the repo" >> $TMP_FILE + echo "# TYPE borg_chunks_unique gauge" >> $TMP_FILE + echo "# HELP borg_chunks_total The total number of chunks in the repo" >> $TMP_FILE + echo "# TYPE borg_chunks_total gauge" >> $TMP_FILE + echo "# HELP borg_last_size The size of the archive (today)" >> $TMP_FILE + echo "# TYPE borg_last_size gauge" >> $TMP_FILE + echo "# HELP borg_last_size_compressed The compressed size of the archive (today)" >> $TMP_FILE + echo "# TYPE borg_last_size_compressed gauge" >> $TMP_FILE + echo "# HELP borg_last_size_dedup The deduplicated size of the archive (today), (size on disk)" >> $TMP_FILE + echo "# TYPE borg_last_size_dedup gauge" >> $TMP_FILE +} + +function getBorgDataForRepository { + REPOSITORY=$1 #repository we're looking into + host=$2 #the host for which the backups are made + declare -A repoData + + ARCHIVES="$(BORG_PASSPHRASE=$BORG_PASSPHRASE borg list $REPOSITORY)" + COUNTER=0 + BACKUPS_TODAY_COUNT=0 + + COUNTER=$(echo "$ARCHIVES" | wc -l) + TODAY=$(date +%Y-%m-%d) + BACKUPS_TODAY=$(echo "$ARCHIVES" | grep ", $TODAY ") + BACKUPS_TODAY_COUNT=$(echo -n "$BACKUPS_TODAY" | wc -l) + + #extract data for last archive + LAST_ARCHIVE=$(BORG_PASSPHRASE=$BORG_PASSPHRASE borg list --last 1 $REPOSITORY) + #we need at least one valid backup to list anything meaningfull + if [ -n "${LAST_ARCHIVE}" ]; then + LAST_ARCHIVE_DATE=$(echo $LAST_ARCHIVE | awk '{print $3" "$4}') + CURRENT_DATE="$(date '+%Y-%m-%d %H:%M:%S')" + NB_HOUR_FROM_LAST_BCK=$($DATEDIFF "$LAST_ARCHIVE_DATE" "$CURRENT_DATE" -f '%H') + + # in case the date parsing from BORG didn't work (e.g. archive with space in it), datediff will output + # a usage message on stdout and will break prometheus formatting. We need to + # check for that here + DATEDIFF_LINES=$(echo "$NB_HOUR_FROM_LAST_BCK" | wc -l) + if [ "${DATEDIFF_LINES}" -eq 1 ]; then + repoData["borg_hours_from_last_archive"]="$NB_HOUR_FROM_LAST_BCK" + repoData["borg_last_archive_timestamp"]=$(date -d "$LAST_ARCHIVE_DATE" +"%s") + + BORG_INFO=$(BORG_PASSPHRASE="$BORG_PASSPHRASE" borg info "$REPOSITORY") + + TOTAL_SIZE=$(calc_bytes $(echo "$BORG_INFO" |grep "All archives" |awk '{print $3}') $(echo "$BORG_INFO" |grep "All archives" |awk '{print $4}')) + TOTAL_SIZE_COMPRESSED=$(calc_bytes $(echo "$BORG_INFO" |grep "All archives" |awk '{print $5}') $(echo "$BORG_INFO" |grep "All archives" |awk '{print $6}')) + TOTAL_SIZE_DEDUP=$(calc_bytes $(echo "$BORG_INFO" |grep "All archives" |awk '{print $7}') $(echo "$BORG_INFO" |grep "All archives" |awk '{print $8}')) + + #echo "Borg REPOSITORY: $REPOSITORY" + #echo "Total size: $TOTAL_SIZE, Compressed size: $TOTAL_SIZE_COMPRESSED, DEDUP size: $TOTAL_SIZE_DEDUP" + + repoData["borg_total_size"]="$TOTAL_SIZE" + repoData["borg_total_size_compressed"]="$TOTAL_SIZE_COMPRESSED" + repoData["borg_total_size_dedup"]="$TOTAL_SIZE_DEDUP" + + repoData["borg_chunks_unique"]=$(echo "$BORG_INFO" | grep "Chunk index" | awk '{print $3}') + repoData["borg_chunks_total"]=$(echo "$BORG_INFO" | grep "Chunk index" | awk '{print $4}') + fi + + repoData["borg_archives_count"]="$COUNTER" + repoData["borg_archives_count_today"]="$BACKUPS_TODAY_COUNT" + + #Write Repo Array to File + writeToFile "{host=\"$host\", backupserver=\"$HOSTNAME\", repo=\"$REPOSITORY\"}" repoData + + #go through the day's archives and count the files/size/etc. + TODAY_ARCHIVES=$(echo -n "$BACKUPS_TODAY" | awk '{print $1}' | xargs echo ) + #echo $TODAY_ARCHIVES + if [ -n "${TODAY_ARCHIVES}" ]; then + for archive in $TODAY_ARCHIVES; do + declare -A archiveData + + echo "Looking at $REPOSITORY::$archive" + #ask for an info on it + CURRENT_INFO=$(BORG_PASSPHRASE="$BORG_PASSPHRASE" borg info "$REPOSITORY::$archive") + #cut out something that looks like a timestamp when reporting: 20210528-1315 + readable_archive=$(echo $archive | sed -r "s/-[0-9]{8}-[0-9]{4,6}//") + + archiveData['borg_files_count']=$(echo "$CURRENT_INFO" | grep "Number of files" | awk '{print $4}') + + # byte size + LAST_SIZE=$(calc_bytes $(echo "$CURRENT_INFO" |grep "This archive" |awk '{print $3}') $(echo "$CURRENT_INFO" |grep "This archive" |awk '{print $4}')) + LAST_SIZE_COMPRESSED=$(calc_bytes $(echo "$CURRENT_INFO" |grep "This archive" |awk '{print $5}') $(echo "$CURRENT_INFO" |grep "This archive" |awk '{print $6}')) + LAST_SIZE_DEDUP=$(calc_bytes $(echo "$CURRENT_INFO" |grep "This archive" |awk '{print $7}') $(echo "$CURRENT_INFO" |grep "This archive" |awk '{print $8}')) + + archiveData['borg_last_size']="$LAST_SIZE" + archiveData['borg_last_size_compressed']="$LAST_SIZE_COMPRESSED" + archiveData['borg_last_size_dedup']="$LAST_SIZE_DEDUP" + + writeToFile "{host=\"$host\", backupserver=\"$HOSTNAME\", repo=\"$REPOSITORY\", archive=\"$readable_archive\"}" archiveData + done + else + echo "Unable to find any archives for today in $REPOSITORY." + fi + else + echo "Unable to find any archives in $REPOSITORY. Processing skipped for it" + fi +} + +function findRepositorysAndGetData() { + REPOS=`find "$BASEREPODIR" -type f -name "README" | grep -v ".cache/borg"` + # e.g. /backup/servers/server_name/README + for REPO in $REPOS; do + #cut out the /README from the name + REPO=$(echo "$REPO" | sed -r "s/\/README//") + #assume the name convention for the repo contains the hostname as the repo name + # e.g. /backup/servers/server_name + host=$(basename "$REPO") + getBorgDataForRepository $REPO $host + done +} + +function sendDataToGatewayOrNodeExplorer() { + if [ -n "${PUSHGATEWAY_URL}" ]; then + #send data via pushgateway + cat $TMP_FILE | curl --data-binary @- ${PUSHGATEWAY_URL}/metrics/job/borg-exporter/host/$HOSTNAME/repository/$REPOSITORY + else + #send data via node_exporter + if [ -d "${NODE_EXPORTER_DIR}" ]; then + cp $TMP_FILE ${NODE_EXPORTER_DIR}/borg_exporter.prom + else + echo "Please configure either PUSHGATEWAY_URL or NODE_EXPORTER_DIR in /etc/borg_exporter.rc" + fi + fi +} + +function cleanup() { + rm -f $TMP_FILE +} + +############################################################################################################################## +# Main Code +############################################################################################################################## +writeDefinitionsToMetrics +if [ -n "${REPOSITORY}" ]; then + getBorgDataForRepository "${REPOSITORY}" "${HOSTNAME}" +else + #discover (recursively) borg repositories starting from a path and extract info for each + #(e.g. when running on the backup server directly) + if [ -d "${BASEREPODIR}" ]; then + findRepositorysAndGetData + else + echo "Error: Either set REPOSITORY or BASEREPODIR in /etc/borg_exporter.rc" + fi +fi + +sendDataToGatewayOrNodeExplorer +cleanup diff --git a/variables.sh b/variables.sh index 144ef5f..b9b74bb 100644 --- a/variables.sh +++ b/variables.sh @@ -1,6 +1,7 @@ -DOCKER_IMAGE_VERSION="1.0.9" +DOCKER_IMAGE_VERSION="2.0.0" BORG_VERSION=$(borg -V) SSH_FOLDERS=( "/sshkeys/clients" "/sshkeys/host" ) +NODE_EXPORTER_DIR="/logs" ############################################################################################################################## # Funktionen ##############################################################################################################################