From 7545e77c1e63271cee082d9a2fe090984f691635 Mon Sep 17 00:00:00 2001 From: Dominik Deutsch Date: Wed, 12 May 2021 14:16:27 +0200 Subject: [PATCH 1/7] Added support for Slack / MS Teams webhooks --- Dockerfile | 4 +++- README.md | 2 ++ docker-entrypoint | 20 ++++++++++++++++++++ 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 18d1230..f667391 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,7 +10,9 @@ ENV AUTOHEAL_CONTAINER_LABEL=autoheal \ AUTOHEAL_INTERVAL=5 \ AUTOHEAL_DEFAULT_STOP_TIMEOUT=10 \ DOCKER_SOCK=/var/run/docker.sock \ - CURL_TIMEOUT=30 + CURL_TIMEOUT=30 \ + WEBHOOK_SLACK="" \ + WEBHOOK_TEAMS="" HEALTHCHECK --interval=5s CMD pgrep -f autoheal || exit 1 diff --git a/README.md b/README.md index 3a749db..4eabdb4 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,8 @@ AUTOHEAL_START_PERIOD=0 # wait 0 seconds before first health check AUTOHEAL_DEFAULT_STOP_TIMEOUT=10 # Docker waits max 10 seconds (the Docker default) for a container to stop before killing during restarts (container overridable via label, see below) DOCKER_SOCK=/var/run/docker.sock # Unix socket for curl requests to Docker API CURL_TIMEOUT=30 # --max-time seconds for curl requests to Docker API +WEBHOOK_SLACK="" # post message to the slack webhook if a container was restarted (or restart failed) +WEBHOOK_TEAMS="" # post message to the teams webhook if a container was restarted (or restart failed) ``` ### Optional Container Labels diff --git a/docker-entrypoint b/docker-entrypoint index 96b494f..9cbd166 100755 --- a/docker-entrypoint +++ b/docker-entrypoint @@ -7,6 +7,8 @@ set -o pipefail DOCKER_SOCK=${DOCKER_SOCK:-/var/run/docker.sock} UNIX_SOCK="" CURL_TIMEOUT=${CURL_TIMEOUT:-30} +WEBHOOK_TEAMS=${WEBHOOK_TEAMS:-""} +WEBHOOK_SLACK=${WEBHOOK_SLACK:-""} # only use unix domain socket if no TCP endpoint is defined case "${DOCKER_SOCK}" in @@ -56,6 +58,21 @@ restart_container() { docker_curl -f -X POST "${HTTP_ENDPOINT}/containers/${container_id}/restart?t=${timeout}" } +notify_webhook() { + local text="$1" + local data="{\"text\": \"${text}\"}" + + if [ -n "$WEBHOOK_SLACK" ] + then + curl -X POST -H "Content-type: application/json" -d $data $WEBHOOK_SLACK + fi + + if [ -n "$WEBHOOK_TEAMS" ] + then + curl -X POST -H "Content-Type: application/json" -d $data $WEBHOOK_TEAMS + fi +} + # SIGTERM-handler term_handler() { exit 143 # 128 + 15 -- SIGTERM @@ -94,6 +111,9 @@ if [ "$1" = "autoheal" ] && [ -e "$DOCKER_SOCK" ];then if ! restart_container "$CONTAINER_ID" "$TIMEOUT" then echo "$DATE Restarting container $CONTAINER_SHORT_ID failed" >&2 + notify_webhook "Container $CONTAINER_NAME (${CONTAINER_SHORT_ID}) found to be unhealthy. Failed to restart the container." + else + notify_webhook "Container $CONTAINER_NAME (${CONTAINER_SHORT_ID}) found to be unhealthy. Successfully restarted the container" fi fi done From a1d0e0e8258f7eec32e15725a0cb0876f47a059c Mon Sep 17 00:00:00 2001 From: Dominik Deutsch Date: Wed, 12 May 2021 15:34:32 +0200 Subject: [PATCH 2/7] Notify webhooks using a generic URL --- Dockerfile | 3 +-- README.md | 3 +-- docker-entrypoint | 12 +++--------- 3 files changed, 5 insertions(+), 13 deletions(-) diff --git a/Dockerfile b/Dockerfile index f667391..44a3d2e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,8 +11,7 @@ ENV AUTOHEAL_CONTAINER_LABEL=autoheal \ AUTOHEAL_DEFAULT_STOP_TIMEOUT=10 \ DOCKER_SOCK=/var/run/docker.sock \ CURL_TIMEOUT=30 \ - WEBHOOK_SLACK="" \ - WEBHOOK_TEAMS="" + WEBHOOK_URL="" HEALTHCHECK --interval=5s CMD pgrep -f autoheal || exit 1 diff --git a/README.md b/README.md index 4eabdb4..f9bcdcb 100644 --- a/README.md +++ b/README.md @@ -60,8 +60,7 @@ AUTOHEAL_START_PERIOD=0 # wait 0 seconds before first health check AUTOHEAL_DEFAULT_STOP_TIMEOUT=10 # Docker waits max 10 seconds (the Docker default) for a container to stop before killing during restarts (container overridable via label, see below) DOCKER_SOCK=/var/run/docker.sock # Unix socket for curl requests to Docker API CURL_TIMEOUT=30 # --max-time seconds for curl requests to Docker API -WEBHOOK_SLACK="" # post message to the slack webhook if a container was restarted (or restart failed) -WEBHOOK_TEAMS="" # post message to the teams webhook if a container was restarted (or restart failed) +WEBHOOK_URL="" # post message to the webhook if a container was restarted (or restart failed) ``` ### Optional Container Labels diff --git a/docker-entrypoint b/docker-entrypoint index 9cbd166..282df19 100755 --- a/docker-entrypoint +++ b/docker-entrypoint @@ -7,8 +7,7 @@ set -o pipefail DOCKER_SOCK=${DOCKER_SOCK:-/var/run/docker.sock} UNIX_SOCK="" CURL_TIMEOUT=${CURL_TIMEOUT:-30} -WEBHOOK_TEAMS=${WEBHOOK_TEAMS:-""} -WEBHOOK_SLACK=${WEBHOOK_SLACK:-""} +WEBHOOK_URL=${WEBHOOK_URL:-""} # only use unix domain socket if no TCP endpoint is defined case "${DOCKER_SOCK}" in @@ -62,14 +61,9 @@ notify_webhook() { local text="$1" local data="{\"text\": \"${text}\"}" - if [ -n "$WEBHOOK_SLACK" ] + if [ -n "$WEBHOOK_URL" ] then - curl -X POST -H "Content-type: application/json" -d $data $WEBHOOK_SLACK - fi - - if [ -n "$WEBHOOK_TEAMS" ] - then - curl -X POST -H "Content-Type: application/json" -d $data $WEBHOOK_TEAMS + curl -X POST -H "Content-type: application/json" -d $data $WEBHOOK_URL fi } From 6adc217055d4654ec1e1ffa6a0b8236b13720c3a Mon Sep 17 00:00:00 2001 From: Dominik Deutsch Date: Wed, 12 May 2021 17:06:11 +0200 Subject: [PATCH 3/7] Fixed alert messages --- docker-entrypoint | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/docker-entrypoint b/docker-entrypoint index 282df19..34e39a0 100755 --- a/docker-entrypoint +++ b/docker-entrypoint @@ -58,15 +58,24 @@ restart_container() { } notify_webhook() { - local text="$1" - local data="{\"text\": \"${text}\"}" + local text="$@" if [ -n "$WEBHOOK_URL" ] then - curl -X POST -H "Content-type: application/json" -d $data $WEBHOOK_URL + curl -X POST -H "Content-type: application/json" -d "$(generate_webhook_payload $text)" $WEBHOOK_URL fi } +# https://towardsdatascience.com/proper-ways-to-pass-environment-variables-in-json-for-curl-post-f797d2698bf3 +generate_webhook_payload() { + local text="$@" + cat <&2 - notify_webhook "Container $CONTAINER_NAME (${CONTAINER_SHORT_ID}) found to be unhealthy. Failed to restart the container." + notify_webhook "Container ${CONTAINER_NAME:1} (${CONTAINER_SHORT_ID}) found to be unhealthy. Failed to restart the container!" else - notify_webhook "Container $CONTAINER_NAME (${CONTAINER_SHORT_ID}) found to be unhealthy. Successfully restarted the container" + notify_webhook "Container ${CONTAINER_NAME:1} (${CONTAINER_SHORT_ID}) found to be unhealthy. Successfully restarted the container!" fi fi done From 449db175005a0dee56d2e84665c67f0695280f8c Mon Sep 17 00:00:00 2001 From: Dominik Deutsch Date: Wed, 12 May 2021 20:37:34 +0200 Subject: [PATCH 4/7] Execute webhook requests as background processes --- docker-entrypoint | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker-entrypoint b/docker-entrypoint index 34e39a0..386282f 100755 --- a/docker-entrypoint +++ b/docker-entrypoint @@ -62,7 +62,8 @@ notify_webhook() { if [ -n "$WEBHOOK_URL" ] then - curl -X POST -H "Content-type: application/json" -d "$(generate_webhook_payload $text)" $WEBHOOK_URL + # execute webhook requests as background process to prevent healer from blocking + curl -X POST -H "Content-type: application/json" -d "$(generate_webhook_payload $text)" $WEBHOOK_URL & fi } From 5bf4edcac8ed7e61692ecfd50a816f4c2695e7d6 Mon Sep 17 00:00:00 2001 From: Dominik Deutsch Date: Wed, 12 May 2021 22:31:42 +0200 Subject: [PATCH 5/7] Run notify_webhook as background process instead of just the curl command --- docker-entrypoint | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker-entrypoint b/docker-entrypoint index 386282f..eb482bf 100755 --- a/docker-entrypoint +++ b/docker-entrypoint @@ -63,7 +63,7 @@ notify_webhook() { if [ -n "$WEBHOOK_URL" ] then # execute webhook requests as background process to prevent healer from blocking - curl -X POST -H "Content-type: application/json" -d "$(generate_webhook_payload $text)" $WEBHOOK_URL & + curl -X POST -H "Content-type: application/json" -d "$(generate_webhook_payload $text)" $WEBHOOK_URL fi } @@ -115,9 +115,9 @@ if [ "$1" = "autoheal" ] && [ -e "$DOCKER_SOCK" ];then if ! restart_container "$CONTAINER_ID" "$TIMEOUT" then echo "$DATE Restarting container $CONTAINER_SHORT_ID failed" >&2 - notify_webhook "Container ${CONTAINER_NAME:1} (${CONTAINER_SHORT_ID}) found to be unhealthy. Failed to restart the container!" + notify_webhook "Container ${CONTAINER_NAME:1} (${CONTAINER_SHORT_ID}) found to be unhealthy. Failed to restart the container!" & else - notify_webhook "Container ${CONTAINER_NAME:1} (${CONTAINER_SHORT_ID}) found to be unhealthy. Successfully restarted the container!" + notify_webhook "Container ${CONTAINER_NAME:1} (${CONTAINER_SHORT_ID}) found to be unhealthy. Successfully restarted the container!" & fi fi done From 5b5d43e913f4be64ab3826429268fc4cb7fffda5 Mon Sep 17 00:00:00 2001 From: Dominik Deutsch Date: Fri, 11 Jun 2021 14:11:06 +0200 Subject: [PATCH 6/7] Added NOTIFY_ONLY option to disable automated container restarts --- README.md | 1 + docker-entrypoint | 19 +++++++++++++------ 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index f9bcdcb..048ba84 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,7 @@ AUTOHEAL_DEFAULT_STOP_TIMEOUT=10 # Docker waits max 10 seconds (the Docker def DOCKER_SOCK=/var/run/docker.sock # Unix socket for curl requests to Docker API CURL_TIMEOUT=30 # --max-time seconds for curl requests to Docker API WEBHOOK_URL="" # post message to the webhook if a container was restarted (or restart failed) +NOTIFY_ONLY="" # set this to true if you want to disable the automated restart ``` ### Optional Container Labels diff --git a/docker-entrypoint b/docker-entrypoint index eb482bf..dcf8267 100755 --- a/docker-entrypoint +++ b/docker-entrypoint @@ -8,6 +8,7 @@ DOCKER_SOCK=${DOCKER_SOCK:-/var/run/docker.sock} UNIX_SOCK="" CURL_TIMEOUT=${CURL_TIMEOUT:-30} WEBHOOK_URL=${WEBHOOK_URL:-""} +NOTIFY_ONLY=${NOTIFY_ONLY:-""} # only use unix domain socket if no TCP endpoint is defined case "${DOCKER_SOCK}" in @@ -111,13 +112,19 @@ if [ "$1" = "autoheal" ] && [ -e "$DOCKER_SOCK" ];then then echo "$DATE Container $CONTAINER_NAME (${CONTAINER_SHORT_ID}) found to be restarting - don't restart" else - echo "$DATE Container $CONTAINER_NAME (${CONTAINER_SHORT_ID}) found to be unhealthy - Restarting container now with ${TIMEOUT}s timeout" - if ! restart_container "$CONTAINER_ID" "$TIMEOUT" + + if [ -n "$NOTIFY_ONLY" ] then - echo "$DATE Restarting container $CONTAINER_SHORT_ID failed" >&2 - notify_webhook "Container ${CONTAINER_NAME:1} (${CONTAINER_SHORT_ID}) found to be unhealthy. Failed to restart the container!" & - else - notify_webhook "Container ${CONTAINER_NAME:1} (${CONTAINER_SHORT_ID}) found to be unhealthy. Successfully restarted the container!" & + notify_webhook "Container ${CONTAINER_NAME:1} (${CONTAINER_SHORT_ID}) found to be unhealthy. Automated container restard is disabled by NOTIFY_ONLY env variable." & + else + echo "$DATE Container $CONTAINER_NAME (${CONTAINER_SHORT_ID}) found to be unhealthy - Restarting container now with ${TIMEOUT}s timeout" + if ! restart_container "$CONTAINER_ID" "$TIMEOUT" + then + echo "$DATE Restarting container $CONTAINER_SHORT_ID failed" >&2 + notify_webhook "Container ${CONTAINER_NAME:1} (${CONTAINER_SHORT_ID}) found to be unhealthy. Failed to restart the container!" & + else + notify_webhook "Container ${CONTAINER_NAME:1} (${CONTAINER_SHORT_ID}) found to be unhealthy. Successfully restarted the container!" & + fi fi fi done From 7f8ed3acd1d70a57f0a36a7369b6c9b0be8799a2 Mon Sep 17 00:00:00 2001 From: Dominik Deutsch Date: Sun, 11 Jul 2021 16:54:52 +0200 Subject: [PATCH 7/7] Removed notify only option --- README.md | 1 - docker-entrypoint | 19 ++++++------------- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 048ba84..f9bcdcb 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,6 @@ AUTOHEAL_DEFAULT_STOP_TIMEOUT=10 # Docker waits max 10 seconds (the Docker def DOCKER_SOCK=/var/run/docker.sock # Unix socket for curl requests to Docker API CURL_TIMEOUT=30 # --max-time seconds for curl requests to Docker API WEBHOOK_URL="" # post message to the webhook if a container was restarted (or restart failed) -NOTIFY_ONLY="" # set this to true if you want to disable the automated restart ``` ### Optional Container Labels diff --git a/docker-entrypoint b/docker-entrypoint index dcf8267..eb482bf 100755 --- a/docker-entrypoint +++ b/docker-entrypoint @@ -8,7 +8,6 @@ DOCKER_SOCK=${DOCKER_SOCK:-/var/run/docker.sock} UNIX_SOCK="" CURL_TIMEOUT=${CURL_TIMEOUT:-30} WEBHOOK_URL=${WEBHOOK_URL:-""} -NOTIFY_ONLY=${NOTIFY_ONLY:-""} # only use unix domain socket if no TCP endpoint is defined case "${DOCKER_SOCK}" in @@ -112,19 +111,13 @@ if [ "$1" = "autoheal" ] && [ -e "$DOCKER_SOCK" ];then then echo "$DATE Container $CONTAINER_NAME (${CONTAINER_SHORT_ID}) found to be restarting - don't restart" else - - if [ -n "$NOTIFY_ONLY" ] + echo "$DATE Container $CONTAINER_NAME (${CONTAINER_SHORT_ID}) found to be unhealthy - Restarting container now with ${TIMEOUT}s timeout" + if ! restart_container "$CONTAINER_ID" "$TIMEOUT" then - notify_webhook "Container ${CONTAINER_NAME:1} (${CONTAINER_SHORT_ID}) found to be unhealthy. Automated container restard is disabled by NOTIFY_ONLY env variable." & - else - echo "$DATE Container $CONTAINER_NAME (${CONTAINER_SHORT_ID}) found to be unhealthy - Restarting container now with ${TIMEOUT}s timeout" - if ! restart_container "$CONTAINER_ID" "$TIMEOUT" - then - echo "$DATE Restarting container $CONTAINER_SHORT_ID failed" >&2 - notify_webhook "Container ${CONTAINER_NAME:1} (${CONTAINER_SHORT_ID}) found to be unhealthy. Failed to restart the container!" & - else - notify_webhook "Container ${CONTAINER_NAME:1} (${CONTAINER_SHORT_ID}) found to be unhealthy. Successfully restarted the container!" & - fi + echo "$DATE Restarting container $CONTAINER_SHORT_ID failed" >&2 + notify_webhook "Container ${CONTAINER_NAME:1} (${CONTAINER_SHORT_ID}) found to be unhealthy. Failed to restart the container!" & + else + notify_webhook "Container ${CONTAINER_NAME:1} (${CONTAINER_SHORT_ID}) found to be unhealthy. Successfully restarted the container!" & fi fi done