diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6560709 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ + + +# IDE +.idea +*.iml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..283b24d --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,15 @@ +image: docker:latest + +services: + - docker:dind +stages: + - build_and_deploy +job: + stage: build_and_deploy + script: + - mkdir ~/.docker + - echo '{"experimental":"enabled"}' | tee ~/.docker/config.json + - apk add --no-cache bash + - ls -al ~/.docker/config.json + - cat ~/.docker/config.json + - bash build.sh $DOCKER_ENV_CI_PROJECT_PATH $CI_REGISTRY_USER $CI_REGISTRY_PASSWORD $CI_REGISTRY diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..b6d5c6f --- /dev/null +++ b/.travis.yml @@ -0,0 +1,24 @@ +sudo: required +services: + - docker +language: bash + +env: + global: + - TRAVIS_SECURE_ENV_VARS=true + +before_install: + - sudo apt-get remove docker docker-engine docker.io + - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - + - sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + - sudo apt-get update + - sudo apt-get -y install docker-ce + - echo '{"experimental":true}' | sudo tee /etc/docker/daemon.json + - sudo service docker restart + - mkdir ~/.docker + - echo '{"experimental":"enabled"}' | tee ~/.docker/config.json + - ls -al ~/.docker/config.json + - cat ~/.docker/config.json + +script: + - bash build.sh $TRAVIS_REPO_SLUG $DOCKER_USERNAME $DOCKER_PASSWORD index.docker.io diff --git a/Dockerfile b/Dockerfile index 209ee3c..ae825de 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,5 @@ -FROM alpine:3.5 +ARG arch=x86_64 +FROM multiarch/alpine:${arch}-v3.8 RUN apk add --no-cache curl jq @@ -7,4 +8,4 @@ ENTRYPOINT ["/docker-entrypoint"] HEALTHCHECK --interval=5s CMD exit 0 -CMD ["autoheal"] \ No newline at end of file +CMD ["autoheal"] diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..0043319 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2018 will Farrell + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index 7041491..ffaab78 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # docker-autoheal Monitor and restart unhealthy docker containers. -This functionality was propose to be included with the addition of `HEALTHCHECK`, however didn't make the cut. +This functionality was proposed to be included with the addition of `HEALTHCHECK`, however didn't make the cut. This container is a stand-in till there is native support for `--exit-on-unhealthy` https://github.com/docker/docker/pull/22719. ## Supported tags and Dockerfile links @@ -10,19 +10,35 @@ This container is a stand-in till there is native support for `--exit-on-unhealt [![](https://images.microbadger.com/badges/version/willfarrell/autoheal.svg)](http://microbadger.com/images/willfarrell/autoheal "Get your own version badge on microbadger.com") [![](https://images.microbadger.com/badges/image/willfarrell/autoheal.svg)](http://microbadger.com/images/willfarrell/autoheal "Get your own image badge on microbadger.com") ## How to use +```bash +docker run -d \ + --name autoheal \ + --restart=always \ + -e AUTOHEAL_CONTAINER_LABEL=all \ + -v /var/run/docker.sock:/var/run/docker.sock \ + willfarrell/autoheal +``` a) Apply the label `autoheal=true` to your container to have it watched. b) Set ENV `AUTOHEAL_CONTAINER_LABEL=all` to watch all running containers. c) Set ENV `AUTOHEAL_CONTAINER_LABEL` to existing label name that has the value `true`. -Note: You must apply `HEALTHCHECK` to your docker images first. See https://docs.docker.com/engine/reference/builder/#/healthcheck for details. +Note: You must apply `HEALTHCHECK` to your docker images first. See https://docs.docker.com/engine/reference/builder/#healthcheck for details. ## ENV Defaults ``` AUTOHEAL_CONTAINER_LABEL=autoheal -AUTOHEAL_INTERVAL=5 -DOCKER_SOCK=/var/run/docker.sock +AUTOHEAL_INTERVAL=5 # check every 5 seconds +AUTOHEAL_START_PERIOD=0 # wait 0 seconds before first health check +AUTOHEAL_DEFAULT_STOP_TIMEOUT=10 # Docker waits max 10 seconds (the Docker default) for a container to stop before killing during restarts (container overridable via label, see below) +DOCKER_SOCK=/var/run/docker.sock # Unix socket for curl requests to Docker API +CURL_TIMEOUT=30 # --max-time seconds for curl requests to Docker API +``` + +### Optional Container Labels +``` +autoheal.stop.timeout=20 # Per containers override for stop timeout seconds during restart ``` ## Testing diff --git a/build.sh b/build.sh new file mode 100644 index 0000000..ea15696 --- /dev/null +++ b/build.sh @@ -0,0 +1,36 @@ +#!/bin/sh + +REPO=${1} +DOCKER_USERNAME=${2} +DOCKER_PASSWORD=${3} +DOCKER_REGESTRY=${4} +declare -A ARCH_MAP +ARCH_MAP[x86_64]=x86_64 +ARCH_MAP[armhf]=arm +ARCH_MAP[arm64]=arm64 + +docker run --rm --privileged multiarch/qemu-user-static:register --reset +echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin + +echo "Building docker images" +for i in "${!ARCH_MAP[@]}"; do + echo "################################################################################" + echo "Building and pushing: ${REPO}-${ARCH_MAP[$i]}:latest " + echo "################################################################################" + docker build --build-arg arch=${i} -t ${REPO}-${ARCH_MAP[$i]}:latest .; + docker push ${REPO}-${ARCH_MAP[$i]}:latest; +done; + +MANIFESTS="" +for arch in "${ARCH_MAP[@]}"; do MANIFESTS="${MANIFESTS} ${REPO}-${arch}:latest"; done + +echo "################################################################################" +echo "Building and pushing: ${REPO}-${ARCH_MAP[$i]}:latest " +echo "################################################################################" +docker manifest create ${REPO}:latest ${MANIFESTS}; + +for arch in "${ARCH_MAP[@]}"; do + docker manifest annotate --os linux --arch ${arch} ${REPO}:latest ${REPO}-${arch}:latest; +done; + +docker manifest push ${REPO}:latest diff --git a/docker-entrypoint b/docker-entrypoint index 497a823..a1bf8ab 100755 --- a/docker-entrypoint +++ b/docker-entrypoint @@ -1,50 +1,54 @@ #!/usr/bin/env sh + set -e +set -o pipefail DOCKER_SOCK=${DOCKER_SOCK:-/var/run/docker.sock} -TMP_DIR=/tmp/restart +CURL_TIMEOUT=${CURL_TIMEOUT:-30} # SIGTERM-handler term_handler() { exit 143; # 128 + 15 -- SIGTERM } +docker_curl() { + curl --max-time "${CURL_TIMEOUT}" --no-buffer -s --unix-socket "${DOCKER_SOCK}" "$@" +} + trap 'kill ${!}; term_handler' SIGTERM if [ "$1" = 'autoheal' ] && [ -e ${DOCKER_SOCK} ]; then - mkdir -p $TMP_DIR - # https://docs.docker.com/engine/api/v1.25/ # Set container selector if [ "$AUTOHEAL_CONTAINER_LABEL" == "all" ]; then - selector() { - jq -r .[].Id - } + labelFilter="" else - selector() { - jq -r '.[] | select(.Labels["'${AUTOHEAL_CONTAINER_LABEL:=autoheal}'"] == "true") | .Id' - } + labelFilter=",\"label\":\[\"${AUTOHEAL_CONTAINER_LABEL:=autoheal}=true\"\]" fi - echo "Monitoring containers for unhealthy status" + AUTOHEAL_START_PERIOD=${AUTOHEAL_START_PERIOD:=0} + echo "Monitoring containers for unhealthy status in ${AUTOHEAL_START_PERIOD} second(s)" + sleep ${AUTOHEAL_START_PERIOD} + while true; do sleep ${AUTOHEAL_INTERVAL:=5} - - CONTAINERS=$(curl --no-buffer -s -XGET --unix-socket ${DOCKER_SOCK} http://localhost/containers/json | selector) - for CONTAINER in $CONTAINERS; do - HEALTH=$(curl --no-buffer -s -XGET --unix-socket ${DOCKER_SOCK} http://localhost/containers/${CONTAINER}/json | jq -r .State.Health.Status) - if [ "unhealthy" = "$HEALTH" ]; then + + apiUrl="http://localhost/containers/json?filters=\{\"health\":\[\"unhealthy\"\]${labelFilter}\}" + stopTimeout=".Labels[\"autoheal.stop.timeout\"] // ${AUTOHEAL_DEFAULT_STOP_TIMEOUT:-10}" + docker_curl "$apiUrl" | \ + jq -r "foreach .[] as \$CONTAINER([];[]; \$CONTAINER | .Id, .Names[0], $stopTimeout )" | \ + while read -r CONTAINER_ID && read -r CONTAINER_NAME && read -r TIMEOUT; do + CONTAINER_SHORT_ID=${CONTAINER_ID:0:12} DATE=$(date +%d-%m-%Y" "%H:%M:%S) - echo "$DATE Container ${CONTAINER:0:12} found to be unhealthy" - touch "$TMP_DIR/$CONTAINER" - fi - done - for CONTAINER in `ls $TMP_DIR`; do - DATE=$(date +%d-%m-%Y" "%H:%M:%S) - echo "$DATE Restarting container ${CONTAINER:0:12}" - curl -f --no-buffer -s -XPOST --unix-socket ${DOCKER_SOCK} http://localhost/containers/${CONTAINER}/restart && rm "$TMP_DIR/$CONTAINER" || echo "$DATE Restarting container ${CONTAINER:0:12} failed" + if [ "null" = "$CONTAINER_NAME" ]; then + echo "$DATE Container name of ($CONTAINER_SHORT_ID) is null, which implies container does not exist - don't restart" + else + echo "$DATE Container ${CONTAINER_NAME} ($CONTAINER_SHORT_ID) found to be unhealthy - Restarting container now with ${TIMEOUT}s timeout" + docker_curl -f -XPOST "http://localhost/containers/${CONTAINER_ID}/restart?t=${TIMEOUT}" \ + || echo "$DATE Restarting container $CONTAINER_SHORT_ID failed" + fi done done