Add internal health endpoint and health check command

.golangci.yaml

@@ -309,6 +309,8 @@ linters:
           disabled: true
         - name: flag-parameter
           disabled: true
+        - name: deep-exit
+          disabled: true
 
     gosec:
       excludes:

Dockerfile

@@ -28,5 +28,10 @@ COPY --link --from=golang /app/nebula-sync /usr/local/bin/
 
 USER 1001
 
+ENV API_ENABLED=true
+
+HEALTHCHECK --interval=30s --timeout=3s --start-period=3s --retries=3 \
+  CMD ["nebula-sync", "healthcheck"]
+
 ENTRYPOINT ["nebula-sync"]
 CMD ["run"]

cmd/health.go

@@ -0,0 +1,25 @@
+package cmd
+
+import (
+	"os"
+
+	"github.com/spf13/cobra"
+
+	"github.com/lovelaze/nebula-sync/internal/health"
+)
+
+const healthURL = "http://127.0.0.1:8080/health"
+
+var healthCmd = &cobra.Command{
+	Use: "healthcheck",
+	Run: func(cmd *cobra.Command, args []string) {
+		if err := health.Check(healthURL); err != nil {
+			os.Exit(1)
+		}
+	},
+}
+
+func init() {
+	healthCmd.Hidden = true
+	rootCmd.AddCommand(healthCmd)
+}

go.mod

@@ -29,6 +29,7 @@ require (
 	github.com/docker/go-units v0.5.0 // indirect
 	github.com/ebitengine/purego v0.8.2 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/go-chi/chi/v5 v5.2.1 // indirect
 	github.com/go-logr/logr v1.4.2 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-ole/go-ole v1.3.0 // indirect

go.sum

@@ -35,6 +35,8 @@ github.com/ebitengine/purego v0.8.2 h1:jPPGWs2sZ1UgOSgD2bClL0MJIqu58nOmIcBuXr62z
 github.com/ebitengine/purego v0.8.2/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
 github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
+github.com/go-chi/chi/v5 v5.2.1 h1:KOIHODQj58PmL80G2Eak4WdvUzjSJSm0vG72crDCqb8=
+github.com/go-chi/chi/v5 v5.2.1/go.mod h1:L2yAIGWB3H+phAw1NxKwWM+7eUH/lU8pOMm5hHcoops=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
 github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=

internal/api/handler.go

@@ -0,0 +1,17 @@
+package api
+
+import (
+	"net/http"
+)
+
+func (s *Server) healthHandler(w http.ResponseWriter, r *http.Request) {
+	if s.healthy() {
+		w.WriteHeader(http.StatusOK)
+	} else {
+		w.WriteHeader(http.StatusInternalServerError)
+	}
+}
+
+func (s *Server) healthy() bool {
+	return len(s.state.Stack) > 0 && s.state.Stack[0].Success
+}

internal/api/handler_test.go

@@ -0,0 +1,53 @@
+package api
+
+import (
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/lovelaze/nebula-sync/internal/sync"
+)
+
+func TestHealthHandler_healthy(t *testing.T) {
+	state := sync.NewState()
+	state.OnSuccess()
+
+	require.Len(t, state.Stack, 1)
+	require.True(t, state.Stack[0].Success)
+
+	server := NewServer(state)
+
+	req := httptest.NewRequest(http.MethodGet, "/health", nil)
+	resp := httptest.NewRecorder()
+
+	server.router.ServeHTTP(resp, req)
+
+	result := resp.Result()
+	defer result.Body.Close()
+
+	assert.Equal(t, 200, result.StatusCode)
+}
+
+func TestHealthHandler_unhealthy(t *testing.T) {
+	state := sync.NewState()
+	state.OnFailure(errors.New(("test error")))
+
+	require.Len(t, state.Stack, 1)
+	require.False(t, state.Stack[0].Success)
+
+	server := NewServer(state)
+
+	req := httptest.NewRequest(http.MethodGet, "/health", nil)
+	resp := httptest.NewRecorder()
+
+	server.router.ServeHTTP(resp, req)
+
+	result := resp.Result()
+	defer result.Body.Close()
+
+	assert.Equal(t, 500, result.StatusCode)
+}

internal/api/server.go

@@ -0,0 +1,50 @@
+package api
+
+import (
+	"fmt"
+	"net/http"
+	"time"
+
+	"github.com/go-chi/chi/v5"
+	"github.com/rs/zerolog/log"
+
+	"github.com/lovelaze/nebula-sync/internal/sync"
+)
+
+const (
+	port              = 8080
+	readHeaderTimeout = 10 * time.Second
+)
+
+type Server struct {
+	state  *sync.State
+	router *chi.Mux
+}
+
+func NewServer(state *sync.State) *Server {
+	router := chi.NewRouter()
+	server := &Server{
+		state:  state,
+		router: router,
+	}
+
+	router.Get("/health", server.healthHandler)
+
+	return server
+}
+
+func (s *Server) Start() {
+	go func() {
+		log.Debug().Msg("Starting http server")
+
+		server := &http.Server{
+			Handler:           s.router,
+			Addr:              fmt.Sprintf(":%d", port),
+			ReadHeaderTimeout: readHeaderTimeout,
+		}
+
+		if err := server.ListenAndServe(); err != nil {
+			log.Fatal().Err(err).Msg("Failed to start http server")
+		}
+	}()
+}

internal/config/api.go

@@ -0,0 +1,5 @@
+package config
+
+type API struct {
+	Enabled bool `default:"false" envconfig:"ENABLED"` // internal use only
+}

internal/config/config.go

@@ -10,10 +10,11 @@ import (
 )
 
 type Config struct {
-	Primary  model.PiHole   `required:"true" envconfig:"PRIMARY"`
-	Replicas []model.PiHole `required:"true" envconfig:"REPLICAS"`
-	Client   *Client        `                                     ignored:"true"`
-	Sync     *Sync          `                                     ignored:"true"`
+	Primary  model.PiHole   `ignored:"true" required:"true" envconfig:"PRIMARY"`
+	Replicas []model.PiHole `ignored:"true" required:"true" envconfig:"REPLICAS"`
+	Client   *Client        `ignored:"true"`
+	Sync     *Sync          `ignored:"true"`
+	API      *API           `                               envconfig:"API"`
 }
 
 type Sync struct {
@@ -158,6 +159,10 @@ func NewConfigSetting(enabled bool, included, excluded []string) *ConfigSetting
 }
 
 func (c *Config) Load() error {
+	if err := envconfig.Process("", c); err != nil {
+		return err
+	}
+
 	if err := c.loadTargets(); err != nil {
 		return err
 	}
@@ -211,6 +216,10 @@ func (s *Sync) String() string {
 	return fmt.Sprintf("%+v", *s)
 }
 
+func (a *API) String() string {
+	return fmt.Sprintf("%+v", *a)
+}
+
 func (gs *GravitySettings) String() string {
 	return fmt.Sprintf("%+v", *gs)
 }

internal/health/health.go

@@ -0,0 +1,26 @@
+package health
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+)
+
+func Check(url string) error {
+	req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, url, nil)
+	if err != nil {
+		return fmt.Errorf("health check failed: %w", err)
+	}
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return fmt.Errorf("health check failed: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("health check failed: %s", resp.Status)
+	}
+
+	return nil
+}

internal/health/health_test.go

@@ -0,0 +1,29 @@
+package health
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestHealth_CheckSuccess(t *testing.T) {
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer ts.Close()
+
+	err := Check(ts.URL)
+	require.NoError(t, err)
+}
+
+func TestHealth_CheckFailure(t *testing.T) {
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusInternalServerError)
+	}))
+	defer ts.Close()
+
+	err := Check(ts.URL)
+	require.Error(t, err)
+}

internal/service/service.go

@@ -6,6 +6,7 @@ import (
 	"github.com/robfig/cron/v3"
 	"github.com/rs/zerolog/log"
 
+	"github.com/lovelaze/nebula-sync/internal/api"
 	"github.com/lovelaze/nebula-sync/internal/config"
 	"github.com/lovelaze/nebula-sync/internal/pihole"
 	"github.com/lovelaze/nebula-sync/internal/sync"
@@ -18,13 +19,18 @@ type Service struct {
 	target    sync.Target
 	conf      config.Config
 	callbacks []sync.Callback
+	State     *sync.State
 }
 
 func NewService(target sync.Target, conf config.Config, callbacks ...sync.Callback) *Service {
+	state := sync.NewState()
+	cbs := append([]sync.Callback{state}, callbacks...)
+
 	return &Service{
 		target:    target,
 		conf:      conf,
-		callbacks: callbacks,
+		callbacks: cbs,
+		State:     state,
 	}
 }
 
@@ -47,6 +53,12 @@ func Init() (*Service, error) {
 
 	target := sync.NewTarget(primary, replicas)
 	service := NewService(target, conf, webhookClient)
+
+	if conf.API.Enabled && conf.Sync.Cron != nil {
+		server := api.NewServer(service.State)
+		server.Start()
+	}
+
 	return service, nil
 }
 
@@ -54,13 +66,13 @@ func (service *Service) Run() error {
 	log.Info().Msgf("Starting nebula-sync %s", version.Version)
 	log.Debug().Str("config", service.conf.String()).Msgf("Settings")
 
-	if err := service.doSync(service.target); err != nil {
+	if err := service.sync(service.target); err != nil {
 		return err
 	}
 
 	if service.conf.Sync.Cron != nil {
 		return service.startCron(func() {
-			if err := service.doSync(service.target); err != nil {
+			if err := service.sync(service.target); err != nil {
 				log.Error().Err(err).Msg("Sync failed")
 			}
 		})
@@ -69,29 +81,31 @@ func (service *Service) Run() error {
 	return nil
 }
 
-func (service *Service) doSync(t sync.Target) error {
-	err := service.selectSyncMethod(t)
-
-	if err != nil {
-		for _, callback := range service.callbacks {
-			callback.OnFailure(err)
-		}
+func (service *Service) sync(t sync.Target) error {
+	var err error
+	if service.conf.Sync.FullSync {
+		err = t.FullSync(service.conf.Sync)
 	} else {
-		for _, callback := range service.callbacks {
-			callback.OnSuccess()
-		}
+		err = t.SelectiveSync(service.conf.Sync)
+	}
+
+	service.runCallbacks(err)
+
+	if err == nil {
 		log.Info().Msg("Sync completed")
 	}
 
 	return err
 }
 
-func (service *Service) selectSyncMethod(t sync.Target) error {
-	if service.conf.Sync.FullSync {
-		return t.FullSync(service.conf.Sync)
+func (service *Service) runCallbacks(syncError error) {
+	for _, callback := range service.callbacks {
+		if syncError != nil {
+			callback.OnFailure(syncError)
+		} else {
+			callback.OnSuccess()
+		}
 	}
-
-	return t.SelectiveSync(service.conf.Sync)
 }
 
 func (service *Service) startCron(cmd func()) error {

internal/sync/state.go

@@ -0,0 +1,43 @@
+package sync
+
+import "time"
+
+const stackSize = 5
+
+type State struct {
+	Stack []Outcome
+}
+
+func NewState() *State {
+	return &State{
+		Stack: []Outcome{},
+	}
+}
+
+type Outcome struct {
+	Timestamp time.Time
+	Success   bool
+}
+
+func NewOutcome(success bool) *Outcome {
+	return &Outcome{
+		Timestamp: time.Now(),
+		Success:   success,
+	}
+}
+
+func (s *State) Add(outcome Outcome) {
+	s.Stack = append([]Outcome{outcome}, s.Stack...)
+
+	if len(s.Stack) > stackSize {
+		s.Stack = s.Stack[:stackSize]
+	}
+}
+
+func (s *State) OnSuccess() {
+	s.Add(*NewOutcome(true))
+}
+
+func (s *State) OnFailure(err error) {
+	s.Add(*NewOutcome(false))
+}

internal/sync/state_test.go

@@ -0,0 +1,45 @@
+package sync
+
+import (
+	"errors"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestState_Add(t *testing.T) {
+	s := NewState()
+	require.Empty(t, s.Stack)
+
+	outcome := *NewOutcome(true)
+	s.Add(outcome)
+
+	assert.Len(t, s.Stack, 1)
+	assert.Contains(t, s.Stack, outcome)
+}
+
+func TestState_OnSuccess(t *testing.T) {
+	s := NewState()
+	require.Empty(t, s.Stack)
+
+	s.OnSuccess()
+
+	assert.Len(t, s.Stack, 1)
+	assert.True(t, s.Stack[0].Success)
+	now := time.Now()
+	assert.WithinRange(t, s.Stack[0].Timestamp, now.Add(-1*time.Second), now.Add(1*time.Second))
+}
+
+func TestState_OnFailure(t *testing.T) {
+	s := NewState()
+	require.Empty(t, s.Stack)
+
+	s.OnFailure(errors.New("test error"))
+
+	assert.Len(t, s.Stack, 1)
+	assert.False(t, s.Stack[0].Success)
+	now := time.Now()
+	assert.WithinRange(t, s.Stack[0].Timestamp, now.Add(-1*time.Second), now.Add(1*time.Second))
+}