feat(chromium): add health check (closes #633)

Makefile

@@ -33,6 +33,7 @@ API_TIMEOUT=30s
 API_ROOT_PATH=/
 API_TRACE_HEADER=Gotenberg-Trace
 API_DISABLE_HEALTH_CHECK_LOGGING=false
+CHROMIUM_FAILED_STARTS_THRESHOLD=5
 CHROMIUM_INCOGNITO=false
 CHROMIUM_ALLOW_INSECURE_LOCALHOST=false
 CHROMIUM_IGNORE_CERTIFICATE_ERRORS=false
@@ -78,6 +79,7 @@ run: ## Start a Gotenberg container
 	--api-root-path=$(API_ROOT_PATH) \
 	--api-trace-header=$(API_TRACE_HEADER) \
 	--api-disable-health-check-logging=$(API_DISABLE_HEALTH_CHECK_LOGGING) \
+	--chromium-failed-starts-threshold=$(CHROMIUM_FAILED_STARTS_THRESHOLD) \
 	--chromium-incognito=$(CHROMIUM_INCOGNITO) \
 	--chromium-allow-insecure-localhost=$(CHROMIUM_ALLOW_INSECURE_LOCALHOST) \
 	--chromium-ignore-certificate-errors=$(CHROMIUM_IGNORE_CERTIFICATE_ERRORS) \

pkg/modules/chromium/chromium.go

@@ -11,6 +11,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/alexliesenfeld/health"
 	"github.com/chromedp/cdproto/cdp"
 	"github.com/chromedp/cdproto/emulation"
 	"github.com/chromedp/cdproto/fetch"
@@ -69,6 +70,7 @@ var (
 type Chromium struct {
 	binPath                  string
 	engine                   gotenberg.PDFEngine
+	failedStartsThreshold    int
 	userAgent                string
 	incognito                bool
 	allowInsecureLocalhost   bool
@@ -261,6 +263,7 @@ func (mod Chromium) Descriptor() gotenberg.ModuleDescriptor {
 		ID: "chromium",
 		FlagSet: func() *flag.FlagSet {
 			fs := flag.NewFlagSet("chromium", flag.ExitOnError)
+			fs.Int("chromium-failed-starts-threshold", 5, "Set the number of consecutive failed starts after which the module is considered unhealthy - 0 means ignore")
 			fs.String("chromium-user-agent", "", "Override the default User-Agent header")
 			fs.Bool("chromium-incognito", false, "Start Chromium with incognito mode")
 			fs.Bool("chromium-allow-insecure-localhost", false, "Ignore TLS/SSL errors on localhost")
@@ -288,6 +291,7 @@ func (mod Chromium) Descriptor() gotenberg.ModuleDescriptor {
 // Provision sets the module properties.
 func (mod *Chromium) Provision(ctx *gotenberg.Context) error {
 	flags := ctx.ParsedFlags()
+	mod.failedStartsThreshold = flags.MustInt("chromium-failed-starts-threshold")
 	mod.userAgent = flags.MustString("chromium-user-agent")
 	mod.allowInsecureLocalhost = flags.MustBool("chromium-allow-insecure-localhost")
 	mod.ignoreCertificateErrors = flags.MustBool("chromium-ignore-certificate-errors")
@@ -345,6 +349,41 @@ func (mod Chromium) Metrics() ([]gotenberg.Metric, error) {
 				return activeInstancesCount
 			},
 		},
+		{
+			Name:        "chromium_failed_starts_count",
+			Description: "Current number of Chromium consecutive starting failures.",
+			Read: func() float64 {
+				failedStartsCountMu.RLock()
+				defer failedStartsCountMu.RUnlock()
+
+				return failedStartsCount
+			},
+		},
+	}, nil
+}
+
+// Checks adds a health check that verifies if Chromium consecutive failed
+// starts threshold has been reached or not.
+// See https://github.com/gotenberg/gotenberg/issues/633.
+func (mod Chromium) Checks() ([]health.CheckerOption, error) {
+	if mod.failedStartsThreshold == 0 {
+		return nil, nil
+	}
+
+	return []health.CheckerOption{
+		health.WithCheck(health.Check{
+			Name: "chromium",
+			Check: func(_ context.Context) error {
+				failedStartsCountMu.RLock()
+				defer failedStartsCountMu.RUnlock()
+
+				if int(failedStartsCount) < mod.failedStartsThreshold {
+					return nil
+				}
+
+				return errors.New("failed starts threshold reached")
+			},
+		}),
 	}, nil
 }
 
@@ -906,9 +945,21 @@ func (mod Chromium) PDF(ctx context.Context, logger *zap.Logger, URL, outputPath
 			return ErrRpccMessageTooLarge
 		}
 
+		// See https://github.com/gotenberg/gotenberg/issues/633.
+		if strings.Contains(errMessage, "chrome failed to start") {
+			failedStartsCountMu.Lock()
+			failedStartsCount += 1
+			failedStartsCountMu.Unlock()
+		}
+
 		return fmt.Errorf("chromium PDF: %w", err)
 	}
 
+	// See https://github.com/gotenberg/gotenberg/issues/633.
+	failedStartsCountMu.Lock()
+	failedStartsCount = 0
+	failedStartsCountMu.Unlock()
+
 	// See https://github.com/gotenberg/gotenberg/issues/262.
 	consoleExceptionsMu.RLock()
 	defer consoleExceptionsMu.RUnlock()
@@ -923,6 +974,8 @@ func (mod Chromium) PDF(ctx context.Context, logger *zap.Logger, URL, outputPath
 var (
 	activeInstancesCount   float64
 	activeInstancesCountMu sync.RWMutex
+	failedStartsCount      float64
+	failedStartsCountMu    sync.RWMutex
 )
 
 // Interface guards.
@@ -931,6 +984,7 @@ var (
 	_ gotenberg.Provisioner     = (*Chromium)(nil)
 	_ gotenberg.Validator       = (*Chromium)(nil)
 	_ gotenberg.MetricsProvider = (*Chromium)(nil)
+	_ api.HealthChecker         = (*Chromium)(nil)
 	_ api.Router                = (*Chromium)(nil)
 	_ API                       = (*Chromium)(nil)
 	_ Provider                  = (*Chromium)(nil)

pkg/modules/chromium/chromium_test.go

@@ -9,6 +9,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/alexliesenfeld/health"
 	"github.com/gotenberg/gotenberg/v7/pkg/gotenberg"
 	"go.uber.org/zap"
 )
@@ -178,14 +179,88 @@ func TestChromium_Metrics(t *testing.T) {
 		t.Fatalf("expected no error but got: %v", err)
 	}
 
-	if len(metrics) != 1 {
-		t.Fatalf("expected %d metrics, but got %d", 1, len(metrics))
+	if len(metrics) != 2 {
+		t.Fatalf("expected %d metrics, but got %d", 2, len(metrics))
 	}
 
 	actual := metrics[0].Read()
 	if actual != 0 {
 		t.Errorf("expected %d Chromium instances, but got %f", 0, actual)
 	}
+
+	actual = metrics[1].Read()
+	if actual != 0 {
+		t.Errorf("expected %d Chromium failed starts, but got %f", 0, actual)
+	}
+}
+
+func TestChromium_Checks(t *testing.T) {
+	tests := []struct {
+		name                     string
+		mod                      Chromium
+		tearUp                   func()
+		tearDown                 func()
+		expectAvailabilityStatus health.AvailabilityStatus
+	}{
+		{
+			name: "ignore Chromium failed starts",
+			mod: Chromium{
+				failedStartsThreshold: 0,
+			},
+		},
+		{
+			name: "with Chromium failed starts threshold not reached",
+			mod: Chromium{
+				failedStartsThreshold: 1,
+			},
+			expectAvailabilityStatus: health.StatusUp,
+		},
+		{
+			name: "with Chromium failed starts threshold reached",
+			mod: Chromium{
+				failedStartsThreshold: 1,
+			},
+			tearUp: func() {
+				failedStartsCount = 1
+			},
+			tearDown: func() {
+				failedStartsCount = 0
+			},
+			expectAvailabilityStatus: health.StatusDown,
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			if tc.tearUp != nil {
+				tc.tearUp()
+			}
+
+			checks, err := tc.mod.Checks()
+			if err != nil {
+				t.Fatalf("expected no error from mod.Checks(), but got: %v", err)
+			}
+
+			if len(checks) == 0 {
+				return
+			}
+
+			if len(checks) != 1 {
+				t.Fatalf("expected 1 check from mod.Checks(), but got %d", len(checks))
+			}
+
+			checker := health.NewChecker(checks...)
+			result := checker.Check(context.Background())
+
+			if result.Status != tc.expectAvailabilityStatus {
+				t.Errorf("expected '%s' as availability status, but got '%s'", tc.expectAvailabilityStatus, result.Status)
+			}
+
+			if tc.tearDown != nil {
+				tc.tearDown()
+			}
+		})
+	}
 }
 
 func TestChromium_Chromium(t *testing.T) {