From 84334ace6a228771d98b6b114973971f4f02b31e Mon Sep 17 00:00:00 2001 From: Julien Neuhart Date: Mon, 24 Apr 2023 11:25:20 +0200 Subject: [PATCH] feat(chromium): add health check (closes #633) --- Makefile | 2 + pkg/modules/chromium/chromium.go | 54 ++++++++++++++++++ pkg/modules/chromium/chromium_test.go | 79 ++++++++++++++++++++++++++- 3 files changed, 133 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index d9dc6fb1..ae79409f 100644 --- a/Makefile +++ b/Makefile @@ -33,6 +33,7 @@ API_TIMEOUT=30s API_ROOT_PATH=/ API_TRACE_HEADER=Gotenberg-Trace API_DISABLE_HEALTH_CHECK_LOGGING=false +CHROMIUM_FAILED_STARTS_THRESHOLD=5 CHROMIUM_INCOGNITO=false CHROMIUM_ALLOW_INSECURE_LOCALHOST=false CHROMIUM_IGNORE_CERTIFICATE_ERRORS=false @@ -78,6 +79,7 @@ run: ## Start a Gotenberg container --api-root-path=$(API_ROOT_PATH) \ --api-trace-header=$(API_TRACE_HEADER) \ --api-disable-health-check-logging=$(API_DISABLE_HEALTH_CHECK_LOGGING) \ + --chromium-failed-starts-threshold=$(CHROMIUM_FAILED_STARTS_THRESHOLD) \ --chromium-incognito=$(CHROMIUM_INCOGNITO) \ --chromium-allow-insecure-localhost=$(CHROMIUM_ALLOW_INSECURE_LOCALHOST) \ --chromium-ignore-certificate-errors=$(CHROMIUM_IGNORE_CERTIFICATE_ERRORS) \ diff --git a/pkg/modules/chromium/chromium.go b/pkg/modules/chromium/chromium.go index a9c9c693..1258a468 100644 --- a/pkg/modules/chromium/chromium.go +++ b/pkg/modules/chromium/chromium.go @@ -11,6 +11,7 @@ import ( "sync" "time" + "github.com/alexliesenfeld/health" "github.com/chromedp/cdproto/cdp" "github.com/chromedp/cdproto/emulation" "github.com/chromedp/cdproto/fetch" @@ -69,6 +70,7 @@ var ( type Chromium struct { binPath string engine gotenberg.PDFEngine + failedStartsThreshold int userAgent string incognito bool allowInsecureLocalhost bool @@ -261,6 +263,7 @@ func (mod Chromium) Descriptor() gotenberg.ModuleDescriptor { ID: "chromium", FlagSet: func() *flag.FlagSet { fs := flag.NewFlagSet("chromium", flag.ExitOnError) + fs.Int("chromium-failed-starts-threshold", 5, "Set the number of consecutive failed starts after which the module is considered unhealthy - 0 means ignore") fs.String("chromium-user-agent", "", "Override the default User-Agent header") fs.Bool("chromium-incognito", false, "Start Chromium with incognito mode") fs.Bool("chromium-allow-insecure-localhost", false, "Ignore TLS/SSL errors on localhost") @@ -288,6 +291,7 @@ func (mod Chromium) Descriptor() gotenberg.ModuleDescriptor { // Provision sets the module properties. func (mod *Chromium) Provision(ctx *gotenberg.Context) error { flags := ctx.ParsedFlags() + mod.failedStartsThreshold = flags.MustInt("chromium-failed-starts-threshold") mod.userAgent = flags.MustString("chromium-user-agent") mod.allowInsecureLocalhost = flags.MustBool("chromium-allow-insecure-localhost") mod.ignoreCertificateErrors = flags.MustBool("chromium-ignore-certificate-errors") @@ -345,6 +349,41 @@ func (mod Chromium) Metrics() ([]gotenberg.Metric, error) { return activeInstancesCount }, }, + { + Name: "chromium_failed_starts_count", + Description: "Current number of Chromium consecutive starting failures.", + Read: func() float64 { + failedStartsCountMu.RLock() + defer failedStartsCountMu.RUnlock() + + return failedStartsCount + }, + }, + }, nil +} + +// Checks adds a health check that verifies if Chromium consecutive failed +// starts threshold has been reached or not. +// See https://github.com/gotenberg/gotenberg/issues/633. +func (mod Chromium) Checks() ([]health.CheckerOption, error) { + if mod.failedStartsThreshold == 0 { + return nil, nil + } + + return []health.CheckerOption{ + health.WithCheck(health.Check{ + Name: "chromium", + Check: func(_ context.Context) error { + failedStartsCountMu.RLock() + defer failedStartsCountMu.RUnlock() + + if int(failedStartsCount) < mod.failedStartsThreshold { + return nil + } + + return errors.New("failed starts threshold reached") + }, + }), }, nil } @@ -906,9 +945,21 @@ func (mod Chromium) PDF(ctx context.Context, logger *zap.Logger, URL, outputPath return ErrRpccMessageTooLarge } + // See https://github.com/gotenberg/gotenberg/issues/633. + if strings.Contains(errMessage, "chrome failed to start") { + failedStartsCountMu.Lock() + failedStartsCount += 1 + failedStartsCountMu.Unlock() + } + return fmt.Errorf("chromium PDF: %w", err) } + // See https://github.com/gotenberg/gotenberg/issues/633. + failedStartsCountMu.Lock() + failedStartsCount = 0 + failedStartsCountMu.Unlock() + // See https://github.com/gotenberg/gotenberg/issues/262. consoleExceptionsMu.RLock() defer consoleExceptionsMu.RUnlock() @@ -923,6 +974,8 @@ func (mod Chromium) PDF(ctx context.Context, logger *zap.Logger, URL, outputPath var ( activeInstancesCount float64 activeInstancesCountMu sync.RWMutex + failedStartsCount float64 + failedStartsCountMu sync.RWMutex ) // Interface guards. @@ -931,6 +984,7 @@ var ( _ gotenberg.Provisioner = (*Chromium)(nil) _ gotenberg.Validator = (*Chromium)(nil) _ gotenberg.MetricsProvider = (*Chromium)(nil) + _ api.HealthChecker = (*Chromium)(nil) _ api.Router = (*Chromium)(nil) _ API = (*Chromium)(nil) _ Provider = (*Chromium)(nil) diff --git a/pkg/modules/chromium/chromium_test.go b/pkg/modules/chromium/chromium_test.go index 60c38361..a5190e1c 100644 --- a/pkg/modules/chromium/chromium_test.go +++ b/pkg/modules/chromium/chromium_test.go @@ -9,6 +9,7 @@ import ( "testing" "time" + "github.com/alexliesenfeld/health" "github.com/gotenberg/gotenberg/v7/pkg/gotenberg" "go.uber.org/zap" ) @@ -178,14 +179,88 @@ func TestChromium_Metrics(t *testing.T) { t.Fatalf("expected no error but got: %v", err) } - if len(metrics) != 1 { - t.Fatalf("expected %d metrics, but got %d", 1, len(metrics)) + if len(metrics) != 2 { + t.Fatalf("expected %d metrics, but got %d", 2, len(metrics)) } actual := metrics[0].Read() if actual != 0 { t.Errorf("expected %d Chromium instances, but got %f", 0, actual) } + + actual = metrics[1].Read() + if actual != 0 { + t.Errorf("expected %d Chromium failed starts, but got %f", 0, actual) + } +} + +func TestChromium_Checks(t *testing.T) { + tests := []struct { + name string + mod Chromium + tearUp func() + tearDown func() + expectAvailabilityStatus health.AvailabilityStatus + }{ + { + name: "ignore Chromium failed starts", + mod: Chromium{ + failedStartsThreshold: 0, + }, + }, + { + name: "with Chromium failed starts threshold not reached", + mod: Chromium{ + failedStartsThreshold: 1, + }, + expectAvailabilityStatus: health.StatusUp, + }, + { + name: "with Chromium failed starts threshold reached", + mod: Chromium{ + failedStartsThreshold: 1, + }, + tearUp: func() { + failedStartsCount = 1 + }, + tearDown: func() { + failedStartsCount = 0 + }, + expectAvailabilityStatus: health.StatusDown, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if tc.tearUp != nil { + tc.tearUp() + } + + checks, err := tc.mod.Checks() + if err != nil { + t.Fatalf("expected no error from mod.Checks(), but got: %v", err) + } + + if len(checks) == 0 { + return + } + + if len(checks) != 1 { + t.Fatalf("expected 1 check from mod.Checks(), but got %d", len(checks)) + } + + checker := health.NewChecker(checks...) + result := checker.Check(context.Background()) + + if result.Status != tc.expectAvailabilityStatus { + t.Errorf("expected '%s' as availability status, but got '%s'", tc.expectAvailabilityStatus, result.Status) + } + + if tc.tearDown != nil { + tc.tearDown() + } + }) + } } func TestChromium_Chromium(t *testing.T) {