mirror of
https://github.com/gotenberg/gotenberg.git
synced 2026-10-07 21:13:18 +01:00
fix(chromium): fail fast with 503 when Chromium crashes (#1641)
This commit is contained in:
@@ -523,8 +523,45 @@ func (b *chromiumBrowser) do(ctx context.Context, logger *slog.Logger, url strin
|
||||
cancelOnMainPageError: taskCancel,
|
||||
})
|
||||
|
||||
var (
|
||||
crashed error
|
||||
crashedMu sync.RWMutex
|
||||
)
|
||||
|
||||
// See https://github.com/gotenberg/gotenberg/issues/1640.
|
||||
listenForEventTargetCrashed(taskCtx, logger, eventTargetCrashedOptions{
|
||||
crashed: &crashed,
|
||||
crashedMu: &crashedMu,
|
||||
cancel: taskCancel,
|
||||
})
|
||||
|
||||
runErr := chromedp.Run(taskCtx, tasks...)
|
||||
|
||||
// A crashed renderer is the root cause of every other failure this
|
||||
// conversion may have recorded, so check it first.
|
||||
// See https://github.com/gotenberg/gotenberg/issues/1640.
|
||||
crashedMu.RLock()
|
||||
defer crashedMu.RUnlock()
|
||||
|
||||
if crashed != nil {
|
||||
return fmt.Errorf("handle tasks: %w", crashed)
|
||||
}
|
||||
|
||||
// The browser context is only ever canceled when the browser process
|
||||
// dies or is stopped, never on a request timeout. If the run failed
|
||||
// and the browser context is done, the conversion failed because the
|
||||
// browser went away mid-flight; fail fast with the same crash error
|
||||
// instead of letting the error fall through as a generic context
|
||||
// cancellation. The check is gated on runErr so a successful
|
||||
// conversion is never discarded by a browser death that lands right
|
||||
// after it.
|
||||
// See https://github.com/gotenberg/gotenberg/issues/1640.
|
||||
if runErr != nil {
|
||||
if err := b.ctx.Err(); err != nil {
|
||||
return fmt.Errorf("handle tasks: %w", ErrChromiumCrashed)
|
||||
}
|
||||
}
|
||||
|
||||
// Check event-driven errors first — they take priority over chromedp.Run
|
||||
// errors because they carry the actual root cause (e.g., HTTP 500 from
|
||||
// the main page). When we cancel taskCtx on a main page error,
|
||||
|
||||
@@ -71,6 +71,10 @@ var (
|
||||
// ErrResourceLoadingFailed happens when one or more resources failed to load.
|
||||
ErrResourceLoadingFailed = errors.New("resource loading failed")
|
||||
|
||||
// ErrChromiumCrashed happens when the Chromium renderer crashes during a
|
||||
// conversion.
|
||||
ErrChromiumCrashed = errors.New("chromium crashed")
|
||||
|
||||
// PDF specific.
|
||||
|
||||
// ErrOmitBackgroundWithoutPrintBackground happens if
|
||||
@@ -1106,6 +1110,8 @@ func chromiumErrorType(err error, queueReason string) string {
|
||||
errors.Is(err, ErrInvalidEvaluationExpression),
|
||||
errors.Is(err, ErrInvalidSelectorQuery):
|
||||
return gotenberg.ErrorTypeInvalidInput
|
||||
case errors.Is(err, ErrChromiumCrashed):
|
||||
return "chromium_unavailable"
|
||||
case errors.Is(err, gotenberg.ErrMaximumQueueSizeExceeded):
|
||||
return queueReason
|
||||
case errors.Is(err, gotenberg.ErrProcessAlreadyRestarting):
|
||||
|
||||
@@ -21,6 +21,7 @@ func TestChromiumErrorType(t *testing.T) {
|
||||
{"invalid resource http status", ErrInvalidResourceHttpStatusCode, "chromium_unavailable", "invalid_input"},
|
||||
{"loading failed", ErrLoadingFailed, "chromium_unavailable", "invalid_input"},
|
||||
{"resource loading failed", ErrResourceLoadingFailed, "chromium_unavailable", "invalid_input"},
|
||||
{"crashed", ErrChromiumCrashed, "chromium_unavailable", "chromium_unavailable"},
|
||||
{"invalid evaluation expression", ErrInvalidEvaluationExpression, "chromium_unavailable", "invalid_input"},
|
||||
{"invalid selector query", ErrInvalidSelectorQuery, "chromium_unavailable", "invalid_input"},
|
||||
{"pdf queue", gotenberg.ErrMaximumQueueSizeExceeded, "chromium_unavailable", "chromium_unavailable"},
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
|
||||
"github.com/chromedp/cdproto/cdp"
|
||||
"github.com/chromedp/cdproto/fetch"
|
||||
"github.com/chromedp/cdproto/inspector"
|
||||
"github.com/chromedp/cdproto/network"
|
||||
"github.com/chromedp/cdproto/page"
|
||||
"github.com/chromedp/cdproto/runtime"
|
||||
@@ -521,6 +522,38 @@ func listenForEventExceptionThrown(ctx context.Context, logger *slog.Logger, con
|
||||
})
|
||||
}
|
||||
|
||||
type eventTargetCrashedOptions struct {
|
||||
crashed *error
|
||||
crashedMu *sync.RWMutex
|
||||
cancel context.CancelFunc
|
||||
}
|
||||
|
||||
// listenForEventTargetCrashed listens for the Inspector.targetCrashed event,
|
||||
// which Chromium sends when the renderer serving the conversion's tab
|
||||
// crashes. chromedp enables the Inspector domain on every target but does
|
||||
// not handle this event: left alone, the in-flight CDP command never
|
||||
// receives a response and the conversion blocks until the request deadline.
|
||||
// Record the crash and cancel the task context so the conversion fails fast
|
||||
// instead.
|
||||
// See https://github.com/gotenberg/gotenberg/issues/1640.
|
||||
func listenForEventTargetCrashed(ctx context.Context, logger *slog.Logger, options eventTargetCrashedOptions) {
|
||||
chromedp.ListenTarget(ctx, func(ev any) {
|
||||
if _, ok := ev.(*inspector.EventTargetCrashed); ok {
|
||||
logger.DebugContext(ctx, "event EventTargetCrashed fired")
|
||||
|
||||
options.crashedMu.Lock()
|
||||
defer options.crashedMu.Unlock()
|
||||
|
||||
*options.crashed = ErrChromiumCrashed
|
||||
|
||||
// Cancel the task context so the in-flight CDP command aborts
|
||||
// immediately instead of waiting for a response the crashed
|
||||
// renderer can never send.
|
||||
options.cancel()
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// waitForEventDomContentEventFired registers a listener for the
|
||||
// DomContentEventFired event and returns a waiter that blocks until the
|
||||
// event fires or ctx is done. The listener registers at call time, not
|
||||
|
||||
@@ -1020,6 +1020,16 @@ func handleChromiumError(err error, options Options) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
if errors.Is(err, ErrChromiumCrashed) {
|
||||
return api.WrapError(
|
||||
err,
|
||||
api.NewSentinelHttpError(
|
||||
http.StatusServiceUnavailable,
|
||||
"Chromium crashed while processing the request. Retry, or reduce the workload if the problem persists.",
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
if errors.Is(err, ErrInvalidEvaluationExpression) {
|
||||
if options.WaitForExpression == "" {
|
||||
// We do not expect the 'waitWindowStatus' form field to return
|
||||
|
||||
51
pkg/modules/chromium/routes_test.go
Normal file
51
pkg/modules/chromium/routes_test.go
Normal file
@@ -0,0 +1,51 @@
|
||||
package chromium
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"github.com/gotenberg/gotenberg/v8/pkg/modules/api"
|
||||
)
|
||||
|
||||
// TestHandleChromiumError_Crashed pins the mapping of a Chromium renderer
|
||||
// crash to a 503 Service Unavailable. When the renderer crashes mid-conversion,
|
||||
// the request must fail fast with 503 rather than hang until the deadline and
|
||||
// surface as a generic timeout.
|
||||
// See https://github.com/gotenberg/gotenberg/issues/1640.
|
||||
func TestHandleChromiumError_Crashed(t *testing.T) {
|
||||
// Mirror the wrapping done by [chromiumBrowser.do].
|
||||
err := handleChromiumError(fmt.Errorf("handle tasks: %w", ErrChromiumCrashed), Options{})
|
||||
if err == nil {
|
||||
t.Fatal("expected an error, got none")
|
||||
}
|
||||
|
||||
status, message := api.ParseError(err)
|
||||
if status != http.StatusServiceUnavailable {
|
||||
t.Errorf("status = %d, want %d (message: %s)", status, http.StatusServiceUnavailable, message)
|
||||
}
|
||||
|
||||
want := "Chromium crashed while processing the request. Retry, or reduce the workload if the problem persists."
|
||||
if message != want {
|
||||
t.Errorf("message = %q, want %q", message, want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestHandleChromiumError_CrashedTakesPrecedence guards the ordering in
|
||||
// [handleChromiumError]: a crash is a server-side failure and must map to 503
|
||||
// even when the error chain also carries a marker that another branch would
|
||||
// map to a client-error status.
|
||||
func TestHandleChromiumError_CrashedTakesPrecedence(t *testing.T) {
|
||||
err := handleChromiumError(
|
||||
fmt.Errorf("handle tasks: %w; %w", ErrChromiumCrashed, ErrInvalidHttpStatusCode),
|
||||
Options{},
|
||||
)
|
||||
if err == nil {
|
||||
t.Fatal("expected an error, got none")
|
||||
}
|
||||
|
||||
status, _ := api.ParseError(err)
|
||||
if status != http.StatusServiceUnavailable {
|
||||
t.Errorf("status = %d, want %d", status, http.StatusServiceUnavailable)
|
||||
}
|
||||
}
|
||||
@@ -1401,3 +1401,19 @@ Feature: /forms/chromium/convert/url
|
||||
Then the response header "Content-Type" should be "application/pdf"
|
||||
Then there should be 1 PDF(s) in the response
|
||||
Then the "foo.pdf" PDF should have 1 page(s)
|
||||
|
||||
# chrome://crash makes the renderer crash deterministically, the same
|
||||
# failure class as a renderer crash triggered by the page content. The
|
||||
# request must fail fast with a 503 instead of hanging until the API
|
||||
# timeout.
|
||||
# See https://github.com/gotenberg/gotenberg/issues/1640.
|
||||
Scenario: POST /forms/chromium/convert/url (Chromium crash fails fast with 503)
|
||||
Given I have a default Gotenberg container
|
||||
When I make a "POST" request to Gotenberg at the "/forms/chromium/convert/url" endpoint with the following form data and header(s):
|
||||
| url | chrome://crash | field |
|
||||
Then the response status code should be 503
|
||||
Then the response header "Content-Type" should be "text/plain; charset=UTF-8"
|
||||
Then the response body should contain string:
|
||||
"""
|
||||
Chromium crashed while processing the request
|
||||
"""
|
||||
|
||||
Reference in New Issue
Block a user