package chromium import ( "context" "errors" "fmt" "log/slog" "net/http" "net/url" "slices" "strings" "sync" "github.com/chromedp/cdproto/cdp" "github.com/chromedp/cdproto/fetch" "github.com/chromedp/cdproto/network" "github.com/chromedp/cdproto/page" "github.com/chromedp/cdproto/runtime" "github.com/chromedp/chromedp" "github.com/dlclark/regexp2" "golang.org/x/sync/errgroup" "github.com/gotenberg/gotenberg/v8/pkg/gotenberg" ) // listenForNetworkActivity accumulates per-conversion network activity into // aggregate from the always-on Network domain events. It is a no-op when // aggregate is nil. func listenForNetworkActivity(ctx context.Context, aggregate *networkAggregate) { if aggregate == nil { return } chromedp.ListenTarget(ctx, func(ev any) { switch e := ev.(type) { case *network.EventResponseReceived: aggregate.onResponseReceived(e) case *network.EventLoadingFinished: aggregate.onLoadingFinished(e) case *network.EventLoadingFailed: aggregate.onLoadingFailed(e) } }) } type eventRequestPausedOptions struct { allowList, denyList []*regexp2.Regexp denyPrivateIPs bool denyPublicIPs bool allowedFilePrefixes []string extraHttpHeaders []ExtraHttpHeader } // listenForEventRequestPaused listens for requests to check if they are // allowed or not. It also set the extra HTTP headers, if any. // See https://github.com/gotenberg/gotenberg/issues/1011. // TODO: https://chromedevtools.github.io/devtools-protocol/tot/Network/#method-setBlockedURLs (experimental for now). func listenForEventRequestPaused(ctx context.Context, logger *slog.Logger, options eventRequestPausedOptions) { if len(options.extraHttpHeaders) == 0 { logger.DebugContext(ctx, "no extra HTTP headers") } else { logger.DebugContext(ctx, fmt.Sprintf("extra HTTP headers: %+v", options.extraHttpHeaders)) } chromedp.ListenTarget(ctx, func(ev any) { if e, ok := ev.(*fetch.EventRequestPaused); ok { go func() { logger.DebugContext(ctx, fmt.Sprintf("event EventRequestPaused fired for '%s'", e.Request.URL)) allow := true deadline, ok := ctx.Deadline() if !ok { logger.ErrorContext(ctx, "context has no deadline, cannot filter URL") return } err := gotenberg.FilterOutboundURL(ctx, e.Request.URL, options.allowList, options.denyList, deadline, gotenberg.WithDenyPrivateIPs(options.denyPrivateIPs), gotenberg.WithDenyPublicIPs(options.denyPublicIPs), ) if err != nil { logger.WarnContext(ctx, err.Error()) allow = false } // Sub-resource file:// URLs are opt-in per route. A route // that renders local files (HTML, Markdown) populates // allowedFilePrefixes with the request working directory // so its own assets load while sibling requests' /tmp // paths stay out of reach. Every other route leaves the // slice empty; treat that as default-deny so a file:// // sub-resource that slips past the deny-list (which // exempts /tmp/) still cannot read the working // directories of other in-flight conversions. if allow && strings.HasPrefix(e.Request.URL, "file://") && !isAllowedFileSubResource(e.Request.URL, options.allowedFilePrefixes) { logger.WarnContext(ctx, fmt.Sprintf("'%s' is not within any allowed file prefix", e.Request.URL)) allow = false } cctx := chromedp.FromContext(ctx) executorCtx := cdp.WithExecutor(ctx, cctx.Target) if !allow { // Use AccessDenied so Chromium emits net::ERR_ACCESS_DENIED, // which is intentionally absent from the EventLoadingFailed // known-errors list. Routing through BlockedByClient would // surface the failure, but the Document-type dispatcher in // listenForEventLoadingFailed cannot distinguish a blocked // iframe (sub-frame Document) from a main-page Document, and // would attribute the iframe failure to the main page. // Filter-block observability is provided by the warn log // above instead. req := fetch.FailRequest(e.RequestID, network.ErrorReasonAccessDenied) err = req.Do(executorCtx) if err != nil { logger.ErrorContext(ctx, fmt.Sprintf("fail request: %s", err)) } return } req := fetch.ContinueRequest(e.RequestID) var extraHttpHeadersToSet []ExtraHttpHeader if len(options.extraHttpHeaders) > 0 { // The user wants to set extra HTTP headers. // First, we have to check if at least one header has to be // set for the current request. for _, header := range options.extraHttpHeaders { if header.Scope == nil { // Non-scoped header. logger.DebugContext(ctx, fmt.Sprintf("extra HTTP header '%s' will be set for request URL '%s'", header.Name, e.Request.URL)) extraHttpHeadersToSet = append(extraHttpHeadersToSet, header) continue } ok, err := header.Scope.MatchString(e.Request.URL) switch { case err != nil: logger.ErrorContext(ctx, fmt.Sprintf("fail to match extra HTTP header '%s' scope with URL '%s': %s", header.Name, e.Request.URL, err)) case ok: logger.DebugContext(ctx, fmt.Sprintf("extra HTTP header '%s' (scoped) will be set for request URL '%s'", header.Name, e.Request.URL)) extraHttpHeadersToSet = append(extraHttpHeadersToSet, header) default: logger.DebugContext(ctx, fmt.Sprintf("scoped extra HTTP header '%s' (scoped) will not be set for request URL '%s'", header.Name, e.Request.URL)) } } } if len(extraHttpHeadersToSet) > 0 { logger.DebugContext(ctx, fmt.Sprintf("setting extra HTTP headers for request URL '%s': %+v", e.Request.URL, extraHttpHeadersToSet)) originalHeaders := e.Request.Headers headers := make(map[string]string) for key, value := range originalHeaders { strValue, ok := value.(string) if ok { headers[key] = strValue } else { logger.ErrorContext(ctx, fmt.Sprintf("ignoring header '%s' for URL '%s' since it cannot be cast to a string", key, e.Request.URL)) } } var headersEntries []*fetch.HeaderEntry for key, value := range headers { headersEntries = append(headersEntries, &fetch.HeaderEntry{ Name: key, Value: value, }) } for _, header := range extraHttpHeadersToSet { headersEntries = append(headersEntries, &fetch.HeaderEntry{ Name: header.Name, Value: header.Value, }) } req.Headers = headersEntries } err = req.Do(executorCtx) if err != nil { logger.ErrorContext(ctx, fmt.Sprintf("continue request: %s", err)) } }() } }) } type eventResponseReceivedOptions struct { mainPageUrl string failOnHttpStatusCodes []int64 invalidHttpStatusCode *error invalidHttpStatusCodeMu *sync.RWMutex failOnResourceOnHttpStatusCode []int64 ignoreResourceHttpStatusDomains []string invalidResourceHttpStatusCode *error invalidResourceHttpStatusCodeMu *sync.RWMutex cancelOnMainPageError context.CancelFunc } // listenForEventResponseReceived listens for an invalid HTTP status code // returned by the main page or by one or more resources. // See: // https://github.com/gotenberg/gotenberg/issues/613. // https://github.com/gotenberg/gotenberg/issues/1021. func listenForEventResponseReceived( ctx context.Context, logger *slog.Logger, options eventResponseReceivedOptions, ) { normalizedIgnoreDomains := normalizeDomains(options.ignoreResourceHttpStatusDomains) for _, code := range []int64{199, 299, 399, 499, 599} { if slices.Contains(options.failOnHttpStatusCodes, code) { for i := code - 99; i <= code; i++ { options.failOnHttpStatusCodes = append(options.failOnHttpStatusCodes, i) } } if slices.Contains(options.failOnResourceOnHttpStatusCode, code) { for i := code - 99; i <= code; i++ { options.failOnResourceOnHttpStatusCode = append(options.failOnResourceOnHttpStatusCode, i) } } } chromedp.ListenTarget(ctx, func(ev any) { if ev, ok := ev.(*network.EventResponseReceived); ok { if ev.Response.URL == options.mainPageUrl { logger.DebugContext(ctx, fmt.Sprintf("event EventResponseReceived fired for main page: %+v", ev.Response)) if slices.Contains(options.failOnHttpStatusCodes, ev.Response.Status) { options.invalidHttpStatusCodeMu.Lock() defer options.invalidHttpStatusCodeMu.Unlock() *options.invalidHttpStatusCode = fmt.Errorf("%d: %s", ev.Response.Status, ev.Response.StatusText) // Cancel the task context so that any in-flight wait // operations (waitForSelector, waitForExpression, etc.) // abort immediately instead of polling until timeout. // See https://github.com/gotenberg/gotenberg/issues/1492. if options.cancelOnMainPageError != nil { options.cancelOnMainPageError() } } return } logger.DebugContext(ctx, fmt.Sprintf("event EventResponseReceived fired for a resource: %+v", ev.Response)) if slices.Contains(options.failOnResourceOnHttpStatusCode, ev.Response.Status) { if !shouldCheckResourceHttpStatusCode(ev.Response.URL, normalizedIgnoreDomains) { logger.DebugContext(ctx, fmt.Sprintf("skip resource HTTP status code check for '%s' due to domain filtering", ev.Response.URL)) return } options.invalidResourceHttpStatusCodeMu.Lock() defer options.invalidResourceHttpStatusCodeMu.Unlock() *options.invalidResourceHttpStatusCode = errors.Join( *options.invalidResourceHttpStatusCode, fmt.Errorf("%s - %d: %s", ev.Response.URL, ev.Response.Status, http.StatusText(int(ev.Response.Status))), ) } } }) } // isAllowedFileSubResource reports whether a file:// sub-resource URL is // within at least one prefix. An empty prefix list rejects every // file:// URL so routes that never populate the list (for example // /forms/chromium/convert/url) default-deny reads from /tmp/, blocking // cross-request enumeration. func isAllowedFileSubResource(rawURL string, allowedFilePrefixes []string) bool { if len(allowedFilePrefixes) == 0 { return false } for _, prefix := range allowedFilePrefixes { if strings.HasPrefix(rawURL, "file://"+prefix) { return true } } return false } func shouldCheckResourceHttpStatusCode(rawURL string, ignoreDomains []string) bool { host := hostnameFromURL(rawURL) if len(ignoreDomains) > 0 && matchesAnyDomain(host, ignoreDomains) { return false } return true } func hostnameFromURL(rawURL string) string { u, err := url.Parse(rawURL) if err != nil { return "" } return strings.ToLower(u.Hostname()) } func normalizeDomains(domains []string) []string { normalized := make([]string, 0, len(domains)) for _, domain := range domains { d := normalizeDomain(domain) if d == "" { continue } normalized = append(normalized, d) } return normalized } func normalizeDomain(domain string) string { d := strings.ToLower(strings.TrimSpace(domain)) if d == "" { return "" } // Accept "example.com", "*.example.com", ".example.com", "https://example.com/path", // or "example.com:443". if strings.Contains(d, "://") || strings.HasPrefix(d, "//") { u, err := url.Parse(d) if err == nil && u.Hostname() != "" { d = strings.ToLower(u.Hostname()) } } else { // Make it parseable as a URL to extract the hostname and drop any port/path. u, err := url.Parse("https://" + d) if err == nil && u.Hostname() != "" { d = strings.ToLower(u.Hostname()) } } d = strings.TrimPrefix(d, "*.") d = strings.TrimPrefix(d, ".") return d } func matchesAnyDomain(host string, domains []string) bool { if host == "" || len(domains) == 0 { return false } for _, domain := range domains { if host == domain || strings.HasSuffix(host, "."+domain) { return true } } return false } type eventLoadingFailedOptions struct { loadingFailed *error loadingFailedMu *sync.RWMutex resourceLoadingFailed *error resourceLoadingFailedMu *sync.RWMutex cancelOnMainPageError context.CancelFunc } // listenForEventLoadingFailed listens for an event indicating that the main // page or one or more resources failed to load. // See: // https://github.com/gotenberg/gotenberg/issues/913. // https://github.com/gotenberg/gotenberg/issues/959. // https://github.com/gotenberg/gotenberg/issues/1021. func listenForEventLoadingFailed(ctx context.Context, logger *slog.Logger, options eventLoadingFailedOptions) { chromedp.ListenTarget(ctx, func(ev any) { if ev, ok := ev.(*network.EventLoadingFailed); ok { logger.DebugContext(ctx, fmt.Sprintf("event EventLoadingFailed fired: %+v", ev.ErrorText)) // We are looking for common errors. // TODO: sufficient? knownErrors := []string{ "net::ERR_CONNECTION_CLOSED", "net::ERR_CONNECTION_RESET", "net::ERR_CONNECTION_REFUSED", "net::ERR_CONNECTION_ABORTED", "net::ERR_CONNECTION_FAILED", "net::ERR_NAME_NOT_RESOLVED", "net::ERR_INTERNET_DISCONNECTED", "net::ERR_ADDRESS_UNREACHABLE", "net::ERR_BLOCKED_BY_CLIENT", "net::ERR_BLOCKED_BY_RESPONSE", "net::ERR_FILE_NOT_FOUND", "net::ERR_HTTP2_PROTOCOL_ERROR", } if !slices.Contains(knownErrors, ev.ErrorText) { logger.DebugContext(ctx, fmt.Sprintf("skip EventLoadingFailed: '%s' is not part of %+v", ev.ErrorText, knownErrors)) return } if ev.Type == network.ResourceTypeDocument { // Supposition: except iframe, an event loading failed with a // resource type Document is about the main page. logger.DebugContext(ctx, "event EventLoadingFailed fired for main page") options.loadingFailedMu.Lock() defer options.loadingFailedMu.Unlock() *options.loadingFailed = fmt.Errorf("%s", ev.ErrorText) // Cancel the task context so that any in-flight wait // operations abort immediately. // See https://github.com/gotenberg/gotenberg/issues/1492. if options.cancelOnMainPageError != nil { options.cancelOnMainPageError() } return } logger.DebugContext(ctx, "event EventLoadingFailed fired for a resource") options.resourceLoadingFailedMu.Lock() defer options.resourceLoadingFailedMu.Unlock() *options.resourceLoadingFailed = errors.Join( *options.resourceLoadingFailed, fmt.Errorf("resource %s: %s", ev.Type, ev.ErrorText), ) } }) } // listenForEventExceptionThrown listens for exceptions in the console and // appends those exceptions to the given error pointer. // See https://github.com/gotenberg/gotenberg/issues/262. func listenForEventExceptionThrown(ctx context.Context, logger *slog.Logger, consoleExceptions *error, consoleExceptionsMu *sync.RWMutex) { chromedp.ListenTarget(ctx, func(ev any) { if ev, ok := ev.(*runtime.EventExceptionThrown); ok { logger.DebugContext(ctx, fmt.Sprintf("event EventExceptionThrown fired: %+v", ev.ExceptionDetails)) consoleExceptionsMu.Lock() defer consoleExceptionsMu.Unlock() *consoleExceptions = errors.Join(*consoleExceptions, fmt.Errorf("\n%+v", ev.ExceptionDetails)) } }) } // waitForEventDomContentEventFired registers a listener for the // DomContentEventFired event and returns a waiter that blocks until the // event fires or ctx is done. The listener registers at call time, not // inside the waiter, so callers must invoke this before triggering the // action that may emit the event. Registering inside the waiter would // open a race: for fast loads (typically file:// pages with no external // sub-resources), the event can fire before the waiter goroutine starts // and the listener never sees a record. func waitForEventDomContentEventFired(ctx context.Context, logger *slog.Logger) func() error { ch := make(chan struct{}) cctx, cancel := context.WithCancel(ctx) chromedp.ListenTarget(cctx, func(ev any) { if _, ok := ev.(*page.EventDomContentEventFired); ok { cancel() close(ch) } }) return func() error { select { case <-ch: logger.DebugContext(ctx, "event DomContentEventFired fired") return nil case <-ctx.Done(): return fmt.Errorf("wait for event DomContentEventFired: %w", ctx.Err()) } } } // waitForEventLoadEventFired registers a listener for the LoadEventFired // event and returns a waiter that blocks until the event fires or ctx is // done. See [waitForEventDomContentEventFired] for the rationale on // registering at call time rather than inside the waiter. func waitForEventLoadEventFired(ctx context.Context, logger *slog.Logger) func() error { ch := make(chan struct{}) cctx, cancel := context.WithCancel(ctx) chromedp.ListenTarget(cctx, func(ev any) { if _, ok := ev.(*page.EventLoadEventFired); ok { cancel() close(ch) } }) return func() error { select { case <-ch: logger.DebugContext(ctx, "event LoadEventFired fired") return nil case <-ctx.Done(): return fmt.Errorf("wait for event LoadEventFired: %w", ctx.Err()) } } } // waitForEventNetworkIdle registers a listener for the networkIdle // lifecycle event and returns a waiter that blocks until the event fires // or ctx is done. See [waitForEventDomContentEventFired] for the // rationale on registering at call time rather than inside the waiter. func waitForEventNetworkIdle(ctx context.Context, logger *slog.Logger) func() error { ch := make(chan struct{}) cctx, cancel := context.WithCancel(ctx) chromedp.ListenTarget(cctx, func(ev any) { if e, ok := ev.(*page.EventLifecycleEvent); ok && e.Name == "networkIdle" { cancel() close(ch) } }) return func() error { select { case <-ch: logger.DebugContext(ctx, "event networkIdle fired") return nil case <-ctx.Done(): return fmt.Errorf("wait for event networkIdle: %w", ctx.Err()) } } } // waitForEventNetworkAlmostIdle registers a listener for the networkIdle2 // lifecycle event and returns a waiter that blocks until the event fires // or ctx is done. See [waitForEventDomContentEventFired] for the // rationale on registering at call time rather than inside the waiter. func waitForEventNetworkAlmostIdle(ctx context.Context, logger *slog.Logger) func() error { ch := make(chan struct{}) cctx, cancel := context.WithCancel(ctx) chromedp.ListenTarget(cctx, func(ev any) { if e, ok := ev.(*page.EventLifecycleEvent); ok && e.Name == "networkIdle2" { cancel() close(ch) } }) return func() error { select { case <-ch: logger.DebugContext(ctx, "event networkAlmostIdle fired") return nil case <-ctx.Done(): return fmt.Errorf("wait for event networkAlmostIdle: %w", ctx.Err()) } } } // waitForEventLoadingFinished registers a listener for the // LoadingFinished event and returns a waiter that blocks until the event // fires or ctx is done. See [waitForEventDomContentEventFired] for the // rationale on registering at call time rather than inside the waiter. func waitForEventLoadingFinished(ctx context.Context, logger *slog.Logger) func() error { ch := make(chan struct{}) cctx, cancel := context.WithCancel(ctx) chromedp.ListenTarget(cctx, func(ev any) { if _, ok := ev.(*network.EventLoadingFinished); ok { cancel() close(ch) } }) return func() error { select { case <-ch: logger.DebugContext(ctx, "event LoadingFinished fired") return nil case <-ctx.Done(): return fmt.Errorf("wait for event LoadingFinished: %w", ctx.Err()) } } } // runBatch runs all functions simultaneously and waits until all of them are // completed or an error is encountered. func runBatch(ctx context.Context, fn ...func() error) error { eg, _ := errgroup.WithContext(ctx) for _, f := range fn { eg.Go(f) } return eg.Wait() }