feat(chromium): skip or waits for networkIdle2 event

pkg/modules/chromium/browser.go

@@ -285,7 +285,7 @@ func (b *chromiumBrowser) pdf(ctx context.Context, logger *zap.Logger, url, outp
 		disableJavaScriptActionFunc(logger, b.arguments.disableJavaScript),
 		setCookiesActionFunc(logger, options.Cookies),
 		userAgentOverride(logger, options.UserAgent),
-		navigateActionFunc(logger, url, options.SkipNetworkIdleEvent),
+		navigateActionFunc(logger, url, options.SkipNetworkIdleEvent, options.SkipNetworkAlmostIdleEvent),
 		hideDefaultWhiteBackgroundActionFunc(logger, options.OmitBackground, options.PrintBackground),
 		forceExactColorsActionFunc(logger, options.PrintBackground),
 		emulateMediaTypeActionFunc(logger, options.EmulatedMediaType, options.EmulatedMediaFeatures),
@@ -311,7 +311,7 @@ func (b *chromiumBrowser) screenshot(ctx context.Context, logger *zap.Logger, ur
 		disableJavaScriptActionFunc(logger, b.arguments.disableJavaScript),
 		setCookiesActionFunc(logger, options.Cookies),
 		userAgentOverride(logger, options.UserAgent),
-		navigateActionFunc(logger, url, options.SkipNetworkIdleEvent),
+		navigateActionFunc(logger, url, options.SkipNetworkIdleEvent, options.SkipNetworkAlmostIdleEvent),
 		hideDefaultWhiteBackgroundActionFunc(logger, options.OmitBackground, true),
 		forceExactColorsActionFunc(logger, true),
 		emulateMediaTypeActionFunc(logger, options.EmulatedMediaType, options.EmulatedMediaFeatures),

pkg/modules/chromium/chromium.go

@@ -105,6 +105,10 @@ type Options struct {
 	// rendered until this event is fired.
 	SkipNetworkIdleEvent bool
 
+	// SkipNetworkAlmostIdleEvent set if the conversion should wait for the
+	// "networkAlmostIdle" event.
+	SkipNetworkAlmostIdleEvent bool
+
 	// FailOnHttpStatusCodes sets if the conversion should fail if the status
 	// code from the main page matches with one of its entries.
 	FailOnHttpStatusCodes []int64
@@ -197,6 +201,7 @@ type EmulatedMediaFeature struct {
 func DefaultOptions() Options {
 	return Options{
 		SkipNetworkIdleEvent:            true,
+		SkipNetworkAlmostIdleEvent:      true,
 		FailOnHttpStatusCodes:           []int64{499, 599},
 		FailOnResourceHttpStatusCodes:   nil,
 		IgnoreResourceHttpStatusDomains: nil,

pkg/modules/chromium/events.go

@@ -462,6 +462,32 @@ func waitForEventNetworkIdle(ctx context.Context, logger *zap.Logger) func() err
 	}
 }
 
+// waitForEventNetworkAlmostIdle waits until the event networkIdle2 is fired
+// or the context timeout.
+func waitForEventNetworkAlmostIdle(ctx context.Context, logger *slog.Logger) func() error {
+	return func() error {
+		ch := make(chan struct{})
+		cctx, cancel := context.WithCancel(ctx)
+		chromedp.ListenTarget(cctx, func(ev any) {
+			switch e := ev.(type) {
+			case *page.EventLifecycleEvent:
+				if e.Name == "networkIdle2" {
+					cancel()
+					close(ch)
+				}
+			}
+		})
+
+		select {
+		case <-ch:
+			logger.DebugContext(ctx, "event networkAlmostIdle fired")
+			return nil
+		case <-ctx.Done():
+			return fmt.Errorf("wait for event networkAlmostIdle: %w", ctx.Err())
+		}
+	}
+}
+
 // waitForEventLoadingFinished waits until the event LoadingFinished is fired
 // or the context timeout.
 func waitForEventLoadingFinished(ctx context.Context, logger *zap.Logger) func() error {

pkg/modules/chromium/routes.go

@@ -48,6 +48,7 @@ func FormDataChromiumOptions(ctx *api.Context) (*api.FormData, Options) {
 
 	var (
 		skipNetworkIdleEvent            bool
+		skipNetworkAlmostIdleEvent      bool
 		failOnHttpStatusCodes           []int64
 		failOnResourceHttpStatusCodes   []int64
 		ignoreResourceHttpStatusDomains []string
@@ -67,6 +68,7 @@ func FormDataChromiumOptions(ctx *api.Context) (*api.FormData, Options) {
 
 	form := ctx.FormData().
 		Bool("skipNetworkIdleEvent", &skipNetworkIdleEvent, defaultOptions.SkipNetworkIdleEvent).
+		Bool("skipNetworkAlmostIdleEvent", &skipNetworkAlmostIdleEvent, defaultOptions.SkipNetworkAlmostIdleEvent).
 		Custom("failOnHttpStatusCodes", func(value string) error {
 			if value == "" {
 				failOnHttpStatusCodes = defaultOptions.FailOnHttpStatusCodes
@@ -254,6 +256,7 @@ func FormDataChromiumOptions(ctx *api.Context) (*api.FormData, Options) {
 
 	options := Options{
 		SkipNetworkIdleEvent:            skipNetworkIdleEvent,
+		SkipNetworkAlmostIdleEvent:      skipNetworkAlmostIdleEvent,
 		FailOnHttpStatusCodes:           failOnHttpStatusCodes,
 		FailOnResourceHttpStatusCodes:   failOnResourceHttpStatusCodes,
 		IgnoreResourceHttpStatusDomains: ignoreResourceHttpStatusDomains,

pkg/modules/chromium/tasks.go

@@ -326,7 +326,7 @@ func userAgentOverride(logger *zap.Logger, userAgent string) chromedp.ActionFunc
 //	}
 //}
 
-func navigateActionFunc(logger *zap.Logger, url string, skipNetworkIdleEvent bool) chromedp.ActionFunc {
+func navigateActionFunc(logger *zap.Logger, url string, skipNetworkIdleEvent, skipNetworkAlmostIdleEvent bool) chromedp.ActionFunc {
 	return func(ctx context.Context) error {
 		logger.Debug(fmt.Sprintf("navigate to '%s'", url))
 
@@ -347,6 +347,12 @@ func navigateActionFunc(logger *zap.Logger, url string, skipNetworkIdleEvent boo
 			logger.Debug("skipping network idle event")
 		}
 
+		if !skipNetworkAlmostIdleEvent {
+			waitFunc = append(waitFunc, waitForEventNetworkAlmostIdle(ctx, logger))
+		} else {
+			logger.Debug("skipping network almost idle event")
+		}
+
 		err = runBatch(
 			ctx,
 			waitFunc...,