diff --git a/pkg/modules/chromium/browser.go b/pkg/modules/chromium/browser.go index 2fc491c2..409941e0 100644 --- a/pkg/modules/chromium/browser.go +++ b/pkg/modules/chromium/browser.go @@ -25,8 +25,8 @@ import ( type browser interface { gotenberg.Process - pdf(ctx context.Context, logger *slog.Logger, url, outputPath string, options PdfOptions) error - screenshot(ctx context.Context, logger *slog.Logger, url, outputPath string, options ScreenshotOptions) error + pdf(ctx context.Context, logger *slog.Logger, url, outputPath string, options PdfOptions, aggregate *networkAggregate) error + screenshot(ctx context.Context, logger *slog.Logger, url, outputPath string, options ScreenshotOptions, aggregate *networkAggregate) error } type browserArguments struct { @@ -314,10 +314,10 @@ func (b *chromiumBrowser) Healthy(logger *slog.Logger) bool { return true } -func (b *chromiumBrowser) pdf(ctx context.Context, logger *slog.Logger, url, outputPath string, options PdfOptions) error { +func (b *chromiumBrowser) pdf(ctx context.Context, logger *slog.Logger, url, outputPath string, options PdfOptions, aggregate *networkAggregate) error { // Note: no error wrapping because it leaks on errors we want to display to // the end user. - return b.do(ctx, logger, url, options.Options, chromedp.Tasks{ + return b.do(ctx, logger, url, options.Options, aggregate, chromedp.Tasks{ network.Enable(), fetch.Enable(), runtime.Enable(), @@ -340,10 +340,10 @@ func (b *chromiumBrowser) pdf(ctx context.Context, logger *slog.Logger, url, out }) } -func (b *chromiumBrowser) screenshot(ctx context.Context, logger *slog.Logger, url, outputPath string, options ScreenshotOptions) error { +func (b *chromiumBrowser) screenshot(ctx context.Context, logger *slog.Logger, url, outputPath string, options ScreenshotOptions, aggregate *networkAggregate) error { // Note: no error wrapping because it leaks on errors we want to display to // the end user. - return b.do(ctx, logger, url, options.Options, chromedp.Tasks{ + return b.do(ctx, logger, url, options.Options, aggregate, chromedp.Tasks{ network.Enable(), fetch.Enable(), runtime.Enable(), @@ -367,7 +367,7 @@ func (b *chromiumBrowser) screenshot(ctx context.Context, logger *slog.Logger, u }) } -func (b *chromiumBrowser) do(ctx context.Context, logger *slog.Logger, url string, options Options, tasks chromedp.Tasks) error { +func (b *chromiumBrowser) do(ctx context.Context, logger *slog.Logger, url string, options Options, aggregate *networkAggregate, tasks chromedp.Tasks) error { if !b.isStarted.Load() { return errors.New("browser not started, cannot handle tasks") } @@ -396,6 +396,9 @@ func (b *chromiumBrowser) do(ctx context.Context, logger *slog.Logger, url strin taskCtx, taskCancel := chromedp.NewContext(timeoutCtx) defer taskCancel() + // Accumulate per-conversion network activity for telemetry. + listenForNetworkActivity(taskCtx, aggregate) + // We validate all other requests against our allowed / deny lists. // If a request does not pass the validation, we make it fail. It also set // the extra HTTP headers, if any. diff --git a/pkg/modules/chromium/chromium.go b/pkg/modules/chromium/chromium.go index d8ee2162..06565c32 100644 --- a/pkg/modules/chromium/chromium.go +++ b/pkg/modules/chromium/chromium.go @@ -107,6 +107,8 @@ type Chromium struct { queueWaitDurationCounter metric.Float64Histogram pdfOutputSizeCounter metric.Int64Histogram imageOutputSizeCounter metric.Int64Histogram + networkRequestsCounter metric.Int64Counter + networkBytesCounter metric.Int64Histogram } // Options are the common options for all conversions. @@ -633,6 +635,24 @@ func (mod *Chromium) Provision(ctx *gotenberg.Context) error { return fmt.Errorf("create chromium.image.output.size histogram: %w", err) } + mod.networkRequestsCounter, err = meter.Int64Counter( + "chromium.network.requests.total", + metric.WithDescription("Total number of network requests made during Chromium conversions"), + metric.WithUnit("{request}"), + ) + if err != nil { + return fmt.Errorf("create chromium.network.requests.total counter: %w", err) + } + + mod.networkBytesCounter, err = meter.Int64Histogram( + "chromium.network.bytes", + metric.WithDescription("Bytes fetched over the network during a Chromium conversion"), + metric.WithUnit("By"), + ) + if err != nil { + return fmt.Errorf("create chromium.network.bytes histogram: %w", err) + } + return nil } @@ -821,9 +841,10 @@ func (mod *Chromium) Pdf(ctx context.Context, logger *slog.Logger, url, outputPa start := time.Now() var conversionStart time.Time + aggregate := newNetworkAggregate() err := mod.supervisor.Run(ctx, logger, func() error { conversionStart = time.Now() - return mod.browser.pdf(ctx, logger, url, outputPath, options) + return mod.browser.pdf(ctx, logger, url, outputPath, options, aggregate) }) end := time.Now() @@ -865,6 +886,8 @@ func (mod *Chromium) Pdf(ctx context.Context, logger *slog.Logger, url, outputPa attribute.String("status", status), )) + mod.recordNetwork(ctx, span, aggregate) + if err == nil { if fileInfo, statErr := os.Stat(outputPath); statErr == nil { mod.pdfOutputSizeCounter.Record(ctx, fileInfo.Size()) @@ -898,9 +921,10 @@ func (mod *Chromium) Screenshot(ctx context.Context, logger *slog.Logger, url, o start := time.Now() var conversionStart time.Time + aggregate := newNetworkAggregate() err := mod.supervisor.Run(ctx, logger, func() error { conversionStart = time.Now() - return mod.browser.screenshot(ctx, logger, url, outputPath, options) + return mod.browser.screenshot(ctx, logger, url, outputPath, options, aggregate) }) end := time.Now() @@ -942,6 +966,8 @@ func (mod *Chromium) Screenshot(ctx context.Context, logger *slog.Logger, url, o attribute.String("status", status), )) + mod.recordNetwork(ctx, span, aggregate) + if err == nil { if fileInfo, statErr := os.Stat(outputPath); statErr == nil { mod.imageOutputSizeCounter.Record(ctx, fileInfo.Size()) @@ -956,6 +982,42 @@ func (mod *Chromium) Screenshot(ctx context.Context, logger *slog.Logger, url, o return err } +// recordNetwork lifts per-conversion network aggregates onto the span and the +// network metrics. Counts are dimensioned by outcome and bytes feed a +// histogram; both are recorded with the conversion context so the SDK attaches +// trace exemplars. The heaviest resource URL is redacted before it lands on the +// span event. +func (mod *Chromium) recordNetwork(ctx context.Context, span trace.Span, aggregate *networkAggregate) { + if aggregate == nil { + return + } + + stats := aggregate.snapshot() + + span.SetAttributes( + attribute.Int64("gotenberg.chromium.resources.count", stats.requestCount), + attribute.Int64("gotenberg.chromium.resources.bytes_total", stats.bytesTotal), + attribute.Int64("gotenberg.chromium.resources.failed_count", stats.failedCount), + attribute.Int64("gotenberg.chromium.resources.unique_origins", stats.uniqueOrigins), + ) + + if stats.heaviestURL != "" { + span.AddEvent("chromium.heaviest_resource", trace.WithAttributes( + attribute.String("url", gotenberg.RedactURL(stats.heaviestURL)), + attribute.Int64("bytes", stats.heaviestBytes), + )) + } + + if ok := stats.requestCount - stats.failedCount; ok > 0 { + mod.networkRequestsCounter.Add(ctx, ok, metric.WithAttributes(attribute.String("outcome", "ok"))) + } + if stats.failedCount > 0 { + mod.networkRequestsCounter.Add(ctx, stats.failedCount, metric.WithAttributes(attribute.String("outcome", "failed"))) + } + + mod.networkBytesCounter.Record(ctx, stats.bytesTotal) +} + // conversionInputAttrs derives low-cardinality input attributes for a // conversion span: the number of received files (when ctx is an [api.Context]) // and the size of the local HTML input (when url is a file:// URL). Remote URL diff --git a/pkg/modules/chromium/events.go b/pkg/modules/chromium/events.go index c34aff5e..40d6c2b5 100644 --- a/pkg/modules/chromium/events.go +++ b/pkg/modules/chromium/events.go @@ -23,6 +23,26 @@ import ( "github.com/gotenberg/gotenberg/v8/pkg/gotenberg" ) +// listenForNetworkActivity accumulates per-conversion network activity into +// aggregate from the always-on Network domain events. It is a no-op when +// aggregate is nil. +func listenForNetworkActivity(ctx context.Context, aggregate *networkAggregate) { + if aggregate == nil { + return + } + + chromedp.ListenTarget(ctx, func(ev any) { + switch e := ev.(type) { + case *network.EventResponseReceived: + aggregate.onResponseReceived(e) + case *network.EventLoadingFinished: + aggregate.onLoadingFinished(e) + case *network.EventLoadingFailed: + aggregate.onLoadingFailed(e) + } + }) +} + type eventRequestPausedOptions struct { allowList, denyList []*regexp2.Regexp denyPrivateIPs bool diff --git a/pkg/modules/chromium/mocks.go b/pkg/modules/chromium/mocks.go index 36a2f5f1..a0440964 100644 --- a/pkg/modules/chromium/mocks.go +++ b/pkg/modules/chromium/mocks.go @@ -24,16 +24,16 @@ func (api *ApiMock) Screenshot(ctx context.Context, logger *slog.Logger, url, ou // browserMock is a mock for the [browser] interface. type browserMock struct { gotenberg.ProcessMock - pdfMock func(ctx context.Context, logger *slog.Logger, url, outputPath string, options PdfOptions) error - screenshotMock func(ctx context.Context, logger *slog.Logger, url, outputPath string, options ScreenshotOptions) error + pdfMock func(ctx context.Context, logger *slog.Logger, url, outputPath string, options PdfOptions, aggregate *networkAggregate) error + screenshotMock func(ctx context.Context, logger *slog.Logger, url, outputPath string, options ScreenshotOptions, aggregate *networkAggregate) error } -func (b *browserMock) pdf(ctx context.Context, logger *slog.Logger, url, outputPath string, options PdfOptions) error { - return b.pdfMock(ctx, logger, url, outputPath, options) +func (b *browserMock) pdf(ctx context.Context, logger *slog.Logger, url, outputPath string, options PdfOptions, aggregate *networkAggregate) error { + return b.pdfMock(ctx, logger, url, outputPath, options, aggregate) } -func (b *browserMock) screenshot(ctx context.Context, logger *slog.Logger, url, outputPath string, options ScreenshotOptions) error { - return b.screenshotMock(ctx, logger, url, outputPath, options) +func (b *browserMock) screenshot(ctx context.Context, logger *slog.Logger, url, outputPath string, options ScreenshotOptions, aggregate *networkAggregate) error { + return b.screenshotMock(ctx, logger, url, outputPath, options, aggregate) } // Interface guards. diff --git a/pkg/modules/chromium/network_aggregate.go b/pkg/modules/chromium/network_aggregate.go new file mode 100644 index 00000000..3431f261 --- /dev/null +++ b/pkg/modules/chromium/network_aggregate.go @@ -0,0 +1,121 @@ +package chromium + +import ( + "net/url" + "sync" + + "github.com/chromedp/cdproto/network" +) + +// maxTrackedOrigins bounds the distinct origins kept per conversion so a +// pathological page cannot grow the set without limit. +const maxTrackedOrigins = 64 + +// networkAggregate accumulates per-conversion network activity from Chromium +// DevTools events. It is safe for concurrent use by the chromedp event listener +// goroutine and the conversion goroutine that reads the snapshot afterwards. +type networkAggregate struct { + mu sync.Mutex + + requestCount int64 + bytesTotal int64 + failedCount int64 + + origins map[string]struct{} + requestURLByID map[network.RequestID]string + + heaviestURL string + heaviestBytes int64 +} + +// networkStats is an immutable snapshot of a [networkAggregate]. +type networkStats struct { + requestCount int64 + bytesTotal int64 + failedCount int64 + uniqueOrigins int64 + heaviestURL string + heaviestBytes int64 +} + +func newNetworkAggregate() *networkAggregate { + return &networkAggregate{ + origins: make(map[string]struct{}), + requestURLByID: make(map[network.RequestID]string), + } +} + +// onResponseReceived records the response origin and remembers the URL for the +// request id, so a later loading-finished event can attribute its bytes. +func (a *networkAggregate) onResponseReceived(ev *network.EventResponseReceived) { + if ev == nil || ev.Response == nil { + return + } + + a.mu.Lock() + defer a.mu.Unlock() + + if origin := originOf(ev.Response.URL); origin != "" { + if _, ok := a.origins[origin]; !ok && len(a.origins) < maxTrackedOrigins { + a.origins[origin] = struct{}{} + } + } + a.requestURLByID[ev.RequestID] = ev.Response.URL +} + +// onLoadingFinished records a successfully completed request and its size, +// tracking the single heaviest resource. +func (a *networkAggregate) onLoadingFinished(ev *network.EventLoadingFinished) { + if ev == nil { + return + } + + a.mu.Lock() + defer a.mu.Unlock() + + a.requestCount++ + + size := int64(ev.EncodedDataLength) + a.bytesTotal += size + if size > a.heaviestBytes { + a.heaviestBytes = size + a.heaviestURL = a.requestURLByID[ev.RequestID] + } +} + +// onLoadingFailed records a request that failed to complete. +func (a *networkAggregate) onLoadingFailed(ev *network.EventLoadingFailed) { + if ev == nil { + return + } + + a.mu.Lock() + defer a.mu.Unlock() + + a.requestCount++ + a.failedCount++ +} + +func (a *networkAggregate) snapshot() networkStats { + a.mu.Lock() + defer a.mu.Unlock() + + return networkStats{ + requestCount: a.requestCount, + bytesTotal: a.bytesTotal, + failedCount: a.failedCount, + uniqueOrigins: int64(len(a.origins)), + heaviestURL: a.heaviestURL, + heaviestBytes: a.heaviestBytes, + } +} + +// originOf returns the scheme://host of rawURL, or an empty string when it has +// no host (for example data: or file: URLs). +func originOf(rawURL string) string { + parsed, err := url.Parse(rawURL) + if err != nil || parsed.Host == "" { + return "" + } + return parsed.Scheme + "://" + parsed.Host +} diff --git a/pkg/modules/chromium/network_aggregate_test.go b/pkg/modules/chromium/network_aggregate_test.go new file mode 100644 index 00000000..67db6dc8 --- /dev/null +++ b/pkg/modules/chromium/network_aggregate_test.go @@ -0,0 +1,100 @@ +package chromium + +import ( + "fmt" + "sync" + "testing" + + "github.com/chromedp/cdproto/network" +) + +func TestOriginOf(t *testing.T) { + for _, tc := range []struct { + raw string + want string + }{ + {"https://example.com/path?q=1", "https://example.com"}, + {"http://cdn.example.com:8080/a.js", "http://cdn.example.com:8080"}, + {"data:image/png;base64,AAAA", ""}, + {"file:///tmp/index.html", ""}, + {"not a url", ""}, + } { + if got := originOf(tc.raw); got != tc.want { + t.Errorf("originOf(%q) = %q, want %q", tc.raw, got, tc.want) + } + } +} + +func TestNetworkAggregate_Snapshot(t *testing.T) { + a := newNetworkAggregate() + + a.onResponseReceived(&network.EventResponseReceived{ + RequestID: "1", + Response: &network.Response{URL: "https://example.com/a.js"}, + }) + a.onResponseReceived(&network.EventResponseReceived{ + RequestID: "2", + Response: &network.Response{URL: "https://cdn.example.com/b.png"}, + }) + // Duplicate origin must not grow the set. + a.onResponseReceived(&network.EventResponseReceived{ + RequestID: "3", + Response: &network.Response{URL: "https://example.com/c.css"}, + }) + + a.onLoadingFinished(&network.EventLoadingFinished{RequestID: "1", EncodedDataLength: 100}) + a.onLoadingFinished(&network.EventLoadingFinished{RequestID: "2", EncodedDataLength: 900}) + a.onLoadingFailed(&network.EventLoadingFailed{RequestID: "3"}) + + got := a.snapshot() + if got.requestCount != 3 { + t.Errorf("requestCount = %d, want 3", got.requestCount) + } + if got.bytesTotal != 1000 { + t.Errorf("bytesTotal = %d, want 1000", got.bytesTotal) + } + if got.failedCount != 1 { + t.Errorf("failedCount = %d, want 1", got.failedCount) + } + if got.uniqueOrigins != 2 { + t.Errorf("uniqueOrigins = %d, want 2", got.uniqueOrigins) + } + if got.heaviestBytes != 900 || got.heaviestURL != "https://cdn.example.com/b.png" { + t.Errorf("heaviest = (%q, %d), want (%q, 900)", got.heaviestURL, got.heaviestBytes, "https://cdn.example.com/b.png") + } +} + +func TestNetworkAggregate_OriginCap(t *testing.T) { + a := newNetworkAggregate() + for i := 0; i < maxTrackedOrigins+50; i++ { + a.onResponseReceived(&network.EventResponseReceived{ + RequestID: network.RequestID(fmt.Sprintf("r%d", i)), + Response: &network.Response{URL: fmt.Sprintf("https://host%d.example.com/x", i)}, + }) + } + if got := a.snapshot().uniqueOrigins; got != maxTrackedOrigins { + t.Errorf("uniqueOrigins = %d, want %d (capped)", got, maxTrackedOrigins) + } +} + +func TestNetworkAggregate_ConcurrentSafe(t *testing.T) { + a := newNetworkAggregate() + var wg sync.WaitGroup + for i := 0; i < 100; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + id := network.RequestID(fmt.Sprintf("r%d", i)) + a.onResponseReceived(&network.EventResponseReceived{ + RequestID: id, + Response: &network.Response{URL: fmt.Sprintf("https://host%d.example.com/x", i)}, + }) + a.onLoadingFinished(&network.EventLoadingFinished{RequestID: id, EncodedDataLength: 10}) + }(i) + } + wg.Wait() + + if got := a.snapshot().requestCount; got != 100 { + t.Errorf("requestCount = %d, want 100", got) + } +}