feat(chromium): add per-conversion network observability with metrics and exemplars

pkg/modules/chromium/browser.go

@@ -25,8 +25,8 @@ import (
 
 type browser interface {
 	gotenberg.Process
-	pdf(ctx context.Context, logger *slog.Logger, url, outputPath string, options PdfOptions) error
-	screenshot(ctx context.Context, logger *slog.Logger, url, outputPath string, options ScreenshotOptions) error
+	pdf(ctx context.Context, logger *slog.Logger, url, outputPath string, options PdfOptions, aggregate *networkAggregate) error
+	screenshot(ctx context.Context, logger *slog.Logger, url, outputPath string, options ScreenshotOptions, aggregate *networkAggregate) error
 }
 
 type browserArguments struct {
@@ -314,10 +314,10 @@ func (b *chromiumBrowser) Healthy(logger *slog.Logger) bool {
 	return true
 }
 
-func (b *chromiumBrowser) pdf(ctx context.Context, logger *slog.Logger, url, outputPath string, options PdfOptions) error {
+func (b *chromiumBrowser) pdf(ctx context.Context, logger *slog.Logger, url, outputPath string, options PdfOptions, aggregate *networkAggregate) error {
 	// Note: no error wrapping because it leaks on errors we want to display to
 	// the end user.
-	return b.do(ctx, logger, url, options.Options, chromedp.Tasks{
+	return b.do(ctx, logger, url, options.Options, aggregate, chromedp.Tasks{
 		network.Enable(),
 		fetch.Enable(),
 		runtime.Enable(),
@@ -340,10 +340,10 @@ func (b *chromiumBrowser) pdf(ctx context.Context, logger *slog.Logger, url, out
 	})
 }
 
-func (b *chromiumBrowser) screenshot(ctx context.Context, logger *slog.Logger, url, outputPath string, options ScreenshotOptions) error {
+func (b *chromiumBrowser) screenshot(ctx context.Context, logger *slog.Logger, url, outputPath string, options ScreenshotOptions, aggregate *networkAggregate) error {
 	// Note: no error wrapping because it leaks on errors we want to display to
 	// the end user.
-	return b.do(ctx, logger, url, options.Options, chromedp.Tasks{
+	return b.do(ctx, logger, url, options.Options, aggregate, chromedp.Tasks{
 		network.Enable(),
 		fetch.Enable(),
 		runtime.Enable(),
@@ -367,7 +367,7 @@ func (b *chromiumBrowser) screenshot(ctx context.Context, logger *slog.Logger, u
 	})
 }
 
-func (b *chromiumBrowser) do(ctx context.Context, logger *slog.Logger, url string, options Options, tasks chromedp.Tasks) error {
+func (b *chromiumBrowser) do(ctx context.Context, logger *slog.Logger, url string, options Options, aggregate *networkAggregate, tasks chromedp.Tasks) error {
 	if !b.isStarted.Load() {
 		return errors.New("browser not started, cannot handle tasks")
 	}
@@ -396,6 +396,9 @@ func (b *chromiumBrowser) do(ctx context.Context, logger *slog.Logger, url strin
 	taskCtx, taskCancel := chromedp.NewContext(timeoutCtx)
 	defer taskCancel()
 
+	// Accumulate per-conversion network activity for telemetry.
+	listenForNetworkActivity(taskCtx, aggregate)
+
 	// We validate all other requests against our allowed / deny lists.
 	// If a request does not pass the validation, we make it fail. It also set
 	// the extra HTTP headers, if any.

pkg/modules/chromium/chromium.go

@@ -107,6 +107,8 @@ type Chromium struct {
 	queueWaitDurationCounter  metric.Float64Histogram
 	pdfOutputSizeCounter      metric.Int64Histogram
 	imageOutputSizeCounter    metric.Int64Histogram
+	networkRequestsCounter    metric.Int64Counter
+	networkBytesCounter       metric.Int64Histogram
 }
 
 // Options are the common options for all conversions.
@@ -633,6 +635,24 @@ func (mod *Chromium) Provision(ctx *gotenberg.Context) error {
 		return fmt.Errorf("create chromium.image.output.size histogram: %w", err)
 	}
 
+	mod.networkRequestsCounter, err = meter.Int64Counter(
+		"chromium.network.requests.total",
+		metric.WithDescription("Total number of network requests made during Chromium conversions"),
+		metric.WithUnit("{request}"),
+	)
+	if err != nil {
+		return fmt.Errorf("create chromium.network.requests.total counter: %w", err)
+	}
+
+	mod.networkBytesCounter, err = meter.Int64Histogram(
+		"chromium.network.bytes",
+		metric.WithDescription("Bytes fetched over the network during a Chromium conversion"),
+		metric.WithUnit("By"),
+	)
+	if err != nil {
+		return fmt.Errorf("create chromium.network.bytes histogram: %w", err)
+	}
+
 	return nil
 }
 
@@ -821,9 +841,10 @@ func (mod *Chromium) Pdf(ctx context.Context, logger *slog.Logger, url, outputPa
 	start := time.Now()
 	var conversionStart time.Time
 
+	aggregate := newNetworkAggregate()
 	err := mod.supervisor.Run(ctx, logger, func() error {
 		conversionStart = time.Now()
-		return mod.browser.pdf(ctx, logger, url, outputPath, options)
+		return mod.browser.pdf(ctx, logger, url, outputPath, options, aggregate)
 	})
 
 	end := time.Now()
@@ -865,6 +886,8 @@ func (mod *Chromium) Pdf(ctx context.Context, logger *slog.Logger, url, outputPa
 		attribute.String("status", status),
 	))
 
+	mod.recordNetwork(ctx, span, aggregate)
+
 	if err == nil {
 		if fileInfo, statErr := os.Stat(outputPath); statErr == nil {
 			mod.pdfOutputSizeCounter.Record(ctx, fileInfo.Size())
@@ -898,9 +921,10 @@ func (mod *Chromium) Screenshot(ctx context.Context, logger *slog.Logger, url, o
 	start := time.Now()
 	var conversionStart time.Time
 
+	aggregate := newNetworkAggregate()
 	err := mod.supervisor.Run(ctx, logger, func() error {
 		conversionStart = time.Now()
-		return mod.browser.screenshot(ctx, logger, url, outputPath, options)
+		return mod.browser.screenshot(ctx, logger, url, outputPath, options, aggregate)
 	})
 
 	end := time.Now()
@@ -942,6 +966,8 @@ func (mod *Chromium) Screenshot(ctx context.Context, logger *slog.Logger, url, o
 		attribute.String("status", status),
 	))
 
+	mod.recordNetwork(ctx, span, aggregate)
+
 	if err == nil {
 		if fileInfo, statErr := os.Stat(outputPath); statErr == nil {
 			mod.imageOutputSizeCounter.Record(ctx, fileInfo.Size())
@@ -956,6 +982,42 @@ func (mod *Chromium) Screenshot(ctx context.Context, logger *slog.Logger, url, o
 	return err
 }
 
+// recordNetwork lifts per-conversion network aggregates onto the span and the
+// network metrics. Counts are dimensioned by outcome and bytes feed a
+// histogram; both are recorded with the conversion context so the SDK attaches
+// trace exemplars. The heaviest resource URL is redacted before it lands on the
+// span event.
+func (mod *Chromium) recordNetwork(ctx context.Context, span trace.Span, aggregate *networkAggregate) {
+	if aggregate == nil {
+		return
+	}
+
+	stats := aggregate.snapshot()
+
+	span.SetAttributes(
+		attribute.Int64("gotenberg.chromium.resources.count", stats.requestCount),
+		attribute.Int64("gotenberg.chromium.resources.bytes_total", stats.bytesTotal),
+		attribute.Int64("gotenberg.chromium.resources.failed_count", stats.failedCount),
+		attribute.Int64("gotenberg.chromium.resources.unique_origins", stats.uniqueOrigins),
+	)
+
+	if stats.heaviestURL != "" {
+		span.AddEvent("chromium.heaviest_resource", trace.WithAttributes(
+			attribute.String("url", gotenberg.RedactURL(stats.heaviestURL)),
+			attribute.Int64("bytes", stats.heaviestBytes),
+		))
+	}
+
+	if ok := stats.requestCount - stats.failedCount; ok > 0 {
+		mod.networkRequestsCounter.Add(ctx, ok, metric.WithAttributes(attribute.String("outcome", "ok")))
+	}
+	if stats.failedCount > 0 {
+		mod.networkRequestsCounter.Add(ctx, stats.failedCount, metric.WithAttributes(attribute.String("outcome", "failed")))
+	}
+
+	mod.networkBytesCounter.Record(ctx, stats.bytesTotal)
+}
+
 // conversionInputAttrs derives low-cardinality input attributes for a
 // conversion span: the number of received files (when ctx is an [api.Context])
 // and the size of the local HTML input (when url is a file:// URL). Remote URL

pkg/modules/chromium/events.go

@@ -23,6 +23,26 @@ import (
 	"github.com/gotenberg/gotenberg/v8/pkg/gotenberg"
 )
 
+// listenForNetworkActivity accumulates per-conversion network activity into
+// aggregate from the always-on Network domain events. It is a no-op when
+// aggregate is nil.
+func listenForNetworkActivity(ctx context.Context, aggregate *networkAggregate) {
+	if aggregate == nil {
+		return
+	}
+
+	chromedp.ListenTarget(ctx, func(ev any) {
+		switch e := ev.(type) {
+		case *network.EventResponseReceived:
+			aggregate.onResponseReceived(e)
+		case *network.EventLoadingFinished:
+			aggregate.onLoadingFinished(e)
+		case *network.EventLoadingFailed:
+			aggregate.onLoadingFailed(e)
+		}
+	})
+}
+
 type eventRequestPausedOptions struct {
 	allowList, denyList []*regexp2.Regexp
 	denyPrivateIPs      bool

pkg/modules/chromium/mocks.go

@@ -24,16 +24,16 @@ func (api *ApiMock) Screenshot(ctx context.Context, logger *slog.Logger, url, ou
 // browserMock is a mock for the [browser] interface.
 type browserMock struct {
 	gotenberg.ProcessMock
-	pdfMock        func(ctx context.Context, logger *slog.Logger, url, outputPath string, options PdfOptions) error
-	screenshotMock func(ctx context.Context, logger *slog.Logger, url, outputPath string, options ScreenshotOptions) error
+	pdfMock        func(ctx context.Context, logger *slog.Logger, url, outputPath string, options PdfOptions, aggregate *networkAggregate) error
+	screenshotMock func(ctx context.Context, logger *slog.Logger, url, outputPath string, options ScreenshotOptions, aggregate *networkAggregate) error
 }
 
-func (b *browserMock) pdf(ctx context.Context, logger *slog.Logger, url, outputPath string, options PdfOptions) error {
-	return b.pdfMock(ctx, logger, url, outputPath, options)
+func (b *browserMock) pdf(ctx context.Context, logger *slog.Logger, url, outputPath string, options PdfOptions, aggregate *networkAggregate) error {
+	return b.pdfMock(ctx, logger, url, outputPath, options, aggregate)
 }
 
-func (b *browserMock) screenshot(ctx context.Context, logger *slog.Logger, url, outputPath string, options ScreenshotOptions) error {
-	return b.screenshotMock(ctx, logger, url, outputPath, options)
+func (b *browserMock) screenshot(ctx context.Context, logger *slog.Logger, url, outputPath string, options ScreenshotOptions, aggregate *networkAggregate) error {
+	return b.screenshotMock(ctx, logger, url, outputPath, options, aggregate)
 }
 
 // Interface guards.

pkg/modules/chromium/network_aggregate.go

@@ -0,0 +1,121 @@
+package chromium
+
+import (
+	"net/url"
+	"sync"
+
+	"github.com/chromedp/cdproto/network"
+)
+
+// maxTrackedOrigins bounds the distinct origins kept per conversion so a
+// pathological page cannot grow the set without limit.
+const maxTrackedOrigins = 64
+
+// networkAggregate accumulates per-conversion network activity from Chromium
+// DevTools events. It is safe for concurrent use by the chromedp event listener
+// goroutine and the conversion goroutine that reads the snapshot afterwards.
+type networkAggregate struct {
+	mu sync.Mutex
+
+	requestCount int64
+	bytesTotal   int64
+	failedCount  int64
+
+	origins        map[string]struct{}
+	requestURLByID map[network.RequestID]string
+
+	heaviestURL   string
+	heaviestBytes int64
+}
+
+// networkStats is an immutable snapshot of a [networkAggregate].
+type networkStats struct {
+	requestCount  int64
+	bytesTotal    int64
+	failedCount   int64
+	uniqueOrigins int64
+	heaviestURL   string
+	heaviestBytes int64
+}
+
+func newNetworkAggregate() *networkAggregate {
+	return &networkAggregate{
+		origins:        make(map[string]struct{}),
+		requestURLByID: make(map[network.RequestID]string),
+	}
+}
+
+// onResponseReceived records the response origin and remembers the URL for the
+// request id, so a later loading-finished event can attribute its bytes.
+func (a *networkAggregate) onResponseReceived(ev *network.EventResponseReceived) {
+	if ev == nil || ev.Response == nil {
+		return
+	}
+
+	a.mu.Lock()
+	defer a.mu.Unlock()
+
+	if origin := originOf(ev.Response.URL); origin != "" {
+		if _, ok := a.origins[origin]; !ok && len(a.origins) < maxTrackedOrigins {
+			a.origins[origin] = struct{}{}
+		}
+	}
+	a.requestURLByID[ev.RequestID] = ev.Response.URL
+}
+
+// onLoadingFinished records a successfully completed request and its size,
+// tracking the single heaviest resource.
+func (a *networkAggregate) onLoadingFinished(ev *network.EventLoadingFinished) {
+	if ev == nil {
+		return
+	}
+
+	a.mu.Lock()
+	defer a.mu.Unlock()
+
+	a.requestCount++
+
+	size := int64(ev.EncodedDataLength)
+	a.bytesTotal += size
+	if size > a.heaviestBytes {
+		a.heaviestBytes = size
+		a.heaviestURL = a.requestURLByID[ev.RequestID]
+	}
+}
+
+// onLoadingFailed records a request that failed to complete.
+func (a *networkAggregate) onLoadingFailed(ev *network.EventLoadingFailed) {
+	if ev == nil {
+		return
+	}
+
+	a.mu.Lock()
+	defer a.mu.Unlock()
+
+	a.requestCount++
+	a.failedCount++
+}
+
+func (a *networkAggregate) snapshot() networkStats {
+	a.mu.Lock()
+	defer a.mu.Unlock()
+
+	return networkStats{
+		requestCount:  a.requestCount,
+		bytesTotal:    a.bytesTotal,
+		failedCount:   a.failedCount,
+		uniqueOrigins: int64(len(a.origins)),
+		heaviestURL:   a.heaviestURL,
+		heaviestBytes: a.heaviestBytes,
+	}
+}
+
+// originOf returns the scheme://host of rawURL, or an empty string when it has
+// no host (for example data: or file: URLs).
+func originOf(rawURL string) string {
+	parsed, err := url.Parse(rawURL)
+	if err != nil || parsed.Host == "" {
+		return ""
+	}
+	return parsed.Scheme + "://" + parsed.Host
+}

pkg/modules/chromium/network_aggregate_test.go

@@ -0,0 +1,100 @@
+package chromium
+
+import (
+	"fmt"
+	"sync"
+	"testing"
+
+	"github.com/chromedp/cdproto/network"
+)
+
+func TestOriginOf(t *testing.T) {
+	for _, tc := range []struct {
+		raw  string
+		want string
+	}{
+		{"https://example.com/path?q=1", "https://example.com"},
+		{"http://cdn.example.com:8080/a.js", "http://cdn.example.com:8080"},
+		{"data:image/png;base64,AAAA", ""},
+		{"file:///tmp/index.html", ""},
+		{"not a url", ""},
+	} {
+		if got := originOf(tc.raw); got != tc.want {
+			t.Errorf("originOf(%q) = %q, want %q", tc.raw, got, tc.want)
+		}
+	}
+}
+
+func TestNetworkAggregate_Snapshot(t *testing.T) {
+	a := newNetworkAggregate()
+
+	a.onResponseReceived(&network.EventResponseReceived{
+		RequestID: "1",
+		Response:  &network.Response{URL: "https://example.com/a.js"},
+	})
+	a.onResponseReceived(&network.EventResponseReceived{
+		RequestID: "2",
+		Response:  &network.Response{URL: "https://cdn.example.com/b.png"},
+	})
+	// Duplicate origin must not grow the set.
+	a.onResponseReceived(&network.EventResponseReceived{
+		RequestID: "3",
+		Response:  &network.Response{URL: "https://example.com/c.css"},
+	})
+
+	a.onLoadingFinished(&network.EventLoadingFinished{RequestID: "1", EncodedDataLength: 100})
+	a.onLoadingFinished(&network.EventLoadingFinished{RequestID: "2", EncodedDataLength: 900})
+	a.onLoadingFailed(&network.EventLoadingFailed{RequestID: "3"})
+
+	got := a.snapshot()
+	if got.requestCount != 3 {
+		t.Errorf("requestCount = %d, want 3", got.requestCount)
+	}
+	if got.bytesTotal != 1000 {
+		t.Errorf("bytesTotal = %d, want 1000", got.bytesTotal)
+	}
+	if got.failedCount != 1 {
+		t.Errorf("failedCount = %d, want 1", got.failedCount)
+	}
+	if got.uniqueOrigins != 2 {
+		t.Errorf("uniqueOrigins = %d, want 2", got.uniqueOrigins)
+	}
+	if got.heaviestBytes != 900 || got.heaviestURL != "https://cdn.example.com/b.png" {
+		t.Errorf("heaviest = (%q, %d), want (%q, 900)", got.heaviestURL, got.heaviestBytes, "https://cdn.example.com/b.png")
+	}
+}
+
+func TestNetworkAggregate_OriginCap(t *testing.T) {
+	a := newNetworkAggregate()
+	for i := 0; i < maxTrackedOrigins+50; i++ {
+		a.onResponseReceived(&network.EventResponseReceived{
+			RequestID: network.RequestID(fmt.Sprintf("r%d", i)),
+			Response:  &network.Response{URL: fmt.Sprintf("https://host%d.example.com/x", i)},
+		})
+	}
+	if got := a.snapshot().uniqueOrigins; got != maxTrackedOrigins {
+		t.Errorf("uniqueOrigins = %d, want %d (capped)", got, maxTrackedOrigins)
+	}
+}
+
+func TestNetworkAggregate_ConcurrentSafe(t *testing.T) {
+	a := newNetworkAggregate()
+	var wg sync.WaitGroup
+	for i := 0; i < 100; i++ {
+		wg.Add(1)
+		go func(i int) {
+			defer wg.Done()
+			id := network.RequestID(fmt.Sprintf("r%d", i))
+			a.onResponseReceived(&network.EventResponseReceived{
+				RequestID: id,
+				Response:  &network.Response{URL: fmt.Sprintf("https://host%d.example.com/x", i)},
+			})
+			a.onLoadingFinished(&network.EventLoadingFinished{RequestID: id, EncodedDataLength: 10})
+		}(i)
+	}
+	wg.Wait()
+
+	if got := a.snapshot().requestCount; got != 100 {
+		t.Errorf("requestCount = %d, want 100", got)
+	}
+}