diff --git a/pkg/gotenberg/mocks.go b/pkg/gotenberg/mocks.go index dc25c9b1..32cbb50a 100644 --- a/pkg/gotenberg/mocks.go +++ b/pkg/gotenberg/mocks.go @@ -117,12 +117,13 @@ func (p *ProcessMock) Healthy(logger *slog.Logger) bool { // ProcessSupervisorMock is a mock for the [ProcessSupervisor] interface. type ProcessSupervisorMock struct { - LaunchMock func() error - ShutdownMock func() error - HealthyMock func() bool - RunMock func(ctx context.Context, logger *slog.Logger, task func() error) error - ReqQueueSizeMock func() int64 - RestartsCountMock func() int64 + LaunchMock func() error + ShutdownMock func() error + HealthyMock func() bool + RunMock func(ctx context.Context, logger *slog.Logger, task func() error) error + ReqQueueSizeMock func() int64 + ActiveTasksCountMock func() int64 + RestartsCountMock func() int64 } func (s *ProcessSupervisorMock) Launch() error { @@ -145,6 +146,10 @@ func (s *ProcessSupervisorMock) ReqQueueSize() int64 { return s.ReqQueueSizeMock() } +func (s *ProcessSupervisorMock) ActiveTasksCount() int64 { + return s.ActiveTasksCountMock() +} + func (s *ProcessSupervisorMock) RestartsCount() int64 { return s.RestartsCountMock() } diff --git a/pkg/gotenberg/supervisor.go b/pkg/gotenberg/supervisor.go index e63a35ef..e1495d6e 100644 --- a/pkg/gotenberg/supervisor.go +++ b/pkg/gotenberg/supervisor.go @@ -70,6 +70,9 @@ type ProcessSupervisor interface { // ReqQueueSize returns the current size of the request queue. ReqQueueSize() int64 + // ActiveTasksCount returns the current number of active tasks. + ActiveTasksCount() int64 + // RestartsCount returns the current number of restart. RestartsCount() int64 } @@ -343,6 +346,10 @@ func (s *processSupervisor) ReqQueueSize() int64 { return s.reqQueueSize.Load() } +func (s *processSupervisor) ActiveTasksCount() int64 { + return s.activeTasks.Load() +} + func (s *processSupervisor) RestartsCount() int64 { return s.restartsCounter.Load() } diff --git a/pkg/modules/chromium/chromium.go b/pkg/modules/chromium/chromium.go index 52ffb7d1..e0a8af1b 100644 --- a/pkg/modules/chromium/chromium.go +++ b/pkg/modules/chromium/chromium.go @@ -15,6 +15,7 @@ import ( "github.com/chromedp/cdproto/network" "github.com/dlclark/regexp2" flag "github.com/spf13/pflag" + "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/codes" "go.opentelemetry.io/otel/metric" @@ -97,6 +98,13 @@ type Chromium struct { browser browser supervisor gotenberg.ProcessSupervisor engine gotenberg.PdfEngine + + reqsCounter metric.Int64Counter + errsCounter metric.Int64Counter + conversionDurationCounter metric.Float64Histogram + queueWaitDurationCounter metric.Float64Histogram + pdfOutputSizeCounter metric.Int64Histogram + imageOutputSizeCounter metric.Int64Histogram } // Options are the common options for all conversions. @@ -519,6 +527,76 @@ func (mod *Chromium) Provision(ctx *gotenberg.Context) error { return fmt.Errorf("create requests queue size observable gauge: %w", err) } + _, err = meter.Int64ObservableGauge( + "chromium.requests.active", + metric.WithDescription("Current number of Chromium conversion requests actively being processed."), + metric.WithUnit("{request}"), + metric.WithInt64Callback(func(_ context.Context, o metric.Int64Observer) error { + val := mod.supervisor.ActiveTasksCount() + o.Observe(val) + return nil + }), + ) + if err != nil { + return fmt.Errorf("create requests active observable gauge: %w", err) + } + + mod.reqsCounter, err = meter.Int64Counter( + "chromium.requests.total", + metric.WithDescription("Total number of Chromium conversion requests."), + metric.WithUnit("{request}"), + ) + if err != nil { + return fmt.Errorf("create requests total counter: %w", err) + } + + mod.errsCounter, err = meter.Int64Counter( + "chromium.errors.total", + metric.WithDescription("Total number of Chromium errors."), + metric.WithUnit("{error}"), + ) + if err != nil { + return fmt.Errorf("create errors total counter: %w", err) + } + + mod.conversionDurationCounter, err = meter.Float64Histogram( + "chromium.conversion.duration", + metric.WithDescription("Duration of each HTML-to-PDF conversion."), + metric.WithUnit("s"), + metric.WithExplicitBucketBoundaries(0.5, 1, 2, 5, 10, 30, 60), + ) + if err != nil { + return fmt.Errorf("create conversion duration histogram: %w", err) + } + + mod.queueWaitDurationCounter, err = meter.Float64Histogram( + "chromium.queue.wait.duration", + metric.WithDescription("Time a request spends waiting in the queue before processing starts."), + metric.WithUnit("s"), + metric.WithExplicitBucketBoundaries(0.5, 1, 2, 5, 10, 30, 60), + ) + if err != nil { + return fmt.Errorf("create queue wait duration histogram: %w", err) + } + + mod.pdfOutputSizeCounter, err = meter.Int64Histogram( + "chromium.pdf.output.size", + metric.WithDescription("Size of the generated PDF files."), + metric.WithUnit("By"), + ) + if err != nil { + return fmt.Errorf("create pdf output size histogram: %w", err) + } + + mod.imageOutputSizeCounter, err = meter.Int64Histogram( + "chromium.image.output.size", + metric.WithDescription("Size of the generated image files."), + metric.WithUnit("By"), + ) + if err != nil { + return fmt.Errorf("create image output size histogram: %w", err) + } + return nil } @@ -671,12 +749,68 @@ func (mod *Chromium) Pdf(ctx context.Context, logger *slog.Logger, url, outputPa ctx, span := gotenberg.Tracer().Start(ctx, "Chromium.Pdf") defer span.End() + start := time.Now() + var conversionStart time.Time + err := mod.supervisor.Run(ctx, logger, func() error { + conversionStart = time.Now() return mod.browser.pdf(ctx, logger, url, outputPath, options) }) + + end := time.Now() + + status := "success" if err != nil { + if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) { + status = "timeout" + } else { + status = "error" + } + span.RecordError(err) span.SetStatus(codes.Error, err.Error()) + + reason := "unknown" + if errors.Is(err, context.DeadlineExceeded) { + reason = "timeout" + } else if errors.Is(err, context.Canceled) { + reason = "context_cancelled" + } else if errors.Is(err, ErrInvalidHttpStatusCode) || errors.Is(err, ErrInvalidResourceHttpStatusCode) || errors.Is(err, ErrLoadingFailed) || errors.Is(err, ErrResourceLoadingFailed) || errors.Is(err, ErrInvalidEvaluationExpression) || errors.Is(err, ErrInvalidSelectorQuery) { + reason = "invalid_input" + } else if errors.Is(err, gotenberg.ErrMaximumQueueSizeExceeded) || errors.Is(err, gotenberg.ErrProcessAlreadyRestarting) { + reason = "chromium_unavailable" + } + + mod.errsCounter.Add(ctx, 1, metric.WithAttributes( + attribute.String("reason", reason), + )) + } + + if !conversionStart.IsZero() { + waitDuration := conversionStart.Sub(start).Seconds() + conversionDuration := end.Sub(conversionStart).Seconds() + + mod.queueWaitDurationCounter.Record(ctx, waitDuration, metric.WithAttributes( + attribute.String("status", status), + )) + mod.conversionDurationCounter.Record(ctx, conversionDuration, metric.WithAttributes( + attribute.String("status", status), + )) + } else { + waitDuration := end.Sub(start).Seconds() + mod.queueWaitDurationCounter.Record(ctx, waitDuration, metric.WithAttributes( + attribute.String("status", status), + )) + } + + mod.reqsCounter.Add(ctx, 1, metric.WithAttributes( + attribute.String("status", status), + )) + + if err == nil { + if fileInfo, statErr := os.Stat(outputPath); statErr == nil { + mod.pdfOutputSizeCounter.Record(ctx, fileInfo.Size()) + } } return err @@ -688,12 +822,70 @@ func (mod *Chromium) Screenshot(ctx context.Context, logger *slog.Logger, url, o ctx, span := gotenberg.Tracer().Start(ctx, "Chromium.Screenshot") defer span.End() + start := time.Now() + var conversionStart time.Time + err := mod.supervisor.Run(ctx, logger, func() error { + conversionStart = time.Now() return mod.browser.screenshot(ctx, logger, url, outputPath, options) }) + + end := time.Now() + + status := "success" if err != nil { + if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) { + status = "timeout" + } else { + status = "error" + } + span.RecordError(err) span.SetStatus(codes.Error, err.Error()) + + reason := "unknown" + if errors.Is(err, context.DeadlineExceeded) { + reason = "timeout" + } else if errors.Is(err, context.Canceled) { + reason = "context_cancelled" + } else if errors.Is(err, ErrInvalidHttpStatusCode) || errors.Is(err, ErrInvalidResourceHttpStatusCode) || errors.Is(err, ErrLoadingFailed) || errors.Is(err, ErrResourceLoadingFailed) || errors.Is(err, ErrInvalidEvaluationExpression) || errors.Is(err, ErrInvalidSelectorQuery) { + reason = "invalid_input" + } else if errors.Is(err, gotenberg.ErrMaximumQueueSizeExceeded) { + reason = "chromium_maximum_queue_size_exceeded" + } else if errors.Is(err, gotenberg.ErrProcessAlreadyRestarting) { + reason = "chromium_unavailable" + } + + mod.errsCounter.Add(ctx, 1, metric.WithAttributes( + attribute.String("reason", reason), + )) + } + + if !conversionStart.IsZero() { + waitDuration := conversionStart.Sub(start).Seconds() + conversionDuration := end.Sub(conversionStart).Seconds() + + mod.queueWaitDurationCounter.Record(ctx, waitDuration, metric.WithAttributes( + attribute.String("status", status), + )) + mod.conversionDurationCounter.Record(ctx, conversionDuration, metric.WithAttributes( + attribute.String("status", status), + )) + } else { + waitDuration := end.Sub(start).Seconds() + mod.queueWaitDurationCounter.Record(ctx, waitDuration, metric.WithAttributes( + attribute.String("status", status), + )) + } + + mod.reqsCounter.Add(ctx, 1, metric.WithAttributes( + attribute.String("status", status), + )) + + if err == nil { + if fileInfo, statErr := os.Stat(outputPath); statErr == nil { + mod.imageOutputSizeCounter.Record(ctx, fileInfo.Size()) + } } return err diff --git a/pkg/modules/libreoffice/api/api.go b/pkg/modules/libreoffice/api/api.go index 00f2f08a..34b17263 100644 --- a/pkg/modules/libreoffice/api/api.go +++ b/pkg/modules/libreoffice/api/api.go @@ -13,6 +13,7 @@ import ( "github.com/alexliesenfeld/health" flag "github.com/spf13/pflag" + "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/codes" "go.opentelemetry.io/otel/metric" "go.uber.org/multierr" @@ -50,6 +51,12 @@ type Api struct { logger *slog.Logger libreOffice libreOffice supervisor gotenberg.ProcessSupervisor + + reqsCounter metric.Int64Counter + errsCounter metric.Int64Counter + conversionDurationCounter metric.Float64Histogram + queueWaitDurationCounter metric.Float64Histogram + pdfOutputSizeCounter metric.Int64Histogram } // Options gathers available options when converting a document to PDF. @@ -279,6 +286,67 @@ func (a *Api) Provision(ctx *gotenberg.Context) error { return fmt.Errorf("create requests queue size observable gauge: %w", err) } + _, err = meter.Int64ObservableGauge( + "libreoffice.requests.active", + metric.WithDescription("Current number of LibreOffice conversion requests actively being processed."), + metric.WithUnit("{request}"), + metric.WithInt64Callback(func(_ context.Context, o metric.Int64Observer) error { + val := a.supervisor.ActiveTasksCount() + o.Observe(val) + return nil + }), + ) + if err != nil { + return fmt.Errorf("create requests active observable gauge: %w", err) + } + + a.reqsCounter, err = meter.Int64Counter( + "libreoffice.requests.total", + metric.WithDescription("Total number of LibreOffice conversion requests."), + metric.WithUnit("{request}"), + ) + if err != nil { + return fmt.Errorf("create requests total counter: %w", err) + } + + a.errsCounter, err = meter.Int64Counter( + "libreoffice.errors.total", + metric.WithDescription("Total number of LibreOffice errors."), + metric.WithUnit("{error}"), + ) + if err != nil { + return fmt.Errorf("create errors total counter: %w", err) + } + + a.conversionDurationCounter, err = meter.Float64Histogram( + "libreoffice.conversion.duration", + metric.WithDescription("Duration of each PDF conversion."), + metric.WithUnit("s"), + metric.WithExplicitBucketBoundaries(0.5, 1, 2, 5, 10, 30, 60), + ) + if err != nil { + return fmt.Errorf("create conversion duration histogram: %w", err) + } + + a.queueWaitDurationCounter, err = meter.Float64Histogram( + "libreoffice.queue.wait.duration", + metric.WithDescription("Time a request spends waiting in the queue before processing starts."), + metric.WithUnit("s"), + metric.WithExplicitBucketBoundaries(0.5, 1, 2, 5, 10, 30, 60), + ) + if err != nil { + return fmt.Errorf("create queue wait duration histogram: %w", err) + } + + a.pdfOutputSizeCounter, err = meter.Int64Histogram( + "libreoffice.pdf.output.size", + metric.WithDescription("Size of the generated PDF files."), + metric.WithUnit("By"), + ) + if err != nil { + return fmt.Errorf("create pdf output size histogram: %w", err) + } + return nil } @@ -410,14 +478,70 @@ func (a *Api) Pdf(ctx context.Context, logger *slog.Logger, inputPath, outputPat ctx, span := gotenberg.Tracer().Start(ctx, "LibreOffice.Pdf") defer span.End() + start := time.Now() + var conversionStart time.Time + err := a.supervisor.Run(ctx, logger, func() error { + conversionStart = time.Now() return a.libreOffice.pdf(ctx, logger, inputPath, outputPath, options) }) + end := time.Now() + + status := "success" + if err != nil { + if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) { + status = "timeout" + } else { + status = "error" + } + } + + if !conversionStart.IsZero() { + waitDuration := conversionStart.Sub(start).Seconds() + conversionDuration := end.Sub(conversionStart).Seconds() + + a.queueWaitDurationCounter.Record(ctx, waitDuration, metric.WithAttributes( + attribute.String("status", status), + )) + a.conversionDurationCounter.Record(ctx, conversionDuration, metric.WithAttributes( + attribute.String("status", status), + )) + } else { + waitDuration := end.Sub(start).Seconds() + a.queueWaitDurationCounter.Record(ctx, waitDuration, metric.WithAttributes( + attribute.String("status", status), + )) + } + + a.reqsCounter.Add(ctx, 1, metric.WithAttributes( + attribute.String("status", status), + )) + if err == nil { + if fileInfo, statErr := os.Stat(outputPath); statErr == nil { + a.pdfOutputSizeCounter.Record(ctx, fileInfo.Size()) + } return nil } + reason := "unknown" + if errors.Is(err, context.DeadlineExceeded) { + reason = "timeout" + } else if errors.Is(err, context.Canceled) { + reason = "context_cancelled" + } else if errors.Is(err, ErrInvalidPdfFormats) { + reason = "invalid_input" + } else if errors.Is(err, gotenberg.ErrMaximumQueueSizeExceeded) { + reason = "libreoffice_maximum_queue_size_exceeded" + } else if errors.Is(err, gotenberg.ErrProcessAlreadyRestarting) { + reason = "libreoffice_unavailable" + } + + a.errsCounter.Add(ctx, 1, metric.WithAttributes( + attribute.String("reason", reason), + )) + // See https://github.com/gotenberg/gotenberg/issues/639. if errors.Is(err, ErrCoreDumped) { logger.DebugContext(ctx, fmt.Sprintf("got a '%s' error, retry conversion", err))