refactor(pdfengines): route fallback ops through a generic runWithFallback helper

pkg/modules/pdfengines/fallback_test.go

@@ -0,0 +1,171 @@
+package pdfengines
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"testing"
+
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/codes"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+	"go.opentelemetry.io/otel/sdk/trace/tracetest"
+
+	"github.com/gotenberg/gotenberg/v8/pkg/gotenberg"
+)
+
+func newFallbackRecorder(t *testing.T) *tracetest.SpanRecorder {
+	t.Helper()
+	recorder := tracetest.NewSpanRecorder()
+	provider := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(recorder))
+	previous := otel.GetTracerProvider()
+	otel.SetTracerProvider(provider)
+	t.Cleanup(func() { otel.SetTracerProvider(previous) })
+	return recorder
+}
+
+func findFallbackSpan(recorder *tracetest.SpanRecorder, name string) sdktrace.ReadOnlySpan {
+	for _, s := range recorder.Ended() {
+		if s.Name() == name {
+			return s
+		}
+	}
+	return nil
+}
+
+func wrapTest(err error) error { return fmt.Errorf("test op with multi PDF engines: %w", err) }
+
+func TestRunWithFallback_FirstSucceeds(t *testing.T) {
+	recorder := newFallbackRecorder(t)
+	engines := []gotenberg.PdfEngine{&gotenberg.PdfEngineMock{}, &gotenberg.PdfEngineMock{}}
+
+	calls := 0
+	got, err := runWithFallback(context.Background(), "pdfengines.Test", engines,
+		func(_ context.Context, _ gotenberg.PdfEngine) (string, error) {
+			calls++
+			return "ok", nil
+		}, wrapTest)
+
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if got != "ok" {
+		t.Errorf("got %q, want ok", got)
+	}
+	if calls != 1 {
+		t.Errorf("expected 1 engine call, got %d", calls)
+	}
+
+	span := findFallbackSpan(recorder, "pdfengines.Test")
+	if span.Status().Code != codes.Ok {
+		t.Errorf("status = %v, want Ok", span.Status().Code)
+	}
+	attrs := map[string]string{}
+	for _, kv := range span.Attributes() {
+		attrs[string(kv.Key)] = kv.Value.Emit()
+	}
+	if attrs["gotenberg.pdf_engine.attempts"] != "1" {
+		t.Errorf("attempts = %q, want 1", attrs["gotenberg.pdf_engine.attempts"])
+	}
+	if attrs["gotenberg.pdf_engine.selected"] == "" {
+		t.Error("expected a selected engine attribute")
+	}
+}
+
+func TestRunWithFallback_SecondSucceeds(t *testing.T) {
+	recorder := newFallbackRecorder(t)
+	engines := []gotenberg.PdfEngine{&gotenberg.PdfEngineMock{}, &gotenberg.PdfEngineMock{}}
+
+	calls := 0
+	got, err := runWithFallback(context.Background(), "pdfengines.Test", engines,
+		func(_ context.Context, _ gotenberg.PdfEngine) (string, error) {
+			calls++
+			if calls == 1 {
+				return "", errors.New("first engine failed")
+			}
+			return "ok", nil
+		}, wrapTest)
+
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if got != "ok" || calls != 2 {
+		t.Errorf("got %q after %d calls, want ok after 2", got, calls)
+	}
+
+	span := findFallbackSpan(recorder, "pdfengines.Test")
+	var failedEvents int
+	for _, e := range span.Events() {
+		if e.Name == "pdf_engine.attempt_failed" {
+			failedEvents++
+		}
+	}
+	if failedEvents != 1 {
+		t.Errorf("expected 1 attempt_failed event, got %d", failedEvents)
+	}
+	for _, kv := range span.Attributes() {
+		if string(kv.Key) == "gotenberg.pdf_engine.attempts" && kv.Value.Emit() != "2" {
+			t.Errorf("attempts = %q, want 2", kv.Value.Emit())
+		}
+	}
+}
+
+func TestRunWithFallback_AllFail(t *testing.T) {
+	recorder := newFallbackRecorder(t)
+	engines := []gotenberg.PdfEngine{&gotenberg.PdfEngineMock{}, &gotenberg.PdfEngineMock{}}
+
+	sentinel := errors.New("engine failed")
+	_, err := runWithFallback(context.Background(), "pdfengines.Test", engines,
+		func(_ context.Context, _ gotenberg.PdfEngine) (string, error) {
+			return "", sentinel
+		}, wrapTest)
+
+	if err == nil {
+		t.Fatal("expected an error when all engines fail")
+	}
+	if !errors.Is(err, sentinel) {
+		t.Errorf("expected the joined engine error to be wrapped, got %v", err)
+	}
+
+	span := findFallbackSpan(recorder, "pdfengines.Test")
+	if span.Status().Code != codes.Error {
+		t.Errorf("status = %v, want Error", span.Status().Code)
+	}
+}
+
+func TestRunWithFallback_ZeroEngines(t *testing.T) {
+	newFallbackRecorder(t)
+	_, err := runWithFallback(context.Background(), "pdfengines.Test", nil,
+		func(_ context.Context, _ gotenberg.PdfEngine) (string, error) {
+			return "ok", nil
+		}, wrapTest)
+	if err == nil {
+		t.Error("expected an error with no engines")
+	}
+}
+
+func TestRunWithFallback_ContextDone(t *testing.T) {
+	recorder := newFallbackRecorder(t)
+	engines := []gotenberg.PdfEngine{&gotenberg.PdfEngineMock{}}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+
+	release := make(chan struct{})
+	t.Cleanup(func() { close(release) })
+
+	_, err := runWithFallback(ctx, "pdfengines.Test", engines,
+		func(_ context.Context, _ gotenberg.PdfEngine) (string, error) {
+			<-release // never returns during the call, forcing the ctx.Done branch
+			return "", nil
+		}, wrapTest)
+
+	if !errors.Is(err, context.Canceled) {
+		t.Errorf("expected context.Canceled, got %v", err)
+	}
+
+	span := findFallbackSpan(recorder, "pdfengines.Test")
+	if span.Status().Code != codes.Error {
+		t.Errorf("status = %v, want Error (the cancellation must mark the span)", span.Status().Code)
+	}
+}

pkg/modules/pdfengines/multi.go

@@ -5,8 +5,8 @@ import (
 	"errors"
 	"fmt"
 	"log/slog"
-	"sync"
 
+	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/codes"
 	"go.opentelemetry.io/otel/trace"
 
@@ -64,583 +64,260 @@ func newMultiPdfEngines(
 	}
 }
 
-// Merge combines multiple PDF files into a single document using the first
+// engineName returns the module ID of a PDF engine for telemetry, falling back
-// available engine that supports PDF merging.
+// to its type name when it does not expose a descriptor.
-//
+func engineName(engine gotenberg.PdfEngine) string {
-//nolint:dupl
+	if module, ok := engine.(gotenberg.Module); ok {
-func (multi *multiPdfEngines) Merge(ctx context.Context, logger *slog.Logger, inputPaths []string, outputPath string) error {
+		return module.Descriptor().ID
-	tracer := gotenberg.Tracer()
+	}
-	ctx, span := tracer.Start(ctx, "pdfengines.Merge", trace.WithSpanKind(trace.SpanKindInternal))
+	return fmt.Sprintf("%T", engine)
+}
+
+// runWithFallback runs op against each engine in order and returns the first
+// success. It wraps the attempts in a pdfengines span, records the winning
+// engine and attempt count, emits a pdf_engine.attempt_failed event for each
+// failed engine, and joins all engine errors on total failure. A context
+// cancellation marks the span as errored too. wrap applies the op-specific
+// final error message.
+func runWithFallback[T any](
+	ctx context.Context,
+	spanName string,
+	engines []gotenberg.PdfEngine,
+	op func(ctx context.Context, engine gotenberg.PdfEngine) (T, error),
+	wrap func(err error) error,
+) (T, error) {
+	var zero T
+
+	ctx, span := gotenberg.Tracer().Start(ctx, spanName, trace.WithSpanKind(trace.SpanKindInternal))
 	defer span.End()
 
-	var err error
+	type attemptResult struct {
-	errChan := make(chan error, 1)
+		value T
+		err   error
+	}
+
+	var joined error
+	for attempt, engine := range engines {
+		resultChan := make(chan attemptResult, 1)
 
-	for _, engine := range multi.mergeEngines {
 		go func(engine gotenberg.PdfEngine) {
-			errChan <- engine.Merge(ctx, logger, inputPaths, outputPath)
+			value, err := op(ctx, engine)
+			resultChan <- attemptResult{value: value, err: err}
 		}(engine)
 
 		select {
-		case mergeErr := <-errChan:
+		case result := <-resultChan:
-			if mergeErr != nil {
+			if result.err == nil {
-				err = errors.Join(err, mergeErr)
+				span.SetAttributes(
-			} else {
+					attribute.String("gotenberg.pdf_engine.selected", engineName(engine)),
+					attribute.Int("gotenberg.pdf_engine.attempts", attempt+1),
+				)
 				span.SetStatus(codes.Ok, "")
-				return nil
+				return result.value, nil
 			}
+
+			joined = errors.Join(joined, result.err)
+			span.AddEvent("pdf_engine.attempt_failed", trace.WithAttributes(
+				attribute.String("engine", engineName(engine)),
+			))
 		case <-ctx.Done():
-			return ctx.Err()
+			err := ctx.Err()
+			span.RecordError(err)
+			span.SetStatus(codes.Error, err.Error())
+			return zero, err
 		}
 	}
 
-	err = fmt.Errorf("merge PDFs with multi PDF engines: %w", err)
+	err := wrap(joined)
 	span.RecordError(err)
 	span.SetStatus(codes.Error, err.Error())
 
+	return zero, err
+}
+
+// runWithFallbackVoid adapts [runWithFallback] for operations that return no
+// value beyond an error.
+func runWithFallbackVoid(
+	ctx context.Context,
+	spanName string,
+	engines []gotenberg.PdfEngine,
+	op func(ctx context.Context, engine gotenberg.PdfEngine) error,
+	wrap func(err error) error,
+) error {
+	_, err := runWithFallback(ctx, spanName, engines,
+		func(ctx context.Context, engine gotenberg.PdfEngine) (struct{}, error) {
+			return struct{}{}, op(ctx, engine)
+		},
+		wrap,
+	)
 	return err
 }
 
-type splitResult struct {
+// Merge combines multiple PDF files into a single document using the first
-	outputPaths []string
+// available engine that supports PDF merging.
-	err         error
+func (multi *multiPdfEngines) Merge(ctx context.Context, logger *slog.Logger, inputPaths []string, outputPath string) error {
+	return runWithFallbackVoid(ctx, "pdfengines.Merge", multi.mergeEngines,
+		func(ctx context.Context, engine gotenberg.PdfEngine) error {
+			return engine.Merge(ctx, logger, inputPaths, outputPath)
+		},
+		func(err error) error { return fmt.Errorf("merge PDFs with multi PDF engines: %w", err) },
+	)
 }
 
 // Split divides the PDF into separate pages using the first available engine
 // that supports PDF splitting.
 func (multi *multiPdfEngines) Split(ctx context.Context, logger *slog.Logger, mode gotenberg.SplitMode, inputPath, outputDirPath string) ([]string, error) {
-	tracer := gotenberg.Tracer()
+	return runWithFallback(ctx, "pdfengines.Split", multi.splitEngines,
-	ctx, span := tracer.Start(ctx, "pdfengines.Split", trace.WithSpanKind(trace.SpanKindInternal))
+		func(ctx context.Context, engine gotenberg.PdfEngine) ([]string, error) {
-	defer span.End()
+			return engine.Split(ctx, logger, mode, inputPath, outputDirPath)
-
+		},
-	var err error
+		func(err error) error { return fmt.Errorf("split PDF with multi PDF engines: %w", err) },
-	var mu sync.Mutex // to safely append errors.
+	)
-
-	for _, engine := range multi.splitEngines {
-		resultChan := make(chan splitResult, 1)
-
-		go func(engine gotenberg.PdfEngine) {
-			outputPaths, err := engine.Split(ctx, logger, mode, inputPath, outputDirPath)
-			resultChan <- splitResult{outputPaths: outputPaths, err: err}
-		}(engine)
-
-		select {
-		case result := <-resultChan:
-			if result.err != nil {
-				mu.Lock()
-				err = errors.Join(err, result.err)
-				mu.Unlock()
-			} else {
-				span.SetStatus(codes.Ok, "")
-				return result.outputPaths, nil
-			}
-		case <-ctx.Done():
-			return nil, ctx.Err()
-		}
-	}
-
-	err = fmt.Errorf("split PDF with multi PDF engines: %w", err)
-	span.RecordError(err)
-	span.SetStatus(codes.Error, err.Error())
-
-	return nil, err
 }
 
 // Flatten merges existing annotation appearances with page content using the
 // first available engine that supports flattening.
 func (multi *multiPdfEngines) Flatten(ctx context.Context, logger *slog.Logger, inputPath string) error {
-	tracer := gotenberg.Tracer()
+	return runWithFallbackVoid(ctx, "pdfengines.Flatten", multi.flattenEngines,
-	ctx, span := tracer.Start(ctx, "pdfengines.Flatten", trace.WithSpanKind(trace.SpanKindInternal))
+		func(ctx context.Context, engine gotenberg.PdfEngine) error {
-	defer span.End()
+			return engine.Flatten(ctx, logger, inputPath)
-
+		},
-	var err error
+		func(err error) error { return fmt.Errorf("flatten PDF with multi PDF engines: %w", err) },
-	errChan := make(chan error, 1)
+	)
-
-	for _, engine := range multi.flattenEngines {
-		go func(engine gotenberg.PdfEngine) {
-			errChan <- engine.Flatten(ctx, logger, inputPath)
-		}(engine)
-
-		select {
-		case mergeErr := <-errChan:
-			if mergeErr != nil {
-				err = errors.Join(err, mergeErr)
-			} else {
-				span.SetStatus(codes.Ok, "")
-				return nil
-			}
-		case <-ctx.Done():
-			return ctx.Err()
-		}
-	}
-
-	err = fmt.Errorf("flatten PDF with multi PDF engines: %w", err)
-	span.RecordError(err)
-	span.SetStatus(codes.Error, err.Error())
-
-	return err
 }
 
 // Convert transforms the given PDF to a specific PDF format using the first
 // available engine that supports PDF conversion.
 func (multi *multiPdfEngines) Convert(ctx context.Context, logger *slog.Logger, formats gotenberg.PdfFormats, inputPath, outputPath string) error {
-	tracer := gotenberg.Tracer()
+	return runWithFallbackVoid(ctx, "pdfengines.Convert", multi.convertEngines,
-	ctx, span := tracer.Start(ctx, "pdfengines.Convert", trace.WithSpanKind(trace.SpanKindInternal))
+		func(ctx context.Context, engine gotenberg.PdfEngine) error {
-	defer span.End()
+			return engine.Convert(ctx, logger, formats, inputPath, outputPath)
-
+		},
-	var err error
+		func(err error) error {
-	errChan := make(chan error, 1)
+			return fmt.Errorf("convert PDF to '%+v' with multi PDF engines: %w", formats, err)
-
+		},
-	for _, engine := range multi.convertEngines {
+	)
-		go func(engine gotenberg.PdfEngine) {
-			errChan <- engine.Convert(ctx, logger, formats, inputPath, outputPath)
-		}(engine)
-
-		select {
-		case mergeErr := <-errChan:
-			if mergeErr != nil {
-				err = errors.Join(err, mergeErr)
-			} else {
-				span.SetStatus(codes.Ok, "")
-				return nil
-			}
-		case <-ctx.Done():
-			return ctx.Err()
-		}
-	}
-
-	err = fmt.Errorf("convert PDF to '%+v' with multi PDF engines: %w", formats, err)
-	span.RecordError(err)
-	span.SetStatus(codes.Error, err.Error())
-
-	return err
-}
-
-type readMetadataResult struct {
-	metadata map[string]any
-	err      error
 }
 
 // ReadMetadata extracts metadata from a PDF file using the first available
 // engine that supports metadata reading.
-//
-//nolint:dupl
 func (multi *multiPdfEngines) ReadMetadata(ctx context.Context, logger *slog.Logger, inputPath string) (map[string]any, error) {
-	tracer := gotenberg.Tracer()
+	return runWithFallback(ctx, "pdfengines.ReadMetadata", multi.readMetadataEngines,
-	ctx, span := tracer.Start(ctx, "pdfengines.ReadMetadata", trace.WithSpanKind(trace.SpanKindInternal))
+		func(ctx context.Context, engine gotenberg.PdfEngine) (map[string]any, error) {
-	defer span.End()
+			return engine.ReadMetadata(ctx, logger, inputPath)
-
+		},
-	var err error
+		func(err error) error { return fmt.Errorf("read PDF metadata with multi PDF engines: %w", err) },
-	var mu sync.Mutex // to safely append errors.
+	)
-
-	for _, engine := range multi.readMetadataEngines {
-		resultChan := make(chan readMetadataResult, 1)
-
-		go func(engine gotenberg.PdfEngine) {
-			metadata, err := engine.ReadMetadata(ctx, logger, inputPath)
-			resultChan <- readMetadataResult{metadata: metadata, err: err}
-		}(engine)
-
-		select {
-		case result := <-resultChan:
-			if result.err != nil {
-				mu.Lock()
-				err = errors.Join(err, result.err)
-				mu.Unlock()
-			} else {
-				span.SetStatus(codes.Ok, "")
-				return result.metadata, nil
-			}
-		case <-ctx.Done():
-			return nil, ctx.Err()
-		}
-	}
-
-	err = fmt.Errorf("read PDF metadata with multi PDF engines: %w", err)
-	span.RecordError(err)
-	span.SetStatus(codes.Error, err.Error())
-
-	return nil, err
 }
 
 // WriteMetadata embeds metadata into a PDF file using the first available
 // engine that supports metadata writing.
 func (multi *multiPdfEngines) WriteMetadata(ctx context.Context, logger *slog.Logger, metadata map[string]any, inputPath string) error {
-	tracer := gotenberg.Tracer()
+	return runWithFallbackVoid(ctx, "pdfengines.WriteMetadata", multi.writeMetadataEngines,
-	ctx, span := tracer.Start(ctx, "pdfengines.WriteMetadata", trace.WithSpanKind(trace.SpanKindInternal))
+		func(ctx context.Context, engine gotenberg.PdfEngine) error {
-	defer span.End()
+			return engine.WriteMetadata(ctx, logger, metadata, inputPath)
-
+		},
-	var err error
+		func(err error) error { return fmt.Errorf("write PDF metadata with multi PDF engines: %w", err) },
-	errChan := make(chan error, 1)
+	)
-
-	for _, engine := range multi.writeMetadataEngines {
-		go func(engine gotenberg.PdfEngine) {
-			errChan <- engine.WriteMetadata(ctx, logger, metadata, inputPath)
-		}(engine)
-
-		select {
-		case writeMetadataErr := <-errChan:
-			if writeMetadataErr != nil {
-				err = errors.Join(err, writeMetadataErr)
-			} else {
-				span.SetStatus(codes.Ok, "")
-				return nil
-			}
-		case <-ctx.Done():
-			return ctx.Err()
-		}
-	}
-
-	err = fmt.Errorf("write PDF metadata with multi PDF engines: %w", err)
-	span.RecordError(err)
-	span.SetStatus(codes.Error, err.Error())
-
-	return err
-}
-
-type pageCountResult struct {
-	pageCount int
-	err       error
 }
 
 // PageCount returns the number of pages in a PDF file using the first available
 // engine that supports metadata reading.
 func (multi *multiPdfEngines) PageCount(ctx context.Context, logger *slog.Logger, inputPath string) (int, error) {
-	tracer := gotenberg.Tracer()
+	return runWithFallback(ctx, "pdfengines.PageCount", multi.readMetadataEngines,
-	ctx, span := tracer.Start(ctx, "pdfengines.PageCount", trace.WithSpanKind(trace.SpanKindInternal))
+		func(ctx context.Context, engine gotenberg.PdfEngine) (int, error) {
-	defer span.End()
+			return engine.PageCount(ctx, logger, inputPath)
-
+		},
-	var err error
+		func(err error) error { return fmt.Errorf("page count with multi PDF engines: %w", err) },
-	var mu sync.Mutex // to safely append errors.
+	)
-
-	for _, engine := range multi.readMetadataEngines {
-		resultChan := make(chan pageCountResult, 1)
-
-		go func(engine gotenberg.PdfEngine) {
-			pageCount, err := engine.PageCount(ctx, logger, inputPath)
-			resultChan <- pageCountResult{pageCount: pageCount, err: err}
-		}(engine)
-
-		select {
-		case result := <-resultChan:
-			if result.err != nil {
-				mu.Lock()
-				err = errors.Join(err, result.err)
-				mu.Unlock()
-			} else {
-				span.SetStatus(codes.Ok, "")
-				return result.pageCount, nil
-			}
-		case <-ctx.Done():
-			return 0, ctx.Err()
-		}
-	}
-
-	err = fmt.Errorf("page count with multi PDF engines: %w", err)
-	span.RecordError(err)
-	span.SetStatus(codes.Error, err.Error())
-
-	return 0, err
-}
-
-type readBookmarksResult struct {
-	bookmarks []gotenberg.Bookmark
-	err       error
 }
 
 // ReadBookmarks reads bookmarks from a PDF file using the first available
 // engine that supports bookmarks reading.
-//
-//nolint:dupl
 func (multi *multiPdfEngines) ReadBookmarks(ctx context.Context, logger *slog.Logger, inputPath string) ([]gotenberg.Bookmark, error) {
-	tracer := gotenberg.Tracer()
+	return runWithFallback(ctx, "pdfengines.ReadBookmarks", multi.readBookmarksEngines,
-	ctx, span := tracer.Start(ctx, "pdfengines.ReadBookmarks", trace.WithSpanKind(trace.SpanKindInternal))
+		func(ctx context.Context, engine gotenberg.PdfEngine) ([]gotenberg.Bookmark, error) {
-	defer span.End()
+			return engine.ReadBookmarks(ctx, logger, inputPath)
-
+		},
-	var err error
+		func(err error) error { return fmt.Errorf("read PDF bookmarks with multi PDF engines: %w", err) },
-	var mu sync.Mutex // to safely append errors.
+	)
-
-	for _, engine := range multi.readBookmarksEngines {
-		resultChan := make(chan readBookmarksResult, 1)
-
-		go func(engine gotenberg.PdfEngine) {
-			bookmarks, err := engine.ReadBookmarks(ctx, logger, inputPath)
-			resultChan <- readBookmarksResult{bookmarks: bookmarks, err: err}
-		}(engine)
-
-		select {
-		case result := <-resultChan:
-			if result.err != nil {
-				mu.Lock()
-				err = errors.Join(err, result.err)
-				mu.Unlock()
-			} else {
-				span.SetStatus(codes.Ok, "")
-				return result.bookmarks, nil
-			}
-		case <-ctx.Done():
-			return nil, ctx.Err()
-		}
-	}
-
-	err = fmt.Errorf("read PDF bookmarks with multi PDF engines: %w", err)
-	span.RecordError(err)
-	span.SetStatus(codes.Error, err.Error())
-
-	return nil, err
 }
 
 // WriteBookmarks adds a document outline (bookmarks) to a PDF file using the
 // first available engine that supports bookmarks writing.
 func (multi *multiPdfEngines) WriteBookmarks(ctx context.Context, logger *slog.Logger, inputPath string, bookmarks []gotenberg.Bookmark) error {
-	tracer := gotenberg.Tracer()
+	return runWithFallbackVoid(ctx, "pdfengines.WriteBookmarks", multi.writeBookmarksEngines,
-	ctx, span := tracer.Start(ctx, "pdfengines.WriteBookmarks", trace.WithSpanKind(trace.SpanKindInternal))
+		func(ctx context.Context, engine gotenberg.PdfEngine) error {
-	defer span.End()
+			return engine.WriteBookmarks(ctx, logger, inputPath, bookmarks)
-
+		},
-	var err error
+		func(err error) error { return fmt.Errorf("write PDF bookmarks with multi PDF engines: %w", err) },
-	errChan := make(chan error, 1)
+	)
-
-	for _, engine := range multi.writeBookmarksEngines {
-		go func(engine gotenberg.PdfEngine) {
-			errChan <- engine.WriteBookmarks(ctx, logger, inputPath, bookmarks)
-		}(engine)
-
-		select {
-		case writeBookmarksErr := <-errChan:
-			if writeBookmarksErr != nil {
-				err = errors.Join(err, writeBookmarksErr)
-			} else {
-				span.SetStatus(codes.Ok, "")
-				return nil
-			}
-		case <-ctx.Done():
-			return ctx.Err()
-		}
-	}
-
-	err = fmt.Errorf("write PDF bookmarks with multi PDF engines: %w", err)
-	span.RecordError(err)
-	span.SetStatus(codes.Error, err.Error())
-
-	return err
 }
 
 // Encrypt adds password protection to a PDF file using the first available
 // engine that supports password protection.
 func (multi *multiPdfEngines) Encrypt(ctx context.Context, logger *slog.Logger, inputPath, userPassword, ownerPassword string) error {
-	tracer := gotenberg.Tracer()
+	return runWithFallbackVoid(ctx, "pdfengines.Encrypt", multi.passwordEngines,
-	ctx, span := tracer.Start(ctx, "pdfengines.Encrypt", trace.WithSpanKind(trace.SpanKindInternal))
+		func(ctx context.Context, engine gotenberg.PdfEngine) error {
-	defer span.End()
+			return engine.Encrypt(ctx, logger, inputPath, userPassword, ownerPassword)
-
+		},
-	var err error
+		func(err error) error { return fmt.Errorf("encrypt PDF using multi PDF engines: %w", err) },
-	errChan := make(chan error, 1)
+	)
-
-	for _, engine := range multi.passwordEngines {
-		go func(engine gotenberg.PdfEngine) {
-			errChan <- engine.Encrypt(ctx, logger, inputPath, userPassword, ownerPassword)
-		}(engine)
-
-		select {
-		case protectErr := <-errChan:
-			if protectErr != nil {
-				err = errors.Join(err, protectErr)
-			} else {
-				span.SetStatus(codes.Ok, "")
-				return nil
-			}
-		case <-ctx.Done():
-			return ctx.Err()
-		}
-	}
-
-	err = fmt.Errorf("encrypt PDF using multi PDF engines: %w", err)
-	span.RecordError(err)
-	span.SetStatus(codes.Error, err.Error())
-
-	return err
 }
 
-// EmbedFiles embeds files into a PDF using the first available
+// EmbedFiles embeds files into a PDF using the first available engine that
-// engine that supports file embedding.
+// supports file embedding.
-//
-//nolint:dupl
 func (multi *multiPdfEngines) EmbedFiles(ctx context.Context, logger *slog.Logger, filePaths []string, inputPath string) error {
-	tracer := gotenberg.Tracer()
+	return runWithFallbackVoid(ctx, "pdfengines.EmbedFiles", multi.embedEngines,
-	ctx, span := tracer.Start(ctx, "pdfengines.EmbedFiles", trace.WithSpanKind(trace.SpanKindInternal))
+		func(ctx context.Context, engine gotenberg.PdfEngine) error {
-	defer span.End()
+			return engine.EmbedFiles(ctx, logger, filePaths, inputPath)
-
+		},
-	var err error
+		func(err error) error { return fmt.Errorf("embed files into PDF using multi PDF engines: %w", err) },
-	errChan := make(chan error, 1)
+	)
-
-	for _, engine := range multi.embedEngines {
-		go func(engine gotenberg.PdfEngine) {
-			errChan <- engine.EmbedFiles(ctx, logger, filePaths, inputPath)
-		}(engine)
-
-		select {
-		case embedErr := <-errChan:
-			if embedErr != nil {
-				err = errors.Join(err, embedErr)
-			} else {
-				span.SetStatus(codes.Ok, "")
-				return nil
-			}
-		case <-ctx.Done():
-			return ctx.Err()
-		}
-	}
-
-	err = fmt.Errorf("embed files into PDF using multi PDF engines: %w", err)
-	span.RecordError(err)
-	span.SetStatus(codes.Error, err.Error())
-
-	return err
 }
 
 // Watermark applies a watermark (behind page content) to a PDF file using the
 // first available engine that supports watermarking.
-//
-//nolint:dupl
 func (multi *multiPdfEngines) Watermark(ctx context.Context, logger *slog.Logger, inputPath string, stamp gotenberg.Stamp) error {
-	tracer := gotenberg.Tracer()
+	return runWithFallbackVoid(ctx, "pdfengines.Watermark", multi.watermarkEngines,
-	ctx, span := tracer.Start(ctx, "pdfengines.Watermark", trace.WithSpanKind(trace.SpanKindInternal))
+		func(ctx context.Context, engine gotenberg.PdfEngine) error {
-	defer span.End()
+			return engine.Watermark(ctx, logger, inputPath, stamp)
-
+		},
-	var err error
+		func(err error) error { return fmt.Errorf("watermark PDF with multi PDF engines: %w", err) },
-	errChan := make(chan error, 1)
+	)
-
-	for _, engine := range multi.watermarkEngines {
-		go func(engine gotenberg.PdfEngine) {
-			errChan <- engine.Watermark(ctx, logger, inputPath, stamp)
-		}(engine)
-
-		select {
-		case watermarkErr := <-errChan:
-			if watermarkErr != nil {
-				err = errors.Join(err, watermarkErr)
-			} else {
-				span.SetStatus(codes.Ok, "")
-				return nil
-			}
-		case <-ctx.Done():
-			return ctx.Err()
-		}
-	}
-
-	err = fmt.Errorf("watermark PDF with multi PDF engines: %w", err)
-	span.RecordError(err)
-	span.SetStatus(codes.Error, err.Error())
-
-	return err
 }
 
-// Stamp applies a stamp (on top of page content) to a PDF file using the
+// Stamp applies a stamp (on top of page content) to a PDF file using the first
-// first available engine that supports stamping.
+// available engine that supports stamping.
-//
-//nolint:dupl
 func (multi *multiPdfEngines) Stamp(ctx context.Context, logger *slog.Logger, inputPath string, stamp gotenberg.Stamp) error {
-	tracer := gotenberg.Tracer()
+	return runWithFallbackVoid(ctx, "pdfengines.Stamp", multi.stampEngines,
-	ctx, span := tracer.Start(ctx, "pdfengines.Stamp", trace.WithSpanKind(trace.SpanKindInternal))
+		func(ctx context.Context, engine gotenberg.PdfEngine) error {
-	defer span.End()
+			return engine.Stamp(ctx, logger, inputPath, stamp)
-
+		},
-	var err error
+		func(err error) error { return fmt.Errorf("stamp PDF with multi PDF engines: %w", err) },
-	errChan := make(chan error, 1)
+	)
-
-	for _, engine := range multi.stampEngines {
-		go func(engine gotenberg.PdfEngine) {
-			errChan <- engine.Stamp(ctx, logger, inputPath, stamp)
-		}(engine)
-
-		select {
-		case stampErr := <-errChan:
-			if stampErr != nil {
-				err = errors.Join(err, stampErr)
-			} else {
-				span.SetStatus(codes.Ok, "")
-				return nil
-			}
-		case <-ctx.Done():
-			return ctx.Err()
-		}
-	}
-
-	err = fmt.Errorf("stamp PDF with multi PDF engines: %w", err)
-	span.RecordError(err)
-	span.SetStatus(codes.Error, err.Error())
-
-	return err
 }
 
 // Rotate rotates pages of a PDF file using the first available engine that
 // supports rotation.
 func (multi *multiPdfEngines) Rotate(ctx context.Context, logger *slog.Logger, inputPath string, angle int, pages string) error {
-	tracer := gotenberg.Tracer()
+	return runWithFallbackVoid(ctx, "pdfengines.Rotate", multi.rotateEngines,
-	ctx, span := tracer.Start(ctx, "pdfengines.Rotate", trace.WithSpanKind(trace.SpanKindInternal))
+		func(ctx context.Context, engine gotenberg.PdfEngine) error {
-	defer span.End()
+			return engine.Rotate(ctx, logger, inputPath, angle, pages)
-
+		},
-	var err error
+		func(err error) error { return fmt.Errorf("rotate PDF with multi PDF engines: %w", err) },
-	errChan := make(chan error, 1)
+	)
-
-	for _, engine := range multi.rotateEngines {
-		go func(engine gotenberg.PdfEngine) {
-			errChan <- engine.Rotate(ctx, logger, inputPath, angle, pages)
-		}(engine)
-
-		select {
-		case rotateErr := <-errChan:
-			if rotateErr != nil {
-				err = errors.Join(err, rotateErr)
-			} else {
-				span.SetStatus(codes.Ok, "")
-				return nil
-			}
-		case <-ctx.Done():
-			return ctx.Err()
-		}
-	}
-
-	err = fmt.Errorf("rotate PDF with multi PDF engines: %w", err)
-	span.RecordError(err)
-	span.SetStatus(codes.Error, err.Error())
-
-	return err
 }
 
 // EmbedFilesMetadata sets metadata on embedded files using the first available
 // engine that supports it.
-//
-//nolint:dupl
 func (multi *multiPdfEngines) EmbedFilesMetadata(ctx context.Context, logger *slog.Logger, metadata map[string]map[string]string, inputPath string) error {
-	tracer := gotenberg.Tracer()
+	return runWithFallbackVoid(ctx, "pdfengines.EmbedFilesMetadata", multi.embedMetadataEngines,
-	ctx, span := tracer.Start(ctx, "pdfengines.EmbedFilesMetadata", trace.WithSpanKind(trace.SpanKindInternal))
+		func(ctx context.Context, engine gotenberg.PdfEngine) error {
-	defer span.End()
+			return engine.EmbedFilesMetadata(ctx, logger, metadata, inputPath)
-
+		},
-	var err error
+		func(err error) error { return fmt.Errorf("set embeds metadata using multi PDF engines: %w", err) },
-	errChan := make(chan error, 1)
+	)
-
-	for _, engine := range multi.embedMetadataEngines {
-		go func(engine gotenberg.PdfEngine) {
-			errChan <- engine.EmbedFilesMetadata(ctx, logger, metadata, inputPath)
-		}(engine)
-
-		select {
-		case setErr := <-errChan:
-			if setErr != nil {
-				err = errors.Join(err, setErr)
-			} else {
-				span.SetStatus(codes.Ok, "")
-				return nil
-			}
-		case <-ctx.Done():
-			return ctx.Err()
-		}
-	}
-
-	err = fmt.Errorf("set embeds metadata using multi PDF engines: %w", err)
-	span.RecordError(err)
-	span.SetStatus(codes.Error, err.Error())
-
-	return err
 }
 
 // Interface guards.