feat(chromium): add 'extraLinkTags' and 'extraScriptTags' form fields, add auto-detection of files for URL endpoint, disable some functionalities if JavaScript is disabled

pkg/modules/chromium/chromium.go

@@ -21,6 +21,7 @@ import (
 	"github.com/gotenberg/gotenberg/v7/pkg/modules/api"
 	flag "github.com/spf13/pflag"
 	"go.uber.org/zap"
+	"golang.org/x/sync/errgroup"
 )
 
 func init() {
@@ -75,10 +76,25 @@ type Chromium struct {
 	disableRoutes            bool
 }
 
+// LinkTag represents an HTML <link> element.
+type LinkTag struct {
+	// Href is the "href" attribute of the HTML <link> element.
+	// Required.
+	Href string `json:"href"`
+}
+
+// ScriptTag represents an HTML <script> element.
+type ScriptTag struct {
+	// Src is the "src" attribute of the HTML <link> element.
+	// Required.
+	Src string `json:"src"`
+}
+
 // Options are the available options for converting HTML document to PDF.
 type Options struct {
 	// FailOnConsoleExceptions sets if the conversion should fail if there are
 	// exceptions in the Chromium console.
+	// Optional.
 	FailOnConsoleExceptions bool
 
 	// WaitDelay is the duration to wait when loading an HTML document before
@@ -105,11 +121,19 @@ type Options struct {
 	// Optional.
 	ExtraHTTPHeaders map[string]string
 
+	// ExtraLinkTags are HTML <link> attributes that are added on the fly.
+	// Optional.
+	ExtraLinkTags []LinkTag
+
 	// EmulatedMediaType is the media type to emulate, either "screen" or
 	// "print".
 	// Optional.
 	EmulatedMediaType string
 
+	// ExtraScriptTags are HTML <script> attributes that are added on the fly.
+	// Optional.
+	ExtraScriptTags []ScriptTag
+
 	// Landscape sets the paper orientation.
 	// Optional.
 	Landscape bool
@@ -183,7 +207,9 @@ func DefaultOptions() Options {
 		WaitForExpression:       "",
 		UserAgent:               "",
 		ExtraHTTPHeaders:        nil,
+		ExtraLinkTags:           nil,
 		EmulatedMediaType:       "",
+		ExtraScriptTags:         nil,
 		Landscape:               false,
 		PrintBackground:         false,
 		Scale:                   1.0,
@@ -499,6 +525,50 @@ func (mod Chromium) PDF(ctx context.Context, logger *zap.Logger, URL, outputPath
 
 				return fmt.Errorf("add CSS for exact colors: %w", err)
 			}),
+			chromedp.ActionFunc(func(ctx context.Context) error {
+				if len(options.ExtraLinkTags) == 0 {
+					logger.Debug("no extra link tags")
+
+					return nil
+				}
+
+				logger.Debug(fmt.Sprintf("extra link tags: %+v", options.ExtraLinkTags))
+
+				addLinkTag := func(i int, linkTag LinkTag) func() error {
+					return func() error {
+						script := `
+(() => {
+	const link = document.createElement('link');
+	link.href = '%s';
+	link.rel = 'stylesheet'
+	document.head.appendChild(link);
+})();
+`
+
+						evaluate := chromedp.Evaluate(fmt.Sprintf(script, linkTag.Href), nil)
+						err := evaluate.Do(ctx)
+
+						if err == nil {
+							return nil
+						}
+
+						return fmt.Errorf("add extra link tag %d: %w", i, err)
+					}
+				}
+
+				eg, _ := errgroup.WithContext(ctx)
+
+				for i, linkTag := range options.ExtraLinkTags {
+					eg.Go(addLinkTag(i, linkTag))
+				}
+
+				err := eg.Wait()
+				if err == nil {
+					return nil
+				}
+
+				return fmt.Errorf("add extra link tags: %w", err)
+			}),
 			chromedp.ActionFunc(func(ctx context.Context) error {
 				if options.EmulatedMediaType == "" {
 					logger.Debug("no emulated media type")
@@ -522,24 +592,89 @@ func (mod Chromium) PDF(ctx context.Context, logger *zap.Logger, URL, outputPath
 				return fmt.Errorf("emulate media type '%s': %w", options.EmulatedMediaType, err)
 			}),
 			chromedp.ActionFunc(func(ctx context.Context) error {
-				if options.WaitDelay > 0 {
-					// We wait for a given amount of time so that JavaScript
-					// scripts have a chance to finish before printing the page
-					// to PDF.
-					logger.Debug(fmt.Sprintf("wait '%s' before print", options.WaitDelay))
+				if mod.disableJavaScript {
+					logger.Debug("JavaScript disabled, skipping extra script tags")
 
-					select {
-					case <-ctx.Done():
-						return fmt.Errorf("wait delay: %w", ctx.Err())
-					case <-time.After(options.WaitDelay):
-						return nil
+					return nil
+				}
+
+				if len(options.ExtraScriptTags) == 0 {
+					logger.Debug("no extra script tags")
+
+					return nil
+				}
+
+				logger.Debug(fmt.Sprintf("extra script tags: %+v", options.ExtraScriptTags))
+
+				addScriptTag := func(i int, scriptTag ScriptTag) func() error {
+					return func() error {
+						script := `
+(() => {
+	const script = document.createElement('script');
+	script.src = '%s';
+	document.head.appendChild(script);
+})();
+`
+
+						evaluate := chromedp.Evaluate(fmt.Sprintf(script, scriptTag.Src), nil)
+						err := evaluate.Do(ctx)
+
+						if err == nil {
+							return nil
+						}
+
+						return fmt.Errorf("add extra script tag %d: %w", i, err)
 					}
 				}
 
-				return nil
+				eg, _ := errgroup.WithContext(ctx)
+
+				for i, scriptTag := range options.ExtraScriptTags {
+					eg.Go(addScriptTag(i, scriptTag))
+				}
+
+				err := eg.Wait()
+				if err == nil {
+					return nil
+				}
+
+				return fmt.Errorf("add extra script tags: %w", err)
 			}),
 			chromedp.ActionFunc(func(ctx context.Context) error {
+				if mod.disableJavaScript {
+					logger.Debug("JavaScript disabled, skipping wait delay")
+
+					return nil
+				}
+
+				if options.WaitDelay <= 0 {
+					logger.Debug("no wait delay")
+
+					return nil
+				}
+
+				// We wait for a given amount of time so that JavaScript
+				// scripts have a chance to finish before printing the page
+				// to PDF.
+				logger.Debug(fmt.Sprintf("wait '%s' before print", options.WaitDelay))
+
+				select {
+				case <-ctx.Done():
+					return fmt.Errorf("wait delay: %w", ctx.Err())
+				case <-time.After(options.WaitDelay):
+					return nil
+				}
+			}),
+			chromedp.ActionFunc(func(ctx context.Context) error {
+				if mod.disableJavaScript {
+					logger.Debug("JavaScript disabled, skipping wait for window status / wait for expression")
+
+					return nil
+				}
+
 				if options.WaitWindowStatus == "" && options.WaitForExpression == "" {
+					logger.Debug("no wait for window status nor wait for expression")
+
 					return nil
 				}
 

pkg/modules/chromium/chromium_test.go

@@ -276,6 +276,19 @@ func TestChromium_PDF(t *testing.T) {
 				},
 			},
 		},
+		{
+			URL: "file:///tests/test/testdata/chromium/html/sample11/index.html",
+			options: Options{
+				ExtraLinkTags: []LinkTag{
+					{
+						Href: "font.woff",
+					},
+					{
+						Href: "style.css",
+					},
+				},
+			},
+		},
 		{
 			URL: "file:///tests/test/testdata/chromium/html/sample8/index.html",
 			options: Options{
@@ -295,6 +308,16 @@ func TestChromium_PDF(t *testing.T) {
 				EmulatedMediaType: "print",
 			},
 		},
+		{
+			URL: "file:///tests/test/testdata/chromium/html/sample11/index.html",
+			options: Options{
+				ExtraScriptTags: []ScriptTag{
+					{
+						Src: "script.js",
+					},
+				},
+			},
+		},
 		{
 			URL: "file:///tests/test/testdata/chromium/html/sample4/index.html",
 			options: Options{

pkg/modules/chromium/routes.go

@@ -84,7 +84,9 @@ func FormDataChromiumPDFOptions(ctx *api.Context) (*api.FormData, Options) {
 		WaitForExpression:       waitForExpression,
 		UserAgent:               userAgent,
 		ExtraHTTPHeaders:        extraHTTPHeaders,
+		ExtraLinkTags:           defaultOptions.ExtraLinkTags,
 		EmulatedMediaType:       emulatedMediaType,
+		ExtraScriptTags:         defaultOptions.ExtraScriptTags,
 		Landscape:               landscape,
 		PrintBackground:         printBackground,
 		Scale:                   scale,
@@ -114,19 +116,209 @@ func convertURLRoute(chromium API, engine gotenberg.PDFEngine) api.Route {
 			form, options := FormDataChromiumPDFOptions(ctx)
 
 			var (
-				URL       string
-				PDFformat string
+				URL         string
+				PDFformat   string
+				linkPaths   []string
+				scriptPaths []string
 			)
 
 			err := form.
 				MandatoryString("url", &URL).
 				String("pdfFormat", &PDFformat, "").
+				Custom("extraLinkTags", func(value string) error {
+					if value == "" {
+						return nil
+					}
+
+					err := json.Unmarshal([]byte(value), &options.ExtraLinkTags)
+					if err != nil {
+						return fmt.Errorf("unmarshal extra link tags: %w", err)
+					}
+
+					return nil
+				}).
+				Custom("extraScriptTags", func(value string) error {
+					if value == "" {
+						return nil
+					}
+
+					err := json.Unmarshal([]byte(value), &options.ExtraScriptTags)
+					if err != nil {
+						return fmt.Errorf("unmarshal extra script tags: %w", err)
+					}
+
+					return nil
+				}).
+				Paths([]string{".woff2", ".woff", ".ttf", ".css"}, &linkPaths).
+				Paths([]string{".js"}, &scriptPaths).
 				Validate()
 
 			if err != nil {
 				return fmt.Errorf("validate form data: %w", err)
 			}
 
+			// Thanks to Options.LinkTags and Options.ScriptTags, one may
+			// "hijack" the content of a remote HTML document (for instance, a
+			// website) by loading external assets like scripts or CSS
+			// stylesheets.
+			//
+			// There are two possibilities:
+			//
+			// 1. We auto-detect all files sent in the request that match the
+			// following file extensions: ".woff2", ".woff", ".ttf", ".css",
+			// ".css", and ".js".
+			//
+			// 2. The user has sent both files and a JSON mapping via the
+			// "extraLinkTags" and/or "extraScriptTags" form fields. In such a
+			// scenario, the JSON mapping has the priority for ordering, and
+			// files which are a not mapped are added at the end. The user may
+			// also have sent remote URLs in the JSON mapping.
+
+			// First, let's handle the HTML <link> elements.
+			hasExtraLinkTags := len(options.ExtraLinkTags) > 0
+			hasLinkPaths := len(linkPaths) > 0
+
+			if !hasExtraLinkTags && hasLinkPaths {
+				// First scenario: there is no JSON mapping, we simply add the
+				// paths.
+				options.ExtraLinkTags = make([]LinkTag, len(linkPaths))
+
+				for i, path := range linkPaths {
+					options.ExtraLinkTags[i] = LinkTag{
+						Href: filepath.Base(path),
+					}
+				}
+			} else if hasExtraLinkTags && hasLinkPaths {
+				// Second scenario: there are both files and a JSON mapping.
+
+				// First, find the filenames of the files.
+				filenames := make([]string, len(linkPaths))
+				for i, path := range linkPaths {
+					filenames[i] = filepath.Base(path)
+				}
+
+				var extraLinkTags []LinkTag
+
+				// Then, let's find the filenames that exist in the JSON
+				// mapping, plus the entries that do only exist in the JSON
+				// mapping.
+				for _, linkTagFromMapping := range options.ExtraLinkTags {
+					found := false
+
+					for _, filename := range filenames {
+						if linkTagFromMapping.Href == filename {
+							extraLinkTags = append(extraLinkTags, linkTagFromMapping)
+							found = true
+
+							break
+						}
+					}
+
+					if !found {
+						// This entry only exist in the JSON mapping.
+						extraLinkTags = append(extraLinkTags, linkTagFromMapping)
+					}
+				}
+
+				// Then, add the remaining filenames.
+				for _, filename := range filenames {
+					found := false
+
+					for _, linkTag := range extraLinkTags {
+						if linkTag.Href == filename {
+							found = true
+
+							break
+						}
+					}
+
+					if !found {
+						extraLinkTags = append(extraLinkTags, LinkTag{
+							Href: filename,
+						})
+					}
+				}
+
+				// Last but not least, update the options.
+				options.ExtraLinkTags = extraLinkTags
+			}
+
+			// Next, let's handle the HTML <script> elements.
+			hasExtraScriptTags := len(options.ExtraScriptTags) > 0
+			hasScriptPaths := len(scriptPaths) > 0
+
+			if !hasExtraScriptTags && hasScriptPaths {
+				// First scenario: there is no JSON mapping, we simply add the
+				// paths.
+				options.ExtraScriptTags = make([]ScriptTag, len(scriptPaths))
+
+				for i, path := range scriptPaths {
+					options.ExtraScriptTags[i] = ScriptTag{
+						Src: filepath.Base(path),
+					}
+				}
+			} else if hasExtraScriptTags && hasScriptPaths {
+				// Second scenario: there are both files and a JSON mapping.
+
+				// First, find the filenames of the files.
+				filenames := make([]string, len(scriptPaths))
+				for i, path := range scriptPaths {
+					filenames[i] = filepath.Base(path)
+				}
+
+				var extraScriptTags []ScriptTag
+
+				// Then, let's find the filenames that exist in the JSON
+				// mapping, plus the entries that do only exist in the JSON
+				// mapping.
+				for _, scriptTagFromMapping := range options.ExtraScriptTags {
+					found := false
+
+					for _, filename := range filenames {
+						if scriptTagFromMapping.Src == filename {
+							extraScriptTags = append(extraScriptTags, scriptTagFromMapping)
+							found = true
+
+							break
+						}
+					}
+
+					if !found {
+						// This entry only exist in the JSON mapping.
+						extraScriptTags = append(extraScriptTags, scriptTagFromMapping)
+					}
+				}
+
+				// Then, add the remaining filenames.
+				for _, filename := range filenames {
+					found := false
+
+					for _, scriptTag := range extraScriptTags {
+						if scriptTag.Src == filename {
+							found = true
+
+							break
+						}
+					}
+
+					if !found {
+						extraScriptTags = append(extraScriptTags, ScriptTag{
+							Src: filename,
+						})
+					}
+				}
+
+				// Last but not least, update the options.
+				options.ExtraScriptTags = extraScriptTags
+			}
+
+			// For both <link> and <script> HTML elements, other scenarios are:
+			//
+			// 1. No files and no JSON mapping.
+			// 2. No files but JSON mapping.
+			//
+			// Nothing to do in such cases.
+
 			err = convertURL(ctx, chromium, engine, URL, PDFformat, options)
 			if err != nil {
 				return fmt.Errorf("convert URL to PDF: %w", err)

pkg/modules/chromium/routes_test.go

@@ -3,6 +3,7 @@ package chromium
 import (
 	"context"
 	"errors"
+	"fmt"
 	"net/http"
 	"os"
 	"reflect"
@@ -60,7 +61,7 @@ func TestFormDataChromiumPDFOptions(t *testing.T) {
 		_, actual := FormDataChromiumPDFOptions(tc.ctx.Context)
 
 		if !reflect.DeepEqual(actual, tc.options) {
-			t.Errorf("test %d: expected %v but got: %v", i, tc.options, actual)
+			t.Errorf("test %d: expected %+v but got: %+v", i, tc.options, actual)
 		}
 	}
 }
@@ -137,6 +138,226 @@ func TestConvertURLHandler(t *testing.T) {
 			}(),
 			expectOutputPathsCount: 1,
 		},
+		{
+			ctx: func() *api.MockContext {
+				ctx := &api.MockContext{Context: &api.Context{}}
+				ctx.SetValues(map[string][]string{
+					"url": {
+						"foo",
+					},
+					"extraLinkTags": {
+						"foo",
+					},
+				})
+
+				return ctx
+			}(),
+			api: func() API {
+				chromiumAPI := struct{ ProtoAPI }{}
+				chromiumAPI.pdf = func(_ context.Context, _ *zap.Logger, _, _ string, _ Options) error {
+					return nil
+				}
+
+				return chromiumAPI
+			}(),
+			expectErr:        true,
+			expectHTTPErr:    true,
+			expectHTTPStatus: http.StatusBadRequest,
+		},
+		{
+			ctx: func() *api.MockContext {
+				ctx := &api.MockContext{Context: &api.Context{}}
+				ctx.SetValues(map[string][]string{
+					"url": {
+						"foo",
+					},
+				})
+				ctx.SetFiles(map[string]string{
+					"b.css":  "/b.css",
+					"a.woff": "/a.woff",
+				})
+
+				return ctx
+			}(),
+			api: func() API {
+				chromiumAPI := struct{ ProtoAPI }{}
+				chromiumAPI.pdf = func(_ context.Context, _ *zap.Logger, _, _ string, options Options) error {
+					expectOptions := DefaultOptions()
+					expectOptions.ExtraLinkTags = []LinkTag{
+						{
+							Href: "a.woff",
+						},
+						{
+							Href: "b.css",
+						},
+					}
+
+					if !reflect.DeepEqual(options, expectOptions) {
+						return fmt.Errorf("expected options %+v, but got: %+v", expectOptions, options)
+					}
+
+					return nil
+				}
+
+				return chromiumAPI
+			}(),
+			expectOutputPathsCount: 1,
+		},
+		{
+			ctx: func() *api.MockContext {
+				ctx := &api.MockContext{Context: &api.Context{}}
+				ctx.SetValues(map[string][]string{
+					"url": {
+						"foo",
+					},
+					"extraLinkTags": {
+						`[{"href":"https://cdn.foo"},{"href":"b.css"}]`,
+					},
+				})
+				ctx.SetFiles(map[string]string{
+					"b.css":  "/b.css",
+					"a.woff": "/a.woff",
+				})
+
+				return ctx
+			}(),
+			api: func() API {
+				chromiumAPI := struct{ ProtoAPI }{}
+				chromiumAPI.pdf = func(_ context.Context, _ *zap.Logger, _, _ string, options Options) error {
+					expectOptions := DefaultOptions()
+					expectOptions.ExtraLinkTags = []LinkTag{
+						{
+							Href: "https://cdn.foo",
+						},
+						{
+							Href: "b.css",
+						},
+						{
+							Href: "a.woff",
+						},
+					}
+
+					if !reflect.DeepEqual(options, expectOptions) {
+						return fmt.Errorf("expected options %+v, but got: %+v", expectOptions, options)
+					}
+
+					return nil
+				}
+
+				return chromiumAPI
+			}(),
+			expectOutputPathsCount: 1,
+		},
+		{
+			ctx: func() *api.MockContext {
+				ctx := &api.MockContext{Context: &api.Context{}}
+				ctx.SetValues(map[string][]string{
+					"url": {
+						"foo",
+					},
+					"extraScriptTags": {
+						"foo",
+					},
+				})
+
+				return ctx
+			}(),
+			api: func() API {
+				chromiumAPI := struct{ ProtoAPI }{}
+				chromiumAPI.pdf = func(_ context.Context, _ *zap.Logger, _, _ string, _ Options) error {
+					return nil
+				}
+
+				return chromiumAPI
+			}(),
+			expectErr:        true,
+			expectHTTPErr:    true,
+			expectHTTPStatus: http.StatusBadRequest,
+		},
+		{
+			ctx: func() *api.MockContext {
+				ctx := &api.MockContext{Context: &api.Context{}}
+				ctx.SetValues(map[string][]string{
+					"url": {
+						"foo",
+					},
+				})
+				ctx.SetFiles(map[string]string{
+					"b.js": "/b.js",
+					"a.js": "/a.js",
+				})
+
+				return ctx
+			}(),
+			api: func() API {
+				chromiumAPI := struct{ ProtoAPI }{}
+				chromiumAPI.pdf = func(_ context.Context, _ *zap.Logger, _, _ string, options Options) error {
+					expectOptions := DefaultOptions()
+					expectOptions.ExtraScriptTags = []ScriptTag{
+						{
+							Src: "a.js",
+						},
+						{
+							Src: "b.js",
+						},
+					}
+
+					if !reflect.DeepEqual(options, expectOptions) {
+						return fmt.Errorf("expected options %+v, but got: %+v", expectOptions, options)
+					}
+
+					return nil
+				}
+
+				return chromiumAPI
+			}(),
+			expectOutputPathsCount: 1,
+		},
+		{
+			ctx: func() *api.MockContext {
+				ctx := &api.MockContext{Context: &api.Context{}}
+				ctx.SetValues(map[string][]string{
+					"url": {
+						"foo",
+					},
+					"extraScriptTags": {
+						`[{"src":"https://cdn.foo"},{"src":"b.js"}]`,
+					},
+				})
+				ctx.SetFiles(map[string]string{
+					"b.js": "/b.js",
+					"a.js": "/a.js",
+				})
+
+				return ctx
+			}(),
+			api: func() API {
+				chromiumAPI := struct{ ProtoAPI }{}
+				chromiumAPI.pdf = func(_ context.Context, _ *zap.Logger, _, _ string, options Options) error {
+					expectOptions := DefaultOptions()
+					expectOptions.ExtraScriptTags = []ScriptTag{
+						{
+							Src: "https://cdn.foo",
+						},
+						{
+							Src: "b.js",
+						},
+						{
+							Src: "a.js",
+						},
+					}
+
+					if !reflect.DeepEqual(options, expectOptions) {
+						return fmt.Errorf("expected options %+v, but got: %+v", expectOptions, options)
+					}
+
+					return nil
+				}
+
+				return chromiumAPI
+			}(),
+			expectOutputPathsCount: 1,
+		},
 	} {
 		c := echo.New().NewContext(nil, nil)
 		c.Set("context", tc.ctx.Context)

test/testdata/chromium/html/sample11/index.html

@@ -0,0 +1,12 @@
+<!doctype html>
+<html lang="en">
+<head>
+    <meta charset=v"utf-8">
+    <title>Gutenberg</title>
+</head>
+<body>
+
+<p id="extra">Extra link tags <span id="not">not</span> working</p>
+
+</body>
+</html>

test/testdata/chromium/html/sample11/script.js

@@ -0,0 +1,2 @@
+let extra = document.getElementById('extra')
+extra.innerText = extra.innerText + ' and extra JavaScript tags too!'

test/testdata/chromium/html/sample11/style.css

@@ -0,0 +1,14 @@
+@font-face {
+    font-family: 'Local';
+    src: url('font.woff') format('woff');
+    font-weight: normal;
+    font-style: normal;
+}
+
+#extra {
+    font-family: 'Local'
+}
+
+#not {
+    display: none;
+}