rpcc buffer size may now be set from env var or form field

2026-08-14 19:32:15 +01:00 · 2019-10-07 15:31:05 +02:00
parent ab475d65ee
commit 8d89b2e80a
11 changed files with 405 additions and 53 deletions
--- a/internal/app/xhttp/handler_test.go
+++ b/internal/app/xhttp/handler_test.go
@@ -184,6 +184,24 @@ func TestHTMLHandler(t *testing.T) {
 	req = httptest.NewRequest(http.MethodPost, endpoint, body)
 	req.Header.Set(echo.HeaderContentType, contentType)
 	test.AssertStatusCode(t, http.StatusBadRequest, srv, req)
+	// should return 400 as "googleChromeRpccBufferSize" form field
+	// value is < 0.
+	body, contentType = test.HTMLMultipartForm(t, map[string]string{string(resource.GoogleChromeRpccBufferSizeArgKey): "-1"})
+	req = httptest.NewRequest(http.MethodPost, endpoint, body)
+	req.Header.Set(echo.HeaderContentType, contentType)
+	test.AssertStatusCode(t, http.StatusBadRequest, srv, req)
+	// should return 400 as "googleChromeRpccBufferSize" form field
+	// value is is > config.MaximumGoogleChromeRpccBufferSize().
+	body, contentType = test.HTMLMultipartForm(t, map[string]string{string(resource.GoogleChromeRpccBufferSizeArgKey): "104857601"})
+	req = httptest.NewRequest(http.MethodPost, endpoint, body)
+	req.Header.Set(echo.HeaderContentType, contentType)
+	test.AssertStatusCode(t, http.StatusBadRequest, srv, req)
+	// should return 400 as "googleChromeRpccBufferSize" form field
+	// value is invalid.
+	body, contentType = test.HTMLMultipartForm(t, map[string]string{string(resource.GoogleChromeRpccBufferSizeArgKey): "not an int"})
+	req = httptest.NewRequest(http.MethodPost, endpoint, body)
+	req.Header.Set(echo.HeaderContentType, contentType)
+	test.AssertStatusCode(t, http.StatusBadRequest, srv, req)
 }

 func TestURLHandler(t *testing.T) {
@@ -314,6 +332,24 @@ func TestURLHandler(t *testing.T) {
 	req = httptest.NewRequest(http.MethodPost, endpoint, body)
 	req.Header.Set(echo.HeaderContentType, contentType)
 	test.AssertStatusCode(t, http.StatusBadRequest, srv, req)
+	// should return 400 as "googleChromeRpccBufferSize" form field
+	// value is < 0.
+	body, contentType = test.URLMultipartForm(t, map[string]string{string(resource.GoogleChromeRpccBufferSizeArgKey): "-1"})
+	req = httptest.NewRequest(http.MethodPost, endpoint, body)
+	req.Header.Set(echo.HeaderContentType, contentType)
+	test.AssertStatusCode(t, http.StatusBadRequest, srv, req)
+	// should return 400 as "googleChromeRpccBufferSize" form field
+	// value is is > config.MaximumGoogleChromeRpccBufferSize().
+	body, contentType = test.URLMultipartForm(t, map[string]string{string(resource.GoogleChromeRpccBufferSizeArgKey): "104857601"})
+	req = httptest.NewRequest(http.MethodPost, endpoint, body)
+	req.Header.Set(echo.HeaderContentType, contentType)
+	test.AssertStatusCode(t, http.StatusBadRequest, srv, req)
+	// should return 400 as "googleChromeRpccBufferSize" form field
+	// value is invalid.
+	body, contentType = test.URLMultipartForm(t, map[string]string{string(resource.GoogleChromeRpccBufferSizeArgKey): "not an int"})
+	req = httptest.NewRequest(http.MethodPost, endpoint, body)
+	req.Header.Set(echo.HeaderContentType, contentType)
+	test.AssertStatusCode(t, http.StatusBadRequest, srv, req)
 }

 func TestMarkdownHandler(t *testing.T) {
@@ -444,6 +480,24 @@ func TestMarkdownHandler(t *testing.T) {
 	req = httptest.NewRequest(http.MethodPost, endpoint, body)
 	req.Header.Set(echo.HeaderContentType, contentType)
 	test.AssertStatusCode(t, http.StatusBadRequest, srv, req)
+	// should return 400 as "googleChromeRpccBufferSize" form field
+	// value is < 0.
+	body, contentType = test.MarkdownMultipartForm(t, map[string]string{string(resource.GoogleChromeRpccBufferSizeArgKey): "-1"})
+	req = httptest.NewRequest(http.MethodPost, endpoint, body)
+	req.Header.Set(echo.HeaderContentType, contentType)
+	test.AssertStatusCode(t, http.StatusBadRequest, srv, req)
+	// should return 400 as "googleChromeRpccBufferSize" form field
+	// value is is > config.MaximumGoogleChromeRpccBufferSize().
+	body, contentType = test.MarkdownMultipartForm(t, map[string]string{string(resource.GoogleChromeRpccBufferSizeArgKey): "104857601"})
+	req = httptest.NewRequest(http.MethodPost, endpoint, body)
+	req.Header.Set(echo.HeaderContentType, contentType)
+	test.AssertStatusCode(t, http.StatusBadRequest, srv, req)
+	// should return 400 as "googleChromeRpccBufferSize" form field
+	// value is invalid.
+	body, contentType = test.MarkdownMultipartForm(t, map[string]string{string(resource.GoogleChromeRpccBufferSizeArgKey): "not an int"})
+	req = httptest.NewRequest(http.MethodPost, endpoint, body)
+	req.Header.Set(echo.HeaderContentType, contentType)
+	test.AssertStatusCode(t, http.StatusBadRequest, srv, req)
 }

 func TestOfficeHandler(t *testing.T) {
--- a/internal/app/xhttp/option.go
+++ b/internal/app/xhttp/option.go
@@ -48,18 +48,23 @@ func chromePrinterOptions(r resource.Resource, config conf.Config) (printer.Chro
 		if err != nil {
 			return printer.ChromePrinterOptions{}, err
 		}
+		googleChromeRpccBufferSize, err := resource.GoogleChromeRpccBufferSizeArg(r, config)
+		if err != nil {
+			return printer.ChromePrinterOptions{}, err
+		}
 		return printer.ChromePrinterOptions{
-			WaitTimeout:  waitTimeout,
-			WaitDelay:    waitDelay,
-			HeaderHTML:   headerHTML,
-			FooterHTML:   footerHTML,
-			PaperWidth:   paperWidth,
-			PaperHeight:  paperHeight,
-			MarginTop:    marginTop,
-			MarginBottom: marginBottom,
-			MarginLeft:   marginLeft,
-			MarginRight:  marginRight,
-			Landscape:    landscape,
+			WaitTimeout:    waitTimeout,
+			WaitDelay:      waitDelay,
+			HeaderHTML:     headerHTML,
+			FooterHTML:     footerHTML,
+			PaperWidth:     paperWidth,
+			PaperHeight:    paperHeight,
+			MarginTop:      marginTop,
+			MarginBottom:   marginBottom,
+			MarginLeft:     marginLeft,
+			MarginRight:    marginRight,
+			Landscape:      landscape,
+			RpccBufferSize: googleChromeRpccBufferSize,
 		}, nil
 	}
 	opts, err := resolver()
--- a/internal/app/xhttp/pkg/resource/arg.go
+++ b/internal/app/xhttp/pkg/resource/arg.go
@@ -51,6 +51,9 @@ const (
 	// LandscapeArgKey is the key
 	// of the argument "landscape".
 	LandscapeArgKey ArgKey = "landscape"
+	// GoogleChromeRpccBufferSizeArgKey is the key
+	// of the argument "googleChromeRpccBufferSize".
+	GoogleChromeRpccBufferSizeArgKey ArgKey = "googleChromeRpccBufferSize"
 )

 /*
@@ -73,6 +76,7 @@ func ArgKeys() []ArgKey {
 		MarginLeftArgKey,
 		MarginRightArgKey,
 		LandscapeArgKey,
+		GoogleChromeRpccBufferSizeArgKey,
 	}
 }

@@ -265,3 +269,24 @@ func MarginArgs(r Resource, config conf.Config) (float64, float64, float64, floa
 		marginRight,
 		nil
 }
+
+/*
+GoogleChromeRpccBufferSizeArg is a helper for retrieving
+the "googleChromeRpccBufferSize" argument as int64.
+
+It also validates it against the application
+configuration.
+*/
+func GoogleChromeRpccBufferSizeArg(r Resource, config conf.Config) (int64, error) {
+	const op string = "resource.GoogleChromeRpccBufferSizeArg"
+	result, err := r.Int64Arg(
+		GoogleChromeRpccBufferSizeArgKey,
+		config.DefaultGoogleChromeRpccBufferSize(),
+		xassert.Int64NotInferiorTo(0.0),
+		xassert.Int64NotSuperiorTo(config.MaximumGoogleChromeRpccBufferSize()),
+	)
+	if err != nil {
+		return result, xerror.New(op, err)
+	}
+	return result, nil
+}
--- a/internal/app/xhttp/pkg/resource/arg_test.go
+++ b/internal/app/xhttp/pkg/resource/arg_test.go
@@ -24,6 +24,7 @@ func TestArgKeys(t *testing.T) {
 		MarginLeftArgKey,
 		MarginRightArgKey,
 		LandscapeArgKey,
+		GoogleChromeRpccBufferSizeArgKey,
 	}
 	assert.Equal(t, expected, ArgKeys())
 }
@@ -300,3 +301,47 @@ func TestMarginArgs(t *testing.T) {
 	err = r.Close()
 	assert.Nil(t, err)
 }
+
+func TestGoogleChromeRpccBufferSizeArg(t *testing.T) {
+	const resourceDirectoryName string = "foo"
+	var expected int64
+	logger := test.DebugLogger()
+	config := conf.DefaultConfig()
+	r, err := New(logger, resourceDirectoryName)
+	assert.Nil(t, err)
+	// argument does not exist.
+	expected = config.DefaultGoogleChromeRpccBufferSize()
+	v, err := GoogleChromeRpccBufferSizeArg(r, config)
+	assert.Nil(t, err)
+	assert.Equal(t, expected, v)
+	// argument exist.
+	expected = 10
+	r.WithArg(GoogleChromeRpccBufferSizeArgKey, "10")
+	v, err = GoogleChromeRpccBufferSizeArg(r, config)
+	assert.Nil(t, err)
+	assert.Equal(t, expected, v)
+	// should not be OK as argument
+	// value is < 0.
+	expected = config.DefaultGoogleChromeRpccBufferSize()
+	r.WithArg(GoogleChromeRpccBufferSizeArgKey, "-1")
+	v, err = GoogleChromeRpccBufferSizeArg(r, config)
+	test.AssertError(t, err)
+	assert.Equal(t, expected, v)
+	// should not be OK as argument
+	// value is > config.MaximumGoogleChromeRpccBufferSize().
+	expected = config.DefaultGoogleChromeRpccBufferSize()
+	r.WithArg(GoogleChromeRpccBufferSizeArgKey, "104857601")
+	v, err = GoogleChromeRpccBufferSizeArg(r, config)
+	test.AssertError(t, err)
+	assert.Equal(t, expected, v)
+	// should not be OK as
+	// argument value is invalid.
+	expected = config.DefaultGoogleChromeRpccBufferSize()
+	r.WithArg(GoogleChromeRpccBufferSizeArgKey, "foo")
+	v, err = GoogleChromeRpccBufferSizeArg(r, config)
+	test.AssertError(t, err)
+	assert.Equal(t, expected, v)
+	// finally...
+	err = r.Close()
+	assert.Nil(t, err)
+}
--- a/internal/pkg/conf/conf.go
+++ b/internal/pkg/conf/conf.go
@@ -34,35 +34,42 @@ const (
 	// LogLevelEnvVar contains the name
 	// of the environment variable "LOG_LEVEL".
 	LogLevelEnvVar string = "LOG_LEVEL"
+	// DefaultGoogleChromeRpccBufferSizeEnvVar contains the name
+	// of the environment variable "DEFAULT_GOOGLE_CHROME_RPCC_BUFFER_SIZE".
+	DefaultGoogleChromeRpccBufferSizeEnvVar string = "DEFAULT_GOOGLE_CHROME_RPCC_BUFFER_SIZE"
 )

 // Config contains the application
 // configuration.
 type Config struct {
-	maximumWaitTimeout       float64
-	maximumWaitDelay         float64
-	maximumWebhookURLTimeout float64
-	defaultWaitTimeout       float64
-	defaultWebhookURLTimeout float64
-	defaultListenPort        int64
-	disableGoogleChrome      bool
-	disableUnoconv           bool
-	logLevel                 xlog.Level
+	maximumWaitTimeout                float64
+	maximumWaitDelay                  float64
+	maximumWebhookURLTimeout          float64
+	defaultWaitTimeout                float64
+	defaultWebhookURLTimeout          float64
+	defaultListenPort                 int64
+	disableGoogleChrome               bool
+	disableUnoconv                    bool
+	logLevel                          xlog.Level
+	maximumGoogleChromeRpccBufferSize int64
+	defaultGoogleChromeRpccBufferSize int64
 }

 // DefaultConfig returns the default
 // configuration.
 func DefaultConfig() Config {
 	return Config{
-		maximumWaitTimeout:       30.0,
-		maximumWaitDelay:         10.0,
-		maximumWebhookURLTimeout: 30.0,
-		defaultWaitTimeout:       10.0,
-		defaultWebhookURLTimeout: 10.0,
-		defaultListenPort:        3000,
-		disableGoogleChrome:      false,
-		disableUnoconv:           false,
-		logLevel:                 xlog.InfoLevel,
+		maximumWaitTimeout:                30.0,
+		maximumWaitDelay:                  10.0,
+		maximumWebhookURLTimeout:          30.0,
+		defaultWaitTimeout:                10.0,
+		defaultWebhookURLTimeout:          10.0,
+		defaultListenPort:                 3000,
+		disableGoogleChrome:               false,
+		disableUnoconv:                    false,
+		logLevel:                          xlog.InfoLevel,
+		maximumGoogleChromeRpccBufferSize: 104857600, // ~100 MB
+		defaultGoogleChromeRpccBufferSize: 1048576,   // 1 MB
 	}
 }

@@ -156,6 +163,16 @@ func FromEnv() (Config, error) {
 		if err != nil {
 			return c, err
 		}
+		defaultGoogleChromeRpccBufferSize, err := xassert.Int64FromEnv(
+			DefaultGoogleChromeRpccBufferSizeEnvVar,
+			c.defaultGoogleChromeRpccBufferSize,
+			xassert.Int64NotInferiorTo(0),
+			xassert.Int64NotSuperiorTo(c.MaximumGoogleChromeRpccBufferSize()),
+		)
+		c.defaultGoogleChromeRpccBufferSize = defaultGoogleChromeRpccBufferSize
+		if err != nil {
+			return c, err
+		}
 		return c, nil
 	}
 	result, err := resolver()
@@ -224,3 +241,15 @@ func (c Config) DisableUnoconv() bool {
 func (c Config) LogLevel() xlog.Level {
 	return c.logLevel
 }
+
+// MaximumGoogleChromeRpccBufferSize returns the maximum
+// Google Chrome rpcc buffer size from the configuration.
+func (c Config) MaximumGoogleChromeRpccBufferSize() int64 {
+	return c.maximumGoogleChromeRpccBufferSize
+}
+
+// DefaultGoogleChromeRpccBufferSize returns the default
+//  Google Chrome rpcc buffer size from the configuration.
+func (c Config) DefaultGoogleChromeRpccBufferSize() int64 {
+	return c.defaultGoogleChromeRpccBufferSize
+}
--- a/internal/pkg/conf/conf_test.go
+++ b/internal/pkg/conf/conf_test.go
@@ -320,6 +320,43 @@ func TestLogLevelFromEnv(t *testing.T) {
 	os.Unsetenv(LogLevelEnvVar)
 }

+func TestDefaultGoogleChromeRpccBufferSizeFromEnv(t *testing.T) {
+	var (
+		expected Config
+		result   Config
+		err      error
+	)
+	// DEFAULT_GOOGLE_CHROME_RPCC_BUFFER_SIZE correctly set.
+	os.Setenv(DefaultGoogleChromeRpccBufferSizeEnvVar, "100")
+	expected = DefaultConfig()
+	expected.defaultGoogleChromeRpccBufferSize = 100
+	result, err = FromEnv()
+	assert.Nil(t, err)
+	assert.Equal(t, expected, result)
+	os.Unsetenv(DefaultGoogleChromeRpccBufferSizeEnvVar)
+	// DEFAULT_GOOGLE_CHROME_RPCC_BUFFER_SIZE wrongly set.
+	os.Setenv(DefaultGoogleChromeRpccBufferSizeEnvVar, "foo")
+	expected = DefaultConfig()
+	result, err = FromEnv()
+	test.AssertError(t, err)
+	assert.Equal(t, expected, result)
+	os.Unsetenv(DefaultGoogleChromeRpccBufferSizeEnvVar)
+	// DEFAULT_GOOGLE_CHROME_RPCC_BUFFER_SIZE < 0.
+	os.Setenv(DefaultGoogleChromeRpccBufferSizeEnvVar, "-1")
+	expected = DefaultConfig()
+	result, err = FromEnv()
+	test.AssertError(t, err)
+	assert.Equal(t, expected, result)
+	os.Unsetenv(DefaultGoogleChromeRpccBufferSizeEnvVar)
+	// DEFAULT_GOOGLE_CHROME_RPCC_BUFFER_SIZE > 100 MB (maximumGoogleChromeRpccBufferSize).
+	os.Setenv(DefaultGoogleChromeRpccBufferSizeEnvVar, "104857601")
+	expected = DefaultConfig()
+	result, err = FromEnv()
+	test.AssertError(t, err)
+	assert.Equal(t, expected, result)
+	os.Unsetenv(DefaultGoogleChromeRpccBufferSizeEnvVar)
+}
+
 func TestGetters(t *testing.T) {
 	result := DefaultConfig()
 	assert.Equal(t, result.maximumWaitTimeout, result.MaximumWaitTimeout())
@@ -331,4 +368,6 @@ func TestGetters(t *testing.T) {
 	assert.Equal(t, result.disableGoogleChrome, result.DisableGoogleChrome())
 	assert.Equal(t, result.disableUnoconv, result.DisableUnoconv())
 	assert.Equal(t, result.logLevel, result.LogLevel())
+	assert.Equal(t, result.maximumGoogleChromeRpccBufferSize, result.MaximumGoogleChromeRpccBufferSize())
+	assert.Equal(t, result.defaultGoogleChromeRpccBufferSize, result.DefaultGoogleChromeRpccBufferSize())
 }
--- a/internal/pkg/printer/chrome.go
+++ b/internal/pkg/printer/chrome.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"io/ioutil"
+	"strings"
 	"time"

 	"github.com/mafredri/cdp"
@@ -29,17 +30,18 @@ type chromePrinter struct {
 // ChromePrinterOptions helps customizing the
 // Google Chrome Printer behaviour.
 type ChromePrinterOptions struct {
-	WaitTimeout  float64
-	WaitDelay    float64
-	HeaderHTML   string
-	FooterHTML   string
-	PaperWidth   float64
-	PaperHeight  float64
-	MarginTop    float64
-	MarginBottom float64
-	MarginLeft   float64
-	MarginRight  float64
-	Landscape    bool
+	WaitTimeout    float64
+	WaitDelay      float64
+	HeaderHTML     string
+	FooterHTML     string
+	PaperWidth     float64
+	PaperHeight    float64
+	MarginTop      float64
+	MarginBottom   float64
+	MarginLeft     float64
+	MarginRight    float64
+	Landscape      bool
+	RpccBufferSize int64
 }

 // DefaultChromePrinterOptions returns the default
@@ -47,17 +49,18 @@ type ChromePrinterOptions struct {
 func DefaultChromePrinterOptions(config conf.Config) ChromePrinterOptions {
 	const defaultHeaderFooterHTML string = "<html><head></head><body></body></html>"
 	return ChromePrinterOptions{
-		WaitTimeout:  config.DefaultWaitTimeout(),
-		WaitDelay:    0.0,
-		HeaderHTML:   defaultHeaderFooterHTML,
-		FooterHTML:   defaultHeaderFooterHTML,
-		PaperWidth:   8.27,
-		PaperHeight:  11.7,
-		MarginTop:    1.0,
-		MarginBottom: 1.0,
-		MarginLeft:   1.0,
-		MarginRight:  1.0,
-		Landscape:    false,
+		WaitTimeout:    config.DefaultWaitTimeout(),
+		WaitDelay:      0.0,
+		HeaderHTML:     defaultHeaderFooterHTML,
+		FooterHTML:     defaultHeaderFooterHTML,
+		PaperWidth:     8.27,
+		PaperHeight:    11.7,
+		MarginTop:      1.0,
+		MarginBottom:   1.0,
+		MarginLeft:     1.0,
+		MarginRight:    1.0,
+		Landscape:      false,
+		RpccBufferSize: config.DefaultGoogleChromeRpccBufferSize(),
 	}
 }

@@ -119,7 +122,7 @@ func (p chromePrinter) Print(destination string) error {
 				https://github.com/mafredri/cdp/issues/4
 				https://github.com/ChromeDevTools/devtools-protocol/issues/24
 			*/
-			rpcc.WithWriteBufferSize(1024*1024),
+			rpcc.WithWriteBufferSize(int(p.opts.RpccBufferSize)),
 			rpcc.WithCompression(),
 		)
 		if err != nil {
@@ -170,6 +173,16 @@ func (p chromePrinter) Print(destination string) error {
 				SetPrintBackground(true),
 		)
 		if err != nil {
+			if strings.Contains(err.Error(), "rpcc: message too large") {
+				return xerror.Invalid(
+					op,
+					fmt.Sprintf(
+						"'%d' bytes are not enough: increase the Google Chrome rpcc buffer size (up to 100 MB)",
+						p.opts.RpccBufferSize,
+					),
+					err,
+				)
+			}
 			return err
 		}
 		if err := ioutil.WriteFile(destination, print.Data, 0644); err != nil {