From 171f93662f749a8689922ff61dbe0a227f07e059 Mon Sep 17 00:00:00 2001 From: Julien Neuhart Date: Thu, 19 Sep 2019 17:28:29 +0200 Subject: [PATCH] process load balancing: broken but in progress --- Makefile | 2 +- README.md | 2 +- build/docs/content/00-introduction.md | 2 +- cmd/gotenberg/main.go | 40 +-- internal/app/xhttp/handler.go | 102 +++++-- internal/app/xhttp/handler_test.go | 28 +- internal/app/xhttp/middleware.go | 12 +- internal/app/xhttp/option.go | 53 ++-- internal/app/xhttp/pkg/context/context.go | 69 ++++- internal/app/xhttp/pkg/resource/arg.go | 28 ++ internal/app/xhttp/xhttp.go | 12 +- internal/app/xhttp/xhttp_test.go | 9 +- internal/pkg/prinery/doc.go | 1 + internal/pkg/prinery/prinery.go | 123 ++++++++ internal/pkg/print/chrome.go | 265 ++++++++++++++++++ internal/pkg/print/doc.go | 1 + internal/pkg/print/html.go | 18 ++ internal/pkg/print/markdown.go | 70 +++++ internal/pkg/print/merge.go | 44 +++ internal/pkg/print/office.go | 109 ++++++++ internal/pkg/print/print.go | 13 + internal/pkg/print/url.go | 15 + internal/pkg/process/chrome.go | 106 +++++++ internal/pkg/process/doc.go | 1 + internal/pkg/process/pm2.go | 324 ++++++++++++++++++++++ internal/pkg/process/process.go | 40 +++ internal/pkg/process/soffice.go | 74 +++++ internal/pkg/xerrgroup/doc.go | 6 + internal/pkg/xerrgroup/xerrgroup.go | 20 ++ loadtesting/README.md | 113 ++++++++ loadtesting/html.js | 44 +++ loadtesting/merge.js | 18 ++ loadtesting/office.js | 34 +++ 33 files changed, 1681 insertions(+), 117 deletions(-) create mode 100644 internal/pkg/prinery/doc.go create mode 100644 internal/pkg/prinery/prinery.go create mode 100644 internal/pkg/print/chrome.go create mode 100644 internal/pkg/print/doc.go create mode 100644 internal/pkg/print/html.go create mode 100644 internal/pkg/print/markdown.go create mode 100644 internal/pkg/print/merge.go create mode 100644 internal/pkg/print/office.go create mode 100644 internal/pkg/print/print.go create mode 100644 internal/pkg/print/url.go create mode 100644 internal/pkg/process/chrome.go create mode 100644 internal/pkg/process/doc.go create mode 100644 internal/pkg/process/pm2.go create mode 100644 internal/pkg/process/process.go create mode 100644 internal/pkg/process/soffice.go create mode 100644 internal/pkg/xerrgroup/doc.go create mode 100644 internal/pkg/xerrgroup/xerrgroup.go create mode 100644 loadtesting/README.md create mode 100644 loadtesting/html.js create mode 100644 loadtesting/merge.js create mode 100644 loadtesting/office.js diff --git a/Makefile b/Makefile index 85ec5f69..ac5afacd 100644 --- a/Makefile +++ b/Makefile @@ -52,7 +52,7 @@ image: # start the API using previously built Docker image. gotenberg: - docker run -it --rm -e MAXIMUM_WAIT_TIMEOUT=$(MAXIMUM_WAIT_TIMEOUT) -e MAXIMUM_WAIT_DELAY=$(MAXIMUM_WAIT_DELAY) -e MAXIMUM_WEBHOOK_URL_TIMEOUT=$(MAXIMUM_WEBHOOK_URL_TIMEOUT) -e DEFAULT_WEBHOOK_URL_TIMEOUT=$(DEFAULT_WEBHOOK_URL_TIMEOUT) -e MAXIMUM_WEBHOOK_URL_TIMEOUT=$(MAXIMUM_WEBHOOK_URL_TIMEOUT) -e DEFAULT_LISTEN_PORT=$(DEFAULT_LISTEN_PORT) -e DISABLE_GOOGLE_CHROME=$(DISABLE_GOOGLE_CHROME) -e DISABLE_UNOCONV=$(DISABLE_UNOCONV) -e LOG_LEVEL=$(LOG_LEVEL) -p "3000:$(DEFAULT_LISTEN_PORT)" $(DOCKER_REPOSITORY)/gotenberg:$(VERSION) + docker run -it --rm -e MAXIMUM_WAIT_TIMEOUT=$(MAXIMUM_WAIT_TIMEOUT) -e MAXIMUM_WAIT_DELAY=$(MAXIMUM_WAIT_DELAY) -e MAXIMUM_WEBHOOK_URL_TIMEOUT=$(MAXIMUM_WEBHOOK_URL_TIMEOUT) -e DEFAULT_WEBHOOK_URL_TIMEOUT=$(DEFAULT_WEBHOOK_URL_TIMEOUT) -e MAXIMUM_WEBHOOK_URL_TIMEOUT=$(MAXIMUM_WEBHOOK_URL_TIMEOUT) -e DEFAULT_LISTEN_PORT=$(DEFAULT_LISTEN_PORT) -e DISABLE_GOOGLE_CHROME=$(DISABLE_GOOGLE_CHROME) -e DISABLE_UNOCONV=$(DISABLE_UNOCONV) -e LOG_LEVEL=$(LOG_LEVEL) -p "$(DEFAULT_LISTEN_PORT):$(DEFAULT_LISTEN_PORT)" $(DOCKER_REPOSITORY)/gotenberg:$(VERSION) # publish Gotenberg images according to version. publish: diff --git a/README.md b/README.md index f2bfc700..174208ba 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ At TheCodingMachine, we build a lot of web applications (intranets, extranets an * HTML and Markdown conversions using Google Chrome headless * Office conversions (.txt, .rtf, .docx, .doc, .odt, .pptx, .ppt, .odp and so on) using [unoconv](https://github.com/dagwieers/unoconv) -* Performance :zap:: Google Chrome and LibreOffice (unoconv) started once in the background thanks to PM2 +* Performance :zap:: Google Chrome and LibreOffice started once in the background thanks to PM2 * Failure prevention :broken_heart:: PM2 automatically restarts previous processes if they fail * Assets :package:: send your header, footer, images, fonts, stylesheets and so on for converting your HTML and Markdown to beaufitul PDFs! * Easily interact with the API using our [Go](https://github.com/thecodingmachine/gotenberg-go-client) and [PHP](https://github.com/thecodingmachine/gotenberg-php-client) libraries diff --git a/build/docs/content/00-introduction.md b/build/docs/content/00-introduction.md index ac1bf49a..1c6728e1 100644 --- a/build/docs/content/00-introduction.md +++ b/build/docs/content/00-introduction.md @@ -6,7 +6,7 @@ title: Introduction * HTML and Markdown conversions using Google Chrome headless * Office conversions (.txt, .rtf, .docx, .doc, .odt, .pptx, .ppt, .odp and so on) using [unoconv](https://github.com/dagwieers/unoconv) -* Performance: Google Chrome and LibreOffice (unoconv) started once in the background thanks to PM2 +* Performance: Google Chrome and LibreOffice started once in the background thanks to PM2 * Failure prevention: PM2 automatically restarts previous processes if they fail * Assets: send your header, footer, images, fonts, stylesheets and so on for converting your HTML and Markdown to beaufitul PDFs! * Easily interact with the API using our [Go](https://github.com/thecodingmachine/gotenberg-go-client) and [PHP](https://github.com/thecodingmachine/gotenberg-php-client) libraries \ No newline at end of file diff --git a/cmd/gotenberg/main.go b/cmd/gotenberg/main.go index 3ffb2696..1cbf44f2 100644 --- a/cmd/gotenberg/main.go +++ b/cmd/gotenberg/main.go @@ -8,7 +8,8 @@ import ( "github.com/thecodingmachine/gotenberg/internal/app/xhttp" "github.com/thecodingmachine/gotenberg/internal/pkg/conf" - "github.com/thecodingmachine/gotenberg/internal/pkg/pm2" + "github.com/thecodingmachine/gotenberg/internal/pkg/prinery" + "github.com/thecodingmachine/gotenberg/internal/pkg/process" "github.com/thecodingmachine/gotenberg/internal/pkg/xcontext" "github.com/thecodingmachine/gotenberg/internal/pkg/xlog" ) @@ -26,22 +27,30 @@ func main() { } systemLogger.InfofOp(op, "Gotenberg %s", version) systemLogger.DebugfOp(op, "configuration: %+v", config) - // start PM2 processes. - var processes []pm2.Process - if !config.DisableGoogleChrome() { - processes = append(processes, pm2.NewChromeProcess(systemLogger)) + // create PM2 manager and start processes. + manager := process.NewPM2Manager(systemLogger, config) + if err := manager.Start(); err != nil { + systemLogger.FatalOp(op, err) } - /*if !config.DisableUnoconv() { - processes = append(processes, pm2.NewUnoconvProcess(systemLogger)) - }*/ - for _, p := range processes { - systemLogger.InfofOp(op, "starting '%s' with PM2...", p.Fullname()) - if err := p.Start(); err != nil { + // create prineries. + var chromePrinery *prinery.Prinery + if !config.DisableGoogleChrome() { + chromePrinery, err = prinery.New(systemLogger, manager, process.ChromeKey) + if err != nil { systemLogger.FatalOp(op, err) } + go chromePrinery.Start() + } + var sofficePrinery *prinery.Prinery + if !config.DisableUnoconv() { + sofficePrinery, err = prinery.New(systemLogger, manager, process.SofficeKey) + if err != nil { + systemLogger.FatalOp(op, err) + } + go sofficePrinery.Start() } // create our API. - srv := xhttp.New(config, processes...) + srv := xhttp.New(config, manager, chromePrinery, sofficePrinery) // run our API in a goroutine so that it doesn't block. go func() { systemLogger.InfofOp(op, "http server started on port '%d'", config.DefaultListenPort()) @@ -66,13 +75,6 @@ func main() { if err := srv.Shutdown(ctx); err != nil { systemLogger.FatalOp(op, err) } - // shutdown PM2 processes. - for _, p := range processes { - systemLogger.InfofOp(op, "shutting down '%s' with PM2...", p.Fullname()) - if err := p.Stop(); err != nil { - systemLogger.FatalOp(op, err) - } - } systemLogger.InfoOp(op, "bye!") os.Exit(0) } diff --git a/internal/app/xhttp/handler.go b/internal/app/xhttp/handler.go index 345dafef..fce3bf6f 100644 --- a/internal/app/xhttp/handler.go +++ b/internal/app/xhttp/handler.go @@ -9,7 +9,9 @@ import ( "github.com/thecodingmachine/gotenberg/internal/app/xhttp/pkg/context" "github.com/thecodingmachine/gotenberg/internal/app/xhttp/pkg/resource" "github.com/thecodingmachine/gotenberg/internal/pkg/pm2" - "github.com/thecodingmachine/gotenberg/internal/pkg/printer" + "github.com/thecodingmachine/gotenberg/internal/pkg/prinery" + "github.com/thecodingmachine/gotenberg/internal/pkg/print" + "github.com/thecodingmachine/gotenberg/internal/pkg/xcontext" "github.com/thecodingmachine/gotenberg/internal/pkg/xerror" "github.com/thecodingmachine/gotenberg/internal/pkg/xlog" "github.com/thecodingmachine/gotenberg/internal/pkg/xrand" @@ -39,6 +41,7 @@ func pingHandler(c echo.Context) error { if logger.Level() != xlog.DebugLevel { return nil } + // TODO list, err := pm2.List() if err != nil { return err @@ -59,17 +62,18 @@ func mergeHandler(c echo.Context) error { ctx := context.MustCastFromEchoContext(c) logger := ctx.XLogger() logger.DebugOp(op, "handling merge request...") + config := ctx.Config() r := ctx.MustResource() - opts, err := mergePrinterOptions(r, ctx.Config()) + timeout, err := resource.WaitTimeoutAndWaitDelayArg(r, config) if err != nil { - return xerror.New(op, err) + return err } fpaths, err := r.Fpaths(".pdf") if err != nil { return err } - p := printer.NewMergePrinter(logger, fpaths, opts) - return convert(ctx, p) + p := print.NewMergePrint(logger, fpaths) + return convert(ctx, nil, p, timeout) } if err := resolver(); err != nil { return xerror.New(op, err) @@ -83,10 +87,16 @@ func htmlHandler(c echo.Context) error { const op string = "xhttp.htmlHandler" resolver := func() error { ctx := context.MustCastFromEchoContext(c) + prinry := ctx.MustChromePrinery() logger := ctx.XLogger() logger.DebugOp(op, "handling HTML request...") + config := ctx.Config() r := ctx.MustResource() - opts, err := chromePrinterOptions(r, ctx.Config()) + timeout, err := resource.WaitTimeoutAndWaitDelayArg(r, config) + if err != nil { + return err + } + opts, err := chromePrintOptions(r, config) if err != nil { return err } @@ -94,8 +104,8 @@ func htmlHandler(c echo.Context) error { if err != nil { return err } - p := printer.NewHTMLPrinter(logger, fpath, opts) - return convert(ctx, p) + p := print.NewHTMLPrint(logger, fpath, opts) + return convert(ctx, prinry, p, timeout) } if err := resolver(); err != nil { return xerror.New(op, err) @@ -109,10 +119,16 @@ func urlHandler(c echo.Context) error { const op string = "xhttp.urlHandler" resolver := func() error { ctx := context.MustCastFromEchoContext(c) + prinry := ctx.MustChromePrinery() logger := ctx.XLogger() logger.DebugOp(op, "handling URL request...") + config := ctx.Config() r := ctx.MustResource() - opts, err := chromePrinterOptions(r, ctx.Config()) + timeout, err := resource.WaitTimeoutAndWaitDelayArg(r, config) + if err != nil { + return err + } + opts, err := chromePrintOptions(r, config) if err != nil { return err } @@ -127,8 +143,8 @@ func urlHandler(c echo.Context) error { if err != nil { return err } - p := printer.NewURLPrinter(logger, remoteURL, opts) - return convert(ctx, p) + p := print.NewURLPrint(logger, remoteURL, opts) + return convert(ctx, prinry, p, timeout) } if err := resolver(); err != nil { return xerror.New(op, err) @@ -142,10 +158,16 @@ func markdownHandler(c echo.Context) error { const op string = "xhttp.markdownHandler" resolver := func() error { ctx := context.MustCastFromEchoContext(c) + prinry := ctx.MustChromePrinery() logger := ctx.XLogger() logger.DebugOp(op, "handling Markdown request...") + config := ctx.Config() r := ctx.MustResource() - opts, err := chromePrinterOptions(r, ctx.Config()) + timeout, err := resource.WaitTimeoutAndWaitDelayArg(r, config) + if err != nil { + return err + } + opts, err := chromePrintOptions(r, config) if err != nil { return err } @@ -153,11 +175,11 @@ func markdownHandler(c echo.Context) error { if err != nil { return err } - p, err := printer.NewMarkdownPrinter(logger, fpath, opts) + p, err := print.NewMarkdownPrint(logger, fpath, opts) if err != nil { return err } - return convert(ctx, p) + return convert(ctx, prinry, p, timeout) } if err := resolver(); err != nil { return xerror.New(op, err) @@ -171,10 +193,16 @@ func officeHandler(c echo.Context) error { const op string = "xhttp.officeHandler" resolver := func() error { ctx := context.MustCastFromEchoContext(c) + prinry := ctx.MustSofficePrinery() logger := ctx.XLogger() logger.DebugOp(op, "handling Office request...") + config := ctx.Config() r := ctx.MustResource() - opts, err := officePrinterOptions(r, ctx.Config()) + timeout, err := resource.WaitTimeoutAndWaitDelayArg(r, config) + if err != nil { + return err + } + opts, err := officePrintOptions(r, config) if err != nil { return err } @@ -195,8 +223,8 @@ func officeHandler(c echo.Context) error { if err != nil { return err } - p := printer.NewOfficePrinter(logger, fpaths, opts) - return convert(ctx, p) + p := print.NewOfficePrint(logger, fpaths, opts) + return convert(ctx, prinry, p, timeout) } if err := resolver(); err != nil { return xerror.New(op, err) @@ -204,7 +232,7 @@ func officeHandler(c echo.Context) error { return nil } -func convert(ctx context.Context, p printer.Printer) error { +func convert(ctx context.Context, prinry *prinery.Prinery, prnt print.Print, timeout float64) error { const op string = "xhttp.convert" resolver := func() error { logger := ctx.XLogger() @@ -217,13 +245,13 @@ func convert(ctx context.Context, p printer.Printer) error { // or an error. if !r.HasArg(resource.WebhookURLArgKey) { logger.DebugfOp(op, "no '%s' found, converting synchronously", resource.WebhookURLArgKey) - return convertSync(ctx, p, filename, fpath) + return convertSync(ctx, prinry, prnt, timeout, filename, fpath) } // as a webhook URL has been given, we // run the following lines in a goroutine so that // it doesn't block. logger.DebugfOp(op, "'%s' found, converting asynchronously", resource.WebhookURLArgKey) - return convertAsync(ctx, p, filename, fpath) + return convertAsync(ctx, prinry, prnt, timeout, filename, fpath) } if err := resolver(); err != nil { return xerror.New(op, err) @@ -231,13 +259,19 @@ func convert(ctx context.Context, p printer.Printer) error { return nil } -func convertSync(ctx context.Context, p printer.Printer, filename, fpath string) error { +func convertSync(ctx context.Context, prinry *prinery.Prinery, prnt print.Print, timeout float64, filename, fpath string) error { const op = "xhttp.convertSync" + logger := ctx.XLogger() + r := ctx.MustResource() + timeoutCtx, cancel := xcontext.WithTimeout(logger, timeout) + defer cancel() resolver := func() error { - logger := ctx.XLogger() - r := ctx.MustResource() - - if err := p.Print(fpath); err != nil { + if prinry == nil { + // case: merge. + if err := prnt.Print(timeoutCtx, fpath, nil); err != nil { + return err + } + } else if err := prinry.PrintRequest(timeoutCtx, logger, prnt, fpath); err != nil { return err } if !r.HasArg(resource.ResultFilenameArgKey) { @@ -267,12 +301,15 @@ func convertSync(ctx context.Context, p printer.Printer, filename, fpath string) return nil } if err := resolver(); err != nil { - return xerror.New(op, err) + return xcontext.MustHandleError( + timeoutCtx, + xerror.New(op, err), + ) } return nil } -func convertAsync(ctx context.Context, p printer.Printer, filename, fpath string) error { +func convertAsync(ctx context.Context, prinry *prinery.Prinery, prnt print.Print, timeout float64, filename, fpath string) error { const op = "xhttp.convertAsync" logger := ctx.XLogger() r := ctx.MustResource() @@ -286,7 +323,16 @@ func convertAsync(ctx context.Context, p printer.Printer, filename, fpath string } go func() { defer r.Close() // nolint: errcheck - if err := p.Print(fpath); err != nil { + timeoutCtx, cancel := xcontext.WithTimeout(logger, timeout) + defer cancel() + if prinry == nil { + // case: merge. + if err := prnt.Print(timeoutCtx, fpath, nil); err != nil { + xerr := xerror.New(op, err) + logger.ErrorOp(xerror.Op(xerr), xerr) + return + } + } else if err := prinry.PrintRequest(timeoutCtx, logger, prnt, fpath); err != nil { xerr := xerror.New(op, err) logger.ErrorOp(xerror.Op(xerr), xerr) return diff --git a/internal/app/xhttp/handler_test.go b/internal/app/xhttp/handler_test.go index c4c78add..c1a5355b 100644 --- a/internal/app/xhttp/handler_test.go +++ b/internal/app/xhttp/handler_test.go @@ -19,8 +19,8 @@ import ( func TestPingHandler(t *testing.T) { // should return 200. config := conf.DefaultConfig() - srv := New(config) - srv = New(config) + // TODO + srv := New(config, nil, nil) req := httptest.NewRequest(http.MethodGet, pingEndpoint, nil) test.AssertStatusCode(t, http.StatusOK, srv, req) // should returns a JSON as @@ -28,7 +28,8 @@ func TestPingHandler(t *testing.T) { os.Setenv(conf.LogLevelEnvVar, "DEBUG") config, err := conf.FromEnv() assert.Nil(t, err) - srv = New(config) + // TODO + srv = New(config, nil, nil) req = httptest.NewRequest(http.MethodGet, pingEndpoint, nil) rec := httptest.NewRecorder() srv.ServeHTTP(rec, req) @@ -39,7 +40,8 @@ func TestPingHandler(t *testing.T) { func TestMergeHandler(t *testing.T) { config := conf.DefaultConfig() - srv := New(config) + // TODO + srv := New(config, nil, nil) // should return 200. body, contentType := test.MergeMultipartForm(t, nil) req := httptest.NewRequest(http.MethodPost, mergeEndpoint, body) @@ -72,7 +74,8 @@ func TestMergeHandler(t *testing.T) { func TestHTMLHandler(t *testing.T) { config := conf.DefaultConfig() - srv := New(config) + // TODO + srv := New(config, nil, nil) endpoint := fmt.Sprintf("%s%s", convertGroupEndpoint, htmlEndpoint) // should return 200. body, contentType := test.HTMLMultipartForm(t, nil) @@ -202,7 +205,8 @@ func TestHTMLHandler(t *testing.T) { func TestURLHandler(t *testing.T) { config := conf.DefaultConfig() - srv := New(config) + // TODO + srv := New(config, nil, nil) endpoint := fmt.Sprintf("%s%s", convertGroupEndpoint, urlEndpoint) // should return 200. body, contentType := test.URLMultipartForm(t, nil) @@ -332,7 +336,8 @@ func TestURLHandler(t *testing.T) { func TestMarkdownHandler(t *testing.T) { config := conf.DefaultConfig() - srv := New(config) + // TODO + srv := New(config, nil, nil) endpoint := fmt.Sprintf("%s%s", convertGroupEndpoint, markdownEndpoint) // should return 200. body, contentType := test.MarkdownMultipartForm(t, nil) @@ -462,7 +467,8 @@ func TestMarkdownHandler(t *testing.T) { func TestOfficeHandler(t *testing.T) { config := conf.DefaultConfig() - srv := New(config) + // TODO + srv := New(config, nil, nil) endpoint := fmt.Sprintf("%s%s", convertGroupEndpoint, officeEndpoint) // should return 200. body, contentType := test.OfficeMultipartForm(t, nil) @@ -524,7 +530,8 @@ func TestWebhook(t *testing.T) { rcv.Start(":3001") }() config := conf.DefaultConfig() - srv := New(config) + // TODO + srv := New(config, nil, nil) // our custom server should receive the PDF. body, contentType := test.MergeMultipartForm(t, map[string]string{string(resource.WebhookURLArgKey): "http://localhost:3001/foo"}) req := httptest.NewRequest(http.MethodPost, mergeEndpoint, body) @@ -536,7 +543,8 @@ func TestWebhook(t *testing.T) { func TestResultFilename(t *testing.T) { config := conf.DefaultConfig() - srv := New(config) + // TODO + srv := New(config, nil, nil) body, contentType := test.MergeMultipartForm(t, map[string]string{string(resource.ResultFilenameArgKey): "foo.pdf"}) req := httptest.NewRequest(http.MethodPost, mergeEndpoint, body) req.Header.Set(echo.HeaderContentType, contentType) diff --git a/internal/app/xhttp/middleware.go b/internal/app/xhttp/middleware.go index 4c6f2262..69f7b232 100644 --- a/internal/app/xhttp/middleware.go +++ b/internal/app/xhttp/middleware.go @@ -7,7 +7,8 @@ import ( "github.com/thecodingmachine/gotenberg/internal/app/xhttp/pkg/context" "github.com/thecodingmachine/gotenberg/internal/app/xhttp/pkg/resource" "github.com/thecodingmachine/gotenberg/internal/pkg/conf" - "github.com/thecodingmachine/gotenberg/internal/pkg/pm2" + "github.com/thecodingmachine/gotenberg/internal/pkg/prinery" + "github.com/thecodingmachine/gotenberg/internal/pkg/process" "github.com/thecodingmachine/gotenberg/internal/pkg/xerror" "github.com/thecodingmachine/gotenberg/internal/pkg/xlog" "github.com/thecodingmachine/gotenberg/internal/pkg/xrand" @@ -15,7 +16,12 @@ import ( // contextMiddleware extends the default echo.Context with // our custom context.Context. -func contextMiddleware(config conf.Config, processes ...pm2.Process) echo.MiddlewareFunc { +func contextMiddleware( + config conf.Config, + manager process.Manager, + chromePrinery *prinery.Prinery, + sofficePrinery *prinery.Prinery, +) echo.MiddlewareFunc { return func(next echo.HandlerFunc) echo.HandlerFunc { return func(c echo.Context) error { // generate a unique identifier for the request. @@ -25,7 +31,7 @@ func contextMiddleware(config conf.Config, processes ...pm2.Process) echo.Middle logger := xlog.New(config.LogLevel(), trace) // extend the current echo context with our custom // context. - ctx := context.New(c, logger, config, processes...) + ctx := context.New(c, logger, config, manager, chromePrinery, sofficePrinery) // if its an healthcheck request, there // is no need to create a Resource. if ctx.Path() == pingEndpoint { diff --git a/internal/app/xhttp/option.go b/internal/app/xhttp/option.go index d79841e8..0f1131a4 100644 --- a/internal/app/xhttp/option.go +++ b/internal/app/xhttp/option.go @@ -3,53 +3,37 @@ package xhttp import ( "github.com/thecodingmachine/gotenberg/internal/app/xhttp/pkg/resource" "github.com/thecodingmachine/gotenberg/internal/pkg/conf" - "github.com/thecodingmachine/gotenberg/internal/pkg/printer" + "github.com/thecodingmachine/gotenberg/internal/pkg/print" "github.com/thecodingmachine/gotenberg/internal/pkg/xerror" ) -func mergePrinterOptions(r resource.Resource, config conf.Config) (printer.MergePrinterOptions, error) { - const op string = "xhttp.mergePrinterOptions" - waitTimeout, err := resource.WaitTimeoutArg(r, config) - if err != nil { - return printer.MergePrinterOptions{}, xerror.New(op, err) - } - return printer.MergePrinterOptions{ - WaitTimeout: waitTimeout, - }, nil -} - -func chromePrinterOptions(r resource.Resource, config conf.Config) (printer.ChromePrinterOptions, error) { - const op string = "xhttp.chromePrinterOptions" - resolver := func() (printer.ChromePrinterOptions, error) { - waitTimeout, err := resource.WaitTimeoutArg(r, config) - if err != nil { - return printer.ChromePrinterOptions{}, err - } +func chromePrintOptions(r resource.Resource, config conf.Config) (print.ChromePrintOptions, error) { + const op string = "xhttp.chromePrintOptions" + resolver := func() (print.ChromePrintOptions, error) { waitDelay, err := resource.WaitDelayArg(r, config) if err != nil { - return printer.ChromePrinterOptions{}, err + return print.ChromePrintOptions{}, err } headerHTML, footerHTML, err := resource.HeaderFooterContents(r, config) if err != nil { - return printer.ChromePrinterOptions{}, err + return print.ChromePrintOptions{}, err } paperWidth, paperHeight, err := resource.PaperSizeArgs(r, config) if err != nil { - return printer.ChromePrinterOptions{}, err + return print.ChromePrintOptions{}, err } marginTop, marginBottom, marginLeft, marginRight, err := resource.MarginArgs(r, config) if err != nil { - return printer.ChromePrinterOptions{}, err + return print.ChromePrintOptions{}, err } landscape, err := r.BoolArg(resource.LandscapeArgKey, false) if err != nil { - return printer.ChromePrinterOptions{}, err + return print.ChromePrintOptions{}, err } - return printer.ChromePrinterOptions{ - WaitTimeout: waitTimeout, + return print.ChromePrintOptions{ WaitDelay: waitDelay, HeaderHTML: headerHTML, FooterHTML: footerHTML, @@ -69,20 +53,15 @@ func chromePrinterOptions(r resource.Resource, config conf.Config) (printer.Chro return opts, nil } -func officePrinterOptions(r resource.Resource, config conf.Config) (printer.OfficePrinterOptions, error) { - const op string = "xhttp.officePrinterOptions" - resolver := func() (printer.OfficePrinterOptions, error) { - waitTimeout, err := resource.WaitTimeoutArg(r, config) - if err != nil { - return printer.OfficePrinterOptions{}, err - } +func officePrintOptions(r resource.Resource, config conf.Config) (print.OfficePrintOptions, error) { + const op string = "xhttp.officePrintOptions" + resolver := func() (print.OfficePrintOptions, error) { landscape, err := r.BoolArg(resource.LandscapeArgKey, false) if err != nil { - return printer.OfficePrinterOptions{}, err + return print.OfficePrintOptions{}, err } - return printer.OfficePrinterOptions{ - WaitTimeout: waitTimeout, - Landscape: landscape, + return print.OfficePrintOptions{ + Landscape: landscape, }, nil } opts, err := resolver() diff --git a/internal/app/xhttp/pkg/context/context.go b/internal/app/xhttp/pkg/context/context.go index d702e9d8..0012b73c 100644 --- a/internal/app/xhttp/pkg/context/context.go +++ b/internal/app/xhttp/pkg/context/context.go @@ -13,7 +13,8 @@ import ( "github.com/thecodingmachine/gotenberg/internal/app/xhttp/pkg/resource" "github.com/thecodingmachine/gotenberg/internal/pkg/conf" "github.com/thecodingmachine/gotenberg/internal/pkg/normalize" - "github.com/thecodingmachine/gotenberg/internal/pkg/pm2" + "github.com/thecodingmachine/gotenberg/internal/pkg/prinery" + "github.com/thecodingmachine/gotenberg/internal/pkg/process" "github.com/thecodingmachine/gotenberg/internal/pkg/xerror" "github.com/thecodingmachine/gotenberg/internal/pkg/xlog" ) @@ -21,20 +22,31 @@ import ( // Context extends the default echo.Context. type Context struct { echo.Context - logger xlog.Logger - config conf.Config - processes []pm2.Process - resource resource.Resource - startTime time.Time + logger xlog.Logger + config conf.Config + manager process.Manager + chromePrinery *prinery.Prinery + sofficePrinery *prinery.Prinery + resource resource.Resource + startTime time.Time } // New creates a new Context. -func New(c echo.Context, logger xlog.Logger, config conf.Config, processes ...pm2.Process) Context { +func New( + c echo.Context, + logger xlog.Logger, + config conf.Config, + manager process.Manager, + chromePrinery *prinery.Prinery, + sofficePrinery *prinery.Prinery, +) Context { return Context{ c, logger, config, - processes, + manager, + chromePrinery, + sofficePrinery, resource.Resource{}, time.Now(), } @@ -77,17 +89,52 @@ func (ctx Context) Config() conf.Config { // one of the processes is not viable. func (ctx Context) ProcessesHealthcheck() error { const op string = "context.Context.ProcessesHealthcheck" - for _, process := range ctx.processes { - if !process.IsViable() { + processes := ctx.manager.All() + for _, p := range processes { + if !ctx.manager.IsViable(p) { return xerror.New( op, - fmt.Errorf("'%s' is not viable", process.Fullname()), + fmt.Errorf("'%s' is not viable", p.ID()), ) } } return nil } +/* +MustChromePrinery returns the instance of +prinery.Prinery associated with the Context. + +This prinery.Prinery handles Google Chrome +headless. + +It panics if no instance of prinery.Prinery. +*/ +func (ctx Context) MustChromePrinery() *prinery.Prinery { + const op string = "context.Context.MustChromePrinery" + if ctx.chromePrinery == nil { + panic(fmt.Sprintf("%s: unable to retrieve the instance of Google Chrome Headless prinery.Prinery from our custom context.Context", op)) + } + return ctx.chromePrinery +} + +/* +MustSofficePrinery returns the instance of +prinery.Prinery associated with the Context. + +This prinery.Prinery handles LibreOffice +headless. + +It panics if no instance of prinery.Prinery. +*/ +func (ctx Context) MustSofficePrinery() *prinery.Prinery { + const op string = "context.Context.MustSofficePrinery" + if ctx.sofficePrinery == nil { + panic(fmt.Sprintf("%s: unable to retrieve the instance of LibreOffice Headless prinery.Prinery from our custom context.Context", op)) + } + return ctx.sofficePrinery +} + // WithResource creates a resource.Resource and // adds it to the Context. func (ctx *Context) WithResource(directoryName string) error { diff --git a/internal/app/xhttp/pkg/resource/arg.go b/internal/app/xhttp/pkg/resource/arg.go index 35c7800b..b196281d 100644 --- a/internal/app/xhttp/pkg/resource/arg.go +++ b/internal/app/xhttp/pkg/resource/arg.go @@ -76,6 +76,34 @@ func ArgKeys() []ArgKey { } } +/* +WaitTimeoutAndWaitDelayArg is a helper for retrieving +the sum of "waitTimeout" and "waitDelay" arguments +as float64. + +It also validates them against the application +configuration. +*/ +func WaitTimeoutAndWaitDelayArg(r Resource, config conf.Config) (float64, error) { + const op string = "resource.WaitTimeoutAndWaitDelayArg" + resolver := func() (float64, error) { + waitTimeout, err := WaitTimeoutArg(r, config) + if err != nil { + return waitTimeout, err + } + waitDelay, err := WaitDelayArg(r, config) + if err != nil { + return waitDelay, err + } + return waitTimeout + waitDelay, nil + } + combined, err := resolver() + if err != nil { + return combined, xerror.New(op, err) + } + return combined, nil +} + /* WaitTimeoutArg is a helper for retrieving the "waitTimeout" argument as float64. diff --git a/internal/app/xhttp/xhttp.go b/internal/app/xhttp/xhttp.go index 73241d9b..5d402964 100644 --- a/internal/app/xhttp/xhttp.go +++ b/internal/app/xhttp/xhttp.go @@ -3,15 +3,21 @@ package xhttp import ( "github.com/labstack/echo/v4" "github.com/thecodingmachine/gotenberg/internal/pkg/conf" - "github.com/thecodingmachine/gotenberg/internal/pkg/pm2" + "github.com/thecodingmachine/gotenberg/internal/pkg/prinery" + "github.com/thecodingmachine/gotenberg/internal/pkg/process" ) // New returns a custom echo.Echo. -func New(config conf.Config, processes ...pm2.Process) *echo.Echo { +func New( + config conf.Config, + manager process.Manager, + chromePrinery *prinery.Prinery, + sofficePrinery *prinery.Prinery, +) *echo.Echo { srv := echo.New() srv.HideBanner = true srv.HidePort = true - srv.Use(contextMiddleware(config, processes...)) + srv.Use(contextMiddleware(config, manager, chromePrinery, sofficePrinery)) srv.Use(loggerMiddleware()) srv.Use(cleanupMiddleware()) srv.Use(errorMiddleware()) diff --git a/internal/app/xhttp/xhttp_test.go b/internal/app/xhttp/xhttp_test.go index 685fb0a1..0d317e3d 100644 --- a/internal/app/xhttp/xhttp_test.go +++ b/internal/app/xhttp/xhttp_test.go @@ -17,7 +17,8 @@ func TestDisableChromeEndpoints(t *testing.T) { os.Setenv(conf.DisableGoogleChromeEnvVar, "1") config, err := conf.FromEnv() assert.Nil(t, err) - srv := New(config) + // TODO + srv := New(config, nil, nil) // Ping endpoint should return 200. req := httptest.NewRequest(http.MethodGet, pingEndpoint, nil) test.AssertStatusCode(t, http.StatusOK, srv, req) @@ -54,7 +55,8 @@ func TestDisableUnoconvEndpoints(t *testing.T) { os.Setenv(conf.DisableUnoconvEnvVar, "1") config, err := conf.FromEnv() assert.Nil(t, err) - srv := New(config) + // TODO + srv := New(config, nil, nil) // Ping endpoint should return 200. req := httptest.NewRequest(http.MethodGet, pingEndpoint, nil) test.AssertStatusCode(t, http.StatusOK, srv, req) @@ -91,7 +93,8 @@ func TestDisableChromeAndUnoconvEndpoints(t *testing.T) { os.Setenv(conf.DisableUnoconvEnvVar, "1") config, err := conf.FromEnv() assert.Nil(t, err) - srv := New(config) + // TODO + srv := New(config, nil, nil) // Ping endpoint should return 200. req := httptest.NewRequest(http.MethodGet, pingEndpoint, nil) test.AssertStatusCode(t, http.StatusOK, srv, req) diff --git a/internal/pkg/prinery/doc.go b/internal/pkg/prinery/doc.go new file mode 100644 index 00000000..d16971e5 --- /dev/null +++ b/internal/pkg/prinery/doc.go @@ -0,0 +1 @@ +package prinery diff --git a/internal/pkg/prinery/prinery.go b/internal/pkg/prinery/prinery.go new file mode 100644 index 00000000..da535903 --- /dev/null +++ b/internal/pkg/prinery/prinery.go @@ -0,0 +1,123 @@ +package prinery + +import ( + "context" + "fmt" + + "github.com/thecodingmachine/gotenberg/internal/pkg/print" + "github.com/thecodingmachine/gotenberg/internal/pkg/process" + "github.com/thecodingmachine/gotenberg/internal/pkg/xerror" + "github.com/thecodingmachine/gotenberg/internal/pkg/xlog" +) + +type request struct { + ctx context.Context + logger xlog.Logger + print print.Print + dest string + result chan error +} + +type worker struct { + work chan request + proc process.Process +} + +func (w *worker) do(done chan *worker) { + for { + req := <-w.work + req.result <- req.print.Print(req.ctx, req.dest, w.proc) + done <- w + } +} + +type Prinery struct { + logger xlog.Logger + manager process.Manager + work chan request + pool chan *worker + done chan *worker +} + +func New(logger xlog.Logger, manager process.Manager, key process.Key) (*Prinery, error) { + const op string = "prinery.New" + processes := manager.Processes(key) + nWorkers := len(processes) + if nWorkers == 0 { + err := fmt.Errorf("no processes found for key '%s'", string(key)) + return nil, xerror.New(op, err) + } + logger.DebugfOp(op, "found '%d' processes for key '%s'", nWorkers, string(key)) + work := make(chan request, 1) + pool := make(chan *worker, nWorkers) + done := make(chan *worker, nWorkers) + for _, p := range processes { + w := &worker{ + work: work, + proc: p, + } + pool <- w + go w.do(done) + } + return &Prinery{ + logger: logger, + manager: manager, + work: work, + pool: pool, + done: done, + }, nil +} + +func (p *Prinery) PrintRequest(ctx context.Context, logger xlog.Logger, prnt print.Print, dest string) error { + const op string = "prinery.Prinery.PrintRequest" + req := request{ + ctx: ctx, + logger: logger, + print: prnt, + dest: dest, + result: make(chan error), + } + p.dispatch(req) + err := <-req.result + if err != nil { + return xerror.New(op, err) + } + return nil +} + +func (p *Prinery) Start() { + for { + select { + case req := <-p.work: + p.dispatch(req) + case w := <-p.done: + p.completed(w) + } + } +} + +func (p *Prinery) dispatch(req request) { + const op string = "prinery.Prinery.dispatch" + select { + case w := <-p.pool: + w.work <- req + case <-req.ctx.Done(): + req.result <- xerror.New(op, req.ctx.Err()) + } +} + +func (p *Prinery) completed(w *worker) { + const op string = "prinery.Prinerty.completed" + go func() { + // check process viability. + isViable := p.manager.IsViable(w.proc) + // check process memory usage. + // TODO handle error. + memory, _ := p.manager.Memory(w.proc) + p.logger.DebugfOp(op, "%s: isViable = %t, memory = %d", w.proc.ID(), isViable, memory) + // TODO manage viability and memory usage. + // pushing back the worker. + p.pool <- w + }() + +} diff --git a/internal/pkg/print/chrome.go b/internal/pkg/print/chrome.go new file mode 100644 index 00000000..b1abdf8a --- /dev/null +++ b/internal/pkg/print/chrome.go @@ -0,0 +1,265 @@ +package print + +import ( + "context" + "fmt" + "io/ioutil" + "time" + + "github.com/mafredri/cdp" + "github.com/mafredri/cdp/devtool" + "github.com/mafredri/cdp/protocol/network" + "github.com/mafredri/cdp/protocol/page" + "github.com/mafredri/cdp/protocol/target" + "github.com/mafredri/cdp/rpcc" + "github.com/thecodingmachine/gotenberg/internal/pkg/process" + "github.com/thecodingmachine/gotenberg/internal/pkg/xerrgroup" + "github.com/thecodingmachine/gotenberg/internal/pkg/xerror" + "github.com/thecodingmachine/gotenberg/internal/pkg/xlog" + "github.com/thecodingmachine/gotenberg/internal/pkg/xtime" +) + +type chromePrint struct { + logger xlog.Logger + url string + opts ChromePrintOptions +} + +// ChromePrintOptions helps customizing the +// Google Chrome Print result. +type ChromePrintOptions struct { + WaitDelay float64 + HeaderHTML string + FooterHTML string + PaperWidth float64 + PaperHeight float64 + MarginTop float64 + MarginBottom float64 + MarginLeft float64 + MarginRight float64 + Landscape bool +} + +// DefaultChromePrintOptions returns the default +// Google Chrome Print options. +func DefaultChromePrintOptions() ChromePrintOptions { + const defaultHeaderFooterHTML string = "" + return ChromePrintOptions{ + WaitDelay: 0.0, + HeaderHTML: defaultHeaderFooterHTML, + FooterHTML: defaultHeaderFooterHTML, + PaperWidth: 8.27, + PaperHeight: 11.7, + MarginTop: 1.0, + MarginBottom: 1.0, + MarginLeft: 1.0, + MarginRight: 1.0, + Landscape: false, + } +} + +func (p chromePrint) Print(ctx context.Context, dest string, proc process.Process) error { + const op string = "print.chromePrint.Print" + resolver := func() error { + devtEndpoint := fmt.Sprintf("http://%s:%d", proc.Host(), proc.Port()) + devt, err := devtool.New(devtEndpoint).Version(ctx) + if err != nil { + return err + } + // connect to WebSocket URL (page) that speaks the Chrome DevTools Protocol. + devtConn, err := rpcc.DialContext(ctx, devt.WebSocketDebuggerURL) + if err != nil { + return err + } + defer devtConn.Close() // nolint: errcheck + // create a new CDP Client that uses conn. + devtClient := cdp.NewClient(devtConn) + newContextTarget, err := devtClient.Target.CreateBrowserContext(ctx) + if err != nil { + return err + } + /* + close the browser context when done. + we're not using the "default" context + as it may timeout before actually closing + the browser context. + see: https://github.com/mafredri/cdp/issues/101#issuecomment-524533670 + */ + disposeBrowserContextArgs := target.NewDisposeBrowserContextArgs(newContextTarget.BrowserContextID) + defer devtClient.Target.DisposeBrowserContext(context.Background(), disposeBrowserContextArgs) // nolint: errcheck + // create a new blank target with the new browser context. + createTargetArgs := target. + NewCreateTargetArgs("about:blank"). + SetBrowserContextID(newContextTarget.BrowserContextID) + newTarget, err := devtClient.Target.CreateTarget(ctx, createTargetArgs) + if err != nil { + return err + } + // connect the client to the new target. + newTargetWsURL := fmt.Sprintf("ws://%s:%d/devtools/page/%s", proc.Host(), proc.Port(), newTarget.TargetID) + newContextConn, err := rpcc.DialContext(ctx, newTargetWsURL) + if err != nil { + return err + } + defer newContextConn.Close() // nolint: errcheck + // create a new CDP Client that uses newContextConn. + targetClient := cdp.NewClient(newContextConn) + /* + close the target when done. + we're not using the "default" context + as it may timeout before actually closing + the target. + see: https://github.com/mafredri/cdp/issues/101#issuecomment-524533670 + */ + closeTargetArgs := target.NewCloseTargetArgs(newTarget.TargetID) + defer targetClient.Target.CloseTarget(context.Background(), closeTargetArgs) // nolint: errcheck + // enable all events. + if err := p.enableEvents(ctx, targetClient); err != nil { + return err + } + // listen for all events. + if err := p.listenEvents(ctx, targetClient); err != nil { + return err + } + // apply a wait delay (if any). + if p.opts.WaitDelay > 0.0 { + // wait for a given amount of time (useful for javascript delay). + p.logger.DebugfOp(op, "applying a wait delay of '%.2fs'...", p.opts.WaitDelay) + time.Sleep(xtime.Duration(p.opts.WaitDelay)) + } else { + p.logger.DebugOp(op, "no wait delay to apply, moving on...") + } + // print the page to PDF. + print, err := targetClient.Page.PrintToPDF( + ctx, + page.NewPrintToPDFArgs(). + SetPaperWidth(p.opts.PaperWidth). + SetPaperHeight(p.opts.PaperHeight). + SetMarginTop(p.opts.MarginTop). + SetMarginBottom(p.opts.MarginBottom). + SetMarginLeft(p.opts.MarginLeft). + SetMarginRight(p.opts.MarginRight). + SetLandscape(p.opts.Landscape). + SetDisplayHeaderFooter(true). + SetHeaderTemplate(p.opts.HeaderHTML). + SetFooterTemplate(p.opts.FooterHTML). + SetPrintBackground(true), + ) + if err != nil { + return err + } + if err := ioutil.WriteFile(dest, print.Data, 0644); err != nil { + return err + } + return nil + } + if err := resolver(); err != nil { + return xerror.New(op, err) + } + return nil +} + +func (p chromePrint) enableEvents(ctx context.Context, client *cdp.Client) error { + const op string = "print.chromePrint.enableEvents" + // enable all the domain events that we're interested in. + if err := xerrgroup.Run( + func() error { return client.DOM.Enable(ctx) }, + func() error { return client.Network.Enable(ctx, network.NewEnableArgs()) }, + func() error { return client.Page.Enable(ctx) }, + func() error { + return client.Page.SetLifecycleEventsEnabled(ctx, page.NewSetLifecycleEventsEnabledArgs(true)) + }, + func() error { return client.Runtime.Enable(ctx) }, + ); err != nil { + return xerror.New(op, err) + } + return nil +} + +func (p chromePrint) listenEvents(ctx context.Context, client *cdp.Client) error { + const op string = "print.chromePrint.listenEvents" + resolver := func() error { + // make sure Page events are enabled. + if err := client.Page.Enable(ctx); err != nil { + return err + } + // make sure Network events are enabled. + if err := client.Network.Enable(ctx, nil); err != nil { + return err + } + // create all clients for events. + domContentEventFired, err := client.Page.DOMContentEventFired(ctx) + if err != nil { + return err + } + defer domContentEventFired.Close() // nolint: errcheck + loadEventFired, err := client.Page.LoadEventFired(ctx) + if err != nil { + return err + } + defer loadEventFired.Close() // nolint: errcheck + lifecycleEvent, err := client.Page.LifecycleEvent(ctx) + if err != nil { + return err + } + defer lifecycleEvent.Close() // nolint: errcheck + loadingFinished, err := client.Network.LoadingFinished(ctx) + if err != nil { + return err + } + defer loadingFinished.Close() // nolint: errcheck + if _, err := client.Page.Navigate(ctx, page.NewNavigateArgs(p.url)); err != nil { + return err + } + // wait for all events. + return xerrgroup.Run( + func() error { + _, err := domContentEventFired.Recv() + if err != nil { + return err + } + p.logger.DebugOp(op, "event 'domContentEventFired' received") + return nil + }, + func() error { + _, err := loadEventFired.Recv() + if err != nil { + return err + } + p.logger.DebugOp(op, "event 'loadEventFired' received") + return nil + }, + func() error { + const networkIdleEventName string = "networkIdle" + for { + ev, err := lifecycleEvent.Recv() + if err != nil { + return err + } + p.logger.DebugfOp(op, "event '%s' received", ev.Name) + if ev.Name == networkIdleEventName { + break + } + } + return nil + }, + func() error { + _, err := loadingFinished.Recv() + if err != nil { + return err + } + p.logger.DebugOp(op, "event 'loadingFinished' received") + return nil + }, + ) + } + if err := resolver(); err != nil { + return xerror.New(op, err) + } + return nil +} + +// Compile-time checks to ensure type implements desired interfaces. +var ( + _ = Print(new(chromePrint)) +) diff --git a/internal/pkg/print/doc.go b/internal/pkg/print/doc.go new file mode 100644 index 00000000..91d3830c --- /dev/null +++ b/internal/pkg/print/doc.go @@ -0,0 +1 @@ +package print diff --git a/internal/pkg/print/html.go b/internal/pkg/print/html.go new file mode 100644 index 00000000..9a3e04f5 --- /dev/null +++ b/internal/pkg/print/html.go @@ -0,0 +1,18 @@ +package print + +import ( + "fmt" + + "github.com/thecodingmachine/gotenberg/internal/pkg/xlog" +) + +// NewHTMLPrint returns a Print for +// converting an HTML file to PDF. +func NewHTMLPrint(logger xlog.Logger, fpath string, opts ChromePrintOptions) Print { + URL := fmt.Sprintf("file://%s", fpath) + return chromePrint{ + logger: logger, + url: URL, + opts: opts, + } +} diff --git a/internal/pkg/print/markdown.go b/internal/pkg/print/markdown.go new file mode 100644 index 00000000..0be02349 --- /dev/null +++ b/internal/pkg/print/markdown.go @@ -0,0 +1,70 @@ +package print + +import ( + "bytes" + "fmt" + "html/template" + "io/ioutil" + "path/filepath" + + "github.com/microcosm-cc/bluemonday" + "github.com/russross/blackfriday/v2" + "github.com/thecodingmachine/gotenberg/internal/pkg/xerror" + "github.com/thecodingmachine/gotenberg/internal/pkg/xlog" + "github.com/thecodingmachine/gotenberg/internal/pkg/xrand" +) + +// NewMarkdownPrint returns a Print for +// converting a Markdown file to PDF. +func NewMarkdownPrint(logger xlog.Logger, fpath string, opts ChromePrintOptions) (Print, error) { + const op string = "print.NewMarkdownPrint" + resolver := func() (string, error) { + tmpl, err := template. + New(filepath.Base(fpath)). + Funcs(template.FuncMap{"toHTML": markdownToHTML}). + ParseFiles(fpath) + if err != nil { + return "", err + } + dirPath := filepath.Dir(fpath) + data := &templateData{DirPath: dirPath} + logger.DebugOp(op, "converting Markdown files to HTML...") + var buffer bytes.Buffer + if err := tmpl.Execute(&buffer, data); err != nil { + return "", err + } + baseFilename := xrand.Get() + dst := fmt.Sprintf("%s/%s.html", dirPath, baseFilename) + logger.DebugOp(op, "writing the HTML from previous conversion(s) into new file...") + if err := ioutil.WriteFile(dst, buffer.Bytes(), 0644); err != nil { + return "", err + } + return fmt.Sprintf("file://%s", dst), nil + } + URL, err := resolver() + if err != nil { + return chromePrint{}, xerror.New(op, err) + } + return chromePrint{ + logger: logger, + url: URL, + opts: opts, + }, nil +} + +type templateData struct { + DirPath string +} + +func markdownToHTML(dirPath, filename string) (template.HTML, error) { + const op string = "print.markdownToHTML" + fpath := fmt.Sprintf("%s/%s", dirPath, filename) + b, err := ioutil.ReadFile(fpath) + if err != nil { + return "", xerror.New(op, err) + } + unsafe := blackfriday.Run(b) + content := bluemonday.UGCPolicy().SanitizeBytes(unsafe) + /* #nosec */ + return template.HTML(content), nil +} diff --git a/internal/pkg/print/merge.go b/internal/pkg/print/merge.go new file mode 100644 index 00000000..a961c4fd --- /dev/null +++ b/internal/pkg/print/merge.go @@ -0,0 +1,44 @@ +package print + +import ( + "context" + + "github.com/thecodingmachine/gotenberg/internal/pkg/process" + "github.com/thecodingmachine/gotenberg/internal/pkg/xerror" + "github.com/thecodingmachine/gotenberg/internal/pkg/xexec" + "github.com/thecodingmachine/gotenberg/internal/pkg/xlog" +) + +type mergePrint struct { + logger xlog.Logger + fpaths []string +} + +// NewMergePrint returns a Print for +// merging PDF files. +func NewMergePrint(logger xlog.Logger, fpaths []string) Print { + return mergePrint{ + logger: logger, + fpaths: fpaths, + } +} + +func (p mergePrint) Print(ctx context.Context, dest string, proc process.Process) error { + const op string = "print.mergePrint.Print" + p.logger.DebugfOp(op, "merging '%v'...", p.fpaths) + resolver := func() error { + var args []string + args = append(args, p.fpaths...) + args = append(args, "cat", "output", dest) + cmd, err := xexec.CommandContext(ctx, p.logger, "pdftk", args...) + if err != nil { + return err + } + xexec.LogBeforeExecute(p.logger, cmd) + return cmd.Run() + } + if err := resolver(); err != nil { + return xerror.New(op, err) + } + return nil +} diff --git a/internal/pkg/print/office.go b/internal/pkg/print/office.go new file mode 100644 index 00000000..5bc524f8 --- /dev/null +++ b/internal/pkg/print/office.go @@ -0,0 +1,109 @@ +package print + +import ( + "context" + "fmt" + "os" + "path/filepath" + + "github.com/thecodingmachine/gotenberg/internal/pkg/process" + "github.com/thecodingmachine/gotenberg/internal/pkg/xerror" + "github.com/thecodingmachine/gotenberg/internal/pkg/xexec" + "github.com/thecodingmachine/gotenberg/internal/pkg/xlog" + "github.com/thecodingmachine/gotenberg/internal/pkg/xrand" +) + +type officePrint struct { + logger xlog.Logger + fpaths []string + opts OfficePrintOptions +} + +// OfficePrintOptions helps customizing the +// LibreOffice Print result. +type OfficePrintOptions struct { + Landscape bool +} + +// DefaultOfficePrinterOptions returns the default +// LibreOffice Print options. +func DefaultOfficePrinterOptions() OfficePrintOptions { + return OfficePrintOptions{ + Landscape: false, + } +} + +// NewOfficePrint returns a Print for +// converting Office documents to PDF. +func NewOfficePrint(logger xlog.Logger, fpaths []string, opts OfficePrintOptions) Print { + return officePrint{ + logger: logger, + fpaths: fpaths, + opts: opts, + } +} + +func (p officePrint) Print(ctx context.Context, dest string, proc process.Process) error { + const op string = "print.officePrint.Print" + resolver := func() error { + fpaths := make([]string, len(p.fpaths)) + dirPath := filepath.Dir(dest) + for i, fpath := range p.fpaths { + baseFilename := xrand.Get() + tmpDest := fmt.Sprintf("%s/%d%s.pdf", dirPath, i, baseFilename) + p.logger.DebugfOp(op, "converting '%s' to PDF...", fpath) + if err := unoconv(ctx, p.logger, fpath, tmpDest, p.opts); err != nil { + return err + } + p.logger.DebugfOp(op, "'%s.pdf' created", baseFilename) + fpaths[i] = tmpDest + } + if len(fpaths) == 1 { + p.logger.DebugOp(op, "only one PDF created, nothing to merge") + if err := os.Rename(fpaths[0], dest); err != nil { + return err + } + return nil + } + merger := NewMergePrint(p.logger, fpaths) + return merger.Print(ctx, dest, nil) + } + if err := resolver(); err != nil { + return xerror.New(op, err) + } + return nil +} + +func unoconv(ctx context.Context, logger xlog.Logger, fpath, dest string, opts OfficePrintOptions) error { + const op string = "print.unoconv" + resolver := func() error { + args := []string{ + "--format", + "pdf", + } + if opts.Landscape { + args = append(args, "--printer", "PaperOrientation=landscape") + } + args = append(args, "--output", dest, fpath) + cmd, err := xexec.CommandContext( + ctx, + logger, + "unoconv", + args..., + ) + if err != nil { + return err + } + xexec.LogBeforeExecute(logger, cmd) + return cmd.Run() + } + if err := resolver(); err != nil { + return xerror.New(op, err) + } + return nil +} + +// Compile-time checks to ensure type implements desired interfaces. +var ( + _ = Print(new(officePrint)) +) diff --git a/internal/pkg/print/print.go b/internal/pkg/print/print.go new file mode 100644 index 00000000..4b641c8e --- /dev/null +++ b/internal/pkg/print/print.go @@ -0,0 +1,13 @@ +package print + +import ( + "context" + + "github.com/thecodingmachine/gotenberg/internal/pkg/process" +) + +// Print is a type that can create a PDF file from a source. +// The source is defined in the underlying implementation. +type Print interface { + Print(ctx context.Context, dest string, proc process.Process) error +} diff --git a/internal/pkg/print/url.go b/internal/pkg/print/url.go new file mode 100644 index 00000000..9031e21d --- /dev/null +++ b/internal/pkg/print/url.go @@ -0,0 +1,15 @@ +package print + +import ( + "github.com/thecodingmachine/gotenberg/internal/pkg/xlog" +) + +// NewURLPrint returns a Print for +// converting a URL to PDF. +func NewURLPrint(logger xlog.Logger, url string, opts ChromePrintOptions) Print { + return chromePrint{ + logger: logger, + url: url, + opts: opts, + } +} diff --git a/internal/pkg/process/chrome.go b/internal/pkg/process/chrome.go new file mode 100644 index 00000000..c93dbe51 --- /dev/null +++ b/internal/pkg/process/chrome.go @@ -0,0 +1,106 @@ +package process + +import ( + "context" + "fmt" + "time" + + "github.com/mafredri/cdp/devtool" + "github.com/thecodingmachine/gotenberg/internal/pkg/xlog" +) + +const ChromeKey Key = "chrome" + +type chromeProcess struct { + id string + host string + port int +} + +// NewChromeProcess returns a Google Chrome +// headless process. +func NewChromeProcess(id, host string, port int) Process { + return chromeProcess{ + id: id, + host: host, + port: port, + } +} + +func (p chromeProcess) ID() string { + return p.id +} + +func (p chromeProcess) Host() string { + return p.host +} + +func (p chromeProcess) Port() int { + return p.port +} + +func (p chromeProcess) binary() string { + return "google-chrome-stable" +} + +func (p chromeProcess) args() []string { + return []string{ + "--no-sandbox", + "--headless", + // see https://github.com/GoogleChrome/puppeteer/issues/2410. + "--font-render-hinting=medium", + fmt.Sprintf("--remote-debugging-port=%d", p.port), + "--disable-gpu", + "--disable-translate", + "--disable-extensions", + "--disable-background-networking", + "--safebrowsing-disable-auto-update", + "--disable-sync", + "--disable-default-apps", + "--hide-scrollbars", + "--metrics-recording-only", + "--mute-audio", + "--no-first-run", + } +} + +func (p chromeProcess) warmupTime() time.Duration { + return 10 * time.Second +} + +func (p chromeProcess) viabilityFunc() func(logger xlog.Logger) bool { + const op string = "process.chromeProcess.viabilityFunc" + return func(logger xlog.Logger) bool { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + endpoint := fmt.Sprintf("http://%s:%d" /*p.host*/, "localhost", p.port) + logger.DebugfOp( + op, + "checking '%s' viability via endpoint '%s/json/version'", + p.ID(), + endpoint, + ) + v, err := devtool.New(endpoint).Version(ctx) + if err != nil { + logger.ErrorfOp( + op, + "'%s' is not viable as endpoint returned '%v'", + p.ID(), + err, + ) + return false + } + logger.DebugfOp( + op, + "'%s' is viable as endpoint returned '%v'", + p.ID(), + v, + ) + return true + } +} + +// Compile-time checks to ensure type implements desired interfaces. +var ( + _ = Process(new(chromeProcess)) +) diff --git a/internal/pkg/process/doc.go b/internal/pkg/process/doc.go new file mode 100644 index 00000000..f2ef9d28 --- /dev/null +++ b/internal/pkg/process/doc.go @@ -0,0 +1 @@ +package process diff --git a/internal/pkg/process/pm2.go b/internal/pkg/process/pm2.go new file mode 100644 index 00000000..8be78c26 --- /dev/null +++ b/internal/pkg/process/pm2.go @@ -0,0 +1,324 @@ +package process + +import ( + "encoding/json" + "fmt" + "os/exec" + "sync" + "time" + + "github.com/thecodingmachine/gotenberg/internal/pkg/conf" + "github.com/thecodingmachine/gotenberg/internal/pkg/xerror" + "github.com/thecodingmachine/gotenberg/internal/pkg/xexec" + "github.com/thecodingmachine/gotenberg/internal/pkg/xlog" + "github.com/thecodingmachine/gotenberg/internal/pkg/xtime" +) + +type jlistItem struct { + Name string `json:"name"` + PM2Env struct { + Status string `json:"status"` + RestartTime int64 `json:"restart_time"` + } `json:"pm2_env"` + Monit struct { + Memory int64 `json:"memory"` + CPU float64 `json:"cpu"` + } `json:"monit"` +} + +type jlist []jlistItem + +func (list jlist) toList() List { + var result List + for _, current := range list { + item := ListItem{ + Name: current.Name, + Status: current.PM2Env.Status, + Restart: current.PM2Env.RestartTime, + Memory: current.Monit.Memory, // TODO humanize? + CPU: current.Monit.CPU, + } + result = append(result, item) + } + return result +} + +func (list jlist) isOnline(p Process) bool { + const onlineStatus string = "online" + for _, item := range list { + if item.Name == p.ID() { + return item.PM2Env.Status == onlineStatus + } + } + return false +} + +func (list jlist) memory(p Process) (int64, error) { + const op string = "process.jlist.memory" + for _, item := range list { + if item.Name == p.ID() { + return item.Monit.Memory, nil + } + } + return 0, xerror.New( + op, + fmt.Errorf("'%s' does not exist in the list of PM2 processes", p.ID()), + ) +} + +type command string + +const ( + startCommand command = "start" + restartCommand command = "restart" + stopCommand command = "stop" + logsCommand command = "logs" + jlistCommand command = "jlist" +) + +const maximumRestartAttempts uint = 3 + +type pm2Manager struct { + logger xlog.Logger + config conf.Config + pool map[Key][]Process + list *jlist + listLock *sync.Mutex +} + +// NewPM2Manager returns a PM2 manager. +func NewPM2Manager(logger xlog.Logger, config conf.Config) Manager { + const op string = "process.NewPM2Manager" + m := &pm2Manager{ + logger: logger, + config: config, + pool: make(map[Key][]Process), + listLock: &sync.Mutex{}, + } + if !config.DisableGoogleChrome() { + processes := make([]Process, 2) + availablePort := 9222 + // TODO from config + for i := 0; i < 2; i++ { + proc := chromeProcess{ + host: "127.0.0.1", + port: availablePort, + } + proc.id = fmt.Sprintf("%s-%d", proc.binary(), proc.port) + processes[i] = proc + logger.DebugfOp(op, "added new process %v", proc) + availablePort++ + } + m.pool[ChromeKey] = processes + } + if !config.DisableUnoconv() { + processes := make([]Process, 2) + availablePort := 2002 + // TODO from config + for i := 0; i < 2; i++ { + proc := sofficeProcess{ + host: "127.0.0.1", + port: availablePort, + } + proc.id = fmt.Sprintf("%s-%d", proc.binary(), proc.port) + processes[i] = proc + logger.DebugfOp(op, "added new process %v", proc) + availablePort++ + } + m.pool[SofficeKey] = processes + } + // update the manager processes list + // only if there are processes. + if !config.DisableGoogleChrome() || !config.DisableUnoconv() { + go m.jlistTimer() + } + return m +} + +func (m *pm2Manager) jlistTimer() { + const op string = "process.pm2Manager.jlistTimer" + duration := xtime.Duration(10) + resolver := func() error { + m.listLock.Lock() + defer m.listLock.Unlock() + out, err := exec. + Command("pm2", string(jlistCommand)). + Output() + if err != nil { + return err + } + data := &jlist{} + if err := json.Unmarshal(out, data); err != nil { + return err + } + m.list = data + return nil + } + // update every x seconds the + // list from the manager. + for range time.Tick(duration) { + if err := resolver(); err != nil { + m.logger.ErrorOp(op, xerror.New(op, err)) + } + } +} + +func (m *pm2Manager) Start() error { + const op string = "process.pm2Manager.Start" + for _, processes := range m.pool { + for _, proc := range processes { + if err := m.start(proc); err != nil { + return xerror.New(op, err) + } + } + } + return nil +} + +func (m *pm2Manager) start(p Process) error { + const op string = "process.pm2Manager.start" + resolver := func() error { + // first, we try to start the process. + if err := m.run(startCommand, p); err != nil { + return err + } + // we wait the process to be ready. + m.warmup(p) + // if the process failed to start correctly, + // we have to restart it. + if !m.IsViable(p) && maximumRestartAttempts > 0 { + return m.Restart(p) + } + // the process is viable, let's log its + // output. + return m.logs(p) + } + if err := resolver(); err != nil { + return xerror.New(op, err) + } + return nil +} + +func (m *pm2Manager) List() List { + m.listLock.Lock() + defer m.listLock.Unlock() + return m.list.toList() +} + +func (m *pm2Manager) All() []Process { + var result []Process + for _, processes := range m.pool { + result = append(result, processes...) + } + return result +} + +func (m *pm2Manager) Processes(key Key) []Process { + return m.pool[key] +} + +func (m *pm2Manager) Restart(p Process) error { + const op string = "process.pm2Manager.Restart" + resolver := func() error { + var attempts uint + for attempts < maximumRestartAttempts { + // we restart the process. + if err := m.run(restartCommand, p); err != nil { + return err + } + // we wait the process to be ready. + m.warmup(p) + attempts++ + // if the process is viable, we + // leave. + if m.IsViable(p) { + return m.logs(p) + } + } + return fmt.Errorf("failed to start '%s'", p.ID()) + } + if err := resolver(); err != nil { + return xerror.New(op, err) + } + return nil +} + +func (m *pm2Manager) Stop(p Process) error { + const op string = "process.pm2Manager.Stop" + if err := m.run(stopCommand, p); err != nil { + return xerror.New(op, err) + } + return nil +} + +func (m *pm2Manager) IsViable(p Process) bool { + if !p.viabilityFunc()(m.logger) { + return false + } + m.listLock.Lock() + defer m.listLock.Unlock() + return m.list.isOnline(p) +} + +func (m *pm2Manager) Memory(p Process) (int64, error) { + const op string = "process.pm2Manager.Memory" + m.listLock.Lock() + defer m.listLock.Unlock() + memory, err := m.list.memory(p) + if err != nil { + return 0, xerror.New(op, err) + } + return memory, nil +} + +func (m *pm2Manager) warmup(p Process) { + const op string = "process.pm2Manager.warmup" + warmupTime := p.warmupTime() + m.logger.DebugfOp( + op, + "waiting '%v' for allowing '%s' to warmup", + warmupTime, + p.ID(), + ) + time.Sleep(warmupTime) +} + +func (m *pm2Manager) logs(p Process) error { + const op string = "process.pm2Manager.logs" + if m.config.LogLevel() == xlog.DebugLevel { + if err := m.run(logsCommand, p); err != nil { + return xerror.New(op, err) + } + } + return nil +} + +func (m *pm2Manager) run(pm2Cmd command, p Process) error { + const op string = "process.pm2Manager.run" + resolver := func() error { + args := []string{ + string(pm2Cmd), + p.binary(), + } + if pm2Cmd == startCommand { + args = append(args, fmt.Sprintf("--name=%s", p.ID())) + args = append(args, "--interpreter=none", "--") + args = append(args, p.args()...) + } + cmd, err := xexec.Command(m.logger, "pm2", args...) + if err != nil { + return err + } + xexec.LogBeforeExecute(m.logger, cmd) + return cmd.Start() + } + if err := resolver(); err != nil { + return xerror.New(op, err) + } + return nil +} + +// Compile-time checks to ensure type implements desired interfaces. +var ( + _ = Manager(new(pm2Manager)) +) diff --git a/internal/pkg/process/process.go b/internal/pkg/process/process.go new file mode 100644 index 00000000..898d53bc --- /dev/null +++ b/internal/pkg/process/process.go @@ -0,0 +1,40 @@ +package process + +import ( + "time" + + "github.com/thecodingmachine/gotenberg/internal/pkg/xlog" +) + +type Process interface { + ID() string + Host() string + Port() int + binary() string + args() []string + warmupTime() time.Duration + viabilityFunc() func(logger xlog.Logger) bool +} + +type ListItem struct { + Name string `json:"name"` + Status string `json:"status"` + Restart int64 `json:"restart"` + Memory int64 `json:"memory"` + CPU float64 `json:"cpu"` +} + +type List []ListItem + +type Key string + +type Manager interface { + Start() error + List() List + All() []Process + Processes(key Key) []Process + Restart(proc Process) error + Stop(proc Process) error + IsViable(proc Process) bool + Memory(proc Process) (int64, error) +} diff --git a/internal/pkg/process/soffice.go b/internal/pkg/process/soffice.go new file mode 100644 index 00000000..4f7a3be9 --- /dev/null +++ b/internal/pkg/process/soffice.go @@ -0,0 +1,74 @@ +package process + +import ( + "fmt" + "time" + + "github.com/thecodingmachine/gotenberg/internal/pkg/xlog" +) + +const SofficeKey Key = "soffice" + +type sofficeProcess struct { + id string + host string + port int +} + +// NewSofficeProcess returns a LibreOffice +// headless process. +func NewSofficeProcess(id, host string, port int) Process { + return sofficeProcess{ + id: id, + host: host, + port: port, + } +} + +func (p sofficeProcess) ID() string { + return p.id +} + +func (p sofficeProcess) Host() string { + return p.host +} + +func (p sofficeProcess) Port() int { + return p.port +} + +func (p sofficeProcess) binary() string { + return "soffice" +} + +func (p sofficeProcess) args() []string { + return []string{ + // see https://ask.libreoffice.org/en/question/42975/how-can-i-run-multiple-instances-of-sofficebin-at-a-time/. + fmt.Sprintf("-env:UserInstallation=file:///tmp/%d", p.port), + "--headless", + "--invisible", + "--nocrashreport", + "--nodefault", + "--nofirststartwizard", + "--nologo", + "--norestore", + fmt.Sprintf("--accept=socket,host=%s,port=%d,tcpNoDelay=1;urp;StarOffice.ComponentContext", p.host, p.port), + } +} + +func (p sofficeProcess) warmupTime() time.Duration { + return 3 * time.Second +} + +func (p sofficeProcess) viabilityFunc() func(logger xlog.Logger) bool { + const op string = "process.sofficeProcess.viabilityFunc" + return func(logger xlog.Logger) bool { + // TODO find a way to check. + return true + } +} + +// Compile-time checks to ensure type implements desired interfaces. +var ( + _ = Process(new(sofficeProcess)) +) diff --git a/internal/pkg/xerrgroup/doc.go b/internal/pkg/xerrgroup/doc.go new file mode 100644 index 00000000..2134c8cb --- /dev/null +++ b/internal/pkg/xerrgroup/doc.go @@ -0,0 +1,6 @@ +/* +Package xerrgroup helps running +many functions simultaneously and wait until +execution has completed or an error is encountered. +*/ +package xerrgroup diff --git a/internal/pkg/xerrgroup/xerrgroup.go b/internal/pkg/xerrgroup/xerrgroup.go new file mode 100644 index 00000000..fb4d0e24 --- /dev/null +++ b/internal/pkg/xerrgroup/xerrgroup.go @@ -0,0 +1,20 @@ +package xerrgroup + +import ( + "github.com/thecodingmachine/gotenberg/internal/pkg/xerror" + "golang.org/x/sync/errgroup" +) + +// Run runs all functions simultaneously and wait until +// execution has completed or an error is encountered. +func Run(fn ...func() error) error { + const op string = "xerrgroup.Run" + eg := errgroup.Group{} + for _, f := range fn { + eg.Go(f) + } + if err := eg.Wait(); err != nil { + return xerror.New(op, err) + } + return nil +} diff --git a/loadtesting/README.md b/loadtesting/README.md new file mode 100644 index 00000000..a55a3e1b --- /dev/null +++ b/loadtesting/README.md @@ -0,0 +1,113 @@ +# Load testing + +You may wonder how Gotenberg behaves under load. + +In order to help you having an idea, we created a bunch of scenarios for the +[k6](https://docs.k6.io/docs) load testing tool. + +The Gotenberg container (version `6.0.0` and default options) was hosted on a AWS EC2 `t2.micro` instance (1 vCPU and 1 Go of RAM, low to average network performances). + +The k6 scenarios have been performed on a MacBook Pro 2016 (2 GHz Intel Core i5 and 16 Go 1867 MHz LPDDR3). + +## HTML + +The HTML scenario is quite simple: + +* Ramp up from 0 to 100 virtual users, each one uploading as many time as possible the [HTML test data](../test/testdata/html). +* Stop when at least one response is not an HTTP 200 code. + +```bash +$ k6 run --env MAX_VUS=100 --env BASE_URL=http://ec2-foo.eu-west-1.compute.amazonaws.com html.js + + /\ |‾‾| /‾‾/ /‾/ + /\ / \ | |_/ / / / + / \/ \ | | / ‾‾\ + / \ | |‾\ \ | (_) | + / __________ \ |__| \__\ \___/ .io + + execution: local + output: - + script: html.js + + duration: -, iterations: - + vus: 1, max: 100 + + done [==========================================================] 1m11.9s / 10m0s + + ✗ is status 200 + ↳ 99% — ✓ 179 / ✗ 1 + ✗ is not status 504 + ↳ 99% — ✓ 179 / ✗ 1 + ✓ is not status 500 + + checks.....................: 99.62% ✓ 538 ✗ 2 + data_received..............: 41 MB 576 kB/s + data_sent..................: 8.5 MB 118 kB/s + ✗ failed requests............: 1 0.013895/s + http_req_blocked...........: avg=1.58ms min=2µs med=5µs max=59.73ms p(90)=10.19µs p(95)=20.57ms + http_req_connecting........: avg=1.56ms min=0s med=0s max=59.61ms p(90)=0s p(95)=20.48ms + http_req_duration..........: avg=2.3s min=1.32s med=1.67s max=10.2s p(90)=4.41s p(95)=7.54s + http_req_receiving.........: avg=99.01ms min=127µs med=98.4ms max=154.86ms p(90)=115.92ms p(95)=122.42ms + http_req_sending...........: avg=232.69µs min=133µs med=207µs max=1.16ms p(90)=335.4µs p(95)=382.64µs + http_req_tls_handshaking...: avg=0s min=0s med=0s max=0s p(90)=0s p(95)=0s + http_req_waiting...........: avg=2.2s min=1.21s med=1.56s max=10.2s p(90)=4.32s p(95)=7.45s + http_reqs..................: 180 2.501138/s + iteration_duration.........: avg=2.3s min=1.33s med=1.67s max=10.22s p(90)=4.45s p(95)=7.54s + iterations.................: 180 2.501138/s + vus........................: 12 min=1 max=12 + vus_max....................: 100 min=100 max=100 +``` + +In our use case, when reaching 12 virtual users (~2,5 requests per second), some incoming requests cannot be fulfilled before 10 seconds (`DEFAULT_WAIT_TIMEOUT` value). +During this test, CPU usage went from 0 to 100% and memory usage stayed low (Google Chrome go from 64.1 MB to 64.9 MB). + +## Office + +The Office scenario is the same as the HTML scenario, but with a [document.docx](../test/testdata/office/document.docx). + +```bash +$ k6 run --env MAX_VUS=100 --env BASE_URL=http://ec2-foo.eu-west-1.compute.amazonaws.com office.js + + /\ |‾‾| /‾‾/ /‾/ + /\ / \ | |_/ / / / + / \/ \ | | / ‾‾\ + / \ | |‾\ \ | (_) | + / __________ \ |__| \__\ \___/ .io + + execution: local + output: - + script: office.js + + duration: -, iterations: - + vus: 1, max: 100 + + done [==========================================================] 2m7.9s / 10m0s + + ✗ is status 200 + ↳ 99% — ✓ 481 / ✗ 3 + ✗ is not status 504 + ↳ 99% — ✓ 481 / ✗ 3 + ✓ is not status 500 + + checks.....................: 99.58% ✓ 1446 ✗ 6 + data_received..............: 40 MB 312 kB/s + data_sent..................: 45 MB 348 kB/s + ✗ failed requests............: 3 0.023446/s + http_req_blocked...........: avg=1.85ms min=2µs med=5µs max=373.57ms p(90)=12.69µs p(95)=23.84µs + http_req_connecting........: avg=1.11ms min=0s med=0s max=47.62ms p(90)=0s p(95)=0s + http_req_duration..........: avg=2.65s min=289.89ms med=2.48s max=10.08s p(90)=4.51s p(95)=5.43s + http_req_receiving.........: avg=57.15ms min=79µs med=51.53ms max=200.89ms p(90)=81.12ms p(95)=91.61ms + http_req_sending...........: avg=387.86µs min=159µs med=332µs max=3.87ms p(90)=513.1µs p(95)=695µs + http_req_tls_handshaking...: avg=0s min=0s med=0s max=0s p(90)=0s p(95)=0s + http_req_waiting...........: avg=2.59s min=264.48ms med=2.4s max=10.08s p(90)=4.45s p(95)=5.37s + http_reqs..................: 484 3.782635/s + iteration_duration.........: avg=2.65s min=290.24ms med=2.48s max=10.08s p(90)=4.51s p(95)=5.43s + iterations.................: 484 3.782635/s + vus........................: 22 min=1 max=22 + vus_max....................: 100 min=100 max=100 +``` + +In our use case, when reaching 22 virtual users (~3,7 requests per second), some incoming requests cannot be fulfilled before 10 seconds (`DEFAULT_WAIT_TIMEOUT` value). +During this test, CPU usage went from 0 to 100% and memory usage stayed low. + +## Merge \ No newline at end of file diff --git a/loadtesting/html.js b/loadtesting/html.js new file mode 100644 index 00000000..b6965a35 --- /dev/null +++ b/loadtesting/html.js @@ -0,0 +1,44 @@ +import http from "k6/http"; +import { Counter } from "k6/metrics"; +import { check } from "k6"; + +let indexFile = open("../test/testdata/html/index.html", "b"), + styleFile = open("../test/testdata/html/style.css", "b"), + headerFile = open("../test/testdata/html/header.html", "b"), + footerFile = open("../test/testdata/html/footer.html", "b"), + fontFile = open("../test/testdata/html/font.woff", "b"), + imgFile = open("../test/testdata/html/img.gif", "b"); + +let failCounter = new Counter("failed requests"); + +export let options = { + stages: [ + { duration: "10m", target: __ENV.MAX_VUS } + ], + thresholds: { + "failed requests": [{ + threshold: "count<1", + abortOnFail: true, + }] + } +} + +export default function() { + let data = { + "index.html": http.file(indexFile, "index.html"), + "style.css": http.file(styleFile, "style.css"), + "header.html": http.file(headerFile, "header.html"), + "footer.html": http.file(footerFile, "footer.html"), + "font.woff": http.file(fontFile, "font.woff"), + "img.gif": http.file(imgFile, "img.gif") + } + let res = http.post(__ENV.BASE_URL + '/convert/html', data); + check(res, { + "is status 200": (r) => r.status === 200, + "is not status 504": (r) => r.status !== 504, + "is not status 500": (r) => r.status !== 500 + }); + if (res.status !== 200) { + failCounter.add(1); + } +} \ No newline at end of file diff --git a/loadtesting/merge.js b/loadtesting/merge.js new file mode 100644 index 00000000..6229743a --- /dev/null +++ b/loadtesting/merge.js @@ -0,0 +1,18 @@ +import http from "k6/http"; +import { check } from "k6"; + +let pdf1File = open("../test/testdata/pdf/gotenberg.pdf", "b"), + pdf2File = open("../test/testdata/pdf/gotenberg_bis.pdf", "b"); + +export default function() { + var data = { + "gotenberg.pdf": http.file(pdf1File, "gotenberg.pdf"), + "gotenberg_bis.pdf": http.file(pdf2File, "gotenberg_bis.pdf") + } + var res = http.post(__ENV.BASE_URL + '/merge', data); + check(res, { + "is status 200": (r) => r.status === 200, + "is not status 504": (r) => r.status !== 504, + "is not status 500": (r) => r.status !== 500, + }); +} \ No newline at end of file diff --git a/loadtesting/office.js b/loadtesting/office.js new file mode 100644 index 00000000..6df88069 --- /dev/null +++ b/loadtesting/office.js @@ -0,0 +1,34 @@ +import http from "k6/http"; +import { Counter } from "k6/metrics"; +import { check } from "k6"; + +let documentFile = open("../test/testdata/office/document.docx", "b"); + +let failCounter = new Counter("failed requests"); + +export let options = { + stages: [ + { duration: "10m", target: __ENV.MAX_VUS } + ], + thresholds: { + "failed requests": [{ + threshold: "count<1", + abortOnFail: true, + }] + } +} + +export default function() { + let data = { + "document.docx": http.file(documentFile, "document.docx") + } + let res = http.post(__ENV.BASE_URL + '/convert/office', data); + check(res, { + "is status 200": (r) => r.status === 200, + "is not status 504": (r) => r.status !== 504, + "is not status 500": (r) => r.status !== 500, + }); + if (res.status !== 200) { + failCounter.add(1); + } +} \ No newline at end of file