process balancing: better implementation (wip)

This commit is contained in:
Julien Neuhart
2019-09-23 18:29:24 +02:00
parent 171f93662f
commit 88f74d5e9c
16 changed files with 1270 additions and 262 deletions

View File

@@ -1,6 +1,7 @@
package xhttp
import (
timeoutContext "context"
"fmt"
"net/http"
"os"
@@ -9,8 +10,6 @@ import (
"github.com/thecodingmachine/gotenberg/internal/app/xhttp/pkg/context"
"github.com/thecodingmachine/gotenberg/internal/app/xhttp/pkg/resource"
"github.com/thecodingmachine/gotenberg/internal/pkg/pm2"
"github.com/thecodingmachine/gotenberg/internal/pkg/prinery"
"github.com/thecodingmachine/gotenberg/internal/pkg/print"
"github.com/thecodingmachine/gotenberg/internal/pkg/xcontext"
"github.com/thecodingmachine/gotenberg/internal/pkg/xerror"
"github.com/thecodingmachine/gotenberg/internal/pkg/xlog"
@@ -72,8 +71,11 @@ func mergeHandler(c echo.Context) error {
if err != nil {
return err
}
p := print.NewMergePrint(logger, fpaths)
return convert(ctx, nil, p, timeout)
prinry := ctx.Prinery()
printFunc := func(ctx timeoutContext.Context, logger xlog.Logger, dest string) error {
return prinry.Merge(ctx, logger, dest, fpaths)
}
return convert(ctx, timeout, printFunc)
}
if err := resolver(); err != nil {
return xerror.New(op, err)
@@ -87,7 +89,6 @@ func htmlHandler(c echo.Context) error {
const op string = "xhttp.htmlHandler"
resolver := func() error {
ctx := context.MustCastFromEchoContext(c)
prinry := ctx.MustChromePrinery()
logger := ctx.XLogger()
logger.DebugOp(op, "handling HTML request...")
config := ctx.Config()
@@ -104,8 +105,11 @@ func htmlHandler(c echo.Context) error {
if err != nil {
return err
}
p := print.NewHTMLPrint(logger, fpath, opts)
return convert(ctx, prinry, p, timeout)
prinry := ctx.Prinery()
printFunc := func(ctx timeoutContext.Context, logger xlog.Logger, dest string) error {
return prinry.HTML(ctx, logger, dest, fpath, opts)
}
return convert(ctx, timeout, printFunc)
}
if err := resolver(); err != nil {
return xerror.New(op, err)
@@ -119,7 +123,6 @@ func urlHandler(c echo.Context) error {
const op string = "xhttp.urlHandler"
resolver := func() error {
ctx := context.MustCastFromEchoContext(c)
prinry := ctx.MustChromePrinery()
logger := ctx.XLogger()
logger.DebugOp(op, "handling URL request...")
config := ctx.Config()
@@ -143,8 +146,11 @@ func urlHandler(c echo.Context) error {
if err != nil {
return err
}
p := print.NewURLPrint(logger, remoteURL, opts)
return convert(ctx, prinry, p, timeout)
prinry := ctx.Prinery()
printFunc := func(ctx timeoutContext.Context, logger xlog.Logger, dest string) error {
return prinry.URL(ctx, logger, dest, remoteURL, opts)
}
return convert(ctx, timeout, printFunc)
}
if err := resolver(); err != nil {
return xerror.New(op, err)
@@ -158,7 +164,6 @@ func markdownHandler(c echo.Context) error {
const op string = "xhttp.markdownHandler"
resolver := func() error {
ctx := context.MustCastFromEchoContext(c)
prinry := ctx.MustChromePrinery()
logger := ctx.XLogger()
logger.DebugOp(op, "handling Markdown request...")
config := ctx.Config()
@@ -175,11 +180,11 @@ func markdownHandler(c echo.Context) error {
if err != nil {
return err
}
p, err := print.NewMarkdownPrint(logger, fpath, opts)
if err != nil {
return err
prinry := ctx.Prinery()
printFunc := func(ctx timeoutContext.Context, logger xlog.Logger, dest string) error {
return prinry.Markdown(ctx, logger, dest, fpath, opts)
}
return convert(ctx, prinry, p, timeout)
return convert(ctx, timeout, printFunc)
}
if err := resolver(); err != nil {
return xerror.New(op, err)
@@ -193,7 +198,6 @@ func officeHandler(c echo.Context) error {
const op string = "xhttp.officeHandler"
resolver := func() error {
ctx := context.MustCastFromEchoContext(c)
prinry := ctx.MustSofficePrinery()
logger := ctx.XLogger()
logger.DebugOp(op, "handling Office request...")
config := ctx.Config()
@@ -202,7 +206,7 @@ func officeHandler(c echo.Context) error {
if err != nil {
return err
}
opts, err := officePrintOptions(r, config)
opts, err := unoconvPrintOptions(r, config)
if err != nil {
return err
}
@@ -223,8 +227,11 @@ func officeHandler(c echo.Context) error {
if err != nil {
return err
}
p := print.NewOfficePrint(logger, fpaths, opts)
return convert(ctx, prinry, p, timeout)
prinry := ctx.Prinery()
printFunc := func(ctx timeoutContext.Context, logger xlog.Logger, dest string) error {
return prinry.Office(ctx, logger, dest, fpaths, opts)
}
return convert(ctx, timeout, printFunc)
}
if err := resolver(); err != nil {
return xerror.New(op, err)
@@ -232,7 +239,11 @@ func officeHandler(c echo.Context) error {
return nil
}
func convert(ctx context.Context, prinry *prinery.Prinery, prnt print.Print, timeout float64) error {
func convert(
ctx context.Context,
timeout float64,
printFunc func(ctx timeoutContext.Context, logger xlog.Logger, dest string) error,
) error {
const op string = "xhttp.convert"
resolver := func() error {
logger := ctx.XLogger()
@@ -245,13 +256,13 @@ func convert(ctx context.Context, prinry *prinery.Prinery, prnt print.Print, tim
// or an error.
if !r.HasArg(resource.WebhookURLArgKey) {
logger.DebugfOp(op, "no '%s' found, converting synchronously", resource.WebhookURLArgKey)
return convertSync(ctx, prinry, prnt, timeout, filename, fpath)
return convertSync(ctx, timeout, filename, fpath, printFunc)
}
// as a webhook URL has been given, we
// run the following lines in a goroutine so that
// it doesn't block.
logger.DebugfOp(op, "'%s' found, converting asynchronously", resource.WebhookURLArgKey)
return convertAsync(ctx, prinry, prnt, timeout, filename, fpath)
return convertAsync(ctx, timeout, filename, fpath, printFunc)
}
if err := resolver(); err != nil {
return xerror.New(op, err)
@@ -259,19 +270,19 @@ func convert(ctx context.Context, prinry *prinery.Prinery, prnt print.Print, tim
return nil
}
func convertSync(ctx context.Context, prinry *prinery.Prinery, prnt print.Print, timeout float64, filename, fpath string) error {
func convertSync(
ctx context.Context,
timeout float64,
filename, dest string,
printFunc func(ctx timeoutContext.Context, logger xlog.Logger, dest string) error,
) error {
const op = "xhttp.convertSync"
logger := ctx.XLogger()
r := ctx.MustResource()
timeoutCtx, cancel := xcontext.WithTimeout(logger, timeout)
defer cancel()
resolver := func() error {
if prinry == nil {
// case: merge.
if err := prnt.Print(timeoutCtx, fpath, nil); err != nil {
return err
}
} else if err := prinry.PrintRequest(timeoutCtx, logger, prnt, fpath); err != nil {
if err := printFunc(timeoutCtx, logger, dest); err != nil {
return err
}
if !r.HasArg(resource.ResultFilenameArgKey) {
@@ -281,7 +292,7 @@ func convertSync(ctx context.Context, prinry *prinery.Prinery, prnt print.Print,
resource.RemoteURLArgKey,
filename,
)
if err := ctx.Attachment(fpath, filename); err != nil {
if err := ctx.Attachment(dest, filename); err != nil {
return err
}
return nil
@@ -295,7 +306,7 @@ func convertSync(ctx context.Context, prinry *prinery.Prinery, prnt print.Print,
if err != nil {
return err
}
if err := ctx.Attachment(fpath, filename); err != nil {
if err := ctx.Attachment(dest, filename); err != nil {
return err
}
return nil
@@ -309,7 +320,12 @@ func convertSync(ctx context.Context, prinry *prinery.Prinery, prnt print.Print,
return nil
}
func convertAsync(ctx context.Context, prinry *prinery.Prinery, prnt print.Print, timeout float64, filename, fpath string) error {
func convertAsync(
ctx context.Context,
timeout float64,
filename, dest string,
printFunc func(ctx timeoutContext.Context, logger xlog.Logger, dest string) error,
) error {
const op = "xhttp.convertAsync"
logger := ctx.XLogger()
r := ctx.MustResource()
@@ -325,19 +341,12 @@ func convertAsync(ctx context.Context, prinry *prinery.Prinery, prnt print.Print
defer r.Close() // nolint: errcheck
timeoutCtx, cancel := xcontext.WithTimeout(logger, timeout)
defer cancel()
if prinry == nil {
// case: merge.
if err := prnt.Print(timeoutCtx, fpath, nil); err != nil {
xerr := xerror.New(op, err)
logger.ErrorOp(xerror.Op(xerr), xerr)
return
}
} else if err := prinry.PrintRequest(timeoutCtx, logger, prnt, fpath); err != nil {
if err := printFunc(timeoutCtx, logger, dest); err != nil {
xerr := xerror.New(op, err)
logger.ErrorOp(xerror.Op(xerr), xerr)
return
}
f, err := os.Open(fpath)
f, err := os.Open(dest)
if err != nil {
xerr := xerror.New(op, err)
logger.ErrorOp(xerror.Op(xerr), xerr)

View File

@@ -8,7 +8,6 @@ import (
"github.com/thecodingmachine/gotenberg/internal/app/xhttp/pkg/resource"
"github.com/thecodingmachine/gotenberg/internal/pkg/conf"
"github.com/thecodingmachine/gotenberg/internal/pkg/prinery"
"github.com/thecodingmachine/gotenberg/internal/pkg/process"
"github.com/thecodingmachine/gotenberg/internal/pkg/xerror"
"github.com/thecodingmachine/gotenberg/internal/pkg/xlog"
"github.com/thecodingmachine/gotenberg/internal/pkg/xrand"
@@ -16,12 +15,7 @@ import (
// contextMiddleware extends the default echo.Context with
// our custom context.Context.
func contextMiddleware(
config conf.Config,
manager process.Manager,
chromePrinery *prinery.Prinery,
sofficePrinery *prinery.Prinery,
) echo.MiddlewareFunc {
func contextMiddleware(config conf.Config, prinry prinery.Prinery) echo.MiddlewareFunc {
return func(next echo.HandlerFunc) echo.HandlerFunc {
return func(c echo.Context) error {
// generate a unique identifier for the request.
@@ -31,7 +25,7 @@ func contextMiddleware(
logger := xlog.New(config.LogLevel(), trace)
// extend the current echo context with our custom
// context.
ctx := context.New(c, logger, config, manager, chromePrinery, sofficePrinery)
ctx := context.New(c, logger, config, prinry)
// if its an healthcheck request, there
// is no need to create a Resource.
if ctx.Path() == pingEndpoint {

View File

@@ -3,37 +3,37 @@ package xhttp
import (
"github.com/thecodingmachine/gotenberg/internal/app/xhttp/pkg/resource"
"github.com/thecodingmachine/gotenberg/internal/pkg/conf"
"github.com/thecodingmachine/gotenberg/internal/pkg/print"
"github.com/thecodingmachine/gotenberg/internal/pkg/prinery"
"github.com/thecodingmachine/gotenberg/internal/pkg/xerror"
)
func chromePrintOptions(r resource.Resource, config conf.Config) (print.ChromePrintOptions, error) {
func chromePrintOptions(r resource.Resource, config conf.Config) (prinery.ChromePrintOptions, error) {
const op string = "xhttp.chromePrintOptions"
resolver := func() (print.ChromePrintOptions, error) {
resolver := func() (prinery.ChromePrintOptions, error) {
waitDelay, err := resource.WaitDelayArg(r, config)
if err != nil {
return print.ChromePrintOptions{}, err
return prinery.ChromePrintOptions{}, err
}
headerHTML, footerHTML,
err := resource.HeaderFooterContents(r, config)
if err != nil {
return print.ChromePrintOptions{}, err
return prinery.ChromePrintOptions{}, err
}
paperWidth, paperHeight,
err := resource.PaperSizeArgs(r, config)
if err != nil {
return print.ChromePrintOptions{}, err
return prinery.ChromePrintOptions{}, err
}
marginTop, marginBottom, marginLeft, marginRight,
err := resource.MarginArgs(r, config)
if err != nil {
return print.ChromePrintOptions{}, err
return prinery.ChromePrintOptions{}, err
}
landscape, err := r.BoolArg(resource.LandscapeArgKey, false)
if err != nil {
return print.ChromePrintOptions{}, err
return prinery.ChromePrintOptions{}, err
}
return print.ChromePrintOptions{
return prinery.ChromePrintOptions{
WaitDelay: waitDelay,
HeaderHTML: headerHTML,
FooterHTML: footerHTML,
@@ -53,14 +53,14 @@ func chromePrintOptions(r resource.Resource, config conf.Config) (print.ChromePr
return opts, nil
}
func officePrintOptions(r resource.Resource, config conf.Config) (print.OfficePrintOptions, error) {
const op string = "xhttp.officePrintOptions"
resolver := func() (print.OfficePrintOptions, error) {
func unoconvPrintOptions(r resource.Resource, config conf.Config) (prinery.UnoconvPrintOptions, error) {
const op string = "xhttp.unoconvPrintOptions"
resolver := func() (prinery.UnoconvPrintOptions, error) {
landscape, err := r.BoolArg(resource.LandscapeArgKey, false)
if err != nil {
return print.OfficePrintOptions{}, err
return prinery.UnoconvPrintOptions{}, err
}
return print.OfficePrintOptions{
return prinery.UnoconvPrintOptions{
Landscape: landscape,
}, nil
}

View File

@@ -14,7 +14,6 @@ import (
"github.com/thecodingmachine/gotenberg/internal/pkg/conf"
"github.com/thecodingmachine/gotenberg/internal/pkg/normalize"
"github.com/thecodingmachine/gotenberg/internal/pkg/prinery"
"github.com/thecodingmachine/gotenberg/internal/pkg/process"
"github.com/thecodingmachine/gotenberg/internal/pkg/xerror"
"github.com/thecodingmachine/gotenberg/internal/pkg/xlog"
)
@@ -22,13 +21,11 @@ import (
// Context extends the default echo.Context.
type Context struct {
echo.Context
logger xlog.Logger
config conf.Config
manager process.Manager
chromePrinery *prinery.Prinery
sofficePrinery *prinery.Prinery
resource resource.Resource
startTime time.Time
logger xlog.Logger
config conf.Config
prinry prinery.Prinery
resource resource.Resource
startTime time.Time
}
// New creates a new Context.
@@ -36,17 +33,13 @@ func New(
c echo.Context,
logger xlog.Logger,
config conf.Config,
manager process.Manager,
chromePrinery *prinery.Prinery,
sofficePrinery *prinery.Prinery,
prinry prinery.Prinery,
) Context {
return Context{
c,
logger,
config,
manager,
chromePrinery,
sofficePrinery,
prinry,
resource.Resource{},
time.Now(),
}
@@ -89,7 +82,8 @@ func (ctx Context) Config() conf.Config {
// one of the processes is not viable.
func (ctx Context) ProcessesHealthcheck() error {
const op string = "context.Context.ProcessesHealthcheck"
processes := ctx.manager.All()
// TODO
/*processes := ctx.manager.All()
for _, p := range processes {
if !ctx.manager.IsViable(p) {
return xerror.New(
@@ -97,42 +91,14 @@ func (ctx Context) ProcessesHealthcheck() error {
fmt.Errorf("'%s' is not viable", p.ID()),
)
}
}
}*/
return nil
}
/*
MustChromePrinery returns the instance of
prinery.Prinery associated with the Context.
This prinery.Prinery handles Google Chrome
headless.
It panics if no instance of prinery.Prinery.
*/
func (ctx Context) MustChromePrinery() *prinery.Prinery {
const op string = "context.Context.MustChromePrinery"
if ctx.chromePrinery == nil {
panic(fmt.Sprintf("%s: unable to retrieve the instance of Google Chrome Headless prinery.Prinery from our custom context.Context", op))
}
return ctx.chromePrinery
}
/*
MustSofficePrinery returns the instance of
prinery.Prinery associated with the Context.
This prinery.Prinery handles LibreOffice
headless.
It panics if no instance of prinery.Prinery.
*/
func (ctx Context) MustSofficePrinery() *prinery.Prinery {
const op string = "context.Context.MustSofficePrinery"
if ctx.sofficePrinery == nil {
panic(fmt.Sprintf("%s: unable to retrieve the instance of LibreOffice Headless prinery.Prinery from our custom context.Context", op))
}
return ctx.sofficePrinery
// Prinery returns the instance of prinery.Prinery
// associated with the Context.
func (ctx Context) Prinery() prinery.Prinery {
return ctx.prinry
}
// WithResource creates a resource.Resource and

View File

@@ -4,20 +4,14 @@ import (
"github.com/labstack/echo/v4"
"github.com/thecodingmachine/gotenberg/internal/pkg/conf"
"github.com/thecodingmachine/gotenberg/internal/pkg/prinery"
"github.com/thecodingmachine/gotenberg/internal/pkg/process"
)
// New returns a custom echo.Echo.
func New(
config conf.Config,
manager process.Manager,
chromePrinery *prinery.Prinery,
sofficePrinery *prinery.Prinery,
) *echo.Echo {
func New(config conf.Config, prinry prinery.Prinery) *echo.Echo {
srv := echo.New()
srv.HideBanner = true
srv.HidePort = true
srv.Use(contextMiddleware(config, manager, chromePrinery, sofficePrinery))
srv.Use(contextMiddleware(config, prinry))
srv.Use(loggerMiddleware())
srv.Use(cleanupMiddleware())
srv.Use(errorMiddleware())