process balancing: better implementation (wip)

This commit is contained in:
Julien Neuhart
2019-09-23 18:29:24 +02:00
parent 171f93662f
commit 88f74d5e9c
16 changed files with 1270 additions and 262 deletions

View File

@@ -201,6 +201,16 @@ func (c Config) DefaultListenPort() int64 {
return c.defaultListenPort
}
func (c Config) GoogleChromeInstances() int64 {
// TODO: replace DisableGoogleChrome
return 2
}
func (c Config) SofficeInstances() int64 {
// TODO: replace DisableUnoconv
return 2
}
/*
DisableGoogleChrome returns true if
Google Chrome is disabled in the

View File

@@ -0,0 +1,327 @@
package prinery
import (
"context"
"fmt"
"io/ioutil"
"time"
"github.com/mafredri/cdp"
"github.com/mafredri/cdp/devtool"
"github.com/mafredri/cdp/protocol/network"
"github.com/mafredri/cdp/protocol/page"
"github.com/mafredri/cdp/protocol/target"
"github.com/mafredri/cdp/rpcc"
"github.com/thecodingmachine/gotenberg/internal/pkg/xerrgroup"
"github.com/thecodingmachine/gotenberg/internal/pkg/xerror"
"github.com/thecodingmachine/gotenberg/internal/pkg/xlog"
"github.com/thecodingmachine/gotenberg/internal/pkg/xtime"
)
type chromeProcess struct {
devtPort uint
}
func newChromeProcesses(nInstances int64) []process {
processes := make([]process, nInstances)
var i int64
var currentPort uint = 9222
for i = 0; i < nInstances; i++ {
processes[i] = chromeProcess{
devtPort: currentPort,
}
currentPort++
}
return processes
}
func (p chromeProcess) id() string {
return fmt.Sprintf("%s-%d", p.binary(), p.port())
}
func (p chromeProcess) host() string {
return "127.0.0.1"
}
func (p chromeProcess) port() uint {
return p.devtPort
}
func (p chromeProcess) spec() processSpec {
return p
}
func (p chromeProcess) binary() string {
return "google-chrome-stable"
}
func (p chromeProcess) args() []string {
return []string{
"--no-sandbox",
"--headless",
// see https://github.com/GoogleChrome/puppeteer/issues/2410.
"--font-render-hinting=medium",
fmt.Sprintf("--remote-debugging-port=%d", p.port()),
"--disable-gpu",
"--disable-translate",
"--disable-extensions",
"--disable-background-networking",
"--safebrowsing-disable-auto-update",
"--disable-sync",
"--disable-default-apps",
"--hide-scrollbars",
"--metrics-recording-only",
"--mute-audio",
"--no-first-run",
}
}
func (p chromeProcess) warmupTime() time.Duration {
return 10 * time.Second
}
func (p chromeProcess) viabilityFunc() func(logger xlog.Logger) bool {
const op string = "prinery.chromeProcess.viabilityFunc"
return func(logger xlog.Logger) bool {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
endpoint := fmt.Sprintf("http://%s:%d" /*p.host()*/, "localhost", p.port())
logger.DebugfOp(
op,
"checking '%s' viability via endpoint '%s/json/version'",
p.id(),
endpoint,
)
v, err := devtool.New(endpoint).Version(ctx)
if err != nil {
logger.ErrorfOp(
op,
"'%s' is not viable as endpoint returned '%v'",
p.id(),
err,
)
return false
}
logger.DebugfOp(
op,
"'%s' is viable as endpoint returned '%v'",
p.id(),
v,
)
return true
}
}
type chromePrinter struct {
logger xlog.Logger
url string
opts ChromePrintOptions
}
func (p chromePrinter) print(ctx context.Context, spec processSpec, dest string) error {
const op string = "prinery.chromePrinter.print"
resolver := func() error {
devtEndpoint := fmt.Sprintf("http://%s:%d", spec.host(), spec.port())
devt, err := devtool.New(devtEndpoint).Version(ctx)
if err != nil {
return err
}
// connect to WebSocket URL (page) that speaks the Chrome DevTools Protocol.
devtConn, err := rpcc.DialContext(ctx, devt.WebSocketDebuggerURL)
if err != nil {
return err
}
defer devtConn.Close() // nolint: errcheck
// create a new CDP Client that uses conn.
devtClient := cdp.NewClient(devtConn)
newContextTarget, err := devtClient.Target.CreateBrowserContext(ctx)
if err != nil {
return err
}
/*
close the browser context when done.
we're not using the "default" context
as it may timeout before actually closing
the browser context.
see: https://github.com/mafredri/cdp/issues/101#issuecomment-524533670
*/
disposeBrowserContextArgs := target.NewDisposeBrowserContextArgs(newContextTarget.BrowserContextID)
defer devtClient.Target.DisposeBrowserContext(context.Background(), disposeBrowserContextArgs) // nolint: errcheck
// create a new blank target with the new browser context.
createTargetArgs := target.
NewCreateTargetArgs("about:blank").
SetBrowserContextID(newContextTarget.BrowserContextID)
newTarget, err := devtClient.Target.CreateTarget(ctx, createTargetArgs)
if err != nil {
return err
}
// connect the client to the new target.
newTargetWsURL := fmt.Sprintf("ws://%s:%d/devtools/page/%s", spec.host(), spec.port(), newTarget.TargetID)
newContextConn, err := rpcc.DialContext(ctx, newTargetWsURL)
if err != nil {
return err
}
defer newContextConn.Close() // nolint: errcheck
// create a new CDP Client that uses newContextConn.
targetClient := cdp.NewClient(newContextConn)
/*
close the target when done.
we're not using the "default" context
as it may timeout before actually closing
the target.
see: https://github.com/mafredri/cdp/issues/101#issuecomment-524533670
*/
closeTargetArgs := target.NewCloseTargetArgs(newTarget.TargetID)
defer targetClient.Target.CloseTarget(context.Background(), closeTargetArgs) // nolint: errcheck
// enable all events.
if err := p.enableEvents(ctx, targetClient); err != nil {
return err
}
// listen for all events.
if err := p.listenEvents(ctx, targetClient); err != nil {
return err
}
// apply a wait delay (if any).
if p.opts.WaitDelay > 0.0 {
// wait for a given amount of time (useful for javascript delay).
p.logger.DebugfOp(op, "applying a wait delay of '%.2fs'...", p.opts.WaitDelay)
time.Sleep(xtime.Duration(p.opts.WaitDelay))
} else {
p.logger.DebugOp(op, "no wait delay to apply, moving on...")
}
// print the page to PDF.
print, err := targetClient.Page.PrintToPDF(
ctx,
page.NewPrintToPDFArgs().
SetPaperWidth(p.opts.PaperWidth).
SetPaperHeight(p.opts.PaperHeight).
SetMarginTop(p.opts.MarginTop).
SetMarginBottom(p.opts.MarginBottom).
SetMarginLeft(p.opts.MarginLeft).
SetMarginRight(p.opts.MarginRight).
SetLandscape(p.opts.Landscape).
SetDisplayHeaderFooter(true).
SetHeaderTemplate(p.opts.HeaderHTML).
SetFooterTemplate(p.opts.FooterHTML).
SetPrintBackground(true),
)
if err != nil {
return err
}
if err := ioutil.WriteFile(dest, print.Data, 0644); err != nil {
return err
}
return nil
}
if err := resolver(); err != nil {
return xerror.New(op, err)
}
return nil
}
func (p chromePrinter) enableEvents(ctx context.Context, client *cdp.Client) error {
const op string = "prinery.chromePrinter.enableEvents"
// enable all the domain events that we're interested in.
if err := xerrgroup.Run(
func() error { return client.DOM.Enable(ctx) },
func() error { return client.Network.Enable(ctx, network.NewEnableArgs()) },
func() error { return client.Page.Enable(ctx) },
func() error {
return client.Page.SetLifecycleEventsEnabled(ctx, page.NewSetLifecycleEventsEnabledArgs(true))
},
func() error { return client.Runtime.Enable(ctx) },
); err != nil {
return xerror.New(op, err)
}
return nil
}
func (p chromePrinter) listenEvents(ctx context.Context, client *cdp.Client) error {
const op string = "prinery.chromePrinter.listenEvents"
resolver := func() error {
// make sure Page events are enabled.
if err := client.Page.Enable(ctx); err != nil {
return err
}
// make sure Network events are enabled.
if err := client.Network.Enable(ctx, nil); err != nil {
return err
}
// create all clients for events.
domContentEventFired, err := client.Page.DOMContentEventFired(ctx)
if err != nil {
return err
}
defer domContentEventFired.Close() // nolint: errcheck
loadEventFired, err := client.Page.LoadEventFired(ctx)
if err != nil {
return err
}
defer loadEventFired.Close() // nolint: errcheck
lifecycleEvent, err := client.Page.LifecycleEvent(ctx)
if err != nil {
return err
}
defer lifecycleEvent.Close() // nolint: errcheck
loadingFinished, err := client.Network.LoadingFinished(ctx)
if err != nil {
return err
}
defer loadingFinished.Close() // nolint: errcheck
if _, err := client.Page.Navigate(ctx, page.NewNavigateArgs(p.url)); err != nil {
return err
}
// wait for all events.
return xerrgroup.Run(
func() error {
_, err := domContentEventFired.Recv()
if err != nil {
return err
}
p.logger.DebugOp(op, "event 'domContentEventFired' received")
return nil
},
func() error {
_, err := loadEventFired.Recv()
if err != nil {
return err
}
p.logger.DebugOp(op, "event 'loadEventFired' received")
return nil
},
func() error {
const networkIdleEventName string = "networkIdle"
for {
ev, err := lifecycleEvent.Recv()
if err != nil {
return err
}
p.logger.DebugfOp(op, "event '%s' received", ev.Name)
if ev.Name == networkIdleEventName {
break
}
}
return nil
},
func() error {
_, err := loadingFinished.Recv()
if err != nil {
return err
}
p.logger.DebugOp(op, "event 'loadingFinished' received")
return nil
},
)
}
if err := resolver(); err != nil {
return xerror.New(op, err)
}
return nil
}
// Compile-time checks to ensure type implements desired interfaces.
var (
_ = processSpec(new(chromeProcess))
_ = process(new(chromeProcess))
_ = printer(new(chromePrinter))
)

View File

@@ -0,0 +1,68 @@
package prinery
import (
"bytes"
"fmt"
"html/template"
"io/ioutil"
"path/filepath"
"github.com/microcosm-cc/bluemonday"
"github.com/russross/blackfriday/v2"
"github.com/thecodingmachine/gotenberg/internal/pkg/xerror"
"github.com/thecodingmachine/gotenberg/internal/pkg/xlog"
"github.com/thecodingmachine/gotenberg/internal/pkg/xrand"
)
func newMarkdownPrinter(logger xlog.Logger, fpath string, opts ChromePrintOptions) (printer, error) {
const op string = "prinery.newMarkdownPrinter"
resolver := func() (string, error) {
tmpl, err := template.
New(filepath.Base(fpath)).
Funcs(template.FuncMap{"toHTML": markdownToHTML}).
ParseFiles(fpath)
if err != nil {
return "", err
}
dirPath := filepath.Dir(fpath)
data := &templateData{DirPath: dirPath}
logger.DebugOp(op, "converting Markdown files to HTML...")
var buffer bytes.Buffer
if err := tmpl.Execute(&buffer, data); err != nil {
return "", err
}
baseFilename := xrand.Get()
dst := fmt.Sprintf("%s/%s.html", dirPath, baseFilename)
logger.DebugOp(op, "writing the HTML from previous conversion(s) into new file...")
if err := ioutil.WriteFile(dst, buffer.Bytes(), 0644); err != nil {
return "", err
}
return fmt.Sprintf("file://%s", dst), nil
}
URL, err := resolver()
if err != nil {
return nil, xerror.New(op, err)
}
return chromePrinter{
logger: logger,
url: URL,
opts: opts,
}, nil
}
type templateData struct {
DirPath string
}
func markdownToHTML(dirPath, filename string) (template.HTML, error) {
const op string = "prinery.markdownToHTML"
fpath := fmt.Sprintf("%s/%s", dirPath, filename)
b, err := ioutil.ReadFile(fpath)
if err != nil {
return "", xerror.New(op, err)
}
unsafe := blackfriday.Run(b)
content := bluemonday.UGCPolicy().SanitizeBytes(unsafe)
/* #nosec */
return template.HTML(content), nil
}

View File

@@ -0,0 +1,39 @@
package prinery
import (
"context"
"github.com/thecodingmachine/gotenberg/internal/pkg/xerror"
"github.com/thecodingmachine/gotenberg/internal/pkg/xexec"
"github.com/thecodingmachine/gotenberg/internal/pkg/xlog"
)
type mergePrinter struct {
logger xlog.Logger
fpaths []string
}
func (p mergePrinter) print(ctx context.Context, spec processSpec, dest string) error {
const op string = "prinery.mergePrinter.print"
p.logger.DebugfOp(op, "merging '%v'...", p.fpaths)
resolver := func() error {
var args []string
args = append(args, p.fpaths...)
args = append(args, "cat", "output", dest)
cmd, err := xexec.CommandContext(ctx, p.logger, "pdftk", args...)
if err != nil {
return err
}
xexec.LogBeforeExecute(p.logger, cmd)
return cmd.Run()
}
if err := resolver(); err != nil {
return xerror.New(op, err)
}
return nil
}
// Compile-time checks to ensure type implements desired interfaces.
var (
_ = printer(new(mergePrinter))
)

365
internal/pkg/prinery/pm2.go Normal file
View File

@@ -0,0 +1,365 @@
package prinery
import (
"context"
"errors"
"fmt"
"time"
"github.com/thecodingmachine/gotenberg/internal/pkg/conf"
"github.com/thecodingmachine/gotenberg/internal/pkg/xerror"
"github.com/thecodingmachine/gotenberg/internal/pkg/xexec"
"github.com/thecodingmachine/gotenberg/internal/pkg/xlog"
)
type command string
const (
startCommand command = "start"
restartCommand command = "restart"
stopCommand command = "stop"
logsCommand command = "logs"
jlistCommand command = "jlist"
)
const maximumRestartAttempts uint = 3
type pm2 struct {
logger xlog.Logger
config conf.Config
chromeUnit *unit
sofficeUnit *unit
processes []process
}
func NewPM2Prinery(logger xlog.Logger, config conf.Config) (Prinery, error) {
const op string = "prinery.NewPM2Prinery"
resolver := func() (*pm2, error) {
m := &pm2{
logger: logger,
config: config,
}
chromeProcesses := newChromeProcesses(config.GoogleChromeInstances())
if len(chromeProcesses) > 0 {
unit, err := newUnit(logger, processesToSpecs(chromeProcesses))
if err != nil {
return nil, err
}
m.chromeUnit = unit
}
sofficeProcesses := newSofficesProcesses(config.SofficeInstances())
if len(sofficeProcesses) > 0 {
unit, err := newUnit(logger, processesToSpecs(sofficeProcesses))
if err != nil {
return nil, err
}
m.sofficeUnit = unit
}
m.processes = append(chromeProcesses, sofficeProcesses...)
return m, nil
}
p, err := resolver()
if err != nil {
return nil, xerror.New(op, err)
}
return p, nil
}
func (m *pm2) Start(emergency chan error) error {
const op string = "prinery.pm2.Start"
// start processes.
// those lines "works" but processes
// fail to start...
/*var wg sync.WaitGroup
result := make(chan error, len(m.processes))
for _, proc := range m.processes {
wg.Add(1)
go func(proc process, result chan error) {
result <- m.start(proc)
wg.Done()
}(proc, result)
}
wg.Wait()
close(result)
for err := range result {
if err != nil {
return xerror.New(op, err)
}
}*/
for _, proc := range m.processes {
if err := m.start(proc); err != nil {
return xerror.New(op, err)
}
}
// start units.
if m.chromeUnit != nil {
go m.chromeUnit.start(emergency)
}
if m.sofficeUnit != nil {
go m.sofficeUnit.start(emergency)
}
return nil
}
func (m *pm2) HTML(
ctx context.Context,
logger xlog.Logger,
dest, fpath string,
opts ChromePrintOptions,
) error {
const op string = "prinery.pm2.HTML"
resolver := func() error {
if m.chromeUnit == nil {
return errors.New("cannot handle HTML print request as there is no chromeUnit")
}
URL := fmt.Sprintf("file://%s", fpath)
p := chromePrinter{
logger: logger,
url: URL,
opts: opts,
}
req := request{
ctx: ctx,
logger: logger,
printer: p,
dest: dest,
result: make(chan error),
}
m.chromeUnit.dispatch(req)
err := <-req.result
return err
}
if err := resolver(); err != nil {
return xerror.New(op, err)
}
return nil
}
func (m *pm2) URL(
ctx context.Context,
logger xlog.Logger, dest, URL string,
opts ChromePrintOptions,
) error {
const op string = "prinery.pm2.URL"
resolver := func() error {
if m.chromeUnit == nil {
return errors.New("cannot handle URL print request as there is no chromeUnit")
}
p := chromePrinter{
logger: logger,
url: URL,
opts: opts,
}
req := request{
ctx: ctx,
logger: logger,
printer: p,
dest: dest,
result: make(chan error),
}
m.chromeUnit.dispatch(req)
err := <-req.result
return err
}
if err := resolver(); err != nil {
return xerror.New(op, err)
}
return nil
}
func (m *pm2) Markdown(
ctx context.Context,
logger xlog.Logger,
dest, fpath string,
opts ChromePrintOptions,
) error {
const op string = "prinery.pm2.Markdown"
resolver := func() error {
if m.chromeUnit == nil {
return errors.New("cannot handle Markdown print request as there is no chromeUnit")
}
p, err := newMarkdownPrinter(logger, fpath, opts)
if err != nil {
return err
}
req := request{
ctx: ctx,
logger: logger,
printer: p,
dest: dest,
result: make(chan error),
}
m.chromeUnit.dispatch(req)
err = <-req.result
return err
}
if err := resolver(); err != nil {
return xerror.New(op, err)
}
return nil
}
func (m *pm2) Office(
ctx context.Context,
logger xlog.Logger,
dest string,
fpaths []string,
opts UnoconvPrintOptions,
) error {
const op string = "prinery.pm2.Office"
resolver := func() error {
if m.sofficeUnit == nil {
return errors.New("cannot handle Office print request as there is no sofficeUnit")
}
p := unoconvPrinter{
logger: logger,
fpaths: fpaths,
opts: opts,
}
req := request{
ctx: ctx,
logger: logger,
printer: p,
dest: dest,
result: make(chan error),
}
m.sofficeUnit.dispatch(req)
err := <-req.result
return err
}
if err := resolver(); err != nil {
return xerror.New(op, err)
}
return nil
}
func (m *pm2) Merge(
ctx context.Context,
logger xlog.Logger,
dest string,
fpaths []string,
) error {
const op string = "prinery.pm2.Merge"
p := mergePrinter{
logger: logger,
fpaths: fpaths,
}
if err := p.print(ctx, nil, dest); err != nil {
return xerror.New(op, err)
}
return nil
}
func (m *pm2) start(p process) error {
const op string = "prinery.pm2.start"
resolver := func() error {
// first, we try to start the process.
if err := m.run(startCommand, p); err != nil {
return err
}
// we wait the process to be ready.
m.warmup(p)
// if the process failed to start correctly,
// we have to restart it.
if !m.isViable(p) && maximumRestartAttempts > 0 {
return m.restart(p)
}
// the process is viable, let's log its
// output.
return m.logs(p)
}
if err := resolver(); err != nil {
return xerror.New(op, err)
}
return nil
}
func (m *pm2) restart(p process) error {
const op string = "prinery.pm2.restart"
resolver := func() error {
var attempts uint
for attempts < maximumRestartAttempts {
// we restart the process.
if err := m.run(restartCommand, p); err != nil {
return err
}
// we wait the process to be ready.
m.warmup(p)
attempts++
// if the process is viable, we
// leave.
if m.isViable(p) {
return m.logs(p)
}
}
return fmt.Errorf("failed to start '%s'", p.spec().id())
}
if err := resolver(); err != nil {
return xerror.New(op, err)
}
return nil
}
func (m *pm2) isViable(p process) bool {
if !p.viabilityFunc()(m.logger) {
return false
}
/*m.listLock.Lock()
defer m.listLock.Unlock()
return m.list.isOnline(p)*/
return true
}
func (m *pm2) warmup(p process) {
const op string = "prinery.pm2.warmup"
warmupTime := p.warmupTime()
m.logger.DebugfOp(
op,
"waiting '%v' for allowing '%s' to warmup",
warmupTime,
p.spec().id(),
)
time.Sleep(warmupTime)
}
func (m *pm2) logs(p process) error {
const op string = "prinery.pm2.logs"
if m.config.LogLevel() == xlog.DebugLevel {
if err := m.run(logsCommand, p); err != nil {
return xerror.New(op, err)
}
}
return nil
}
func (m *pm2) run(pm2Cmd command, p process) error {
const op string = "prinery.pm2.run"
resolver := func() error {
args := []string{
string(pm2Cmd),
}
if pm2Cmd == startCommand {
args = append(args, p.binary())
args = append(args, fmt.Sprintf("--name=%s", p.spec().id()))
args = append(args, "--interpreter=none", "--")
args = append(args, p.args()...)
} else {
args = append(args, p.spec().id())
}
cmd, err := xexec.Command(m.logger, "pm2", args...)
if err != nil {
return err
}
xexec.LogBeforeExecute(m.logger, cmd)
return cmd.Start()
}
if err := resolver(); err != nil {
return xerror.New(op, err)
}
return nil
}
// Compile-time checks to ensure type implements desired interfaces.
var (
_ = Prinery(new(pm2))
)

View File

@@ -2,122 +2,89 @@ package prinery
import (
"context"
"fmt"
"time"
"github.com/thecodingmachine/gotenberg/internal/pkg/print"
"github.com/thecodingmachine/gotenberg/internal/pkg/process"
"github.com/thecodingmachine/gotenberg/internal/pkg/xerror"
"github.com/thecodingmachine/gotenberg/internal/pkg/xlog"
)
type request struct {
ctx context.Context
logger xlog.Logger
print print.Print
dest string
result chan error
type processSpec interface {
id() string
host() string
port() uint
}
type worker struct {
work chan request
proc process.Process
type process interface {
spec() processSpec
binary() string
args() []string
warmupTime() time.Duration
viabilityFunc() func(logger xlog.Logger) bool
}
func (w *worker) do(done chan *worker) {
for {
req := <-w.work
req.result <- req.print.Print(req.ctx, req.dest, w.proc)
done <- w
func processesToSpecs(processes []process) []processSpec {
specs := make([]processSpec, len(processes))
for i, p := range processes {
specs[i] = p.spec()
}
return specs
}
type printer interface {
print(ctx context.Context, spec processSpec, dest string) error
}
// ChromePrintOptions helps customizing the
// Google Chrome print result.
type ChromePrintOptions struct {
WaitDelay float64
HeaderHTML string
FooterHTML string
PaperWidth float64
PaperHeight float64
MarginTop float64
MarginBottom float64
MarginLeft float64
MarginRight float64
Landscape bool
}
// DefaultChromePrintOptions returns the default
// Google Chrome print options.
func DefaultChromePrintOptions() ChromePrintOptions {
const defaultHeaderFooterHTML string = "<html><head></head><body></body></html>"
return ChromePrintOptions{
WaitDelay: 0.0,
HeaderHTML: defaultHeaderFooterHTML,
FooterHTML: defaultHeaderFooterHTML,
PaperWidth: 8.27,
PaperHeight: 11.7,
MarginTop: 1.0,
MarginBottom: 1.0,
MarginLeft: 1.0,
MarginRight: 1.0,
Landscape: false,
}
}
type Prinery struct {
logger xlog.Logger
manager process.Manager
work chan request
pool chan *worker
done chan *worker
// UnoconvPrintOptions helps customizing the
// LibreOffice print result.
type UnoconvPrintOptions struct {
Landscape bool
}
func New(logger xlog.Logger, manager process.Manager, key process.Key) (*Prinery, error) {
const op string = "prinery.New"
processes := manager.Processes(key)
nWorkers := len(processes)
if nWorkers == 0 {
err := fmt.Errorf("no processes found for key '%s'", string(key))
return nil, xerror.New(op, err)
}
logger.DebugfOp(op, "found '%d' processes for key '%s'", nWorkers, string(key))
work := make(chan request, 1)
pool := make(chan *worker, nWorkers)
done := make(chan *worker, nWorkers)
for _, p := range processes {
w := &worker{
work: work,
proc: p,
}
pool <- w
go w.do(done)
}
return &Prinery{
logger: logger,
manager: manager,
work: work,
pool: pool,
done: done,
}, nil
}
func (p *Prinery) PrintRequest(ctx context.Context, logger xlog.Logger, prnt print.Print, dest string) error {
const op string = "prinery.Prinery.PrintRequest"
req := request{
ctx: ctx,
logger: logger,
print: prnt,
dest: dest,
result: make(chan error),
}
p.dispatch(req)
err := <-req.result
if err != nil {
return xerror.New(op, err)
}
return nil
}
func (p *Prinery) Start() {
for {
select {
case req := <-p.work:
p.dispatch(req)
case w := <-p.done:
p.completed(w)
}
// DefaultUnoconvPrinterOptions returns the default
// LibreOffice print options.
func DefaultUnoconvPrinterOptions() UnoconvPrintOptions {
return UnoconvPrintOptions{
Landscape: false,
}
}
func (p *Prinery) dispatch(req request) {
const op string = "prinery.Prinery.dispatch"
select {
case w := <-p.pool:
w.work <- req
case <-req.ctx.Done():
req.result <- xerror.New(op, req.ctx.Err())
}
}
func (p *Prinery) completed(w *worker) {
const op string = "prinery.Prinerty.completed"
go func() {
// check process viability.
isViable := p.manager.IsViable(w.proc)
// check process memory usage.
// TODO handle error.
memory, _ := p.manager.Memory(w.proc)
p.logger.DebugfOp(op, "%s: isViable = %t, memory = %d", w.proc.ID(), isViable, memory)
// TODO manage viability and memory usage.
// pushing back the worker.
p.pool <- w
}()
type Prinery interface {
Start(emergency chan error) error
HTML(ctx context.Context, logger xlog.Logger, dest, fpath string, opts ChromePrintOptions) error
URL(ctx context.Context, logger xlog.Logger, dest, URL string, opts ChromePrintOptions) error
Markdown(ctx context.Context, logger xlog.Logger, dest, fpath string, opts ChromePrintOptions) error
Office(ctx context.Context, logger xlog.Logger, dest string, fpaths []string, opts UnoconvPrintOptions) error
Merge(ctx context.Context, logger xlog.Logger, dest string, fpaths []string) error
}

View File

@@ -0,0 +1,158 @@
package prinery
import (
"context"
"fmt"
"os"
"path/filepath"
"time"
"github.com/thecodingmachine/gotenberg/internal/pkg/xerror"
"github.com/thecodingmachine/gotenberg/internal/pkg/xexec"
"github.com/thecodingmachine/gotenberg/internal/pkg/xlog"
"github.com/thecodingmachine/gotenberg/internal/pkg/xrand"
)
type sofficeProcess struct {
unoPort uint
}
func newSofficesProcesses(nInstances int64) []process {
processes := make([]process, nInstances)
var i int64
var currentPort uint = 2002
for i = 0; i < nInstances; i++ {
processes[i] = sofficeProcess{
unoPort: currentPort,
}
currentPort++
}
return processes
}
func (p sofficeProcess) id() string {
return fmt.Sprintf("%s-%d", p.binary(), p.port())
}
func (p sofficeProcess) host() string {
return "127.0.0.1"
}
func (p sofficeProcess) port() uint {
return p.unoPort
}
func (p sofficeProcess) spec() processSpec {
return p
}
func (p sofficeProcess) binary() string {
return "soffice"
}
func (p sofficeProcess) args() []string {
return []string{
// see https://ask.libreoffice.org/en/question/42975/how-can-i-run-multiple-instances-of-sofficebin-at-a-time/.
fmt.Sprintf("-env:UserInstallation=file:///tmp/%d", p.port()),
"--headless",
"--invisible",
"--nocrashreport",
"--nodefault",
"--nofirststartwizard",
"--nologo",
"--norestore",
fmt.Sprintf("--accept=socket,host=%s,port=%d,tcpNoDelay=1;urp;StarOffice.ComponentContext", p.host(), p.port()),
}
}
func (p sofficeProcess) warmupTime() time.Duration {
return 3 * time.Second
}
func (p sofficeProcess) viabilityFunc() func(logger xlog.Logger) bool {
const op string = "prinery.sofficeProcess.viabilityFunc"
return func(logger xlog.Logger) bool {
// TODO find a way to check.
return true
}
}
type unoconvPrinter struct {
logger xlog.Logger
fpaths []string
opts UnoconvPrintOptions
}
func (p unoconvPrinter) print(ctx context.Context, spec processSpec, dest string) error {
const op string = "prinery.unoconvPrinter.print"
resolver := func() error {
fpaths := make([]string, len(p.fpaths))
dirPath := filepath.Dir(dest)
for i, fpath := range p.fpaths {
baseFilename := xrand.Get()
tmpDest := fmt.Sprintf("%s/%d%s.pdf", dirPath, i, baseFilename)
p.logger.DebugfOp(op, "converting '%s' to PDF...", fpath)
if err := p.unoconv(ctx, spec, fpath, tmpDest); err != nil {
return err
}
p.logger.DebugfOp(op, "'%s.pdf' created", baseFilename)
fpaths[i] = tmpDest
}
if len(fpaths) == 1 {
p.logger.DebugOp(op, "only one PDF created, nothing to merge")
if err := os.Rename(fpaths[0], dest); err != nil {
return err
}
return nil
}
merger := mergePrinter{
logger: p.logger,
fpaths: fpaths,
}
return merger.print(ctx, nil, dest)
}
if err := resolver(); err != nil {
return xerror.New(op, err)
}
return nil
}
func (p unoconvPrinter) unoconv(ctx context.Context, spec processSpec, fpath, dest string) error {
const op string = "prinery.unoconvPrinter.unoconv"
resolver := func() error {
args := []string{
"--server",
spec.host(),
"--port",
fmt.Sprintf("%d", spec.port()),
"--format",
"pdf",
}
if p.opts.Landscape {
args = append(args, "--printer", "PaperOrientation=landscape")
}
args = append(args, "--output", dest, fpath)
cmd, err := xexec.CommandContext(
ctx,
p.logger,
"unoconv",
args...,
)
if err != nil {
return err
}
xexec.LogBeforeExecute(p.logger, cmd)
return cmd.Run()
}
if err := resolver(); err != nil {
return xerror.New(op, err)
}
return nil
}
// Compile-time checks to ensure type implements desired interfaces.
var (
_ = processSpec(new(sofficeProcess))
_ = process(new(sofficeProcess))
_ = printer(new(unoconvPrinter))
)

View File

@@ -0,0 +1,125 @@
package prinery
import (
"context"
"errors"
"github.com/thecodingmachine/gotenberg/internal/pkg/xerror"
"github.com/thecodingmachine/gotenberg/internal/pkg/xlog"
)
type request struct {
ctx context.Context
logger xlog.Logger
printer printer
dest string
result chan error
}
type worker struct {
work chan request
spec processSpec
}
func (w *worker) do(done chan *worker) {
for {
req := <-w.work
req.result <- req.printer.print(req.ctx, w.spec, req.dest)
done <- w
}
}
type unit struct {
logger xlog.Logger
workers []*worker
work chan request
pool chan *worker
done chan *worker
}
func newUnit(logger xlog.Logger, specs []processSpec) (*unit, error) {
const op string = "prinery.newUnit"
resolver := func() (*unit, error) {
nWorkers := len(specs)
if nWorkers == 0 {
return nil, errors.New("no workers to create")
}
logger.DebugfOp(op, "creating '%d' workers...", nWorkers)
workers := make([]*worker, nWorkers)
work := make(chan request, 1)
pool := make(chan *worker, nWorkers)
done := make(chan *worker, nWorkers)
for i, spec := range specs {
w := &worker{
work: work,
spec: spec,
}
logger.DebugfOp(op, "worker '%s' created", w.spec.id())
workers[i] = w
}
return &unit{
logger: logger,
workers: workers,
work: work,
pool: pool,
done: done,
}, nil
}
u, err := resolver()
if err != nil {
return nil, xerror.New(op, err)
}
return u, nil
}
func (u *unit) start(emergency chan error) {
const op string = "prinery.unit.start"
for _, w := range u.workers {
u.logger.DebugfOp(op, "starting worker '%s'...", w.spec.id())
go w.do(u.done)
u.pool <- w
}
for {
select {
case req := <-u.work:
u.dispatch(req)
case w := <-u.done:
u.completed(w, emergency)
}
}
}
func (u *unit) request(ctx context.Context, logger xlog.Logger, printer printer, dest string) error {
const op string = "prinery.unit.request"
req := request{
ctx: ctx,
logger: logger,
printer: printer,
dest: dest,
result: make(chan error),
}
u.dispatch(req)
err := <-req.result
if err != nil {
return xerror.New(op, err)
}
return nil
}
func (u *unit) dispatch(req request) {
const op string = "prinery.unit.dispatch"
select {
case w := <-u.pool:
u.logger.DebugfOp(op, "worker '%s' is now in use", w.spec.id())
w.work <- req
case <-req.ctx.Done():
req.result <- xerror.New(op, req.ctx.Err())
}
}
func (u *unit) completed(w *worker, emergency chan error) {
const op string = "prinery.unit.completed"
// TODO check viability, handles errors via emergency chan.
u.logger.DebugfOp(op, "worker '%s' is ready for a new request", w.spec.id())
u.pool <- w
}

View File

@@ -1,18 +0,0 @@
package print
import (
"fmt"
"github.com/thecodingmachine/gotenberg/internal/pkg/xlog"
)
// NewHTMLPrint returns a Print for
// converting an HTML file to PDF.
func NewHTMLPrint(logger xlog.Logger, fpath string, opts ChromePrintOptions) Print {
URL := fmt.Sprintf("file://%s", fpath)
return chromePrint{
logger: logger,
url: URL,
opts: opts,
}
}

View File

@@ -52,7 +52,14 @@ func (p officePrint) Print(ctx context.Context, dest string, proc process.Proces
baseFilename := xrand.Get()
tmpDest := fmt.Sprintf("%s/%d%s.pdf", dirPath, i, baseFilename)
p.logger.DebugfOp(op, "converting '%s' to PDF...", fpath)
if err := unoconv(ctx, p.logger, fpath, tmpDest, p.opts); err != nil {
if err := unoconv(
ctx,
p.logger,
proc,
p.opts,
fpath,
tmpDest,
); err != nil {
return err
}
p.logger.DebugfOp(op, "'%s.pdf' created", baseFilename)
@@ -74,10 +81,20 @@ func (p officePrint) Print(ctx context.Context, dest string, proc process.Proces
return nil
}
func unoconv(ctx context.Context, logger xlog.Logger, fpath, dest string, opts OfficePrintOptions) error {
func unoconv(
ctx context.Context,
logger xlog.Logger,
proc process.Process,
opts OfficePrintOptions,
fpath, dest string,
) error {
const op string = "print.unoconv"
resolver := func() error {
args := []string{
"--server",
proc.Host(),
"--port",
fmt.Sprintf("%d", proc.Port()),
"--format",
"pdf",
}