This commit is contained in:
Julien Neuhart
2018-12-10 16:09:19 +01:00
committed by GitHub
parent c1f6100382
commit 495203c112
473 changed files with 4631 additions and 174161 deletions

View File

@@ -0,0 +1,5 @@
/*
Package notify is used across the application
to display nice outputs to the user.
*/
package notify

View File

@@ -0,0 +1,30 @@
package notify
import (
"fmt"
"os"
"github.com/labstack/gommon/color"
)
var (
stdout *color.Color
stderr *color.Color
)
func init() {
stdout = color.New()
stdout.SetOutput(os.Stdout)
stderr = color.New()
stderr.SetOutput(os.Stderr)
}
// Println prints a message to stdout.
func Println(message string) {
stdout.Printf("⇨ %s\n", message)
}
// ErrPrintln prints an error to stderr.
func ErrPrintln(err error) {
stderr.Printf("%s\n", color.Red(fmt.Sprintf("⇨ error: %v", err)))
}

View File

@@ -0,0 +1,70 @@
package pm2
import (
"context"
"fmt"
"time"
"github.com/mafredri/cdp/devtool"
"github.com/thecodingmachine/gotenberg/internal/pkg/notify"
)
// Chrome facilitates starting or shutting down
// Chrome headless with PM2.
type Chrome struct{}
// Launch starts Chrome headless with PM2.
func (c *Chrome) Launch() error {
return launch(c)
}
// Shutdown stops Chrome headless and
// removes it from the list of PM2
// processes.
func (c *Chrome) Shutdown() error {
return shutdown(c)
}
func (c *Chrome) getArgs() []string {
return []string{
"--no-sandbox",
"--headless",
"--remote-debugging-port=9222",
"--disable-gpu",
"--disable-translate",
"--disable-extensions",
"--disable-background-networking",
"--safebrowsing-disable-auto-update",
"--disable-sync",
"--disable-default-apps",
"--hide-scrollbars",
"--metrics-recording-only",
"--mute-audio",
"--no-first-run",
}
}
func (c *Chrome) getName() string {
return "google-chrome-stable"
}
func (c *Chrome) getFullname() string {
return "Chrome headless"
}
func (c *Chrome) isViable() bool {
// check if Chrome is correctly running.
devt := devtool.New("http://127.0.0.1:9222")
_, err := devt.Create(context.TODO())
return err == nil
}
func (c *Chrome) warmup() {
notify.Println(fmt.Sprintf("warming-up %s", c.getFullname()))
time.Sleep(5 * time.Second)
}
// Compile-time checks to ensure type implements desired interfaces.
var (
_ = Process(new(Chrome))
)

View File

@@ -0,0 +1,19 @@
package pm2
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestChromeLaunch(t *testing.T) {
p := &Chrome{}
err := p.Launch()
require.Nil(t, err)
}
func TestChromeShutdown(t *testing.T) {
p := &Chrome{}
err := p.Shutdown()
require.Nil(t, err)
}

12
internal/pkg/pm2/doc.go Normal file
View File

@@ -0,0 +1,12 @@
/*
Package pm2 facilitates starting external
processes on which our API depends.
For instance, it starts Chrome headless and
unoconv listener with PM2.
The PM2 process manager launch those processes and keep
them running in the background. If for some reason they
crash, it will also restart them.
*/
package pm2

71
internal/pkg/pm2/pm2.go Normal file
View File

@@ -0,0 +1,71 @@
package pm2
import (
"fmt"
"os/exec"
"github.com/thecodingmachine/gotenberg/internal/pkg/notify"
)
// Process is a type that can launch or
// shutdown a process with PM2.
type Process interface {
Launch() error
Shutdown() error
getArgs() []string
getName() string
getFullname() string
isViable() bool
warmup()
}
const maxRestartAttempts int = 5
var humanNames = map[string]string{
"start": "started",
"restart": "restarted",
"stop": "stopped",
}
func launch(p Process) error {
if err := run(p, "start"); err != nil {
return err
}
p.warmup()
if !p.isViable() {
attempts := 0
for attempts < maxRestartAttempts && !p.isViable() {
run(p, "restart")
p.warmup()
attempts++
}
if !p.isViable() {
return fmt.Errorf("failed to launch %s", p.getFullname())
}
}
return nil
}
func shutdown(p Process) error {
return run(p, "stop")
}
func run(p Process, cmdName string) error {
cmdArgs := []string{
cmdName,
p.getName(),
}
if cmdName == "start" {
cmdArgs = append(cmdArgs, "--interpreter none", "--")
cmdArgs = append(cmdArgs, p.getArgs()...)
}
cmd := exec.Command(
"pm2",
cmdArgs...,
)
if err := cmd.Start(); err != nil {
return fmt.Errorf("%s %s with PM2: %v", cmdName, p.getFullname(), err)
}
notify.Println(fmt.Sprintf("%s %s with PM2", p.getFullname(), humanNames[cmdName]))
return nil
}

View File

@@ -0,0 +1,47 @@
package pm2
// Unoconv facilitates starting or shutting down
// unoconv listener with PM2.
type Unoconv struct{}
// Launch starts unoconv listener with PM2.
func (u *Unoconv) Launch() error {
return launch(u)
}
// Shutdown stops unoconv listener and
// removes it from the list of PM2
// processes.
func (u *Unoconv) Shutdown() error {
return shutdown(u)
}
func (u *Unoconv) getArgs() []string {
return []string{
"--listener",
"--verbose",
}
}
func (u *Unoconv) getName() string {
return "unoconv"
}
func (u *Unoconv) getFullname() string {
return "unoconv listener"
}
func (u *Unoconv) isViable() bool {
// TODO find a way to check if
// unoconv is correctly started?
return true
}
func (u *Unoconv) warmup() {
// let's do nothing.
}
// Compile-time checks to ensure type implements desired interfaces.
var (
_ = Process(new(Unoconv))
)

View File

@@ -0,0 +1,19 @@
package pm2
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestUnoconvLaunch(t *testing.T) {
p := &Unoconv{}
err := p.Launch()
require.Nil(t, err)
}
func TestUnoconvShutdown(t *testing.T) {
p := &Unoconv{}
err := p.Shutdown()
require.Nil(t, err)
}

View File

@@ -0,0 +1,52 @@
/*
Package printer contains structs which convert
a specific file type to PDF:
// converting HTML to PDF.
p := &printer.HTML{
Context: context.Background(),
PaperWidth: 8.27,
PaperHeight: 11.7,
Landscape: false,
}
p.WithLocalURL("index.html")
if err := p.Print("result.pdf"); err != nil {
return err
}
// converting Markdown to PDF:
// it assumes here that our template "index.html"
// will call toHTML method to convert
// markdown files to HTML.
p := &printer.Markdown{
Context: context.Background(),
TemplatePath: "index.html",
PaperWidth: 8.27,
PaperHeight: 11.7,
Landscape: false,
}
if err := p.Print("result.pdf"); err != nil {
return err
}
// converting Office documents to PDF:
// it converts each files independently and
// then merge them.
//
// Also, as unoconv cannot perform
// concurrent conversions, a lock is applied.
p := &printer.Office{
Context: ctx,
FilePaths: []string{"document.docx", "presentation.pptx"}
}
if err := p.Print("result.pdf"); err != nil {
return err
}
It is also able to merge a list of PDF files:
if err := printer.Merge([]string{"foo.pdf", "bar.pdf"}, "result.pdf"); err != nil {
return err
}
*/
package printer

View File

@@ -0,0 +1,188 @@
package printer
import (
"context"
"errors"
"fmt"
"io/ioutil"
"strings"
"github.com/mafredri/cdp"
"github.com/mafredri/cdp/devtool"
"github.com/mafredri/cdp/protocol/network"
"github.com/mafredri/cdp/protocol/page"
"github.com/mafredri/cdp/protocol/runtime"
"github.com/mafredri/cdp/protocol/target"
"github.com/mafredri/cdp/rpcc"
)
// HTML facilitates HTML to PDF conversion.
type HTML struct {
Context context.Context
URL string
HeaderHTML string
FooterHTML string
PaperWidth float64
PaperHeight float64
MarginTop float64
MarginBottom float64
MarginLeft float64
MarginRight float64
Landscape bool
}
const defaultHeaderFooterHTML string = "<html><head></head><body></body></html>"
// Print converts HTML to PDF.
// Credits: https://medium.com/compass-true-north/go-service-to-convert-web-pages-to-pdf-using-headless-chrome-5fd9ffbae1af
func (html *HTML) Print(destination string) error {
// use the DevTools HTTP/JSON API to manage targets (e.g. pages, webworkers).
devt := devtool.New("http://127.0.0.1:9222")
pt, err := devt.Create(html.Context)
if err != nil {
return fmt.Errorf("creating DevTools target: %v", err)
}
// open a new RPC connection to the Chrome Debugging Protocol target.
conn, err := rpcc.DialContext(html.Context, pt.WebSocketDebuggerURL)
if err != nil {
return fmt.Errorf("creating RPC connection: %v", err)
}
defer conn.Close()
// create new browser context.
baseBrowser := cdp.NewClient(conn)
newContextTarget, err := baseBrowser.Target.CreateBrowserContext(html.Context)
if err != nil {
return fmt.Errorf("creating new browser context: %v", err)
}
// create a new blank target with the new browser context.
newTargetArgs := target.NewCreateTargetArgs("about:blank").
SetBrowserContextID(newContextTarget.BrowserContextID)
newTarget, err := baseBrowser.Target.CreateTarget(html.Context, newTargetArgs)
if err != nil {
return fmt.Errorf("creating new blank target: %v", err)
}
// connect the client to the new target.
newTargetWsURL := fmt.Sprintf("ws://127.0.0.1:9222/devtools/page/%s", newTarget.TargetID)
newContextConn, err := rpcc.DialContext(html.Context, newTargetWsURL)
if err != nil {
return fmt.Errorf("connecting client to blank target: %v", err)
}
defer newContextConn.Close()
// close the target when done.
closeTargetArgs := target.NewCloseTargetArgs(newTarget.TargetID)
defer baseBrowser.Target.CloseTarget(html.Context, closeTargetArgs)
c := cdp.NewClient(newContextConn)
// enable the runtime.
if err := c.Runtime.Enable(html.Context); err != nil {
return fmt.Errorf("enabling runtime: %v", err)
}
// enable the network.
if err := c.Network.Enable(html.Context, network.NewEnableArgs()); err != nil {
return fmt.Errorf("enabling network: %v", err)
}
// enable events on the page domain.
if err := c.Page.Enable(html.Context); err != nil {
return fmt.Errorf("enabling events on page domain: %v", err)
}
// create a client to listen for the load event to be fired.
loadEventFiredClient, err := c.Page.LoadEventFired(html.Context)
if err != nil {
return fmt.Errorf("creating client listening for load event: %v", err)
}
defer loadEventFiredClient.Close()
// tell the page to navigate to the URL.
navArgs := page.NewNavigateArgs(html.URL)
_, err = c.Page.Navigate(html.Context, navArgs)
if err != nil {
return fmt.Errorf("%s: navigating to URL: %v", html.URL, err)
}
// wait for the page to finish loading.
_, err = loadEventFiredClient.Recv()
if err != nil {
return fmt.Errorf("waiting for page loading: %v", err)
}
// inject a script to make sure web fonts are loaded.
script := `new Promise((resolve, reject) => {
document.fonts.ready.then(function () {
resolve('fonts loaded');
});
setTimeout(resolve.bind(resolve, 'timeout'), %.0f);
});`
scriptArg := runtime.NewEvaluateArgs(script).SetAwaitPromise(true)
returnObj, _ := c.Runtime.Evaluate(html.Context, scriptArg)
loadFontsResult := string(returnObj.Result.Value)
if strings.Contains(loadFontsResult, "timeout") {
return errors.New("timed out loading fonts")
}
// if no header or footer, use the default template
// for avoiding displaying default Chrome templates.
if html.HeaderHTML == "" {
html.HeaderHTML = defaultHeaderFooterHTML
}
if html.FooterHTML == "" {
html.FooterHTML = defaultHeaderFooterHTML
}
print, err := c.Page.PrintToPDF(
html.Context,
page.NewPrintToPDFArgs().
SetPaperWidth(html.PaperWidth).
SetPaperHeight(html.PaperHeight).
SetMarginTop(html.MarginTop).
SetMarginBottom(html.MarginBottom).
SetMarginLeft(html.MarginLeft).
SetMarginRight(html.MarginRight).
SetLandscape(html.Landscape).
SetDisplayHeaderFooter(true).
SetHeaderTemplate(html.HeaderHTML).
SetFooterTemplate(html.FooterHTML).
SetPrintBackground(true),
)
if err != nil {
return fmt.Errorf("printing page to PDF: %v", err)
}
return writeBytesToFile(destination, print.Data)
}
// WithLocalURL sets a local URL from a file path.
func (html *HTML) WithLocalURL(fpath string) {
html.URL = fmt.Sprintf("file://%s", fpath)
}
// WithHeaderFile sets header content from a file.
func (html *HTML) WithHeaderFile(fpath string) error {
if fpath == "" {
return nil
}
contentHTML, err := fileContentToString(fpath)
if err != nil {
return err
}
html.HeaderHTML = contentHTML
return nil
}
// WithFooterFile sets footer content from a file.
func (html *HTML) WithFooterFile(fpath string) error {
if fpath == "" {
return nil
}
contentHTML, err := fileContentToString(fpath)
if err != nil {
return err
}
html.FooterHTML = contentHTML
return nil
}
func fileContentToString(fpath string) (string, error) {
b, err := ioutil.ReadFile(fpath)
if err != nil {
return "", fmt.Errorf("%s: reading file: %v", fpath, err)
}
return string(b), nil
}
// Compile-time checks to ensure type implements desired interfaces.
var (
_ = Printer(new(HTML))
)

View File

@@ -0,0 +1,39 @@
package printer
import (
"context"
"fmt"
"os"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/thecodingmachine/gotenberg/test"
)
func TestHTML(t *testing.T) {
dirPath := test.HTMLTestDirPath(t)
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
html := &HTML{
Context: ctx,
PaperWidth: 8.27,
PaperHeight: 11.7,
MarginTop: 1,
MarginBottom: 1,
MarginLeft: 1,
MarginRight: 1,
}
html.WithLocalURL(fmt.Sprintf("%s/%s", dirPath, "index.html"))
err := html.WithHeaderFile(fmt.Sprintf("%s/%s", dirPath, "header.html"))
require.Nil(t, err)
err = html.WithFooterFile(fmt.Sprintf("%s/%s", dirPath, "footer.html"))
require.Nil(t, err)
dst := fmt.Sprintf("%s/%s", dirPath, "foo.pdf")
err = html.Print(dst)
require.Nil(t, err)
require.FileExists(t, dst)
err = os.RemoveAll(dirPath)
assert.Nil(t, err)
}

View File

@@ -0,0 +1,111 @@
package printer
import (
"bytes"
"context"
"fmt"
"html/template"
"io/ioutil"
"path/filepath"
"github.com/microcosm-cc/bluemonday"
"github.com/russross/blackfriday/v2"
"github.com/thecodingmachine/gotenberg/internal/pkg/rand"
)
// Markdown facilitates Markdown to PDF conversion.
type Markdown struct {
Context context.Context
TemplatePath string
HeaderHTML string
FooterHTML string
PaperWidth float64
PaperHeight float64
MarginTop float64
MarginBottom float64
MarginLeft float64
MarginRight float64
Landscape bool
html *HTML
}
type templateData struct {
DirPath string
}
// Print converts markdown to PDF.
func (md *Markdown) Print(destination string) error {
if md.html == nil {
md.html = &HTML{Context: md.Context}
}
if md.HeaderHTML != "" {
md.html.HeaderHTML = md.HeaderHTML
}
if md.FooterHTML != "" {
md.html.FooterHTML = md.FooterHTML
}
md.html.PaperWidth = md.PaperWidth
md.html.PaperHeight = md.PaperHeight
md.html.MarginTop = md.MarginTop
md.html.MarginBottom = md.MarginBottom
md.html.MarginLeft = md.MarginLeft
md.html.MarginRight = md.MarginRight
md.html.Landscape = md.Landscape
tmpl, err := template.
New(filepath.Base(md.TemplatePath)).
Funcs(template.FuncMap{"toHTML": toHTML}).
ParseFiles(md.TemplatePath)
if err != nil {
return fmt.Errorf("%s: parsing template: %v", md.TemplatePath, err)
}
dirPath := filepath.Dir(md.TemplatePath)
data := &templateData{DirPath: dirPath}
var buffer bytes.Buffer
if err := tmpl.Execute(&buffer, data); err != nil {
return fmt.Errorf("%s: executing template: %v", md.TemplatePath, err)
}
baseFilename, err := rand.Get()
if err != nil {
return err
}
dst := fmt.Sprintf("%s/%s.html", dirPath, baseFilename)
if err := writeBytesToFile(dst, buffer.Bytes()); err != nil {
return err
}
md.html.WithLocalURL(dst)
return md.html.Print(destination)
}
func toHTML(dirPath, filename string) (template.HTML, error) {
fpath := fmt.Sprintf("%s/%s", dirPath, filename)
b, err := ioutil.ReadFile(fpath)
if err != nil {
return "", fmt.Errorf("%s: reading file: %v", fpath, err)
}
unsafe := blackfriday.Run(b)
contentHTML := bluemonday.UGCPolicy().SanitizeBytes(unsafe)
return template.HTML(contentHTML), nil
}
// WithHeaderFile sets header content from a file.
func (md *Markdown) WithHeaderFile(fpath string) error {
if md.html == nil {
md.html = &HTML{Context: md.Context}
}
return md.html.WithHeaderFile(fpath)
}
// WithFooterFile sets footer content from a file.
func (md *Markdown) WithFooterFile(fpath string) error {
if md.html == nil {
md.html = &HTML{Context: md.Context}
}
return md.html.WithFooterFile(fpath)
}
// Compile-time checks to ensure type implements desired interfaces.
var (
_ = Printer(new(Markdown))
)

View File

@@ -0,0 +1,39 @@
package printer
import (
"context"
"fmt"
"os"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/thecodingmachine/gotenberg/test"
)
func TestMarkdown(t *testing.T) {
dirPath := test.MarkdownTestDirPath(t)
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
markdown := &Markdown{
Context: ctx,
TemplatePath: fmt.Sprintf("%s/%s", dirPath, "index.html"),
PaperWidth: 8.27,
PaperHeight: 11.7,
MarginTop: 1,
MarginBottom: 1,
MarginLeft: 1,
MarginRight: 1,
}
err := markdown.WithHeaderFile(fmt.Sprintf("%s/%s", dirPath, "header.html"))
require.Nil(t, err)
err = markdown.WithFooterFile(fmt.Sprintf("%s/%s", dirPath, "footer.html"))
require.Nil(t, err)
dst := fmt.Sprintf("%s/%s", dirPath, "foo.pdf")
err = markdown.Print(dst)
require.Nil(t, err)
require.FileExists(t, dst)
err = os.RemoveAll(dirPath)
assert.Nil(t, err)
}

View File

@@ -0,0 +1,62 @@
package printer
import (
"context"
"errors"
"fmt"
"os"
"os/exec"
"path/filepath"
"sync"
"github.com/thecodingmachine/gotenberg/internal/pkg/rand"
)
var mu sync.Mutex
// Office facilitates Office documents to PDF conversion.
type Office struct {
Context context.Context
FilePaths []string
}
// Print converts Office documents to PDF.
func (o *Office) Print(destination string) error {
mu.Lock()
defer mu.Unlock()
fpaths := make([]string, len(o.FilePaths))
dirPath := filepath.Dir(destination)
for i, fpath := range o.FilePaths {
baseFilename, err := rand.Get()
if err != nil {
return err
}
tmpDest := fmt.Sprintf("%s/%s.pdf", dirPath, baseFilename)
cmd := exec.CommandContext(
o.Context,
"unoconv",
"--format",
"pdf",
"--output",
tmpDest,
fpath,
)
_, err = cmd.Output()
if o.Context.Err() == context.DeadlineExceeded {
return errors.New("unoconv: command timed out")
}
if err != nil {
return fmt.Errorf("unoconv: non-zero exit code: %v", err)
}
fpaths[i] = tmpDest
}
if len(fpaths) == 1 {
return os.Rename(fpaths[0], destination)
}
return Merge(fpaths, destination)
}
// Compile-time checks to ensure type implements desired interfaces.
var (
_ = Printer(new(Office))
)

View File

@@ -0,0 +1,31 @@
package printer
import (
"context"
"fmt"
"os"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/thecodingmachine/gotenberg/test"
)
func TestOffice(t *testing.T) {
dirPath := test.OfficeTestDirPath(t)
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
office := &Office{
Context: ctx,
FilePaths: []string{
fmt.Sprintf("%s/%s", dirPath, "document.docx"),
},
}
dst := fmt.Sprintf("%s/%s", dirPath, "foo.pdf")
err := office.Print(dst)
require.Nil(t, err)
require.FileExists(t, dst)
err = os.RemoveAll(dirPath)
assert.Nil(t, err)
}

View File

@@ -0,0 +1,29 @@
package printer
import (
"fmt"
"io/ioutil"
pdfcpuAPI "github.com/hhrutter/pdfcpu/pkg/api"
pdfcpuConfig "github.com/hhrutter/pdfcpu/pkg/pdfcpu"
)
// Printer is a type that can create a PDF file from a source.
// The source is defined in the underlying implementation.
type Printer interface {
Print(destination string) error
}
// Merge merges PDF files.
func Merge(fpaths []string, destination string) error {
cmd := pdfcpuAPI.MergeCommand(fpaths, destination, pdfcpuConfig.NewDefaultConfiguration())
_, err := pdfcpuAPI.Merge(cmd)
return err
}
func writeBytesToFile(dst string, b []byte) error {
if err := ioutil.WriteFile(dst, b, 0644); err != nil {
return fmt.Errorf("%s: writting file: %v", dst, err)
}
return nil
}

View File

@@ -0,0 +1,27 @@
package printer
import (
"fmt"
"os"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/thecodingmachine/gotenberg/test"
)
func TestMerge(t *testing.T) {
dirPath := test.PDFTestDirPath(t)
dst := fmt.Sprintf("%s/%s", dirPath, "foo.pdf")
err := Merge(
[]string{
fmt.Sprintf("%s/%s", dirPath, "gotenberg.pdf"),
fmt.Sprintf("%s/%s", dirPath, "gotenberg.pdf"),
},
dst,
)
require.Nil(t, err)
require.FileExists(t, dst)
err = os.RemoveAll(dirPath)
assert.Nil(t, err)
}

7
internal/pkg/rand/doc.go Normal file
View File

@@ -0,0 +1,7 @@
/*
Package rand helps generating a random string.
It should be used for creating directory and
file names in order to avoid collision.
*/
package rand

17
internal/pkg/rand/rand.go Normal file
View File

@@ -0,0 +1,17 @@
package rand
import (
"crypto/rand"
"encoding/hex"
"fmt"
)
// Get returns a random string.
func Get() (string, error) {
randBytes := make([]byte, 16)
_, err := rand.Read(randBytes)
if err != nil {
return "", fmt.Errorf("creating random string: %v", err)
}
return hex.EncodeToString(randBytes), nil
}

View File

@@ -0,0 +1,16 @@
package rand
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestGet(t *testing.T) {
rand1, err := Get()
require.Nil(t, err)
rand2, err := Get()
require.Nil(t, err)
assert.NotEqual(t, rand1, rand2)
}