Compare commits

..

16 Commits

Author SHA1 Message Date
Julien Neuhart
872263d253 feat: switch from top to htop 2022-01-05 15:46:20 +01:00
Julien Neuhart
1c491fb1c5 fix: use bullseye-backports distribution instead of sid for installing LibreOffice (fix #403) 2022-01-05 14:59:38 +01:00
Julien Neuhart
29374c579a chore: update Go dependencies 2021-12-30 17:45:16 +01:00
Julien Neuhart
206d7b6ca9 fix(chromium): typo in godoc <link> -> <script> 2021-12-27 17:49:06 +01:00
Julien Neuhart
ab6ca8c894 chore(README): improve Gotenberg description 2021-12-27 17:34:10 +01:00
Kaijia Feng
e7f5fde051 feat: add Noto CJK fonts 2021-12-27 17:21:20 +01:00
Julien Neuhart
030b358b0e fix: rework api and webhook module timeout properties (fixes #396) 2021-12-17 19:13:14 +01:00
Julien Neuhart
4b1791d35c fix(release): add noto-emoji arg 2021-12-16 17:49:48 +01:00
Julien Neuhart
b788cf823d chore: update Go dependencies 2021-12-08 17:22:24 +01:00
Julien Neuhart
11cbc0288f chore: update Go dependencies 2021-12-08 17:17:40 +01:00
Julien Neuhart
2269c80b02 feat: start a unoconv listener by default, but allow stateless mode as before with the --unoconv-disable-listener flag. Also improve the shutdown process 2021-12-08 17:17:40 +01:00
Julien Neuhart
c922cc042e chore: improve some tests 2021-12-08 17:17:40 +01:00
Julien Neuhart
9cb6f44f72 feat: improve prometheus startup message 2021-12-08 17:17:40 +01:00
Julien Neuhart
f376cd5ab1 feat: improve 'logger' field from logger 2021-12-08 15:41:08 +01:00
Julien Neuhart
8b8a02b68c fix(qpdf): typo in godoc 2021-12-08 14:39:20 +01:00
Rodrigo Delduca
5759b1336a feat: add qpdf engine (#380)
Co-authored-by: Andrei Regiani <andrei.cpp@gmail.com>
2021-12-07 19:48:41 +01:00
28 changed files with 936 additions and 214 deletions

View File

@@ -29,9 +29,7 @@ build: ## Build the Gotenberg's Docker image
GOTENBERG_GRACEFUL_SHUTDOWN_DURATION=30s
API_PORT=3000
API_PORT_FROM_ENV=
API_READ_TIMEOUT=30s
API_PROCESS_TIMEOUT=30s
API_WRITE_TIMEOUT=30s
API_TIMEOUT=30s
API_ROOT_PATH=/
API_TRACE_HEADER=Gotenberg-Trace
API_DISABLE_HEALTH_CHECK_LOGGING=false
@@ -53,6 +51,7 @@ PROMETHEUS_NAMESPACE=gotenberg
PROMETHEUS_COLLECT_INTERVAL=1s
PROMETHEUS_DISABLE_ROUTE_LOGGING=false
PROMETHEUS_DISABLE_COLLECT=false
UNOCONV_DISABLE_LISTENER=false
WEBHOOK_ALLOW_LIST=
WEBHOOK_DENY_LIST=
WEBHOOK_ERROR_ALLOW_LIST=
@@ -60,6 +59,7 @@ WEBHOOK_ERROR_DENY_LIST=
WEBHOOK_MAX_RETRY=4
WEBHOOK_RETRY_MIN_WAIT=1s
WEBHOOK_RETRY_MAX_WAIT=30s
WEBHOOK_CLIENT_TIMEOUT=30s
WEBHOOK_DISABLE=false
.PHONY: run
@@ -71,9 +71,7 @@ run: ## Start a Gotenberg container
--gotenberg-graceful-shutdown-duration=$(GOTENBERG_GRACEFUL_SHUTDOWN_DURATION) \
--api-port=$(API_PORT) \
--api-port-from-env=$(API_PORT_FROM_ENV) \
--api-read-timeout=$(API_READ_TIMEOUT) \
--api-process-timeout=$(API_PROCESS_TIMEOUT) \
--api-write-timeout=$(API_WRITE_TIMEOUT) \
--api-timeout=$(API_TIMEOUT) \
--api-root-path=$(API_ROOT_PATH) \
--api-trace-header=$(API_TRACE_HEADER) \
--api-disable-health-check-logging=$(API_DISABLE_HEALTH_CHECK_LOGGING) \
@@ -95,6 +93,7 @@ run: ## Start a Gotenberg container
--prometheus-collect-interval=$(PROMETHEUS_COLLECT_INTERVAL) \
--prometheus-disable-route-logging=$(PROMETHEUS_DISABLE_ROUTE_LOGGING) \
--prometheus-disable-collect=$(PROMETHEUS_DISABLE_COLLECT) \
--unoconv-disable-listener=$(UNOCONV_DISABLE_LISTENER) \
--webhook-allow-list=$(WEBHOOK_ALLOW_LIST) \
--webhook-deny-list=$(WEBHOOK_DENY_LIST) \
--webhook-error-allow-list=$(WEBHOOK_ERROR_ALLOW_LIST) \
@@ -102,6 +101,7 @@ run: ## Start a Gotenberg container
--webhook-max-retry=$(WEBHOOK_MAX_RETRY) \
--webhook-retry-min-wait=$(WEBHOOK_RETRY_MIN_WAIT) \
--webhook-retry-max-wait=$(WEBHOOK_RETRY_MAX_WAIT) \
--webhook-client-timeout=$(WEBHOOK_CLIENT_TIMEOUT) \
--webhook-disable=$(WEBHOOK_DISABLE)
.PHONY: build-tests
@@ -145,5 +145,6 @@ release: ## Build the Gotenberg's Docker image for many platforms, then push it
$(GOTENBERG_VERSION) \
$(GOTENBERG_USER_GID) \
$(GOTENBERG_USER_UID) \
$(NOTO_COLOR_EMOJI_VERSION) \
$(PDFTK_VERSION) \
$(DOCKER_REPOSITORY)

View File

@@ -7,8 +7,8 @@
---
Gotenberg provides a developer-friendly API to interact with powerful tools like Chromium and LibreOffice to convert many
documents (HTML, Markdown, Word, Excel, etc.) to PDF, transform them, merge them, and more!
Gotenberg provides a developer-friendly API to interact with powerful tools like Chromium and LibreOffice for converting
numerous document formats (HTML, Markdown, Word, Excel, etc.) into PDF files, and more!
## Quick Start

View File

@@ -64,10 +64,10 @@ RUN \
mkdir /home/gotenberg &&\
chown gotenberg: /home/gotenberg &&\
# Install dependencies required for the next instructions or debugging.
# Note: procps for "top" command (useful when debugging processes).
# Note: tini is a helper for reaping zombie processes.
apt-get update -qq &&\
DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends curl gnupg procps tini python3 default-jre-headless &&\
DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends curl gnupg htop tini python3 default-jre-headless &&\
ln -s /usr/bin/htop /usr/bin/top &&\
# Install fonts.
# Credits:
# https://github.com/arachnys/athenapdf/blob/master/cli/Dockerfile.
@@ -103,6 +103,7 @@ RUN \
fonts-liberation \
fonts-liberation2 \
fonts-linuxlibertine \
fonts-noto-cjk \
fonts-noto-core \
fonts-noto-mono \
fonts-noto-ui-core \
@@ -117,11 +118,13 @@ RUN \
# Install Google Chrome / Chromium.
/tmp/install-chromium.sh &&\
# Install LibreOffice.
# Note: we use the sid distribution to get the latest LibreOffice version.
# See https://github.com/gotenberg/gotenberg/pull/322.
echo "deb https://httpredir.debian.org/debian/ sid main contrib non-free" >> /etc/apt/sources.list &&\
# Note: we use the bullseye-backports distribution to get the latest LibreOffice version.
# See:
# https://github.com/gotenberg/gotenberg/pull/322.
# https://github.com/gotenberg/gotenberg/issues/403.
echo "deb https://httpredir.debian.org/debian/ bullseye-backports main contrib non-free" >> /etc/apt/sources.list &&\
apt-get update -qq &&\
DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends -t sid libreoffice &&\
DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends -t bullseye-backports libreoffice &&\
# Download unoconv (Python script).
curl -Ls https://raw.githubusercontent.com/dagwieers/unoconv/master/unoconv -o /usr/bin/unoconv &&\
chmod +x /usr/bin/unoconv &&\
@@ -131,6 +134,8 @@ RUN \
# See https://github.com/gotenberg/gotenberg/pull/273.
curl -o /usr/bin/pdftk-all.jar "https://gitlab.com/pdftk-java/pdftk/-/jobs/$PDFTK_VERSION/artifacts/raw/build/libs/pdftk-all.jar" &&\
chmod a+x /usr/bin/pdftk-all.jar &&\
# Download QPDF.
DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends qpdf &&\
# See https://github.com/nextcloud/docker/issues/380.
mkdir -p /usr/share/man/man1mkdir -p /usr/share/man/man1 &&\
# Cleanup.
@@ -142,7 +147,8 @@ RUN \
chromium --version &&\
libreoffice --version &&\
unoconv --version &&\
pdftk --version
pdftk --version &&\
qpdf --version
# Copy the Gotenberg binary from the builder stage.
COPY --from=builder /home/gotenberg /usr/bin/
@@ -152,6 +158,7 @@ ENV GC_EXCLUDE_SUBSTR "hsperfdata_root,hsperfdata_gotenberg"
ENV CHROMIUM_BIN_PATH /usr/bin/chromium
ENV UNOCONV_BIN_PATH /usr/bin/unoconv
ENV PDFTK_BIN_PATH /usr/bin/pdftk
ENV QPDF_BIN_PATH /usr/bin/qpdf
USER gotenberg
WORKDIR /home/gotenberg
@@ -160,4 +167,4 @@ WORKDIR /home/gotenberg
EXPOSE 3000
ENTRYPOINT [ "/usr/bin/tini", "--" ]
CMD [ "gotenberg" ]
CMD [ "gotenberg" ]

View File

@@ -9,6 +9,7 @@ import (
"github.com/gotenberg/gotenberg/v7/pkg/gotenberg"
flag "github.com/spf13/pflag"
"golang.org/x/sync/errgroup"
)
// See https://patorjk.com/software/taag/#p=display&f=Small%20Slant&t=Gotenberg.
@@ -117,18 +118,40 @@ func Run() {
gracefulShutdownCtx, cancel := context.WithTimeout(context.Background(), gracefulShutdownDuration)
defer cancel()
forceQuit := make(chan os.Signal, 1)
signal.Notify(forceQuit, os.Interrupt)
go func() {
// In case of force quit, cancel the context.
<-forceQuit
cancel()
}()
fmt.Printf("[SYSTEM] graceful shutdown of %s\n", gracefulShutdownDuration)
eg, _ := errgroup.WithContext(gracefulShutdownCtx)
for _, a := range apps {
id := a.(gotenberg.Module).Descriptor().ID
app := a.(gotenberg.App)
eg.Go(func(app gotenberg.App) func() error {
return func() error {
id := app.(gotenberg.Module).Descriptor().ID
err = app.Stop(gracefulShutdownCtx)
if err != nil {
fmt.Printf("[ERROR] stopping %s: %s\n", id, err)
}
err = app.Stop(gracefulShutdownCtx)
if err != nil {
return fmt.Errorf("stopping %s: %w", id, err)
}
fmt.Printf("[SYSTEM] %s: application stopped\n", id)
fmt.Printf("[SYSTEM] %s: application stopped\n", id)
return nil
}
}(a.(gotenberg.App)))
}
err = eg.Wait()
if err != nil {
fmt.Printf("[FATAL] %v\n", err)
os.Exit(1)
}
os.Exit(0)

8
go.mod
View File

@@ -5,8 +5,8 @@ go 1.17
require (
github.com/alexliesenfeld/health v0.6.0
github.com/andybalholm/brotli v1.0.4 // indirect
github.com/chromedp/cdproto v0.0.0-20211112223831-e508c7bf6446
github.com/chromedp/chromedp v0.7.4
github.com/chromedp/cdproto v0.0.0-20211226214557-5a9a0564ad26
github.com/chromedp/chromedp v0.7.6
github.com/golang/snappy v0.0.4 // indirect
github.com/google/uuid v1.3.0
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
@@ -20,7 +20,7 @@ require (
github.com/microcosm-cc/bluemonday v1.0.16
github.com/nwaples/rardecode v1.1.2 // indirect
github.com/pdfcpu/pdfcpu v0.3.12
github.com/pierrec/lz4/v4 v4.1.11 // indirect
github.com/pierrec/lz4/v4 v4.1.12 // indirect
github.com/prometheus/client_golang v1.11.0
github.com/russross/blackfriday/v2 v2.1.0
github.com/spf13/pflag v1.0.5
@@ -32,7 +32,7 @@ require (
golang.org/x/image v0.0.0-20211028202545-6944b10bf410 // indirect
golang.org/x/net v0.0.0-20210913180222-943fd674d43e
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
golang.org/x/sys v0.0.0-20211103235746-7861aae1554b // indirect
golang.org/x/sys v0.0.0-20211124211545-fe61309f8881 // indirect
golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b
golang.org/x/text v0.3.7
)

18
go.sum
View File

@@ -55,11 +55,11 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/chromedp/cdproto v0.0.0-20210713064928-7d28b402946a/go.mod h1:At5TxYYdxkbQL0TSefRjhLE3Q0lgvqKKMSFUglJ7i1U=
github.com/chromedp/cdproto v0.0.0-20211112223831-e508c7bf6446 h1:uG7ssld4pEDC5O+1UoKqMb0djjJb8qjuKZVnCskGY+s=
github.com/chromedp/cdproto v0.0.0-20211112223831-e508c7bf6446/go.mod h1:At5TxYYdxkbQL0TSefRjhLE3Q0lgvqKKMSFUglJ7i1U=
github.com/chromedp/chromedp v0.7.4 h1:U+0d3WbB/Oj4mDuBOI0P7S3PJEued5UZIl5AJ3QulwU=
github.com/chromedp/chromedp v0.7.4/go.mod h1:dBj+SXuQHznp6ZPwZeDDEBZKwclUwDLbZ0hjMialMYs=
github.com/chromedp/cdproto v0.0.0-20211126220118-81fa0469ad77/go.mod h1:At5TxYYdxkbQL0TSefRjhLE3Q0lgvqKKMSFUglJ7i1U=
github.com/chromedp/cdproto v0.0.0-20211226214557-5a9a0564ad26 h1:sj/nGJwGadNGvxuA/dKFC+3o9hHi0PTGb+iIMhlCX6A=
github.com/chromedp/cdproto v0.0.0-20211226214557-5a9a0564ad26/go.mod h1:At5TxYYdxkbQL0TSefRjhLE3Q0lgvqKKMSFUglJ7i1U=
github.com/chromedp/chromedp v0.7.6 h1:2juGaktzjwULlsn+DnvIZXFUckEp5xs+GOBroaea+jA=
github.com/chromedp/chromedp v0.7.6/go.mod h1:ayT4YU/MGAALNfOg9gNrpGSAdnU51PMx+FCeuT1iXzo=
github.com/chromedp/sysutil v1.0.0 h1:+ZxhTpfpZlmchB58ih/LBHX52ky7w2VhQVKQMucy3Ic=
github.com/chromedp/sysutil v1.0.0/go.mod h1:kgWmDdq8fTzXYcKIBqIYvRRTnYb9aNS9moAV0xufSww=
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
@@ -233,8 +233,8 @@ github.com/orisano/pixelmatch v0.0.0-20210112091706-4fa4c7ba91d5/go.mod h1:nZgzb
github.com/pdfcpu/pdfcpu v0.3.12 h1:B+MdKisilWNSk5OCO58Z9U6H93usH73xqk6hMOaZCls=
github.com/pdfcpu/pdfcpu v0.3.12/go.mod h1:8XVBtVxuuIuSZL4Ez15Q4QoC+H8zeAaGnuiOEwAk8jA=
github.com/pierrec/lz4/v4 v4.1.2/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
github.com/pierrec/lz4/v4 v4.1.11 h1:LVs17FAZJFOjgmJXl9Tf13WfLUvZq7/RjfEJrnwZ9OE=
github.com/pierrec/lz4/v4 v4.1.11/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
github.com/pierrec/lz4/v4 v4.1.12 h1:44l88ehTZAUGW4VlO1QC4zkilL99M6Y9MXNwEs0uzP8=
github.com/pierrec/lz4/v4 v4.1.12/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
@@ -442,14 +442,14 @@ golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210525143221-35b2ab0089ea/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210910150752-751e447fb3d0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211103235746-7861aae1554b h1:1VkfZQv42XQlA/jchYumAnv1UPo6RgF9rJFkTgZIxO4=
golang.org/x/sys v0.0.0-20211103235746-7861aae1554b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211124211545-fe61309f8881 h1:TyHqChC80pFkXWraUUf6RuB5IqFdQieMLwwCJokV2pc=
golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b h1:9zKuko04nR4gjZ4+DNjHqRlAJqbJETHwiNKDqTfOjfE=
golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=

View File

@@ -31,7 +31,7 @@ func Command(logger *zap.Logger, binPath string, args ...string) Cmd {
return Cmd{
ctx: nil,
logger: logger.Named("cmd"),
logger: logger.Named(strings.ReplaceAll(binPath, "/", "")),
process: cmd,
}
}
@@ -51,7 +51,7 @@ func CommandContext(ctx context.Context, logger *zap.Logger, binPath string, arg
return Cmd{
ctx: ctx,
logger: logger.Named("cmd"),
logger: logger.Named(strings.ReplaceAll(binPath, "/", "")),
process: cmd,
}, nil
}

View File

@@ -30,8 +30,8 @@ func init() {
type API struct {
port int
readTimeout time.Duration
processTimeout time.Duration
writeTimeout time.Duration
timeout time.Duration
rootPath string
traceHeader string
disableHealthCheckLogging bool
@@ -166,10 +166,20 @@ func (API) Descriptor() gotenberg.ModuleDescriptor {
fs.Duration("api-read-timeout", time.Duration(30)*time.Second, "Set the maximum duration allowed to read a complete request, including the body")
fs.Duration("api-process-timeout", time.Duration(30)*time.Second, "Set the maximum duration allowed to process a request")
fs.Duration("api-write-timeout", time.Duration(30)*time.Second, "Set the maximum duration before timing out writes of the response")
fs.Duration("api-timeout", time.Duration(30)*time.Second, "Set the time limit for requests")
fs.String("api-root-path", "/", "Set the root path of the API - for service discovery via URL paths")
fs.String("api-trace-header", "Gotenberg-Trace", "Set the header name to use for identifying requests")
fs.Bool("api-disable-health-check-logging", false, "Disable health check logging")
var err error
err = multierr.Append(err, fs.MarkDeprecated("api-read-timeout", "use api-timeout instead"))
err = multierr.Append(err, fs.MarkDeprecated("api-process-timeout", "use api-timeout instead"))
err = multierr.Append(err, fs.MarkDeprecated("api-write-timeout", "use api-timeout instead"))
if err != nil {
panic(fmt.Errorf("create deprecated flags for the api module: %v", err))
}
return fs
}(),
New: func() gotenberg.Module { return new(API) },
@@ -180,9 +190,9 @@ func (API) Descriptor() gotenberg.ModuleDescriptor {
func (a *API) Provision(ctx *gotenberg.Context) error {
flags := ctx.ParsedFlags()
a.port = flags.MustInt("api-port")
a.readTimeout = flags.MustDuration("api-read-timeout")
a.processTimeout = flags.MustDuration("api-process-timeout")
a.writeTimeout = flags.MustDuration("api-write-timeout")
a.readTimeout = flags.MustDeprecatedDuration("api-read-timeout", "api-timeout")
a.writeTimeout = flags.MustDeprecatedDuration("api-write-timeout", "api-timeout")
a.timeout = flags.MustDeprecatedDuration("api-process-timeout", "api-timeout")
a.rootPath = flags.MustString("api-root-path")
a.traceHeader = flags.MustString("api-trace-header")
a.disableHealthCheckLogging = flags.MustBool("api-disable-health-check-logging")
@@ -274,7 +284,7 @@ func (a *API) Provision(ctx *gotenberg.Context) error {
}
// Grace duration.
a.gcGraceDuration = a.readTimeout + a.processTimeout + a.writeTimeout
a.gcGraceDuration = a.timeout
mods, err = ctx.Modules(new(GarbageCollectorGraceDurationIncrementer))
if err != nil {
@@ -378,7 +388,9 @@ func (a *API) Start() error {
a.srv.HideBanner = true
a.srv.HidePort = true
a.srv.Server.ReadTimeout = a.readTimeout
a.srv.Server.WriteTimeout = a.writeTimeout
a.srv.Server.IdleTimeout = a.timeout
// See https://github.com/gotenberg/gotenberg/issues/396.
a.srv.Server.WriteTimeout = a.writeTimeout + a.writeTimeout
a.srv.HTTPErrorHandler = httpErrorHandler()
// Let's prepare the modules' routes.
@@ -402,7 +414,6 @@ func (a *API) Start() error {
latencyMiddleware(),
rootPathMiddleware(a.rootPath),
traceMiddleware(a.traceHeader),
timeoutsMiddleware(a.readTimeout, a.processTimeout, a.writeTimeout),
loggerMiddleware(a.logger, disableLoggingForPaths),
)
@@ -419,14 +430,14 @@ func (a *API) Start() error {
}
}
hardTimeout := a.processTimeout + (time.Duration(5) * time.Second)
hardTimeout := a.timeout + (time.Duration(5) * time.Second)
// Add the modules' routes and their specific middlewares.
for _, route := range a.routes {
var middlewares []echo.MiddlewareFunc
if route.IsMultipart {
middlewares = append(middlewares, contextMiddleware(a.processTimeout))
middlewares = append(middlewares, contextMiddleware(a.timeout))
for _, externalMultipartMiddleware := range externalMultipartMiddlewares {
middlewares = append(middlewares, externalMultipartMiddleware.Handler)
@@ -447,7 +458,7 @@ func (a *API) Start() error {
a.srv.GET(
fmt.Sprintf("%s%s", a.rootPath, "health"),
func() echo.HandlerFunc {
checks := append(a.healthChecks, health.WithTimeout(a.processTimeout))
checks := append(a.healthChecks, health.WithTimeout(a.timeout))
checker := health.NewChecker(checks...)
return echo.WrapHandler(health.NewHandler(checker))

View File

@@ -386,7 +386,7 @@ func TestAPI_Provision(t *testing.T) {
},
)
}(),
expectGraceDuration: time.Duration(93) * time.Second,
expectGraceDuration: time.Duration(33) * time.Second,
expectErr: true,
},
{

View File

@@ -5,6 +5,7 @@ import (
"errors"
"fmt"
"net/http"
"strings"
"time"
"github.com/google/uuid"
@@ -119,25 +120,6 @@ func traceMiddleware(header string) echo.MiddlewareFunc {
}
}
// timeoutsMiddleware sets the read, process and write timeouts in the
// echo.Context under "readTimeout", "processTimeout" and "writeTimeout".
//
// readTimeout := c.Get("readTimeout").(time.Duration)
// processTimeout := c.Get("processTimeout").(time.Duration)
// writeTimeout := c.Get("writeTimeout").(time.Duration)
func timeoutsMiddleware(readTimeout, processTimeout, writeTimeout time.Duration) echo.MiddlewareFunc {
return func(next echo.HandlerFunc) echo.HandlerFunc {
return func(c echo.Context) error {
c.Set("readTimeout", readTimeout)
c.Set("processTimeout", processTimeout)
c.Set("writeTimeout", writeTimeout)
// Call the next middleware in the chain.
return next(c)
}
}
}
// loggerMiddleware sets the logger in the echo.Context under "logger" and logs
// a synchronous request result.
//
@@ -147,10 +129,17 @@ func loggerMiddleware(logger *zap.Logger, disableLoggingForPaths []string) echo.
return func(c echo.Context) error {
startTime := c.Get("startTime").(time.Time)
trace := c.Get("trace").(string)
rootPath := c.Get("rootPath").(string)
// Create the request logger and add it to our locals.
reqLogger := logger.With(zap.String("trace", trace))
c.Set("logger", reqLogger)
c.Set("logger", reqLogger.Named(func() string {
return strings.ReplaceAll(
strings.ReplaceAll(c.Request().URL.Path, rootPath, ""),
"/",
"",
)
}()))
// Call the next middleware in the chain.
err := next(c)
@@ -159,7 +148,6 @@ func loggerMiddleware(logger *zap.Logger, disableLoggingForPaths []string) echo.
}
for _, path := range disableLoggingForPaths {
rootPath := c.Get("rootPath").(string)
URI := fmt.Sprintf("%s%s", rootPath, path)
if c.Request().RequestURI == URI {
@@ -211,14 +199,14 @@ func loggerMiddleware(logger *zap.Logger, disableLoggingForPaths []string) echo.
//
// ctx := c.Get("context").(*api.Context)
// cancel := c.Get("cancel").(context.CancelFunc)
func contextMiddleware(processTimeout time.Duration) echo.MiddlewareFunc {
func contextMiddleware(timeout time.Duration) echo.MiddlewareFunc {
return func(next echo.HandlerFunc) echo.HandlerFunc {
return func(c echo.Context) error {
logger := c.Get("logger").(*zap.Logger)
// We create a context with a timeout so that underlying processes are
// able to stop early and handle correctly a timeout scenario.
ctx, cancel, err := newContext(c, logger, processTimeout)
ctx, cancel, err := newContext(c, logger, timeout)
if err != nil {
cancel()

View File

@@ -222,47 +222,6 @@ func TestTraceMiddleware(t *testing.T) {
}
}
func TestTimeoutsMiddleware(t *testing.T) {
recorder := httptest.NewRecorder()
request := httptest.NewRequest(http.MethodGet, "/foo", nil)
srv := echo.New()
srv.HideBanner = true
srv.HidePort = true
c := srv.NewContext(request, recorder)
expectReadTimeout := time.Duration(1) * time.Second
expectProcessTimeout := time.Duration(2) * time.Second
expectWriteTimeout := time.Duration(3) * time.Second
err := timeoutsMiddleware(expectReadTimeout, expectProcessTimeout, expectWriteTimeout)(
func(c echo.Context) error {
return nil
},
)(c)
if err != nil {
t.Fatalf("expected no error but got: %v", err)
}
actualReadTimeout := c.Get("readTimeout").(time.Duration)
actualProcessTimeout := c.Get("processTimeout").(time.Duration)
actualWriteTimeout := c.Get("writeTimeout").(time.Duration)
if actualReadTimeout != expectReadTimeout {
t.Errorf("expected '%s' but got '%s", expectReadTimeout, actualReadTimeout)
}
if actualProcessTimeout != expectProcessTimeout {
t.Errorf("expected '%s' but got '%s", expectProcessTimeout, actualProcessTimeout)
}
if actualWriteTimeout != expectWriteTimeout {
t.Errorf("expected '%s' but got '%s", actualWriteTimeout, expectWriteTimeout)
}
}
func TestLoggerMiddleware(t *testing.T) {
for i, tc := range []struct {
request *http.Request

View File

@@ -85,7 +85,7 @@ type LinkTag struct {
// ScriptTag represents an HTML <script> element.
type ScriptTag struct {
// Src is the "src" attribute of the HTML <link> element.
// Src is the "src" attribute of the HTML <script> element.
// Required.
Src string `json:"src"`
}
@@ -355,7 +355,7 @@ func (mod Chromium) Routes() ([]api.Route, error) {
// drastically. In such a scenario, the given context may also be done before
// the end of the conversion.
func (mod Chromium) PDF(ctx context.Context, logger *zap.Logger, URL, outputPath string, options Options) error {
debug := debugLogger{logger: logger.Named("chromium.debug")}
debug := debugLogger{logger: logger.Named("browser")}
userProfileDirPath := gotenberg.NewDirPath()
args := append(chromedp.DefaultExecAllocatorOptions[:],

View File

@@ -176,11 +176,11 @@ func TestChromium_Validate(t *testing.T) {
func TestChromium_Metrics(t *testing.T) {
metrics, err := new(Chromium).Metrics()
if err != nil {
t.Errorf("expected no error but got: %v", err)
t.Fatalf("expected no error but got: %v", err)
}
if len(metrics) != 1 {
t.Errorf("expected %d metrics, but got %d", 1, len(metrics))
t.Fatalf("expected %d metrics, but got %d", 1, len(metrics))
}
actual := metrics[0].Read()

View File

@@ -5,7 +5,6 @@ import (
"errors"
"fmt"
"os"
"os/signal"
"path/filepath"
"strings"
"sync"
@@ -202,19 +201,10 @@ func (gc *GarbageCollector) Stop(ctx context.Context) error {
}
// Block until the context is done so that other module may gracefully stop
// before we do a shutdown cleanup. We skip this step if we receive a
// SIGINT in the meantime.
// before we do a shutdown cleanup.
gc.logger.Debug("wait for the end of grace duration")
quit := make(chan os.Signal, 1)
signal.Notify(quit, os.Interrupt)
select {
case <-quit:
return nil
case <-ctx.Done():
break
}
<-ctx.Done()
gc.ticker.Stop()
gc.done <- true

View File

@@ -0,0 +1,31 @@
package unoconv
import (
"fmt"
"net"
"strconv"
"go.uber.org/zap"
)
func freePort(logger *zap.Logger) (int, error) {
listener, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
return 0, fmt.Errorf("listen on the local network address: %w", err)
}
defer func() {
err := listener.Close()
if err != nil {
logger.Error(fmt.Sprintf("close listener: %s", err.Error()))
}
}()
addr := listener.Addr().String()
_, portStr, err := net.SplitHostPort(addr)
if err != nil {
return 0, fmt.Errorf("get free port from host: %w", err)
}
return strconv.Atoi(portStr)
}

View File

@@ -4,13 +4,12 @@ import (
"context"
"errors"
"fmt"
"net"
"os"
"strconv"
"strings"
"sync"
"github.com/gotenberg/gotenberg/v7/pkg/gotenberg"
flag "github.com/spf13/pflag"
"go.uber.org/zap"
)
@@ -24,7 +23,12 @@ var ErrMalformedPageRanges = errors.New("page ranges are malformed")
// Unoconv is a module which provides an API to interact with unoconv.
type Unoconv struct {
binPath string
binPath string
disableListener bool
listenerCmd gotenberg.Cmd
listenerPort int
logger *zap.Logger
}
// Options gathers available options when converting a document to PDF.
@@ -67,14 +71,23 @@ type Provider interface {
// Descriptor returns a Unoconv's module descriptor.
func (Unoconv) Descriptor() gotenberg.ModuleDescriptor {
return gotenberg.ModuleDescriptor{
ID: "unoconv",
ID: "unoconv",
FlagSet: func() *flag.FlagSet {
fs := flag.NewFlagSet("unoconv", flag.ExitOnError)
fs.Bool("unoconv-disable-listener", false, "Do not start a unoconv listener - save resources in detriment of performance")
return fs
}(),
New: func() gotenberg.Module { return new(Unoconv) },
}
}
// Provision sets the module properties. It returns an error if the environment
// variable UNOCONV_BIN_PATH is not set.
func (mod *Unoconv) Provision(_ *gotenberg.Context) error {
func (mod *Unoconv) Provision(ctx *gotenberg.Context) error {
flags := ctx.ParsedFlags()
mod.disableListener = flags.MustBool("unoconv-disable-listener")
binPath, ok := os.LookupEnv("UNOCONV_BIN_PATH")
if !ok {
return errors.New("UNOCONV_BIN_PATH environment variable is not set")
@@ -82,6 +95,18 @@ func (mod *Unoconv) Provision(_ *gotenberg.Context) error {
mod.binPath = binPath
loggerProvider, err := ctx.Module(new(gotenberg.LoggerProvider))
if err != nil {
return fmt.Errorf("get logger provider: %w", err)
}
logger, err := loggerProvider.(gotenberg.LoggerProvider).Logger(mod)
if err != nil {
return fmt.Errorf("get logger: %w", err)
}
mod.logger = logger
return nil
}
@@ -95,12 +120,92 @@ func (mod Unoconv) Validate() error {
return nil
}
func (mod *Unoconv) Start() error {
if mod.disableListener {
return nil
}
port, err := freePort(mod.logger)
if err != nil {
return fmt.Errorf("get free port: %w", err)
}
mod.listenerPort = port
args := []string{
"--listener",
"--user-profile",
// Just to make sure LibreOffice does not leak files in an unknown
// directory. The directory will be removed anyway by the garbage
// collector.
fmt.Sprintf("//%s", gotenberg.NewDirPath()),
"--port",
fmt.Sprintf("%d", mod.listenerPort),
}
checkedEntry := mod.logger.Check(zap.DebugLevel, "check for debug level before setting high verbosity")
if checkedEntry != nil {
args = append(args, "-vvv")
}
mod.listenerCmd = gotenberg.Command(mod.logger, mod.binPath, args...)
err = mod.listenerCmd.Start()
if err != nil {
return fmt.Errorf("start unoconv listener: %w", err)
}
listenerActiveInstancesCountMu.Lock()
listenerActiveInstancesCount += 1
listenerActiveInstancesCountMu.Unlock()
return nil
}
// StartupMessage returns a custom startup message.
func (mod Unoconv) StartupMessage() string {
if mod.disableListener {
return "listener disabled"
}
return fmt.Sprintf("listener started on port %d", mod.listenerPort)
}
// Stop stops the HTTP server.
func (mod *Unoconv) Stop(ctx context.Context) error {
if mod.disableListener {
return nil
}
_, ok := ctx.Deadline()
if !ok {
return errors.New("no context dead line")
}
// Block until the context is done so that other module may gracefully stop
// before we do a shutdown cleanup.
mod.logger.Debug("wait for the end of grace duration")
<-ctx.Done()
err := mod.listenerCmd.Kill()
if err != nil {
return fmt.Errorf("kill unoconv listener: %w", err)
}
listenerActiveInstancesCountMu.Lock()
listenerActiveInstancesCount -= 1
listenerActiveInstancesCountMu.Unlock()
return nil
}
// Metrics returns the metrics.
func (mod Unoconv) Metrics() ([]gotenberg.Metric, error) {
return []gotenberg.Metric{
{
Name: "unoconv_active_instances_count",
Description: "Current number of active LibreOffice instances.",
Description: "Current number of active unoconv instances.",
Read: func() float64 {
activeInstancesCountMu.RLock()
defer activeInstancesCountMu.RUnlock()
@@ -108,57 +213,70 @@ func (mod Unoconv) Metrics() ([]gotenberg.Metric, error) {
return activeInstancesCount
},
},
{
Name: "unoconv_listener_active_instances_count",
Description: "Current number of active unoconv listener instances.",
Read: func() float64 {
listenerActiveInstancesCountMu.RLock()
defer listenerActiveInstancesCountMu.RUnlock()
return listenerActiveInstancesCount
},
},
{
Name: "unoconv_listener_queue_length",
Description: "Current number of processes in the queue.",
Read: func() float64 {
listenerQueueLengthMu.RLock()
defer listenerQueueLengthMu.RUnlock()
return listenerQueueLength
},
},
}, nil
}
// Unoconv returns an API for interacting with unoconv.
func (mod Unoconv) Unoconv() (API, error) {
func (mod *Unoconv) Unoconv() (API, error) {
return mod, nil
}
// PDF converts a document to PDF. It creates a dedicated LibreOffice instance
// thanks to a custom user profile directory and a free port. Substantial calls
// to this method may increase CPU and memory usage drastically. In such a
// scenario, the given context may also be done before the end of the
// conversion.
// PDF converts a document to PDF.
//
// In stateless mode, it creates a dedicated LibreOffice instance thanks to a
// custom user profile directory and a free port. Substantial calls to this
// method may increase CPU and memory usage drastically. In such a scenario,
// the given context may also be done before the end of the conversion.
//
// In listener mode, it calls the unoconv listener to interact with
// LibreOffice, improving substantially the performance. However, it cannot
// perform parallel operations and have to wait for the lock to be available.
func (mod Unoconv) PDF(ctx context.Context, logger *zap.Logger, inputPath, outputPath string, options Options) error {
port, err := func() (int, error) {
listener, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
return 0, fmt.Errorf("listen on the local network address: %w", err)
}
defer func() {
err := listener.Close()
if err != nil {
logger.Error(fmt.Sprintf("close listener: %s", err.Error()))
}
}()
addr := listener.Addr().String()
_, portStr, err := net.SplitHostPort(addr)
if err != nil {
return 0, fmt.Errorf("get free port from host: %w", err)
}
return strconv.Atoi(portStr)
}()
if err != nil {
return fmt.Errorf("get free port: %w", err)
}
userProfileDirPath := gotenberg.NewDirPath()
args := []string{
"--user-profile",
fmt.Sprintf("//%s", userProfileDirPath),
"--port",
fmt.Sprintf("%d", port),
"--format",
"pdf",
}
var userProfileDirPath string
if mod.disableListener {
port, err := freePort(logger)
if err != nil {
return fmt.Errorf("get free port: %w", err)
}
userProfileDirPath = gotenberg.NewDirPath()
args = append(args,
"--port",
fmt.Sprintf("%d", port),
"--user-profile",
fmt.Sprintf("//%s", userProfileDirPath),
)
} else {
args = append(args, "--port", fmt.Sprintf("%d", mod.listenerPort))
}
checkedEntry := logger.Check(zap.DebugLevel, "check for debug level before setting high verbosity")
if checkedEntry != nil {
args = append(args, "-vvv")
@@ -178,6 +296,36 @@ func (mod Unoconv) PDF(ctx context.Context, logger *zap.Logger, inputPath, outpu
args = append(args, "--output", outputPath, inputPath)
if !mod.disableListener {
listenerQueueLengthMu.Lock()
listenerQueueLength += 1
listenerQueueLengthMu.Unlock()
select {
case listenerLock <- struct{}{}:
logger.Debug("unoconv lock acquired")
listenerQueueLengthMu.Lock()
listenerQueueLength -= 1
listenerQueueLengthMu.Unlock()
break
case <-ctx.Done():
logger.Debug("failed to acquire the unoconv lock before deadline")
listenerQueueLengthMu.Lock()
listenerQueueLength -= 1
listenerQueueLengthMu.Unlock()
return fmt.Errorf("acquire unoconv lock: %w", ctx.Err())
}
defer func() {
<-listenerLock
logger.Debug("unoconv lock released")
}()
}
cmd, err := gotenberg.CommandContext(ctx, logger, mod.binPath, args...)
if err != nil {
return fmt.Errorf("create unoconv command: %w", err)
@@ -195,16 +343,18 @@ func (mod Unoconv) PDF(ctx context.Context, logger *zap.Logger, inputPath, outpu
activeInstancesCount -= 1
activeInstancesCountMu.Unlock()
// Always remove the user profile directory created by LibreOffice.
// See https://github.com/gotenberg/gotenberg/issues/192.
go func() {
logger.Debug(fmt.Sprintf("remove user profile directory '%s'", userProfileDirPath))
if mod.disableListener {
// Always remove the user profile directory created by LibreOffice.
// See https://github.com/gotenberg/gotenberg/issues/192.
go func() {
logger.Debug(fmt.Sprintf("remove user profile directory '%s'", userProfileDirPath))
err := os.RemoveAll(userProfileDirPath)
if err != nil {
logger.Error(fmt.Sprintf("remove user profile directory: %s", err))
}
}()
err := os.RemoveAll(userProfileDirPath)
if err != nil {
logger.Error(fmt.Sprintf("remove user profile directory: %s", err))
}
}()
}
if err == nil {
return nil
@@ -311,8 +461,13 @@ func (mod Unoconv) Extensions() []string {
}
var (
activeInstancesCount float64
activeInstancesCountMu sync.RWMutex
listenerLock = make(chan struct{}, 1)
listenerQueueLength float64
listenerQueueLengthMu sync.RWMutex
listenerActiveInstancesCount float64
listenerActiveInstancesCountMu sync.RWMutex
activeInstancesCount float64
activeInstancesCountMu sync.RWMutex
)
// Interface guards.
@@ -320,6 +475,7 @@ var (
_ gotenberg.Module = (*Unoconv)(nil)
_ gotenberg.Provisioner = (*Unoconv)(nil)
_ gotenberg.Validator = (*Unoconv)(nil)
_ gotenberg.App = (*Unoconv)(nil)
_ gotenberg.MetricsProvider = (*Unoconv)(nil)
_ API = (*Unoconv)(nil)
_ Provider = (*Unoconv)(nil)

View File

@@ -2,14 +2,42 @@ package unoconv
import (
"context"
"errors"
"os"
"reflect"
"testing"
"time"
"github.com/gotenberg/gotenberg/v7/pkg/gotenberg"
"go.uber.org/zap"
)
type ProtoModule struct {
descriptor func() gotenberg.ModuleDescriptor
}
func (mod ProtoModule) Descriptor() gotenberg.ModuleDescriptor {
return mod.descriptor()
}
type ProtoValidator struct {
ProtoModule
validate func() error
}
func (mod ProtoValidator) Validate() error {
return mod.validate()
}
type ProtoLoggerProvider struct {
ProtoModule
logger func(mod gotenberg.Module) (*zap.Logger, error)
}
func (factory ProtoLoggerProvider) Logger(mod gotenberg.Module) (*zap.Logger, error) {
return factory.logger(mod)
}
func TestUnoconv_Descriptor(t *testing.T) {
descriptor := Unoconv{}.Descriptor()
@@ -22,12 +50,69 @@ func TestUnoconv_Descriptor(t *testing.T) {
}
func TestUnoconv_Provision(t *testing.T) {
mod := new(Unoconv)
ctx := gotenberg.NewContext(gotenberg.ParsedFlags{}, nil)
for i, tc := range []struct {
ctx *gotenberg.Context
expectErr bool
}{
{
ctx: gotenberg.NewContext(
gotenberg.ParsedFlags{FlagSet: new(Unoconv).Descriptor().FlagSet},
make([]gotenberg.ModuleDescriptor, 0),
),
expectErr: true,
},
{
ctx: func() *gotenberg.Context {
mod := struct {
ProtoLoggerProvider
}{}
mod.descriptor = func() gotenberg.ModuleDescriptor {
return gotenberg.ModuleDescriptor{ID: "foo", New: func() gotenberg.Module { return mod }}
}
mod.logger = func(mod gotenberg.Module) (*zap.Logger, error) { return nil, errors.New("foo") }
err := mod.Provision(ctx)
if err != nil {
t.Errorf("expected no error but got: %v", err)
return gotenberg.NewContext(
gotenberg.ParsedFlags{
FlagSet: new(Unoconv).Descriptor().FlagSet,
},
[]gotenberg.ModuleDescriptor{
mod.Descriptor(),
},
)
}(),
expectErr: true,
},
{
ctx: func() *gotenberg.Context {
mod := struct {
ProtoLoggerProvider
}{}
mod.descriptor = func() gotenberg.ModuleDescriptor {
return gotenberg.ModuleDescriptor{ID: "foo", New: func() gotenberg.Module { return mod }}
}
mod.logger = func(mod gotenberg.Module) (*zap.Logger, error) { return zap.NewNop(), nil }
return gotenberg.NewContext(
gotenberg.ParsedFlags{
FlagSet: new(Unoconv).Descriptor().FlagSet,
},
[]gotenberg.ModuleDescriptor{
mod.Descriptor(),
},
)
}(),
},
} {
mod := new(Unoconv)
err := mod.Provision(tc.ctx)
if tc.expectErr && err == nil {
t.Errorf("test %d: expected error but got: %v", i, err)
}
if !tc.expectErr && err != nil {
t.Errorf("test %d: expected no error but got: %v", i, err)
}
}
}
@@ -47,8 +132,10 @@ func TestUnoconv_Validate(t *testing.T) {
binPath: os.Getenv("UNOCONV_BIN_PATH"),
},
} {
mod := new(Unoconv)
mod.binPath = tc.binPath
mod := Unoconv{
binPath: tc.binPath,
}
err := mod.Validate()
if tc.expectErr && err == nil {
@@ -61,20 +148,149 @@ func TestUnoconv_Validate(t *testing.T) {
}
}
func TestChromium_Metrics(t *testing.T) {
func TestUnoconv_Start(t *testing.T) {
for i, tc := range []struct {
mod *Unoconv
expectErr bool
}{
{
mod: &Unoconv{
disableListener: true,
logger: zap.NewNop(),
},
},
{
mod: &Unoconv{
binPath: os.Getenv("UNOCONV_BIN_PATH"),
disableListener: false,
logger: zap.NewExample(),
},
},
} {
func() {
err := tc.mod.Start()
if tc.expectErr && err == nil {
t.Errorf("test %d: expected error but got: %v", i, err)
}
if !tc.expectErr && err != nil {
t.Errorf("test %d: expected no error but got: %v", i, err)
}
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(1)*time.Nanosecond)
defer cancel()
err = tc.mod.Stop(ctx)
if err != nil {
t.Errorf("test %d: expected not error but got: %v", i, err)
}
}()
}
}
func TestUnoconv_StartupMessage(t *testing.T) {
for i, tc := range []struct {
disableListener bool
expectMessage string
}{
{
disableListener: true,
expectMessage: "listener disabled",
},
{
expectMessage: "listener started on port 0",
},
} {
mod := Unoconv{
disableListener: tc.disableListener,
}
actual := mod.StartupMessage()
if actual != tc.expectMessage {
t.Errorf("test %d: expected '%s' but got '%s'", i, tc.expectMessage, actual)
}
}
}
func TestUnoconv_Stop(t *testing.T) {
for i, tc := range []struct {
start bool
disableListener bool
timeout time.Duration
expectErr bool
}{
{
disableListener: true,
},
{
expectErr: true,
},
{
start: true,
timeout: time.Duration(1) * time.Nanosecond,
},
} {
func() {
mod := &Unoconv{
binPath: os.Getenv("UNOCONV_BIN_PATH"),
disableListener: tc.disableListener,
logger: zap.NewNop(),
}
if tc.start {
err := mod.Start()
if err != nil {
t.Fatalf("test %d: expected no error but got: %v", i, err)
}
}
var err error
if tc.timeout == 0 {
err = mod.Stop(context.TODO())
} else {
ctx, cancel := context.WithTimeout(context.Background(), tc.timeout)
defer cancel()
err = mod.Stop(ctx)
}
if tc.expectErr && err == nil {
t.Errorf("test %d: expected error but got: %v", i, err)
}
if !tc.expectErr && err != nil {
t.Errorf("test %d: expected no error but got: %v", i, err)
}
}()
}
}
func TestUnoconv_Metrics(t *testing.T) {
metrics, err := new(Unoconv).Metrics()
if err != nil {
t.Errorf("expected no error but got: %v", err)
t.Fatalf("expected no error but got: %v", err)
}
if len(metrics) != 1 {
t.Errorf("expected %d metrics, but got %d", 1, len(metrics))
if len(metrics) != 3 {
t.Fatalf("expected %d metrics, but got %d", 1, len(metrics))
}
actual := metrics[0].Read()
if actual != 0 {
t.Errorf("expected %d unoconv instances, but got %f", 0, actual)
}
actual = metrics[1].Read()
if actual != 0 {
t.Errorf("expected %d unoconv listener instances, but got %f", 0, actual)
}
actual = metrics[2].Read()
if actual != 0 {
t.Errorf("expected %d processes in the queue, but got %f", 0, actual)
}
}
func TestUnoconv_Unoconv(t *testing.T) {
@@ -89,17 +305,26 @@ func TestUnoconv_Unoconv(t *testing.T) {
func TestUnoconv_PDF(t *testing.T) {
for i, tc := range []struct {
ctx context.Context
mod Unoconv
logger *zap.Logger
inputPath string
options Options
expectErr bool
}{
{
mod: Unoconv{
binPath: os.Getenv("UNOCONV_BIN_PATH"),
disableListener: true,
},
logger: zap.NewNop(),
expectErr: true,
},
{
ctx: context.Background(),
ctx: context.Background(),
mod: Unoconv{
binPath: os.Getenv("UNOCONV_BIN_PATH"),
disableListener: true,
},
logger: zap.NewExample(),
inputPath: "/tests/test/testdata/libreoffice/sample1.docx",
options: Options{
@@ -109,7 +334,11 @@ func TestUnoconv_PDF(t *testing.T) {
},
},
{
ctx: context.Background(),
ctx: context.Background(),
mod: Unoconv{
binPath: os.Getenv("UNOCONV_BIN_PATH"),
disableListener: true,
},
logger: zap.NewNop(),
inputPath: "/tests/test/testdata/libreoffice/sample1.docx",
options: Options{
@@ -124,19 +353,40 @@ func TestUnoconv_PDF(t *testing.T) {
return ctx
}(),
mod: Unoconv{
binPath: os.Getenv("UNOCONV_BIN_PATH"),
disableListener: true,
},
logger: zap.NewNop(),
inputPath: "/tests/test/testdata/libreoffice/sample1.docx",
expectErr: true,
},
{
ctx: context.Background(),
mod: Unoconv{
binPath: os.Getenv("UNOCONV_BIN_PATH"),
logger: zap.NewNop(),
},
logger: zap.NewNop(),
inputPath: "/tests/test/testdata/libreoffice/sample1.docx",
},
{
ctx: func() context.Context {
ctx, cancel := context.WithCancel(context.TODO())
defer cancel()
return ctx
}(),
mod: Unoconv{
binPath: os.Getenv("UNOCONV_BIN_PATH"),
logger: zap.NewNop(),
},
logger: zap.NewNop(),
inputPath: "/tests/test/testdata/libreoffice/sample1.docx",
expectErr: true,
},
} {
func() {
mod := new(Unoconv)
err := mod.Provision(nil)
if err != nil {
t.Fatalf("test %d: expected error but got: %v", i, err)
}
outputDir, err := gotenberg.MkdirAll()
if err != nil {
t.Fatalf("test %d: expected error but got: %v", i, err)
@@ -149,7 +399,17 @@ func TestUnoconv_PDF(t *testing.T) {
}
}()
err = mod.PDF(tc.ctx, tc.logger, tc.inputPath, outputDir+"/foo.pdf", tc.options)
if !tc.mod.disableListener {
err = tc.mod.Start()
if err != nil {
t.Fatalf("test %d: expected no error but got: %v", i, err)
}
// Let's give it some room to start.
time.Sleep(time.Duration(1) * time.Second)
}
err = tc.mod.PDF(tc.ctx, tc.logger, tc.inputPath, outputDir+"/foo.pdf", tc.options)
if tc.expectErr && err == nil {
t.Errorf("test %d: expected error but got: %v", i, err)
@@ -158,6 +418,16 @@ func TestUnoconv_PDF(t *testing.T) {
if !tc.expectErr && err != nil {
t.Errorf("test %d: expected no error but got: %v", i, err)
}
if !tc.mod.disableListener {
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(1)*time.Nanosecond)
defer cancel()
err = tc.mod.Stop(ctx)
if err != nil {
t.Fatalf("test %d: expected no error but got: %v", i, err)
}
}
}()
}
}
@@ -173,3 +443,11 @@ func TestUnoconv_Extensions(t *testing.T) {
t.Errorf("expected %d extensions but got %d", expect, actual)
}
}
// Interface guards.
var (
_ gotenberg.Module = (*ProtoModule)(nil)
_ gotenberg.Validator = (*ProtoValidator)(nil)
_ gotenberg.LoggerProvider = (*ProtoLoggerProvider)(nil)
_ gotenberg.Module = (*ProtoLoggerProvider)(nil)
)

View File

@@ -65,11 +65,11 @@ func TestPDFtk_Validate(t *testing.T) {
func TestPDFtk_Metrics(t *testing.T) {
metrics, err := new(PDFtk).Metrics()
if err != nil {
t.Errorf("expected no error but got: %v", err)
t.Fatalf("expected no error but got: %v", err)
}
if len(metrics) != 1 {
t.Errorf("expected %d metrics, but got %d", 1, len(metrics))
t.Fatalf("expected %d metrics, but got %d", 1, len(metrics))
}
actual := metrics[0].Read()

View File

@@ -150,7 +150,7 @@ func (mod Prometheus) Start() error {
// StartupMessage returns a custom startup message.
func (mod Prometheus) StartupMessage() string {
if mod.disableCollect {
return "application not started (collect disabled by user)"
return "collect disabled"
}
return "collecting metrics"

View File

@@ -296,8 +296,8 @@ func TestPrometheus_StartupMessage(t *testing.T) {
expectMessage string
}{
{
expectMessage: "application not started (collect disabled by user)",
disableCollect: true,
expectMessage: "collect disabled",
},
{
expectMessage: "collecting metrics",

3
pkg/modules/qpdf/doc.go Normal file
View File

@@ -0,0 +1,3 @@
// Package qpdf provides a module which abstracts the CLI tool QPDF and
// implements the gotenberg.PDFEngine interface.
package qpdf

117
pkg/modules/qpdf/qpdf.go Normal file
View File

@@ -0,0 +1,117 @@
package qpdf
import (
"context"
"errors"
"fmt"
"os"
"sync"
"github.com/gotenberg/gotenberg/v7/pkg/gotenberg"
"go.uber.org/zap"
)
func init() {
gotenberg.MustRegisterModule(QPDF{})
}
// QPDF abstracts the CLI tool QPDF and implements the gotenberg.PDFEngine
// interface.
type QPDF struct {
binPath string
}
// Descriptor returns a QPDF's module descriptor.
func (QPDF) Descriptor() gotenberg.ModuleDescriptor {
return gotenberg.ModuleDescriptor{
ID: "qpdf",
New: func() gotenberg.Module { return new(QPDF) },
}
}
// Provision sets the modules properties. It returns an error if the
// environment variable QPDF_BIN_PATH is not set.
func (engine *QPDF) Provision(_ *gotenberg.Context) error {
binPath, ok := os.LookupEnv("QPDF_BIN_PATH")
if !ok {
return errors.New("QPDF_BIN_PATH environment variable is not set")
}
engine.binPath = binPath
return nil
}
// Validate validates the module properties.
func (engine QPDF) Validate() error {
_, err := os.Stat(engine.binPath)
if os.IsNotExist(err) {
return fmt.Errorf("QPDF binary path does not exist: %w", err)
}
return nil
}
// Metrics returns the metrics.
func (engine QPDF) Metrics() ([]gotenberg.Metric, error) {
return []gotenberg.Metric{
{
Name: "qpdf_active_instances_count",
Description: "Current number of active QPDF instances.",
Read: func() float64 {
activeInstancesCountMu.RLock()
defer activeInstancesCountMu.RUnlock()
return activeInstancesCount
},
},
}, nil
}
// Merge merges the given PDFs into a unique PDF.
func (engine QPDF) Merge(ctx context.Context, logger *zap.Logger, inputPaths []string, outputPath string) error {
var args []string
args = append(args, "--empty")
args = append(args, "--pages")
args = append(args, inputPaths...)
args = append(args, "--", outputPath)
cmd, err := gotenberg.CommandContext(ctx, logger, engine.binPath, args...)
if err != nil {
return fmt.Errorf("create command: %w", err)
}
activeInstancesCountMu.Lock()
activeInstancesCount += 1
activeInstancesCountMu.Unlock()
err = cmd.Exec()
activeInstancesCountMu.Lock()
activeInstancesCount -= 1
activeInstancesCountMu.Unlock()
if err == nil {
return nil
}
return fmt.Errorf("merge PDFs with QPDF: %w", err)
}
// Convert is not available for this PDF engine.
func (engine QPDF) Convert(_ context.Context, _ *zap.Logger, format, _, _ string) error {
return fmt.Errorf("convert PDF to '%s' with QPDF: %w", format, gotenberg.ErrPDFEngineMethodNotAvailable)
}
var (
activeInstancesCount float64
activeInstancesCountMu sync.RWMutex
)
var (
_ gotenberg.Module = (*QPDF)(nil)
_ gotenberg.Provisioner = (*QPDF)(nil)
_ gotenberg.Validator = (*QPDF)(nil)
_ gotenberg.MetricsProvider = (*QPDF)(nil)
_ gotenberg.PDFEngine = (*QPDF)(nil)
)

View File

@@ -0,0 +1,152 @@
package qpdf
import (
"context"
"errors"
"os"
"reflect"
"testing"
"github.com/gotenberg/gotenberg/v7/pkg/gotenberg"
"go.uber.org/zap"
)
func TestQPDF_Descriptor(t *testing.T) {
descriptor := QPDF{}.Descriptor()
actual := reflect.TypeOf(descriptor.New())
expect := reflect.TypeOf(new(QPDF))
if actual != expect {
t.Errorf("expected '%s' but got '%s'", expect, actual)
}
}
func TestQPDF_Provision(t *testing.T) {
mod := new(QPDF)
ctx := gotenberg.NewContext(gotenberg.ParsedFlags{}, nil)
err := mod.Provision(ctx)
if err != nil {
t.Errorf("expected no error but got: %v", err)
}
}
func TestQPDF_Validate(t *testing.T) {
for i, tc := range []struct {
binPath string
expectErr bool
}{
{
expectErr: true,
},
{
binPath: "/foo",
expectErr: true,
},
{
binPath: os.Getenv("QPDF_BIN_PATH"),
},
} {
mod := new(QPDF)
mod.binPath = tc.binPath
err := mod.Validate()
if tc.expectErr && err == nil {
t.Errorf("test %d: expected error but got: %v", i, err)
}
if !tc.expectErr && err != nil {
t.Errorf("test %d: expected no error but got: %v", i, err)
}
}
}
func TestQPDF_Metrics(t *testing.T) {
metrics, err := new(QPDF).Metrics()
if err != nil {
t.Errorf("expected no error but got: %v", err)
}
if len(metrics) != 1 {
t.Errorf("expected %d metrics, but got %d", 1, len(metrics))
}
actual := metrics[0].Read()
if actual != 0 {
t.Errorf("expected %d QPDF instances, but got %f", 0, actual)
}
}
func TestQPDF_Merge(t *testing.T) {
for i, tc := range []struct {
ctx context.Context
inputPaths []string
expectErr bool
}{
{
ctx: context.TODO(),
inputPaths: []string{
"/tests/test/testdata/pdfengines/sample1.pdf",
},
},
{
ctx: context.TODO(),
inputPaths: []string{
"/tests/test/testdata/pdfengines/sample1.pdf",
"/tests/test/testdata/pdfengines/sample2.pdf",
},
},
{
ctx: nil,
expectErr: true,
},
{
ctx: context.TODO(),
inputPaths: []string{
"foo",
},
expectErr: true,
},
} {
func() {
mod := new(QPDF)
err := mod.Provision(nil)
if err != nil {
t.Fatalf("test %d: expected error but got: %v", i, err)
}
outputDir, err := gotenberg.MkdirAll()
if err != nil {
t.Fatalf("test %d: expected error but got: %v", i, err)
}
defer func() {
err := os.RemoveAll(outputDir)
if err != nil {
t.Fatalf("test %d: expected no error but got: %v", i, err)
}
}()
err = mod.Merge(tc.ctx, zap.NewNop(), tc.inputPaths, outputDir+"/foo.pdf")
if tc.expectErr && err == nil {
t.Errorf("test %d: expected error but got: %v", i, err)
}
if !tc.expectErr && err != nil {
t.Errorf("test %d: expected no error but got: %v", i, err)
}
}()
}
}
func TestQPDF_Convert(t *testing.T) {
mod := new(QPDF)
err := mod.Convert(context.TODO(), zap.NewNop(), "", "", "")
if !errors.Is(err, gotenberg.ErrPDFEngineMethodNotAvailable) {
t.Errorf("expected error %v, but got: %v", gotenberg.ErrPDFEngineMethodNotAvailable, err)
}
}

View File

@@ -143,7 +143,7 @@ func webhookMiddleware(w Webhook) api.Middleware {
client: &retryablehttp.Client{
HTTPClient: &http.Client{
Timeout: c.Get("writeTimeout").(time.Duration),
Timeout: w.clientTimeout,
},
RetryMax: w.maxRetry,
RetryWaitMin: w.retryMinWait,

View File

@@ -324,6 +324,7 @@ func TestWebhookMiddlewareAsynchronousProcess(t *testing.T) {
maxRetry: 0,
retryMinWait: 0,
retryMaxWait: 0,
clientTimeout: time.Duration(30) * time.Second,
disable: false,
}
}
@@ -397,7 +398,6 @@ func TestWebhookMiddlewareAsynchronousProcess(t *testing.T) {
c.Set("traceHeader", "Gotenberg-Trace")
c.Set("trace", "foo")
c.Set("startTime", time.Now())
c.Set("writeTimeout", time.Duration(10)*time.Second)
ctx := &api.MockContext{Context: &api.Context{}}
ctx.SetLogger(zap.NewNop())

View File

@@ -25,6 +25,7 @@ type Webhook struct {
maxRetry int
retryMinWait time.Duration
retryMaxWait time.Duration
clientTimeout time.Duration
disable bool
}
@@ -55,7 +56,7 @@ func (Webhook) Descriptor() gotenberg.ModuleDescriptor {
err = multierr.Append(err, fs.MarkDeprecated("api-disable-webhook", "use webhook-disable instead"))
if err != nil {
panic(fmt.Errorf("create deprecated flags for webhook module: %v", err))
panic(fmt.Errorf("create deprecated flags for the webhook module: %v", err))
}
// New flags.
@@ -66,6 +67,7 @@ func (Webhook) Descriptor() gotenberg.ModuleDescriptor {
fs.Int("webhook-max-retry", 4, "Set the maximum number of retries for the webhook feature")
fs.Duration("webhook-retry-min-wait", time.Duration(1)*time.Second, "Set the minimum duration to wait before trying to call the webhook again")
fs.Duration("webhook-retry-max-wait", time.Duration(30)*time.Second, "Set the maximum duration to wait before trying to call the webhook again")
fs.Duration("webhook-client-timeout", time.Duration(30)*time.Second, "Set the time limit for requests to the webhook")
fs.Bool("webhook-disable", false, "Disable the webhook feature")
return fs
@@ -84,6 +86,7 @@ func (w *Webhook) Provision(ctx *gotenberg.Context) error {
w.maxRetry = flags.MustDeprecatedInt("api-webhook-max-retry", "webhook-max-retry")
w.retryMinWait = flags.MustDeprecatedDuration("api-webhook-retry-min-wait", "webhook-retry-min-wait")
w.retryMaxWait = flags.MustDeprecatedDuration("api-webhook-retry-min-wait", "webhook-retry-max-wait")
w.clientTimeout = flags.MustDuration("webhook-client-timeout")
w.disable = flags.MustDeprecatedBool("api-disable-webhook", "webhook-disable")
return nil

View File

@@ -13,5 +13,6 @@ import (
_ "github.com/gotenberg/gotenberg/v7/pkg/modules/pdfengines"
_ "github.com/gotenberg/gotenberg/v7/pkg/modules/pdftk"
_ "github.com/gotenberg/gotenberg/v7/pkg/modules/prometheus"
_ "github.com/gotenberg/gotenberg/v7/pkg/modules/qpdf"
_ "github.com/gotenberg/gotenberg/v7/pkg/modules/webhook"
)

View File

@@ -6,8 +6,9 @@ GOLANG_VERSION="$1"
GOTENBERG_VERSION="$2"
GOTENBERG_USER_GID="$3"
GOTENBERG_USER_UID="$4"
PDFTK_VERSION="$5"
DOCKER_REPOSITORY="$6"
NOTO_COLOR_EMOJI_VERSION="$5"
PDFTK_VERSION="$6"
DOCKER_REPOSITORY="$7"
GOTENBERG_VERSION="${GOTENBERG_VERSION//v}"
SEMVER=( ${GOTENBERG_VERSION//./ } )
@@ -23,6 +24,7 @@ docker buildx build \
--build-arg GOTENBERG_VERSION="$GOTENBERG_VERSION" \
--build-arg GOTENBERG_USER_GID="$GOTENBERG_USER_GID" \
--build-arg GOTENBERG_USER_UID="$GOTENBERG_USER_UID" \
--build-arg NOTO_COLOR_EMOJI_VERSION="$NOTO_COLOR_EMOJI_VERSION" \
--build-arg PDFTK_VERSION="$PDFTK_VERSION" \
--platform linux/amd64 \
--platform linux/arm64 \