Compare commits

...

30 Commits

Author SHA1 Message Date
Julien Neuhart
430ed38cc5 build(makefile): re-enable the embedlit go fix modernizer 2026-09-11 16:13:06 +02:00
Julien Neuhart
623e93bf3e chore(deps): update Go to 1.27.1 2026-09-11 16:13:06 +02:00
dependabot[bot]
fe4fb9416d chore(deps): bump golang.org/x/sync from 0.22.0 to 0.23.0 (#1658)
Bumps [golang.org/x/sync](https://github.com/golang/sync) from 0.22.0 to 0.23.0.
- [Commits](https://github.com/golang/sync/compare/v0.22.0...v0.23.0)

---
updated-dependencies:
- dependency-name: golang.org/x/sync
  dependency-version: 0.23.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2026-09-11 07:20:22 +02:00
Julien Neuhart
fb7536a526 fix(chromium): disable the WebUI omnibox popup preloaded at browser start 2026-09-09 19:38:40 +02:00
Julien Neuhart
b16ce08da7 perf(chromium): skip formatting a discarded debug message per response 2026-09-08 18:35:03 +02:00
Julien Neuhart
fcfd590169 perf(api): build the access log with a single record 2026-09-08 18:35:03 +02:00
Julien Neuhart
b39b8c76aa perf(gotenberg): match allow and deny patterns without recompiling them 2026-09-08 18:34:58 +02:00
Julien Neuhart
06ed58b6e7 ci(build): reuse the Docker build cache across runs 2026-09-08 18:34:54 +02:00
Julien Neuhart
34b7b4845e perf(pdfengines): process a request's files concurrently behind an opt-in ceiling 2026-09-08 18:34:51 +02:00
Julien Neuhart
ab3832d9e5 test(integration): drop the per-scenario Docker network 2026-09-08 18:34:43 +02:00
Julien Neuhart
cddaa0fa57 fix(chromium): ceiling the CONNECT tunnels in flight and bound tunnel writes 2026-09-07 18:13:29 +02:00
Julien Neuhart
ade6a327a4 fix(chromium): drop settled requests from the per-conversion network map 2026-09-07 18:13:29 +02:00
Julien Neuhart
35ddf81812 fix(libreoffice): stop the outbound proxy when the daemon fails to start 2026-09-07 18:13:29 +02:00
Julien Neuhart
1891a9ea68 fix(chromium): bound CONNECT tunnels so a silent upstream cannot pin them forever 2026-09-07 18:13:29 +02:00
Julien Neuhart
e1e0a80883 fix(webhook): close the response body when the callback returns an error status 2026-09-07 18:13:29 +02:00
Max Freedom Pollard
c21ceacd4b fix(libreoffice): report an encrypted .xlsb as password-protected (#1655)
Uploading a password-protected .xlsb workbook without its password
answered 500 with the unattributable-failure message instead of 400 with
the remedy.

DetectPasswordProtection in pkg/modules/libreoffice/api/protection.go
infers encryption from a compound-file header carried by an extension
whose unencrypted form is always a ZIP package. The ooxmlExtensions map
listed .xlsx, .xlsm, .xltx and .xltm but not .xlsb, so an encrypted
workbook under that extension fell through to PasswordProtectionUnknown.
The convert route in pkg/modules/libreoffice/routes.go then matched
neither password branch of its exit-code switch and returned the 500
default.

An Excel Binary Workbook is an Open Packaging Conventions ZIP holding
binary parts, so a compound file under that extension is encrypted for
the same reason .xlsx is. Adding .xlsb to the map restores the 400 that
names the 'password' form field.
2026-09-06 13:05:20 +02:00
Julien Neuhart
ac825a2c03 feat(api): warn at startup when the debug route has no authentication 2026-09-05 14:04:21 +02:00
Julien Neuhart
2c9fa6b6ed fix(exiftool): reject metadata keys that collide with ExifTool options 2026-09-05 14:04:21 +02:00
Julien Neuhart
ca8b45cd3a fix(api): keep the filter verdict generic when a redirect is blocked 2026-09-05 14:04:21 +02:00
Julien Neuhart
40cf48442f fix(outbound): treat CGNAT and benchmarking ranges as non-public 2026-09-05 11:34:03 +02:00
Julien Neuhart
df3bac99ed fix(api): keep upload order when de-duplicating repeated filenames 2026-09-05 11:26:10 +02:00
Julien Neuhart
17868b8c02 fix(api): stop upload filenames from failing or silently dropping a request 2026-09-05 10:07:47 +02:00
Julien Neuhart
78284df590 fix(api): snapshot the output filename before echo recycles its context 2026-09-05 10:05:12 +02:00
Julien Neuhart
8f415186d5 fix(api): bound downloadFrom decoding by the entry limit 2026-09-05 10:03:56 +02:00
Julien Neuhart
f675f78f77 fix(webhook): bound webhook delivery with its own retry budget 2026-09-05 10:01:29 +02:00
Julien Neuhart
83b01c2baa fix(api): bound downloadFrom requests by the request deadline 2026-09-05 09:59:29 +02:00
Julien Neuhart
9a46fdd681 style: escape backslashes in the Chromium allow-list scenario table 2026-09-04 20:00:22 +02:00
Julien Neuhart
4de9b0f68b docs(outbound): document the allow-list bypass on every allow-list flag and terminate the example patterns 2026-09-04 19:55:57 +02:00
Julien Neuhart
201e80b9d7 feat(outbound): warn at startup about allow-list patterns that grant more than intended 2026-09-04 19:52:41 +02:00
Julien Neuhart
86a013b664 fix(outbound): strip URL userinfo before allow and deny list matching 2026-09-04 19:45:58 +02:00
51 changed files with 4147 additions and 492 deletions

View File

@@ -83,12 +83,16 @@ runs:
INPUT_PLATFORM: ${{ inputs.platform }}
INPUT_ALTERNATE_REPOSITORY: ${{ inputs.alternate_repository }}
INPUT_DRY_RUN: ${{ inputs.dry_run }}
# Exporting the build cache needs a registry login. Forks run without
# credentials, so they import the cache but never export it.
INPUT_CACHE_WRITABLE: ${{ inputs.docker_hub_username != '' }}
run: |
.github/actions/build-test-push/build.sh \
--version "$INPUT_VERSION" \
--platform "$INPUT_PLATFORM" \
--alternate-repository "$INPUT_ALTERNATE_REPOSITORY" \
--dry-run "$INPUT_DRY_RUN"
--dry-run "$INPUT_DRY_RUN" \
--cache-writable "$INPUT_CACHE_WRITABLE"
- name: Run integration tests
if: inputs.skip_integrations_tests != 'true'

View File

@@ -12,6 +12,7 @@ version=""
platform=""
alternate_repository=""
dry_run=""
cache_writable=""
while [[ $# -gt 0 ]]; do
case $1 in
@@ -31,6 +32,10 @@ while [[ $# -gt 0 ]]; do
dry_run="$2"
shift 2
;;
--cache-writable)
cache_writable="$2"
shift 2
;;
*)
echo "Unknown option $1"
exit 1
@@ -44,11 +49,41 @@ echo
echo "Gotenberg version: $version"
echo "Target platform: $platform"
# The build cache lives under the canonical repository, captured before the
# alternate-repository override below. Pull requests build into "snapshot", so
# deriving the cache ref after the override would give them a cache namespace
# of their own and they would never import what main published, which is the
# population that benefits most.
cache_image="$DOCKER_REGISTRY/$DOCKER_REPOSITORY"
# Layers are per-architecture, so each platform keeps its own cache manifest.
cache_platform="${platform//\//-}"
# Layers running "apt-get upgrade" install whatever versions are current at
# build time, and the packages are deliberately not pinned. A persistent cache
# would turn those into hits and freeze security patches into a published
# image until debian:13-slim itself changes digest. Keying them on the ISO week
# bounds that staleness to seven days while leaving every build within a week
# free to reuse the cache.
apt_snapshot="$(date -u +%G-W%V)"
# Only a build that is not redirected to an alternate repository writes the
# cache, so a pull request cannot make its own state the baseline for main.
# Reading stays enabled everywhere, including forks, since the cache ref is
# public and needs no credentials.
cache_to_enabled="false"
if [ "$cache_writable" = "true" ] && [ -z "$alternate_repository" ]; then
cache_to_enabled="true"
fi
if [ -n "$alternate_repository" ]; then
DOCKER_REPOSITORY=$alternate_repository
echo "⚠️ Using $alternate_repository for DOCKER_REPOSITORY"
fi
echo "Build cache: $cache_image:buildcache-<target>-$cache_platform (write: $cache_to_enabled)"
echo "APT snapshot: $apt_snapshot"
if [ "$dry_run" = "true" ]; then
echo "🚧 Dry run"
fi
@@ -189,12 +224,36 @@ join() {
echo "$*"
}
# cache_flags echoes the buildx cache arguments for a build target. Each target
# keeps its own manifest so that the Chromium and LibreOffice variants, which
# branch from common-stage rather than from each other, do not overwrite one
# another's entry.
#
# mode=max exports intermediate stages too, not just the final layers, which is
# what makes the expensive apt and jlink stages reusable. type=registry, not
# type=gha: the GitHub Actions cache is capped at 10 GB per repository and is
# already carrying the Go and golangci-lint caches that the lint and test jobs
# depend on. Multi-GB image layers across five platforms would evict them.
cache_flags() {
local target="$1"
local ref="$cache_image:buildcache-$target-$cache_platform"
local flags="--cache-from type=registry,ref=$ref"
if [ "$cache_to_enabled" = "true" ]; then
flags="$flags --cache-to type=registry,ref=$ref,mode=max"
fi
echo "$flags"
}
no_arch_tag="$DOCKER_REGISTRY/$DOCKER_REPOSITORY:$version"
# Full variant.
cmd="docker buildx build \
--target gotenberg \
--build-arg GOTENBERG_VERSION=$version \
--build-arg APT_SNAPSHOT=$apt_snapshot \
$(cache_flags gotenberg) \
--platform $platform \
--load \
${tags_flags[*]} \
@@ -207,6 +266,8 @@ run_cmd "$cmd"
cmd="docker buildx build \
--target gotenberg-chromium \
--build-arg GOTENBERG_VERSION=$version \
--build-arg APT_SNAPSHOT=$apt_snapshot \
$(cache_flags gotenberg-chromium) \
--platform $platform \
--load \
${tags_chromium_flags[*]} \
@@ -218,6 +279,8 @@ run_cmd "$cmd"
cmd="docker buildx build \
--target gotenberg-libreoffice \
--build-arg GOTENBERG_VERSION=$version \
--build-arg APT_SNAPSHOT=$apt_snapshot \
$(cache_flags gotenberg-libreoffice) \
--platform $platform \
--load \
${tags_libreoffice_flags[*]} \
@@ -230,6 +293,8 @@ if [ "$platform" = "linux/amd64" ]; then
cmd="docker buildx build \
--target gotenberg-cloudrun \
--build-arg GOTENBERG_VERSION=$version \
--build-arg APT_SNAPSHOT=$apt_snapshot \
$(cache_flags gotenberg-cloudrun) \
--platform $platform \
--load \
${tags_cloud_run_flags[*]} \
@@ -240,6 +305,8 @@ if [ "$platform" = "linux/amd64" ]; then
cmd="docker buildx build \
--target gotenberg-cloudrun-chromium \
--build-arg GOTENBERG_VERSION=$version \
--build-arg APT_SNAPSHOT=$apt_snapshot \
$(cache_flags gotenberg-cloudrun-chromium) \
--platform $platform \
--load \
${tags_cloud_run_chromium_flags[*]} \
@@ -250,6 +317,8 @@ if [ "$platform" = "linux/amd64" ]; then
cmd="docker buildx build \
--target gotenberg-cloudrun-libreoffice \
--build-arg GOTENBERG_VERSION=$version \
--build-arg APT_SNAPSHOT=$apt_snapshot \
$(cache_flags gotenberg-cloudrun-libreoffice) \
--platform $platform \
--load \
${tags_cloud_run_libreoffice_flags[*]} \
@@ -263,6 +332,8 @@ if [ "$platform" = "linux/amd64" ] || [ "$platform" = "linux/arm64" ]; then
cmd="docker buildx build \
--target gotenberg-aws-lambda \
--build-arg GOTENBERG_VERSION=$version \
--build-arg APT_SNAPSHOT=$apt_snapshot \
$(cache_flags gotenberg-aws-lambda) \
--platform $platform \
--load \
${tags_aws_lambda_flags[*]} \
@@ -273,6 +344,8 @@ if [ "$platform" = "linux/amd64" ] || [ "$platform" = "linux/arm64" ]; then
cmd="docker buildx build \
--target gotenberg-aws-lambda-chromium \
--build-arg GOTENBERG_VERSION=$version \
--build-arg APT_SNAPSHOT=$apt_snapshot \
$(cache_flags gotenberg-aws-lambda-chromium) \
--platform $platform \
--load \
${tags_aws_lambda_chromium_flags[*]} \
@@ -283,6 +356,8 @@ if [ "$platform" = "linux/amd64" ] || [ "$platform" = "linux/arm64" ]; then
cmd="docker buildx build \
--target gotenberg-aws-lambda-libreoffice \
--build-arg GOTENBERG_VERSION=$version \
--build-arg APT_SNAPSHOT=$apt_snapshot \
$(cache_flags gotenberg-aws-lambda-libreoffice) \
--platform $platform \
--load \
${tags_aws_lambda_libreoffice_flags[*]} \

View File

@@ -33,7 +33,10 @@ API_OIDC_ISSUER=
API_OIDC_AUDIENCE=
API_OIDC_JWKS_URL=
API_DOWNLOAD_FROM_ALLOW_LIST=
API_DOWNLOAD_FROM_DENY_LIST=^https?://(10\.|172\.(1[6-9]|2[0-9]|3[01])\.|192\.168\.|169\.254\.|0\.0\.0\.0|127\.|localhost|\[::1\]|\[fd)
# Empty, like the flag default since 8.32.0. A textual deny-list cannot
# enumerate every way to write a private address, so *_DENY_PRIVATE_IPS is the
# control to reach for. Left false here so local testing can reach the host.
API_DOWNLOAD_FROM_DENY_LIST=
API_DOWNLOAD_FROM_DENY_PRIVATE_IPS=false
API_DOWNLOAD_FROM_DENY_PUBLIC_IPS=false
API_DOWNLOAD_FROM_ENABLE_ENVIRONMENT_PROXY=false
@@ -84,6 +87,7 @@ LOG_STD_FORMAT=auto
LOG_STD_ENABLE_GCP_FIELDS=false
LOG_STD_LEVEL_CASE=lower
PDFENGINES_DISABLE_ROUTES=false
PDFENGINES_MAX_CONCURRENCY=1
PDFENGINES_MERGE_ENGINES=qpdf,pdfcpu,pdftk
PDFENGINES_SPLIT_ENGINES=pdfcpu,qpdf,pdftk
PDFENGINES_FLATTEN_ENGINES=qpdf
@@ -114,7 +118,8 @@ OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317
OTEL_EXPORTER_OTLP_INSECURE=true
WEBHOOK_ENABLE_SYNC_MODE=false
WEBHOOK_ALLOW_LIST=
WEBHOOK_DENY_LIST=^https?://(10\.|172\.(1[6-9]|2[0-9]|3[01])\.|192\.168\.|169\.254\.|0\.0\.0\.0|127\.|localhost|\[::1\]|\[fd)
# See the note on API_DOWNLOAD_FROM_DENY_LIST.
WEBHOOK_DENY_LIST=
WEBHOOK_DENY_PRIVATE_IPS=false
WEBHOOK_DENY_PUBLIC_IPS=false
WEBHOOK_ENABLE_ENVIRONMENT_PROXY=false
@@ -214,20 +219,17 @@ lint-prettier: ## Lint non-Golang codebase
lint-todo: ## Find TODOs in Golang codebase
golangci-lint run --no-config --default=none --enable godox
# TODO: restore a plain "go fix ./..." once both modernizers stop rewriting this
# codebase into code that does not compile. Re-check by dropping a flag and
# running "make fmt && make lint". Removing an analyzer upstream makes go fix
# fail with "flag provided but not defined", so this cannot rot silently.
# embedlit folds a post-literal field assignment into a literal that already
# sets that key, producing a duplicate field name (cmd/gotenberg.go). Fixed in
# Go 1.27.1, so drop that flag once the toolchain moves past 1.27.0.
# See https://github.com/golang/go/issues/81101.
# TODO: restore a plain "go fix ./..." once the errorsastype modernizer stops
# rewriting this codebase into code that does not compile. Re-check by dropping
# the flag and running "make fmt && make lint". Removing the analyzer upstream
# makes go fix fail with "flag provided but not defined", so this cannot rot
# silently.
# errorsastype rewrites errors.As to errors.AsType[T] without checking that T
# satisfies error, which breaks on api.HttpError since it does not embed
# error. Still broken as of Go 1.27.1.
.PHONY: fmt
fmt: ## Format Golang codebase and "optimize" the dependencies
go fix -embedlit=false -errorsastype=false ./...
go fix -errorsastype=false ./...
golangci-lint fmt
go mod tidy

View File

@@ -1,7 +1,7 @@
# ARG instructions do not create additional layers. Instead, next layers will
# concatenate them. Also, we have to repeat ARG instructions in each build
# stage that uses them.
ARG GOLANG_VERSION=1.27.0
ARG GOLANG_VERSION=1.27.1
# ----------------------------------------------
# pdfcpu binary build stage
@@ -59,7 +59,14 @@ RUN go build -o gotenberg -ldflags "-s -w -X 'github.com/gotenberg/gotenberg/v8/
# ----------------------------------------------
FROM debian:13-slim AS custom-jre-stage
RUN apt-get update -qq \
# APT_SNAPSHOT busts every layer below it when CI rotates the value, weekly.
# Without it a persistent build cache turns the unpinned "apt-get upgrade" into
# a cache hit and the published image keeps shipping the package versions that
# were current when the cache was first populated.
ARG APT_SNAPSHOT=""
RUN echo "apt snapshot: $APT_SNAPSHOT" \
&& apt-get update -qq \
&& apt-get upgrade -yqq \
&& DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends default-jdk-headless binutils
@@ -114,9 +121,15 @@ FROM base-image-stage AS common-stage
ARG GOTENBERG_USER_GID=1001
ARG GOTENBERG_USER_UID=1001
# See the note on APT_SNAPSHOT in custom-jre-stage. Declaring it here covers
# every "apt-get upgrade" in the gotenberg, gotenberg-chromium and
# gotenberg-libreoffice targets too, since all three branch from this stage.
ARG APT_SNAPSHOT=""
# Create a non-root user.
# All processes in the Docker container will run with this dedicated user.
RUN groupadd --gid "$GOTENBERG_USER_GID" gotenberg \
RUN echo "apt snapshot: $APT_SNAPSHOT" \
&& groupadd --gid "$GOTENBERG_USER_GID" gotenberg \
&& useradd --uid "$GOTENBERG_USER_UID" --gid gotenberg --shell /bin/bash --home /home/gotenberg --no-create-home gotenberg \
&& mkdir /home/gotenberg \
&& chown gotenberg: /home/gotenberg

View File

@@ -80,7 +80,7 @@ func Run() {
// Override their values if the corresponding environment variables are
// set.
fs.VisitAll(func(f *flag.Flag) {
envName := strings.ToUpper(strings.ReplaceAll(f.Name, "-", "_"))
envName := gotenberg.EnvVarName(f.Name)
val, ok := os.LookupEnv(envName)
if !ok {
return

View File

@@ -96,6 +96,7 @@ services:
- "--pdfengines-embed-engines=${PDFENGINES_EMBED_ENGINES}"
- "--pdfengines-embed-metadata-engines=${PDFENGINES_EMBED_METADATA_ENGINES}"
- "--pdfengines-factur-x-engines=${PDFENGINES_FACTUR_X_ENGINES}"
- "--pdfengines-max-concurrency=${PDFENGINES_MAX_CONCURRENCY}"
- "--pdfengines-disable-routes=${PDFENGINES_DISABLE_ROUTES}"
- "--prometheus-namespace=${PROMETHEUS_NAMESPACE}"
- "--prometheus-collect-interval=${PROMETHEUS_COLLECT_INTERVAL}"

4
go.mod
View File

@@ -1,6 +1,6 @@
module github.com/gotenberg/gotenberg/v8
go 1.27.0
go 1.27.1
require (
github.com/alexliesenfeld/health v0.8.1
@@ -34,7 +34,7 @@ require (
go.opentelemetry.io/otel/sdk/metric v1.46.0
go.opentelemetry.io/otel/trace v1.46.0
golang.org/x/net v0.58.0
golang.org/x/sync v0.22.0
golang.org/x/sync v0.23.0
golang.org/x/term v0.45.0
golang.org/x/text v0.41.0
)

4
go.sum
View File

@@ -305,8 +305,8 @@ golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To=
golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU=
golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk=
golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

635
pkg/gotenberg/allowlist.go Normal file
View File

@@ -0,0 +1,635 @@
package gotenberg
import (
"strings"
)
// AllowListRisk classifies why an allow-list pattern is dangerous. A URL that
// matches an allow-list skips the private and public IP checks, so a pattern
// that matches more than its author intended silently widens outbound access.
// See [AuditAllowList].
type AllowListRisk string
const (
// AllowListRiskUnanchored marks a pattern with no leading "^". regexp2
// searches rather than matches, so the pattern hits anywhere in the URL,
// including the query string.
AllowListRiskUnanchored AllowListRisk = "unanchored"
// AllowListRiskUnanchoredBranch marks an alternation whose later branches
// have no leading "^". Anchoring only the first branch is a common slip.
AllowListRiskUnanchoredBranch AllowListRisk = "unanchored-branch"
// AllowListRiskCatchAll marks a pattern with no literal prefix, such as
// ".+", which matches every URL and disables filtering entirely.
AllowListRiskCatchAll AllowListRisk = "catch-all"
// AllowListRiskOpenHost marks a pattern whose host is not terminated, so
// it also matches attacker-chosen suffix hosts. For example
// "^https://trusted\.example\.com" matches
// "https://trusted.example.com.attacker.example/".
AllowListRiskOpenHost AllowListRisk = "open-host"
)
// AllowListFinding reports one risky entry of an allow-list.
type AllowListFinding struct {
// Index is the zero-based position of the pattern within the flag value.
Index int
// Pattern is the operator's pattern, verbatim.
Pattern string
// Risk is why the pattern is dangerous.
Risk AllowListRisk
}
// maxAuditedPatternLength bounds the patterns [AuditAllowList] inspects. A
// pathological pattern is not worth walking, and reporting nothing is better
// than reporting a partial verdict.
const maxAuditedPatternLength = 4096
// AuditAllowList reports the entries of an allow-list that match more URLs
// than their author is likely to intend. It is a lint over the pattern source,
// not a parser: it recognizes the shapes that are dangerous in practice and
// stays silent when it cannot be sure.
//
// Callers use the findings to warn operators. Never use them to reject a
// configuration: existing deployments rely on loose patterns, and a pattern
// this function does not flag is not thereby safe.
func AuditAllowList(patterns []string) []AllowListFinding {
var findings []AllowListFinding
for i, pattern := range patterns {
if pattern == "" || len(pattern) > maxAuditedPatternLength {
continue
}
risk, ok := auditPattern(pattern)
if ok {
findings = append(findings, AllowListFinding{Index: i, Pattern: pattern, Risk: risk})
}
}
return findings
}
// auditPattern classifies a single pattern, reporting the first risk found.
func auditPattern(pattern string) (AllowListRisk, bool) {
body := trimInlineFlags(pattern)
branches := splitTopLevelAlternation(body)
for i, branch := range branches {
branch = strings.TrimSpace(branch)
anchored := hasStartAnchor(branch)
rest := strings.TrimPrefix(strings.TrimPrefix(branch, `\A`), "^")
// Catch-all first: a pattern that constrains nothing matches every URL
// whether or not it is anchored, and saying so is more useful than
// telling the operator to anchor it.
if literalPrefix(rest) == "" {
return AllowListRiskCatchAll, true
}
if !anchored {
if i == 0 {
return AllowListRiskUnanchored, true
}
return AllowListRiskUnanchoredBranch, true
}
// A lookaround invalidates the token walk, so skip the host check for
// this branch rather than guess. The anchor and catch-all checks above
// still applied.
if containsLookaround(rest) {
continue
}
if hostIsOpen(rest) {
return AllowListRiskOpenHost, true
}
}
return "", false
}
// trimInlineFlags removes a leading inline flag group such as "(?i)" so that
// the anchor check sees the pattern proper.
func trimInlineFlags(pattern string) string {
if !strings.HasPrefix(pattern, "(?") {
return pattern
}
end := strings.Index(pattern, ")")
if end == -1 {
return pattern
}
// Only a flag group qualifies. "(?:", "(?=", "(?!" and "(?<" open a real
// group and must stay.
flags := pattern[2:end]
if flags == "" || strings.ContainsAny(flags, ":=!<") {
return pattern
}
for _, r := range flags {
if !strings.ContainsRune("imsUx-", r) {
return pattern
}
}
return pattern[end+1:]
}
// hasStartAnchor reports whether branch begins with a start-of-input anchor.
func hasStartAnchor(branch string) bool {
return strings.HasPrefix(branch, "^") || strings.HasPrefix(branch, `\A`)
}
// containsLookaround reports whether the pattern uses a lookaround, which the
// token walk in [hostIsOpen] cannot reason about.
func containsLookaround(s string) bool {
return strings.Contains(s, "(?=") || strings.Contains(s, "(?!") || strings.Contains(s, "(?<")
}
// splitTopLevelAlternation splits on "|" at paren depth zero, honoring escapes
// and character classes.
func splitTopLevelAlternation(s string) []string {
var (
parts []string
current strings.Builder
depth int
inClass bool
)
for i := 0; i < len(s); i++ {
c := s[i]
switch {
case c == '\\' && i+1 < len(s):
current.WriteByte(c)
current.WriteByte(s[i+1])
i++
continue
case c == '[' && !inClass:
inClass = true
case c == ']' && inClass:
inClass = false
case c == '(' && !inClass:
depth++
case c == ')' && !inClass:
depth--
case c == '|' && !inClass && depth == 0:
parts = append(parts, current.String())
current.Reset()
continue
}
current.WriteByte(c)
}
parts = append(parts, current.String())
return parts
}
// literalPrefix returns the characters a matching URL must start with. It
// stops at the first optional or non-literal token, and descends one level
// into a leading mandatory group so that "^(https|http)://" is not mistaken
// for a catch-all. An empty result means the pattern constrains nothing.
func literalPrefix(s string) string {
var prefix strings.Builder
for i := 0; i < len(s); {
// A group: descend once when it is mandatory, otherwise stop.
if s[i] == '(' {
end := matchingParen(s, i)
if end == -1 {
break
}
if isQuantified(s, end+1) {
break
}
inner := trimInlineFlags(s[i+1 : end])
branches := splitTopLevelAlternation(inner)
common := literalPrefix(branches[0])
for _, b := range branches[1:] {
common = commonPrefix(common, literalPrefix(b))
}
prefix.WriteString(common)
// Only the leading group is worth descending into.
break
}
var token string
switch {
case s[i] == '\\' && i+1 < len(s):
token = s[i : i+2]
case s[i] == '[':
end := matchingBracket(s, i)
if end == -1 {
return prefix.String()
}
token = s[i : end+1]
default:
token = s[i : i+1]
}
next := i + len(token)
if isQuantified(s, next) {
break
}
// Only a plain literal or an escaped literal contributes.
switch {
case len(token) == 2 && token[0] == '\\' && !isEscapeClass(token[1]):
prefix.WriteByte(token[1])
case len(token) == 1 && !strings.ContainsAny(token, `.[]()^$*+?{}|`):
prefix.WriteByte(token[0])
default:
return prefix.String()
}
i = next
}
return prefix.String()
}
// hostIsOpen reports whether the authority part of the pattern can be left
// without crossing a terminator, which means the pattern also matches
// attacker-chosen suffix hosts or userinfo.
//
// It walks the tokens after "://" and classifies each one. A terminator ends
// the authority, so the pattern is safe. A crosser can match "@", "?" or "#"
// and therefore lets a matching URL escape the authority, so the pattern is
// open. Reaching the end without a terminator is open too, which is the
// classic "^https://trusted\.example\.com" case.
func hostIsOpen(s string) bool {
_, after, ok := strings.Cut(s, "://")
if !ok {
// No authority to reason about, for example "^file:///tmp/".
return false
}
rest := after
for i := 0; i < len(rest); {
var token string
switch {
case rest[i] == '\\' && i+1 < len(rest):
token = rest[i : i+2]
case rest[i] == '[':
end := matchingBracket(rest, i)
if end == -1 {
return true
}
token = rest[i : end+1]
case rest[i] == '(':
end := matchingParen(rest, i)
if end == -1 {
return true
}
token = rest[i : end+1]
default:
token = rest[i : i+1]
}
next := i + len(token)
optional := isOptionalQuantifier(rest, next)
switch classifyHostToken(token) {
case hostTokenTerminator:
// An optional terminator does not end anything, since the URL may
// match without it.
if !optional {
return false
}
case hostTokenCrosser:
return true
case hostTokenNeutral:
// Part of the host itself, so keep walking.
}
i = next
for i < len(rest) && isQuantifierByte(rest[i]) {
if rest[i] == '{' {
end := strings.IndexByte(rest[i:], '}')
if end == -1 {
return true
}
i += end + 1
continue
}
i++
}
}
return true
}
// hostTokenKind is how a token affects the walk in [hostIsOpen].
type hostTokenKind int
const (
hostTokenNeutral hostTokenKind = iota
hostTokenTerminator
hostTokenCrosser
)
// hostTerminators are the characters that end the authority of a URL.
const hostTerminators = "/:#?"
// crosserClassChars are the characters that, if a class can match them, let a
// match escape the authority. "/" is deliberately absent: it ends the
// authority rather than escaping it, so a class such as "[:/]" is safe.
const crosserClassChars = "@?#"
// classifyHostToken classifies one token of the authority walk.
func classifyHostToken(token string) hostTokenKind {
switch {
case token == ".":
// The wildcard matches "@", "#" and "?", so a host built on it can be
// left without ever reaching a terminator.
return hostTokenCrosser
case token == "$":
return hostTokenTerminator
case len(token) == 1 && strings.Contains(hostTerminators, token):
return hostTokenTerminator
case len(token) == 2 && token[0] == '\\':
switch token[1] {
case 'S', 'D', 'W':
return hostTokenCrosser
case 'd', 'w', 's':
return hostTokenNeutral
case 'p', 'P':
return hostTokenCrosser
}
if strings.Contains(hostTerminators, token[1:]) {
return hostTokenTerminator
}
return hostTokenNeutral
case strings.HasPrefix(token, "["):
inner := strings.TrimSuffix(strings.TrimPrefix(token, "["), "]")
if strings.HasPrefix(inner, "^") {
// A negated class almost always admits "@".
return hostTokenCrosser
}
if classContainsAny(inner, crosserClassChars) {
return hostTokenCrosser
}
if classOnlyTerminators(inner) {
return hostTokenTerminator
}
return hostTokenNeutral
case strings.HasPrefix(token, "("):
return classifyGroup(token)
}
return hostTokenNeutral
}
// classifyGroup classifies a parenthesized group. A group whose every branch
// starts with a terminator ends the authority, which is what makes the
// idiomatic "(:|/|$)" safe. A group containing a crosser is a crosser.
func classifyGroup(token string) hostTokenKind {
inner := trimInlineFlags(strings.TrimSuffix(strings.TrimPrefix(token, "("), ")"))
inner = strings.TrimPrefix(inner, "?:")
branches := splitTopLevelAlternation(inner)
allTerminate := true
for _, branch := range branches {
if branch == "" {
allTerminate = false
continue
}
kind := classifyHostToken(firstToken(branch))
if kind == hostTokenCrosser {
return hostTokenCrosser
}
if kind != hostTokenTerminator {
allTerminate = false
}
// A crosser anywhere inside the branch still escapes the authority.
if branchHasCrosser(branch) {
return hostTokenCrosser
}
}
if allTerminate {
return hostTokenTerminator
}
return hostTokenNeutral
}
// branchHasCrosser reports whether any token of branch is a crosser.
func branchHasCrosser(branch string) bool {
for i := 0; i < len(branch); {
token := tokenAt(branch, i)
if token == "" {
return true
}
if classifyHostToken(token) == hostTokenCrosser {
return true
}
i += len(token)
}
return false
}
// firstToken returns the first regex token of s.
func firstToken(s string) string {
return tokenAt(s, 0)
}
// tokenAt returns the regex token starting at index i, or "" if it is
// malformed.
func tokenAt(s string, i int) string {
if i >= len(s) {
return ""
}
switch {
case s[i] == '\\' && i+1 < len(s):
return s[i : i+2]
case s[i] == '[':
end := matchingBracket(s, i)
if end == -1 {
return ""
}
return s[i : end+1]
case s[i] == '(':
end := matchingParen(s, i)
if end == -1 {
return ""
}
return s[i : end+1]
}
return s[i : i+1]
}
// classOnlyTerminators reports whether every character a class can match ends
// the authority, which makes the class itself a terminator. A range is never
// treated as one.
func classOnlyTerminators(class string) bool {
if class == "" {
return false
}
for i := 0; i < len(class); i++ {
if class[i] == '\\' && i+1 < len(class) {
if !strings.Contains(hostTerminators, class[i+1:i+2]) {
return false
}
i++
continue
}
if i+2 < len(class) && class[i+1] == '-' {
return false
}
if !strings.Contains(hostTerminators, class[i:i+1]) {
return false
}
}
return true
}
// classContainsAny reports whether a character class body can match any of the
// given characters, expanding simple ranges.
func classContainsAny(class, chars string) bool {
for i := 0; i < len(class); i++ {
if class[i] == '\\' && i+1 < len(class) {
// An escape class such as \S inside a class admits everything.
if strings.ContainsRune("SDW", rune(class[i+1])) {
return true
}
if strings.ContainsRune(chars, rune(class[i+1])) {
return true
}
i++
continue
}
if i+2 < len(class) && class[i+1] == '-' {
lo, hi := class[i], class[i+2]
for _, c := range []byte(chars) {
if c >= lo && c <= hi {
return true
}
}
i += 2
continue
}
if strings.ContainsRune(chars, rune(class[i])) {
return true
}
}
return false
}
// matchingParen returns the index of the ")" closing the "(" at start.
func matchingParen(s string, start int) int {
depth := 0
inClass := false
for i := start; i < len(s); i++ {
switch {
case s[i] == '\\' && i+1 < len(s):
i++
case s[i] == '[' && !inClass:
inClass = true
case s[i] == ']' && inClass:
inClass = false
case s[i] == '(' && !inClass:
depth++
case s[i] == ')' && !inClass:
depth--
if depth == 0 {
return i
}
}
}
return -1
}
// matchingBracket returns the index of the "]" closing the "[" at start.
func matchingBracket(s string, start int) int {
for i := start + 1; i < len(s); i++ {
switch {
case s[i] == '\\' && i+1 < len(s):
i++
case s[i] == ']':
// A "]" immediately after "[" or "[^" is a literal.
if i == start+1 || (i == start+2 && s[start+1] == '^') {
continue
}
return i
}
}
return -1
}
// isQuantifierByte reports whether c opens a quantifier.
func isQuantifierByte(c byte) bool {
return c == '?' || c == '*' || c == '+' || c == '{'
}
// isQuantified reports whether a quantifier starts at index i.
func isQuantified(s string, i int) bool {
return i < len(s) && isQuantifierByte(s[i])
}
// isOptionalQuantifier reports whether the quantifier at index i lets the
// preceding token match nothing.
func isOptionalQuantifier(s string, i int) bool {
if i >= len(s) {
return false
}
switch s[i] {
case '?', '*':
return true
case '{':
return strings.HasPrefix(s[i:], "{0")
}
return false
}
// isEscapeClass reports whether c after a backslash denotes a character class
// rather than a literal.
func isEscapeClass(c byte) bool {
return strings.ContainsRune("dDwWsSbBAzZpP", rune(c))
}
// commonPrefix returns the longest common prefix of a and b.
func commonPrefix(a, b string) string {
n := min(len(a), len(b))
for i := range n {
if a[i] != b[i] {
return a[:i]
}
}
return a[:n]
}

View File

@@ -0,0 +1,179 @@
package gotenberg
import (
"testing"
"github.com/dlclark/regexp2"
)
func TestAuditAllowList(t *testing.T) {
for _, tc := range []struct {
scenario string
pattern string
want AllowListRisk
}{
// Safe: the host is terminated before anything can leave it.
{"idiomatic terminator group", `^https?://internal\.svc(:|/|$)`, ""},
{"trailing slash", `^https://trusted\.example\.com/`, ""},
{"optional port then terminator", `^https://example\.com(:[0-9]+)?(/|$)`, ""},
{"positive class cannot leave authority", `^https://[a-z0-9.-]+\.s3\.amazonaws\.com/`, ""},
{"leading mandatory group", `^(https|http)://a\.example\.com/`, ""},
{"optional subdomain group", `^https://(www\.)?example\.com/`, ""},
{"port terminator", `^https://example\.com:8443/`, ""},
{"end anchor", `^https://example\.com$`, ""},
{"alternation both anchored and terminated", `^https://a\.example/|^https://b\.example/`, ""},
{"no authority to check", `^file:///tmp/`, ""},
{"digit class in host", `^https://node\d+\.example\.com/`, ""},
{"class of only terminators", `^https://example\.com[:/]`, ""},
{"feature file pattern, fixed", `^https?://host\.docker\.internal(:[0-9]+)?/`, ""},
// Unanchored: regexp2 searches, so these match anywhere in the URL.
{"no anchor", `trusted\.example\.com`, AllowListRiskUnanchored},
{"no anchor with scheme", `https://trusted\.example\.com/`, AllowListRiskUnanchored},
// Only the first branch anchored.
{"second branch unanchored", `^http://a\.example/|http://b\.example/`, AllowListRiskUnanchoredBranch},
// Catch-all: matches every URL.
{"dot plus", `.+`, AllowListRiskCatchAll},
{"dot star", `.*`, AllowListRiskCatchAll},
{"anchored dot star", `^.*`, AllowListRiskCatchAll},
{"anchored dot plus", `^.+`, AllowListRiskCatchAll},
// Open host: the reported vulnerability class.
{"advisory pattern", `^http://trusted\.example\.com`, AllowListRiskOpenHost},
{"gotenberg.dev internet-facing recipe", `^https?://[^/]+\.internal\.example\.com`, AllowListRiskOpenHost},
{"gotenberg.dev strict whitelist recipe", `^https://(api|cdn|images)\.internal\.example\.com`, AllowListRiskOpenHost},
{"gotenberg.dev hooks recipe", `^https?://hooks\.internal\.example\.com`, AllowListRiskOpenHost},
{"feature file pattern", `^https?://host.docker.internal.*`, AllowListRiskOpenHost},
{"scheme only", `^https?://`, AllowListRiskOpenHost},
{"wildcard subdomain", `^https://.+\.example\.com/`, AllowListRiskOpenHost},
{"escaped dot is not a terminator", `^https?://example\.com\.`, AllowListRiskOpenHost},
{"negated class in host", `^https://[^.]+\.example\.com/`, AllowListRiskOpenHost},
} {
t.Run(tc.scenario, func(t *testing.T) {
findings := AuditAllowList([]string{tc.pattern})
if tc.want == "" {
if len(findings) != 0 {
t.Fatalf("AuditAllowList(%q) = %+v, want no finding", tc.pattern, findings)
}
return
}
if len(findings) != 1 {
t.Fatalf("AuditAllowList(%q) returned %d findings, want 1", tc.pattern, len(findings))
}
if findings[0].Risk != tc.want {
t.Fatalf("AuditAllowList(%q) risk = %q, want %q", tc.pattern, findings[0].Risk, tc.want)
}
})
}
}
// TestAuditAllowList_FlaggedPatternsAreActuallyExploitable proves the audit is
// not merely syntactic: every pattern it flags as open-host really does admit
// a host the operator did not intend.
func TestAuditAllowList_FlaggedPatternsAreActuallyExploitable(t *testing.T) {
for _, tc := range []struct {
pattern string
attack string
}{
{`^http://trusted\.example\.com`, "http://trusted.example.com.attacker.example/"},
{`^https?://[^/]+\.internal\.example\.com`, "http://a.internal.example.com.attacker.example/"},
{`^https://(api|cdn|images)\.internal\.example\.com`, "https://api.internal.example.com.attacker.example/"},
{`^https?://hooks\.internal\.example\.com`, "http://hooks.internal.example.com.attacker.example/"},
{`^https?://host.docker.internal.*`, "http://host.docker.internal.attacker.example/"},
{`^https://.+\.example\.com/`, "https://attacker.example/#x.example.com/"},
{`^https?://example\.com\.`, "http://example.com.attacker.example/"},
} {
t.Run(tc.pattern, func(t *testing.T) {
findings := AuditAllowList([]string{tc.pattern})
if len(findings) == 0 {
t.Fatalf("pattern %q was not flagged", tc.pattern)
}
ok, err := regexp2.MustCompile(tc.pattern, 0).MatchString(tc.attack)
if err != nil {
t.Fatalf("match %q: %v", tc.attack, err)
}
if !ok {
t.Fatalf("pattern %q does not match %q, so the finding is a false positive", tc.pattern, tc.attack)
}
})
}
}
// TestAuditAllowList_SafePatternsRejectTheAttacks is the converse: the shapes
// the audit stays silent about really do reject the same attacks.
func TestAuditAllowList_SafePatternsRejectTheAttacks(t *testing.T) {
safe := []string{
`^https?://internal\.svc(:|/|$)`,
`^https://trusted\.example\.com/`,
`^https://example\.com(:[0-9]+)?(/|$)`,
`^https://[a-z0-9.-]+\.s3\.amazonaws\.com/`,
}
attacks := []string{
"https://internal.svc.attacker.example/",
"https://trusted.example.com.attacker.example/",
"https://trusted.example.com@169.254.169.254/",
"https://example.com.attacker.example/",
"https://example.com@10.0.0.5/",
"https://bucket.s3.amazonaws.com.attacker.example/",
"https://bucket.s3.amazonaws.com@127.0.0.1/",
}
for _, pattern := range safe {
t.Run(pattern, func(t *testing.T) {
if findings := AuditAllowList([]string{pattern}); len(findings) != 0 {
t.Fatalf("safe pattern %q was flagged as %q", pattern, findings[0].Risk)
}
re := regexp2.MustCompile(pattern, 0)
for _, attack := range attacks {
ok, err := re.MatchString(attack)
if err != nil {
t.Fatalf("match %q: %v", attack, err)
}
if ok {
t.Fatalf("pattern %q matches attack %q but was not flagged", pattern, attack)
}
}
})
}
}
func TestAuditAllowList_SkipsEmptyAndOversized(t *testing.T) {
oversized := make([]byte, maxAuditedPatternLength+1)
for i := range oversized {
oversized[i] = 'a'
}
findings := AuditAllowList([]string{"", string(oversized)})
if len(findings) != 0 {
t.Fatalf("AuditAllowList returned %+v, want no finding", findings)
}
}
func TestAuditAllowList_ReportsIndex(t *testing.T) {
findings := AuditAllowList([]string{
`^https://ok\.example\.com/`,
`^https://open\.example\.com`,
})
if len(findings) != 1 {
t.Fatalf("got %d findings, want 1", len(findings))
}
if findings[0].Index != 1 {
t.Fatalf("findings[0].Index = %d, want 1", findings[0].Index)
}
}
// TestAuditAllowList_ShippedChromiumDenyListIsNotAudited guards the rule that
// deny-lists are never audited. The shipped Chromium deny-list uses a
// lookaround and has no authority, so auditing it would produce noise.
func TestAuditAllowList_LookaroundIsNotFlaggedForHost(t *testing.T) {
findings := AuditAllowList([]string{`^file:(?!//\/tmp/).*`})
if len(findings) != 0 {
t.Fatalf("AuditAllowList returned %+v, want no finding", findings)
}
}

View File

@@ -1,70 +0,0 @@
package gotenberg
import (
"context"
"errors"
"fmt"
"time"
"github.com/dlclark/regexp2"
)
// ErrFiltered happens if a value is filtered by the [FilterDeadline] function.
var ErrFiltered = errors.New("value filtered")
// FilterDeadline checks if the given value is allowed and not denied according
// to regex patterns. The allowed list uses OR semantics (value must match at
// least one pattern). The denied list uses OR semantics (value is denied if it
// matches any pattern). It returns a [context.DeadlineExceeded] if it takes
// too long to process.
func FilterDeadline(allowed, denied []*regexp2.Regexp, s string, deadline time.Time) error {
if len(allowed) > 0 {
matched := false
for _, pattern := range allowed {
// FIXME: not ideal to compile everytime, but is there another way to create a clone?
clone := regexp2.MustCompile(pattern.String(), 0)
clone.MatchTimeout = time.Until(deadline)
ok, err := clone.MatchString(s)
if err != nil {
if time.Now().After(deadline) {
return context.DeadlineExceeded
}
return fmt.Errorf("'%s' cannot handle '%s': %w", clone.String(), s, err)
}
if ok {
matched = true
break
}
}
if !matched {
return fmt.Errorf("'%s' does not match any expression from the allowed list: %w", s, ErrFiltered)
}
}
if len(denied) > 0 {
for _, pattern := range denied {
clone := regexp2.MustCompile(pattern.String(), 0)
clone.MatchTimeout = time.Until(deadline)
ok, err := clone.MatchString(s)
if err != nil {
if time.Now().After(deadline) {
return context.DeadlineExceeded
}
return fmt.Errorf("'%s' cannot handle '%s': %w", clone.String(), s, err)
}
if ok {
return fmt.Errorf("'%s' matches the expression from the denied list: %w", s, ErrFiltered)
}
}
}
return nil
}

View File

@@ -1,117 +0,0 @@
package gotenberg
import (
"context"
"errors"
"testing"
"time"
"github.com/dlclark/regexp2"
)
func TestFilterDeadline(t *testing.T) {
for _, tc := range []struct {
scenario string
allowed []*regexp2.Regexp
denied []*regexp2.Regexp
s string
deadline time.Time
expectError bool
expectedError error
}{
{
scenario: "DeadlineExceeded (allowed)",
allowed: []*regexp2.Regexp{regexp2.MustCompile("foo", 0)},
denied: nil,
s: "foo",
deadline: time.Now().Add(time.Duration(-1) * time.Hour),
expectError: true,
expectedError: context.DeadlineExceeded,
},
{
scenario: "ErrFiltered (allowed, no match)",
allowed: []*regexp2.Regexp{regexp2.MustCompile("foo", 0)},
denied: nil,
s: "bar",
deadline: time.Now().Add(time.Duration(5) * time.Second),
expectError: true,
expectedError: ErrFiltered,
},
{
scenario: "DeadlineExceeded (denied)",
allowed: nil,
denied: []*regexp2.Regexp{regexp2.MustCompile("foo", 0)},
s: "foo",
deadline: time.Now().Add(time.Duration(-1) * time.Hour),
expectError: true,
expectedError: context.DeadlineExceeded,
},
{
scenario: "ErrFiltered (denied)",
allowed: nil,
denied: []*regexp2.Regexp{regexp2.MustCompile("foo", 0)},
s: "foo",
deadline: time.Now().Add(time.Duration(5) * time.Second),
expectError: true,
expectedError: ErrFiltered,
},
{
scenario: "success (empty lists)",
allowed: nil,
denied: nil,
s: "foo",
deadline: time.Now().Add(time.Duration(5) * time.Second),
expectError: false,
},
{
scenario: "multi-pattern allow list, second matches",
allowed: []*regexp2.Regexp{regexp2.MustCompile("^https://", 0), regexp2.MustCompile("^file:///tmp/", 0)},
denied: nil,
s: "file:///tmp/abc/index.html",
deadline: time.Now().Add(time.Duration(5) * time.Second),
expectError: false,
},
{
scenario: "multi-pattern allow list, none matches",
allowed: []*regexp2.Regexp{regexp2.MustCompile("^https://", 0), regexp2.MustCompile("^ftp://", 0)},
denied: nil,
s: "file:///tmp/abc/index.html",
deadline: time.Now().Add(time.Duration(5) * time.Second),
expectError: true,
expectedError: ErrFiltered,
},
{
scenario: "multi-pattern deny list, second matches",
allowed: nil,
denied: []*regexp2.Regexp{regexp2.MustCompile("^ftp://", 0), regexp2.MustCompile("^file:.*", 0)},
s: "file:///etc/passwd",
deadline: time.Now().Add(time.Duration(5) * time.Second),
expectError: true,
expectedError: ErrFiltered,
},
{
scenario: "https URL passes deny list targeting file://",
allowed: nil,
denied: []*regexp2.Regexp{regexp2.MustCompile("^file:.*", 0)},
s: "https://example.com",
deadline: time.Now().Add(time.Duration(5) * time.Second),
expectError: false,
},
} {
t.Run(tc.scenario, func(t *testing.T) {
err := FilterDeadline(tc.allowed, tc.denied, tc.s, tc.deadline)
if tc.expectError && err == nil {
t.Fatal("expected an error but got none")
}
if !tc.expectError && err != nil {
t.Fatalf("expected no error but got: %v", err)
}
if tc.expectedError != nil && !errors.Is(err, tc.expectedError) {
t.Fatalf("expected error %v but got: %v", tc.expectedError, err)
}
})
}
}

View File

@@ -1,11 +1,17 @@
package gotenberg
import (
"context"
"fmt"
"log/slog"
"strings"
"time"
"github.com/dlclark/regexp2"
"github.com/labstack/gommon/bytes"
flag "github.com/spf13/pflag"
"github.com/gotenberg/gotenberg/v8/pkg/gotenberg/internal/log"
)
// ParsedFlags wraps a [flag.FlagSet] so that retrieving the typed values is
@@ -201,15 +207,38 @@ func (f *ParsedFlags) MustDeprecatedHumanReadableBytes(deprecated string, newNam
return f.MustHumanReadableBytes(newName)
}
// PatternMatchTimeout bounds a single match against an operator-supplied
// allow-list or deny-list pattern.
//
// regexp2 backtracks, and the strings matched against these patterns are
// client-controlled: a request URL, a CONNECT host. A pattern that backtracks
// catastrophically would otherwise burn a core for as long as the caller's
// deadline allows, which is --api-timeout (env API_TIMEOUT), 30 seconds by
// default. The ceiling mirrors the one the Chromium module already applies to
// the per-request extraHttpHeaders scope pattern.
//
// [ParsedFlags.MustRegexp] and [ParsedFlags.MustRegexpSlice] stamp this onto
// every pattern they compile, which is how all four production lists are
// built. Patterns compiled any other way keep regexp2's default of
// math.MaxInt64, which it treats as no timeout at all, so a hand-built slice
// must set this itself before reaching [DecideOutbound].
const PatternMatchTimeout = 250 * time.Millisecond
// MustRegexp returns the regular expression of a flag given by name.
// It panics if an error occurs.
//
// The returned expression carries [PatternMatchTimeout] and is safe to match
// on concurrently: callers must not compile a private copy per match.
func (f *ParsedFlags) MustRegexp(name string) *regexp2.Regexp {
val, err := f.GetString(name)
if err != nil {
panic(err)
}
return regexp2.MustCompile(val, 0)
re := regexp2.MustCompile(val, 0)
re.MatchTimeout = PatternMatchTimeout
return re
}
// MustDeprecatedRegexp returns the regular expression of a deprecated flag if
@@ -226,21 +255,140 @@ func (f *ParsedFlags) MustDeprecatedRegexp(deprecated string, newName string) *r
// MustRegexpSlice returns a slice of compiled regular expressions from a
// string-slice flag given by name. Empty strings are skipped.
// It panics if an error occurs.
//
// Every allow-list and deny-list in Gotenberg is read through this method, so
// it is also where allow-list patterns are audited. See [AuditAllowList].
//
// The returned expressions carry [PatternMatchTimeout] and are safe to match
// on concurrently: callers must not compile a private copy per match.
func (f *ParsedFlags) MustRegexpSlice(name string) []*regexp2.Regexp {
vals := f.MustStringSlice(name)
f.warnRiskyAllowList(name, vals)
var regexps []*regexp2.Regexp
for _, val := range vals {
if val == "" {
continue
}
regexps = append(regexps, regexp2.MustCompile(val, 0))
re := regexp2.MustCompile(val, 0)
re.MatchTimeout = PatternMatchTimeout
regexps = append(regexps, re)
}
return regexps
}
// allowListFlagSuffix identifies the flags whose patterns grant an IP-check
// bypass. Deny-lists are never audited: they always apply, cannot be bypassed,
// and a loose deny-list is safe rather than dangerous.
const allowListFlagSuffix = "-allow-list"
// warnRiskyAllowList logs one warning per allow-list entry that matches more
// URLs than its author is likely to intend.
//
// It warns and never fails: operators depend on loose patterns today, and
// rejecting them at startup would break running deployments.
func (f *ParsedFlags) warnRiskyAllowList(name string, vals []string) {
if !strings.HasSuffix(name, allowListFlagSuffix) {
return
}
findings := AuditAllowList(vals)
if len(findings) == 0 {
return
}
// The logger is nil until the entry point initializes it, which happens
// before any module is provisioned. Tests and embedders that call this
// method directly get no logger, and must not panic for it.
logger := log.Logger()
if logger == nil {
return
}
for _, finding := range findings {
// Provision has no context.Context to propagate, so the trace-aware
// logging convention is satisfied with a background context.
logger.WarnContext(
context.Background(),
f.allowListWarning(name, finding),
slog.String("flag", "--"+name),
slog.String("env", EnvVarName(name)),
slog.Int("entry", finding.Index+1),
slog.String("reason", string(finding.Risk)),
)
}
}
// allowListWarning builds the operator-facing message for a finding. It names
// the flag and its environment variable, and, when they exist, the IP-check
// flags the entry silently disables.
func (f *ParsedFlags) allowListWarning(name string, finding AllowListFinding) string {
var b strings.Builder
// Print the pattern raw rather than quoted: %q escapes every backslash, so
// the operator would not recognize the value they set.
fmt.Fprintf(&b, "--%s (%s) entry %d '%s' ", name, EnvVarName(name), finding.Index+1, finding.Pattern)
switch finding.Risk {
case AllowListRiskUnanchored:
b.WriteString("is not anchored with ^, so it matches anywhere in the URL and a URL such as http://attacker.example/?u=trusted.example.com passes. ")
case AllowListRiskUnanchoredBranch:
b.WriteString("has an alternation branch that is not anchored with ^, and that branch matches anywhere in the URL. ")
case AllowListRiskCatchAll:
b.WriteString("matches every URL. ")
case AllowListRiskOpenHost:
b.WriteString("does not terminate the host, so it also matches suffix hosts such as http://trusted.example.com.attacker.example/. ")
}
b.WriteString(f.bypassSentence(name))
switch finding.Risk {
case AllowListRiskUnanchored, AllowListRiskUnanchoredBranch:
b.WriteString("Anchor every branch with ^ and end the host with /, :, or $.")
case AllowListRiskCatchAll:
b.WriteString("Restrict the entry to the hosts you trust, or unset the flag.")
case AllowListRiskOpenHost:
b.WriteString("End the host with /, :, $, or a group such as (:|/|$).")
}
return b.String()
}
// bypassSentence names the IP-check flags an allow-list match skips, when the
// module registers them.
func (f *ParsedFlags) bypassSentence(name string) string {
prefix := strings.TrimSuffix(name, allowListFlagSuffix)
private, public := prefix+"-deny-private-ips", prefix+"-deny-public-ips"
if f.Lookup(private) == nil || f.Lookup(public) == nil {
// A deprecated alias such as webhook-error-allow-list carries an extra
// segment that the IP-check flags do not have.
if i := strings.LastIndex(prefix, "-"); i != -1 {
private, public = prefix[:i]+"-deny-private-ips", prefix[:i]+"-deny-public-ips"
}
}
if f.Lookup(private) == nil || f.Lookup(public) == nil {
return "A URL that matches the allow-list skips the private and public IP checks. "
}
return fmt.Sprintf(
"A URL that matches the allow-list skips --%s (%s) and --%s (%s). ",
private, EnvVarName(private), public, EnvVarName(public),
)
}
// EnvVarName returns the environment variable that overrides the flag given by
// name. The entry point derives the same name when it applies environment
// overrides, so operator-facing messages can name both without drifting.
func EnvVarName(name string) string {
return strings.ToUpper(strings.ReplaceAll(name, "-", "_"))
}
// MustDeprecatedRegexpSlice returns the slice of compiled regular expressions
// of a deprecated flag if it was explicitly set or the slice of the new flag.
// It panics if an error occurs.

View File

@@ -3,6 +3,7 @@ package gotenberg
import (
"reflect"
"regexp"
"strings"
"testing"
"time"
@@ -951,3 +952,138 @@ func TestParsedFlags_MustDeprecatedRegexpSlice(t *testing.T) {
_ = regexp2.None // Keep import alive.
}
func TestParsedFlags_AllowListWarning(t *testing.T) {
fs := flag.NewFlagSet("tests", flag.ContinueOnError)
fs.StringSlice("chromium-allow-list", []string{}, "")
fs.Bool("chromium-deny-private-ips", false, "")
fs.Bool("chromium-deny-public-ips", false, "")
fs.StringSlice("standalone-allow-list", []string{}, "")
parsedFlags := ParsedFlags{FlagSet: fs}
for _, tc := range []struct {
scenario string
name string
finding AllowListFinding
contains []string
}{
{
scenario: "open host names both IP-check flags and their env vars",
name: "chromium-allow-list",
finding: AllowListFinding{Index: 0, Pattern: `^https://trusted\.example\.com`, Risk: AllowListRiskOpenHost},
contains: []string{
"--chromium-allow-list (CHROMIUM_ALLOW_LIST)",
"entry 1",
`^https://trusted\.example\.com`,
"does not terminate the host",
"--chromium-deny-private-ips (CHROMIUM_DENY_PRIVATE_IPS)",
"--chromium-deny-public-ips (CHROMIUM_DENY_PUBLIC_IPS)",
"End the host with",
},
},
{
scenario: "catch-all tells the operator to restrict or unset",
name: "chromium-allow-list",
finding: AllowListFinding{Index: 2, Pattern: ".+", Risk: AllowListRiskCatchAll},
contains: []string{"entry 3", "matches every URL", "Restrict the entry"},
},
{
scenario: "unanchored explains the search semantics",
name: "chromium-allow-list",
finding: AllowListFinding{Index: 0, Pattern: `trusted\.example\.com`, Risk: AllowListRiskUnanchored},
contains: []string{"is not anchored with ^", "Anchor every branch with ^"},
},
{
scenario: "module without IP-check flags falls back to a generic sentence",
name: "standalone-allow-list",
finding: AllowListFinding{Index: 0, Pattern: `^https://a\.example\.com`, Risk: AllowListRiskOpenHost},
contains: []string{"skips the private and public IP checks"},
},
} {
t.Run(tc.scenario, func(t *testing.T) {
msg := parsedFlags.allowListWarning(tc.name, tc.finding)
for _, want := range tc.contains {
if !strings.Contains(msg, want) {
t.Fatalf("message %q does not contain %q", msg, want)
}
}
if strings.Contains(msg, "—") {
t.Fatalf("message must not contain an em dash: %q", msg)
}
})
}
}
func TestParsedFlags_WarnRiskyAllowList_SkipsDenyLists(t *testing.T) {
fs := flag.NewFlagSet("tests", flag.ContinueOnError)
fs.StringSlice("chromium-deny-list", []string{}, "")
parsedFlags := ParsedFlags{FlagSet: fs}
// A deny-list is never audited: it always applies and cannot be bypassed,
// so a loose one is safe. This must also not panic on a nil logger.
parsedFlags.warnRiskyAllowList("chromium-deny-list", []string{".+", `^file:(?!//\/tmp/).*`})
}
func TestParsedFlags_WarnRiskyAllowList_NilLoggerDoesNotPanic(t *testing.T) {
fs := flag.NewFlagSet("tests", flag.ContinueOnError)
fs.StringSlice("chromium-allow-list", []string{}, "")
parsedFlags := ParsedFlags{FlagSet: fs}
// Provision runs after the entry point initializes the logger, but tests
// and embedders reach this path with no logger at all.
parsedFlags.warnRiskyAllowList("chromium-allow-list", []string{".+"})
}
func TestEnvVarName(t *testing.T) {
for _, tc := range []struct {
name string
want string
}{
{"chromium-allow-list", "CHROMIUM_ALLOW_LIST"},
{"api-download-from-deny-private-ips", "API_DOWNLOAD_FROM_DENY_PRIVATE_IPS"},
{"log-level", "LOG_LEVEL"},
} {
t.Run(tc.name, func(t *testing.T) {
if got := EnvVarName(tc.name); got != tc.want {
t.Fatalf("EnvVarName(%q) = %q, want %q", tc.name, got, tc.want)
}
})
}
}
func TestParsedFlags_RegexpMatchTimeout(t *testing.T) {
// [DecideOutbound] matches on these patterns directly instead of compiling
// a private copy per call, so the bound has to come from here. regexp2's
// own default is math.MaxInt64, which it treats as no
// timeout at all, so a pattern built without this stamp runs unbounded
// against a client-controlled string.
fs := flag.NewFlagSet("tests", flag.ContinueOnError)
fs.StringSlice("some-deny-list", []string{`^file:`, `^https?://`}, "")
fs.String("some-pattern", `^file:`, "")
err := fs.Parse(nil)
if err != nil {
t.Fatalf("expected no error but got: %v", err)
}
parsedFlags := ParsedFlags{FlagSet: fs}
regexps := parsedFlags.MustRegexpSlice("some-deny-list")
if len(regexps) != 2 {
t.Fatalf("expected 2 patterns but got %d", len(regexps))
}
for _, re := range regexps {
if re.MatchTimeout != PatternMatchTimeout {
t.Fatalf("pattern '%s' has MatchTimeout %s, expected %s", re.String(), re.MatchTimeout, PatternMatchTimeout)
}
}
if got := parsedFlags.MustRegexp("some-pattern").MatchTimeout; got != PatternMatchTimeout {
t.Fatalf("expected MustRegexp MatchTimeout %s but got %s", PatternMatchTimeout, got)
}
}

View File

@@ -16,6 +16,7 @@ import (
"time"
"github.com/dlclark/regexp2"
"github.com/hashicorp/go-retryablehttp"
"golang.org/x/net/http/httpproxy"
)
@@ -26,6 +27,12 @@ import (
// example [::ffff:127.0.0.1]).
var ErrNonPublicIP = errors.New("non-public IP")
// ErrFiltered happens when a value is rejected by an allow-list or a
// deny-list, or when it cannot be validated and [DecideOutbound] fails closed.
// Callers map it to a generic 403: the specific reason stays in the operator
// logs so a client cannot probe the lists.
var ErrFiltered = errors.New("value filtered")
// ErrPublicIP indicates that an outbound URL targets an IP address that is
// reachable on the public internet. It is returned when a caller opts
// into denying public destinations via [WithDenyPublicIPs]; typical use
@@ -78,6 +85,18 @@ var nonPublicIPv6Prefixes = []netip.Prefix{
netip.MustParsePrefix("100::/64"),
}
// nonPublicIPv4Prefixes lists IPv4 ranges that the [netip.Addr] helpers do
// not classify but that must not be considered public:
//
// - 100.64.0.0/10 Carrier-grade NAT (RFC 6598). Routable inside provider
// and cluster networks, and Alibaba Cloud serves instance metadata from
// 100.100.100.200.
// - 198.18.0.0/15 Benchmarking (RFC 2544). Never routed on the internet.
var nonPublicIPv4Prefixes = []netip.Prefix{
netip.MustParsePrefix("100.64.0.0/10"),
netip.MustParsePrefix("198.18.0.0/15"),
}
// IsPublicIP reports whether addr is reachable on the public internet. It
// returns false for loopback, private (RFC1918), link-local, unspecified,
// multicast, and unique-local addresses. IPv4-mapped IPv6 addresses are
@@ -88,7 +107,8 @@ var nonPublicIPv6Prefixes = []netip.Prefix{
// (6to4, Teredo, NAT64) are rejected wholesale rather than recursed into,
// because a host that routes them implicitly trusts the IPv4 mapping and
// the prefixes themselves are deprecated or translation-only. See
// [nonPublicIPv6Prefixes] for the full list and rationale.
// [nonPublicIPv6Prefixes] and [nonPublicIPv4Prefixes] for the full lists
// and rationale.
func IsPublicIP(addr netip.Addr) bool {
if !addr.IsValid() {
return false
@@ -104,6 +124,13 @@ func IsPublicIP(addr netip.Addr) bool {
addr.IsInterfaceLocalMulticast():
return false
}
if addr.Is4() {
for _, p := range nonPublicIPv4Prefixes {
if p.Contains(addr) {
return false
}
}
}
if addr.Is6() {
for _, p := range nonPublicIPv6Prefixes {
if p.Contains(addr) {
@@ -249,7 +276,9 @@ func httpLikeScheme(scheme string) bool {
//
// The semantics:
//
// 1. The URL is parsed and its scheme and host lowercased.
// 1. The URL is parsed, its scheme and host lowercased, and any userinfo
// dropped from the form the regexes see. The request still carries the
// credentials.
// 2. allowList and denyList apply against the normalized form with OR
// semantics. The deny-list always applies.
// 3. For http, https, ws, and wss, the host is resolved and every
@@ -268,26 +297,44 @@ func DecideOutbound(ctx context.Context, rawURL string, allowList, denyList []*r
opt(&cfg)
}
// Each match is bounded by [PatternMatchTimeout] rather than by the
// remaining budget, so an already-spent deadline no longer surfaces from
// the match itself. Schemes that resolve a host still learn about it from
// resolveHost, but a non-matching file:// or data: URL returns before that
// point, so check it here to keep failing closed on every path.
if !time.Now().Before(deadline) {
return OutboundDecision{}, context.DeadlineExceeded
}
parsed, err := url.Parse(rawURL)
if err != nil {
return OutboundDecision{}, fmt.Errorf("parse URL %q: %w", rawURL, ErrFiltered)
}
parsed.Scheme = strings.ToLower(parsed.Scheme)
parsed.Host = strings.ToLower(parsed.Host)
normalized := parsed.String()
// Match on a credential-free form. [url.URL.String] re-emits userinfo
// between "scheme://" and the host, so keeping it would let any
// "^https?://<host>" pattern be shifted past its own anchor:
// http://a@127.0.0.1/ escapes a deny-list anchored on 127\. and
// http://trusted.example.com@10.0.0.1/ satisfies an allow-list anchored on
// trusted\.example\.com. The host checks below already read
// [url.URL.Hostname], which ignores userinfo, so only the regex layer was
// affected. Dropping the credentials here also keeps them out of the error
// strings below, which reach operator logs and any OTEL log exporter.
matchable := *parsed
matchable.User = nil
normalized := matchable.String()
allowMatched := false
if len(allowList) > 0 {
for _, pattern := range allowList {
clone := regexp2.MustCompile(pattern.String(), 0)
clone.MatchTimeout = time.Until(deadline)
ok, err := clone.MatchString(normalized)
ok, err := pattern.MatchString(normalized)
if err != nil {
if time.Now().After(deadline) {
return OutboundDecision{}, context.DeadlineExceeded
}
return OutboundDecision{}, fmt.Errorf("'%s' cannot handle '%s': %w", clone.String(), normalized, err)
return OutboundDecision{}, fmt.Errorf("'%s' cannot handle '%s': %w", pattern.String(), normalized, err)
}
if ok {
@@ -302,15 +349,12 @@ func DecideOutbound(ctx context.Context, rawURL string, allowList, denyList []*r
}
for _, pattern := range denyList {
clone := regexp2.MustCompile(pattern.String(), 0)
clone.MatchTimeout = time.Until(deadline)
ok, err := clone.MatchString(normalized)
ok, err := pattern.MatchString(normalized)
if err != nil {
if time.Now().After(deadline) {
return OutboundDecision{}, context.DeadlineExceeded
}
return OutboundDecision{}, fmt.Errorf("'%s' cannot handle '%s': %w", clone.String(), normalized, err)
return OutboundDecision{}, fmt.Errorf("'%s' cannot handle '%s': %w", pattern.String(), normalized, err)
}
if ok {
@@ -357,9 +401,8 @@ func DecideOutbound(ctx context.Context, rawURL string, allowList, denyList []*r
}
// FilterOutboundURL validates that rawURL is acceptable for an outbound
// request from Gotenberg. It is the URL-aware replacement for
// [FilterDeadline] and should be preferred for any new code that filters
// a URL before issuing or instructing an outbound request.
// request from Gotenberg. Prefer it for any new code that filters a URL
// before issuing or instructing an outbound request.
//
// The default behavior is permissive: the URL passes as long as it clears
// the regex allow-list and deny-list. Callers that need IP-class checks
@@ -436,6 +479,16 @@ func (rt *outboundRoundTripper) RoundTrip(req *http.Request) (*http.Response, er
// gate this behind their module's opt-in flag. See
// https://github.com/gotenberg/gotenberg/issues/1592.
func NewOutboundHttpClient(timeout time.Duration, allowList, denyList []*regexp2.Regexp, enableEnvironmentProxy bool, opts ...DecideOption) *http.Client {
// A negative timeout means the caller's budget is already spent, which
// happens when it derives one from a deadline that has passed. [http.Client]
// treats any non-positive Timeout as no deadline at all, so passing it
// through would silently produce an unbounded client. Fail closed instead.
// Zero keeps meaning unbounded: callers that own the connection lifetime
// themselves pass it deliberately.
if timeout < 0 {
timeout = time.Nanosecond
}
base := http.DefaultTransport.(*http.Transport).Clone()
var proxyFunc func(*url.URL) (*url.URL, error)
@@ -479,6 +532,25 @@ func NewOutboundHttpClient(timeout time.Duration, allowList, denyList []*regexp2
}
}
// ClampedBackoff is a [retryablehttp.Backoff] that honors max on every path.
//
// [retryablehttp.DefaultBackoff] returns a Retry-After header from the remote
// verbatim for 429 and 503, and returns it before applying its own max clamp.
// A hostile origin therefore decides how long Gotenberg waits, and the wait is
// not interruptible. Retry-After is still respected here, just never beyond
// the ceiling the caller set.
func ClampedBackoff(min, max time.Duration, attemptNum int, resp *http.Response) time.Duration {
wait := retryablehttp.DefaultBackoff(min, max, attemptNum, resp)
if wait > max {
return max
}
if wait < 0 {
return 0
}
return wait
}
// environmentProxyVariables are the variables golang.org/x/net/http/httpproxy
// reads, in the casing precedence it applies.
var environmentProxyVariables = []string{

View File

@@ -3,7 +3,9 @@ package gotenberg
import (
"context"
"errors"
"net/http"
"net/netip"
"strings"
"testing"
"time"
@@ -38,6 +40,24 @@ func TestIsPublicIP(t *testing.T) {
// Link-local.
{"169.254.169.254", false},
{"169.254.170.2", false},
// Carrier-grade NAT (RFC 6598). Alibaba Cloud serves instance
// metadata from 100.100.100.200.
{"100.64.0.0", false},
{"100.100.100.200", false},
{"100.127.255.255", false},
{"::ffff:100.100.100.200", false},
// Benchmarking (RFC 2544).
{"198.18.0.1", false},
{"198.19.255.255", false},
// Adjacent to the ranges above, and public.
{"100.63.255.255", true},
{"100.128.0.0", true},
{"198.17.255.255", true},
{"198.20.0.0", true},
{"fe80::1", false},
// Unique-local.
@@ -520,3 +540,229 @@ func TestDecideOutbound_Permissive_AllowsPrivate(t *testing.T) {
t.Fatalf("decision.Pinned = %v, want [10.0.0.5]", decision.Pinned)
}
}
// privateIPsDenyList is the textual private-IP deny-list that shipped as the
// default for api-download-from-deny-list and webhook-deny-list in v8.31.0 and
// is still published as a migration recipe. Every alternative is anchored on
// "://", so userinfo used to slide the private address past the anchor.
const privateIPsDenyList = `^https?://(10\.|172\.(1[6-9]|2[0-9]|3[01])\.|192\.168\.|169\.254\.|0\.0\.0\.0|127\.|localhost|\[::1\]|\[fd)`
func TestDecideOutbound_UserinfoDoesNotEvadeDenyList(t *testing.T) {
for _, rawURL := range []string{
"http://127.0.0.1:9999/",
"http://a@127.0.0.1:9999/",
"http://@127.0.0.1:9999/",
"http://:@127.0.0.1:9999/",
"http://%61@127.0.0.1:9999/",
"http://user:pass@127.0.0.1:9999/",
"HTTP://A@127.0.0.1:9999/",
"http://a@169.254.169.254/latest/meta-data/",
// url.Parse takes the last "@" as the userinfo separator, so the host
// here is the second literal.
"http://a@127.0.0.1:9999@127.0.0.1:9999/",
} {
t.Run(rawURL, func(t *testing.T) {
withStubResolver(t, func(host string) ([]netip.Addr, error) {
t.Fatalf("unexpected DNS lookup for %q: the deny-list must reject before resolution", host)
return nil, nil
})
// Deny-list only, with the permissive IP defaults the modules ship.
_, err := DecideOutbound(
context.Background(),
rawURL,
nil,
[]*regexp2.Regexp{regexp2.MustCompile(privateIPsDenyList, 0)},
time.Now().Add(5*time.Second),
)
if !errors.Is(err, ErrFiltered) {
t.Fatalf("userinfo must not evade the deny-list, got: %v", err)
}
})
}
}
func TestDecideOutbound_UserinfoDoesNotSatisfyAllowList(t *testing.T) {
// A host-terminated allow-list, the shape the documentation recommends.
allowList := []*regexp2.Regexp{regexp2.MustCompile(`^https://trusted\.example\.com(:[0-9]+)?(/|$)`, 0)}
for _, rawURL := range []string{
"https://trusted.example.com@169.254.169.254/latest/meta-data/",
"https://trusted.example.com@10.0.0.5/",
"https://trusted.example.com:443@10.0.0.5/",
} {
t.Run(rawURL, func(t *testing.T) {
withStubResolver(t, func(host string) ([]netip.Addr, error) {
return mustAddrs(t, "10.0.0.5"), nil
})
decision, err := DecideOutbound(
context.Background(),
rawURL,
allowList, nil,
time.Now().Add(5*time.Second),
WithDenyPrivateIPs(true),
)
if err == nil {
t.Fatalf("userinfo must not satisfy the allow-list, got decision %+v", decision)
}
if decision.Bypass {
t.Fatal("userinfo must never produce a bypass")
}
})
}
}
func TestDecideOutbound_UserinfoKeptOutOfErrorMessages(t *testing.T) {
withStubResolver(t, func(host string) ([]netip.Addr, error) {
t.Fatalf("unexpected DNS lookup for %q", host)
return nil, nil
})
_, err := DecideOutbound(
context.Background(),
"http://alice:hunter2@127.0.0.1:9999/",
nil,
[]*regexp2.Regexp{regexp2.MustCompile(privateIPsDenyList, 0)},
time.Now().Add(5*time.Second),
)
if err == nil {
t.Fatal("expected the URL to be filtered")
}
if strings.Contains(err.Error(), "hunter2") || strings.Contains(err.Error(), "alice") {
t.Fatalf("error message must not leak URL credentials: %v", err)
}
}
func TestDecideOutbound_LegitimateCredentialsStillReachTheHost(t *testing.T) {
withStubResolver(t, func(host string) ([]netip.Addr, error) {
if host != "example.com" {
t.Fatalf("host = %q, want example.com: userinfo must not reach resolution", host)
}
return mustAddrs(t, "93.184.216.34"), nil
})
// Stripping userinfo is a matching concern only. A credentialed URL that
// breaks no rule must still be allowed through.
decision, err := DecideOutbound(
context.Background(),
"https://alice:hunter2@example.com/report.pdf",
[]*regexp2.Regexp{regexp2.MustCompile(`^https://example\.com(:[0-9]+)?(/|$)`, 0)},
nil,
time.Now().Add(5*time.Second),
WithDenyPrivateIPs(true),
)
if err != nil {
t.Fatalf("credentialed URL matching the allow-list must pass, got: %v", err)
}
if !decision.Bypass {
t.Fatalf("decision.Bypass = false, want true")
}
}
func TestClampedBackoff(t *testing.T) {
const (
min = 1 * time.Second
max = 30 * time.Second
)
retryAfter := func(status int, seconds string) *http.Response {
return &http.Response{StatusCode: status, Header: http.Header{"Retry-After": []string{seconds}}}
}
for _, tc := range []struct {
scenario string
resp *http.Response
want time.Duration
}{
{"429 with an hour is clamped", retryAfter(http.StatusTooManyRequests, "3600"), max},
{"429 with a day is clamped", retryAfter(http.StatusTooManyRequests, "86400"), max},
{"503 with an hour is clamped", retryAfter(http.StatusServiceUnavailable, "3600"), max},
{"429 under the ceiling is honored", retryAfter(http.StatusTooManyRequests, "5"), 5 * time.Second},
{"no response falls back to exponential", nil, min},
} {
t.Run(tc.scenario, func(t *testing.T) {
got := ClampedBackoff(min, max, 0, tc.resp)
if got != tc.want {
t.Fatalf("ClampedBackoff = %s, want %s", got, tc.want)
}
if got > max {
t.Fatalf("ClampedBackoff = %s, which exceeds max %s", got, max)
}
})
}
}
// A negative max means the caller's budget is spent. The backoff must not
// return a negative duration, which would make the retry loop spin.
func TestClampedBackoff_NegativeMaxIsNotNegative(t *testing.T) {
got := ClampedBackoff(1*time.Second, -5*time.Second, 0, nil)
if got < 0 {
t.Fatalf("ClampedBackoff = %s, want a non-negative duration", got)
}
}
func TestNewOutboundHttpClient_NonPositiveTimeout(t *testing.T) {
// Zero stays unbounded: the LibreOffice proxy owns its own lifetime and
// passes it deliberately.
if got := NewOutboundHttpClient(0, nil, nil, false).Timeout; got != 0 {
t.Fatalf("timeout for 0 = %s, want 0", got)
}
// Negative means an expired budget. http.Client reads any non-positive
// Timeout as no deadline at all, so it must not be passed through.
if got := NewOutboundHttpClient(-5*time.Second, nil, nil, false).Timeout; got <= 0 {
t.Fatalf("timeout for a negative budget = %s, want a positive value so the client fails closed", got)
}
}
func TestDecideOutboundExpiredDeadline(t *testing.T) {
// Patterns are matched under the fixed PatternMatchTimeout rather than
// under the caller's remaining budget, so an expired deadline no longer
// surfaces from the match itself. Every scheme must still fail closed,
// including the ones that return before a host is resolved.
expired := time.Now().Add(-time.Second)
for _, rawURL := range []string{
"https://example.com/",
"file:///tmp/foo.html",
"data:text/html,hello",
} {
_, err := DecideOutbound(context.Background(), rawURL, nil, nil, expired)
if !errors.Is(err, context.DeadlineExceeded) {
t.Fatalf("DecideOutbound(%q) with an expired deadline = %v, want context.DeadlineExceeded", rawURL, err)
}
}
}
func TestDecideOutboundBoundsCatastrophicPatterns(t *testing.T) {
// A deny-list pattern that backtracks catastrophically, matched against a
// client-controlled URL. Before PatternMatchTimeout the ceiling was the
// caller's whole budget, so a 30s API_TIMEOUT bought a 30s CPU burn.
// The trailing "!" makes the match fail only after the nested quantifier
// has explored every way to split the run of "a"s.
pattern := regexp2.MustCompile(`^https://example\.com/(a+)+$`, 0)
pattern.MatchTimeout = PatternMatchTimeout
rawURL := "https://example.com/" + strings.Repeat("a", 40) + "!"
start := time.Now()
_, err := DecideOutbound(
context.Background(),
rawURL,
nil,
[]*regexp2.Regexp{pattern},
time.Now().Add(30*time.Second),
)
elapsed := time.Since(start)
if err == nil {
t.Fatal("expected an error from a catastrophic deny-list pattern")
}
// Generous headroom over the 250ms ceiling, still far below the 30s
// deadline the match would otherwise have been allowed to consume.
if elapsed > 5*time.Second {
t.Fatalf("match took %s, want it aborted near PatternMatchTimeout (%s)", elapsed, PatternMatchTimeout)
}
}

View File

@@ -201,17 +201,13 @@ func TestNewServerRecordMetrics(t *testing.T) {
server.RecordMetrics(t.Context(), semconv.ServerMetricData{
ServerName: "stuff",
ResponseSize: 200,
MetricAttributes: semconv.MetricAttributes{
Req: req,
StatusCode: 301,
AdditionalAttributes: []attribute.KeyValue{
attribute.String("key", "value"),
},
},
MetricData: semconv.MetricData{
RequestSize: 100,
ElapsedTime: 300,
Req: req,
StatusCode: 301,
AdditionalAttributes: []attribute.KeyValue{
attribute.String("key", "value"),
},
RequestSize: 100,
ElapsedTime: 300,
})
rm := metricdata.ResourceMetrics{}

View File

@@ -208,7 +208,7 @@ func (a *Api) Descriptor() gotenberg.ModuleDescriptor {
fs.String("api-oidc-issuer", "", "Set the OIDC issuer URL, e.g. https://tenant.example.com/ - the token 'iss' claim must match")
fs.String("api-oidc-audience", "", "Set the expected OIDC audience - the token 'aud' claim must contain it")
fs.String("api-oidc-jwks-url", "", "Set the OIDC JWKS URL - discovered from the issuer's well-known configuration when empty")
fs.StringSlice("api-download-from-allow-list", []string{}, "Set the allowed URLs for the download from feature using regular expressions - supports multiple values")
fs.StringSlice("api-download-from-allow-list", []string{}, `Set the allowed URLs for the download from feature using regular expressions - supports multiple values. A match bypasses --api-download-from-deny-private-ips (API_DOWNLOAD_FROM_DENY_PRIVATE_IPS) and --api-download-from-deny-public-ips (API_DOWNLOAD_FROM_DENY_PUBLIC_IPS), so terminate the host or the pattern also matches suffix hosts, for example ^https?://internal\.svc(:|/|$)`)
fs.StringSlice("api-download-from-deny-list", []string{}, "Set the denied URLs for the download from feature using regular expressions - supports multiple values")
fs.Bool("api-download-from-deny-private-ips", false, "Reject downloadFrom URLs whose host resolves to a non-public IP address (loopback, RFC1918, link-local, unique-local). Enable on deployments that accept untrusted downloadFrom sources to mitigate SSRF against internal services")
fs.Bool("api-download-from-deny-public-ips", false, "Reject downloadFrom URLs whose host resolves to a public IP address. Enable on air-gapped or data-governed deployments to prevent downloads from reaching the public internet")
@@ -383,12 +383,37 @@ func (a *Api) Provision(ctx *gotenberg.Context) error {
// Logger.
a.logger = gotenberg.Logger(a)
a.warnInsecureDebugRoute()
// File system.
a.fs = gotenberg.NewFileSystem(new(gotenberg.OsMkdirAll))
return nil
}
// warnInsecureDebugRoute logs a warning when the debug route is reachable
// without authentication.
//
// The route reports the resolved configuration of every module, which is
// useful to an operator and equally useful to anyone else who can reach it.
// This warns rather than refuses: an operator may sit behind a gateway that
// authenticates on Gotenberg's behalf, and failing startup would break them.
func (a *Api) warnInsecureDebugRoute() {
if !a.enableDebugRoute || a.basicAuthUsername != "" || a.oidcEnabled {
return
}
if a.logger == nil {
return
}
a.logger.WarnContext(
context.Background(),
"--api-enable-debug-route (API_ENABLE_DEBUG_ROUTE) is enabled but no authentication is configured, so anyone who can reach Gotenberg can read its configuration. Set --api-enable-basic-auth (API_ENABLE_BASIC_AUTH) with GOTENBERG_API_BASIC_AUTH_USERNAME and GOTENBERG_API_BASIC_AUTH_PASSWORD, set --api-enable-oidc-auth (API_ENABLE_OIDC_AUTH), or disable the route.",
slog.String("flag", "--api-enable-debug-route"),
slog.String("env", "API_ENABLE_DEBUG_ROUTE"),
)
}
// Validate validates the module properties.
func (a *Api) Validate() error {
var err error

View File

@@ -1,6 +1,8 @@
package api
import (
"bytes"
"log/slog"
"strings"
"testing"
)
@@ -65,3 +67,68 @@ func TestApi_Validate_Auth(t *testing.T) {
})
}
}
func TestApi_warnInsecureDebugRoute(t *testing.T) {
for _, tc := range []struct {
scenario string
api Api
expectWarn bool
expectFields []string
}{
{
scenario: "debug route on with no auth warns",
api: Api{enableDebugRoute: true},
expectWarn: true,
expectFields: []string{"--api-enable-debug-route", "API_ENABLE_DEBUG_ROUTE", "--api-enable-basic-auth", "API_ENABLE_BASIC_AUTH", "--api-enable-oidc-auth", "API_ENABLE_OIDC_AUTH"},
},
{
scenario: "debug route off is silent",
api: Api{enableDebugRoute: false},
expectWarn: false,
},
{
scenario: "basic auth silences it",
api: Api{enableDebugRoute: true, basicAuthUsername: "foo"},
expectWarn: false,
},
{
scenario: "oidc silences it",
api: Api{enableDebugRoute: true, oidcEnabled: true},
expectWarn: false,
},
} {
t.Run(tc.scenario, func(t *testing.T) {
buf := new(bytes.Buffer)
tc.api.logger = slog.New(slog.NewJSONHandler(buf, &slog.HandlerOptions{Level: slog.LevelWarn}))
tc.api.warnInsecureDebugRoute()
logged := buf.String()
if !tc.expectWarn {
if logged != "" {
t.Fatalf("expected no warning, got: %s", logged)
}
return
}
if logged == "" {
t.Fatal("expected a warning, got none")
}
// Every flag named must carry its environment variable.
for _, want := range tc.expectFields {
if !strings.Contains(logged, want) {
t.Fatalf("warning does not mention %q: %s", want, logged)
}
}
if strings.Contains(logged, "—") {
t.Fatalf("warning must not contain an em dash: %s", logged)
}
})
}
}
// The warning reads a.logger, which is nil until Provision assigns it.
func TestApi_warnInsecureDebugRoute_NilLoggerDoesNotPanic(t *testing.T) {
api := Api{enableDebugRoute: true}
api.warnInsecureDebugRoute()
}

View File

@@ -51,6 +51,24 @@ type Context struct {
outputPaths []string
cancelled bool
// fileOrder records the order files were received in, keyed by disk path.
// It breaks ties when two uploads share an original filename, so that
// de-duplicated files keep their upload order instead of being ordered by
// the suffix uniqueFilename added.
fileOrder map[string]int
// fileBase maps a disk path to the original filename as received, before
// de-duplication. Sorting on it keeps a de-duplicated file next to its
// twin rather than wherever its numbered name would land.
fileBase map[string]string
// outputFilename is the sanitized Gotenberg-Output-Filename header,
// snapshotted while the [echo.Context] is still live. Echo returns that
// context to a pool as soon as the handler returns, and an asynchronous
// conversion outlives it, so reading the header from the pooled store later
// yields whichever request happens to own it by then.
outputFilename string
logger *slog.Logger
echoCtx echo.Context
mkdirAll gotenberg.MkdirAll
@@ -80,6 +98,48 @@ func (t *trackingReader) Read(p []byte) (int, error) {
return n, nil
}
// errTooManyDownloadFromEntries is returned by [decodeDownloadFrom] when the
// array holds more entries than the configured maximum.
var errTooManyDownloadFromEntries = errors.New("too many downloadFrom entries")
// decodeDownloadFrom decodes the downloadFrom form field, refusing to
// accumulate more than maxEntries. A maxEntries of 0 means no limit.
//
// It decodes element by element rather than calling [json.Unmarshal] on the
// whole value. A compact array such as "[{},{},{}]" costs three bytes per
// entry on the wire and expands to roughly seventy times that once
// unmarshalled, so counting the entries afterwards is too late to bound the
// allocation. Streaming keeps the cost proportional to maxEntries no matter
// how long the array is.
func decodeDownloadFrom(raw string, maxEntries int) ([]downloadFrom, error) {
dec := json.NewDecoder(strings.NewReader(raw))
token, err := dec.Token()
if err != nil {
return nil, err
}
if delim, ok := token.(json.Delim); !ok || delim != '[' {
return nil, fmt.Errorf("expected a JSON array, got '%v'", token)
}
var dls []downloadFrom
for dec.More() {
if maxEntries > 0 && len(dls) >= maxEntries {
return nil, errTooManyDownloadFromEntries
}
var dl downloadFrom
err = dec.Decode(&dl)
if err != nil {
return nil, err
}
dls = append(dls, dl)
}
return dls, nil
}
type downloadFrom struct {
// Url is the URL to download a file from.
Url string `json:"url"`
@@ -117,14 +177,18 @@ func newContext(echoCtx echo.Context, logger *slog.Logger, fs *gotenberg.FileSys
return nil
}
// Snapshot now, while echoCtx still belongs to this request.
outputFilename, _ := echoCtx.Get("outputFilename").(string)
ctx := &Context{
outputPaths: make([]string, 0),
cancelled: false,
logger: logger,
echoCtx: echoCtx,
mkdirAll: new(gotenberg.OsMkdirAll),
pathRename: new(gotenberg.OsPathRename),
Context: processCtx,
outputPaths: make([]string, 0),
cancelled: false,
outputFilename: outputFilename,
logger: logger,
echoCtx: echoCtx,
mkdirAll: new(gotenberg.OsMkdirAll),
pathRename: new(gotenberg.OsPathRename),
Context: processCtx,
}
// A custom cancel function which removes the context's working directory
@@ -213,8 +277,13 @@ func newContext(echoCtx echo.Context, logger *slog.Logger, fs *gotenberg.FileSys
// any.
raw, ok := ctx.values["downloadFrom"]
if !downloadFromCfg.disable && ok {
var dls []downloadFrom
err = json.Unmarshal([]byte(raw[0]), &dls)
dls, err := decodeDownloadFrom(raw[0], downloadFromCfg.maxEntries)
if errors.Is(err, errTooManyDownloadFromEntries) {
return nil, cancel, WrapError(
fmt.Errorf("decode downloadFrom: %w", err),
NewSentinelHttpError(http.StatusBadRequest, fmt.Sprintf("Invalid 'downloadFrom' form field value: too many entries, the maximum is %d", downloadFromCfg.maxEntries)),
)
}
if err != nil {
return nil, cancel, WrapError(
fmt.Errorf("unmarshal json: %w", err),
@@ -222,16 +291,6 @@ func newContext(echoCtx echo.Context, logger *slog.Logger, fs *gotenberg.FileSys
)
}
// Reject oversized arrays at the trust boundary, before allocating
// the results slice or spawning any goroutine, so a compact request
// cannot inflate into an unbounded fan-out.
if downloadFromCfg.maxEntries > 0 && len(dls) > downloadFromCfg.maxEntries {
return nil, cancel, WrapError(
fmt.Errorf("too many downloadFrom entries: got %d, max %d", len(dls), downloadFromCfg.maxEntries),
NewSentinelHttpError(http.StatusBadRequest, fmt.Sprintf("Invalid 'downloadFrom' form field value: too many entries, the maximum is %d", downloadFromCfg.maxEntries)),
)
}
// Each goroutine writes to its own results slot. The main
// goroutine merges into ctx.files, ctx.diskToOriginal, and
// ctx.filesByField after eg.Wait() to avoid concurrent map
@@ -280,7 +339,12 @@ func newContext(echoCtx echo.Context, logger *slog.Logger, fs *gotenberg.FileSys
logger.DebugContext(dlCtx, fmt.Sprintf("download file from '%s'", dl.Url))
req, err := retryablehttp.NewRequest(http.MethodGet, dl.Url, nil)
// The request must carry dlCtx: retryablehttp.NewRequest builds
// on context.Background(), and its wait between attempts is a
// select on the request context, so a contextless request cannot
// be interrupted by --api-timeout (env API_TIMEOUT) or by the
// caller going away.
req, err := retryablehttp.NewRequestWithContext(dlCtx, http.MethodGet, dl.Url, nil)
if err != nil {
dlSpan.RecordError(err)
dlSpan.SetStatus(codes.Error, err.Error())
@@ -303,14 +367,28 @@ func newContext(echoCtx echo.Context, logger *slog.Logger, fs *gotenberg.FileSys
}
}
// Entries are serialized by the concurrency limit above, so a
// late one can start after the deadline has already passed.
// Fail closed rather than derive a non-positive timeout, which
// [http.Client] reads as no deadline at all.
remaining := time.Until(deadline)
if remaining <= 0 {
dlSpan.RecordError(context.DeadlineExceeded)
dlSpan.SetStatus(codes.Error, context.DeadlineExceeded.Error())
dlSpan.End()
return fmt.Errorf("download file from '%s': %w", dl.Url, context.DeadlineExceeded)
}
client := &retryablehttp.Client{
HTTPClient: gotenberg.NewOutboundHttpClient(time.Until(deadline), downloadFromCfg.allowList, downloadFromCfg.denyList, downloadFromCfg.enableEnvironmentProxy, ipOpts...),
HTTPClient: gotenberg.NewOutboundHttpClient(remaining, downloadFromCfg.allowList, downloadFromCfg.denyList, downloadFromCfg.enableEnvironmentProxy, ipOpts...),
RetryMax: downloadFromCfg.maxRetry,
RetryWaitMin: time.Duration(1) * time.Second,
RetryWaitMax: time.Until(deadline),
RetryWaitMax: remaining,
Logger: gotenberg.NewLeveledLogger(logger),
CheckRetry: retryablehttp.DefaultRetryPolicy,
Backoff: retryablehttp.DefaultBackoff,
// Not DefaultBackoff: it hands a hostile origin control of
// the wait via Retry-After.
Backoff: gotenberg.ClampedBackoff,
}
resp, err := client.Do(req)
@@ -318,6 +396,17 @@ func newContext(echoCtx echo.Context, logger *slog.Logger, fs *gotenberg.FileSys
dlSpan.RecordError(err)
dlSpan.SetStatus(codes.Error, err.Error())
dlSpan.End()
// A redirect target is filtered inside the client, so the
// policy verdict surfaces here rather than from the
// pre-flight above. Keep it out of the response: the first
// hop answers a filtered URL with a generic 403, and a
// later hop must not describe the allow-list, the deny-list
// or the IP policy instead.
if errors.Is(err, gotenberg.ErrFiltered) {
return fmt.Errorf("download file from '%s': %w", dl.Url, err)
}
return WrapError(
fmt.Errorf("download file from to '%s': %w", dl.Url, err),
NewSentinelHttpError(http.StatusBadRequest, fmt.Sprintf("Unable to download file from '%s': %s", dl.Url, err)),
@@ -391,7 +480,7 @@ func newContext(echoCtx echo.Context, logger *slog.Logger, fs *gotenberg.FileSys
// Use a UUID-based name on disk to avoid filesystem
// NAME_MAX limits with long filenames.
// See: https://github.com/gotenberg/gotenberg/issues/1500.
safeName := uuid.New().String() + filepath.Ext(filename)
safeName := uuid.New().String() + safeExt(filename)
path := fmt.Sprintf("%s/%s", ctx.dirPath, safeName)
out, err := os.Create(path)
@@ -445,18 +534,20 @@ func newContext(echoCtx echo.Context, logger *slog.Logger, fs *gotenberg.FileSys
}
for _, r := range results {
ctx.files[r.filename] = r.path
ctx.diskToOriginal[r.path] = r.filename
filename := ctx.uniqueFilename(r.filename)
ctx.files[filename] = r.path
ctx.diskToOriginal[r.path] = filename
ctx.trackFileOrder(r.path, r.filename)
if r.formField != "" {
ctx.filesByField[r.formField] = append(ctx.filesByField[r.formField], r.path)
}
}
}
copyToDisk := func(fh *multipart.FileHeader) error {
copyToDisk := func(fh *multipart.FileHeader) (string, error) {
in, err := fh.Open()
if err != nil {
return fmt.Errorf("open multipart file: %w", err)
return "", fmt.Errorf("open multipart file: %w", err)
}
defer func() {
@@ -479,12 +570,12 @@ func newContext(echoCtx echo.Context, logger *slog.Logger, fs *gotenberg.FileSys
// Use a UUID-based name on disk to avoid filesystem
// NAME_MAX limits with long filenames.
// See: https://github.com/gotenberg/gotenberg/issues/1500.
safeName := uuid.New().String() + filepath.Ext(filename)
safeName := uuid.New().String() + safeExt(filename)
path := fmt.Sprintf("%s/%s", ctx.dirPath, safeName)
out, err := os.Create(path)
if err != nil {
return fmt.Errorf("create local file: %w", err)
return "", fmt.Errorf("create local file: %w", err)
}
defer func() {
err := out.Close()
@@ -495,26 +586,28 @@ func newContext(echoCtx echo.Context, logger *slog.Logger, fs *gotenberg.FileSys
_, err = io.Copy(out, reader)
if err != nil {
return fmt.Errorf("copy multipart file to local file: %w", err)
return "", fmt.Errorf("copy multipart file to local file: %w", err)
}
base := filename
filename = ctx.uniqueFilename(filename)
ctx.files[filename] = path
ctx.diskToOriginal[path] = filename
ctx.trackFileOrder(path, base)
return nil
return filename, nil
}
// Then, copy the form files, if any.
for fieldName, files := range form.File {
for _, fh := range files {
err = copyToDisk(fh)
if err != nil {
return ctx, cancel, fmt.Errorf("copy to disk: %w", err)
filename, errCopy := copyToDisk(fh)
if errCopy != nil {
return ctx, cancel, fmt.Errorf("copy to disk: %w", errCopy)
}
// Track files by field name
filename := sanitizeFilename(fh.Filename)
filePath := ctx.files[filename]
ctx.filesByField[fieldName] = append(ctx.filesByField[fieldName], filePath)
// Track files by field name, under the name copyToDisk actually
// stored, which may be a de-duplicated variant.
ctx.filesByField[fieldName] = append(ctx.filesByField[fieldName], ctx.files[filename])
}
}
@@ -529,9 +622,9 @@ func newContext(echoCtx echo.Context, logger *slog.Logger, fs *gotenberg.FileSys
if symlinkPath == diskPath {
continue
}
err = os.Symlink(filepath.Base(diskPath), symlinkPath)
if err != nil {
logger.DebugContext(context.Background(), fmt.Sprintf("skip symlink for '%s': %s", originalName, err))
errSymlink := os.Symlink(filepath.Base(diskPath), symlinkPath)
if errSymlink != nil {
logger.DebugContext(context.Background(), fmt.Sprintf("skip symlink for '%s': %s", originalName, errSymlink))
}
}
@@ -540,7 +633,10 @@ func newContext(echoCtx echo.Context, logger *slog.Logger, fs *gotenberg.FileSys
ctx.Log().DebugContext(ctx, fmt.Sprintf("form files by field: %+v", ctx.filesByField))
ctx.Log().DebugContext(ctx, fmt.Sprintf("total bytes: %d", totalBytesRead.Load()))
return ctx, cancel, err
// Explicitly nil: the best-effort symlink loop above must not decide the
// outcome of the request. Its failure used to escape here as a bare 500,
// non-deterministically, because ctx.files iterates in random order.
return ctx, cancel, nil
}
// Request returns the [http.Request].
@@ -555,6 +651,8 @@ func (ctx *Context) FormData() *FormData {
files: ctx.files,
filesByField: ctx.filesByField,
diskToOriginal: ctx.diskToOriginal,
fileOrder: ctx.fileOrder,
fileBase: ctx.fileBase,
errors: nil,
}
}
@@ -595,7 +693,7 @@ func (ctx *Context) GeneratePath(extension string) string {
// limits but registers the given filename so that [Context.OriginalFilename]
// can resolve it. It does not create a file.
func (ctx *Context) GeneratePathFromFilename(filename string) string {
safeName := uuid.New().String() + filepath.Ext(filename)
safeName := uuid.New().String() + safeExt(filename)
path := fmt.Sprintf("%s/%s", ctx.dirPath, safeName)
ctx.diskToOriginal[path] = filename
return path
@@ -698,15 +796,72 @@ func (ctx *Context) BuildOutputFile() (string, error) {
// OutputFilename returns the filename based on the given output path or the
// "Gotenberg-Output-Filename" header's value.
func (ctx *Context) OutputFilename(outputPath string) string {
filename := ctx.echoCtx.Get("outputFilename").(string)
if filename == "" {
if ctx.outputFilename == "" {
return ctx.OriginalFilename(outputPath)
}
filename := ctx.outputFilename
return fmt.Sprintf("%s%s", filename, filepath.Ext(outputPath))
}
// maxDiskExtLength bounds the extension copied onto a UUID-based disk name.
// The UUID stem is 36 characters, so a longer extension risks NAME_MAX, which
// is 255 on ext4 and overlayfs. The untruncated name is kept in
// [Context.diskToOriginal], which never reaches the filesystem.
const maxDiskExtLength = 32
// safeExt returns the extension to append to a UUID-based disk name. It drops
// an extension too long to be safe rather than let [os.Create] fail with
// ENAMETOOLONG, which surfaced to the caller as a bare 500.
func safeExt(filename string) string {
ext := filepath.Ext(filename)
if len(ext) > maxDiskExtLength {
return ""
}
return ext
}
// uniqueFilename returns filename, or a numbered variant of it when the
// request already carries a file by that name.
//
// Uploads are keyed by their sanitized original filename, so two files sharing
// one name used to collide: the second overwrote the first and only one
// reached the conversion, while both stayed on disk and counted against the
// body limit. Sanitizing strips directories, so "a/doc.pdf" and "b/doc.pdf"
// collide too.
func (ctx *Context) uniqueFilename(filename string) string {
_, exists := ctx.files[filename]
if !exists {
return filename
}
ext := filepath.Ext(filename)
stem := strings.TrimSuffix(filename, ext)
for i := 2; ; i++ {
candidate := fmt.Sprintf("%s (%d)%s", stem, i, ext)
_, exists = ctx.files[candidate]
if !exists {
return candidate
}
}
}
// trackFileOrder records where a file arrived in the request and the filename
// it arrived under, so [FormData.paths] can order it the way the caller sent
// it.
func (ctx *Context) trackFileOrder(path, base string) {
if ctx.fileOrder == nil {
ctx.fileOrder = make(map[string]int)
ctx.fileBase = make(map[string]string)
}
ctx.fileOrder[path] = len(ctx.fileOrder)
ctx.fileBase[path] = base
}
// sanitizeFilename strips path separators (including backslashes, which
// [filepath.Base] ignores on Linux) and control characters from a
// caller-supplied filename, then NFC-normalizes the result. This prevents a

View File

@@ -11,11 +11,15 @@ import (
"net/http"
"net/http/httptest"
"os"
"regexp"
"runtime"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/dlclark/regexp2"
"github.com/labstack/echo/v4"
"github.com/gotenberg/gotenberg/v8/pkg/gotenberg"
@@ -414,3 +418,487 @@ func TestContext_FileCount(t *testing.T) {
t.Errorf("expected 3 files, got %d", got)
}
}
// A hostile origin must not choose how long Gotenberg waits.
// [retryablehttp.DefaultBackoff] returns a Retry-After header verbatim for 429
// and 503, and the wait between attempts is a select on the request context.
// Building the request without a context therefore pinned the goroutine, its
// connection, and its working directory for the attacker's chosen duration,
// well past --api-timeout (env API_TIMEOUT).
func TestNewContext_DownloadFromHostileRetryAfterIsBounded(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Retry-After", "3600")
w.WriteHeader(http.StatusTooManyRequests)
}))
defer server.Close()
payload, err := json.Marshal([]downloadFrom{{Url: server.URL + "/file"}})
if err != nil {
t.Fatalf("marshal downloadFrom payload: %v", err)
}
body := new(bytes.Buffer)
writer := multipart.NewWriter(body)
err = writer.WriteField("downloadFrom", string(payload))
if err != nil {
t.Fatalf("write downloadFrom field: %v", err)
}
err = writer.Close()
if err != nil {
t.Fatalf("close multipart writer: %v", err)
}
req := httptest.NewRequest(http.MethodPost, "/forms/libreoffice/convert", body)
req.Header.Set("Content-Type", writer.FormDataContentType())
echoCtx := echo.New().NewContext(req, httptest.NewRecorder())
logger := slog.New(slog.DiscardHandler)
fs := gotenberg.NewFileSystem(new(gotenberg.OsMkdirAll))
const timeout = 500 * time.Millisecond
start := time.Now()
_, cancel, err := newContext(echoCtx, logger, fs, timeout, 0, downloadFromConfig{maxRetry: 2})
elapsed := time.Since(start)
if cancel != nil {
defer cancel()
}
if err == nil {
t.Fatal("expected newContext to fail against an origin that only answers 429")
}
// Generous: the deadline is 500ms and Retry-After asks for an hour. Any
// value in seconds means the remote is still in control.
if elapsed > 10*time.Second {
t.Fatalf("newContext took %s with Retry-After 3600; --api-timeout must bound it", elapsed)
}
}
// An entry that starts after the deadline has passed must fail closed. It used
// to derive a negative client timeout, which [http.Client] reads as no
// deadline at all, leaving the download unbounded.
func TestNewContext_DownloadFromExpiredBudgetFailsClosed(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
<-r.Context().Done()
}))
defer server.Close()
// Two entries, serialized by the concurrency limit, so the second one
// starts once the first has burned the whole budget.
payload, err := json.Marshal([]downloadFrom{
{Url: server.URL + "/first"},
{Url: server.URL + "/second"},
})
if err != nil {
t.Fatalf("marshal downloadFrom payload: %v", err)
}
body := new(bytes.Buffer)
writer := multipart.NewWriter(body)
err = writer.WriteField("downloadFrom", string(payload))
if err != nil {
t.Fatalf("write downloadFrom field: %v", err)
}
err = writer.Close()
if err != nil {
t.Fatalf("close multipart writer: %v", err)
}
req := httptest.NewRequest(http.MethodPost, "/forms/libreoffice/convert", body)
req.Header.Set("Content-Type", writer.FormDataContentType())
echoCtx := echo.New().NewContext(req, httptest.NewRecorder())
logger := slog.New(slog.DiscardHandler)
fs := gotenberg.NewFileSystem(new(gotenberg.OsMkdirAll))
done := make(chan error, 1)
go func() {
_, cancel, err := newContext(echoCtx, logger, fs, 400*time.Millisecond, 0, downloadFromConfig{
maxRetry: 0,
maxConcurrency: 1,
})
if cancel != nil {
cancel()
}
done <- err
}()
select {
case err := <-done:
if err == nil {
t.Fatal("expected newContext to fail against a stalling origin")
}
case <-time.After(15 * time.Second):
t.Fatal("newContext never returned: an entry starting past the deadline built an unbounded client")
}
}
func TestDecodeDownloadFrom(t *testing.T) {
for _, tc := range []struct {
scenario string
raw string
maxEntries int
expectErr error
expectLen int
}{
{"empty array", `[]`, 10, nil, 0},
{"under the limit", `[{"url":"http://a"},{"url":"http://b"}]`, 10, nil, 2},
{"exactly the limit", `[{"url":"http://a"},{"url":"http://b"}]`, 2, nil, 2},
{"over the limit", `[{"url":"http://a"},{"url":"http://b"}]`, 1, errTooManyDownloadFromEntries, 0},
{"no limit", `[{"url":"http://a"},{"url":"http://b"}]`, 0, nil, 2},
{"not an array", `{"url":"http://a"}`, 10, nil, 0},
{"malformed", `[{"url":`, 10, nil, 0},
{"not json", `nope`, 10, nil, 0},
} {
t.Run(tc.scenario, func(t *testing.T) {
dls, err := decodeDownloadFrom(tc.raw, tc.maxEntries)
if tc.expectErr != nil {
if !errors.Is(err, tc.expectErr) {
t.Fatalf("error = %v, want %v", err, tc.expectErr)
}
return
}
if tc.scenario == "not an array" || tc.scenario == "malformed" || tc.scenario == "not json" {
if err == nil {
t.Fatalf("expected an error for %q", tc.raw)
}
return
}
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if len(dls) != tc.expectLen {
t.Fatalf("decoded %d entries, want %d", len(dls), tc.expectLen)
}
})
}
}
// A compact array costs three bytes per entry on the wire and expands by
// roughly seventy times once unmarshalled. Decoding must stop at the limit
// rather than materialize the whole array and count afterwards.
func TestDecodeDownloadFrom_StopsBeforeMaterializingTheArray(t *testing.T) {
const entries = 2_000_000
raw := "[" + strings.Repeat("{},", entries) + "{}]"
var before, after runtime.MemStats
runtime.GC()
runtime.ReadMemStats(&before)
_, err := decodeDownloadFrom(raw, 1000)
runtime.ReadMemStats(&after)
if !errors.Is(err, errTooManyDownloadFromEntries) {
t.Fatalf("error = %v, want errTooManyDownloadFromEntries", err)
}
// json.Unmarshal on the same input allocates hundreds of MiB. Bounded
// decoding should stay in the low single-digit MiB, so this threshold is
// deliberately loose and still fails loudly on a regression.
allocated := after.TotalAlloc - before.TotalAlloc
if allocated > 32<<20 {
t.Fatalf("decoding allocated %d MiB for a %d-entry array, want the limit to bound it", allocated>>20, entries)
}
t.Logf("allocated %d KiB decoding a %d-entry array with a limit of 1000", allocated>>10, entries)
}
// An asynchronous conversion outlives the [echo.Context]. Echo returns that
// context to a sync.Pool as soon as the handler returns, and
// outputFilenameMiddleware runs in srv.Pre on every request, including
// /health, so a later request overwrites the store. Reading the output
// filename from it after the fact returned another caller's value.
func TestContext_OutputFilename_SurvivesEchoContextRecycling(t *testing.T) {
body := new(bytes.Buffer)
writer := multipart.NewWriter(body)
err := writer.Close()
if err != nil {
t.Fatalf("close multipart writer: %v", err)
}
req := httptest.NewRequest(http.MethodPost, "/forms/libreoffice/convert", body)
req.Header.Set("Content-Type", writer.FormDataContentType())
echoCtx := echo.New().NewContext(req, httptest.NewRecorder())
// What outputFilenameMiddleware does for this request.
echoCtx.Set("outputFilename", "victim")
logger := slog.New(slog.DiscardHandler)
fs := gotenberg.NewFileSystem(new(gotenberg.OsMkdirAll))
ctx, cancel, err := newContext(echoCtx, logger, fs, 10*time.Second, 0, downloadFromConfig{disable: true})
if err != nil {
t.Fatalf("newContext returned error: %v", err)
}
defer cancel()
// Echo recycles the context and another request claims the store.
echoCtx.Set("outputFilename", "attacker-controlled")
if got := ctx.OutputFilename("/tmp/out.pdf"); got != "victim.pdf" {
t.Fatalf("OutputFilename = %q, want %q", got, "victim.pdf")
}
}
// A recycled context has a nil store, so the previous unguarded type assertion
// could panic. The snapshot must tolerate an absent value.
func TestContext_OutputFilename_NoHeader(t *testing.T) {
body := new(bytes.Buffer)
writer := multipart.NewWriter(body)
err := writer.Close()
if err != nil {
t.Fatalf("close multipart writer: %v", err)
}
req := httptest.NewRequest(http.MethodPost, "/forms/libreoffice/convert", body)
req.Header.Set("Content-Type", writer.FormDataContentType())
// No Set call at all: the store holds nothing for "outputFilename".
echoCtx := echo.New().NewContext(req, httptest.NewRecorder())
logger := slog.New(slog.DiscardHandler)
fs := gotenberg.NewFileSystem(new(gotenberg.OsMkdirAll))
ctx, cancel, err := newContext(echoCtx, logger, fs, 10*time.Second, 0, downloadFromConfig{disable: true})
if err != nil {
t.Fatalf("newContext returned error: %v", err)
}
defer cancel()
if got := ctx.OutputFilename("/tmp/out.pdf"); got != "out.pdf" {
t.Fatalf("OutputFilename = %q, want the original filename %q", got, "out.pdf")
}
}
func TestSafeExt(t *testing.T) {
for _, tc := range []struct {
scenario string
filename string
want string
}{
{"ordinary extension", "report.pdf", ".pdf"},
{"no extension", "report", ""},
{"at the limit", "a." + strings.Repeat("x", maxDiskExtLength-1), "." + strings.Repeat("x", maxDiskExtLength-1)},
{"over the limit is dropped", "a." + strings.Repeat("x", 300), ""},
} {
t.Run(tc.scenario, func(t *testing.T) {
got := safeExt(tc.filename)
if got != tc.want {
t.Fatalf("safeExt(%q) = %q, want %q", tc.filename, got, tc.want)
}
// A UUID stem is 36 characters. The whole disk name must stay
// under NAME_MAX.
if len(got)+36 > 255 {
t.Fatalf("disk name would be %d characters, over NAME_MAX", len(got)+36)
}
})
}
}
// An upload whose extension exceeds NAME_MAX used to fail os.Create and return
// a bare 500. The extension is bounded, and the original name survives in
// diskToOriginal.
func TestNewContext_LongExtensionIsAccepted(t *testing.T) {
filename := "invoice." + strings.Repeat("x", 300)
body := new(bytes.Buffer)
writer := multipart.NewWriter(body)
part, err := writer.CreateFormFile("files", filename)
if err != nil {
t.Fatalf("create multipart file: %v", err)
}
_, err = part.Write([]byte("%PDF-1.4"))
if err != nil {
t.Fatalf("write multipart file: %v", err)
}
err = writer.Close()
if err != nil {
t.Fatalf("close multipart writer: %v", err)
}
req := httptest.NewRequest(http.MethodPost, "/forms/libreoffice/convert", body)
req.Header.Set("Content-Type", writer.FormDataContentType())
echoCtx := echo.New().NewContext(req, httptest.NewRecorder())
logger := slog.New(slog.DiscardHandler)
fs := gotenberg.NewFileSystem(new(gotenberg.OsMkdirAll))
ctx, cancel, err := newContext(echoCtx, logger, fs, 10*time.Second, 0, downloadFromConfig{disable: true})
if cancel != nil {
defer cancel()
}
if err != nil {
t.Fatalf("newContext returned error for a long extension: %v", err)
}
if got := len(ctx.files); got != 1 {
t.Fatalf("files = %d, want 1", got)
}
}
// A filename that cannot become a symlink (too long, "..", "/") must not fail
// the request. The symlink loop is best-effort, but its error escaped through
// the shared err variable, and ctx.files iterates randomly, so byte-identical
// requests gave different HTTP outcomes.
func TestNewContext_UnsymlinkableFilenameStillSucceeds(t *testing.T) {
for _, filename := range []string{
strings.Repeat("a", 300) + ".txt",
"..",
"/",
} {
t.Run(filename[:min(len(filename), 12)], func(t *testing.T) {
body := new(bytes.Buffer)
writer := multipart.NewWriter(body)
part, err := writer.CreateFormFile("files", filename)
if err != nil {
t.Fatalf("create multipart file: %v", err)
}
_, err = part.Write([]byte("%PDF-1.4"))
if err != nil {
t.Fatalf("write multipart file: %v", err)
}
err = writer.Close()
if err != nil {
t.Fatalf("close multipart writer: %v", err)
}
req := httptest.NewRequest(http.MethodPost, "/forms/libreoffice/convert", body)
req.Header.Set("Content-Type", writer.FormDataContentType())
echoCtx := echo.New().NewContext(req, httptest.NewRecorder())
logger := slog.New(slog.DiscardHandler)
fs := gotenberg.NewFileSystem(new(gotenberg.OsMkdirAll))
_, cancel, err := newContext(echoCtx, logger, fs, 10*time.Second, 0, downloadFromConfig{disable: true})
if cancel != nil {
defer cancel()
}
if err != nil {
t.Fatalf("newContext failed on a best-effort symlink for %q: %v", filename, err)
}
})
}
}
// Two uploads sharing a filename must both reach the conversion. The second
// used to overwrite the first in ctx.files, so one file was silently dropped
// while both stayed on disk and counted against the body limit.
func TestNewContext_DuplicateFilenamesAreBothKept(t *testing.T) {
body := new(bytes.Buffer)
writer := multipart.NewWriter(body)
for _, content := range []string{"FIRST", "SECOND"} {
part, err := writer.CreateFormFile("files", "doc.pdf")
if err != nil {
t.Fatalf("create multipart file: %v", err)
}
_, err = part.Write([]byte(content))
if err != nil {
t.Fatalf("write multipart file: %v", err)
}
}
err := writer.Close()
if err != nil {
t.Fatalf("close multipart writer: %v", err)
}
req := httptest.NewRequest(http.MethodPost, "/forms/pdfengines/merge", body)
req.Header.Set("Content-Type", writer.FormDataContentType())
echoCtx := echo.New().NewContext(req, httptest.NewRecorder())
logger := slog.New(slog.DiscardHandler)
fs := gotenberg.NewFileSystem(new(gotenberg.OsMkdirAll))
ctx, cancel, err := newContext(echoCtx, logger, fs, 10*time.Second, 0, downloadFromConfig{disable: true})
if err != nil {
t.Fatalf("newContext returned error: %v", err)
}
defer cancel()
if got := len(ctx.files); got != 2 {
t.Fatalf("ctx.files = %d entries, want 2: a duplicate filename dropped a file", got)
}
if got := len(ctx.filesByField["files"]); got != 2 {
t.Fatalf("filesByField[files] = %d entries, want 2", got)
}
// The two maps must agree, and both files must be distinct on disk.
seen := make(map[string]struct{})
for _, path := range ctx.files {
if _, ok := ctx.diskToOriginal[path]; !ok {
t.Fatalf("path %q has no diskToOriginal entry", path)
}
seen[path] = struct{}{}
}
if len(seen) != 2 {
t.Fatalf("distinct disk paths = %d, want 2", len(seen))
}
}
// A redirect target is filtered inside the HTTP client, so the policy verdict
// surfaces from client.Do rather than from the pre-flight check. It used to be
// interpolated into the response body, so a redirect described the allow-list,
// the deny-list or the IP policy where the first hop returns a generic 403.
func TestNewContext_DownloadFromRedirectVerdictStaysGeneric(t *testing.T) {
private := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Disposition", `attachment; filename="secret.txt"`)
_, _ = w.Write([]byte("internal"))
}))
defer private.Close()
redirector := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
http.Redirect(w, r, private.URL+"/secret", http.StatusFound)
}))
defer redirector.Close()
payload, err := json.Marshal([]downloadFrom{{Url: redirector.URL + "/start"}})
if err != nil {
t.Fatalf("marshal downloadFrom payload: %v", err)
}
body := new(bytes.Buffer)
writer := multipart.NewWriter(body)
err = writer.WriteField("downloadFrom", string(payload))
if err != nil {
t.Fatalf("write downloadFrom field: %v", err)
}
err = writer.Close()
if err != nil {
t.Fatalf("close multipart writer: %v", err)
}
req := httptest.NewRequest(http.MethodPost, "/forms/libreoffice/convert", body)
req.Header.Set("Content-Type", writer.FormDataContentType())
echoCtx := echo.New().NewContext(req, httptest.NewRecorder())
logger := slog.New(slog.DiscardHandler)
fs := gotenberg.NewFileSystem(new(gotenberg.OsMkdirAll))
// The first hop is allowed, the redirect target is denied by the deny-list.
denyList := []*regexp2.Regexp{regexp2.MustCompile("^"+regexp.QuoteMeta(private.URL), 0)}
_, cancel, err := newContext(echoCtx, logger, fs, 10*time.Second, 0, downloadFromConfig{
denyList: denyList,
maxRetry: 0,
})
if cancel != nil {
defer cancel()
}
if err == nil {
t.Fatal("expected the redirect to a denied host to fail")
}
status, message := ParseError(err)
if status != http.StatusForbidden {
t.Fatalf("status = %d, want %d: a filtered redirect must answer like a filtered first hop", status, http.StatusForbidden)
}
if message != http.StatusText(http.StatusForbidden) {
t.Fatalf("message = %q, want the generic %q", message, http.StatusText(http.StatusForbidden))
}
// The response must not name the policy, the pattern, or the blocked host.
for _, leak := range []string{"denied list", "allowed list", "non-public", "expression", private.URL} {
if strings.Contains(message, leak) {
t.Fatalf("response message %q leaks %q", message, leak)
}
}
}

View File

@@ -41,6 +41,8 @@ type FormData struct {
files map[string]string
filesByField map[string][]string
diskToOriginal map[string]string
fileOrder map[string]int
fileBase map[string]string
errors error
}
@@ -582,24 +584,42 @@ func (form *FormData) paths(extensions []string, target *[]string) *FormData {
}
// See https://github.com/gotenberg/gotenberg/issues/139.
originals := make(gotenberg.AlphanumericSort, len(entries))
for i, e := range entries {
originals[i] = e.original
}
sort.Sort(originals)
//
// Sort on the filename as received rather than on the map key. The key
// carries the suffix uniqueFilename adds when two uploads share a name,
// and that suffix would otherwise decide the order: "doc (2).pdf" sorts
// before "doc.pdf". Ordering on the received name keeps the pair adjacent,
// and the arrival index breaks the tie, so duplicates merge in the order
// the caller sent them. A file with a unique name is unaffected, since its
// received name and its key are the same string.
sort.SliceStable(entries, func(i, j int) bool {
nameI := form.receivedName(entries[i].disk, entries[i].original)
nameJ := form.receivedName(entries[j].disk, entries[j].original)
if nameI != nameJ {
return gotenberg.AlphanumericSort{nameI, nameJ}.Less(0, 1)
}
return form.fileOrder[entries[i].disk] < form.fileOrder[entries[j].disk]
})
// Build a lookup from original name to disk path.
lookup := make(map[string]string, len(entries))
for _, e := range entries {
lookup[e.original] = e.disk
}
for _, o := range originals {
*target = append(*target, lookup[o])
*target = append(*target, e.disk)
}
return form
}
// receivedName returns the filename the file at disk arrived under, before
// de-duplication, falling back to fallback.
func (form *FormData) receivedName(disk, fallback string) string {
base, ok := form.fileBase[disk]
if ok {
return base
}
return fallback
}
// append adds an error to the list of errors.
func (form *FormData) append(err error) {
form.errors = errors.Join(form.errors, err)

View File

@@ -1895,3 +1895,81 @@ func TestFormData_Watermarks(t *testing.T) {
t.Errorf("expected %+v, got %+v", want, got)
}
}
// De-duplicating a repeated filename must not change merge order. Files with
// unique names keep exactly the order they had before de-duplication existed,
// and two files sharing a name merge in the order the caller sent them.
func TestFormData_paths_DuplicateFilenamesKeepUploadOrder(t *testing.T) {
for _, tc := range []struct {
scenario string
files map[string]string
fileBase map[string]string
order map[string]int
want []string
}{
{
scenario: "unique names sort exactly as before",
files: map[string]string{"b.pdf": "/w/2", "a.pdf": "/w/1", "c.pdf": "/w/3"},
fileBase: map[string]string{"/w/1": "a.pdf", "/w/2": "b.pdf", "/w/3": "c.pdf"},
order: map[string]int{"/w/1": 0, "/w/2": 1, "/w/3": 2},
want: []string{"/w/1", "/w/2", "/w/3"},
},
{
scenario: "numeric prefixes still win",
files: map[string]string{"10_x.pdf": "/w/3", "2_x.pdf": "/w/2", "1_x.pdf": "/w/1"},
fileBase: map[string]string{"/w/1": "1_x.pdf", "/w/2": "2_x.pdf", "/w/3": "10_x.pdf"},
order: map[string]int{"/w/1": 0, "/w/2": 1, "/w/3": 2},
want: []string{"/w/1", "/w/2", "/w/3"},
},
{
scenario: "duplicates merge in upload order, not suffix order",
files: map[string]string{"doc.pdf": "/w/1", "doc (2).pdf": "/w/2"},
fileBase: map[string]string{"/w/1": "doc.pdf", "/w/2": "doc.pdf"},
order: map[string]int{"/w/1": 0, "/w/2": 1},
want: []string{"/w/1", "/w/2"},
},
{
scenario: "duplicates stay adjacent and in position",
files: map[string]string{
"a.pdf": "/w/1", "doc.pdf": "/w/2", "doc (2).pdf": "/w/3", "z.pdf": "/w/4",
},
fileBase: map[string]string{
"/w/1": "a.pdf", "/w/2": "doc.pdf", "/w/3": "doc.pdf", "/w/4": "z.pdf",
},
order: map[string]int{"/w/1": 0, "/w/2": 1, "/w/3": 2, "/w/4": 3},
want: []string{"/w/1", "/w/2", "/w/3", "/w/4"},
},
{
scenario: "three copies keep their order",
files: map[string]string{"r.pdf": "/w/1", "r (2).pdf": "/w/2", "r (3).pdf": "/w/3"},
fileBase: map[string]string{"/w/1": "r.pdf", "/w/2": "r.pdf", "/w/3": "r.pdf"},
order: map[string]int{"/w/1": 0, "/w/2": 1, "/w/3": 2},
want: []string{"/w/1", "/w/2", "/w/3"},
},
} {
t.Run(tc.scenario, func(t *testing.T) {
form := &FormData{
files: tc.files,
filesByField: map[string][]string{},
fileBase: tc.fileBase,
fileOrder: tc.order,
}
// Map iteration is randomised, so run it repeatedly: an unstable
// comparator shows up as a differing result across runs.
for range 50 {
var got []string
form.paths([]string{".pdf"}, &got)
if len(got) != len(tc.want) {
t.Fatalf("paths() returned %d entries, want %d", len(got), len(tc.want))
}
for i := range got {
if got[i] != tc.want[i] {
t.Fatalf("paths() = %v, want %v", got, tc.want)
}
}
}
})
}
}

View File

@@ -309,49 +309,59 @@ func telemetryMiddleware(logger *slog.Logger, serverName, correlationIdHeader st
WriteBytes: c.Response().Size,
})...)
accessLogger := logger.
With(slog.String("log_type", "access")).
With(slog.String("correlation_id", correlationId)).
With(slog.String("remote_ip", c.RealIP())).
With(slog.String("host", c.Request().Host)).
With(slog.String("uri", c.Request().RequestURI)).
With(slog.String("method", c.Request().Method)).
With(slog.String("path", routePath)).
With(slog.String("referer", c.Request().Referer())).
With(slog.String("user_agent", c.Request().UserAgent())).
With(slog.Int("status", c.Response().Status)).
With(slog.Int64("latency", int64(finishTime.Sub(startTime)))).
With(slog.String("latency_human", finishTime.Sub(startTime).String())).
With(slog.Int64("bytes_in", c.Request().ContentLength)).
With(slog.Int64("bytes_out", c.Response().Size))
// Pick the level and message before building the record: err.Error
// walks a joined error chain, and the nil-error branch has no use
// for it.
level := slog.LevelInfo
msg := "request handled"
switch {
case err == nil:
accessLogger.InfoContext(ctx, "request handled")
case canceled:
// A client abort is expected, not a server failure; keep it
// visible but out of the error stream.
accessLogger.InfoContext(ctx, err.Error())
msg = err.Error()
default:
accessLogger.ErrorContext(ctx, err.Error())
level = slog.LevelError
msg = err.Error()
}
// One record rather than a chain of With calls. Each With clones
// the whole handler chain, and this logger fans out to a JSON
// handler and an OpenTelemetry bridge that is wired in even when no
// exporter is configured, so a 14-deep chain clones both sub-chains
// 14 times to emit a single line.
latency := finishTime.Sub(startTime)
logger.LogAttrs(ctx, level, msg,
slog.String("log_type", "access"),
slog.String("correlation_id", correlationId),
slog.String("remote_ip", c.RealIP()),
slog.String("host", c.Request().Host),
slog.String("uri", c.Request().RequestURI),
slog.String("method", c.Request().Method),
slog.String("path", routePath),
slog.String("referer", c.Request().Referer()),
slog.String("user_agent", c.Request().UserAgent()),
slog.Int("status", c.Response().Status),
slog.Int64("latency", int64(latency)),
slog.String("latency_human", latency.String()),
slog.Int64("bytes_in", c.Request().ContentLength),
slog.Int64("bytes_out", c.Response().Size),
)
additionalAttributes := []attribute.KeyValue{
semconvSrv.Route(routePath),
}
semconvSrv.RecordMetrics(ctx, semconvutil.ServerMetricData{
ServerName: serverName,
ResponseSize: c.Response().Size,
MetricAttributes: semconvutil.MetricAttributes{
Req: request,
StatusCode: status,
AdditionalAttributes: additionalAttributes,
},
MetricData: semconvutil.MetricData{
RequestSize: request.ContentLength,
ElapsedTime: float64(time.Since(startTime)) / float64(time.Millisecond),
},
ServerName: serverName,
ResponseSize: c.Response().Size,
Req: request,
StatusCode: status,
AdditionalAttributes: additionalAttributes,
RequestSize: request.ContentLength,
ElapsedTime: float64(time.Since(startTime)) / float64(time.Millisecond),
})
return nil

View File

@@ -23,6 +23,25 @@ import (
"github.com/gotenberg/gotenberg/v8/pkg/gotenberg"
)
// chromiumDisableFeatures is the value of Chromium's --disable-features
// switch.
//
// It restates the "site-per-process,Translate,BlinkGenPropertyTrees" default
// from chromedp.DefaultExecAllocatorOptions (chromedp v0.14.2) on purpose:
// chromedp.Flag keys its flags by switch name, so a second --disable-features
// replaces chromedp's value instead of merging with it. Revisit this list when
// bumping chromedp.
//
// WebUIOmniboxPopup and WebUIOmniboxAimPopup became enabled by default in
// Chromium 151.0.7922.132. Their presenters build the address-bar popup WebUI
// at browser start, headless included, which leaves a renderer process holding
// ~85 MB of anonymous memory for a UI a PDF service can never show. Chromium
// silently ignores feature names it does not know, so both stay harmless on
// older builds (they exist but default to disabled on the Chromium pinned for
// ppc64el) and once upstream eventually removes them.
// See https://github.com/gotenberg/gotenberg/issues/1656.
const chromiumDisableFeatures = "site-per-process,Translate,BlinkGenPropertyTrees,WebUIOmniboxPopup,WebUIOmniboxAimPopup"
type browser interface {
gotenberg.Process
pdf(ctx context.Context, logger *slog.Logger, url, outputPath string, options PdfOptions, aggregate *networkAggregate) error
@@ -133,6 +152,8 @@ func (b *chromiumBrowser) Start(logger *slog.Logger) error {
chromedp.Flag("disable-dev-shm-usage", true),
// See https://github.com/gotenberg/gotenberg/issues/1293.
chromedp.Flag("disable-component-update", false),
// See https://github.com/gotenberg/gotenberg/issues/1656.
chromedp.Flag("disable-features", chromiumDisableFeatures),
)
if b.arguments.allowInsecureLocalhost {

View File

@@ -3,6 +3,7 @@ package chromium
import (
"context"
"log/slog"
"slices"
"strings"
"testing"
)
@@ -37,3 +38,24 @@ func TestChromiumBrowser_Start_rejectsOverlappingStart(t *testing.T) {
t.Fatal("expected the browser to stay not started")
}
}
// TestChromiumDisableFeatures guards the override described in
// https://github.com/gotenberg/gotenberg/issues/1656. Gotenberg replaces
// chromedp's --disable-features value rather than extending it, as
// chromedp.Flag keys its flags by switch name. Dropping one of chromedp's own
// entries while editing this list would silently re-enable it.
func TestChromiumDisableFeatures(t *testing.T) {
for _, feature := range []string{
// chromedp.DefaultExecAllocatorOptions.
"site-per-process",
"Translate",
"BlinkGenPropertyTrees",
// The address-bar popup WebUI, built even in headless.
"WebUIOmniboxPopup",
"WebUIOmniboxAimPopup",
} {
if !slices.Contains(strings.Split(chromiumDisableFeatures, ","), feature) {
t.Errorf("expected %q to be disabled, got %q", feature, chromiumDisableFeatures)
}
}
}

View File

@@ -475,7 +475,7 @@ func (mod *Chromium) Descriptor() gotenberg.ModuleDescriptor {
fs.String("chromium-host-resolver-rules", "", "Set custom mappings to the host resolver")
fs.String("chromium-proxy-server", "", "Set the outbound proxy server; this switch only affects HTTP and HTTPS requests")
fs.Bool("chromium-enable-environment-proxy", false, "Route Chromium's outbound requests through the proxy defined by the standard HTTP_PROXY, HTTPS_PROXY, and NO_PROXY variables, including credentials. Use this instead of --chromium-proxy-server for authenticated proxies, and leave --chromium-proxy-server and --chromium-host-resolver-rules unset")
fs.StringSlice("chromium-allow-list", []string{}, "Set the allowed URLs for Chromium using regular expressions - supports multiple values")
fs.StringSlice("chromium-allow-list", []string{}, `Set the allowed URLs for Chromium using regular expressions - supports multiple values. A match bypasses --chromium-deny-private-ips (CHROMIUM_DENY_PRIVATE_IPS) and --chromium-deny-public-ips (CHROMIUM_DENY_PUBLIC_IPS), so terminate the host or the pattern also matches suffix hosts, for example ^https?://internal\.svc(:|/|$)`)
fs.StringSlice("chromium-deny-list", []string{`^file:(?!//\/tmp/).*`}, "Set the denied URLs for Chromium using regular expressions - supports multiple values")
fs.Bool("chromium-deny-private-ips", false, "Reject URLs whose host resolves to a non-public IP address (loopback, RFC1918, link-local, unique-local). Enable on deployments that accept untrusted form input to mitigate SSRF against internal services")
fs.Bool("chromium-deny-public-ips", false, "Reject URLs whose host resolves to a public IP address. Enable on air-gapped or data-governed deployments to prevent outbound traffic from leaving a private network")

View File

@@ -323,7 +323,15 @@ func listenForEventResponseReceived(
return
}
logger.DebugContext(ctx, fmt.Sprintf("event EventResponseReceived fired for a resource: %+v", ev.Response))
// Formatting the whole response is the most expensive thing this
// listener does, and it runs per sub-resource on chromedp's single
// per-target event goroutine while that goroutine holds the mutex
// it also takes to dispatch command responses. At the default log
// level the result is discarded, so gate it on the level rather
// than let slog drop it after the fact.
if logger.Enabled(ctx, slog.LevelDebug) {
logger.DebugContext(ctx, fmt.Sprintf("event EventResponseReceived fired for a resource: %+v", ev.Response))
}
if slices.Contains(options.failOnResourceOnHttpStatusCode, ev.Response.Status) {
if !shouldCheckResourceHttpStatusCode(ev.Response.URL, normalizedIgnoreDomains) {

View File

@@ -11,6 +11,15 @@ import (
// pathological page cannot grow the set without limit.
const maxTrackedOrigins = 64
// maxTrackedRequests bounds the request id to URL map. Entries are dropped as
// soon as the request settles, so the map normally holds only what is in
// flight, but a request that never reports a loading-finished or
// loading-failed event never settles. Without a cap, a page that opens
// requests it never resolves would grow the map for the whole conversion, at
// the cost of one full response URL per entry. Losing an entry only costs the
// heaviest-resource URL attribution for that request.
const maxTrackedRequests = 1024
// networkAggregate accumulates per-conversion network activity from Chromium
// DevTools events. It is safe for concurrent use by the chromedp event listener
// goroutine and the conversion goroutine that reads the snapshot afterwards.
@@ -60,7 +69,9 @@ func (a *networkAggregate) onResponseReceived(ev *network.EventResponseReceived)
a.origins[origin] = struct{}{}
}
}
a.requestURLByID[ev.RequestID] = ev.Response.URL
if len(a.requestURLByID) < maxTrackedRequests {
a.requestURLByID[ev.RequestID] = ev.Response.URL
}
}
// onLoadingFinished records a successfully completed request and its size,
@@ -81,6 +92,11 @@ func (a *networkAggregate) onLoadingFinished(ev *network.EventLoadingFinished) {
a.heaviestBytes = size
a.heaviestURL = a.requestURLByID[ev.RequestID]
}
// The request has settled and nothing reads its URL again. Dropping it
// keeps the map proportional to the requests in flight rather than to
// every request the page ever made.
delete(a.requestURLByID, ev.RequestID)
}
// onLoadingFailed records a request that failed to complete.
@@ -94,6 +110,9 @@ func (a *networkAggregate) onLoadingFailed(ev *network.EventLoadingFailed) {
a.requestCount++
a.failedCount++
// Settled, like a finished request: its URL is never read again.
delete(a.requestURLByID, ev.RequestID)
}
func (a *networkAggregate) snapshot() networkStats {

View File

@@ -98,3 +98,86 @@ func TestNetworkAggregate_ConcurrentSafe(t *testing.T) {
t.Errorf("requestCount = %d, want 100", got)
}
}
// TestNetworkAggregate_SettledRequestsAreDropped covers the growth where every
// response URL stayed in the map for the whole conversion even though nothing
// reads it again once the request settles.
func TestNetworkAggregate_SettledRequestsAreDropped(t *testing.T) {
a := newNetworkAggregate()
for i := range 500 {
id := network.RequestID(fmt.Sprintf("r%d", i))
a.onResponseReceived(&network.EventResponseReceived{
RequestID: id,
Response: &network.Response{URL: fmt.Sprintf("https://host.example.com/%d", i)},
})
if i%2 == 0 {
a.onLoadingFinished(&network.EventLoadingFinished{RequestID: id, EncodedDataLength: 10})
continue
}
a.onLoadingFailed(&network.EventLoadingFailed{RequestID: id})
}
a.mu.Lock()
tracked := len(a.requestURLByID)
a.mu.Unlock()
if tracked != 0 {
t.Errorf("tracked requests = %d, want 0: settled requests must not be retained", tracked)
}
// The bookkeeping the map feeds must survive the pruning.
got := a.snapshot()
if got.requestCount != 500 {
t.Errorf("requestCount = %d, want 500", got.requestCount)
}
if got.failedCount != 250 {
t.Errorf("failedCount = %d, want 250", got.failedCount)
}
}
// TestNetworkAggregate_UnsettledRequestCap verifies the ceiling that applies
// when requests never settle, which is the only way the map can still grow.
func TestNetworkAggregate_UnsettledRequestCap(t *testing.T) {
a := newNetworkAggregate()
for i := range maxTrackedRequests + 500 {
a.onResponseReceived(&network.EventResponseReceived{
RequestID: network.RequestID(fmt.Sprintf("r%d", i)),
Response: &network.Response{URL: fmt.Sprintf("https://host.example.com/%d", i)},
})
}
a.mu.Lock()
tracked := len(a.requestURLByID)
a.mu.Unlock()
if tracked != maxTrackedRequests {
t.Errorf("tracked requests = %d, want %d (capped)", tracked, maxTrackedRequests)
}
}
// TestNetworkAggregate_HeaviestURLSurvivesPruning guards the attribution the
// map exists for: the URL must still be resolved before the entry is dropped.
func TestNetworkAggregate_HeaviestURLSurvivesPruning(t *testing.T) {
a := newNetworkAggregate()
a.onResponseReceived(&network.EventResponseReceived{
RequestID: "small",
Response: &network.Response{URL: "https://example.com/small.css"},
})
a.onLoadingFinished(&network.EventLoadingFinished{RequestID: "small", EncodedDataLength: 10})
a.onResponseReceived(&network.EventResponseReceived{
RequestID: "big",
Response: &network.Response{URL: "https://example.com/big.png"},
})
a.onLoadingFinished(&network.EventLoadingFinished{RequestID: "big", EncodedDataLength: 4096})
got := a.snapshot()
if got.heaviestURL != "https://example.com/big.png" || got.heaviestBytes != 4096 {
t.Errorf("heaviest = (%q, %d), want (%q, 4096)", got.heaviestURL, got.heaviestBytes, "https://example.com/big.png")
}
}

View File

@@ -11,6 +11,7 @@ import (
"net/netip"
"net/url"
"sync"
"sync/atomic"
"time"
"github.com/dlclark/regexp2"
@@ -57,6 +58,18 @@ type pinningProxy struct {
server *http.Server
wg sync.WaitGroup
// closing is closed by Stop to force in-flight CONNECT tunnels shut.
// [http.Server.Shutdown] cannot do it: net/http untracks a connection once
// a handler hijacks it, so a tunnel would otherwise outlive the proxy that
// created it. Recreated on every Start.
closing chan struct{}
// maxTunnels ceilings the CONNECT handlers in flight. Tests may lower it.
maxTunnels int64
// tunnels counts the CONNECT handlers in flight.
tunnels atomic.Int64
logger *slog.Logger
started bool
mu sync.Mutex
@@ -82,6 +95,7 @@ func newPinningProxy(allowList, denyList []*regexp2.Regexp, denyPrivateIPs, deny
dialer := &net.Dialer{Timeout: 10 * time.Second}
return dialer.DialContext(ctx, network, addr)
},
maxTunnels: maxConcurrentTunnels,
}
if enableEnvironmentProxy {
@@ -110,6 +124,7 @@ func (p *pinningProxy) Start(logger *slog.Logger) error {
}
p.listener = l
p.closing = make(chan struct{})
p.logger = logger.With(slog.String("logger", "pinning-proxy"))
p.server = &http.Server{
Handler: http.HandlerFunc(p.serveHTTP),
@@ -140,9 +155,18 @@ func (p *pinningProxy) Stop(logger *slog.Logger) error {
return nil
}
srv := p.server
closing := p.closing
p.closing = nil
p.started = false
p.mu.Unlock()
// Force in-flight tunnels shut before draining the server. Shutdown does
// not reach them, so a tunnel whose upstream never answers would otherwise
// survive the proxy, and with it every Chromium restart.
if closing != nil {
close(closing)
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
@@ -181,6 +205,25 @@ func (p *pinningProxy) serveHTTP(w http.ResponseWriter, req *http.Request) {
// Chromium then negotiates TLS end-to-end with the original hostname in
// SNI.
func (p *pinningProxy) handleConnect(w http.ResponseWriter, req *http.Request) {
// A ceiling, not a tuning knob: it bounds what a tunnel that refuses to end
// can accumulate, whatever keeps it alive. [spliceIdleTimeout] ends a silent
// tunnel, but a peer trickling a byte just under it stays "active" forever,
// and a compromised renderer can hold the client side open to match.
//
// Chromium caps itself well below this. Its socket pool manager allows 128
// sockets per proxy chain for normal traffic plus 128 for WebSocket
// traffic, and every request Gotenberg's Chromium makes traverses this one
// proxy chain, so an honest browser cannot exceed 256 tunnels here. At
// double that, a real page never meets the ceiling and a hostile one stops
// at it.
if !p.acquireTunnel() {
p.logger.WarnContext(req.Context(), fmt.Sprintf("CONNECT to '%s' refused: %d tunnels already in flight", req.Host, p.maxTunnels))
http.Error(w, "too many tunnels", http.StatusServiceUnavailable)
return
}
defer p.releaseTunnel()
_, port, err := net.SplitHostPort(req.Host)
if err != nil {
http.Error(w, "bad CONNECT target", http.StatusBadRequest)
@@ -264,24 +307,142 @@ func (p *pinningProxy) handleConnect(w http.ResponseWriter, req *http.Request) {
return
}
// Splice bytes in both directions until either side closes.
var splice sync.WaitGroup
splice.Add(2)
p.mu.Lock()
closing := p.closing
p.mu.Unlock()
spliceTunnel(client, upstream, closing, spliceIdleTimeout)
}
// maxConcurrentTunnels is the default for [pinningProxy.maxTunnels]. See
// [pinningProxy.handleConnect] for how the value is derived.
const maxConcurrentTunnels = 512
// acquireTunnel reserves a slot for one CONNECT handler, reporting false when
// the proxy is already at [pinningProxy.maxTunnels]. The compare-and-swap loop
// keeps the check and the increment atomic, so concurrent handlers cannot
// overshoot the ceiling between them.
func (p *pinningProxy) acquireTunnel() bool {
for {
current := p.tunnels.Load()
if current >= p.maxTunnels {
return false
}
if p.tunnels.CompareAndSwap(current, current+1) {
return true
}
}
}
// releaseTunnel returns a slot taken by [pinningProxy.acquireTunnel].
func (p *pinningProxy) releaseTunnel() {
p.tunnels.Add(-1)
}
// spliceIdleTimeout bounds a CONNECT tunnel in which no byte has moved in
// either direction.
//
// Nothing else bounds one. The hijacked connections carry no deadline: the
// server clears the header read deadline once the request line is in, and
// net.Dialer.Timeout only covers the connect. net/http also untracks a
// connection once it is hijacked, so neither Server.Shutdown nor a Chromium
// restart reaps it. Left alone, an upstream that accepts the tunnel and then
// answers nothing holds two goroutines and two sockets until the process dies.
//
// Sized well above any legitimate pause between a request and its response, so
// a slow origin is never cut off. A transfer that keeps making progress
// refreshes the deadline and runs for as long as it needs.
const spliceIdleTimeout = 2 * time.Minute
// spliceTunnel copies bytes between the two ends of a CONNECT tunnel until
// both directions finish, the tunnel sits idle for idleTimeout, or closing is
// closed because the proxy is shutting down. Callers pass
// [spliceIdleTimeout]; only tests shorten it.
//
// Each direction half-closes its destination once its source reaches EOF, so a
// peer that waits for the request to end before answering still sees the EOF.
// Idleness is tracked across both directions rather than per direction: the
// client sends nothing for the length of a download, and half-closing its write
// side then would tell the origin the client had gone away.
func spliceTunnel(client, upstream net.Conn, closing <-chan struct{}, idleTimeout time.Duration) {
var lastActivity atomic.Int64
lastActivity.Store(time.Now().UnixNano())
var wg sync.WaitGroup
wg.Add(2)
go func() {
defer splice.Done()
_, _ = io.Copy(upstream, client)
defer wg.Done()
copyTracking(upstream, client, &lastActivity, idleTimeout)
if cw, ok := upstream.(interface{ CloseWrite() error }); ok {
_ = cw.CloseWrite()
}
}()
go func() {
defer splice.Done()
_, _ = io.Copy(client, upstream)
defer wg.Done()
copyTracking(client, upstream, &lastActivity, idleTimeout)
if cw, ok := client.(interface{ CloseWrite() error }); ok {
_ = cw.CloseWrite()
}
}()
splice.Wait()
done := make(chan struct{})
go func() {
wg.Wait()
close(done)
}()
ticker := time.NewTicker(idleTimeout / 4)
defer ticker.Stop()
for {
select {
case <-done:
return
case <-closing:
case <-ticker.C:
if time.Since(time.Unix(0, lastActivity.Load())) < idleTimeout {
continue
}
}
// Closing both ends unblocks whichever copy is still reading. The
// caller's own deferred Close calls then become no-ops.
_ = client.Close()
_ = upstream.Close()
<-done
return
}
}
// copyTracking copies src into dst, recording the time of every chunk that
// moves so [spliceTunnel] can tell a busy tunnel from an idle one.
func copyTracking(dst, src net.Conn, lastActivity *atomic.Int64, writeTimeout time.Duration) {
buf := make([]byte, 32*1024)
for {
n, readErr := src.Read(buf)
if n > 0 {
lastActivity.Store(time.Now().UnixNano())
// Bound the write. A destination that has gone away accepts the
// first chunk into its send buffer and only fails on the next one,
// so without a deadline this direction keeps a dead tunnel alive
// for one more chunk. A destination that stops reading altogether
// would block here forever.
_ = dst.SetWriteDeadline(time.Now().Add(writeTimeout))
_, writeErr := dst.Write(buf[:n])
if writeErr != nil {
return
}
lastActivity.Store(time.Now().UnixNano())
}
if readErr != nil {
return
}
}
}
// handleForward handles plain HTTP requests sent to the proxy as absolute

View File

@@ -898,3 +898,280 @@ func TestPinningProxy_StopIdempotent(t *testing.T) {
t.Fatalf("second Stop on stopped proxy: %v", err)
}
}
// tcpPair returns the two ends of a connected loopback TCP connection. Both
// ends are closed when the test finishes.
func tcpPair(t *testing.T) (net.Conn, net.Conn) {
t.Helper()
listener, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatalf("listen: %v", err)
}
defer func() { _ = listener.Close() }()
type accepted struct {
conn net.Conn
err error
}
acceptChan := make(chan accepted, 1)
go func() {
conn, acceptErr := listener.Accept()
acceptChan <- accepted{conn: conn, err: acceptErr}
}()
dialed, err := net.Dial("tcp", listener.Addr().String())
if err != nil {
t.Fatalf("dial: %v", err)
}
res := <-acceptChan
if res.err != nil {
t.Fatalf("accept: %v", res.err)
}
t.Cleanup(func() {
_ = dialed.Close()
_ = res.conn.Close()
})
return dialed, res.conn
}
// TestSpliceTunnel_IdleTunnelIsClosed covers the leak where an upstream that
// accepted a CONNECT tunnel and then never spoke pinned both splice goroutines
// and both sockets for the lifetime of the process. A hijacked connection
// carries no deadline and net/http stops tracking it, so the idle bound in
// spliceTunnel is the only thing that ends such a tunnel.
func TestSpliceTunnel_IdleTunnelIsClosed(t *testing.T) {
// The peers are kept open by the pair's cleanup: the tunnel is silent, not
// finished.
client, _ := tcpPair(t)
upstream, _ := tcpPair(t)
done := make(chan struct{})
go func() {
spliceTunnel(client, upstream, nil, 100*time.Millisecond)
close(done)
}()
select {
case <-done:
case <-time.After(10 * time.Second):
t.Fatal("spliceTunnel did not return on an idle tunnel")
}
}
// TestSpliceTunnel_ClosingShutsTunnelDown verifies that stopping the proxy
// reaps in-flight tunnels. http.Server.Shutdown cannot: it stops tracking a
// connection once a handler hijacks it, so without this signal a tunnel would
// outlive the proxy and every Chromium restart after it.
func TestSpliceTunnel_ClosingShutsTunnelDown(t *testing.T) {
client, _ := tcpPair(t)
upstream, _ := tcpPair(t)
closing := make(chan struct{})
done := make(chan struct{})
go func() {
// An idle timeout far beyond the test: only closing can end this.
spliceTunnel(client, upstream, closing, time.Hour)
close(done)
}()
close(closing)
select {
case <-done:
case <-time.After(10 * time.Second):
t.Fatal("spliceTunnel did not return when the proxy shut down")
}
}
// TestSpliceTunnel_ActiveTransferOutlivesIdleTimeout guards the idle bound
// against cutting a healthy transfer. Idleness is tracked across both
// directions, so a download that keeps making progress must survive well past
// the timeout even though the client sends nothing throughout.
func TestSpliceTunnel_ActiveTransferOutlivesIdleTimeout(t *testing.T) {
const (
idleTimeout = 100 * time.Millisecond
chunks = 10
interval = 30 * time.Millisecond
)
client, clientPeer := tcpPair(t)
upstream, upstreamPeer := tcpPair(t)
done := make(chan struct{})
go func() {
spliceTunnel(client, upstream, nil, idleTimeout)
close(done)
}()
// Trickle a response for well over the idle timeout, then finish.
go func() {
for range chunks {
_, _ = upstreamPeer.Write([]byte("x"))
time.Sleep(interval)
}
_ = upstreamPeer.Close()
}()
received := 0
buf := make([]byte, chunks)
for received < chunks {
err := clientPeer.SetReadDeadline(time.Now().Add(10 * time.Second))
if err != nil {
t.Fatalf("set read deadline: %v", err)
}
n, readErr := clientPeer.Read(buf)
received += n
if readErr != nil {
break
}
}
if received != chunks {
t.Fatalf("received %d bytes, want %d: the tunnel was cut while still transferring", received, chunks)
}
select {
case <-done:
case <-time.After(10 * time.Second):
t.Fatal("spliceTunnel did not return after the upstream closed")
}
}
// TestPinningProxy_CONNECT_TunnelCeiling verifies the ceiling that bounds what
// tunnels refusing to end can accumulate. The idle bound cannot cover a peer
// that trickles just under it, so the count is what stops the growth.
func TestPinningProxy_CONNECT_TunnelCeiling(t *testing.T) {
// An upstream that accepts and then says nothing: the tunnel stays open.
upstreamAddr, stop := newRawTCPServer(t, func(c net.Conn) {
<-make(chan struct{})
})
t.Cleanup(stop)
p := newPinningProxy(nil, nil, false, false, false)
p.maxTunnels = 1
p.decide = func(_ context.Context, _ string, _, _ []*regexp2.Regexp, _ time.Time) (gotenberg.OutboundDecision, error) {
return gotenberg.OutboundDecision{Pinned: []netip.Addr{netip.MustParseAddr("127.0.0.1")}}, nil
}
p.dialPinned = func(_ context.Context, network string, _ []netip.Addr, _ string) (net.Conn, error) {
return net.Dial(network, upstreamAddr)
}
proxyURL := newProxyForTest(t, p)
proxyAddr := strings.TrimPrefix(proxyURL, "http://")
connect := func(t *testing.T) *bufio.Reader {
t.Helper()
conn, err := net.Dial("tcp", proxyAddr)
if err != nil {
t.Fatalf("dial proxy: %v", err)
}
t.Cleanup(func() { _ = conn.Close() })
err = conn.SetDeadline(time.Now().Add(10 * time.Second))
if err != nil {
t.Fatalf("set deadline: %v", err)
}
_, err = fmt.Fprintf(conn, "CONNECT example.com:443 HTTP/1.1\r\nHost: example.com:443\r\n\r\n")
if err != nil {
t.Fatalf("write CONNECT: %v", err)
}
return bufio.NewReader(conn)
}
first := connect(t)
statusLine, err := first.ReadString('\n')
if err != nil {
t.Fatalf("read first status: %v", err)
}
if !strings.Contains(statusLine, " 200 ") {
t.Fatalf("first CONNECT status = %q, want 200", statusLine)
}
// The first tunnel now holds the only slot.
second := connect(t)
resp, err := http.ReadResponse(second, nil)
if err != nil {
t.Fatalf("read second response: %v", err)
}
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode != http.StatusServiceUnavailable {
t.Fatalf("second CONNECT status = %d, want %d", resp.StatusCode, http.StatusServiceUnavailable)
}
if got := p.tunnels.Load(); got != 1 {
t.Errorf("tunnels in flight = %d, want 1: a refused CONNECT must not consume a slot", got)
}
}
// TestPinningProxy_TunnelSlotIsReleased verifies a completed tunnel gives its
// slot back, so the ceiling bounds concurrency rather than lifetime totals.
func TestPinningProxy_TunnelSlotIsReleased(t *testing.T) {
upstreamAddr, stop := newRawTCPServer(t, func(c net.Conn) {
defer c.Close()
_, _ = c.Write([]byte("HI"))
})
t.Cleanup(stop)
p := newPinningProxy(nil, nil, false, false, false)
p.maxTunnels = 1
p.decide = func(_ context.Context, _ string, _, _ []*regexp2.Regexp, _ time.Time) (gotenberg.OutboundDecision, error) {
return gotenberg.OutboundDecision{Pinned: []netip.Addr{netip.MustParseAddr("127.0.0.1")}}, nil
}
p.dialPinned = func(_ context.Context, network string, _ []netip.Addr, _ string) (net.Conn, error) {
return net.Dial(network, upstreamAddr)
}
proxyURL := newProxyForTest(t, p)
proxyAddr := strings.TrimPrefix(proxyURL, "http://")
for attempt := range 3 {
conn, err := net.Dial("tcp", proxyAddr)
if err != nil {
t.Fatalf("attempt %d dial proxy: %v", attempt, err)
}
err = conn.SetDeadline(time.Now().Add(10 * time.Second))
if err != nil {
t.Fatalf("attempt %d set deadline: %v", attempt, err)
}
_, err = fmt.Fprintf(conn, "CONNECT example.com:443 HTTP/1.1\r\nHost: example.com:443\r\n\r\n")
if err != nil {
t.Fatalf("attempt %d write CONNECT: %v", attempt, err)
}
br := bufio.NewReader(conn)
statusLine, err := br.ReadString('\n')
if err != nil {
t.Fatalf("attempt %d read status: %v", attempt, err)
}
if !strings.Contains(statusLine, " 200 ") {
t.Fatalf("attempt %d CONNECT status = %q, want 200: the slot was not released", attempt, statusLine)
}
// Drain until the upstream's close ends the tunnel, then release it.
_, _ = io.ReadAll(br)
_ = conn.Close()
// The handler returns just after the splice ends.
for range 100 {
if p.tunnels.Load() == 0 {
break
}
time.Sleep(20 * time.Millisecond)
}
if got := p.tunnels.Load(); got != 0 {
t.Fatalf("attempt %d: tunnels in flight = %d, want 0", attempt, got)
}
}
}

View File

@@ -80,6 +80,37 @@ var dangerousTags = []string{
"FilePermissions", // Writing this changes the file's permissions
}
// controlOptions lists ExifTool command-line option names that collide with a
// tag assignment. A metadata key of "csv" becomes the argv entry "-csv=value",
// which exiftool reads as its own option rather than as a tag, so the value
// becomes a filename exiftool opens. Only an unprefixed key can collide:
// "-XMP:csv=value" is unambiguously a tag.
//
// See https://exiftool.org/exiftool_pod.html.
var controlOptions = []string{
"api", "argfile", "charset", "common_args", "config", "csv", "diff",
"echo", "efile", "execute", "ext", "fileorder", "geotag", "geosync",
"htmldump", "if", "json", "lang", "listitem", "o", "out", "p", "php",
"require", "srcfile", "stay_open", "tagsfromfile", "textout", "use", "w",
"wm", "xmlformat",
}
// isControlOption reports whether an unprefixed key would reach exiftool as
// one of its own options instead of as a tag assignment.
func isControlOption(key string) bool {
if strings.Contains(key, ":") {
return false
}
for _, option := range controlOptions {
if strings.EqualFold(key, option) {
return true
}
}
return false
}
// isDangerousTag reports whether key matches one of the [dangerousTags]
// after case-insensitive comparison with any group prefix stripped.
func isDangerousTag(key string) bool {
@@ -114,6 +145,9 @@ func buildExifToolWriteArgs(metadata map[string]any) ([]string, error) {
if !safeKeyPattern.MatchString(key) {
return nil, fmt.Errorf("write PDF metadata with ExifTool: invalid metadata key %q: %w", key, gotenberg.ErrPdfEngineMetadataValueNotSupported)
}
if isControlOption(key) {
return nil, fmt.Errorf("write PDF metadata with ExifTool: metadata key %q is an ExifTool option, prefix it with a group such as %q: %w", key, "XMP:"+key, gotenberg.ErrPdfEngineMetadataValueNotSupported)
}
tag := key
if key == "Trapped" {

View File

@@ -2,6 +2,7 @@ package exiftool
import (
"errors"
"fmt"
"slices"
"testing"
@@ -211,3 +212,48 @@ func TestSafeKeyPattern(t *testing.T) {
}
}
}
// A metadata key that collides with an ExifTool option becomes a bare argv
// entry such as "-csv=/etc/passwd", which exiftool reads as its own option and
// treats the value as a filename to open.
func TestBuildExifToolWriteArgs_RejectsControlOptions(t *testing.T) {
for _, key := range []string{"csv", "CSV", "json", "geotag", "config", "tagsFromFile", "execute", "stay_open", "o", "w", "if", "p"} {
t.Run(key, func(t *testing.T) {
_, err := buildExifToolWriteArgs(map[string]any{key: "/etc/passwd"})
if err == nil {
t.Fatalf("buildExifToolWriteArgs accepted the control option %q", key)
}
if !errors.Is(err, gotenberg.ErrPdfEngineMetadataValueNotSupported) {
t.Fatalf("error %v does not wrap ErrPdfEngineMetadataValueNotSupported", err)
}
})
}
}
// A group prefix makes the key unambiguous, so it must still be accepted.
func TestBuildExifToolWriteArgs_AcceptsPrefixedOptionNames(t *testing.T) {
for _, key := range []string{"XMP:csv", "XMP-dc:json", "IPTC:p"} {
t.Run(key, func(t *testing.T) {
args, err := buildExifToolWriteArgs(map[string]any{key: "value"})
if err != nil {
t.Fatalf("buildExifToolWriteArgs rejected the prefixed key %q: %v", key, err)
}
want := fmt.Sprintf("-%s=value", key)
if len(args) != 1 || args[0] != want {
t.Fatalf("args = %v, want [%s]", args, want)
}
})
}
}
// Ordinary tags must be unaffected.
func TestBuildExifToolWriteArgs_AcceptsOrdinaryTags(t *testing.T) {
for _, key := range []string{"Author", "Title", "Subject", "Keywords", "Producer", "Creator"} {
t.Run(key, func(t *testing.T) {
_, err := buildExifToolWriteArgs(map[string]any{key: "value"})
if err != nil {
t.Fatalf("buildExifToolWriteArgs rejected the ordinary tag %q: %v", key, err)
}
})
}
}

View File

@@ -353,7 +353,7 @@ func (a *Api) Descriptor() gotenberg.ModuleDescriptor {
fs.Duration("libreoffice-idle-shutdown-timeout", 0, "Shutdown LibreOffice after being idle for the given duration. Set to 0 to disable this feature")
fs.Bool("libreoffice-auto-start", false, "Automatically launch LibreOffice upon initialization if set to true; otherwise, LibreOffice will start at the time of the first conversion")
fs.Duration("libreoffice-start-timeout", time.Duration(20)*time.Second, "Maximum duration to wait for LibreOffice to start or restart")
fs.StringSlice("libreoffice-allow-list", []string{}, "Set the allowed URLs for LibreOffice outbound fetches (embedded images, linked content) using regular expressions - supports multiple values")
fs.StringSlice("libreoffice-allow-list", []string{}, `Set the allowed URLs for LibreOffice outbound fetches (embedded images, linked content) using regular expressions - supports multiple values. A match bypasses --libreoffice-deny-private-ips (LIBREOFFICE_DENY_PRIVATE_IPS) and --libreoffice-deny-public-ips (LIBREOFFICE_DENY_PUBLIC_IPS), so terminate the host or the pattern also matches suffix hosts, for example ^https?://internal\.svc(:|/|$)`)
fs.StringSlice("libreoffice-deny-list", []string{}, "Set the denied URLs for LibreOffice outbound fetches using regular expressions - supports multiple values")
fs.Bool("libreoffice-deny-private-ips", false, "Reject LibreOffice outbound URLs whose host resolves to a non-public IP address (loopback, RFC1918, link-local, unique-local). Enable on deployments that accept untrusted documents to mitigate SSRF against internal services")
fs.Bool("libreoffice-deny-public-ips", false, "Reject LibreOffice outbound URLs whose host resolves to a public IP address. Enable on air-gapped or data-governed deployments to prevent outbound traffic from leaving a private network")

View File

@@ -68,6 +68,59 @@ func (p *libreOfficeProcess) Start(logger *slog.Logger) error {
userProfileDirPath := p.fs.NewDirPath()
var (
cmd *gotenberg.Cmd
success bool
)
// Registered here, right after the proxy starts listening, so that every
// failure below tears it down. A return between the proxy start and this
// point strands it: its listener stays bound, its Serve goroutine and HTTP
// client stay alive, and p.proxy is only assigned on success, so nothing
// could ever reach it to stop it. exec.Cmd.Start fails precisely under fd
// or memory pressure, and the supervisor retries the launch on the next
// request, so each stranded proxy compounds the condition that caused it.
defer func() {
if success {
p.cfgMu.Lock()
defer p.cfgMu.Unlock()
p.socketPort = port
p.userProfileDirPath = userProfileDirPath
p.cmd = cmd
p.proxy = proxy
p.isStarted.Store(true)
return
}
// LibreOffice failed to start; tear the proxy down too.
stopErr := proxy.Stop(context.Background())
if stopErr != nil {
logger.WarnContext(context.Background(), fmt.Sprintf("stop LibreOffice outbound proxy after failed start: %s", stopErr))
}
// Let's make sure the process is killed. It is nil when the failure
// happened before the command was built.
if cmd != nil {
killErr := cmd.Kill()
if killErr != nil {
logger.DebugContext(context.Background(), fmt.Sprintf("kill LibreOffice process: %v", killErr))
}
}
// And the user profile directory is deleted. It may never have been
// created, which RemoveAll reports as success.
removeErr := os.RemoveAll(userProfileDirPath)
if removeErr != nil {
logger.ErrorContext(context.Background(), fmt.Sprintf("remove LibreOffice's user profile directory: %v", removeErr))
return
}
logger.DebugContext(context.Background(), fmt.Sprintf("'%s' LibreOffice's user profile directory removed", userProfileDirPath))
}()
// LibreOffice fetches external content (OOXML images via
// TargetMode=External, RTF INCLUDEPICTURE, ODT linked images) inside
// its own libcurl. The profile config routes those fetches through the
@@ -75,7 +128,6 @@ func (p *libreOfficeProcess) Start(logger *slog.Logger) error {
// blocks content linked from untrusted locations so absolute-path
// (file://) and direct fetches are dropped at the source.
if err := writeSofficeProfileConfig(userProfileDirPath, proxy.Addr()); err != nil {
_ = proxy.Stop(context.Background())
return fmt.Errorf("write soffice profile config: %w", err)
}
sofficeEnv := sofficeProxyEnv(os.Environ(), proxy.Addr())
@@ -95,9 +147,8 @@ func (p *libreOfficeProcess) Start(logger *slog.Logger) error {
ctx, cancel := context.WithTimeout(context.Background(), p.arguments.startTimeout)
defer cancel()
cmd, err := gotenberg.CommandContext(ctx, logger, p.arguments.binPath, args...)
cmd, err = gotenberg.CommandContext(ctx, logger, p.arguments.binPath, args...)
if err != nil {
_ = proxy.Stop(context.Background())
return fmt.Errorf("create LibreOffice command: %w", err)
}
cmd.SetEnv(sofficeEnv)
@@ -106,7 +157,6 @@ func (p *libreOfficeProcess) Start(logger *slog.Logger) error {
// able to run as a daemon.
exitCode, err := cmd.Exec()
if err != nil && exitCode != 81 {
_ = proxy.Stop(context.Background())
return fmt.Errorf("execute LibreOffice: %w", err)
}
@@ -155,43 +205,6 @@ func (p *libreOfficeProcess) Start(logger *slog.Logger) error {
}
}()
var success bool
defer func() {
if success {
p.cfgMu.Lock()
defer p.cfgMu.Unlock()
p.socketPort = port
p.userProfileDirPath = userProfileDirPath
p.cmd = cmd
p.proxy = proxy
p.isStarted.Store(true)
return
}
// LibreOffice failed to start; tear the proxy down too.
stopErr := proxy.Stop(context.Background())
if stopErr != nil {
logger.WarnContext(context.Background(), fmt.Sprintf("stop LibreOffice outbound proxy after failed start: %s", stopErr))
}
// Let's make sure the process is killed.
err = cmd.Kill()
if err != nil {
logger.DebugContext(context.Background(), fmt.Sprintf("kill LibreOffice process: %v", err))
}
// And the user profile directory is deleted.
err = os.RemoveAll(userProfileDirPath)
if err != nil {
logger.ErrorContext(context.Background(), fmt.Sprintf("remove LibreOffice's user profile directory: %v", err))
}
logger.DebugContext(context.Background(), fmt.Sprintf("'%s' LibreOffice's user profile directory removed", userProfileDirPath))
}()
logger.DebugContext(context.Background(), "waiting for the LibreOffice socket to be available...")
for {

View File

@@ -35,12 +35,13 @@ var (
zipMagic = []byte{0x50, 0x4b, 0x03, 0x04}
// An unencrypted OOXML document is always a ZIP package, so any of these
// extensions over a compound file means the payload is encrypted. Legacy
// binary formats (.doc, .xls, .ppt) are compound files either way and are
// deliberately absent.
// extensions over a compound file means the payload is encrypted. A .xlsb
// workbook stores binary parts inside that same ZIP package, so it belongs
// here too. Legacy binary formats (.doc, .xls, .ppt) are compound files
// either way and are deliberately absent.
ooxmlExtensions = map[string]struct{}{
".docx": {}, ".docm": {}, ".dotx": {}, ".dotm": {},
".xlsx": {}, ".xlsm": {}, ".xltx": {}, ".xltm": {},
".xlsx": {}, ".xlsm": {}, ".xltx": {}, ".xltm": {}, ".xlsb": {},
".pptx": {}, ".pptm": {}, ".potx": {}, ".potm": {},
".ppsx": {}, ".ppsm": {},
}

View File

@@ -76,6 +76,11 @@ func TestDetectPasswordProtection(t *testing.T) {
path: ole2("encrypted.xlsx"),
want: PasswordProtectionRequired,
},
{
name: "encrypted binary workbook",
path: ole2("encrypted.xlsb"),
want: PasswordProtectionRequired,
},
{
name: "legacy binary document is inconclusive",
path: ole2("legacy.doc"),

View File

@@ -0,0 +1,172 @@
package pdfengines
import (
"sync"
"github.com/gotenberg/gotenberg/v8/pkg/modules/api"
)
// defaultMaxConcurrency is the number of PDF files a single stub processes at
// once when --pdfengines-max-concurrency (env PDFENGINES_MAX_CONCURRENCY) is
// not set. Each unit of work forks an external binary (qpdf, pdfcpu, pdftk or
// exiftool), so the ceiling trades wall clock against process count and RSS.
//
// It defaults to one, which processes files exactly as the sequential loops
// this package used to run did. Raising it only ever affects a request that
// carries several files, or one that splits into several outputs: a
// single-file request never reaches the concurrent path at all. Operators who
// send multi-file batches and have the memory headroom opt in.
//
// This never covers LibreOffice. libreoffice-pdfengine implements Convert and
// nothing else, every other [gotenberg.PdfEngine] method on it returns
// [gotenberg.ErrPdfEngineMethodNotSupported], and [ConvertStub] deliberately
// does not use this package's helpers. A soffice instance costs too much
// memory to run several of per container, so LibreOffice throughput is scaled
// by adding Gotenberg containers, not by raising this number.
const defaultMaxConcurrency = 1
// maxFileConcurrency is how many files one request may have in flight at once.
// It is replaced during [PdfEngines.Provision].
var maxFileConcurrency = defaultMaxConcurrency
// engineExtraSlots bounds the concurrency this package ADDS, across the whole
// process rather than per request, and holds one fewer slot than
// [maxFileConcurrency] because every request already owns one unit of its own.
//
// Bounding the added concurrency rather than the total is what keeps the
// ceiling from becoming a throughput regression. The sequential loops this
// helper replaced had no ceiling at all: X concurrent requests ran X engine
// binaries, one apiece. A pool covering the total would cut those X requests
// down to the ceiling, so an operator raising the flag to speed up a single
// multi-file request would slow the server down under real load. Reserving
// each request the unit it always had makes the worst case "what happened
// before, plus at most maxFileConcurrency-1".
//
// A per-request limit would have the opposite failure: X simultaneous requests
// forking X times the limit, trading the timeouts this exists to prevent for
// memory exhaustion.
var engineExtraSlots = make(chan struct{}, defaultMaxConcurrency-1)
// acquireEngineSlot waits for the first unit of capacity to become available,
// either this request's reserved unit or a slot from the shared pool, and
// returns the function that gives it back.
//
// Waiting on both at once is the whole point. Committing to one source and
// blocking on it strands the other: a goroutine parked on an exhausted pool
// cannot pick up its own request's reserved unit when the file before it
// finishes, so the reserved units sit idle while every file queues on the
// pool, which is slower than having no pool at all.
func acquireEngineSlot(ctx *api.Context, reserved chan struct{}) (func(), error) {
// An [api.Context] carries a request context in production, but one built
// as a literal, which the unit tests do, embeds a nil [context.Context]
// and would panic on Done. A nil channel never fires, which correctly
// leaves the two capacity sources as the only things to wait on.
var done <-chan struct{}
if ctx != nil && ctx.Context != nil {
done = ctx.Done()
}
// A select whose cancellation and capacity cases are both ready picks
// between them at random, so an already-dead request would start more
// files on a coin flip. Check first and stop taking on work.
if done != nil {
select {
case <-done:
return nil, ctx.Err()
default:
}
}
select {
case <-reserved:
return func() { reserved <- struct{}{} }, nil
case engineExtraSlots <- struct{}{}:
return func() { <-engineExtraSlots }, nil
case <-done:
return nil, ctx.Err()
}
}
// forEachInputPath runs fn against every input path, up to
// --pdfengines-max-concurrency (env PDFENGINES_MAX_CONCURRENCY) at a time.
//
// The stubs mutate each PDF in place, so distinct input paths never touch the
// same file and may run together. Callers that layer operations on one file,
// like [WatermarkStub] applying several watermarks in order, must keep that
// outer sequence and parallelize only the file dimension.
//
// Every path is attempted even after one fails, and the error returned is the
// first in input order rather than the first to arrive. That keeps the failing
// filename in the error message identical to what the sequential form
// reported, which the integration scenarios assert on.
func forEachInputPath(ctx *api.Context, inputPaths []string, fn func(inputPath string) error) error {
return forEachInputPathIndexed(ctx, inputPaths, func(_ int, inputPath string) error {
return fn(inputPath)
})
}
// forEachInputPathIndexed is [forEachInputPath] with the input path's index,
// for callers collecting a result per file. Writing into a preallocated slice
// at the given index needs no further synchronization; writing into a shared
// map does and must not be done from fn.
func forEachInputPathIndexed(ctx *api.Context, inputPaths []string, fn func(i int, inputPath string) error) error {
if len(inputPaths) == 0 {
return nil
}
// The common case is a single file. Skip the goroutine and the slot: the
// caller is already inside whatever bound its own route applies.
if len(inputPaths) == 1 {
return fn(0, inputPaths[0])
}
// At the default ceiling of one, run the plain sequential loop this helper
// replaced. Racing goroutines for a single slot would serialize the work
// just the same, but the order files are picked up in would be down to the
// scheduler, and every file would be attempted even once one has failed.
// Taking the old path keeps the default a genuine no-op: same order, same
// early return, no goroutines.
if maxFileConcurrency < 2 {
for i, inputPath := range inputPaths {
err := fn(i, inputPath)
if err != nil {
return err
}
}
return nil
}
// The unit this request would have had all to itself before any of this
// existed. Whichever file claims it runs without touching the shared pool,
// so concurrent requests can never throttle each other below the
// one-binary-apiece they already got. See [engineExtraSlots].
reserved := make(chan struct{}, 1)
reserved <- struct{}{}
errs := make([]error, len(inputPaths))
var wg sync.WaitGroup
for i, inputPath := range inputPaths {
wg.Go(func() {
release, err := acquireEngineSlot(ctx, reserved)
if err != nil {
errs[i] = err
return
}
defer release()
errs[i] = fn(i, inputPath)
})
}
wg.Wait()
for _, err := range errs {
if err != nil {
return err
}
}
return nil
}

View File

@@ -0,0 +1,341 @@
package pdfengines
import (
"context"
"errors"
"fmt"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/gotenberg/gotenberg/v8/pkg/modules/api"
)
func TestForEachInputPath(t *testing.T) {
for _, tc := range []struct {
scenario string
inputPaths []string
fn func(inputPath string) error
expectErr string
}{
{
scenario: "no input path",
inputPaths: nil,
fn: func(string) error { return errors.New("must not run") },
},
{
scenario: "single input path",
inputPaths: []string{"a.pdf"},
fn: func(string) error { return nil },
},
{
scenario: "single input path with error",
inputPaths: []string{"a.pdf"},
fn: func(p string) error { return fmt.Errorf("boom %s", p) },
expectErr: "boom a.pdf",
},
{
scenario: "many input paths",
inputPaths: []string{"a.pdf", "b.pdf", "c.pdf", "d.pdf", "e.pdf"},
fn: func(string) error { return nil },
},
{
scenario: "error is the first in input order, not the first to arrive",
inputPaths: []string{"a.pdf", "b.pdf", "c.pdf"},
fn: func(p string) error {
// "c.pdf" fails without delay so that it lands well before
// "b.pdf"; the reported error must still be "b.pdf".
if p == "b.pdf" {
var counter int
for i := range 5_000_000 {
counter += i
}
return fmt.Errorf("slow failure %s (%d)", p, counter%1)
}
if p == "c.pdf" {
return fmt.Errorf("fast failure %s", p)
}
return nil
},
expectErr: "slow failure b.pdf (0)",
},
} {
t.Run(tc.scenario, func(t *testing.T) {
err := forEachInputPath(new(api.Context), tc.inputPaths, tc.fn)
if tc.expectErr == "" {
if err != nil {
t.Fatalf("expected no error but got: %v", err)
}
return
}
if err == nil {
t.Fatalf("expected error %q but got none", tc.expectErr)
}
if err.Error() != tc.expectErr {
t.Fatalf("expected error %q but got %q", tc.expectErr, err.Error())
}
})
}
}
func TestForEachInputPathRunsEveryPath(t *testing.T) {
inputPaths := make([]string, 50)
for i := range inputPaths {
inputPaths[i] = fmt.Sprintf("%d.pdf", i)
}
var (
mu sync.Mutex
seen = make(map[string]int)
)
err := forEachInputPath(new(api.Context), inputPaths, func(inputPath string) error {
mu.Lock()
defer mu.Unlock()
seen[inputPath]++
return nil
})
if err != nil {
t.Fatalf("expected no error but got: %v", err)
}
if len(seen) != len(inputPaths) {
t.Fatalf("expected %d distinct paths but got %d", len(inputPaths), len(seen))
}
for path, count := range seen {
if count != 1 {
t.Fatalf("expected '%s' to run once but it ran %d times", path, count)
}
}
}
func TestForEachInputPathRespectsTheSlotCeiling(t *testing.T) {
previousSlots, previousMax := engineExtraSlots, maxFileConcurrency
defer func() { engineExtraSlots, maxFileConcurrency = previousSlots, previousMax }()
// One request may run its reserved unit plus ceiling-1 borrowed ones.
const ceiling = 3
maxFileConcurrency = ceiling
engineExtraSlots = make(chan struct{}, ceiling-1)
inputPaths := make([]string, 40)
for i := range inputPaths {
inputPaths[i] = fmt.Sprintf("%d.pdf", i)
}
var inFlight, peak atomic.Int64
err := forEachInputPath(new(api.Context), inputPaths, func(string) error {
current := inFlight.Add(1)
defer inFlight.Add(-1)
for {
observed := peak.Load()
if current <= observed || peak.CompareAndSwap(observed, current) {
break
}
}
// Hold the slot long enough that the ceiling would be exceeded if it
// were not enforced.
var counter int
for i := range 200_000 {
counter += i
}
_ = counter
return nil
})
if err != nil {
t.Fatalf("expected no error but got: %v", err)
}
if peak.Load() > ceiling {
t.Fatalf("expected at most %d concurrent runs but observed %d", ceiling, peak.Load())
}
}
func TestForEachInputPathHonorsCancellation(t *testing.T) {
previousSlots, previousMax := engineExtraSlots, maxFileConcurrency
defer func() { engineExtraSlots, maxFileConcurrency = previousSlots, previousMax }()
// Concurrent path, with the shared pool exhausted by another request, so
// only this request's reserved unit is available.
maxFileConcurrency = 3
engineExtraSlots = make(chan struct{}, 2)
engineExtraSlots <- struct{}{}
engineExtraSlots <- struct{}{}
cancelledCtx, cancel := context.WithCancel(context.Background())
cancel()
// Neither source of capacity is available: the pool is exhausted by other
// requests and this request's reserved unit is already in use by one of its
// own files. A waiter must observe the cancelled context rather than block
// forever. Driving acquireEngineSlot directly keeps that deterministic:
// through forEachInputPath the reserved unit is reusable, so whether a
// given file waits at all depends on how fast the file before it finishes.
inUse := make(chan struct{}, 1)
release, err := acquireEngineSlot(&api.Context{Context: cancelledCtx}, inUse)
if !errors.Is(err, context.Canceled) {
t.Fatalf("expected context.Canceled but got: %v", err)
}
if release != nil {
t.Fatal("expected no release function when acquisition fails")
}
// A cancelled request stops taking on work even when capacity is free,
// rather than deciding on the coin flip a ready select would give.
inUse <- struct{}{}
_, err = acquireEngineSlot(&api.Context{Context: cancelledCtx}, inUse)
if !errors.Is(err, context.Canceled) {
t.Fatalf("expected context.Canceled with the reserved unit free but got: %v", err)
}
// Live request, free reserved unit: acquired and handed back.
release, err = acquireEngineSlot(&api.Context{Context: context.Background()}, inUse)
if err != nil {
t.Fatalf("expected the reserved unit to be acquired but got: %v", err)
}
release()
if len(inUse) != 1 {
t.Fatalf("expected the reserved unit to be returned but the channel holds %d", len(inUse))
}
}
func TestForEachInputPathCompletesWithACancelledContext(t *testing.T) {
previousSlots, previousMax := engineExtraSlots, maxFileConcurrency
defer func() { engineExtraSlots, maxFileConcurrency = previousSlots, previousMax }()
maxFileConcurrency = 3
engineExtraSlots = make(chan struct{}, 2)
engineExtraSlots <- struct{}{}
engineExtraSlots <- struct{}{}
cancelledCtx, cancel := context.WithCancel(context.Background())
cancel()
done := make(chan struct{})
go func() {
defer close(done)
_ = forEachInputPath(&api.Context{Context: cancelledCtx}, []string{"a.pdf", "b.pdf", "c.pdf"}, func(string) error {
return nil
})
}()
select {
case <-done:
case <-time.After(10 * time.Second):
t.Fatal("forEachInputPath hung on a cancelled context with the shared pool exhausted")
}
}
func TestForEachInputPathNeverThrottlesBelowOnePerRequest(t *testing.T) {
previousSlots, previousMax := engineExtraSlots, maxFileConcurrency
defer func() { engineExtraSlots, maxFileConcurrency = previousSlots, previousMax }()
// A small ceiling against far more concurrent requests than it covers.
// Before the shared pool existed each of these ran a binary of its own, so
// the pool must not drop aggregate concurrency below one per request.
const (
ceiling = 2
requests = 8
)
maxFileConcurrency = ceiling
engineExtraSlots = make(chan struct{}, ceiling-1)
// Every runner announces itself and then blocks, so the count of arrivals
// is the true simultaneous concurrency rather than whatever the scheduler
// happened to overlap.
arrived := make(chan struct{}, requests*3)
release := make(chan struct{})
var wg sync.WaitGroup
for range requests {
wg.Go(func() {
_ = forEachInputPath(new(api.Context), []string{"a.pdf", "b.pdf", "c.pdf"}, func(string) error {
arrived <- struct{}{}
<-release
return nil
})
})
}
// One runner per request must be able to start without waiting on the
// shared pool. If the pool governed the total instead of the surplus, only
// `ceiling` runners would ever arrive and this would time out.
for i := range requests {
select {
case <-arrived:
case <-time.After(10 * time.Second):
t.Fatalf("only %d runners started concurrently, expected at least one per request (%d); the shared pool is throttling requests against each other", i, requests)
}
}
close(release)
wg.Wait()
}
func TestForEachInputPathIsSequentialAtTheDefaultCeiling(t *testing.T) {
if defaultMaxConcurrency != 1 {
t.Fatalf("this test pins the default as a no-op, but defaultMaxConcurrency is %d", defaultMaxConcurrency)
}
previousSlots, previousMax := engineExtraSlots, maxFileConcurrency
defer func() { engineExtraSlots, maxFileConcurrency = previousSlots, previousMax }()
maxFileConcurrency = defaultMaxConcurrency
engineExtraSlots = make(chan struct{}, defaultMaxConcurrency-1)
// At the default the helper must behave exactly like the sequential loops
// it replaced: files in input order, and no file attempted once one has
// failed.
var order []string
err := forEachInputPath(new(api.Context), []string{"a.pdf", "b.pdf", "c.pdf", "d.pdf"}, func(inputPath string) error {
order = append(order, inputPath)
if inputPath == "b.pdf" {
return errors.New("boom")
}
return nil
})
if err == nil || err.Error() != "boom" {
t.Fatalf("expected error \"boom\" but got: %v", err)
}
if len(order) != 2 || order[0] != "a.pdf" || order[1] != "b.pdf" {
t.Fatalf("expected the run to stop after b.pdf in input order but got %v", order)
}
}
func TestForEachInputPathIndexed(t *testing.T) {
inputPaths := []string{"a.pdf", "b.pdf", "c.pdf", "d.pdf"}
collected := make([]string, len(inputPaths))
err := forEachInputPathIndexed(new(api.Context), inputPaths, func(i int, inputPath string) error {
collected[i] = inputPath
return nil
})
if err != nil {
t.Fatalf("expected no error but got: %v", err)
}
for i, inputPath := range inputPaths {
if collected[i] != inputPath {
t.Fatalf("expected index %d to hold '%s' but got '%s'", i, inputPath, collected[i])
}
}
}

View File

@@ -45,6 +45,7 @@ type PdfEngines struct {
rotateNames []string
facturXNames []string
engines []gotenberg.PdfEngine
maxConcurrency int
disableRoutes bool
}
@@ -70,6 +71,7 @@ func (mod *PdfEngines) Descriptor() gotenberg.ModuleDescriptor {
fs.StringSlice("pdfengines-stamp-engines", []string{"pdfcpu", "pdftk"}, "Set the PDF engines and their order for the stamp feature - empty means all")
fs.StringSlice("pdfengines-rotate-engines", []string{"pdfcpu", "pdftk"}, "Set the PDF engines and their order for the rotate feature - empty means all")
fs.StringSlice("pdfengines-factur-x-engines", []string{"qpdf"}, "Set the PDF engines and their order for the Factur-X XMP feature - empty means all")
fs.Int("pdfengines-max-concurrency", defaultMaxConcurrency, "Set the maximum number of PDF files a feature processes concurrently, across all requests - bounds how many qpdf, pdfcpu, pdftk and exiftool processes run at once, so raising it trades memory for speed. Does not apply to LibreOffice: scale Gotenberg containers instead")
fs.Bool("pdfengines-disable-routes", false, "Disable the routes")
// Deprecated flags.
@@ -105,8 +107,16 @@ func (mod *PdfEngines) Provision(ctx *gotenberg.Context) error {
stampNames := flags.MustStringSlice("pdfengines-stamp-engines")
rotateNames := flags.MustStringSlice("pdfengines-rotate-engines")
facturXNames := flags.MustStringSlice("pdfengines-factur-x-engines")
mod.maxConcurrency = flags.MustInt("pdfengines-max-concurrency")
mod.disableRoutes = flags.MustBool("pdfengines-disable-routes")
if mod.maxConcurrency > 0 {
maxFileConcurrency = mod.maxConcurrency
// One fewer than the ceiling: each request already reserves a unit of
// its own. See [engineExtraSlots].
engineExtraSlots = make(chan struct{}, mod.maxConcurrency-1)
}
engines, err := ctx.Modules(new(gotenberg.PdfEngine))
if err != nil {
return fmt.Errorf("get PDF engines: %w", err)
@@ -222,6 +232,10 @@ func (mod *PdfEngines) Validate() error {
return errors.New("no PDF engine is available; enable at least one engine module (e.g. qpdf, pdfcpu, pdftk, libreoffice-pdfengine, exiftool)")
}
if mod.maxConcurrency < 1 {
return fmt.Errorf("PDF engines max concurrency must be at least 1, got %d; set --pdfengines-max-concurrency (env PDFENGINES_MAX_CONCURRENCY) to a positive value", mod.maxConcurrency)
}
availableEngines := make([]string, len(mod.engines))
for i, engine := range mod.engines {
@@ -296,6 +310,7 @@ func (mod *PdfEngines) SystemMessages() []string {
fmt.Sprintf("stamp engines - %s", strings.Join(mod.stampNames, " ")),
fmt.Sprintf("rotate engines - %s", strings.Join(mod.rotateNames, " ")),
fmt.Sprintf("factur-x engines - %s", strings.Join(mod.facturXNames, " ")),
fmt.Sprintf("max concurrency - %d", mod.maxConcurrency),
}
}

View File

@@ -208,14 +208,14 @@ func RotateStub(ctx *api.Context, engine gotenberg.PdfEngine, angle int, pages s
return nil
}
for _, inputPath := range inputPaths {
return forEachInputPath(ctx, inputPaths, func(inputPath string) error {
err := engine.Rotate(ctx, ctx.Log(), inputPath, angle, pages)
if err != nil {
return fmt.Errorf("rotate '%s': %w", inputPath, err)
}
}
return nil
return nil
})
}
// ValidatePdfFormatsCompat checks for incompatible combinations of PDF formats
@@ -334,14 +334,14 @@ func SplitPdfStub(ctx *api.Context, engine gotenberg.PdfEngine, mode gotenberg.S
// FlattenStub merges annotation appearances with page content for each given
// PDF, effectively deleting the original annotations.
func FlattenStub(ctx *api.Context, engine gotenberg.PdfEngine, inputPaths []string) error {
for _, inputPath := range inputPaths {
return forEachInputPath(ctx, inputPaths, func(inputPath string) error {
err := engine.Flatten(ctx, ctx.Log(), inputPath)
if err != nil {
return fmt.Errorf("flatten '%s': %w", inputPath, err)
}
}
return nil
return nil
})
}
// defaultImageQuality is the JPEG quality applied by the image optimization
@@ -393,19 +393,27 @@ func OptimizeStub(ctx *api.Context, engine gotenberg.PdfEngine, optimizeImages b
return nil
}
for _, inputPath := range inputPaths {
return forEachInputPath(ctx, inputPaths, func(inputPath string) error {
err := engine.OptimizeImages(ctx, ctx.Log(), imageQuality, inputPath)
if err != nil {
return fmt.Errorf("optimize images of '%s': %w", inputPath, err)
}
}
return nil
return nil
})
}
// ConvertStub transforms a given PDF to the specified formats defined in
// [gotenberg.PdfFormats]. If no format, it does nothing and returns the input
// paths.
//
// This loop stays sequential on purpose. Convert is the one PDF engine method
// LibreOffice implements, and libreoffice-pdfengine is the default and only
// convert engine, so every iteration here drives the single soffice daemon. A
// LibreOffice instance is far too memory-hungry to run several of per
// container: the way to convert more documents at once is to scale Gotenberg
// containers, not to widen this loop. Do not route it through
// [forEachInputPath].
func ConvertStub(ctx *api.Context, engine gotenberg.PdfEngine, formats gotenberg.PdfFormats, inputPaths []string) ([]string, error) {
zeroValued := gotenberg.PdfFormats{}
if formats == zeroValued {
@@ -432,14 +440,14 @@ func WriteMetadataStub(ctx *api.Context, engine gotenberg.PdfEngine, metadata ma
return nil
}
for _, inputPath := range inputPaths {
return forEachInputPath(ctx, inputPaths, func(inputPath string) error {
err := engine.WriteMetadata(ctx, ctx.Log(), metadata, inputPath)
if err != nil {
return fmt.Errorf("write metadata into '%s': %w", inputPath, err)
}
}
return nil
return nil
})
}
// documentTitle returns the input PDF's Title metadata entry, falling back to
@@ -491,28 +499,33 @@ func WriteBookmarksStub(ctx *api.Context, engine gotenberg.PdfEngine, bookmarks
return nil
}
for _, inputPath := range inputPaths {
return forEachInputPath(ctx, inputPaths, func(inputPath string) error {
err := engine.WriteBookmarks(ctx, ctx.Log(), inputPath, b)
if err != nil {
return fmt.Errorf("write bookmarks into '%s': %w", inputPath, err)
}
}
return nil
})
case map[string][]gotenberg.Bookmark:
for _, inputPath := range inputPaths {
return forEachInputPath(ctx, inputPaths, func(inputPath string) error {
filename := ctx.OriginalFilename(inputPath)
if specificBookmarks, ok := b[filename]; ok {
err := engine.WriteBookmarks(ctx, ctx.Log(), inputPath, specificBookmarks)
if err != nil {
return fmt.Errorf("write bookmarks into '%s': %w", inputPath, err)
}
specificBookmarks, ok := b[filename]
if !ok {
return nil
}
}
err := engine.WriteBookmarks(ctx, ctx.Log(), inputPath, specificBookmarks)
if err != nil {
return fmt.Errorf("write bookmarks into '%s': %w", inputPath, err)
}
return nil
})
default:
// Should not happen.
return fmt.Errorf("bookmarks type '%T' not supported", bookmarks)
}
return nil
}
// FormDataPdfEmbeds extracts embedded file paths from form data.
@@ -537,14 +550,14 @@ func EmbedFilesMetadataStub(ctx *api.Context, engine gotenberg.PdfEngine, metada
return nil
}
for _, inputPath := range inputPaths {
return forEachInputPath(ctx, inputPaths, func(inputPath string) error {
err := engine.EmbedFilesMetadata(ctx, ctx.Log(), metadata, inputPath)
if err != nil {
return fmt.Errorf("set embeds metadata on PDF '%s': %w", inputPath, err)
}
}
return nil
return nil
})
}
// FormDataPdfFacturX extracts the Factur-X parameters and the invoice XML path
@@ -737,14 +750,14 @@ func InjectFacturXXMPStub(ctx *api.Context, engine gotenberg.PdfEngine, facturX
return nil
}
for _, inputPath := range inputPaths {
return forEachInputPath(ctx, inputPaths, func(inputPath string) error {
err := engine.InjectFacturXXMP(ctx, ctx.Log(), facturX, inputPath)
if err != nil {
return fmt.Errorf("inject Factur-X XMP into PDF '%s': %w", inputPath, err)
}
}
return nil
return nil
})
}
// FormDataPdfEncrypt extracts the encryption parameters and permissions from
@@ -783,14 +796,14 @@ func EncryptPdfStub(ctx *api.Context, engine gotenberg.PdfEngine, opts gotenberg
return nil
}
for _, inputPath := range inputPaths {
return forEachInputPath(ctx, inputPaths, func(inputPath string) error {
err := engine.Encrypt(ctx, ctx.Log(), inputPath, opts)
if err != nil {
return fmt.Errorf("encrypt PDF '%s': %w", inputPath, err)
}
}
return nil
return nil
})
}
// EmbedFilesStub embeds files into PDF files.
@@ -819,14 +832,14 @@ func EmbedFilesStub(ctx *api.Context, engine gotenberg.PdfEngine, embedPaths []s
resolvedPaths[i] = resolvedPath
}
for _, inputPath := range inputPaths {
return forEachInputPath(ctx, inputPaths, func(inputPath string) error {
err := engine.EmbedFiles(ctx, ctx.Log(), resolvedPaths, inputPath)
if err != nil {
return fmt.Errorf("embed files into PDF '%s': %w", inputPath, err)
}
}
return nil
return nil
})
}
// FormDataPdfStamps builds the ordered list of stamps from the repeated stamp
@@ -943,16 +956,23 @@ func bindStampOrWatermarkFiles(stamps []gotenberg.Stamp, files []string, kind st
// WatermarkStub applies each watermark to a list of PDF files, in order.
// Entries with no source are skipped, so an empty list does nothing.
func WatermarkStub(ctx *api.Context, engine gotenberg.PdfEngine, watermarks []gotenberg.Stamp, inputPaths []string) error {
// Watermarks stack on the same file, so the outer loop stays sequential;
// only the file dimension is parallel.
for _, watermark := range watermarks {
if watermark.Source == "" {
continue
}
for _, inputPath := range inputPaths {
err := engine.Watermark(ctx, ctx.Log(), inputPath, watermark)
if err != nil {
return fmt.Errorf("watermark '%s': %w", inputPath, err)
err := forEachInputPath(ctx, inputPaths, func(inputPath string) error {
errWatermark := engine.Watermark(ctx, ctx.Log(), inputPath, watermark)
if errWatermark != nil {
return fmt.Errorf("watermark '%s': %w", inputPath, errWatermark)
}
return nil
})
if err != nil {
return err
}
}
@@ -962,16 +982,23 @@ func WatermarkStub(ctx *api.Context, engine gotenberg.PdfEngine, watermarks []go
// StampStub applies each stamp to a list of PDF files, in order. Entries with
// no source are skipped, so an empty list does nothing.
func StampStub(ctx *api.Context, engine gotenberg.PdfEngine, stamps []gotenberg.Stamp, inputPaths []string) error {
// Stamps stack on the same file, so the outer loop stays sequential; only
// the file dimension is parallel.
for _, stamp := range stamps {
if stamp.Source == "" {
continue
}
for _, inputPath := range inputPaths {
err := engine.Stamp(ctx, ctx.Log(), inputPath, stamp)
if err != nil {
return fmt.Errorf("stamp '%s': %w", inputPath, err)
err := forEachInputPath(ctx, inputPaths, func(inputPath string) error {
errStamp := engine.Stamp(ctx, ctx.Log(), inputPath, stamp)
if errStamp != nil {
return fmt.Errorf("stamp '%s': %w", inputPath, errStamp)
}
return nil
})
if err != nil {
return err
}
}
@@ -1476,14 +1503,25 @@ func readMetadataRoute(engine gotenberg.PdfEngine) api.Route {
return fmt.Errorf("validate form data: %w", err)
}
res := make(map[string]map[string]any, len(inputPaths))
for _, inputPath := range inputPaths {
metadata, err := engine.ReadMetadata(ctx, ctx.Log(), inputPath)
if err != nil {
return fmt.Errorf("read metadata: %w", err)
// Collected per index, then folded into the map on this
// goroutine: a shared map cannot be written concurrently.
collected := make([]map[string]any, len(inputPaths))
err = forEachInputPathIndexed(ctx, inputPaths, func(i int, inputPath string) error {
metadata, errRead := engine.ReadMetadata(ctx, ctx.Log(), inputPath)
if errRead != nil {
return fmt.Errorf("read metadata: %w", errRead)
}
res[ctx.OriginalFilename(inputPath)] = metadata
collected[i] = metadata
return nil
})
if err != nil {
return err
}
res := make(map[string]map[string]any, len(inputPaths))
for i, inputPath := range inputPaths {
res[ctx.OriginalFilename(inputPath)] = collected[i]
}
err = c.JSON(http.StatusOK, res)
@@ -1554,14 +1592,25 @@ func readBookmarksRoute(engine gotenberg.PdfEngine) api.Route {
return fmt.Errorf("validate form data: %w", err)
}
res := make(map[string][]gotenberg.Bookmark, len(inputPaths))
for _, inputPath := range inputPaths {
bookmarks, err := engine.ReadBookmarks(ctx, ctx.Log(), inputPath)
if err != nil {
return fmt.Errorf("read bookmarks: %w", err)
// Collected per index, then folded into the map on this
// goroutine: a shared map cannot be written concurrently.
collected := make([][]gotenberg.Bookmark, len(inputPaths))
err = forEachInputPathIndexed(ctx, inputPaths, func(i int, inputPath string) error {
bookmarks, errRead := engine.ReadBookmarks(ctx, ctx.Log(), inputPath)
if errRead != nil {
return fmt.Errorf("read bookmarks: %w", errRead)
}
res[ctx.OriginalFilename(inputPath)] = bookmarks
collected[i] = bookmarks
return nil
})
if err != nil {
return err
}
res := make(map[string][]gotenberg.Bookmark, len(inputPaths))
for i, inputPath := range inputPaths {
res[ctx.OriginalFilename(inputPath)] = collected[i]
}
err = c.JSON(http.StatusOK, res)

View File

@@ -120,6 +120,30 @@ func (engine *QPdf) spanAttrs(extra ...attribute.KeyValue) []attribute.KeyValue
return append(attrs, extra...)
}
// qpdfPageRange matches the page range syntax qpdf accepts, and nothing else.
//
// A term is a page number, "z" for the last page, or "rN" counting from the
// end, optionally prefixed with "x" to exclude it. Terms combine into ranges
// with "-", ranges join with ",", and the whole thing takes an optional ":odd"
// or ":even".
var qpdfPageRange = regexp.MustCompile(`^x?(?:z|r\d+|\d+)(?:-x?(?:z|r\d+|\d+))?(?:,x?(?:z|r\d+|\d+)(?:-x?(?:z|r\d+|\d+))?)*(?::odd|:even)?$`)
// validateSplitSpan returns span when it is a qpdf page range.
//
// qpdf reads the argument after "--pages ." as either a page range or another
// source file, so a span carrying a path makes qpdf append that file's pages
// to the output. Other engines in the split chain accept spellings qpdf does
// not, such as pdfcpu's "2-end", so a span this rejects is not necessarily
// invalid. Returning an error lets the chain move on to an engine that
// understands it.
func validateSplitSpan(span string) error {
if qpdfPageRange.MatchString(span) {
return nil
}
return fmt.Errorf("split span '%s' is not a QPDF page range: %w", span, gotenberg.ErrPdfSplitModeNotSupported)
}
// Split splits a given PDF file.
func (engine *QPdf) Split(ctx context.Context, logger *slog.Logger, mode gotenberg.SplitMode, inputPath, outputDirPath string) ([]string, error) {
ctx, span := gotenberg.Tracer().Start(ctx, "qpdf.Split",
@@ -139,6 +163,12 @@ func (engine *QPdf) Split(ctx context.Context, logger *slog.Logger, mode gotenbe
span.SetStatus(codes.Error, err.Error())
return nil, err
}
err := validateSplitSpan(mode.Span)
if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
return nil, err
}
args = append(args, inputPath)
args = append(args, engine.globalArgs...)
args = append(args, "--pages", ".", mode.Span)

View File

@@ -3,9 +3,12 @@ package qpdf
import (
"context"
"encoding/json"
"errors"
"log/slog"
"os"
"testing"
"github.com/gotenberg/gotenberg/v8/pkg/gotenberg"
)
func TestStripQpdfStringPrefix(t *testing.T) {
@@ -270,3 +273,54 @@ func TestSetStreamSubtype(t *testing.T) {
}
})
}
func TestValidateSplitSpan(t *testing.T) {
for _, tc := range []struct {
span string
valid bool
}{
// qpdf page ranges.
{"1", true},
{"12", true},
{"1-5", true},
{"2-z", true},
{"z", true},
{"r1", true},
{"r3-r1", true},
{"1,3,5-9", true},
{"1-5,x3", true},
{"1-z:odd", true},
{"1-z:even", true},
// Other engines' spellings. Not valid here, so the chain moves on.
{"2-end", false},
{"2-", false},
{"foo", false},
// A span qpdf would read as a source file.
{"/tmp/secret.pdf", false},
{"secret.pdf", false},
{"./secret.pdf", false},
{"../../etc/hosts", false},
{"1,/tmp/secret.pdf", false},
{"1 /tmp/secret.pdf", false},
{"", false},
{"--password=x", false},
} {
t.Run(tc.span, func(t *testing.T) {
err := validateSplitSpan(tc.span)
if tc.valid && err != nil {
t.Fatalf("validateSplitSpan(%q) = %v, want nil", tc.span, err)
}
if !tc.valid {
if err == nil {
t.Fatalf("validateSplitSpan(%q) = nil, want an error", tc.span)
}
// The chain must be able to try the next engine.
if !errors.Is(err, gotenberg.ErrPdfSplitModeNotSupported) {
t.Fatalf("error %v does not wrap ErrPdfSplitModeNotSupported", err)
}
}
})
}
}

View File

@@ -31,10 +31,33 @@ type client struct {
extraHttpHeaders map[string]string
startTime time.Time
// deliveryTimeout bounds one delivery including retries. See
// [Webhook.deliveryTimeout].
deliveryTimeout time.Duration
client *retryablehttp.Client
logger *slog.Logger
}
// deliveryContext returns the context one delivery runs on.
//
// It keeps the values of ctx, so trace propagation and logging correlation
// survive, and replaces its cancellation with a fresh budget. Threading the
// conversion context straight through does not work: a delivery starts after
// the handler returned, so that deadline may already be spent and the callback
// would fail without a single attempt.
func (c client) deliveryContext(ctx context.Context) (context.Context, context.CancelFunc) {
timeout := c.deliveryTimeout
if timeout <= 0 {
// An unset budget would expire the delivery before its first attempt.
// [Webhook.deliveryTimeout] never returns a non-positive value, so this
// only guards a caller that builds a client without one.
timeout = minDeliveryTimeout
}
return context.WithTimeout(context.WithoutCancel(ctx), timeout)
}
// send call the webhook either to send the success response or the error response.
func (c client) send(ctx context.Context, body io.Reader, headers map[string]string, errored bool) error {
url := c.url
@@ -57,6 +80,9 @@ func (c client) send(ctx context.Context, body io.Reader, headers map[string]str
spanName = fmt.Sprintf("%s Webhook Error", method)
}
ctx, cancel := c.deliveryContext(ctx)
defer cancel()
tracer := gotenberg.Tracer()
ctx, span := tracer.Start(ctx, spanName,
trace.WithSpanKind(trace.SpanKindClient),
@@ -64,7 +90,10 @@ func (c client) send(ctx context.Context, body io.Reader, headers map[string]str
)
defer span.End()
req, err := retryablehttp.NewRequest(method, url, body)
// The request must carry ctx: retryablehttp.NewRequest builds on
// [context.Background], and its wait between attempts is a select on the
// request context, so a contextless request cannot be interrupted.
req, err := retryablehttp.NewRequestWithContext(ctx, method, url, body)
if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
@@ -114,13 +143,12 @@ func (c client) send(ctx context.Context, body io.Reader, headers map[string]str
return fmt.Errorf("send '%s' request to '%s': %w", method, url, err)
}
if resp.StatusCode >= http.StatusBadRequest {
err := fmt.Errorf("send '%s' request to '%s': got status: '%s'", method, url, resp.Status)
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
return err
}
// Registered before the status check below. [retryablehttp.Client.Do] hands
// back a live body for a status it does not retry, which is every 4xx but
// 429, so returning early without closing it strands the connection and the
// transport goroutines that serve it for the lifetime of the process. The
// transport is built per delivery in [gotenberg.NewOutboundHttpClient], so
// nothing reclaims it later either.
defer func() {
err := resp.Body.Close()
if err != nil {
@@ -128,6 +156,13 @@ func (c client) send(ctx context.Context, body io.Reader, headers map[string]str
}
}()
if resp.StatusCode >= http.StatusBadRequest {
err := fmt.Errorf("send '%s' request to '%s': got status: '%s'", method, url, resp.Status)
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
return err
}
// Last piece for calculating the latency.
finishTime := time.Now()
@@ -165,6 +200,9 @@ func (c client) sendEvent(ctx context.Context, correlationIdHeader, correlationI
return
}
ctx, cancel := c.deliveryContext(ctx)
defer cancel()
tracer := gotenberg.Tracer()
ctx, span := tracer.Start(ctx, "POST Webhook Event",
trace.WithSpanKind(trace.SpanKindClient),
@@ -172,7 +210,7 @@ func (c client) sendEvent(ctx context.Context, correlationIdHeader, correlationI
)
defer span.End()
req, err := retryablehttp.NewRequest(http.MethodPost, c.eventsUrl, b)
req, err := retryablehttp.NewRequestWithContext(ctx, http.MethodPost, c.eventsUrl, b)
if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())

View File

@@ -222,6 +222,7 @@ func webhookMiddleware(w *Webhook) api.Middleware {
eventsUrl: webhookEventsUrl,
extraHttpHeaders: extraHttpHeaders,
startTime: startTime,
deliveryTimeout: w.deliveryTimeout(),
client: &retryablehttp.Client{
HTTPClient: gotenberg.NewOutboundHttpClient(w.clientTimeout, w.allowList, w.denyList, w.enableEnvironmentProxy, ipOpts...),
@@ -230,7 +231,10 @@ func webhookMiddleware(w *Webhook) api.Middleware {
RetryWaitMax: w.retryMaxWait,
Logger: gotenberg.NewLeveledLogger(ctx.Log()),
CheckRetry: retryablehttp.DefaultRetryPolicy,
Backoff: retryablehttp.DefaultBackoff,
// Not DefaultBackoff: it returns a remote Retry-After
// verbatim, ignoring --webhook-retry-max-wait (env
// WEBHOOK_RETRY_MAX_WAIT).
Backoff: gotenberg.ClampedBackoff,
},
logger: ctx.Log(),
}

View File

@@ -42,7 +42,7 @@ func (w *Webhook) Descriptor() gotenberg.ModuleDescriptor {
FlagSet: func() *flag.FlagSet {
fs := flag.NewFlagSet("webhook", flag.ExitOnError)
fs.Bool("webhook-enable-sync-mode", false, "Enable synchronous mode for the webhook feature")
fs.StringSlice("webhook-allow-list", []string{}, "Set the allowed URLs for the webhook feature using regular expressions - supports multiple values")
fs.StringSlice("webhook-allow-list", []string{}, `Set the allowed URLs for the webhook feature using regular expressions - supports multiple values. A match bypasses --webhook-deny-private-ips (WEBHOOK_DENY_PRIVATE_IPS) and --webhook-deny-public-ips (WEBHOOK_DENY_PUBLIC_IPS), so terminate the host or the pattern also matches suffix hosts, for example ^https?://internal\.svc(:|/|$)`)
fs.StringSlice("webhook-deny-list", []string{}, "Set the denied URLs for the webhook feature using regular expressions - supports multiple values")
fs.Bool("webhook-deny-private-ips", false, "Reject webhook URLs whose host resolves to a non-public IP address (loopback, RFC1918, link-local, unique-local). Enable on deployments that accept untrusted webhook destinations to mitigate SSRF against internal services")
fs.Bool("webhook-deny-public-ips", false, "Reject webhook URLs whose host resolves to a public IP address. Enable on air-gapped or data-governed deployments to prevent callbacks from leaving a private network")
@@ -50,7 +50,7 @@ func (w *Webhook) Descriptor() gotenberg.ModuleDescriptor {
fs.Int("webhook-max-retry", 4, "Set the maximum number of retries for the webhook feature")
// Deprecated flags.
fs.StringSlice("webhook-error-allow-list", []string{}, "Set the allowed URLs in case of an error for the webhook feature using regular expressions - supports multiple values")
fs.StringSlice("webhook-error-allow-list", []string{}, `Set the allowed URLs in case of an error for the webhook feature using regular expressions - supports multiple values. A match bypasses --webhook-deny-private-ips (WEBHOOK_DENY_PRIVATE_IPS) and --webhook-deny-public-ips (WEBHOOK_DENY_PUBLIC_IPS), so terminate the host or the pattern also matches suffix hosts, for example ^https?://internal\.svc(:|/|$)`)
fs.StringSlice("webhook-error-deny-list", []string{}, "Set the denied URLs in case of an error for the webhook feature using regular expressions - supports multiple values")
err := fs.MarkDeprecated("webhook-error-allow-list", "use --webhook-allow-list instead")
if err != nil {
@@ -92,6 +92,27 @@ func (w *Webhook) Provision(ctx *gotenberg.Context) error {
return nil
}
// minDeliveryTimeout is the floor for [Webhook.deliveryTimeout], so that a
// deliberately tiny --webhook-client-timeout (env WEBHOOK_CLIENT_TIMEOUT) never
// leaves a delivery with no budget at all.
const minDeliveryTimeout = 1 * time.Second
// deliveryTimeout bounds one webhook delivery including its retries. It is the
// worst case a correctly behaving remote produces: one client timeout per
// attempt, plus the capped wait between attempts.
//
// A delivery runs after the handler returned, so it cannot borrow the
// conversion deadline. Without this budget it would have none, because
// [retryablehttp] builds its requests on [context.Background].
func (w *Webhook) deliveryTimeout() time.Duration {
timeout := w.clientTimeout*time.Duration(w.maxRetry+1) + w.retryMaxWait*time.Duration(w.maxRetry)
if timeout < minDeliveryTimeout {
return minDeliveryTimeout
}
return timeout
}
// Middlewares returns the middleware.
func (w *Webhook) Middlewares() ([]api.Middleware, error) {
if w.disable {

View File

@@ -0,0 +1,95 @@
package webhook
import (
"context"
"testing"
"time"
)
func TestWebhook_deliveryTimeout(t *testing.T) {
for _, tc := range []struct {
scenario string
clientTimeout time.Duration
maxRetry int
retryMaxWait time.Duration
want time.Duration
}{
{
scenario: "shipped defaults",
clientTimeout: 30 * time.Second,
maxRetry: 4,
retryMaxWait: 30 * time.Second,
want: 270 * time.Second,
},
{
scenario: "no retry is one client timeout",
clientTimeout: 30 * time.Second,
maxRetry: 0,
retryMaxWait: 30 * time.Second,
want: 30 * time.Second,
},
{
scenario: "a zero client timeout still gets a budget",
clientTimeout: 0,
maxRetry: 0,
retryMaxWait: 0,
want: minDeliveryTimeout,
},
} {
t.Run(tc.scenario, func(t *testing.T) {
w := &Webhook{
clientTimeout: tc.clientTimeout,
maxRetry: tc.maxRetry,
retryMaxWait: tc.retryMaxWait,
}
if got := w.deliveryTimeout(); got != tc.want {
t.Fatalf("deliveryTimeout() = %s, want %s", got, tc.want)
}
if w.deliveryTimeout() <= 0 {
t.Fatal("deliveryTimeout() must always be positive")
}
})
}
}
// A delivery must not inherit the conversion deadline. It runs after the
// handler returned, so that deadline is often already spent, which would fail
// the callback without a single attempt.
func TestClient_deliveryContext_IgnoresAnExpiredParentDeadline(t *testing.T) {
expired, cancelExpired := context.WithTimeout(context.Background(), -1*time.Second)
defer cancelExpired()
if expired.Err() == nil {
t.Fatal("expected the parent context to be expired")
}
c := client{deliveryTimeout: 30 * time.Second}
ctx, cancel := c.deliveryContext(expired)
defer cancel()
if ctx.Err() != nil {
t.Fatalf("delivery context inherited the expired parent: %v", ctx.Err())
}
deadline, ok := ctx.Deadline()
if !ok {
t.Fatal("delivery context has no deadline, so a delivery would be unbounded")
}
if remaining := time.Until(deadline); remaining <= 0 {
t.Fatalf("delivery budget = %s, want a positive value", remaining)
}
}
// The delivery context must still be bounded, so a hostile remote cannot hold
// the goroutine and its output file open indefinitely.
func TestClient_deliveryContext_IsBounded(t *testing.T) {
c := client{deliveryTimeout: 50 * time.Millisecond}
ctx, cancel := c.deliveryContext(context.Background())
defer cancel()
select {
case <-ctx.Done():
case <-time.After(5 * time.Second):
t.Fatal("delivery context never expired")
}
}

View File

@@ -589,7 +589,7 @@ Feature: /forms/chromium/convert/url
@chromium-ssrf
Scenario: POST /forms/chromium/convert/url (Redirect to a non-allow-listed address is re-filtered)
Given I have a Gotenberg container with the following environment variable(s):
| CHROMIUM_ALLOW_LIST | ^https?://host.docker.internal.* |
| CHROMIUM_ALLOW_LIST | ^https?://host\\.docker\\.internal(:[0-9]+)?/ |
Given I have a static server
When I make a "POST" request to Gotenberg at the "/forms/chromium/convert/url" endpoint with the following form data and header(s):
| url | http://host.docker.internal:%d/redirect-to-private | field |

View File

@@ -11,7 +11,6 @@ import (
"github.com/moby/moby/client"
"github.com/testcontainers/testcontainers-go"
"github.com/testcontainers/testcontainers-go/exec"
"github.com/testcontainers/testcontainers-go/network"
"github.com/testcontainers/testcontainers-go/wait"
)
@@ -21,11 +20,13 @@ import (
const testcontainersLabel = "org.testcontainers"
// PruneOrphanedNetworks removes dangling networks created by the test suite.
// Each scenario spins a dedicated network, and a failed container start can
// leak one before teardown records it. Leaked networks consume Docker's
// predefined address pools until none remain and every later scenario fails
// with "all predefined address pools have been fully subnetted". Call this
// before a run and between retries to reclaim the subnets.
// Scenarios no longer create one: the Gotenberg container is reached over its
// mapped port and the host-side helper over host.docker.internal, so the
// default bridge suffices. This stays as cheap insurance against networks
// leaked by an older suite version or an interrupted run, which consume
// Docker's predefined address pools until none remain and every later
// scenario fails with "all predefined address pools have been fully
// subnetted".
//
// Only unused networks bearing the testcontainers label are removed, so
// running containers and operator networks are never affected.
@@ -99,17 +100,18 @@ func applyDefaultEnv(env map[string]string) map[string]string {
return env
}
func startGotenbergContainer(ctx context.Context, env map[string]string) (*testcontainers.DockerNetwork, testcontainers.Container, error) {
// startGotenbergContainer starts a Gotenberg container on Docker's default
// bridge. No dedicated network is created: the suite addresses the container
// through container.Host plus its mapped port, and the container reaches the
// host-side webhook and static file server through the host.docker.internal
// alias below, so a per-scenario network would carry no traffic while still
// consuming one of Docker's predefined subnets.
func startGotenbergContainer(ctx context.Context, env map[string]string) (testcontainers.Container, error) {
ctx, cancel := context.WithTimeout(ctx, 2*time.Minute)
defer cancel()
env = applyDefaultEnv(env)
n, err := network.New(ctx)
if err != nil {
return nil, nil, fmt.Errorf("create Gotenberg container network: %w", err)
}
healthPath := "/health"
if env["API_ROOT_PATH"] != "" {
healthPath = fmt.Sprintf("%shealth", env["API_ROOT_PATH"])
@@ -122,7 +124,6 @@ func startGotenbergContainer(ctx context.Context, env map[string]string) (*testc
HostConfigModifier: func(hostConfig *container.HostConfig) {
hostConfig.ExtraHosts = []string{"host.docker.internal:host-gateway"}
},
Networks: []string{n.Name},
WaitingFor: wait.ForHTTP(healthPath),
Env: env,
}
@@ -148,19 +149,10 @@ func startGotenbergContainer(ctx context.Context, env map[string]string) (*testc
}
}
// The network is already created. The scenario teardown only
// removes networks it knows about, and the caller discards n on
// error, so remove it here to avoid leaking a subnet on every
// failed start. Leaked networks accumulate until Docker's address
// pools are fully subnetted and all later scenarios fail.
if errRemove := n.Remove(ctx); errRemove != nil {
err = fmt.Errorf("%w (also failed to remove network: %v)", err, errRemove)
}
return nil, nil, err
return nil, err
}
return n, c, nil
return c, nil
}
func execCommandInIntegrationToolsContainer(ctx context.Context, cmd []string, path string) (string, error) {

View File

@@ -86,16 +86,15 @@ func findScenarioLine(filePath, name string) int {
}
type scenario struct {
resp *httptest.ResponseRecorder
concurrentResps []*httptest.ResponseRecorder
probeResps []*httptest.ResponseRecorder
sequentialResps []*httptest.ResponseRecorder
workdir string
teststoreDir string
gotenbergContainer testcontainers.Container
gotenbergContainerNetwork *testcontainers.DockerNetwork
server *server
hostPort int
resp *httptest.ResponseRecorder
concurrentResps []*httptest.ResponseRecorder
probeResps []*httptest.ResponseRecorder
sequentialResps []*httptest.ResponseRecorder
workdir string
teststoreDir string
gotenbergContainer testcontainers.Container
server *server
hostPort int
}
func (s *scenario) reset(ctx context.Context) error {
@@ -123,11 +122,10 @@ func (s *scenario) reset(ctx context.Context) error {
}
func (s *scenario) iHaveADefaultGotenbergContainer(ctx context.Context) error {
n, c, err := startGotenbergContainer(ctx, nil)
c, err := startGotenbergContainer(ctx, nil)
if err != nil {
return fmt.Errorf("create Gotenberg container: %s", err)
}
s.gotenbergContainerNetwork = n
s.gotenbergContainer = c
return nil
}
@@ -137,11 +135,10 @@ func (s *scenario) iHaveAGotenbergContainerWithTheFollowingEnvironmentVariables(
for _, row := range envTable.Rows {
env[row.Cells[0].Value] = row.Cells[1].Value
}
n, c, err := startGotenbergContainer(ctx, env)
c, err := startGotenbergContainer(ctx, env)
if err != nil {
return fmt.Errorf("create Gotenberg container: %s", err)
}
s.gotenbergContainerNetwork = n
s.gotenbergContainer = c
return nil
}
@@ -1812,12 +1809,6 @@ func InitializeScenario(ctx *godog.ScenarioContext) {
return ctx, fmt.Errorf("terminate Gotenberg container: %w", errTerminate)
}
}
if s.gotenbergContainerNetwork != nil {
errRemove := s.gotenbergContainerNetwork.Remove(ctx)
if errRemove != nil {
return ctx, fmt.Errorf("remove Gotenberg container network: %w", errRemove)
}
}
return ctx, nil
})
ctx.After(func(ctx context.Context, sc *godog.Scenario, err error) (context.Context, error) {