mirror of
https://github.com/gotenberg/gotenberg.git
synced 2026-10-08 05:23:18 +01:00
Compare commits
54 Commits
v8.36.0
...
ff817f6cf8
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ff817f6cf8 | ||
|
|
ab18384a51 | ||
|
|
70783a01c6 | ||
|
|
2f9020455a | ||
|
|
8e850b1c93 | ||
|
|
430ed38cc5 | ||
|
|
623e93bf3e | ||
|
|
fe4fb9416d | ||
|
|
fb7536a526 | ||
|
|
b16ce08da7 | ||
|
|
fcfd590169 | ||
|
|
b39b8c76aa | ||
|
|
06ed58b6e7 | ||
|
|
34b7b4845e | ||
|
|
ab3832d9e5 | ||
|
|
cddaa0fa57 | ||
|
|
ade6a327a4 | ||
|
|
35ddf81812 | ||
|
|
1891a9ea68 | ||
|
|
e1e0a80883 | ||
|
|
c21ceacd4b | ||
|
|
ac825a2c03 | ||
|
|
2c9fa6b6ed | ||
|
|
ca8b45cd3a | ||
|
|
40cf48442f | ||
|
|
df3bac99ed | ||
|
|
17868b8c02 | ||
|
|
78284df590 | ||
|
|
8f415186d5 | ||
|
|
f675f78f77 | ||
|
|
83b01c2baa | ||
|
|
9a46fdd681 | ||
|
|
4de9b0f68b | ||
|
|
201e80b9d7 | ||
|
|
86a013b664 | ||
|
|
9c5acd7418 | ||
|
|
e29b7cb4f5 | ||
|
|
23d59f3133 | ||
|
|
8a0de7d5d7 | ||
|
|
3c691cbebc | ||
|
|
0e83f737b4 | ||
|
|
57b048c611 | ||
|
|
d79e174c6f | ||
|
|
88ddaed09b | ||
|
|
8944db131c | ||
|
|
676570074a | ||
|
|
11179cb271 | ||
|
|
c2a85f92d0 | ||
|
|
7dbff18e65 | ||
|
|
923e5f71eb | ||
|
|
334f859d95 | ||
|
|
0819514b7b | ||
|
|
c636a52666 | ||
|
|
0b16b0ab34 |
6
.github/actions/build-test-push/action.yml
vendored
6
.github/actions/build-test-push/action.yml
vendored
@@ -83,12 +83,16 @@ runs:
|
||||
INPUT_PLATFORM: ${{ inputs.platform }}
|
||||
INPUT_ALTERNATE_REPOSITORY: ${{ inputs.alternate_repository }}
|
||||
INPUT_DRY_RUN: ${{ inputs.dry_run }}
|
||||
# Exporting the build cache needs a registry login. Forks run without
|
||||
# credentials, so they import the cache but never export it.
|
||||
INPUT_CACHE_WRITABLE: ${{ inputs.docker_hub_username != '' }}
|
||||
run: |
|
||||
.github/actions/build-test-push/build.sh \
|
||||
--version "$INPUT_VERSION" \
|
||||
--platform "$INPUT_PLATFORM" \
|
||||
--alternate-repository "$INPUT_ALTERNATE_REPOSITORY" \
|
||||
--dry-run "$INPUT_DRY_RUN"
|
||||
--dry-run "$INPUT_DRY_RUN" \
|
||||
--cache-writable "$INPUT_CACHE_WRITABLE"
|
||||
|
||||
- name: Run integration tests
|
||||
if: inputs.skip_integrations_tests != 'true'
|
||||
|
||||
75
.github/actions/build-test-push/build.sh
vendored
75
.github/actions/build-test-push/build.sh
vendored
@@ -12,6 +12,7 @@ version=""
|
||||
platform=""
|
||||
alternate_repository=""
|
||||
dry_run=""
|
||||
cache_writable=""
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
@@ -31,6 +32,10 @@ while [[ $# -gt 0 ]]; do
|
||||
dry_run="$2"
|
||||
shift 2
|
||||
;;
|
||||
--cache-writable)
|
||||
cache_writable="$2"
|
||||
shift 2
|
||||
;;
|
||||
*)
|
||||
echo "Unknown option $1"
|
||||
exit 1
|
||||
@@ -44,11 +49,41 @@ echo
|
||||
echo "Gotenberg version: $version"
|
||||
echo "Target platform: $platform"
|
||||
|
||||
# The build cache lives under the canonical repository, captured before the
|
||||
# alternate-repository override below. Pull requests build into "snapshot", so
|
||||
# deriving the cache ref after the override would give them a cache namespace
|
||||
# of their own and they would never import what main published, which is the
|
||||
# population that benefits most.
|
||||
cache_image="$DOCKER_REGISTRY/$DOCKER_REPOSITORY"
|
||||
|
||||
# Layers are per-architecture, so each platform keeps its own cache manifest.
|
||||
cache_platform="${platform//\//-}"
|
||||
|
||||
# Layers running "apt-get upgrade" install whatever versions are current at
|
||||
# build time, and the packages are deliberately not pinned. A persistent cache
|
||||
# would turn those into hits and freeze security patches into a published
|
||||
# image until debian:13-slim itself changes digest. Keying them on the ISO week
|
||||
# bounds that staleness to seven days while leaving every build within a week
|
||||
# free to reuse the cache.
|
||||
apt_snapshot="$(date -u +%G-W%V)"
|
||||
|
||||
# Only a build that is not redirected to an alternate repository writes the
|
||||
# cache, so a pull request cannot make its own state the baseline for main.
|
||||
# Reading stays enabled everywhere, including forks, since the cache ref is
|
||||
# public and needs no credentials.
|
||||
cache_to_enabled="false"
|
||||
if [ "$cache_writable" = "true" ] && [ -z "$alternate_repository" ]; then
|
||||
cache_to_enabled="true"
|
||||
fi
|
||||
|
||||
if [ -n "$alternate_repository" ]; then
|
||||
DOCKER_REPOSITORY=$alternate_repository
|
||||
echo "⚠️ Using $alternate_repository for DOCKER_REPOSITORY"
|
||||
fi
|
||||
|
||||
echo "Build cache: $cache_image:buildcache-<target>-$cache_platform (write: $cache_to_enabled)"
|
||||
echo "APT snapshot: $apt_snapshot"
|
||||
|
||||
if [ "$dry_run" = "true" ]; then
|
||||
echo "🚧 Dry run"
|
||||
fi
|
||||
@@ -189,12 +224,36 @@ join() {
|
||||
echo "$*"
|
||||
}
|
||||
|
||||
# cache_flags echoes the buildx cache arguments for a build target. Each target
|
||||
# keeps its own manifest so that the Chromium and LibreOffice variants, which
|
||||
# branch from common-stage rather than from each other, do not overwrite one
|
||||
# another's entry.
|
||||
#
|
||||
# mode=max exports intermediate stages too, not just the final layers, which is
|
||||
# what makes the expensive apt and jlink stages reusable. type=registry, not
|
||||
# type=gha: the GitHub Actions cache is capped at 10 GB per repository and is
|
||||
# already carrying the Go and golangci-lint caches that the lint and test jobs
|
||||
# depend on. Multi-GB image layers across five platforms would evict them.
|
||||
cache_flags() {
|
||||
local target="$1"
|
||||
local ref="$cache_image:buildcache-$target-$cache_platform"
|
||||
local flags="--cache-from type=registry,ref=$ref"
|
||||
|
||||
if [ "$cache_to_enabled" = "true" ]; then
|
||||
flags="$flags --cache-to type=registry,ref=$ref,mode=max"
|
||||
fi
|
||||
|
||||
echo "$flags"
|
||||
}
|
||||
|
||||
no_arch_tag="$DOCKER_REGISTRY/$DOCKER_REPOSITORY:$version"
|
||||
|
||||
# Full variant.
|
||||
cmd="docker buildx build \
|
||||
--target gotenberg \
|
||||
--build-arg GOTENBERG_VERSION=$version \
|
||||
--build-arg APT_SNAPSHOT=$apt_snapshot \
|
||||
$(cache_flags gotenberg) \
|
||||
--platform $platform \
|
||||
--load \
|
||||
${tags_flags[*]} \
|
||||
@@ -207,6 +266,8 @@ run_cmd "$cmd"
|
||||
cmd="docker buildx build \
|
||||
--target gotenberg-chromium \
|
||||
--build-arg GOTENBERG_VERSION=$version \
|
||||
--build-arg APT_SNAPSHOT=$apt_snapshot \
|
||||
$(cache_flags gotenberg-chromium) \
|
||||
--platform $platform \
|
||||
--load \
|
||||
${tags_chromium_flags[*]} \
|
||||
@@ -218,6 +279,8 @@ run_cmd "$cmd"
|
||||
cmd="docker buildx build \
|
||||
--target gotenberg-libreoffice \
|
||||
--build-arg GOTENBERG_VERSION=$version \
|
||||
--build-arg APT_SNAPSHOT=$apt_snapshot \
|
||||
$(cache_flags gotenberg-libreoffice) \
|
||||
--platform $platform \
|
||||
--load \
|
||||
${tags_libreoffice_flags[*]} \
|
||||
@@ -230,6 +293,8 @@ if [ "$platform" = "linux/amd64" ]; then
|
||||
cmd="docker buildx build \
|
||||
--target gotenberg-cloudrun \
|
||||
--build-arg GOTENBERG_VERSION=$version \
|
||||
--build-arg APT_SNAPSHOT=$apt_snapshot \
|
||||
$(cache_flags gotenberg-cloudrun) \
|
||||
--platform $platform \
|
||||
--load \
|
||||
${tags_cloud_run_flags[*]} \
|
||||
@@ -240,6 +305,8 @@ if [ "$platform" = "linux/amd64" ]; then
|
||||
cmd="docker buildx build \
|
||||
--target gotenberg-cloudrun-chromium \
|
||||
--build-arg GOTENBERG_VERSION=$version \
|
||||
--build-arg APT_SNAPSHOT=$apt_snapshot \
|
||||
$(cache_flags gotenberg-cloudrun-chromium) \
|
||||
--platform $platform \
|
||||
--load \
|
||||
${tags_cloud_run_chromium_flags[*]} \
|
||||
@@ -250,6 +317,8 @@ if [ "$platform" = "linux/amd64" ]; then
|
||||
cmd="docker buildx build \
|
||||
--target gotenberg-cloudrun-libreoffice \
|
||||
--build-arg GOTENBERG_VERSION=$version \
|
||||
--build-arg APT_SNAPSHOT=$apt_snapshot \
|
||||
$(cache_flags gotenberg-cloudrun-libreoffice) \
|
||||
--platform $platform \
|
||||
--load \
|
||||
${tags_cloud_run_libreoffice_flags[*]} \
|
||||
@@ -263,6 +332,8 @@ if [ "$platform" = "linux/amd64" ] || [ "$platform" = "linux/arm64" ]; then
|
||||
cmd="docker buildx build \
|
||||
--target gotenberg-aws-lambda \
|
||||
--build-arg GOTENBERG_VERSION=$version \
|
||||
--build-arg APT_SNAPSHOT=$apt_snapshot \
|
||||
$(cache_flags gotenberg-aws-lambda) \
|
||||
--platform $platform \
|
||||
--load \
|
||||
${tags_aws_lambda_flags[*]} \
|
||||
@@ -273,6 +344,8 @@ if [ "$platform" = "linux/amd64" ] || [ "$platform" = "linux/arm64" ]; then
|
||||
cmd="docker buildx build \
|
||||
--target gotenberg-aws-lambda-chromium \
|
||||
--build-arg GOTENBERG_VERSION=$version \
|
||||
--build-arg APT_SNAPSHOT=$apt_snapshot \
|
||||
$(cache_flags gotenberg-aws-lambda-chromium) \
|
||||
--platform $platform \
|
||||
--load \
|
||||
${tags_aws_lambda_chromium_flags[*]} \
|
||||
@@ -283,6 +356,8 @@ if [ "$platform" = "linux/amd64" ] || [ "$platform" = "linux/arm64" ]; then
|
||||
cmd="docker buildx build \
|
||||
--target gotenberg-aws-lambda-libreoffice \
|
||||
--build-arg GOTENBERG_VERSION=$version \
|
||||
--build-arg APT_SNAPSHOT=$apt_snapshot \
|
||||
$(cache_flags gotenberg-aws-lambda-libreoffice) \
|
||||
--platform $platform \
|
||||
--load \
|
||||
${tags_aws_lambda_libreoffice_flags[*]} \
|
||||
|
||||
2
.github/workflows/continuous-integration.yml
vendored
2
.github/workflows/continuous-integration.yml
vendored
@@ -31,7 +31,7 @@ jobs:
|
||||
- name: Run linters
|
||||
uses: golangci/golangci-lint-action@v9
|
||||
with:
|
||||
version: v2.12.2
|
||||
version: v2.13.2
|
||||
|
||||
lint-prettier:
|
||||
name: Lint non-Golang codebase
|
||||
|
||||
23
Makefile
23
Makefile
@@ -33,11 +33,16 @@ API_OIDC_ISSUER=
|
||||
API_OIDC_AUDIENCE=
|
||||
API_OIDC_JWKS_URL=
|
||||
API_DOWNLOAD_FROM_ALLOW_LIST=
|
||||
API_DOWNLOAD_FROM_DENY_LIST=^https?://(10\.|172\.(1[6-9]|2[0-9]|3[01])\.|192\.168\.|169\.254\.|0\.0\.0\.0|127\.|localhost|\[::1\]|\[fd)
|
||||
# Empty, like the flag default since 8.32.0. A textual deny-list cannot
|
||||
# enumerate every way to write a private address, so *_DENY_PRIVATE_IPS is the
|
||||
# control to reach for. Left false here so local testing can reach the host.
|
||||
API_DOWNLOAD_FROM_DENY_LIST=
|
||||
API_DOWNLOAD_FROM_DENY_PRIVATE_IPS=false
|
||||
API_DOWNLOAD_FROM_DENY_PUBLIC_IPS=false
|
||||
API_DOWNLOAD_FROM_ENABLE_ENVIRONMENT_PROXY=false
|
||||
API_DOWNLOAD_FROM_MAX_RETRY=4
|
||||
API_DOWNLOAD_FROM_MAX_CONCURRENCY=10
|
||||
API_DOWNLOAD_FROM_MAX_ENTRIES=0
|
||||
API_DISABLE_DOWNLOAD_FROM=false
|
||||
API_DISABLE_HEALTH_CHECK_ROUTE_TELEMETRY=true
|
||||
API_DISABLE_ROOT_ROUTE_TELEMETRY=true
|
||||
@@ -82,6 +87,7 @@ LOG_STD_FORMAT=auto
|
||||
LOG_STD_ENABLE_GCP_FIELDS=false
|
||||
LOG_STD_LEVEL_CASE=lower
|
||||
PDFENGINES_DISABLE_ROUTES=false
|
||||
PDFENGINES_MAX_CONCURRENCY=1
|
||||
PDFENGINES_MERGE_ENGINES=qpdf,pdfcpu,pdftk
|
||||
PDFENGINES_SPLIT_ENGINES=pdfcpu,qpdf,pdftk
|
||||
PDFENGINES_FLATTEN_ENGINES=qpdf
|
||||
@@ -112,7 +118,8 @@ OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317
|
||||
OTEL_EXPORTER_OTLP_INSECURE=true
|
||||
WEBHOOK_ENABLE_SYNC_MODE=false
|
||||
WEBHOOK_ALLOW_LIST=
|
||||
WEBHOOK_DENY_LIST=^https?://(10\.|172\.(1[6-9]|2[0-9]|3[01])\.|192\.168\.|169\.254\.|0\.0\.0\.0|127\.|localhost|\[::1\]|\[fd)
|
||||
# See the note on API_DOWNLOAD_FROM_DENY_LIST.
|
||||
WEBHOOK_DENY_LIST=
|
||||
WEBHOOK_DENY_PRIVATE_IPS=false
|
||||
WEBHOOK_DENY_PUBLIC_IPS=false
|
||||
WEBHOOK_ENABLE_ENVIRONMENT_PROXY=false
|
||||
@@ -210,11 +217,19 @@ lint-prettier: ## Lint non-Golang codebase
|
||||
|
||||
.PHONY: lint-todo
|
||||
lint-todo: ## Find TODOs in Golang codebase
|
||||
golangci-lint run --no-config --disable-all --enable godox
|
||||
golangci-lint run --no-config --default=none --enable godox
|
||||
|
||||
# TODO: restore a plain "go fix ./..." once the errorsastype modernizer stops
|
||||
# rewriting this codebase into code that does not compile. Re-check by dropping
|
||||
# the flag and running "make fmt && make lint". Removing the analyzer upstream
|
||||
# makes go fix fail with "flag provided but not defined", so this cannot rot
|
||||
# silently.
|
||||
# errorsastype rewrites errors.As to errors.AsType[T] without checking that T
|
||||
# satisfies error, which breaks on api.HttpError since it does not embed
|
||||
# error. Still broken as of Go 1.27.1.
|
||||
.PHONY: fmt
|
||||
fmt: ## Format Golang codebase and "optimize" the dependencies
|
||||
go fix ./...
|
||||
go fix -errorsastype=false ./...
|
||||
golangci-lint fmt
|
||||
go mod tidy
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# ARG instructions do not create additional layers. Instead, next layers will
|
||||
# concatenate them. Also, we have to repeat ARG instructions in each build
|
||||
# stage that uses them.
|
||||
ARG GOLANG_VERSION=1.26.5
|
||||
ARG GOLANG_VERSION=1.27.1
|
||||
|
||||
# ----------------------------------------------
|
||||
# pdfcpu binary build stage
|
||||
@@ -59,7 +59,14 @@ RUN go build -o gotenberg -ldflags "-s -w -X 'github.com/gotenberg/gotenberg/v8/
|
||||
# ----------------------------------------------
|
||||
FROM debian:13-slim AS custom-jre-stage
|
||||
|
||||
RUN apt-get update -qq \
|
||||
# APT_SNAPSHOT busts every layer below it when CI rotates the value, weekly.
|
||||
# Without it a persistent build cache turns the unpinned "apt-get upgrade" into
|
||||
# a cache hit and the published image keeps shipping the package versions that
|
||||
# were current when the cache was first populated.
|
||||
ARG APT_SNAPSHOT=""
|
||||
|
||||
RUN echo "apt snapshot: $APT_SNAPSHOT" \
|
||||
&& apt-get update -qq \
|
||||
&& apt-get upgrade -yqq \
|
||||
&& DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends default-jdk-headless binutils
|
||||
|
||||
@@ -88,7 +95,7 @@ RUN apt-get update -qq \
|
||||
|
||||
WORKDIR /downloads
|
||||
|
||||
RUN curl -Ls https://raw.githubusercontent.com/gotenberg/unoconverter/v0.4.0/unoconv -o unoconverter \
|
||||
RUN curl -Ls https://raw.githubusercontent.com/gotenberg/unoconverter/v0.5.0/unoconv -o unoconverter \
|
||||
&& chmod +x unoconverter
|
||||
|
||||
RUN curl -o pdftk-all.jar "https://gitlab.com/api/v4/projects/5024297/packages/generic/pdftk-java/$PDFTK_VERSION/pdftk-all.jar" \
|
||||
@@ -114,9 +121,15 @@ FROM base-image-stage AS common-stage
|
||||
ARG GOTENBERG_USER_GID=1001
|
||||
ARG GOTENBERG_USER_UID=1001
|
||||
|
||||
# See the note on APT_SNAPSHOT in custom-jre-stage. Declaring it here covers
|
||||
# every "apt-get upgrade" in the gotenberg, gotenberg-chromium and
|
||||
# gotenberg-libreoffice targets too, since all three branch from this stage.
|
||||
ARG APT_SNAPSHOT=""
|
||||
|
||||
# Create a non-root user.
|
||||
# All processes in the Docker container will run with this dedicated user.
|
||||
RUN groupadd --gid "$GOTENBERG_USER_GID" gotenberg \
|
||||
RUN echo "apt snapshot: $APT_SNAPSHOT" \
|
||||
&& groupadd --gid "$GOTENBERG_USER_GID" gotenberg \
|
||||
&& useradd --uid "$GOTENBERG_USER_UID" --gid gotenberg --shell /bin/bash --home /home/gotenberg --no-create-home gotenberg \
|
||||
&& mkdir /home/gotenberg \
|
||||
&& chown gotenberg: /home/gotenberg
|
||||
@@ -259,7 +272,7 @@ RUN echo "deb http://deb.debian.org/debian trixie-backports main" >> /etc/apt/so
|
||||
hyphen-no hyphen-or hyphen-pa hyphen-pl hyphen-pt-br hyphen-pt-pt hyphen-ro hyphen-ru hyphen-sk hyphen-sl \
|
||||
hyphen-sr hyphen-sv hyphen-ta hyphen-te hyphen-th hyphen-uk hyphen-zu \
|
||||
&& DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends -t trixie-backports \
|
||||
libreoffice-writer libreoffice-calc libreoffice-impress libreoffice-draw python3-uno \
|
||||
libreoffice-writer libreoffice-calc libreoffice-impress libreoffice-draw libreoffice-math python3-uno \
|
||||
# unoconverter will look for the Python binary, which has to be at version 3.
|
||||
&& ln -s /usr/bin/python3 /usr/bin/python \
|
||||
# Cleanup.
|
||||
@@ -386,7 +399,7 @@ RUN echo "deb http://deb.debian.org/debian trixie-backports main" >> /etc/apt/so
|
||||
hyphen-no hyphen-or hyphen-pa hyphen-pl hyphen-pt-br hyphen-pt-pt hyphen-ro hyphen-ru hyphen-sk hyphen-sl \
|
||||
hyphen-sr hyphen-sv hyphen-ta hyphen-te hyphen-th hyphen-uk hyphen-zu \
|
||||
&& DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends -t trixie-backports \
|
||||
libreoffice-writer libreoffice-calc libreoffice-impress libreoffice-draw python3-uno \
|
||||
libreoffice-writer libreoffice-calc libreoffice-impress libreoffice-draw libreoffice-math python3-uno \
|
||||
# unoconverter will look for the Python binary, which has to be at version 3.
|
||||
&& ln -s /usr/bin/python3 /usr/bin/python \
|
||||
# Cleanup.
|
||||
@@ -485,7 +498,7 @@ FROM gotenberg AS gotenberg-aws-lambda
|
||||
|
||||
USER root
|
||||
|
||||
COPY --from=public.ecr.aws/awsguru/aws-lambda-adapter:0.9.1 /lambda-adapter /opt/extensions/lambda-adapter
|
||||
COPY --from=public.ecr.aws/awsguru/aws-lambda-adapter:1.0.1 /lambda-adapter /opt/extensions/lambda-adapter
|
||||
|
||||
ENV AWS_LWA_PORT=3000
|
||||
ENV AWS_LWA_READINESS_CHECK_PATH=/health
|
||||
@@ -503,7 +516,7 @@ FROM gotenberg-chromium AS gotenberg-aws-lambda-chromium
|
||||
|
||||
USER root
|
||||
|
||||
COPY --from=public.ecr.aws/awsguru/aws-lambda-adapter:0.9.1 /lambda-adapter /opt/extensions/lambda-adapter
|
||||
COPY --from=public.ecr.aws/awsguru/aws-lambda-adapter:1.0.1 /lambda-adapter /opt/extensions/lambda-adapter
|
||||
|
||||
ENV AWS_LWA_PORT=3000
|
||||
ENV AWS_LWA_READINESS_CHECK_PATH=/health
|
||||
@@ -521,7 +534,7 @@ FROM gotenberg-libreoffice AS gotenberg-aws-lambda-libreoffice
|
||||
|
||||
USER root
|
||||
|
||||
COPY --from=public.ecr.aws/awsguru/aws-lambda-adapter:0.9.1 /lambda-adapter /opt/extensions/lambda-adapter
|
||||
COPY --from=public.ecr.aws/awsguru/aws-lambda-adapter:1.0.1 /lambda-adapter /opt/extensions/lambda-adapter
|
||||
|
||||
ENV AWS_LWA_PORT=3000
|
||||
ENV AWS_LWA_READINESS_CHECK_PATH=/health
|
||||
|
||||
@@ -80,7 +80,7 @@ func Run() {
|
||||
// Override their values if the corresponding environment variables are
|
||||
// set.
|
||||
fs.VisitAll(func(f *flag.Flag) {
|
||||
envName := strings.ToUpper(strings.ReplaceAll(f.Name, "-", "_"))
|
||||
envName := gotenberg.EnvVarName(f.Name)
|
||||
val, ok := os.LookupEnv(envName)
|
||||
if !ok {
|
||||
return
|
||||
|
||||
@@ -35,6 +35,8 @@ services:
|
||||
- "--api-download-from-deny-public-ips=${API_DOWNLOAD_FROM_DENY_PUBLIC_IPS}"
|
||||
- "--api-download-from-enable-environment-proxy=${API_DOWNLOAD_FROM_ENABLE_ENVIRONMENT_PROXY}"
|
||||
- "--api-download-from-max-retry=${API_DOWNLOAD_FROM_MAX_RETRY}"
|
||||
- "--api-download-from-max-concurrency=${API_DOWNLOAD_FROM_MAX_CONCURRENCY}"
|
||||
- "--api-download-from-max-entries=${API_DOWNLOAD_FROM_MAX_ENTRIES}"
|
||||
- "--api-disable-download-from=${API_DISABLE_DOWNLOAD_FROM}"
|
||||
- "--api-disable-health-check-route-telemetry=${API_DISABLE_HEALTH_CHECK_ROUTE_TELEMETRY}"
|
||||
- "--api-disable-root-route-telemetry=${API_DISABLE_ROOT_ROUTE_TELEMETRY}"
|
||||
@@ -94,6 +96,7 @@ services:
|
||||
- "--pdfengines-embed-engines=${PDFENGINES_EMBED_ENGINES}"
|
||||
- "--pdfengines-embed-metadata-engines=${PDFENGINES_EMBED_METADATA_ENGINES}"
|
||||
- "--pdfengines-factur-x-engines=${PDFENGINES_FACTUR_X_ENGINES}"
|
||||
- "--pdfengines-max-concurrency=${PDFENGINES_MAX_CONCURRENCY}"
|
||||
- "--pdfengines-disable-routes=${PDFENGINES_DISABLE_ROUTES}"
|
||||
- "--prometheus-namespace=${PROMETHEUS_NAMESPACE}"
|
||||
- "--prometheus-collect-interval=${PROMETHEUS_COLLECT_INTERVAL}"
|
||||
|
||||
103
go.mod
103
go.mod
@@ -1,53 +1,50 @@
|
||||
module github.com/gotenberg/gotenberg/v8
|
||||
|
||||
go 1.26.5
|
||||
go 1.27.1
|
||||
|
||||
require (
|
||||
github.com/alexliesenfeld/health v0.8.1
|
||||
github.com/chromedp/cdproto v0.0.0-20250803210736-d308e07a266d // pinned with chromedp v0.14.2, see below
|
||||
github.com/chromedp/chromedp v0.14.2 // pinned: v0.15.x breaks the headless print-mode paint pipeline (rAF / ResizeObserver / IntersectionObserver stop firing, blank charts). See https://github.com/gotenberg/gotenberg/issues/1535.
|
||||
github.com/coreos/go-oidc/v3 v3.21.0
|
||||
github.com/cucumber/godog v0.16.0
|
||||
github.com/dlclark/regexp2 v1.12.0
|
||||
github.com/gomarkdown/markdown v0.0.0-20260725000948-8435af3f5984
|
||||
github.com/dlclark/regexp2/v2 v2.8.0
|
||||
github.com/gomarkdown/markdown v0.0.0-20260824154242-13c5cf49db8d
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/hashicorp/go-retryablehttp v0.7.8
|
||||
github.com/labstack/echo/v4 v4.15.4
|
||||
github.com/labstack/echo/v5 v5.3.1
|
||||
github.com/labstack/gommon v0.5.0
|
||||
github.com/mholt/archives v0.1.5
|
||||
github.com/microcosm-cc/bluemonday v1.0.27
|
||||
github.com/moby/moby/api v1.55.0
|
||||
github.com/moby/moby/client v0.5.1
|
||||
github.com/moby/moby/api v1.56.0
|
||||
github.com/moby/moby/client v0.6.0
|
||||
github.com/prometheus/client_golang v1.24.1
|
||||
github.com/shirou/gopsutil/v4 v4.26.7
|
||||
github.com/shirou/gopsutil/v4 v4.26.8
|
||||
github.com/spf13/pflag v1.0.10
|
||||
github.com/stretchr/testify v1.11.1
|
||||
github.com/stretchr/testify v1.12.1
|
||||
github.com/testcontainers/testcontainers-go v0.44.0
|
||||
go.opentelemetry.io/contrib/bridges/otelslog v0.20.0
|
||||
go.opentelemetry.io/contrib/exporters/autoexport v0.70.0
|
||||
go.opentelemetry.io/otel v1.45.0
|
||||
go.opentelemetry.io/otel/log v0.21.0
|
||||
go.opentelemetry.io/otel/metric v1.45.0
|
||||
go.opentelemetry.io/otel/sdk v1.45.0
|
||||
go.opentelemetry.io/otel/sdk/log v0.21.0
|
||||
go.opentelemetry.io/otel/sdk/metric v1.45.0
|
||||
go.opentelemetry.io/otel/trace v1.45.0
|
||||
go.opentelemetry.io/contrib/bridges/otelslog v0.20.1
|
||||
go.opentelemetry.io/contrib/exporters/autoexport v0.71.0
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0
|
||||
go.opentelemetry.io/otel v1.46.0
|
||||
go.opentelemetry.io/otel/log v0.22.0
|
||||
go.opentelemetry.io/otel/metric v1.46.0
|
||||
go.opentelemetry.io/otel/sdk v1.46.0
|
||||
go.opentelemetry.io/otel/sdk/log v0.22.0
|
||||
go.opentelemetry.io/otel/sdk/metric v1.46.0
|
||||
go.opentelemetry.io/otel/trace v1.46.0
|
||||
golang.org/x/net v0.58.0
|
||||
golang.org/x/sync v0.22.0
|
||||
golang.org/x/sync v0.23.0
|
||||
golang.org/x/term v0.45.0
|
||||
golang.org/x/text v0.41.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/coreos/go-oidc/v3 v3.20.0
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0
|
||||
)
|
||||
|
||||
require (
|
||||
dario.cat/mergo v1.0.2 // indirect
|
||||
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect
|
||||
github.com/Microsoft/go-winio v0.6.2 // indirect
|
||||
github.com/STARRY-S/zip v0.2.3 // indirect
|
||||
github.com/andybalholm/brotli v1.2.2 // indirect
|
||||
github.com/andybalholm/brotli v1.2.3 // indirect
|
||||
github.com/aymerick/douceur v0.2.0 // indirect
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/bodgit/plumbing v1.3.0 // indirect
|
||||
@@ -64,15 +61,14 @@ require (
|
||||
github.com/cpuguy83/dockercfg v0.3.2 // indirect
|
||||
github.com/cucumber/gherkin/go/v42 v42.0.1 // indirect
|
||||
github.com/cucumber/messages/go/v34 v34.2.1 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/distribution/reference v0.6.0 // indirect
|
||||
github.com/docker/go-connections v0.8.1 // indirect
|
||||
github.com/docker/go-units v0.5.0 // indirect
|
||||
github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 // indirect
|
||||
github.com/ebitengine/purego v0.10.2 // indirect
|
||||
github.com/ebitengine/purego v0.11.0 // indirect
|
||||
github.com/felixge/httpsnoop v1.1.0 // indirect
|
||||
github.com/go-jose/go-jose/v4 v4.1.4 // indirect
|
||||
github.com/go-json-experiment/json v0.0.0-20260623181947-01eb4420fa68 // indirect
|
||||
github.com/go-jose/go-jose/v4 v4.1.5 // indirect
|
||||
github.com/go-json-experiment/json v0.0.0-20260820222146-c27c302e5fc3 // indirect
|
||||
github.com/go-logr/logr v1.4.4 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/go-ole/go-ole v1.3.0 // indirect
|
||||
@@ -86,7 +82,7 @@ require (
|
||||
github.com/hashicorp/go-memdb v1.3.5 // indirect
|
||||
github.com/hashicorp/golang-lru v1.0.2 // indirect
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
|
||||
github.com/klauspost/compress v1.19.2 // indirect
|
||||
github.com/klauspost/compress v1.20.0 // indirect
|
||||
github.com/klauspost/pgzip v1.2.6 // indirect
|
||||
github.com/lufia/plan9stats v0.0.0-20260802145828-341c2f0c90b5 // indirect
|
||||
github.com/magiconair/properties v1.18.11 // indirect
|
||||
@@ -102,48 +98,45 @@ require (
|
||||
github.com/moby/sys/userns v0.2.0 // indirect
|
||||
github.com/moby/term v0.5.2 // indirect
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
||||
github.com/nwaples/rardecode/v2 v2.3.0 // indirect
|
||||
github.com/nwaples/rardecode/v2 v2.4.1 // indirect
|
||||
github.com/opencontainers/go-digest v1.0.0 // indirect
|
||||
github.com/opencontainers/image-spec v1.1.1 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.28 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.29 // indirect
|
||||
github.com/power-devops/perfstat v0.0.0-20260805114148-88456608a4f6 // indirect
|
||||
github.com/prometheus/client_model v0.6.2 // indirect
|
||||
github.com/prometheus/common v0.70.1 // indirect
|
||||
github.com/prometheus/client_model v0.6.3 // indirect
|
||||
github.com/prometheus/common v0.71.0 // indirect
|
||||
github.com/prometheus/otlptranslator v1.0.0 // indirect
|
||||
github.com/prometheus/procfs v0.21.1 // indirect
|
||||
github.com/sirupsen/logrus v1.10.0 // indirect
|
||||
github.com/prometheus/procfs v0.22.0 // indirect
|
||||
github.com/sirupsen/logrus v1.10.2 // indirect
|
||||
github.com/sorairolake/lzip-go v0.3.8 // indirect
|
||||
github.com/spf13/afero v1.15.0 // indirect
|
||||
github.com/stangelandcl/ppmd v0.1.1 // indirect
|
||||
github.com/tklauser/go-sysconf v0.4.0 // indirect
|
||||
github.com/tklauser/numcpus v0.12.0 // indirect
|
||||
github.com/ulikunitz/xz v0.5.16 // indirect
|
||||
github.com/valyala/bytebufferpool v1.0.0 // indirect
|
||||
github.com/valyala/fasttemplate v1.2.2 // indirect
|
||||
github.com/yusufpapurcu/wmi v1.2.4 // indirect
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
|
||||
go.opentelemetry.io/contrib/bridges/prometheus v0.70.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.21.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.21.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.45.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.45.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.45.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.45.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/prometheus v0.67.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.21.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.45.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.45.0 // indirect
|
||||
go.opentelemetry.io/contrib/bridges/prometheus v0.71.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.22.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.22.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.46.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.46.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.46.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.46.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.46.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/prometheus v0.68.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.22.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.46.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.46.0 // indirect
|
||||
go.opentelemetry.io/proto/otlp v1.11.0 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.5 // indirect
|
||||
go4.org v0.0.0-20260112195520-a5071408f32f // indirect
|
||||
golang.org/x/crypto v0.55.0 // indirect
|
||||
golang.org/x/crypto v0.56.0 // indirect
|
||||
golang.org/x/oauth2 v0.36.0 // indirect
|
||||
golang.org/x/sys v0.47.0 // indirect
|
||||
golang.org/x/time v0.15.0 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260810153831-ec0a7760b754 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260810153831-ec0a7760b754 // indirect
|
||||
google.golang.org/grpc v1.83.0 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260831171406-18b4a7587f8a // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260831171406-18b4a7587f8a // indirect
|
||||
google.golang.org/grpc v1.83.2 // indirect
|
||||
google.golang.org/protobuf v1.36.12 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
)
|
||||
|
||||
209
go.sum
209
go.sum
@@ -10,8 +10,8 @@ github.com/STARRY-S/zip v0.2.3 h1:luE4dMvRPDOWQdeDdUxUoZkzUIpTccdKdhHHsQJ1fm4=
|
||||
github.com/STARRY-S/zip v0.2.3/go.mod h1:lqJ9JdeRipyOQJrYSOtpNAiaesFO6zVDsE8GIGFaoSk=
|
||||
github.com/alexliesenfeld/health v0.8.1 h1:wdE3vt+cbJotiR8DGDBZPKHDFoJbAoWEfQTcqrmedUg=
|
||||
github.com/alexliesenfeld/health v0.8.1/go.mod h1:TfNP0f+9WQVWMQRzvMUjlws4ceXKEL3WR+6Hp95HUFc=
|
||||
github.com/andybalholm/brotli v1.2.2 h1:HzTuoo2ErYQqf5qvcJInB8uvqSVxRttzkFexPWtnceM=
|
||||
github.com/andybalholm/brotli v1.2.2/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
|
||||
github.com/andybalholm/brotli v1.2.3 h1:8H1qwOkl2LPfjf3YezB90JnCliZb6SInJ/OJkEbA5NQ=
|
||||
github.com/andybalholm/brotli v1.2.3/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
|
||||
github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
|
||||
github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
|
||||
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
||||
@@ -42,8 +42,8 @@ github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=
|
||||
github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo=
|
||||
github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A=
|
||||
github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw=
|
||||
github.com/coreos/go-oidc/v3 v3.20.0 h1:EtE0WIBHk03N+DqGkY4+UONzzZHk7amKt6IyNd7OsZE=
|
||||
github.com/coreos/go-oidc/v3 v3.20.0/go.mod h1:DYCf24+ncYi+XkIH97GY1+dqoRlbaSI26KVTCI9SrY4=
|
||||
github.com/coreos/go-oidc/v3 v3.21.0 h1:wZo4Q9Pum8dYEj0eMUPrqR+kvuGkeUplbLpNCkBqoWM=
|
||||
github.com/coreos/go-oidc/v3 v3.21.0/go.mod h1:DYCf24+ncYi+XkIH97GY1+dqoRlbaSI26KVTCI9SrY4=
|
||||
github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA=
|
||||
github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc=
|
||||
github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s=
|
||||
@@ -55,12 +55,11 @@ github.com/cucumber/godog v0.16.0/go.mod h1:EDUX9yCqANK+GpbftMDeu61sUDtdLuo1JJgX
|
||||
github.com/cucumber/messages/go/v34 v34.2.1 h1:qBPEl+HhNJuRX8Kjaw1Pm60KOODZf2/4WQAi0SR/neE=
|
||||
github.com/cucumber/messages/go/v34 v34.2.1/go.mod h1:LYUPjqlTS1kS0pdkdf6sS5uirnjwiIzEGyXPezXNhL8=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
|
||||
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
|
||||
github.com/dlclark/regexp2 v1.12.0 h1:0j4c5qQmnC6XOWNjP3PIXURXN2gWx76rd3KvgdPkCz8=
|
||||
github.com/dlclark/regexp2 v1.12.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
|
||||
github.com/dlclark/regexp2/v2 v2.8.0 h1:CekDhPLGfm+GAJmPVFIG+5dqMIQPkyHJll7BbdneDfw=
|
||||
github.com/dlclark/regexp2/v2 v2.8.0/go.mod h1:avUrQvPaLz2DrFNHJF0taWAFFX2C1GMSSoeiqFjcBmU=
|
||||
github.com/docker/go-connections v0.8.1 h1:JibmG5hULs5qXSr/cp/w3Pw5fZuStt4MOHMUExb29/M=
|
||||
github.com/docker/go-connections v0.8.1/go.mod h1:no1qkHdjq7kLMGUXYAduOhYPSJxxvgWBh7ogVvptn3Q=
|
||||
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
|
||||
@@ -68,16 +67,16 @@ github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDD
|
||||
github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 h1:2tV76y6Q9BB+NEBasnqvs7e49aEBFI8ejC89PSnWH+4=
|
||||
github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707/go.mod h1:qssHWj60/X5sZFNxpG4HBPDHVqxNm4DfnCKgrbZOT+s=
|
||||
github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY=
|
||||
github.com/ebitengine/purego v0.10.2 h1:W809HbnvzAxgdm+aOvlSekrM16wGCdT/e76+9tS7gzE=
|
||||
github.com/ebitengine/purego v0.10.2/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
|
||||
github.com/ebitengine/purego v0.11.0 h1:jhp/D+Nyv7UUW8HAcmcjt2N2rYrYi9m3SL21k0Ua/NI=
|
||||
github.com/ebitengine/purego v0.11.0/go.mod h1:DCHPP08djqhNSoTfImcnHYQRZmd0qhakvrozqaEYhGQ=
|
||||
github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
|
||||
github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
|
||||
github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc=
|
||||
github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE=
|
||||
github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA=
|
||||
github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
|
||||
github.com/go-json-experiment/json v0.0.0-20260623181947-01eb4420fa68 h1:KZaTBSyshWX3MP5jukJcNSuXDQTO+rNpt0J564dX/eg=
|
||||
github.com/go-json-experiment/json v0.0.0-20260623181947-01eb4420fa68/go.mod h1:tphK2c80bpPhMOI4v6bIc2xWywPfbqi1Z06+RcrMkDg=
|
||||
github.com/go-jose/go-jose/v4 v4.1.5 h1:RjgjO2LOtWOJKUC5wpwY9LR3B3vwVAz6JS2YHfYU6eA=
|
||||
github.com/go-jose/go-jose/v4 v4.1.5/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
|
||||
github.com/go-json-experiment/json v0.0.0-20260820222146-c27c302e5fc3 h1:UADEEmDKgfXbtnGJZ97beY5XLo9ZechG1nlU4KnRrkE=
|
||||
github.com/go-json-experiment/json v0.0.0-20260820222146-c27c302e5fc3/go.mod h1:tphK2c80bpPhMOI4v6bIc2xWywPfbqi1Z06+RcrMkDg=
|
||||
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
|
||||
github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8=
|
||||
github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
|
||||
@@ -94,8 +93,8 @@ github.com/gobwas/ws v1.4.0 h1:CTaoG1tojrh4ucGPcoJFiAQUAsEWekEWvLy7GsVNqGs=
|
||||
github.com/gobwas/ws v1.4.0/go.mod h1:G3gNqMNtPppf5XUz7O4shetPpcZ1VJ7zt18dlUeakrc=
|
||||
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
|
||||
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
|
||||
github.com/gomarkdown/markdown v0.0.0-20260725000948-8435af3f5984 h1:6DE2PprLLZelQJu7AeebZLxjcENwC1plRZHDvd5uMCU=
|
||||
github.com/gomarkdown/markdown v0.0.0-20260725000948-8435af3f5984/go.mod h1:JDGcbDT52eL4fju3sZ4TeHGsQwhG9nbDV21aMyhwPoA=
|
||||
github.com/gomarkdown/markdown v0.0.0-20260824154242-13c5cf49db8d h1:8VtgBGEPLZ2Yn0Fuh6Pwmy3qF6indeaqy8mrBMbUKRQ=
|
||||
github.com/gomarkdown/markdown v0.0.0-20260824154242-13c5cf49db8d/go.mod h1:JDGcbDT52eL4fju3sZ4TeHGsQwhG9nbDV21aMyhwPoA=
|
||||
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
@@ -125,19 +124,15 @@ github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uG
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
|
||||
github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
|
||||
github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8=
|
||||
github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
||||
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
|
||||
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
|
||||
github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU=
|
||||
github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
|
||||
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
|
||||
github.com/labstack/echo/v4 v4.15.4 h1:DL45vVYa+BWE+XuW+zZNd9H0YEdZ80UAWJGcTVW4EVs=
|
||||
github.com/labstack/echo/v4 v4.15.4/go.mod h1:CuMetKIRwsuO/qlAgMq+KTAalwGoB/h4tC+yPdrTj1g=
|
||||
github.com/labstack/echo/v5 v5.3.1 h1:75maCxkQVGualckLc/5s/ihgpH1a1Dc6AuGWNVNs6bw=
|
||||
github.com/labstack/echo/v5 v5.3.1/go.mod h1:4iEGNQiPPZnkfYpNR/L6fINd3NLiGWUD5+eBotFALas=
|
||||
github.com/labstack/gommon v0.5.0 h1:6VSQ2NOzsnEJ5W6+84E0RbcaDDmgB6NIAzWCczTEe6c=
|
||||
github.com/labstack/gommon v0.5.0/go.mod h1:Rzlg7HHy1maLfzBYGg9NZcVuz1sA68HHhLjhcEllYE0=
|
||||
github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80 h1:6Yzfa6GP0rIo/kULo2bwGEkFvCePZ3qHDDTC3/J9Swo=
|
||||
@@ -162,10 +157,10 @@ github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3N
|
||||
github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
|
||||
github.com/moby/go-archive v0.3.3 h1:OxxR9paxsluYi+zDUEXTTaIxtkK3viymW+Ka7vRhhME=
|
||||
github.com/moby/go-archive v0.3.3/go.mod h1:Npdv43fFqlhZW7Xo8fbm3ZMYFvAGNviUPqX21VERbcE=
|
||||
github.com/moby/moby/api v1.55.0 h1:2/sexvQyqIWS8pRSCFddBfpW2qE7vR7FCL+vN8pxwMc=
|
||||
github.com/moby/moby/api v1.55.0/go.mod h1:+RQ6wluLwtYaTd1WnPLykIDPekkuyD/ROWQClE83pzs=
|
||||
github.com/moby/moby/client v0.5.1 h1:tYNaJno4c0HXz12y5BiqEDy0rVTYkWzI26lGvnTMiJw=
|
||||
github.com/moby/moby/client v0.5.1/go.mod h1:odLstlZ6uSnfvAgVxMpvgmb8SUdd+siH2T0GBuxVAlM=
|
||||
github.com/moby/moby/api v1.56.0 h1:GQzua3NA599ASSIICx0iFgiJeO9YkdDARvQsm23ZZuQ=
|
||||
github.com/moby/moby/api v1.56.0/go.mod h1:sZ+THbVWkjOmBPPfbnzdD/G1LuIexWhqlSHHPTDQ1Uk=
|
||||
github.com/moby/moby/client v0.6.0 h1:AJjEB21QPbXSXjDsZorFBoDZPhMrfbpaPLgSMAW9Bgs=
|
||||
github.com/moby/moby/client v0.6.0/go.mod h1:OCo00wNRyA3m4lmJ228W3JbyCN4ZNNYjpOXiJydBdcQ=
|
||||
github.com/moby/patternmatcher v0.6.1 h1:qlhtafmr6kgMIJjKJMDmMWq7WLkKIo23hsrpR3x084U=
|
||||
github.com/moby/patternmatcher v0.6.1/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc=
|
||||
github.com/moby/sys/mount v0.3.5 h1:eS3fsZTjHaBihwjp4/+5Z3jxqLXYsbwxqpVSfFv3M00=
|
||||
@@ -182,36 +177,33 @@ github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ=
|
||||
github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
|
||||
github.com/nwaples/rardecode/v2 v2.3.0 h1:CtgyxWm8ClLcSh1u4M58fOz6lmeb/j4V7KpaEi/6UtM=
|
||||
github.com/nwaples/rardecode/v2 v2.3.0/go.mod h1:7uz379lSxPe6j9nvzxUZ+n7mnJNgjsRNb6IbvGVHRmw=
|
||||
github.com/nwaples/rardecode/v2 v2.4.1 h1:F7zNW2LdAuuBThHWXQaiFUGVD/sef299NfWSB1nHAl4=
|
||||
github.com/nwaples/rardecode/v2 v2.4.1/go.mod h1:7uz379lSxPe6j9nvzxUZ+n7mnJNgjsRNb6IbvGVHRmw=
|
||||
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
|
||||
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
|
||||
github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040=
|
||||
github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M=
|
||||
github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw=
|
||||
github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0=
|
||||
github.com/pierrec/lz4/v4 v4.1.28 h1:pPEPwRJ4kybBTfGt28q7lQsRJQHhC08axprdLD5Ppio=
|
||||
github.com/pierrec/lz4/v4 v4.1.28/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pierrec/lz4/v4 v4.1.29 h1:CDQY6qZOLI4DW0Nx6R1vRrifrCeQHnNXkMb0hZWXFjg=
|
||||
github.com/pierrec/lz4/v4 v4.1.29/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/power-devops/perfstat v0.0.0-20260805114148-88456608a4f6 h1:jL3a8soXdzuTCcRnKhOmtcsVOObdDTFf4O2B403HPRU=
|
||||
github.com/power-devops/perfstat v0.0.0-20260805114148-88456608a4f6/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
|
||||
github.com/prometheus/client_golang v1.24.1 h1:JnJkREXzWxUdCuPFpIWZiPispT9xVV59uiuyR2bPlnU=
|
||||
github.com/prometheus/client_golang v1.24.1/go.mod h1:F+oSRECHg4sse5ucfYpYDeIv/hu68Zo0uoHKetWnzcE=
|
||||
github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
|
||||
github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
|
||||
github.com/prometheus/common v0.70.1 h1:1HvjP4D5oL3t8RsPlwxA9onvvStjtIHYE5XuuwOi/PY=
|
||||
github.com/prometheus/common v0.70.1/go.mod h1:VdFUQDMZK3VLkurFUVhia6uys/0suUp86TJz5qbJRhc=
|
||||
github.com/prometheus/client_model v0.6.3 h1:O0jaTVAYNxTHYInEPFJt5I3+sN8zqBtVMPTB1qyxiEo=
|
||||
github.com/prometheus/client_model v0.6.3/go.mod h1:gpN5P9S7Rr6Yr92PiQ+Ixvhf6JZEkF1dnxsYL2aPBEM=
|
||||
github.com/prometheus/common v0.71.0 h1:9KDAKb7Mj3HEVKyFCK6Dc/HIwlBzZIN2l7/lrHl3KK8=
|
||||
github.com/prometheus/common v0.71.0/go.mod h1:CLJ5H8TEsGX8bl31BdMkfhIZ+QmZ9tBPPotUxUbfcmk=
|
||||
github.com/prometheus/otlptranslator v1.0.0 h1:s0LJW/iN9dkIH+EnhiD3BlkkP5QVIUVEoIwkU+A6qos=
|
||||
github.com/prometheus/otlptranslator v1.0.0/go.mod h1:vRYWnXvI6aWGpsdY/mOT/cbeVRBlPWtBNDb7kGR3uKM=
|
||||
github.com/prometheus/procfs v0.21.1 h1:GljZCt+zSTS+NZq88cyQ1LjZ+RCHp3uVuabBWA5+OJI=
|
||||
github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY=
|
||||
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
|
||||
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
|
||||
github.com/shirou/gopsutil/v4 v4.26.7 h1:IXzpHz/dkMRYAhKkOXr1HB6SuzWU3eoyyeWe7g3bNZc=
|
||||
github.com/shirou/gopsutil/v4 v4.26.7/go.mod h1:5O9FjBiXoTDFatIWjZZosqj4pV0DRtLx598xGbBehzM=
|
||||
github.com/sirupsen/logrus v1.10.0 h1:T8MxJJXVZkfcC5zSRMRAg2F8+lxjmUCGGWPzFxO+Msc=
|
||||
github.com/sirupsen/logrus v1.10.0/go.mod h1:FXZFonkDAnFozmO+5hGAFvB0Yg9/j2SIhA/QuIkP180=
|
||||
github.com/prometheus/procfs v0.22.0 h1:6q9+/JL9IKAPbCmBrv9n5O5Ty3NKnciV5X7YGw0oics=
|
||||
github.com/prometheus/procfs v0.22.0/go.mod h1:CvmFr/GVhIjIvWJZW3tgkODBQMRIf0EyWMQLHCHab58=
|
||||
github.com/shirou/gopsutil/v4 v4.26.8 h1:YQMTF/1J50B5+Y0vlo1eDRf5DoR7Gk69hY+8wjYkQeo=
|
||||
github.com/shirou/gopsutil/v4 v4.26.8/go.mod h1:5O9FjBiXoTDFatIWjZZosqj4pV0DRtLx598xGbBehzM=
|
||||
github.com/sirupsen/logrus v1.10.2 h1:G2SED73/qrAu6YwbdxOD6peLkCBI3z7L+ykJFTXJBBo=
|
||||
github.com/sirupsen/logrus v1.10.2/go.mod h1:SLEg8TqYulVKKfIGHldVp2K2aYz2DKSVBq4g/H5bR7Q=
|
||||
github.com/sorairolake/lzip-go v0.3.8 h1:j5Q2313INdTA80ureWYRhX+1K78mUXfMoPZCw/ivWik=
|
||||
github.com/sorairolake/lzip-go v0.3.8/go.mod h1:JcBqGMV0frlxwrsE9sMWXDjqn3EeVf0/54YPsw66qkU=
|
||||
github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I=
|
||||
@@ -228,8 +220,8 @@ github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+Q
|
||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE=
|
||||
github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg=
|
||||
github.com/testcontainers/testcontainers-go v0.44.0 h1:/Fwh6HY1mIikhnm9e7HwoxGycx0lzRAE0f5VQpjFxzI=
|
||||
github.com/testcontainers/testcontainers-go v0.44.0/go.mod h1:IcnwQrYTO86xHXu5bvMaBH7ATlbS3Qn1M1QWW3c66rE=
|
||||
github.com/tklauser/go-sysconf v0.4.0 h1:7H0uAN+7RkwWRaxhYXDLqa5V3LPrJeV8wmD9dRUgPQU=
|
||||
@@ -239,80 +231,78 @@ github.com/tklauser/numcpus v0.12.0/go.mod h1:ABHeXzJnr/qqwguhClkZKT1/8VABcYrsyU
|
||||
github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
|
||||
github.com/ulikunitz/xz v0.5.16 h1:ld6NyySjx5lowVKwJvMRLnW5nxKX/xnpSiFYZ/Lxur0=
|
||||
github.com/ulikunitz/xz v0.5.16/go.mod h1:H9Rt/W6/Qj27PGauhQc6nfCDy7vHpzsOThBSaYDoEhw=
|
||||
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
|
||||
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
|
||||
github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo=
|
||||
github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
|
||||
github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
|
||||
github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
|
||||
github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
|
||||
github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
|
||||
go.opentelemetry.io/contrib/bridges/otelslog v0.20.0 h1:oEl2Pw/i4OQwhAuda2pAHFAcOMivA+Xa+iTccBfab/g=
|
||||
go.opentelemetry.io/contrib/bridges/otelslog v0.20.0/go.mod h1:yMSQaiiq5dpfrSJCYLBcqFeJkFFI67seT4ngvx6jfVo=
|
||||
go.opentelemetry.io/contrib/bridges/prometheus v0.70.0 h1:qU2CqTGdlstwoVhu1WfjJJ3z2ntcNjTJO0ksTsFKzPI=
|
||||
go.opentelemetry.io/contrib/bridges/prometheus v0.70.0/go.mod h1:Ekh3I2XXfhdWkqbRq4PrivJS4BS/se7Er9ZsbK6YEtQ=
|
||||
go.opentelemetry.io/contrib/exporters/autoexport v0.70.0 h1:wpCLEJ/4RHUadR11UOdznbmyyih5/OPYFcsehAh6PYI=
|
||||
go.opentelemetry.io/contrib/exporters/autoexport v0.70.0/go.mod h1:x7MbNOwoKV5Hj6uYMXQksHlQdTNOP3hoFPvqWISiu6s=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0 h1:LMuyCAyfalSjDyjdC65nK6N0zoTT63+E/u95X0JovZI=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0/go.mod h1:085m8qbm4hgc8rZWGDEa4vmyyo2c3nPxUslYUKUIU04=
|
||||
go.opentelemetry.io/otel v1.45.0 h1:pdrWmLHofpubmArBv1LgFSv1Z0Ie/ppdZzu+kUN5EeU=
|
||||
go.opentelemetry.io/otel v1.45.0/go.mod h1:XZxIqPapzEYnhNSScF5DIqXhm/rYi0FzCe2XddAwZfQ=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.21.0 h1:WseeVYf5dJZTsyPiyW5L14k5qsSibqXAMTSiFEDiWr0=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.21.0/go.mod h1:SiLZnQS6Qk2eCpvr2CH/XMAOa64TWGXxEZJZCpD2Lmc=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.21.0 h1:fvNHGyo3CdRv/DQveXqhqBxnKTDyRaC5sMSQxilX/A0=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.21.0/go.mod h1:zyGrjRKL2B/6+Jc/m4/otPoZqV2MY9ZjC/aBraRO7zc=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.45.0 h1:klTViGcsvLCd1xN3rZzfZ12NslC/OimbmR+k+A006RI=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.45.0/go.mod h1:jRsK04CWmXuY8A0O+wMpSf+t90RHZ53o5Qmxn2PQPfk=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.45.0 h1:pnxy6c/kvNBWdNNFzqpjuJLm9Hjhgk/Q0nY221rwuk0=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.45.0/go.mod h1:qw6YsFapotRwoDhXRZvljzaOvCQB7UfnafEJagpN2TA=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0 h1:QRefszxJmfPdjXUUm3j6iDzY03mTPXMjqErFqQ67vUg=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0/go.mod h1:Tiz03lTBVBrm7eWZBOidzEaYaJa8tjwGUGv6d8mlTyk=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.45.0 h1:fG5MCxGz8+2VtrN/WgqSpJFctVz24gpxj8CxkKmc8Ww=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.45.0/go.mod h1:BmAYTn+3ysbRe+IU2msxmf5Rx3g6DHvex+tWI3LdhYI=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.45.0 h1:QBajQ2SrwQijzHyZbQlPsuIzpl/ll8DY6wPWsajeGcI=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.45.0/go.mod h1:08ZQLjrPLQ6R4kAXvuOvODEer5Yh4CoFvll5qB2BCI8=
|
||||
go.opentelemetry.io/otel/exporters/prometheus v0.67.0 h1:7IefDa35e6V3NoiqIeLDMDxMFyZDk5qcoC0Ax4cC16E=
|
||||
go.opentelemetry.io/otel/exporters/prometheus v0.67.0/go.mod h1:nsPI1awTg5Vmg1YrommL2mVarVGlqc4yXOoKAkPRD0c=
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.21.0 h1:2lpf4hnrasYIsUyEXwnTZq5lsxrMm4T2Bwb06IctAZQ=
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.21.0/go.mod h1:YWOW6h7jwApz9Pl76ie/izUsSPj0s2MdIlpqbPqaf3U=
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.45.0 h1:dm9iyzn6tioYZtwqaiBSU0TSI8Yu/8dTIbfG0+B49DY=
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.45.0/go.mod h1:xAvxYjYK28qvt+yu4BYZ/zMmAjwMXINXD6JiMyeB8iI=
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.45.0 h1:lsA/S1bxgdbyFGkTj+3meEdJ6ADVU7QoFstV6MXgE68=
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.45.0/go.mod h1:L7u+MirGoB1bjeLH66+xDykF4RC8C3RN7lIFpBiewUo=
|
||||
go.opentelemetry.io/otel/log v0.21.0 h1:SLsVDGmtyBrdw8/a2Z0bOIxou/+bN4z56GebH7T0LvA=
|
||||
go.opentelemetry.io/otel/log v0.21.0/go.mod h1:iReetQrZL9Wyg84cCkOoCmqDHS5RCFfyxC7J+r8fn8g=
|
||||
go.opentelemetry.io/otel/metric v1.45.0 h1:7Eg1uH7CJ5cXv9is6tnBe1FI6rj1nwUdbFypRm3br/M=
|
||||
go.opentelemetry.io/otel/metric v1.45.0/go.mod h1:HAPbm1nd3p1PmFH7v2dR+6BjXxw+Lq4a2+pndMAm08s=
|
||||
go.opentelemetry.io/otel/metric/x v0.67.0 h1:PcicCNZFkZ4bXfSooXdo3WN7RBOVOtjVdo1wD358Uns=
|
||||
go.opentelemetry.io/otel/metric/x v0.67.0/go.mod h1:FBjCWZe6wgcqxcMtjdGiClDKXb2YxxXii0CXftE4QtI=
|
||||
go.opentelemetry.io/otel/sdk v1.45.0 h1:4VVSMgQ83dUgW2aoX5f6JgLvHwIvzcuLnF9lUdCSpCw=
|
||||
go.opentelemetry.io/otel/sdk v1.45.0/go.mod h1:Sr40LgXV7DsKMMJMKOhUWOgMWTfAaqvm2kF0g7ilwuA=
|
||||
go.opentelemetry.io/otel/sdk/log v0.21.0 h1:QsE7XSR0ktQdKmRKGnR+f1ObGF32WG+7MER/P9KgmYc=
|
||||
go.opentelemetry.io/otel/sdk/log v0.21.0/go.mod h1:m9mApjCoD2/1QuKCAptjv+BrG9WKOvQLVdNx+iBldTo=
|
||||
go.opentelemetry.io/otel/sdk/log/logtest v0.21.0 h1:X+JBBgKlswCGYsmgL0CnoUUtlE//VB345c84jYAYkdQ=
|
||||
go.opentelemetry.io/otel/sdk/log/logtest v0.21.0/go.mod h1:HD1575K8e6sIFBBDd5tZB3t9DlMytWXq9FuR+Y4rfjE=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.45.0 h1:oVFszMfyj1Am6s24Vtc7wBb8BKLcwepJjNEYILuiE3o=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.45.0/go.mod h1:vUWUxDZvu1WVRj8JA8S0AdhsPrZoDpA2DdZauIh4mDA=
|
||||
go.opentelemetry.io/otel/trace v1.45.0 h1:l/mP6Uv7oNO7/TblbhpbgMidxhq1uO/rPsikOyVhxag=
|
||||
go.opentelemetry.io/otel/trace v1.45.0/go.mod h1:qoJJA2xNMnxRrdISU/kLtfUH2wNeQbiv+jhs/CxI8bc=
|
||||
go.opentelemetry.io/contrib/bridges/otelslog v0.20.1 h1:5sHc4ToTFjfSZCtGAAM6jPunICAmJX73htv372T4ipc=
|
||||
go.opentelemetry.io/contrib/bridges/otelslog v0.20.1/go.mod h1:oa6kgvyz/3GYW04dohd0++xJIH4xdQY8PAbpeCMaM8M=
|
||||
go.opentelemetry.io/contrib/bridges/prometheus v0.71.0 h1:9qgxsFLskbDMXl8WMqThoF6w8yGJgCumn9qRc67OmnI=
|
||||
go.opentelemetry.io/contrib/bridges/prometheus v0.71.0/go.mod h1:2rCjF4F2siiTeLCzJsaGZ3CK0XIoimCSKXEBPdv+Je0=
|
||||
go.opentelemetry.io/contrib/exporters/autoexport v0.71.0 h1:VCsJbp0YLyPtx2tu5Vgv2a2/qLoaMCj8hT2uZ34+Mx0=
|
||||
go.opentelemetry.io/contrib/exporters/autoexport v0.71.0/go.mod h1:qxZqn7e10f6ajmMCkg/47rMS7qQYfaOl2nj/4aytHUQ=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0 h1:3g7B90UzBltIDKq1/5mrTGxTnOFDV0ICOhLoxiZ8jlg=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0/go.mod h1:Ef8SuTh59BT7+ofpDxN9z+yOlc4t2GjLmKDgYNJL/NU=
|
||||
go.opentelemetry.io/otel v1.46.0 h1:FHt5/CDyVxi/8IM1CH7VE/rRgq3kLHa2mSTVMO8AWyc=
|
||||
go.opentelemetry.io/otel v1.46.0/go.mod h1:Gj3SEScelsNC45tp4nSxRYlS+f5iez7W8XPMCt905kE=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.22.0 h1:Bu39F5tzJct+f2IZbB8989fwyTps3c8e7EsUQsz+vs8=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.22.0/go.mod h1:dJUwod88EsFgYCqrDHaSPzhiY9pBUpt0d85/qSfua7k=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.22.0 h1:lYk7RmxdLK865qLwibroNGldHa1U7SWKYYvNjlK7PIo=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.22.0/go.mod h1:6GvlND0H0xdUJanOtIAn0xfwLkauh1tmsYEEVSMDdqY=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.46.0 h1:qkDYCAFiZXLcs1L4aY+tP2wguQ4kURANqHOQMA2et2s=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.46.0/go.mod h1:tkipS4DRzmpAmvg+Gw4++O1IdDq6TVDnvnYU6cmbQVs=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.46.0 h1:AP23h/mFgb/lc7tdck1Kfn9qxsM8TAeNPCU5C3pzaps=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.46.0/go.mod h1:K4EqCe1b4kGk5WR690ntg9LaBfsPoV32FwthbyoptuA=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.46.0 h1:OFnwLJr+pF3iHrlGSzbxyuo6/6HyBlnlN1CWEJmBVcw=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.46.0/go.mod h1:716wFneO0ov19A2beH5hjfh9AK5z/VWNAtDijp1Y0/g=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.46.0 h1:w53CDeOA/Kurp7yRsegSr6pbbr759dOvJ+yNmWM6Hxs=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.46.0/go.mod h1:BOmGMCbAtvcJiSJ+hLuhgPLdDbimnraSl8irz3iY8sY=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.46.0 h1:KrC1YrQeSt46ITMWAbgQx1M1eV1/1TKzttrBzymPmss=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.46.0/go.mod h1:zDSEzoEqsOrgBeGvH66KRgxh90VonFyJqBHA0Pk3+rM=
|
||||
go.opentelemetry.io/otel/exporters/prometheus v0.68.0 h1:QOf2IftqQwITVRJpnn0M7M9ZCbgWfxz4P7i9C9yc2N4=
|
||||
go.opentelemetry.io/otel/exporters/prometheus v0.68.0/go.mod h1:bgSvqu2TWGXiz7yr5UTMfObH8oqxJWHTnubQ3ef9BO4=
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.22.0 h1:kvMAiLEudKmk+CSG+iYbU8vTUGNNDaf/V09OO5lrTwI=
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.22.0/go.mod h1:L9Dlksri+MdT1cb2gIiA1cJJYW3Y92ipvDjNxYEyaDI=
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.46.0 h1:PR9eAf7o0dQs3hshZNZpE9aW2dXWX/KdDf6pJilVD3U=
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.46.0/go.mod h1:2Z4KyNdH1uuzivdinyfGsxzNNT/Rl45pwtVwfYVI0xk=
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.46.0 h1:KdRxPiAoMptR3vfWzvjjvutTsSiwbC2uG0496rzZNfo=
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.46.0/go.mod h1:K/qSA+3G7Eovxi4K09wzrAgkWRnosS0DAOZeEpve7sM=
|
||||
go.opentelemetry.io/otel/log v0.22.0 h1:5DBNnfvaJ6CVdkJ+Jle8Tzs50aSSv49TXGj9XRsEYw0=
|
||||
go.opentelemetry.io/otel/log v0.22.0/go.mod h1:gzOt/R67vF2GniAqWu8Qv0SXy89f71muHcrkz76PCdc=
|
||||
go.opentelemetry.io/otel/metric v1.46.0 h1:yBnkXvgV7AXFILZc5K6IZe/CBFF3OS7BJ8ov6/lj0K8=
|
||||
go.opentelemetry.io/otel/metric v1.46.0/go.mod h1:iPmdWqifKUdzziPkvvzIJXITl56fQx2mGM/DHLB3/2o=
|
||||
go.opentelemetry.io/otel/metric/x v0.68.0 h1:TA/cBT23D3MnxYPwHL7YFOdYGdx0A0v+s7Mzotpd1dU=
|
||||
go.opentelemetry.io/otel/metric/x v0.68.0/go.mod h1:agudOmvWhwUTjgibWDzxD2PoWYnpw5Ht5jISYOD2Hd4=
|
||||
go.opentelemetry.io/otel/sdk v1.46.0 h1:h5CNQQjEbuQXY/JfZtgt3i7HVFV3aHPO2OAwO2eTYPI=
|
||||
go.opentelemetry.io/otel/sdk v1.46.0/go.mod h1:GAERFXFt5SYCEB+YiKUbMBeza6UaDH7GmGOZEfh2gSM=
|
||||
go.opentelemetry.io/otel/sdk/log v0.22.0 h1:PRL+s6P63XT4E/bheEflopPUpVxuvANqZwtt89yhoGk=
|
||||
go.opentelemetry.io/otel/sdk/log v0.22.0/go.mod h1:JNp0sBELrjCTcu5W3GzABVypeU6vDJjBS+X0JISuz+g=
|
||||
go.opentelemetry.io/otel/sdk/log/logtest v0.22.0 h1:infPnfNrhCNgOUZRs3gWUg8vhoBUHihq02gwK05gzlg=
|
||||
go.opentelemetry.io/otel/sdk/log/logtest v0.22.0/go.mod h1:gkQZA3z15Bv3KU9vigBTi8dFechSozRP7v94X4VZv+s=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.46.0 h1:0piZ26EG4RBfebb2jhDH6ERCYHoVWduc3kLgPCwSnSE=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.46.0/go.mod h1:I1PbKrdVc8Qu8HYVDNtqVIwLwjNrhsV/uFuxfwg8mO4=
|
||||
go.opentelemetry.io/otel/trace v1.46.0 h1:OULy7ccdJnZtJ0UDYFOIGaCmiWzJ8Vi2G/Rsu60qs1c=
|
||||
go.opentelemetry.io/otel/trace v1.46.0/go.mod h1:J7GAXweO77XSFkB/rmAqk9D6ihszhFjLU+d9WuUxDLI=
|
||||
go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk=
|
||||
go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E=
|
||||
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
||||
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
||||
go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ=
|
||||
go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ=
|
||||
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
|
||||
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
|
||||
go4.org v0.0.0-20260112195520-a5071408f32f h1:ziUVAjmTPwQMBmYR1tbdRFJPtTcQUI12fH9QQjfb0Sw=
|
||||
go4.org v0.0.0-20260112195520-a5071408f32f/go.mod h1:ZRJnO5ZI4zAwMFp+dS1+V6J6MSyAowhRqAE+DPa1Xp0=
|
||||
golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M=
|
||||
golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis=
|
||||
golang.org/x/crypto v0.56.0 h1:GUh5Ii4J5jtcseSMiRqr1jXCNHoxjeV9Fmekc2oLy6Y=
|
||||
golang.org/x/crypto v0.56.0/go.mod h1:OMW5y6CY9l38uPLmxU6l6pwcXp1obtLo3e6gT7gQR2I=
|
||||
golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To=
|
||||
golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU=
|
||||
golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
|
||||
golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=
|
||||
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
|
||||
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk=
|
||||
golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
|
||||
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
@@ -329,19 +319,16 @@ golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
|
||||
gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260810153831-ec0a7760b754 h1:dWeMvEJ3JhYgqSCAHUZZJgMUyfniiiCvDc72x5EqJP0=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260810153831-ec0a7760b754/go.mod h1:q/3oV3jAi5vwelxsVAprMBC8BcM2zmNe+IjRGd+9/ks=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260810153831-ec0a7760b754 h1:k5CJw9e5ONCcA/u0webKt092npXuY+KeGh3Q8NAVf0g=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260810153831-ec0a7760b754/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
|
||||
google.golang.org/grpc v1.83.0 h1:JeNZEKJFbQxArAMl+hiytHauacDNqJUllNfmIMmpqnQ=
|
||||
google.golang.org/grpc v1.83.0/go.mod h1:kDyl6SKsiHKt0uylY5gtn5cEjkrIOhQOGDgIc4JGwzQ=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260831171406-18b4a7587f8a h1:i3TAXhpKc7TUP1VAPiBBrv45kamjoizCC3rOC0cAbOs=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260831171406-18b4a7587f8a/go.mod h1:CvYJHpbzPlT0fb/PsgtAamdwru/GVxUsomFdXTpOTI8=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260831171406-18b4a7587f8a h1:3Dnd1cDaZlB68lziofO+bJXpjOy8UfRv8Unt+yH8tQ4=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260831171406-18b4a7587f8a/go.mod h1:DjtHYE8FKJLivXcBEjGwndXfIC23G0VpXiXKqG179uA=
|
||||
google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU=
|
||||
google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8=
|
||||
google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc=
|
||||
google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q=
|
||||
gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA=
|
||||
|
||||
635
pkg/gotenberg/allowlist.go
Normal file
635
pkg/gotenberg/allowlist.go
Normal file
@@ -0,0 +1,635 @@
|
||||
package gotenberg
|
||||
|
||||
import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
// AllowListRisk classifies why an allow-list pattern is dangerous. A URL that
|
||||
// matches an allow-list skips the private and public IP checks, so a pattern
|
||||
// that matches more than its author intended silently widens outbound access.
|
||||
// See [AuditAllowList].
|
||||
type AllowListRisk string
|
||||
|
||||
const (
|
||||
// AllowListRiskUnanchored marks a pattern with no leading "^". regexp2
|
||||
// searches rather than matches, so the pattern hits anywhere in the URL,
|
||||
// including the query string.
|
||||
AllowListRiskUnanchored AllowListRisk = "unanchored"
|
||||
|
||||
// AllowListRiskUnanchoredBranch marks an alternation whose later branches
|
||||
// have no leading "^". Anchoring only the first branch is a common slip.
|
||||
AllowListRiskUnanchoredBranch AllowListRisk = "unanchored-branch"
|
||||
|
||||
// AllowListRiskCatchAll marks a pattern with no literal prefix, such as
|
||||
// ".+", which matches every URL and disables filtering entirely.
|
||||
AllowListRiskCatchAll AllowListRisk = "catch-all"
|
||||
|
||||
// AllowListRiskOpenHost marks a pattern whose host is not terminated, so
|
||||
// it also matches attacker-chosen suffix hosts. For example
|
||||
// "^https://trusted\.example\.com" matches
|
||||
// "https://trusted.example.com.attacker.example/".
|
||||
AllowListRiskOpenHost AllowListRisk = "open-host"
|
||||
)
|
||||
|
||||
// AllowListFinding reports one risky entry of an allow-list.
|
||||
type AllowListFinding struct {
|
||||
// Index is the zero-based position of the pattern within the flag value.
|
||||
Index int
|
||||
|
||||
// Pattern is the operator's pattern, verbatim.
|
||||
Pattern string
|
||||
|
||||
// Risk is why the pattern is dangerous.
|
||||
Risk AllowListRisk
|
||||
}
|
||||
|
||||
// maxAuditedPatternLength bounds the patterns [AuditAllowList] inspects. A
|
||||
// pathological pattern is not worth walking, and reporting nothing is better
|
||||
// than reporting a partial verdict.
|
||||
const maxAuditedPatternLength = 4096
|
||||
|
||||
// AuditAllowList reports the entries of an allow-list that match more URLs
|
||||
// than their author is likely to intend. It is a lint over the pattern source,
|
||||
// not a parser: it recognizes the shapes that are dangerous in practice and
|
||||
// stays silent when it cannot be sure.
|
||||
//
|
||||
// Callers use the findings to warn operators. Never use them to reject a
|
||||
// configuration: existing deployments rely on loose patterns, and a pattern
|
||||
// this function does not flag is not thereby safe.
|
||||
func AuditAllowList(patterns []string) []AllowListFinding {
|
||||
var findings []AllowListFinding
|
||||
|
||||
for i, pattern := range patterns {
|
||||
if pattern == "" || len(pattern) > maxAuditedPatternLength {
|
||||
continue
|
||||
}
|
||||
|
||||
risk, ok := auditPattern(pattern)
|
||||
if ok {
|
||||
findings = append(findings, AllowListFinding{Index: i, Pattern: pattern, Risk: risk})
|
||||
}
|
||||
}
|
||||
|
||||
return findings
|
||||
}
|
||||
|
||||
// auditPattern classifies a single pattern, reporting the first risk found.
|
||||
func auditPattern(pattern string) (AllowListRisk, bool) {
|
||||
body := trimInlineFlags(pattern)
|
||||
|
||||
branches := splitTopLevelAlternation(body)
|
||||
for i, branch := range branches {
|
||||
branch = strings.TrimSpace(branch)
|
||||
|
||||
anchored := hasStartAnchor(branch)
|
||||
rest := strings.TrimPrefix(strings.TrimPrefix(branch, `\A`), "^")
|
||||
|
||||
// Catch-all first: a pattern that constrains nothing matches every URL
|
||||
// whether or not it is anchored, and saying so is more useful than
|
||||
// telling the operator to anchor it.
|
||||
if literalPrefix(rest) == "" {
|
||||
return AllowListRiskCatchAll, true
|
||||
}
|
||||
|
||||
if !anchored {
|
||||
if i == 0 {
|
||||
return AllowListRiskUnanchored, true
|
||||
}
|
||||
return AllowListRiskUnanchoredBranch, true
|
||||
}
|
||||
|
||||
// A lookaround invalidates the token walk, so skip the host check for
|
||||
// this branch rather than guess. The anchor and catch-all checks above
|
||||
// still applied.
|
||||
if containsLookaround(rest) {
|
||||
continue
|
||||
}
|
||||
|
||||
if hostIsOpen(rest) {
|
||||
return AllowListRiskOpenHost, true
|
||||
}
|
||||
}
|
||||
|
||||
return "", false
|
||||
}
|
||||
|
||||
// trimInlineFlags removes a leading inline flag group such as "(?i)" so that
|
||||
// the anchor check sees the pattern proper.
|
||||
func trimInlineFlags(pattern string) string {
|
||||
if !strings.HasPrefix(pattern, "(?") {
|
||||
return pattern
|
||||
}
|
||||
|
||||
end := strings.Index(pattern, ")")
|
||||
if end == -1 {
|
||||
return pattern
|
||||
}
|
||||
|
||||
// Only a flag group qualifies. "(?:", "(?=", "(?!" and "(?<" open a real
|
||||
// group and must stay.
|
||||
flags := pattern[2:end]
|
||||
if flags == "" || strings.ContainsAny(flags, ":=!<") {
|
||||
return pattern
|
||||
}
|
||||
for _, r := range flags {
|
||||
if !strings.ContainsRune("imsUx-", r) {
|
||||
return pattern
|
||||
}
|
||||
}
|
||||
|
||||
return pattern[end+1:]
|
||||
}
|
||||
|
||||
// hasStartAnchor reports whether branch begins with a start-of-input anchor.
|
||||
func hasStartAnchor(branch string) bool {
|
||||
return strings.HasPrefix(branch, "^") || strings.HasPrefix(branch, `\A`)
|
||||
}
|
||||
|
||||
// containsLookaround reports whether the pattern uses a lookaround, which the
|
||||
// token walk in [hostIsOpen] cannot reason about.
|
||||
func containsLookaround(s string) bool {
|
||||
return strings.Contains(s, "(?=") || strings.Contains(s, "(?!") || strings.Contains(s, "(?<")
|
||||
}
|
||||
|
||||
// splitTopLevelAlternation splits on "|" at paren depth zero, honoring escapes
|
||||
// and character classes.
|
||||
func splitTopLevelAlternation(s string) []string {
|
||||
var (
|
||||
parts []string
|
||||
current strings.Builder
|
||||
depth int
|
||||
inClass bool
|
||||
)
|
||||
|
||||
for i := 0; i < len(s); i++ {
|
||||
c := s[i]
|
||||
|
||||
switch {
|
||||
case c == '\\' && i+1 < len(s):
|
||||
current.WriteByte(c)
|
||||
current.WriteByte(s[i+1])
|
||||
i++
|
||||
continue
|
||||
case c == '[' && !inClass:
|
||||
inClass = true
|
||||
case c == ']' && inClass:
|
||||
inClass = false
|
||||
case c == '(' && !inClass:
|
||||
depth++
|
||||
case c == ')' && !inClass:
|
||||
depth--
|
||||
case c == '|' && !inClass && depth == 0:
|
||||
parts = append(parts, current.String())
|
||||
current.Reset()
|
||||
continue
|
||||
}
|
||||
|
||||
current.WriteByte(c)
|
||||
}
|
||||
|
||||
parts = append(parts, current.String())
|
||||
|
||||
return parts
|
||||
}
|
||||
|
||||
// literalPrefix returns the characters a matching URL must start with. It
|
||||
// stops at the first optional or non-literal token, and descends one level
|
||||
// into a leading mandatory group so that "^(https|http)://" is not mistaken
|
||||
// for a catch-all. An empty result means the pattern constrains nothing.
|
||||
func literalPrefix(s string) string {
|
||||
var prefix strings.Builder
|
||||
|
||||
for i := 0; i < len(s); {
|
||||
// A group: descend once when it is mandatory, otherwise stop.
|
||||
if s[i] == '(' {
|
||||
end := matchingParen(s, i)
|
||||
if end == -1 {
|
||||
break
|
||||
}
|
||||
if isQuantified(s, end+1) {
|
||||
break
|
||||
}
|
||||
|
||||
inner := trimInlineFlags(s[i+1 : end])
|
||||
branches := splitTopLevelAlternation(inner)
|
||||
|
||||
common := literalPrefix(branches[0])
|
||||
for _, b := range branches[1:] {
|
||||
common = commonPrefix(common, literalPrefix(b))
|
||||
}
|
||||
prefix.WriteString(common)
|
||||
|
||||
// Only the leading group is worth descending into.
|
||||
break
|
||||
}
|
||||
|
||||
var token string
|
||||
switch {
|
||||
case s[i] == '\\' && i+1 < len(s):
|
||||
token = s[i : i+2]
|
||||
case s[i] == '[':
|
||||
end := matchingBracket(s, i)
|
||||
if end == -1 {
|
||||
return prefix.String()
|
||||
}
|
||||
token = s[i : end+1]
|
||||
default:
|
||||
token = s[i : i+1]
|
||||
}
|
||||
|
||||
next := i + len(token)
|
||||
if isQuantified(s, next) {
|
||||
break
|
||||
}
|
||||
|
||||
// Only a plain literal or an escaped literal contributes.
|
||||
switch {
|
||||
case len(token) == 2 && token[0] == '\\' && !isEscapeClass(token[1]):
|
||||
prefix.WriteByte(token[1])
|
||||
case len(token) == 1 && !strings.ContainsAny(token, `.[]()^$*+?{}|`):
|
||||
prefix.WriteByte(token[0])
|
||||
default:
|
||||
return prefix.String()
|
||||
}
|
||||
|
||||
i = next
|
||||
}
|
||||
|
||||
return prefix.String()
|
||||
}
|
||||
|
||||
// hostIsOpen reports whether the authority part of the pattern can be left
|
||||
// without crossing a terminator, which means the pattern also matches
|
||||
// attacker-chosen suffix hosts or userinfo.
|
||||
//
|
||||
// It walks the tokens after "://" and classifies each one. A terminator ends
|
||||
// the authority, so the pattern is safe. A crosser can match "@", "?" or "#"
|
||||
// and therefore lets a matching URL escape the authority, so the pattern is
|
||||
// open. Reaching the end without a terminator is open too, which is the
|
||||
// classic "^https://trusted\.example\.com" case.
|
||||
func hostIsOpen(s string) bool {
|
||||
_, after, ok := strings.Cut(s, "://")
|
||||
if !ok {
|
||||
// No authority to reason about, for example "^file:///tmp/".
|
||||
return false
|
||||
}
|
||||
|
||||
rest := after
|
||||
|
||||
for i := 0; i < len(rest); {
|
||||
var token string
|
||||
switch {
|
||||
case rest[i] == '\\' && i+1 < len(rest):
|
||||
token = rest[i : i+2]
|
||||
case rest[i] == '[':
|
||||
end := matchingBracket(rest, i)
|
||||
if end == -1 {
|
||||
return true
|
||||
}
|
||||
token = rest[i : end+1]
|
||||
case rest[i] == '(':
|
||||
end := matchingParen(rest, i)
|
||||
if end == -1 {
|
||||
return true
|
||||
}
|
||||
token = rest[i : end+1]
|
||||
default:
|
||||
token = rest[i : i+1]
|
||||
}
|
||||
|
||||
next := i + len(token)
|
||||
optional := isOptionalQuantifier(rest, next)
|
||||
|
||||
switch classifyHostToken(token) {
|
||||
case hostTokenTerminator:
|
||||
// An optional terminator does not end anything, since the URL may
|
||||
// match without it.
|
||||
if !optional {
|
||||
return false
|
||||
}
|
||||
case hostTokenCrosser:
|
||||
return true
|
||||
case hostTokenNeutral:
|
||||
// Part of the host itself, so keep walking.
|
||||
}
|
||||
|
||||
i = next
|
||||
for i < len(rest) && isQuantifierByte(rest[i]) {
|
||||
if rest[i] == '{' {
|
||||
end := strings.IndexByte(rest[i:], '}')
|
||||
if end == -1 {
|
||||
return true
|
||||
}
|
||||
i += end + 1
|
||||
continue
|
||||
}
|
||||
i++
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
// hostTokenKind is how a token affects the walk in [hostIsOpen].
|
||||
type hostTokenKind int
|
||||
|
||||
const (
|
||||
hostTokenNeutral hostTokenKind = iota
|
||||
hostTokenTerminator
|
||||
hostTokenCrosser
|
||||
)
|
||||
|
||||
// hostTerminators are the characters that end the authority of a URL.
|
||||
const hostTerminators = "/:#?"
|
||||
|
||||
// crosserClassChars are the characters that, if a class can match them, let a
|
||||
// match escape the authority. "/" is deliberately absent: it ends the
|
||||
// authority rather than escaping it, so a class such as "[:/]" is safe.
|
||||
const crosserClassChars = "@?#"
|
||||
|
||||
// classifyHostToken classifies one token of the authority walk.
|
||||
func classifyHostToken(token string) hostTokenKind {
|
||||
switch {
|
||||
case token == ".":
|
||||
// The wildcard matches "@", "#" and "?", so a host built on it can be
|
||||
// left without ever reaching a terminator.
|
||||
return hostTokenCrosser
|
||||
|
||||
case token == "$":
|
||||
return hostTokenTerminator
|
||||
|
||||
case len(token) == 1 && strings.Contains(hostTerminators, token):
|
||||
return hostTokenTerminator
|
||||
|
||||
case len(token) == 2 && token[0] == '\\':
|
||||
switch token[1] {
|
||||
case 'S', 'D', 'W':
|
||||
return hostTokenCrosser
|
||||
case 'd', 'w', 's':
|
||||
return hostTokenNeutral
|
||||
case 'p', 'P':
|
||||
return hostTokenCrosser
|
||||
}
|
||||
if strings.Contains(hostTerminators, token[1:]) {
|
||||
return hostTokenTerminator
|
||||
}
|
||||
return hostTokenNeutral
|
||||
|
||||
case strings.HasPrefix(token, "["):
|
||||
inner := strings.TrimSuffix(strings.TrimPrefix(token, "["), "]")
|
||||
if strings.HasPrefix(inner, "^") {
|
||||
// A negated class almost always admits "@".
|
||||
return hostTokenCrosser
|
||||
}
|
||||
if classContainsAny(inner, crosserClassChars) {
|
||||
return hostTokenCrosser
|
||||
}
|
||||
if classOnlyTerminators(inner) {
|
||||
return hostTokenTerminator
|
||||
}
|
||||
return hostTokenNeutral
|
||||
|
||||
case strings.HasPrefix(token, "("):
|
||||
return classifyGroup(token)
|
||||
}
|
||||
|
||||
return hostTokenNeutral
|
||||
}
|
||||
|
||||
// classifyGroup classifies a parenthesized group. A group whose every branch
|
||||
// starts with a terminator ends the authority, which is what makes the
|
||||
// idiomatic "(:|/|$)" safe. A group containing a crosser is a crosser.
|
||||
func classifyGroup(token string) hostTokenKind {
|
||||
inner := trimInlineFlags(strings.TrimSuffix(strings.TrimPrefix(token, "("), ")"))
|
||||
inner = strings.TrimPrefix(inner, "?:")
|
||||
|
||||
branches := splitTopLevelAlternation(inner)
|
||||
|
||||
allTerminate := true
|
||||
for _, branch := range branches {
|
||||
if branch == "" {
|
||||
allTerminate = false
|
||||
continue
|
||||
}
|
||||
|
||||
kind := classifyHostToken(firstToken(branch))
|
||||
if kind == hostTokenCrosser {
|
||||
return hostTokenCrosser
|
||||
}
|
||||
if kind != hostTokenTerminator {
|
||||
allTerminate = false
|
||||
}
|
||||
|
||||
// A crosser anywhere inside the branch still escapes the authority.
|
||||
if branchHasCrosser(branch) {
|
||||
return hostTokenCrosser
|
||||
}
|
||||
}
|
||||
|
||||
if allTerminate {
|
||||
return hostTokenTerminator
|
||||
}
|
||||
|
||||
return hostTokenNeutral
|
||||
}
|
||||
|
||||
// branchHasCrosser reports whether any token of branch is a crosser.
|
||||
func branchHasCrosser(branch string) bool {
|
||||
for i := 0; i < len(branch); {
|
||||
token := tokenAt(branch, i)
|
||||
if token == "" {
|
||||
return true
|
||||
}
|
||||
if classifyHostToken(token) == hostTokenCrosser {
|
||||
return true
|
||||
}
|
||||
i += len(token)
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// firstToken returns the first regex token of s.
|
||||
func firstToken(s string) string {
|
||||
return tokenAt(s, 0)
|
||||
}
|
||||
|
||||
// tokenAt returns the regex token starting at index i, or "" if it is
|
||||
// malformed.
|
||||
func tokenAt(s string, i int) string {
|
||||
if i >= len(s) {
|
||||
return ""
|
||||
}
|
||||
|
||||
switch {
|
||||
case s[i] == '\\' && i+1 < len(s):
|
||||
return s[i : i+2]
|
||||
case s[i] == '[':
|
||||
end := matchingBracket(s, i)
|
||||
if end == -1 {
|
||||
return ""
|
||||
}
|
||||
return s[i : end+1]
|
||||
case s[i] == '(':
|
||||
end := matchingParen(s, i)
|
||||
if end == -1 {
|
||||
return ""
|
||||
}
|
||||
return s[i : end+1]
|
||||
}
|
||||
|
||||
return s[i : i+1]
|
||||
}
|
||||
|
||||
// classOnlyTerminators reports whether every character a class can match ends
|
||||
// the authority, which makes the class itself a terminator. A range is never
|
||||
// treated as one.
|
||||
func classOnlyTerminators(class string) bool {
|
||||
if class == "" {
|
||||
return false
|
||||
}
|
||||
|
||||
for i := 0; i < len(class); i++ {
|
||||
if class[i] == '\\' && i+1 < len(class) {
|
||||
if !strings.Contains(hostTerminators, class[i+1:i+2]) {
|
||||
return false
|
||||
}
|
||||
i++
|
||||
continue
|
||||
}
|
||||
|
||||
if i+2 < len(class) && class[i+1] == '-' {
|
||||
return false
|
||||
}
|
||||
|
||||
if !strings.Contains(hostTerminators, class[i:i+1]) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
// classContainsAny reports whether a character class body can match any of the
|
||||
// given characters, expanding simple ranges.
|
||||
func classContainsAny(class, chars string) bool {
|
||||
for i := 0; i < len(class); i++ {
|
||||
if class[i] == '\\' && i+1 < len(class) {
|
||||
// An escape class such as \S inside a class admits everything.
|
||||
if strings.ContainsRune("SDW", rune(class[i+1])) {
|
||||
return true
|
||||
}
|
||||
if strings.ContainsRune(chars, rune(class[i+1])) {
|
||||
return true
|
||||
}
|
||||
i++
|
||||
continue
|
||||
}
|
||||
|
||||
if i+2 < len(class) && class[i+1] == '-' {
|
||||
lo, hi := class[i], class[i+2]
|
||||
for _, c := range []byte(chars) {
|
||||
if c >= lo && c <= hi {
|
||||
return true
|
||||
}
|
||||
}
|
||||
i += 2
|
||||
continue
|
||||
}
|
||||
|
||||
if strings.ContainsRune(chars, rune(class[i])) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// matchingParen returns the index of the ")" closing the "(" at start.
|
||||
func matchingParen(s string, start int) int {
|
||||
depth := 0
|
||||
inClass := false
|
||||
|
||||
for i := start; i < len(s); i++ {
|
||||
switch {
|
||||
case s[i] == '\\' && i+1 < len(s):
|
||||
i++
|
||||
case s[i] == '[' && !inClass:
|
||||
inClass = true
|
||||
case s[i] == ']' && inClass:
|
||||
inClass = false
|
||||
case s[i] == '(' && !inClass:
|
||||
depth++
|
||||
case s[i] == ')' && !inClass:
|
||||
depth--
|
||||
if depth == 0 {
|
||||
return i
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return -1
|
||||
}
|
||||
|
||||
// matchingBracket returns the index of the "]" closing the "[" at start.
|
||||
func matchingBracket(s string, start int) int {
|
||||
for i := start + 1; i < len(s); i++ {
|
||||
switch {
|
||||
case s[i] == '\\' && i+1 < len(s):
|
||||
i++
|
||||
case s[i] == ']':
|
||||
// A "]" immediately after "[" or "[^" is a literal.
|
||||
if i == start+1 || (i == start+2 && s[start+1] == '^') {
|
||||
continue
|
||||
}
|
||||
return i
|
||||
}
|
||||
}
|
||||
|
||||
return -1
|
||||
}
|
||||
|
||||
// isQuantifierByte reports whether c opens a quantifier.
|
||||
func isQuantifierByte(c byte) bool {
|
||||
return c == '?' || c == '*' || c == '+' || c == '{'
|
||||
}
|
||||
|
||||
// isQuantified reports whether a quantifier starts at index i.
|
||||
func isQuantified(s string, i int) bool {
|
||||
return i < len(s) && isQuantifierByte(s[i])
|
||||
}
|
||||
|
||||
// isOptionalQuantifier reports whether the quantifier at index i lets the
|
||||
// preceding token match nothing.
|
||||
func isOptionalQuantifier(s string, i int) bool {
|
||||
if i >= len(s) {
|
||||
return false
|
||||
}
|
||||
|
||||
switch s[i] {
|
||||
case '?', '*':
|
||||
return true
|
||||
case '{':
|
||||
return strings.HasPrefix(s[i:], "{0")
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// isEscapeClass reports whether c after a backslash denotes a character class
|
||||
// rather than a literal.
|
||||
func isEscapeClass(c byte) bool {
|
||||
return strings.ContainsRune("dDwWsSbBAzZpP", rune(c))
|
||||
}
|
||||
|
||||
// commonPrefix returns the longest common prefix of a and b.
|
||||
func commonPrefix(a, b string) string {
|
||||
n := min(len(a), len(b))
|
||||
for i := range n {
|
||||
if a[i] != b[i] {
|
||||
return a[:i]
|
||||
}
|
||||
}
|
||||
|
||||
return a[:n]
|
||||
}
|
||||
179
pkg/gotenberg/allowlist_test.go
Normal file
179
pkg/gotenberg/allowlist_test.go
Normal file
@@ -0,0 +1,179 @@
|
||||
package gotenberg
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/dlclark/regexp2/v2"
|
||||
)
|
||||
|
||||
func TestAuditAllowList(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
scenario string
|
||||
pattern string
|
||||
want AllowListRisk
|
||||
}{
|
||||
// Safe: the host is terminated before anything can leave it.
|
||||
{"idiomatic terminator group", `^https?://internal\.svc(:|/|$)`, ""},
|
||||
{"trailing slash", `^https://trusted\.example\.com/`, ""},
|
||||
{"optional port then terminator", `^https://example\.com(:[0-9]+)?(/|$)`, ""},
|
||||
{"positive class cannot leave authority", `^https://[a-z0-9.-]+\.s3\.amazonaws\.com/`, ""},
|
||||
{"leading mandatory group", `^(https|http)://a\.example\.com/`, ""},
|
||||
{"optional subdomain group", `^https://(www\.)?example\.com/`, ""},
|
||||
{"port terminator", `^https://example\.com:8443/`, ""},
|
||||
{"end anchor", `^https://example\.com$`, ""},
|
||||
{"alternation both anchored and terminated", `^https://a\.example/|^https://b\.example/`, ""},
|
||||
{"no authority to check", `^file:///tmp/`, ""},
|
||||
{"digit class in host", `^https://node\d+\.example\.com/`, ""},
|
||||
{"class of only terminators", `^https://example\.com[:/]`, ""},
|
||||
{"feature file pattern, fixed", `^https?://host\.docker\.internal(:[0-9]+)?/`, ""},
|
||||
|
||||
// Unanchored: regexp2 searches, so these match anywhere in the URL.
|
||||
{"no anchor", `trusted\.example\.com`, AllowListRiskUnanchored},
|
||||
{"no anchor with scheme", `https://trusted\.example\.com/`, AllowListRiskUnanchored},
|
||||
|
||||
// Only the first branch anchored.
|
||||
{"second branch unanchored", `^http://a\.example/|http://b\.example/`, AllowListRiskUnanchoredBranch},
|
||||
|
||||
// Catch-all: matches every URL.
|
||||
{"dot plus", `.+`, AllowListRiskCatchAll},
|
||||
{"dot star", `.*`, AllowListRiskCatchAll},
|
||||
{"anchored dot star", `^.*`, AllowListRiskCatchAll},
|
||||
{"anchored dot plus", `^.+`, AllowListRiskCatchAll},
|
||||
|
||||
// Open host: the reported vulnerability class.
|
||||
{"advisory pattern", `^http://trusted\.example\.com`, AllowListRiskOpenHost},
|
||||
{"gotenberg.dev internet-facing recipe", `^https?://[^/]+\.internal\.example\.com`, AllowListRiskOpenHost},
|
||||
{"gotenberg.dev strict whitelist recipe", `^https://(api|cdn|images)\.internal\.example\.com`, AllowListRiskOpenHost},
|
||||
{"gotenberg.dev hooks recipe", `^https?://hooks\.internal\.example\.com`, AllowListRiskOpenHost},
|
||||
{"feature file pattern", `^https?://host.docker.internal.*`, AllowListRiskOpenHost},
|
||||
{"scheme only", `^https?://`, AllowListRiskOpenHost},
|
||||
{"wildcard subdomain", `^https://.+\.example\.com/`, AllowListRiskOpenHost},
|
||||
{"escaped dot is not a terminator", `^https?://example\.com\.`, AllowListRiskOpenHost},
|
||||
{"negated class in host", `^https://[^.]+\.example\.com/`, AllowListRiskOpenHost},
|
||||
} {
|
||||
t.Run(tc.scenario, func(t *testing.T) {
|
||||
findings := AuditAllowList([]string{tc.pattern})
|
||||
|
||||
if tc.want == "" {
|
||||
if len(findings) != 0 {
|
||||
t.Fatalf("AuditAllowList(%q) = %+v, want no finding", tc.pattern, findings)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if len(findings) != 1 {
|
||||
t.Fatalf("AuditAllowList(%q) returned %d findings, want 1", tc.pattern, len(findings))
|
||||
}
|
||||
if findings[0].Risk != tc.want {
|
||||
t.Fatalf("AuditAllowList(%q) risk = %q, want %q", tc.pattern, findings[0].Risk, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestAuditAllowList_FlaggedPatternsAreActuallyExploitable proves the audit is
|
||||
// not merely syntactic: every pattern it flags as open-host really does admit
|
||||
// a host the operator did not intend.
|
||||
func TestAuditAllowList_FlaggedPatternsAreActuallyExploitable(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
pattern string
|
||||
attack string
|
||||
}{
|
||||
{`^http://trusted\.example\.com`, "http://trusted.example.com.attacker.example/"},
|
||||
{`^https?://[^/]+\.internal\.example\.com`, "http://a.internal.example.com.attacker.example/"},
|
||||
{`^https://(api|cdn|images)\.internal\.example\.com`, "https://api.internal.example.com.attacker.example/"},
|
||||
{`^https?://hooks\.internal\.example\.com`, "http://hooks.internal.example.com.attacker.example/"},
|
||||
{`^https?://host.docker.internal.*`, "http://host.docker.internal.attacker.example/"},
|
||||
{`^https://.+\.example\.com/`, "https://attacker.example/#x.example.com/"},
|
||||
{`^https?://example\.com\.`, "http://example.com.attacker.example/"},
|
||||
} {
|
||||
t.Run(tc.pattern, func(t *testing.T) {
|
||||
findings := AuditAllowList([]string{tc.pattern})
|
||||
if len(findings) == 0 {
|
||||
t.Fatalf("pattern %q was not flagged", tc.pattern)
|
||||
}
|
||||
|
||||
ok, err := regexp2.MustCompile(tc.pattern, regexp2.None).MatchString(tc.attack)
|
||||
if err != nil {
|
||||
t.Fatalf("match %q: %v", tc.attack, err)
|
||||
}
|
||||
if !ok {
|
||||
t.Fatalf("pattern %q does not match %q, so the finding is a false positive", tc.pattern, tc.attack)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestAuditAllowList_SafePatternsRejectTheAttacks is the converse: the shapes
|
||||
// the audit stays silent about really do reject the same attacks.
|
||||
func TestAuditAllowList_SafePatternsRejectTheAttacks(t *testing.T) {
|
||||
safe := []string{
|
||||
`^https?://internal\.svc(:|/|$)`,
|
||||
`^https://trusted\.example\.com/`,
|
||||
`^https://example\.com(:[0-9]+)?(/|$)`,
|
||||
`^https://[a-z0-9.-]+\.s3\.amazonaws\.com/`,
|
||||
}
|
||||
attacks := []string{
|
||||
"https://internal.svc.attacker.example/",
|
||||
"https://trusted.example.com.attacker.example/",
|
||||
"https://trusted.example.com@169.254.169.254/",
|
||||
"https://example.com.attacker.example/",
|
||||
"https://example.com@10.0.0.5/",
|
||||
"https://bucket.s3.amazonaws.com.attacker.example/",
|
||||
"https://bucket.s3.amazonaws.com@127.0.0.1/",
|
||||
}
|
||||
|
||||
for _, pattern := range safe {
|
||||
t.Run(pattern, func(t *testing.T) {
|
||||
if findings := AuditAllowList([]string{pattern}); len(findings) != 0 {
|
||||
t.Fatalf("safe pattern %q was flagged as %q", pattern, findings[0].Risk)
|
||||
}
|
||||
|
||||
re := regexp2.MustCompile(pattern, regexp2.None)
|
||||
for _, attack := range attacks {
|
||||
ok, err := re.MatchString(attack)
|
||||
if err != nil {
|
||||
t.Fatalf("match %q: %v", attack, err)
|
||||
}
|
||||
if ok {
|
||||
t.Fatalf("pattern %q matches attack %q but was not flagged", pattern, attack)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAuditAllowList_SkipsEmptyAndOversized(t *testing.T) {
|
||||
oversized := make([]byte, maxAuditedPatternLength+1)
|
||||
for i := range oversized {
|
||||
oversized[i] = 'a'
|
||||
}
|
||||
|
||||
findings := AuditAllowList([]string{"", string(oversized)})
|
||||
if len(findings) != 0 {
|
||||
t.Fatalf("AuditAllowList returned %+v, want no finding", findings)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAuditAllowList_ReportsIndex(t *testing.T) {
|
||||
findings := AuditAllowList([]string{
|
||||
`^https://ok\.example\.com/`,
|
||||
`^https://open\.example\.com`,
|
||||
})
|
||||
if len(findings) != 1 {
|
||||
t.Fatalf("got %d findings, want 1", len(findings))
|
||||
}
|
||||
if findings[0].Index != 1 {
|
||||
t.Fatalf("findings[0].Index = %d, want 1", findings[0].Index)
|
||||
}
|
||||
}
|
||||
|
||||
// TestAuditAllowList_ShippedChromiumDenyListIsNotAudited guards the rule that
|
||||
// deny-lists are never audited. The shipped Chromium deny-list uses a
|
||||
// lookaround and has no authority, so auditing it would produce noise.
|
||||
func TestAuditAllowList_LookaroundIsNotFlaggedForHost(t *testing.T) {
|
||||
findings := AuditAllowList([]string{`^file:(?!//\/tmp/).*`})
|
||||
if len(findings) != 0 {
|
||||
t.Fatalf("AuditAllowList returned %+v, want no finding", findings)
|
||||
}
|
||||
}
|
||||
@@ -1,70 +0,0 @@
|
||||
package gotenberg
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/dlclark/regexp2"
|
||||
)
|
||||
|
||||
// ErrFiltered happens if a value is filtered by the [FilterDeadline] function.
|
||||
var ErrFiltered = errors.New("value filtered")
|
||||
|
||||
// FilterDeadline checks if the given value is allowed and not denied according
|
||||
// to regex patterns. The allowed list uses OR semantics (value must match at
|
||||
// least one pattern). The denied list uses OR semantics (value is denied if it
|
||||
// matches any pattern). It returns a [context.DeadlineExceeded] if it takes
|
||||
// too long to process.
|
||||
func FilterDeadline(allowed, denied []*regexp2.Regexp, s string, deadline time.Time) error {
|
||||
if len(allowed) > 0 {
|
||||
matched := false
|
||||
|
||||
for _, pattern := range allowed {
|
||||
// FIXME: not ideal to compile everytime, but is there another way to create a clone?
|
||||
clone := regexp2.MustCompile(pattern.String(), 0)
|
||||
clone.MatchTimeout = time.Until(deadline)
|
||||
|
||||
ok, err := clone.MatchString(s)
|
||||
if err != nil {
|
||||
if time.Now().After(deadline) {
|
||||
return context.DeadlineExceeded
|
||||
}
|
||||
|
||||
return fmt.Errorf("'%s' cannot handle '%s': %w", clone.String(), s, err)
|
||||
}
|
||||
|
||||
if ok {
|
||||
matched = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if !matched {
|
||||
return fmt.Errorf("'%s' does not match any expression from the allowed list: %w", s, ErrFiltered)
|
||||
}
|
||||
}
|
||||
|
||||
if len(denied) > 0 {
|
||||
for _, pattern := range denied {
|
||||
clone := regexp2.MustCompile(pattern.String(), 0)
|
||||
clone.MatchTimeout = time.Until(deadline)
|
||||
|
||||
ok, err := clone.MatchString(s)
|
||||
if err != nil {
|
||||
if time.Now().After(deadline) {
|
||||
return context.DeadlineExceeded
|
||||
}
|
||||
|
||||
return fmt.Errorf("'%s' cannot handle '%s': %w", clone.String(), s, err)
|
||||
}
|
||||
|
||||
if ok {
|
||||
return fmt.Errorf("'%s' matches the expression from the denied list: %w", s, ErrFiltered)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -1,117 +0,0 @@
|
||||
package gotenberg
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/dlclark/regexp2"
|
||||
)
|
||||
|
||||
func TestFilterDeadline(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
scenario string
|
||||
allowed []*regexp2.Regexp
|
||||
denied []*regexp2.Regexp
|
||||
s string
|
||||
deadline time.Time
|
||||
expectError bool
|
||||
expectedError error
|
||||
}{
|
||||
{
|
||||
scenario: "DeadlineExceeded (allowed)",
|
||||
allowed: []*regexp2.Regexp{regexp2.MustCompile("foo", 0)},
|
||||
denied: nil,
|
||||
s: "foo",
|
||||
deadline: time.Now().Add(time.Duration(-1) * time.Hour),
|
||||
expectError: true,
|
||||
expectedError: context.DeadlineExceeded,
|
||||
},
|
||||
{
|
||||
scenario: "ErrFiltered (allowed, no match)",
|
||||
allowed: []*regexp2.Regexp{regexp2.MustCompile("foo", 0)},
|
||||
denied: nil,
|
||||
s: "bar",
|
||||
deadline: time.Now().Add(time.Duration(5) * time.Second),
|
||||
expectError: true,
|
||||
expectedError: ErrFiltered,
|
||||
},
|
||||
{
|
||||
scenario: "DeadlineExceeded (denied)",
|
||||
allowed: nil,
|
||||
denied: []*regexp2.Regexp{regexp2.MustCompile("foo", 0)},
|
||||
s: "foo",
|
||||
deadline: time.Now().Add(time.Duration(-1) * time.Hour),
|
||||
expectError: true,
|
||||
expectedError: context.DeadlineExceeded,
|
||||
},
|
||||
{
|
||||
scenario: "ErrFiltered (denied)",
|
||||
allowed: nil,
|
||||
denied: []*regexp2.Regexp{regexp2.MustCompile("foo", 0)},
|
||||
s: "foo",
|
||||
deadline: time.Now().Add(time.Duration(5) * time.Second),
|
||||
expectError: true,
|
||||
expectedError: ErrFiltered,
|
||||
},
|
||||
{
|
||||
scenario: "success (empty lists)",
|
||||
allowed: nil,
|
||||
denied: nil,
|
||||
s: "foo",
|
||||
deadline: time.Now().Add(time.Duration(5) * time.Second),
|
||||
expectError: false,
|
||||
},
|
||||
{
|
||||
scenario: "multi-pattern allow list, second matches",
|
||||
allowed: []*regexp2.Regexp{regexp2.MustCompile("^https://", 0), regexp2.MustCompile("^file:///tmp/", 0)},
|
||||
denied: nil,
|
||||
s: "file:///tmp/abc/index.html",
|
||||
deadline: time.Now().Add(time.Duration(5) * time.Second),
|
||||
expectError: false,
|
||||
},
|
||||
{
|
||||
scenario: "multi-pattern allow list, none matches",
|
||||
allowed: []*regexp2.Regexp{regexp2.MustCompile("^https://", 0), regexp2.MustCompile("^ftp://", 0)},
|
||||
denied: nil,
|
||||
s: "file:///tmp/abc/index.html",
|
||||
deadline: time.Now().Add(time.Duration(5) * time.Second),
|
||||
expectError: true,
|
||||
expectedError: ErrFiltered,
|
||||
},
|
||||
{
|
||||
scenario: "multi-pattern deny list, second matches",
|
||||
allowed: nil,
|
||||
denied: []*regexp2.Regexp{regexp2.MustCompile("^ftp://", 0), regexp2.MustCompile("^file:.*", 0)},
|
||||
s: "file:///etc/passwd",
|
||||
deadline: time.Now().Add(time.Duration(5) * time.Second),
|
||||
expectError: true,
|
||||
expectedError: ErrFiltered,
|
||||
},
|
||||
{
|
||||
scenario: "https URL passes deny list targeting file://",
|
||||
allowed: nil,
|
||||
denied: []*regexp2.Regexp{regexp2.MustCompile("^file:.*", 0)},
|
||||
s: "https://example.com",
|
||||
deadline: time.Now().Add(time.Duration(5) * time.Second),
|
||||
expectError: false,
|
||||
},
|
||||
} {
|
||||
t.Run(tc.scenario, func(t *testing.T) {
|
||||
err := FilterDeadline(tc.allowed, tc.denied, tc.s, tc.deadline)
|
||||
|
||||
if tc.expectError && err == nil {
|
||||
t.Fatal("expected an error but got none")
|
||||
}
|
||||
|
||||
if !tc.expectError && err != nil {
|
||||
t.Fatalf("expected no error but got: %v", err)
|
||||
}
|
||||
|
||||
if tc.expectedError != nil && !errors.Is(err, tc.expectedError) {
|
||||
t.Fatalf("expected error %v but got: %v", tc.expectedError, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,11 +1,17 @@
|
||||
package gotenberg
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/dlclark/regexp2"
|
||||
"github.com/dlclark/regexp2/v2"
|
||||
"github.com/labstack/gommon/bytes"
|
||||
flag "github.com/spf13/pflag"
|
||||
|
||||
"github.com/gotenberg/gotenberg/v8/pkg/gotenberg/internal/log"
|
||||
)
|
||||
|
||||
// ParsedFlags wraps a [flag.FlagSet] so that retrieving the typed values is
|
||||
@@ -201,15 +207,42 @@ func (f *ParsedFlags) MustDeprecatedHumanReadableBytes(deprecated string, newNam
|
||||
return f.MustHumanReadableBytes(newName)
|
||||
}
|
||||
|
||||
// PatternMatchTimeout bounds a single match against an operator-supplied
|
||||
// allow-list or deny-list pattern.
|
||||
//
|
||||
// regexp2 backtracks, and the strings matched against these patterns are
|
||||
// client-controlled: a request URL, a CONNECT host. A pattern that backtracks
|
||||
// catastrophically would otherwise burn a core for as long as the caller's
|
||||
// deadline allows, which is --api-timeout (env API_TIMEOUT), 30 seconds by
|
||||
// default. The ceiling mirrors the one the Chromium module already applies to
|
||||
// the per-request extraHttpHeaders scope pattern.
|
||||
//
|
||||
// [ParsedFlags.MustRegexp] and [ParsedFlags.MustRegexpSlice] stamp this onto
|
||||
// every pattern they compile, which is how all four production lists are
|
||||
// built. Patterns compiled any other way keep regexp2's default of
|
||||
// math.MaxInt64, which it treats as no timeout at all, so a hand-built slice
|
||||
// must set this itself before reaching [DecideOutbound].
|
||||
//
|
||||
// The ceiling is wall-clock. Match through [MatchPattern] rather than calling
|
||||
// regexp2 directly: a match that never approaches the ceiling still aborts if
|
||||
// the process loses the CPU at the wrong moment.
|
||||
const PatternMatchTimeout = 250 * time.Millisecond
|
||||
|
||||
// MustRegexp returns the regular expression of a flag given by name.
|
||||
// It panics if an error occurs.
|
||||
//
|
||||
// The returned expression carries [PatternMatchTimeout] and is safe to match
|
||||
// on concurrently: callers must not compile a private copy per match.
|
||||
func (f *ParsedFlags) MustRegexp(name string) *regexp2.Regexp {
|
||||
val, err := f.GetString(name)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
return regexp2.MustCompile(val, 0)
|
||||
re := regexp2.MustCompile(val, regexp2.None)
|
||||
re.MatchTimeout = PatternMatchTimeout
|
||||
|
||||
return re
|
||||
}
|
||||
|
||||
// MustDeprecatedRegexp returns the regular expression of a deprecated flag if
|
||||
@@ -226,21 +259,140 @@ func (f *ParsedFlags) MustDeprecatedRegexp(deprecated string, newName string) *r
|
||||
// MustRegexpSlice returns a slice of compiled regular expressions from a
|
||||
// string-slice flag given by name. Empty strings are skipped.
|
||||
// It panics if an error occurs.
|
||||
//
|
||||
// Every allow-list and deny-list in Gotenberg is read through this method, so
|
||||
// it is also where allow-list patterns are audited. See [AuditAllowList].
|
||||
//
|
||||
// The returned expressions carry [PatternMatchTimeout] and are safe to match
|
||||
// on concurrently: callers must not compile a private copy per match.
|
||||
func (f *ParsedFlags) MustRegexpSlice(name string) []*regexp2.Regexp {
|
||||
vals := f.MustStringSlice(name)
|
||||
|
||||
f.warnRiskyAllowList(name, vals)
|
||||
|
||||
var regexps []*regexp2.Regexp
|
||||
for _, val := range vals {
|
||||
if val == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
regexps = append(regexps, regexp2.MustCompile(val, 0))
|
||||
re := regexp2.MustCompile(val, regexp2.None)
|
||||
re.MatchTimeout = PatternMatchTimeout
|
||||
|
||||
regexps = append(regexps, re)
|
||||
}
|
||||
|
||||
return regexps
|
||||
}
|
||||
|
||||
// allowListFlagSuffix identifies the flags whose patterns grant an IP-check
|
||||
// bypass. Deny-lists are never audited: they always apply, cannot be bypassed,
|
||||
// and a loose deny-list is safe rather than dangerous.
|
||||
const allowListFlagSuffix = "-allow-list"
|
||||
|
||||
// warnRiskyAllowList logs one warning per allow-list entry that matches more
|
||||
// URLs than its author is likely to intend.
|
||||
//
|
||||
// It warns and never fails: operators depend on loose patterns today, and
|
||||
// rejecting them at startup would break running deployments.
|
||||
func (f *ParsedFlags) warnRiskyAllowList(name string, vals []string) {
|
||||
if !strings.HasSuffix(name, allowListFlagSuffix) {
|
||||
return
|
||||
}
|
||||
|
||||
findings := AuditAllowList(vals)
|
||||
if len(findings) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
// The logger is nil until the entry point initializes it, which happens
|
||||
// before any module is provisioned. Tests and embedders that call this
|
||||
// method directly get no logger, and must not panic for it.
|
||||
logger := log.Logger()
|
||||
if logger == nil {
|
||||
return
|
||||
}
|
||||
|
||||
for _, finding := range findings {
|
||||
// Provision has no context.Context to propagate, so the trace-aware
|
||||
// logging convention is satisfied with a background context.
|
||||
logger.WarnContext(
|
||||
context.Background(),
|
||||
f.allowListWarning(name, finding),
|
||||
slog.String("flag", "--"+name),
|
||||
slog.String("env", EnvVarName(name)),
|
||||
slog.Int("entry", finding.Index+1),
|
||||
slog.String("reason", string(finding.Risk)),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// allowListWarning builds the operator-facing message for a finding. It names
|
||||
// the flag and its environment variable, and, when they exist, the IP-check
|
||||
// flags the entry silently disables.
|
||||
func (f *ParsedFlags) allowListWarning(name string, finding AllowListFinding) string {
|
||||
var b strings.Builder
|
||||
|
||||
// Print the pattern raw rather than quoted: %q escapes every backslash, so
|
||||
// the operator would not recognize the value they set.
|
||||
fmt.Fprintf(&b, "--%s (%s) entry %d '%s' ", name, EnvVarName(name), finding.Index+1, finding.Pattern)
|
||||
|
||||
switch finding.Risk {
|
||||
case AllowListRiskUnanchored:
|
||||
b.WriteString("is not anchored with ^, so it matches anywhere in the URL and a URL such as http://attacker.example/?u=trusted.example.com passes. ")
|
||||
case AllowListRiskUnanchoredBranch:
|
||||
b.WriteString("has an alternation branch that is not anchored with ^, and that branch matches anywhere in the URL. ")
|
||||
case AllowListRiskCatchAll:
|
||||
b.WriteString("matches every URL. ")
|
||||
case AllowListRiskOpenHost:
|
||||
b.WriteString("does not terminate the host, so it also matches suffix hosts such as http://trusted.example.com.attacker.example/. ")
|
||||
}
|
||||
|
||||
b.WriteString(f.bypassSentence(name))
|
||||
|
||||
switch finding.Risk {
|
||||
case AllowListRiskUnanchored, AllowListRiskUnanchoredBranch:
|
||||
b.WriteString("Anchor every branch with ^ and end the host with /, :, or $.")
|
||||
case AllowListRiskCatchAll:
|
||||
b.WriteString("Restrict the entry to the hosts you trust, or unset the flag.")
|
||||
case AllowListRiskOpenHost:
|
||||
b.WriteString("End the host with /, :, $, or a group such as (:|/|$).")
|
||||
}
|
||||
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// bypassSentence names the IP-check flags an allow-list match skips, when the
|
||||
// module registers them.
|
||||
func (f *ParsedFlags) bypassSentence(name string) string {
|
||||
prefix := strings.TrimSuffix(name, allowListFlagSuffix)
|
||||
|
||||
private, public := prefix+"-deny-private-ips", prefix+"-deny-public-ips"
|
||||
if f.Lookup(private) == nil || f.Lookup(public) == nil {
|
||||
// A deprecated alias such as webhook-error-allow-list carries an extra
|
||||
// segment that the IP-check flags do not have.
|
||||
if i := strings.LastIndex(prefix, "-"); i != -1 {
|
||||
private, public = prefix[:i]+"-deny-private-ips", prefix[:i]+"-deny-public-ips"
|
||||
}
|
||||
}
|
||||
|
||||
if f.Lookup(private) == nil || f.Lookup(public) == nil {
|
||||
return "A URL that matches the allow-list skips the private and public IP checks. "
|
||||
}
|
||||
|
||||
return fmt.Sprintf(
|
||||
"A URL that matches the allow-list skips --%s (%s) and --%s (%s). ",
|
||||
private, EnvVarName(private), public, EnvVarName(public),
|
||||
)
|
||||
}
|
||||
|
||||
// EnvVarName returns the environment variable that overrides the flag given by
|
||||
// name. The entry point derives the same name when it applies environment
|
||||
// overrides, so operator-facing messages can name both without drifting.
|
||||
func EnvVarName(name string) string {
|
||||
return strings.ToUpper(strings.ReplaceAll(name, "-", "_"))
|
||||
}
|
||||
|
||||
// MustDeprecatedRegexpSlice returns the slice of compiled regular expressions
|
||||
// of a deprecated flag if it was explicitly set or the slice of the new flag.
|
||||
// It panics if an error occurs.
|
||||
|
||||
@@ -3,10 +3,11 @@ package gotenberg
|
||||
import (
|
||||
"reflect"
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/dlclark/regexp2"
|
||||
"github.com/dlclark/regexp2/v2"
|
||||
flag "github.com/spf13/pflag"
|
||||
)
|
||||
|
||||
@@ -951,3 +952,138 @@ func TestParsedFlags_MustDeprecatedRegexpSlice(t *testing.T) {
|
||||
|
||||
_ = regexp2.None // Keep import alive.
|
||||
}
|
||||
|
||||
func TestParsedFlags_AllowListWarning(t *testing.T) {
|
||||
fs := flag.NewFlagSet("tests", flag.ContinueOnError)
|
||||
fs.StringSlice("chromium-allow-list", []string{}, "")
|
||||
fs.Bool("chromium-deny-private-ips", false, "")
|
||||
fs.Bool("chromium-deny-public-ips", false, "")
|
||||
fs.StringSlice("standalone-allow-list", []string{}, "")
|
||||
|
||||
parsedFlags := ParsedFlags{FlagSet: fs}
|
||||
|
||||
for _, tc := range []struct {
|
||||
scenario string
|
||||
name string
|
||||
finding AllowListFinding
|
||||
contains []string
|
||||
}{
|
||||
{
|
||||
scenario: "open host names both IP-check flags and their env vars",
|
||||
name: "chromium-allow-list",
|
||||
finding: AllowListFinding{Index: 0, Pattern: `^https://trusted\.example\.com`, Risk: AllowListRiskOpenHost},
|
||||
contains: []string{
|
||||
"--chromium-allow-list (CHROMIUM_ALLOW_LIST)",
|
||||
"entry 1",
|
||||
`^https://trusted\.example\.com`,
|
||||
"does not terminate the host",
|
||||
"--chromium-deny-private-ips (CHROMIUM_DENY_PRIVATE_IPS)",
|
||||
"--chromium-deny-public-ips (CHROMIUM_DENY_PUBLIC_IPS)",
|
||||
"End the host with",
|
||||
},
|
||||
},
|
||||
{
|
||||
scenario: "catch-all tells the operator to restrict or unset",
|
||||
name: "chromium-allow-list",
|
||||
finding: AllowListFinding{Index: 2, Pattern: ".+", Risk: AllowListRiskCatchAll},
|
||||
contains: []string{"entry 3", "matches every URL", "Restrict the entry"},
|
||||
},
|
||||
{
|
||||
scenario: "unanchored explains the search semantics",
|
||||
name: "chromium-allow-list",
|
||||
finding: AllowListFinding{Index: 0, Pattern: `trusted\.example\.com`, Risk: AllowListRiskUnanchored},
|
||||
contains: []string{"is not anchored with ^", "Anchor every branch with ^"},
|
||||
},
|
||||
{
|
||||
scenario: "module without IP-check flags falls back to a generic sentence",
|
||||
name: "standalone-allow-list",
|
||||
finding: AllowListFinding{Index: 0, Pattern: `^https://a\.example\.com`, Risk: AllowListRiskOpenHost},
|
||||
contains: []string{"skips the private and public IP checks"},
|
||||
},
|
||||
} {
|
||||
t.Run(tc.scenario, func(t *testing.T) {
|
||||
msg := parsedFlags.allowListWarning(tc.name, tc.finding)
|
||||
|
||||
for _, want := range tc.contains {
|
||||
if !strings.Contains(msg, want) {
|
||||
t.Fatalf("message %q does not contain %q", msg, want)
|
||||
}
|
||||
}
|
||||
if strings.Contains(msg, "—") {
|
||||
t.Fatalf("message must not contain an em dash: %q", msg)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParsedFlags_WarnRiskyAllowList_SkipsDenyLists(t *testing.T) {
|
||||
fs := flag.NewFlagSet("tests", flag.ContinueOnError)
|
||||
fs.StringSlice("chromium-deny-list", []string{}, "")
|
||||
|
||||
parsedFlags := ParsedFlags{FlagSet: fs}
|
||||
|
||||
// A deny-list is never audited: it always applies and cannot be bypassed,
|
||||
// so a loose one is safe. This must also not panic on a nil logger.
|
||||
parsedFlags.warnRiskyAllowList("chromium-deny-list", []string{".+", `^file:(?!//\/tmp/).*`})
|
||||
}
|
||||
|
||||
func TestParsedFlags_WarnRiskyAllowList_NilLoggerDoesNotPanic(t *testing.T) {
|
||||
fs := flag.NewFlagSet("tests", flag.ContinueOnError)
|
||||
fs.StringSlice("chromium-allow-list", []string{}, "")
|
||||
|
||||
parsedFlags := ParsedFlags{FlagSet: fs}
|
||||
|
||||
// Provision runs after the entry point initializes the logger, but tests
|
||||
// and embedders reach this path with no logger at all.
|
||||
parsedFlags.warnRiskyAllowList("chromium-allow-list", []string{".+"})
|
||||
}
|
||||
|
||||
func TestEnvVarName(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
want string
|
||||
}{
|
||||
{"chromium-allow-list", "CHROMIUM_ALLOW_LIST"},
|
||||
{"api-download-from-deny-private-ips", "API_DOWNLOAD_FROM_DENY_PRIVATE_IPS"},
|
||||
{"log-level", "LOG_LEVEL"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := EnvVarName(tc.name); got != tc.want {
|
||||
t.Fatalf("EnvVarName(%q) = %q, want %q", tc.name, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParsedFlags_RegexpMatchTimeout(t *testing.T) {
|
||||
// [DecideOutbound] matches on these patterns directly instead of compiling
|
||||
// a private copy per call, so the bound has to come from here. regexp2's
|
||||
// own default is math.MaxInt64, which it treats as no
|
||||
// timeout at all, so a pattern built without this stamp runs unbounded
|
||||
// against a client-controlled string.
|
||||
fs := flag.NewFlagSet("tests", flag.ContinueOnError)
|
||||
fs.StringSlice("some-deny-list", []string{`^file:`, `^https?://`}, "")
|
||||
fs.String("some-pattern", `^file:`, "")
|
||||
|
||||
err := fs.Parse(nil)
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error but got: %v", err)
|
||||
}
|
||||
|
||||
parsedFlags := ParsedFlags{FlagSet: fs}
|
||||
|
||||
regexps := parsedFlags.MustRegexpSlice("some-deny-list")
|
||||
if len(regexps) != 2 {
|
||||
t.Fatalf("expected 2 patterns but got %d", len(regexps))
|
||||
}
|
||||
|
||||
for _, re := range regexps {
|
||||
if re.MatchTimeout != PatternMatchTimeout {
|
||||
t.Fatalf("pattern '%s' has MatchTimeout %s, expected %s", re.String(), re.MatchTimeout, PatternMatchTimeout)
|
||||
}
|
||||
}
|
||||
|
||||
if got := parsedFlags.MustRegexp("some-pattern").MatchTimeout; got != PatternMatchTimeout {
|
||||
t.Fatalf("expected MustRegexp MatchTimeout %s but got %s", PatternMatchTimeout, got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
|
||||
"go.opentelemetry.io/contrib/bridges/otelslog"
|
||||
@@ -74,6 +75,24 @@ func buildResource(ctx context.Context, logger *slog.Logger, serviceName, servic
|
||||
return merged
|
||||
}
|
||||
|
||||
// OTEL_*_EXPORTER select the exporter for each signal. autoexport treats an
|
||||
// unset or empty value as a request for the OTLP exporter, which then fails
|
||||
// against the default localhost:4318 endpoint when nothing listens there and,
|
||||
// for metrics, keeps retrying on the periodic reader's timer. Gotenberg keeps
|
||||
// telemetry opt-in: a signal with no exporter configured is built without one
|
||||
// and stays inert. See https://github.com/gotenberg/gotenberg/issues/1643.
|
||||
const (
|
||||
tracesExporterEnvKey = "OTEL_TRACES_EXPORTER"
|
||||
metricsExporterEnvKey = "OTEL_METRICS_EXPORTER"
|
||||
logsExporterEnvKey = "OTEL_LOGS_EXPORTER"
|
||||
)
|
||||
|
||||
// exporterConfigured reports whether the operator selected an exporter for the
|
||||
// signal owning envKey. An unset or blank value keeps that signal off.
|
||||
func exporterConfigured(envKey string) bool {
|
||||
return strings.TrimSpace(os.Getenv(envKey)) != ""
|
||||
}
|
||||
|
||||
// InitTracerProvider initializes the OpenTelemetry tracer provider.
|
||||
func InitTracerProvider(logger *slog.Logger, serviceName, serviceVersion string) (shutdown func(context.Context) error, err error) {
|
||||
initOtelLogger(logger)
|
||||
@@ -86,13 +105,14 @@ func InitTracerProvider(logger *slog.Logger, serviceName, serviceVersion string)
|
||||
trace.WithResource(res),
|
||||
}
|
||||
|
||||
traceExporter, err := autoexport.NewSpanExporter(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if !autoexport.IsNoneSpanExporter(traceExporter) {
|
||||
traceOpts = append(traceOpts, trace.WithBatcher(traceExporter))
|
||||
if exporterConfigured(tracesExporterEnvKey) {
|
||||
traceExporter, err := autoexport.NewSpanExporter(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !autoexport.IsNoneSpanExporter(traceExporter) {
|
||||
traceOpts = append(traceOpts, trace.WithBatcher(traceExporter))
|
||||
}
|
||||
}
|
||||
|
||||
traceProvider := trace.NewTracerProvider(traceOpts...)
|
||||
@@ -119,13 +139,14 @@ func InitMeterProvider(logger *slog.Logger, serviceName, serviceVersion string)
|
||||
}
|
||||
metricOpts = append(metricOpts, exemplarFilterOptions()...)
|
||||
|
||||
metricReader, err := autoexport.NewMetricReader(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if !autoexport.IsNoneMetricReader(metricReader) {
|
||||
metricOpts = append(metricOpts, metric.WithReader(metricReader))
|
||||
if exporterConfigured(metricsExporterEnvKey) {
|
||||
metricReader, err := autoexport.NewMetricReader(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !autoexport.IsNoneMetricReader(metricReader) {
|
||||
metricOpts = append(metricOpts, metric.WithReader(metricReader))
|
||||
}
|
||||
}
|
||||
|
||||
meterProvider := metric.NewMeterProvider(metricOpts...)
|
||||
@@ -157,13 +178,14 @@ func InitLoggerProvider(logger *slog.Logger, serviceName, serviceVersion string)
|
||||
log.WithResource(res),
|
||||
}
|
||||
|
||||
logExporter, err := autoexport.NewLogExporter(ctx)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
if !autoexport.IsNoneLogExporter(logExporter) {
|
||||
logOpts = append(logOpts, log.WithProcessor(log.NewBatchProcessor(logExporter)))
|
||||
if exporterConfigured(logsExporterEnvKey) {
|
||||
logExporter, err := autoexport.NewLogExporter(ctx)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if !autoexport.IsNoneLogExporter(logExporter) {
|
||||
logOpts = append(logOpts, log.WithProcessor(log.NewBatchProcessor(logExporter)))
|
||||
}
|
||||
}
|
||||
|
||||
loggerProvider := log.NewLoggerProvider(logOpts...)
|
||||
|
||||
@@ -89,6 +89,49 @@ func TestInitTracerProvider_HonorsSamplerEnv(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestExporterConfigured pins the opt-in gate: an unset or blank
|
||||
// OTEL_*_EXPORTER keeps the signal off, so Gotenberg never wires the OTLP
|
||||
// exporter that autoexport would otherwise default to and fail to reach at
|
||||
// localhost:4318. See https://github.com/gotenberg/gotenberg/issues/1643.
|
||||
func TestExporterConfigured(t *testing.T) {
|
||||
const key = "OTEL_METRICS_EXPORTER"
|
||||
|
||||
orig, had := os.LookupEnv(key)
|
||||
t.Cleanup(func() {
|
||||
if had {
|
||||
os.Setenv(key, orig)
|
||||
return
|
||||
}
|
||||
os.Unsetenv(key)
|
||||
})
|
||||
|
||||
for _, tc := range []struct {
|
||||
scenario string
|
||||
unset bool
|
||||
value string
|
||||
want bool
|
||||
}{
|
||||
{"unset", true, "", false},
|
||||
{"empty", false, "", false},
|
||||
{"whitespace only", false, " ", false},
|
||||
{"none", false, "none", true},
|
||||
{"otlp", false, "otlp", true},
|
||||
{"padded value", false, " otlp ", true},
|
||||
} {
|
||||
t.Run(tc.scenario, func(t *testing.T) {
|
||||
if tc.unset {
|
||||
os.Unsetenv(key)
|
||||
} else {
|
||||
os.Setenv(key, tc.value)
|
||||
}
|
||||
|
||||
if got := exporterConfigured(key); got != tc.want {
|
||||
t.Errorf("exporterConfigured(%q) = %v, want %v", key, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExemplarFilterOptions(t *testing.T) {
|
||||
t.Run("default pins trace-based", func(t *testing.T) {
|
||||
if v, ok := os.LookupEnv("OTEL_METRICS_EXEMPLAR_FILTER"); ok {
|
||||
|
||||
@@ -15,7 +15,8 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/dlclark/regexp2"
|
||||
"github.com/dlclark/regexp2/v2"
|
||||
"github.com/hashicorp/go-retryablehttp"
|
||||
"golang.org/x/net/http/httpproxy"
|
||||
)
|
||||
|
||||
@@ -26,6 +27,12 @@ import (
|
||||
// example [::ffff:127.0.0.1]).
|
||||
var ErrNonPublicIP = errors.New("non-public IP")
|
||||
|
||||
// ErrFiltered happens when a value is rejected by an allow-list or a
|
||||
// deny-list, or when it cannot be validated and [DecideOutbound] fails closed.
|
||||
// Callers map it to a generic 403: the specific reason stays in the operator
|
||||
// logs so a client cannot probe the lists.
|
||||
var ErrFiltered = errors.New("value filtered")
|
||||
|
||||
// ErrPublicIP indicates that an outbound URL targets an IP address that is
|
||||
// reachable on the public internet. It is returned when a caller opts
|
||||
// into denying public destinations via [WithDenyPublicIPs]; typical use
|
||||
@@ -78,6 +85,18 @@ var nonPublicIPv6Prefixes = []netip.Prefix{
|
||||
netip.MustParsePrefix("100::/64"),
|
||||
}
|
||||
|
||||
// nonPublicIPv4Prefixes lists IPv4 ranges that the [netip.Addr] helpers do
|
||||
// not classify but that must not be considered public:
|
||||
//
|
||||
// - 100.64.0.0/10 Carrier-grade NAT (RFC 6598). Routable inside provider
|
||||
// and cluster networks, and Alibaba Cloud serves instance metadata from
|
||||
// 100.100.100.200.
|
||||
// - 198.18.0.0/15 Benchmarking (RFC 2544). Never routed on the internet.
|
||||
var nonPublicIPv4Prefixes = []netip.Prefix{
|
||||
netip.MustParsePrefix("100.64.0.0/10"),
|
||||
netip.MustParsePrefix("198.18.0.0/15"),
|
||||
}
|
||||
|
||||
// IsPublicIP reports whether addr is reachable on the public internet. It
|
||||
// returns false for loopback, private (RFC1918), link-local, unspecified,
|
||||
// multicast, and unique-local addresses. IPv4-mapped IPv6 addresses are
|
||||
@@ -88,12 +107,20 @@ var nonPublicIPv6Prefixes = []netip.Prefix{
|
||||
// (6to4, Teredo, NAT64) are rejected wholesale rather than recursed into,
|
||||
// because a host that routes them implicitly trusts the IPv4 mapping and
|
||||
// the prefixes themselves are deprecated or translation-only. See
|
||||
// [nonPublicIPv6Prefixes] for the full list and rationale.
|
||||
// [nonPublicIPv6Prefixes] and [nonPublicIPv4Prefixes] for the full lists
|
||||
// and rationale.
|
||||
//
|
||||
// An IPv6 zone identifier is ignored, so [::%1] classifies the same as [::].
|
||||
func IsPublicIP(addr netip.Addr) bool {
|
||||
if !addr.IsValid() {
|
||||
return false
|
||||
}
|
||||
addr = addr.Unmap()
|
||||
// A zone does not change where a non-link-local address routes, but
|
||||
// [netip.Prefix.Contains] never matches a zoned address and
|
||||
// [netip.Addr.IsUnspecified] compares the zone too. Keeping it would let
|
||||
// http://[::%251]/ or any zoned address in the prefixes below pass as
|
||||
// public.
|
||||
addr = addr.WithZone("").Unmap()
|
||||
switch {
|
||||
case addr.IsLoopback(),
|
||||
addr.IsPrivate(),
|
||||
@@ -104,6 +131,13 @@ func IsPublicIP(addr netip.Addr) bool {
|
||||
addr.IsInterfaceLocalMulticast():
|
||||
return false
|
||||
}
|
||||
if addr.Is4() {
|
||||
for _, p := range nonPublicIPv4Prefixes {
|
||||
if p.Contains(addr) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
if addr.Is6() {
|
||||
for _, p := range nonPublicIPv6Prefixes {
|
||||
if p.Contains(addr) {
|
||||
@@ -249,7 +283,9 @@ func httpLikeScheme(scheme string) bool {
|
||||
//
|
||||
// The semantics:
|
||||
//
|
||||
// 1. The URL is parsed and its scheme and host lowercased.
|
||||
// 1. The URL is parsed, its scheme and host lowercased, and any userinfo
|
||||
// dropped from the form the regexes see. The request still carries the
|
||||
// credentials.
|
||||
// 2. allowList and denyList apply against the normalized form with OR
|
||||
// semantics. The deny-list always applies.
|
||||
// 3. For http, https, ws, and wss, the host is resolved and every
|
||||
@@ -268,26 +304,48 @@ func DecideOutbound(ctx context.Context, rawURL string, allowList, denyList []*r
|
||||
opt(&cfg)
|
||||
}
|
||||
|
||||
// Each match is bounded by [PatternMatchTimeout] rather than by the
|
||||
// remaining budget, so an already-spent deadline no longer surfaces from
|
||||
// the match itself. Schemes that resolve a host still learn about it from
|
||||
// resolveHost, but a non-matching file:// or data: URL returns before that
|
||||
// point, so check it here to keep failing closed on every path.
|
||||
if !time.Now().Before(deadline) {
|
||||
return OutboundDecision{}, context.DeadlineExceeded
|
||||
}
|
||||
|
||||
parsed, err := url.Parse(rawURL)
|
||||
if err != nil {
|
||||
return OutboundDecision{}, fmt.Errorf("parse URL %q: %w", rawURL, ErrFiltered)
|
||||
}
|
||||
parsed.Scheme = strings.ToLower(parsed.Scheme)
|
||||
parsed.Host = strings.ToLower(parsed.Host)
|
||||
normalized := parsed.String()
|
||||
|
||||
// Match on a credential-free form. [url.URL.String] re-emits userinfo
|
||||
// between "scheme://" and the host, so keeping it would let any
|
||||
// "^https?://<host>" pattern be shifted past its own anchor:
|
||||
// http://a@127.0.0.1/ escapes a deny-list anchored on 127\. and
|
||||
// http://trusted.example.com@10.0.0.1/ satisfies an allow-list anchored on
|
||||
// trusted\.example\.com. The host checks below already read
|
||||
// [url.URL.Hostname], which ignores userinfo, so only the regex layer was
|
||||
// affected. Dropping the credentials here also keeps them out of the error
|
||||
// strings below, which reach operator logs and any OTEL log exporter.
|
||||
matchable := *parsed
|
||||
matchable.User = nil
|
||||
normalized := matchable.String()
|
||||
|
||||
allowMatched := false
|
||||
if len(allowList) > 0 {
|
||||
for _, pattern := range allowList {
|
||||
clone := regexp2.MustCompile(pattern.String(), 0)
|
||||
clone.MatchTimeout = time.Until(deadline)
|
||||
|
||||
ok, err := clone.MatchString(normalized)
|
||||
ok, err := MatchPattern(pattern, normalized)
|
||||
if err != nil {
|
||||
if time.Now().After(deadline) {
|
||||
return OutboundDecision{}, context.DeadlineExceeded
|
||||
}
|
||||
return OutboundDecision{}, fmt.Errorf("'%s' cannot handle '%s': %w", clone.String(), normalized, err)
|
||||
// The pattern could not be evaluated, so the URL cannot be
|
||||
// cleared for the IP-check bypass an allow-list match grants.
|
||||
// Fail closed like an unresolvable host does below, so the
|
||||
// client gets a generic 403 rather than a 500.
|
||||
return OutboundDecision{}, fmt.Errorf("'%s' cannot handle '%s': %v: %w", pattern.String(), normalized, err, ErrFiltered)
|
||||
}
|
||||
|
||||
if ok {
|
||||
@@ -302,15 +360,15 @@ func DecideOutbound(ctx context.Context, rawURL string, allowList, denyList []*r
|
||||
}
|
||||
|
||||
for _, pattern := range denyList {
|
||||
clone := regexp2.MustCompile(pattern.String(), 0)
|
||||
clone.MatchTimeout = time.Until(deadline)
|
||||
|
||||
ok, err := clone.MatchString(normalized)
|
||||
ok, err := MatchPattern(pattern, normalized)
|
||||
if err != nil {
|
||||
if time.Now().After(deadline) {
|
||||
return OutboundDecision{}, context.DeadlineExceeded
|
||||
}
|
||||
return OutboundDecision{}, fmt.Errorf("'%s' cannot handle '%s': %w", clone.String(), normalized, err)
|
||||
// The pattern could not be evaluated, so the URL cannot be proven
|
||||
// to fall outside the deny-list. Fail closed rather than letting a
|
||||
// deny-list that never ran pass the request through.
|
||||
return OutboundDecision{}, fmt.Errorf("'%s' cannot handle '%s': %v: %w", pattern.String(), normalized, err, ErrFiltered)
|
||||
}
|
||||
|
||||
if ok {
|
||||
@@ -357,9 +415,8 @@ func DecideOutbound(ctx context.Context, rawURL string, allowList, denyList []*r
|
||||
}
|
||||
|
||||
// FilterOutboundURL validates that rawURL is acceptable for an outbound
|
||||
// request from Gotenberg. It is the URL-aware replacement for
|
||||
// [FilterDeadline] and should be preferred for any new code that filters
|
||||
// a URL before issuing or instructing an outbound request.
|
||||
// request from Gotenberg. Prefer it for any new code that filters a URL
|
||||
// before issuing or instructing an outbound request.
|
||||
//
|
||||
// The default behavior is permissive: the URL passes as long as it clears
|
||||
// the regex allow-list and deny-list. Callers that need IP-class checks
|
||||
@@ -436,6 +493,16 @@ func (rt *outboundRoundTripper) RoundTrip(req *http.Request) (*http.Response, er
|
||||
// gate this behind their module's opt-in flag. See
|
||||
// https://github.com/gotenberg/gotenberg/issues/1592.
|
||||
func NewOutboundHttpClient(timeout time.Duration, allowList, denyList []*regexp2.Regexp, enableEnvironmentProxy bool, opts ...DecideOption) *http.Client {
|
||||
// A negative timeout means the caller's budget is already spent, which
|
||||
// happens when it derives one from a deadline that has passed. [http.Client]
|
||||
// treats any non-positive Timeout as no deadline at all, so passing it
|
||||
// through would silently produce an unbounded client. Fail closed instead.
|
||||
// Zero keeps meaning unbounded: callers that own the connection lifetime
|
||||
// themselves pass it deliberately.
|
||||
if timeout < 0 {
|
||||
timeout = time.Nanosecond
|
||||
}
|
||||
|
||||
base := http.DefaultTransport.(*http.Transport).Clone()
|
||||
|
||||
var proxyFunc func(*url.URL) (*url.URL, error)
|
||||
@@ -479,6 +546,25 @@ func NewOutboundHttpClient(timeout time.Duration, allowList, denyList []*regexp2
|
||||
}
|
||||
}
|
||||
|
||||
// ClampedBackoff is a [retryablehttp.Backoff] that honors max on every path.
|
||||
//
|
||||
// [retryablehttp.DefaultBackoff] returns a Retry-After header from the remote
|
||||
// verbatim for 429 and 503, and returns it before applying its own max clamp.
|
||||
// A hostile origin therefore decides how long Gotenberg waits, and the wait is
|
||||
// not interruptible. Retry-After is still respected here, just never beyond
|
||||
// the ceiling the caller set.
|
||||
func ClampedBackoff(min, max time.Duration, attemptNum int, resp *http.Response) time.Duration {
|
||||
wait := retryablehttp.DefaultBackoff(min, max, attemptNum, resp)
|
||||
if wait > max {
|
||||
return max
|
||||
}
|
||||
if wait < 0 {
|
||||
return 0
|
||||
}
|
||||
|
||||
return wait
|
||||
}
|
||||
|
||||
// environmentProxyVariables are the variables golang.org/x/net/http/httpproxy
|
||||
// reads, in the casing precedence it applies.
|
||||
var environmentProxyVariables = []string{
|
||||
|
||||
@@ -3,11 +3,13 @@ package gotenberg
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/netip"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/dlclark/regexp2"
|
||||
"github.com/dlclark/regexp2/v2"
|
||||
)
|
||||
|
||||
func TestIsPublicIP(t *testing.T) {
|
||||
@@ -38,6 +40,24 @@ func TestIsPublicIP(t *testing.T) {
|
||||
|
||||
// Link-local.
|
||||
{"169.254.169.254", false},
|
||||
{"169.254.170.2", false},
|
||||
|
||||
// Carrier-grade NAT (RFC 6598). Alibaba Cloud serves instance
|
||||
// metadata from 100.100.100.200.
|
||||
{"100.64.0.0", false},
|
||||
{"100.100.100.200", false},
|
||||
{"100.127.255.255", false},
|
||||
{"::ffff:100.100.100.200", false},
|
||||
|
||||
// Benchmarking (RFC 2544).
|
||||
{"198.18.0.1", false},
|
||||
{"198.19.255.255", false},
|
||||
|
||||
// Adjacent to the ranges above, and public.
|
||||
{"100.63.255.255", true},
|
||||
{"100.128.0.0", true},
|
||||
{"198.17.255.255", true},
|
||||
{"198.20.0.0", true},
|
||||
{"fe80::1", false},
|
||||
|
||||
// Unique-local.
|
||||
@@ -83,6 +103,23 @@ func TestIsPublicIP(t *testing.T) {
|
||||
|
||||
// Discard prefix (RFC 6666).
|
||||
{"100::1", false},
|
||||
|
||||
// A zone identifier must not change the classification.
|
||||
{"::%1", false},
|
||||
{"::%lo", false},
|
||||
{"::1%1", false},
|
||||
{"fe80::1%eth0", false},
|
||||
{"fc00::1%1", false},
|
||||
{"::ffff:127.0.0.1%1", false},
|
||||
{"fec0::1%eth0", false},
|
||||
{"2002:a9fe:a9fe::%1", false},
|
||||
{"2001:0:abcd:ef12:3456:7890:a9fe:a9fe%1", false},
|
||||
{"64:ff9b::a9fe:a9fe%1", false},
|
||||
{"64:ff9b:1::a9fe:a9fe%1", false},
|
||||
{"::a9fe:a9fe%1", false},
|
||||
{"2001:db8::1%1", false},
|
||||
{"100::1%1", false},
|
||||
{"2606:4700:4700::1111%1", true},
|
||||
} {
|
||||
t.Run(tc.addr, func(t *testing.T) {
|
||||
addr, err := netip.ParseAddr(tc.addr)
|
||||
@@ -127,10 +164,10 @@ func mustAddrs(t *testing.T, ss ...string) []netip.Addr {
|
||||
|
||||
func TestFilterOutboundURL(t *testing.T) {
|
||||
defaultDeny := []*regexp2.Regexp{
|
||||
regexp2.MustCompile(`^https?://(10\.|172\.(1[6-9]|2[0-9]|3[01])\.|192\.168\.|169\.254\.|0\.0\.0\.0|127\.|localhost|\[::1\]|\[fd)`, 0),
|
||||
regexp2.MustCompile(`^https?://(10\.|172\.(1[6-9]|2[0-9]|3[01])\.|192\.168\.|169\.254\.|0\.0\.0\.0|127\.|localhost|\[::1\]|\[fd)`, regexp2.None),
|
||||
}
|
||||
chromiumDeny := []*regexp2.Regexp{
|
||||
regexp2.MustCompile(`^file:(?!//\/tmp/).*`, 0),
|
||||
regexp2.MustCompile(`^file:(?!//\/tmp/).*`, regexp2.None),
|
||||
}
|
||||
|
||||
for _, tc := range []struct {
|
||||
@@ -209,7 +246,7 @@ func TestFilterOutboundURL(t *testing.T) {
|
||||
{
|
||||
scenario: "allow-list match bypasses IP check",
|
||||
rawURL: "http://internal.service/api",
|
||||
allow: []*regexp2.Regexp{regexp2.MustCompile(`^http://internal\.service`, 0)},
|
||||
allow: []*regexp2.Regexp{regexp2.MustCompile(`^http://internal\.service`, regexp2.None)},
|
||||
deny: defaultDeny,
|
||||
opts: []DecideOption{WithDenyPrivateIPs(true)},
|
||||
expectErr: false,
|
||||
@@ -217,15 +254,15 @@ func TestFilterOutboundURL(t *testing.T) {
|
||||
{
|
||||
scenario: "deny-list still wins over allow-list match",
|
||||
rawURL: "http://internal.service/api",
|
||||
allow: []*regexp2.Regexp{regexp2.MustCompile(`^http://internal`, 0)},
|
||||
deny: []*regexp2.Regexp{regexp2.MustCompile(`/api$`, 0)},
|
||||
allow: []*regexp2.Regexp{regexp2.MustCompile(`^http://internal`, regexp2.None)},
|
||||
deny: []*regexp2.Regexp{regexp2.MustCompile(`/api$`, regexp2.None)},
|
||||
expectErr: true,
|
||||
expectIs: ErrFiltered,
|
||||
},
|
||||
{
|
||||
scenario: "allow-list non-empty and no match rejects",
|
||||
rawURL: "https://other.example/",
|
||||
allow: []*regexp2.Regexp{regexp2.MustCompile(`^https://allowed\.example`, 0)},
|
||||
allow: []*regexp2.Regexp{regexp2.MustCompile(`^https://allowed\.example`, regexp2.None)},
|
||||
expectErr: true,
|
||||
expectIs: ErrFiltered,
|
||||
},
|
||||
@@ -284,6 +321,22 @@ func TestFilterOutboundURL(t *testing.T) {
|
||||
expectErr: true,
|
||||
expectIs: ErrFiltered,
|
||||
},
|
||||
{
|
||||
scenario: "zoned unspecified address blocked with deny-private-ips",
|
||||
rawURL: "http://[::%251]:9999/",
|
||||
deny: defaultDeny,
|
||||
opts: []DecideOption{WithDenyPrivateIPs(true)},
|
||||
expectErr: true,
|
||||
expectIs: ErrFiltered,
|
||||
},
|
||||
{
|
||||
scenario: "zoned site-local address blocked with deny-private-ips",
|
||||
rawURL: "http://[fec0:1234::3%25eth0]:8080/",
|
||||
deny: defaultDeny,
|
||||
opts: []DecideOption{WithDenyPrivateIPs(true)},
|
||||
expectErr: true,
|
||||
expectIs: ErrFiltered,
|
||||
},
|
||||
} {
|
||||
t.Run(tc.scenario, func(t *testing.T) {
|
||||
if tc.stub != nil {
|
||||
@@ -486,7 +539,7 @@ func TestDecideOutbound_DenyLists_WinOverDenyPrivateIPs(t *testing.T) {
|
||||
// The regex deny-list fires before any resolution; verifies that
|
||||
// operator-supplied deny patterns remain effective regardless of
|
||||
// IP-class options.
|
||||
deny := []*regexp2.Regexp{regexp2.MustCompile(`^http://evil\.`, 0)}
|
||||
deny := []*regexp2.Regexp{regexp2.MustCompile(`^http://evil\.`, regexp2.None)}
|
||||
|
||||
_, err := DecideOutbound(
|
||||
context.Background(),
|
||||
@@ -520,3 +573,236 @@ func TestDecideOutbound_Permissive_AllowsPrivate(t *testing.T) {
|
||||
t.Fatalf("decision.Pinned = %v, want [10.0.0.5]", decision.Pinned)
|
||||
}
|
||||
}
|
||||
|
||||
// privateIPsDenyList is the textual private-IP deny-list that shipped as the
|
||||
// default for api-download-from-deny-list and webhook-deny-list in v8.31.0 and
|
||||
// is still published as a migration recipe. Every alternative is anchored on
|
||||
// "://", so userinfo used to slide the private address past the anchor.
|
||||
const privateIPsDenyList = `^https?://(10\.|172\.(1[6-9]|2[0-9]|3[01])\.|192\.168\.|169\.254\.|0\.0\.0\.0|127\.|localhost|\[::1\]|\[fd)`
|
||||
|
||||
func TestDecideOutbound_UserinfoDoesNotEvadeDenyList(t *testing.T) {
|
||||
for _, rawURL := range []string{
|
||||
"http://127.0.0.1:9999/",
|
||||
"http://a@127.0.0.1:9999/",
|
||||
"http://@127.0.0.1:9999/",
|
||||
"http://:@127.0.0.1:9999/",
|
||||
"http://%61@127.0.0.1:9999/",
|
||||
"http://user:pass@127.0.0.1:9999/",
|
||||
"HTTP://A@127.0.0.1:9999/",
|
||||
"http://a@169.254.169.254/latest/meta-data/",
|
||||
// url.Parse takes the last "@" as the userinfo separator, so the host
|
||||
// here is the second literal.
|
||||
"http://a@127.0.0.1:9999@127.0.0.1:9999/",
|
||||
} {
|
||||
t.Run(rawURL, func(t *testing.T) {
|
||||
withStubResolver(t, func(host string) ([]netip.Addr, error) {
|
||||
t.Fatalf("unexpected DNS lookup for %q: the deny-list must reject before resolution", host)
|
||||
return nil, nil
|
||||
})
|
||||
|
||||
// Deny-list only, with the permissive IP defaults the modules ship.
|
||||
_, err := DecideOutbound(
|
||||
context.Background(),
|
||||
rawURL,
|
||||
nil,
|
||||
[]*regexp2.Regexp{regexp2.MustCompile(privateIPsDenyList, regexp2.None)},
|
||||
time.Now().Add(5*time.Second),
|
||||
)
|
||||
if !errors.Is(err, ErrFiltered) {
|
||||
t.Fatalf("userinfo must not evade the deny-list, got: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecideOutbound_UserinfoDoesNotSatisfyAllowList(t *testing.T) {
|
||||
// A host-terminated allow-list, the shape the documentation recommends.
|
||||
allowList := []*regexp2.Regexp{regexp2.MustCompile(`^https://trusted\.example\.com(:[0-9]+)?(/|$)`, regexp2.None)}
|
||||
|
||||
for _, rawURL := range []string{
|
||||
"https://trusted.example.com@169.254.169.254/latest/meta-data/",
|
||||
"https://trusted.example.com@10.0.0.5/",
|
||||
"https://trusted.example.com:443@10.0.0.5/",
|
||||
} {
|
||||
t.Run(rawURL, func(t *testing.T) {
|
||||
withStubResolver(t, func(host string) ([]netip.Addr, error) {
|
||||
return mustAddrs(t, "10.0.0.5"), nil
|
||||
})
|
||||
|
||||
decision, err := DecideOutbound(
|
||||
context.Background(),
|
||||
rawURL,
|
||||
allowList, nil,
|
||||
time.Now().Add(5*time.Second),
|
||||
WithDenyPrivateIPs(true),
|
||||
)
|
||||
if err == nil {
|
||||
t.Fatalf("userinfo must not satisfy the allow-list, got decision %+v", decision)
|
||||
}
|
||||
if decision.Bypass {
|
||||
t.Fatal("userinfo must never produce a bypass")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecideOutbound_UserinfoKeptOutOfErrorMessages(t *testing.T) {
|
||||
withStubResolver(t, func(host string) ([]netip.Addr, error) {
|
||||
t.Fatalf("unexpected DNS lookup for %q", host)
|
||||
return nil, nil
|
||||
})
|
||||
|
||||
_, err := DecideOutbound(
|
||||
context.Background(),
|
||||
"http://alice:hunter2@127.0.0.1:9999/",
|
||||
nil,
|
||||
[]*regexp2.Regexp{regexp2.MustCompile(privateIPsDenyList, regexp2.None)},
|
||||
time.Now().Add(5*time.Second),
|
||||
)
|
||||
if err == nil {
|
||||
t.Fatal("expected the URL to be filtered")
|
||||
}
|
||||
if strings.Contains(err.Error(), "hunter2") || strings.Contains(err.Error(), "alice") {
|
||||
t.Fatalf("error message must not leak URL credentials: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecideOutbound_LegitimateCredentialsStillReachTheHost(t *testing.T) {
|
||||
withStubResolver(t, func(host string) ([]netip.Addr, error) {
|
||||
if host != "example.com" {
|
||||
t.Fatalf("host = %q, want example.com: userinfo must not reach resolution", host)
|
||||
}
|
||||
return mustAddrs(t, "93.184.216.34"), nil
|
||||
})
|
||||
|
||||
// Stripping userinfo is a matching concern only. A credentialed URL that
|
||||
// breaks no rule must still be allowed through.
|
||||
decision, err := DecideOutbound(
|
||||
context.Background(),
|
||||
"https://alice:hunter2@example.com/report.pdf",
|
||||
[]*regexp2.Regexp{regexp2.MustCompile(`^https://example\.com(:[0-9]+)?(/|$)`, regexp2.None)},
|
||||
nil,
|
||||
time.Now().Add(5*time.Second),
|
||||
WithDenyPrivateIPs(true),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("credentialed URL matching the allow-list must pass, got: %v", err)
|
||||
}
|
||||
if !decision.Bypass {
|
||||
t.Fatalf("decision.Bypass = false, want true")
|
||||
}
|
||||
}
|
||||
|
||||
func TestClampedBackoff(t *testing.T) {
|
||||
const (
|
||||
min = 1 * time.Second
|
||||
max = 30 * time.Second
|
||||
)
|
||||
|
||||
retryAfter := func(status int, seconds string) *http.Response {
|
||||
return &http.Response{StatusCode: status, Header: http.Header{"Retry-After": []string{seconds}}}
|
||||
}
|
||||
|
||||
for _, tc := range []struct {
|
||||
scenario string
|
||||
resp *http.Response
|
||||
want time.Duration
|
||||
}{
|
||||
{"429 with an hour is clamped", retryAfter(http.StatusTooManyRequests, "3600"), max},
|
||||
{"429 with a day is clamped", retryAfter(http.StatusTooManyRequests, "86400"), max},
|
||||
{"503 with an hour is clamped", retryAfter(http.StatusServiceUnavailable, "3600"), max},
|
||||
{"429 under the ceiling is honored", retryAfter(http.StatusTooManyRequests, "5"), 5 * time.Second},
|
||||
{"no response falls back to exponential", nil, min},
|
||||
} {
|
||||
t.Run(tc.scenario, func(t *testing.T) {
|
||||
got := ClampedBackoff(min, max, 0, tc.resp)
|
||||
if got != tc.want {
|
||||
t.Fatalf("ClampedBackoff = %s, want %s", got, tc.want)
|
||||
}
|
||||
if got > max {
|
||||
t.Fatalf("ClampedBackoff = %s, which exceeds max %s", got, max)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A negative max means the caller's budget is spent. The backoff must not
|
||||
// return a negative duration, which would make the retry loop spin.
|
||||
func TestClampedBackoff_NegativeMaxIsNotNegative(t *testing.T) {
|
||||
got := ClampedBackoff(1*time.Second, -5*time.Second, 0, nil)
|
||||
if got < 0 {
|
||||
t.Fatalf("ClampedBackoff = %s, want a non-negative duration", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewOutboundHttpClient_NonPositiveTimeout(t *testing.T) {
|
||||
// Zero stays unbounded: the LibreOffice proxy owns its own lifetime and
|
||||
// passes it deliberately.
|
||||
if got := NewOutboundHttpClient(0, nil, nil, false).Timeout; got != 0 {
|
||||
t.Fatalf("timeout for 0 = %s, want 0", got)
|
||||
}
|
||||
|
||||
// Negative means an expired budget. http.Client reads any non-positive
|
||||
// Timeout as no deadline at all, so it must not be passed through.
|
||||
if got := NewOutboundHttpClient(-5*time.Second, nil, nil, false).Timeout; got <= 0 {
|
||||
t.Fatalf("timeout for a negative budget = %s, want a positive value so the client fails closed", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecideOutboundExpiredDeadline(t *testing.T) {
|
||||
// Patterns are matched under the fixed PatternMatchTimeout rather than
|
||||
// under the caller's remaining budget, so an expired deadline no longer
|
||||
// surfaces from the match itself. Every scheme must still fail closed,
|
||||
// including the ones that return before a host is resolved.
|
||||
expired := time.Now().Add(-time.Second)
|
||||
|
||||
for _, rawURL := range []string{
|
||||
"https://example.com/",
|
||||
"file:///tmp/foo.html",
|
||||
"data:text/html,hello",
|
||||
} {
|
||||
_, err := DecideOutbound(context.Background(), rawURL, nil, nil, expired)
|
||||
if !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Fatalf("DecideOutbound(%q) with an expired deadline = %v, want context.DeadlineExceeded", rawURL, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecideOutboundBoundsCatastrophicPatterns(t *testing.T) {
|
||||
// A deny-list pattern that backtracks catastrophically, matched against a
|
||||
// client-controlled URL. Before PatternMatchTimeout the ceiling was the
|
||||
// caller's whole budget, so a 30s API_TIMEOUT bought a 30s CPU burn.
|
||||
// The trailing "!" makes the match fail only after the nested quantifier
|
||||
// has explored every way to split the run of "a"s.
|
||||
pattern := regexp2.MustCompile(`^https://example\.com/(a+)+$`, regexp2.None)
|
||||
pattern.MatchTimeout = PatternMatchTimeout
|
||||
|
||||
rawURL := "https://example.com/" + strings.Repeat("a", 40) + "!"
|
||||
|
||||
start := time.Now()
|
||||
_, err := DecideOutbound(
|
||||
context.Background(),
|
||||
rawURL,
|
||||
nil,
|
||||
[]*regexp2.Regexp{pattern},
|
||||
time.Now().Add(30*time.Second),
|
||||
)
|
||||
elapsed := time.Since(start)
|
||||
|
||||
if err == nil {
|
||||
t.Fatal("expected an error from a catastrophic deny-list pattern")
|
||||
}
|
||||
|
||||
// A deny-list that could not be evaluated cannot clear the URL, so the
|
||||
// decision fails closed and the client gets a generic 403 rather than a
|
||||
// 500 naming the pattern.
|
||||
if !errors.Is(err, ErrFiltered) {
|
||||
t.Fatalf("expected ErrFiltered from an unevaluable deny-list pattern but got: %v", err)
|
||||
}
|
||||
|
||||
// Generous headroom over the 250ms ceiling, still far below the 30s
|
||||
// deadline the match would otherwise have been allowed to consume.
|
||||
if elapsed > 5*time.Second {
|
||||
t.Fatalf("match took %s, want it aborted near PatternMatchTimeout (%s)", elapsed, PatternMatchTimeout)
|
||||
}
|
||||
}
|
||||
|
||||
55
pkg/gotenberg/pattern.go
Normal file
55
pkg/gotenberg/pattern.go
Normal file
@@ -0,0 +1,55 @@
|
||||
package gotenberg
|
||||
|
||||
import (
|
||||
"github.com/dlclark/regexp2/v2"
|
||||
)
|
||||
|
||||
// patternMatchAttempts caps how many times [MatchPattern] runs one pattern
|
||||
// against one string. It is what keeps a pattern that is genuinely out of
|
||||
// budget from retrying forever: three attempts bound its cost at three
|
||||
// [PatternMatchTimeout], which is still two orders of magnitude below the
|
||||
// --api-timeout (env API_TIMEOUT) the ceiling exists to protect.
|
||||
const patternMatchAttempts = 3
|
||||
|
||||
// MatchPattern reports whether s matches pattern. It bounds the match by the
|
||||
// pattern's MatchTimeout without the false timeouts that the bound alone
|
||||
// produces.
|
||||
//
|
||||
// regexp2 does not time a match against [time.Now]. It derives the deadline
|
||||
// from a process-global clock that a background goroutine advances every
|
||||
// 100ms, and it tests that deadline on the very first step of the match.
|
||||
// Anything that stops the whole process, a cgroup CPU-quota throttle or a long
|
||||
// stop-the-world pause, also stops that goroutine, which then advances the
|
||||
// clock by the full pause in a single write. A match holding a deadline from
|
||||
// before that jump aborts whatever work it had done: a 366ns match against a
|
||||
// short URL reports "match timeout after 250ms". The abort lands on whichever
|
||||
// match straddles the jump rather than on a match that was slow, which is why
|
||||
// it fires on an idle instance and against Gotenberg's own file:///tmp/ URLs.
|
||||
// See https://github.com/gotenberg/gotenberg/issues/1659.
|
||||
//
|
||||
// Retrying separates the two cases. Catastrophic backtracking is
|
||||
// deterministic: the same pattern against the same string exhausts the same
|
||||
// budget on every attempt, so a genuine runaway still aborts, and costs at
|
||||
// most patternMatchAttempts ceilings to prove it. A clock-induced abort needs
|
||||
// the process to lose the CPU inside one specific match, which the next
|
||||
// attempt does not reproduce.
|
||||
//
|
||||
// Elapsed time cannot make that call instead. A match frozen mid-flight
|
||||
// reports the freeze as its own cost, 806ms against a 250ms ceiling in one
|
||||
// measured run, so it is indistinguishable by wall clock from a match that
|
||||
// really did spend its budget. Go exposes no per-goroutine CPU time, and
|
||||
// process CPU time counts every other request in flight.
|
||||
func MatchPattern(pattern *regexp2.Regexp, s string) (bool, error) {
|
||||
var err error
|
||||
|
||||
for range patternMatchAttempts {
|
||||
var ok bool
|
||||
|
||||
ok, err = pattern.MatchString(s)
|
||||
if err == nil {
|
||||
return ok, nil
|
||||
}
|
||||
}
|
||||
|
||||
return false, err
|
||||
}
|
||||
99
pkg/gotenberg/pattern_test.go
Normal file
99
pkg/gotenberg/pattern_test.go
Normal file
@@ -0,0 +1,99 @@
|
||||
package gotenberg
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/dlclark/regexp2/v2"
|
||||
)
|
||||
|
||||
// mustPattern compiles a pattern the way the production lists are built.
|
||||
func mustPattern(t *testing.T, expr string) *regexp2.Regexp {
|
||||
t.Helper()
|
||||
|
||||
re := regexp2.MustCompile(expr, regexp2.None)
|
||||
re.MatchTimeout = PatternMatchTimeout
|
||||
|
||||
return re
|
||||
}
|
||||
|
||||
func TestMatchPattern(t *testing.T) {
|
||||
// The abort [MatchPattern] absorbs cannot be staged here: it needs the
|
||||
// whole process to lose the CPU around one specific match, which no test
|
||||
// can schedule. What is testable is the other half of the contract, that
|
||||
// retrying never turns a genuine runaway into a pass.
|
||||
// See https://github.com/gotenberg/gotenberg/issues/1659.
|
||||
for _, tc := range []struct {
|
||||
scenario string
|
||||
pattern *regexp2.Regexp
|
||||
s string
|
||||
expectMatch bool
|
||||
expectError bool
|
||||
}{
|
||||
{
|
||||
scenario: "deny-list match",
|
||||
pattern: mustPattern(t, `^file:(?!//\/tmp/).*`),
|
||||
s: "file:///etc/passwd",
|
||||
expectMatch: true,
|
||||
},
|
||||
{
|
||||
scenario: "no match against Gotenberg's own working directory",
|
||||
pattern: mustPattern(t, `^file:(?!//\/tmp/).*`),
|
||||
s: "file:///tmp/1a2b3c4d/5e6f7a8b/9c0d1e2f.html",
|
||||
expectMatch: false,
|
||||
},
|
||||
{
|
||||
scenario: "no match",
|
||||
pattern: mustPattern(t, `^https://example\.com/`),
|
||||
s: "https://example.org/",
|
||||
expectMatch: false,
|
||||
},
|
||||
{
|
||||
scenario: "catastrophic backtracking still aborts",
|
||||
pattern: mustPattern(t, `^https://example\.com/(a+)+$`),
|
||||
s: "https://example.com/" + strings.Repeat("a", 40) + "!",
|
||||
expectError: true,
|
||||
},
|
||||
} {
|
||||
t.Run(tc.scenario, func(t *testing.T) {
|
||||
ok, err := MatchPattern(tc.pattern, tc.s)
|
||||
|
||||
if tc.expectError && err == nil {
|
||||
t.Fatal("expected an error but got none")
|
||||
}
|
||||
|
||||
if !tc.expectError && err != nil {
|
||||
t.Fatalf("expected no error but got: %v", err)
|
||||
}
|
||||
|
||||
if ok != tc.expectMatch {
|
||||
t.Fatalf("expected match %t but got %t", tc.expectMatch, ok)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMatchPatternBoundsCatastrophicPattern(t *testing.T) {
|
||||
// Proving a runaway is genuine costs one ceiling per attempt, so the
|
||||
// worst case is patternMatchAttempts of them plus regexp2's clock period
|
||||
// on each, roughly a second. The bound that matters is the one this
|
||||
// replaced: before the ceiling existed, the same match was allowed to
|
||||
// burn a core for the caller's whole 30s budget.
|
||||
pattern := mustPattern(t, `^https://example\.com/(a+)+$`)
|
||||
s := "https://example.com/" + strings.Repeat("a", 40) + "!"
|
||||
|
||||
start := time.Now()
|
||||
_, err := MatchPattern(pattern, s)
|
||||
elapsed := time.Since(start)
|
||||
|
||||
if err == nil {
|
||||
t.Fatal("expected an error from a catastrophic pattern")
|
||||
}
|
||||
|
||||
// Generous headroom over the expected second keeps this stable on a
|
||||
// loaded CI box while still failing if the bound is gone.
|
||||
if elapsed > 5*time.Second {
|
||||
t.Fatalf("match took %s, want at most %d ceilings of %s", elapsed, patternMatchAttempts, PatternMatchTimeout)
|
||||
}
|
||||
}
|
||||
@@ -201,17 +201,13 @@ func TestNewServerRecordMetrics(t *testing.T) {
|
||||
server.RecordMetrics(t.Context(), semconv.ServerMetricData{
|
||||
ServerName: "stuff",
|
||||
ResponseSize: 200,
|
||||
MetricAttributes: semconv.MetricAttributes{
|
||||
Req: req,
|
||||
StatusCode: 301,
|
||||
AdditionalAttributes: []attribute.KeyValue{
|
||||
attribute.String("key", "value"),
|
||||
},
|
||||
},
|
||||
MetricData: semconv.MetricData{
|
||||
RequestSize: 100,
|
||||
ElapsedTime: 300,
|
||||
Req: req,
|
||||
StatusCode: 301,
|
||||
AdditionalAttributes: []attribute.KeyValue{
|
||||
attribute.String("key", "value"),
|
||||
},
|
||||
RequestSize: 100,
|
||||
ElapsedTime: 300,
|
||||
})
|
||||
|
||||
rm := metricdata.ResourceMetrics{}
|
||||
|
||||
@@ -59,8 +59,9 @@ type ProcessSupervisor interface {
|
||||
// Healthy checks and returns the health status of the managed [Process].
|
||||
//
|
||||
// A non-started process is considered healthy (startup is deferred until
|
||||
// the first request). Returns false if the process is currently restarting
|
||||
// or is reported unhealthy by the underlying [Process].
|
||||
// the first request), as is one going through a planned restart, since it
|
||||
// keeps serving traffic. Returns false during an unplanned restart or when
|
||||
// the underlying [Process] reports unhealthy.
|
||||
Healthy() bool
|
||||
|
||||
// Run executes a provided task while managing the state of the [Process].
|
||||
@@ -103,6 +104,28 @@ const healthCheckCacheTTL = 2 * time.Second
|
||||
// this. See https://github.com/gotenberg/gotenberg/issues/1561.
|
||||
const healthFailureThreshold = 2
|
||||
|
||||
// Restart reasons, also reported as the gotenberg.process.start.reason span
|
||||
// attribute by [processSupervisor.tracedLaunch]. Only
|
||||
// [restartReasonMaxRequests] is a planned restart: it fires on a healthy
|
||||
// process that reached its conversion limit, so the node keeps serving
|
||||
// traffic throughout. The others signal a process that cannot serve.
|
||||
const (
|
||||
restartReasonFirstStart = "first_start"
|
||||
restartReasonUnhealthy = "unhealthy"
|
||||
restartReasonMaxRequests = "max_requests"
|
||||
)
|
||||
|
||||
// defaultEagerRestartTimeout bounds the restart triggered after the maximum
|
||||
// request limit. That restart runs on a background context, unlike the one from
|
||||
// ensureHealthy which inherits the request deadline, so without a deadline of
|
||||
// its own the drain loop in [processSupervisor.doRestartLocked] would wait
|
||||
// forever on a task that never completes. That would pin isRestarting and,
|
||||
// with it, the health reported by [processSupervisor.Healthy]. Sized well above
|
||||
// --api-timeout (30s by default) plus the engine start timeouts (20s by
|
||||
// default) so it never fires while tasks are merely slow. The eager restart is
|
||||
// opportunistic: on expiry it aborts, and the next task retries it.
|
||||
const defaultEagerRestartTimeout = 2 * time.Minute
|
||||
|
||||
type processSupervisor struct {
|
||||
logger *slog.Logger
|
||||
engine string
|
||||
@@ -118,11 +141,16 @@ type processSupervisor struct {
|
||||
// transient failure (such as a cold-start timeout) must not poison the
|
||||
// supervisor for the rest of the container's lifetime. See
|
||||
// https://github.com/gotenberg/gotenberg/issues/1538.
|
||||
firstStartMu sync.Mutex
|
||||
reqCounter atomic.Int64
|
||||
reqQueueSize atomic.Int64
|
||||
restartsCounter atomic.Int64
|
||||
isRestarting atomic.Bool
|
||||
firstStartMu sync.Mutex
|
||||
reqCounter atomic.Int64
|
||||
reqQueueSize atomic.Int64
|
||||
restartsCounter atomic.Int64
|
||||
isRestarting atomic.Bool
|
||||
// restartPlanned records whether the in-flight restart is a planned one
|
||||
// (see [restartReasonMaxRequests]). Written before isRestarting and never
|
||||
// cleared, so a reader that observed isRestarting always sees the matching
|
||||
// kind. See [processSupervisor.Healthy].
|
||||
restartPlanned atomic.Bool
|
||||
activeTasks atomic.Int64
|
||||
restartMutex sync.Mutex
|
||||
idleShutdownTimeout time.Duration
|
||||
@@ -135,6 +163,9 @@ type processSupervisor struct {
|
||||
consecutiveHealthFailures atomic.Int64 // reset to 0 on every successful probe
|
||||
idleMu sync.Mutex // protects idleStopChan
|
||||
idleStopChan chan struct{} // signal to stop the idle ticker goroutine
|
||||
// eagerRestartTimeout bounds the restart from maybeRestartAfterTask.
|
||||
// Defaults to [defaultEagerRestartTimeout]; only tests shorten it.
|
||||
eagerRestartTimeout time.Duration
|
||||
}
|
||||
|
||||
// NewProcessSupervisor initializes a new [ProcessSupervisor]. engine names the
|
||||
@@ -158,6 +189,7 @@ func NewProcessSupervisor(logger *slog.Logger, engine string, process Process, m
|
||||
maxQueueSize: maxQueueSize,
|
||||
maxConcurrency: maxConcurrency,
|
||||
idleShutdownTimeout: idleShutdownTimeout,
|
||||
eagerRestartTimeout: defaultEagerRestartTimeout,
|
||||
}
|
||||
b.reqCounter.Store(0)
|
||||
b.reqQueueSize.Store(0)
|
||||
@@ -212,12 +244,18 @@ func (s *processSupervisor) restart() error {
|
||||
s.logger.WarnContext(context.Background(), fmt.Sprintf("stop process before restart: %s", err))
|
||||
}
|
||||
|
||||
// Reset the counter on the attempt, not on its outcome. Leaving it at the
|
||||
// limit after a failed launch re-triggers maybeRestartAfterTask on every
|
||||
// subsequent task, producing back-to-back restarts. Recovering a process
|
||||
// that will not start is ensureHealthy's job: it restarts synchronously
|
||||
// before running a task, and reports the failure to the caller.
|
||||
s.reqCounter.Store(0)
|
||||
|
||||
err = s.Launch()
|
||||
if err != nil {
|
||||
return fmt.Errorf("restart process: %w", err)
|
||||
}
|
||||
|
||||
s.reqCounter.Store(0)
|
||||
s.restartsCounter.Add(1)
|
||||
s.logger.DebugContext(context.Background(), "process successfully restarted")
|
||||
|
||||
@@ -234,9 +272,17 @@ func (s *processSupervisor) Healthy() bool {
|
||||
}
|
||||
|
||||
if s.isRestarting.Load() {
|
||||
// A restarting process is not yet healthy. This gives load balancers
|
||||
// honest information so they can avoid routing traffic to this node.
|
||||
return false
|
||||
// A planned restart is routine maintenance: the process reached the
|
||||
// limit set by --chromium-restart-after (env CHROMIUM_RESTART_AFTER) or
|
||||
// --libreoffice-restart-after (env LIBREOFFICE_RESTART_AFTER) while
|
||||
// healthy. Tasks arriving during it are requeued by acquireSlot, not
|
||||
// rejected, so the node still serves traffic and must report healthy. A
|
||||
// probe sent between two conversions used to fail here.
|
||||
// See https://github.com/gotenberg/gotenberg/issues/1648.
|
||||
//
|
||||
// An unplanned restart keeps reporting unhealthy, which gives load
|
||||
// balancers honest information so they can avoid routing traffic here.
|
||||
return s.restartPlanned.Load()
|
||||
}
|
||||
|
||||
// Cache hit: a recent probe succeeded. Skip the CDP roundtrip so probe
|
||||
@@ -484,7 +530,7 @@ func (s *processSupervisor) ensureStarted(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
err := s.tracedLaunch(ctx, "first_start", func() error {
|
||||
err := s.tracedLaunch(ctx, restartReasonFirstStart, func() error {
|
||||
return s.runWithDeadline(ctx, s.Launch)
|
||||
})
|
||||
if err != nil {
|
||||
@@ -525,7 +571,7 @@ func (s *processSupervisor) ensureHealthy(ctx context.Context) error {
|
||||
|
||||
s.logger.DebugContext(context.Background(), "process is unhealthy, cannot handle task, restarting...")
|
||||
|
||||
if err := s.doRestart(ctx, "unhealthy"); err != nil {
|
||||
if err := s.doRestart(ctx, restartReasonUnhealthy); err != nil {
|
||||
return fmt.Errorf("process restart before task: %w", err)
|
||||
}
|
||||
|
||||
@@ -533,9 +579,10 @@ func (s *processSupervisor) ensureHealthy(ctx context.Context) error {
|
||||
}
|
||||
|
||||
// maybeRestartAfterTask checks if the maximum request limit has been reached
|
||||
// and, if so, triggers an asynchronous restart. If a restart is initiated, it
|
||||
// takes ownership of the caller's semaphore slot (the caller must not release
|
||||
// it). Returns true if ownership was taken.
|
||||
// and, if so, triggers an asynchronous restart bounded by
|
||||
// [defaultEagerRestartTimeout]. If a restart is initiated, it takes ownership
|
||||
// of the caller's semaphore slot (the caller must not release it). Returns true
|
||||
// if ownership was taken.
|
||||
func (s *processSupervisor) maybeRestartAfterTask(logger *slog.Logger) bool {
|
||||
if s.maxReqLimit <= 0 || s.reqCounter.Load() < s.maxReqLimit {
|
||||
return false
|
||||
@@ -548,7 +595,10 @@ func (s *processSupervisor) maybeRestartAfterTask(logger *slog.Logger) bool {
|
||||
s.logger.DebugContext(context.Background(), "max request limit reached, restarting eagerly...")
|
||||
|
||||
go func() {
|
||||
restartErr := s.doRestartLocked(context.Background(), "max_requests")
|
||||
ctx, cancel := context.WithTimeout(context.Background(), s.eagerRestartTimeout)
|
||||
defer cancel()
|
||||
|
||||
restartErr := s.doRestartLocked(ctx, restartReasonMaxRequests)
|
||||
s.restartMutex.Unlock()
|
||||
if restartErr != nil {
|
||||
s.logger.ErrorContext(context.Background(), fmt.Sprintf("process restart after task: %v", restartErr))
|
||||
@@ -571,6 +621,10 @@ func (s *processSupervisor) doRestart(ctx context.Context, reason string) error
|
||||
|
||||
// doRestartLocked performs the restart drain logic. The caller must hold restartMutex.
|
||||
func (s *processSupervisor) doRestartLocked(ctx context.Context, reason string) error {
|
||||
// Publish the kind before raising the flag. [processSupervisor.Healthy]
|
||||
// reads restartPlanned only after it observes isRestarting, so this
|
||||
// ordering keeps it from pairing a new restart with a stale kind.
|
||||
s.restartPlanned.Store(reason == restartReasonMaxRequests)
|
||||
s.isRestarting.Store(true)
|
||||
defer s.isRestarting.Store(false)
|
||||
|
||||
|
||||
@@ -160,13 +160,49 @@ func TestProcessSupervisor_restart(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestProcessSupervisor_restart_ResetsCounterOnFailedLaunch verifies that a
|
||||
// restart whose launch fails still clears the request counter. Leaving it at
|
||||
// the limit makes maybeRestartAfterTask re-fire on every subsequent task.
|
||||
func TestProcessSupervisor_restart_ResetsCounterOnFailedLaunch(t *testing.T) {
|
||||
logger := slog.New(slog.DiscardHandler)
|
||||
|
||||
const maxReqLimit = 5
|
||||
|
||||
process := &ProcessMock{
|
||||
StartMock: func(_ *slog.Logger) error { return errors.New("start error") },
|
||||
StopMock: func(_ *slog.Logger) error { return nil },
|
||||
HealthyMock: func(_ *slog.Logger) bool { return true },
|
||||
}
|
||||
|
||||
ps := NewProcessSupervisor(logger, "test", process, maxReqLimit, 0, 1, 0).(*processSupervisor)
|
||||
ps.reqCounter.Store(maxReqLimit)
|
||||
|
||||
err := ps.restart()
|
||||
if err == nil {
|
||||
t.Fatal("expected error but got none")
|
||||
}
|
||||
|
||||
if got := ps.reqCounter.Load(); got != 0 {
|
||||
t.Fatalf("expected the request counter to be reset but got %d", got)
|
||||
}
|
||||
|
||||
if got := ps.restartsCounter.Load(); got != 0 {
|
||||
t.Fatalf("expected the restarts counter to stay at 0 but got %d", got)
|
||||
}
|
||||
|
||||
if ps.maybeRestartAfterTask(logger) {
|
||||
t.Fatal("expected no further eager restart to be triggered")
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessSupervisor_Healthy(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
scenario string
|
||||
initiallyStarted bool
|
||||
initiallyRestarting bool
|
||||
processHealthy bool
|
||||
expectHealthy bool
|
||||
scenario string
|
||||
initiallyStarted bool
|
||||
initiallyRestarting bool
|
||||
initiallyRestartPlanned bool
|
||||
processHealthy bool
|
||||
expectHealthy bool
|
||||
}{
|
||||
{
|
||||
scenario: "non-started process is healthy",
|
||||
@@ -179,6 +215,13 @@ func TestProcessSupervisor_Healthy(t *testing.T) {
|
||||
initiallyRestarting: true,
|
||||
expectHealthy: false,
|
||||
},
|
||||
{
|
||||
scenario: "process going through a planned restart is healthy",
|
||||
initiallyStarted: true,
|
||||
initiallyRestarting: true,
|
||||
initiallyRestartPlanned: true,
|
||||
expectHealthy: true,
|
||||
},
|
||||
{
|
||||
scenario: "process reports as healthy",
|
||||
initiallyStarted: true,
|
||||
@@ -208,6 +251,9 @@ func TestProcessSupervisor_Healthy(t *testing.T) {
|
||||
if tc.initiallyRestarting {
|
||||
ps.isRestarting.Store(true)
|
||||
}
|
||||
if tc.initiallyRestartPlanned {
|
||||
ps.restartPlanned.Store(true)
|
||||
}
|
||||
|
||||
healthy := ps.Healthy()
|
||||
|
||||
@@ -257,6 +303,204 @@ func TestProcessSupervisor_Healthy_ConsecutiveFailures(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestProcessSupervisor_Healthy_PlannedRestart reproduces
|
||||
// https://github.com/gotenberg/gotenberg/issues/1648. It drives the real
|
||||
// Run() path until the maximum request limit triggers the eager restart, then
|
||||
// asserts the supervisor reports healthy while that restart is in flight.
|
||||
// Tasks arriving during it are requeued by acquireSlot, not rejected, so the
|
||||
// node still serves traffic.
|
||||
func TestProcessSupervisor_Healthy_PlannedRestart(t *testing.T) {
|
||||
logger := slog.New(slog.DiscardHandler)
|
||||
|
||||
const maxReqLimit = 10
|
||||
|
||||
restarting := make(chan struct{})
|
||||
release := make(chan struct{})
|
||||
|
||||
var (
|
||||
starts atomic.Int64
|
||||
signalOne sync.Once
|
||||
)
|
||||
process := &ProcessMock{
|
||||
StartMock: func(_ *slog.Logger) error {
|
||||
// Hold the restart open so the assertions below run inside the
|
||||
// window that used to report unhealthy.
|
||||
if starts.Add(1) > 1 {
|
||||
signalOne.Do(func() { close(restarting) })
|
||||
<-release
|
||||
}
|
||||
|
||||
return nil
|
||||
},
|
||||
StopMock: func(_ *slog.Logger) error { return nil },
|
||||
HealthyMock: func(_ *slog.Logger) bool { return true },
|
||||
}
|
||||
|
||||
ps := NewProcessSupervisor(logger, "test", process, maxReqLimit, 0, 1, 0).(*processSupervisor)
|
||||
|
||||
for i := range maxReqLimit {
|
||||
err := ps.Run(context.Background(), logger, func() error { return nil })
|
||||
if err != nil {
|
||||
t.Fatalf("task %d: expected no error but got: %v", i+1, err)
|
||||
}
|
||||
}
|
||||
|
||||
select {
|
||||
case <-restarting:
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatalf("expected an eager restart after %d tasks", maxReqLimit)
|
||||
}
|
||||
|
||||
if !ps.isRestarting.Load() {
|
||||
t.Fatal("expected the supervisor to be restarting")
|
||||
}
|
||||
|
||||
if !ps.restartPlanned.Load() {
|
||||
t.Fatal("expected the restart to be flagged as planned")
|
||||
}
|
||||
|
||||
if !ps.Healthy() {
|
||||
t.Fatal("expected a planned restart to report healthy")
|
||||
}
|
||||
|
||||
close(release)
|
||||
}
|
||||
|
||||
// TestProcessSupervisor_Healthy_UnplannedRestart verifies the counterpart of
|
||||
// [TestProcessSupervisor_Healthy_PlannedRestart]: a restart triggered by an
|
||||
// unhealthy process keeps reporting unhealthy, so load balancers get honest
|
||||
// information.
|
||||
func TestProcessSupervisor_Healthy_UnplannedRestart(t *testing.T) {
|
||||
logger := slog.New(slog.DiscardHandler)
|
||||
|
||||
restarting := make(chan struct{})
|
||||
release := make(chan struct{})
|
||||
|
||||
var signalOne sync.Once
|
||||
process := &ProcessMock{
|
||||
StartMock: func(_ *slog.Logger) error {
|
||||
signalOne.Do(func() { close(restarting) })
|
||||
<-release
|
||||
|
||||
return nil
|
||||
},
|
||||
StopMock: func(_ *slog.Logger) error { return nil },
|
||||
HealthyMock: func(_ *slog.Logger) bool { return false },
|
||||
}
|
||||
|
||||
ps := NewProcessSupervisor(logger, "test", process, 0, 0, 1, 0).(*processSupervisor)
|
||||
ps.firstStart.Store(true)
|
||||
|
||||
go func() {
|
||||
_ = ps.ensureHealthy(context.Background())
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-restarting:
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("expected an unhealthy restart to be triggered")
|
||||
}
|
||||
|
||||
if !ps.isRestarting.Load() {
|
||||
t.Fatal("expected the supervisor to be restarting")
|
||||
}
|
||||
|
||||
if ps.restartPlanned.Load() {
|
||||
t.Fatal("expected the restart not to be flagged as planned")
|
||||
}
|
||||
|
||||
if ps.Healthy() {
|
||||
t.Fatal("expected an unplanned restart to report unhealthy")
|
||||
}
|
||||
|
||||
close(release)
|
||||
}
|
||||
|
||||
// TestProcessSupervisor_doRestartLocked_DrainDeadline verifies that a drain
|
||||
// unable to acquire every slot gives up on the context deadline and clears
|
||||
// isRestarting. Without a deadline on the eager restart, a task that never
|
||||
// completes would pin the flag and, since a planned restart reports healthy,
|
||||
// leave the supervisor claiming health forever.
|
||||
func TestProcessSupervisor_doRestartLocked_DrainDeadline(t *testing.T) {
|
||||
logger := slog.New(slog.DiscardHandler)
|
||||
|
||||
var starts atomic.Int64
|
||||
process := &ProcessMock{
|
||||
StartMock: func(_ *slog.Logger) error {
|
||||
starts.Add(1)
|
||||
|
||||
return nil
|
||||
},
|
||||
StopMock: func(_ *slog.Logger) error { return nil },
|
||||
HealthyMock: func(_ *slog.Logger) bool { return true },
|
||||
}
|
||||
|
||||
// A concurrency of 2 makes the drain acquire one slot on top of the one the
|
||||
// triggering task hands over. Fill the semaphore so it never can, mimicking
|
||||
// a concurrent task that never completes.
|
||||
ps := NewProcessSupervisor(logger, "test", process, 1, 0, 2, 0).(*processSupervisor)
|
||||
ps.semaphore <- struct{}{}
|
||||
ps.semaphore <- struct{}{}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
|
||||
defer cancel()
|
||||
|
||||
err := ps.doRestartLocked(ctx, restartReasonMaxRequests)
|
||||
if err == nil {
|
||||
t.Fatal("expected the drain to fail on the context deadline")
|
||||
}
|
||||
|
||||
if ps.isRestarting.Load() {
|
||||
t.Fatal("expected isRestarting to be cleared after a failed drain")
|
||||
}
|
||||
|
||||
if starts.Load() != 0 {
|
||||
t.Fatalf("expected no restart attempt after a failed drain but got %d", starts.Load())
|
||||
}
|
||||
}
|
||||
|
||||
// TestProcessSupervisor_maybeRestartAfterTask_Bounded verifies that the eager
|
||||
// restart runs under a deadline. A concurrent task that never completes blocks
|
||||
// the drain, and without a bound the restart goroutine would wait forever with
|
||||
// isRestarting pinned, leaving Healthy() reporting a planned restart for good.
|
||||
func TestProcessSupervisor_maybeRestartAfterTask_Bounded(t *testing.T) {
|
||||
logger := slog.New(slog.DiscardHandler)
|
||||
|
||||
process := &ProcessMock{
|
||||
StartMock: func(_ *slog.Logger) error { return nil },
|
||||
StopMock: func(_ *slog.Logger) error { return nil },
|
||||
HealthyMock: func(_ *slog.Logger) bool { return true },
|
||||
}
|
||||
|
||||
ps := NewProcessSupervisor(logger, "test", process, 1, 0, 2, 0).(*processSupervisor)
|
||||
ps.eagerRestartTimeout = 100 * time.Millisecond
|
||||
ps.firstStart.Store(true)
|
||||
|
||||
// Wedge one slot so the drain, which needs one on top of the slot the
|
||||
// triggering task hands over, can never complete.
|
||||
ps.semaphore <- struct{}{}
|
||||
|
||||
err := ps.Run(context.Background(), logger, func() error { return nil })
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error but got: %v", err)
|
||||
}
|
||||
|
||||
waitFor := func(what string, want bool) {
|
||||
t.Helper()
|
||||
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for ps.isRestarting.Load() != want {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("timed out waiting for the eager restart to %s", what)
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
waitFor("start", true)
|
||||
waitFor("give up on its deadline", false)
|
||||
}
|
||||
|
||||
// TestProcessSupervisor_Healthy_CachesPositiveResult verifies that a
|
||||
// successful probe is cached for [healthCheckCacheTTL] so subsequent
|
||||
// supervisor.Healthy() calls do not re-issue the underlying process
|
||||
|
||||
@@ -12,10 +12,9 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/alexliesenfeld/health"
|
||||
"github.com/dlclark/regexp2"
|
||||
"github.com/labstack/echo/v4"
|
||||
"github.com/dlclark/regexp2/v2"
|
||||
"github.com/labstack/echo/v5"
|
||||
flag "github.com/spf13/pflag"
|
||||
"golang.org/x/net/http2"
|
||||
"golang.org/x/sync/errgroup"
|
||||
|
||||
"github.com/gotenberg/gotenberg/v8/pkg/gotenberg"
|
||||
@@ -58,6 +57,12 @@ type Api struct {
|
||||
fs *gotenberg.FileSystem
|
||||
logger *slog.Logger
|
||||
srv *echo.Echo
|
||||
|
||||
// shutdownCancel triggers the graceful shutdown of the server started by
|
||||
// [echo.StartConfig]. Echo v5 drives shutdown from a context instead of an
|
||||
// Echo.Shutdown method. serveDone closes once that shutdown completed.
|
||||
shutdownCancel context.CancelFunc
|
||||
serveDone chan struct{}
|
||||
}
|
||||
|
||||
type downloadFromConfig struct {
|
||||
@@ -67,6 +72,8 @@ type downloadFromConfig struct {
|
||||
denyPublicIPs bool
|
||||
enableEnvironmentProxy bool
|
||||
maxRetry int
|
||||
maxConcurrency int
|
||||
maxEntries int
|
||||
disable bool
|
||||
}
|
||||
|
||||
@@ -132,7 +139,7 @@ const (
|
||||
// middleware := Middleware{
|
||||
// Handler: func() echo.MiddlewareFunc {
|
||||
// return func(next echo.HandlerFunc) echo.HandlerFunc {
|
||||
// return func(c echo.Context) error {
|
||||
// return func(c *echo.Context) error {
|
||||
// rootPath := c.Get("rootPath").(string)
|
||||
// healthURI := fmt.Sprintf("%shealth", rootPath)
|
||||
//
|
||||
@@ -206,12 +213,14 @@ func (a *Api) Descriptor() gotenberg.ModuleDescriptor {
|
||||
fs.String("api-oidc-issuer", "", "Set the OIDC issuer URL, e.g. https://tenant.example.com/ - the token 'iss' claim must match")
|
||||
fs.String("api-oidc-audience", "", "Set the expected OIDC audience - the token 'aud' claim must contain it")
|
||||
fs.String("api-oidc-jwks-url", "", "Set the OIDC JWKS URL - discovered from the issuer's well-known configuration when empty")
|
||||
fs.StringSlice("api-download-from-allow-list", []string{}, "Set the allowed URLs for the download from feature using regular expressions - supports multiple values")
|
||||
fs.StringSlice("api-download-from-allow-list", []string{}, `Set the allowed URLs for the download from feature using regular expressions - supports multiple values. A match bypasses --api-download-from-deny-private-ips (API_DOWNLOAD_FROM_DENY_PRIVATE_IPS) and --api-download-from-deny-public-ips (API_DOWNLOAD_FROM_DENY_PUBLIC_IPS), so terminate the host or the pattern also matches suffix hosts, for example ^https?://internal\.svc(:|/|$)`)
|
||||
fs.StringSlice("api-download-from-deny-list", []string{}, "Set the denied URLs for the download from feature using regular expressions - supports multiple values")
|
||||
fs.Bool("api-download-from-deny-private-ips", false, "Reject downloadFrom URLs whose host resolves to a non-public IP address (loopback, RFC1918, link-local, unique-local). Enable on deployments that accept untrusted downloadFrom sources to mitigate SSRF against internal services")
|
||||
fs.Bool("api-download-from-deny-public-ips", false, "Reject downloadFrom URLs whose host resolves to a public IP address. Enable on air-gapped or data-governed deployments to prevent downloads from reaching the public internet")
|
||||
fs.Bool("api-download-from-enable-environment-proxy", false, "Route downloadFrom fetches through the proxy defined by the standard HTTP_PROXY, HTTPS_PROXY, and NO_PROXY variables, including credentials")
|
||||
fs.Int("api-download-from-max-retry", 4, "Set the maximum number of retries for the download from feature")
|
||||
fs.Int("api-download-from-max-concurrency", 10, "Set the maximum number of downloadFrom entries fetched concurrently per request - bounds the outbound fan-out. Set to 0 to disable this feature")
|
||||
fs.Int("api-download-from-max-entries", 0, "Set the maximum number of downloadFrom entries allowed per request. Set to 0 to disable this feature")
|
||||
fs.Bool("api-disable-download-from", false, "Disable the download from feature")
|
||||
fs.Bool("api-disable-health-check-route-telemetry", true, "Disable telemetry for health check route")
|
||||
fs.Bool("api-disable-root-route-telemetry", true, "Disable telemetry for the root route")
|
||||
@@ -255,6 +264,8 @@ func (a *Api) Provision(ctx *gotenberg.Context) error {
|
||||
denyPublicIPs: flags.MustBool("api-download-from-deny-public-ips"),
|
||||
enableEnvironmentProxy: flags.MustBool("api-download-from-enable-environment-proxy"),
|
||||
maxRetry: flags.MustInt("api-download-from-max-retry"),
|
||||
maxConcurrency: flags.MustInt("api-download-from-max-concurrency"),
|
||||
maxEntries: flags.MustInt("api-download-from-max-entries"),
|
||||
disable: flags.MustBool("api-disable-download-from"),
|
||||
}
|
||||
a.disableHealthCheckRouteTelemetry = flags.MustDeprecatedBool("api-disable-health-check-logging", "api-disable-health-check-route-telemetry")
|
||||
@@ -377,12 +388,37 @@ func (a *Api) Provision(ctx *gotenberg.Context) error {
|
||||
// Logger.
|
||||
a.logger = gotenberg.Logger(a)
|
||||
|
||||
a.warnInsecureDebugRoute()
|
||||
|
||||
// File system.
|
||||
a.fs = gotenberg.NewFileSystem(new(gotenberg.OsMkdirAll))
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// warnInsecureDebugRoute logs a warning when the debug route is reachable
|
||||
// without authentication.
|
||||
//
|
||||
// The route reports the resolved configuration of every module, which is
|
||||
// useful to an operator and equally useful to anyone else who can reach it.
|
||||
// This warns rather than refuses: an operator may sit behind a gateway that
|
||||
// authenticates on Gotenberg's behalf, and failing startup would break them.
|
||||
func (a *Api) warnInsecureDebugRoute() {
|
||||
if !a.enableDebugRoute || a.basicAuthUsername != "" || a.oidcEnabled {
|
||||
return
|
||||
}
|
||||
if a.logger == nil {
|
||||
return
|
||||
}
|
||||
|
||||
a.logger.WarnContext(
|
||||
context.Background(),
|
||||
"--api-enable-debug-route (API_ENABLE_DEBUG_ROUTE) is enabled but no authentication is configured, so anyone who can reach Gotenberg can read its configuration. Set --api-enable-basic-auth (API_ENABLE_BASIC_AUTH) with GOTENBERG_API_BASIC_AUTH_USERNAME and GOTENBERG_API_BASIC_AUTH_PASSWORD, set --api-enable-oidc-auth (API_ENABLE_OIDC_AUTH), or disable the route.",
|
||||
slog.String("flag", "--api-enable-debug-route"),
|
||||
slog.String("env", "API_ENABLE_DEBUG_ROUTE"),
|
||||
)
|
||||
}
|
||||
|
||||
// Validate validates the module properties.
|
||||
func (a *Api) Validate() error {
|
||||
var err error
|
||||
@@ -404,6 +440,18 @@ func (a *Api) Validate() error {
|
||||
}
|
||||
}
|
||||
|
||||
if a.downloadFromCfg.maxConcurrency < 0 {
|
||||
err = errors.Join(err,
|
||||
fmt.Errorf("download from max concurrency must not be negative, got %d; set --api-download-from-max-concurrency (env API_DOWNLOAD_FROM_MAX_CONCURRENCY) to 0 to disable the limit", a.downloadFromCfg.maxConcurrency),
|
||||
)
|
||||
}
|
||||
|
||||
if a.downloadFromCfg.maxEntries < 0 {
|
||||
err = errors.Join(err,
|
||||
fmt.Errorf("download from max entries must not be negative, got %d; set --api-download-from-max-entries (env API_DOWNLOAD_FROM_MAX_ENTRIES) to 0 to disable the limit", a.downloadFromCfg.maxEntries),
|
||||
)
|
||||
}
|
||||
|
||||
if (a.tlsCertFile != "" && a.tlsKeyFile == "") || (a.tlsCertFile == "" && a.tlsKeyFile != "") {
|
||||
err = errors.Join(err,
|
||||
errors.New("both TLS certificate and key files must be set"),
|
||||
@@ -493,16 +541,29 @@ func (a *Api) Validate() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// newEchoServer builds the [echo.Echo] instance with the settings Gotenberg
|
||||
// relies on, independently of the routes and middlewares added by [Api.Start].
|
||||
func newEchoServer() *echo.Echo {
|
||||
srv := echo.New()
|
||||
srv.HTTPErrorHandler = httpErrorHandler()
|
||||
// Echo v5 serves files through Echo.Filesystem, an [fs.FS] rooted at the
|
||||
// working directory, and [fs.FS] rejects absolute names. Every output file
|
||||
// lives under the request's temporary directory, so Context.Attachment gets
|
||||
// an absolute path and the default filesystem answers 404 for every
|
||||
// conversion. Rooting at "/" restores the v4 behavior, where the path was
|
||||
// opened as-is.
|
||||
srv.Filesystem = echo.NewDefaultFS("/")
|
||||
// Echo v5 dropped the X-Forwarded-For and X-Real-IP fallbacks from
|
||||
// Context.RealIP. Keep the previous behavior so that the access log still
|
||||
// reports the client IP when Gotenberg sits behind a reverse proxy.
|
||||
srv.IPExtractor = echo.LegacyIPExtractor()
|
||||
|
||||
return srv
|
||||
}
|
||||
|
||||
// Start starts the HTTP server.
|
||||
func (a *Api) Start() error {
|
||||
a.srv = echo.New()
|
||||
a.srv.HideBanner = true
|
||||
a.srv.HidePort = true
|
||||
a.srv.Server.ReadTimeout = a.timeout
|
||||
a.srv.Server.IdleTimeout = a.timeout
|
||||
// See https://github.com/gotenberg/gotenberg/issues/396.
|
||||
a.srv.Server.WriteTimeout = a.timeout + a.timeout
|
||||
a.srv.HTTPErrorHandler = httpErrorHandler()
|
||||
a.srv = newEchoServer()
|
||||
|
||||
// Let's prepare the modules' routes.
|
||||
var disableTelemetryForPaths []string
|
||||
@@ -566,7 +627,7 @@ func (a *Api) Start() error {
|
||||
securityMiddleware = oidcAuthMiddleware(verifier)
|
||||
default:
|
||||
securityMiddleware = func(next echo.HandlerFunc) echo.HandlerFunc {
|
||||
return func(c echo.Context) error {
|
||||
return func(c *echo.Context) error {
|
||||
return next(c)
|
||||
}
|
||||
}
|
||||
@@ -598,7 +659,7 @@ func (a *Api) Start() error {
|
||||
// Root route.
|
||||
a.srv.GET(
|
||||
a.rootPath,
|
||||
func(c echo.Context) error {
|
||||
func(c *echo.Context) error {
|
||||
return c.HTML(http.StatusOK, `Hey, Gotenberg has no UI, it's an API. Head to the <a href="https://gotenberg.dev">documentation</a> to learn how to interact with it 🚀`)
|
||||
},
|
||||
securityMiddleware,
|
||||
@@ -607,7 +668,7 @@ func (a *Api) Start() error {
|
||||
// Favicon route.
|
||||
a.srv.GET(
|
||||
fmt.Sprintf("%s%s", a.rootPath, "favicon.ico"),
|
||||
func(c echo.Context) error {
|
||||
func(c *echo.Context) error {
|
||||
return c.NoContent(http.StatusNoContent)
|
||||
},
|
||||
securityMiddleware,
|
||||
@@ -638,7 +699,7 @@ func (a *Api) Start() error {
|
||||
// ...the version route.
|
||||
a.srv.GET(
|
||||
fmt.Sprintf("%s%s", a.rootPath, "version"),
|
||||
func(c echo.Context) error {
|
||||
func(c *echo.Context) error {
|
||||
return c.String(http.StatusOK, gotenberg.Version)
|
||||
},
|
||||
securityMiddleware,
|
||||
@@ -648,7 +709,7 @@ func (a *Api) Start() error {
|
||||
if a.enableDebugRoute {
|
||||
a.srv.GET(
|
||||
fmt.Sprintf("%s%s", a.rootPath, "debug"),
|
||||
func(c echo.Context) error {
|
||||
func(c *echo.Context) error {
|
||||
return c.JSONPretty(http.StatusOK, gotenberg.Debug(), " ")
|
||||
},
|
||||
securityMiddleware,
|
||||
@@ -669,18 +730,50 @@ func (a *Api) Start() error {
|
||||
return fmt.Errorf("waiting for modules readiness: %w", err)
|
||||
}
|
||||
|
||||
startConfig := echo.StartConfig{
|
||||
Address: fmt.Sprintf("%s:%d", a.bindIp, a.port),
|
||||
HideBanner: true,
|
||||
HidePort: true,
|
||||
BeforeServeFunc: func(s *http.Server) error {
|
||||
s.ReadTimeout = a.timeout
|
||||
s.IdleTimeout = a.timeout
|
||||
// See https://github.com/gotenberg/gotenberg/issues/396.
|
||||
s.WriteTimeout = a.timeout + a.timeout
|
||||
|
||||
if a.tlsCertFile == "" || a.tlsKeyFile == "" {
|
||||
// Serve HTTP/2 Cleartext (h2c). Echo v5 dropped
|
||||
// StartH2CServer and golang.org/x/net/http2/h2c is deprecated,
|
||||
// so the standard library serves h2c through Server.Protocols
|
||||
// instead.
|
||||
protocols := new(http.Protocols)
|
||||
protocols.SetHTTP1(true)
|
||||
protocols.SetUnencryptedHTTP2(true)
|
||||
s.Protocols = protocols
|
||||
}
|
||||
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
// Not named cancel: that would reassign the readiness timeout's cancel from
|
||||
// above, whose deferred call is already bound to the old value.
|
||||
serveCtx, serveCancel := context.WithCancel(context.Background())
|
||||
a.shutdownCancel = serveCancel
|
||||
a.serveDone = make(chan struct{})
|
||||
|
||||
// As the following code is blocking, run it in a goroutine.
|
||||
go func() {
|
||||
defer close(a.serveDone)
|
||||
|
||||
var err error
|
||||
if a.tlsCertFile != "" && a.tlsKeyFile != "" {
|
||||
// Start an HTTPS server (supports HTTP/2).
|
||||
err = a.srv.StartTLS(fmt.Sprintf("%s:%d", a.bindIp, a.port), a.tlsCertFile, a.tlsKeyFile)
|
||||
err = startConfig.StartTLS(serveCtx, a.srv, a.tlsCertFile, a.tlsKeyFile)
|
||||
} else {
|
||||
// Start an HTTP/2 Cleartext (non-HTTPS) server.
|
||||
server := &http2.Server{}
|
||||
err = a.srv.StartH2CServer(fmt.Sprintf("%s:%d", a.bindIp, a.port), server)
|
||||
err = startConfig.Start(serveCtx, a.srv)
|
||||
}
|
||||
if !errors.Is(err, http.ErrServerClosed) {
|
||||
if err != nil && !errors.Is(err, http.ErrServerClosed) {
|
||||
a.logger.ErrorContext(context.Background(), err.Error())
|
||||
}
|
||||
}()
|
||||
@@ -697,6 +790,26 @@ func (a *Api) StartupMessage() string {
|
||||
return fmt.Sprintf("server started on %s:%d", ip, a.port)
|
||||
}
|
||||
|
||||
// shutdown triggers the server's graceful shutdown and waits for it to
|
||||
// complete, or for ctx to be done. Cancelling the serve context is what Echo v5
|
||||
// exposes in place of Echo.Shutdown: [echo.StartConfig] shuts the server down
|
||||
// on its own deadline, so the shutdown proceeds even when ctx is already done.
|
||||
func (a *Api) shutdown(ctx context.Context) error {
|
||||
if a.shutdownCancel == nil {
|
||||
// Start never ran, so there is nothing to shut down.
|
||||
return nil
|
||||
}
|
||||
|
||||
a.shutdownCancel()
|
||||
|
||||
select {
|
||||
case <-a.serveDone:
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
// Stop stops the HTTP server.
|
||||
func (a *Api) Stop(ctx context.Context) error {
|
||||
for {
|
||||
@@ -706,7 +819,7 @@ func (a *Api) Stop(ctx context.Context) error {
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return a.srv.Shutdown(ctx)
|
||||
return a.shutdown(ctx)
|
||||
default:
|
||||
a.logger.DebugContext(ctx, fmt.Sprintf("%d asynchronous requests", count))
|
||||
if count > 0 {
|
||||
@@ -714,7 +827,7 @@ func (a *Api) Stop(ctx context.Context) error {
|
||||
continue
|
||||
}
|
||||
a.logger.DebugContext(ctx, "no more asynchronous requests, continue with shutdown")
|
||||
err := a.srv.Shutdown(ctx)
|
||||
err := a.shutdown(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("shutdown: %w", err)
|
||||
}
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
@@ -65,3 +67,68 @@ func TestApi_Validate_Auth(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestApi_warnInsecureDebugRoute(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
scenario string
|
||||
api Api
|
||||
expectWarn bool
|
||||
expectFields []string
|
||||
}{
|
||||
{
|
||||
scenario: "debug route on with no auth warns",
|
||||
api: Api{enableDebugRoute: true},
|
||||
expectWarn: true,
|
||||
expectFields: []string{"--api-enable-debug-route", "API_ENABLE_DEBUG_ROUTE", "--api-enable-basic-auth", "API_ENABLE_BASIC_AUTH", "--api-enable-oidc-auth", "API_ENABLE_OIDC_AUTH"},
|
||||
},
|
||||
{
|
||||
scenario: "debug route off is silent",
|
||||
api: Api{enableDebugRoute: false},
|
||||
expectWarn: false,
|
||||
},
|
||||
{
|
||||
scenario: "basic auth silences it",
|
||||
api: Api{enableDebugRoute: true, basicAuthUsername: "foo"},
|
||||
expectWarn: false,
|
||||
},
|
||||
{
|
||||
scenario: "oidc silences it",
|
||||
api: Api{enableDebugRoute: true, oidcEnabled: true},
|
||||
expectWarn: false,
|
||||
},
|
||||
} {
|
||||
t.Run(tc.scenario, func(t *testing.T) {
|
||||
buf := new(bytes.Buffer)
|
||||
tc.api.logger = slog.New(slog.NewJSONHandler(buf, &slog.HandlerOptions{Level: slog.LevelWarn}))
|
||||
|
||||
tc.api.warnInsecureDebugRoute()
|
||||
|
||||
logged := buf.String()
|
||||
if !tc.expectWarn {
|
||||
if logged != "" {
|
||||
t.Fatalf("expected no warning, got: %s", logged)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if logged == "" {
|
||||
t.Fatal("expected a warning, got none")
|
||||
}
|
||||
// Every flag named must carry its environment variable.
|
||||
for _, want := range tc.expectFields {
|
||||
if !strings.Contains(logged, want) {
|
||||
t.Fatalf("warning does not mention %q: %s", want, logged)
|
||||
}
|
||||
}
|
||||
if strings.Contains(logged, "—") {
|
||||
t.Fatalf("warning must not contain an em dash: %s", logged)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The warning reads a.logger, which is nil until Provision assigns it.
|
||||
func TestApi_warnInsecureDebugRoute_NilLoggerDoesNotPanic(t *testing.T) {
|
||||
api := Api{enableDebugRoute: true}
|
||||
api.warnInsecureDebugRoute()
|
||||
}
|
||||
|
||||
@@ -18,7 +18,7 @@ import (
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/hashicorp/go-retryablehttp"
|
||||
"github.com/labstack/echo/v4"
|
||||
"github.com/labstack/echo/v5"
|
||||
"github.com/mholt/archives"
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
@@ -51,8 +51,26 @@ type Context struct {
|
||||
outputPaths []string
|
||||
cancelled bool
|
||||
|
||||
// fileOrder records the order files were received in, keyed by disk path.
|
||||
// It breaks ties when two uploads share an original filename, so that
|
||||
// de-duplicated files keep their upload order instead of being ordered by
|
||||
// the suffix uniqueFilename added.
|
||||
fileOrder map[string]int
|
||||
|
||||
// fileBase maps a disk path to the original filename as received, before
|
||||
// de-duplication. Sorting on it keeps a de-duplicated file next to its
|
||||
// twin rather than wherever its numbered name would land.
|
||||
fileBase map[string]string
|
||||
|
||||
// outputFilename is the sanitized Gotenberg-Output-Filename header,
|
||||
// snapshotted while the [echo.Context] is still live. Echo returns that
|
||||
// context to a pool as soon as the handler returns, and an asynchronous
|
||||
// conversion outlives it, so reading the header from the pooled store later
|
||||
// yields whichever request happens to own it by then.
|
||||
outputFilename string
|
||||
|
||||
logger *slog.Logger
|
||||
echoCtx echo.Context
|
||||
echoCtx *echo.Context
|
||||
mkdirAll gotenberg.MkdirAll
|
||||
pathRename gotenberg.PathRename
|
||||
context.Context
|
||||
@@ -80,6 +98,48 @@ func (t *trackingReader) Read(p []byte) (int, error) {
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// errTooManyDownloadFromEntries is returned by [decodeDownloadFrom] when the
|
||||
// array holds more entries than the configured maximum.
|
||||
var errTooManyDownloadFromEntries = errors.New("too many downloadFrom entries")
|
||||
|
||||
// decodeDownloadFrom decodes the downloadFrom form field, refusing to
|
||||
// accumulate more than maxEntries. A maxEntries of 0 means no limit.
|
||||
//
|
||||
// It decodes element by element rather than calling [json.Unmarshal] on the
|
||||
// whole value. A compact array such as "[{},{},{}]" costs three bytes per
|
||||
// entry on the wire and expands to roughly seventy times that once
|
||||
// unmarshalled, so counting the entries afterwards is too late to bound the
|
||||
// allocation. Streaming keeps the cost proportional to maxEntries no matter
|
||||
// how long the array is.
|
||||
func decodeDownloadFrom(raw string, maxEntries int) ([]downloadFrom, error) {
|
||||
dec := json.NewDecoder(strings.NewReader(raw))
|
||||
|
||||
token, err := dec.Token()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if delim, ok := token.(json.Delim); !ok || delim != '[' {
|
||||
return nil, fmt.Errorf("expected a JSON array, got '%v'", token)
|
||||
}
|
||||
|
||||
var dls []downloadFrom
|
||||
for dec.More() {
|
||||
if maxEntries > 0 && len(dls) >= maxEntries {
|
||||
return nil, errTooManyDownloadFromEntries
|
||||
}
|
||||
|
||||
var dl downloadFrom
|
||||
err = dec.Decode(&dl)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
dls = append(dls, dl)
|
||||
}
|
||||
|
||||
return dls, nil
|
||||
}
|
||||
|
||||
type downloadFrom struct {
|
||||
// Url is the URL to download a file from.
|
||||
Url string `json:"url"`
|
||||
@@ -98,7 +158,7 @@ type downloadFrom struct {
|
||||
}
|
||||
|
||||
// newContext returns a [Context] by parsing a "multipart/form-data" request.
|
||||
func newContext(echoCtx echo.Context, logger *slog.Logger, fs *gotenberg.FileSystem, timeout time.Duration, bodyLimit int64, downloadFromCfg downloadFromConfig) (*Context, context.CancelFunc, error) {
|
||||
func newContext(echoCtx *echo.Context, logger *slog.Logger, fs *gotenberg.FileSystem, timeout time.Duration, bodyLimit int64, downloadFromCfg downloadFromConfig) (*Context, context.CancelFunc, error) {
|
||||
processCtx, processCancel := context.WithTimeout(echoCtx.Request().Context(), timeout)
|
||||
|
||||
// We want to make sure the multipart/form-data does not exceed a given
|
||||
@@ -117,14 +177,18 @@ func newContext(echoCtx echo.Context, logger *slog.Logger, fs *gotenberg.FileSys
|
||||
return nil
|
||||
}
|
||||
|
||||
// Snapshot now, while echoCtx still belongs to this request.
|
||||
outputFilename, _ := echoCtx.Get("outputFilename").(string)
|
||||
|
||||
ctx := &Context{
|
||||
outputPaths: make([]string, 0),
|
||||
cancelled: false,
|
||||
logger: logger,
|
||||
echoCtx: echoCtx,
|
||||
mkdirAll: new(gotenberg.OsMkdirAll),
|
||||
pathRename: new(gotenberg.OsPathRename),
|
||||
Context: processCtx,
|
||||
outputPaths: make([]string, 0),
|
||||
cancelled: false,
|
||||
outputFilename: outputFilename,
|
||||
logger: logger,
|
||||
echoCtx: echoCtx,
|
||||
mkdirAll: new(gotenberg.OsMkdirAll),
|
||||
pathRename: new(gotenberg.OsPathRename),
|
||||
Context: processCtx,
|
||||
}
|
||||
|
||||
// A custom cancel function which removes the context's working directory
|
||||
@@ -213,8 +277,13 @@ func newContext(echoCtx echo.Context, logger *slog.Logger, fs *gotenberg.FileSys
|
||||
// any.
|
||||
raw, ok := ctx.values["downloadFrom"]
|
||||
if !downloadFromCfg.disable && ok {
|
||||
var dls []downloadFrom
|
||||
err = json.Unmarshal([]byte(raw[0]), &dls)
|
||||
dls, err := decodeDownloadFrom(raw[0], downloadFromCfg.maxEntries)
|
||||
if errors.Is(err, errTooManyDownloadFromEntries) {
|
||||
return nil, cancel, WrapError(
|
||||
fmt.Errorf("decode downloadFrom: %w", err),
|
||||
NewSentinelHttpError(http.StatusBadRequest, fmt.Sprintf("Invalid 'downloadFrom' form field value: too many entries, the maximum is %d", downloadFromCfg.maxEntries)),
|
||||
)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, cancel, WrapError(
|
||||
fmt.Errorf("unmarshal json: %w", err),
|
||||
@@ -232,6 +301,13 @@ func newContext(echoCtx echo.Context, logger *slog.Logger, fs *gotenberg.FileSys
|
||||
results := make([]downloadFromResult, len(dls))
|
||||
|
||||
eg, _ := errgroup.WithContext(ctx)
|
||||
// Bound the number of in-flight downloads. Each entry allocates a
|
||||
// retryable client, an outbound transport, a span, and logger state,
|
||||
// so an unbounded array would otherwise exhaust process memory. A
|
||||
// value of 0 keeps the fan-out unbounded.
|
||||
if downloadFromCfg.maxConcurrency > 0 {
|
||||
eg.SetLimit(downloadFromCfg.maxConcurrency)
|
||||
}
|
||||
for i, dl := range dls {
|
||||
eg.Go(func() error {
|
||||
deadline, ok := ctx.Deadline()
|
||||
@@ -263,7 +339,12 @@ func newContext(echoCtx echo.Context, logger *slog.Logger, fs *gotenberg.FileSys
|
||||
|
||||
logger.DebugContext(dlCtx, fmt.Sprintf("download file from '%s'", dl.Url))
|
||||
|
||||
req, err := retryablehttp.NewRequest(http.MethodGet, dl.Url, nil)
|
||||
// The request must carry dlCtx: retryablehttp.NewRequest builds
|
||||
// on context.Background(), and its wait between attempts is a
|
||||
// select on the request context, so a contextless request cannot
|
||||
// be interrupted by --api-timeout (env API_TIMEOUT) or by the
|
||||
// caller going away.
|
||||
req, err := retryablehttp.NewRequestWithContext(dlCtx, http.MethodGet, dl.Url, nil)
|
||||
if err != nil {
|
||||
dlSpan.RecordError(err)
|
||||
dlSpan.SetStatus(codes.Error, err.Error())
|
||||
@@ -286,14 +367,28 @@ func newContext(echoCtx echo.Context, logger *slog.Logger, fs *gotenberg.FileSys
|
||||
}
|
||||
}
|
||||
|
||||
// Entries are serialized by the concurrency limit above, so a
|
||||
// late one can start after the deadline has already passed.
|
||||
// Fail closed rather than derive a non-positive timeout, which
|
||||
// [http.Client] reads as no deadline at all.
|
||||
remaining := time.Until(deadline)
|
||||
if remaining <= 0 {
|
||||
dlSpan.RecordError(context.DeadlineExceeded)
|
||||
dlSpan.SetStatus(codes.Error, context.DeadlineExceeded.Error())
|
||||
dlSpan.End()
|
||||
return fmt.Errorf("download file from '%s': %w", dl.Url, context.DeadlineExceeded)
|
||||
}
|
||||
|
||||
client := &retryablehttp.Client{
|
||||
HTTPClient: gotenberg.NewOutboundHttpClient(time.Until(deadline), downloadFromCfg.allowList, downloadFromCfg.denyList, downloadFromCfg.enableEnvironmentProxy, ipOpts...),
|
||||
HTTPClient: gotenberg.NewOutboundHttpClient(remaining, downloadFromCfg.allowList, downloadFromCfg.denyList, downloadFromCfg.enableEnvironmentProxy, ipOpts...),
|
||||
RetryMax: downloadFromCfg.maxRetry,
|
||||
RetryWaitMin: time.Duration(1) * time.Second,
|
||||
RetryWaitMax: time.Until(deadline),
|
||||
RetryWaitMax: remaining,
|
||||
Logger: gotenberg.NewLeveledLogger(logger),
|
||||
CheckRetry: retryablehttp.DefaultRetryPolicy,
|
||||
Backoff: retryablehttp.DefaultBackoff,
|
||||
// Not DefaultBackoff: it hands a hostile origin control of
|
||||
// the wait via Retry-After.
|
||||
Backoff: gotenberg.ClampedBackoff,
|
||||
}
|
||||
|
||||
resp, err := client.Do(req)
|
||||
@@ -301,6 +396,17 @@ func newContext(echoCtx echo.Context, logger *slog.Logger, fs *gotenberg.FileSys
|
||||
dlSpan.RecordError(err)
|
||||
dlSpan.SetStatus(codes.Error, err.Error())
|
||||
dlSpan.End()
|
||||
|
||||
// A redirect target is filtered inside the client, so the
|
||||
// policy verdict surfaces here rather than from the
|
||||
// pre-flight above. Keep it out of the response: the first
|
||||
// hop answers a filtered URL with a generic 403, and a
|
||||
// later hop must not describe the allow-list, the deny-list
|
||||
// or the IP policy instead.
|
||||
if errors.Is(err, gotenberg.ErrFiltered) {
|
||||
return fmt.Errorf("download file from '%s': %w", dl.Url, err)
|
||||
}
|
||||
|
||||
return WrapError(
|
||||
fmt.Errorf("download file from to '%s': %w", dl.Url, err),
|
||||
NewSentinelHttpError(http.StatusBadRequest, fmt.Sprintf("Unable to download file from '%s': %s", dl.Url, err)),
|
||||
@@ -374,7 +480,7 @@ func newContext(echoCtx echo.Context, logger *slog.Logger, fs *gotenberg.FileSys
|
||||
// Use a UUID-based name on disk to avoid filesystem
|
||||
// NAME_MAX limits with long filenames.
|
||||
// See: https://github.com/gotenberg/gotenberg/issues/1500.
|
||||
safeName := uuid.New().String() + filepath.Ext(filename)
|
||||
safeName := uuid.New().String() + safeExt(filename)
|
||||
path := fmt.Sprintf("%s/%s", ctx.dirPath, safeName)
|
||||
|
||||
out, err := os.Create(path)
|
||||
@@ -428,18 +534,20 @@ func newContext(echoCtx echo.Context, logger *slog.Logger, fs *gotenberg.FileSys
|
||||
}
|
||||
|
||||
for _, r := range results {
|
||||
ctx.files[r.filename] = r.path
|
||||
ctx.diskToOriginal[r.path] = r.filename
|
||||
filename := ctx.uniqueFilename(r.filename)
|
||||
ctx.files[filename] = r.path
|
||||
ctx.diskToOriginal[r.path] = filename
|
||||
ctx.trackFileOrder(r.path, r.filename)
|
||||
if r.formField != "" {
|
||||
ctx.filesByField[r.formField] = append(ctx.filesByField[r.formField], r.path)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
copyToDisk := func(fh *multipart.FileHeader) error {
|
||||
copyToDisk := func(fh *multipart.FileHeader) (string, error) {
|
||||
in, err := fh.Open()
|
||||
if err != nil {
|
||||
return fmt.Errorf("open multipart file: %w", err)
|
||||
return "", fmt.Errorf("open multipart file: %w", err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
@@ -462,12 +570,12 @@ func newContext(echoCtx echo.Context, logger *slog.Logger, fs *gotenberg.FileSys
|
||||
// Use a UUID-based name on disk to avoid filesystem
|
||||
// NAME_MAX limits with long filenames.
|
||||
// See: https://github.com/gotenberg/gotenberg/issues/1500.
|
||||
safeName := uuid.New().String() + filepath.Ext(filename)
|
||||
safeName := uuid.New().String() + safeExt(filename)
|
||||
path := fmt.Sprintf("%s/%s", ctx.dirPath, safeName)
|
||||
|
||||
out, err := os.Create(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create local file: %w", err)
|
||||
return "", fmt.Errorf("create local file: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
err := out.Close()
|
||||
@@ -478,26 +586,28 @@ func newContext(echoCtx echo.Context, logger *slog.Logger, fs *gotenberg.FileSys
|
||||
|
||||
_, err = io.Copy(out, reader)
|
||||
if err != nil {
|
||||
return fmt.Errorf("copy multipart file to local file: %w", err)
|
||||
return "", fmt.Errorf("copy multipart file to local file: %w", err)
|
||||
}
|
||||
|
||||
base := filename
|
||||
filename = ctx.uniqueFilename(filename)
|
||||
ctx.files[filename] = path
|
||||
ctx.diskToOriginal[path] = filename
|
||||
ctx.trackFileOrder(path, base)
|
||||
|
||||
return nil
|
||||
return filename, nil
|
||||
}
|
||||
|
||||
// Then, copy the form files, if any.
|
||||
for fieldName, files := range form.File {
|
||||
for _, fh := range files {
|
||||
err = copyToDisk(fh)
|
||||
if err != nil {
|
||||
return ctx, cancel, fmt.Errorf("copy to disk: %w", err)
|
||||
filename, errCopy := copyToDisk(fh)
|
||||
if errCopy != nil {
|
||||
return ctx, cancel, fmt.Errorf("copy to disk: %w", errCopy)
|
||||
}
|
||||
// Track files by field name
|
||||
filename := sanitizeFilename(fh.Filename)
|
||||
filePath := ctx.files[filename]
|
||||
ctx.filesByField[fieldName] = append(ctx.filesByField[fieldName], filePath)
|
||||
// Track files by field name, under the name copyToDisk actually
|
||||
// stored, which may be a de-duplicated variant.
|
||||
ctx.filesByField[fieldName] = append(ctx.filesByField[fieldName], ctx.files[filename])
|
||||
}
|
||||
}
|
||||
|
||||
@@ -512,9 +622,9 @@ func newContext(echoCtx echo.Context, logger *slog.Logger, fs *gotenberg.FileSys
|
||||
if symlinkPath == diskPath {
|
||||
continue
|
||||
}
|
||||
err = os.Symlink(filepath.Base(diskPath), symlinkPath)
|
||||
if err != nil {
|
||||
logger.DebugContext(context.Background(), fmt.Sprintf("skip symlink for '%s': %s", originalName, err))
|
||||
errSymlink := os.Symlink(filepath.Base(diskPath), symlinkPath)
|
||||
if errSymlink != nil {
|
||||
logger.DebugContext(context.Background(), fmt.Sprintf("skip symlink for '%s': %s", originalName, errSymlink))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -523,7 +633,10 @@ func newContext(echoCtx echo.Context, logger *slog.Logger, fs *gotenberg.FileSys
|
||||
ctx.Log().DebugContext(ctx, fmt.Sprintf("form files by field: %+v", ctx.filesByField))
|
||||
ctx.Log().DebugContext(ctx, fmt.Sprintf("total bytes: %d", totalBytesRead.Load()))
|
||||
|
||||
return ctx, cancel, err
|
||||
// Explicitly nil: the best-effort symlink loop above must not decide the
|
||||
// outcome of the request. Its failure used to escape here as a bare 500,
|
||||
// non-deterministically, because ctx.files iterates in random order.
|
||||
return ctx, cancel, nil
|
||||
}
|
||||
|
||||
// Request returns the [http.Request].
|
||||
@@ -538,6 +651,8 @@ func (ctx *Context) FormData() *FormData {
|
||||
files: ctx.files,
|
||||
filesByField: ctx.filesByField,
|
||||
diskToOriginal: ctx.diskToOriginal,
|
||||
fileOrder: ctx.fileOrder,
|
||||
fileBase: ctx.fileBase,
|
||||
errors: nil,
|
||||
}
|
||||
}
|
||||
@@ -578,7 +693,7 @@ func (ctx *Context) GeneratePath(extension string) string {
|
||||
// limits but registers the given filename so that [Context.OriginalFilename]
|
||||
// can resolve it. It does not create a file.
|
||||
func (ctx *Context) GeneratePathFromFilename(filename string) string {
|
||||
safeName := uuid.New().String() + filepath.Ext(filename)
|
||||
safeName := uuid.New().String() + safeExt(filename)
|
||||
path := fmt.Sprintf("%s/%s", ctx.dirPath, safeName)
|
||||
ctx.diskToOriginal[path] = filename
|
||||
return path
|
||||
@@ -681,15 +796,72 @@ func (ctx *Context) BuildOutputFile() (string, error) {
|
||||
// OutputFilename returns the filename based on the given output path or the
|
||||
// "Gotenberg-Output-Filename" header's value.
|
||||
func (ctx *Context) OutputFilename(outputPath string) string {
|
||||
filename := ctx.echoCtx.Get("outputFilename").(string)
|
||||
|
||||
if filename == "" {
|
||||
if ctx.outputFilename == "" {
|
||||
return ctx.OriginalFilename(outputPath)
|
||||
}
|
||||
|
||||
filename := ctx.outputFilename
|
||||
|
||||
return fmt.Sprintf("%s%s", filename, filepath.Ext(outputPath))
|
||||
}
|
||||
|
||||
// maxDiskExtLength bounds the extension copied onto a UUID-based disk name.
|
||||
// The UUID stem is 36 characters, so a longer extension risks NAME_MAX, which
|
||||
// is 255 on ext4 and overlayfs. The untruncated name is kept in
|
||||
// [Context.diskToOriginal], which never reaches the filesystem.
|
||||
const maxDiskExtLength = 32
|
||||
|
||||
// safeExt returns the extension to append to a UUID-based disk name. It drops
|
||||
// an extension too long to be safe rather than let [os.Create] fail with
|
||||
// ENAMETOOLONG, which surfaced to the caller as a bare 500.
|
||||
func safeExt(filename string) string {
|
||||
ext := filepath.Ext(filename)
|
||||
if len(ext) > maxDiskExtLength {
|
||||
return ""
|
||||
}
|
||||
|
||||
return ext
|
||||
}
|
||||
|
||||
// uniqueFilename returns filename, or a numbered variant of it when the
|
||||
// request already carries a file by that name.
|
||||
//
|
||||
// Uploads are keyed by their sanitized original filename, so two files sharing
|
||||
// one name used to collide: the second overwrote the first and only one
|
||||
// reached the conversion, while both stayed on disk and counted against the
|
||||
// body limit. Sanitizing strips directories, so "a/doc.pdf" and "b/doc.pdf"
|
||||
// collide too.
|
||||
func (ctx *Context) uniqueFilename(filename string) string {
|
||||
_, exists := ctx.files[filename]
|
||||
if !exists {
|
||||
return filename
|
||||
}
|
||||
|
||||
ext := filepath.Ext(filename)
|
||||
stem := strings.TrimSuffix(filename, ext)
|
||||
|
||||
for i := 2; ; i++ {
|
||||
candidate := fmt.Sprintf("%s (%d)%s", stem, i, ext)
|
||||
_, exists = ctx.files[candidate]
|
||||
if !exists {
|
||||
return candidate
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// trackFileOrder records where a file arrived in the request and the filename
|
||||
// it arrived under, so [FormData.paths] can order it the way the caller sent
|
||||
// it.
|
||||
func (ctx *Context) trackFileOrder(path, base string) {
|
||||
if ctx.fileOrder == nil {
|
||||
ctx.fileOrder = make(map[string]int)
|
||||
ctx.fileBase = make(map[string]string)
|
||||
}
|
||||
|
||||
ctx.fileOrder[path] = len(ctx.fileOrder)
|
||||
ctx.fileBase[path] = base
|
||||
}
|
||||
|
||||
// sanitizeFilename strips path separators (including backslashes, which
|
||||
// [filepath.Base] ignores on Linux) and control characters from a
|
||||
// caller-supplied filename, then NFC-normalizes the result. This prevents a
|
||||
|
||||
@@ -4,17 +4,23 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"regexp"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/labstack/echo/v4"
|
||||
"github.com/dlclark/regexp2/v2"
|
||||
"github.com/labstack/echo/v5"
|
||||
|
||||
"github.com/gotenberg/gotenberg/v8/pkg/gotenberg"
|
||||
)
|
||||
@@ -209,6 +215,137 @@ func TestNewContext_DownloadFromConcurrentMapWrites(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// An oversized downloadFrom array must be rejected at the trust boundary with
|
||||
// a 400, before any download goroutine is spawned.
|
||||
// https://github.com/gotenberg/gotenberg/security/advisories/GHSA-6vqw-2jgm-4x88
|
||||
func TestNewContext_DownloadFromMaxEntries(t *testing.T) {
|
||||
var hits atomic.Int64
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
hits.Add(1)
|
||||
w.Header().Set("Content-Disposition", `attachment; filename="download.txt"`)
|
||||
_, _ = w.Write([]byte("downloaded"))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
dls := make([]downloadFrom, 3)
|
||||
for i := range dls {
|
||||
dls[i] = downloadFrom{Url: fmt.Sprintf("%s/file?i=%d", server.URL, i)}
|
||||
}
|
||||
|
||||
payload, err := json.Marshal(dls)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal downloadFrom payload: %v", err)
|
||||
}
|
||||
|
||||
body := new(bytes.Buffer)
|
||||
writer := multipart.NewWriter(body)
|
||||
err = writer.WriteField("downloadFrom", string(payload))
|
||||
if err != nil {
|
||||
t.Fatalf("write downloadFrom field: %v", err)
|
||||
}
|
||||
err = writer.Close()
|
||||
if err != nil {
|
||||
t.Fatalf("close multipart writer: %v", err)
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/forms/libreoffice/convert", body)
|
||||
req.Header.Set("Content-Type", writer.FormDataContentType())
|
||||
|
||||
echoCtx := echo.New().NewContext(req, httptest.NewRecorder())
|
||||
logger := slog.New(slog.DiscardHandler)
|
||||
fs := gotenberg.NewFileSystem(new(gotenberg.OsMkdirAll))
|
||||
downloadFromCfg := downloadFromConfig{maxEntries: 2}
|
||||
|
||||
_, cancel, err := newContext(echoCtx, logger, fs, 10*time.Second, 0, downloadFromCfg)
|
||||
if cancel != nil {
|
||||
defer cancel()
|
||||
}
|
||||
if err == nil {
|
||||
t.Fatal("newContext returned no error, want a 400 for too many entries")
|
||||
}
|
||||
|
||||
var httpErr HttpError
|
||||
if !errors.As(err, &httpErr) {
|
||||
t.Fatalf("error %v is not an HttpError", err)
|
||||
}
|
||||
if status, _ := httpErr.HttpError(); status != http.StatusBadRequest {
|
||||
t.Fatalf("HTTP status = %d, want %d", status, http.StatusBadRequest)
|
||||
}
|
||||
if got := hits.Load(); got != 0 {
|
||||
t.Fatalf("server hits = %d, want 0 (rejected before any download)", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The number of in-flight downloadFrom fetches must never exceed the
|
||||
// configured concurrency limit, regardless of array length.
|
||||
// https://github.com/gotenberg/gotenberg/security/advisories/GHSA-6vqw-2jgm-4x88
|
||||
func TestNewContext_DownloadFromMaxConcurrency(t *testing.T) {
|
||||
const (
|
||||
downloads = 8
|
||||
maxConcurrency = 2
|
||||
)
|
||||
|
||||
var current, peak atomic.Int64
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
inFlight := current.Add(1)
|
||||
for {
|
||||
observed := peak.Load()
|
||||
if inFlight <= observed || peak.CompareAndSwap(observed, inFlight) {
|
||||
break
|
||||
}
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
current.Add(-1)
|
||||
|
||||
filename := fmt.Sprintf("download-%s.txt", r.URL.Query().Get("i"))
|
||||
w.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"`, filename))
|
||||
_, _ = w.Write([]byte("downloaded"))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
dls := make([]downloadFrom, downloads)
|
||||
for i := range dls {
|
||||
dls[i] = downloadFrom{Url: fmt.Sprintf("%s/file?i=%d", server.URL, i)}
|
||||
}
|
||||
|
||||
payload, err := json.Marshal(dls)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal downloadFrom payload: %v", err)
|
||||
}
|
||||
|
||||
body := new(bytes.Buffer)
|
||||
writer := multipart.NewWriter(body)
|
||||
err = writer.WriteField("downloadFrom", string(payload))
|
||||
if err != nil {
|
||||
t.Fatalf("write downloadFrom field: %v", err)
|
||||
}
|
||||
err = writer.Close()
|
||||
if err != nil {
|
||||
t.Fatalf("close multipart writer: %v", err)
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/forms/libreoffice/convert", body)
|
||||
req.Header.Set("Content-Type", writer.FormDataContentType())
|
||||
|
||||
echoCtx := echo.New().NewContext(req, httptest.NewRecorder())
|
||||
logger := slog.New(slog.DiscardHandler)
|
||||
fs := gotenberg.NewFileSystem(new(gotenberg.OsMkdirAll))
|
||||
downloadFromCfg := downloadFromConfig{maxConcurrency: maxConcurrency}
|
||||
|
||||
ctx, cancel, err := newContext(echoCtx, logger, fs, 10*time.Second, 0, downloadFromCfg)
|
||||
if err != nil {
|
||||
t.Fatalf("newContext returned error: %v", err)
|
||||
}
|
||||
defer cancel()
|
||||
|
||||
if got := len(ctx.files); got != downloads {
|
||||
t.Fatalf("downloaded files = %d, want %d", got, downloads)
|
||||
}
|
||||
if got := peak.Load(); got > maxConcurrency {
|
||||
t.Fatalf("peak concurrency = %d, want <= %d", got, maxConcurrency)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSanitizeFilename(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
scenario string
|
||||
@@ -281,3 +418,487 @@ func TestContext_FileCount(t *testing.T) {
|
||||
t.Errorf("expected 3 files, got %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A hostile origin must not choose how long Gotenberg waits.
|
||||
// [retryablehttp.DefaultBackoff] returns a Retry-After header verbatim for 429
|
||||
// and 503, and the wait between attempts is a select on the request context.
|
||||
// Building the request without a context therefore pinned the goroutine, its
|
||||
// connection, and its working directory for the attacker's chosen duration,
|
||||
// well past --api-timeout (env API_TIMEOUT).
|
||||
func TestNewContext_DownloadFromHostileRetryAfterIsBounded(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Retry-After", "3600")
|
||||
w.WriteHeader(http.StatusTooManyRequests)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
payload, err := json.Marshal([]downloadFrom{{Url: server.URL + "/file"}})
|
||||
if err != nil {
|
||||
t.Fatalf("marshal downloadFrom payload: %v", err)
|
||||
}
|
||||
|
||||
body := new(bytes.Buffer)
|
||||
writer := multipart.NewWriter(body)
|
||||
err = writer.WriteField("downloadFrom", string(payload))
|
||||
if err != nil {
|
||||
t.Fatalf("write downloadFrom field: %v", err)
|
||||
}
|
||||
err = writer.Close()
|
||||
if err != nil {
|
||||
t.Fatalf("close multipart writer: %v", err)
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/forms/libreoffice/convert", body)
|
||||
req.Header.Set("Content-Type", writer.FormDataContentType())
|
||||
|
||||
echoCtx := echo.New().NewContext(req, httptest.NewRecorder())
|
||||
logger := slog.New(slog.DiscardHandler)
|
||||
fs := gotenberg.NewFileSystem(new(gotenberg.OsMkdirAll))
|
||||
|
||||
const timeout = 500 * time.Millisecond
|
||||
|
||||
start := time.Now()
|
||||
_, cancel, err := newContext(echoCtx, logger, fs, timeout, 0, downloadFromConfig{maxRetry: 2})
|
||||
elapsed := time.Since(start)
|
||||
if cancel != nil {
|
||||
defer cancel()
|
||||
}
|
||||
|
||||
if err == nil {
|
||||
t.Fatal("expected newContext to fail against an origin that only answers 429")
|
||||
}
|
||||
// Generous: the deadline is 500ms and Retry-After asks for an hour. Any
|
||||
// value in seconds means the remote is still in control.
|
||||
if elapsed > 10*time.Second {
|
||||
t.Fatalf("newContext took %s with Retry-After 3600; --api-timeout must bound it", elapsed)
|
||||
}
|
||||
}
|
||||
|
||||
// An entry that starts after the deadline has passed must fail closed. It used
|
||||
// to derive a negative client timeout, which [http.Client] reads as no
|
||||
// deadline at all, leaving the download unbounded.
|
||||
func TestNewContext_DownloadFromExpiredBudgetFailsClosed(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
<-r.Context().Done()
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
// Two entries, serialized by the concurrency limit, so the second one
|
||||
// starts once the first has burned the whole budget.
|
||||
payload, err := json.Marshal([]downloadFrom{
|
||||
{Url: server.URL + "/first"},
|
||||
{Url: server.URL + "/second"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("marshal downloadFrom payload: %v", err)
|
||||
}
|
||||
|
||||
body := new(bytes.Buffer)
|
||||
writer := multipart.NewWriter(body)
|
||||
err = writer.WriteField("downloadFrom", string(payload))
|
||||
if err != nil {
|
||||
t.Fatalf("write downloadFrom field: %v", err)
|
||||
}
|
||||
err = writer.Close()
|
||||
if err != nil {
|
||||
t.Fatalf("close multipart writer: %v", err)
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/forms/libreoffice/convert", body)
|
||||
req.Header.Set("Content-Type", writer.FormDataContentType())
|
||||
|
||||
echoCtx := echo.New().NewContext(req, httptest.NewRecorder())
|
||||
logger := slog.New(slog.DiscardHandler)
|
||||
fs := gotenberg.NewFileSystem(new(gotenberg.OsMkdirAll))
|
||||
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, cancel, err := newContext(echoCtx, logger, fs, 400*time.Millisecond, 0, downloadFromConfig{
|
||||
maxRetry: 0,
|
||||
maxConcurrency: 1,
|
||||
})
|
||||
if cancel != nil {
|
||||
cancel()
|
||||
}
|
||||
done <- err
|
||||
}()
|
||||
|
||||
select {
|
||||
case err := <-done:
|
||||
if err == nil {
|
||||
t.Fatal("expected newContext to fail against a stalling origin")
|
||||
}
|
||||
case <-time.After(15 * time.Second):
|
||||
t.Fatal("newContext never returned: an entry starting past the deadline built an unbounded client")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeDownloadFrom(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
scenario string
|
||||
raw string
|
||||
maxEntries int
|
||||
expectErr error
|
||||
expectLen int
|
||||
}{
|
||||
{"empty array", `[]`, 10, nil, 0},
|
||||
{"under the limit", `[{"url":"http://a"},{"url":"http://b"}]`, 10, nil, 2},
|
||||
{"exactly the limit", `[{"url":"http://a"},{"url":"http://b"}]`, 2, nil, 2},
|
||||
{"over the limit", `[{"url":"http://a"},{"url":"http://b"}]`, 1, errTooManyDownloadFromEntries, 0},
|
||||
{"no limit", `[{"url":"http://a"},{"url":"http://b"}]`, 0, nil, 2},
|
||||
{"not an array", `{"url":"http://a"}`, 10, nil, 0},
|
||||
{"malformed", `[{"url":`, 10, nil, 0},
|
||||
{"not json", `nope`, 10, nil, 0},
|
||||
} {
|
||||
t.Run(tc.scenario, func(t *testing.T) {
|
||||
dls, err := decodeDownloadFrom(tc.raw, tc.maxEntries)
|
||||
|
||||
if tc.expectErr != nil {
|
||||
if !errors.Is(err, tc.expectErr) {
|
||||
t.Fatalf("error = %v, want %v", err, tc.expectErr)
|
||||
}
|
||||
return
|
||||
}
|
||||
if tc.scenario == "not an array" || tc.scenario == "malformed" || tc.scenario == "not json" {
|
||||
if err == nil {
|
||||
t.Fatalf("expected an error for %q", tc.raw)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if len(dls) != tc.expectLen {
|
||||
t.Fatalf("decoded %d entries, want %d", len(dls), tc.expectLen)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A compact array costs three bytes per entry on the wire and expands by
|
||||
// roughly seventy times once unmarshalled. Decoding must stop at the limit
|
||||
// rather than materialize the whole array and count afterwards.
|
||||
func TestDecodeDownloadFrom_StopsBeforeMaterializingTheArray(t *testing.T) {
|
||||
const entries = 2_000_000
|
||||
|
||||
raw := "[" + strings.Repeat("{},", entries) + "{}]"
|
||||
|
||||
var before, after runtime.MemStats
|
||||
runtime.GC()
|
||||
runtime.ReadMemStats(&before)
|
||||
|
||||
_, err := decodeDownloadFrom(raw, 1000)
|
||||
|
||||
runtime.ReadMemStats(&after)
|
||||
|
||||
if !errors.Is(err, errTooManyDownloadFromEntries) {
|
||||
t.Fatalf("error = %v, want errTooManyDownloadFromEntries", err)
|
||||
}
|
||||
|
||||
// json.Unmarshal on the same input allocates hundreds of MiB. Bounded
|
||||
// decoding should stay in the low single-digit MiB, so this threshold is
|
||||
// deliberately loose and still fails loudly on a regression.
|
||||
allocated := after.TotalAlloc - before.TotalAlloc
|
||||
if allocated > 32<<20 {
|
||||
t.Fatalf("decoding allocated %d MiB for a %d-entry array, want the limit to bound it", allocated>>20, entries)
|
||||
}
|
||||
t.Logf("allocated %d KiB decoding a %d-entry array with a limit of 1000", allocated>>10, entries)
|
||||
}
|
||||
|
||||
// An asynchronous conversion outlives the [echo.Context]. Echo returns that
|
||||
// context to a sync.Pool as soon as the handler returns, and
|
||||
// outputFilenameMiddleware runs in srv.Pre on every request, including
|
||||
// /health, so a later request overwrites the store. Reading the output
|
||||
// filename from it after the fact returned another caller's value.
|
||||
func TestContext_OutputFilename_SurvivesEchoContextRecycling(t *testing.T) {
|
||||
body := new(bytes.Buffer)
|
||||
writer := multipart.NewWriter(body)
|
||||
err := writer.Close()
|
||||
if err != nil {
|
||||
t.Fatalf("close multipart writer: %v", err)
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/forms/libreoffice/convert", body)
|
||||
req.Header.Set("Content-Type", writer.FormDataContentType())
|
||||
|
||||
echoCtx := echo.New().NewContext(req, httptest.NewRecorder())
|
||||
// What outputFilenameMiddleware does for this request.
|
||||
echoCtx.Set("outputFilename", "victim")
|
||||
|
||||
logger := slog.New(slog.DiscardHandler)
|
||||
fs := gotenberg.NewFileSystem(new(gotenberg.OsMkdirAll))
|
||||
|
||||
ctx, cancel, err := newContext(echoCtx, logger, fs, 10*time.Second, 0, downloadFromConfig{disable: true})
|
||||
if err != nil {
|
||||
t.Fatalf("newContext returned error: %v", err)
|
||||
}
|
||||
defer cancel()
|
||||
|
||||
// Echo recycles the context and another request claims the store.
|
||||
echoCtx.Set("outputFilename", "attacker-controlled")
|
||||
|
||||
if got := ctx.OutputFilename("/tmp/out.pdf"); got != "victim.pdf" {
|
||||
t.Fatalf("OutputFilename = %q, want %q", got, "victim.pdf")
|
||||
}
|
||||
}
|
||||
|
||||
// A recycled context has a nil store, so the previous unguarded type assertion
|
||||
// could panic. The snapshot must tolerate an absent value.
|
||||
func TestContext_OutputFilename_NoHeader(t *testing.T) {
|
||||
body := new(bytes.Buffer)
|
||||
writer := multipart.NewWriter(body)
|
||||
err := writer.Close()
|
||||
if err != nil {
|
||||
t.Fatalf("close multipart writer: %v", err)
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/forms/libreoffice/convert", body)
|
||||
req.Header.Set("Content-Type", writer.FormDataContentType())
|
||||
|
||||
// No Set call at all: the store holds nothing for "outputFilename".
|
||||
echoCtx := echo.New().NewContext(req, httptest.NewRecorder())
|
||||
|
||||
logger := slog.New(slog.DiscardHandler)
|
||||
fs := gotenberg.NewFileSystem(new(gotenberg.OsMkdirAll))
|
||||
|
||||
ctx, cancel, err := newContext(echoCtx, logger, fs, 10*time.Second, 0, downloadFromConfig{disable: true})
|
||||
if err != nil {
|
||||
t.Fatalf("newContext returned error: %v", err)
|
||||
}
|
||||
defer cancel()
|
||||
|
||||
if got := ctx.OutputFilename("/tmp/out.pdf"); got != "out.pdf" {
|
||||
t.Fatalf("OutputFilename = %q, want the original filename %q", got, "out.pdf")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSafeExt(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
scenario string
|
||||
filename string
|
||||
want string
|
||||
}{
|
||||
{"ordinary extension", "report.pdf", ".pdf"},
|
||||
{"no extension", "report", ""},
|
||||
{"at the limit", "a." + strings.Repeat("x", maxDiskExtLength-1), "." + strings.Repeat("x", maxDiskExtLength-1)},
|
||||
{"over the limit is dropped", "a." + strings.Repeat("x", 300), ""},
|
||||
} {
|
||||
t.Run(tc.scenario, func(t *testing.T) {
|
||||
got := safeExt(tc.filename)
|
||||
if got != tc.want {
|
||||
t.Fatalf("safeExt(%q) = %q, want %q", tc.filename, got, tc.want)
|
||||
}
|
||||
// A UUID stem is 36 characters. The whole disk name must stay
|
||||
// under NAME_MAX.
|
||||
if len(got)+36 > 255 {
|
||||
t.Fatalf("disk name would be %d characters, over NAME_MAX", len(got)+36)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// An upload whose extension exceeds NAME_MAX used to fail os.Create and return
|
||||
// a bare 500. The extension is bounded, and the original name survives in
|
||||
// diskToOriginal.
|
||||
func TestNewContext_LongExtensionIsAccepted(t *testing.T) {
|
||||
filename := "invoice." + strings.Repeat("x", 300)
|
||||
|
||||
body := new(bytes.Buffer)
|
||||
writer := multipart.NewWriter(body)
|
||||
part, err := writer.CreateFormFile("files", filename)
|
||||
if err != nil {
|
||||
t.Fatalf("create multipart file: %v", err)
|
||||
}
|
||||
_, err = part.Write([]byte("%PDF-1.4"))
|
||||
if err != nil {
|
||||
t.Fatalf("write multipart file: %v", err)
|
||||
}
|
||||
err = writer.Close()
|
||||
if err != nil {
|
||||
t.Fatalf("close multipart writer: %v", err)
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/forms/libreoffice/convert", body)
|
||||
req.Header.Set("Content-Type", writer.FormDataContentType())
|
||||
|
||||
echoCtx := echo.New().NewContext(req, httptest.NewRecorder())
|
||||
logger := slog.New(slog.DiscardHandler)
|
||||
fs := gotenberg.NewFileSystem(new(gotenberg.OsMkdirAll))
|
||||
|
||||
ctx, cancel, err := newContext(echoCtx, logger, fs, 10*time.Second, 0, downloadFromConfig{disable: true})
|
||||
if cancel != nil {
|
||||
defer cancel()
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("newContext returned error for a long extension: %v", err)
|
||||
}
|
||||
if got := len(ctx.files); got != 1 {
|
||||
t.Fatalf("files = %d, want 1", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A filename that cannot become a symlink (too long, "..", "/") must not fail
|
||||
// the request. The symlink loop is best-effort, but its error escaped through
|
||||
// the shared err variable, and ctx.files iterates randomly, so byte-identical
|
||||
// requests gave different HTTP outcomes.
|
||||
func TestNewContext_UnsymlinkableFilenameStillSucceeds(t *testing.T) {
|
||||
for _, filename := range []string{
|
||||
strings.Repeat("a", 300) + ".txt",
|
||||
"..",
|
||||
"/",
|
||||
} {
|
||||
t.Run(filename[:min(len(filename), 12)], func(t *testing.T) {
|
||||
body := new(bytes.Buffer)
|
||||
writer := multipart.NewWriter(body)
|
||||
part, err := writer.CreateFormFile("files", filename)
|
||||
if err != nil {
|
||||
t.Fatalf("create multipart file: %v", err)
|
||||
}
|
||||
_, err = part.Write([]byte("%PDF-1.4"))
|
||||
if err != nil {
|
||||
t.Fatalf("write multipart file: %v", err)
|
||||
}
|
||||
err = writer.Close()
|
||||
if err != nil {
|
||||
t.Fatalf("close multipart writer: %v", err)
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/forms/libreoffice/convert", body)
|
||||
req.Header.Set("Content-Type", writer.FormDataContentType())
|
||||
|
||||
echoCtx := echo.New().NewContext(req, httptest.NewRecorder())
|
||||
logger := slog.New(slog.DiscardHandler)
|
||||
fs := gotenberg.NewFileSystem(new(gotenberg.OsMkdirAll))
|
||||
|
||||
_, cancel, err := newContext(echoCtx, logger, fs, 10*time.Second, 0, downloadFromConfig{disable: true})
|
||||
if cancel != nil {
|
||||
defer cancel()
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("newContext failed on a best-effort symlink for %q: %v", filename, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Two uploads sharing a filename must both reach the conversion. The second
|
||||
// used to overwrite the first in ctx.files, so one file was silently dropped
|
||||
// while both stayed on disk and counted against the body limit.
|
||||
func TestNewContext_DuplicateFilenamesAreBothKept(t *testing.T) {
|
||||
body := new(bytes.Buffer)
|
||||
writer := multipart.NewWriter(body)
|
||||
for _, content := range []string{"FIRST", "SECOND"} {
|
||||
part, err := writer.CreateFormFile("files", "doc.pdf")
|
||||
if err != nil {
|
||||
t.Fatalf("create multipart file: %v", err)
|
||||
}
|
||||
_, err = part.Write([]byte(content))
|
||||
if err != nil {
|
||||
t.Fatalf("write multipart file: %v", err)
|
||||
}
|
||||
}
|
||||
err := writer.Close()
|
||||
if err != nil {
|
||||
t.Fatalf("close multipart writer: %v", err)
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/forms/pdfengines/merge", body)
|
||||
req.Header.Set("Content-Type", writer.FormDataContentType())
|
||||
|
||||
echoCtx := echo.New().NewContext(req, httptest.NewRecorder())
|
||||
logger := slog.New(slog.DiscardHandler)
|
||||
fs := gotenberg.NewFileSystem(new(gotenberg.OsMkdirAll))
|
||||
|
||||
ctx, cancel, err := newContext(echoCtx, logger, fs, 10*time.Second, 0, downloadFromConfig{disable: true})
|
||||
if err != nil {
|
||||
t.Fatalf("newContext returned error: %v", err)
|
||||
}
|
||||
defer cancel()
|
||||
|
||||
if got := len(ctx.files); got != 2 {
|
||||
t.Fatalf("ctx.files = %d entries, want 2: a duplicate filename dropped a file", got)
|
||||
}
|
||||
if got := len(ctx.filesByField["files"]); got != 2 {
|
||||
t.Fatalf("filesByField[files] = %d entries, want 2", got)
|
||||
}
|
||||
|
||||
// The two maps must agree, and both files must be distinct on disk.
|
||||
seen := make(map[string]struct{})
|
||||
for _, path := range ctx.files {
|
||||
if _, ok := ctx.diskToOriginal[path]; !ok {
|
||||
t.Fatalf("path %q has no diskToOriginal entry", path)
|
||||
}
|
||||
seen[path] = struct{}{}
|
||||
}
|
||||
if len(seen) != 2 {
|
||||
t.Fatalf("distinct disk paths = %d, want 2", len(seen))
|
||||
}
|
||||
}
|
||||
|
||||
// A redirect target is filtered inside the HTTP client, so the policy verdict
|
||||
// surfaces from client.Do rather than from the pre-flight check. It used to be
|
||||
// interpolated into the response body, so a redirect described the allow-list,
|
||||
// the deny-list or the IP policy where the first hop returns a generic 403.
|
||||
func TestNewContext_DownloadFromRedirectVerdictStaysGeneric(t *testing.T) {
|
||||
private := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Disposition", `attachment; filename="secret.txt"`)
|
||||
_, _ = w.Write([]byte("internal"))
|
||||
}))
|
||||
defer private.Close()
|
||||
|
||||
redirector := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
http.Redirect(w, r, private.URL+"/secret", http.StatusFound)
|
||||
}))
|
||||
defer redirector.Close()
|
||||
|
||||
payload, err := json.Marshal([]downloadFrom{{Url: redirector.URL + "/start"}})
|
||||
if err != nil {
|
||||
t.Fatalf("marshal downloadFrom payload: %v", err)
|
||||
}
|
||||
|
||||
body := new(bytes.Buffer)
|
||||
writer := multipart.NewWriter(body)
|
||||
err = writer.WriteField("downloadFrom", string(payload))
|
||||
if err != nil {
|
||||
t.Fatalf("write downloadFrom field: %v", err)
|
||||
}
|
||||
err = writer.Close()
|
||||
if err != nil {
|
||||
t.Fatalf("close multipart writer: %v", err)
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/forms/libreoffice/convert", body)
|
||||
req.Header.Set("Content-Type", writer.FormDataContentType())
|
||||
|
||||
echoCtx := echo.New().NewContext(req, httptest.NewRecorder())
|
||||
logger := slog.New(slog.DiscardHandler)
|
||||
fs := gotenberg.NewFileSystem(new(gotenberg.OsMkdirAll))
|
||||
|
||||
// The first hop is allowed, the redirect target is denied by the deny-list.
|
||||
denyList := []*regexp2.Regexp{regexp2.MustCompile("^"+regexp.QuoteMeta(private.URL), regexp2.None)}
|
||||
|
||||
_, cancel, err := newContext(echoCtx, logger, fs, 10*time.Second, 0, downloadFromConfig{
|
||||
denyList: denyList,
|
||||
maxRetry: 0,
|
||||
})
|
||||
if cancel != nil {
|
||||
defer cancel()
|
||||
}
|
||||
if err == nil {
|
||||
t.Fatal("expected the redirect to a denied host to fail")
|
||||
}
|
||||
|
||||
status, message := ParseError(err)
|
||||
if status != http.StatusForbidden {
|
||||
t.Fatalf("status = %d, want %d: a filtered redirect must answer like a filtered first hop", status, http.StatusForbidden)
|
||||
}
|
||||
if message != http.StatusText(http.StatusForbidden) {
|
||||
t.Fatalf("message = %q, want the generic %q", message, http.StatusText(http.StatusForbidden))
|
||||
}
|
||||
// The response must not name the policy, the pattern, or the blocked host.
|
||||
for _, leak := range []string{"denied list", "allowed list", "non-public", "expression", private.URL} {
|
||||
if strings.Contains(message, leak) {
|
||||
t.Fatalf("response message %q leaks %q", message, leak)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -41,6 +41,8 @@ type FormData struct {
|
||||
files map[string]string
|
||||
filesByField map[string][]string
|
||||
diskToOriginal map[string]string
|
||||
fileOrder map[string]int
|
||||
fileBase map[string]string
|
||||
errors error
|
||||
}
|
||||
|
||||
@@ -582,24 +584,42 @@ func (form *FormData) paths(extensions []string, target *[]string) *FormData {
|
||||
}
|
||||
|
||||
// See https://github.com/gotenberg/gotenberg/issues/139.
|
||||
originals := make(gotenberg.AlphanumericSort, len(entries))
|
||||
for i, e := range entries {
|
||||
originals[i] = e.original
|
||||
}
|
||||
sort.Sort(originals)
|
||||
//
|
||||
// Sort on the filename as received rather than on the map key. The key
|
||||
// carries the suffix uniqueFilename adds when two uploads share a name,
|
||||
// and that suffix would otherwise decide the order: "doc (2).pdf" sorts
|
||||
// before "doc.pdf". Ordering on the received name keeps the pair adjacent,
|
||||
// and the arrival index breaks the tie, so duplicates merge in the order
|
||||
// the caller sent them. A file with a unique name is unaffected, since its
|
||||
// received name and its key are the same string.
|
||||
sort.SliceStable(entries, func(i, j int) bool {
|
||||
nameI := form.receivedName(entries[i].disk, entries[i].original)
|
||||
nameJ := form.receivedName(entries[j].disk, entries[j].original)
|
||||
if nameI != nameJ {
|
||||
return gotenberg.AlphanumericSort{nameI, nameJ}.Less(0, 1)
|
||||
}
|
||||
|
||||
return form.fileOrder[entries[i].disk] < form.fileOrder[entries[j].disk]
|
||||
})
|
||||
|
||||
// Build a lookup from original name to disk path.
|
||||
lookup := make(map[string]string, len(entries))
|
||||
for _, e := range entries {
|
||||
lookup[e.original] = e.disk
|
||||
}
|
||||
for _, o := range originals {
|
||||
*target = append(*target, lookup[o])
|
||||
*target = append(*target, e.disk)
|
||||
}
|
||||
|
||||
return form
|
||||
}
|
||||
|
||||
// receivedName returns the filename the file at disk arrived under, before
|
||||
// de-duplication, falling back to fallback.
|
||||
func (form *FormData) receivedName(disk, fallback string) string {
|
||||
base, ok := form.fileBase[disk]
|
||||
if ok {
|
||||
return base
|
||||
}
|
||||
|
||||
return fallback
|
||||
}
|
||||
|
||||
// append adds an error to the list of errors.
|
||||
func (form *FormData) append(err error) {
|
||||
form.errors = errors.Join(form.errors, err)
|
||||
@@ -612,7 +632,7 @@ func (form *FormData) mustValue(key string, target any, defaultValue any) *FormD
|
||||
val, ok := form.values[key]
|
||||
|
||||
if !ok || val[0] == "" {
|
||||
switch t := (target).(type) {
|
||||
switch t := target.(type) {
|
||||
case *string:
|
||||
*t = defaultValue.(string)
|
||||
case *bool:
|
||||
@@ -659,7 +679,7 @@ func (form *FormData) mustMandatoryField(key string, target any) *FormData {
|
||||
func (form *FormData) mustAssign(key, value string, target any) *FormData {
|
||||
var err error
|
||||
|
||||
switch t := (target).(type) {
|
||||
switch t := target.(type) {
|
||||
case *string:
|
||||
*t = value
|
||||
case *bool:
|
||||
|
||||
@@ -1895,3 +1895,81 @@ func TestFormData_Watermarks(t *testing.T) {
|
||||
t.Errorf("expected %+v, got %+v", want, got)
|
||||
}
|
||||
}
|
||||
|
||||
// De-duplicating a repeated filename must not change merge order. Files with
|
||||
// unique names keep exactly the order they had before de-duplication existed,
|
||||
// and two files sharing a name merge in the order the caller sent them.
|
||||
func TestFormData_paths_DuplicateFilenamesKeepUploadOrder(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
scenario string
|
||||
files map[string]string
|
||||
fileBase map[string]string
|
||||
order map[string]int
|
||||
want []string
|
||||
}{
|
||||
{
|
||||
scenario: "unique names sort exactly as before",
|
||||
files: map[string]string{"b.pdf": "/w/2", "a.pdf": "/w/1", "c.pdf": "/w/3"},
|
||||
fileBase: map[string]string{"/w/1": "a.pdf", "/w/2": "b.pdf", "/w/3": "c.pdf"},
|
||||
order: map[string]int{"/w/1": 0, "/w/2": 1, "/w/3": 2},
|
||||
want: []string{"/w/1", "/w/2", "/w/3"},
|
||||
},
|
||||
{
|
||||
scenario: "numeric prefixes still win",
|
||||
files: map[string]string{"10_x.pdf": "/w/3", "2_x.pdf": "/w/2", "1_x.pdf": "/w/1"},
|
||||
fileBase: map[string]string{"/w/1": "1_x.pdf", "/w/2": "2_x.pdf", "/w/3": "10_x.pdf"},
|
||||
order: map[string]int{"/w/1": 0, "/w/2": 1, "/w/3": 2},
|
||||
want: []string{"/w/1", "/w/2", "/w/3"},
|
||||
},
|
||||
{
|
||||
scenario: "duplicates merge in upload order, not suffix order",
|
||||
files: map[string]string{"doc.pdf": "/w/1", "doc (2).pdf": "/w/2"},
|
||||
fileBase: map[string]string{"/w/1": "doc.pdf", "/w/2": "doc.pdf"},
|
||||
order: map[string]int{"/w/1": 0, "/w/2": 1},
|
||||
want: []string{"/w/1", "/w/2"},
|
||||
},
|
||||
{
|
||||
scenario: "duplicates stay adjacent and in position",
|
||||
files: map[string]string{
|
||||
"a.pdf": "/w/1", "doc.pdf": "/w/2", "doc (2).pdf": "/w/3", "z.pdf": "/w/4",
|
||||
},
|
||||
fileBase: map[string]string{
|
||||
"/w/1": "a.pdf", "/w/2": "doc.pdf", "/w/3": "doc.pdf", "/w/4": "z.pdf",
|
||||
},
|
||||
order: map[string]int{"/w/1": 0, "/w/2": 1, "/w/3": 2, "/w/4": 3},
|
||||
want: []string{"/w/1", "/w/2", "/w/3", "/w/4"},
|
||||
},
|
||||
{
|
||||
scenario: "three copies keep their order",
|
||||
files: map[string]string{"r.pdf": "/w/1", "r (2).pdf": "/w/2", "r (3).pdf": "/w/3"},
|
||||
fileBase: map[string]string{"/w/1": "r.pdf", "/w/2": "r.pdf", "/w/3": "r.pdf"},
|
||||
order: map[string]int{"/w/1": 0, "/w/2": 1, "/w/3": 2},
|
||||
want: []string{"/w/1", "/w/2", "/w/3"},
|
||||
},
|
||||
} {
|
||||
t.Run(tc.scenario, func(t *testing.T) {
|
||||
form := &FormData{
|
||||
files: tc.files,
|
||||
filesByField: map[string][]string{},
|
||||
fileBase: tc.fileBase,
|
||||
fileOrder: tc.order,
|
||||
}
|
||||
|
||||
// Map iteration is randomised, so run it repeatedly: an unstable
|
||||
// comparator shows up as a differing result across runs.
|
||||
for range 50 {
|
||||
var got []string
|
||||
form.paths([]string{".pdf"}, &got)
|
||||
|
||||
if len(got) != len(tc.want) {
|
||||
t.Fatalf("paths() returned %d entries, want %d", len(got), len(tc.want))
|
||||
}
|
||||
for i := range got {
|
||||
if got[i] != tc.want[i] {
|
||||
t.Fatalf("paths() = %v, want %v", got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,8 +12,8 @@ import (
|
||||
|
||||
"github.com/coreos/go-oidc/v3/oidc"
|
||||
"github.com/google/uuid"
|
||||
"github.com/labstack/echo/v4"
|
||||
"github.com/labstack/echo/v4/middleware"
|
||||
"github.com/labstack/echo/v5"
|
||||
"github.com/labstack/echo/v5/middleware"
|
||||
"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
@@ -37,10 +37,12 @@ var (
|
||||
// ParseError parses an error and returns the corresponding HTTP status and
|
||||
// HTTP message.
|
||||
func ParseError(err error) (int, string) {
|
||||
var echoErr *echo.HTTPError
|
||||
ok := errors.As(err, &echoErr)
|
||||
if ok {
|
||||
return echoErr.Code, http.StatusText(echoErr.Code)
|
||||
// [echo.StatusCode] also matches the router's ErrNotFound and
|
||||
// ErrMethodNotAllowed sentinels, which Echo v5 no longer models as
|
||||
// [echo.HTTPError]. Matching that type alone would let every unrouted
|
||||
// request fall through to a 500.
|
||||
if code := echo.StatusCode(err); code != 0 {
|
||||
return code, http.StatusText(code)
|
||||
}
|
||||
|
||||
if errors.Is(err, context.DeadlineExceeded) {
|
||||
@@ -100,14 +102,14 @@ const statusClientClosedRequest = 499
|
||||
// A server-side timeout is [context.DeadlineExceeded], mapped to 503 by
|
||||
// [ParseError], and is deliberately not treated as a client abort.
|
||||
// See https://github.com/gotenberg/gotenberg/issues/1627.
|
||||
func requestCanceled(c echo.Context, err error) bool {
|
||||
func requestCanceled(c *echo.Context, err error) bool {
|
||||
return errors.Is(err, context.Canceled) && errors.Is(c.Request().Context().Err(), context.Canceled)
|
||||
}
|
||||
|
||||
// httpErrorHandler is the centralized HTTP error handler. It parses the error,
|
||||
// returns a response as "text/plain; charset=UTF-8".
|
||||
func httpErrorHandler() echo.HTTPErrorHandler {
|
||||
return func(err error, c echo.Context) {
|
||||
return func(c *echo.Context, err error) {
|
||||
logger := c.Get("logger").(*slog.Logger)
|
||||
|
||||
if requestCanceled(c, err) {
|
||||
@@ -134,7 +136,7 @@ func httpErrorHandler() echo.HTTPErrorHandler {
|
||||
// startTime := c.Get("startTime").(time.Time)
|
||||
func latencyMiddleware() echo.MiddlewareFunc {
|
||||
return func(next echo.HandlerFunc) echo.HandlerFunc {
|
||||
return func(c echo.Context) error {
|
||||
return func(c *echo.Context) error {
|
||||
// First piece for calculating the latency.
|
||||
startTime := time.Now()
|
||||
c.Set("startTime", startTime)
|
||||
@@ -159,7 +161,7 @@ func latencyMiddleware() echo.MiddlewareFunc {
|
||||
// }
|
||||
func rootPathMiddleware(rootPath string) echo.MiddlewareFunc {
|
||||
return func(next echo.HandlerFunc) echo.HandlerFunc {
|
||||
return func(c echo.Context) error {
|
||||
return func(c *echo.Context) error {
|
||||
c.Set("rootPath", rootPath)
|
||||
// Call the next middleware in the chain.
|
||||
return next(c)
|
||||
@@ -173,7 +175,7 @@ func rootPathMiddleware(rootPath string) echo.MiddlewareFunc {
|
||||
// outputFilename := c.Get("outputFilename").(string)
|
||||
func outputFilenameMiddleware() echo.MiddlewareFunc {
|
||||
return func(next echo.HandlerFunc) echo.HandlerFunc {
|
||||
return func(c echo.Context) error {
|
||||
return func(c *echo.Context) error {
|
||||
filename := c.Request().Header.Get("Gotenberg-Output-Filename")
|
||||
// Keep only the last path segment, so that a caller cannot name an
|
||||
// output file after a path.
|
||||
@@ -203,10 +205,18 @@ func telemetryMiddleware(logger *slog.Logger, serverName, correlationIdHeader st
|
||||
semconvSrv := semconvutil.NewHTTPServer(meter)
|
||||
|
||||
return func(next echo.HandlerFunc) echo.HandlerFunc {
|
||||
return func(c echo.Context) error {
|
||||
return func(c *echo.Context) error {
|
||||
startTime := c.Get("startTime").(time.Time)
|
||||
rootPath := c.Get("rootPath").(string)
|
||||
|
||||
// Echo v5 returns the bare [http.ResponseWriter] from
|
||||
// Context.Response, so unwrap it to read the recorded status and
|
||||
// size below.
|
||||
response, errUnwrap := echo.UnwrapResponse(c.Response())
|
||||
if errUnwrap != nil {
|
||||
return fmt.Errorf("unwrap response: %w", errUnwrap)
|
||||
}
|
||||
|
||||
request := c.Request()
|
||||
savedCtx := request.Context()
|
||||
defer func() {
|
||||
@@ -239,7 +249,7 @@ func telemetryMiddleware(logger *slog.Logger, serverName, correlationIdHeader st
|
||||
|
||||
err := next(c)
|
||||
if err != nil {
|
||||
c.Error(err)
|
||||
c.Echo().HTTPErrorHandler(c, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -288,7 +298,7 @@ func telemetryMiddleware(logger *slog.Logger, serverName, correlationIdHeader st
|
||||
err := next(c)
|
||||
finishTime := time.Now()
|
||||
|
||||
status := c.Response().Status
|
||||
status := response.Status
|
||||
canceled := false
|
||||
if err != nil {
|
||||
canceled = requestCanceled(c, err)
|
||||
@@ -300,58 +310,68 @@ func telemetryMiddleware(logger *slog.Logger, serverName, correlationIdHeader st
|
||||
}
|
||||
|
||||
span.SetAttributes(attribute.String("error", err.Error()))
|
||||
c.Error(err)
|
||||
c.Echo().HTTPErrorHandler(c, err)
|
||||
}
|
||||
|
||||
span.SetStatus(semconvSrv.Status(status))
|
||||
span.SetAttributes(semconvSrv.ResponseTraceAttrs(semconvutil.ResponseTelemetry{
|
||||
StatusCode: status,
|
||||
WriteBytes: c.Response().Size,
|
||||
WriteBytes: response.Size,
|
||||
})...)
|
||||
|
||||
accessLogger := logger.
|
||||
With(slog.String("log_type", "access")).
|
||||
With(slog.String("correlation_id", correlationId)).
|
||||
With(slog.String("remote_ip", c.RealIP())).
|
||||
With(slog.String("host", c.Request().Host)).
|
||||
With(slog.String("uri", c.Request().RequestURI)).
|
||||
With(slog.String("method", c.Request().Method)).
|
||||
With(slog.String("path", routePath)).
|
||||
With(slog.String("referer", c.Request().Referer())).
|
||||
With(slog.String("user_agent", c.Request().UserAgent())).
|
||||
With(slog.Int("status", c.Response().Status)).
|
||||
With(slog.Int64("latency", int64(finishTime.Sub(startTime)))).
|
||||
With(slog.String("latency_human", finishTime.Sub(startTime).String())).
|
||||
With(slog.Int64("bytes_in", c.Request().ContentLength)).
|
||||
With(slog.Int64("bytes_out", c.Response().Size))
|
||||
// Pick the level and message before building the record: err.Error
|
||||
// walks a joined error chain, and the nil-error branch has no use
|
||||
// for it.
|
||||
level := slog.LevelInfo
|
||||
msg := "request handled"
|
||||
|
||||
switch {
|
||||
case err == nil:
|
||||
accessLogger.InfoContext(ctx, "request handled")
|
||||
case canceled:
|
||||
// A client abort is expected, not a server failure; keep it
|
||||
// visible but out of the error stream.
|
||||
accessLogger.InfoContext(ctx, err.Error())
|
||||
msg = err.Error()
|
||||
default:
|
||||
accessLogger.ErrorContext(ctx, err.Error())
|
||||
level = slog.LevelError
|
||||
msg = err.Error()
|
||||
}
|
||||
|
||||
// One record rather than a chain of With calls. Each With clones
|
||||
// the whole handler chain, and this logger fans out to a JSON
|
||||
// handler and an OpenTelemetry bridge that is wired in even when no
|
||||
// exporter is configured, so a 14-deep chain clones both sub-chains
|
||||
// 14 times to emit a single line.
|
||||
latency := finishTime.Sub(startTime)
|
||||
|
||||
logger.LogAttrs(ctx, level, msg,
|
||||
slog.String("log_type", "access"),
|
||||
slog.String("correlation_id", correlationId),
|
||||
slog.String("remote_ip", c.RealIP()),
|
||||
slog.String("host", c.Request().Host),
|
||||
slog.String("uri", c.Request().RequestURI),
|
||||
slog.String("method", c.Request().Method),
|
||||
slog.String("path", routePath),
|
||||
slog.String("referer", c.Request().Referer()),
|
||||
slog.String("user_agent", c.Request().UserAgent()),
|
||||
slog.Int("status", response.Status),
|
||||
slog.Int64("latency", int64(latency)),
|
||||
slog.String("latency_human", latency.String()),
|
||||
slog.Int64("bytes_in", c.Request().ContentLength),
|
||||
slog.Int64("bytes_out", response.Size),
|
||||
)
|
||||
|
||||
additionalAttributes := []attribute.KeyValue{
|
||||
semconvSrv.Route(routePath),
|
||||
}
|
||||
|
||||
semconvSrv.RecordMetrics(ctx, semconvutil.ServerMetricData{
|
||||
ServerName: serverName,
|
||||
ResponseSize: c.Response().Size,
|
||||
MetricAttributes: semconvutil.MetricAttributes{
|
||||
Req: request,
|
||||
StatusCode: status,
|
||||
AdditionalAttributes: additionalAttributes,
|
||||
},
|
||||
MetricData: semconvutil.MetricData{
|
||||
RequestSize: request.ContentLength,
|
||||
ElapsedTime: float64(time.Since(startTime)) / float64(time.Millisecond),
|
||||
},
|
||||
ServerName: serverName,
|
||||
ResponseSize: response.Size,
|
||||
Req: request,
|
||||
StatusCode: status,
|
||||
AdditionalAttributes: additionalAttributes,
|
||||
RequestSize: request.ContentLength,
|
||||
ElapsedTime: float64(time.Since(startTime)) / float64(time.Millisecond),
|
||||
})
|
||||
|
||||
return nil
|
||||
@@ -361,7 +381,7 @@ func telemetryMiddleware(logger *slog.Logger, serverName, correlationIdHeader st
|
||||
|
||||
// basicAuthMiddleware manages basic authentication.
|
||||
func basicAuthMiddleware(username, password string) echo.MiddlewareFunc {
|
||||
return middleware.BasicAuth(func(u string, p string, e echo.Context) (bool, error) {
|
||||
return middleware.BasicAuth(func(c *echo.Context, u string, p string) (bool, error) {
|
||||
if subtle.ConstantTimeCompare([]byte(u), []byte(username)) == 1 &&
|
||||
subtle.ConstantTimeCompare([]byte(p), []byte(password)) == 1 {
|
||||
return true, nil
|
||||
@@ -407,7 +427,7 @@ func (a *Api) buildOidcVerifier() (*oidc.IDTokenVerifier, error) {
|
||||
// it to the client.
|
||||
func oidcAuthMiddleware(verifier *oidc.IDTokenVerifier) echo.MiddlewareFunc {
|
||||
return func(next echo.HandlerFunc) echo.HandlerFunc {
|
||||
return func(c echo.Context) error {
|
||||
return func(c *echo.Context) error {
|
||||
rawToken, ok := strings.CutPrefix(c.Request().Header.Get("Authorization"), "Bearer ")
|
||||
if !ok || rawToken == "" {
|
||||
return echo.NewHTTPError(http.StatusUnauthorized, "a Bearer token is required in the Authorization header")
|
||||
@@ -436,7 +456,7 @@ func oidcAuthMiddleware(verifier *oidc.IDTokenVerifier) echo.MiddlewareFunc {
|
||||
// cancel := c.Get("cancel").(context.CancelFunc)
|
||||
func contextMiddleware(fs *gotenberg.FileSystem, timeout time.Duration, bodyLimit int64, downloadFromCfg downloadFromConfig) echo.MiddlewareFunc {
|
||||
return func(next echo.HandlerFunc) echo.HandlerFunc {
|
||||
return func(c echo.Context) error {
|
||||
return func(c *echo.Context) error {
|
||||
logger, _ := c.Get("logger").(*slog.Logger)
|
||||
if logger == nil {
|
||||
return errors.New("no logger in context (possible pool reuse)")
|
||||
@@ -497,7 +517,7 @@ func contextMiddleware(fs *gotenberg.FileSystem, timeout time.Duration, bodyLimi
|
||||
// handler fails to timeout as expected.
|
||||
func hardTimeoutMiddleware(hardTimeout time.Duration) echo.MiddlewareFunc {
|
||||
return func(next echo.HandlerFunc) echo.HandlerFunc {
|
||||
return func(c echo.Context) error {
|
||||
return func(c *echo.Context) error {
|
||||
// Guard the type assertion so a pooled [echo.Context] whose
|
||||
// store has been recycled under us does not crash the process.
|
||||
// See the webhook async handler for the race this protects
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/rsa"
|
||||
@@ -9,13 +10,15 @@ import (
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/coreos/go-oidc/v3/oidc"
|
||||
"github.com/coreos/go-oidc/v3/oidc/oidctest"
|
||||
"github.com/labstack/echo/v4"
|
||||
"github.com/labstack/echo/v5"
|
||||
)
|
||||
|
||||
// TestRequestCanceled pins the client-abort discriminator: only a
|
||||
@@ -73,7 +76,7 @@ func TestHttpErrorHandler_ClientClosedRequest(t *testing.T) {
|
||||
c := echo.New().NewContext(req, rec)
|
||||
c.Set("logger", slog.New(slog.DiscardHandler))
|
||||
|
||||
httpErrorHandler()(tc.err, c)
|
||||
httpErrorHandler()(c, tc.err)
|
||||
|
||||
if rec.Code != tc.wantStatus {
|
||||
t.Fatalf("status = %d, want %d", rec.Code, tc.wantStatus)
|
||||
@@ -105,7 +108,7 @@ func TestOutputFilenameMiddleware(t *testing.T) {
|
||||
{"control characters", "fo\x01o\x7f", "foo"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
handler := outputFilenameMiddleware()(func(c echo.Context) error { return nil })
|
||||
handler := outputFilenameMiddleware()(func(c *echo.Context) error { return nil })
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/", nil)
|
||||
if tc.header != "" {
|
||||
@@ -131,7 +134,7 @@ func TestOutputFilenameMiddleware(t *testing.T) {
|
||||
|
||||
func TestHardTimeoutMiddleware_MissingLoggerReturnsErrorInsteadOfPanicking(t *testing.T) {
|
||||
mw := hardTimeoutMiddleware(100 * time.Millisecond)
|
||||
handler := mw(func(c echo.Context) error { return nil })
|
||||
handler := mw(func(c *echo.Context) error { return nil })
|
||||
|
||||
e := echo.New()
|
||||
req := httptest.NewRequest(http.MethodGet, "/", nil)
|
||||
@@ -220,7 +223,7 @@ func TestOidcAuthMiddleware(t *testing.T) {
|
||||
}
|
||||
c := echo.New().NewContext(req, httptest.NewRecorder())
|
||||
|
||||
handler := oidcAuthMiddleware(verifier)(func(c echo.Context) error {
|
||||
handler := oidcAuthMiddleware(verifier)(func(c *echo.Context) error {
|
||||
return c.NoContent(http.StatusOK)
|
||||
})
|
||||
|
||||
@@ -243,3 +246,114 @@ func TestOidcAuthMiddleware(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestParseError_StatusMapping pins the statuses [ParseError] derives from the
|
||||
// errors Echo and Gotenberg produce.
|
||||
//
|
||||
// Echo v5 models the router's ErrNotFound and ErrMethodNotAllowed as an
|
||||
// unexported type rather than [echo.HTTPError], so matching that type alone
|
||||
// would turn every unrouted request into a 500. It also guards the ordering:
|
||||
// Gotenberg's own [SentinelHttpError] carries a client-facing message and must
|
||||
// not be shadowed by the generic status lookup.
|
||||
func TestParseError_StatusMapping(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
scenario string
|
||||
err error
|
||||
wantStatus int
|
||||
wantMessage string
|
||||
}{
|
||||
{"router not found", echo.ErrNotFound, http.StatusNotFound, http.StatusText(http.StatusNotFound)},
|
||||
{"router method not allowed", echo.ErrMethodNotAllowed, http.StatusMethodNotAllowed, http.StatusText(http.StatusMethodNotAllowed)},
|
||||
{"explicit HTTP error", echo.NewHTTPError(http.StatusUnauthorized, "nope"), http.StatusUnauthorized, http.StatusText(http.StatusUnauthorized)},
|
||||
{"wrapped HTTP error", fmt.Errorf("authenticate request: %w", echo.NewHTTPError(http.StatusUnauthorized, "nope")), http.StatusUnauthorized, http.StatusText(http.StatusUnauthorized)},
|
||||
{"sentinel keeps its message", NewSentinelHttpError(http.StatusBadRequest, "Invalid 'foo' form field value"), http.StatusBadRequest, "Invalid 'foo' form field value"},
|
||||
{"unknown error", errors.New("boom"), http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)},
|
||||
} {
|
||||
t.Run(tc.scenario, func(t *testing.T) {
|
||||
status, message := ParseError(tc.err)
|
||||
if status != tc.wantStatus {
|
||||
t.Fatalf("status = %d, want %d", status, tc.wantStatus)
|
||||
}
|
||||
if message != tc.wantMessage {
|
||||
t.Fatalf("message = %q, want %q", message, tc.wantMessage)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestNewEchoServer_RealIP pins the client IP extraction that the access log's
|
||||
// "remote_ip" field depends on.
|
||||
//
|
||||
// Echo v5.1.0 dropped the X-Forwarded-For and X-Real-IP fallbacks from
|
||||
// Context.RealIP, so without an explicit extractor a Gotenberg behind a reverse
|
||||
// proxy would log the proxy's address for every request. [newEchoServer]
|
||||
// restores the previous behavior.
|
||||
func TestNewEchoServer_RealIP(t *testing.T) {
|
||||
srv := newEchoServer()
|
||||
if srv.IPExtractor == nil {
|
||||
t.Fatal("no IPExtractor configured: remote_ip would report the proxy address")
|
||||
}
|
||||
|
||||
for _, tc := range []struct {
|
||||
scenario string
|
||||
headers map[string]string
|
||||
want string
|
||||
}{
|
||||
{"x-forwarded-for keeps the client, not the proxy", map[string]string{"X-Forwarded-For": "203.0.113.7, 70.41.3.18"}, "203.0.113.7"},
|
||||
{"single x-forwarded-for", map[string]string{"X-Forwarded-For": "203.0.113.7"}, "203.0.113.7"},
|
||||
{"bracketed IPv6 is unwrapped", map[string]string{"X-Forwarded-For": "[2001:db8::1], 70.41.3.18"}, "2001:db8::1"},
|
||||
{"x-real-ip when no x-forwarded-for", map[string]string{"X-Real-IP": "203.0.113.9"}, "203.0.113.9"},
|
||||
{"no headers falls back to the remote address", nil, "192.0.2.1"},
|
||||
} {
|
||||
t.Run(tc.scenario, func(t *testing.T) {
|
||||
req := httptest.NewRequest(http.MethodGet, "/", nil)
|
||||
req.RemoteAddr = "192.0.2.1:1234"
|
||||
for key, value := range tc.headers {
|
||||
req.Header.Set(key, value)
|
||||
}
|
||||
|
||||
c := srv.NewContext(req, httptest.NewRecorder())
|
||||
|
||||
if got := c.RealIP(); got != tc.want {
|
||||
t.Fatalf("RealIP = %q, want %q", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestNewEchoServer_AttachmentServesAbsolutePath pins the filesystem that every
|
||||
// conversion response is sent through.
|
||||
//
|
||||
// Echo v5 serves files through Echo.Filesystem, an [fs.FS] rooted at the working
|
||||
// directory, and [fs.FS] rejects absolute names. Gotenberg builds every output
|
||||
// file under the request's temporary directory and hands Context.Attachment an
|
||||
// absolute path, so with the default filesystem every conversion route answers
|
||||
// 404 while still reading the whole upload.
|
||||
func TestNewEchoServer_AttachmentServesAbsolutePath(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "output.pdf")
|
||||
want := []byte("%PDF-1.7 not really a PDF")
|
||||
|
||||
err := os.WriteFile(path, want, 0o600)
|
||||
if err != nil {
|
||||
t.Fatalf("write fixture: %v", err)
|
||||
}
|
||||
|
||||
srv := newEchoServer()
|
||||
rec := httptest.NewRecorder()
|
||||
c := srv.NewContext(httptest.NewRequest(http.MethodPost, "/", nil), rec)
|
||||
|
||||
err = c.Attachment(path, "output.pdf")
|
||||
if err != nil {
|
||||
t.Fatalf("Attachment(%q) = %v, want nil", path, err)
|
||||
}
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want %d", rec.Code, http.StatusOK)
|
||||
}
|
||||
if got := rec.Body.Bytes(); !bytes.Equal(got, want) {
|
||||
t.Fatalf("body = %q, want %q", got, want)
|
||||
}
|
||||
if got := rec.Header().Get("Content-Disposition"); !strings.Contains(got, `filename="output.pdf"`) {
|
||||
t.Fatalf("Content-Disposition = %q, want it to carry filename=\"output.pdf\"", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,7 +4,7 @@ import (
|
||||
"log/slog"
|
||||
|
||||
"github.com/alexliesenfeld/health"
|
||||
"github.com/labstack/echo/v4"
|
||||
"github.com/labstack/echo/v5"
|
||||
|
||||
"github.com/gotenberg/gotenberg/v8/pkg/gotenberg"
|
||||
)
|
||||
@@ -83,7 +83,7 @@ func (ctx *ContextMock) SetLogger(logger *slog.Logger) {
|
||||
//
|
||||
// ctx := &api.ContextMock{Context: &api.Context{}}
|
||||
// ctx.setEchoContext(c)
|
||||
func (ctx *ContextMock) SetEchoContext(c echo.Context) {
|
||||
func (ctx *ContextMock) SetEchoContext(c *echo.Context) {
|
||||
ctx.echoCtx = c
|
||||
}
|
||||
|
||||
|
||||
@@ -17,12 +17,31 @@ import (
|
||||
"github.com/chromedp/cdproto/page"
|
||||
"github.com/chromedp/cdproto/runtime"
|
||||
"github.com/chromedp/chromedp"
|
||||
"github.com/dlclark/regexp2"
|
||||
"github.com/dlclark/regexp2/v2"
|
||||
"github.com/shirou/gopsutil/v4/process"
|
||||
|
||||
"github.com/gotenberg/gotenberg/v8/pkg/gotenberg"
|
||||
)
|
||||
|
||||
// chromiumDisableFeatures is the value of Chromium's --disable-features
|
||||
// switch.
|
||||
//
|
||||
// It restates the "site-per-process,Translate,BlinkGenPropertyTrees" default
|
||||
// from chromedp.DefaultExecAllocatorOptions (chromedp v0.14.2) on purpose:
|
||||
// chromedp.Flag keys its flags by switch name, so a second --disable-features
|
||||
// replaces chromedp's value instead of merging with it. Revisit this list when
|
||||
// bumping chromedp.
|
||||
//
|
||||
// WebUIOmniboxPopup and WebUIOmniboxAimPopup became enabled by default in
|
||||
// Chromium 151.0.7922.132. Their presenters build the address-bar popup WebUI
|
||||
// at browser start, headless included, which leaves a renderer process holding
|
||||
// ~85 MB of anonymous memory for a UI a PDF service can never show. Chromium
|
||||
// silently ignores feature names it does not know, so both stay harmless on
|
||||
// older builds (they exist but default to disabled on the Chromium pinned for
|
||||
// ppc64el) and once upstream eventually removes them.
|
||||
// See https://github.com/gotenberg/gotenberg/issues/1656.
|
||||
const chromiumDisableFeatures = "site-per-process,Translate,BlinkGenPropertyTrees,WebUIOmniboxPopup,WebUIOmniboxAimPopup"
|
||||
|
||||
type browser interface {
|
||||
gotenberg.Process
|
||||
pdf(ctx context.Context, logger *slog.Logger, url, outputPath string, options PdfOptions, aggregate *networkAggregate) error
|
||||
@@ -133,6 +152,8 @@ func (b *chromiumBrowser) Start(logger *slog.Logger) error {
|
||||
chromedp.Flag("disable-dev-shm-usage", true),
|
||||
// See https://github.com/gotenberg/gotenberg/issues/1293.
|
||||
chromedp.Flag("disable-component-update", false),
|
||||
// See https://github.com/gotenberg/gotenberg/issues/1656.
|
||||
chromedp.Flag("disable-features", chromiumDisableFeatures),
|
||||
)
|
||||
|
||||
if b.arguments.allowInsecureLocalhost {
|
||||
@@ -523,8 +544,45 @@ func (b *chromiumBrowser) do(ctx context.Context, logger *slog.Logger, url strin
|
||||
cancelOnMainPageError: taskCancel,
|
||||
})
|
||||
|
||||
var (
|
||||
crashed error
|
||||
crashedMu sync.RWMutex
|
||||
)
|
||||
|
||||
// See https://github.com/gotenberg/gotenberg/issues/1640.
|
||||
listenForEventTargetCrashed(taskCtx, logger, eventTargetCrashedOptions{
|
||||
crashed: &crashed,
|
||||
crashedMu: &crashedMu,
|
||||
cancel: taskCancel,
|
||||
})
|
||||
|
||||
runErr := chromedp.Run(taskCtx, tasks...)
|
||||
|
||||
// A crashed renderer is the root cause of every other failure this
|
||||
// conversion may have recorded, so check it first.
|
||||
// See https://github.com/gotenberg/gotenberg/issues/1640.
|
||||
crashedMu.RLock()
|
||||
defer crashedMu.RUnlock()
|
||||
|
||||
if crashed != nil {
|
||||
return fmt.Errorf("handle tasks: %w", crashed)
|
||||
}
|
||||
|
||||
// The browser context is only ever canceled when the browser process
|
||||
// dies or is stopped, never on a request timeout. If the run failed
|
||||
// and the browser context is done, the conversion failed because the
|
||||
// browser went away mid-flight; fail fast with the same crash error
|
||||
// instead of letting the error fall through as a generic context
|
||||
// cancellation. The check is gated on runErr so a successful
|
||||
// conversion is never discarded by a browser death that lands right
|
||||
// after it.
|
||||
// See https://github.com/gotenberg/gotenberg/issues/1640.
|
||||
if runErr != nil {
|
||||
if err := b.ctx.Err(); err != nil {
|
||||
return fmt.Errorf("handle tasks: %w", ErrChromiumCrashed)
|
||||
}
|
||||
}
|
||||
|
||||
// Check event-driven errors first — they take priority over chromedp.Run
|
||||
// errors because they carry the actual root cause (e.g., HTTP 500 from
|
||||
// the main page). When we cancel taskCtx on a main page error,
|
||||
|
||||
@@ -3,6 +3,7 @@ package chromium
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
@@ -37,3 +38,24 @@ func TestChromiumBrowser_Start_rejectsOverlappingStart(t *testing.T) {
|
||||
t.Fatal("expected the browser to stay not started")
|
||||
}
|
||||
}
|
||||
|
||||
// TestChromiumDisableFeatures guards the override described in
|
||||
// https://github.com/gotenberg/gotenberg/issues/1656. Gotenberg replaces
|
||||
// chromedp's --disable-features value rather than extending it, as
|
||||
// chromedp.Flag keys its flags by switch name. Dropping one of chromedp's own
|
||||
// entries while editing this list would silently re-enable it.
|
||||
func TestChromiumDisableFeatures(t *testing.T) {
|
||||
for _, feature := range []string{
|
||||
// chromedp.DefaultExecAllocatorOptions.
|
||||
"site-per-process",
|
||||
"Translate",
|
||||
"BlinkGenPropertyTrees",
|
||||
// The address-bar popup WebUI, built even in headless.
|
||||
"WebUIOmniboxPopup",
|
||||
"WebUIOmniboxAimPopup",
|
||||
} {
|
||||
if !slices.Contains(strings.Split(chromiumDisableFeatures, ","), feature) {
|
||||
t.Errorf("expected %q to be disabled, got %q", feature, chromiumDisableFeatures)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,7 +14,7 @@ import (
|
||||
|
||||
"github.com/alexliesenfeld/health"
|
||||
"github.com/chromedp/cdproto/network"
|
||||
"github.com/dlclark/regexp2"
|
||||
"github.com/dlclark/regexp2/v2"
|
||||
flag "github.com/spf13/pflag"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
@@ -71,6 +71,10 @@ var (
|
||||
// ErrResourceLoadingFailed happens when one or more resources failed to load.
|
||||
ErrResourceLoadingFailed = errors.New("resource loading failed")
|
||||
|
||||
// ErrChromiumCrashed happens when the Chromium renderer crashes during a
|
||||
// conversion.
|
||||
ErrChromiumCrashed = errors.New("chromium crashed")
|
||||
|
||||
// PDF specific.
|
||||
|
||||
// ErrOmitBackgroundWithoutPrintBackground happens if
|
||||
@@ -471,7 +475,7 @@ func (mod *Chromium) Descriptor() gotenberg.ModuleDescriptor {
|
||||
fs.String("chromium-host-resolver-rules", "", "Set custom mappings to the host resolver")
|
||||
fs.String("chromium-proxy-server", "", "Set the outbound proxy server; this switch only affects HTTP and HTTPS requests")
|
||||
fs.Bool("chromium-enable-environment-proxy", false, "Route Chromium's outbound requests through the proxy defined by the standard HTTP_PROXY, HTTPS_PROXY, and NO_PROXY variables, including credentials. Use this instead of --chromium-proxy-server for authenticated proxies, and leave --chromium-proxy-server and --chromium-host-resolver-rules unset")
|
||||
fs.StringSlice("chromium-allow-list", []string{}, "Set the allowed URLs for Chromium using regular expressions - supports multiple values")
|
||||
fs.StringSlice("chromium-allow-list", []string{}, `Set the allowed URLs for Chromium using regular expressions - supports multiple values. A match bypasses --chromium-deny-private-ips (CHROMIUM_DENY_PRIVATE_IPS) and --chromium-deny-public-ips (CHROMIUM_DENY_PUBLIC_IPS), so terminate the host or the pattern also matches suffix hosts, for example ^https?://internal\.svc(:|/|$)`)
|
||||
fs.StringSlice("chromium-deny-list", []string{`^file:(?!//\/tmp/).*`}, "Set the denied URLs for Chromium using regular expressions - supports multiple values")
|
||||
fs.Bool("chromium-deny-private-ips", false, "Reject URLs whose host resolves to a non-public IP address (loopback, RFC1918, link-local, unique-local). Enable on deployments that accept untrusted form input to mitigate SSRF against internal services")
|
||||
fs.Bool("chromium-deny-public-ips", false, "Reject URLs whose host resolves to a public IP address. Enable on air-gapped or data-governed deployments to prevent outbound traffic from leaving a private network")
|
||||
@@ -1106,6 +1110,8 @@ func chromiumErrorType(err error, queueReason string) string {
|
||||
errors.Is(err, ErrInvalidEvaluationExpression),
|
||||
errors.Is(err, ErrInvalidSelectorQuery):
|
||||
return gotenberg.ErrorTypeInvalidInput
|
||||
case errors.Is(err, ErrChromiumCrashed):
|
||||
return "chromium_unavailable"
|
||||
case errors.Is(err, gotenberg.ErrMaximumQueueSizeExceeded):
|
||||
return queueReason
|
||||
case errors.Is(err, gotenberg.ErrProcessAlreadyRestarting):
|
||||
|
||||
@@ -21,6 +21,7 @@ func TestChromiumErrorType(t *testing.T) {
|
||||
{"invalid resource http status", ErrInvalidResourceHttpStatusCode, "chromium_unavailable", "invalid_input"},
|
||||
{"loading failed", ErrLoadingFailed, "chromium_unavailable", "invalid_input"},
|
||||
{"resource loading failed", ErrResourceLoadingFailed, "chromium_unavailable", "invalid_input"},
|
||||
{"crashed", ErrChromiumCrashed, "chromium_unavailable", "chromium_unavailable"},
|
||||
{"invalid evaluation expression", ErrInvalidEvaluationExpression, "chromium_unavailable", "invalid_input"},
|
||||
{"invalid selector query", ErrInvalidSelectorQuery, "chromium_unavailable", "invalid_input"},
|
||||
{"pdf queue", gotenberg.ErrMaximumQueueSizeExceeded, "chromium_unavailable", "chromium_unavailable"},
|
||||
|
||||
@@ -14,11 +14,12 @@ import (
|
||||
|
||||
"github.com/chromedp/cdproto/cdp"
|
||||
"github.com/chromedp/cdproto/fetch"
|
||||
"github.com/chromedp/cdproto/inspector"
|
||||
"github.com/chromedp/cdproto/network"
|
||||
"github.com/chromedp/cdproto/page"
|
||||
"github.com/chromedp/cdproto/runtime"
|
||||
"github.com/chromedp/chromedp"
|
||||
"github.com/dlclark/regexp2"
|
||||
"github.com/dlclark/regexp2/v2"
|
||||
"golang.org/x/sync/errgroup"
|
||||
|
||||
"github.com/gotenberg/gotenberg/v8/pkg/gotenberg"
|
||||
@@ -205,7 +206,7 @@ func listenForEventRequestPaused(ctx context.Context, logger *slog.Logger, optio
|
||||
}
|
||||
|
||||
matchStart := time.Now()
|
||||
ok, err := header.Scope.MatchString(e.Request.URL)
|
||||
ok, err := gotenberg.MatchPattern(header.Scope, e.Request.URL)
|
||||
budget.consume(time.Since(matchStart))
|
||||
|
||||
switch {
|
||||
@@ -322,7 +323,15 @@ func listenForEventResponseReceived(
|
||||
return
|
||||
}
|
||||
|
||||
logger.DebugContext(ctx, fmt.Sprintf("event EventResponseReceived fired for a resource: %+v", ev.Response))
|
||||
// Formatting the whole response is the most expensive thing this
|
||||
// listener does, and it runs per sub-resource on chromedp's single
|
||||
// per-target event goroutine while that goroutine holds the mutex
|
||||
// it also takes to dispatch command responses. At the default log
|
||||
// level the result is discarded, so gate it on the level rather
|
||||
// than let slog drop it after the fact.
|
||||
if logger.Enabled(ctx, slog.LevelDebug) {
|
||||
logger.DebugContext(ctx, fmt.Sprintf("event EventResponseReceived fired for a resource: %+v", ev.Response))
|
||||
}
|
||||
|
||||
if slices.Contains(options.failOnResourceOnHttpStatusCode, ev.Response.Status) {
|
||||
if !shouldCheckResourceHttpStatusCode(ev.Response.URL, normalizedIgnoreDomains) {
|
||||
@@ -521,6 +530,38 @@ func listenForEventExceptionThrown(ctx context.Context, logger *slog.Logger, con
|
||||
})
|
||||
}
|
||||
|
||||
type eventTargetCrashedOptions struct {
|
||||
crashed *error
|
||||
crashedMu *sync.RWMutex
|
||||
cancel context.CancelFunc
|
||||
}
|
||||
|
||||
// listenForEventTargetCrashed listens for the Inspector.targetCrashed event,
|
||||
// which Chromium sends when the renderer serving the conversion's tab
|
||||
// crashes. chromedp enables the Inspector domain on every target but does
|
||||
// not handle this event: left alone, the in-flight CDP command never
|
||||
// receives a response and the conversion blocks until the request deadline.
|
||||
// Record the crash and cancel the task context so the conversion fails fast
|
||||
// instead.
|
||||
// See https://github.com/gotenberg/gotenberg/issues/1640.
|
||||
func listenForEventTargetCrashed(ctx context.Context, logger *slog.Logger, options eventTargetCrashedOptions) {
|
||||
chromedp.ListenTarget(ctx, func(ev any) {
|
||||
if _, ok := ev.(*inspector.EventTargetCrashed); ok {
|
||||
logger.DebugContext(ctx, "event EventTargetCrashed fired")
|
||||
|
||||
options.crashedMu.Lock()
|
||||
defer options.crashedMu.Unlock()
|
||||
|
||||
*options.crashed = ErrChromiumCrashed
|
||||
|
||||
// Cancel the task context so the in-flight CDP command aborts
|
||||
// immediately instead of waiting for a response the crashed
|
||||
// renderer can never send.
|
||||
options.cancel()
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// waitForEventDomContentEventFired registers a listener for the
|
||||
// DomContentEventFired event and returns a waiter that blocks until the
|
||||
// event fires or ctx is done. The listener registers at call time, not
|
||||
|
||||
@@ -11,6 +11,15 @@ import (
|
||||
// pathological page cannot grow the set without limit.
|
||||
const maxTrackedOrigins = 64
|
||||
|
||||
// maxTrackedRequests bounds the request id to URL map. Entries are dropped as
|
||||
// soon as the request settles, so the map normally holds only what is in
|
||||
// flight, but a request that never reports a loading-finished or
|
||||
// loading-failed event never settles. Without a cap, a page that opens
|
||||
// requests it never resolves would grow the map for the whole conversion, at
|
||||
// the cost of one full response URL per entry. Losing an entry only costs the
|
||||
// heaviest-resource URL attribution for that request.
|
||||
const maxTrackedRequests = 1024
|
||||
|
||||
// networkAggregate accumulates per-conversion network activity from Chromium
|
||||
// DevTools events. It is safe for concurrent use by the chromedp event listener
|
||||
// goroutine and the conversion goroutine that reads the snapshot afterwards.
|
||||
@@ -60,7 +69,9 @@ func (a *networkAggregate) onResponseReceived(ev *network.EventResponseReceived)
|
||||
a.origins[origin] = struct{}{}
|
||||
}
|
||||
}
|
||||
a.requestURLByID[ev.RequestID] = ev.Response.URL
|
||||
if len(a.requestURLByID) < maxTrackedRequests {
|
||||
a.requestURLByID[ev.RequestID] = ev.Response.URL
|
||||
}
|
||||
}
|
||||
|
||||
// onLoadingFinished records a successfully completed request and its size,
|
||||
@@ -81,6 +92,11 @@ func (a *networkAggregate) onLoadingFinished(ev *network.EventLoadingFinished) {
|
||||
a.heaviestBytes = size
|
||||
a.heaviestURL = a.requestURLByID[ev.RequestID]
|
||||
}
|
||||
|
||||
// The request has settled and nothing reads its URL again. Dropping it
|
||||
// keeps the map proportional to the requests in flight rather than to
|
||||
// every request the page ever made.
|
||||
delete(a.requestURLByID, ev.RequestID)
|
||||
}
|
||||
|
||||
// onLoadingFailed records a request that failed to complete.
|
||||
@@ -94,6 +110,9 @@ func (a *networkAggregate) onLoadingFailed(ev *network.EventLoadingFailed) {
|
||||
|
||||
a.requestCount++
|
||||
a.failedCount++
|
||||
|
||||
// Settled, like a finished request: its URL is never read again.
|
||||
delete(a.requestURLByID, ev.RequestID)
|
||||
}
|
||||
|
||||
func (a *networkAggregate) snapshot() networkStats {
|
||||
|
||||
@@ -98,3 +98,86 @@ func TestNetworkAggregate_ConcurrentSafe(t *testing.T) {
|
||||
t.Errorf("requestCount = %d, want 100", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestNetworkAggregate_SettledRequestsAreDropped covers the growth where every
|
||||
// response URL stayed in the map for the whole conversion even though nothing
|
||||
// reads it again once the request settles.
|
||||
func TestNetworkAggregate_SettledRequestsAreDropped(t *testing.T) {
|
||||
a := newNetworkAggregate()
|
||||
|
||||
for i := range 500 {
|
||||
id := network.RequestID(fmt.Sprintf("r%d", i))
|
||||
a.onResponseReceived(&network.EventResponseReceived{
|
||||
RequestID: id,
|
||||
Response: &network.Response{URL: fmt.Sprintf("https://host.example.com/%d", i)},
|
||||
})
|
||||
|
||||
if i%2 == 0 {
|
||||
a.onLoadingFinished(&network.EventLoadingFinished{RequestID: id, EncodedDataLength: 10})
|
||||
continue
|
||||
}
|
||||
|
||||
a.onLoadingFailed(&network.EventLoadingFailed{RequestID: id})
|
||||
}
|
||||
|
||||
a.mu.Lock()
|
||||
tracked := len(a.requestURLByID)
|
||||
a.mu.Unlock()
|
||||
|
||||
if tracked != 0 {
|
||||
t.Errorf("tracked requests = %d, want 0: settled requests must not be retained", tracked)
|
||||
}
|
||||
|
||||
// The bookkeeping the map feeds must survive the pruning.
|
||||
got := a.snapshot()
|
||||
if got.requestCount != 500 {
|
||||
t.Errorf("requestCount = %d, want 500", got.requestCount)
|
||||
}
|
||||
if got.failedCount != 250 {
|
||||
t.Errorf("failedCount = %d, want 250", got.failedCount)
|
||||
}
|
||||
}
|
||||
|
||||
// TestNetworkAggregate_UnsettledRequestCap verifies the ceiling that applies
|
||||
// when requests never settle, which is the only way the map can still grow.
|
||||
func TestNetworkAggregate_UnsettledRequestCap(t *testing.T) {
|
||||
a := newNetworkAggregate()
|
||||
|
||||
for i := range maxTrackedRequests + 500 {
|
||||
a.onResponseReceived(&network.EventResponseReceived{
|
||||
RequestID: network.RequestID(fmt.Sprintf("r%d", i)),
|
||||
Response: &network.Response{URL: fmt.Sprintf("https://host.example.com/%d", i)},
|
||||
})
|
||||
}
|
||||
|
||||
a.mu.Lock()
|
||||
tracked := len(a.requestURLByID)
|
||||
a.mu.Unlock()
|
||||
|
||||
if tracked != maxTrackedRequests {
|
||||
t.Errorf("tracked requests = %d, want %d (capped)", tracked, maxTrackedRequests)
|
||||
}
|
||||
}
|
||||
|
||||
// TestNetworkAggregate_HeaviestURLSurvivesPruning guards the attribution the
|
||||
// map exists for: the URL must still be resolved before the entry is dropped.
|
||||
func TestNetworkAggregate_HeaviestURLSurvivesPruning(t *testing.T) {
|
||||
a := newNetworkAggregate()
|
||||
|
||||
a.onResponseReceived(&network.EventResponseReceived{
|
||||
RequestID: "small",
|
||||
Response: &network.Response{URL: "https://example.com/small.css"},
|
||||
})
|
||||
a.onLoadingFinished(&network.EventLoadingFinished{RequestID: "small", EncodedDataLength: 10})
|
||||
|
||||
a.onResponseReceived(&network.EventResponseReceived{
|
||||
RequestID: "big",
|
||||
Response: &network.Response{URL: "https://example.com/big.png"},
|
||||
})
|
||||
a.onLoadingFinished(&network.EventLoadingFinished{RequestID: "big", EncodedDataLength: 4096})
|
||||
|
||||
got := a.snapshot()
|
||||
if got.heaviestURL != "https://example.com/big.png" || got.heaviestBytes != 4096 {
|
||||
t.Errorf("heaviest = (%q, %d), want (%q, 4096)", got.heaviestURL, got.heaviestBytes, "https://example.com/big.png")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,9 +11,10 @@ import (
|
||||
"net/netip"
|
||||
"net/url"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/dlclark/regexp2"
|
||||
"github.com/dlclark/regexp2/v2"
|
||||
"golang.org/x/net/http/httpproxy"
|
||||
|
||||
"github.com/gotenberg/gotenberg/v8/pkg/gotenberg"
|
||||
@@ -57,6 +58,18 @@ type pinningProxy struct {
|
||||
server *http.Server
|
||||
wg sync.WaitGroup
|
||||
|
||||
// closing is closed by Stop to force in-flight CONNECT tunnels shut.
|
||||
// [http.Server.Shutdown] cannot do it: net/http untracks a connection once
|
||||
// a handler hijacks it, so a tunnel would otherwise outlive the proxy that
|
||||
// created it. Recreated on every Start.
|
||||
closing chan struct{}
|
||||
|
||||
// maxTunnels ceilings the CONNECT handlers in flight. Tests may lower it.
|
||||
maxTunnels int64
|
||||
|
||||
// tunnels counts the CONNECT handlers in flight.
|
||||
tunnels atomic.Int64
|
||||
|
||||
logger *slog.Logger
|
||||
started bool
|
||||
mu sync.Mutex
|
||||
@@ -82,6 +95,7 @@ func newPinningProxy(allowList, denyList []*regexp2.Regexp, denyPrivateIPs, deny
|
||||
dialer := &net.Dialer{Timeout: 10 * time.Second}
|
||||
return dialer.DialContext(ctx, network, addr)
|
||||
},
|
||||
maxTunnels: maxConcurrentTunnels,
|
||||
}
|
||||
|
||||
if enableEnvironmentProxy {
|
||||
@@ -110,6 +124,7 @@ func (p *pinningProxy) Start(logger *slog.Logger) error {
|
||||
}
|
||||
|
||||
p.listener = l
|
||||
p.closing = make(chan struct{})
|
||||
p.logger = logger.With(slog.String("logger", "pinning-proxy"))
|
||||
p.server = &http.Server{
|
||||
Handler: http.HandlerFunc(p.serveHTTP),
|
||||
@@ -140,9 +155,18 @@ func (p *pinningProxy) Stop(logger *slog.Logger) error {
|
||||
return nil
|
||||
}
|
||||
srv := p.server
|
||||
closing := p.closing
|
||||
p.closing = nil
|
||||
p.started = false
|
||||
p.mu.Unlock()
|
||||
|
||||
// Force in-flight tunnels shut before draining the server. Shutdown does
|
||||
// not reach them, so a tunnel whose upstream never answers would otherwise
|
||||
// survive the proxy, and with it every Chromium restart.
|
||||
if closing != nil {
|
||||
close(closing)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
@@ -181,6 +205,25 @@ func (p *pinningProxy) serveHTTP(w http.ResponseWriter, req *http.Request) {
|
||||
// Chromium then negotiates TLS end-to-end with the original hostname in
|
||||
// SNI.
|
||||
func (p *pinningProxy) handleConnect(w http.ResponseWriter, req *http.Request) {
|
||||
// A ceiling, not a tuning knob: it bounds what a tunnel that refuses to end
|
||||
// can accumulate, whatever keeps it alive. [spliceIdleTimeout] ends a silent
|
||||
// tunnel, but a peer trickling a byte just under it stays "active" forever,
|
||||
// and a compromised renderer can hold the client side open to match.
|
||||
//
|
||||
// Chromium caps itself well below this. Its socket pool manager allows 128
|
||||
// sockets per proxy chain for normal traffic plus 128 for WebSocket
|
||||
// traffic, and every request Gotenberg's Chromium makes traverses this one
|
||||
// proxy chain, so an honest browser cannot exceed 256 tunnels here. At
|
||||
// double that, a real page never meets the ceiling and a hostile one stops
|
||||
// at it.
|
||||
if !p.acquireTunnel() {
|
||||
p.logger.WarnContext(req.Context(), fmt.Sprintf("CONNECT to '%s' refused: %d tunnels already in flight", req.Host, p.maxTunnels))
|
||||
http.Error(w, "too many tunnels", http.StatusServiceUnavailable)
|
||||
|
||||
return
|
||||
}
|
||||
defer p.releaseTunnel()
|
||||
|
||||
_, port, err := net.SplitHostPort(req.Host)
|
||||
if err != nil {
|
||||
http.Error(w, "bad CONNECT target", http.StatusBadRequest)
|
||||
@@ -264,24 +307,142 @@ func (p *pinningProxy) handleConnect(w http.ResponseWriter, req *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
// Splice bytes in both directions until either side closes.
|
||||
var splice sync.WaitGroup
|
||||
splice.Add(2)
|
||||
p.mu.Lock()
|
||||
closing := p.closing
|
||||
p.mu.Unlock()
|
||||
|
||||
spliceTunnel(client, upstream, closing, spliceIdleTimeout)
|
||||
}
|
||||
|
||||
// maxConcurrentTunnels is the default for [pinningProxy.maxTunnels]. See
|
||||
// [pinningProxy.handleConnect] for how the value is derived.
|
||||
const maxConcurrentTunnels = 512
|
||||
|
||||
// acquireTunnel reserves a slot for one CONNECT handler, reporting false when
|
||||
// the proxy is already at [pinningProxy.maxTunnels]. The compare-and-swap loop
|
||||
// keeps the check and the increment atomic, so concurrent handlers cannot
|
||||
// overshoot the ceiling between them.
|
||||
func (p *pinningProxy) acquireTunnel() bool {
|
||||
for {
|
||||
current := p.tunnels.Load()
|
||||
if current >= p.maxTunnels {
|
||||
return false
|
||||
}
|
||||
if p.tunnels.CompareAndSwap(current, current+1) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// releaseTunnel returns a slot taken by [pinningProxy.acquireTunnel].
|
||||
func (p *pinningProxy) releaseTunnel() {
|
||||
p.tunnels.Add(-1)
|
||||
}
|
||||
|
||||
// spliceIdleTimeout bounds a CONNECT tunnel in which no byte has moved in
|
||||
// either direction.
|
||||
//
|
||||
// Nothing else bounds one. The hijacked connections carry no deadline: the
|
||||
// server clears the header read deadline once the request line is in, and
|
||||
// net.Dialer.Timeout only covers the connect. net/http also untracks a
|
||||
// connection once it is hijacked, so neither Server.Shutdown nor a Chromium
|
||||
// restart reaps it. Left alone, an upstream that accepts the tunnel and then
|
||||
// answers nothing holds two goroutines and two sockets until the process dies.
|
||||
//
|
||||
// Sized well above any legitimate pause between a request and its response, so
|
||||
// a slow origin is never cut off. A transfer that keeps making progress
|
||||
// refreshes the deadline and runs for as long as it needs.
|
||||
const spliceIdleTimeout = 2 * time.Minute
|
||||
|
||||
// spliceTunnel copies bytes between the two ends of a CONNECT tunnel until
|
||||
// both directions finish, the tunnel sits idle for idleTimeout, or closing is
|
||||
// closed because the proxy is shutting down. Callers pass
|
||||
// [spliceIdleTimeout]; only tests shorten it.
|
||||
//
|
||||
// Each direction half-closes its destination once its source reaches EOF, so a
|
||||
// peer that waits for the request to end before answering still sees the EOF.
|
||||
// Idleness is tracked across both directions rather than per direction: the
|
||||
// client sends nothing for the length of a download, and half-closing its write
|
||||
// side then would tell the origin the client had gone away.
|
||||
func spliceTunnel(client, upstream net.Conn, closing <-chan struct{}, idleTimeout time.Duration) {
|
||||
var lastActivity atomic.Int64
|
||||
lastActivity.Store(time.Now().UnixNano())
|
||||
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(2)
|
||||
go func() {
|
||||
defer splice.Done()
|
||||
_, _ = io.Copy(upstream, client)
|
||||
defer wg.Done()
|
||||
copyTracking(upstream, client, &lastActivity, idleTimeout)
|
||||
if cw, ok := upstream.(interface{ CloseWrite() error }); ok {
|
||||
_ = cw.CloseWrite()
|
||||
}
|
||||
}()
|
||||
go func() {
|
||||
defer splice.Done()
|
||||
_, _ = io.Copy(client, upstream)
|
||||
defer wg.Done()
|
||||
copyTracking(client, upstream, &lastActivity, idleTimeout)
|
||||
if cw, ok := client.(interface{ CloseWrite() error }); ok {
|
||||
_ = cw.CloseWrite()
|
||||
}
|
||||
}()
|
||||
splice.Wait()
|
||||
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
wg.Wait()
|
||||
close(done)
|
||||
}()
|
||||
|
||||
ticker := time.NewTicker(idleTimeout / 4)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-done:
|
||||
return
|
||||
case <-closing:
|
||||
case <-ticker.C:
|
||||
if time.Since(time.Unix(0, lastActivity.Load())) < idleTimeout {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// Closing both ends unblocks whichever copy is still reading. The
|
||||
// caller's own deferred Close calls then become no-ops.
|
||||
_ = client.Close()
|
||||
_ = upstream.Close()
|
||||
<-done
|
||||
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// copyTracking copies src into dst, recording the time of every chunk that
|
||||
// moves so [spliceTunnel] can tell a busy tunnel from an idle one.
|
||||
func copyTracking(dst, src net.Conn, lastActivity *atomic.Int64, writeTimeout time.Duration) {
|
||||
buf := make([]byte, 32*1024)
|
||||
|
||||
for {
|
||||
n, readErr := src.Read(buf)
|
||||
if n > 0 {
|
||||
lastActivity.Store(time.Now().UnixNano())
|
||||
|
||||
// Bound the write. A destination that has gone away accepts the
|
||||
// first chunk into its send buffer and only fails on the next one,
|
||||
// so without a deadline this direction keeps a dead tunnel alive
|
||||
// for one more chunk. A destination that stops reading altogether
|
||||
// would block here forever.
|
||||
_ = dst.SetWriteDeadline(time.Now().Add(writeTimeout))
|
||||
|
||||
_, writeErr := dst.Write(buf[:n])
|
||||
if writeErr != nil {
|
||||
return
|
||||
}
|
||||
|
||||
lastActivity.Store(time.Now().UnixNano())
|
||||
}
|
||||
if readErr != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// handleForward handles plain HTTP requests sent to the proxy as absolute
|
||||
|
||||
@@ -18,7 +18,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/dlclark/regexp2"
|
||||
"github.com/dlclark/regexp2/v2"
|
||||
|
||||
"github.com/gotenberg/gotenberg/v8/pkg/gotenberg"
|
||||
)
|
||||
@@ -898,3 +898,280 @@ func TestPinningProxy_StopIdempotent(t *testing.T) {
|
||||
t.Fatalf("second Stop on stopped proxy: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// tcpPair returns the two ends of a connected loopback TCP connection. Both
|
||||
// ends are closed when the test finishes.
|
||||
func tcpPair(t *testing.T) (net.Conn, net.Conn) {
|
||||
t.Helper()
|
||||
|
||||
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatalf("listen: %v", err)
|
||||
}
|
||||
defer func() { _ = listener.Close() }()
|
||||
|
||||
type accepted struct {
|
||||
conn net.Conn
|
||||
err error
|
||||
}
|
||||
|
||||
acceptChan := make(chan accepted, 1)
|
||||
go func() {
|
||||
conn, acceptErr := listener.Accept()
|
||||
acceptChan <- accepted{conn: conn, err: acceptErr}
|
||||
}()
|
||||
|
||||
dialed, err := net.Dial("tcp", listener.Addr().String())
|
||||
if err != nil {
|
||||
t.Fatalf("dial: %v", err)
|
||||
}
|
||||
|
||||
res := <-acceptChan
|
||||
if res.err != nil {
|
||||
t.Fatalf("accept: %v", res.err)
|
||||
}
|
||||
|
||||
t.Cleanup(func() {
|
||||
_ = dialed.Close()
|
||||
_ = res.conn.Close()
|
||||
})
|
||||
|
||||
return dialed, res.conn
|
||||
}
|
||||
|
||||
// TestSpliceTunnel_IdleTunnelIsClosed covers the leak where an upstream that
|
||||
// accepted a CONNECT tunnel and then never spoke pinned both splice goroutines
|
||||
// and both sockets for the lifetime of the process. A hijacked connection
|
||||
// carries no deadline and net/http stops tracking it, so the idle bound in
|
||||
// spliceTunnel is the only thing that ends such a tunnel.
|
||||
func TestSpliceTunnel_IdleTunnelIsClosed(t *testing.T) {
|
||||
// The peers are kept open by the pair's cleanup: the tunnel is silent, not
|
||||
// finished.
|
||||
client, _ := tcpPair(t)
|
||||
upstream, _ := tcpPair(t)
|
||||
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
spliceTunnel(client, upstream, nil, 100*time.Millisecond)
|
||||
close(done)
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("spliceTunnel did not return on an idle tunnel")
|
||||
}
|
||||
}
|
||||
|
||||
// TestSpliceTunnel_ClosingShutsTunnelDown verifies that stopping the proxy
|
||||
// reaps in-flight tunnels. http.Server.Shutdown cannot: it stops tracking a
|
||||
// connection once a handler hijacks it, so without this signal a tunnel would
|
||||
// outlive the proxy and every Chromium restart after it.
|
||||
func TestSpliceTunnel_ClosingShutsTunnelDown(t *testing.T) {
|
||||
client, _ := tcpPair(t)
|
||||
upstream, _ := tcpPair(t)
|
||||
|
||||
closing := make(chan struct{})
|
||||
done := make(chan struct{})
|
||||
|
||||
go func() {
|
||||
// An idle timeout far beyond the test: only closing can end this.
|
||||
spliceTunnel(client, upstream, closing, time.Hour)
|
||||
close(done)
|
||||
}()
|
||||
|
||||
close(closing)
|
||||
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("spliceTunnel did not return when the proxy shut down")
|
||||
}
|
||||
}
|
||||
|
||||
// TestSpliceTunnel_ActiveTransferOutlivesIdleTimeout guards the idle bound
|
||||
// against cutting a healthy transfer. Idleness is tracked across both
|
||||
// directions, so a download that keeps making progress must survive well past
|
||||
// the timeout even though the client sends nothing throughout.
|
||||
func TestSpliceTunnel_ActiveTransferOutlivesIdleTimeout(t *testing.T) {
|
||||
const (
|
||||
idleTimeout = 100 * time.Millisecond
|
||||
chunks = 10
|
||||
interval = 30 * time.Millisecond
|
||||
)
|
||||
|
||||
client, clientPeer := tcpPair(t)
|
||||
upstream, upstreamPeer := tcpPair(t)
|
||||
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
spliceTunnel(client, upstream, nil, idleTimeout)
|
||||
close(done)
|
||||
}()
|
||||
|
||||
// Trickle a response for well over the idle timeout, then finish.
|
||||
go func() {
|
||||
for range chunks {
|
||||
_, _ = upstreamPeer.Write([]byte("x"))
|
||||
time.Sleep(interval)
|
||||
}
|
||||
_ = upstreamPeer.Close()
|
||||
}()
|
||||
|
||||
received := 0
|
||||
buf := make([]byte, chunks)
|
||||
|
||||
for received < chunks {
|
||||
err := clientPeer.SetReadDeadline(time.Now().Add(10 * time.Second))
|
||||
if err != nil {
|
||||
t.Fatalf("set read deadline: %v", err)
|
||||
}
|
||||
|
||||
n, readErr := clientPeer.Read(buf)
|
||||
received += n
|
||||
if readErr != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if received != chunks {
|
||||
t.Fatalf("received %d bytes, want %d: the tunnel was cut while still transferring", received, chunks)
|
||||
}
|
||||
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("spliceTunnel did not return after the upstream closed")
|
||||
}
|
||||
}
|
||||
|
||||
// TestPinningProxy_CONNECT_TunnelCeiling verifies the ceiling that bounds what
|
||||
// tunnels refusing to end can accumulate. The idle bound cannot cover a peer
|
||||
// that trickles just under it, so the count is what stops the growth.
|
||||
func TestPinningProxy_CONNECT_TunnelCeiling(t *testing.T) {
|
||||
// An upstream that accepts and then says nothing: the tunnel stays open.
|
||||
upstreamAddr, stop := newRawTCPServer(t, func(c net.Conn) {
|
||||
<-make(chan struct{})
|
||||
})
|
||||
t.Cleanup(stop)
|
||||
|
||||
p := newPinningProxy(nil, nil, false, false, false)
|
||||
p.maxTunnels = 1
|
||||
p.decide = func(_ context.Context, _ string, _, _ []*regexp2.Regexp, _ time.Time) (gotenberg.OutboundDecision, error) {
|
||||
return gotenberg.OutboundDecision{Pinned: []netip.Addr{netip.MustParseAddr("127.0.0.1")}}, nil
|
||||
}
|
||||
p.dialPinned = func(_ context.Context, network string, _ []netip.Addr, _ string) (net.Conn, error) {
|
||||
return net.Dial(network, upstreamAddr)
|
||||
}
|
||||
proxyURL := newProxyForTest(t, p)
|
||||
proxyAddr := strings.TrimPrefix(proxyURL, "http://")
|
||||
|
||||
connect := func(t *testing.T) *bufio.Reader {
|
||||
t.Helper()
|
||||
|
||||
conn, err := net.Dial("tcp", proxyAddr)
|
||||
if err != nil {
|
||||
t.Fatalf("dial proxy: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = conn.Close() })
|
||||
|
||||
err = conn.SetDeadline(time.Now().Add(10 * time.Second))
|
||||
if err != nil {
|
||||
t.Fatalf("set deadline: %v", err)
|
||||
}
|
||||
|
||||
_, err = fmt.Fprintf(conn, "CONNECT example.com:443 HTTP/1.1\r\nHost: example.com:443\r\n\r\n")
|
||||
if err != nil {
|
||||
t.Fatalf("write CONNECT: %v", err)
|
||||
}
|
||||
|
||||
return bufio.NewReader(conn)
|
||||
}
|
||||
|
||||
first := connect(t)
|
||||
statusLine, err := first.ReadString('\n')
|
||||
if err != nil {
|
||||
t.Fatalf("read first status: %v", err)
|
||||
}
|
||||
if !strings.Contains(statusLine, " 200 ") {
|
||||
t.Fatalf("first CONNECT status = %q, want 200", statusLine)
|
||||
}
|
||||
|
||||
// The first tunnel now holds the only slot.
|
||||
second := connect(t)
|
||||
resp, err := http.ReadResponse(second, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("read second response: %v", err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
if resp.StatusCode != http.StatusServiceUnavailable {
|
||||
t.Fatalf("second CONNECT status = %d, want %d", resp.StatusCode, http.StatusServiceUnavailable)
|
||||
}
|
||||
|
||||
if got := p.tunnels.Load(); got != 1 {
|
||||
t.Errorf("tunnels in flight = %d, want 1: a refused CONNECT must not consume a slot", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestPinningProxy_TunnelSlotIsReleased verifies a completed tunnel gives its
|
||||
// slot back, so the ceiling bounds concurrency rather than lifetime totals.
|
||||
func TestPinningProxy_TunnelSlotIsReleased(t *testing.T) {
|
||||
upstreamAddr, stop := newRawTCPServer(t, func(c net.Conn) {
|
||||
defer c.Close()
|
||||
_, _ = c.Write([]byte("HI"))
|
||||
})
|
||||
t.Cleanup(stop)
|
||||
|
||||
p := newPinningProxy(nil, nil, false, false, false)
|
||||
p.maxTunnels = 1
|
||||
p.decide = func(_ context.Context, _ string, _, _ []*regexp2.Regexp, _ time.Time) (gotenberg.OutboundDecision, error) {
|
||||
return gotenberg.OutboundDecision{Pinned: []netip.Addr{netip.MustParseAddr("127.0.0.1")}}, nil
|
||||
}
|
||||
p.dialPinned = func(_ context.Context, network string, _ []netip.Addr, _ string) (net.Conn, error) {
|
||||
return net.Dial(network, upstreamAddr)
|
||||
}
|
||||
proxyURL := newProxyForTest(t, p)
|
||||
proxyAddr := strings.TrimPrefix(proxyURL, "http://")
|
||||
|
||||
for attempt := range 3 {
|
||||
conn, err := net.Dial("tcp", proxyAddr)
|
||||
if err != nil {
|
||||
t.Fatalf("attempt %d dial proxy: %v", attempt, err)
|
||||
}
|
||||
|
||||
err = conn.SetDeadline(time.Now().Add(10 * time.Second))
|
||||
if err != nil {
|
||||
t.Fatalf("attempt %d set deadline: %v", attempt, err)
|
||||
}
|
||||
|
||||
_, err = fmt.Fprintf(conn, "CONNECT example.com:443 HTTP/1.1\r\nHost: example.com:443\r\n\r\n")
|
||||
if err != nil {
|
||||
t.Fatalf("attempt %d write CONNECT: %v", attempt, err)
|
||||
}
|
||||
|
||||
br := bufio.NewReader(conn)
|
||||
statusLine, err := br.ReadString('\n')
|
||||
if err != nil {
|
||||
t.Fatalf("attempt %d read status: %v", attempt, err)
|
||||
}
|
||||
if !strings.Contains(statusLine, " 200 ") {
|
||||
t.Fatalf("attempt %d CONNECT status = %q, want 200: the slot was not released", attempt, statusLine)
|
||||
}
|
||||
|
||||
// Drain until the upstream's close ends the tunnel, then release it.
|
||||
_, _ = io.ReadAll(br)
|
||||
_ = conn.Close()
|
||||
|
||||
// The handler returns just after the splice ends.
|
||||
for range 100 {
|
||||
if p.tunnels.Load() == 0 {
|
||||
break
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
if got := p.tunnels.Load(); got != 0 {
|
||||
t.Fatalf("attempt %d: tunnels in flight = %d, want 0", attempt, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,7 +14,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/dlclark/regexp2"
|
||||
"github.com/dlclark/regexp2/v2"
|
||||
|
||||
"github.com/gotenberg/gotenberg/v8/pkg/gotenberg"
|
||||
)
|
||||
|
||||
@@ -14,9 +14,9 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/dlclark/regexp2"
|
||||
"github.com/dlclark/regexp2/v2"
|
||||
"github.com/gomarkdown/markdown"
|
||||
"github.com/labstack/echo/v4"
|
||||
"github.com/labstack/echo/v5"
|
||||
"github.com/microcosm-cc/bluemonday"
|
||||
|
||||
"github.com/gotenberg/gotenberg/v8/pkg/gotenberg"
|
||||
@@ -466,7 +466,7 @@ func convertUrlRoute(chromium Api, engine gotenberg.PdfEngine) api.Route {
|
||||
Method: http.MethodPost,
|
||||
Path: "/forms/chromium/convert/url",
|
||||
IsMultipart: true,
|
||||
Handler: func(c echo.Context) error {
|
||||
Handler: func(c *echo.Context) error {
|
||||
ctx := c.Get("context").(*api.Context)
|
||||
form, options := FormDataChromiumPdfOptions(ctx)
|
||||
mode := pdfengines.FormDataPdfSplitMode(form, false)
|
||||
@@ -528,7 +528,7 @@ func screenshotUrlRoute(chromium Api) api.Route {
|
||||
Method: http.MethodPost,
|
||||
Path: "/forms/chromium/screenshot/url",
|
||||
IsMultipart: true,
|
||||
Handler: func(c echo.Context) error {
|
||||
Handler: func(c *echo.Context) error {
|
||||
ctx := c.Get("context").(*api.Context)
|
||||
form, options := FormDataChromiumScreenshotOptions(ctx)
|
||||
|
||||
@@ -562,7 +562,7 @@ func convertHtmlRoute(chromium Api, engine gotenberg.PdfEngine) api.Route {
|
||||
Method: http.MethodPost,
|
||||
Path: "/forms/chromium/convert/html",
|
||||
IsMultipart: true,
|
||||
Handler: func(c echo.Context) error {
|
||||
Handler: func(c *echo.Context) error {
|
||||
ctx := c.Get("context").(*api.Context)
|
||||
form, options := FormDataChromiumPdfOptions(ctx)
|
||||
mode := pdfengines.FormDataPdfSplitMode(form, false)
|
||||
@@ -621,7 +621,7 @@ func screenshotHtmlRoute(chromium Api) api.Route {
|
||||
Method: http.MethodPost,
|
||||
Path: "/forms/chromium/screenshot/html",
|
||||
IsMultipart: true,
|
||||
Handler: func(c echo.Context) error {
|
||||
Handler: func(c *echo.Context) error {
|
||||
ctx := c.Get("context").(*api.Context)
|
||||
form, options := FormDataChromiumScreenshotOptions(ctx)
|
||||
|
||||
@@ -652,7 +652,7 @@ func convertMarkdownRoute(chromium Api, engine gotenberg.PdfEngine) api.Route {
|
||||
Method: http.MethodPost,
|
||||
Path: "/forms/chromium/convert/markdown",
|
||||
IsMultipart: true,
|
||||
Handler: func(c echo.Context) error {
|
||||
Handler: func(c *echo.Context) error {
|
||||
ctx := c.Get("context").(*api.Context)
|
||||
form, options := FormDataChromiumPdfOptions(ctx)
|
||||
mode := pdfengines.FormDataPdfSplitMode(form, false)
|
||||
@@ -720,7 +720,7 @@ func screenshotMarkdownRoute(chromium Api) api.Route {
|
||||
Method: http.MethodPost,
|
||||
Path: "/forms/chromium/screenshot/markdown",
|
||||
IsMultipart: true,
|
||||
Handler: func(c echo.Context) error {
|
||||
Handler: func(c *echo.Context) error {
|
||||
ctx := c.Get("context").(*api.Context)
|
||||
form, options := FormDataChromiumScreenshotOptions(ctx)
|
||||
|
||||
@@ -1020,6 +1020,16 @@ func handleChromiumError(err error, options Options) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
if errors.Is(err, ErrChromiumCrashed) {
|
||||
return api.WrapError(
|
||||
err,
|
||||
api.NewSentinelHttpError(
|
||||
http.StatusServiceUnavailable,
|
||||
"Chromium crashed while processing the request. Retry, or reduce the workload if the problem persists.",
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
if errors.Is(err, ErrInvalidEvaluationExpression) {
|
||||
if options.WaitForExpression == "" {
|
||||
// We do not expect the 'waitWindowStatus' form field to return
|
||||
|
||||
51
pkg/modules/chromium/routes_test.go
Normal file
51
pkg/modules/chromium/routes_test.go
Normal file
@@ -0,0 +1,51 @@
|
||||
package chromium
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"github.com/gotenberg/gotenberg/v8/pkg/modules/api"
|
||||
)
|
||||
|
||||
// TestHandleChromiumError_Crashed pins the mapping of a Chromium renderer
|
||||
// crash to a 503 Service Unavailable. When the renderer crashes mid-conversion,
|
||||
// the request must fail fast with 503 rather than hang until the deadline and
|
||||
// surface as a generic timeout.
|
||||
// See https://github.com/gotenberg/gotenberg/issues/1640.
|
||||
func TestHandleChromiumError_Crashed(t *testing.T) {
|
||||
// Mirror the wrapping done by [chromiumBrowser.do].
|
||||
err := handleChromiumError(fmt.Errorf("handle tasks: %w", ErrChromiumCrashed), Options{})
|
||||
if err == nil {
|
||||
t.Fatal("expected an error, got none")
|
||||
}
|
||||
|
||||
status, message := api.ParseError(err)
|
||||
if status != http.StatusServiceUnavailable {
|
||||
t.Errorf("status = %d, want %d (message: %s)", status, http.StatusServiceUnavailable, message)
|
||||
}
|
||||
|
||||
want := "Chromium crashed while processing the request. Retry, or reduce the workload if the problem persists."
|
||||
if message != want {
|
||||
t.Errorf("message = %q, want %q", message, want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestHandleChromiumError_CrashedTakesPrecedence guards the ordering in
|
||||
// [handleChromiumError]: a crash is a server-side failure and must map to 503
|
||||
// even when the error chain also carries a marker that another branch would
|
||||
// map to a client-error status.
|
||||
func TestHandleChromiumError_CrashedTakesPrecedence(t *testing.T) {
|
||||
err := handleChromiumError(
|
||||
fmt.Errorf("handle tasks: %w; %w", ErrChromiumCrashed, ErrInvalidHttpStatusCode),
|
||||
Options{},
|
||||
)
|
||||
if err == nil {
|
||||
t.Fatal("expected an error, got none")
|
||||
}
|
||||
|
||||
status, _ := api.ParseError(err)
|
||||
if status != http.StatusServiceUnavailable {
|
||||
t.Errorf("status = %d, want %d", status, http.StatusServiceUnavailable)
|
||||
}
|
||||
}
|
||||
@@ -6,7 +6,9 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/dlclark/regexp2"
|
||||
"github.com/dlclark/regexp2/v2"
|
||||
|
||||
"github.com/gotenberg/gotenberg/v8/pkg/gotenberg"
|
||||
)
|
||||
|
||||
func TestScopeMatchBudget(t *testing.T) {
|
||||
@@ -90,7 +92,7 @@ func TestScopeMatchBudget_BoundsCatastrophicBacktracking(t *testing.T) {
|
||||
break
|
||||
}
|
||||
matchStart := time.Now()
|
||||
_, _ = pattern.MatchString(url)
|
||||
_, _ = gotenberg.MatchPattern(pattern, url)
|
||||
b.consume(time.Since(matchStart))
|
||||
matched++
|
||||
}
|
||||
@@ -100,10 +102,13 @@ func TestScopeMatchBudget_BoundsCatastrophicBacktracking(t *testing.T) {
|
||||
t.Errorf("all %d headers were matched, want the budget to stop matching early", headers)
|
||||
}
|
||||
|
||||
// Each match is separately capped at extraHttpHeaderScopeMatchTimeout, so
|
||||
// the worst case is the budget plus one final match that started with the
|
||||
// last of the credit. Generous slack keeps this stable on a loaded CI box.
|
||||
ceiling := budget + extraHttpHeaderScopeMatchTimeout + time.Second
|
||||
// A match that is genuinely out of budget costs a few
|
||||
// extraHttpHeaderScopeMatchTimeout rather than one:
|
||||
// [gotenberg.MatchPattern] retries an abort to tell a real runaway from
|
||||
// one caused by the process losing the CPU. The worst case is the budget
|
||||
// plus one final match that started with the last of the credit. Generous
|
||||
// slack keeps this stable on a loaded CI box.
|
||||
ceiling := budget + 4*extraHttpHeaderScopeMatchTimeout + time.Second
|
||||
if elapsed > ceiling {
|
||||
t.Errorf("matching took %s, want at most %s", elapsed, ceiling)
|
||||
}
|
||||
|
||||
@@ -80,6 +80,37 @@ var dangerousTags = []string{
|
||||
"FilePermissions", // Writing this changes the file's permissions
|
||||
}
|
||||
|
||||
// controlOptions lists ExifTool command-line option names that collide with a
|
||||
// tag assignment. A metadata key of "csv" becomes the argv entry "-csv=value",
|
||||
// which exiftool reads as its own option rather than as a tag, so the value
|
||||
// becomes a filename exiftool opens. Only an unprefixed key can collide:
|
||||
// "-XMP:csv=value" is unambiguously a tag.
|
||||
//
|
||||
// See https://exiftool.org/exiftool_pod.html.
|
||||
var controlOptions = []string{
|
||||
"api", "argfile", "charset", "common_args", "config", "csv", "diff",
|
||||
"echo", "efile", "execute", "ext", "fileorder", "geotag", "geosync",
|
||||
"htmldump", "if", "json", "lang", "listitem", "o", "out", "p", "php",
|
||||
"require", "srcfile", "stay_open", "tagsfromfile", "textout", "use", "w",
|
||||
"wm", "xmlformat",
|
||||
}
|
||||
|
||||
// isControlOption reports whether an unprefixed key would reach exiftool as
|
||||
// one of its own options instead of as a tag assignment.
|
||||
func isControlOption(key string) bool {
|
||||
if strings.Contains(key, ":") {
|
||||
return false
|
||||
}
|
||||
|
||||
for _, option := range controlOptions {
|
||||
if strings.EqualFold(key, option) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// isDangerousTag reports whether key matches one of the [dangerousTags]
|
||||
// after case-insensitive comparison with any group prefix stripped.
|
||||
func isDangerousTag(key string) bool {
|
||||
@@ -114,6 +145,9 @@ func buildExifToolWriteArgs(metadata map[string]any) ([]string, error) {
|
||||
if !safeKeyPattern.MatchString(key) {
|
||||
return nil, fmt.Errorf("write PDF metadata with ExifTool: invalid metadata key %q: %w", key, gotenberg.ErrPdfEngineMetadataValueNotSupported)
|
||||
}
|
||||
if isControlOption(key) {
|
||||
return nil, fmt.Errorf("write PDF metadata with ExifTool: metadata key %q is an ExifTool option, prefix it with a group such as %q: %w", key, "XMP:"+key, gotenberg.ErrPdfEngineMetadataValueNotSupported)
|
||||
}
|
||||
|
||||
tag := key
|
||||
if key == "Trapped" {
|
||||
|
||||
@@ -2,6 +2,7 @@ package exiftool
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"testing"
|
||||
|
||||
@@ -211,3 +212,48 @@ func TestSafeKeyPattern(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A metadata key that collides with an ExifTool option becomes a bare argv
|
||||
// entry such as "-csv=/etc/passwd", which exiftool reads as its own option and
|
||||
// treats the value as a filename to open.
|
||||
func TestBuildExifToolWriteArgs_RejectsControlOptions(t *testing.T) {
|
||||
for _, key := range []string{"csv", "CSV", "json", "geotag", "config", "tagsFromFile", "execute", "stay_open", "o", "w", "if", "p"} {
|
||||
t.Run(key, func(t *testing.T) {
|
||||
_, err := buildExifToolWriteArgs(map[string]any{key: "/etc/passwd"})
|
||||
if err == nil {
|
||||
t.Fatalf("buildExifToolWriteArgs accepted the control option %q", key)
|
||||
}
|
||||
if !errors.Is(err, gotenberg.ErrPdfEngineMetadataValueNotSupported) {
|
||||
t.Fatalf("error %v does not wrap ErrPdfEngineMetadataValueNotSupported", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A group prefix makes the key unambiguous, so it must still be accepted.
|
||||
func TestBuildExifToolWriteArgs_AcceptsPrefixedOptionNames(t *testing.T) {
|
||||
for _, key := range []string{"XMP:csv", "XMP-dc:json", "IPTC:p"} {
|
||||
t.Run(key, func(t *testing.T) {
|
||||
args, err := buildExifToolWriteArgs(map[string]any{key: "value"})
|
||||
if err != nil {
|
||||
t.Fatalf("buildExifToolWriteArgs rejected the prefixed key %q: %v", key, err)
|
||||
}
|
||||
want := fmt.Sprintf("-%s=value", key)
|
||||
if len(args) != 1 || args[0] != want {
|
||||
t.Fatalf("args = %v, want [%s]", args, want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Ordinary tags must be unaffected.
|
||||
func TestBuildExifToolWriteArgs_AcceptsOrdinaryTags(t *testing.T) {
|
||||
for _, key := range []string{"Author", "Title", "Subject", "Keywords", "Producer", "Creator"} {
|
||||
t.Run(key, func(t *testing.T) {
|
||||
_, err := buildExifToolWriteArgs(map[string]any{key: "value"})
|
||||
if err != nil {
|
||||
t.Fatalf("buildExifToolWriteArgs rejected the ordinary tag %q: %v", key, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -353,7 +353,7 @@ func (a *Api) Descriptor() gotenberg.ModuleDescriptor {
|
||||
fs.Duration("libreoffice-idle-shutdown-timeout", 0, "Shutdown LibreOffice after being idle for the given duration. Set to 0 to disable this feature")
|
||||
fs.Bool("libreoffice-auto-start", false, "Automatically launch LibreOffice upon initialization if set to true; otherwise, LibreOffice will start at the time of the first conversion")
|
||||
fs.Duration("libreoffice-start-timeout", time.Duration(20)*time.Second, "Maximum duration to wait for LibreOffice to start or restart")
|
||||
fs.StringSlice("libreoffice-allow-list", []string{}, "Set the allowed URLs for LibreOffice outbound fetches (embedded images, linked content) using regular expressions - supports multiple values")
|
||||
fs.StringSlice("libreoffice-allow-list", []string{}, `Set the allowed URLs for LibreOffice outbound fetches (embedded images, linked content) using regular expressions - supports multiple values. A match bypasses --libreoffice-deny-private-ips (LIBREOFFICE_DENY_PRIVATE_IPS) and --libreoffice-deny-public-ips (LIBREOFFICE_DENY_PUBLIC_IPS), so terminate the host or the pattern also matches suffix hosts, for example ^https?://internal\.svc(:|/|$)`)
|
||||
fs.StringSlice("libreoffice-deny-list", []string{}, "Set the denied URLs for LibreOffice outbound fetches using regular expressions - supports multiple values")
|
||||
fs.Bool("libreoffice-deny-private-ips", false, "Reject LibreOffice outbound URLs whose host resolves to a non-public IP address (loopback, RFC1918, link-local, unique-local). Enable on deployments that accept untrusted documents to mitigate SSRF against internal services")
|
||||
fs.Bool("libreoffice-deny-public-ips", false, "Reject LibreOffice outbound URLs whose host resolves to a public IP address. Enable on air-gapped or data-governed deployments to prevent outbound traffic from leaving a private network")
|
||||
|
||||
@@ -68,6 +68,59 @@ func (p *libreOfficeProcess) Start(logger *slog.Logger) error {
|
||||
|
||||
userProfileDirPath := p.fs.NewDirPath()
|
||||
|
||||
var (
|
||||
cmd *gotenberg.Cmd
|
||||
success bool
|
||||
)
|
||||
|
||||
// Registered here, right after the proxy starts listening, so that every
|
||||
// failure below tears it down. A return between the proxy start and this
|
||||
// point strands it: its listener stays bound, its Serve goroutine and HTTP
|
||||
// client stay alive, and p.proxy is only assigned on success, so nothing
|
||||
// could ever reach it to stop it. exec.Cmd.Start fails precisely under fd
|
||||
// or memory pressure, and the supervisor retries the launch on the next
|
||||
// request, so each stranded proxy compounds the condition that caused it.
|
||||
defer func() {
|
||||
if success {
|
||||
p.cfgMu.Lock()
|
||||
defer p.cfgMu.Unlock()
|
||||
|
||||
p.socketPort = port
|
||||
p.userProfileDirPath = userProfileDirPath
|
||||
p.cmd = cmd
|
||||
p.proxy = proxy
|
||||
p.isStarted.Store(true)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// LibreOffice failed to start; tear the proxy down too.
|
||||
stopErr := proxy.Stop(context.Background())
|
||||
if stopErr != nil {
|
||||
logger.WarnContext(context.Background(), fmt.Sprintf("stop LibreOffice outbound proxy after failed start: %s", stopErr))
|
||||
}
|
||||
|
||||
// Let's make sure the process is killed. It is nil when the failure
|
||||
// happened before the command was built.
|
||||
if cmd != nil {
|
||||
killErr := cmd.Kill()
|
||||
if killErr != nil {
|
||||
logger.DebugContext(context.Background(), fmt.Sprintf("kill LibreOffice process: %v", killErr))
|
||||
}
|
||||
}
|
||||
|
||||
// And the user profile directory is deleted. It may never have been
|
||||
// created, which RemoveAll reports as success.
|
||||
removeErr := os.RemoveAll(userProfileDirPath)
|
||||
if removeErr != nil {
|
||||
logger.ErrorContext(context.Background(), fmt.Sprintf("remove LibreOffice's user profile directory: %v", removeErr))
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
logger.DebugContext(context.Background(), fmt.Sprintf("'%s' LibreOffice's user profile directory removed", userProfileDirPath))
|
||||
}()
|
||||
|
||||
// LibreOffice fetches external content (OOXML images via
|
||||
// TargetMode=External, RTF INCLUDEPICTURE, ODT linked images) inside
|
||||
// its own libcurl. The profile config routes those fetches through the
|
||||
@@ -75,7 +128,6 @@ func (p *libreOfficeProcess) Start(logger *slog.Logger) error {
|
||||
// blocks content linked from untrusted locations so absolute-path
|
||||
// (file://) and direct fetches are dropped at the source.
|
||||
if err := writeSofficeProfileConfig(userProfileDirPath, proxy.Addr()); err != nil {
|
||||
_ = proxy.Stop(context.Background())
|
||||
return fmt.Errorf("write soffice profile config: %w", err)
|
||||
}
|
||||
sofficeEnv := sofficeProxyEnv(os.Environ(), proxy.Addr())
|
||||
@@ -95,9 +147,8 @@ func (p *libreOfficeProcess) Start(logger *slog.Logger) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), p.arguments.startTimeout)
|
||||
defer cancel()
|
||||
|
||||
cmd, err := gotenberg.CommandContext(ctx, logger, p.arguments.binPath, args...)
|
||||
cmd, err = gotenberg.CommandContext(ctx, logger, p.arguments.binPath, args...)
|
||||
if err != nil {
|
||||
_ = proxy.Stop(context.Background())
|
||||
return fmt.Errorf("create LibreOffice command: %w", err)
|
||||
}
|
||||
cmd.SetEnv(sofficeEnv)
|
||||
@@ -106,7 +157,6 @@ func (p *libreOfficeProcess) Start(logger *slog.Logger) error {
|
||||
// able to run as a daemon.
|
||||
exitCode, err := cmd.Exec()
|
||||
if err != nil && exitCode != 81 {
|
||||
_ = proxy.Stop(context.Background())
|
||||
return fmt.Errorf("execute LibreOffice: %w", err)
|
||||
}
|
||||
|
||||
@@ -155,43 +205,6 @@ func (p *libreOfficeProcess) Start(logger *slog.Logger) error {
|
||||
}
|
||||
}()
|
||||
|
||||
var success bool
|
||||
|
||||
defer func() {
|
||||
if success {
|
||||
p.cfgMu.Lock()
|
||||
defer p.cfgMu.Unlock()
|
||||
|
||||
p.socketPort = port
|
||||
p.userProfileDirPath = userProfileDirPath
|
||||
p.cmd = cmd
|
||||
p.proxy = proxy
|
||||
p.isStarted.Store(true)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// LibreOffice failed to start; tear the proxy down too.
|
||||
stopErr := proxy.Stop(context.Background())
|
||||
if stopErr != nil {
|
||||
logger.WarnContext(context.Background(), fmt.Sprintf("stop LibreOffice outbound proxy after failed start: %s", stopErr))
|
||||
}
|
||||
|
||||
// Let's make sure the process is killed.
|
||||
err = cmd.Kill()
|
||||
if err != nil {
|
||||
logger.DebugContext(context.Background(), fmt.Sprintf("kill LibreOffice process: %v", err))
|
||||
}
|
||||
|
||||
// And the user profile directory is deleted.
|
||||
err = os.RemoveAll(userProfileDirPath)
|
||||
if err != nil {
|
||||
logger.ErrorContext(context.Background(), fmt.Sprintf("remove LibreOffice's user profile directory: %v", err))
|
||||
}
|
||||
|
||||
logger.DebugContext(context.Background(), fmt.Sprintf("'%s' LibreOffice's user profile directory removed", userProfileDirPath))
|
||||
}()
|
||||
|
||||
logger.DebugContext(context.Background(), "waiting for the LibreOffice socket to be available...")
|
||||
|
||||
for {
|
||||
|
||||
@@ -35,12 +35,13 @@ var (
|
||||
zipMagic = []byte{0x50, 0x4b, 0x03, 0x04}
|
||||
|
||||
// An unencrypted OOXML document is always a ZIP package, so any of these
|
||||
// extensions over a compound file means the payload is encrypted. Legacy
|
||||
// binary formats (.doc, .xls, .ppt) are compound files either way and are
|
||||
// deliberately absent.
|
||||
// extensions over a compound file means the payload is encrypted. A .xlsb
|
||||
// workbook stores binary parts inside that same ZIP package, so it belongs
|
||||
// here too. Legacy binary formats (.doc, .xls, .ppt) are compound files
|
||||
// either way and are deliberately absent.
|
||||
ooxmlExtensions = map[string]struct{}{
|
||||
".docx": {}, ".docm": {}, ".dotx": {}, ".dotm": {},
|
||||
".xlsx": {}, ".xlsm": {}, ".xltx": {}, ".xltm": {},
|
||||
".xlsx": {}, ".xlsm": {}, ".xltx": {}, ".xltm": {}, ".xlsb": {},
|
||||
".pptx": {}, ".pptm": {}, ".potx": {}, ".potm": {},
|
||||
".ppsx": {}, ".ppsm": {},
|
||||
}
|
||||
|
||||
@@ -76,6 +76,11 @@ func TestDetectPasswordProtection(t *testing.T) {
|
||||
path: ole2("encrypted.xlsx"),
|
||||
want: PasswordProtectionRequired,
|
||||
},
|
||||
{
|
||||
name: "encrypted binary workbook",
|
||||
path: ole2("encrypted.xlsb"),
|
||||
want: PasswordProtectionRequired,
|
||||
},
|
||||
{
|
||||
name: "legacy binary document is inconclusive",
|
||||
path: ole2("legacy.doc"),
|
||||
|
||||
@@ -14,7 +14,7 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/dlclark/regexp2"
|
||||
"github.com/dlclark/regexp2/v2"
|
||||
"golang.org/x/net/http/httpproxy"
|
||||
|
||||
"github.com/gotenberg/gotenberg/v8/pkg/gotenberg"
|
||||
|
||||
@@ -17,14 +17,14 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/dlclark/regexp2"
|
||||
"github.com/dlclark/regexp2/v2"
|
||||
)
|
||||
|
||||
func compileRegexes(t *testing.T, patterns ...string) []*regexp2.Regexp {
|
||||
t.Helper()
|
||||
out := make([]*regexp2.Regexp, 0, len(patterns))
|
||||
for _, p := range patterns {
|
||||
r, err := regexp2.Compile(p, 0)
|
||||
r, err := regexp2.Compile(p, regexp2.None)
|
||||
if err != nil {
|
||||
t.Fatalf("compile %q: %v", p, err)
|
||||
}
|
||||
@@ -361,7 +361,7 @@ func TestSofficeProxyEnv_OverridesExisting(t *testing.T) {
|
||||
// Old proxy values must be gone, not duplicated. Count exact-case keys.
|
||||
counts := map[string]int{}
|
||||
for _, kv := range out {
|
||||
key := strings.SplitN(kv, "=", 2)[0]
|
||||
key, _, _ := strings.Cut(kv, "=")
|
||||
counts[key]++
|
||||
}
|
||||
for _, key := range []string{"http_proxy", "HTTP_PROXY", "https_proxy", "HTTPS_PROXY", "no_proxy", "NO_PROXY"} {
|
||||
|
||||
@@ -7,7 +7,7 @@ import (
|
||||
"slices"
|
||||
"strconv"
|
||||
|
||||
"github.com/labstack/echo/v4"
|
||||
"github.com/labstack/echo/v5"
|
||||
|
||||
"github.com/gotenberg/gotenberg/v8/pkg/gotenberg"
|
||||
"github.com/gotenberg/gotenberg/v8/pkg/modules/api"
|
||||
@@ -27,7 +27,7 @@ func convertRoute(libreOffice libreofficeapi.Uno, engine gotenberg.PdfEngine) ap
|
||||
Method: http.MethodPost,
|
||||
Path: "/forms/libreoffice/convert",
|
||||
IsMultipart: true,
|
||||
Handler: func(c echo.Context) error {
|
||||
Handler: func(c *echo.Context) error {
|
||||
ctx := c.Get("context").(*api.Context)
|
||||
defaultOptions := libreofficeapi.DefaultOptions()
|
||||
|
||||
|
||||
@@ -12,7 +12,7 @@ import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/labstack/echo/v4"
|
||||
"github.com/labstack/echo/v5"
|
||||
|
||||
"github.com/gotenberg/gotenberg/v8/pkg/gotenberg"
|
||||
"github.com/gotenberg/gotenberg/v8/pkg/modules/api"
|
||||
|
||||
@@ -180,7 +180,7 @@ func (engine *PdfCpu) listImages(ctx context.Context, inputPath string) ([]pdfcp
|
||||
// their second column is not a numeric object number.
|
||||
func parseImagesList(output string) []pdfcpuImage {
|
||||
var images []pdfcpuImage
|
||||
for _, line := range strings.Split(output, "\n") {
|
||||
for line := range strings.SplitSeq(output, "\n") {
|
||||
cols := strings.Split(line, "│")
|
||||
if len(cols) < 9 {
|
||||
continue
|
||||
|
||||
172
pkg/modules/pdfengines/concurrency.go
Normal file
172
pkg/modules/pdfengines/concurrency.go
Normal file
@@ -0,0 +1,172 @@
|
||||
package pdfengines
|
||||
|
||||
import (
|
||||
"sync"
|
||||
|
||||
"github.com/gotenberg/gotenberg/v8/pkg/modules/api"
|
||||
)
|
||||
|
||||
// defaultMaxConcurrency is the number of PDF files a single stub processes at
|
||||
// once when --pdfengines-max-concurrency (env PDFENGINES_MAX_CONCURRENCY) is
|
||||
// not set. Each unit of work forks an external binary (qpdf, pdfcpu, pdftk or
|
||||
// exiftool), so the ceiling trades wall clock against process count and RSS.
|
||||
//
|
||||
// It defaults to one, which processes files exactly as the sequential loops
|
||||
// this package used to run did. Raising it only ever affects a request that
|
||||
// carries several files, or one that splits into several outputs: a
|
||||
// single-file request never reaches the concurrent path at all. Operators who
|
||||
// send multi-file batches and have the memory headroom opt in.
|
||||
//
|
||||
// This never covers LibreOffice. libreoffice-pdfengine implements Convert and
|
||||
// nothing else, every other [gotenberg.PdfEngine] method on it returns
|
||||
// [gotenberg.ErrPdfEngineMethodNotSupported], and [ConvertStub] deliberately
|
||||
// does not use this package's helpers. A soffice instance costs too much
|
||||
// memory to run several of per container, so LibreOffice throughput is scaled
|
||||
// by adding Gotenberg containers, not by raising this number.
|
||||
const defaultMaxConcurrency = 1
|
||||
|
||||
// maxFileConcurrency is how many files one request may have in flight at once.
|
||||
// It is replaced during [PdfEngines.Provision].
|
||||
var maxFileConcurrency = defaultMaxConcurrency
|
||||
|
||||
// engineExtraSlots bounds the concurrency this package ADDS, across the whole
|
||||
// process rather than per request, and holds one fewer slot than
|
||||
// [maxFileConcurrency] because every request already owns one unit of its own.
|
||||
//
|
||||
// Bounding the added concurrency rather than the total is what keeps the
|
||||
// ceiling from becoming a throughput regression. The sequential loops this
|
||||
// helper replaced had no ceiling at all: X concurrent requests ran X engine
|
||||
// binaries, one apiece. A pool covering the total would cut those X requests
|
||||
// down to the ceiling, so an operator raising the flag to speed up a single
|
||||
// multi-file request would slow the server down under real load. Reserving
|
||||
// each request the unit it always had makes the worst case "what happened
|
||||
// before, plus at most maxFileConcurrency-1".
|
||||
//
|
||||
// A per-request limit would have the opposite failure: X simultaneous requests
|
||||
// forking X times the limit, trading the timeouts this exists to prevent for
|
||||
// memory exhaustion.
|
||||
var engineExtraSlots = make(chan struct{}, defaultMaxConcurrency-1)
|
||||
|
||||
// acquireEngineSlot waits for the first unit of capacity to become available,
|
||||
// either this request's reserved unit or a slot from the shared pool, and
|
||||
// returns the function that gives it back.
|
||||
//
|
||||
// Waiting on both at once is the whole point. Committing to one source and
|
||||
// blocking on it strands the other: a goroutine parked on an exhausted pool
|
||||
// cannot pick up its own request's reserved unit when the file before it
|
||||
// finishes, so the reserved units sit idle while every file queues on the
|
||||
// pool, which is slower than having no pool at all.
|
||||
func acquireEngineSlot(ctx *api.Context, reserved chan struct{}) (func(), error) {
|
||||
// An [api.Context] carries a request context in production, but one built
|
||||
// as a literal, which the unit tests do, embeds a nil [context.Context]
|
||||
// and would panic on Done. A nil channel never fires, which correctly
|
||||
// leaves the two capacity sources as the only things to wait on.
|
||||
var done <-chan struct{}
|
||||
if ctx != nil && ctx.Context != nil {
|
||||
done = ctx.Done()
|
||||
}
|
||||
|
||||
// A select whose cancellation and capacity cases are both ready picks
|
||||
// between them at random, so an already-dead request would start more
|
||||
// files on a coin flip. Check first and stop taking on work.
|
||||
if done != nil {
|
||||
select {
|
||||
case <-done:
|
||||
return nil, ctx.Err()
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
select {
|
||||
case <-reserved:
|
||||
return func() { reserved <- struct{}{} }, nil
|
||||
case engineExtraSlots <- struct{}{}:
|
||||
return func() { <-engineExtraSlots }, nil
|
||||
case <-done:
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
// forEachInputPath runs fn against every input path, up to
|
||||
// --pdfengines-max-concurrency (env PDFENGINES_MAX_CONCURRENCY) at a time.
|
||||
//
|
||||
// The stubs mutate each PDF in place, so distinct input paths never touch the
|
||||
// same file and may run together. Callers that layer operations on one file,
|
||||
// like [WatermarkStub] applying several watermarks in order, must keep that
|
||||
// outer sequence and parallelize only the file dimension.
|
||||
//
|
||||
// Every path is attempted even after one fails, and the error returned is the
|
||||
// first in input order rather than the first to arrive. That keeps the failing
|
||||
// filename in the error message identical to what the sequential form
|
||||
// reported, which the integration scenarios assert on.
|
||||
func forEachInputPath(ctx *api.Context, inputPaths []string, fn func(inputPath string) error) error {
|
||||
return forEachInputPathIndexed(ctx, inputPaths, func(_ int, inputPath string) error {
|
||||
return fn(inputPath)
|
||||
})
|
||||
}
|
||||
|
||||
// forEachInputPathIndexed is [forEachInputPath] with the input path's index,
|
||||
// for callers collecting a result per file. Writing into a preallocated slice
|
||||
// at the given index needs no further synchronization; writing into a shared
|
||||
// map does and must not be done from fn.
|
||||
func forEachInputPathIndexed(ctx *api.Context, inputPaths []string, fn func(i int, inputPath string) error) error {
|
||||
if len(inputPaths) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// The common case is a single file. Skip the goroutine and the slot: the
|
||||
// caller is already inside whatever bound its own route applies.
|
||||
if len(inputPaths) == 1 {
|
||||
return fn(0, inputPaths[0])
|
||||
}
|
||||
|
||||
// At the default ceiling of one, run the plain sequential loop this helper
|
||||
// replaced. Racing goroutines for a single slot would serialize the work
|
||||
// just the same, but the order files are picked up in would be down to the
|
||||
// scheduler, and every file would be attempted even once one has failed.
|
||||
// Taking the old path keeps the default a genuine no-op: same order, same
|
||||
// early return, no goroutines.
|
||||
if maxFileConcurrency < 2 {
|
||||
for i, inputPath := range inputPaths {
|
||||
err := fn(i, inputPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// The unit this request would have had all to itself before any of this
|
||||
// existed. Whichever file claims it runs without touching the shared pool,
|
||||
// so concurrent requests can never throttle each other below the
|
||||
// one-binary-apiece they already got. See [engineExtraSlots].
|
||||
reserved := make(chan struct{}, 1)
|
||||
reserved <- struct{}{}
|
||||
|
||||
errs := make([]error, len(inputPaths))
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for i, inputPath := range inputPaths {
|
||||
wg.Go(func() {
|
||||
release, err := acquireEngineSlot(ctx, reserved)
|
||||
if err != nil {
|
||||
errs[i] = err
|
||||
return
|
||||
}
|
||||
defer release()
|
||||
|
||||
errs[i] = fn(i, inputPath)
|
||||
})
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
|
||||
for _, err := range errs {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
341
pkg/modules/pdfengines/concurrency_test.go
Normal file
341
pkg/modules/pdfengines/concurrency_test.go
Normal file
@@ -0,0 +1,341 @@
|
||||
package pdfengines
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gotenberg/gotenberg/v8/pkg/modules/api"
|
||||
)
|
||||
|
||||
func TestForEachInputPath(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
scenario string
|
||||
inputPaths []string
|
||||
fn func(inputPath string) error
|
||||
expectErr string
|
||||
}{
|
||||
{
|
||||
scenario: "no input path",
|
||||
inputPaths: nil,
|
||||
fn: func(string) error { return errors.New("must not run") },
|
||||
},
|
||||
{
|
||||
scenario: "single input path",
|
||||
inputPaths: []string{"a.pdf"},
|
||||
fn: func(string) error { return nil },
|
||||
},
|
||||
{
|
||||
scenario: "single input path with error",
|
||||
inputPaths: []string{"a.pdf"},
|
||||
fn: func(p string) error { return fmt.Errorf("boom %s", p) },
|
||||
expectErr: "boom a.pdf",
|
||||
},
|
||||
{
|
||||
scenario: "many input paths",
|
||||
inputPaths: []string{"a.pdf", "b.pdf", "c.pdf", "d.pdf", "e.pdf"},
|
||||
fn: func(string) error { return nil },
|
||||
},
|
||||
{
|
||||
scenario: "error is the first in input order, not the first to arrive",
|
||||
inputPaths: []string{"a.pdf", "b.pdf", "c.pdf"},
|
||||
fn: func(p string) error {
|
||||
// "c.pdf" fails without delay so that it lands well before
|
||||
// "b.pdf"; the reported error must still be "b.pdf".
|
||||
if p == "b.pdf" {
|
||||
var counter int
|
||||
for i := range 5_000_000 {
|
||||
counter += i
|
||||
}
|
||||
return fmt.Errorf("slow failure %s (%d)", p, counter%1)
|
||||
}
|
||||
if p == "c.pdf" {
|
||||
return fmt.Errorf("fast failure %s", p)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
expectErr: "slow failure b.pdf (0)",
|
||||
},
|
||||
} {
|
||||
t.Run(tc.scenario, func(t *testing.T) {
|
||||
err := forEachInputPath(new(api.Context), tc.inputPaths, tc.fn)
|
||||
|
||||
if tc.expectErr == "" {
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error but got: %v", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if err == nil {
|
||||
t.Fatalf("expected error %q but got none", tc.expectErr)
|
||||
}
|
||||
|
||||
if err.Error() != tc.expectErr {
|
||||
t.Fatalf("expected error %q but got %q", tc.expectErr, err.Error())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestForEachInputPathRunsEveryPath(t *testing.T) {
|
||||
inputPaths := make([]string, 50)
|
||||
for i := range inputPaths {
|
||||
inputPaths[i] = fmt.Sprintf("%d.pdf", i)
|
||||
}
|
||||
|
||||
var (
|
||||
mu sync.Mutex
|
||||
seen = make(map[string]int)
|
||||
)
|
||||
|
||||
err := forEachInputPath(new(api.Context), inputPaths, func(inputPath string) error {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
seen[inputPath]++
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error but got: %v", err)
|
||||
}
|
||||
|
||||
if len(seen) != len(inputPaths) {
|
||||
t.Fatalf("expected %d distinct paths but got %d", len(inputPaths), len(seen))
|
||||
}
|
||||
|
||||
for path, count := range seen {
|
||||
if count != 1 {
|
||||
t.Fatalf("expected '%s' to run once but it ran %d times", path, count)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestForEachInputPathRespectsTheSlotCeiling(t *testing.T) {
|
||||
previousSlots, previousMax := engineExtraSlots, maxFileConcurrency
|
||||
defer func() { engineExtraSlots, maxFileConcurrency = previousSlots, previousMax }()
|
||||
|
||||
// One request may run its reserved unit plus ceiling-1 borrowed ones.
|
||||
const ceiling = 3
|
||||
maxFileConcurrency = ceiling
|
||||
engineExtraSlots = make(chan struct{}, ceiling-1)
|
||||
|
||||
inputPaths := make([]string, 40)
|
||||
for i := range inputPaths {
|
||||
inputPaths[i] = fmt.Sprintf("%d.pdf", i)
|
||||
}
|
||||
|
||||
var inFlight, peak atomic.Int64
|
||||
|
||||
err := forEachInputPath(new(api.Context), inputPaths, func(string) error {
|
||||
current := inFlight.Add(1)
|
||||
defer inFlight.Add(-1)
|
||||
|
||||
for {
|
||||
observed := peak.Load()
|
||||
if current <= observed || peak.CompareAndSwap(observed, current) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// Hold the slot long enough that the ceiling would be exceeded if it
|
||||
// were not enforced.
|
||||
var counter int
|
||||
for i := range 200_000 {
|
||||
counter += i
|
||||
}
|
||||
_ = counter
|
||||
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error but got: %v", err)
|
||||
}
|
||||
|
||||
if peak.Load() > ceiling {
|
||||
t.Fatalf("expected at most %d concurrent runs but observed %d", ceiling, peak.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestForEachInputPathHonorsCancellation(t *testing.T) {
|
||||
previousSlots, previousMax := engineExtraSlots, maxFileConcurrency
|
||||
defer func() { engineExtraSlots, maxFileConcurrency = previousSlots, previousMax }()
|
||||
|
||||
// Concurrent path, with the shared pool exhausted by another request, so
|
||||
// only this request's reserved unit is available.
|
||||
maxFileConcurrency = 3
|
||||
engineExtraSlots = make(chan struct{}, 2)
|
||||
engineExtraSlots <- struct{}{}
|
||||
engineExtraSlots <- struct{}{}
|
||||
|
||||
cancelledCtx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
// Neither source of capacity is available: the pool is exhausted by other
|
||||
// requests and this request's reserved unit is already in use by one of its
|
||||
// own files. A waiter must observe the cancelled context rather than block
|
||||
// forever. Driving acquireEngineSlot directly keeps that deterministic:
|
||||
// through forEachInputPath the reserved unit is reusable, so whether a
|
||||
// given file waits at all depends on how fast the file before it finishes.
|
||||
inUse := make(chan struct{}, 1)
|
||||
|
||||
release, err := acquireEngineSlot(&api.Context{Context: cancelledCtx}, inUse)
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("expected context.Canceled but got: %v", err)
|
||||
}
|
||||
|
||||
if release != nil {
|
||||
t.Fatal("expected no release function when acquisition fails")
|
||||
}
|
||||
|
||||
// A cancelled request stops taking on work even when capacity is free,
|
||||
// rather than deciding on the coin flip a ready select would give.
|
||||
inUse <- struct{}{}
|
||||
|
||||
_, err = acquireEngineSlot(&api.Context{Context: cancelledCtx}, inUse)
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("expected context.Canceled with the reserved unit free but got: %v", err)
|
||||
}
|
||||
|
||||
// Live request, free reserved unit: acquired and handed back.
|
||||
release, err = acquireEngineSlot(&api.Context{Context: context.Background()}, inUse)
|
||||
if err != nil {
|
||||
t.Fatalf("expected the reserved unit to be acquired but got: %v", err)
|
||||
}
|
||||
|
||||
release()
|
||||
|
||||
if len(inUse) != 1 {
|
||||
t.Fatalf("expected the reserved unit to be returned but the channel holds %d", len(inUse))
|
||||
}
|
||||
}
|
||||
|
||||
func TestForEachInputPathCompletesWithACancelledContext(t *testing.T) {
|
||||
previousSlots, previousMax := engineExtraSlots, maxFileConcurrency
|
||||
defer func() { engineExtraSlots, maxFileConcurrency = previousSlots, previousMax }()
|
||||
|
||||
maxFileConcurrency = 3
|
||||
engineExtraSlots = make(chan struct{}, 2)
|
||||
engineExtraSlots <- struct{}{}
|
||||
engineExtraSlots <- struct{}{}
|
||||
|
||||
cancelledCtx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
done := make(chan struct{})
|
||||
|
||||
go func() {
|
||||
defer close(done)
|
||||
_ = forEachInputPath(&api.Context{Context: cancelledCtx}, []string{"a.pdf", "b.pdf", "c.pdf"}, func(string) error {
|
||||
return nil
|
||||
})
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("forEachInputPath hung on a cancelled context with the shared pool exhausted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestForEachInputPathNeverThrottlesBelowOnePerRequest(t *testing.T) {
|
||||
previousSlots, previousMax := engineExtraSlots, maxFileConcurrency
|
||||
defer func() { engineExtraSlots, maxFileConcurrency = previousSlots, previousMax }()
|
||||
|
||||
// A small ceiling against far more concurrent requests than it covers.
|
||||
// Before the shared pool existed each of these ran a binary of its own, so
|
||||
// the pool must not drop aggregate concurrency below one per request.
|
||||
const (
|
||||
ceiling = 2
|
||||
requests = 8
|
||||
)
|
||||
|
||||
maxFileConcurrency = ceiling
|
||||
engineExtraSlots = make(chan struct{}, ceiling-1)
|
||||
|
||||
// Every runner announces itself and then blocks, so the count of arrivals
|
||||
// is the true simultaneous concurrency rather than whatever the scheduler
|
||||
// happened to overlap.
|
||||
arrived := make(chan struct{}, requests*3)
|
||||
release := make(chan struct{})
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for range requests {
|
||||
wg.Go(func() {
|
||||
_ = forEachInputPath(new(api.Context), []string{"a.pdf", "b.pdf", "c.pdf"}, func(string) error {
|
||||
arrived <- struct{}{}
|
||||
<-release
|
||||
return nil
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
// One runner per request must be able to start without waiting on the
|
||||
// shared pool. If the pool governed the total instead of the surplus, only
|
||||
// `ceiling` runners would ever arrive and this would time out.
|
||||
for i := range requests {
|
||||
select {
|
||||
case <-arrived:
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatalf("only %d runners started concurrently, expected at least one per request (%d); the shared pool is throttling requests against each other", i, requests)
|
||||
}
|
||||
}
|
||||
|
||||
close(release)
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func TestForEachInputPathIsSequentialAtTheDefaultCeiling(t *testing.T) {
|
||||
if defaultMaxConcurrency != 1 {
|
||||
t.Fatalf("this test pins the default as a no-op, but defaultMaxConcurrency is %d", defaultMaxConcurrency)
|
||||
}
|
||||
|
||||
previousSlots, previousMax := engineExtraSlots, maxFileConcurrency
|
||||
defer func() { engineExtraSlots, maxFileConcurrency = previousSlots, previousMax }()
|
||||
|
||||
maxFileConcurrency = defaultMaxConcurrency
|
||||
engineExtraSlots = make(chan struct{}, defaultMaxConcurrency-1)
|
||||
|
||||
// At the default the helper must behave exactly like the sequential loops
|
||||
// it replaced: files in input order, and no file attempted once one has
|
||||
// failed.
|
||||
var order []string
|
||||
|
||||
err := forEachInputPath(new(api.Context), []string{"a.pdf", "b.pdf", "c.pdf", "d.pdf"}, func(inputPath string) error {
|
||||
order = append(order, inputPath)
|
||||
if inputPath == "b.pdf" {
|
||||
return errors.New("boom")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
if err == nil || err.Error() != "boom" {
|
||||
t.Fatalf("expected error \"boom\" but got: %v", err)
|
||||
}
|
||||
|
||||
if len(order) != 2 || order[0] != "a.pdf" || order[1] != "b.pdf" {
|
||||
t.Fatalf("expected the run to stop after b.pdf in input order but got %v", order)
|
||||
}
|
||||
}
|
||||
|
||||
func TestForEachInputPathIndexed(t *testing.T) {
|
||||
inputPaths := []string{"a.pdf", "b.pdf", "c.pdf", "d.pdf"}
|
||||
collected := make([]string, len(inputPaths))
|
||||
|
||||
err := forEachInputPathIndexed(new(api.Context), inputPaths, func(i int, inputPath string) error {
|
||||
collected[i] = inputPath
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error but got: %v", err)
|
||||
}
|
||||
|
||||
for i, inputPath := range inputPaths {
|
||||
if collected[i] != inputPath {
|
||||
t.Fatalf("expected index %d to hold '%s' but got '%s'", i, inputPath, collected[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -45,6 +45,7 @@ type PdfEngines struct {
|
||||
rotateNames []string
|
||||
facturXNames []string
|
||||
engines []gotenberg.PdfEngine
|
||||
maxConcurrency int
|
||||
disableRoutes bool
|
||||
}
|
||||
|
||||
@@ -70,6 +71,7 @@ func (mod *PdfEngines) Descriptor() gotenberg.ModuleDescriptor {
|
||||
fs.StringSlice("pdfengines-stamp-engines", []string{"pdfcpu", "pdftk"}, "Set the PDF engines and their order for the stamp feature - empty means all")
|
||||
fs.StringSlice("pdfengines-rotate-engines", []string{"pdfcpu", "pdftk"}, "Set the PDF engines and their order for the rotate feature - empty means all")
|
||||
fs.StringSlice("pdfengines-factur-x-engines", []string{"qpdf"}, "Set the PDF engines and their order for the Factur-X XMP feature - empty means all")
|
||||
fs.Int("pdfengines-max-concurrency", defaultMaxConcurrency, "Set the maximum number of PDF files a feature processes concurrently, across all requests - bounds how many qpdf, pdfcpu, pdftk and exiftool processes run at once, so raising it trades memory for speed. Does not apply to LibreOffice: scale Gotenberg containers instead")
|
||||
fs.Bool("pdfengines-disable-routes", false, "Disable the routes")
|
||||
|
||||
// Deprecated flags.
|
||||
@@ -105,8 +107,16 @@ func (mod *PdfEngines) Provision(ctx *gotenberg.Context) error {
|
||||
stampNames := flags.MustStringSlice("pdfengines-stamp-engines")
|
||||
rotateNames := flags.MustStringSlice("pdfengines-rotate-engines")
|
||||
facturXNames := flags.MustStringSlice("pdfengines-factur-x-engines")
|
||||
mod.maxConcurrency = flags.MustInt("pdfengines-max-concurrency")
|
||||
mod.disableRoutes = flags.MustBool("pdfengines-disable-routes")
|
||||
|
||||
if mod.maxConcurrency > 0 {
|
||||
maxFileConcurrency = mod.maxConcurrency
|
||||
// One fewer than the ceiling: each request already reserves a unit of
|
||||
// its own. See [engineExtraSlots].
|
||||
engineExtraSlots = make(chan struct{}, mod.maxConcurrency-1)
|
||||
}
|
||||
|
||||
engines, err := ctx.Modules(new(gotenberg.PdfEngine))
|
||||
if err != nil {
|
||||
return fmt.Errorf("get PDF engines: %w", err)
|
||||
@@ -222,6 +232,10 @@ func (mod *PdfEngines) Validate() error {
|
||||
return errors.New("no PDF engine is available; enable at least one engine module (e.g. qpdf, pdfcpu, pdftk, libreoffice-pdfengine, exiftool)")
|
||||
}
|
||||
|
||||
if mod.maxConcurrency < 1 {
|
||||
return fmt.Errorf("PDF engines max concurrency must be at least 1, got %d; set --pdfengines-max-concurrency (env PDFENGINES_MAX_CONCURRENCY) to a positive value", mod.maxConcurrency)
|
||||
}
|
||||
|
||||
availableEngines := make([]string, len(mod.engines))
|
||||
|
||||
for i, engine := range mod.engines {
|
||||
@@ -296,6 +310,7 @@ func (mod *PdfEngines) SystemMessages() []string {
|
||||
fmt.Sprintf("stamp engines - %s", strings.Join(mod.stampNames, " ")),
|
||||
fmt.Sprintf("rotate engines - %s", strings.Join(mod.rotateNames, " ")),
|
||||
fmt.Sprintf("factur-x engines - %s", strings.Join(mod.facturXNames, " ")),
|
||||
fmt.Sprintf("max concurrency - %d", mod.maxConcurrency),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -11,7 +11,7 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/labstack/echo/v4"
|
||||
"github.com/labstack/echo/v5"
|
||||
|
||||
"github.com/gotenberg/gotenberg/v8/pkg/gotenberg"
|
||||
"github.com/gotenberg/gotenberg/v8/pkg/modules/api"
|
||||
@@ -208,14 +208,14 @@ func RotateStub(ctx *api.Context, engine gotenberg.PdfEngine, angle int, pages s
|
||||
return nil
|
||||
}
|
||||
|
||||
for _, inputPath := range inputPaths {
|
||||
return forEachInputPath(ctx, inputPaths, func(inputPath string) error {
|
||||
err := engine.Rotate(ctx, ctx.Log(), inputPath, angle, pages)
|
||||
if err != nil {
|
||||
return fmt.Errorf("rotate '%s': %w", inputPath, err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// ValidatePdfFormatsCompat checks for incompatible combinations of PDF formats
|
||||
@@ -334,14 +334,14 @@ func SplitPdfStub(ctx *api.Context, engine gotenberg.PdfEngine, mode gotenberg.S
|
||||
// FlattenStub merges annotation appearances with page content for each given
|
||||
// PDF, effectively deleting the original annotations.
|
||||
func FlattenStub(ctx *api.Context, engine gotenberg.PdfEngine, inputPaths []string) error {
|
||||
for _, inputPath := range inputPaths {
|
||||
return forEachInputPath(ctx, inputPaths, func(inputPath string) error {
|
||||
err := engine.Flatten(ctx, ctx.Log(), inputPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("flatten '%s': %w", inputPath, err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// defaultImageQuality is the JPEG quality applied by the image optimization
|
||||
@@ -393,19 +393,27 @@ func OptimizeStub(ctx *api.Context, engine gotenberg.PdfEngine, optimizeImages b
|
||||
return nil
|
||||
}
|
||||
|
||||
for _, inputPath := range inputPaths {
|
||||
return forEachInputPath(ctx, inputPaths, func(inputPath string) error {
|
||||
err := engine.OptimizeImages(ctx, ctx.Log(), imageQuality, inputPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("optimize images of '%s': %w", inputPath, err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// ConvertStub transforms a given PDF to the specified formats defined in
|
||||
// [gotenberg.PdfFormats]. If no format, it does nothing and returns the input
|
||||
// paths.
|
||||
//
|
||||
// This loop stays sequential on purpose. Convert is the one PDF engine method
|
||||
// LibreOffice implements, and libreoffice-pdfengine is the default and only
|
||||
// convert engine, so every iteration here drives the single soffice daemon. A
|
||||
// LibreOffice instance is far too memory-hungry to run several of per
|
||||
// container: the way to convert more documents at once is to scale Gotenberg
|
||||
// containers, not to widen this loop. Do not route it through
|
||||
// [forEachInputPath].
|
||||
func ConvertStub(ctx *api.Context, engine gotenberg.PdfEngine, formats gotenberg.PdfFormats, inputPaths []string) ([]string, error) {
|
||||
zeroValued := gotenberg.PdfFormats{}
|
||||
if formats == zeroValued {
|
||||
@@ -432,14 +440,14 @@ func WriteMetadataStub(ctx *api.Context, engine gotenberg.PdfEngine, metadata ma
|
||||
return nil
|
||||
}
|
||||
|
||||
for _, inputPath := range inputPaths {
|
||||
return forEachInputPath(ctx, inputPaths, func(inputPath string) error {
|
||||
err := engine.WriteMetadata(ctx, ctx.Log(), metadata, inputPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("write metadata into '%s': %w", inputPath, err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// documentTitle returns the input PDF's Title metadata entry, falling back to
|
||||
@@ -491,28 +499,33 @@ func WriteBookmarksStub(ctx *api.Context, engine gotenberg.PdfEngine, bookmarks
|
||||
return nil
|
||||
}
|
||||
|
||||
for _, inputPath := range inputPaths {
|
||||
return forEachInputPath(ctx, inputPaths, func(inputPath string) error {
|
||||
err := engine.WriteBookmarks(ctx, ctx.Log(), inputPath, b)
|
||||
if err != nil {
|
||||
return fmt.Errorf("write bookmarks into '%s': %w", inputPath, err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
case map[string][]gotenberg.Bookmark:
|
||||
for _, inputPath := range inputPaths {
|
||||
return forEachInputPath(ctx, inputPaths, func(inputPath string) error {
|
||||
filename := ctx.OriginalFilename(inputPath)
|
||||
if specificBookmarks, ok := b[filename]; ok {
|
||||
err := engine.WriteBookmarks(ctx, ctx.Log(), inputPath, specificBookmarks)
|
||||
if err != nil {
|
||||
return fmt.Errorf("write bookmarks into '%s': %w", inputPath, err)
|
||||
}
|
||||
specificBookmarks, ok := b[filename]
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
err := engine.WriteBookmarks(ctx, ctx.Log(), inputPath, specificBookmarks)
|
||||
if err != nil {
|
||||
return fmt.Errorf("write bookmarks into '%s': %w", inputPath, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
default:
|
||||
// Should not happen.
|
||||
return fmt.Errorf("bookmarks type '%T' not supported", bookmarks)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// FormDataPdfEmbeds extracts embedded file paths from form data.
|
||||
@@ -537,14 +550,14 @@ func EmbedFilesMetadataStub(ctx *api.Context, engine gotenberg.PdfEngine, metada
|
||||
return nil
|
||||
}
|
||||
|
||||
for _, inputPath := range inputPaths {
|
||||
return forEachInputPath(ctx, inputPaths, func(inputPath string) error {
|
||||
err := engine.EmbedFilesMetadata(ctx, ctx.Log(), metadata, inputPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("set embeds metadata on PDF '%s': %w", inputPath, err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// FormDataPdfFacturX extracts the Factur-X parameters and the invoice XML path
|
||||
@@ -737,14 +750,14 @@ func InjectFacturXXMPStub(ctx *api.Context, engine gotenberg.PdfEngine, facturX
|
||||
return nil
|
||||
}
|
||||
|
||||
for _, inputPath := range inputPaths {
|
||||
return forEachInputPath(ctx, inputPaths, func(inputPath string) error {
|
||||
err := engine.InjectFacturXXMP(ctx, ctx.Log(), facturX, inputPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("inject Factur-X XMP into PDF '%s': %w", inputPath, err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// FormDataPdfEncrypt extracts the encryption parameters and permissions from
|
||||
@@ -783,14 +796,14 @@ func EncryptPdfStub(ctx *api.Context, engine gotenberg.PdfEngine, opts gotenberg
|
||||
return nil
|
||||
}
|
||||
|
||||
for _, inputPath := range inputPaths {
|
||||
return forEachInputPath(ctx, inputPaths, func(inputPath string) error {
|
||||
err := engine.Encrypt(ctx, ctx.Log(), inputPath, opts)
|
||||
if err != nil {
|
||||
return fmt.Errorf("encrypt PDF '%s': %w", inputPath, err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// EmbedFilesStub embeds files into PDF files.
|
||||
@@ -819,14 +832,14 @@ func EmbedFilesStub(ctx *api.Context, engine gotenberg.PdfEngine, embedPaths []s
|
||||
resolvedPaths[i] = resolvedPath
|
||||
}
|
||||
|
||||
for _, inputPath := range inputPaths {
|
||||
return forEachInputPath(ctx, inputPaths, func(inputPath string) error {
|
||||
err := engine.EmbedFiles(ctx, ctx.Log(), resolvedPaths, inputPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("embed files into PDF '%s': %w", inputPath, err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// FormDataPdfStamps builds the ordered list of stamps from the repeated stamp
|
||||
@@ -943,16 +956,23 @@ func bindStampOrWatermarkFiles(stamps []gotenberg.Stamp, files []string, kind st
|
||||
// WatermarkStub applies each watermark to a list of PDF files, in order.
|
||||
// Entries with no source are skipped, so an empty list does nothing.
|
||||
func WatermarkStub(ctx *api.Context, engine gotenberg.PdfEngine, watermarks []gotenberg.Stamp, inputPaths []string) error {
|
||||
// Watermarks stack on the same file, so the outer loop stays sequential;
|
||||
// only the file dimension is parallel.
|
||||
for _, watermark := range watermarks {
|
||||
if watermark.Source == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
for _, inputPath := range inputPaths {
|
||||
err := engine.Watermark(ctx, ctx.Log(), inputPath, watermark)
|
||||
if err != nil {
|
||||
return fmt.Errorf("watermark '%s': %w", inputPath, err)
|
||||
err := forEachInputPath(ctx, inputPaths, func(inputPath string) error {
|
||||
errWatermark := engine.Watermark(ctx, ctx.Log(), inputPath, watermark)
|
||||
if errWatermark != nil {
|
||||
return fmt.Errorf("watermark '%s': %w", inputPath, errWatermark)
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
@@ -962,16 +982,23 @@ func WatermarkStub(ctx *api.Context, engine gotenberg.PdfEngine, watermarks []go
|
||||
// StampStub applies each stamp to a list of PDF files, in order. Entries with
|
||||
// no source are skipped, so an empty list does nothing.
|
||||
func StampStub(ctx *api.Context, engine gotenberg.PdfEngine, stamps []gotenberg.Stamp, inputPaths []string) error {
|
||||
// Stamps stack on the same file, so the outer loop stays sequential; only
|
||||
// the file dimension is parallel.
|
||||
for _, stamp := range stamps {
|
||||
if stamp.Source == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
for _, inputPath := range inputPaths {
|
||||
err := engine.Stamp(ctx, ctx.Log(), inputPath, stamp)
|
||||
if err != nil {
|
||||
return fmt.Errorf("stamp '%s': %w", inputPath, err)
|
||||
err := forEachInputPath(ctx, inputPaths, func(inputPath string) error {
|
||||
errStamp := engine.Stamp(ctx, ctx.Log(), inputPath, stamp)
|
||||
if errStamp != nil {
|
||||
return fmt.Errorf("stamp '%s': %w", inputPath, errStamp)
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
@@ -984,7 +1011,7 @@ func mergeRoute(engine gotenberg.PdfEngine) api.Route {
|
||||
Method: http.MethodPost,
|
||||
Path: "/forms/pdfengines/merge",
|
||||
IsMultipart: true,
|
||||
Handler: func(c echo.Context) error {
|
||||
Handler: func(c *echo.Context) error {
|
||||
ctx := c.Get("context").(*api.Context)
|
||||
|
||||
form := ctx.FormData()
|
||||
@@ -1185,7 +1212,7 @@ func splitRoute(engine gotenberg.PdfEngine) api.Route {
|
||||
Method: http.MethodPost,
|
||||
Path: "/forms/pdfengines/split",
|
||||
IsMultipart: true,
|
||||
Handler: func(c echo.Context) error {
|
||||
Handler: func(c *echo.Context) error {
|
||||
ctx := c.Get("context").(*api.Context)
|
||||
|
||||
form := ctx.FormData()
|
||||
@@ -1337,7 +1364,7 @@ func flattenRoute(engine gotenberg.PdfEngine) api.Route {
|
||||
Method: http.MethodPost,
|
||||
Path: "/forms/pdfengines/flatten",
|
||||
IsMultipart: true,
|
||||
Handler: func(c echo.Context) error {
|
||||
Handler: func(c *echo.Context) error {
|
||||
ctx := c.Get("context").(*api.Context)
|
||||
|
||||
form := ctx.FormData()
|
||||
@@ -1372,7 +1399,7 @@ func optimizeRoute(engine gotenberg.PdfEngine) api.Route {
|
||||
Method: http.MethodPost,
|
||||
Path: "/forms/pdfengines/optimize",
|
||||
IsMultipart: true,
|
||||
Handler: func(c echo.Context) error {
|
||||
Handler: func(c *echo.Context) error {
|
||||
ctx := c.Get("context").(*api.Context)
|
||||
|
||||
form := ctx.FormData()
|
||||
@@ -1408,7 +1435,7 @@ func convertRoute(engine gotenberg.PdfEngine) api.Route {
|
||||
Method: http.MethodPost,
|
||||
Path: "/forms/pdfengines/convert",
|
||||
IsMultipart: true,
|
||||
Handler: func(c echo.Context) error {
|
||||
Handler: func(c *echo.Context) error {
|
||||
ctx := c.Get("context").(*api.Context)
|
||||
|
||||
form := ctx.FormData()
|
||||
@@ -1465,7 +1492,7 @@ func readMetadataRoute(engine gotenberg.PdfEngine) api.Route {
|
||||
Method: http.MethodPost,
|
||||
Path: "/forms/pdfengines/metadata/read",
|
||||
IsMultipart: true,
|
||||
Handler: func(c echo.Context) error {
|
||||
Handler: func(c *echo.Context) error {
|
||||
ctx := c.Get("context").(*api.Context)
|
||||
|
||||
var inputPaths []string
|
||||
@@ -1476,14 +1503,25 @@ func readMetadataRoute(engine gotenberg.PdfEngine) api.Route {
|
||||
return fmt.Errorf("validate form data: %w", err)
|
||||
}
|
||||
|
||||
res := make(map[string]map[string]any, len(inputPaths))
|
||||
for _, inputPath := range inputPaths {
|
||||
metadata, err := engine.ReadMetadata(ctx, ctx.Log(), inputPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read metadata: %w", err)
|
||||
// Collected per index, then folded into the map on this
|
||||
// goroutine: a shared map cannot be written concurrently.
|
||||
collected := make([]map[string]any, len(inputPaths))
|
||||
err = forEachInputPathIndexed(ctx, inputPaths, func(i int, inputPath string) error {
|
||||
metadata, errRead := engine.ReadMetadata(ctx, ctx.Log(), inputPath)
|
||||
if errRead != nil {
|
||||
return fmt.Errorf("read metadata: %w", errRead)
|
||||
}
|
||||
|
||||
res[ctx.OriginalFilename(inputPath)] = metadata
|
||||
collected[i] = metadata
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
res := make(map[string]map[string]any, len(inputPaths))
|
||||
for i, inputPath := range inputPaths {
|
||||
res[ctx.OriginalFilename(inputPath)] = collected[i]
|
||||
}
|
||||
|
||||
err = c.JSON(http.StatusOK, res)
|
||||
@@ -1508,7 +1546,7 @@ func writeMetadataRoute(engine gotenberg.PdfEngine) api.Route {
|
||||
Method: http.MethodPost,
|
||||
Path: "/forms/pdfengines/metadata/write",
|
||||
IsMultipart: true,
|
||||
Handler: func(c echo.Context) error {
|
||||
Handler: func(c *echo.Context) error {
|
||||
ctx := c.Get("context").(*api.Context)
|
||||
|
||||
form := ctx.FormData()
|
||||
@@ -1543,7 +1581,7 @@ func readBookmarksRoute(engine gotenberg.PdfEngine) api.Route {
|
||||
Method: http.MethodPost,
|
||||
Path: "/forms/pdfengines/bookmarks/read",
|
||||
IsMultipart: true,
|
||||
Handler: func(c echo.Context) error {
|
||||
Handler: func(c *echo.Context) error {
|
||||
ctx := c.Get("context").(*api.Context)
|
||||
|
||||
var inputPaths []string
|
||||
@@ -1554,14 +1592,25 @@ func readBookmarksRoute(engine gotenberg.PdfEngine) api.Route {
|
||||
return fmt.Errorf("validate form data: %w", err)
|
||||
}
|
||||
|
||||
res := make(map[string][]gotenberg.Bookmark, len(inputPaths))
|
||||
for _, inputPath := range inputPaths {
|
||||
bookmarks, err := engine.ReadBookmarks(ctx, ctx.Log(), inputPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read bookmarks: %w", err)
|
||||
// Collected per index, then folded into the map on this
|
||||
// goroutine: a shared map cannot be written concurrently.
|
||||
collected := make([][]gotenberg.Bookmark, len(inputPaths))
|
||||
err = forEachInputPathIndexed(ctx, inputPaths, func(i int, inputPath string) error {
|
||||
bookmarks, errRead := engine.ReadBookmarks(ctx, ctx.Log(), inputPath)
|
||||
if errRead != nil {
|
||||
return fmt.Errorf("read bookmarks: %w", errRead)
|
||||
}
|
||||
|
||||
res[ctx.OriginalFilename(inputPath)] = bookmarks
|
||||
collected[i] = bookmarks
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
res := make(map[string][]gotenberg.Bookmark, len(inputPaths))
|
||||
for i, inputPath := range inputPaths {
|
||||
res[ctx.OriginalFilename(inputPath)] = collected[i]
|
||||
}
|
||||
|
||||
err = c.JSON(http.StatusOK, res)
|
||||
@@ -1585,7 +1634,7 @@ func writeBookmarksRoute(engine gotenberg.PdfEngine) api.Route {
|
||||
Method: http.MethodPost,
|
||||
Path: "/forms/pdfengines/bookmarks/write",
|
||||
IsMultipart: true,
|
||||
Handler: func(c echo.Context) error {
|
||||
Handler: func(c *echo.Context) error {
|
||||
ctx := c.Get("context").(*api.Context)
|
||||
|
||||
form := ctx.FormData()
|
||||
@@ -1620,7 +1669,7 @@ func encryptRoute(engine gotenberg.PdfEngine) api.Route {
|
||||
Method: http.MethodPost,
|
||||
Path: "/forms/pdfengines/encrypt",
|
||||
IsMultipart: true,
|
||||
Handler: func(c echo.Context) error {
|
||||
Handler: func(c *echo.Context) error {
|
||||
ctx := c.Get("context").(*api.Context)
|
||||
|
||||
form := ctx.FormData()
|
||||
@@ -1665,7 +1714,7 @@ func embedRoute(engine gotenberg.PdfEngine) api.Route {
|
||||
Method: http.MethodPost,
|
||||
Path: "/forms/pdfengines/embed",
|
||||
IsMultipart: true,
|
||||
Handler: func(c echo.Context) error {
|
||||
Handler: func(c *echo.Context) error {
|
||||
ctx := c.Get("context").(*api.Context)
|
||||
|
||||
form := ctx.FormData()
|
||||
@@ -1727,7 +1776,7 @@ func watermarkRoute(engine gotenberg.PdfEngine) api.Route {
|
||||
Method: http.MethodPost,
|
||||
Path: "/forms/pdfengines/watermark",
|
||||
IsMultipart: true,
|
||||
Handler: func(c echo.Context) error {
|
||||
Handler: func(c *echo.Context) error {
|
||||
ctx := c.Get("context").(*api.Context)
|
||||
|
||||
form := ctx.FormData()
|
||||
@@ -1784,7 +1833,7 @@ func stampRoute(engine gotenberg.PdfEngine) api.Route {
|
||||
Method: http.MethodPost,
|
||||
Path: "/forms/pdfengines/stamp",
|
||||
IsMultipart: true,
|
||||
Handler: func(c echo.Context) error {
|
||||
Handler: func(c *echo.Context) error {
|
||||
ctx := c.Get("context").(*api.Context)
|
||||
|
||||
form := ctx.FormData()
|
||||
@@ -1843,7 +1892,7 @@ func rotateRoute(engine gotenberg.PdfEngine) api.Route {
|
||||
Method: http.MethodPost,
|
||||
Path: "/forms/pdfengines/rotate",
|
||||
IsMultipart: true,
|
||||
Handler: func(c echo.Context) error {
|
||||
Handler: func(c *echo.Context) error {
|
||||
ctx := c.Get("context").(*api.Context)
|
||||
|
||||
form := ctx.FormData()
|
||||
@@ -1880,7 +1929,7 @@ func facturXRoute(engine gotenberg.PdfEngine) api.Route {
|
||||
Method: http.MethodPost,
|
||||
Path: "/forms/pdfengines/factur-x",
|
||||
IsMultipart: true,
|
||||
Handler: func(c echo.Context) error {
|
||||
Handler: func(c *echo.Context) error {
|
||||
ctx := c.Get("context").(*api.Context)
|
||||
|
||||
form := ctx.FormData()
|
||||
|
||||
@@ -7,7 +7,7 @@ import (
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/labstack/echo/v4"
|
||||
"github.com/labstack/echo/v5"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
flag "github.com/spf13/pflag"
|
||||
|
||||
@@ -120,6 +120,30 @@ func (engine *QPdf) spanAttrs(extra ...attribute.KeyValue) []attribute.KeyValue
|
||||
return append(attrs, extra...)
|
||||
}
|
||||
|
||||
// qpdfPageRange matches the page range syntax qpdf accepts, and nothing else.
|
||||
//
|
||||
// A term is a page number, "z" for the last page, or "rN" counting from the
|
||||
// end, optionally prefixed with "x" to exclude it. Terms combine into ranges
|
||||
// with "-", ranges join with ",", and the whole thing takes an optional ":odd"
|
||||
// or ":even".
|
||||
var qpdfPageRange = regexp.MustCompile(`^x?(?:z|r\d+|\d+)(?:-x?(?:z|r\d+|\d+))?(?:,x?(?:z|r\d+|\d+)(?:-x?(?:z|r\d+|\d+))?)*(?::odd|:even)?$`)
|
||||
|
||||
// validateSplitSpan returns span when it is a qpdf page range.
|
||||
//
|
||||
// qpdf reads the argument after "--pages ." as either a page range or another
|
||||
// source file, so a span carrying a path makes qpdf append that file's pages
|
||||
// to the output. Other engines in the split chain accept spellings qpdf does
|
||||
// not, such as pdfcpu's "2-end", so a span this rejects is not necessarily
|
||||
// invalid. Returning an error lets the chain move on to an engine that
|
||||
// understands it.
|
||||
func validateSplitSpan(span string) error {
|
||||
if qpdfPageRange.MatchString(span) {
|
||||
return nil
|
||||
}
|
||||
|
||||
return fmt.Errorf("split span '%s' is not a QPDF page range: %w", span, gotenberg.ErrPdfSplitModeNotSupported)
|
||||
}
|
||||
|
||||
// Split splits a given PDF file.
|
||||
func (engine *QPdf) Split(ctx context.Context, logger *slog.Logger, mode gotenberg.SplitMode, inputPath, outputDirPath string) ([]string, error) {
|
||||
ctx, span := gotenberg.Tracer().Start(ctx, "qpdf.Split",
|
||||
@@ -139,6 +163,12 @@ func (engine *QPdf) Split(ctx context.Context, logger *slog.Logger, mode gotenbe
|
||||
span.SetStatus(codes.Error, err.Error())
|
||||
return nil, err
|
||||
}
|
||||
err := validateSplitSpan(mode.Span)
|
||||
if err != nil {
|
||||
span.RecordError(err)
|
||||
span.SetStatus(codes.Error, err.Error())
|
||||
return nil, err
|
||||
}
|
||||
args = append(args, inputPath)
|
||||
args = append(args, engine.globalArgs...)
|
||||
args = append(args, "--pages", ".", mode.Span)
|
||||
|
||||
@@ -3,9 +3,12 @@ package qpdf
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/gotenberg/gotenberg/v8/pkg/gotenberg"
|
||||
)
|
||||
|
||||
func TestStripQpdfStringPrefix(t *testing.T) {
|
||||
@@ -270,3 +273,54 @@ func TestSetStreamSubtype(t *testing.T) {
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestValidateSplitSpan(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
span string
|
||||
valid bool
|
||||
}{
|
||||
// qpdf page ranges.
|
||||
{"1", true},
|
||||
{"12", true},
|
||||
{"1-5", true},
|
||||
{"2-z", true},
|
||||
{"z", true},
|
||||
{"r1", true},
|
||||
{"r3-r1", true},
|
||||
{"1,3,5-9", true},
|
||||
{"1-5,x3", true},
|
||||
{"1-z:odd", true},
|
||||
{"1-z:even", true},
|
||||
|
||||
// Other engines' spellings. Not valid here, so the chain moves on.
|
||||
{"2-end", false},
|
||||
{"2-", false},
|
||||
{"foo", false},
|
||||
|
||||
// A span qpdf would read as a source file.
|
||||
{"/tmp/secret.pdf", false},
|
||||
{"secret.pdf", false},
|
||||
{"./secret.pdf", false},
|
||||
{"../../etc/hosts", false},
|
||||
{"1,/tmp/secret.pdf", false},
|
||||
{"1 /tmp/secret.pdf", false},
|
||||
{"", false},
|
||||
{"--password=x", false},
|
||||
} {
|
||||
t.Run(tc.span, func(t *testing.T) {
|
||||
err := validateSplitSpan(tc.span)
|
||||
if tc.valid && err != nil {
|
||||
t.Fatalf("validateSplitSpan(%q) = %v, want nil", tc.span, err)
|
||||
}
|
||||
if !tc.valid {
|
||||
if err == nil {
|
||||
t.Fatalf("validateSplitSpan(%q) = nil, want an error", tc.span)
|
||||
}
|
||||
// The chain must be able to try the next engine.
|
||||
if !errors.Is(err, gotenberg.ErrPdfSplitModeNotSupported) {
|
||||
t.Fatalf("error %v does not wrap ErrPdfSplitModeNotSupported", err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,7 +11,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/hashicorp/go-retryablehttp"
|
||||
"github.com/labstack/echo/v4"
|
||||
"github.com/labstack/echo/v5"
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
"go.opentelemetry.io/otel/propagation"
|
||||
@@ -31,10 +31,33 @@ type client struct {
|
||||
extraHttpHeaders map[string]string
|
||||
startTime time.Time
|
||||
|
||||
// deliveryTimeout bounds one delivery including retries. See
|
||||
// [Webhook.deliveryTimeout].
|
||||
deliveryTimeout time.Duration
|
||||
|
||||
client *retryablehttp.Client
|
||||
logger *slog.Logger
|
||||
}
|
||||
|
||||
// deliveryContext returns the context one delivery runs on.
|
||||
//
|
||||
// It keeps the values of ctx, so trace propagation and logging correlation
|
||||
// survive, and replaces its cancellation with a fresh budget. Threading the
|
||||
// conversion context straight through does not work: a delivery starts after
|
||||
// the handler returned, so that deadline may already be spent and the callback
|
||||
// would fail without a single attempt.
|
||||
func (c client) deliveryContext(ctx context.Context) (context.Context, context.CancelFunc) {
|
||||
timeout := c.deliveryTimeout
|
||||
if timeout <= 0 {
|
||||
// An unset budget would expire the delivery before its first attempt.
|
||||
// [Webhook.deliveryTimeout] never returns a non-positive value, so this
|
||||
// only guards a caller that builds a client without one.
|
||||
timeout = minDeliveryTimeout
|
||||
}
|
||||
|
||||
return context.WithTimeout(context.WithoutCancel(ctx), timeout)
|
||||
}
|
||||
|
||||
// send call the webhook either to send the success response or the error response.
|
||||
func (c client) send(ctx context.Context, body io.Reader, headers map[string]string, errored bool) error {
|
||||
url := c.url
|
||||
@@ -57,6 +80,9 @@ func (c client) send(ctx context.Context, body io.Reader, headers map[string]str
|
||||
spanName = fmt.Sprintf("%s Webhook Error", method)
|
||||
}
|
||||
|
||||
ctx, cancel := c.deliveryContext(ctx)
|
||||
defer cancel()
|
||||
|
||||
tracer := gotenberg.Tracer()
|
||||
ctx, span := tracer.Start(ctx, spanName,
|
||||
trace.WithSpanKind(trace.SpanKindClient),
|
||||
@@ -64,7 +90,10 @@ func (c client) send(ctx context.Context, body io.Reader, headers map[string]str
|
||||
)
|
||||
defer span.End()
|
||||
|
||||
req, err := retryablehttp.NewRequest(method, url, body)
|
||||
// The request must carry ctx: retryablehttp.NewRequest builds on
|
||||
// [context.Background], and its wait between attempts is a select on the
|
||||
// request context, so a contextless request cannot be interrupted.
|
||||
req, err := retryablehttp.NewRequestWithContext(ctx, method, url, body)
|
||||
if err != nil {
|
||||
span.RecordError(err)
|
||||
span.SetStatus(codes.Error, err.Error())
|
||||
@@ -114,13 +143,12 @@ func (c client) send(ctx context.Context, body io.Reader, headers map[string]str
|
||||
return fmt.Errorf("send '%s' request to '%s': %w", method, url, err)
|
||||
}
|
||||
|
||||
if resp.StatusCode >= http.StatusBadRequest {
|
||||
err := fmt.Errorf("send '%s' request to '%s': got status: '%s'", method, url, resp.Status)
|
||||
span.RecordError(err)
|
||||
span.SetStatus(codes.Error, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
// Registered before the status check below. [retryablehttp.Client.Do] hands
|
||||
// back a live body for a status it does not retry, which is every 4xx but
|
||||
// 429, so returning early without closing it strands the connection and the
|
||||
// transport goroutines that serve it for the lifetime of the process. The
|
||||
// transport is built per delivery in [gotenberg.NewOutboundHttpClient], so
|
||||
// nothing reclaims it later either.
|
||||
defer func() {
|
||||
err := resp.Body.Close()
|
||||
if err != nil {
|
||||
@@ -128,6 +156,13 @@ func (c client) send(ctx context.Context, body io.Reader, headers map[string]str
|
||||
}
|
||||
}()
|
||||
|
||||
if resp.StatusCode >= http.StatusBadRequest {
|
||||
err := fmt.Errorf("send '%s' request to '%s': got status: '%s'", method, url, resp.Status)
|
||||
span.RecordError(err)
|
||||
span.SetStatus(codes.Error, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
// Last piece for calculating the latency.
|
||||
finishTime := time.Now()
|
||||
|
||||
@@ -165,6 +200,9 @@ func (c client) sendEvent(ctx context.Context, correlationIdHeader, correlationI
|
||||
return
|
||||
}
|
||||
|
||||
ctx, cancel := c.deliveryContext(ctx)
|
||||
defer cancel()
|
||||
|
||||
tracer := gotenberg.Tracer()
|
||||
ctx, span := tracer.Start(ctx, "POST Webhook Event",
|
||||
trace.WithSpanKind(trace.SpanKindClient),
|
||||
@@ -172,7 +210,7 @@ func (c client) sendEvent(ctx context.Context, correlationIdHeader, correlationI
|
||||
)
|
||||
defer span.End()
|
||||
|
||||
req, err := retryablehttp.NewRequest(http.MethodPost, c.eventsUrl, b)
|
||||
req, err := retryablehttp.NewRequestWithContext(ctx, http.MethodPost, c.eventsUrl, b)
|
||||
if err != nil {
|
||||
span.RecordError(err)
|
||||
span.SetStatus(codes.Error, err.Error())
|
||||
|
||||
@@ -14,7 +14,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/hashicorp/go-retryablehttp"
|
||||
"github.com/labstack/echo/v4"
|
||||
"github.com/labstack/echo/v5"
|
||||
|
||||
"github.com/gotenberg/gotenberg/v8/pkg/gotenberg"
|
||||
"github.com/gotenberg/gotenberg/v8/pkg/modules/api"
|
||||
@@ -95,7 +95,7 @@ func webhookMiddleware(w *Webhook) api.Middleware {
|
||||
})
|
||||
}
|
||||
|
||||
return func(c echo.Context) error {
|
||||
return func(c *echo.Context) error {
|
||||
webhookUrl := c.Request().Header.Get("Gotenberg-Webhook-Url")
|
||||
if webhookUrl == "" {
|
||||
// No webhook URL, call the next middleware in the chain.
|
||||
@@ -208,7 +208,7 @@ func webhookMiddleware(w *Webhook) api.Middleware {
|
||||
}
|
||||
}
|
||||
|
||||
// Retrieve values from echo.Context before it gets recycled.
|
||||
// Retrieve values from [echo.Context] before it gets recycled.
|
||||
// See https://github.com/gotenberg/gotenberg/issues/1000.
|
||||
startTime := c.Get("startTime").(time.Time)
|
||||
correlationIdHeader := c.Get("correlationIdHeader").(string)
|
||||
@@ -222,6 +222,7 @@ func webhookMiddleware(w *Webhook) api.Middleware {
|
||||
eventsUrl: webhookEventsUrl,
|
||||
extraHttpHeaders: extraHttpHeaders,
|
||||
startTime: startTime,
|
||||
deliveryTimeout: w.deliveryTimeout(),
|
||||
|
||||
client: &retryablehttp.Client{
|
||||
HTTPClient: gotenberg.NewOutboundHttpClient(w.clientTimeout, w.allowList, w.denyList, w.enableEnvironmentProxy, ipOpts...),
|
||||
@@ -230,7 +231,10 @@ func webhookMiddleware(w *Webhook) api.Middleware {
|
||||
RetryWaitMax: w.retryMaxWait,
|
||||
Logger: gotenberg.NewLeveledLogger(ctx.Log()),
|
||||
CheckRetry: retryablehttp.DefaultRetryPolicy,
|
||||
Backoff: retryablehttp.DefaultBackoff,
|
||||
// Not DefaultBackoff: it returns a remote Retry-After
|
||||
// verbatim, ignoring --webhook-retry-max-wait (env
|
||||
// WEBHOOK_RETRY_MAX_WAIT).
|
||||
Backoff: gotenberg.ClampedBackoff,
|
||||
},
|
||||
logger: ctx.Log(),
|
||||
}
|
||||
@@ -319,16 +323,16 @@ func webhookMiddleware(w *Webhook) api.Middleware {
|
||||
// As a webhook URL has been given, we handle the request in a
|
||||
// goroutine and return immediately.
|
||||
//
|
||||
// Echo returns the echo.Context back to its sync.Pool as
|
||||
// Echo returns the [echo.Context] back to its sync.Pool as
|
||||
// soon as this synchronous handler returns ErrAsyncProcess.
|
||||
// A concurrent request can then claim the recycled context
|
||||
// and c.Reset() wipes the shared store, which would cause
|
||||
// any c.Get("...").(T) assertion downstream of the webhook
|
||||
// goroutine to panic on a nil value and crash the process.
|
||||
// Snapshot the keys downstream reads onto a detached
|
||||
// wrapper before spawning the goroutine so pool reuse
|
||||
// context before spawning the goroutine so pool reuse
|
||||
// cannot reach into our async work.
|
||||
detached := newPoolSafeContext(c, "logger", "context", "correlationId", "correlationIdHeader", "startTime")
|
||||
detached := newDetachedContext(c, "logger", "context", "correlationId", "correlationIdHeader", "startTime")
|
||||
|
||||
w.asyncCount.Add(1)
|
||||
go func() {
|
||||
|
||||
@@ -1,58 +1,34 @@
|
||||
package webhook
|
||||
|
||||
import (
|
||||
"sync"
|
||||
|
||||
"github.com/labstack/echo/v4"
|
||||
"github.com/labstack/echo/v5"
|
||||
)
|
||||
|
||||
// poolSafeContext wraps an [echo.Context] and keeps a private snapshot of
|
||||
// the values that downstream middleware and route handlers read from the
|
||||
// store. Echo returns an [echo.Context] to its sync.Pool as soon as the
|
||||
// synchronous handler returns, including when the webhook middleware
|
||||
// returns [api.ErrAsyncProcess]. A concurrent request can then claim the
|
||||
// recycled context and c.Reset() wipes the shared store out from under
|
||||
// the webhook goroutine, which causes any
|
||||
// `c.Get("logger").(*slog.Logger)`-style assertion further down the
|
||||
// chain to panic on a nil value.
|
||||
// newDetachedContext returns an [echo.Context] carrying a snapshot of the given
|
||||
// keys, detached from Echo's context pool.
|
||||
//
|
||||
// Wrapping c before handing it to the goroutine insulates the async work
|
||||
// from pool reuse: Get/Set read and write the private store while every
|
||||
// other [echo.Context] method delegates to the embedded context for
|
||||
// anything the downstream might still need.
|
||||
type poolSafeContext struct {
|
||||
echo.Context
|
||||
mu sync.RWMutex
|
||||
store map[string]any
|
||||
}
|
||||
|
||||
// newPoolSafeContext snapshots the given keys from c into a detached
|
||||
// store and returns a wrapper whose Get/Set operate on that store
|
||||
// exclusively. Keys absent from c are omitted; the wrapper still
|
||||
// Echo returns an [echo.Context] to its sync.Pool as soon as the synchronous
|
||||
// handler returns, including when the webhook middleware returns
|
||||
// [api.ErrAsyncProcess]. A concurrent request can then claim the recycled
|
||||
// context and c.Reset() wipes the shared store out from under the webhook
|
||||
// goroutine, which causes any `c.Get("logger").(*slog.Logger)`-style assertion
|
||||
// further down the chain to panic on a nil value.
|
||||
//
|
||||
// [echo.NewContext] allocates outside the pool, so recycling cannot reach the
|
||||
// returned context. Keys absent from c are omitted; the returned context still
|
||||
// returns nil for them, matching [echo.Context.Get] behavior.
|
||||
func newPoolSafeContext(c echo.Context, keys ...string) *poolSafeContext {
|
||||
store := make(map[string]any, len(keys))
|
||||
//
|
||||
// Only the asynchronous path uses this. Nothing downstream of the webhook
|
||||
// middleware writes to the response: contextMiddleware sits upstream and has
|
||||
// already answered 204 by the time the goroutine runs.
|
||||
func newDetachedContext(c *echo.Context, keys ...string) *echo.Context {
|
||||
detached := echo.NewContext(c.Request(), c.Response(), c.Echo())
|
||||
|
||||
for _, key := range keys {
|
||||
if v := c.Get(key); v != nil {
|
||||
store[key] = v
|
||||
detached.Set(key, v)
|
||||
}
|
||||
}
|
||||
return &poolSafeContext{Context: c, store: store}
|
||||
}
|
||||
|
||||
// Get returns the value stored in the detached store, not the embedded
|
||||
// context's pooled store.
|
||||
func (p *poolSafeContext) Get(key string) any {
|
||||
p.mu.RLock()
|
||||
defer p.mu.RUnlock()
|
||||
return p.store[key]
|
||||
}
|
||||
|
||||
// Set writes to the detached store, not the embedded context's pooled
|
||||
// store. This prevents downstream middleware writes from leaking into a
|
||||
// later request that claims the same pooled context.
|
||||
func (p *poolSafeContext) Set(key string, val any) {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
p.store[key] = val
|
||||
return detached
|
||||
}
|
||||
|
||||
@@ -6,10 +6,10 @@ import (
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/labstack/echo/v4"
|
||||
"github.com/labstack/echo/v5"
|
||||
)
|
||||
|
||||
func TestPoolSafeContext_SurvivesUnderlyingReset(t *testing.T) {
|
||||
func TestNewDetachedContext_SurvivesUnderlyingReset(t *testing.T) {
|
||||
e := echo.New()
|
||||
req := httptest.NewRequest(http.MethodPost, "/", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
@@ -19,10 +19,10 @@ func TestPoolSafeContext_SurvivesUnderlyingReset(t *testing.T) {
|
||||
c.Set("logger", logger)
|
||||
c.Set("correlationId", "abc-123")
|
||||
|
||||
detached := newPoolSafeContext(c, "logger", "correlationId", "missing")
|
||||
detached := newDetachedContext(c, "logger", "correlationId", "missing")
|
||||
|
||||
// Simulate Echo recycling c for a concurrent request. Reset wipes the
|
||||
// shared store, which is exactly the crash scenario the wrapper
|
||||
// Simulate Echo recycling c for a concurrent request. Reset clears the
|
||||
// pooled store, which is exactly the crash scenario the detached context
|
||||
// guards against.
|
||||
c.Reset(httptest.NewRequest(http.MethodGet, "/", nil), httptest.NewRecorder())
|
||||
|
||||
@@ -38,17 +38,17 @@ func TestPoolSafeContext_SurvivesUnderlyingReset(t *testing.T) {
|
||||
|
||||
// Underlying c must remain clean.
|
||||
if c.Get("logger") != nil {
|
||||
t.Fatalf("underlying c.Get(\"logger\") leaked wrapper state after reset")
|
||||
t.Fatalf("underlying c.Get(\"logger\") leaked detached state after reset")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPoolSafeContext_SetDoesNotTouchUnderlying(t *testing.T) {
|
||||
func TestNewDetachedContext_SetDoesNotTouchUnderlying(t *testing.T) {
|
||||
e := echo.New()
|
||||
req := httptest.NewRequest(http.MethodPost, "/", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
c := e.NewContext(req, rec)
|
||||
|
||||
detached := newPoolSafeContext(c)
|
||||
detached := newDetachedContext(c)
|
||||
detached.Set("foo", "bar")
|
||||
|
||||
if got, _ := detached.Get("foo").(string); got != "bar" {
|
||||
|
||||
@@ -5,7 +5,7 @@ import (
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/dlclark/regexp2"
|
||||
"github.com/dlclark/regexp2/v2"
|
||||
flag "github.com/spf13/pflag"
|
||||
|
||||
"github.com/gotenberg/gotenberg/v8/pkg/gotenberg"
|
||||
@@ -42,7 +42,7 @@ func (w *Webhook) Descriptor() gotenberg.ModuleDescriptor {
|
||||
FlagSet: func() *flag.FlagSet {
|
||||
fs := flag.NewFlagSet("webhook", flag.ExitOnError)
|
||||
fs.Bool("webhook-enable-sync-mode", false, "Enable synchronous mode for the webhook feature")
|
||||
fs.StringSlice("webhook-allow-list", []string{}, "Set the allowed URLs for the webhook feature using regular expressions - supports multiple values")
|
||||
fs.StringSlice("webhook-allow-list", []string{}, `Set the allowed URLs for the webhook feature using regular expressions - supports multiple values. A match bypasses --webhook-deny-private-ips (WEBHOOK_DENY_PRIVATE_IPS) and --webhook-deny-public-ips (WEBHOOK_DENY_PUBLIC_IPS), so terminate the host or the pattern also matches suffix hosts, for example ^https?://internal\.svc(:|/|$)`)
|
||||
fs.StringSlice("webhook-deny-list", []string{}, "Set the denied URLs for the webhook feature using regular expressions - supports multiple values")
|
||||
fs.Bool("webhook-deny-private-ips", false, "Reject webhook URLs whose host resolves to a non-public IP address (loopback, RFC1918, link-local, unique-local). Enable on deployments that accept untrusted webhook destinations to mitigate SSRF against internal services")
|
||||
fs.Bool("webhook-deny-public-ips", false, "Reject webhook URLs whose host resolves to a public IP address. Enable on air-gapped or data-governed deployments to prevent callbacks from leaving a private network")
|
||||
@@ -50,7 +50,7 @@ func (w *Webhook) Descriptor() gotenberg.ModuleDescriptor {
|
||||
fs.Int("webhook-max-retry", 4, "Set the maximum number of retries for the webhook feature")
|
||||
|
||||
// Deprecated flags.
|
||||
fs.StringSlice("webhook-error-allow-list", []string{}, "Set the allowed URLs in case of an error for the webhook feature using regular expressions - supports multiple values")
|
||||
fs.StringSlice("webhook-error-allow-list", []string{}, `Set the allowed URLs in case of an error for the webhook feature using regular expressions - supports multiple values. A match bypasses --webhook-deny-private-ips (WEBHOOK_DENY_PRIVATE_IPS) and --webhook-deny-public-ips (WEBHOOK_DENY_PUBLIC_IPS), so terminate the host or the pattern also matches suffix hosts, for example ^https?://internal\.svc(:|/|$)`)
|
||||
fs.StringSlice("webhook-error-deny-list", []string{}, "Set the denied URLs in case of an error for the webhook feature using regular expressions - supports multiple values")
|
||||
err := fs.MarkDeprecated("webhook-error-allow-list", "use --webhook-allow-list instead")
|
||||
if err != nil {
|
||||
@@ -92,6 +92,27 @@ func (w *Webhook) Provision(ctx *gotenberg.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// minDeliveryTimeout is the floor for [Webhook.deliveryTimeout], so that a
|
||||
// deliberately tiny --webhook-client-timeout (env WEBHOOK_CLIENT_TIMEOUT) never
|
||||
// leaves a delivery with no budget at all.
|
||||
const minDeliveryTimeout = 1 * time.Second
|
||||
|
||||
// deliveryTimeout bounds one webhook delivery including its retries. It is the
|
||||
// worst case a correctly behaving remote produces: one client timeout per
|
||||
// attempt, plus the capped wait between attempts.
|
||||
//
|
||||
// A delivery runs after the handler returned, so it cannot borrow the
|
||||
// conversion deadline. Without this budget it would have none, because
|
||||
// [retryablehttp] builds its requests on [context.Background].
|
||||
func (w *Webhook) deliveryTimeout() time.Duration {
|
||||
timeout := w.clientTimeout*time.Duration(w.maxRetry+1) + w.retryMaxWait*time.Duration(w.maxRetry)
|
||||
if timeout < minDeliveryTimeout {
|
||||
return minDeliveryTimeout
|
||||
}
|
||||
|
||||
return timeout
|
||||
}
|
||||
|
||||
// Middlewares returns the middleware.
|
||||
func (w *Webhook) Middlewares() ([]api.Middleware, error) {
|
||||
if w.disable {
|
||||
|
||||
95
pkg/modules/webhook/webhook_test.go
Normal file
95
pkg/modules/webhook/webhook_test.go
Normal file
@@ -0,0 +1,95 @@
|
||||
package webhook
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestWebhook_deliveryTimeout(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
scenario string
|
||||
clientTimeout time.Duration
|
||||
maxRetry int
|
||||
retryMaxWait time.Duration
|
||||
want time.Duration
|
||||
}{
|
||||
{
|
||||
scenario: "shipped defaults",
|
||||
clientTimeout: 30 * time.Second,
|
||||
maxRetry: 4,
|
||||
retryMaxWait: 30 * time.Second,
|
||||
want: 270 * time.Second,
|
||||
},
|
||||
{
|
||||
scenario: "no retry is one client timeout",
|
||||
clientTimeout: 30 * time.Second,
|
||||
maxRetry: 0,
|
||||
retryMaxWait: 30 * time.Second,
|
||||
want: 30 * time.Second,
|
||||
},
|
||||
{
|
||||
scenario: "a zero client timeout still gets a budget",
|
||||
clientTimeout: 0,
|
||||
maxRetry: 0,
|
||||
retryMaxWait: 0,
|
||||
want: minDeliveryTimeout,
|
||||
},
|
||||
} {
|
||||
t.Run(tc.scenario, func(t *testing.T) {
|
||||
w := &Webhook{
|
||||
clientTimeout: tc.clientTimeout,
|
||||
maxRetry: tc.maxRetry,
|
||||
retryMaxWait: tc.retryMaxWait,
|
||||
}
|
||||
if got := w.deliveryTimeout(); got != tc.want {
|
||||
t.Fatalf("deliveryTimeout() = %s, want %s", got, tc.want)
|
||||
}
|
||||
if w.deliveryTimeout() <= 0 {
|
||||
t.Fatal("deliveryTimeout() must always be positive")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A delivery must not inherit the conversion deadline. It runs after the
|
||||
// handler returned, so that deadline is often already spent, which would fail
|
||||
// the callback without a single attempt.
|
||||
func TestClient_deliveryContext_IgnoresAnExpiredParentDeadline(t *testing.T) {
|
||||
expired, cancelExpired := context.WithTimeout(context.Background(), -1*time.Second)
|
||||
defer cancelExpired()
|
||||
|
||||
if expired.Err() == nil {
|
||||
t.Fatal("expected the parent context to be expired")
|
||||
}
|
||||
|
||||
c := client{deliveryTimeout: 30 * time.Second}
|
||||
ctx, cancel := c.deliveryContext(expired)
|
||||
defer cancel()
|
||||
|
||||
if ctx.Err() != nil {
|
||||
t.Fatalf("delivery context inherited the expired parent: %v", ctx.Err())
|
||||
}
|
||||
|
||||
deadline, ok := ctx.Deadline()
|
||||
if !ok {
|
||||
t.Fatal("delivery context has no deadline, so a delivery would be unbounded")
|
||||
}
|
||||
if remaining := time.Until(deadline); remaining <= 0 {
|
||||
t.Fatalf("delivery budget = %s, want a positive value", remaining)
|
||||
}
|
||||
}
|
||||
|
||||
// The delivery context must still be bounded, so a hostile remote cannot hold
|
||||
// the goroutine and its output file open indefinitely.
|
||||
func TestClient_deliveryContext_IsBounded(t *testing.T) {
|
||||
c := client{deliveryTimeout: 50 * time.Millisecond}
|
||||
ctx, cancel := c.deliveryContext(context.Background())
|
||||
defer cancel()
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("delivery context never expired")
|
||||
}
|
||||
}
|
||||
@@ -589,7 +589,7 @@ Feature: /forms/chromium/convert/url
|
||||
@chromium-ssrf
|
||||
Scenario: POST /forms/chromium/convert/url (Redirect to a non-allow-listed address is re-filtered)
|
||||
Given I have a Gotenberg container with the following environment variable(s):
|
||||
| CHROMIUM_ALLOW_LIST | ^https?://host.docker.internal.* |
|
||||
| CHROMIUM_ALLOW_LIST | ^https?://host\\.docker\\.internal(:[0-9]+)?/ |
|
||||
Given I have a static server
|
||||
When I make a "POST" request to Gotenberg at the "/forms/chromium/convert/url" endpoint with the following form data and header(s):
|
||||
| url | http://host.docker.internal:%d/redirect-to-private | field |
|
||||
@@ -1401,3 +1401,19 @@ Feature: /forms/chromium/convert/url
|
||||
Then the response header "Content-Type" should be "application/pdf"
|
||||
Then there should be 1 PDF(s) in the response
|
||||
Then the "foo.pdf" PDF should have 1 page(s)
|
||||
|
||||
# chrome://crash makes the renderer crash deterministically, the same
|
||||
# failure class as a renderer crash triggered by the page content. The
|
||||
# request must fail fast with a 503 instead of hanging until the API
|
||||
# timeout.
|
||||
# See https://github.com/gotenberg/gotenberg/issues/1640.
|
||||
Scenario: POST /forms/chromium/convert/url (Chromium crash fails fast with 503)
|
||||
Given I have a default Gotenberg container
|
||||
When I make a "POST" request to Gotenberg at the "/forms/chromium/convert/url" endpoint with the following form data and header(s):
|
||||
| url | chrome://crash | field |
|
||||
Then the response status code should be 503
|
||||
Then the response header "Content-Type" should be "text/plain; charset=UTF-8"
|
||||
Then the response body should contain string:
|
||||
"""
|
||||
Chromium crashed while processing the request
|
||||
"""
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
# TODO:
|
||||
# 1. Check if down for each module.
|
||||
# 2. Restarting modules do not make health check fail.
|
||||
|
||||
@health
|
||||
Feature: /health
|
||||
@@ -106,7 +105,19 @@ Feature: /health
|
||||
When I make a "HEAD" request to Gotenberg at the "/foo/health" endpoint
|
||||
Then the response status code should be 200
|
||||
|
||||
# A planned restart, the eager cycle after LIBREOFFICE_RESTART_AFTER
|
||||
# conversions, must not fail the health check: requests arriving during it
|
||||
# are requeued, not rejected. Setting the limit to 1 restarts LibreOffice
|
||||
# after every conversion, so each probe lands right on a restart.
|
||||
# See https://github.com/gotenberg/gotenberg/issues/1648.
|
||||
Scenario: GET /health (Planned LibreOffice Restart)
|
||||
Given I have a Gotenberg container with the following environment variable(s):
|
||||
| LIBREOFFICE_RESTART_AFTER | 1 |
|
||||
When I make 5 sequential "POST" requests to Gotenberg at the "/forms/libreoffice/convert" endpoint, probing "/health" after each, with the following form data and header(s):
|
||||
| files | testdata/page_1.docx | file |
|
||||
Then all sequential response status codes should be 200
|
||||
Then all probe response status codes should be 200
|
||||
|
||||
|
||||
# TODO:
|
||||
# 1. Check if down for each module.
|
||||
# 2. Restarting modules do not make health check fail.
|
||||
# 1. Check if down for each module.
|
||||
@@ -11,7 +11,6 @@ import (
|
||||
"github.com/moby/moby/client"
|
||||
"github.com/testcontainers/testcontainers-go"
|
||||
"github.com/testcontainers/testcontainers-go/exec"
|
||||
"github.com/testcontainers/testcontainers-go/network"
|
||||
"github.com/testcontainers/testcontainers-go/wait"
|
||||
)
|
||||
|
||||
@@ -21,11 +20,13 @@ import (
|
||||
const testcontainersLabel = "org.testcontainers"
|
||||
|
||||
// PruneOrphanedNetworks removes dangling networks created by the test suite.
|
||||
// Each scenario spins a dedicated network, and a failed container start can
|
||||
// leak one before teardown records it. Leaked networks consume Docker's
|
||||
// predefined address pools until none remain and every later scenario fails
|
||||
// with "all predefined address pools have been fully subnetted". Call this
|
||||
// before a run and between retries to reclaim the subnets.
|
||||
// Scenarios no longer create one: the Gotenberg container is reached over its
|
||||
// mapped port and the host-side helper over host.docker.internal, so the
|
||||
// default bridge suffices. This stays as cheap insurance against networks
|
||||
// leaked by an older suite version or an interrupted run, which consume
|
||||
// Docker's predefined address pools until none remain and every later
|
||||
// scenario fails with "all predefined address pools have been fully
|
||||
// subnetted".
|
||||
//
|
||||
// Only unused networks bearing the testcontainers label are removed, so
|
||||
// running containers and operator networks are never affected.
|
||||
@@ -99,17 +100,18 @@ func applyDefaultEnv(env map[string]string) map[string]string {
|
||||
return env
|
||||
}
|
||||
|
||||
func startGotenbergContainer(ctx context.Context, env map[string]string) (*testcontainers.DockerNetwork, testcontainers.Container, error) {
|
||||
// startGotenbergContainer starts a Gotenberg container on Docker's default
|
||||
// bridge. No dedicated network is created: the suite addresses the container
|
||||
// through container.Host plus its mapped port, and the container reaches the
|
||||
// host-side webhook and static file server through the host.docker.internal
|
||||
// alias below, so a per-scenario network would carry no traffic while still
|
||||
// consuming one of Docker's predefined subnets.
|
||||
func startGotenbergContainer(ctx context.Context, env map[string]string) (testcontainers.Container, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, 2*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
env = applyDefaultEnv(env)
|
||||
|
||||
n, err := network.New(ctx)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("create Gotenberg container network: %w", err)
|
||||
}
|
||||
|
||||
healthPath := "/health"
|
||||
if env["API_ROOT_PATH"] != "" {
|
||||
healthPath = fmt.Sprintf("%shealth", env["API_ROOT_PATH"])
|
||||
@@ -122,7 +124,6 @@ func startGotenbergContainer(ctx context.Context, env map[string]string) (*testc
|
||||
HostConfigModifier: func(hostConfig *container.HostConfig) {
|
||||
hostConfig.ExtraHosts = []string{"host.docker.internal:host-gateway"}
|
||||
},
|
||||
Networks: []string{n.Name},
|
||||
WaitingFor: wait.ForHTTP(healthPath),
|
||||
Env: env,
|
||||
}
|
||||
@@ -148,19 +149,10 @@ func startGotenbergContainer(ctx context.Context, env map[string]string) (*testc
|
||||
}
|
||||
}
|
||||
|
||||
// The network is already created. The scenario teardown only
|
||||
// removes networks it knows about, and the caller discards n on
|
||||
// error, so remove it here to avoid leaking a subnet on every
|
||||
// failed start. Leaked networks accumulate until Docker's address
|
||||
// pools are fully subnetted and all later scenarios fail.
|
||||
if errRemove := n.Remove(ctx); errRemove != nil {
|
||||
err = fmt.Errorf("%w (also failed to remove network: %v)", err, errRemove)
|
||||
}
|
||||
|
||||
return nil, nil, err
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return n, c, nil
|
||||
return c, nil
|
||||
}
|
||||
|
||||
func execCommandInIntegrationToolsContainer(ctx context.Context, cmd []string, path string) (string, error) {
|
||||
|
||||
@@ -86,19 +86,22 @@ func findScenarioLine(filePath, name string) int {
|
||||
}
|
||||
|
||||
type scenario struct {
|
||||
resp *httptest.ResponseRecorder
|
||||
concurrentResps []*httptest.ResponseRecorder
|
||||
workdir string
|
||||
teststoreDir string
|
||||
gotenbergContainer testcontainers.Container
|
||||
gotenbergContainerNetwork *testcontainers.DockerNetwork
|
||||
server *server
|
||||
hostPort int
|
||||
resp *httptest.ResponseRecorder
|
||||
concurrentResps []*httptest.ResponseRecorder
|
||||
probeResps []*httptest.ResponseRecorder
|
||||
sequentialResps []*httptest.ResponseRecorder
|
||||
workdir string
|
||||
teststoreDir string
|
||||
gotenbergContainer testcontainers.Container
|
||||
server *server
|
||||
hostPort int
|
||||
}
|
||||
|
||||
func (s *scenario) reset(ctx context.Context) error {
|
||||
s.resp = httptest.NewRecorder()
|
||||
s.concurrentResps = nil
|
||||
s.probeResps = nil
|
||||
s.sequentialResps = nil
|
||||
|
||||
err := os.RemoveAll(s.workdir)
|
||||
if err != nil {
|
||||
@@ -119,11 +122,10 @@ func (s *scenario) reset(ctx context.Context) error {
|
||||
}
|
||||
|
||||
func (s *scenario) iHaveADefaultGotenbergContainer(ctx context.Context) error {
|
||||
n, c, err := startGotenbergContainer(ctx, nil)
|
||||
c, err := startGotenbergContainer(ctx, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create Gotenberg container: %s", err)
|
||||
}
|
||||
s.gotenbergContainerNetwork = n
|
||||
s.gotenbergContainer = c
|
||||
return nil
|
||||
}
|
||||
@@ -133,11 +135,10 @@ func (s *scenario) iHaveAGotenbergContainerWithTheFollowingEnvironmentVariables(
|
||||
for _, row := range envTable.Rows {
|
||||
env[row.Cells[0].Value] = row.Cells[1].Value
|
||||
}
|
||||
n, c, err := startGotenbergContainer(ctx, env)
|
||||
c, err := startGotenbergContainer(ctx, env)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create Gotenberg container: %s", err)
|
||||
}
|
||||
s.gotenbergContainerNetwork = n
|
||||
s.gotenbergContainer = c
|
||||
return nil
|
||||
}
|
||||
@@ -470,6 +471,130 @@ func (s *scenario) iMakeConcurrentRequestsToGotenberg(ctx context.Context, count
|
||||
return nil
|
||||
}
|
||||
|
||||
// iMakeSequentialRequestsToGotenbergProbing mirrors the client loop from
|
||||
// https://github.com/gotenberg/gotenberg/issues/1648: a conversion, then a
|
||||
// probe, repeated. It records both the conversion and the probe responses so a
|
||||
// scenario can assert that a planned process restart neither fails the probe
|
||||
// nor breaks the conversions. Requests are sequential on purpose, since the bug
|
||||
// only surfaces between two conversions.
|
||||
func (s *scenario) iMakeSequentialRequestsToGotenbergProbing(ctx context.Context, count int, method, endpoint, probeEndpoint string, dataTable *godog.Table) error {
|
||||
if s.gotenbergContainer == nil {
|
||||
return errors.New("no Gotenberg container")
|
||||
}
|
||||
|
||||
fields := make(map[string][]string)
|
||||
files := make(map[string][]string)
|
||||
headers := make(map[string]string)
|
||||
|
||||
for _, row := range dataTable.Rows {
|
||||
name := row.Cells[0].Value
|
||||
value := row.Cells[1].Value
|
||||
kind := row.Cells[2].Value
|
||||
|
||||
switch kind {
|
||||
case "field":
|
||||
fields[name] = append(fields[name], value)
|
||||
case "file":
|
||||
wd, err := os.Getwd()
|
||||
if err != nil {
|
||||
return fmt.Errorf("get current directory: %w", err)
|
||||
}
|
||||
value = fmt.Sprintf("%s/%s", wd, value)
|
||||
files[name] = append(files[name], value)
|
||||
case "header":
|
||||
headers[name] = value
|
||||
default:
|
||||
return fmt.Errorf("unexpected %q %q", kind, value)
|
||||
}
|
||||
}
|
||||
|
||||
base, err := containerHttpEndpoint(ctx, s.gotenbergContainer, "3000")
|
||||
if err != nil {
|
||||
return fmt.Errorf("get container HTTP endpoint: %w", err)
|
||||
}
|
||||
|
||||
record := func(resp *http.Response) (*httptest.ResponseRecorder, error) {
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, readErr := io.ReadAll(resp.Body)
|
||||
if readErr != nil {
|
||||
return nil, fmt.Errorf("read response body: %w", readErr)
|
||||
}
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
rec.Code = resp.StatusCode
|
||||
for key, values := range resp.Header {
|
||||
for _, v := range values {
|
||||
rec.Header().Add(key, v)
|
||||
}
|
||||
}
|
||||
_, writeErr := rec.Body.Write(body)
|
||||
if writeErr != nil {
|
||||
return nil, fmt.Errorf("write response body: %w", writeErr)
|
||||
}
|
||||
|
||||
return rec, nil
|
||||
}
|
||||
|
||||
s.probeResps = make([]*httptest.ResponseRecorder, 0, count)
|
||||
s.sequentialResps = make([]*httptest.ResponseRecorder, 0, count)
|
||||
|
||||
for i := range count {
|
||||
resp, reqErr := doFormDataRequest(method, fmt.Sprintf("%s%s", base, endpoint), fields, files, headers)
|
||||
if reqErr != nil {
|
||||
return fmt.Errorf("request %d: do request: %w", i+1, reqErr)
|
||||
}
|
||||
|
||||
rec, recErr := record(resp)
|
||||
if recErr != nil {
|
||||
return fmt.Errorf("request %d: %w", i+1, recErr)
|
||||
}
|
||||
s.resp = rec
|
||||
s.sequentialResps = append(s.sequentialResps, rec)
|
||||
|
||||
probeResp, probeErr := doRequest(http.MethodGet, fmt.Sprintf("%s%s", base, probeEndpoint), nil, nil)
|
||||
if probeErr != nil {
|
||||
return fmt.Errorf("probe %d: do request: %w", i+1, probeErr)
|
||||
}
|
||||
|
||||
probeRec, probeRecErr := record(probeResp)
|
||||
if probeRecErr != nil {
|
||||
return fmt.Errorf("probe %d: %w", i+1, probeRecErr)
|
||||
}
|
||||
s.probeResps = append(s.probeResps, probeRec)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *scenario) allProbeResponseStatusCodesShouldBe(expected int) error {
|
||||
if len(s.probeResps) == 0 {
|
||||
return errors.New("no probe responses recorded")
|
||||
}
|
||||
|
||||
for i, resp := range s.probeResps {
|
||||
if resp.Code != expected {
|
||||
return fmt.Errorf("probe %d: expected status %d, got %d %q", i+1, expected, resp.Code, resp.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *scenario) allSequentialResponseStatusCodesShouldBe(expected int) error {
|
||||
if len(s.sequentialResps) == 0 {
|
||||
return errors.New("no sequential responses recorded")
|
||||
}
|
||||
|
||||
for i, resp := range s.sequentialResps {
|
||||
if resp.Code != expected {
|
||||
return fmt.Errorf("sequential response %d: expected status %d, got %d %q", i+1, expected, resp.Code, resp.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *scenario) allConcurrentResponseStatusCodesShouldBe(expected int) error {
|
||||
if len(s.concurrentResps) == 0 {
|
||||
return errors.New("no concurrent responses recorded")
|
||||
@@ -1644,10 +1769,13 @@ func InitializeScenario(ctx *godog.ScenarioContext) {
|
||||
ctx.When(`^I make a "(GET|HEAD)" request to Gotenberg at the "([^"]*)" endpoint with the following header\(s\):$`, s.iMakeARequestToGotenbergWithTheFollowingHeaders)
|
||||
ctx.When(`^I make a "(POST)" request to Gotenberg at the "([^"]*)" endpoint with the following form data and header\(s\):$`, s.iMakeARequestToGotenbergWithTheFollowingFormDataAndHeaders)
|
||||
ctx.When(`^I make (\d+) concurrent "(POST)" requests to Gotenberg at the "([^"]*)" endpoint with the following form data and header\(s\):$`, s.iMakeConcurrentRequestsToGotenberg)
|
||||
ctx.When(`^I make (\d+) sequential "(POST)" requests to Gotenberg at the "([^"]*)" endpoint, probing "([^"]*)" after each, with the following form data and header\(s\):$`, s.iMakeSequentialRequestsToGotenbergProbing)
|
||||
ctx.When(`^I wait for the asynchronous request to the webhook$`, s.iWaitForTheAsynchronousRequestToWebhook)
|
||||
ctx.Then(`^the Gotenberg container (should|should NOT) log the following entries:$`, s.theGotenbergContainerShouldLogTheFollowingEntries)
|
||||
ctx.Then(`^the response status code should be (\d+)$`, s.theResponseStatusCodeShouldBe)
|
||||
ctx.Then(`^all concurrent response status codes should be (\d+)$`, s.allConcurrentResponseStatusCodesShouldBe)
|
||||
ctx.Then(`^all probe response status codes should be (\d+)$`, s.allProbeResponseStatusCodesShouldBe)
|
||||
ctx.Then(`^all sequential response status codes should be (\d+)$`, s.allSequentialResponseStatusCodesShouldBe)
|
||||
ctx.Then(`^all concurrent responses should have (\d+) PDF\(s\)$`, s.allConcurrentResponsesShouldHavePdfs)
|
||||
ctx.Then(`^the (response|webhook request|file request|server request) header "([^"]*)" should be "([^"]*)"$`, s.theHeaderValueShouldBe)
|
||||
ctx.Then(`^the webhook request header "([^"]*)" should carry trace id "([^"]*)"$`, s.theWebhookRequestHeaderShouldCarryTraceID)
|
||||
@@ -1681,12 +1809,6 @@ func InitializeScenario(ctx *godog.ScenarioContext) {
|
||||
return ctx, fmt.Errorf("terminate Gotenberg container: %w", errTerminate)
|
||||
}
|
||||
}
|
||||
if s.gotenbergContainerNetwork != nil {
|
||||
errRemove := s.gotenbergContainerNetwork.Remove(ctx)
|
||||
if errRemove != nil {
|
||||
return ctx, fmt.Errorf("remove Gotenberg container network: %w", errRemove)
|
||||
}
|
||||
}
|
||||
return ctx, nil
|
||||
})
|
||||
ctx.After(func(ctx context.Context, sc *godog.Scenario, err error) (context.Context, error) {
|
||||
|
||||
@@ -15,7 +15,7 @@ import (
|
||||
|
||||
"github.com/cucumber/godog"
|
||||
"github.com/google/uuid"
|
||||
"github.com/labstack/echo/v4"
|
||||
"github.com/labstack/echo/v5"
|
||||
"github.com/mholt/archives"
|
||||
)
|
||||
|
||||
@@ -26,12 +26,17 @@ type server struct {
|
||||
errChan chan error
|
||||
eventBody []byte
|
||||
eventMu sync.Mutex
|
||||
|
||||
shutdown context.CancelFunc
|
||||
done chan struct{}
|
||||
}
|
||||
|
||||
func newServer(ctx context.Context, workdir string) (*server, error) {
|
||||
srv := echo.New()
|
||||
srv.HideBanner = true
|
||||
srv.HidePort = true
|
||||
// The static file handlers below serve absolute paths, which Echo v5's
|
||||
// default working-directory filesystem rejects. See newEchoServer in
|
||||
// pkg/modules/api.
|
||||
srv.Filesystem = echo.NewDefaultFS("/")
|
||||
s := &server{
|
||||
srv: srv,
|
||||
errChan: make(chan error, 1),
|
||||
@@ -47,7 +52,7 @@ func newServer(ctx context.Context, workdir string) (*server, error) {
|
||||
return err
|
||||
}
|
||||
|
||||
webhookHandler := func(c echo.Context) error {
|
||||
webhookHandler := func(c *echo.Context) error {
|
||||
s.req = c.Request()
|
||||
|
||||
body, err := io.ReadAll(s.req.Body)
|
||||
@@ -131,7 +136,7 @@ func newServer(ctx context.Context, workdir string) (*server, error) {
|
||||
|
||||
return webhookErr(c.String(http.StatusOK, http.StatusText(http.StatusOK)))
|
||||
}
|
||||
webhookErrorHandler := func(c echo.Context) error {
|
||||
webhookErrorHandler := func(c *echo.Context) error {
|
||||
s.req = c.Request()
|
||||
body, err := io.ReadAll(s.req.Body)
|
||||
if err != nil {
|
||||
@@ -148,7 +153,7 @@ func newServer(ctx context.Context, workdir string) (*server, error) {
|
||||
srv.PATCH("/webhook/error", webhookErrorHandler)
|
||||
srv.PUT("/webhook/error", webhookErrorHandler)
|
||||
|
||||
webhookEventsHandler := func(c echo.Context) error {
|
||||
webhookEventsHandler := func(c *echo.Context) error {
|
||||
body, err := io.ReadAll(c.Request().Body)
|
||||
if err != nil {
|
||||
return c.String(http.StatusInternalServerError, err.Error())
|
||||
@@ -159,7 +164,7 @@ func newServer(ctx context.Context, workdir string) (*server, error) {
|
||||
return c.String(http.StatusOK, http.StatusText(http.StatusOK))
|
||||
}
|
||||
srv.POST("/webhook/events", webhookEventsHandler)
|
||||
srv.GET("/static/:path", func(c echo.Context) error {
|
||||
srv.GET("/static/:path", func(c *echo.Context) error {
|
||||
s.req = c.Request()
|
||||
path := c.Param("path")
|
||||
if strings.Contains(path, "teststore") {
|
||||
@@ -167,7 +172,7 @@ func newServer(ctx context.Context, workdir string) (*server, error) {
|
||||
}
|
||||
return c.Attachment(fmt.Sprintf("%s/%s", wd, path), filepath.Base(path))
|
||||
})
|
||||
srv.GET("/html/:path", func(c echo.Context) error {
|
||||
srv.GET("/html/:path", func(c *echo.Context) error {
|
||||
s.req = c.Request()
|
||||
path := fmt.Sprintf("%s/%s", wd, c.Param("path"))
|
||||
f, err := os.Open(path)
|
||||
@@ -181,7 +186,7 @@ func newServer(ctx context.Context, workdir string) (*server, error) {
|
||||
}
|
||||
return c.HTML(http.StatusOK, string(b))
|
||||
})
|
||||
srv.GET("/redirect-to-private", func(c echo.Context) error {
|
||||
srv.GET("/redirect-to-private", func(c *echo.Context) error {
|
||||
s.req = c.Request()
|
||||
// Redirect the browser to a non-public address so the outbound filter
|
||||
// is exercised on the redirected request rather than on this URL.
|
||||
@@ -206,9 +211,20 @@ func (s *server) start(ctx context.Context) (int, error) {
|
||||
|
||||
port := ln.Addr().(*net.TCPAddr).Port
|
||||
|
||||
startConfig := echo.StartConfig{
|
||||
Listener: ln,
|
||||
HideBanner: true,
|
||||
HidePort: true,
|
||||
}
|
||||
|
||||
serveCtx, cancel := context.WithCancel(context.Background())
|
||||
s.shutdown = cancel
|
||||
s.done = make(chan struct{})
|
||||
|
||||
go func() {
|
||||
s.srv.Listener = ln
|
||||
err = s.srv.Start("")
|
||||
defer close(s.done)
|
||||
|
||||
err := startConfig.Start(serveCtx, s.srv)
|
||||
if err != nil && !errors.Is(err, http.ErrServerClosed) {
|
||||
godog.Log(ctx, err.Error())
|
||||
}
|
||||
@@ -219,5 +235,17 @@ func (s *server) start(ctx context.Context) (int, error) {
|
||||
|
||||
func (s *server) stop(ctx context.Context) error {
|
||||
close(s.errChan)
|
||||
return s.srv.Shutdown(ctx)
|
||||
|
||||
if s.shutdown == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
s.shutdown()
|
||||
|
||||
select {
|
||||
case <-s.done:
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user