Compare commits

...

24 Commits

Author SHA1 Message Date
Julien Neuhart
aa57b17254 chore(deps): update Go dependencies 2024-10-06 09:52:48 +02:00
Julien Neuhart
7df786c5c6 fix(webhook): retrieve values from echo.Context before it get recycled 2024-10-05 14:24:22 +02:00
Julien Neuhart
b315464e98 chore(deps): update Noto Color Emoji to Unicode 16.0 2024-10-05 11:46:00 +02:00
Julien Neuhart
1252ea076b feat(chromium): skipNetworkIdleEvent is now true by default 2024-09-29 18:04:37 +02:00
Julien Neuhart
d970e446a8 chore(deps): update Go dependencies 2024-09-29 17:47:51 +02:00
Julien Neuhart
fe40f3727b feat(api): add --api-body-limit flag that set the body limit for multipart/form-data requests 2024-09-29 17:46:20 +02:00
Julien Neuhart
10a290b065 chore(libreoffice): improve HTTP error messages 2024-09-29 13:19:44 +02:00
Julien Neuhart
0abb11b42a chore(deps): update Go dependencies 2024-09-21 09:52:48 +02:00
Julien Neuhart
1eb6eb729a chore(downloadFrom): add godoc for issue https://github.com/golang/go/issues/69551 2024-09-20 21:06:11 +02:00
Julien Neuhart
e371a4a3c3 fix: add DOCKER_REGISTRY for tests image 2024-09-20 21:06:11 +02:00
Julien Neuhart
f2b6bd3d4b feat(api): add 'downloadForm' form field 2024-09-20 21:06:11 +02:00
Julien Neuhart
d21f87b543 feat(ci): add snapshot release 2024-09-20 21:06:11 +02:00
Julien Neuhart
b8cdee4f45 fix(cmd): failing test 2024-09-20 20:58:17 +02:00
Julien Neuhart
b06d7445d9 feat(libreoffice): add password form field for converting protected document 2024-09-20 20:36:45 +02:00
Julien Neuhart
2aa9cceb32 fix(README): wrong link to thecodingmachine/gotenberg repository 2024-09-18 07:57:22 +02:00
Julien Neuhart
ae2a2a8a07 chore: improve README 2024-09-17 16:47:52 +02:00
Julien Neuhart
286555411b chore(supervisor): add a comment about an unlikely issue 2024-09-17 16:47:41 +02:00
Julien Neuhart
3c42aec180 chore(deps): update Go dependencies 2024-09-13 16:47:31 +02:00
Julien Neuhart
a203ae00c6 fix(cmd): leaking error logs if reader already closed 2024-09-13 16:47:03 +02:00
Julien Neuhart
68784616a4 chore(godoc): removing dots 2024-09-13 16:44:12 +02:00
Julien Neuhart
5cd8c6a5da feat(api): add HEAD method for health route (#963) 2024-09-07 17:31:43 +02:00
Julien Neuhart
a3647fea8a fix(chromium): increase the scope of loading failed errors (#962) 2024-09-07 15:54:01 +02:00
Julien Neuhart
300e73f07d chore(deps): update Go dependencies 2024-09-05 09:23:33 +02:00
Gabriel Vasile
03ced23f05 fix(deps): download go1.23 toolchain not available (#957) 2024-09-05 09:22:59 +02:00
36 changed files with 1137 additions and 269 deletions

View File

@@ -23,4 +23,4 @@ jobs:
- name: Build and push Docker image for release - name: Build and push Docker image for release
run: | run: |
make release GOTENBERG_VERSION=${{ github.event.release.tag_name }} make release GOTENBERG_VERSION=${{ github.event.release.tag_name }}
make release GOTENBERG_VERSION=${{ github.event.release.tag_name }} DOCKER_REPOSITORY=thecodingmachine make release GOTENBERG_VERSION=${{ github.event.release.tag_name }} DOCKER_REGISTRY=thecodingmachine

View File

@@ -28,7 +28,7 @@ jobs:
tests: tests:
needs: needs:
- Lint - lint
name: Tests name: Tests
# TODO: once arm64 actions are available, also run the tests on this architecture. # TODO: once arm64 actions are available, also run the tests on this architecture.
# See: https://github.com/actions/virtual-environments/issues/2552#issuecomment-771478000. # See: https://github.com/actions/virtual-environments/issues/2552#issuecomment-771478000.
@@ -50,11 +50,32 @@ jobs:
token: ${{ secrets.CODECOV_TOKEN }} token: ${{ secrets.CODECOV_TOKEN }}
verbose: true verbose: true
multiarch_build: snapshot_release:
if: github.event_name == 'pull_request'
needs:
- tests
name: Snapshot release
runs-on: ubuntu-latest
steps:
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Checkout source code
uses: actions/checkout@v4
- name: Log in to Docker Hub Container Registry
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Build and push snapshot Docker image (linux/amd64)
run: make release GOTENBERG_VERSION=${{ github.head_ref }} DOCKER_REPOSITORY=snapshot LINUX_AMD64_RELEASE=true
edge_release:
if: github.event_name == 'push' && github.ref == 'refs/heads/main' if: github.event_name == 'push' && github.ref == 'refs/heads/main'
needs: needs:
- Tests - tests
name: Multi-arch build name: Edge release
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- name: Set up QEMU - name: Set up QEMU
@@ -71,4 +92,4 @@ jobs:
- name: Build and push Docker image for main branch - name: Build and push Docker image for main branch
run: | run: |
make release GOTENBERG_VERSION=edge make release GOTENBERG_VERSION=edge
make release GOTENBERG_VERSION=edge DOCKER_REPOSITORY=thecodingmachine make release GOTENBERG_VERSION=edge DOCKER_REGISTRY=thecodingmachine

View File

@@ -6,11 +6,12 @@ help: ## Show the help
it: build build-tests ## Initialize the development environment it: build build-tests ## Initialize the development environment
GOLANG_VERSION=1.23 GOLANG_VERSION=1.23
DOCKER_REGISTRY=gotenberg
DOCKER_REPOSITORY=gotenberg DOCKER_REPOSITORY=gotenberg
GOTENBERG_VERSION=snapshot GOTENBERG_VERSION=snapshot
GOTENBERG_USER_GID=1001 GOTENBERG_USER_GID=1001
GOTENBERG_USER_UID=1001 GOTENBERG_USER_UID=1001
NOTO_COLOR_EMOJI_VERSION=v2.042 # See https://github.com/googlefonts/noto-emoji/releases. NOTO_COLOR_EMOJI_VERSION=v2.047 # See https://github.com/googlefonts/noto-emoji/releases.
PDFTK_VERSION=v3.3.3 # See https://gitlab.com/pdftk-java/pdftk/-/releases - Binary package. PDFTK_VERSION=v3.3.3 # See https://gitlab.com/pdftk-java/pdftk/-/releases - Binary package.
GOLANGCI_LINT_VERSION=v1.60.3 # See https://github.com/golangci/golangci-lint/releases. GOLANGCI_LINT_VERSION=v1.60.3 # See https://github.com/golangci/golangci-lint/releases.
@@ -23,7 +24,7 @@ build: ## Build the Gotenberg's Docker image
--build-arg GOTENBERG_USER_UID=$(GOTENBERG_USER_UID) \ --build-arg GOTENBERG_USER_UID=$(GOTENBERG_USER_UID) \
--build-arg NOTO_COLOR_EMOJI_VERSION=$(NOTO_COLOR_EMOJI_VERSION) \ --build-arg NOTO_COLOR_EMOJI_VERSION=$(NOTO_COLOR_EMOJI_VERSION) \
--build-arg PDFTK_VERSION=$(PDFTK_VERSION) \ --build-arg PDFTK_VERSION=$(PDFTK_VERSION) \
-t $(DOCKER_REPOSITORY)/gotenberg:$(GOTENBERG_VERSION) \ -t $(DOCKER_REGISTRY)/$(DOCKER_REPOSITORY):$(GOTENBERG_VERSION) \
-f build/Dockerfile . -f build/Dockerfile .
GOTENBERG_GRACEFUL_SHUTDOWN_DURATION=30s GOTENBERG_GRACEFUL_SHUTDOWN_DURATION=30s
@@ -31,11 +32,16 @@ API_PORT=3000
API_PORT_FROM_ENV= API_PORT_FROM_ENV=
API_START_TIMEOUT=30s API_START_TIMEOUT=30s
API_TIMEOUT=30s API_TIMEOUT=30s
API_BODY_LIMIT=
API_ROOT_PATH=/ API_ROOT_PATH=/
API_TRACE_HEADER=Gotenberg-Trace API_TRACE_HEADER=Gotenberg-Trace
API_ENABLE_BASIC_AUTH=false API_ENABLE_BASIC_AUTH=false
GOTENBERG_API_BASIC_AUTH_USERNAME= GOTENBERG_API_BASIC_AUTH_USERNAME=
GOTENBERG_API_BASIC_AUTH_PASSWORD= GOTENBERG_API_BASIC_AUTH_PASSWORD=
API-DOWNLOAD-FROM-ALLOW-LIST=
API-DOWNLOAD-FROM-DENY-LIST=
API-DOWNLOAD-FROM-FROM-MAX-RETRY=4
API-DISABLE-DOWNLOAD-FROM=false
API_DISABLE_HEALTH_CHECK_LOGGING=false API_DISABLE_HEALTH_CHECK_LOGGING=false
CHROMIUM_RESTART_AFTER=0 CHROMIUM_RESTART_AFTER=0
CHROMIUM_MAX_QUEUE_SIZE=0 CHROMIUM_MAX_QUEUE_SIZE=0
@@ -84,16 +90,21 @@ run: ## Start a Gotenberg container
-p $(API_PORT):$(API_PORT) \ -p $(API_PORT):$(API_PORT) \
-e GOTENBERG_API_BASIC_AUTH_USERNAME=$(GOTENBERG_API_BASIC_AUTH_USERNAME) \ -e GOTENBERG_API_BASIC_AUTH_USERNAME=$(GOTENBERG_API_BASIC_AUTH_USERNAME) \
-e GOTENBERG_API_BASIC_AUTH_PASSWORD=$(GOTENBERG_API_BASIC_AUTH_PASSWORD) \ -e GOTENBERG_API_BASIC_AUTH_PASSWORD=$(GOTENBERG_API_BASIC_AUTH_PASSWORD) \
$(DOCKER_REPOSITORY)/gotenberg:$(GOTENBERG_VERSION) \ $(DOCKER_REGISTRY)/$(DOCKER_REPOSITORY):$(GOTENBERG_VERSION) \
gotenberg \ gotenberg \
--gotenberg-graceful-shutdown-duration=$(GOTENBERG_GRACEFUL_SHUTDOWN_DURATION) \ --gotenberg-graceful-shutdown-duration=$(GOTENBERG_GRACEFUL_SHUTDOWN_DURATION) \
--api-port=$(API_PORT) \ --api-port=$(API_PORT) \
--api-port-from-env=$(API_PORT_FROM_ENV) \ --api-port-from-env=$(API_PORT_FROM_ENV) \
--api-start-timeout=$(API_START_TIMEOUT) \ --api-start-timeout=$(API_START_TIMEOUT) \
--api-timeout=$(API_TIMEOUT) \ --api-timeout=$(API_TIMEOUT) \
--api-body-limit="$(API_BODY_LIMIT)" \
--api-root-path=$(API_ROOT_PATH) \ --api-root-path=$(API_ROOT_PATH) \
--api-trace-header=$(API_TRACE_HEADER) \ --api-trace-header=$(API_TRACE_HEADER) \
--api-enable-basic-auth=$(API_ENABLE_BASIC_AUTH) \ --api-enable-basic-auth=$(API_ENABLE_BASIC_AUTH) \
--api-download-from-allow-list=$(API-DOWNLOAD-FROM-ALLOW-LIST) \
--api-download-from-deny-list=$(API-DOWNLOAD-FROM-DENY-LIST) \
--api-download-from-max-retry=$(API-DOWNLOAD-FROM-FROM-MAX-RETRY) \
--api-disable-download-from=$(API-DISABLE-DOWNLOAD-FROM) \
--api-disable-health-check-logging=$(API_DISABLE_HEALTH_CHECK_LOGGING) \ --api-disable-health-check-logging=$(API_DISABLE_HEALTH_CHECK_LOGGING) \
--chromium-restart-after=$(CHROMIUM_RESTART_AFTER) \ --chromium-restart-after=$(CHROMIUM_RESTART_AFTER) \
--chromium-auto-start=$(CHROMIUM_AUTO_START) \ --chromium-auto-start=$(CHROMIUM_AUTO_START) \
@@ -140,24 +151,25 @@ run: ## Start a Gotenberg container
build-tests: ## Build the tests' Docker image build-tests: ## Build the tests' Docker image
docker build \ docker build \
--build-arg GOLANG_VERSION=$(GOLANG_VERSION) \ --build-arg GOLANG_VERSION=$(GOLANG_VERSION) \
--build-arg DOCKER_REGISTRY=$(DOCKER_REGISTRY) \
--build-arg DOCKER_REPOSITORY=$(DOCKER_REPOSITORY) \ --build-arg DOCKER_REPOSITORY=$(DOCKER_REPOSITORY) \
--build-arg GOTENBERG_VERSION=$(GOTENBERG_VERSION) \ --build-arg GOTENBERG_VERSION=$(GOTENBERG_VERSION) \
--build-arg GOLANGCI_LINT_VERSION=$(GOLANGCI_LINT_VERSION) \ --build-arg GOLANGCI_LINT_VERSION=$(GOLANGCI_LINT_VERSION) \
-t $(DOCKER_REPOSITORY)/gotenberg:$(GOTENBERG_VERSION)-tests \ -t $(DOCKER_REGISTRY)/$(DOCKER_REPOSITORY):$(GOTENBERG_VERSION)-tests \
-f test/Dockerfile . -f test/Dockerfile .
.PHONY: tests .PHONY: tests
tests: ## Start the testing environment tests: ## Start the testing environment
docker run --rm -it \ docker run --rm -it \
-v $(PWD):/tests \ -v $(PWD):/tests \
$(DOCKER_REPOSITORY)/gotenberg:$(GOTENBERG_VERSION)-tests \ $(DOCKER_REGISTRY)/$(DOCKER_REPOSITORY):$(GOTENBERG_VERSION)-tests \
bash bash
.PHONY: tests-once .PHONY: tests-once
tests-once: ## Run the tests once (prefer the "tests" command while developing) tests-once: ## Run the tests once (prefer the "tests" command while developing)
docker run --rm \ docker run --rm \
-v $(PWD):/tests \ -v $(PWD):/tests \
$(DOCKER_REPOSITORY)/gotenberg:$(GOTENBERG_VERSION)-tests \ $(DOCKER_REGISTRY)/$(DOCKER_REPOSITORY):$(GOTENBERG_VERSION)-tests \
gotest gotest
# go install mvdan.cc/gofumpt@latest # go install mvdan.cc/gofumpt@latest
@@ -174,8 +186,10 @@ godoc: ## Run a webserver with Gotenberg godoc
$(info http://localhost:6060/pkg/github.com/gotenberg/gotenberg/v8) $(info http://localhost:6060/pkg/github.com/gotenberg/gotenberg/v8)
godoc -http=:6060 godoc -http=:6060
LINUX_AMD64_RELEASE=false
.PHONY: release .PHONY: release
release: ## Build the Gotenberg's Docker image for many platforms, then push it to a Docker repository release: ## Build the Gotenberg's Docker image and push it to a Docker repository
./scripts/release.sh \ ./scripts/release.sh \
$(GOLANG_VERSION) \ $(GOLANG_VERSION) \
$(GOTENBERG_VERSION) \ $(GOTENBERG_VERSION) \
@@ -183,4 +197,7 @@ release: ## Build the Gotenberg's Docker image for many platforms, then push it
$(GOTENBERG_USER_UID) \ $(GOTENBERG_USER_UID) \
$(NOTO_COLOR_EMOJI_VERSION) \ $(NOTO_COLOR_EMOJI_VERSION) \
$(PDFTK_VERSION) \ $(PDFTK_VERSION) \
$(DOCKER_REPOSITORY) $(DOCKER_REGISTRY) \
$(DOCKER_REPOSITORY) \
$(LINUX_AMD64_RELEASE)

View File

@@ -2,12 +2,20 @@
<img src="https://user-images.githubusercontent.com/8983173/130322857-185831e2-f041-46eb-a17f-0a69d066c4e5.png" alt="Gotenberg Logo" width="150" height="150" /> <img src="https://user-images.githubusercontent.com/8983173/130322857-185831e2-f041-46eb-a17f-0a69d066c4e5.png" alt="Gotenberg Logo" width="150" height="150" />
<h3 align="center">Gotenberg</h3> <h3 align="center">Gotenberg</h3>
<p align="center">A Docker-powered stateless API for PDF files</p> <p align="center">A Docker-powered stateless API for PDF files</p>
<p align="center">
<a href="https://hub.docker.com/r/gotenberg/gotenberg"><img alt="Total downloads (gotenberg/gotenberg)" src="https://img.shields.io/docker/pulls/gotenberg/gotenberg"></a>
<a href="https://hub.docker.com/r/thecodingmachine/gotenberg"><img alt="Total downloads (thecodingmachine/gotenberg)" src="https://img.shields.io/docker/pulls/thecodingmachine/gotenberg"></a>
<br>
<a href="https://github.com/gotenberg/gotenberg/actions/workflows/continuous_integration.yml"><img alt="Continuous Integration" src="https://github.com/gotenberg/gotenberg/actions/workflows/continuous_integration.yml/badge.svg"></a>
<a href="https://pkg.go.dev/github.com/gotenberg/gotenberg/v8"><img alt="Go Reference" src="https://pkg.go.dev/badge/github.com/gotenberg/gotenberg.svg"></a>
<a href="https://codecov.io/gh/gotenberg/gotenberg/branch/main/graph/badge.svg"><img alt="Code coverage" src="https://codecov.io/gh/gotenberg/gotenberg/branch/main/graph/badge.svg"></a>
</p>
<p align="center"><a href="https://gotenberg.dev/docs/getting-started/introduction">Documentation</a> &#183; <a href="https://gotenberg.dev/docs/getting-started/installation#live-demo-">Live Demo</a> 🔥</p> <p align="center"><a href="https://gotenberg.dev/docs/getting-started/introduction">Documentation</a> &#183; <a href="https://gotenberg.dev/docs/getting-started/installation#live-demo-">Live Demo</a> 🔥</p>
</p> </p>
--- ---
Gotenberg provides a developer-friendly API to interact with powerful tools like Chromium and LibreOffice for converting **Gotenberg** provides a developer-friendly API to interact with powerful tools like Chromium and LibreOffice for converting
numerous document formats (HTML, Markdown, Word, Excel, etc.) into PDF files, and more! numerous document formats (HTML, Markdown, Word, Excel, etc.) into PDF files, and more!
## Quick Start ## Quick Start
@@ -40,11 +48,3 @@ Head to the [documentation](https://gotenberg.dev/docs/getting-started/introduct
</p> </p>
Sponsorships help maintaining and improving Gotenberg - [become a sponsor](https://github.com/sponsors/gulien) ❤️ Sponsorships help maintaining and improving Gotenberg - [become a sponsor](https://github.com/sponsors/gulien) ❤️
## Badges
[![Docker pulls](https://img.shields.io/docker/pulls/gotenberg/gotenberg)](https://hub.docker.com/r/gotenberg/gotenberg)
[![Docker pulls](https://img.shields.io/docker/pulls/thecodingmachine/gotenberg)](https://hub.docker.com/r/thecodingmachine/gotenberg)
[![Continuous Integration](https://github.com/gotenberg/gotenberg/actions/workflows/continuous_integration.yml/badge.svg)](https://github.com/gotenberg/gotenberg/actions/workflows/continuous_integration.yml)
[![Go Reference](https://pkg.go.dev/badge/github.com/gotenberg/gotenberg.svg)](https://pkg.go.dev/github.com/gotenberg/gotenberg/v8)
[![Codecov](https://codecov.io/gh/gotenberg/gotenberg/branch/main/graph/badge.svg)](https://codecov.io/gh/gotenberg/gotenberg)

View File

@@ -1,7 +1,8 @@
ARG DOCKER_REGISTRY
ARG DOCKER_REPOSITORY ARG DOCKER_REPOSITORY
ARG GOTENBERG_VERSION ARG GOTENBERG_VERSION
FROM $DOCKER_REPOSITORY/gotenberg:$GOTENBERG_VERSION FROM $DOCKER_REGISTRY/$DOCKER_REPOSITORY:$GOTENBERG_VERSION
USER root USER root

24
go.mod
View File

@@ -1,18 +1,18 @@
module github.com/gotenberg/gotenberg/v8 module github.com/gotenberg/gotenberg/v8
go 1.23 go 1.23.0
require ( require (
github.com/alexliesenfeld/health v0.8.0 github.com/alexliesenfeld/health v0.8.0
github.com/andybalholm/brotli v1.1.0 // indirect github.com/andybalholm/brotli v1.1.0 // indirect
github.com/barasher/go-exiftool v1.10.0 github.com/barasher/go-exiftool v1.10.0
github.com/chromedp/cdproto v0.0.0-20240810084448-b931b754e476 github.com/chromedp/cdproto v0.0.0-20241003230502-a4a8f7c660df
github.com/chromedp/chromedp v0.10.0 github.com/chromedp/chromedp v0.10.0
github.com/golang/snappy v0.0.4 // indirect github.com/golang/snappy v0.0.4 // indirect
github.com/google/uuid v1.6.0 github.com/google/uuid v1.6.0
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
github.com/hashicorp/go-retryablehttp v0.7.7 github.com/hashicorp/go-retryablehttp v0.7.7
github.com/klauspost/compress v1.17.9 // indirect github.com/klauspost/compress v1.17.10 // indirect
github.com/klauspost/pgzip v1.2.6 // indirect github.com/klauspost/pgzip v1.2.6 // indirect
github.com/labstack/echo/v4 v4.12.0 github.com/labstack/echo/v4 v4.12.0
github.com/labstack/gommon v0.4.2 github.com/labstack/gommon v0.4.2
@@ -22,19 +22,19 @@ require (
github.com/nwaples/rardecode v1.1.3 // indirect github.com/nwaples/rardecode v1.1.3 // indirect
github.com/pdfcpu/pdfcpu v0.8.1 github.com/pdfcpu/pdfcpu v0.8.1
github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/pierrec/lz4/v4 v4.1.21 // indirect
github.com/prometheus/client_golang v1.20.2 github.com/prometheus/client_golang v1.20.4
github.com/russross/blackfriday/v2 v2.1.0 github.com/russross/blackfriday/v2 v2.1.0
github.com/spf13/pflag v1.0.5 github.com/spf13/pflag v1.0.5
github.com/ulikunitz/xz v0.5.12 // indirect github.com/ulikunitz/xz v0.5.12 // indirect
go.uber.org/multierr v1.11.0 go.uber.org/multierr v1.11.0
go.uber.org/zap v1.27.0 go.uber.org/zap v1.27.0
golang.org/x/crypto v0.26.0 // indirect golang.org/x/crypto v0.28.0 // indirect
golang.org/x/image v0.19.0 // indirect golang.org/x/image v0.21.0 // indirect
golang.org/x/net v0.28.0 golang.org/x/net v0.30.0
golang.org/x/sync v0.8.0 golang.org/x/sync v0.8.0
golang.org/x/sys v0.24.0 // indirect golang.org/x/sys v0.26.0 // indirect
golang.org/x/term v0.23.0 golang.org/x/term v0.25.0
golang.org/x/text v0.17.0 golang.org/x/text v0.19.0
) )
require github.com/dlclark/regexp2 v1.11.4 require github.com/dlclark/regexp2 v1.11.4
@@ -59,13 +59,13 @@ require (
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/pkg/errors v0.9.1 // indirect github.com/pkg/errors v0.9.1 // indirect
github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/client_model v0.6.1 // indirect
github.com/prometheus/common v0.57.0 // indirect github.com/prometheus/common v0.60.0 // indirect
github.com/prometheus/procfs v0.15.1 // indirect github.com/prometheus/procfs v0.15.1 // indirect
github.com/rivo/uniseg v0.4.7 // indirect github.com/rivo/uniseg v0.4.7 // indirect
github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect
github.com/valyala/fasttemplate v1.2.2 // indirect github.com/valyala/fasttemplate v1.2.2 // indirect
github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
golang.org/x/time v0.6.0 // indirect golang.org/x/time v0.7.0 // indirect
google.golang.org/protobuf v1.34.2 // indirect google.golang.org/protobuf v1.34.2 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect
) )

44
go.sum
View File

@@ -12,8 +12,8 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/chromedp/cdproto v0.0.0-20240801214329-3f85d328b335/go.mod h1:GKljq0VrfU4D5yc+2qA6OVr8pmO/MBbPEWqWQ/oqGEs= github.com/chromedp/cdproto v0.0.0-20240801214329-3f85d328b335/go.mod h1:GKljq0VrfU4D5yc+2qA6OVr8pmO/MBbPEWqWQ/oqGEs=
github.com/chromedp/cdproto v0.0.0-20240810084448-b931b754e476 h1:VnjHsRXCRti7Av7E+j4DCha3kf68echfDzQ+wD11SBU= github.com/chromedp/cdproto v0.0.0-20241003230502-a4a8f7c660df h1:cbtSn19AtqQha1cxmP2Qvgd3fFMz51AeAEKLJMyEUhc=
github.com/chromedp/cdproto v0.0.0-20240810084448-b931b754e476/go.mod h1:GKljq0VrfU4D5yc+2qA6OVr8pmO/MBbPEWqWQ/oqGEs= github.com/chromedp/cdproto v0.0.0-20241003230502-a4a8f7c660df/go.mod h1:GKljq0VrfU4D5yc+2qA6OVr8pmO/MBbPEWqWQ/oqGEs=
github.com/chromedp/chromedp v0.10.0 h1:bRclRYVpMm/UVD76+1HcRW9eV3l58rFfy7AdBvKab1E= github.com/chromedp/chromedp v0.10.0 h1:bRclRYVpMm/UVD76+1HcRW9eV3l58rFfy7AdBvKab1E=
github.com/chromedp/chromedp v0.10.0/go.mod h1:ei/1ncZIqXX1YnAYDkxhD4gzBgavMEUu7JCKvztdomE= github.com/chromedp/chromedp v0.10.0/go.mod h1:ei/1ncZIqXX1YnAYDkxhD4gzBgavMEUu7JCKvztdomE=
github.com/chromedp/sysutil v1.0.0 h1:+ZxhTpfpZlmchB58ih/LBHX52ky7w2VhQVKQMucy3Ic= github.com/chromedp/sysutil v1.0.0 h1:+ZxhTpfpZlmchB58ih/LBHX52ky7w2VhQVKQMucy3Ic=
@@ -60,8 +60,8 @@ github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8Hm
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
github.com/klauspost/compress v1.11.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= github.com/klauspost/compress v1.17.10 h1:oXAz+Vh0PMUvJczoi+flxpnBEPxoER1IaAnU/NMPtT0=
github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/compress v1.17.10/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU=
@@ -107,12 +107,12 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_golang v1.20.2 h1:5ctymQzZlyOON1666svgwn3s6IKWgfbjsejTMiXIyjg= github.com/prometheus/client_golang v1.20.4 h1:Tgh3Yr67PaOv/uTqloMsCEdeuFTatm5zIq5+qNN23vI=
github.com/prometheus/client_golang v1.20.2/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= github.com/prometheus/client_golang v1.20.4/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE=
github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
github.com/prometheus/common v0.57.0 h1:Ro/rKjwdq9mZn1K5QPctzh+MA4Lp0BuYk5ZZEVhoNcY= github.com/prometheus/common v0.60.0 h1:+V9PAREWNvJMAuJ1x1BaWl9dewMW4YrHZQbx0sJNllA=
github.com/prometheus/common v0.57.0/go.mod h1:7uRPFSUTbfZWsJ7MHY56sqt7hLQu3bxXHDnNhl8E9qI= github.com/prometheus/common v0.60.0/go.mod h1:h0LYf1R1deLSKtD4Vdg8gy4RuOvENW2J/h19V5NADQw=
github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
@@ -146,25 +146,25 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw=
golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U=
golang.org/x/image v0.19.0 h1:D9FX4QWkLfkeqaC62SonffIIuYdOk/UE2XKUBgRIBIQ= golang.org/x/image v0.21.0 h1:c5qV36ajHpdj4Qi0GnE0jUc/yuo33OLFaa0d+crTD5s=
golang.org/x/image v0.19.0/go.mod h1:y0zrRqlQRWQ5PXaYCOMLTW2fpsxZ8Qh9I/ohnInJEys= golang.org/x/image v0.21.0/go.mod h1:vUbsLavqK/W303ZroQQVKQ+Af3Yl6Uz1Ppu5J/cLz78=
golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4=
golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU=
golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=
golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU= golang.org/x/term v0.25.0 h1:WtHI/ltw4NvSUig5KARz9h521QvRC8RmF/cuYqifU24=
golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk= golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M=
golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM=
golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= golang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ=
golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=

View File

@@ -156,7 +156,7 @@ func (cmd *Cmd) pipeOutput() error {
r := bufio.NewReader(reader) r := bufio.NewReader(reader)
defer func(reader io.ReadCloser) { defer func(reader io.ReadCloser) {
err := reader.Close() err := reader.Close()
if err != nil { if err != nil && !strings.Contains(err.Error(), "file already closed") {
logger.Error(fmt.Sprintf("close reader: %s", err)) logger.Error(fmt.Sprintf("close reader: %s", err))
} }
}(reader) }(reader)

View File

@@ -104,19 +104,8 @@ func TestCmd_Wait(t *testing.T) {
expectWaitError: false, expectWaitError: false,
}, },
{ {
scenario: "wait error", scenario: "wait error",
cmd: func() *Cmd { cmd: Command(zap.NewNop(), "echo", "Hello", "World"),
cmd := Command(zap.NewNop(), "echo", "Hello", "World")
err := cmd.Start()
if err != nil {
t.Fatalf("expected no error but got: %v", err)
}
err = cmd.Kill()
if err != nil {
t.Fatalf("expected no error but got: %v", err)
}
return cmd
}(),
expectWaitError: true, expectWaitError: true,
}, },
} }

View File

@@ -168,33 +168,37 @@ func (f *ParsedFlags) MustDeprecatedDuration(deprecated string, newName string)
return f.MustDuration(newName) return f.MustDuration(newName)
} }
// MustHumanReadableBytesString returns the human-readable bytes string of a // MustHumanReadableBytes returns the human-readable bytes string of a flag
// flag given by name. // given by name.
// It panics if an error occurs. // It panics if an error occurs.
func (f *ParsedFlags) MustHumanReadableBytesString(name string) string { func (f *ParsedFlags) MustHumanReadableBytes(name string) int64 {
val, err := f.GetString(name) val, err := f.GetString(name)
if err != nil { if err != nil {
panic(err) panic(err)
} }
_, err = bytes.Parse(val) if val == "" {
return 0
}
b, err := bytes.Parse(val)
if err != nil { if err != nil {
panic(err) panic(err)
} }
return val return b
} }
// MustDeprecatedHumanReadableBytesString returns the human-readable bytes // MustDeprecatedHumanReadableBytes returns the human-readable bytes of a
// string of a deprecated flag if it was explicitly set or the human-readable // deprecated flag if it was explicitly set or the human-readable bytes string
// bytes string of the new flag. // of the new flag.
// It panics if an error occurs. // It panics if an error occurs.
func (f *ParsedFlags) MustDeprecatedHumanReadableBytesString(deprecated string, newName string) string { func (f *ParsedFlags) MustDeprecatedHumanReadableBytes(deprecated string, newName string) int64 {
if f.Changed(deprecated) { if f.Changed(deprecated) {
return f.MustHumanReadableBytesString(deprecated) return f.MustHumanReadableBytes(deprecated)
} }
return f.MustHumanReadableBytesString(newName) return f.MustHumanReadableBytes(newName)
} }
// MustRegexp returns the regular expression of a flag given by name. // MustRegexp returns the regular expression of a flag given by name.

View File

@@ -644,10 +644,11 @@ func TestParsedFlags_MustDeprecatedDuration(t *testing.T) {
} }
} }
func TestParsedFlags_MustHumanReadableBytesString(t *testing.T) { func TestParsedFlags_MustHumanReadableBytes(t *testing.T) {
fs := flag.NewFlagSet("tests", flag.ContinueOnError) fs := flag.NewFlagSet("tests", flag.ContinueOnError)
fs.String("foo", "1MB", "") fs.String("foo", "1MB", "")
fs.String("bar", "1MB", "") fs.String("bar", "1MB", "")
fs.String("qux", "", "")
err := fs.Parse([]string{"--foo=1GB", "--bar=foo"}) err := fs.Parse([]string{"--foo=1GB", "--bar=foo"})
if err != nil { if err != nil {
@@ -671,6 +672,11 @@ func TestParsedFlags_MustHumanReadableBytesString(t *testing.T) {
name: "bar", name: "bar",
expectPanic: true, expectPanic: true,
}, },
{
scenario: "success: empty value",
name: "qux",
expectPanic: false,
},
} { } {
t.Run(tc.scenario, func(t *testing.T) { t.Run(tc.scenario, func(t *testing.T) {
if tc.expectPanic { if tc.expectPanic {
@@ -689,31 +695,31 @@ func TestParsedFlags_MustHumanReadableBytesString(t *testing.T) {
}() }()
} }
parsedFlags.MustHumanReadableBytesString(tc.name) parsedFlags.MustHumanReadableBytes(tc.name)
}) })
} }
} }
func TestParsedFlags_MustDeprecatedHumanReadableBytesString(t *testing.T) { func TestParsedFlags_MustDeprecatedHumanReadableBytes(t *testing.T) {
for _, tc := range []struct { for _, tc := range []struct {
scenario string scenario string
rawFlags []string rawFlags []string
expectValue string expectValue int64
}{ }{
{ {
scenario: "deprecated flag value", scenario: "deprecated flag value",
rawFlags: []string{"--foo=1MB"}, rawFlags: []string{"--foo=1MB"},
expectValue: "1MB", expectValue: 1000000,
}, },
{ {
scenario: "non-deprecated flag value", scenario: "non-deprecated flag value",
rawFlags: []string{"--bar=2MB"}, rawFlags: []string{"--bar=2MB"},
expectValue: "2MB", expectValue: 2000000,
}, },
{ {
scenario: "deprecated flag value > non-deprecated flag value", scenario: "deprecated flag value > non-deprecated flag value",
rawFlags: []string{"--foo=1MB", "--bar=2MB"}, rawFlags: []string{"--foo=1MB", "--bar=2MB"},
expectValue: "1MB", expectValue: 1000000,
}, },
} { } {
t.Run(tc.scenario, func(t *testing.T) { t.Run(tc.scenario, func(t *testing.T) {
@@ -728,9 +734,9 @@ func TestParsedFlags_MustDeprecatedHumanReadableBytesString(t *testing.T) {
t.Fatalf("expected no error but got: %v", err) t.Fatalf("expected no error but got: %v", err)
} }
actual := parsedFlags.MustDeprecatedHumanReadableBytesString("foo", "bar") actual := parsedFlags.MustDeprecatedHumanReadableBytes("foo", "bar")
if actual != tc.expectValue { if actual != tc.expectValue {
t.Errorf("expected '%s' but got '%s'", tc.expectValue, actual) t.Errorf("expected %d but got %d", tc.expectValue, actual)
} }
}) })
} }

View File

@@ -1,6 +1,11 @@
package gotenberg package gotenberg
import "go.uber.org/zap" import (
"fmt"
"github.com/hashicorp/go-retryablehttp"
"go.uber.org/zap"
)
// LoggerProvider is an interface for a module that supplies a method for // LoggerProvider is an interface for a module that supplies a method for
// creating a [zap.Logger] instance for use by other modules. // creating a [zap.Logger] instance for use by other modules.
@@ -12,3 +17,41 @@ import "go.uber.org/zap"
type LoggerProvider interface { type LoggerProvider interface {
Logger(mod Module) (*zap.Logger, error) Logger(mod Module) (*zap.Logger, error)
} }
// LeveledLogger is wrapper around a [zap.Logger] so that it may be used by a
// [retryablehttp.Client].
type LeveledLogger struct {
logger *zap.Logger
}
// NewLeveledLogger instantiates a [LeveledLogger].
func NewLeveledLogger(logger *zap.Logger) *LeveledLogger {
return &LeveledLogger{
logger: logger,
}
}
// Error logs a message at error level using the wrapped zap.Logger.
func (leveled LeveledLogger) Error(msg string, keysAndValues ...interface{}) {
leveled.logger.Error(fmt.Sprintf("%s: %+v", msg, keysAndValues))
}
// Warn logs a message at warning level using the wrapped zap.Logger.
func (leveled LeveledLogger) Warn(msg string, keysAndValues ...interface{}) {
leveled.logger.Warn(fmt.Sprintf("%s: %+v", msg, keysAndValues))
}
// Info logs a message at info level using the wrapped zap.Logger.
func (leveled LeveledLogger) Info(msg string, keysAndValues ...interface{}) {
leveled.logger.Info(fmt.Sprintf("%s: %+v", msg, keysAndValues))
}
// Debug logs a message at debug level using the wrapped zap.Logger.
func (leveled LeveledLogger) Debug(msg string, keysAndValues ...interface{}) {
leveled.logger.Debug(fmt.Sprintf("%s: %+v", msg, keysAndValues))
}
// Interface guards.
var (
_ retryablehttp.LeveledLogger = (*LeveledLogger)(nil)
)

View File

@@ -1,4 +1,4 @@
package webhook package gotenberg
import ( import (
"testing" "testing"
@@ -7,17 +7,17 @@ import (
) )
func TestLeveledLogger_Error(t *testing.T) { func TestLeveledLogger_Error(t *testing.T) {
leveledLogger{logger: zap.NewNop()}.Error("foo") NewLeveledLogger(zap.NewNop()).Error("foo")
} }
func TestLeveledLogger_Warn(t *testing.T) { func TestLeveledLogger_Warn(t *testing.T) {
leveledLogger{logger: zap.NewNop()}.Warn("foo") NewLeveledLogger(zap.NewNop()).Warn("foo")
} }
func TestLeveledLogger_Info(t *testing.T) { func TestLeveledLogger_Info(t *testing.T) {
leveledLogger{logger: zap.NewNop()}.Info("foo") NewLeveledLogger(zap.NewNop()).Info("foo")
} }
func TestLeveledLogger_Debug(t *testing.T) { func TestLeveledLogger_Debug(t *testing.T) {
leveledLogger{logger: zap.NewNop()}.Debug("foo") NewLeveledLogger(zap.NewNop()).Debug("foo")
} }

View File

@@ -173,6 +173,20 @@ func (s *processSupervisor) Healthy() bool {
} }
func (s *processSupervisor) Run(ctx context.Context, logger *zap.Logger, task func() error) error { func (s *processSupervisor) Run(ctx context.Context, logger *zap.Logger, task func() error) error {
// A user reported a potential issue:
//
// "Although the counting operation is atomic, nothing prevent 2 concurrent
// goroutines to retrieve the same 'currentQueueSize' and to compare its
// value against the max limit. Then, resulting queue size would be 1 above
// the allowed limit."
//
// However, he was unable to actually trigger this issue, even when sending
// a lot of requests.
//
// For now, the best option is to consider this issue to be unlikely to
// happen, and keep the code as it is because it is more readable this way.
//
// See https://github.com/gotenberg/gotenberg/issues/951.
currentQueueSize := s.reqQueueSize.Load() currentQueueSize := s.reqQueueSize.Load()
if s.maxQueueSize > 0 && currentQueueSize >= s.maxQueueSize { if s.maxQueueSize > 0 && currentQueueSize >= s.maxQueueSize {
return ErrMaximumQueueSizeExceeded return ErrMaximumQueueSizeExceeded

View File

@@ -10,6 +10,7 @@ import (
"time" "time"
"github.com/alexliesenfeld/health" "github.com/alexliesenfeld/health"
"github.com/dlclark/regexp2"
"github.com/labstack/echo/v4" "github.com/labstack/echo/v4"
flag "github.com/spf13/pflag" flag "github.com/spf13/pflag"
"go.uber.org/multierr" "go.uber.org/multierr"
@@ -31,11 +32,13 @@ type Api struct {
tlsCertFile string tlsCertFile string
tlsKeyFile string tlsKeyFile string
startTimeout time.Duration startTimeout time.Duration
bodyLimit int64
timeout time.Duration timeout time.Duration
rootPath string rootPath string
traceHeader string traceHeader string
basicAuthUsername string basicAuthUsername string
basicAuthPassword string basicAuthPassword string
downloadFromCfg downloadFromConfig
disableHealthCheckLogging bool disableHealthCheckLogging bool
routes []Route routes []Route
@@ -47,6 +50,13 @@ type Api struct {
srv *echo.Echo srv *echo.Echo
} }
type downloadFromConfig struct {
allowList *regexp2.Regexp
denyList *regexp2.Regexp
maxRetry int
disable bool
}
// Router is a module interface which adds routes to the [Api]. // Router is a module interface which adds routes to the [Api].
type Router interface { type Router interface {
Routes() ([]Route, error) Routes() ([]Route, error)
@@ -165,9 +175,14 @@ func (a *Api) Descriptor() gotenberg.ModuleDescriptor {
fs.String("api-tls-key-file", "", "Path to the TLS/SSL key file - for HTTPS support") fs.String("api-tls-key-file", "", "Path to the TLS/SSL key file - for HTTPS support")
fs.Duration("api-start-timeout", time.Duration(30)*time.Second, "Set the time limit for the API to start") fs.Duration("api-start-timeout", time.Duration(30)*time.Second, "Set the time limit for the API to start")
fs.Duration("api-timeout", time.Duration(30)*time.Second, "Set the time limit for requests") fs.Duration("api-timeout", time.Duration(30)*time.Second, "Set the time limit for requests")
fs.String("api-body-limit", "", "Set the body limit for multipart/form-data requests")
fs.String("api-root-path", "/", "Set the root path of the API - for service discovery via URL paths") fs.String("api-root-path", "/", "Set the root path of the API - for service discovery via URL paths")
fs.String("api-trace-header", "Gotenberg-Trace", "Set the header name to use for identifying requests") fs.String("api-trace-header", "Gotenberg-Trace", "Set the header name to use for identifying requests")
fs.Bool("api-enable-basic-auth", false, "Enable basic authentication - will look for the GOTENBERG_API_BASIC_AUTH_USERNAME and GOTENBERG_API_BASIC_AUTH_PASSWORD environment variables") fs.Bool("api-enable-basic-auth", false, "Enable basic authentication - will look for the GOTENBERG_API_BASIC_AUTH_USERNAME and GOTENBERG_API_BASIC_AUTH_PASSWORD environment variables")
fs.String("api-download-from-allow-list", "", "Set the allowed URLs for the download from feature using a regular expression")
fs.String("api-download-from-deny-list", "", "Set the denied URLs for the download from feature using a regular expression")
fs.Int("api-download-from-max-retry", 4, "Set the maximum number of retries for the download from feature")
fs.Bool("api-disable-download-from", false, "Disable the download from feature")
fs.Bool("api-disable-health-check-logging", false, "Disable health check logging") fs.Bool("api-disable-health-check-logging", false, "Disable health check logging")
return fs return fs
}(), }(),
@@ -183,8 +198,15 @@ func (a *Api) Provision(ctx *gotenberg.Context) error {
a.tlsKeyFile = flags.MustString("api-tls-key-file") a.tlsKeyFile = flags.MustString("api-tls-key-file")
a.startTimeout = flags.MustDuration("api-start-timeout") a.startTimeout = flags.MustDuration("api-start-timeout")
a.timeout = flags.MustDuration("api-timeout") a.timeout = flags.MustDuration("api-timeout")
a.bodyLimit = flags.MustHumanReadableBytes("api-body-limit")
a.rootPath = flags.MustString("api-root-path") a.rootPath = flags.MustString("api-root-path")
a.traceHeader = flags.MustString("api-trace-header") a.traceHeader = flags.MustString("api-trace-header")
a.downloadFromCfg = downloadFromConfig{
allowList: flags.MustRegexp("api-download-from-allow-list"),
denyList: flags.MustRegexp("api-download-from-deny-list"),
maxRetry: flags.MustInt("api-download-from-max-retry"),
disable: flags.MustBool("api-disable-download-from"),
}
a.disableHealthCheckLogging = flags.MustBool("api-disable-health-check-logging") a.disableHealthCheckLogging = flags.MustBool("api-disable-health-check-logging")
// Port from env? // Port from env?
@@ -436,7 +458,7 @@ func (a *Api) Start() error {
} }
if route.IsMultipart { if route.IsMultipart {
middlewares = append(middlewares, contextMiddleware(a.fs, a.timeout)) middlewares = append(middlewares, contextMiddleware(a.fs, a.timeout, a.bodyLimit, a.downloadFromCfg))
for _, externalMultipartMiddleware := range externalMultipartMiddlewares { for _, externalMultipartMiddleware := range externalMultipartMiddlewares {
middlewares = append(middlewares, externalMultipartMiddleware.Handler) middlewares = append(middlewares, externalMultipartMiddleware.Handler)
@@ -453,13 +475,22 @@ func (a *Api) Start() error {
) )
} }
// Let's not forget the health check route... // Let's not forget the health check routes...
checks := append(a.healthChecks, health.WithTimeout(a.timeout))
checker := health.NewChecker(checks...)
healthCheckHandler := health.NewHandler(checker)
a.srv.GET( a.srv.GET(
fmt.Sprintf("%s%s", a.rootPath, "health"), fmt.Sprintf("%s%s", a.rootPath, "health"),
func() echo.HandlerFunc { func() echo.HandlerFunc {
checks := append(a.healthChecks, health.WithTimeout(a.timeout)) return echo.WrapHandler(healthCheckHandler)
checker := health.NewChecker(checks...) }(),
return echo.WrapHandler(health.NewHandler(checker)) hardTimeoutMiddleware(hardTimeout),
)
a.srv.HEAD(
fmt.Sprintf("%s%s", a.rootPath, "health"),
func() echo.HandlerFunc {
return echo.WrapHandler(healthCheckHandler)
}(), }(),
hardTimeoutMiddleware(hardTimeout), hardTimeoutMiddleware(hardTimeout),
) )

View File

@@ -839,18 +839,23 @@ func TestApi_Start(t *testing.T) {
return return
} }
// health request. // health requests.
recorder := httptest.NewRecorder() recorder := httptest.NewRecorder()
healthRequest := httptest.NewRequest(http.MethodGet, "/health", nil)
mod.srv.ServeHTTP(recorder, healthRequest) healthGetRequest := httptest.NewRequest(http.MethodGet, "/health", nil)
mod.srv.ServeHTTP(recorder, healthGetRequest)
if recorder.Code != http.StatusOK {
t.Errorf("expected %d status code but got %d", http.StatusOK, recorder.Code)
}
healthHeadRequest := httptest.NewRequest(http.MethodHead, "/health", nil)
mod.srv.ServeHTTP(recorder, healthHeadRequest)
if recorder.Code != http.StatusOK { if recorder.Code != http.StatusOK {
t.Errorf("expected %d status code but got %d", http.StatusOK, recorder.Code) t.Errorf("expected %d status code but got %d", http.StatusOK, recorder.Code)
} }
// version request. // version request.
versionRequest := httptest.NewRequest(http.MethodGet, "/version", nil) versionRequest := httptest.NewRequest(http.MethodGet, "/version", nil)
mod.srv.ServeHTTP(recorder, versionRequest) mod.srv.ServeHTTP(recorder, versionRequest)
if recorder.Code != http.StatusOK { if recorder.Code != http.StatusOK {
t.Errorf("expected %d status code but got %d", http.StatusOK, recorder.Code) t.Errorf("expected %d status code but got %d", http.StatusOK, recorder.Code)
@@ -859,7 +864,6 @@ func TestApi_Start(t *testing.T) {
// "multipart/form-data" request. // "multipart/form-data" request.
multipartRequest := func(url string) *http.Request { multipartRequest := func(url string) *http.Request {
body := &bytes.Buffer{} body := &bytes.Buffer{}
writer := multipart.NewWriter(body) writer := multipart.NewWriter(body)
defer func() { defer func() {

View File

@@ -3,20 +3,25 @@ package api
import ( import (
"compress/flate" "compress/flate"
"context" "context"
"encoding/json"
"errors" "errors"
"fmt" "fmt"
"io" "io"
"mime"
"mime/multipart" "mime/multipart"
"net/http" "net/http"
"os" "os"
"path/filepath" "path/filepath"
"strings" "strings"
"sync/atomic"
"time" "time"
"github.com/google/uuid" "github.com/google/uuid"
"github.com/hashicorp/go-retryablehttp"
"github.com/labstack/echo/v4" "github.com/labstack/echo/v4"
"github.com/mholt/archiver/v3" "github.com/mholt/archiver/v3"
"go.uber.org/zap" "go.uber.org/zap"
"golang.org/x/sync/errgroup"
"golang.org/x/text/unicode/norm" "golang.org/x/text/unicode/norm"
"github.com/gotenberg/gotenberg/v8/pkg/gotenberg" "github.com/gotenberg/gotenberg/v8/pkg/gotenberg"
@@ -46,6 +51,36 @@ type Context struct {
context.Context context.Context
} }
type trackingReader struct {
R io.Reader
AddReadBytes func(n int64) error
}
func (t *trackingReader) Read(p []byte) (int, error) {
n, err := t.R.Read(p)
if n > 0 {
errAddRead := t.AddReadBytes(int64(n))
if errAddRead != nil {
return n, fmt.Errorf("add read bytes: %w", errAddRead)
}
}
if err != nil {
// It's a common practice in Go to return io.EOF unwrapped to signal
// the end of a data stream. Wrapping it can lead to unexpected
// behavior in standard library functions.
return n, err
}
return n, nil
}
type downloadFrom struct {
// Url is the URL to download a file from.
Url string `json:"url"`
// ExtraHttpHeaders are the HTTP headers to send alongside.
ExtraHttpHeaders map[string]string `json:"extraHttpHeaders"`
}
type osPathRename struct{} type osPathRename struct{}
func (o *osPathRename) Rename(oldpath, newpath string) error { func (o *osPathRename) Rename(oldpath, newpath string) error {
@@ -53,9 +88,25 @@ func (o *osPathRename) Rename(oldpath, newpath string) error {
} }
// newContext returns a [Context] by parsing a "multipart/form-data" request. // newContext returns a [Context] by parsing a "multipart/form-data" request.
func newContext(echoCtx echo.Context, logger *zap.Logger, fs *gotenberg.FileSystem, timeout time.Duration) (*Context, context.CancelFunc, error) { func newContext(echoCtx echo.Context, logger *zap.Logger, fs *gotenberg.FileSystem, timeout time.Duration, bodyLimit int64, downloadFromCfg downloadFromConfig, traceHeader, trace string) (*Context, context.CancelFunc, error) {
processCtx, processCancel := context.WithTimeout(context.Background(), timeout) processCtx, processCancel := context.WithTimeout(context.Background(), timeout)
// We want to make sure the multipart/form-data does not exceed a given
// limit. We consider: form fields (keys, values, files) and files
// downloaded remotely ("download from" feature).
var totalBytesRead atomic.Int64
addReadBytes := func(n int64) error {
newTotal := totalBytesRead.Add(n)
if bodyLimit != 0 && newTotal > bodyLimit {
return WrapError(
fmt.Errorf("body limit reached (> %d)", bodyLimit),
NewSentinelHttpError(http.StatusRequestEntityTooLarge, http.StatusText(http.StatusRequestEntityTooLarge)),
)
}
return nil
}
ctx := &Context{ ctx := &Context{
outputPaths: make([]string, 0), outputPaths: make([]string, 0),
cancelled: false, cancelled: false,
@@ -117,6 +168,19 @@ func newContext(echoCtx echo.Context, logger *zap.Logger, fs *gotenberg.FileSyst
return nil, cancel, fmt.Errorf("get multipart form: %w", err) return nil, cancel, fmt.Errorf("get multipart form: %w", err)
} }
// This will ensure we do not exceed the body limit.
var formValuesSize int64
for key, valArray := range form.Value {
formValuesSize += int64(len(key))
for _, val := range valArray {
formValuesSize += int64(len(val))
}
}
err = addReadBytes(formValuesSize)
if err != nil {
return nil, cancel, fmt.Errorf("add read bytes: %w", err)
}
dirPath, err := fs.MkdirAll() dirPath, err := fs.MkdirAll()
if err != nil { if err != nil {
return nil, cancel, fmt.Errorf("create working directory: %w", err) return nil, cancel, fmt.Errorf("create working directory: %w", err)
@@ -126,6 +190,150 @@ func newContext(echoCtx echo.Context, logger *zap.Logger, fs *gotenberg.FileSyst
ctx.values = form.Value ctx.values = form.Value
ctx.files = make(map[string]string) ctx.files = make(map[string]string)
// First, try to download files listed in the "downloadFrom" form field, if
// any.
raw, ok := ctx.values["downloadFrom"]
if !downloadFromCfg.disable && ok {
var dls []downloadFrom
err = json.Unmarshal([]byte(raw[0]), &dls)
if err != nil {
return nil, cancel, WrapError(
fmt.Errorf("unmarshal json: %w", err),
NewSentinelHttpError(http.StatusBadRequest, fmt.Sprintf("Invalid 'downloadFrom' form field value: %s", err)),
)
}
eg, _ := errgroup.WithContext(ctx)
for i, dl := range dls {
eg.Go(func() error {
deadline, ok := ctx.Deadline()
if !ok {
// Should not happen, as context is created with a timeout.
return errors.New("context has no deadline")
}
if strings.TrimSpace(dl.Url) == "" {
return WrapError(
errors.New("empty download from URL"),
NewSentinelHttpError(http.StatusBadRequest, fmt.Sprintf("Invalid 'downloadFrom' form field entry %d: URL must be set", i)),
)
}
err := gotenberg.FilterDeadline(downloadFromCfg.allowList, downloadFromCfg.denyList, dl.Url, deadline)
if err != nil {
return fmt.Errorf("filter URL: %w", err)
}
logger.Debug(fmt.Sprintf("download file from '%s'", dl.Url))
req, err := retryablehttp.NewRequest(http.MethodGet, dl.Url, nil)
if err != nil {
return fmt.Errorf("create request to '%s': %w", dl.Url, err)
}
req.Header.Set("User-Agent", "Gotenberg")
for key, value := range dl.ExtraHttpHeaders {
req.Header.Set(key, value)
}
req.Header.Set(traceHeader, trace)
client := &retryablehttp.Client{
HTTPClient: &http.Client{
Timeout: time.Until(deadline),
},
RetryMax: downloadFromCfg.maxRetry,
RetryWaitMin: time.Duration(1) * time.Second,
RetryWaitMax: time.Until(deadline),
Logger: gotenberg.NewLeveledLogger(logger),
CheckRetry: retryablehttp.DefaultRetryPolicy,
Backoff: retryablehttp.DefaultBackoff,
}
resp, err := client.Do(req)
if err != nil {
return WrapError(
fmt.Errorf("download file from to '%s': %w", dl.Url, err),
NewSentinelHttpError(http.StatusBadRequest, fmt.Sprintf("Unable to download file from '%s': %s", dl.Url, err)),
)
}
defer func() {
err := resp.Body.Close()
if err != nil {
logger.Error(fmt.Sprintf("close response body from '%s': %s", dl.Url, err))
}
}()
if resp.StatusCode != http.StatusOK {
return WrapError(
fmt.Errorf("download file from to '%s': got status: '%s'", dl.Url, resp.Status),
NewSentinelHttpError(http.StatusBadRequest, fmt.Sprintf("Unable to download file from '%s': got status: '%s'", dl.Url, resp.Status)),
)
}
contentDisposition := resp.Header.Get("Content-Disposition")
if contentDisposition == "" {
return WrapError(
fmt.Errorf("no 'Content-Disposition' header from '%s'", dl.Url),
NewSentinelHttpError(http.StatusBadRequest, fmt.Sprintf("No 'Content-Disposition' header from '%s'", dl.Url)),
)
}
// FIXME: the implementation of this method might not be
// complete, as it fails to parse an empty mediatype.
// See: https://github.com/golang/go/issues/69551.
_, params, err := mime.ParseMediaType(contentDisposition)
if err != nil {
return WrapError(
fmt.Errorf("parse 'Content-Disposition' header '%s' from '%s': %w", contentDisposition, dl.Url, err),
NewSentinelHttpError(http.StatusBadRequest, fmt.Sprintf("Invalid 'Content-Disposition' header '%s' from '%s': %s", contentDisposition, dl.Url, err)),
)
}
filename, ok := params["filename"]
if !ok {
return WrapError(
fmt.Errorf("get filename from 'Content-Disposition' header '%s' from '%s'", contentDisposition, dl.Url),
NewSentinelHttpError(http.StatusBadRequest, fmt.Sprintf("Invalid 'Content-Disposition' header '%s' from '%s': no filename", contentDisposition, dl.Url)),
)
}
// Avoid directory traversal and make sure filename characters are
// normalized.
// See: https://github.com/gotenberg/gotenberg/issues/662.
filename = norm.NFC.String(filepath.Base(filename))
path := fmt.Sprintf("%s/%s", ctx.dirPath, filename)
out, err := os.Create(path)
if err != nil {
return fmt.Errorf("create local file: %w", err)
}
defer func() {
err := out.Close()
if err != nil {
logger.Error(fmt.Sprintf("close local file: %s", err))
}
}()
// This will ensure we do not exceed the body limit.
reader := &trackingReader{R: resp.Body, AddReadBytes: addReadBytes}
_, err = io.Copy(out, reader)
if err != nil {
return fmt.Errorf("copy downloaded file from '%s' to local file: %w", dl.Url, err)
}
ctx.files[filename] = path
return nil
})
}
err = eg.Wait()
if err != nil {
return ctx, cancel, err
}
}
copyToDisk := func(fh *multipart.FileHeader) error { copyToDisk := func(fh *multipart.FileHeader) error {
in, err := fh.Open() in, err := fh.Open()
if err != nil { if err != nil {
@@ -139,6 +347,9 @@ func newContext(echoCtx echo.Context, logger *zap.Logger, fs *gotenberg.FileSyst
} }
}() }()
// This will ensure we do not exceed the body limit.
reader := &trackingReader{R: in, AddReadBytes: addReadBytes}
// Avoid directory traversal and make sure filename characters are // Avoid directory traversal and make sure filename characters are
// normalized. // normalized.
// See: https://github.com/gotenberg/gotenberg/issues/662. // See: https://github.com/gotenberg/gotenberg/issues/662.
@@ -149,7 +360,6 @@ func newContext(echoCtx echo.Context, logger *zap.Logger, fs *gotenberg.FileSyst
if err != nil { if err != nil {
return fmt.Errorf("create local file: %w", err) return fmt.Errorf("create local file: %w", err)
} }
defer func() { defer func() {
err := out.Close() err := out.Close()
if err != nil { if err != nil {
@@ -157,7 +367,7 @@ func newContext(echoCtx echo.Context, logger *zap.Logger, fs *gotenberg.FileSyst
} }
}() }()
_, err = io.Copy(out, in) _, err = io.Copy(out, reader)
if err != nil { if err != nil {
return fmt.Errorf("copy multipart file to local file: %w", err) return fmt.Errorf("copy multipart file to local file: %w", err)
} }
@@ -167,6 +377,7 @@ func newContext(echoCtx echo.Context, logger *zap.Logger, fs *gotenberg.FileSyst
return nil return nil
} }
// Then, copy the form files, if any.
for _, files := range form.File { for _, files := range form.File {
for _, fh := range files { for _, fh := range files {
err = copyToDisk(fh) err = copyToDisk(fh)
@@ -178,6 +389,7 @@ func newContext(echoCtx echo.Context, logger *zap.Logger, fs *gotenberg.FileSyst
ctx.Log().Debug(fmt.Sprintf("form fields: %+v", ctx.values)) ctx.Log().Debug(fmt.Sprintf("form fields: %+v", ctx.values))
ctx.Log().Debug(fmt.Sprintf("form files: %+v", ctx.files)) ctx.Log().Debug(fmt.Sprintf("form files: %+v", ctx.files))
ctx.Log().Debug(fmt.Sprintf("total bytes: %d", totalBytesRead.Load()))
return ctx, cancel, err return ctx, cancel, err
} }

View File

@@ -2,6 +2,7 @@ package api
import ( import (
"bytes" "bytes"
"context"
"errors" "errors"
"fmt" "fmt"
"io" "io"
@@ -15,6 +16,7 @@ import (
"testing" "testing"
"time" "time"
"github.com/dlclark/regexp2"
"github.com/google/uuid" "github.com/google/uuid"
"github.com/labstack/echo/v4" "github.com/labstack/echo/v4"
"go.uber.org/zap" "go.uber.org/zap"
@@ -75,9 +77,28 @@ func TestOsPathRename_Rename(t *testing.T) {
} }
func TestNewContext(t *testing.T) { func TestNewContext(t *testing.T) {
defaultAllowList, err := regexp2.Compile("", 0)
if err != nil {
t.Fatalf("expected no error but got: %v", err)
}
defaultDenyList, err := regexp2.Compile("", 0)
if err != nil {
t.Fatalf("expected no error but got: %v", err)
}
defaultDownloadFromCfg := downloadFromConfig{
allowList: defaultAllowList,
denyList: defaultDenyList,
maxRetry: 1,
disable: false,
}
for _, tc := range []struct { for _, tc := range []struct {
scenario string scenario string
request *http.Request request *http.Request
bodyLimit int64
downloadFromCfg downloadFromConfig
downloadFromSrv *echo.Echo
expectContext *Context
expectError bool expectError bool
expectHttpError bool expectHttpError bool
expectHttpStatus int expectHttpStatus int
@@ -123,6 +144,325 @@ func TestNewContext(t *testing.T) {
expectHttpError: true, expectHttpError: true,
expectHttpStatus: http.StatusBadRequest, expectHttpStatus: http.StatusBadRequest,
}, },
{
scenario: "request entity too large: form values",
request: func() *http.Request {
body := &bytes.Buffer{}
writer := multipart.NewWriter(body)
defer func() {
err := writer.Close()
if err != nil {
t.Fatalf("expected no error but got: %v", err)
}
}()
err := writer.WriteField("key", "value")
if err != nil {
t.Fatalf("expected no error but got: %v", err)
}
req := httptest.NewRequest(http.MethodPost, "/", body)
req.Header.Set(echo.HeaderContentType, writer.FormDataContentType())
return req
}(),
bodyLimit: 1,
downloadFromCfg: defaultDownloadFromCfg,
expectError: true,
expectHttpError: true,
expectHttpStatus: http.StatusRequestEntityTooLarge,
},
{
scenario: "request entity too large: downloadFrom",
request: func() *http.Request {
body := &bytes.Buffer{}
writer := multipart.NewWriter(body)
defer func() {
err := writer.Close()
if err != nil {
t.Fatalf("expected no error but got: %v", err)
}
}()
err := writer.WriteField("downloadFrom", `[{"url":"http://localhost:80/"}]`)
if err != nil {
t.Fatalf("expected no error but got: %v", err)
}
req := httptest.NewRequest(http.MethodPost, "/", body)
req.Header.Set(echo.HeaderContentType, writer.FormDataContentType())
return req
}(),
bodyLimit: 45, // form values = 44 bytes.
downloadFromSrv: func() *echo.Echo {
srv := echo.New()
srv.HideBanner = true
srv.GET("/", func(c echo.Context) error {
c.Response().Header().Set(echo.HeaderContentDisposition, `attachment; filename="bar.txt"`)
c.Response().Header().Set(echo.HeaderContentType, "text/plain")
return c.String(http.StatusOK, http.StatusText(http.StatusOK))
})
return srv
}(),
downloadFromCfg: defaultDownloadFromCfg,
expectError: true,
expectHttpError: true,
expectHttpStatus: http.StatusRequestEntityTooLarge,
},
{
scenario: "request entity too large: form files",
request: func() *http.Request {
body := &bytes.Buffer{}
writer := multipart.NewWriter(body)
defer func() {
err := writer.Close()
if err != nil {
t.Fatalf("expected no error but got: %v", err)
}
}()
part, err := writer.CreateFormFile("foo.txt", "foo.txt")
if err != nil {
t.Fatalf("expected no error but got: %v", err)
}
_, err = part.Write([]byte("foo"))
if err != nil {
t.Fatalf("expected no error but got: %v", err)
}
req := httptest.NewRequest(http.MethodPost, "/", body)
req.Header.Set(echo.HeaderContentType, writer.FormDataContentType())
return req
}(),
bodyLimit: 1,
downloadFromCfg: defaultDownloadFromCfg,
expectError: true,
expectHttpError: true,
expectHttpStatus: http.StatusRequestEntityTooLarge,
},
{
scenario: "invalid downloadFrom form field: cannot unmarshal",
request: func() *http.Request {
body := &bytes.Buffer{}
writer := multipart.NewWriter(body)
defer func() {
err := writer.Close()
if err != nil {
t.Fatalf("expected no error but got: %v", err)
}
}()
err := writer.WriteField("downloadFrom", "foo")
if err != nil {
t.Fatalf("expected no error but got: %v", err)
}
req := httptest.NewRequest(http.MethodPost, "/", body)
req.Header.Set(echo.HeaderContentType, writer.FormDataContentType())
return req
}(),
downloadFromCfg: defaultDownloadFromCfg,
expectError: true,
expectHttpError: true,
expectHttpStatus: http.StatusBadRequest,
},
{
scenario: "invalid downloadFrom form field: no URL",
request: func() *http.Request {
body := &bytes.Buffer{}
writer := multipart.NewWriter(body)
defer func() {
err := writer.Close()
if err != nil {
t.Fatalf("expected no error but got: %v", err)
}
}()
err := writer.WriteField("downloadFrom", `[{}]`)
if err != nil {
t.Fatalf("expected no error but got: %v", err)
}
req := httptest.NewRequest(http.MethodPost, "/", body)
req.Header.Set(echo.HeaderContentType, writer.FormDataContentType())
return req
}(),
downloadFromCfg: defaultDownloadFromCfg,
expectError: true,
expectHttpError: true,
expectHttpStatus: http.StatusBadRequest,
},
{
scenario: "invalid downloadFrom form field: filtered URL",
request: func() *http.Request {
body := &bytes.Buffer{}
writer := multipart.NewWriter(body)
defer func() {
err := writer.Close()
if err != nil {
t.Fatalf("expected no error but got: %v", err)
}
}()
err := writer.WriteField("downloadFrom", `[{"url":"https://foo.bar"}]`)
if err != nil {
t.Fatalf("expected no error but got: %v", err)
}
req := httptest.NewRequest(http.MethodPost, "/", body)
req.Header.Set(echo.HeaderContentType, writer.FormDataContentType())
return req
}(),
downloadFromCfg: func() downloadFromConfig {
denyList, err := regexp2.Compile("https://foo.bar", 0)
if err != nil {
t.Fatalf("expected no error but got: %v", err)
}
return downloadFromConfig{allowList: defaultAllowList, denyList: denyList, maxRetry: 1, disable: false}
}(),
expectError: true,
},
{
scenario: "invalid downloadFrom form field: unreachable URL",
request: func() *http.Request {
body := &bytes.Buffer{}
writer := multipart.NewWriter(body)
defer func() {
err := writer.Close()
if err != nil {
t.Fatalf("expected no error but got: %v", err)
}
}()
err := writer.WriteField("downloadFrom", `[{"url":"http://localhost:80/"}]`)
if err != nil {
t.Fatalf("expected no error but got: %v", err)
}
req := httptest.NewRequest(http.MethodPost, "/", body)
req.Header.Set(echo.HeaderContentType, writer.FormDataContentType())
return req
}(),
downloadFromCfg: defaultDownloadFromCfg,
expectError: true,
expectHttpError: true,
expectHttpStatus: http.StatusBadRequest,
},
{
scenario: "invalid downloadFrom form field: invalid status code",
request: func() *http.Request {
body := &bytes.Buffer{}
writer := multipart.NewWriter(body)
defer func() {
err := writer.Close()
if err != nil {
t.Fatalf("expected no error but got: %v", err)
}
}()
err := writer.WriteField("downloadFrom", `[{"url":"http://localhost:80/"}]`)
if err != nil {
t.Fatalf("expected no error but got: %v", err)
}
req := httptest.NewRequest(http.MethodPost, "/", body)
req.Header.Set(echo.HeaderContentType, writer.FormDataContentType())
return req
}(),
downloadFromSrv: func() *echo.Echo {
srv := echo.New()
srv.HideBanner = true
srv.GET("/", func(c echo.Context) error {
return c.String(http.StatusNotFound, http.StatusText(http.StatusNotFound))
})
return srv
}(),
downloadFromCfg: defaultDownloadFromCfg,
expectError: true,
expectHttpError: true,
expectHttpStatus: http.StatusBadRequest,
},
{
scenario: "invalid downloadFrom form field: no 'Content-Disposition' header",
request: func() *http.Request {
body := &bytes.Buffer{}
writer := multipart.NewWriter(body)
defer func() {
err := writer.Close()
if err != nil {
t.Fatalf("expected no error but got: %v", err)
}
}()
err := writer.WriteField("downloadFrom", `[{"url":"http://localhost:80/"}]`)
if err != nil {
t.Fatalf("expected no error but got: %v", err)
}
req := httptest.NewRequest(http.MethodPost, "/", body)
req.Header.Set(echo.HeaderContentType, writer.FormDataContentType())
return req
}(),
downloadFromSrv: func() *echo.Echo {
srv := echo.New()
srv.HideBanner = true
srv.GET("/", func(c echo.Context) error {
return c.String(http.StatusOK, http.StatusText(http.StatusOK))
})
return srv
}(),
downloadFromCfg: defaultDownloadFromCfg,
expectError: true,
expectHttpError: true,
expectHttpStatus: http.StatusBadRequest,
},
{
scenario: "invalid downloadFrom form field: malformed 'Content-Disposition' header",
request: func() *http.Request {
body := &bytes.Buffer{}
writer := multipart.NewWriter(body)
defer func() {
err := writer.Close()
if err != nil {
t.Fatalf("expected no error but got: %v", err)
}
}()
err := writer.WriteField("downloadFrom", `[{"url":"http://localhost:80/"}]`)
if err != nil {
t.Fatalf("expected no error but got: %v", err)
}
req := httptest.NewRequest(http.MethodPost, "/", body)
req.Header.Set(echo.HeaderContentType, writer.FormDataContentType())
return req
}(),
downloadFromSrv: func() *echo.Echo {
srv := echo.New()
srv.HideBanner = true
srv.GET("/", func(c echo.Context) error {
c.Response().Header().Set(echo.HeaderContentDisposition, ";;")
return c.String(http.StatusOK, http.StatusText(http.StatusOK))
})
return srv
}(),
downloadFromCfg: defaultDownloadFromCfg,
expectError: true,
expectHttpError: true,
expectHttpStatus: http.StatusBadRequest,
},
{
scenario: "invalid downloadFrom form field: no filename parameter in 'Content-Disposition' header",
request: func() *http.Request {
body := &bytes.Buffer{}
writer := multipart.NewWriter(body)
defer func() {
err := writer.Close()
if err != nil {
t.Fatalf("expected no error but got: %v", err)
}
}()
err := writer.WriteField("downloadFrom", `[{"url":"http://localhost:80/"}]`)
if err != nil {
t.Fatalf("expected no error but got: %v", err)
}
req := httptest.NewRequest(http.MethodPost, "/", body)
req.Header.Set(echo.HeaderContentType, writer.FormDataContentType())
return req
}(),
downloadFromSrv: func() *echo.Echo {
srv := echo.New()
srv.HideBanner = true
srv.GET("/", func(c echo.Context) error {
c.Response().Header().Set(echo.HeaderContentDisposition, "inline;")
return c.String(http.StatusOK, http.StatusText(http.StatusOK))
})
return srv
}(),
downloadFromCfg: defaultDownloadFromCfg,
expectError: true,
expectHttpError: true,
expectHttpStatus: http.StatusBadRequest,
},
{ {
scenario: "success", scenario: "success",
request: func() *http.Request { request: func() *http.Request {
@@ -146,17 +486,69 @@ func TestNewContext(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("expected no error but got: %v", err) t.Fatalf("expected no error but got: %v", err)
} }
err = writer.WriteField("downloadFrom", `[{"url":"http://localhost:80/","extraHttpHeaders":{"X-Foo":"Bar"}}]`)
if err != nil {
t.Fatalf("expected no error but got: %v", err)
}
req := httptest.NewRequest(http.MethodPost, "/", body) req := httptest.NewRequest(http.MethodPost, "/", body)
req.Header.Set(echo.HeaderContentType, writer.FormDataContentType()) req.Header.Set(echo.HeaderContentType, writer.FormDataContentType())
return req return req
}(), }(),
downloadFromSrv: func() *echo.Echo {
srv := echo.New()
srv.HideBanner = true
srv.GET("/", func(c echo.Context) error {
if c.Request().Header.Get("User-Agent") != "Gotenberg" {
t.Fatalf("expected 'Gotenberg' from header 'User-Agent', but got '%s'", c.Request().Header.Get("User-Agent"))
}
if c.Request().Header.Get("X-Foo") != "Bar" {
t.Fatalf("expected 'Bar' from header 'X-Foo', but got '%s'", c.Request().Header.Get("X-Foo"))
}
if c.Request().Header.Get("Gotenberg-Trace") != "123" {
t.Fatalf("expected '123' from header 'Gotenberg-Trace', but got '%s'", c.Request().Header.Get("Gotenberg-Trace"))
}
c.Response().Header().Set(echo.HeaderContentDisposition, `attachment; filename="bar.txt"`)
c.Response().Header().Set(echo.HeaderContentType, "text/plain")
return c.String(http.StatusOK, http.StatusText(http.StatusOK))
})
return srv
}(),
downloadFromCfg: defaultDownloadFromCfg,
expectContext: &Context{
values: map[string][]string{
"foo": {"foo"},
"downloadFrom": {
`[{"url":"http://localhost:80/","extraHttpHeaders":{"X-Foo":"Bar"}}]`,
},
},
files: map[string]string{
"foo.txt": "foo.txt",
"bar.txt": "bar.txt", // downloadFrom.
},
},
expectError: false, expectError: false,
expectHttpError: false, expectHttpError: false,
}, },
} { } {
t.Run(tc.scenario, func(t *testing.T) { t.Run(tc.scenario, func(t *testing.T) {
if tc.downloadFromSrv != nil {
go func() {
err := tc.downloadFromSrv.Start(":80")
if !errors.Is(err, http.ErrServerClosed) {
t.Error(err)
return
}
}()
defer func() {
err := tc.downloadFromSrv.Shutdown(context.TODO())
if err != nil {
t.Error(err)
}
}()
}
handler := func(c echo.Context) error { handler := func(c echo.Context) error {
_, cancel, err := newContext(c, zap.NewNop(), gotenberg.NewFileSystem(), time.Duration(10)*time.Second) ctx, cancel, err := newContext(c, zap.NewNop(), gotenberg.NewFileSystem(), time.Duration(10)*time.Second, tc.bodyLimit, tc.downloadFromCfg, "Gotenberg-Trace", "123")
defer cancel() defer cancel()
// Context already cancelled. // Context already cancelled.
defer cancel() defer cancel()
@@ -165,6 +557,20 @@ func TestNewContext(t *testing.T) {
return err return err
} }
if tc.expectContext != nil {
if !reflect.DeepEqual(tc.expectContext.values, ctx.values) {
t.Fatalf("expected context.values to be %v but got %v", tc.expectContext.values, ctx.values)
}
if len(tc.expectContext.files) != len(ctx.files) {
t.Fatalf("expected context.files to contain %d items but got %d", len(tc.expectContext.files), len(ctx.files))
}
for key, value := range tc.expectContext.files {
if !strings.HasSuffix(ctx.files[key], value) {
t.Fatalf("expected context.files to contain '%s' but got '%s'", value, ctx.files[key])
}
}
}
return nil return nil
} }

View File

@@ -236,14 +236,16 @@ func basicAuthMiddleware(username, password string) echo.MiddlewareFunc {
// //
// ctx := c.Get("context").(*api.Context) // ctx := c.Get("context").(*api.Context)
// cancel := c.Get("cancel").(context.CancelFunc) // cancel := c.Get("cancel").(context.CancelFunc)
func contextMiddleware(fs *gotenberg.FileSystem, timeout time.Duration) echo.MiddlewareFunc { func contextMiddleware(fs *gotenberg.FileSystem, timeout time.Duration, bodyLimit int64, downloadFromCfg downloadFromConfig) echo.MiddlewareFunc {
return func(next echo.HandlerFunc) echo.HandlerFunc { return func(next echo.HandlerFunc) echo.HandlerFunc {
return func(c echo.Context) error { return func(c echo.Context) error {
logger := c.Get("logger").(*zap.Logger) logger := c.Get("logger").(*zap.Logger)
traceHeader := c.Get("traceHeader").(string)
trace := c.Get("trace").(string)
// We create a context with a timeout so that underlying processes are // We create a context with a timeout so that underlying processes are
// able to stop early and handle correctly a timeout scenario. // able to stop early and handle correctly a timeout scenario.
ctx, cancel, err := newContext(c, logger, fs, timeout) ctx, cancel, err := newContext(c, logger, fs, timeout, bodyLimit, downloadFromCfg, traceHeader, trace)
if err != nil { if err != nil {
cancel() cancel()

View File

@@ -458,10 +458,11 @@ func TestContextMiddleware(t *testing.T) {
c := srv.NewContext(tc.request, recorder) c := srv.NewContext(tc.request, recorder)
c.Set("logger", zap.NewNop()) c.Set("logger", zap.NewNop())
c.Set("traceHeader", "Gotenberg-Trace")
c.Set("trace", "foo") c.Set("trace", "foo")
c.Set("startTime", time.Now()) c.Set("startTime", time.Now())
err := contextMiddleware(gotenberg.NewFileSystem(), time.Duration(10)*time.Second)(tc.next)(c) err := contextMiddleware(gotenberg.NewFileSystem(), time.Duration(10)*time.Second, 0, downloadFromConfig{})(tc.next)(c)
if tc.expectErr && err == nil { if tc.expectErr && err == nil {
t.Errorf("test %d: expected error but got: %v", i, err) t.Errorf("test %d: expected error but got: %v", i, err)

View File

@@ -315,12 +315,14 @@ func (b *chromiumBrowser) do(ctx context.Context, logger *zap.Logger, url string
} }
var ( var (
connectionRefused error loadingFailed error
connectionRefusedMu sync.RWMutex loadingFailedMu sync.RWMutex
) )
// See https://github.com/gotenberg/gotenberg/issues/913. // See:
listenForEventLoadingFailedOnConnectionRefused(taskCtx, logger, &connectionRefused, &connectionRefusedMu) // https://github.com/gotenberg/gotenberg/issues/913
// https://github.com/gotenberg/gotenberg/issues/959
listenForEventLoadingFailed(taskCtx, logger, &loadingFailed, &loadingFailedMu)
err = chromedp.Run(taskCtx, tasks...) err = chromedp.Run(taskCtx, tasks...)
if err != nil { if err != nil {
@@ -357,12 +359,14 @@ func (b *chromiumBrowser) do(ctx context.Context, logger *zap.Logger, url string
return fmt.Errorf("%v: %w", consoleExceptions, ErrConsoleExceptions) return fmt.Errorf("%v: %w", consoleExceptions, ErrConsoleExceptions)
} }
// See https://github.com/gotenberg/gotenberg/issues/913. // See:
connectionRefusedMu.RLock() // https://github.com/gotenberg/gotenberg/issues/913
defer connectionRefusedMu.RUnlock() // https://github.com/gotenberg/gotenberg/issues/959
loadingFailedMu.RLock()
defer loadingFailedMu.RUnlock()
if connectionRefused != nil { if loadingFailed != nil {
return fmt.Errorf("%v: %w", connectionRefused, ErrConnectionRefused) return fmt.Errorf("%v: %w", loadingFailed, ErrLoadingFailed)
} }
return nil return nil

View File

@@ -379,7 +379,7 @@ func TestChromiumBrowser_pdf(t *testing.T) {
}, },
}, },
{ {
scenario: "skip networkIdle event", scenario: "do not skip networkIdle event",
browser: newChromiumBrowser( browser: newChromiumBrowser(
browserArguments{ browserArguments{
binPath: os.Getenv("CHROMIUM_BIN_PATH"), binPath: os.Getenv("CHROMIUM_BIN_PATH"),
@@ -404,13 +404,13 @@ func TestChromiumBrowser_pdf(t *testing.T) {
return fs return fs
}(), }(),
options: PdfOptions{ options: PdfOptions{
Options: Options{SkipNetworkIdleEvent: true}, Options: Options{SkipNetworkIdleEvent: false},
}, },
noDeadline: false, noDeadline: false,
start: true, start: true,
expectError: false, expectError: false,
expectedLogEntries: []string{ expectedLogEntries: []string{
"skipping network idle event", "event networkIdle fired",
}, },
}, },
{ {
@@ -480,7 +480,7 @@ func TestChromiumBrowser_pdf(t *testing.T) {
expectedError: ErrConsoleExceptions, expectedError: ErrConsoleExceptions,
}, },
{ {
scenario: "ErrConnectionRefused", scenario: "ErrLoadingFailed",
browser: newChromiumBrowser( browser: newChromiumBrowser(
browserArguments{ browserArguments{
binPath: os.Getenv("CHROMIUM_BIN_PATH"), binPath: os.Getenv("CHROMIUM_BIN_PATH"),
@@ -503,7 +503,7 @@ func TestChromiumBrowser_pdf(t *testing.T) {
noDeadline: false, noDeadline: false,
start: true, start: true,
expectError: true, expectError: true,
expectedError: ErrConnectionRefused, expectedError: ErrLoadingFailed,
}, },
{ {
scenario: "clear cache", scenario: "clear cache",
@@ -1225,6 +1225,7 @@ func TestChromiumBrowser_pdf(t *testing.T) {
"no cookies to set", "no cookies to set",
"no extra HTTP headers", "no extra HTTP headers",
"navigate to", "navigate to",
"skipping network idle event",
"default white background not hidden", "default white background not hidden",
"no emulated media type", "no emulated media type",
"no wait delay", "no wait delay",
@@ -1452,7 +1453,7 @@ func TestChromiumBrowser_screenshot(t *testing.T) {
}, },
}, },
{ {
scenario: "skip networkIdle event", scenario: "do not skip networkIdle event",
browser: newChromiumBrowser( browser: newChromiumBrowser(
browserArguments{ browserArguments{
binPath: os.Getenv("CHROMIUM_BIN_PATH"), binPath: os.Getenv("CHROMIUM_BIN_PATH"),
@@ -1477,13 +1478,13 @@ func TestChromiumBrowser_screenshot(t *testing.T) {
return fs return fs
}(), }(),
options: ScreenshotOptions{ options: ScreenshotOptions{
Options: Options{SkipNetworkIdleEvent: true}, Options: Options{SkipNetworkIdleEvent: false},
}, },
noDeadline: false, noDeadline: false,
start: true, start: true,
expectError: false, expectError: false,
expectedLogEntries: []string{ expectedLogEntries: []string{
"skipping network idle event", "event networkIdle fired",
}, },
}, },
{ {
@@ -1553,7 +1554,7 @@ func TestChromiumBrowser_screenshot(t *testing.T) {
expectedError: ErrConsoleExceptions, expectedError: ErrConsoleExceptions,
}, },
{ {
scenario: "ErrConnectionRefused", scenario: "ErrLoadingFailed",
browser: newChromiumBrowser( browser: newChromiumBrowser(
browserArguments{ browserArguments{
binPath: os.Getenv("CHROMIUM_BIN_PATH"), binPath: os.Getenv("CHROMIUM_BIN_PATH"),
@@ -1577,7 +1578,7 @@ func TestChromiumBrowser_screenshot(t *testing.T) {
noDeadline: false, noDeadline: false,
start: true, start: true,
expectError: true, expectError: true,
expectedError: ErrConnectionRefused, expectedError: ErrLoadingFailed,
}, },
{ {
scenario: "clear cache", scenario: "clear cache",
@@ -2184,6 +2185,7 @@ func TestChromiumBrowser_screenshot(t *testing.T) {
"no user agent override", "no user agent override",
"no extra HTTP headers", "no extra HTTP headers",
"navigate to", "navigate to",
"skipping network idle event",
"default white background not hidden", "default white background not hidden",
"no emulated media type", "no emulated media type",
"no wait delay", "no wait delay",

View File

@@ -42,8 +42,8 @@ var (
// is set to true. // is set to true.
ErrConsoleExceptions = errors.New("console exceptions") ErrConsoleExceptions = errors.New("console exceptions")
// ErrConnectionRefused happens when a URL cannot be reached. // ErrLoadingFailed happens when a URL failed to load.
ErrConnectionRefused = errors.New("connection refused") ErrLoadingFailed = errors.New("loading failed")
// PDF specific. // PDF specific.
@@ -123,7 +123,7 @@ type Options struct {
// DefaultOptions returns the default values for Options. // DefaultOptions returns the default values for Options.
func DefaultOptions() Options { func DefaultOptions() Options {
return Options{ return Options{
SkipNetworkIdleEvent: false, SkipNetworkIdleEvent: true,
FailOnHttpStatusCodes: []int64{499, 599}, FailOnHttpStatusCodes: []int64{499, 599},
FailOnConsoleExceptions: false, FailOnConsoleExceptions: false,
WaitDelay: 0, WaitDelay: 0,

View File

@@ -21,7 +21,8 @@ import (
) )
// listenForEventRequestPaused listens for requests to check if they are // listenForEventRequestPaused listens for requests to check if they are
// allowed or not. // allowed or not.network.SetBlockedURLS()
// TODO: https://chromedevtools.github.io/devtools-protocol/tot/Network/#method-setBlockedURLs (experimental for now).
func listenForEventRequestPaused(ctx context.Context, logger *zap.Logger, allowList *regexp2.Regexp, denyList *regexp2.Regexp) { func listenForEventRequestPaused(ctx context.Context, logger *zap.Logger, allowList *regexp2.Regexp, denyList *regexp2.Regexp) {
chromedp.ListenTarget(ctx, func(ev interface{}) { chromedp.ListenTarget(ctx, func(ev interface{}) {
switch e := ev.(type) { switch e := ev.(type) {
@@ -95,24 +96,48 @@ func listenForEventResponseReceived(ctx context.Context, logger *zap.Logger, url
}) })
} }
// listenForEventLoadingFailedOnConnectionRefused listens for an event // listenForEventLoadingFailed listens for an event indicating that the main
// indicating that the main page failed to load. // page failed to load.
// See https://github.com/gotenberg/gotenberg/issues/913. // See:
func listenForEventLoadingFailedOnConnectionRefused(ctx context.Context, logger *zap.Logger, connectionRefused *error, connectionRefusedMu *sync.RWMutex) { // https://github.com/gotenberg/gotenberg/issues/913.
// https://github.com/gotenberg/gotenberg/issues/959.
func listenForEventLoadingFailed(ctx context.Context, logger *zap.Logger, loadingFailed *error, loadingFailedMu *sync.RWMutex) {
chromedp.ListenTarget(ctx, func(ev interface{}) { chromedp.ListenTarget(ctx, func(ev interface{}) {
switch ev := ev.(type) { switch ev := ev.(type) {
case *network.EventLoadingFailed: case *network.EventLoadingFailed:
logger.Debug(fmt.Sprintf("event EventLoadingFailed fired: %+v", ev.ErrorText)) logger.Debug(fmt.Sprintf("event EventLoadingFailed fired: %+v", ev.ErrorText))
if ev.ErrorText != "net::ERR_CONNECTION_REFUSED" || ev.Type != network.ResourceTypeDocument { if ev.Type != network.ResourceTypeDocument {
logger.Debug("skip EventLoadingFailed: is not net::ERR_CONNECTION_REFUSED and/or resource type Document") logger.Debug("skip EventLoadingFailed: is not resource type Document")
return return
} }
connectionRefusedMu.Lock() // Supposition: except iframe, an event loading failed with a
defer connectionRefusedMu.Unlock() // resource type Document is about the main page.
*connectionRefused = fmt.Errorf("%s", ev.ErrorText) // We are looking for common errors.
// TODO: sufficient?
errors := []string{
"net::ERR_CONNECTION_CLOSED",
"net::ERR_CONNECTION_RESET",
"net::ERR_CONNECTION_REFUSED",
"net::ERR_CONNECTION_ABORTED",
"net::ERR_CONNECTION_FAILED",
"net::ERR_NAME_NOT_RESOLVED",
"net::ERR_INTERNET_DISCONNECTED",
"net::ERR_ADDRESS_UNREACHABLE",
"net::ERR_BLOCKED_BY_CLIENT",
"net::ERR_BLOCKED_BY_RESPONSE",
}
if !slices.Contains(errors, ev.ErrorText) {
logger.Debug(fmt.Sprintf("skip EventLoadingFailed: '%s' is not part of %+v", ev.ErrorText, errors))
return
}
loadingFailedMu.Lock()
defer loadingFailedMu.Unlock()
*loadingFailed = fmt.Errorf("%s", ev.ErrorText)
} }
}) })
} }

View File

@@ -688,12 +688,12 @@ func handleChromiumError(err error, options Options) error {
) )
} }
if errors.Is(err, ErrConnectionRefused) { if errors.Is(err, ErrLoadingFailed) {
return api.WrapError( return api.WrapError(
err, err,
api.NewSentinelHttpError( api.NewSentinelHttpError(
http.StatusBadRequest, http.StatusBadRequest,
"Chromium returned net::ERR_CONNECTION_REFUSED", fmt.Sprintf("Chromium returned %v", err),
), ),
) )
} }

View File

@@ -1436,10 +1436,10 @@ func TestConvertUrl(t *testing.T) {
expectOutputPathsCount: 0, expectOutputPathsCount: 0,
}, },
{ {
scenario: "ErrConnectionRefused", scenario: "ErrLoadingFailed",
ctx: &api.ContextMock{Context: new(api.Context)}, ctx: &api.ContextMock{Context: new(api.Context)},
api: &ApiMock{PdfMock: func(ctx context.Context, logger *zap.Logger, url, outputPath string, options PdfOptions) error { api: &ApiMock{PdfMock: func(ctx context.Context, logger *zap.Logger, url, outputPath string, options PdfOptions) error {
return ErrConnectionRefused return ErrLoadingFailed
}}, }},
options: DefaultPdfOptions(), options: DefaultPdfOptions(),
expectError: true, expectError: true,
@@ -1646,10 +1646,10 @@ func TestScreenshotUrl(t *testing.T) {
expectOutputPathsCount: 0, expectOutputPathsCount: 0,
}, },
{ {
scenario: "ErrConnectionRefused", scenario: "ErrLoadingFailed",
ctx: &api.ContextMock{Context: new(api.Context)}, ctx: &api.ContextMock{Context: new(api.Context)},
api: &ApiMock{ScreenshotMock: func(ctx context.Context, logger *zap.Logger, url, outputPath string, options ScreenshotOptions) error { api: &ApiMock{ScreenshotMock: func(ctx context.Context, logger *zap.Logger, url, outputPath string, options ScreenshotOptions) error {
return ErrConnectionRefused return ErrLoadingFailed
}}, }},
options: DefaultScreenshotOptions(), options: DefaultScreenshotOptions(),
expectError: true, expectError: true,

View File

@@ -25,9 +25,11 @@ var (
// by LibreOffice. // by LibreOffice.
ErrInvalidPdfFormats = errors.New("invalid PDF formats") ErrInvalidPdfFormats = errors.New("invalid PDF formats")
// ErrMalformedPageRanges happens if the page ranges option cannot be // ErrUnoException happens when unoconverter returns an exit code 5.
// interpreted by LibreOffice. ErrUnoException = errors.New("uno exception")
ErrMalformedPageRanges = errors.New("page ranges are malformed")
// ErrRuntimeException happens when unoconverter returns an exit code 6.
ErrRuntimeException = errors.New("uno exception")
// ErrCoreDumped happens randomly; sometime a conversion will work as // ErrCoreDumped happens randomly; sometime a conversion will work as
// expected, and some other time the same conversion will fail. // expected, and some other time the same conversion will fail.
@@ -48,6 +50,9 @@ type Api struct {
// Options gathers available options when converting a document to PDF. // Options gathers available options when converting a document to PDF.
// See: https://help.libreoffice.org/latest/en-US/text/shared/guide/pdf_params.html. // See: https://help.libreoffice.org/latest/en-US/text/shared/guide/pdf_params.html.
type Options struct { type Options struct {
// Password specifies the password for opening the source file.
Password string
// Landscape allows to change the orientation of the resulting PDF. // Landscape allows to change the orientation of the resulting PDF.
Landscape bool Landscape bool
@@ -141,6 +146,7 @@ type Options struct {
// DefaultOptions returns the default values for Options. // DefaultOptions returns the default values for Options.
func DefaultOptions() Options { func DefaultOptions() Options {
return Options{ return Options{
Password: "",
Landscape: false, Landscape: false,
PageRanges: "", PageRanges: "",
ExportFormFields: true, ExportFormFields: true,
@@ -380,6 +386,7 @@ func (a *Api) Pdf(ctx context.Context, logger *zap.Logger, inputPath, outputPath
// See https://github.com/gotenberg/gotenberg/issues/639. // See https://github.com/gotenberg/gotenberg/issues/639.
if errors.Is(err, ErrCoreDumped) { if errors.Is(err, ErrCoreDumped) {
logger.Debug(fmt.Sprintf("got a '%s' error, retry conversion", err))
return a.Pdf(ctx, logger, inputPath, outputPath, options) return a.Pdf(ctx, logger, inputPath, outputPath, options)
} }

View File

@@ -266,6 +266,10 @@ func (p *libreOfficeProcess) pdf(ctx context.Context, logger *zap.Logger, inputP
args = append(args, "-vvv") args = append(args, "-vvv")
} }
if options.Password != "" {
args = append(args, "--password", options.Password)
}
if options.Landscape { if options.Landscape {
args = append(args, "--printer", "PaperOrientation=landscape") args = append(args, "--printer", "PaperOrientation=landscape")
} }
@@ -343,11 +347,8 @@ func (p *libreOfficeProcess) pdf(ctx context.Context, logger *zap.Logger, inputP
} }
// LibreOffice's errors are not explicit. // LibreOffice's errors are not explicit.
// That's why we have to make an educated guess according to the exit code // For instance, an exit code 5 may be explained by a malformed page
// and given inputs. // ranges, but also by a not required password.
if exitCode == 5 && options.PageRanges != "" {
return ErrMalformedPageRanges
}
// We may want to retry in case of a core dumped event. // We may want to retry in case of a core dumped event.
// See https://github.com/gotenberg/gotenberg/issues/639. // See https://github.com/gotenberg/gotenberg/issues/639.
@@ -355,6 +356,15 @@ func (p *libreOfficeProcess) pdf(ctx context.Context, logger *zap.Logger, inputP
return ErrCoreDumped return ErrCoreDumped
} }
if exitCode == 5 {
// Potentially malformed page ranges or password not required.
return ErrUnoException
}
if exitCode == 6 {
// Password potentially required or invalid.
return ErrRuntimeException
}
// Possible errors: // Possible errors:
// 1. LibreOffice failed for some reason. // 1. LibreOffice failed for some reason.
// 2. Context done. // 2. Context done.

View File

@@ -4,6 +4,7 @@ import (
"context" "context"
"errors" "errors"
"fmt" "fmt"
"io"
"os" "os"
"testing" "testing"
"time" "time"
@@ -250,7 +251,7 @@ func TestLibreOfficeProcess_pdf(t *testing.T) {
expectedError: ErrInvalidPdfFormats, expectedError: ErrInvalidPdfFormats,
}, },
{ {
scenario: "ErrMalformedPageRanges", scenario: "ErrUnoException",
libreOffice: newLibreOfficeProcess( libreOffice: newLibreOfficeProcess(
libreOfficeArguments{ libreOfficeArguments{
binPath: os.Getenv("LIBREOFFICE_BIN_PATH"), binPath: os.Getenv("LIBREOFFICE_BIN_PATH"),
@@ -267,7 +268,7 @@ func TestLibreOfficeProcess_pdf(t *testing.T) {
t.Fatalf(fmt.Sprintf("expected no error but got: %v", err)) t.Fatalf(fmt.Sprintf("expected no error but got: %v", err))
} }
err = os.WriteFile(fmt.Sprintf("%s/document.txt", fs.WorkingDirPath()), []byte("ErrMalformedPageRanges"), 0o755) err = os.WriteFile(fmt.Sprintf("%s/document.txt", fs.WorkingDirPath()), []byte("Context done"), 0o755)
if err != nil { if err != nil {
t.Fatalf("expected no error but got: %v", err) t.Fatalf("expected no error but got: %v", err)
} }
@@ -277,7 +278,61 @@ func TestLibreOfficeProcess_pdf(t *testing.T) {
cancelledCtx: false, cancelledCtx: false,
start: true, start: true,
expectError: true, expectError: true,
expectedError: ErrMalformedPageRanges, expectedError: ErrUnoException,
},
{
scenario: "ErrRuntimeException",
libreOffice: newLibreOfficeProcess(
libreOfficeArguments{
binPath: os.Getenv("LIBREOFFICE_BIN_PATH"),
unoBinPath: os.Getenv("UNOCONVERTER_BIN_PATH"),
startTimeout: 5 * time.Second,
},
),
options: Options{Password: "foo"},
fs: func() *gotenberg.FileSystem {
fs := gotenberg.NewFileSystem()
err := os.MkdirAll(fs.WorkingDirPath(), 0o755)
if err != nil {
t.Fatalf(fmt.Sprintf("expected no error but got: %v", err))
}
in, err := os.Open("/tests/test/testdata/libreoffice/protected.docx")
if err != nil {
t.Fatalf(fmt.Sprintf("expected no error but got: %v", err))
}
defer func() {
err := in.Close()
if err != nil {
t.Fatalf(fmt.Sprintf("expected no error but got: %v", err))
}
}()
out, err := os.Create(fmt.Sprintf("%s/protected.docx", fs.WorkingDirPath()))
if err != nil {
t.Fatalf(fmt.Sprintf("expected no error but got: %v", err))
}
defer func() {
err := out.Close()
if err != nil {
t.Fatalf(fmt.Sprintf("expected no error but got: %v", err))
}
}()
_, err = io.Copy(out, in)
if err != nil {
t.Fatalf(fmt.Sprintf("expected no error but got: %v", err))
}
return fs
}(),
cancelledCtx: false,
start: true,
expectError: true,
expectedError: ErrRuntimeException,
}, },
{ {
scenario: "context done", scenario: "context done",
@@ -360,6 +415,7 @@ func TestLibreOfficeProcess_pdf(t *testing.T) {
return fs return fs
}(), }(),
options: Options{ options: Options{
Password: "", // Ok, the only exception in this list.
Landscape: true, Landscape: true,
PageRanges: "1", PageRanges: "1",
ExportFormFields: false, ExportFormFields: false,

View File

@@ -29,6 +29,7 @@ func convertRoute(libreOffice libreofficeapi.Uno, engine gotenberg.PdfEngine) ap
// Let's get the data from the form and validate them. // Let's get the data from the form and validate them.
var ( var (
inputPaths []string inputPaths []string
password string
landscape bool landscape bool
nativePageRanges string nativePageRanges string
exportFormFields bool exportFormFields bool
@@ -59,6 +60,7 @@ func convertRoute(libreOffice libreofficeapi.Uno, engine gotenberg.PdfEngine) ap
err := ctx.FormData(). err := ctx.FormData().
MandatoryPaths(libreOffice.Extensions(), &inputPaths). MandatoryPaths(libreOffice.Extensions(), &inputPaths).
String("password", &password, defaultOptions.Password).
Bool("landscape", &landscape, defaultOptions.Landscape). Bool("landscape", &landscape, defaultOptions.Landscape).
String("nativePageRanges", &nativePageRanges, defaultOptions.PageRanges). String("nativePageRanges", &nativePageRanges, defaultOptions.PageRanges).
Bool("exportFormFields", &exportFormFields, defaultOptions.ExportFormFields). Bool("exportFormFields", &exportFormFields, defaultOptions.ExportFormFields).
@@ -146,6 +148,7 @@ func convertRoute(libreOffice libreofficeapi.Uno, engine gotenberg.PdfEngine) ap
for i, inputPath := range inputPaths { for i, inputPath := range inputPaths {
outputPaths[i] = ctx.GeneratePath(".pdf") outputPaths[i] = ctx.GeneratePath(".pdf")
options := libreofficeapi.Options{ options := libreofficeapi.Options{
Password: password,
Landscape: landscape, Landscape: landscape,
PageRanges: nativePageRanges, PageRanges: nativePageRanges,
ExportFormFields: exportFormFields, ExportFormFields: exportFormFields,
@@ -185,10 +188,17 @@ func convertRoute(libreOffice libreofficeapi.Uno, engine gotenberg.PdfEngine) ap
) )
} }
if errors.Is(err, libreofficeapi.ErrMalformedPageRanges) { if errors.Is(err, libreofficeapi.ErrUnoException) {
return api.WrapError( return api.WrapError(
fmt.Errorf("convert to PDF: %w", err), fmt.Errorf("convert to PDF: %w", err),
api.NewSentinelHttpError(http.StatusBadRequest, fmt.Sprintf("Malformed page ranges '%s' (nativePageRanges)", options.PageRanges)), api.NewSentinelHttpError(http.StatusBadRequest, fmt.Sprintf("LibreOffice failed to process a document: possible causes include malformed page ranges '%s' (nativePageRanges), or, if a password has been provided, it may not be required. In any case, the exact cause is uncertain.", options.PageRanges)),
)
}
if errors.Is(err, libreofficeapi.ErrRuntimeException) {
return api.WrapError(
fmt.Errorf("convert to PDF: %w", err),
api.NewSentinelHttpError(http.StatusBadRequest, "LibreOffice failed to process a document: a password may be required, or, if one has been given, it is invalid. In any case, the exact cause is uncertain."),
) )
} }

View File

@@ -194,14 +194,14 @@ func TestConvertRoute(t *testing.T) {
expectOutputPathsCount: 0, expectOutputPathsCount: 0,
}, },
{ {
scenario: "ErrMalformedPageRanges", scenario: "ErrUnoException",
ctx: func() *api.ContextMock { ctx: func() *api.ContextMock {
ctx := &api.ContextMock{Context: new(api.Context)} ctx := &api.ContextMock{Context: new(api.Context)}
ctx.SetFiles(map[string]string{ ctx.SetFiles(map[string]string{
"document.docx": "/document.docx", "document.docx": "/document.docx",
}) })
ctx.SetValues(map[string][]string{ ctx.SetValues(map[string][]string{
"pdfa": { "nativePageRanges": {
"foo", "foo",
}, },
}) })
@@ -209,7 +209,34 @@ func TestConvertRoute(t *testing.T) {
}(), }(),
libreOffice: &libreofficeapi.ApiMock{ libreOffice: &libreofficeapi.ApiMock{
PdfMock: func(ctx context.Context, logger *zap.Logger, inputPath, outputPath string, options libreofficeapi.Options) error { PdfMock: func(ctx context.Context, logger *zap.Logger, inputPath, outputPath string, options libreofficeapi.Options) error {
return libreofficeapi.ErrMalformedPageRanges return libreofficeapi.ErrUnoException
},
ExtensionsMock: func() []string {
return []string{".docx"}
},
},
expectError: true,
expectHttpError: true,
expectHttpStatus: http.StatusBadRequest,
expectOutputPathsCount: 0,
},
{
scenario: "ErrRuntimeException",
ctx: func() *api.ContextMock {
ctx := &api.ContextMock{Context: new(api.Context)}
ctx.SetFiles(map[string]string{
"document.docx": "/document.docx",
})
ctx.SetValues(map[string][]string{
"password": {
"invalid",
},
})
return ctx
}(),
libreOffice: &libreofficeapi.ApiMock{
PdfMock: func(ctx context.Context, logger *zap.Logger, inputPath, outputPath string, options libreofficeapi.Options) error {
return libreofficeapi.ErrRuntimeException
}, },
ExtensionsMock: func() []string { ExtensionsMock: func() []string {
return []string{".docx"} return []string{".docx"}

View File

@@ -27,9 +27,9 @@ type client struct {
// send call the webhook either to send the success response or the error response. // send call the webhook either to send the success response or the error response.
func (c client) send(body io.Reader, headers map[string]string, erroed bool) error { func (c client) send(body io.Reader, headers map[string]string, erroed bool) error {
URL := c.url url := c.url
if erroed { if erroed {
URL = c.errorUrl url = c.errorUrl
} }
method := c.method method := c.method
@@ -37,9 +37,9 @@ func (c client) send(body io.Reader, headers map[string]string, erroed bool) err
method = c.errorMethod method = c.errorMethod
} }
req, err := retryablehttp.NewRequest(method, URL, body) req, err := retryablehttp.NewRequest(method, url, body)
if err != nil { if err != nil {
return fmt.Errorf("create '%s' request to '%s': %w", method, URL, err) return fmt.Errorf("create '%s' request to '%s': %w", method, url, err)
} }
req.Header.Set("User-Agent", "Gotenberg") req.Header.Set("User-Agent", "Gotenberg")
@@ -75,17 +75,17 @@ func (c client) send(body io.Reader, headers map[string]string, erroed bool) err
resp, err := c.client.Do(req) resp, err := c.client.Do(req)
if err != nil { if err != nil {
return fmt.Errorf("send '%s' request to '%s': %w", method, URL, err) return fmt.Errorf("send '%s' request to '%s': %w", method, url, err)
} }
if resp.StatusCode >= http.StatusBadRequest { if resp.StatusCode >= http.StatusBadRequest {
return fmt.Errorf("send '%s' request to '%s': got status: '%s'", method, URL, resp.Status) return fmt.Errorf("send '%s' request to '%s': got status: '%s'", method, url, resp.Status)
} }
defer func() { defer func() {
err := resp.Body.Close() err := resp.Body.Close()
if err != nil { if err != nil {
c.logger.Error(fmt.Sprintf("close response body from '%s': %s", URL, err)) c.logger.Error(fmt.Sprintf("close response body from '%s': %s", url, err))
} }
}() }()
@@ -94,7 +94,7 @@ func (c client) send(body io.Reader, headers map[string]string, erroed bool) err
// Now let's log! // Now let's log!
fields := make([]zap.Field, 5) fields := make([]zap.Field, 5)
fields[0] = zap.String("webhook_url", URL) fields[0] = zap.String("webhook_url", url)
fields[1] = zap.String("method", method) fields[1] = zap.String("method", method)
fields[2] = zap.Int64("latency", int64(finishTime.Sub(c.startTime))) fields[2] = zap.Int64("latency", int64(finishTime.Sub(c.startTime)))
fields[3] = zap.String("latency_human", finishTime.Sub(c.startTime).String()) fields[3] = zap.String("latency_human", finishTime.Sub(c.startTime).String())
@@ -110,34 +110,3 @@ func (c client) send(body io.Reader, headers map[string]string, erroed bool) err
return nil return nil
} }
// leveledLogger is wrapper around a [zap.Logger] which is used by the
// [retryablehttp.Client].
type leveledLogger struct {
logger *zap.Logger
}
// Error logs a message at error level using the wrapped zap.Logger.
func (leveled leveledLogger) Error(msg string, keysAndValues ...interface{}) {
leveled.logger.Error(fmt.Sprintf("%s: %+v", msg, keysAndValues))
}
// Warn logs a message at warning level using the wrapped zap.Logger.
func (leveled leveledLogger) Warn(msg string, keysAndValues ...interface{}) {
leveled.logger.Warn(fmt.Sprintf("%s: %+v", msg, keysAndValues))
}
// Info logs a message at info level using the wrapped zap.Logger.
func (leveled leveledLogger) Info(msg string, keysAndValues ...interface{}) {
leveled.logger.Info(fmt.Sprintf("%s: %+v", msg, keysAndValues))
}
// Debug logs a message at debug level using the wrapped zap.Logger.
func (leveled leveledLogger) Debug(msg string, keysAndValues ...interface{}) {
leveled.logger.Debug(fmt.Sprintf("%s: %+v", msg, keysAndValues))
}
// Interface guards.
var (
_ retryablehttp.LeveledLogger = (*leveledLogger)(nil)
)

View File

@@ -100,11 +100,11 @@ func webhookMiddleware(w *Webhook) api.Middleware {
} }
// What about extra HTTP headers? // What about extra HTTP headers?
var extraHTTPHeaders map[string]string var extraHttpHeaders map[string]string
extraHTTPHeadersJSON := c.Request().Header.Get("Gotenberg-Webhook-Extra-Http-Headers") extraHttpHeadersJson := c.Request().Header.Get("Gotenberg-Webhook-Extra-Http-Headers")
if extraHTTPHeadersJSON != "" { if extraHttpHeadersJson != "" {
err = json.Unmarshal([]byte(extraHTTPHeadersJSON), &extraHTTPHeaders) err = json.Unmarshal([]byte(extraHttpHeadersJson), &extraHttpHeaders)
if err != nil { if err != nil {
return api.WrapError( return api.WrapError(
fmt.Errorf("unmarshal webhook extra HTTP headers: %w", err), fmt.Errorf("unmarshal webhook extra HTTP headers: %w", err),
@@ -113,13 +113,19 @@ func webhookMiddleware(w *Webhook) api.Middleware {
} }
} }
// Retrieve values from echo.Context before it get recycled.
// See https://github.com/gotenberg/gotenberg/issues/1000.
startTime := c.Get("startTime").(time.Time)
traceHeader := c.Get("traceHeader").(string)
trace := c.Get("trace").(string)
client := &client{ client := &client{
url: webhookUrl, url: webhookUrl,
method: webhookMethod, method: webhookMethod,
errorUrl: webhookErrorUrl, errorUrl: webhookErrorUrl,
errorMethod: webhookErrorMethod, errorMethod: webhookErrorMethod,
extraHttpHeaders: extraHTTPHeaders, extraHttpHeaders: extraHttpHeaders,
startTime: c.Get("startTime").(time.Time), startTime: startTime,
client: &retryablehttp.Client{ client: &retryablehttp.Client{
HTTPClient: &http.Client{ HTTPClient: &http.Client{
@@ -128,11 +134,9 @@ func webhookMiddleware(w *Webhook) api.Middleware {
RetryMax: w.maxRetry, RetryMax: w.maxRetry,
RetryWaitMin: w.retryMinWait, RetryWaitMin: w.retryMinWait,
RetryWaitMax: w.retryMaxWait, RetryWaitMax: w.retryMaxWait,
Logger: leveledLogger{ Logger: gotenberg.NewLeveledLogger(ctx.Log()),
logger: ctx.Log(), CheckRetry: retryablehttp.DefaultRetryPolicy,
}, Backoff: retryablehttp.DefaultBackoff,
CheckRetry: retryablehttp.DefaultRetryPolicy,
Backoff: retryablehttp.DefaultBackoff,
}, },
logger: ctx.Log(), logger: ctx.Log(),
} }
@@ -159,8 +163,8 @@ func webhookMiddleware(w *Webhook) api.Middleware {
} }
headers := map[string]string{ headers := map[string]string{
echo.HeaderContentType: echo.MIMEApplicationJSON, echo.HeaderContentType: echo.MIMEApplicationJSON,
c.Get("traceHeader").(string): c.Get("trace").(string), traceHeader: trace,
} }
err = client.send(bytes.NewReader(b), headers, true) err = client.send(bytes.NewReader(b), headers, true)
@@ -238,7 +242,7 @@ func webhookMiddleware(w *Webhook) api.Middleware {
echo.HeaderContentDisposition: fmt.Sprintf("attachement; filename=%q", ctx.OutputFilename(outputPath)), echo.HeaderContentDisposition: fmt.Sprintf("attachement; filename=%q", ctx.OutputFilename(outputPath)),
echo.HeaderContentType: http.DetectContentType(fileHeader), echo.HeaderContentType: http.DetectContentType(fileHeader),
echo.HeaderContentLength: strconv.FormatInt(fileStat.Size(), 10), echo.HeaderContentLength: strconv.FormatInt(fileStat.Size(), 10),
c.Get("traceHeader").(string): c.Get("trace").(string), traceHeader: trace,
} }
// Send the output file to the webhook. // Send the output file to the webhook.

View File

@@ -2,50 +2,60 @@
set -e set -e
# Args.
GOLANG_VERSION="$1" GOLANG_VERSION="$1"
GOTENBERG_VERSION="$2" GOTENBERG_VERSION="$2"
GOTENBERG_USER_GID="$3" GOTENBERG_USER_GID="$3"
GOTENBERG_USER_UID="$4" GOTENBERG_USER_UID="$4"
NOTO_COLOR_EMOJI_VERSION="$5" NOTO_COLOR_EMOJI_VERSION="$5"
PDFTK_VERSION="$6" PDFTK_VERSION="$6"
DOCKER_REPOSITORY="$7" DOCKER_REGISTRY="$7"
DOCKER_REPOSITORY="$8"
if [ "$GOTENBERG_VERSION" == "edge" ]; then LINUX_AMD64_RELEASE="$9"
docker buildx build \
--build-arg GOLANG_VERSION="$GOLANG_VERSION" \
--build-arg GOTENBERG_VERSION="$GOTENBERG_VERSION" \
--build-arg GOTENBERG_USER_GID="$GOTENBERG_USER_GID" \
--build-arg GOTENBERG_USER_UID="$GOTENBERG_USER_UID" \
--build-arg NOTO_COLOR_EMOJI_VERSION="$NOTO_COLOR_EMOJI_VERSION" \
--build-arg PDFTK_VERSION="$PDFTK_VERSION" \
--platform linux/amd64 \
--platform linux/arm64 \
--platform linux/386 \
--platform linux/arm/v7 \
-t "$DOCKER_REPOSITORY/gotenberg:edge" \
--push \
-f build/Dockerfile .
# Cloud Run variant.
# Only linux/amd64! See https://github.com/gotenberg/gotenberg/issues/505#issuecomment-1264679278.
docker buildx build \
--build-arg DOCKER_REPOSITORY="$DOCKER_REPOSITORY" \
--build-arg GOTENBERG_VERSION="$GOTENBERG_VERSION" \
--platform linux/amd64 \
-t "$DOCKER_REPOSITORY/gotenberg:edge-cloudrun" \
--push \
-f build/Dockerfile.cloudrun .
exit 0
fi
# Find out if given version is "semver".
GOTENBERG_VERSION="${GOTENBERG_VERSION//v}" GOTENBERG_VERSION="${GOTENBERG_VERSION//v}"
IFS='.' read -ra SEMVER <<< "$GOTENBERG_VERSION" IFS='.' read -ra SEMVER <<< "$GOTENBERG_VERSION"
VERSION_LENGTH=${#SEMVER[@]} VERSION_LENGTH=${#SEMVER[@]}
TAGS=()
TAGS_CLOUD_RUN=()
if [ "$VERSION_LENGTH" -ne 3 ]; then if [ "$VERSION_LENGTH" -eq 3 ]; then
echo "$VERSION is not semver." MAJOR="${SEMVER[0]}"
exit 1 MINOR="${SEMVER[1]}"
PATCH="${SEMVER[2]}"
TAGS+=("-t" "$DOCKER_REGISTRY/$DOCKER_REPOSITORY:latest")
TAGS+=("-t" "$DOCKER_REGISTRY/$DOCKER_REPOSITORY:$MAJOR")
TAGS+=("-t" "$DOCKER_REGISTRY/$DOCKER_REPOSITORY:$MAJOR.$MINOR")
TAGS+=("-t" "$DOCKER_REGISTRY/$DOCKER_REPOSITORY:$MAJOR.$MINOR.$PATCH")
TAGS_CLOUD_RUN+=("-t" "$DOCKER_REGISTRY/$DOCKER_REPOSITORY:latest-cloudrun")
TAGS_CLOUD_RUN+=("-t" "$DOCKER_REGISTRY/$DOCKER_REPOSITORY:$MAJOR-cloudrun")
TAGS_CLOUD_RUN+=("-t" "$DOCKER_REGISTRY/$DOCKER_REPOSITORY:$MAJOR.$MINOR-cloudrun")
TAGS_CLOUD_RUN+=("-t" "$DOCKER_REGISTRY/$DOCKER_REPOSITORY:$MAJOR.$MINOR.$PATCH-cloudrun")
else
# Normalizes version.
GOTENBERG_VERSION="${GOTENBERG_VERSION// /-}"
GOTENBERG_VERSION="$(echo "$GOTENBERG_VERSION" | tr -cd '[:alnum:]._\-')"
if [[ "$GOTENBERG_VERSION" =~ ^[\.\-] ]]; then
GOTENBERG_VERSION="_${GOTENBERG_VERSION#?}"
fi
if [ "${#GOTENBERG_VERSION}" -gt 128 ]; then
GOTENBERG_VERSION="${GOTENBERG_VERSION:0:128}"
fi
TAGS+=("-t" "$DOCKER_REGISTRY/$DOCKER_REPOSITORY:$GOTENBERG_VERSION")
TAGS_CLOUD_RUN+=("-t" "$DOCKER_REGISTRY/$DOCKER_REPOSITORY:$GOTENBERG_VERSION-cloudrun")
fi
# Multi-arch build takes a lot of time.
if [ "$LINUX_AMD64_RELEASE" = true ]; then
PLATFORM_FLAG="--platform linux/amd64"
else
PLATFORM_FLAG="--platform linux/amd64,linux/arm64,linux/386,linux/arm/v7"
fi fi
docker buildx build \ docker buildx build \
@@ -55,26 +65,18 @@ docker buildx build \
--build-arg GOTENBERG_USER_UID="$GOTENBERG_USER_UID" \ --build-arg GOTENBERG_USER_UID="$GOTENBERG_USER_UID" \
--build-arg NOTO_COLOR_EMOJI_VERSION="$NOTO_COLOR_EMOJI_VERSION" \ --build-arg NOTO_COLOR_EMOJI_VERSION="$NOTO_COLOR_EMOJI_VERSION" \
--build-arg PDFTK_VERSION="$PDFTK_VERSION" \ --build-arg PDFTK_VERSION="$PDFTK_VERSION" \
--platform linux/amd64 \ $PLATFORM_FLAG \
--platform linux/arm64 \ "${TAGS[@]}" \
--platform linux/386 \
--platform linux/arm/v7 \
-t "$DOCKER_REPOSITORY/gotenberg:latest" \
-t "$DOCKER_REPOSITORY/gotenberg:${SEMVER[0]}" \
-t "$DOCKER_REPOSITORY/gotenberg:${SEMVER[0]}.${SEMVER[1]}" \
-t "$DOCKER_REPOSITORY/gotenberg:${SEMVER[0]}.${SEMVER[1]}.${SEMVER[2]}" \
--push \ --push \
-f build/Dockerfile . -f build/Dockerfile .
# Cloud Run variant. # Cloud Run variant.
# Only linux/amd64! See https://github.com/gotenberg/gotenberg/issues/505#issuecomment-1264679278. # Only linux/amd64! See https://github.com/gotenberg/gotenberg/issues/505#issuecomment-1264679278.
docker buildx build \ docker buildx build \
--build-arg DOCKER_REGISTRY="$DOCKER_REGISTRY" \
--build-arg DOCKER_REPOSITORY="$DOCKER_REPOSITORY" \ --build-arg DOCKER_REPOSITORY="$DOCKER_REPOSITORY" \
--build-arg GOTENBERG_VERSION="$GOTENBERG_VERSION" \ --build-arg GOTENBERG_VERSION="$GOTENBERG_VERSION" \
--platform linux/amd64 \ --platform linux/amd64 \
-t "$DOCKER_REPOSITORY/gotenberg:latest-cloudrun" \ "${TAGS_CLOUD_RUN[@]}" \
-t "$DOCKER_REPOSITORY/gotenberg:${SEMVER[0]}-cloudrun" \
-t "$DOCKER_REPOSITORY/gotenberg:${SEMVER[0]}.${SEMVER[1]}-cloudrun" \
-t "$DOCKER_REPOSITORY/gotenberg:${SEMVER[0]}.${SEMVER[1]}.${SEMVER[2]}-cloudrun" \
--push \ --push \
-f build/Dockerfile.cloudrun . -f build/Dockerfile.cloudrun .

View File

@@ -1,4 +1,5 @@
ARG GOLANG_VERSION ARG GOLANG_VERSION
ARG DOCKER_REGISTRY
ARG DOCKER_REPOSITORY ARG DOCKER_REPOSITORY
ARG GOTENBERG_VERSION ARG GOTENBERG_VERSION
ARG GOLANGCI_LINT_VERSION ARG GOLANGCI_LINT_VERSION
@@ -7,7 +8,7 @@ FROM golang:$GOLANG_VERSION-bookworm AS golang
# We're extending the Gotenberg's Docker image because our code relies on external # We're extending the Gotenberg's Docker image because our code relies on external
# dependencies like Google Chrome, LibreOffice, etc. # dependencies like Google Chrome, LibreOffice, etc.
FROM $DOCKER_REPOSITORY/gotenberg:$GOTENBERG_VERSION FROM $DOCKER_REGISTRY/$DOCKER_REPOSITORY:$GOTENBERG_VERSION
USER root USER root

BIN
test/testdata/libreoffice/protected.docx vendored Normal file

Binary file not shown.