diff --git a/Makefile b/Makefile index 17c77812..d21a286a 100644 --- a/Makefile +++ b/Makefile @@ -11,8 +11,8 @@ GOTENBERG_VERSION=snapshot GOTENBERG_USER_GID=1001 GOTENBERG_USER_UID=1001 NOTO_COLOR_EMOJI_VERSION=v2.028 # See https://github.com/googlefonts/noto-emoji/releases. -PDFTK_VERSION=1353200058 # See https://gitlab.com/pdftk-java/pdftk/-/releases - Binary package. -GOLANGCI_LINT_VERSION=v1.42.0 # See https://github.com/golangci/golangci-lint/releases. +PDFTK_VERSION=1527259628 # See https://gitlab.com/pdftk-java/pdftk/-/releases - Binary package. +GOLANGCI_LINT_VERSION=v1.42.1 # See https://github.com/golangci/golangci-lint/releases. .PHONY: build build: ## Build the Gotenberg's Docker image @@ -35,14 +35,6 @@ API_WRITE_TIMEOUT=30s API_ROOT_PATH=/ API_TRACE_HEADER=Gotenberg-Trace API_DISABLE_HEALTH_CHECK_LOGGING=false -API_WEBHOOK_ALLOW_LIST= -API_WEBHOOK_DENY_LIST= -API_WEBHOOK_ERROR_ALLOW_LIST= -API_WEBHOOK_ERROR_DENY_LIST= -API_WEBHOOK_MAX_RETRY=4 -API_WEBHOOK_RETRY_MIN_WAIT=1s -API_WEBHOOK_RETRY_MAX_WAIT=30s -API_DISABLE_WEBHOOK=false CHROMIUM_USER_AGENT= CHROMIUM_INCOGNITO=false CHROMIUM_IGNORE_CERTIFICATE_ERRORS=false @@ -55,6 +47,18 @@ LOG_LEVEL=info LOG_FORMAT=auto PDFENGINES_ENGINES= PDFENGINES_DISABLE_ROUTES=false +PROMETHEUS_NAMESPACE=gotenberg +PROMETHEUS_COLLECT_INTERVAL=1s +PROMETHEUS_DISABLE_ROUTE_LOGGING=false +PROMETHEUS_DISABLE_COLLECT=false +WEBHOOK_ALLOW_LIST= +WEBHOOK_DENY_LIST= +WEBHOOK_ERROR_ALLOW_LIST= +WEBHOOK_ERROR_DENY_LIST= +WEBHOOK_MAX_RETRY=4 +WEBHOOK_RETRY_MIN_WAIT=1s +WEBHOOK_RETRY_MAX_WAIT=30s +WEBHOOK_DISABLE=false .PHONY: run run: ## Start a Gotenberg container @@ -71,14 +75,6 @@ run: ## Start a Gotenberg container --api-root-path=$(API_ROOT_PATH) \ --api-trace-header=$(API_TRACE_HEADER) \ --api-disable-health-check-logging=$(API_DISABLE_HEALTH_CHECK_LOGGING) \ - --api-webhook-allow-list=$(API_WEBHOOK_ALLOW_LIST) \ - --api-webhook-deny-list=$(API_WEBHOOK_DENY_LIST) \ - --api-webhook-error-allow-list=$(API_WEBHOOK_ERROR_ALLOW_LIST) \ - --api-webhook-error-deny-list=$(API_WEBHOOK_ERROR_DENY_LIST) \ - --api-webhook-max-retry=$(API_WEBHOOK_MAX_RETRY) \ - --api-webhook-retry-min-wait=$(API_WEBHOOK_RETRY_MIN_WAIT) \ - --api-webhook-retry-max-wait=$(API_WEBHOOK_RETRY_MAX_WAIT) \ - --api-disable-webhook=$(API_DISABLE_WEBHOOK) \ --chromium-user-agent=$(CHROMIUM_USER_AGENT) \ --chromium-incognito=$(CHROMIUM_INCOGNITO) \ --chromium-ignore-certificate-errors=$(CHROMIUM_IGNORE_CERTIFICATE_ERRORS) \ @@ -90,7 +86,19 @@ run: ## Start a Gotenberg container --log-level=$(LOG_LEVEL) \ --log-format=$(LOG_FORMAT) \ --pdfengines-engines=$(PDFENGINES_ENGINES) \ - --pdfengines-disable-routes=$(PDFENGINES_DISABLE_ROUTES) + --pdfengines-disable-routes=$(PDFENGINES_DISABLE_ROUTES) \ + --prometheus-namespace=$(PROMETHEUS_NAMESPACE) \ + --prometheus-collect-interval=$(PROMETHEUS_COLLECT_INTERVAL) \ + --prometheus-disable-route-logging=$(PROMETHEUS_DISABLE_ROUTE_LOGGING) \ + --prometheus-disable-collect=$(PROMETHEUS_DISABLE_COLLECT) \ + --webhook-allow-list=$(WEBHOOK_ALLOW_LIST) \ + --webhook-deny-list=$(WEBHOOK_DENY_LIST) \ + --webhook-error-allow-list=$(WEBHOOK_ERROR_ALLOW_LIST) \ + --webhook-error-deny-list=$(WEBHOOK_ERROR_DENY_LIST) \ + --webhook-max-retry=$(WEBHOOK_MAX_RETRY) \ + --webhook-retry-min-wait=$(WEBHOOK_RETRY_MIN_WAIT) \ + --webhook-retry-max-wait=$(WEBHOOK_RETRY_MAX_WAIT) \ + --webhook-disable=$(WEBHOOK_DISABLE) .PHONY: build-tests build-tests: ## Build the tests' Docker image diff --git a/go.mod b/go.mod index cda73a09..af114e6c 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.17 require ( github.com/alexliesenfeld/health v0.6.0 github.com/andybalholm/brotli v1.0.3 // indirect - github.com/chromedp/cdproto v0.0.0-20210910012206-68626162910d + github.com/chromedp/cdproto v0.0.0-20211015214701-1037196e2fdd github.com/chromedp/chromedp v0.7.4 github.com/golang/snappy v0.0.4 // indirect github.com/google/uuid v1.3.0 @@ -13,7 +13,7 @@ require ( github.com/hashicorp/go-retryablehttp v0.7.0 github.com/klauspost/compress v1.13.6 // indirect github.com/klauspost/pgzip v1.2.5 // indirect - github.com/labstack/echo/v4 v4.5.0 + github.com/labstack/echo/v4 v4.6.1 github.com/labstack/gommon v0.3.0 github.com/mattn/go-isatty v0.0.14 // indirect github.com/mholt/archiver/v3 v3.5.0 @@ -21,6 +21,7 @@ require ( github.com/nwaples/rardecode v1.1.2 // indirect github.com/pdfcpu/pdfcpu v0.3.12 github.com/pierrec/lz4/v4 v4.1.8 // indirect + github.com/prometheus/client_golang v1.11.0 github.com/russross/blackfriday/v2 v2.1.0 github.com/spf13/pflag v1.0.5 github.com/ulikunitz/xz v0.5.10 // indirect @@ -29,29 +30,37 @@ require ( go.uber.org/zap v1.19.1 golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 // indirect golang.org/x/image v0.0.0-20210628002857-a66eb6448b8d // indirect - golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d // indirect + golang.org/x/net v0.0.0-20210913180222-943fd674d43e golang.org/x/sync v0.0.0-20210220032951-036812b2e83c - golang.org/x/sys v0.0.0-20210820121016-41cdb8703e55 // indirect + golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6 // indirect golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b golang.org/x/text v0.3.7 ) require ( github.com/aymerick/douceur v0.2.0 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/chromedp/sysutil v1.0.0 // indirect github.com/dsnet/compress v0.0.1 // indirect github.com/gobwas/httphead v0.1.0 // indirect github.com/gobwas/pool v0.2.1 // indirect github.com/gobwas/ws v1.1.0 // indirect + github.com/golang/protobuf v1.5.2 // indirect github.com/gorilla/css v1.0.0 // indirect github.com/hhrutter/lzw v0.0.0-20190829144645-6f07a24e8650 // indirect github.com/hhrutter/tiff v0.0.0-20190829141212-736cae8d0bc7 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect - github.com/mattn/go-colorable v0.1.8 // indirect + github.com/mattn/go-colorable v0.1.11 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/prometheus/client_model v0.2.0 // indirect + github.com/prometheus/common v0.31.1 // indirect + github.com/prometheus/procfs v0.7.3 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasttemplate v1.2.1 // indirect github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect + google.golang.org/protobuf v1.27.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/go.sum b/go.sum index 7732a284..f25b3bcd 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,43 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/alexliesenfeld/health v0.6.0 h1:HRBTCgybNSe4lqGEk7nU82c3bjwh9W+3b46W6UvD4CQ= github.com/alexliesenfeld/health v0.6.0/go.mod h1:N4NDIeQtlWumG+6z1ne1v62eQxktz5ylEgGgH9emdMw= github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= @@ -8,31 +48,112 @@ github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuP github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= +github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chromedp/cdproto v0.0.0-20210713064928-7d28b402946a/go.mod h1:At5TxYYdxkbQL0TSefRjhLE3Q0lgvqKKMSFUglJ7i1U= -github.com/chromedp/cdproto v0.0.0-20210910012206-68626162910d h1:77O+89zb1W9dPNUGjXOGt/J4LP70jPIPz9Rxs3aSX1w= -github.com/chromedp/cdproto v0.0.0-20210910012206-68626162910d/go.mod h1:At5TxYYdxkbQL0TSefRjhLE3Q0lgvqKKMSFUglJ7i1U= +github.com/chromedp/cdproto v0.0.0-20211015214701-1037196e2fdd h1:qAztMIF0bh8h1qH9S/McB5dP1zyKR9+YvvaWdHTOBck= +github.com/chromedp/cdproto v0.0.0-20211015214701-1037196e2fdd/go.mod h1:At5TxYYdxkbQL0TSefRjhLE3Q0lgvqKKMSFUglJ7i1U= github.com/chromedp/chromedp v0.7.4 h1:U+0d3WbB/Oj4mDuBOI0P7S3PJEued5UZIl5AJ3QulwU= github.com/chromedp/chromedp v0.7.4/go.mod h1:dBj+SXuQHznp6ZPwZeDDEBZKwclUwDLbZ0hjMialMYs= github.com/chromedp/sysutil v1.0.0 h1:+ZxhTpfpZlmchB58ih/LBHX52ky7w2VhQVKQMucy3Ic= github.com/chromedp/sysutil v1.0.0/go.mod h1:kgWmDdq8fTzXYcKIBqIYvRRTnYb9aNS9moAV0xufSww= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q= github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo= github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU= github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM= github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og= github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= github.com/gobwas/ws v1.1.0 h1:7RFti/xnNkMJnrK7D1yQ/iCIB5OrrY/54/H930kIbHA= github.com/gobwas/ws v1.1.0/go.mod h1:nzvNcVha5eUziGrbxFCo6qFIojQHjJV5cLYIbezhfL0= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= @@ -42,13 +163,25 @@ github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxC github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= github.com/hashicorp/go-retryablehttp v0.7.0 h1:eu1EI/mbirUgP5C8hVsTNaGZreBDlYiwC1FZWkvQPQ4= github.com/hashicorp/go-retryablehttp v0.7.0/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hhrutter/lzw v0.0.0-20190827003112-58b82c5a41cc/go.mod h1:yJBvOcu1wLQ9q9XZmfiPfur+3dQJuIhYQsMGLYcItZk= github.com/hhrutter/lzw v0.0.0-20190829144645-6f07a24e8650 h1:1yY/RQWNSBjJe2GDCIYoLmpWVidrooriUr4QS/zaATQ= github.com/hhrutter/lzw v0.0.0-20190829144645-6f07a24e8650/go.mod h1:yJBvOcu1wLQ9q9XZmfiPfur+3dQJuIhYQsMGLYcItZk= github.com/hhrutter/tiff v0.0.0-20190829141212-736cae8d0bc7 h1:o1wMw7uTNyA58IlEdDpxIrtFHTgnvYzA8sCQz8luv94= github.com/hhrutter/tiff v0.0.0-20190829141212-736cae8d0bc7/go.mod h1:WkUxfS2JUu3qPo6tRld7ISb8HiC0gVSU91kooBMDVok= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.10.10/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc= @@ -57,29 +190,41 @@ github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgo github.com/klauspost/pgzip v1.2.4/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/labstack/echo/v4 v4.5.0 h1:JXk6H5PAw9I3GwizqUHhYyS4f45iyGebR/c1xNCeOCY= -github.com/labstack/echo/v4 v4.5.0/go.mod h1:czIriw4a0C1dFun+ObrXp7ok03xON0N1awStJ6ArI7Y= +github.com/labstack/echo/v4 v4.6.1 h1:OMVsrnNFzYlGSdaiYGHbgWQnr+JM7NG+B9suCPie14M= +github.com/labstack/echo/v4 v4.6.1/go.mod h1:RnjgMWNDB9g/HucVWhQYNQP9PvbYf6adqftqryo7s9k= github.com/labstack/gommon v0.3.0 h1:JEeO0bvc78PKdyHxloTKiF8BD5iGrH8T6MSeGvSgob0= github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.11 h1:nQ+aFkoE2TMGc0b68U2OKSexC+eq46+XwZzWXHRmPYs= +github.com/mattn/go-colorable v0.1.11/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mholt/archiver/v3 v3.5.0 h1:nE8gZIrw66cu4osS/U7UW7YDuGMHssxKutU8IfWxwWE= github.com/mholt/archiver/v3 v3.5.0/go.mod h1:qqTTPUK/HZPFgFQ/TJ3BzvTpF/dPtFVJXdQbCmeMxwc= github.com/microcosm-cc/bluemonday v1.0.15 h1:J4uN+qPng9rvkBZBoBb8YGR+ijuklIMpSOZZLjYpbeY= github.com/microcosm-cc/bluemonday v1.0.15/go.mod h1:ZLvAzeakRwrGnzQEvstVzVt3ZpqOF2+sdFr0Om+ce30= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nwaples/rardecode v1.1.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= github.com/nwaples/rardecode v1.1.2 h1:Cj0yZY6T1Zx1R7AhTbyGSALm44/Mmq+BAPc4B/p/d3M= github.com/nwaples/rardecode v1.1.2/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= @@ -90,17 +235,44 @@ github.com/pdfcpu/pdfcpu v0.3.12/go.mod h1:8XVBtVxuuIuSZL4Ez15Q4QoC+H8zeAaGnuiOE github.com/pierrec/lz4/v4 v4.0.3/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pierrec/lz4/v4 v4.1.8 h1:ieHkV+i2BRzngO4Wd/3HGowuZStgq6QkPsD1eolNAO4= github.com/pierrec/lz4/v4 v4.1.8/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= +github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ= +github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= +github.com/prometheus/common v0.31.1 h1:d18hG4PkHnNAKNMOmFuXFaiY8Us0nird/2m60uS1AMs= +github.com/prometheus/common v0.31.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= +github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -117,7 +289,15 @@ github.com/valyala/fasttemplate v1.2.1 h1:TVEnxayobAdVkhQfrfes2IzOB6o+z4roRkPF52 github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= @@ -128,68 +308,309 @@ go.uber.org/multierr v1.7.0 h1:zaiO/rmgFjbmCXdSYJWQcdvOCsthmdaHfr3Gm2Kx4Ec= go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= go.uber.org/zap v1.19.1 h1:ue41HOKd1vGURxrmeKIgELGb3jPW9DMUDGtsinblHwI= go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 h1:HWj/xjIHfjYU5nVXpTM0s39J9CbLn7Cc5a7IC5rwsMQ= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.0.0-20190823064033-3a9bac650e44/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.0.0-20210220032944-ac19c3e999fb/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.0.0-20210628002857-a66eb6448b8d h1:RNPAfi2nHY7C2srAV8A49jpsYr0ADedCk1wq6fTMTvs= golang.org/x/image v0.0.0-20210628002857-a66eb6448b8d/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d h1:LO7XpTYMwTqxjLcGWPijK3vRXg1aWdlNOVOHRq45d7c= -golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210913180222-943fd674d43e h1:+b/22bPvDYt4NPDcy4xAGCmON713ONAWFeY3Z7I3tR8= +golang.org/x/net v0.0.0-20210913180222-943fd674d43e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201207223542-d4d67f95c62d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210525143221-35b2ab0089ea/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210820121016-41cdb8703e55 h1:rw6UNGRMfarCepjI8qOepea/SXwIBVfTKjztZ5gBbq4= -golang.org/x/sys v0.0.0-20210820121016-41cdb8703e55/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210910150752-751e447fb3d0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6 h1:foEbQz/B0Oz6YIqu/69kfXPYeFQAuuMYFkjaqXzl5Wo= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b h1:9zKuko04nR4gjZ4+DNjHqRlAJqbJETHwiNKDqTfOjfE= golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/pkg/gotenberg/flags.go b/pkg/gotenberg/flags.go index 18243139..8b2b8d00 100644 --- a/pkg/gotenberg/flags.go +++ b/pkg/gotenberg/flags.go @@ -25,6 +25,17 @@ func (f *ParsedFlags) MustString(name string) string { return val } +// MustDeprecatedString returns the string value of a deprecated flag if it was +// explicitly set or the string value of the new flag. +// It panics if an error occurs. +func (f *ParsedFlags) MustDeprecatedString(deprecated string, newName string) string { + if f.Changed(deprecated) { + return f.MustString(deprecated) + } + + return f.MustString(newName) +} + // MustStringSlice returns the string slice value of a flag given by name. // It panics if an error occurs. func (f *ParsedFlags) MustStringSlice(name string) []string { @@ -36,6 +47,17 @@ func (f *ParsedFlags) MustStringSlice(name string) []string { return val } +// MustDeprecatedStringSlice returns the string slice value of a deprecated +// flag if it was explicitly set or the string slice value of the new flag. +// It panics if an error occurs. +func (f *ParsedFlags) MustDeprecatedStringSlice(deprecated string, newName string) []string { + if f.Changed(deprecated) { + return f.MustStringSlice(deprecated) + } + + return f.MustStringSlice(newName) +} + // MustBool returns the boolean value of a flag given by name. // It panics if an error occurs. func (f *ParsedFlags) MustBool(name string) bool { @@ -47,6 +69,17 @@ func (f *ParsedFlags) MustBool(name string) bool { return val } +// MustDeprecatedBool returns the boolean value of a deprecated flag if it was +// explicitly set or the int value of the new flag. +// It panics if an error occurs. +func (f *ParsedFlags) MustDeprecatedBool(deprecated string, newName string) bool { + if f.Changed(deprecated) { + return f.MustBool(deprecated) + } + + return f.MustBool(newName) +} + // MustInt returns the int value of a flag given by name. // It panics if an error occurs. func (f *ParsedFlags) MustInt(name string) int { @@ -58,6 +91,17 @@ func (f *ParsedFlags) MustInt(name string) int { return val } +// MustDeprecatedInt returns the int value of a deprecated flag if it was +// explicitly set or the int value of the new flag. +// It panics if an error occurs. +func (f *ParsedFlags) MustDeprecatedInt(deprecated string, newName string) int { + if f.Changed(deprecated) { + return f.MustInt(deprecated) + } + + return f.MustInt(newName) +} + // MustFloat64 returns the float value of a flag given by name. // It panics if an error occurs. func (f *ParsedFlags) MustFloat64(name string) float64 { @@ -69,6 +113,17 @@ func (f *ParsedFlags) MustFloat64(name string) float64 { return val } +// MustDeprecatedFloat64 returns the float value of a deprecated flag if it was +// explicitly set or the float value of the new flag. +// It panics if an error occurs. +func (f *ParsedFlags) MustDeprecatedFloat64(deprecated string, newName string) float64 { + if f.Changed(deprecated) { + return f.MustFloat64(deprecated) + } + + return f.MustFloat64(newName) +} + // MustDuration returns the time.Duration value of a flag given by name. // It panics if an error occurs. func (f *ParsedFlags) MustDuration(name string) time.Duration { @@ -80,6 +135,17 @@ func (f *ParsedFlags) MustDuration(name string) time.Duration { return val } +// MustDeprecatedDuration returns the time.Duration value of a deprecated flag +// if it was explicitly set or the time.Duration value of the new flag. +// It panics if an error occurs. +func (f *ParsedFlags) MustDeprecatedDuration(deprecated string, newName string) time.Duration { + if f.Changed(deprecated) { + return f.MustDuration(deprecated) + } + + return f.MustDuration(newName) +} + // MustHumanReadableBytesString returns the human-readable bytes string of a // flag given by name. // It panics if an error occurs. @@ -97,6 +163,18 @@ func (f *ParsedFlags) MustHumanReadableBytesString(name string) string { return val } +// MustDeprecatedHumanReadableBytesString returns the human-readable bytes +// string of a deprecated flag if it was explicitly set or the human-readable +// bytes string of the new flag. +// It panics if an error occurs. +func (f *ParsedFlags) MustDeprecatedHumanReadableBytesString(deprecated string, newName string) string { + if f.Changed(deprecated) { + return f.MustHumanReadableBytesString(deprecated) + } + + return f.MustHumanReadableBytesString(newName) +} + // MustRegexp returns the regular expression of a flag given by name. // It panics if an error occurs. func (f *ParsedFlags) MustRegexp(name string) *regexp.Regexp { @@ -107,3 +185,14 @@ func (f *ParsedFlags) MustRegexp(name string) *regexp.Regexp { return regexp.MustCompile(val) } + +// MustDeprecatedRegexp returns the regular expression of a deprecated flag if +// it was explicitly set or the regular expression of the new flag. +// It panics if an error occurs. +func (f *ParsedFlags) MustDeprecatedRegexp(deprecated string, newName string) *regexp.Regexp { + if f.Changed(deprecated) { + return f.MustRegexp(deprecated) + } + + return f.MustRegexp(newName) +} diff --git a/pkg/gotenberg/flags_test.go b/pkg/gotenberg/flags_test.go index 942ec37d..23361637 100644 --- a/pkg/gotenberg/flags_test.go +++ b/pkg/gotenberg/flags_test.go @@ -1,6 +1,8 @@ package gotenberg import ( + "reflect" + "regexp" "testing" "time" @@ -52,6 +54,42 @@ func TestParsedFlags_MustString(t *testing.T) { } } +func TestParsedFlags_MustDeprecatedString(t *testing.T) { + for i, tc := range []struct { + rawFlags []string + expectValue string + }{ + { + rawFlags: []string{"--foo=foo"}, + expectValue: "foo", + }, + { + rawFlags: []string{"--bar=bar"}, + expectValue: "bar", + }, + { + rawFlags: []string{"--foo=foo", "--bar=bar"}, + expectValue: "foo", + }, + } { + fs := flag.NewFlagSet("tests", flag.ContinueOnError) + fs.String("foo", "", "") + fs.String("bar", "", "") + + parsedFlags := ParsedFlags{FlagSet: fs} + + err := parsedFlags.Parse(tc.rawFlags) + if err != nil { + t.Fatalf("test %d: expected no error but got: %v", i, err) + } + + actual := parsedFlags.MustDeprecatedString("foo", "bar") + if actual != tc.expectValue { + t.Errorf("test %d: expected '%s' but got '%s'", i, tc.expectValue, actual) + } + } +} + func TestParsedFlags_MustStringSlice(t *testing.T) { fs := flag.NewFlagSet("tests", flag.ContinueOnError) fs.StringSlice("foo", make([]string, 0), "") @@ -97,6 +135,42 @@ func TestParsedFlags_MustStringSlice(t *testing.T) { } } +func TestParsedFlags_MustDeprecatedStringSlice(t *testing.T) { + for i, tc := range []struct { + rawFlags []string + expectValue []string + }{ + { + rawFlags: []string{"--foo=foo"}, + expectValue: []string{"foo"}, + }, + { + rawFlags: []string{"--bar=bar"}, + expectValue: []string{"bar"}, + }, + { + rawFlags: []string{"--foo=foo", "--bar=bar"}, + expectValue: []string{"foo"}, + }, + } { + fs := flag.NewFlagSet("tests", flag.ContinueOnError) + fs.StringSlice("foo", make([]string, 0), "") + fs.StringSlice("bar", make([]string, 0), "") + + parsedFlags := ParsedFlags{FlagSet: fs} + + err := parsedFlags.Parse(tc.rawFlags) + if err != nil { + t.Fatalf("test %d: expected no error but got: %v", i, err) + } + + actual := parsedFlags.MustDeprecatedStringSlice("foo", "bar") + if !reflect.DeepEqual(actual, tc.expectValue) { + t.Errorf("test %d: expected %+v but got %+v", i, tc.expectValue, actual) + } + } +} + func TestParsedFlags_MustBool(t *testing.T) { fs := flag.NewFlagSet("tests", flag.ContinueOnError) fs.Bool("foo", false, "") @@ -142,6 +216,42 @@ func TestParsedFlags_MustBool(t *testing.T) { } } +func TestParsedFlags_MustDeprecatedBool(t *testing.T) { + for i, tc := range []struct { + rawFlags []string + expectValue bool + }{ + { + rawFlags: []string{"--foo=true"}, + expectValue: true, + }, + { + rawFlags: []string{"--bar=false"}, + expectValue: false, + }, + { + rawFlags: []string{"--foo=true", "--bar=false"}, + expectValue: true, + }, + } { + fs := flag.NewFlagSet("tests", flag.ContinueOnError) + fs.Bool("foo", false, "") + fs.Bool("bar", true, "") + + parsedFlags := ParsedFlags{FlagSet: fs} + + err := parsedFlags.Parse(tc.rawFlags) + if err != nil { + t.Fatalf("test %d: expected no error but got: %v", i, err) + } + + actual := parsedFlags.MustDeprecatedBool("foo", "bar") + if actual != tc.expectValue { + t.Errorf("test %d: expected %v but got %v", i, tc.expectValue, actual) + } + } +} + func TestParsedFlags_MustInt(t *testing.T) { fs := flag.NewFlagSet("tests", flag.ContinueOnError) fs.Int("foo", 0, "") @@ -187,6 +297,42 @@ func TestParsedFlags_MustInt(t *testing.T) { } } +func TestParsedFlags_MustDeprecatedInt(t *testing.T) { + for i, tc := range []struct { + rawFlags []string + expectValue int + }{ + { + rawFlags: []string{"--foo=1"}, + expectValue: 1, + }, + { + rawFlags: []string{"--bar=2"}, + expectValue: 2, + }, + { + rawFlags: []string{"--foo=1", "--bar=2"}, + expectValue: 1, + }, + } { + fs := flag.NewFlagSet("tests", flag.ContinueOnError) + fs.Int("foo", 0, "") + fs.Int("bar", 0, "") + + parsedFlags := ParsedFlags{FlagSet: fs} + + err := parsedFlags.Parse(tc.rawFlags) + if err != nil { + t.Fatalf("test %d: expected no error but got: %v", i, err) + } + + actual := parsedFlags.MustDeprecatedInt("foo", "bar") + if actual != tc.expectValue { + t.Errorf("test %d: expected %d but got %d", i, tc.expectValue, actual) + } + } +} + func TestParsedFlags_MustFloat64(t *testing.T) { fs := flag.NewFlagSet("tests", flag.ContinueOnError) fs.Float64("foo", 1.0, "") @@ -232,6 +378,42 @@ func TestParsedFlags_MustFloat64(t *testing.T) { } } +func TestParsedFlags_MustDeprecatedFloat64(t *testing.T) { + for i, tc := range []struct { + rawFlags []string + expectValue float64 + }{ + { + rawFlags: []string{"--foo=1.0"}, + expectValue: 1.0, + }, + { + rawFlags: []string{"--bar=2.0"}, + expectValue: 2.0, + }, + { + rawFlags: []string{"--foo=1.0", "--bar=2.0"}, + expectValue: 1.0, + }, + } { + fs := flag.NewFlagSet("tests", flag.ContinueOnError) + fs.Float64("foo", 0, "") + fs.Float64("bar", 0, "") + + parsedFlags := ParsedFlags{FlagSet: fs} + + err := parsedFlags.Parse(tc.rawFlags) + if err != nil { + t.Fatalf("test %d: expected no error but got: %v", i, err) + } + + actual := parsedFlags.MustDeprecatedFloat64("foo", "bar") + if actual != tc.expectValue { + t.Errorf("test %d: expected %f but got %f", i, tc.expectValue, actual) + } + } +} + func TestParsedFlags_MustDuration(t *testing.T) { fs := flag.NewFlagSet("tests", flag.ContinueOnError) fs.Duration("foo", time.Duration(1)*time.Second, "") @@ -277,6 +459,42 @@ func TestParsedFlags_MustDuration(t *testing.T) { } } +func TestParsedFlags_MustDeprecatedDuration(t *testing.T) { + for i, tc := range []struct { + rawFlags []string + expectValue time.Duration + }{ + { + rawFlags: []string{"--foo=1s"}, + expectValue: time.Duration(1) * time.Second, + }, + { + rawFlags: []string{"--bar=2s"}, + expectValue: time.Duration(2) * time.Second, + }, + { + rawFlags: []string{"--foo=1s", "--bar=2s"}, + expectValue: time.Duration(1) * time.Second, + }, + } { + fs := flag.NewFlagSet("tests", flag.ContinueOnError) + fs.Duration("foo", 0, "") + fs.Duration("bar", 0, "") + + parsedFlags := ParsedFlags{FlagSet: fs} + + err := parsedFlags.Parse(tc.rawFlags) + if err != nil { + t.Fatalf("test %d: expected no error but got: %v", i, err) + } + + actual := parsedFlags.MustDeprecatedDuration("foo", "bar") + if actual != tc.expectValue { + t.Errorf("test %d: expected '%s' but got '%s'", i, tc.expectValue, actual) + } + } +} + func TestParsedFlags_MustHumanReadableBytesString(t *testing.T) { fs := flag.NewFlagSet("tests", flag.ContinueOnError) fs.String("foo", "1MB", "") @@ -327,6 +545,42 @@ func TestParsedFlags_MustHumanReadableBytesString(t *testing.T) { } } +func TestParsedFlags_MustDeprecatedHumanReadableBytesString(t *testing.T) { + for i, tc := range []struct { + rawFlags []string + expectValue string + }{ + { + rawFlags: []string{"--foo=1MB"}, + expectValue: "1MB", + }, + { + rawFlags: []string{"--bar=2MB"}, + expectValue: "2MB", + }, + { + rawFlags: []string{"--foo=1MB", "--bar=2MB"}, + expectValue: "1MB", + }, + } { + fs := flag.NewFlagSet("tests", flag.ContinueOnError) + fs.String("foo", "", "") + fs.String("bar", "", "") + + parsedFlags := ParsedFlags{FlagSet: fs} + + err := parsedFlags.Parse(tc.rawFlags) + if err != nil { + t.Fatalf("test %d: expected no error but got: %v", i, err) + } + + actual := parsedFlags.MustDeprecatedHumanReadableBytesString("foo", "bar") + if actual != tc.expectValue { + t.Errorf("test %d: expected '%s' but got '%s'", i, tc.expectValue, actual) + } + } +} + func TestParsedFlags_MustRegexp(t *testing.T) { fs := flag.NewFlagSet("tests", flag.ContinueOnError) fs.String("foo", "", "") @@ -376,3 +630,39 @@ func TestParsedFlags_MustRegexp(t *testing.T) { }() } } + +func TestParsedFlags_MustDeprecatedRegexp(t *testing.T) { + for i, tc := range []struct { + rawFlags []string + expectValue *regexp.Regexp + }{ + { + rawFlags: []string{"--foo=foo"}, + expectValue: regexp.MustCompile("foo"), + }, + { + rawFlags: []string{"--bar=bar"}, + expectValue: regexp.MustCompile("bar"), + }, + { + rawFlags: []string{"--foo=foo", "--bar=bar"}, + expectValue: regexp.MustCompile("foo"), + }, + } { + fs := flag.NewFlagSet("tests", flag.ContinueOnError) + fs.String("foo", "", "") + fs.String("bar", "", "") + + parsedFlags := ParsedFlags{FlagSet: fs} + + err := parsedFlags.Parse(tc.rawFlags) + if err != nil { + t.Fatalf("test %d: expected no error but got: %v", i, err) + } + + actual := parsedFlags.MustDeprecatedRegexp("foo", "bar") + if actual.String() != tc.expectValue.String() { + t.Errorf("test %d: expected '%s' but got '%s'", i, tc.expectValue.String(), actual.String()) + } + } +} diff --git a/pkg/gotenberg/metrics.go b/pkg/gotenberg/metrics.go new file mode 100644 index 00000000..f6603639 --- /dev/null +++ b/pkg/gotenberg/metrics.go @@ -0,0 +1,26 @@ +package gotenberg + +// Metric represents a unitary metric. +type Metric struct { + // Name is the unique identifier. + // Required. + Name string + + // Description describes the metric. + // Optional. + Description string + + // Read returns the current value. + // Required. + Read func() float64 +} + +// MetricsProvider is a module interface which provides a list of Metric. +// +// func (m *YourModule) Provision(ctx *gotenberg.Context) error { +// provider, _ := ctx.Module(new(gotenberg.MetricsProvider)) +// metrics, _ := provider.(gotenberg.MetricsProvider).Metrics() +// } +type MetricsProvider interface { + Metrics() ([]Metric, error) +} diff --git a/pkg/modules/api/api.go b/pkg/modules/api/api.go index a0923d6b..27f35db6 100644 --- a/pkg/modules/api/api.go +++ b/pkg/modules/api/api.go @@ -6,7 +6,6 @@ import ( "fmt" "net/http" "os" - "regexp" "sort" "strconv" "strings" @@ -26,8 +25,8 @@ func init() { gotenberg.MustRegisterModule(API{}) } -// API is a module which provides an HTTP server. Other modules may add -// "multipart/form-data" routes, middlewares or health checks. +// API is a module which provides an HTTP server. Other modules may add routes, +// middlewares or health checks. type API struct { port int readTimeout time.Duration @@ -36,38 +35,41 @@ type API struct { rootPath string traceHeader string disableHealthCheckLogging bool - webhookAllowList *regexp.Regexp - webhookDenyList *regexp.Regexp - webhookErrorAllowList *regexp.Regexp - webhookErrorDenyList *regexp.Regexp - webhookMaxRetry int - webhookRetryMinWait time.Duration - webhookRetryMaxWait time.Duration - disableWebhook bool - multipartFormDataRoutes []MultipartFormDataRoute - externalMiddlewares []Middleware - healthChecks []health.CheckerOption - logger *zap.Logger - srv *echo.Echo + routes []Route + externalMiddlewares []Middleware + healthChecks []health.CheckerOption + gcGraceDuration time.Duration + logger *zap.Logger + srv *echo.Echo } -// MultipartFormDataRouter is a module interface which adds -// "multipart/form-data" routes to the API. -type MultipartFormDataRouter interface { - Routes() ([]MultipartFormDataRoute, error) +// Router is a module interface which adds routes to the API. +type Router interface { + Routes() ([]Route, error) } -// MultipartFormDataRoute represents a "multipart/form-data" route. All routes -// uses the HTTP POST method. -type MultipartFormDataRoute struct { +// Route represents a route from a Router. +type Route struct { + // Method is the HTTP method of the route (i.e., GET, POST, etc.). + // Required. + Method string + // Path is the sub path of the route. Must start with a slash. // Required. Path string + // IsMultipart tells if the route is "multipart/form-data". + // Optional. + IsMultipart bool + + // DisableLogging disables the logging for this route. + // Optional. + DisableLogging bool + // Handler is the function which handles the request. // Required. - Handler func(ctx *Context) error + Handler echo.HandlerFunc } // MiddlewareProvider is a module interface which adds middlewares to the API. @@ -75,8 +77,18 @@ type MiddlewareProvider interface { Middlewares() ([]Middleware, error) } +// MiddlewareStack is a type which helps to determine in which stack the +// middlewares provided by the MiddlewareProvider modules should be located. +type MiddlewareStack uint32 + +const ( + DefaultStack MiddlewareStack = iota + PreRouterStack + MultipartStack +) + // MiddlewarePriority is a type which helps to determine the execution order of -// middlewares provided by the MiddlewareProvider modules. +// middlewares provided by the MiddlewareProvider modules in a stack. type MiddlewarePriority uint32 const ( @@ -113,13 +125,13 @@ const ( // }(), // } type Middleware struct { - // RunBeforeRouter tells if the middleware should run before the router - // process an HTTP request. + // Stack tells in which stack the middleware should be located. + // Default to DefaultStack. // Optional. - RunBeforeRouter bool + Stack MiddlewareStack // Priority tells if the middleware should be positioned high or not in - // the middlewares chain. + // its stack. // Default to VeryLowPriority. // Optional. Priority MiddlewarePriority @@ -137,6 +149,12 @@ type HealthChecker interface { Checks() ([]health.CheckerOption, error) } +// GarbageCollectorGraceDurationIncrementer is a module interface for +// increasing the grace duration provided by the API for the garbage collector. +type GarbageCollectorGraceDurationIncrementer interface { + AddGraceDuration() time.Duration +} + // Descriptor returns an API's module descriptor. func (API) Descriptor() gotenberg.ModuleDescriptor { return gotenberg.ModuleDescriptor{ @@ -151,14 +169,6 @@ func (API) Descriptor() gotenberg.ModuleDescriptor { fs.String("api-root-path", "/", "Set the root path of the API - for service discovery via URL paths") fs.String("api-trace-header", "Gotenberg-Trace", "Set the header name to use for identifying requests") fs.Bool("api-disable-health-check-logging", false, "Disable health check logging") - fs.String("api-webhook-allow-list", "", "Set the allowed URLs for the webhook feature using a regular expression") - fs.String("api-webhook-deny-list", "", "Set the denied URLs for the webhook feature using a regular expression") - fs.String("api-webhook-error-allow-list", "", "Set the allowed URLs in case of an error for the webhook feature using a regular expression") - fs.String("api-webhook-error-deny-list", "", "Set the denied URLs in case of an error for the webhook feature using a regular expression") - fs.Int("api-webhook-max-retry", 4, "Set the maximum number of retries for the webhook feature") - fs.Duration("api-webhook-retry-min-wait", time.Duration(1)*time.Second, "Set the minimum duration to wait before trying to call the webhook again") - fs.Duration("api-webhook-retry-max-wait", time.Duration(30)*time.Second, "Set the maximum duration to wait before trying to call the webhook again") - fs.Bool("api-disable-webhook", false, "Disable the webhook feature") return fs }(), @@ -176,14 +186,6 @@ func (a *API) Provision(ctx *gotenberg.Context) error { a.rootPath = flags.MustString("api-root-path") a.traceHeader = flags.MustString("api-trace-header") a.disableHealthCheckLogging = flags.MustBool("api-disable-health-check-logging") - a.webhookAllowList = flags.MustRegexp("api-webhook-allow-list") - a.webhookDenyList = flags.MustRegexp("api-webhook-deny-list") - a.webhookErrorAllowList = flags.MustRegexp("api-webhook-error-allow-list") - a.webhookErrorDenyList = flags.MustRegexp("api-webhook-error-deny-list") - a.webhookMaxRetry = flags.MustInt("api-webhook-max-retry") - a.webhookRetryMinWait = flags.MustDuration("api-webhook-retry-min-wait") - a.webhookRetryMaxWait = flags.MustDuration("api-webhook-retry-max-wait") - a.disableWebhook = flags.MustBool("api-disable-webhook") // Port from env? portEnvVar := flags.MustString("api-port-from-env") @@ -207,14 +209,14 @@ func (a *API) Provision(ctx *gotenberg.Context) error { } // Get routes from modules. - mods, err := ctx.Modules(new(MultipartFormDataRouter)) + mods, err := ctx.Modules(new(Router)) if err != nil { - return fmt.Errorf("get multipart/form-data routers: %w", err) + return fmt.Errorf("get routers: %w", err) } - routers := make([]MultipartFormDataRouter, len(mods)) + routers := make([]Router, len(mods)) for i, router := range mods { - routers[i] = router.(MultipartFormDataRouter) + routers[i] = router.(Router) } for _, router := range routers { @@ -223,7 +225,7 @@ func (a *API) Provision(ctx *gotenberg.Context) error { return fmt.Errorf("get routes: %w", err) } - a.multipartFormDataRoutes = append(a.multipartFormDataRoutes, routes...) + a.routes = append(a.routes, routes...) } // Get middlewares from modules. @@ -271,6 +273,18 @@ func (a *API) Provision(ctx *gotenberg.Context) error { a.healthChecks = append(a.healthChecks, checks...) } + // Grace duration. + a.gcGraceDuration = a.readTimeout + a.processTimeout + a.writeTimeout + + mods, err = ctx.Modules(new(GarbageCollectorGraceDurationIncrementer)) + if err != nil { + return fmt.Errorf("get garbage collector grace duration increments: %w", err) + } + + for _, incrementer := range mods { + a.gcGraceDuration += incrementer.(GarbageCollectorGraceDurationIncrementer).AddGraceDuration() + } + loggerProvider, err := ctx.Module(new(gotenberg.LoggerProvider)) if err != nil { return fmt.Errorf("get logger provider: %w", err) @@ -318,26 +332,35 @@ func (a API) Validate() error { return err } - routesMap := make(map[string]MultipartFormDataRoute, len(a.multipartFormDataRoutes)) + routesMap := make(map[string]string, len(a.routes)+1) + routesMap["/health"] = "/health" - for _, route := range a.multipartFormDataRoutes { + for _, route := range a.routes { if route.Path == "" { return errors.New("route with empty path cannot be registered") } if !strings.HasPrefix(route.Path, "/") { - return fmt.Errorf("route %s does not start with /", route.Path) + return fmt.Errorf("route '%s' does not start with /", route.Path) + } + + if route.IsMultipart && !strings.HasPrefix(route.Path, "/forms") { + return fmt.Errorf("multipart/form-data route '%s' does not start with /forms", route.Path) + } + + if route.Method == "" { + return fmt.Errorf("route '%s' has an empty method", route.Path) } if route.Handler == nil { - return fmt.Errorf("route %s has a nil handler", route.Path) + return fmt.Errorf("route '%s' has a nil handler", route.Path) } if _, ok := routesMap[route.Path]; ok { - return fmt.Errorf("route %s is already registered", route.Path) + return fmt.Errorf("route '%s' is already registered", route.Path) } - routesMap[route.Path] = route + routesMap[route.Path] = route.Path } for _, middleware := range a.externalMiddlewares { @@ -356,94 +379,83 @@ func (a *API) Start() error { a.srv.HidePort = true a.srv.Server.ReadTimeout = a.readTimeout a.srv.Server.WriteTimeout = a.writeTimeout - a.srv.HTTPErrorHandler = httpErrorHandler(a.traceHeader) + a.srv.HTTPErrorHandler = httpErrorHandler() + // Let's prepare the modules' routes. + var disableLoggingForPaths []string + for i, route := range a.routes { + a.routes[i].Path = strings.TrimPrefix(route.Path, "/") + + if route.DisableLogging { + disableLoggingForPaths = append(disableLoggingForPaths, strings.TrimPrefix(route.Path, "/")) + } + } + + // Check if the user wish to add logging entries related to the health + // check route. + if a.disableHealthCheckLogging { + disableLoggingForPaths = append(disableLoggingForPaths, "health") + } + + // Add the API middlewares. a.srv.Pre( latencyMiddleware(), rootPathMiddleware(a.rootPath), traceMiddleware(a.traceHeader), - loggerMiddleware(a.logger, a.disableHealthCheckLogging), + timeoutsMiddleware(a.readTimeout, a.processTimeout, a.writeTimeout), + loggerMiddleware(a.logger, disableLoggingForPaths), ) + // Add the modules' middlewares in their respective stacks. + var externalMultipartMiddlewares []Middleware for _, externalMiddleware := range a.externalMiddlewares { - if externalMiddleware.RunBeforeRouter { + switch externalMiddleware.Stack { + case PreRouterStack: a.srv.Pre(externalMiddleware.Handler) - - continue + case MultipartStack: + externalMultipartMiddlewares = append(externalMultipartMiddlewares, externalMiddleware) + default: + a.srv.Use(externalMiddleware.Handler) } - - a.srv.Use(externalMiddleware.Handler) } hardTimeout := a.processTimeout + (time.Duration(5) * time.Second) + // Add the modules' routes and their specific middlewares. + for _, route := range a.routes { + var middlewares []echo.MiddlewareFunc + + if route.IsMultipart { + middlewares = append(middlewares, contextMiddleware(a.processTimeout)) + + for _, externalMultipartMiddleware := range externalMultipartMiddlewares { + middlewares = append(middlewares, externalMultipartMiddleware.Handler) + } + } + + middlewares = append(middlewares, hardTimeoutMiddleware(hardTimeout)) + + a.srv.Add( + route.Method, + fmt.Sprintf("%s%s", a.rootPath, route.Path), + route.Handler, + middlewares..., + ) + } + + // Let's not forget the health check route. a.srv.GET( - fmt.Sprintf("%shealth", a.rootPath), + fmt.Sprintf("%s%s", a.rootPath, "health"), func() echo.HandlerFunc { checks := append(a.healthChecks, health.WithTimeout(a.processTimeout)) checker := health.NewChecker(checks...) return echo.WrapHandler(health.NewHandler(checker)) }(), - timeoutMiddleware(hardTimeout), + hardTimeoutMiddleware(hardTimeout), ) - formsGroup := a.srv.Group( - fmt.Sprintf("%sforms", a.rootPath), - contextMiddleware( - contextMiddlewareConfig{ - traceHeader: a.traceHeader, - timeout: struct { - process time.Duration - write time.Duration - }{ - process: a.processTimeout, - write: a.writeTimeout, - }, - webhook: struct { - allowList *regexp.Regexp - denyList *regexp.Regexp - errorAllowList *regexp.Regexp - errorDenyList *regexp.Regexp - maxRetry int - retryMinWait time.Duration - retryMaxWait time.Duration - disable bool - }{ - allowList: a.webhookAllowList, - denyList: a.webhookDenyList, - errorAllowList: a.webhookErrorAllowList, - errorDenyList: a.webhookErrorDenyList, - maxRetry: a.webhookMaxRetry, - retryMinWait: a.webhookRetryMinWait, - retryMaxWait: a.webhookRetryMaxWait, - disable: a.disableWebhook, - }, - }, - ), - timeoutMiddleware(hardTimeout), - ) - - // Add routes from other modules. - for _, route := range a.multipartFormDataRoutes { - formsGroup.POST( - route.Path, - func(route MultipartFormDataRoute) echo.HandlerFunc { - return func(c echo.Context) error { - ctx := c.Get("context").(*Context) - - err := route.Handler(ctx) - if err != nil { - return fmt.Errorf("handle request: %w", err) - } - - return nil - } - }(route), - ) - } - - // As the listen method is blocking, run it in a goroutine. + // As the following code is blocking, run it in a goroutine. go func() { server := &http2.Server{} err := a.srv.StartH2CServer(fmt.Sprintf(":%d", a.port), server) @@ -468,18 +480,7 @@ func (a API) Stop(ctx context.Context) error { // GraceDuration updates the expiration time of files and directories parsed by // the gc.GarbageCollector. func (a API) GraceDuration() time.Duration { - duration := a.readTimeout + a.processTimeout + a.writeTimeout - - if a.disableWebhook { - return duration - } - - for i := 0; i < a.webhookMaxRetry; i++ { - // Yep... Golang does not allow int * time.Duration. - duration += a.webhookRetryMaxWait - } - - return duration + return a.gcGraceDuration } // Interface guards. diff --git a/pkg/modules/api/api_test.go b/pkg/modules/api/api_test.go index e242bc3b..5a85c8d0 100644 --- a/pkg/modules/api/api_test.go +++ b/pkg/modules/api/api_test.go @@ -35,12 +35,12 @@ func (mod ProtoValidator) Validate() error { return mod.validate() } -type ProtoMultipartFormDataRouter struct { +type ProtoRouter struct { ProtoValidator - routes func() ([]MultipartFormDataRoute, error) + routes func() ([]Route, error) } -func (mod ProtoMultipartFormDataRouter) Routes() ([]MultipartFormDataRoute, error) { +func (mod ProtoRouter) Routes() ([]Route, error) { return mod.routes() } @@ -62,6 +62,15 @@ func (mod ProtoHealthChecker) Checks() ([]health.CheckerOption, error) { return mod.checks() } +type ProtoGarbageCollectorGraceDurationIncrementer struct { + ProtoValidator + addGraceDuration func() time.Duration +} + +func (mod ProtoGarbageCollectorGraceDurationIncrementer) AddGraceDuration() time.Duration { + return mod.addGraceDuration() +} + type ProtoLoggerProvider struct { ProtoModule logger func(mod gotenberg.Module) (*zap.Logger, error) @@ -84,11 +93,12 @@ func TestAPI_Descriptor(t *testing.T) { func TestAPI_Provision(t *testing.T) { for i, tc := range []struct { - ctx *gotenberg.Context - setEnv func(i int) - expectPort int - expectMiddlewares []Middleware - expectErr bool + ctx *gotenberg.Context + setEnv func(i int) + expectPort int + expectMiddlewares []Middleware + expectGraceDuration time.Duration + expectErr bool }{ { ctx: func() *gotenberg.Context { @@ -183,14 +193,14 @@ func TestAPI_Provision(t *testing.T) { }, { ctx: func() *gotenberg.Context { - mod := struct{ ProtoMultipartFormDataRouter }{} + mod := struct{ ProtoRouter }{} mod.descriptor = func() gotenberg.ModuleDescriptor { return gotenberg.ModuleDescriptor{ID: "foo", New: func() gotenberg.Module { return mod }} } mod.validate = func() error { return errors.New("foo") } - mod.routes = func() ([]MultipartFormDataRoute, error) { + mod.routes = func() ([]Route, error) { return nil, nil } @@ -255,14 +265,14 @@ func TestAPI_Provision(t *testing.T) { }, { ctx: func() *gotenberg.Context { - mod := struct{ ProtoMultipartFormDataRouter }{} + mod := struct{ ProtoRouter }{} mod.descriptor = func() gotenberg.ModuleDescriptor { return gotenberg.ModuleDescriptor{ID: "foo", New: func() gotenberg.Module { return mod }} } mod.validate = func() error { return nil } - mod.routes = func() ([]MultipartFormDataRoute, error) { + mod.routes = func() ([]Route, error) { return nil, errors.New("foo") } @@ -325,6 +335,60 @@ func TestAPI_Provision(t *testing.T) { }(), expectErr: true, }, + + { + ctx: func() *gotenberg.Context { + mod := struct { + ProtoGarbageCollectorGraceDurationIncrementer + }{} + mod.descriptor = func() gotenberg.ModuleDescriptor { + return gotenberg.ModuleDescriptor{ID: "foo", New: func() gotenberg.Module { return mod }} + } + mod.validate = func() error { + return errors.New("foo") + } + mod.addGraceDuration = func() time.Duration { + return 0 + } + + return gotenberg.NewContext( + gotenberg.ParsedFlags{ + FlagSet: new(API).Descriptor().FlagSet, + }, + []gotenberg.ModuleDescriptor{ + mod.Descriptor(), + }, + ) + }(), + expectErr: true, + }, + { + ctx: func() *gotenberg.Context { + mod := struct { + ProtoGarbageCollectorGraceDurationIncrementer + }{} + mod.descriptor = func() gotenberg.ModuleDescriptor { + return gotenberg.ModuleDescriptor{ID: "foo", New: func() gotenberg.Module { return mod }} + } + mod.validate = func() error { + return nil + } + mod.addGraceDuration = func() time.Duration { + return time.Duration(3) * time.Second + } + + return gotenberg.NewContext( + gotenberg.ParsedFlags{ + FlagSet: new(API).Descriptor().FlagSet, + }, + []gotenberg.ModuleDescriptor{ + mod.Descriptor(), + }, + ) + }(), + expectGraceDuration: time.Duration(93) * time.Second, + expectErr: true, + }, { ctx: func() *gotenberg.Context { return gotenberg.NewContext( @@ -359,15 +423,15 @@ func TestAPI_Provision(t *testing.T) { }, { ctx: func() *gotenberg.Context { - mod1 := struct{ ProtoMultipartFormDataRouter }{} + mod1 := struct{ ProtoRouter }{} mod1.descriptor = func() gotenberg.ModuleDescriptor { return gotenberg.ModuleDescriptor{ID: "foo", New: func() gotenberg.Module { return mod1 }} } mod1.validate = func() error { return nil } - mod1.routes = func() ([]MultipartFormDataRoute, error) { - return []MultipartFormDataRoute{{}}, nil + mod1.routes = func() ([]Route, error) { + return []Route{{}}, nil } mod2 := struct{ ProtoMiddlewareProvider }{} @@ -455,11 +519,15 @@ func TestAPI_Provision(t *testing.T) { err := mod.Provision(tc.ctx) if tc.expectPort != 0 && mod.port != tc.expectPort { - t.Errorf("expected port %d but got %d", tc.expectPort, mod.port) + t.Errorf("test %d: expected port %d but got %d", i, tc.expectPort, mod.port) } if !reflect.DeepEqual(mod.externalMiddlewares, tc.expectMiddlewares) { - t.Errorf("expected %+v, but got: %+v", tc.expectMiddlewares, mod.externalMiddlewares) + t.Errorf("test %d: expected %+v, but got: %+v", i, tc.expectMiddlewares, mod.externalMiddlewares) + } + + if tc.expectGraceDuration != 0 && mod.gcGraceDuration != tc.expectGraceDuration { + t.Errorf("test %d: expected gc grace duration '%s' but got '%s'", i, tc.expectGraceDuration, mod.gcGraceDuration) } if tc.expectErr && err == nil { @@ -477,7 +545,7 @@ func TestAPI_Validate(t *testing.T) { port int rootPath string traceHeader string - routes []MultipartFormDataRoute + routes []Route middlewares []Middleware expectErr bool }{ @@ -494,7 +562,7 @@ func TestAPI_Validate(t *testing.T) { port: 10, rootPath: "/foo/", traceHeader: "foo", - routes: []MultipartFormDataRoute{ + routes: []Route{ { Path: "", }, @@ -505,7 +573,7 @@ func TestAPI_Validate(t *testing.T) { port: 10, rootPath: "/foo/", traceHeader: "foo", - routes: []MultipartFormDataRoute{ + routes: []Route{ { Path: "foo", }, @@ -516,9 +584,10 @@ func TestAPI_Validate(t *testing.T) { port: 10, rootPath: "/foo/", traceHeader: "foo", - routes: []MultipartFormDataRoute{ + routes: []Route{ { - Path: "/foo", + Path: "/foo", + IsMultipart: true, }, }, expectErr: true, @@ -527,14 +596,41 @@ func TestAPI_Validate(t *testing.T) { port: 10, rootPath: "/foo/", traceHeader: "foo", - routes: []MultipartFormDataRoute{ + routes: []Route{ { + Path: "/forms/foo", + IsMultipart: true, + }, + }, + expectErr: true, + }, + { + port: 10, + rootPath: "/foo/", + traceHeader: "foo", + routes: []Route{ + { + Method: http.MethodPost, + Path: "/forms/foo", + IsMultipart: true, + }, + }, + expectErr: true, + }, + { + port: 10, + rootPath: "/foo/", + traceHeader: "foo", + routes: []Route{ + { + Method: http.MethodPost, Path: "/foo", - Handler: func(_ *Context) error { return nil }, + Handler: func(_ echo.Context) error { return nil }, }, { + Method: http.MethodPost, Path: "/foo", - Handler: func(_ *Context) error { return nil }, + Handler: func(_ echo.Context) error { return nil }, }, }, expectErr: true, @@ -554,10 +650,11 @@ func TestAPI_Validate(t *testing.T) { port: 10, rootPath: "/foo/", traceHeader: "foo", - routes: []MultipartFormDataRoute{ + routes: []Route{ { + Method: http.MethodGet, Path: "/foo", - Handler: func(_ *Context) error { return nil }, + Handler: func(_ echo.Context) error { return nil }, }, }, middlewares: []Middleware{ @@ -575,11 +672,11 @@ func TestAPI_Validate(t *testing.T) { }, } { mod := API{ - port: tc.port, - rootPath: tc.rootPath, - traceHeader: tc.traceHeader, - multipartFormDataRoutes: tc.routes, - externalMiddlewares: tc.middlewares, + port: tc.port, + rootPath: tc.rootPath, + traceHeader: tc.traceHeader, + routes: tc.routes, + externalMiddlewares: tc.middlewares, } err := mod.Validate() @@ -598,10 +695,15 @@ func TestAPI_Start(t *testing.T) { mod := new(API) mod.port = 3000 mod.rootPath = "/" - mod.multipartFormDataRoutes = []MultipartFormDataRoute{ + mod.disableHealthCheckLogging = true + mod.routes = []Route{ { - Path: "/foo", - Handler: func(ctx *Context) error { + Method: http.MethodPost, + Path: "/forms/foo", + IsMultipart: true, + DisableLogging: true, + Handler: func(c echo.Context) error { + ctx := c.Get("context").(*Context) ctx.outputPaths = []string{ "/tests/test/testdata/api/sample1.txt", } @@ -610,13 +712,35 @@ func TestAPI_Start(t *testing.T) { }, }, { - Path: "/bar", - Handler: func(_ *Context) error { return errors.New("foo") }, + Method: http.MethodPost, + Path: "/forms/bar", + IsMultipart: true, + Handler: func(_ echo.Context) error { return errors.New("foo") }, }, } mod.externalMiddlewares = []Middleware{ { - RunBeforeRouter: true, + Stack: PreRouterStack, + Handler: func() echo.MiddlewareFunc { + return func(next echo.HandlerFunc) echo.HandlerFunc { + return func(c echo.Context) error { + return next(c) + } + } + }(), + }, + { + Stack: MultipartStack, + Handler: func() echo.MiddlewareFunc { + return func(next echo.HandlerFunc) echo.HandlerFunc { + return func(c echo.Context) error { + return next(c) + } + } + }(), + }, + { + Stack: DefaultStack, Handler: func() echo.MiddlewareFunc { return func(next echo.HandlerFunc) echo.HandlerFunc { return func(c echo.Context) error { @@ -721,10 +845,11 @@ func TestAPI_StartupMessage(t *testing.T) { func TestAPI_Stop(t *testing.T) { mod := API{ port: 3000, - multipartFormDataRoutes: []MultipartFormDataRoute{ + routes: []Route{ { + Method: http.MethodGet, Path: "/foo", - Handler: func(_ *Context) error { return nil }, + Handler: func(_ echo.Context) error { return nil }, }, }, logger: zap.NewNop(), @@ -742,52 +867,35 @@ func TestAPI_Stop(t *testing.T) { } func TestAPI_GraceDuration(t *testing.T) { - for i, tc := range []struct { - mod API - expect time.Duration - }{ - { - mod: API{ - readTimeout: time.Duration(1) * time.Second, - processTimeout: time.Duration(1) * time.Second, - writeTimeout: time.Duration(1) * time.Second, - disableWebhook: true, - }, - expect: time.Duration(3) * time.Second, - }, - { - mod: API{ - readTimeout: time.Duration(1) * time.Second, - processTimeout: time.Duration(1) * time.Second, - writeTimeout: time.Duration(1) * time.Second, - webhookMaxRetry: 5, - webhookRetryMaxWait: time.Duration(5) * time.Second, - }, - expect: time.Duration(28) * time.Second, - }, - } { - actual := tc.mod.GraceDuration() + mod := API{ + gcGraceDuration: time.Duration(3) * time.Second, + } - if actual != tc.expect { - t.Errorf("test %d: expected '%s' but got '%s'", i, tc.expect, actual) - } + expect := time.Duration(3) * time.Second + actual := mod.GraceDuration() + + if actual != expect { + t.Errorf("expected '%s' but got '%s'", expect, actual) } } // Interface guards. var ( - _ gotenberg.Module = (*ProtoModule)(nil) - _ gotenberg.Validator = (*ProtoValidator)(nil) - _ gotenberg.Module = (*ProtoValidator)(nil) - _ MultipartFormDataRouter = (*ProtoMultipartFormDataRouter)(nil) - _ gotenberg.Module = (*ProtoMultipartFormDataRouter)(nil) - _ gotenberg.Validator = (*ProtoMultipartFormDataRouter)(nil) - _ MiddlewareProvider = (*ProtoMiddlewareProvider)(nil) - _ gotenberg.Module = (*ProtoMiddlewareProvider)(nil) - _ gotenberg.Validator = (*ProtoMiddlewareProvider)(nil) - _ HealthChecker = (*ProtoHealthChecker)(nil) - _ gotenberg.Module = (*ProtoHealthChecker)(nil) - _ gotenberg.Validator = (*ProtoHealthChecker)(nil) - _ gotenberg.LoggerProvider = (*ProtoLoggerProvider)(nil) - _ gotenberg.Module = (*ProtoLoggerProvider)(nil) + _ gotenberg.Module = (*ProtoModule)(nil) + _ gotenberg.Validator = (*ProtoValidator)(nil) + _ gotenberg.Module = (*ProtoValidator)(nil) + _ Router = (*ProtoRouter)(nil) + _ gotenberg.Module = (*ProtoRouter)(nil) + _ gotenberg.Validator = (*ProtoRouter)(nil) + _ MiddlewareProvider = (*ProtoMiddlewareProvider)(nil) + _ gotenberg.Module = (*ProtoMiddlewareProvider)(nil) + _ gotenberg.Validator = (*ProtoMiddlewareProvider)(nil) + _ HealthChecker = (*ProtoHealthChecker)(nil) + _ gotenberg.Module = (*ProtoHealthChecker)(nil) + _ gotenberg.Validator = (*ProtoHealthChecker)(nil) + _ GarbageCollectorGraceDurationIncrementer = (*ProtoGarbageCollectorGraceDurationIncrementer)(nil) + _ gotenberg.Module = (*ProtoGarbageCollectorGraceDurationIncrementer)(nil) + _ gotenberg.Validator = (*ProtoGarbageCollectorGraceDurationIncrementer)(nil) + _ gotenberg.LoggerProvider = (*ProtoLoggerProvider)(nil) + _ gotenberg.Module = (*ProtoLoggerProvider)(nil) ) diff --git a/pkg/modules/api/context.go b/pkg/modules/api/context.go index 41e53ce0..44929e75 100644 --- a/pkg/modules/api/context.go +++ b/pkg/modules/api/context.go @@ -227,9 +227,9 @@ func (ctx Context) Log() *zap.Logger { return ctx.logger } -// buildOutputFile builds the output file according to the output paths +// BuildOutputFile builds the output file according to the output paths // registered in the context. If many output paths, an archive is created. -func (ctx Context) buildOutputFile() (string, error) { +func (ctx Context) BuildOutputFile() (string, error) { if ctx.cancelled { return "", ErrContextAlreadyClosed } @@ -265,6 +265,18 @@ func (ctx Context) buildOutputFile() (string, error) { return archivePath, nil } +// OutputFilename returns the filename based on the given output path or the +// "Gotenberg-Output-Filename" header's value. +func (ctx Context) OutputFilename(outputPath string) string { + filename := ctx.echoCtx.Request().Header.Get("Gotenberg-Output-Filename") + + if filename == "" { + return filepath.Base(outputPath) + } + + return fmt.Sprintf("%s%s", filename, filepath.Ext(outputPath)) +} + // MockContext is a helper for tests. // // ctx := &api.MockContext{Context: &api.Context{}} @@ -316,3 +328,19 @@ func (ctx *MockContext) SetCancelled(cancelled bool) { func (ctx MockContext) OutputPaths() []string { return ctx.outputPaths } + +// SetLogger sets the logger. +// +// ctx := &api.MockContext{Context: &api.Context{}} +// ctx.SetLogger(zap.NewNop()) +func (ctx *MockContext) SetLogger(logger *zap.Logger) { + ctx.logger = logger +} + +// SetEchoContext sets the echo.Context. +// +// ctx := &api.MockContext{Context: &api.Context{}} +// ctx.setEchoContext(c) +func (ctx *MockContext) SetEchoContext(c echo.Context) { + ctx.Context.echoCtx = c +} diff --git a/pkg/modules/api/context_test.go b/pkg/modules/api/context_test.go index 3f872790..3ee39699 100644 --- a/pkg/modules/api/context_test.go +++ b/pkg/modules/api/context_test.go @@ -248,7 +248,7 @@ func TestContext_Log(t *testing.T) { } } -func TestContext_buildOutputFile(t *testing.T) { +func TestContext_BuildOutputFile(t *testing.T) { for i, tc := range []struct { ctx *Context expectErr bool @@ -285,7 +285,7 @@ func TestContext_buildOutputFile(t *testing.T) { tc.ctx.dirPath = dirPath tc.ctx.logger = zap.NewNop() - _, err = tc.ctx.buildOutputFile() + _, err = tc.ctx.BuildOutputFile() if tc.expectErr && err == nil { t.Errorf("test %d: expected error but got: %v", i, err) @@ -302,6 +302,39 @@ func TestContext_buildOutputFile(t *testing.T) { } } +func TestContext_OutputFilename(t *testing.T) { + for i, tc := range []struct { + ctx *Context + outputPath string + expectOutputFilename string + }{ + { + ctx: func() *Context { + c := echo.New().NewContext(httptest.NewRequest(http.MethodGet, "/foo", nil), nil) + c.Request().Header.Set("Gotenberg-Output-Filename", "foo") + + return &Context{echoCtx: c} + }(), + outputPath: "/foo/bar.txt", + expectOutputFilename: "foo.txt", + }, + { + ctx: func() *Context { + c := echo.New().NewContext(httptest.NewRequest(http.MethodGet, "/foo", nil), nil) + return &Context{echoCtx: c} + }(), + outputPath: "/foo/foo.txt", + expectOutputFilename: "foo.txt", + }, + } { + actual := tc.ctx.OutputFilename(tc.outputPath) + + if actual != tc.expectOutputFilename { + t.Errorf("test %d: expected '%s' but got '%s'", i, tc.expectOutputFilename, actual) + } + } +} + func TestMockContext_SetDirPath(t *testing.T) { mock := &MockContext{&Context{}} mock.SetDirPath("/foo") @@ -371,3 +404,29 @@ func TestMockContext_OutputPaths(t *testing.T) { t.Errorf("expected %+v but got: %+v", expect, actual) } } + +func TestMockContext_SetLogger(t *testing.T) { + mock := MockContext{&Context{}} + + expect := zap.NewNop() + mock.SetLogger(expect) + + actual := mock.logger + + if actual != expect { + t.Errorf("expected %v but got %v", expect, actual) + } +} + +func TestMockContext_SetEchoContext(t *testing.T) { + mock := MockContext{&Context{}} + + expect := echo.New().NewContext(nil, nil) + mock.SetEchoContext(expect) + + actual := mock.echoCtx + + if actual != expect { + t.Errorf("expected %v but got %v", expect, actual) + } +} diff --git a/pkg/modules/api/middlewares.go b/pkg/modules/api/middlewares.go index 5a0e67b1..c2c818e1 100644 --- a/pkg/modules/api/middlewares.go +++ b/pkg/modules/api/middlewares.go @@ -1,95 +1,54 @@ package api import ( - "bufio" - "bytes" "context" - "encoding/json" "errors" "fmt" "net/http" - "os" - "path/filepath" - "regexp" - "strconv" - "strings" "time" "github.com/google/uuid" - "github.com/hashicorp/go-retryablehttp" "github.com/labstack/echo/v4" "go.uber.org/zap" ) +// ErrAsyncProcess happens when a handler or middleware handles a request in an +// asynchronous fashion. +var ErrAsyncProcess = errors.New("async process") + +// ParseError parses an error and returns the corresponding HTTP status and +// HTTP message. +func ParseError(err error) (int, string) { + echoErr, ok := err.(*echo.HTTPError) + if ok { + return echoErr.Code, http.StatusText(echoErr.Code) + } + + if errors.Is(err, context.DeadlineExceeded) { + return http.StatusServiceUnavailable, http.StatusText(http.StatusServiceUnavailable) + } + + var httpErr HTTPError + if errors.As(err, &httpErr) { + return httpErr.HTTPError() + } + + // Default 500 status code. + return http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError) +} + // httpErrorHandler is the centralized HTTP error handler. It parses the error, -// returns either a response as "text/plain; charset=UTF-8" or, if a webhook -// client exists in the echo.Context, sends a request to the webhook error URL -// with a JSON body containing the trace, the status and the error message. -func httpErrorHandler(traceHeader string) echo.HTTPErrorHandler { +// returns a response as "text/plain; charset=UTF-8". +func httpErrorHandler() echo.HTTPErrorHandler { return func(err error, c echo.Context) { - parseError := func(err error) (int, string) { - echoErr, ok := err.(*echo.HTTPError) - if ok { - return echoErr.Code, http.StatusText(echoErr.Code) - } - - if errors.Is(err, context.DeadlineExceeded) { - return http.StatusServiceUnavailable, http.StatusText(http.StatusServiceUnavailable) - } - - var httpErr HTTPError - if errors.As(err, &httpErr) { - return httpErr.HTTPError() - } - - // Default 500 status code. - return http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError) - } - - status, message := parseError(err) - logger := c.Get("logger").(*zap.Logger) - clientOrNil := c.Get("webhookClient") + status, message := ParseError(err) - // No webhook client, meaning we can send the error as a response. - if clientOrNil == nil { - c.Response().Header().Add(echo.HeaderContentType, echo.MIMETextPlainCharsetUTF8) + c.Response().Header().Add(echo.HeaderContentType, echo.MIMETextPlainCharsetUTF8) - err = c.String(status, message) - - if err != nil { - logger.Error(fmt.Sprintf("send error response: %s", err.Error())) - } - - return - } - - // We have to send the error to the webhook. - client := clientOrNil.(*webhookClient) - - body := struct { - Status int `json:"status"` - Message string `json:"message"` - }{ - Status: status, - Message: message, - } - - b, err := json.Marshal(body) + err = c.String(status, message) if err != nil { - logger.Error(fmt.Sprintf("marshal JSON: %s", err.Error())) - - return - } - - headers := map[string]string{ - echo.HeaderContentType: echo.MIMEApplicationJSONCharsetUTF8, - traceHeader: c.Get("trace").(string), - } - - err = client.send(bytes.NewReader(b), headers, true) - if err != nil { - logger.Error(fmt.Sprintf("send error response to webhook: %s", err.Error())) + logger.Error(fmt.Sprintf("send error response: %s", err.Error())) } } } @@ -116,7 +75,7 @@ func latencyMiddleware() echo.MiddlewareFunc { // URI. // // rootPath := c.Get("rootPath").(string) -// healthURI := fmt.Sprintf("%shealth", rootPath) +// healthURI := fmt.Sprintf("%s/health", rootPath) // // // Skip the middleware if health check URI. // if c.Request().RequestURI == healthURI { @@ -139,6 +98,7 @@ func rootPathMiddleware(rootPath string) echo.MiddlewareFunc { // the header is not present / its value is empty. // // trace := c.Get("trace").(string) +// traceHeader := c.Get("traceHeader").(string). func traceMiddleware(header string) echo.MiddlewareFunc { return func(next echo.HandlerFunc) echo.HandlerFunc { return func(c echo.Context) error { @@ -150,6 +110,7 @@ func traceMiddleware(header string) echo.MiddlewareFunc { } c.Set("trace", trace) + c.Set("traceHeader", header) c.Response().Header().Add(header, trace) // Call the next middleware in the chain. @@ -158,12 +119,30 @@ func traceMiddleware(header string) echo.MiddlewareFunc { } } +// timeoutsMiddleware sets the read, process and write timeouts in the +// echo.Context under "readTimeout", "processTimeout" and "writeTimeout". +// +// readTimeout := c.Get("readTimeout").(time.Duration) +// processTimeout := c.Get("processTimeout").(time.Duration) +// writeTimeout := c.Get("writeTimeout").(time.Duration) +func timeoutsMiddleware(readTimeout, processTimeout, writeTimeout time.Duration) echo.MiddlewareFunc { + return func(next echo.HandlerFunc) echo.HandlerFunc { + return func(c echo.Context) error { + c.Set("readTimeout", readTimeout) + c.Set("processTimeout", processTimeout) + c.Set("writeTimeout", writeTimeout) + + // Call the next middleware in the chain. + return next(c) + } + } +} + // loggerMiddleware sets the logger in the echo.Context under "logger" and logs -// a request result (but does not log a webhook call result, which is the job -// of the webhookClient). +// a synchronous request result. // // logger := c.Get("logger").(*zap.Logger) -func loggerMiddleware(logger *zap.Logger, skipHealthRouteLogging bool) echo.MiddlewareFunc { +func loggerMiddleware(logger *zap.Logger, disableLoggingForPaths []string) echo.MiddlewareFunc { return func(next echo.HandlerFunc) echo.HandlerFunc { return func(c echo.Context) error { startTime := c.Get("startTime").(time.Time) @@ -179,11 +158,11 @@ func loggerMiddleware(logger *zap.Logger, skipHealthRouteLogging bool) echo.Midd c.Error(err) } - if skipHealthRouteLogging { + for _, path := range disableLoggingForPaths { rootPath := c.Get("rootPath").(string) - healthURI := fmt.Sprintf("%shealth", rootPath) + URI := fmt.Sprintf("%s%s", rootPath, path) - if c.Request().RequestURI == healthURI { + if c.Request().RequestURI == URI { return nil } } @@ -225,317 +204,65 @@ func loggerMiddleware(logger *zap.Logger, skipHealthRouteLogging bool) echo.Midd } } -type contextMiddlewareConfig struct { - traceHeader string - timeout struct { - process time.Duration - write time.Duration - } - webhook struct { - allowList *regexp.Regexp - denyList *regexp.Regexp - errorAllowList *regexp.Regexp - errorDenyList *regexp.Regexp - maxRetry int - retryMinWait time.Duration - retryMaxWait time.Duration - disable bool - } -} - -// contextMiddleware handles the result of a "multipart/form-data" request. If -// a webhook URL is present in the headers, exit early and process the result -// in a goroutine. -func contextMiddleware(cfg contextMiddlewareConfig) echo.MiddlewareFunc { +// contextMiddleware, a middleware for "multipart/form-data" requests, sets the +// Context and related context.CancelFunc in the echo.Context under "context" +// and "cancel". If the process is synchronous, it also handles the result of a +// "multipart/form-data" request. +// +// ctx := c.Get("context").(*api.Context) +// cancel := c.Get("cancel").(context.CancelFunc) +func contextMiddleware(processTimeout time.Duration) echo.MiddlewareFunc { return func(next echo.HandlerFunc) echo.HandlerFunc { return func(c echo.Context) error { - webhookURL := c.Request().Header.Get("Gotenberg-Webhook-Url") - logger := c.Get("logger").(*zap.Logger).With(zap.Bool("webhook", webhookURL != "")) + logger := c.Get("logger").(*zap.Logger) // We create a context with a timeout so that underlying processes are // able to stop early and handle correctly a timeout scenario. - ctx, cancel, err := newContext(c, logger, cfg.timeout.process) + ctx, cancel, err := newContext(c, logger, processTimeout) if err != nil { cancel() return fmt.Errorf("create request context: %w", err) } c.Set("context", ctx) + c.Set("cancel", cancel) - // Helper function for retrieving/creating the output filename. - outputFilename := func(outputPath string) string { - filename := c.Request().Header.Get("Gotenberg-Output-Filename") + // Call the next middleware in the chain. + err = next(c) - if filename == "" { - return filepath.Base(outputPath) - } - - return fmt.Sprintf("%s%s", filename, filepath.Ext(outputPath)) + if errors.Is(err, ErrAsyncProcess) { + // A middleware/handler tells us that it's handling the process + // in an asynchronous fashion. Therefore, we must not cancel + // the context nor send an output file. + return c.NoContent(http.StatusNoContent) } - if webhookURL == "" { - defer cancel() + defer cancel() - // No webhook URL, call the next middleware in the chain. - err := next(c) - - if err != nil { - return err - } - - // No error, let's build the output file. - outputPath, err := ctx.buildOutputFile() - if err != nil { - return fmt.Errorf("build output file: %w", err) - } - - // Send the output file. - err = c.Attachment(outputPath, outputFilename(outputPath)) - if err != nil { - return fmt.Errorf("send response: %w", err) - } - - return nil - } - - // Ok, we got a webhook URL. - - if cfg.webhook.disable { - // The client requested the webhook feature, but it has been - // disabled. Let's tell the client about that. - cancel() - - return WrapError( - errors.New("webhook feature requested but it is disabled"), - NewSentinelHTTPError(http.StatusForbidden, "Invalid 'Gotenberg-Webhook-Url' header: feature is disabled"), - ) - } - - // Do we have a webhook error URL in case of... error? - webhookErrorURL := c.Request().Header.Get("Gotenberg-Webhook-Error-Url") - - if webhookErrorURL == "" { - cancel() - - return WrapError( - errors.New("empty webhook error URL"), - NewSentinelHTTPError(http.StatusBadRequest, "Invalid 'Gotenberg-Webhook-Error-Url' header: empty value or header not provided"), - ) - } - - // Let's check if the webhook URLs are acceptable according to our - // allowed/denied lists. - filter := func(URL, header string, allowList, denyList *regexp.Regexp) error { - if !allowList.MatchString(URL) { - return WrapError( - fmt.Errorf("'%s' does not match the expression from the allowed list", URL), - NewSentinelHTTPError( - http.StatusForbidden, - fmt.Sprintf("Invalid '%s' header value: '%s' does not match the authorized URLs", header, URL), - ), - ) - } - - if denyList.String() != "" && denyList.MatchString(URL) { - return WrapError( - fmt.Errorf("'%s' matches the expression from the denied list", URL), - NewSentinelHTTPError( - http.StatusForbidden, - fmt.Sprintf("Invalid '%s' header value: '%s' does not match the authorized URLs", header, URL), - ), - ) - } - - return nil - } - - err = filter(webhookURL, "Gotenberg-Webhook-Url", cfg.webhook.allowList, cfg.webhook.denyList) if err != nil { - cancel() - - return fmt.Errorf("filter webhook URL: %w", err) + return err } - err = filter(webhookErrorURL, "Gotenberg-Webhook-Error-Url", cfg.webhook.errorAllowList, cfg.webhook.errorDenyList) + // No error, let's build the output file. + outputPath, err := ctx.BuildOutputFile() if err != nil { - cancel() - - return fmt.Errorf("filter webhook error URL: %w", err) + return fmt.Errorf("build output file: %w", err) } - // Let's check the HTTP methods for calling the webhook URLs. - methodFromHeader := func(header string) (string, error) { - method := c.Request().Header.Get(header) - - if method == "" { - return http.MethodPost, nil - } - - method = strings.ToUpper(method) - - switch method { - case http.MethodPost: - return method, nil - case http.MethodPatch: - return method, nil - case http.MethodPut: - return method, nil - } - - return "", WrapError( - fmt.Errorf("webhook method '%s' is not '%s', '%s' or '%s'", method, http.MethodPost, http.MethodPatch, http.MethodPut), - NewSentinelHTTPError( - http.StatusBadRequest, - fmt.Sprintf("Invalid '%s' header value: expected '%s', '%s' or '%s', but got '%s'", header, http.MethodPost, http.MethodPatch, http.MethodPut, method), - ), - ) - } - - webhookMethod, err := methodFromHeader("Gotenberg-Webhook-Method") + // Send the output file. + err = c.Attachment(outputPath, ctx.OutputFilename(outputPath)) if err != nil { - cancel() - - return fmt.Errorf("get method to use for webhook: %w", err) + return fmt.Errorf("send response: %w", err) } - webhookErrorMethod, err := methodFromHeader("Gotenberg-Webhook-Error-Method") - if err != nil { - cancel() - - return fmt.Errorf("get method to use for webhook error: %w", err) - } - - // What about extra HTTP headers? - var extraHTTPHeaders map[string]string - - extraHTTPHeadersJSON := c.Request().Header.Get("Gotenberg-Webhook-Extra-Http-Headers") - if extraHTTPHeadersJSON != "" { - err = json.Unmarshal([]byte(extraHTTPHeadersJSON), &extraHTTPHeaders) - if err != nil { - cancel() - - return WrapError( - fmt.Errorf("unmarshal webhook extra HTTP headers: %w", err), - NewSentinelHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid 'Gotenberg-Webhook-Extra-Http-Headers' header value: %s", err.Error())), - ) - } - } - - client := &webhookClient{ - url: webhookURL, - method: webhookMethod, - errorURL: webhookErrorURL, - errorMethod: webhookErrorMethod, - extraHTTPHeaders: extraHTTPHeaders, - startTime: c.Get("startTime").(time.Time), - - client: &retryablehttp.Client{ - HTTPClient: &http.Client{ - Timeout: cfg.timeout.write, - }, - RetryMax: cfg.webhook.maxRetry, - RetryWaitMin: cfg.webhook.retryMinWait, - RetryWaitMax: cfg.webhook.retryMaxWait, - Logger: leveledLogger{ - logger: logger, - }, - CheckRetry: retryablehttp.DefaultRetryPolicy, - Backoff: retryablehttp.DefaultBackoff, - }, - logger: logger, - } - - c.Set("webhookClient", client) - - // As a webhook URL has been given, we handle the request in a - // goroutine and return immediately. - go func() { - defer cancel() - - // Call the next middleware in the chain. - err := next(c) - - if err != nil { - // The process failed for whatever reason. Let's send the - // details to the webhook. - ctx.Log().Error(err.Error()) - c.Error(err) - - return - } - - // No error, let's get build the output file. - outputPath, err := ctx.buildOutputFile() - if err != nil { - ctx.Log().Error(fmt.Sprintf("build output file: %s", err)) - c.Error(err) - - return - } - - outputFile, err := os.Open(outputPath) - if err != nil { - ctx.Log().Error(fmt.Sprintf("open output file: %s", err)) - c.Error(err) - - return - } - - defer func() { - err := outputFile.Close() - if err != nil { - ctx.Log().Error(fmt.Sprintf("close output file: %s", err)) - } - }() - - fileHeader := make([]byte, 512) - _, err = outputFile.Read(fileHeader) - if err != nil { - ctx.Log().Error(fmt.Sprintf("read header of output file: %s", err)) - c.Error(err) - - return - } - - fileStat, err := outputFile.Stat() - if err != nil { - ctx.Log().Error(fmt.Sprintf("get stat from output file: %s", err)) - c.Error(err) - - return - } - - _, err = outputFile.Seek(0, 0) - if err != nil { - ctx.Log().Error(fmt.Sprintf("reset output file reader: %s", err)) - c.Error(err) - - return - } - - headers := map[string]string{ - echo.HeaderContentDisposition: fmt.Sprintf("attachement; filename=%q", outputFilename(outputPath)), - echo.HeaderContentType: http.DetectContentType(fileHeader), - echo.HeaderContentLength: strconv.FormatInt(fileStat.Size(), 10), - cfg.traceHeader: c.Get("trace").(string), - } - - // Send the output file to the webhook. - err = client.send(bufio.NewReader(outputFile), headers, false) - if err != nil { - ctx.Log().Error(fmt.Sprintf("send output file to webhook: %s", err)) - c.Error(err) - } - }() - - return c.NoContent(http.StatusNoContent) + return nil } } } -// timeoutMiddleware manages hard timeout scenarios, i.e., when a route handler -// fails to timeout as expected. -func timeoutMiddleware(hardTimeout time.Duration) echo.MiddlewareFunc { +// hardTimeoutMiddleware manages hard timeout scenarios, i.e., when a route +// handler fails to timeout as expected. +func hardTimeoutMiddleware(hardTimeout time.Duration) echo.MiddlewareFunc { return func(next echo.HandlerFunc) echo.HandlerFunc { return func(c echo.Context) error { logger := c.Get("logger").(*zap.Logger) diff --git a/pkg/modules/api/middlewares_test.go b/pkg/modules/api/middlewares_test.go index d3391eca..578e3e85 100644 --- a/pkg/modules/api/middlewares_test.go +++ b/pkg/modules/api/middlewares_test.go @@ -3,28 +3,21 @@ package api import ( "bytes" "context" - "encoding/json" "errors" - "fmt" - "io/ioutil" - "math/rand" "mime/multipart" "net/http" "net/http/httptest" - "regexp" "strings" "testing" "time" - "github.com/hashicorp/go-retryablehttp" "github.com/labstack/echo/v4" "go.uber.org/zap" ) -func TestHttpErrorHandler(t *testing.T) { +func TestParseError(t *testing.T) { for i, tc := range []struct { err error - webhookClient *webhookClient expectStatus int expectMessage string }{ @@ -46,30 +39,42 @@ func TestHttpErrorHandler(t *testing.T) { expectStatus: http.StatusBadRequest, expectMessage: "foo", }, + } { + actualStatus, actualMessage := ParseError(tc.err) + + if actualStatus != tc.expectStatus { + t.Errorf("test %d: expected HTTP status code %d but got %d", i, tc.expectStatus, actualStatus) + } + + if actualMessage != tc.expectMessage { + t.Errorf("test %d: expected message '%s' but got '%s'", i, tc.expectMessage, actualMessage) + } + } +} + +func TestHttpErrorHandler(t *testing.T) { + for i, tc := range []struct { + err error + expectStatus int + expectMessage string + }{ { - err: echo.ErrInternalServerError, - webhookClient: &webhookClient{ - errorURL: "http://localhost:%d/", - errorMethod: http.MethodPost, - client: retryablehttp.NewClient(), - logger: zap.NewNop(), - }, + err: echo.ErrInternalServerError, expectStatus: http.StatusInternalServerError, expectMessage: http.StatusText(http.StatusInternalServerError), }, { - err: echo.ErrInternalServerError, - webhookClient: &webhookClient{ - errorURL: "non-existent", - errorMethod: http.MethodPost, - client: func() *retryablehttp.Client { - client := retryablehttp.NewClient() - client.RetryMax = 0 - - return client - }(), - logger: zap.NewNop(), - }, + err: context.DeadlineExceeded, + expectStatus: http.StatusServiceUnavailable, + expectMessage: http.StatusText(http.StatusServiceUnavailable), + }, + { + err: WrapError( + errors.New("foo"), + NewSentinelHTTPError(http.StatusBadRequest, "foo"), + ), + expectStatus: http.StatusBadRequest, + expectMessage: "foo", }, } { recorder := httptest.NewRecorder() @@ -81,105 +86,24 @@ func TestHttpErrorHandler(t *testing.T) { c := srv.NewContext(request, recorder) c.Set("logger", zap.NewNop()) - c.Set("trace", "foo") - if tc.webhookClient != nil { - c.Set("webhookClient", tc.webhookClient) + handler := httpErrorHandler() + handler(tc.err, c) + + contentType := recorder.Header().Get(echo.HeaderContentType) + if contentType != echo.MIMETextPlainCharsetUTF8 { + t.Errorf("test %d: expected %s '%s' but got '%s'", i, echo.HeaderContentType, echo.MIMETextPlainCharsetUTF8, contentType) } - if tc.webhookClient == nil { - handler := httpErrorHandler("Gotenberg-Trace") - handler(tc.err, c) + // Note: we cannot test the trace header in the response here, as it is set in the trace middleware. - contentType := recorder.Header().Get(echo.HeaderContentType) - if contentType != echo.MIMETextPlainCharsetUTF8 { - t.Errorf("test %d: expected %s '%s' but got '%s'", i, echo.HeaderContentType, echo.MIMETextPlainCharsetUTF8, contentType) - } - - // Note: we cannot test the trace header in the response here, as it is set in the trace middleware. - - if recorder.Code != tc.expectStatus { - t.Errorf("test %d: expected HTTP status code %d but got %d", i, tc.expectStatus, recorder.Code) - } - - if recorder.Body.String() != tc.expectMessage { - t.Errorf("test %d: expected message '%s' but got '%s'", i, tc.expectMessage, recorder.Body.String()) - } - - continue + if recorder.Code != tc.expectStatus { + t.Errorf("test %d: expected HTTP status code %d but got %d", i, tc.expectStatus, recorder.Code) } - func() { - rand.Seed(time.Now().UnixNano()) - webhookPort := rand.Intn(65535-1025+1) + 1025 - - tc.webhookClient.errorURL = fmt.Sprintf(tc.webhookClient.errorURL, webhookPort) - - c.Set("webhookClient", tc.webhookClient) - - webhook := echo.New() - webhook.HideBanner = true - webhook.HidePort = true - - webhook.POST( - "/", - func() echo.HandlerFunc { - return func(c echo.Context) error { - contentType := c.Request().Header.Get(echo.HeaderContentType) - if contentType != echo.MIMEApplicationJSONCharsetUTF8 { - t.Errorf("test %d: expected %s '%s' but got '%s'", i, echo.HeaderContentType, echo.MIMEApplicationJSONCharsetUTF8, contentType) - } - - trace := c.Request().Header.Get("Gotenberg-Trace") - if trace != "foo" { - t.Errorf("test %d: expected %s '%s' but got '%s'", i, "Gotenberg-Trace", "foo", trace) - } - - body, err := ioutil.ReadAll(c.Request().Body) - if err != nil { - t.Fatalf("test %d: expected not error but got: %v", i, err) - } - - result := struct { - Status int `json:"status"` - Message string `json:"message"` - }{} - - err = json.Unmarshal(body, &result) - if err != nil { - t.Fatalf("test %d: expected not error but got: %v", i, err) - } - - if result.Status != tc.expectStatus { - t.Errorf("test %d: expected status %d from JSON but got %d", i, tc.expectStatus, result.Status) - } - - if result.Message != tc.expectMessage { - t.Errorf("test %d: expected message '%s' from JSON but got '%s'", i, tc.expectMessage, result.Message) - } - - return nil - } - }(), - ) - - go func(server *echo.Echo, port, i int) { - err := webhook.Start(fmt.Sprintf(":%d", port)) - if !errors.Is(err, http.ErrServerClosed) { - t.Errorf("test %d: expected no error but got: %v", i, err) - } - }(webhook, webhookPort, i) - - defer func() { - err := webhook.Shutdown(context.TODO()) - if err != nil { - t.Errorf("test %d: expected no error but got: %v", i, err) - } - }() - - handler := httpErrorHandler("Gotenberg-Trace") - handler(tc.err, c) - }() + if recorder.Body.String() != tc.expectMessage { + t.Errorf("test %d: expected message '%s' but got '%s'", i, tc.expectMessage, recorder.Body.String()) + } } } @@ -298,11 +222,52 @@ func TestTraceMiddleware(t *testing.T) { } } +func TestTimeoutsMiddleware(t *testing.T) { + recorder := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodGet, "/foo", nil) + + srv := echo.New() + srv.HideBanner = true + srv.HidePort = true + + c := srv.NewContext(request, recorder) + + expectReadTimeout := time.Duration(1) * time.Second + expectProcessTimeout := time.Duration(2) * time.Second + expectWriteTimeout := time.Duration(3) * time.Second + + err := timeoutsMiddleware(expectReadTimeout, expectProcessTimeout, expectWriteTimeout)( + func(c echo.Context) error { + return nil + }, + )(c) + + if err != nil { + t.Fatalf("expected no error but got: %v", err) + } + + actualReadTimeout := c.Get("readTimeout").(time.Duration) + actualProcessTimeout := c.Get("processTimeout").(time.Duration) + actualWriteTimeout := c.Get("writeTimeout").(time.Duration) + + if actualReadTimeout != expectReadTimeout { + t.Errorf("expected '%s' but got '%s", expectReadTimeout, actualReadTimeout) + } + + if actualProcessTimeout != expectProcessTimeout { + t.Errorf("expected '%s' but got '%s", expectProcessTimeout, actualProcessTimeout) + } + + if actualWriteTimeout != expectWriteTimeout { + t.Errorf("expected '%s' but got '%s", actualWriteTimeout, expectWriteTimeout) + } +} + func TestLoggerMiddleware(t *testing.T) { for i, tc := range []struct { - request *http.Request - next echo.HandlerFunc - skipHealthRouteLogging bool + request *http.Request + next echo.HandlerFunc + skipLogging bool }{ { request: httptest.NewRequest(http.MethodGet, "/", nil), @@ -319,7 +284,7 @@ func TestLoggerMiddleware(t *testing.T) { return nil } }(), - skipHealthRouteLogging: true, + skipLogging: true, }, { request: httptest.NewRequest(http.MethodGet, "/health", nil), @@ -341,7 +306,12 @@ func TestLoggerMiddleware(t *testing.T) { c.Set("trace", "foo") c.Set("rootPath", "/") - err := loggerMiddleware(zap.NewNop(), tc.skipHealthRouteLogging)(tc.next)(c) + var disableLoggingForPaths []string + if tc.skipLogging { + disableLoggingForPaths = append(disableLoggingForPaths, tc.request.RequestURI) + } + + err := loggerMiddleware(zap.NewNop(), disableLoggingForPaths)(tc.next)(c) if err != nil { t.Errorf("test %d: expected no error but got: %v", i, err) @@ -349,7 +319,7 @@ func TestLoggerMiddleware(t *testing.T) { } } -func TestContextMiddlewareWithoutWebhook(t *testing.T) { +func TestContextMiddleware(t *testing.T) { buildMultipartFormDataRequest := func() *http.Request { body := &bytes.Buffer{} writer := multipart.NewWriter(body) @@ -376,6 +346,7 @@ func TestContextMiddlewareWithoutWebhook(t *testing.T) { request *http.Request next echo.HandlerFunc expectErr bool + expectStatus int expectContentType string expectFilename string }{ @@ -383,6 +354,15 @@ func TestContextMiddlewareWithoutWebhook(t *testing.T) { request: httptest.NewRequest(http.MethodGet, "/", nil), expectErr: true, }, + { + request: buildMultipartFormDataRequest(), + next: func() echo.HandlerFunc { + return func(c echo.Context) error { + return ErrAsyncProcess + } + }(), + expectStatus: http.StatusNoContent, + }, { request: buildMultipartFormDataRequest(), next: func() echo.HandlerFunc { @@ -418,6 +398,7 @@ func TestContextMiddlewareWithoutWebhook(t *testing.T) { return nil } }(), + expectStatus: http.StatusOK, expectContentType: "application/pdf", expectFilename: "foo.pdf", }, @@ -434,6 +415,7 @@ func TestContextMiddlewareWithoutWebhook(t *testing.T) { return nil } }(), + expectStatus: http.StatusOK, expectContentType: "application/zip", }, } { @@ -448,16 +430,7 @@ func TestContextMiddlewareWithoutWebhook(t *testing.T) { c.Set("trace", "foo") c.Set("startTime", time.Now()) - cfg := contextMiddlewareConfig{ - timeout: struct { - process time.Duration - write time.Duration - }{ - process: time.Duration(10) * time.Second, - }, - } - - err := contextMiddleware(cfg)(tc.next)(c) + err := contextMiddleware(time.Duration(10) * time.Second)(tc.next)(c) if tc.expectErr && err == nil { t.Errorf("test %d: expected error but got: %v", i, err) @@ -471,8 +444,12 @@ func TestContextMiddlewareWithoutWebhook(t *testing.T) { continue } - if recorder.Code != http.StatusOK { - t.Errorf("test %d: expected HTTP status code %d but got %d", i, http.StatusOK, recorder.Code) + if recorder.Code != tc.expectStatus { + t.Errorf("test %d: expected HTTP status code %d but got %d", i, tc.expectStatus, recorder.Code) + } + + if tc.expectStatus == http.StatusNoContent { + continue } contentType := recorder.Header().Get(echo.HeaderContentType) @@ -487,470 +464,7 @@ func TestContextMiddlewareWithoutWebhook(t *testing.T) { } } -func TestContextMiddlewareWithWebhook(t *testing.T) { - buildMultipartFormDataRequest := func() *http.Request { - body := &bytes.Buffer{} - writer := multipart.NewWriter(body) - - defer func() { - err := writer.Close() - if err != nil { - t.Fatalf("expected no error but got: %v", err) - } - }() - - err := writer.WriteField("foo", "foo") - if err != nil { - t.Fatalf("expected no error but got: %v", err) - } - - req := httptest.NewRequest(http.MethodPost, "/", body) - req.Header.Set(echo.HeaderContentType, writer.FormDataContentType()) - - return req - } - - buildContextMiddlewareConfig := func() contextMiddlewareConfig { - return contextMiddlewareConfig{ - traceHeader: "Gotenberg-Trace", - timeout: struct { - process time.Duration - write time.Duration - }{ - process: time.Duration(10) * time.Second, - write: time.Duration(10) * time.Second, - }, - webhook: struct { - allowList *regexp.Regexp - denyList *regexp.Regexp - errorAllowList *regexp.Regexp - errorDenyList *regexp.Regexp - maxRetry int - retryMinWait time.Duration - retryMaxWait time.Duration - disable bool - }{ - allowList: regexp.MustCompile(""), - denyList: regexp.MustCompile(""), - errorAllowList: regexp.MustCompile(""), - errorDenyList: regexp.MustCompile(""), - }, - } - } - - for i, tc := range []struct { - request *http.Request - cfg contextMiddlewareConfig - next echo.HandlerFunc - autoWebhookURLs bool - expectErr bool - expectHTTPErr bool - expectHTTPStatus int - expectWebhookContentType string - expectWebhookMethod string - expectWebhookExtraHTTPHeaders map[string]string - expectWebhookFilename string - }{ - { - request: func() *http.Request { - req := buildMultipartFormDataRequest() - req.Header.Set("Gotenberg-Webhook-Url", "foo") - - return req - }(), - cfg: func() contextMiddlewareConfig { - cfg := buildContextMiddlewareConfig() - cfg.webhook.disable = true - - return cfg - }(), - expectErr: true, - expectHTTPErr: true, - expectHTTPStatus: http.StatusForbidden, - }, - { - request: func() *http.Request { - req := buildMultipartFormDataRequest() - req.Header.Set("Gotenberg-Webhook-Url", "foo") - - return req - }(), - cfg: buildContextMiddlewareConfig(), - expectErr: true, - expectHTTPErr: true, - expectHTTPStatus: http.StatusBadRequest, - }, - { - request: func() *http.Request { - req := buildMultipartFormDataRequest() - req.Header.Set("Gotenberg-Webhook-Url", "foo") - req.Header.Set("Gotenberg-Webhook-Error-Url", "bar") - - return req - }(), - cfg: func() contextMiddlewareConfig { - cfg := buildContextMiddlewareConfig() - cfg.webhook.allowList = regexp.MustCompile("bar") - - return cfg - }(), - expectErr: true, - expectHTTPErr: true, - expectHTTPStatus: http.StatusForbidden, - }, - { - request: func() *http.Request { - req := buildMultipartFormDataRequest() - req.Header.Set("Gotenberg-Webhook-Url", "foo") - req.Header.Set("Gotenberg-Webhook-Error-Url", "bar") - - return req - }(), - cfg: func() contextMiddlewareConfig { - cfg := buildContextMiddlewareConfig() - cfg.webhook.denyList = regexp.MustCompile("foo") - - return cfg - }(), - expectErr: true, - expectHTTPErr: true, - expectHTTPStatus: http.StatusForbidden, - }, - { - request: func() *http.Request { - req := buildMultipartFormDataRequest() - req.Header.Set("Gotenberg-Webhook-Url", "foo") - req.Header.Set("Gotenberg-Webhook-Error-Url", "bar") - - return req - }(), - cfg: func() contextMiddlewareConfig { - cfg := buildContextMiddlewareConfig() - cfg.webhook.errorAllowList = regexp.MustCompile("foo") - - return cfg - }(), - expectErr: true, - expectHTTPErr: true, - expectHTTPStatus: http.StatusForbidden, - }, - { - request: func() *http.Request { - req := buildMultipartFormDataRequest() - req.Header.Set("Gotenberg-Webhook-Url", "foo") - req.Header.Set("Gotenberg-Webhook-Error-Url", "bar") - - return req - }(), - cfg: func() contextMiddlewareConfig { - cfg := buildContextMiddlewareConfig() - cfg.webhook.errorDenyList = regexp.MustCompile("bar") - - return cfg - }(), - expectErr: true, - expectHTTPErr: true, - expectHTTPStatus: http.StatusForbidden, - }, - { - request: func() *http.Request { - req := buildMultipartFormDataRequest() - req.Header.Set("Gotenberg-Webhook-Url", "foo") - req.Header.Set("Gotenberg-Webhook-Method", http.MethodGet) - req.Header.Set("Gotenberg-Webhook-Error-Url", "bar") - - return req - }(), - cfg: buildContextMiddlewareConfig(), - expectErr: true, - expectHTTPErr: true, - expectHTTPStatus: http.StatusBadRequest, - }, - { - request: func() *http.Request { - req := buildMultipartFormDataRequest() - req.Header.Set("Gotenberg-Webhook-Url", "foo") - req.Header.Set("Gotenberg-Webhook-Error-Url", "bar") - req.Header.Set("Gotenberg-Webhook-Error-Method", http.MethodGet) - - return req - }(), - cfg: buildContextMiddlewareConfig(), - expectErr: true, - expectHTTPErr: true, - expectHTTPStatus: http.StatusBadRequest, - }, - { - request: func() *http.Request { - req := buildMultipartFormDataRequest() - req.Header.Set("Gotenberg-Webhook-Url", "foo") - req.Header.Set("Gotenberg-Webhook-Method", http.MethodPost) - req.Header.Set("Gotenberg-Webhook-Error-Url", "bar") - req.Header.Set("Gotenberg-Webhook-Error-Method", http.MethodGet) - - return req - }(), - cfg: buildContextMiddlewareConfig(), - expectErr: true, - expectHTTPErr: true, - expectHTTPStatus: http.StatusBadRequest, - }, - { - request: func() *http.Request { - req := buildMultipartFormDataRequest() - req.Header.Set("Gotenberg-Webhook-Url", "foo") - req.Header.Set("Gotenberg-Webhook-Method", http.MethodPatch) - req.Header.Set("Gotenberg-Webhook-Error-Url", "bar") - req.Header.Set("Gotenberg-Webhook-Error-Method", http.MethodGet) - - return req - }(), - cfg: buildContextMiddlewareConfig(), - expectErr: true, - expectHTTPErr: true, - expectHTTPStatus: http.StatusBadRequest, - }, - { - request: func() *http.Request { - req := buildMultipartFormDataRequest() - req.Header.Set("Gotenberg-Webhook-Url", "foo") - req.Header.Set("Gotenberg-Webhook-Method", http.MethodPut) - req.Header.Set("Gotenberg-Webhook-Error-Url", "bar") - req.Header.Set("Gotenberg-Webhook-Error-Method", http.MethodGet) - - return req - }(), - cfg: buildContextMiddlewareConfig(), - expectErr: true, - expectHTTPErr: true, - expectHTTPStatus: http.StatusBadRequest, - }, - { - request: func() *http.Request { - req := buildMultipartFormDataRequest() - req.Header.Set("Gotenberg-Webhook-Url", "foo") - req.Header.Set("Gotenberg-Webhook-Error-Url", "bar") - req.Header.Set("Gotenberg-Webhook-Extra-Http-Headers", "foo") - - return req - }(), - cfg: buildContextMiddlewareConfig(), - expectErr: true, - expectHTTPErr: true, - expectHTTPStatus: http.StatusBadRequest, - }, - { - request: buildMultipartFormDataRequest(), - cfg: buildContextMiddlewareConfig(), - next: func() echo.HandlerFunc { - return func(c echo.Context) error { - return errors.New("foo") - } - }(), - autoWebhookURLs: true, - expectWebhookContentType: echo.MIMEApplicationJSONCharsetUTF8, - expectWebhookMethod: http.MethodPost, - }, - { - request: buildMultipartFormDataRequest(), - cfg: buildContextMiddlewareConfig(), - next: func() echo.HandlerFunc { - return func(c echo.Context) error { - return nil - } - }(), - autoWebhookURLs: true, - expectWebhookContentType: echo.MIMEApplicationJSONCharsetUTF8, - expectWebhookMethod: http.MethodPost, - }, - { - request: func() *http.Request { - req := buildMultipartFormDataRequest() - req.Header.Set("Gotenberg-Output-Filename", "foo") - req.Header.Set("Gotenberg-Webhook-Extra-Http-Headers", `{ "foo": "bar" }`) - - return req - }(), - cfg: buildContextMiddlewareConfig(), - next: func() echo.HandlerFunc { - return func(c echo.Context) error { - ctx := c.Get("context").(*Context) - ctx.outputPaths = []string{ - "/tests/test/testdata/api/sample2.pdf", - } - - return nil - } - }(), - autoWebhookURLs: true, - expectWebhookContentType: "application/pdf", - expectWebhookMethod: http.MethodPost, - expectWebhookFilename: "foo", - expectWebhookExtraHTTPHeaders: map[string]string{"foo": "bar"}, - }, - { - request: buildMultipartFormDataRequest(), - cfg: buildContextMiddlewareConfig(), - next: func() echo.HandlerFunc { - return func(c echo.Context) error { - ctx := c.Get("context").(*Context) - ctx.outputPaths = []string{ - "/tests/test/testdata/api/sample1.txt", - "/tests/test/testdata/api/sample2.pdf", - } - - return nil - } - }(), - autoWebhookURLs: true, - expectWebhookContentType: "application/zip", - expectWebhookMethod: http.MethodPost, - }, - } { - func() { - recorder := httptest.NewRecorder() - - srv := echo.New() - srv.HideBanner = true - srv.HidePort = true - srv.HTTPErrorHandler = httpErrorHandler(tc.cfg.traceHeader) - - c := srv.NewContext(tc.request, recorder) - c.Set("logger", zap.NewNop()) - c.Set("trace", "foo") - c.Set("startTime", time.Now()) - - webhook := echo.New() - webhook.HideBanner = true - webhook.HidePort = true - - rand.Seed(time.Now().UnixNano()) - webhookPort := rand.Intn(65535-1025+1) + 1025 - - if tc.autoWebhookURLs { - c.Request().Header.Set("Gotenberg-Webhook-Url", fmt.Sprintf("http://localhost:%d/", webhookPort)) - c.Request().Header.Set("Gotenberg-Webhook-Error-Url", fmt.Sprintf("http://localhost:%d/", webhookPort)) - } - - errChan := make(chan error, 1) - - webhook.POST( - "/", - func() echo.HandlerFunc { - return func(c echo.Context) error { - contentType := c.Request().Header.Get(echo.HeaderContentType) - if contentType != tc.expectWebhookContentType { - t.Errorf("test %d: expected %s '%s' but got '%s'", i, echo.HeaderContentType, tc.expectWebhookContentType, contentType) - } - - trace := c.Request().Header.Get(tc.cfg.traceHeader) - if trace != "foo" { - t.Errorf("test %d: expected %s '%s' but got '%s'", i, "Gotenberg-Trace", "foo", trace) - } - - method := c.Request().Method - if method != tc.expectWebhookMethod { - t.Errorf("test %d: expected HTTP method '%s' but got '%s'", i, tc.expectWebhookMethod, method) - } - - for key, expect := range tc.expectWebhookExtraHTTPHeaders { - actual := c.Request().Header.Get(key) - - if actual != expect { - t.Errorf("test %d: expected %s '%s' but got '%s'", i, key, expect, actual) - } - } - - if tc.expectWebhookContentType == echo.MIMEApplicationJSONCharsetUTF8 { - errChan <- nil - return nil - } - - contentLength := c.Request().Header.Get(echo.HeaderContentLength) - if contentLength == "" { - t.Errorf("test %d: expected non empty %s", i, echo.HeaderContentLength) - } - - contentDisposition := c.Request().Header.Get(echo.HeaderContentDisposition) - if !strings.Contains(contentDisposition, tc.expectWebhookFilename) { - t.Errorf("test %d: expected %s '%s' to contain '%s'", i, echo.HeaderContentDisposition, contentDisposition, tc.expectWebhookFilename) - } - - body, err := ioutil.ReadAll(c.Request().Body) - if err != nil { - errChan <- err - return nil - } - - if body == nil || len(body) == 0 { - t.Errorf("test %d: expected non nil body", i) - } - - errChan <- nil - return nil - } - }(), - ) - - go func(server *echo.Echo, port, i int) { - err := server.Start(fmt.Sprintf(":%d", webhookPort)) - if !errors.Is(err, http.ErrServerClosed) { - t.Errorf("test %d: expected no error but got: %v", i, err) - } - }(webhook, webhookPort, i) - - defer func() { - err := webhook.Shutdown(context.TODO()) - if err != nil { - t.Errorf("test %d: expected no error but got: %v", i, err) - } - }() - - err := contextMiddleware(tc.cfg)(tc.next)(c) - - if tc.expectErr && err == nil { - t.Errorf("test %d: expected error but got: %v", i, err) - } - - if !tc.expectErr && err != nil { - t.Errorf("test %d: expected no error but got: %v", i, err) - } - - var httpErr HTTPError - isHTTPErr := errors.As(err, &httpErr) - - if tc.expectHTTPErr && !isHTTPErr { - t.Errorf("test %d: expected HTTP error but got: %v", i, err) - } - - if !tc.expectHTTPErr && isHTTPErr { - t.Errorf("test %d: expected no HTTP error but got one: %v", i, httpErr) - } - - if err != nil && tc.expectHTTPErr && isHTTPErr { - status, _ := httpErr.HTTPError() - if status != tc.expectHTTPStatus { - t.Errorf("test %d: expected %d HTTP status code but got %d", i, tc.expectHTTPStatus, status) - } - } - - if err != nil { - return - } - - if recorder.Code != http.StatusNoContent { - t.Errorf("test %d: expected HTTP status code %d but got %d", i, http.StatusNoContent, recorder.Code) - } - - err = <-errChan - if err != nil { - t.Errorf("test %d: expected no error but got: %v", i, err) - } - }() - } -} - -func TestTimeoutMiddleware(t *testing.T) { +func TestHardTimeoutMiddleware(t *testing.T) { for i, tc := range []struct { next echo.HandlerFunc timeout time.Duration @@ -1007,7 +521,7 @@ func TestTimeoutMiddleware(t *testing.T) { c := srv.NewContext(request, recorder) c.Set("logger", zap.NewNop()) - err := timeoutMiddleware(tc.timeout)(tc.next)(c) + err := hardTimeoutMiddleware(tc.timeout)(tc.next)(c) if tc.expectErr && err == nil { t.Errorf("test %d: expected error but got: %v", i, err) diff --git a/pkg/modules/chromium/chromium.go b/pkg/modules/chromium/chromium.go index b72f232e..0ed0d818 100644 --- a/pkg/modules/chromium/chromium.go +++ b/pkg/modules/chromium/chromium.go @@ -8,6 +8,7 @@ import ( "os" "regexp" "strings" + "sync" "time" "github.com/chromedp/cdproto/fetch" @@ -236,19 +237,35 @@ func (mod Chromium) Validate() error { return nil } +// Metrics returns the metrics. +func (mod Chromium) Metrics() ([]gotenberg.Metric, error) { + return []gotenberg.Metric{ + { + Name: "chromium_active_instances_count", + Description: "Current number of active Chromium instances.", + Read: func() float64 { + activeInstancesCountMu.RLock() + defer activeInstancesCountMu.RUnlock() + + return activeInstancesCount + }, + }, + }, nil +} + // Chromium returns an API for interacting with Chromium for converting HTML // documents to PDF. func (mod Chromium) Chromium() (API, error) { return mod, nil } -// Routes returns the API routes. -func (mod Chromium) Routes() ([]api.MultipartFormDataRoute, error) { +// Routes returns the HTTP routes. +func (mod Chromium) Routes() ([]api.Route, error) { if mod.disableRoutes { return nil, nil } - return []api.MultipartFormDataRoute{ + return []api.Route{ convertURLRoute(mod, mod.engine), convertHTMLRoute(mod, mod.engine), convertMarkdownRoute(mod, mod.engine), @@ -475,9 +492,17 @@ func (mod Chromium) PDF(ctx context.Context, logger *zap.Logger, URL, outputPath } } + activeInstancesCountMu.Lock() + activeInstancesCount += 1 + activeInstancesCountMu.Unlock() + var buffer []byte err := chromedp.Run(taskCtx, printToPDF(URL, options, &buffer)) + activeInstancesCountMu.Lock() + activeInstancesCount -= 1 + activeInstancesCountMu.Unlock() + // Always remove the user profile directory created by Chromium. go func() { logger.Debug(fmt.Sprintf("remove user profile directory '%s'", userProfileDirPath)) @@ -514,12 +539,18 @@ func (mod Chromium) PDF(ctx context.Context, logger *zap.Logger, URL, outputPath return nil } +var ( + activeInstancesCount float64 + activeInstancesCountMu sync.RWMutex +) + // Interface guards. var ( - _ gotenberg.Module = (*Chromium)(nil) - _ gotenberg.Provisioner = (*Chromium)(nil) - _ gotenberg.Validator = (*Chromium)(nil) - _ api.MultipartFormDataRouter = (*Chromium)(nil) - _ API = (*Chromium)(nil) - _ Provider = (*Chromium)(nil) + _ gotenberg.Module = (*Chromium)(nil) + _ gotenberg.Provisioner = (*Chromium)(nil) + _ gotenberg.Validator = (*Chromium)(nil) + _ gotenberg.MetricsProvider = (*Chromium)(nil) + _ api.Router = (*Chromium)(nil) + _ API = (*Chromium)(nil) + _ Provider = (*Chromium)(nil) ) diff --git a/pkg/modules/chromium/chromium_test.go b/pkg/modules/chromium/chromium_test.go index f9376a11..49521ba9 100644 --- a/pkg/modules/chromium/chromium_test.go +++ b/pkg/modules/chromium/chromium_test.go @@ -173,6 +173,22 @@ func TestChromium_Validate(t *testing.T) { } } +func TestChromium_Metrics(t *testing.T) { + metrics, err := new(Chromium).Metrics() + if err != nil { + t.Errorf("expected no error but got: %v", err) + } + + if len(metrics) != 1 { + t.Errorf("expected %d metrics, but got %d", 1, len(metrics)) + } + + actual := metrics[0].Read() + if actual != 0 { + t.Errorf("expected %d Chromium instances, but got %f", 0, actual) + } +} + func TestChromium_Chromium(t *testing.T) { mod := new(Chromium) diff --git a/pkg/modules/chromium/routes.go b/pkg/modules/chromium/routes.go index de7f9424..5f4b557c 100644 --- a/pkg/modules/chromium/routes.go +++ b/pkg/modules/chromium/routes.go @@ -14,6 +14,7 @@ import ( "github.com/gotenberg/gotenberg/v7/pkg/gotenberg" "github.com/gotenberg/gotenberg/v7/pkg/modules/api" + "github.com/labstack/echo/v4" "github.com/microcosm-cc/bluemonday" "github.com/russross/blackfriday/v2" "go.uber.org/multierr" @@ -89,12 +90,14 @@ func FormDataChromiumPDFOptions(ctx *api.Context) (*api.FormData, Options) { return form, options } -// convertURLRoute returns an api.MultipartFormDataRoute route which can -// convert a URL to PDF. -func convertURLRoute(chromium API, engine gotenberg.PDFEngine) api.MultipartFormDataRoute { - return api.MultipartFormDataRoute{ - Path: "/chromium/convert/url", - Handler: func(ctx *api.Context) error { +// convertURLRoute returns an api.Route which can convert a URL to PDF. +func convertURLRoute(chromium API, engine gotenberg.PDFEngine) api.Route { + return api.Route{ + Method: http.MethodPost, + Path: "/forms/chromium/convert/url", + IsMultipart: true, + Handler: func(c echo.Context) error { + ctx := c.Get("context").(*api.Context) form, options := FormDataChromiumPDFOptions(ctx) var ( @@ -121,12 +124,14 @@ func convertURLRoute(chromium API, engine gotenberg.PDFEngine) api.MultipartForm } } -// convertHTMLRoute returns an api.MultipartFormDataRoute route which can -// convert an HTML file to PDF. -func convertHTMLRoute(chromium API, engine gotenberg.PDFEngine) api.MultipartFormDataRoute { - return api.MultipartFormDataRoute{ - Path: "/chromium/convert/html", - Handler: func(ctx *api.Context) error { +// convertHTMLRoute returns an api.Route which can convert an HTML file to PDF. +func convertHTMLRoute(chromium API, engine gotenberg.PDFEngine) api.Route { + return api.Route{ + Method: http.MethodPost, + Path: "/forms/chromium/convert/html", + IsMultipart: true, + Handler: func(c echo.Context) error { + ctx := c.Get("context").(*api.Context) form, options := FormDataChromiumPDFOptions(ctx) var ( @@ -155,12 +160,15 @@ func convertHTMLRoute(chromium API, engine gotenberg.PDFEngine) api.MultipartFor } } -// convertMarkdownRoute returns an api.MultipartFormDataRoute route which can -// convert markdown files to PDF. -func convertMarkdownRoute(chromium API, engine gotenberg.PDFEngine) api.MultipartFormDataRoute { - return api.MultipartFormDataRoute{ - Path: "/chromium/convert/markdown", - Handler: func(ctx *api.Context) error { +// convertMarkdownRoute returns an api.Route which can convert markdown files +// to PDF. +func convertMarkdownRoute(chromium API, engine gotenberg.PDFEngine) api.Route { + return api.Route{ + Method: http.MethodPost, + Path: "/forms/chromium/convert/markdown", + IsMultipart: true, + Handler: func(c echo.Context) error { + ctx := c.Get("context").(*api.Context) form, options := FormDataChromiumPDFOptions(ctx) var ( diff --git a/pkg/modules/chromium/routes_test.go b/pkg/modules/chromium/routes_test.go index 0a7a440f..a9c69be2 100644 --- a/pkg/modules/chromium/routes_test.go +++ b/pkg/modules/chromium/routes_test.go @@ -3,13 +3,14 @@ package chromium import ( "context" "errors" - "github.com/gotenberg/gotenberg/v7/pkg/gotenberg" "net/http" "os" "reflect" "testing" + "github.com/gotenberg/gotenberg/v7/pkg/gotenberg" "github.com/gotenberg/gotenberg/v7/pkg/modules/api" + "github.com/labstack/echo/v4" "go.uber.org/zap" ) @@ -137,7 +138,10 @@ func TestConvertURLHandler(t *testing.T) { expectOutputPathsCount: 1, }, } { - err := convertURLRoute(tc.api, nil).Handler(tc.ctx.Context) + c := echo.New().NewContext(nil, nil) + c.Set("context", tc.ctx.Context) + + err := convertURLRoute(tc.api, nil).Handler(c) if tc.expectErr && err == nil { t.Errorf("test %d: expected error but got: %v", i, err) @@ -238,7 +242,10 @@ func TestConvertHTMLHandler(t *testing.T) { expectOutputPathsCount: 1, }, } { - err := convertHTMLRoute(tc.api, nil).Handler(tc.ctx.Context) + c := echo.New().NewContext(nil, nil) + c.Set("context", tc.ctx.Context) + + err := convertHTMLRoute(tc.api, nil).Handler(c) if tc.expectErr && err == nil { t.Errorf("test %d: expected error but got: %v", i, err) @@ -472,7 +479,10 @@ func TestConvertMarkdownHandler(t *testing.T) { }() } - err := convertMarkdownRoute(tc.api, nil).Handler(tc.ctx.Context) + c := echo.New().NewContext(nil, nil) + c.Set("context", tc.ctx.Context) + + err := convertMarkdownRoute(tc.api, nil).Handler(c) if tc.expectErr && err == nil { t.Errorf("test %d: expected error but got: %v", i, err) diff --git a/pkg/modules/libreoffice/libreoffice.go b/pkg/modules/libreoffice/libreoffice.go index 462c3182..cf874d5c 100644 --- a/pkg/modules/libreoffice/libreoffice.go +++ b/pkg/modules/libreoffice/libreoffice.go @@ -67,20 +67,20 @@ func (mod *LibreOffice) Provision(ctx *gotenberg.Context) error { return nil } -// Routes returns the API routes. -func (mod LibreOffice) Routes() ([]api.MultipartFormDataRoute, error) { +// Routes returns the HTTP routes. +func (mod LibreOffice) Routes() ([]api.Route, error) { if mod.disableRoutes { return nil, nil } - return []api.MultipartFormDataRoute{ + return []api.Route{ convertRoute(mod.unoconv, mod.engine), }, nil } // Interface guards. var ( - _ gotenberg.Module = (*LibreOffice)(nil) - _ gotenberg.Provisioner = (*LibreOffice)(nil) - _ api.MultipartFormDataRouter = (*LibreOffice)(nil) + _ gotenberg.Module = (*LibreOffice)(nil) + _ gotenberg.Provisioner = (*LibreOffice)(nil) + _ api.Router = (*LibreOffice)(nil) ) diff --git a/pkg/modules/libreoffice/pdfengine/pdfengine.go b/pkg/modules/libreoffice/pdfengine/pdfengine.go index c2bf62a5..828dbd5a 100644 --- a/pkg/modules/libreoffice/pdfengine/pdfengine.go +++ b/pkg/modules/libreoffice/pdfengine/pdfengine.go @@ -20,7 +20,7 @@ type UnoconvPDFEngine struct { } // Descriptor returns a UnoconvPDFEngine's module descriptor. -func (engine UnoconvPDFEngine) Descriptor() gotenberg.ModuleDescriptor { +func (UnoconvPDFEngine) Descriptor() gotenberg.ModuleDescriptor { return gotenberg.ModuleDescriptor{ ID: "unoconv-pdfengine", New: func() gotenberg.Module { return new(UnoconvPDFEngine) }, diff --git a/pkg/modules/libreoffice/routes.go b/pkg/modules/libreoffice/routes.go index a54f6d9f..70cc01e4 100644 --- a/pkg/modules/libreoffice/routes.go +++ b/pkg/modules/libreoffice/routes.go @@ -8,14 +8,19 @@ import ( "github.com/gotenberg/gotenberg/v7/pkg/gotenberg" "github.com/gotenberg/gotenberg/v7/pkg/modules/api" "github.com/gotenberg/gotenberg/v7/pkg/modules/libreoffice/unoconv" + "github.com/labstack/echo/v4" ) -// convertRoute returns an api.MultipartFormDataRoute which can convert -// LibreOffice documents to PDF. -func convertRoute(uno unoconv.API, engine gotenberg.PDFEngine) api.MultipartFormDataRoute { - return api.MultipartFormDataRoute{ - Path: "/libreoffice/convert", - Handler: func(ctx *api.Context) error { +// convertRoute returns an api.Route which can convert LibreOffice documents +// to PDF. +func convertRoute(uno unoconv.API, engine gotenberg.PDFEngine) api.Route { + return api.Route{ + Method: http.MethodPost, + Path: "/forms/libreoffice/convert", + IsMultipart: true, + Handler: func(c echo.Context) error { + ctx := c.Get("context").(*api.Context) + // Let's get the data from the form and validate them. var ( inputPaths []string diff --git a/pkg/modules/libreoffice/routes_test.go b/pkg/modules/libreoffice/routes_test.go index 4cd9c0d6..54618270 100644 --- a/pkg/modules/libreoffice/routes_test.go +++ b/pkg/modules/libreoffice/routes_test.go @@ -9,6 +9,7 @@ import ( "github.com/gotenberg/gotenberg/v7/pkg/gotenberg" "github.com/gotenberg/gotenberg/v7/pkg/modules/api" "github.com/gotenberg/gotenberg/v7/pkg/modules/libreoffice/unoconv" + "github.com/labstack/echo/v4" "go.uber.org/zap" ) @@ -506,7 +507,10 @@ func TestConvertHandler(t *testing.T) { expectOutputPathsCount: 2, }, } { - err := convertRoute(tc.api, tc.engine).Handler(tc.ctx.Context) + c := echo.New().NewContext(nil, nil) + c.Set("context", tc.ctx.Context) + + err := convertRoute(tc.api, tc.engine).Handler(c) if tc.expectErr && err == nil { t.Errorf("test %d: expected error but got: %v", i, err) diff --git a/pkg/modules/libreoffice/unoconv/unoconv.go b/pkg/modules/libreoffice/unoconv/unoconv.go index 6385f3af..899b4ad2 100644 --- a/pkg/modules/libreoffice/unoconv/unoconv.go +++ b/pkg/modules/libreoffice/unoconv/unoconv.go @@ -8,6 +8,7 @@ import ( "os" "strconv" "strings" + "sync" "github.com/gotenberg/gotenberg/v7/pkg/gotenberg" "go.uber.org/zap" @@ -94,6 +95,22 @@ func (mod Unoconv) Validate() error { return nil } +// Metrics returns the metrics. +func (mod Unoconv) Metrics() ([]gotenberg.Metric, error) { + return []gotenberg.Metric{ + { + Name: "unoconv_active_instances_count", + Description: "Current number of active LibreOffice instances.", + Read: func() float64 { + activeInstancesCountMu.RLock() + defer activeInstancesCountMu.RUnlock() + + return activeInstancesCount + }, + }, + }, nil +} + // Unoconv returns an API for interacting with unoconv. func (mod Unoconv) Unoconv() (API, error) { return mod, nil @@ -163,8 +180,16 @@ func (mod Unoconv) PDF(ctx context.Context, logger *zap.Logger, inputPath, outpu logger.Debug(fmt.Sprintf("print to PDF with: %+v", options)) + activeInstancesCountMu.Lock() + activeInstancesCount += 1 + activeInstancesCountMu.Unlock() + err = cmd.Exec() + activeInstancesCountMu.Lock() + activeInstancesCount -= 1 + activeInstancesCountMu.Unlock() + // Always remove the user profile directory created by LibreOffice. // See https://github.com/gotenberg/gotenberg/issues/192. go func() { @@ -280,11 +305,17 @@ func (mod Unoconv) Extensions() []string { } } +var ( + activeInstancesCount float64 + activeInstancesCountMu sync.RWMutex +) + // Interface guards. var ( - _ gotenberg.Module = (*Unoconv)(nil) - _ gotenberg.Provisioner = (*Unoconv)(nil) - _ gotenberg.Validator = (*Unoconv)(nil) - _ API = (*Unoconv)(nil) - _ Provider = (*Unoconv)(nil) + _ gotenberg.Module = (*Unoconv)(nil) + _ gotenberg.Provisioner = (*Unoconv)(nil) + _ gotenberg.Validator = (*Unoconv)(nil) + _ gotenberg.MetricsProvider = (*Unoconv)(nil) + _ API = (*Unoconv)(nil) + _ Provider = (*Unoconv)(nil) ) diff --git a/pkg/modules/libreoffice/unoconv/unoconv_test.go b/pkg/modules/libreoffice/unoconv/unoconv_test.go index 3526f494..5d72d479 100644 --- a/pkg/modules/libreoffice/unoconv/unoconv_test.go +++ b/pkg/modules/libreoffice/unoconv/unoconv_test.go @@ -61,6 +61,22 @@ func TestUnoconv_Validate(t *testing.T) { } } +func TestChromium_Metrics(t *testing.T) { + metrics, err := new(Unoconv).Metrics() + if err != nil { + t.Errorf("expected no error but got: %v", err) + } + + if len(metrics) != 1 { + t.Errorf("expected %d metrics, but got %d", 1, len(metrics)) + } + + actual := metrics[0].Read() + if actual != 0 { + t.Errorf("expected %d unoconv instances, but got %f", 0, actual) + } +} + func TestUnoconv_Unoconv(t *testing.T) { mod := new(Unoconv) diff --git a/pkg/modules/pdfcpu/pdfcpu.go b/pkg/modules/pdfcpu/pdfcpu.go index 4ab4d86c..a10dd1d9 100644 --- a/pkg/modules/pdfcpu/pdfcpu.go +++ b/pkg/modules/pdfcpu/pdfcpu.go @@ -22,7 +22,7 @@ type PDFcpu struct { } // Descriptor returns a PDFcpu's module descriptor. -func (engine PDFcpu) Descriptor() gotenberg.ModuleDescriptor { +func (PDFcpu) Descriptor() gotenberg.ModuleDescriptor { return gotenberg.ModuleDescriptor{ ID: "pdfcpu", New: func() gotenberg.Module { return new(PDFcpu) }, diff --git a/pkg/modules/pdfengines/pdfengines.go b/pkg/modules/pdfengines/pdfengines.go index c7161287..55e5c7be 100644 --- a/pkg/modules/pdfengines/pdfengines.go +++ b/pkg/modules/pdfengines/pdfengines.go @@ -131,8 +131,8 @@ func (mod PDFEngines) PDFEngine() (gotenberg.PDFEngine, error) { return newMultiPDFEngines(engines...), nil } -// Routes returns the API routes. -func (mod PDFEngines) Routes() ([]api.MultipartFormDataRoute, error) { +// Routes returns the HTTP routes. +func (mod PDFEngines) Routes() ([]api.Route, error) { if mod.disableRoutes { return nil, nil } @@ -144,7 +144,7 @@ func (mod PDFEngines) Routes() ([]api.MultipartFormDataRoute, error) { return nil, fmt.Errorf("get pdf engine: %w", err) } - return []api.MultipartFormDataRoute{ + return []api.Route{ mergeRoute(engine), convertRoute(engine), }, nil @@ -156,5 +156,5 @@ var ( _ gotenberg.Provisioner = (*PDFEngines)(nil) _ gotenberg.Validator = (*PDFEngines)(nil) _ gotenberg.PDFEngineProvider = (*PDFEngines)(nil) - _ api.MultipartFormDataRouter = (*PDFEngines)(nil) + _ api.Router = (*PDFEngines)(nil) ) diff --git a/pkg/modules/pdfengines/routes.go b/pkg/modules/pdfengines/routes.go index 721dba9d..c27cbf52 100644 --- a/pkg/modules/pdfengines/routes.go +++ b/pkg/modules/pdfengines/routes.go @@ -7,13 +7,18 @@ import ( "github.com/gotenberg/gotenberg/v7/pkg/gotenberg" "github.com/gotenberg/gotenberg/v7/pkg/modules/api" + "github.com/labstack/echo/v4" ) -// mergeRoute returns an api.MultipartFormDataRoute which can merge PDFs. -func mergeRoute(engine gotenberg.PDFEngine) api.MultipartFormDataRoute { - return api.MultipartFormDataRoute{ - Path: "/pdfengines/merge", - Handler: func(ctx *api.Context) error { +// mergeRoute returns an api.Route which can merge PDFs. +func mergeRoute(engine gotenberg.PDFEngine) api.Route { + return api.Route{ + Method: http.MethodPost, + Path: "/forms/pdfengines/merge", + IsMultipart: true, + Handler: func(c echo.Context) error { + ctx := c.Get("context").(*api.Context) + // Let's get the data from the form and validate them. var ( inputPaths []string @@ -79,12 +84,16 @@ func mergeRoute(engine gotenberg.PDFEngine) api.MultipartFormDataRoute { } } -// convertRoute returns an api.MultipartFormDataRoute which can convert a PDF -// to a specific PDF format. -func convertRoute(engine gotenberg.PDFEngine) api.MultipartFormDataRoute { - return api.MultipartFormDataRoute{ - Path: "/pdfengines/convert", - Handler: func(ctx *api.Context) error { +// convertRoute returns an api.Route which can convert a PDF to a specific PDF +// format. +func convertRoute(engine gotenberg.PDFEngine) api.Route { + return api.Route{ + Method: http.MethodPost, + Path: "/forms/pdfengines/convert", + IsMultipart: true, + Handler: func(c echo.Context) error { + ctx := c.Get("context").(*api.Context) + // Let's get the data from the form and validate them. var ( inputPaths []string diff --git a/pkg/modules/pdfengines/routes_test.go b/pkg/modules/pdfengines/routes_test.go index 47a86b05..550b21a5 100644 --- a/pkg/modules/pdfengines/routes_test.go +++ b/pkg/modules/pdfengines/routes_test.go @@ -8,6 +8,7 @@ import ( "github.com/gotenberg/gotenberg/v7/pkg/gotenberg" "github.com/gotenberg/gotenberg/v7/pkg/modules/api" + "github.com/labstack/echo/v4" "go.uber.org/zap" ) @@ -168,7 +169,10 @@ func TestMergeHandler(t *testing.T) { expectOutputPathsCount: 1, }, } { - err := mergeRoute(tc.engine).Handler(tc.ctx.Context) + c := echo.New().NewContext(nil, nil) + c.Set("context", tc.ctx.Context) + + err := mergeRoute(tc.engine).Handler(c) if tc.expectErr && err == nil { t.Errorf("test %d: expected error but got: %v", i, err) @@ -357,7 +361,10 @@ func TestConvertHandler(t *testing.T) { expectOutputPathsCount: 2, }, } { - err := convertRoute(tc.engine).Handler(tc.ctx.Context) + c := echo.New().NewContext(nil, nil) + c.Set("context", tc.ctx.Context) + + err := convertRoute(tc.engine).Handler(c) if tc.expectErr && err == nil { t.Errorf("test %d: expected error but got: %v", i, err) diff --git a/pkg/modules/pdftk/pdftk.go b/pkg/modules/pdftk/pdftk.go index f7026db8..adbcea75 100644 --- a/pkg/modules/pdftk/pdftk.go +++ b/pkg/modules/pdftk/pdftk.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "os" + "sync" "github.com/gotenberg/gotenberg/v7/pkg/gotenberg" "go.uber.org/zap" @@ -21,7 +22,7 @@ type PDFtk struct { } // Descriptor returns a PDFtk's module descriptor. -func (engine PDFtk) Descriptor() gotenberg.ModuleDescriptor { +func (PDFtk) Descriptor() gotenberg.ModuleDescriptor { return gotenberg.ModuleDescriptor{ ID: "pdftk", New: func() gotenberg.Module { return new(PDFtk) }, @@ -51,6 +52,22 @@ func (engine PDFtk) Validate() error { return nil } +// Metrics returns the metrics. +func (engine PDFtk) Metrics() ([]gotenberg.Metric, error) { + return []gotenberg.Metric{ + { + Name: "pdftk_active_instances_count", + Description: "Current number of active PDFtk instances.", + Read: func() float64 { + activeInstancesCountMu.RLock() + defer activeInstancesCountMu.RUnlock() + + return activeInstancesCount + }, + }, + }, nil +} + // Merge merges the given PDFs into a unique PDF. func (engine PDFtk) Merge(ctx context.Context, logger *zap.Logger, inputPaths []string, outputPath string) error { var args []string @@ -62,7 +79,16 @@ func (engine PDFtk) Merge(ctx context.Context, logger *zap.Logger, inputPaths [] return fmt.Errorf("create command: %w", err) } + activeInstancesCountMu.Lock() + activeInstancesCount += 1 + activeInstancesCountMu.Unlock() + err = cmd.Exec() + + activeInstancesCountMu.Lock() + activeInstancesCount -= 1 + activeInstancesCountMu.Unlock() + if err == nil { return nil } @@ -75,10 +101,16 @@ func (engine PDFtk) Convert(_ context.Context, _ *zap.Logger, format, _, _ strin return fmt.Errorf("convert PDF to '%s' with PDFtk: %w", format, gotenberg.ErrPDFEngineMethodNotAvailable) } +var ( + activeInstancesCount float64 + activeInstancesCountMu sync.RWMutex +) + // Interface guards. var ( - _ gotenberg.Module = (*PDFtk)(nil) - _ gotenberg.Provisioner = (*PDFtk)(nil) - _ gotenberg.Validator = (*PDFtk)(nil) - _ gotenberg.PDFEngine = (*PDFtk)(nil) + _ gotenberg.Module = (*PDFtk)(nil) + _ gotenberg.Provisioner = (*PDFtk)(nil) + _ gotenberg.Validator = (*PDFtk)(nil) + _ gotenberg.MetricsProvider = (*PDFtk)(nil) + _ gotenberg.PDFEngine = (*PDFtk)(nil) ) diff --git a/pkg/modules/pdftk/pdftk_test.go b/pkg/modules/pdftk/pdftk_test.go index 4d05552c..502a6635 100644 --- a/pkg/modules/pdftk/pdftk_test.go +++ b/pkg/modules/pdftk/pdftk_test.go @@ -62,6 +62,22 @@ func TestPDFtk_Validate(t *testing.T) { } } +func TestPDFtk_Metrics(t *testing.T) { + metrics, err := new(PDFtk).Metrics() + if err != nil { + t.Errorf("expected no error but got: %v", err) + } + + if len(metrics) != 1 { + t.Errorf("expected %d metrics, but got %d", 1, len(metrics)) + } + + actual := metrics[0].Read() + if actual != 0 { + t.Errorf("expected %d PDFtk instances, but got %f", 0, actual) + } +} + func TestPDFtk_Merge(t *testing.T) { for i, tc := range []struct { ctx context.Context diff --git a/pkg/modules/prometheus/doc.go b/pkg/modules/prometheus/doc.go new file mode 100644 index 00000000..9b89d409 --- /dev/null +++ b/pkg/modules/prometheus/doc.go @@ -0,0 +1,3 @@ +// Package prometheus provides a module which collects metrics and exposes them +// via an HTTP route. +package prometheus diff --git a/pkg/modules/prometheus/prometheus.go b/pkg/modules/prometheus/prometheus.go new file mode 100644 index 00000000..897e775b --- /dev/null +++ b/pkg/modules/prometheus/prometheus.go @@ -0,0 +1,189 @@ +package prometheus + +import ( + "context" + "errors" + "fmt" + "net/http" + "time" + + "github.com/gotenberg/gotenberg/v7/pkg/gotenberg" + "github.com/gotenberg/gotenberg/v7/pkg/modules/api" + "github.com/labstack/echo/v4" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + flag "github.com/spf13/pflag" +) + +func init() { + gotenberg.MustRegisterModule(Prometheus{}) +} + +// Prometheus is a module which collects metrics and exposes them via an HTTP +// route. +type Prometheus struct { + namespace string + interval time.Duration + disableRouteLogging bool + disableCollect bool + + metrics []gotenberg.Metric + registry *prometheus.Registry +} + +// Descriptor returns a Prometheus's module descriptor. +func (Prometheus) Descriptor() gotenberg.ModuleDescriptor { + return gotenberg.ModuleDescriptor{ + ID: "prometheus", + FlagSet: func() *flag.FlagSet { + fs := flag.NewFlagSet("prometheus", flag.ExitOnError) + fs.String("prometheus-namespace", "gotenberg", "Set the namespace of modules' metrics") + fs.Duration("prometheus-collect-interval", time.Duration(1)*time.Second, "Set the interval for collecting modules' metrics") + fs.Bool("prometheus-disable-route-logging", false, "Disable the route logging") + fs.Bool("prometheus-disable-collect", false, "Disable the collect of metrics") + + return fs + }(), + New: func() gotenberg.Module { return new(Prometheus) }, + } +} + +// Provision sets the modules properties. +func (mod *Prometheus) Provision(ctx *gotenberg.Context) error { + flags := ctx.ParsedFlags() + mod.namespace = flags.MustString("prometheus-namespace") + mod.interval = flags.MustDuration("prometheus-collect-interval") + mod.disableRouteLogging = flags.MustBool("prometheus-disable-route-logging") + mod.disableCollect = flags.MustBool("prometheus-disable-collect") + + if mod.disableCollect { + // Exit early. + return nil + } + + // Get metrics from modules. + mods, err := ctx.Modules(new(gotenberg.MetricsProvider)) + if err != nil { + return fmt.Errorf("get metrics providers: %w", err) + } + + metricsProviders := make([]gotenberg.MetricsProvider, len(mods)) + for i, metricsProvider := range mods { + metricsProviders[i] = metricsProvider.(gotenberg.MetricsProvider) + } + + for _, metricsProvider := range metricsProviders { + metrics, err := metricsProvider.Metrics() + if err != nil { + return fmt.Errorf("get metrics: %w", err) + } + + mod.metrics = append(mod.metrics, metrics...) + } + + mod.registry = prometheus.NewRegistry() + + return nil +} + +// Validate validates the module properties. +func (mod Prometheus) Validate() error { + if mod.disableCollect { + // Exit early. + return nil + } + + if mod.namespace == "" { + return errors.New("namespace must not be empty") + } + + metricsMap := make(map[string]string, len(mod.metrics)) + + for _, metric := range mod.metrics { + if metric.Name == "" { + return errors.New("metric name cannot be empty") + } + + if metric.Read == nil { + return fmt.Errorf("metric '%s' has nil read method", metric.Name) + } + + if _, ok := metricsMap[metric.Name]; ok { + return fmt.Errorf("metric '%s' is already registered", metric.Name) + } + + metricsMap[metric.Name] = metric.Name + } + + return nil +} + +// Start starts the collect. +func (mod Prometheus) Start() error { + if mod.disableCollect { + // Exit early. + return nil + } + + for _, metric := range mod.metrics { + gauge := prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: mod.namespace, + Name: metric.Name, + Help: metric.Description, + }, + ) + + mod.registry.MustRegister(gauge) + + go func(gauge prometheus.Gauge, metric gotenberg.Metric) { + for { + gauge.Set(metric.Read()) + time.Sleep(mod.interval) + } + }(gauge, metric) + } + + return nil +} + +// StartupMessage returns a custom startup message. +func (mod Prometheus) StartupMessage() string { + if mod.disableCollect { + return "application not started (collect disabled by user)" + } + + return "collecting metrics" +} + +// Stop does nothing. +func (mod Prometheus) Stop(_ context.Context) error { + return nil +} + +// Routes returns the HTTP route. +func (mod Prometheus) Routes() ([]api.Route, error) { + if mod.disableCollect { + return nil, nil + } + + return []api.Route{ + { + Method: http.MethodGet, + Path: "/prometheus/metrics", + DisableLogging: mod.disableRouteLogging, + Handler: echo.WrapHandler( + promhttp.HandlerFor(mod.registry, promhttp.HandlerOpts{}), + ), + }, + }, nil +} + +// Interface guards. +var ( + _ gotenberg.Module = (*Prometheus)(nil) + _ gotenberg.Provisioner = (*Prometheus)(nil) + _ gotenberg.Validator = (*Prometheus)(nil) + _ gotenberg.App = (*Prometheus)(nil) + _ api.Router = (*Prometheus)(nil) +) diff --git a/pkg/modules/prometheus/prometheus_test.go b/pkg/modules/prometheus/prometheus_test.go new file mode 100644 index 00000000..2fbb11a6 --- /dev/null +++ b/pkg/modules/prometheus/prometheus_test.go @@ -0,0 +1,360 @@ +package prometheus + +import ( + "errors" + "reflect" + "testing" + "time" + + "github.com/gotenberg/gotenberg/v7/pkg/gotenberg" + "github.com/prometheus/client_golang/prometheus" +) + +type ProtoModule struct { + descriptor func() gotenberg.ModuleDescriptor +} + +func (mod ProtoModule) Descriptor() gotenberg.ModuleDescriptor { + return mod.descriptor() +} + +type ProtoValidator struct { + ProtoModule + validate func() error +} + +func (mod ProtoValidator) Validate() error { + return mod.validate() +} + +type ProtoMetricsProvider struct { + ProtoValidator + metrics func() ([]gotenberg.Metric, error) +} + +func (mod ProtoMetricsProvider) Metrics() ([]gotenberg.Metric, error) { + return mod.metrics() +} + +func TestPrometheus_Descriptor(t *testing.T) { + descriptor := Prometheus{}.Descriptor() + + actual := reflect.TypeOf(descriptor.New()) + expect := reflect.TypeOf(new(Prometheus)) + + if actual != expect { + t.Errorf("expected '%s' but got '%s'", expect, actual) + } +} + +func TestPrometheus_Provision(t *testing.T) { + for i, tc := range []struct { + ctx *gotenberg.Context + expectMetrics []gotenberg.Metric + expectErr bool + }{ + { + ctx: func() *gotenberg.Context { + fs := new(Prometheus).Descriptor().FlagSet + err := fs.Parse([]string{"--prometheus-disable-collect=true"}) + + if err != nil { + t.Fatalf("expected no error but got: %v", err) + } + + return gotenberg.NewContext( + gotenberg.ParsedFlags{ + FlagSet: fs, + }, + nil, + ) + }(), + }, + { + ctx: func() *gotenberg.Context { + mod := struct{ ProtoMetricsProvider }{} + mod.descriptor = func() gotenberg.ModuleDescriptor { + return gotenberg.ModuleDescriptor{ID: "foo", New: func() gotenberg.Module { return mod }} + } + mod.validate = func() error { + return errors.New("foo") + } + mod.metrics = func() ([]gotenberg.Metric, error) { + return nil, nil + } + + return gotenberg.NewContext( + gotenberg.ParsedFlags{ + FlagSet: new(Prometheus).Descriptor().FlagSet, + }, + []gotenberg.ModuleDescriptor{ + mod.Descriptor(), + }, + ) + }(), + expectErr: true, + }, + { + ctx: func() *gotenberg.Context { + mod := struct{ ProtoMetricsProvider }{} + mod.descriptor = func() gotenberg.ModuleDescriptor { + return gotenberg.ModuleDescriptor{ID: "foo", New: func() gotenberg.Module { return mod }} + } + mod.validate = func() error { + return nil + } + mod.metrics = func() ([]gotenberg.Metric, error) { + return nil, errors.New("foo") + } + + return gotenberg.NewContext( + gotenberg.ParsedFlags{ + FlagSet: new(Prometheus).Descriptor().FlagSet, + }, + []gotenberg.ModuleDescriptor{ + mod.Descriptor(), + }, + ) + }(), + expectErr: true, + }, + { + ctx: func() *gotenberg.Context { + mod := struct{ ProtoMetricsProvider }{} + mod.descriptor = func() gotenberg.ModuleDescriptor { + return gotenberg.ModuleDescriptor{ID: "foo", New: func() gotenberg.Module { return mod }} + } + mod.validate = func() error { + return nil + } + mod.metrics = func() ([]gotenberg.Metric, error) { + return []gotenberg.Metric{ + { + Name: "foo", + Description: "Bar.", + }, + }, nil + } + + return gotenberg.NewContext( + gotenberg.ParsedFlags{ + FlagSet: new(Prometheus).Descriptor().FlagSet, + }, + []gotenberg.ModuleDescriptor{ + mod.Descriptor(), + }, + ) + }(), + expectMetrics: []gotenberg.Metric{ + { + Name: "foo", + Description: "Bar.", + }, + }, + }, + } { + mod := new(Prometheus) + err := mod.Provision(tc.ctx) + + if !reflect.DeepEqual(mod.metrics, tc.expectMetrics) { + t.Errorf("test %d: expected %+v, but got: %+v", i, tc.expectMetrics, mod.metrics) + } + + if tc.expectErr && err == nil { + t.Errorf("test %d: expected error but got: %v", i, err) + } + + if !tc.expectErr && err != nil { + t.Errorf("test %d: expected no error but got: %v", i, err) + } + } +} + +func TestPrometheus_Validate(t *testing.T) { + for i, tc := range []struct { + namespace string + metrics []gotenberg.Metric + disableCollect bool + expectErr bool + }{ + { + disableCollect: true, + }, + { + namespace: "", + expectErr: true, + }, + { + namespace: "foo", + metrics: []gotenberg.Metric{ + { + Name: "", + }, + }, + expectErr: true, + }, + { + namespace: "foo", + metrics: []gotenberg.Metric{ + { + Name: "foo", + }, + }, + expectErr: true, + }, + { + namespace: "foo", + metrics: []gotenberg.Metric{ + { + Name: "foo", + Read: func() float64 { + return 0 + }, + }, + { + Name: "foo", + Read: func() float64 { + return 0 + }, + }, + }, + expectErr: true, + }, + { + namespace: "foo", + metrics: []gotenberg.Metric{ + { + Name: "foo", + Read: func() float64 { + return 0 + }, + }, + { + Name: "bar", + Read: func() float64 { + return 0 + }, + }, + }, + }, + } { + mod := Prometheus{ + namespace: tc.namespace, + metrics: tc.metrics, + disableCollect: tc.disableCollect, + } + + err := mod.Validate() + + if tc.expectErr && err == nil { + t.Errorf("test %d: expected error but got: %v", i, err) + } + + if !tc.expectErr && err != nil { + t.Errorf("test %d: expected no error but got: %v", i, err) + } + } +} + +func TestPrometheus_Start(t *testing.T) { + for i, tc := range []struct { + metrics []gotenberg.Metric + disableCollect bool + }{ + { + disableCollect: true, + }, + { + metrics: []gotenberg.Metric{ + { + Name: "foo", + Read: func() float64 { + return 0 + }, + }, + }, + }, + } { + mod := Prometheus{ + namespace: "foo", + interval: time.Duration(1) * time.Second, + metrics: tc.metrics, + disableCollect: tc.disableCollect, + registry: prometheus.NewRegistry(), + } + + err := mod.Start() + if err != nil { + t.Errorf("test %d: expected no error but got: %v", i, err) + } + } +} + +func TestPrometheus_StartupMessage(t *testing.T) { + for i, tc := range []struct { + disableCollect bool + expectMessage string + }{ + { + expectMessage: "application not started (collect disabled by user)", + disableCollect: true, + }, + { + expectMessage: "collecting metrics", + }, + } { + mod := Prometheus{ + disableCollect: tc.disableCollect, + } + + actual := mod.StartupMessage() + if actual != tc.expectMessage { + t.Errorf("test %d: expected '%s' but got '%s'", i, tc.expectMessage, actual) + } + } +} + +func TestPrometheus_Stop(t *testing.T) { + err := Prometheus{}.Stop(nil) + if err != nil { + t.Errorf("expected no error but got: %v", err) + } +} + +func TestPrometheus_Routes(t *testing.T) { + for i, tc := range []struct { + expectRoutes int + disableCollect bool + }{ + { + disableCollect: true, + }, + { + expectRoutes: 1, + }, + } { + mod := Prometheus{ + disableCollect: tc.disableCollect, + registry: prometheus.NewRegistry(), + } + + routes, err := mod.Routes() + if err != nil { + t.Fatalf("test %d: expected no error but got: %v", i, err) + } + + if tc.expectRoutes != len(routes) { + t.Errorf("test %d: expected %d routes but got %d", i, tc.expectRoutes, len(routes)) + } + } +} + +// Interface guards. +var ( + _ gotenberg.Module = (*ProtoModule)(nil) + _ gotenberg.Validator = (*ProtoValidator)(nil) + _ gotenberg.Module = (*ProtoValidator)(nil) + _ gotenberg.MetricsProvider = (*ProtoMetricsProvider)(nil) + _ gotenberg.Module = (*ProtoMetricsProvider)(nil) + _ gotenberg.Validator = (*ProtoMetricsProvider)(nil) +) diff --git a/pkg/modules/api/webhook.go b/pkg/modules/webhook/client.go similarity index 79% rename from pkg/modules/api/webhook.go rename to pkg/modules/webhook/client.go index f186b21a..6815fdd6 100644 --- a/pkg/modules/api/webhook.go +++ b/pkg/modules/webhook/client.go @@ -1,4 +1,4 @@ -package api +package webhook import ( "fmt" @@ -11,8 +11,8 @@ import ( "go.uber.org/zap" ) -// webhookClient gathers all the data required to send a request to a webhook. -type webhookClient struct { +// client gathers all the data required to send a request to a webhook. +type client struct { url string method string errorURL string @@ -25,15 +25,15 @@ type webhookClient struct { } // send call the webhook either to send the success response or the error response. -func (webhook webhookClient) send(body io.Reader, headers map[string]string, erroed bool) error { - URL := webhook.url +func (c client) send(body io.Reader, headers map[string]string, erroed bool) error { + URL := c.url if erroed { - URL = webhook.errorURL + URL = c.errorURL } - method := webhook.method + method := c.method if erroed { - method = webhook.errorMethod + method = c.errorMethod } req, err := retryablehttp.NewRequest(method, URL, body) @@ -44,7 +44,7 @@ func (webhook webhookClient) send(body io.Reader, headers map[string]string, err req.Header.Set("User-Agent", "Gotenberg") // Extra HTTP headers are the custom headers from the user. - for key, value := range webhook.extraHTTPHeaders { + for key, value := range c.extraHTTPHeaders { req.Header.Set(key, value) } @@ -73,7 +73,7 @@ func (webhook webhookClient) send(body io.Reader, headers map[string]string, err req.Header.Set(key, value) } - resp, err := webhook.client.Do(req) + resp, err := c.client.Do(req) if err != nil { return fmt.Errorf("send '%s' request to '%s': %w", method, URL, err) } @@ -81,7 +81,7 @@ func (webhook webhookClient) send(body io.Reader, headers map[string]string, err defer func() { err := resp.Body.Close() if err != nil { - webhook.logger.Error(fmt.Sprintf("close response body from '%s': %s", URL, err)) + c.logger.Error(fmt.Sprintf("close response body from '%s': %s", URL, err)) } }() @@ -92,17 +92,17 @@ func (webhook webhookClient) send(body io.Reader, headers map[string]string, err fields := make([]zap.Field, 5) fields[0] = zap.String("webhook_url", URL) fields[1] = zap.String("method", method) - fields[2] = zap.Int64("latency", int64(finishTime.Sub(webhook.startTime))) - fields[3] = zap.String("latency_human", finishTime.Sub(webhook.startTime).String()) + fields[2] = zap.Int64("latency", int64(finishTime.Sub(c.startTime))) + fields[3] = zap.String("latency_human", finishTime.Sub(c.startTime).String()) fields[4] = zap.Int64("bytes_out", req.ContentLength) if erroed { - webhook.logger.Warn("request to webhook with error details handled", fields...) + c.logger.Warn("request to webhook with error details handled", fields...) return nil } - webhook.logger.Info("request to webhook handled", fields...) + c.logger.Info("request to webhook handled", fields...) return nil } diff --git a/pkg/modules/api/webhook_test.go b/pkg/modules/webhook/client_test.go similarity index 96% rename from pkg/modules/api/webhook_test.go rename to pkg/modules/webhook/client_test.go index 5c72df3c..3bc62f75 100644 --- a/pkg/modules/api/webhook_test.go +++ b/pkg/modules/webhook/client_test.go @@ -1,4 +1,4 @@ -package api +package webhook import ( "testing" diff --git a/pkg/modules/webhook/doc.go b/pkg/modules/webhook/doc.go new file mode 100644 index 00000000..8416ec9c --- /dev/null +++ b/pkg/modules/webhook/doc.go @@ -0,0 +1,3 @@ +// Package webhook provides a module which adds a middleware for uploading +// output files to any destinations in an asynchronous fashion. +package webhook diff --git a/pkg/modules/webhook/middleware.go b/pkg/modules/webhook/middleware.go new file mode 100644 index 00000000..17588efc --- /dev/null +++ b/pkg/modules/webhook/middleware.go @@ -0,0 +1,278 @@ +package webhook + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "os" + "regexp" + "strconv" + "strings" + "time" + + "github.com/gotenberg/gotenberg/v7/pkg/modules/api" + "github.com/hashicorp/go-retryablehttp" + "github.com/labstack/echo/v4" +) + +func webhookMiddleware(w Webhook) api.Middleware { + return api.Middleware{ + Stack: api.MultipartStack, + Handler: func() echo.MiddlewareFunc { + return func(next echo.HandlerFunc) echo.HandlerFunc { + return func(c echo.Context) error { + webhookURL := c.Request().Header.Get("Gotenberg-Webhook-Url") + + if webhookURL == "" { + // No webhook URL, call the next middleware in the chain. + return next(c) + } + + ctx := c.Get("context").(*api.Context) + cancel := c.Get("cancel").(context.CancelFunc) + + // Do we have a webhook error URL in case of... error? + webhookErrorURL := c.Request().Header.Get("Gotenberg-Webhook-Error-Url") + + if webhookErrorURL == "" { + return api.WrapError( + errors.New("empty webhook error URL"), + api.NewSentinelHTTPError(http.StatusBadRequest, "Invalid 'Gotenberg-Webhook-Error-Url' header: empty value or header not provided"), + ) + } + + // Let's check if the webhook URLs are acceptable according to our + // allowed/denied lists. + filter := func(URL, header string, allowList, denyList *regexp.Regexp) error { + if !allowList.MatchString(URL) { + return api.WrapError( + fmt.Errorf("'%s' does not match the expression from the allowed list", URL), + api.NewSentinelHTTPError( + http.StatusForbidden, + fmt.Sprintf("Invalid '%s' header value: '%s' does not match the authorized URLs", header, URL), + ), + ) + } + + if denyList.String() != "" && denyList.MatchString(URL) { + return api.WrapError( + fmt.Errorf("'%s' matches the expression from the denied list", URL), + api.NewSentinelHTTPError( + http.StatusForbidden, + fmt.Sprintf("Invalid '%s' header value: '%s' does not match the authorized URLs", header, URL), + ), + ) + } + + return nil + } + + err := filter(webhookURL, "Gotenberg-Webhook-Url", w.allowList, w.denyList) + if err != nil { + return fmt.Errorf("filter webhook URL: %w", err) + } + + err = filter(webhookErrorURL, "Gotenberg-Webhook-Error-Url", w.errorAllowList, w.errorDenyList) + if err != nil { + return fmt.Errorf("filter webhook error URL: %w", err) + } + + // Let's check the HTTP methods for calling the webhook URLs. + methodFromHeader := func(header string) (string, error) { + method := c.Request().Header.Get(header) + + if method == "" { + return http.MethodPost, nil + } + + method = strings.ToUpper(method) + + switch method { + case http.MethodPost: + return method, nil + case http.MethodPatch: + return method, nil + case http.MethodPut: + return method, nil + } + + return "", api.WrapError( + fmt.Errorf("webhook method '%s' is not '%s', '%s' or '%s'", method, http.MethodPost, http.MethodPatch, http.MethodPut), + api.NewSentinelHTTPError( + http.StatusBadRequest, + fmt.Sprintf("Invalid '%s' header value: expected '%s', '%s' or '%s', but got '%s'", header, http.MethodPost, http.MethodPatch, http.MethodPut, method), + ), + ) + } + + webhookMethod, err := methodFromHeader("Gotenberg-Webhook-Method") + if err != nil { + return fmt.Errorf("get method to use for webhook: %w", err) + } + + webhookErrorMethod, err := methodFromHeader("Gotenberg-Webhook-Error-Method") + if err != nil { + return fmt.Errorf("get method to use for webhook error: %w", err) + } + + // What about extra HTTP headers? + var extraHTTPHeaders map[string]string + + extraHTTPHeadersJSON := c.Request().Header.Get("Gotenberg-Webhook-Extra-Http-Headers") + if extraHTTPHeadersJSON != "" { + err = json.Unmarshal([]byte(extraHTTPHeadersJSON), &extraHTTPHeaders) + if err != nil { + return api.WrapError( + fmt.Errorf("unmarshal webhook extra HTTP headers: %w", err), + api.NewSentinelHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid 'Gotenberg-Webhook-Extra-Http-Headers' header value: %s", err.Error())), + ) + } + } + + client := &client{ + url: webhookURL, + method: webhookMethod, + errorURL: webhookErrorURL, + errorMethod: webhookErrorMethod, + extraHTTPHeaders: extraHTTPHeaders, + startTime: c.Get("startTime").(time.Time), + + client: &retryablehttp.Client{ + HTTPClient: &http.Client{ + Timeout: c.Get("writeTimeout").(time.Duration), + }, + RetryMax: w.maxRetry, + RetryWaitMin: w.retryMinWait, + RetryWaitMax: w.retryMaxWait, + Logger: leveledLogger{ + logger: ctx.Log(), + }, + CheckRetry: retryablehttp.DefaultRetryPolicy, + Backoff: retryablehttp.DefaultBackoff, + }, + logger: ctx.Log(), + } + + // This method parses an "asynchronous" error and sends a + // request to the webhook error URL with a JSON body + // containing the status and the error message. + handleAsyncError := func(err error) { + status, message := api.ParseError(err) + + body := struct { + Status int `json:"status"` + Message string `json:"message"` + }{ + Status: status, + Message: message, + } + + b, err := json.Marshal(body) + if err != nil { + ctx.Log().Error(fmt.Sprintf("marshal JSON: %s", err.Error())) + + return + } + + headers := map[string]string{ + echo.HeaderContentType: echo.MIMEApplicationJSONCharsetUTF8, + c.Get("traceHeader").(string): c.Get("trace").(string), + } + + err = client.send(bytes.NewReader(b), headers, true) + if err != nil { + ctx.Log().Error(fmt.Sprintf("send error response to webhook: %s", err.Error())) + } + } + + // As a webhook URL has been given, we handle the request in a + // goroutine and return immediately. + go func() { + defer cancel() + + // Call the next middleware in the chain. + err := next(c) + + if err != nil { + // The process failed for whatever reason. Let's send the + // details to the webhook. + ctx.Log().Error(err.Error()) + handleAsyncError(err) + + return + } + + // No error, let's get build the output file. + outputPath, err := ctx.BuildOutputFile() + if err != nil { + ctx.Log().Error(fmt.Sprintf("build output file: %s", err)) + handleAsyncError(err) + + return + } + + outputFile, err := os.Open(outputPath) + if err != nil { + ctx.Log().Error(fmt.Sprintf("open output file: %s", err)) + handleAsyncError(err) + + return + } + + defer func() { + err := outputFile.Close() + if err != nil { + ctx.Log().Error(fmt.Sprintf("close output file: %s", err)) + } + }() + + fileHeader := make([]byte, 512) + _, err = outputFile.Read(fileHeader) + if err != nil { + ctx.Log().Error(fmt.Sprintf("read header of output file: %s", err)) + handleAsyncError(err) + + return + } + + fileStat, err := outputFile.Stat() + if err != nil { + ctx.Log().Error(fmt.Sprintf("get stat from output file: %s", err)) + handleAsyncError(err) + + return + } + + _, err = outputFile.Seek(0, 0) + if err != nil { + ctx.Log().Error(fmt.Sprintf("reset output file reader: %s", err)) + handleAsyncError(err) + + return + } + + headers := map[string]string{ + echo.HeaderContentDisposition: fmt.Sprintf("attachement; filename=%q", ctx.OutputFilename(outputPath)), + echo.HeaderContentType: http.DetectContentType(fileHeader), + echo.HeaderContentLength: strconv.FormatInt(fileStat.Size(), 10), + c.Get("traceHeader").(string): c.Get("trace").(string), + } + + // Send the output file to the webhook. + err = client.send(bufio.NewReader(outputFile), headers, false) + if err != nil { + ctx.Log().Error(fmt.Sprintf("send output file to webhook: %s", err)) + handleAsyncError(err) + } + }() + + return api.ErrAsyncProcess + } + } + }(), + } +} diff --git a/pkg/modules/webhook/middleware_test.go b/pkg/modules/webhook/middleware_test.go new file mode 100644 index 00000000..ec736021 --- /dev/null +++ b/pkg/modules/webhook/middleware_test.go @@ -0,0 +1,534 @@ +package webhook + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "math/rand" + "mime/multipart" + "net/http" + "net/http/httptest" + "regexp" + "strings" + "testing" + "time" + + "github.com/gotenberg/gotenberg/v7/pkg/modules/api" + "github.com/labstack/echo/v4" + "go.uber.org/zap" +) + +func TestWebhookMiddlewareGuards(t *testing.T) { + buildMultipartFormDataRequest := func() *http.Request { + body := &bytes.Buffer{} + writer := multipart.NewWriter(body) + + defer func() { + err := writer.Close() + if err != nil { + t.Fatalf("expected no error but got: %v", err) + } + }() + + err := writer.WriteField("foo", "foo") + if err != nil { + t.Fatalf("expected no error but got: %v", err) + } + + req := httptest.NewRequest(http.MethodPost, "/", body) + req.Header.Set(echo.HeaderContentType, writer.FormDataContentType()) + + return req + } + + buildWebhookModule := func() Webhook { + return Webhook{ + allowList: regexp.MustCompile(""), + denyList: regexp.MustCompile(""), + errorAllowList: regexp.MustCompile(""), + errorDenyList: regexp.MustCompile(""), + maxRetry: 0, + retryMinWait: 0, + retryMaxWait: 0, + disable: false, + } + } + + for i, tc := range []struct { + request *http.Request + mod Webhook + next echo.HandlerFunc + expectErr bool + expectHTTPErr bool + expectHTTPStatus int + }{ + { + request: buildMultipartFormDataRequest(), + mod: buildWebhookModule(), + next: func() echo.HandlerFunc { + return func(c echo.Context) error { + return nil + } + }(), + }, + { + request: func() *http.Request { + req := buildMultipartFormDataRequest() + req.Header.Set("Gotenberg-Webhook-Url", "foo") + + return req + }(), + mod: buildWebhookModule(), + expectErr: true, + expectHTTPErr: true, + expectHTTPStatus: http.StatusBadRequest, + }, + { + request: func() *http.Request { + req := buildMultipartFormDataRequest() + req.Header.Set("Gotenberg-Webhook-Url", "foo") + req.Header.Set("Gotenberg-Webhook-Error-Url", "bar") + + return req + }(), + mod: func() Webhook { + mod := buildWebhookModule() + mod.allowList = regexp.MustCompile("bar") + + return mod + }(), + expectErr: true, + expectHTTPErr: true, + expectHTTPStatus: http.StatusForbidden, + }, + { + request: func() *http.Request { + req := buildMultipartFormDataRequest() + req.Header.Set("Gotenberg-Webhook-Url", "foo") + req.Header.Set("Gotenberg-Webhook-Error-Url", "bar") + + return req + }(), + mod: func() Webhook { + mod := buildWebhookModule() + mod.denyList = regexp.MustCompile("foo") + + return mod + }(), + expectErr: true, + expectHTTPErr: true, + expectHTTPStatus: http.StatusForbidden, + }, + { + request: func() *http.Request { + req := buildMultipartFormDataRequest() + req.Header.Set("Gotenberg-Webhook-Url", "foo") + req.Header.Set("Gotenberg-Webhook-Error-Url", "bar") + + return req + }(), + mod: func() Webhook { + mod := buildWebhookModule() + mod.errorAllowList = regexp.MustCompile("foo") + + return mod + }(), + expectErr: true, + expectHTTPErr: true, + expectHTTPStatus: http.StatusForbidden, + }, + { + request: func() *http.Request { + req := buildMultipartFormDataRequest() + req.Header.Set("Gotenberg-Webhook-Url", "foo") + req.Header.Set("Gotenberg-Webhook-Error-Url", "bar") + + return req + }(), + mod: func() Webhook { + mod := buildWebhookModule() + mod.errorDenyList = regexp.MustCompile("bar") + + return mod + }(), + expectErr: true, + expectHTTPErr: true, + expectHTTPStatus: http.StatusForbidden, + }, + { + request: func() *http.Request { + req := buildMultipartFormDataRequest() + req.Header.Set("Gotenberg-Webhook-Url", "foo") + req.Header.Set("Gotenberg-Webhook-Method", http.MethodGet) + req.Header.Set("Gotenberg-Webhook-Error-Url", "bar") + + return req + }(), + mod: buildWebhookModule(), + expectErr: true, + expectHTTPErr: true, + expectHTTPStatus: http.StatusBadRequest, + }, + { + request: func() *http.Request { + req := buildMultipartFormDataRequest() + req.Header.Set("Gotenberg-Webhook-Url", "foo") + req.Header.Set("Gotenberg-Webhook-Error-Url", "bar") + req.Header.Set("Gotenberg-Webhook-Error-Method", http.MethodGet) + + return req + }(), + mod: buildWebhookModule(), + expectErr: true, + expectHTTPErr: true, + expectHTTPStatus: http.StatusBadRequest, + }, + { + request: func() *http.Request { + req := buildMultipartFormDataRequest() + req.Header.Set("Gotenberg-Webhook-Url", "foo") + req.Header.Set("Gotenberg-Webhook-Method", http.MethodPost) + req.Header.Set("Gotenberg-Webhook-Error-Url", "bar") + req.Header.Set("Gotenberg-Webhook-Error-Method", http.MethodGet) + + return req + }(), + mod: buildWebhookModule(), + expectErr: true, + expectHTTPErr: true, + expectHTTPStatus: http.StatusBadRequest, + }, + { + request: func() *http.Request { + req := buildMultipartFormDataRequest() + req.Header.Set("Gotenberg-Webhook-Url", "foo") + req.Header.Set("Gotenberg-Webhook-Method", http.MethodPatch) + req.Header.Set("Gotenberg-Webhook-Error-Url", "bar") + req.Header.Set("Gotenberg-Webhook-Error-Method", http.MethodGet) + + return req + }(), + mod: buildWebhookModule(), + expectErr: true, + expectHTTPErr: true, + expectHTTPStatus: http.StatusBadRequest, + }, + { + request: func() *http.Request { + req := buildMultipartFormDataRequest() + req.Header.Set("Gotenberg-Webhook-Url", "foo") + req.Header.Set("Gotenberg-Webhook-Method", http.MethodPut) + req.Header.Set("Gotenberg-Webhook-Error-Url", "bar") + req.Header.Set("Gotenberg-Webhook-Error-Method", http.MethodGet) + + return req + }(), + mod: buildWebhookModule(), + expectErr: true, + expectHTTPErr: true, + expectHTTPStatus: http.StatusBadRequest, + }, + { + request: func() *http.Request { + req := buildMultipartFormDataRequest() + req.Header.Set("Gotenberg-Webhook-Url", "foo") + req.Header.Set("Gotenberg-Webhook-Error-Url", "bar") + req.Header.Set("Gotenberg-Webhook-Extra-Http-Headers", "foo") + + return req + }(), + mod: buildWebhookModule(), + expectErr: true, + expectHTTPErr: true, + expectHTTPStatus: http.StatusBadRequest, + }, + } { + srv := echo.New() + srv.HideBanner = true + srv.HidePort = true + + c := srv.NewContext(tc.request, httptest.NewRecorder()) + + ctx := &api.MockContext{Context: &api.Context{}} + ctx.SetEchoContext(c) + + c.Set("context", ctx.Context) + c.Set("cancel", func() context.CancelFunc { + return func() { + return + } + }()) + + err := webhookMiddleware(tc.mod).Handler(tc.next)(c) + + if tc.expectErr && err == nil { + t.Errorf("test %d: expected error but got: %v", i, err) + } + + if !tc.expectErr && err != nil { + t.Errorf("test %d: expected no error but got: %v", i, err) + } + + var httpErr api.HTTPError + isHTTPErr := errors.As(err, &httpErr) + + if tc.expectHTTPErr && !isHTTPErr { + t.Errorf("test %d: expected HTTP error but got: %v", i, err) + } + + if !tc.expectHTTPErr && isHTTPErr { + t.Errorf("test %d: expected no HTTP error but got one: %v", i, httpErr) + } + + if err != nil && tc.expectHTTPErr && isHTTPErr { + status, _ := httpErr.HTTPError() + if status != tc.expectHTTPStatus { + t.Errorf("test %d: expected %d HTTP status code but got %d", i, tc.expectHTTPStatus, status) + } + } + } +} + +func TestWebhookMiddlewareAsynchronousProcess(t *testing.T) { + buildMultipartFormDataRequest := func() *http.Request { + body := &bytes.Buffer{} + writer := multipart.NewWriter(body) + + defer func() { + err := writer.Close() + if err != nil { + t.Fatalf("expected no error but got: %v", err) + } + }() + + err := writer.WriteField("foo", "foo") + if err != nil { + t.Fatalf("expected no error but got: %v", err) + } + + req := httptest.NewRequest(http.MethodPost, "/", body) + req.Header.Set(echo.HeaderContentType, writer.FormDataContentType()) + + return req + } + + buildWebhookModule := func() Webhook { + return Webhook{ + allowList: regexp.MustCompile(""), + denyList: regexp.MustCompile(""), + errorAllowList: regexp.MustCompile(""), + errorDenyList: regexp.MustCompile(""), + maxRetry: 0, + retryMinWait: 0, + retryMaxWait: 0, + disable: false, + } + } + + for i, tc := range []struct { + request *http.Request + mod Webhook + next echo.HandlerFunc + expectWebhookContentType string + expectWebhookMethod string + expectWebhookExtraHTTPHeaders map[string]string + expectWebhookFilename string + expectWebhookErrorStatus int + expectWebhookErrorMessage string + }{ + { + request: buildMultipartFormDataRequest(), + mod: buildWebhookModule(), + next: func() echo.HandlerFunc { + return func(c echo.Context) error { + return errors.New("foo") + } + }(), + expectWebhookContentType: echo.MIMEApplicationJSONCharsetUTF8, + expectWebhookMethod: http.MethodPost, + expectWebhookErrorStatus: http.StatusInternalServerError, + expectWebhookErrorMessage: http.StatusText(http.StatusInternalServerError), + }, + { + request: buildMultipartFormDataRequest(), + mod: buildWebhookModule(), + next: func() echo.HandlerFunc { + return func(c echo.Context) error { + return nil + } + }(), + expectWebhookContentType: echo.MIMEApplicationJSONCharsetUTF8, + expectWebhookMethod: http.MethodPost, + expectWebhookErrorStatus: http.StatusInternalServerError, + expectWebhookErrorMessage: http.StatusText(http.StatusInternalServerError), + }, + { + request: func() *http.Request { + req := buildMultipartFormDataRequest() + req.Header.Set("Gotenberg-Output-Filename", "foo") + req.Header.Set("Gotenberg-Webhook-Extra-Http-Headers", `{ "foo": "bar" }`) + + return req + }(), + mod: buildWebhookModule(), + next: func() echo.HandlerFunc { + return func(c echo.Context) error { + ctx := c.Get("context").(*api.Context) + + return ctx.AddOutputPaths("/tests/test/testdata/api/sample2.pdf") + } + }(), + expectWebhookContentType: "application/pdf", + expectWebhookMethod: http.MethodPost, + expectWebhookFilename: "foo", + expectWebhookExtraHTTPHeaders: map[string]string{"foo": "bar"}, + }, + } { + func() { + srv := echo.New() + srv.HideBanner = true + srv.HidePort = true + + c := srv.NewContext(tc.request, httptest.NewRecorder()) + c.Set("logger", zap.NewNop()) + c.Set("traceHeader", "Gotenberg-Trace") + c.Set("trace", "foo") + c.Set("startTime", time.Now()) + c.Set("writeTimeout", time.Duration(10)*time.Second) + + ctx := &api.MockContext{Context: &api.Context{}} + ctx.SetLogger(zap.NewNop()) + ctx.SetEchoContext(c) + + c.Set("context", ctx.Context) + c.Set("cancel", func() context.CancelFunc { + return func() { + return + } + }()) + + webhook := echo.New() + webhook.HideBanner = true + webhook.HidePort = true + + rand.Seed(time.Now().UnixNano()) + webhookPort := rand.Intn(65535-1025+1) + 1025 + + c.Request().Header.Set("Gotenberg-Webhook-Url", fmt.Sprintf("http://localhost:%d/", webhookPort)) + c.Request().Header.Set("Gotenberg-Webhook-Error-Url", fmt.Sprintf("http://localhost:%d/", webhookPort)) + + errChan := make(chan error, 1) + + webhook.POST( + "/", + func() echo.HandlerFunc { + return func(c echo.Context) error { + contentType := c.Request().Header.Get(echo.HeaderContentType) + if contentType != tc.expectWebhookContentType { + t.Errorf("test %d: expected '%s' '%s' but got '%s'", i, echo.HeaderContentType, tc.expectWebhookContentType, contentType) + } + + trace := c.Request().Header.Get("Gotenberg-Trace") + if trace != "foo" { + t.Errorf("test %d: expected '%s' '%s' but got '%s'", i, "Gotenberg-Trace", "foo", trace) + } + + method := c.Request().Method + if method != tc.expectWebhookMethod { + t.Errorf("test %d: expected HTTP method '%s' but got '%s'", i, tc.expectWebhookMethod, method) + } + + for key, expect := range tc.expectWebhookExtraHTTPHeaders { + actual := c.Request().Header.Get(key) + + if actual != expect { + t.Errorf("test %d: expected '%s' '%s' but got '%s'", i, key, expect, actual) + } + } + + if contentType == echo.MIMEApplicationJSONCharsetUTF8 { + body, err := ioutil.ReadAll(c.Request().Body) + if err != nil { + errChan <- err + return nil + } + + result := struct { + Status int `json:"status"` + Message string `json:"message"` + }{} + + err = json.Unmarshal(body, &result) + if err != nil { + errChan <- err + return nil + } + + if result.Status != tc.expectWebhookErrorStatus { + t.Errorf("test %d: expected status %d from JSON but got %d", i, tc.expectWebhookErrorStatus, result.Status) + } + + if result.Message != tc.expectWebhookErrorMessage { + t.Errorf("test %d: expected message '%s' from JSON but got '%s'", i, tc.expectWebhookErrorMessage, result.Message) + } + + errChan <- nil + return nil + } + + contentLength := c.Request().Header.Get(echo.HeaderContentLength) + if contentLength == "" { + t.Errorf("test %d: expected non empty '%s'", i, echo.HeaderContentLength) + } + + contentDisposition := c.Request().Header.Get(echo.HeaderContentDisposition) + if !strings.Contains(contentDisposition, tc.expectWebhookFilename) { + t.Errorf("test %d: expected '%s' '%s' to contain '%s'", i, echo.HeaderContentDisposition, contentDisposition, tc.expectWebhookFilename) + } + + body, err := ioutil.ReadAll(c.Request().Body) + if err != nil { + errChan <- err + return nil + } + + if body == nil || len(body) == 0 { + t.Errorf("test %d: expected non nil body", i) + } + + errChan <- nil + return nil + } + }(), + ) + + go func() { + err := webhook.Start(fmt.Sprintf(":%d", webhookPort)) + if !errors.Is(err, http.ErrServerClosed) { + t.Errorf("test %d: expected no error but got: %v", i, err) + } + }() + + defer func() { + err := webhook.Shutdown(context.TODO()) + if err != nil { + t.Errorf("test %d: expected no error but got: %v", i, err) + } + }() + + err := webhookMiddleware(tc.mod).Handler(tc.next)(c) + if err != nil && err != api.ErrAsyncProcess { + t.Errorf("test %d: expected no error but got: %v", i, err) + } + + err = <-errChan + if err != nil { + t.Errorf("test %d: expected no error but got: %v", i, err) + } + }() + } + +} diff --git a/pkg/modules/webhook/webhook.go b/pkg/modules/webhook/webhook.go new file mode 100644 index 00000000..4138bbc6 --- /dev/null +++ b/pkg/modules/webhook/webhook.go @@ -0,0 +1,126 @@ +package webhook + +import ( + "fmt" + "regexp" + "time" + + "github.com/gotenberg/gotenberg/v7/pkg/gotenberg" + "github.com/gotenberg/gotenberg/v7/pkg/modules/api" + flag "github.com/spf13/pflag" + "go.uber.org/multierr" +) + +func init() { + gotenberg.MustRegisterModule(Webhook{}) +} + +// Webhook is a module which provides a middleware for uploading output files +// to any destinations in an asynchronous fashion. +type Webhook struct { + allowList *regexp.Regexp + denyList *regexp.Regexp + errorAllowList *regexp.Regexp + errorDenyList *regexp.Regexp + maxRetry int + retryMinWait time.Duration + retryMaxWait time.Duration + disable bool +} + +// Descriptor returns an Webhook's module descriptor. +func (Webhook) Descriptor() gotenberg.ModuleDescriptor { + return gotenberg.ModuleDescriptor{ + ID: "webhook", + FlagSet: func() *flag.FlagSet { + fs := flag.NewFlagSet("webhook", flag.ExitOnError) + // Deprecated flags. + fs.String("api-webhook-allow-list", "", "Set the allowed URLs for the webhook feature using a regular expression") + fs.String("api-webhook-deny-list", "", "Set the denied URLs for the webhook feature using a regular expression") + fs.String("api-webhook-error-allow-list", "", "Set the allowed URLs in case of an error for the webhook feature using a regular expression") + fs.String("api-webhook-error-deny-list", "", "Set the denied URLs in case of an error for the webhook feature using a regular expression") + fs.Int("api-webhook-max-retry", 4, "Set the maximum number of retries for the webhook feature") + fs.Duration("api-webhook-retry-min-wait", time.Duration(1)*time.Second, "Set the minimum duration to wait before trying to call the webhook again") + fs.Duration("api-webhook-retry-max-wait", time.Duration(30)*time.Second, "Set the maximum duration to wait before trying to call the webhook again") + fs.Bool("api-disable-webhook", false, "Disable the webhook feature") + + var err error + err = multierr.Append(err, fs.MarkDeprecated("api-webhook-allow-list", "use webhook-allow-list instead")) + err = multierr.Append(err, fs.MarkDeprecated("api-webhook-deny-list", "use webhook-deny-list instead")) + err = multierr.Append(err, fs.MarkDeprecated("api-webhook-error-allow-list", "use webhook-error-allow-list instead")) + err = multierr.Append(err, fs.MarkDeprecated("api-webhook-error-deny-list", "use webhook-error-deny-list instead")) + err = multierr.Append(err, fs.MarkDeprecated("api-webhook-max-retry", "use webhook-max-retry instead")) + err = multierr.Append(err, fs.MarkDeprecated("api-webhook-retry-min-wait", "use webhook-retry-min-wait instead")) + err = multierr.Append(err, fs.MarkDeprecated("api-webhook-retry-max-wait", "use webhook-retry-max-wait instead")) + err = multierr.Append(err, fs.MarkDeprecated("api-disable-webhook", "use webhook-disable instead")) + + if err != nil { + panic(fmt.Errorf("create deprecated flags for webhook module: %v", err)) + } + + // New flags. + fs.String("webhook-allow-list", "", "Set the allowed URLs for the webhook feature using a regular expression") + fs.String("webhook-deny-list", "", "Set the denied URLs for the webhook feature using a regular expression") + fs.String("webhook-error-allow-list", "", "Set the allowed URLs in case of an error for the webhook feature using a regular expression") + fs.String("webhook-error-deny-list", "", "Set the denied URLs in case of an error for the webhook feature using a regular expression") + fs.Int("webhook-max-retry", 4, "Set the maximum number of retries for the webhook feature") + fs.Duration("webhook-retry-min-wait", time.Duration(1)*time.Second, "Set the minimum duration to wait before trying to call the webhook again") + fs.Duration("webhook-retry-max-wait", time.Duration(30)*time.Second, "Set the maximum duration to wait before trying to call the webhook again") + fs.Bool("webhook-disable", false, "Disable the webhook feature") + + return fs + }(), + New: func() gotenberg.Module { return new(Webhook) }, + } +} + +// Provision sets the module properties. +func (w *Webhook) Provision(ctx *gotenberg.Context) error { + flags := ctx.ParsedFlags() + w.allowList = flags.MustDeprecatedRegexp("api-webhook-allow-list", "webhook-allow-list") + w.denyList = flags.MustDeprecatedRegexp("api-webhook-deny-list", "webhook-deny-list") + w.errorAllowList = flags.MustDeprecatedRegexp("api-webhook-error-allow-list", "webhook-error-allow-list") + w.errorDenyList = flags.MustDeprecatedRegexp("api-webhook-error-deny-list", "webhook-error-deny-list") + w.maxRetry = flags.MustDeprecatedInt("api-webhook-max-retry", "webhook-max-retry") + w.retryMinWait = flags.MustDeprecatedDuration("api-webhook-retry-min-wait", "webhook-retry-min-wait") + w.retryMaxWait = flags.MustDeprecatedDuration("api-webhook-retry-min-wait", "webhook-retry-max-wait") + w.disable = flags.MustDeprecatedBool("api-disable-webhook", "webhook-disable") + + return nil +} + +// Middlewares returns the middleware. +func (w Webhook) Middlewares() ([]api.Middleware, error) { + if w.disable { + return nil, nil + } + + return []api.Middleware{ + webhookMiddleware(w), + }, nil +} + +// AddGraceDuration increases the grace duration provided by the API for the +// garbage collector. +func (w Webhook) AddGraceDuration() time.Duration { + var duration time.Duration + + if w.disable { + return duration + } + + for i := 0; i < w.maxRetry; i++ { + // Yep... Golang does not allow int * time.Duration. + duration += w.retryMaxWait + } + + return duration +} + +// Interface guards. +var ( + _ gotenberg.Module = (*Webhook)(nil) + _ gotenberg.Provisioner = (*Webhook)(nil) + _ api.MiddlewareProvider = (*Webhook)(nil) + _ api.GarbageCollectorGraceDurationIncrementer = (*Webhook)(nil) +) diff --git a/pkg/modules/webhook/webhook_test.go b/pkg/modules/webhook/webhook_test.go new file mode 100644 index 00000000..32fe9306 --- /dev/null +++ b/pkg/modules/webhook/webhook_test.go @@ -0,0 +1,89 @@ +package webhook + +import ( + "reflect" + "testing" + "time" + + "github.com/gotenberg/gotenberg/v7/pkg/gotenberg" +) + +func TestWebhook_Descriptor(t *testing.T) { + descriptor := Webhook{}.Descriptor() + + actual := reflect.TypeOf(descriptor.New()) + expect := reflect.TypeOf(new(Webhook)) + + if actual != expect { + t.Errorf("expected '%s' but got '%s'", expect, actual) + } +} + +func TestWebhook_Provision(t *testing.T) { + mod := new(Webhook) + ctx := gotenberg.NewContext( + gotenberg.ParsedFlags{ + FlagSet: new(Webhook).Descriptor().FlagSet, + }, + nil, + ) + + err := mod.Provision(ctx) + if err != nil { + t.Errorf("expected no error but got: %v", err) + } +} + +func TestWebhook_Middlewares(t *testing.T) { + for i, tc := range []struct { + expectMiddlewares int + disable bool + }{ + { + expectMiddlewares: 1, + }, + { + disable: true, + }, + } { + mod := new(Webhook) + mod.disable = tc.disable + + middlewares, err := mod.Middlewares() + if err != nil { + t.Fatalf("test %d: expected no error but got: %v", i, err) + } + + if tc.expectMiddlewares != len(middlewares) { + t.Errorf("test %d: expected %d middlewares but got %d", i, tc.expectMiddlewares, len(middlewares)) + } + } +} + +func TestWebhook_AddGraceDuration(t *testing.T) { + for i, tc := range []struct { + maxRetry int + retryMaxWait time.Duration + expectDuration time.Duration + disable bool + }{ + { + maxRetry: 3, + retryMaxWait: time.Duration(1) * time.Second, + expectDuration: time.Duration(3) * time.Second, + }, + { + disable: true, + }, + } { + mod := new(Webhook) + mod.maxRetry = tc.maxRetry + mod.retryMaxWait = tc.retryMaxWait + mod.disable = tc.disable + + actual := mod.AddGraceDuration() + if actual != tc.expectDuration { + t.Errorf("test %d: expected '%s' but got '%s'", i, tc.expectDuration, actual) + } + } +} diff --git a/pkg/standard/imports.go b/pkg/standard/imports.go index 4b1d3f6a..8d1b25d8 100644 --- a/pkg/standard/imports.go +++ b/pkg/standard/imports.go @@ -1,6 +1,7 @@ package standard import ( + // Standard Gotenberg modules. _ "github.com/gotenberg/gotenberg/v7/pkg/modules/api" _ "github.com/gotenberg/gotenberg/v7/pkg/modules/chromium" _ "github.com/gotenberg/gotenberg/v7/pkg/modules/gc" @@ -11,4 +12,6 @@ import ( _ "github.com/gotenberg/gotenberg/v7/pkg/modules/pdfcpu" _ "github.com/gotenberg/gotenberg/v7/pkg/modules/pdfengines" _ "github.com/gotenberg/gotenberg/v7/pkg/modules/pdftk" + _ "github.com/gotenberg/gotenberg/v7/pkg/modules/prometheus" + _ "github.com/gotenberg/gotenberg/v7/pkg/modules/webhook" )