feat: add metrics system, move webhook feature to dedicated module (#372)

This commit is contained in:
Julien Neuhart
2021-10-19 18:35:24 +02:00
committed by GitHub
parent b839069af1
commit d4c47cecf2
39 changed files with 3359 additions and 1309 deletions

View File

@@ -0,0 +1,139 @@
package webhook
import (
"fmt"
"io"
"strconv"
"time"
"github.com/hashicorp/go-retryablehttp"
"github.com/labstack/echo/v4"
"go.uber.org/zap"
)
// client gathers all the data required to send a request to a webhook.
type client struct {
url string
method string
errorURL string
errorMethod string
extraHTTPHeaders map[string]string
startTime time.Time
client *retryablehttp.Client
logger *zap.Logger
}
// send call the webhook either to send the success response or the error response.
func (c client) send(body io.Reader, headers map[string]string, erroed bool) error {
URL := c.url
if erroed {
URL = c.errorURL
}
method := c.method
if erroed {
method = c.errorMethod
}
req, err := retryablehttp.NewRequest(method, URL, body)
if err != nil {
return fmt.Errorf("create '%s' request to '%s': %w", method, URL, err)
}
req.Header.Set("User-Agent", "Gotenberg")
// Extra HTTP headers are the custom headers from the user.
for key, value := range c.extraHTTPHeaders {
req.Header.Set(key, value)
}
// Middleware caller's headers > extra HTTP headers from the user.
contentLength, ok := headers[echo.HeaderContentLength]
if ok {
// Golang "http" package should automatically calculate the size of the
// body. But, when using a buffered file reader, it does not work.
// Worse, the "Content-Length" header is also removed. Therefore, in
// order to keep this valuable information, we have to trust the caller
// by reading the value of the "Content-Length" entry and set it as the
// content length of the request. It's kinda sub-optimal, but hey, at
// least it works.
bodySize, err := strconv.ParseInt(contentLength, 10, 64)
if err != nil {
return fmt.Errorf("parse content length entry: %w", err)
}
req.ContentLength = bodySize
}
for key, value := range headers {
req.Header.Set(key, value)
}
resp, err := c.client.Do(req)
if err != nil {
return fmt.Errorf("send '%s' request to '%s': %w", method, URL, err)
}
defer func() {
err := resp.Body.Close()
if err != nil {
c.logger.Error(fmt.Sprintf("close response body from '%s': %s", URL, err))
}
}()
// Last piece for calculating the latency.
finishTime := time.Now()
// Now let's log!
fields := make([]zap.Field, 5)
fields[0] = zap.String("webhook_url", URL)
fields[1] = zap.String("method", method)
fields[2] = zap.Int64("latency", int64(finishTime.Sub(c.startTime)))
fields[3] = zap.String("latency_human", finishTime.Sub(c.startTime).String())
fields[4] = zap.Int64("bytes_out", req.ContentLength)
if erroed {
c.logger.Warn("request to webhook with error details handled", fields...)
return nil
}
c.logger.Info("request to webhook handled", fields...)
return nil
}
// leveledLogger is wrapper around a zap.Logger which is used by the
// retryablehttp.Client.
type leveledLogger struct {
logger *zap.Logger
}
// Error logs a message at error level using the wrapped zap.Logger.
func (leveled leveledLogger) Error(msg string, keysAndValues ...interface{}) {
leveled.logger.Error(fmt.Sprintf("%s: %+v", msg, keysAndValues))
}
// Warn logs a message at warning level using the wrapped zap.Logger.
func (leveled leveledLogger) Warn(msg string, keysAndValues ...interface{}) {
leveled.logger.Warn(fmt.Sprintf("%s: %+v", msg, keysAndValues))
}
// Info logs a message at info level using the wrapped zap.Logger.
func (leveled leveledLogger) Info(msg string, keysAndValues ...interface{}) {
leveled.logger.Info(fmt.Sprintf("%s: %+v", msg, keysAndValues))
}
// Debug logs a message at debug level using the wrapped zap.Logger.
func (leveled leveledLogger) Debug(msg string, keysAndValues ...interface{}) {
leveled.logger.Debug(fmt.Sprintf("%s: %+v", msg, keysAndValues))
}
// Interface guards.
var (
_ retryablehttp.LeveledLogger = (*leveledLogger)(nil)
)

View File

@@ -0,0 +1,23 @@
package webhook
import (
"testing"
"go.uber.org/zap"
)
func TestLeveledLogger_Error(t *testing.T) {
leveledLogger{logger: zap.NewNop()}.Error("foo")
}
func TestLeveledLogger_Warn(t *testing.T) {
leveledLogger{logger: zap.NewNop()}.Warn("foo")
}
func TestLeveledLogger_Info(t *testing.T) {
leveledLogger{logger: zap.NewNop()}.Info("foo")
}
func TestLeveledLogger_Debug(t *testing.T) {
leveledLogger{logger: zap.NewNop()}.Debug("foo")
}

View File

@@ -0,0 +1,3 @@
// Package webhook provides a module which adds a middleware for uploading
// output files to any destinations in an asynchronous fashion.
package webhook

View File

@@ -0,0 +1,278 @@
package webhook
import (
"bufio"
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"os"
"regexp"
"strconv"
"strings"
"time"
"github.com/gotenberg/gotenberg/v7/pkg/modules/api"
"github.com/hashicorp/go-retryablehttp"
"github.com/labstack/echo/v4"
)
func webhookMiddleware(w Webhook) api.Middleware {
return api.Middleware{
Stack: api.MultipartStack,
Handler: func() echo.MiddlewareFunc {
return func(next echo.HandlerFunc) echo.HandlerFunc {
return func(c echo.Context) error {
webhookURL := c.Request().Header.Get("Gotenberg-Webhook-Url")
if webhookURL == "" {
// No webhook URL, call the next middleware in the chain.
return next(c)
}
ctx := c.Get("context").(*api.Context)
cancel := c.Get("cancel").(context.CancelFunc)
// Do we have a webhook error URL in case of... error?
webhookErrorURL := c.Request().Header.Get("Gotenberg-Webhook-Error-Url")
if webhookErrorURL == "" {
return api.WrapError(
errors.New("empty webhook error URL"),
api.NewSentinelHTTPError(http.StatusBadRequest, "Invalid 'Gotenberg-Webhook-Error-Url' header: empty value or header not provided"),
)
}
// Let's check if the webhook URLs are acceptable according to our
// allowed/denied lists.
filter := func(URL, header string, allowList, denyList *regexp.Regexp) error {
if !allowList.MatchString(URL) {
return api.WrapError(
fmt.Errorf("'%s' does not match the expression from the allowed list", URL),
api.NewSentinelHTTPError(
http.StatusForbidden,
fmt.Sprintf("Invalid '%s' header value: '%s' does not match the authorized URLs", header, URL),
),
)
}
if denyList.String() != "" && denyList.MatchString(URL) {
return api.WrapError(
fmt.Errorf("'%s' matches the expression from the denied list", URL),
api.NewSentinelHTTPError(
http.StatusForbidden,
fmt.Sprintf("Invalid '%s' header value: '%s' does not match the authorized URLs", header, URL),
),
)
}
return nil
}
err := filter(webhookURL, "Gotenberg-Webhook-Url", w.allowList, w.denyList)
if err != nil {
return fmt.Errorf("filter webhook URL: %w", err)
}
err = filter(webhookErrorURL, "Gotenberg-Webhook-Error-Url", w.errorAllowList, w.errorDenyList)
if err != nil {
return fmt.Errorf("filter webhook error URL: %w", err)
}
// Let's check the HTTP methods for calling the webhook URLs.
methodFromHeader := func(header string) (string, error) {
method := c.Request().Header.Get(header)
if method == "" {
return http.MethodPost, nil
}
method = strings.ToUpper(method)
switch method {
case http.MethodPost:
return method, nil
case http.MethodPatch:
return method, nil
case http.MethodPut:
return method, nil
}
return "", api.WrapError(
fmt.Errorf("webhook method '%s' is not '%s', '%s' or '%s'", method, http.MethodPost, http.MethodPatch, http.MethodPut),
api.NewSentinelHTTPError(
http.StatusBadRequest,
fmt.Sprintf("Invalid '%s' header value: expected '%s', '%s' or '%s', but got '%s'", header, http.MethodPost, http.MethodPatch, http.MethodPut, method),
),
)
}
webhookMethod, err := methodFromHeader("Gotenberg-Webhook-Method")
if err != nil {
return fmt.Errorf("get method to use for webhook: %w", err)
}
webhookErrorMethod, err := methodFromHeader("Gotenberg-Webhook-Error-Method")
if err != nil {
return fmt.Errorf("get method to use for webhook error: %w", err)
}
// What about extra HTTP headers?
var extraHTTPHeaders map[string]string
extraHTTPHeadersJSON := c.Request().Header.Get("Gotenberg-Webhook-Extra-Http-Headers")
if extraHTTPHeadersJSON != "" {
err = json.Unmarshal([]byte(extraHTTPHeadersJSON), &extraHTTPHeaders)
if err != nil {
return api.WrapError(
fmt.Errorf("unmarshal webhook extra HTTP headers: %w", err),
api.NewSentinelHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid 'Gotenberg-Webhook-Extra-Http-Headers' header value: %s", err.Error())),
)
}
}
client := &client{
url: webhookURL,
method: webhookMethod,
errorURL: webhookErrorURL,
errorMethod: webhookErrorMethod,
extraHTTPHeaders: extraHTTPHeaders,
startTime: c.Get("startTime").(time.Time),
client: &retryablehttp.Client{
HTTPClient: &http.Client{
Timeout: c.Get("writeTimeout").(time.Duration),
},
RetryMax: w.maxRetry,
RetryWaitMin: w.retryMinWait,
RetryWaitMax: w.retryMaxWait,
Logger: leveledLogger{
logger: ctx.Log(),
},
CheckRetry: retryablehttp.DefaultRetryPolicy,
Backoff: retryablehttp.DefaultBackoff,
},
logger: ctx.Log(),
}
// This method parses an "asynchronous" error and sends a
// request to the webhook error URL with a JSON body
// containing the status and the error message.
handleAsyncError := func(err error) {
status, message := api.ParseError(err)
body := struct {
Status int `json:"status"`
Message string `json:"message"`
}{
Status: status,
Message: message,
}
b, err := json.Marshal(body)
if err != nil {
ctx.Log().Error(fmt.Sprintf("marshal JSON: %s", err.Error()))
return
}
headers := map[string]string{
echo.HeaderContentType: echo.MIMEApplicationJSONCharsetUTF8,
c.Get("traceHeader").(string): c.Get("trace").(string),
}
err = client.send(bytes.NewReader(b), headers, true)
if err != nil {
ctx.Log().Error(fmt.Sprintf("send error response to webhook: %s", err.Error()))
}
}
// As a webhook URL has been given, we handle the request in a
// goroutine and return immediately.
go func() {
defer cancel()
// Call the next middleware in the chain.
err := next(c)
if err != nil {
// The process failed for whatever reason. Let's send the
// details to the webhook.
ctx.Log().Error(err.Error())
handleAsyncError(err)
return
}
// No error, let's get build the output file.
outputPath, err := ctx.BuildOutputFile()
if err != nil {
ctx.Log().Error(fmt.Sprintf("build output file: %s", err))
handleAsyncError(err)
return
}
outputFile, err := os.Open(outputPath)
if err != nil {
ctx.Log().Error(fmt.Sprintf("open output file: %s", err))
handleAsyncError(err)
return
}
defer func() {
err := outputFile.Close()
if err != nil {
ctx.Log().Error(fmt.Sprintf("close output file: %s", err))
}
}()
fileHeader := make([]byte, 512)
_, err = outputFile.Read(fileHeader)
if err != nil {
ctx.Log().Error(fmt.Sprintf("read header of output file: %s", err))
handleAsyncError(err)
return
}
fileStat, err := outputFile.Stat()
if err != nil {
ctx.Log().Error(fmt.Sprintf("get stat from output file: %s", err))
handleAsyncError(err)
return
}
_, err = outputFile.Seek(0, 0)
if err != nil {
ctx.Log().Error(fmt.Sprintf("reset output file reader: %s", err))
handleAsyncError(err)
return
}
headers := map[string]string{
echo.HeaderContentDisposition: fmt.Sprintf("attachement; filename=%q", ctx.OutputFilename(outputPath)),
echo.HeaderContentType: http.DetectContentType(fileHeader),
echo.HeaderContentLength: strconv.FormatInt(fileStat.Size(), 10),
c.Get("traceHeader").(string): c.Get("trace").(string),
}
// Send the output file to the webhook.
err = client.send(bufio.NewReader(outputFile), headers, false)
if err != nil {
ctx.Log().Error(fmt.Sprintf("send output file to webhook: %s", err))
handleAsyncError(err)
}
}()
return api.ErrAsyncProcess
}
}
}(),
}
}

View File

@@ -0,0 +1,534 @@
package webhook
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"math/rand"
"mime/multipart"
"net/http"
"net/http/httptest"
"regexp"
"strings"
"testing"
"time"
"github.com/gotenberg/gotenberg/v7/pkg/modules/api"
"github.com/labstack/echo/v4"
"go.uber.org/zap"
)
func TestWebhookMiddlewareGuards(t *testing.T) {
buildMultipartFormDataRequest := func() *http.Request {
body := &bytes.Buffer{}
writer := multipart.NewWriter(body)
defer func() {
err := writer.Close()
if err != nil {
t.Fatalf("expected no error but got: %v", err)
}
}()
err := writer.WriteField("foo", "foo")
if err != nil {
t.Fatalf("expected no error but got: %v", err)
}
req := httptest.NewRequest(http.MethodPost, "/", body)
req.Header.Set(echo.HeaderContentType, writer.FormDataContentType())
return req
}
buildWebhookModule := func() Webhook {
return Webhook{
allowList: regexp.MustCompile(""),
denyList: regexp.MustCompile(""),
errorAllowList: regexp.MustCompile(""),
errorDenyList: regexp.MustCompile(""),
maxRetry: 0,
retryMinWait: 0,
retryMaxWait: 0,
disable: false,
}
}
for i, tc := range []struct {
request *http.Request
mod Webhook
next echo.HandlerFunc
expectErr bool
expectHTTPErr bool
expectHTTPStatus int
}{
{
request: buildMultipartFormDataRequest(),
mod: buildWebhookModule(),
next: func() echo.HandlerFunc {
return func(c echo.Context) error {
return nil
}
}(),
},
{
request: func() *http.Request {
req := buildMultipartFormDataRequest()
req.Header.Set("Gotenberg-Webhook-Url", "foo")
return req
}(),
mod: buildWebhookModule(),
expectErr: true,
expectHTTPErr: true,
expectHTTPStatus: http.StatusBadRequest,
},
{
request: func() *http.Request {
req := buildMultipartFormDataRequest()
req.Header.Set("Gotenberg-Webhook-Url", "foo")
req.Header.Set("Gotenberg-Webhook-Error-Url", "bar")
return req
}(),
mod: func() Webhook {
mod := buildWebhookModule()
mod.allowList = regexp.MustCompile("bar")
return mod
}(),
expectErr: true,
expectHTTPErr: true,
expectHTTPStatus: http.StatusForbidden,
},
{
request: func() *http.Request {
req := buildMultipartFormDataRequest()
req.Header.Set("Gotenberg-Webhook-Url", "foo")
req.Header.Set("Gotenberg-Webhook-Error-Url", "bar")
return req
}(),
mod: func() Webhook {
mod := buildWebhookModule()
mod.denyList = regexp.MustCompile("foo")
return mod
}(),
expectErr: true,
expectHTTPErr: true,
expectHTTPStatus: http.StatusForbidden,
},
{
request: func() *http.Request {
req := buildMultipartFormDataRequest()
req.Header.Set("Gotenberg-Webhook-Url", "foo")
req.Header.Set("Gotenberg-Webhook-Error-Url", "bar")
return req
}(),
mod: func() Webhook {
mod := buildWebhookModule()
mod.errorAllowList = regexp.MustCompile("foo")
return mod
}(),
expectErr: true,
expectHTTPErr: true,
expectHTTPStatus: http.StatusForbidden,
},
{
request: func() *http.Request {
req := buildMultipartFormDataRequest()
req.Header.Set("Gotenberg-Webhook-Url", "foo")
req.Header.Set("Gotenberg-Webhook-Error-Url", "bar")
return req
}(),
mod: func() Webhook {
mod := buildWebhookModule()
mod.errorDenyList = regexp.MustCompile("bar")
return mod
}(),
expectErr: true,
expectHTTPErr: true,
expectHTTPStatus: http.StatusForbidden,
},
{
request: func() *http.Request {
req := buildMultipartFormDataRequest()
req.Header.Set("Gotenberg-Webhook-Url", "foo")
req.Header.Set("Gotenberg-Webhook-Method", http.MethodGet)
req.Header.Set("Gotenberg-Webhook-Error-Url", "bar")
return req
}(),
mod: buildWebhookModule(),
expectErr: true,
expectHTTPErr: true,
expectHTTPStatus: http.StatusBadRequest,
},
{
request: func() *http.Request {
req := buildMultipartFormDataRequest()
req.Header.Set("Gotenberg-Webhook-Url", "foo")
req.Header.Set("Gotenberg-Webhook-Error-Url", "bar")
req.Header.Set("Gotenberg-Webhook-Error-Method", http.MethodGet)
return req
}(),
mod: buildWebhookModule(),
expectErr: true,
expectHTTPErr: true,
expectHTTPStatus: http.StatusBadRequest,
},
{
request: func() *http.Request {
req := buildMultipartFormDataRequest()
req.Header.Set("Gotenberg-Webhook-Url", "foo")
req.Header.Set("Gotenberg-Webhook-Method", http.MethodPost)
req.Header.Set("Gotenberg-Webhook-Error-Url", "bar")
req.Header.Set("Gotenberg-Webhook-Error-Method", http.MethodGet)
return req
}(),
mod: buildWebhookModule(),
expectErr: true,
expectHTTPErr: true,
expectHTTPStatus: http.StatusBadRequest,
},
{
request: func() *http.Request {
req := buildMultipartFormDataRequest()
req.Header.Set("Gotenberg-Webhook-Url", "foo")
req.Header.Set("Gotenberg-Webhook-Method", http.MethodPatch)
req.Header.Set("Gotenberg-Webhook-Error-Url", "bar")
req.Header.Set("Gotenberg-Webhook-Error-Method", http.MethodGet)
return req
}(),
mod: buildWebhookModule(),
expectErr: true,
expectHTTPErr: true,
expectHTTPStatus: http.StatusBadRequest,
},
{
request: func() *http.Request {
req := buildMultipartFormDataRequest()
req.Header.Set("Gotenberg-Webhook-Url", "foo")
req.Header.Set("Gotenberg-Webhook-Method", http.MethodPut)
req.Header.Set("Gotenberg-Webhook-Error-Url", "bar")
req.Header.Set("Gotenberg-Webhook-Error-Method", http.MethodGet)
return req
}(),
mod: buildWebhookModule(),
expectErr: true,
expectHTTPErr: true,
expectHTTPStatus: http.StatusBadRequest,
},
{
request: func() *http.Request {
req := buildMultipartFormDataRequest()
req.Header.Set("Gotenberg-Webhook-Url", "foo")
req.Header.Set("Gotenberg-Webhook-Error-Url", "bar")
req.Header.Set("Gotenberg-Webhook-Extra-Http-Headers", "foo")
return req
}(),
mod: buildWebhookModule(),
expectErr: true,
expectHTTPErr: true,
expectHTTPStatus: http.StatusBadRequest,
},
} {
srv := echo.New()
srv.HideBanner = true
srv.HidePort = true
c := srv.NewContext(tc.request, httptest.NewRecorder())
ctx := &api.MockContext{Context: &api.Context{}}
ctx.SetEchoContext(c)
c.Set("context", ctx.Context)
c.Set("cancel", func() context.CancelFunc {
return func() {
return
}
}())
err := webhookMiddleware(tc.mod).Handler(tc.next)(c)
if tc.expectErr && err == nil {
t.Errorf("test %d: expected error but got: %v", i, err)
}
if !tc.expectErr && err != nil {
t.Errorf("test %d: expected no error but got: %v", i, err)
}
var httpErr api.HTTPError
isHTTPErr := errors.As(err, &httpErr)
if tc.expectHTTPErr && !isHTTPErr {
t.Errorf("test %d: expected HTTP error but got: %v", i, err)
}
if !tc.expectHTTPErr && isHTTPErr {
t.Errorf("test %d: expected no HTTP error but got one: %v", i, httpErr)
}
if err != nil && tc.expectHTTPErr && isHTTPErr {
status, _ := httpErr.HTTPError()
if status != tc.expectHTTPStatus {
t.Errorf("test %d: expected %d HTTP status code but got %d", i, tc.expectHTTPStatus, status)
}
}
}
}
func TestWebhookMiddlewareAsynchronousProcess(t *testing.T) {
buildMultipartFormDataRequest := func() *http.Request {
body := &bytes.Buffer{}
writer := multipart.NewWriter(body)
defer func() {
err := writer.Close()
if err != nil {
t.Fatalf("expected no error but got: %v", err)
}
}()
err := writer.WriteField("foo", "foo")
if err != nil {
t.Fatalf("expected no error but got: %v", err)
}
req := httptest.NewRequest(http.MethodPost, "/", body)
req.Header.Set(echo.HeaderContentType, writer.FormDataContentType())
return req
}
buildWebhookModule := func() Webhook {
return Webhook{
allowList: regexp.MustCompile(""),
denyList: regexp.MustCompile(""),
errorAllowList: regexp.MustCompile(""),
errorDenyList: regexp.MustCompile(""),
maxRetry: 0,
retryMinWait: 0,
retryMaxWait: 0,
disable: false,
}
}
for i, tc := range []struct {
request *http.Request
mod Webhook
next echo.HandlerFunc
expectWebhookContentType string
expectWebhookMethod string
expectWebhookExtraHTTPHeaders map[string]string
expectWebhookFilename string
expectWebhookErrorStatus int
expectWebhookErrorMessage string
}{
{
request: buildMultipartFormDataRequest(),
mod: buildWebhookModule(),
next: func() echo.HandlerFunc {
return func(c echo.Context) error {
return errors.New("foo")
}
}(),
expectWebhookContentType: echo.MIMEApplicationJSONCharsetUTF8,
expectWebhookMethod: http.MethodPost,
expectWebhookErrorStatus: http.StatusInternalServerError,
expectWebhookErrorMessage: http.StatusText(http.StatusInternalServerError),
},
{
request: buildMultipartFormDataRequest(),
mod: buildWebhookModule(),
next: func() echo.HandlerFunc {
return func(c echo.Context) error {
return nil
}
}(),
expectWebhookContentType: echo.MIMEApplicationJSONCharsetUTF8,
expectWebhookMethod: http.MethodPost,
expectWebhookErrorStatus: http.StatusInternalServerError,
expectWebhookErrorMessage: http.StatusText(http.StatusInternalServerError),
},
{
request: func() *http.Request {
req := buildMultipartFormDataRequest()
req.Header.Set("Gotenberg-Output-Filename", "foo")
req.Header.Set("Gotenberg-Webhook-Extra-Http-Headers", `{ "foo": "bar" }`)
return req
}(),
mod: buildWebhookModule(),
next: func() echo.HandlerFunc {
return func(c echo.Context) error {
ctx := c.Get("context").(*api.Context)
return ctx.AddOutputPaths("/tests/test/testdata/api/sample2.pdf")
}
}(),
expectWebhookContentType: "application/pdf",
expectWebhookMethod: http.MethodPost,
expectWebhookFilename: "foo",
expectWebhookExtraHTTPHeaders: map[string]string{"foo": "bar"},
},
} {
func() {
srv := echo.New()
srv.HideBanner = true
srv.HidePort = true
c := srv.NewContext(tc.request, httptest.NewRecorder())
c.Set("logger", zap.NewNop())
c.Set("traceHeader", "Gotenberg-Trace")
c.Set("trace", "foo")
c.Set("startTime", time.Now())
c.Set("writeTimeout", time.Duration(10)*time.Second)
ctx := &api.MockContext{Context: &api.Context{}}
ctx.SetLogger(zap.NewNop())
ctx.SetEchoContext(c)
c.Set("context", ctx.Context)
c.Set("cancel", func() context.CancelFunc {
return func() {
return
}
}())
webhook := echo.New()
webhook.HideBanner = true
webhook.HidePort = true
rand.Seed(time.Now().UnixNano())
webhookPort := rand.Intn(65535-1025+1) + 1025
c.Request().Header.Set("Gotenberg-Webhook-Url", fmt.Sprintf("http://localhost:%d/", webhookPort))
c.Request().Header.Set("Gotenberg-Webhook-Error-Url", fmt.Sprintf("http://localhost:%d/", webhookPort))
errChan := make(chan error, 1)
webhook.POST(
"/",
func() echo.HandlerFunc {
return func(c echo.Context) error {
contentType := c.Request().Header.Get(echo.HeaderContentType)
if contentType != tc.expectWebhookContentType {
t.Errorf("test %d: expected '%s' '%s' but got '%s'", i, echo.HeaderContentType, tc.expectWebhookContentType, contentType)
}
trace := c.Request().Header.Get("Gotenberg-Trace")
if trace != "foo" {
t.Errorf("test %d: expected '%s' '%s' but got '%s'", i, "Gotenberg-Trace", "foo", trace)
}
method := c.Request().Method
if method != tc.expectWebhookMethod {
t.Errorf("test %d: expected HTTP method '%s' but got '%s'", i, tc.expectWebhookMethod, method)
}
for key, expect := range tc.expectWebhookExtraHTTPHeaders {
actual := c.Request().Header.Get(key)
if actual != expect {
t.Errorf("test %d: expected '%s' '%s' but got '%s'", i, key, expect, actual)
}
}
if contentType == echo.MIMEApplicationJSONCharsetUTF8 {
body, err := ioutil.ReadAll(c.Request().Body)
if err != nil {
errChan <- err
return nil
}
result := struct {
Status int `json:"status"`
Message string `json:"message"`
}{}
err = json.Unmarshal(body, &result)
if err != nil {
errChan <- err
return nil
}
if result.Status != tc.expectWebhookErrorStatus {
t.Errorf("test %d: expected status %d from JSON but got %d", i, tc.expectWebhookErrorStatus, result.Status)
}
if result.Message != tc.expectWebhookErrorMessage {
t.Errorf("test %d: expected message '%s' from JSON but got '%s'", i, tc.expectWebhookErrorMessage, result.Message)
}
errChan <- nil
return nil
}
contentLength := c.Request().Header.Get(echo.HeaderContentLength)
if contentLength == "" {
t.Errorf("test %d: expected non empty '%s'", i, echo.HeaderContentLength)
}
contentDisposition := c.Request().Header.Get(echo.HeaderContentDisposition)
if !strings.Contains(contentDisposition, tc.expectWebhookFilename) {
t.Errorf("test %d: expected '%s' '%s' to contain '%s'", i, echo.HeaderContentDisposition, contentDisposition, tc.expectWebhookFilename)
}
body, err := ioutil.ReadAll(c.Request().Body)
if err != nil {
errChan <- err
return nil
}
if body == nil || len(body) == 0 {
t.Errorf("test %d: expected non nil body", i)
}
errChan <- nil
return nil
}
}(),
)
go func() {
err := webhook.Start(fmt.Sprintf(":%d", webhookPort))
if !errors.Is(err, http.ErrServerClosed) {
t.Errorf("test %d: expected no error but got: %v", i, err)
}
}()
defer func() {
err := webhook.Shutdown(context.TODO())
if err != nil {
t.Errorf("test %d: expected no error but got: %v", i, err)
}
}()
err := webhookMiddleware(tc.mod).Handler(tc.next)(c)
if err != nil && err != api.ErrAsyncProcess {
t.Errorf("test %d: expected no error but got: %v", i, err)
}
err = <-errChan
if err != nil {
t.Errorf("test %d: expected no error but got: %v", i, err)
}
}()
}
}

View File

@@ -0,0 +1,126 @@
package webhook
import (
"fmt"
"regexp"
"time"
"github.com/gotenberg/gotenberg/v7/pkg/gotenberg"
"github.com/gotenberg/gotenberg/v7/pkg/modules/api"
flag "github.com/spf13/pflag"
"go.uber.org/multierr"
)
func init() {
gotenberg.MustRegisterModule(Webhook{})
}
// Webhook is a module which provides a middleware for uploading output files
// to any destinations in an asynchronous fashion.
type Webhook struct {
allowList *regexp.Regexp
denyList *regexp.Regexp
errorAllowList *regexp.Regexp
errorDenyList *regexp.Regexp
maxRetry int
retryMinWait time.Duration
retryMaxWait time.Duration
disable bool
}
// Descriptor returns an Webhook's module descriptor.
func (Webhook) Descriptor() gotenberg.ModuleDescriptor {
return gotenberg.ModuleDescriptor{
ID: "webhook",
FlagSet: func() *flag.FlagSet {
fs := flag.NewFlagSet("webhook", flag.ExitOnError)
// Deprecated flags.
fs.String("api-webhook-allow-list", "", "Set the allowed URLs for the webhook feature using a regular expression")
fs.String("api-webhook-deny-list", "", "Set the denied URLs for the webhook feature using a regular expression")
fs.String("api-webhook-error-allow-list", "", "Set the allowed URLs in case of an error for the webhook feature using a regular expression")
fs.String("api-webhook-error-deny-list", "", "Set the denied URLs in case of an error for the webhook feature using a regular expression")
fs.Int("api-webhook-max-retry", 4, "Set the maximum number of retries for the webhook feature")
fs.Duration("api-webhook-retry-min-wait", time.Duration(1)*time.Second, "Set the minimum duration to wait before trying to call the webhook again")
fs.Duration("api-webhook-retry-max-wait", time.Duration(30)*time.Second, "Set the maximum duration to wait before trying to call the webhook again")
fs.Bool("api-disable-webhook", false, "Disable the webhook feature")
var err error
err = multierr.Append(err, fs.MarkDeprecated("api-webhook-allow-list", "use webhook-allow-list instead"))
err = multierr.Append(err, fs.MarkDeprecated("api-webhook-deny-list", "use webhook-deny-list instead"))
err = multierr.Append(err, fs.MarkDeprecated("api-webhook-error-allow-list", "use webhook-error-allow-list instead"))
err = multierr.Append(err, fs.MarkDeprecated("api-webhook-error-deny-list", "use webhook-error-deny-list instead"))
err = multierr.Append(err, fs.MarkDeprecated("api-webhook-max-retry", "use webhook-max-retry instead"))
err = multierr.Append(err, fs.MarkDeprecated("api-webhook-retry-min-wait", "use webhook-retry-min-wait instead"))
err = multierr.Append(err, fs.MarkDeprecated("api-webhook-retry-max-wait", "use webhook-retry-max-wait instead"))
err = multierr.Append(err, fs.MarkDeprecated("api-disable-webhook", "use webhook-disable instead"))
if err != nil {
panic(fmt.Errorf("create deprecated flags for webhook module: %v", err))
}
// New flags.
fs.String("webhook-allow-list", "", "Set the allowed URLs for the webhook feature using a regular expression")
fs.String("webhook-deny-list", "", "Set the denied URLs for the webhook feature using a regular expression")
fs.String("webhook-error-allow-list", "", "Set the allowed URLs in case of an error for the webhook feature using a regular expression")
fs.String("webhook-error-deny-list", "", "Set the denied URLs in case of an error for the webhook feature using a regular expression")
fs.Int("webhook-max-retry", 4, "Set the maximum number of retries for the webhook feature")
fs.Duration("webhook-retry-min-wait", time.Duration(1)*time.Second, "Set the minimum duration to wait before trying to call the webhook again")
fs.Duration("webhook-retry-max-wait", time.Duration(30)*time.Second, "Set the maximum duration to wait before trying to call the webhook again")
fs.Bool("webhook-disable", false, "Disable the webhook feature")
return fs
}(),
New: func() gotenberg.Module { return new(Webhook) },
}
}
// Provision sets the module properties.
func (w *Webhook) Provision(ctx *gotenberg.Context) error {
flags := ctx.ParsedFlags()
w.allowList = flags.MustDeprecatedRegexp("api-webhook-allow-list", "webhook-allow-list")
w.denyList = flags.MustDeprecatedRegexp("api-webhook-deny-list", "webhook-deny-list")
w.errorAllowList = flags.MustDeprecatedRegexp("api-webhook-error-allow-list", "webhook-error-allow-list")
w.errorDenyList = flags.MustDeprecatedRegexp("api-webhook-error-deny-list", "webhook-error-deny-list")
w.maxRetry = flags.MustDeprecatedInt("api-webhook-max-retry", "webhook-max-retry")
w.retryMinWait = flags.MustDeprecatedDuration("api-webhook-retry-min-wait", "webhook-retry-min-wait")
w.retryMaxWait = flags.MustDeprecatedDuration("api-webhook-retry-min-wait", "webhook-retry-max-wait")
w.disable = flags.MustDeprecatedBool("api-disable-webhook", "webhook-disable")
return nil
}
// Middlewares returns the middleware.
func (w Webhook) Middlewares() ([]api.Middleware, error) {
if w.disable {
return nil, nil
}
return []api.Middleware{
webhookMiddleware(w),
}, nil
}
// AddGraceDuration increases the grace duration provided by the API for the
// garbage collector.
func (w Webhook) AddGraceDuration() time.Duration {
var duration time.Duration
if w.disable {
return duration
}
for i := 0; i < w.maxRetry; i++ {
// Yep... Golang does not allow int * time.Duration.
duration += w.retryMaxWait
}
return duration
}
// Interface guards.
var (
_ gotenberg.Module = (*Webhook)(nil)
_ gotenberg.Provisioner = (*Webhook)(nil)
_ api.MiddlewareProvider = (*Webhook)(nil)
_ api.GarbageCollectorGraceDurationIncrementer = (*Webhook)(nil)
)

View File

@@ -0,0 +1,89 @@
package webhook
import (
"reflect"
"testing"
"time"
"github.com/gotenberg/gotenberg/v7/pkg/gotenberg"
)
func TestWebhook_Descriptor(t *testing.T) {
descriptor := Webhook{}.Descriptor()
actual := reflect.TypeOf(descriptor.New())
expect := reflect.TypeOf(new(Webhook))
if actual != expect {
t.Errorf("expected '%s' but got '%s'", expect, actual)
}
}
func TestWebhook_Provision(t *testing.T) {
mod := new(Webhook)
ctx := gotenberg.NewContext(
gotenberg.ParsedFlags{
FlagSet: new(Webhook).Descriptor().FlagSet,
},
nil,
)
err := mod.Provision(ctx)
if err != nil {
t.Errorf("expected no error but got: %v", err)
}
}
func TestWebhook_Middlewares(t *testing.T) {
for i, tc := range []struct {
expectMiddlewares int
disable bool
}{
{
expectMiddlewares: 1,
},
{
disable: true,
},
} {
mod := new(Webhook)
mod.disable = tc.disable
middlewares, err := mod.Middlewares()
if err != nil {
t.Fatalf("test %d: expected no error but got: %v", i, err)
}
if tc.expectMiddlewares != len(middlewares) {
t.Errorf("test %d: expected %d middlewares but got %d", i, tc.expectMiddlewares, len(middlewares))
}
}
}
func TestWebhook_AddGraceDuration(t *testing.T) {
for i, tc := range []struct {
maxRetry int
retryMaxWait time.Duration
expectDuration time.Duration
disable bool
}{
{
maxRetry: 3,
retryMaxWait: time.Duration(1) * time.Second,
expectDuration: time.Duration(3) * time.Second,
},
{
disable: true,
},
} {
mod := new(Webhook)
mod.maxRetry = tc.maxRetry
mod.retryMaxWait = tc.retryMaxWait
mod.disable = tc.disable
actual := mod.AddGraceDuration()
if actual != tc.expectDuration {
t.Errorf("test %d: expected '%s' but got '%s'", i, tc.expectDuration, actual)
}
}
}