From 782f6ac27e8a13196b9b3b7513fac20ef80c8772 Mon Sep 17 00:00:00 2001 From: Julien Neuhart Date: Thu, 5 Dec 2019 18:16:02 +0100 Subject: [PATCH] rolling back to golangci-lint 1.20.1 + moving normize filename to resource & tests with a file with special chars in its name + webhookurl custom headers done + preparing remote url custom headers --- Makefile | 2 +- build/lint/Dockerfile | 2 +- internal/app/xhttp/handler.go | 31 +++++++++- internal/app/xhttp/handler_test.go | 14 ++++- internal/app/xhttp/pkg/context/context.go | 11 ++-- internal/app/xhttp/pkg/resource/header.go | 37 +++++++++++ .../app/xhttp/pkg/resource/header_test.go | 62 +++++++++++++++++++ internal/app/xhttp/pkg/resource/resource.go | 54 ++++++++++++---- test/testdata.go | 1 + .../office/document_with_special_éà.txt | 3 + 10 files changed, 191 insertions(+), 26 deletions(-) create mode 100644 internal/app/xhttp/pkg/resource/header.go create mode 100644 internal/app/xhttp/pkg/resource/header_test.go create mode 100644 test/testdata/office/document_with_special_éà.txt diff --git a/Makefile b/Makefile index 876e90db..f3d59a02 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ VERSION=snapshot DOCKER_USER= DOCKER_PASSWORD= DOCKER_REPOSITORY=thecodingmachine -GOLANGCI_LINT_VERSION=1.21.0 +GOLANGCI_LINT_VERSION=1.20.1 CODE_COVERAGE=0 TINI_VERSION=0.18.0 MAXIMUM_WAIT_TIMEOUT=30.0 diff --git a/build/lint/Dockerfile b/build/lint/Dockerfile index fd08c9c5..42a8c054 100644 --- a/build/lint/Dockerfile +++ b/build/lint/Dockerfile @@ -33,4 +33,4 @@ RUN go mod download &&\ # Copy our code source. COPY --chown=gotenberg:gotenberg . . -CMD ["golangci-lint", "run" ,"--tests=false", "--enable-all", "--disable=dupl", "--disable=funlen" ] \ No newline at end of file +CMD ["golangci-lint", "run" ,"--tests=false", "--enable-all", "--disable=dupl", "--disable=funlen", "--disable=wsl", "--disable=gocognit" ] \ No newline at end of file diff --git a/internal/app/xhttp/handler.go b/internal/app/xhttp/handler.go index 7c819bd2..234c0903 100644 --- a/internal/app/xhttp/handler.go +++ b/internal/app/xhttp/handler.go @@ -308,20 +308,47 @@ func convertAsync(ctx context.Context, p printer.Printer, filename, fpath string defer f.Close() // nolint: errcheck logger.DebugfOp( op, - "sending result file '%s' to '%s'", + "preparing to send result file '%s' to '%s'...", filename, webhookURL, ) httpClient := &http.Client{ Timeout: xtime.Duration(webhookURLTimeout), } - resp, err := httpClient.Post(webhookURL, "application/pdf", f) /* #nosec */ + req, err := http.NewRequest(http.MethodPost, webhookURL, f) + if err != nil { + xerr := xerror.New(op, err) + logger.ErrorOp(xerror.Op(xerr), xerr) + return + } + req.Header.Set(echo.HeaderContentType, "application/pdf") + // set custom headers (if any). + for key, value := range resource.WebhookURLCustomHeaders(r) { + for _, v := range value { + req.Header.Add(key, v) + logger.DebugfOp(op, "added '%s' to custom header '%s'", v, key) + } + } + // send the result file. + logger.DebugfOp( + op, + "sending result file '%s' to '%s'...", + filename, + webhookURL, + ) + resp, err := httpClient.Do(req) /* #nosec */ if err != nil { xerr := xerror.New(op, err) logger.ErrorOp(xerror.Op(xerr), xerr) return } defer resp.Body.Close() // nolint: errcheck + logger.DebugfOp( + op, + "result file '%s' sent to '%s'", + filename, + webhookURL, + ) }() return nil } diff --git a/internal/app/xhttp/handler_test.go b/internal/app/xhttp/handler_test.go index 5602d90b..f3264567 100644 --- a/internal/app/xhttp/handler_test.go +++ b/internal/app/xhttp/handler_test.go @@ -579,11 +579,18 @@ func TestOfficeHandler(t *testing.T) { } func TestWebhook(t *testing.T) { + customHeaderRealKey := http.CanonicalHeaderKey("MyCustomHeader") + customHeaderKey := fmt.Sprintf("%s%s", resource.WebhookURLCustomHeaderCanonicalBaseKey, customHeaderRealKey) + customHeaderValue := "foo" status := make(chan error, 2) rcv := echo.New() rcv.POST("/foo", func(c echo.Context) error { - if c.Request().Header.Get("Content-type") != "application/pdf" { - status <- fmt.Errorf("wrong Content-type: got %s want %s", c.Request().Header.Get("Content-type"), "application/pdf") + if c.Request().Header.Get(echo.HeaderContentType) != "application/pdf" { + status <- fmt.Errorf("wrong Content-type: got '%s' want '%s'", c.Request().Header.Get(echo.HeaderContentType), "application/pdf") + return nil + } + if c.Request().Header.Get(customHeaderRealKey) != customHeaderValue { + status <- fmt.Errorf("wrong '%s': got '%s' want '%s'", customHeaderRealKey, c.Request().Header.Get(customHeaderRealKey), customHeaderValue) return nil } body, err := ioutil.ReadAll(c.Request().Body) @@ -607,6 +614,7 @@ func TestWebhook(t *testing.T) { body, contentType := test.MergeMultipartForm(t, map[string]string{string(resource.WebhookURLArgKey): "http://localhost:3001/foo"}) req := httptest.NewRequest(http.MethodPost, mergeEndpoint, body) req.Header.Set(echo.HeaderContentType, contentType) + req.Header.Set(customHeaderKey, customHeaderValue) test.AssertStatusCode(t, http.StatusOK, srv, req) err := <-status assert.NoError(t, err) @@ -620,5 +628,5 @@ func TestResultFilename(t *testing.T) { req.Header.Set(echo.HeaderContentType, contentType) rec := httptest.NewRecorder() srv.ServeHTTP(rec, req) - assert.Equal(t, "attachment; filename=\"foo.pdf\"", rec.Header().Get("Content-Disposition")) + assert.Equal(t, "attachment; filename=\"foo.pdf\"", rec.Header().Get(echo.HeaderContentDisposition)) } diff --git a/internal/app/xhttp/pkg/context/context.go b/internal/app/xhttp/pkg/context/context.go index d1065b58..6a8122d1 100644 --- a/internal/app/xhttp/pkg/context/context.go +++ b/internal/app/xhttp/pkg/context/context.go @@ -12,7 +12,6 @@ import ( "github.com/labstack/echo/v4" "github.com/thecodingmachine/gotenberg/internal/app/xhttp/pkg/resource" "github.com/thecodingmachine/gotenberg/internal/pkg/conf" - "github.com/thecodingmachine/gotenberg/internal/pkg/normalize" "github.com/thecodingmachine/gotenberg/internal/pkg/xerror" "github.com/thecodingmachine/gotenberg/internal/pkg/xlog" ) @@ -79,6 +78,10 @@ func (ctx *Context) WithResource(directoryName string) error { if err != nil { return r, err } + // retrieve custom headers from request. + for name, value := range ctx.Request().Header { + r.WithCustomHeader(name, value) + } // retrieve form values from request. for _, key := range resource.ArgKeys() { r.WithArg(key, ctx.FormValue(string(key))) @@ -103,11 +106,7 @@ func (ctx *Context) WithResource(directoryName string) error { return r, err } defer in.Close() // nolint: errcheck - filename, err := normalize.String(fh.Filename) - if err != nil { - return r, err - } - if err := r.WithFile(filename, in); err != nil { + if err := r.WithFile(fh.Filename, in); err != nil { return r, err } } diff --git a/internal/app/xhttp/pkg/resource/header.go b/internal/app/xhttp/pkg/resource/header.go new file mode 100644 index 00000000..ac284643 --- /dev/null +++ b/internal/app/xhttp/pkg/resource/header.go @@ -0,0 +1,37 @@ +package resource + +import ( + "strings" +) + +const ( + // RemoteURLCustomHeaderCanonicalBaseKey is the base key + // of custom headers send to the remote URL. + RemoteURLCustomHeaderCanonicalBaseKey string = "Gotenberg-Remoteurl-" + // WebhookURLCustomHeaderCanonicalBaseKey is the base key + // of custom headers send to the webhook URL. + WebhookURLCustomHeaderCanonicalBaseKey string = "Gotenberg-Webhookurl-" +) + +func fetchCustomHeaders(r Resource, baseKey string) map[string][]string { + customHeaders := make(map[string][]string) + for key, value := range r.customHeaders { + if strings.Contains(key, baseKey) { + realKey := strings.Replace(key, baseKey, "", 1) + customHeaders[realKey] = value + } + } + return customHeaders +} + +// RemoteURLCustomHeaders is a helper for retrieving +// the custom headers for the URL conversion. +func RemoteURLCustomHeaders(r Resource) map[string][]string { + return fetchCustomHeaders(r, RemoteURLCustomHeaderCanonicalBaseKey) +} + +// WebhookURLCustomHeaders is a helper for retrieving +// the custom headers for the webhook URL. +func WebhookURLCustomHeaders(r Resource) map[string][]string { + return fetchCustomHeaders(r, WebhookURLCustomHeaderCanonicalBaseKey) +} diff --git a/internal/app/xhttp/pkg/resource/header_test.go b/internal/app/xhttp/pkg/resource/header_test.go new file mode 100644 index 00000000..07c71f0a --- /dev/null +++ b/internal/app/xhttp/pkg/resource/header_test.go @@ -0,0 +1,62 @@ +package resource + +import ( + "fmt" + "net/http" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/thecodingmachine/gotenberg/test" +) + +func TestRemoteURLCustomHeaders(t *testing.T) { + const resourceDirectoryName string = "foo" + logger := test.DebugLogger() + r, err := New(logger, resourceDirectoryName) + assert.Nil(t, err) + // should find the custom header. + customHeaderValue := "bar" + customHeaderCanonicalRealKey := "Foo" + customHeaderCanonicalKey := http.CanonicalHeaderKey(fmt.Sprintf("%s%s", RemoteURLCustomHeaderCanonicalBaseKey, customHeaderCanonicalRealKey)) + r.WithCustomHeader(customHeaderCanonicalKey, []string{customHeaderValue}) + r.WithCustomHeader("Bar", []string{"Bar"}) + expected := map[string][]string{ + customHeaderCanonicalRealKey: []string{ + customHeaderValue, + }, + } + notExpected := map[string][]string{ + customHeaderCanonicalKey: []string{ + customHeaderValue, + }, + } + v := RemoteURLCustomHeaders(r) + assert.Equal(t, expected, v) + assert.NotEqual(t, notExpected, v) +} + +func TestWebhookURLCustomHeaders(t *testing.T) { + const resourceDirectoryName string = "foo" + logger := test.DebugLogger() + r, err := New(logger, resourceDirectoryName) + assert.Nil(t, err) + // should find the custom header. + customHeaderValue := "bar" + customHeaderCanonicalRealKey := "Foo" + customHeaderCanonicalKey := http.CanonicalHeaderKey(fmt.Sprintf("%s%s", WebhookURLCustomHeaderCanonicalBaseKey, customHeaderCanonicalRealKey)) + r.WithCustomHeader(customHeaderCanonicalKey, []string{customHeaderValue}) + r.WithCustomHeader("Bar", []string{"Bar"}) + expected := map[string][]string{ + customHeaderCanonicalRealKey: []string{ + customHeaderValue, + }, + } + notExpected := map[string][]string{ + customHeaderCanonicalKey: []string{ + customHeaderValue, + }, + } + v := WebhookURLCustomHeaders(r) + assert.Equal(t, expected, v) + assert.NotEqual(t, notExpected, v) +} diff --git a/internal/app/xhttp/pkg/resource/resource.go b/internal/app/xhttp/pkg/resource/resource.go index 81cf001a..39d1229b 100644 --- a/internal/app/xhttp/pkg/resource/resource.go +++ b/internal/app/xhttp/pkg/resource/resource.go @@ -5,7 +5,9 @@ import ( "io" "os" "path/filepath" + "strings" + "github.com/thecodingmachine/gotenberg/internal/pkg/normalize" "github.com/thecodingmachine/gotenberg/internal/pkg/xassert" "github.com/thecodingmachine/gotenberg/internal/pkg/xerror" "github.com/thecodingmachine/gotenberg/internal/pkg/xlog" @@ -21,10 +23,11 @@ const TemporaryDirectory string = "tmp" // Resource helps managing // arguments and files for a conversion. type Resource struct { - logger xlog.Logger - dirPath string - args map[ArgKey]string - files map[string]file + logger xlog.Logger + dirPath string + customHeaders map[string][]string + args map[ArgKey]string + files map[string]file } // New creates a Resource where its files will @@ -48,10 +51,11 @@ func New(logger xlog.Logger, directoryName string) (Resource, error) { } logger.DebugfOp(op, "resource directory '%s' created", directoryName) return Resource{ - logger: logger, - dirPath: dirPath, - args: make(map[ArgKey]string), - files: make(map[string]file), + logger: logger, + dirPath: dirPath, + customHeaders: make(map[string][]string), + args: make(map[ArgKey]string), + files: make(map[string]file), }, nil } @@ -70,6 +74,19 @@ func (r Resource) Close() error { return nil } +// WithCustomHeader add a new custom header to the Resource. +// Given key should be in canonical format. +func (r *Resource) WithCustomHeader(key string, value []string) { + const op string = "resource.Resource.WithCustomHeader" + if strings.Contains(key, RemoteURLCustomHeaderCanonicalBaseKey) || + strings.Contains(key, WebhookURLCustomHeaderCanonicalBaseKey) { + r.customHeaders[key] = value + r.logger.DebugfOp(op, "added '%s' with value '%s' to resource custom headers", key, value) + return + } + r.logger.DebugfOp(op, "skipping '%s' as it is not a custom header...", key) +} + // WithArg add a new argument to the Resource. func (r *Resource) WithArg(key ArgKey, value string) { const op string = "resource.Resource.WithArg" @@ -80,13 +97,24 @@ func (r *Resource) WithArg(key ArgKey, value string) { // WithFile add a new file to the Resource. func (r *Resource) WithFile(filename string, in io.Reader) error { const op string = "resource.Resource.WithFile" - fpath := fmt.Sprintf("%s/%s", r.dirPath, filename) - file := file{fpath: fpath} - if err := file.write(in); err != nil { + resolver := func() error { + // see https://github.com/thecodingmachine/gotenberg/issues/104. + normalized, err := normalize.String(filename) + if err != nil { + return err + } + fpath := fmt.Sprintf("%s/%s", r.dirPath, normalized) + file := file{fpath: fpath} + if err := file.write(in); err != nil { + return err + } + r.files[filename] = file + r.logger.DebugfOp(op, "resource file '%s' created", filename) + return nil + } + if err := resolver(); err != nil { return xerror.New(op, err) } - r.files[filename] = file - r.logger.DebugfOp(op, "resource file '%s' created", filename) return nil } diff --git a/test/testdata.go b/test/testdata.go index e40ab818..fc72472b 100644 --- a/test/testdata.go +++ b/test/testdata.go @@ -75,6 +75,7 @@ func OfficeFpaths(t *testing.T) []string { fpath(t, "office", "document.docx"), fpath(t, "office", "document.rtf"), fpath(t, "office", "document.txt"), + fpath(t, "office", "document_with_special_éà.txt"), } } diff --git a/test/testdata/office/document_with_special_éà.txt b/test/testdata/office/document_with_special_éà.txt new file mode 100644 index 00000000..16eab9d7 --- /dev/null +++ b/test/testdata/office/document_with_special_éà.txt @@ -0,0 +1,3 @@ +Gutenberg + +Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. \ No newline at end of file