adalal/gotenberg
1
0
Fork 0
mirror of https://github.com/gotenberg/gotenberg.git synced 2026-08-10 09:32:13 +01:00
Code Issues Packages Projects Releases Wiki Activity

feat: add 7.x source code

Browse Source
This commit is contained in:
Julien Neuhart
2021-08-22 12:52:44 +02:00
parent e457155950
commit 0f5e8fd314
111 changed files with 31188 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

494
pkg/modules/api/api.go Normal file
View File

@@ -0,0 +1,494 @@
package api
import (
"context"
"errors"
"fmt"
"net/http"
"os"
"regexp"
"sort"
"strconv"
"strings"
"time"
"github.com/alexliesenfeld/health"
"github.com/gotenberg/gotenberg/v7/pkg/gotenberg"
"github.com/gotenberg/gotenberg/v7/pkg/modules/gc"
"github.com/labstack/echo/v4"
flag "github.com/spf13/pflag"
"go.uber.org/multierr"
"go.uber.org/zap"
)
func init() {
gotenberg.MustRegisterModule(API{})
}
// API is a module which provides an HTTP server. Other modules may add
// "multipart/form-data" routes, middlewares or health checks.
type API struct {
port int
readTimeout time.Duration
processTimeout time.Duration
writeTimeout time.Duration
rootPath string
traceHeader string
disableHealthCheckLogging bool
webhookAllowList *regexp.Regexp
webhookDenyList *regexp.Regexp
webhookErrorAllowList *regexp.Regexp
webhookErrorDenyList *regexp.Regexp
webhookMaxRetry int
webhookRetryMinWait time.Duration
webhookRetryMaxWait time.Duration
disableWebhook bool
multipartFormDataRoutes []MultipartFormDataRoute
externalMiddlewares []Middleware
healthChecks []health.CheckerOption
logger *zap.Logger
srv *echo.Echo
}
// MultipartFormDataRouter is a module interface which adds
// "multipart/form-data" routes to the API.
type MultipartFormDataRouter interface {
Routes() ([]MultipartFormDataRoute, error)
}
// MultipartFormDataRoute represents a "multipart/form-data" route. All routes
// uses the HTTP POST method.
type MultipartFormDataRoute struct {
// Path is the sub path of the route. Must start with a slash.
// Required.
Path string
// Handler is the function which handles the request.
// Required.
Handler func(ctx *Context) error
}
// MiddlewareProvider is a module interface which adds middlewares to the API.
type MiddlewareProvider interface {
Middlewares() ([]Middleware, error)
}
// MiddlewarePriority is a type which helps to determine the execution order of
// middlewares provided by the MiddlewareProvider modules.
type MiddlewarePriority uint32
const (
VeryLowPriority MiddlewarePriority = iota
LowPriority
MediumPriority
HighPriority
VeryHighPriority
)
// Middleware is a middleware which can be added to the API's middlewares
// chain.
//
// middleware := &Middleware{
// Handler: func() echo.MiddlewareFunc {
// return func(next echo.HandlerFunc) echo.HandlerFunc {
// return func(c echo.Context) error {
// rootPath := c.Get("rootPath").(string)
// healthURI := fmt.Sprintf("%shealth", rootPath)
//
// // Skip the middleware if health check URI.
// if c.Request().RequestURI == healthURI {
// // Call the next middleware in the chain.
// return next(c)
// }
//
// // Your middleware process.
// // ...
//
// // Call the next middleware in the chain.
// return next(c)
// }
// }
// }(),
// }
type Middleware struct {
// RunBeforeRouter tells if the middleware should run before the router
// process an HTTP request.
// Optional.
RunBeforeRouter bool
// Priority tells if the middleware should be positioned high or not in
// the middlewares chain.
// Default to VeryLowPriority.
// Optional.
Priority MiddlewarePriority
// Handler is the function of the middleware.
// Required.
Handler echo.MiddlewareFunc
}
// HealthChecker is a module interface which allows adding health checks to the
// API.
//
// See https://github.com/alexliesenfeld/health for more details.
type HealthChecker interface {
Checks() ([]health.CheckerOption, error)
}
// Descriptor returns an API's module descriptor.
func (API) Descriptor() gotenberg.ModuleDescriptor {
return gotenberg.ModuleDescriptor{
ID: "api",
FlagSet: func() *flag.FlagSet {
fs := flag.NewFlagSet("api", flag.ExitOnError)
fs.Int("api-port", 3000, "Set the port on which the API should listen")
fs.String("api-port-from-env", "", "Set the environment variable with the port on which the API should listen - override the default port")
fs.Duration("api-read-timeout", time.Duration(30)*time.Second, "Set the maximum duration allowed to read a complete request, including the body")
fs.Duration("api-process-timeout", time.Duration(30)*time.Second, "Set the maximum duration allowed to process a request")
fs.Duration("api-write-timeout", time.Duration(30)*time.Second, "Set the maximum duration before timing out writes of the response")
fs.String("api-root-path", "/", "Set the root path of the API - for service discovery via URL paths")
fs.String("api-trace-header", "Gotenberg-Trace", "Set the header name to use for identifying requests")
fs.Bool("api-disable-health-check-logging", false, "Disable health check logging")
fs.String("api-webhook-allow-list", "", "Set the allowed URLs for the webhook feature using a regular expression")
fs.String("api-webhook-deny-list", "", "Set the denied URLs for the webhook feature using a regular expression")
fs.String("api-webhook-error-allow-list", "", "Set the allowed URLs in case of an error for the webhook feature using a regular expression")
fs.String("api-webhook-error-deny-list", "", "Set the denied URLs in case of an error for the webhook feature using a regular expression")
fs.Int("api-webhook-max-retry", 4, "Set the maximum number of retries for the webhook feature")
fs.Duration("api-webhook-retry-min-wait", time.Duration(1)*time.Second, "Set the minimum duration to wait before trying to call the webhook again")
fs.Duration("api-webhook-retry-max-wait", time.Duration(30)*time.Second, "Set the maximum duration to wait before trying to call the webhook again")
fs.Bool("api-disable-webhook", false, "Disable the webhook feature")
return fs
}(),
New: func() gotenberg.Module { return new(API) },
}
}
// Provision sets the module properties.
func (a *API) Provision(ctx *gotenberg.Context) error {
flags := ctx.ParsedFlags()
a.port = flags.MustInt("api-port")
a.readTimeout = flags.MustDuration("api-read-timeout")
a.processTimeout = flags.MustDuration("api-process-timeout")
a.writeTimeout = flags.MustDuration("api-write-timeout")
a.rootPath = flags.MustString("api-root-path")
a.traceHeader = flags.MustString("api-trace-header")
a.disableHealthCheckLogging = flags.MustBool("api-disable-health-check-logging")
a.webhookAllowList = flags.MustRegexp("api-webhook-allow-list")
a.webhookDenyList = flags.MustRegexp("api-webhook-deny-list")
a.webhookErrorAllowList = flags.MustRegexp("api-webhook-error-allow-list")
a.webhookErrorDenyList = flags.MustRegexp("api-webhook-error-deny-list")
a.webhookMaxRetry = flags.MustInt("api-webhook-max-retry")
a.webhookRetryMinWait = flags.MustDuration("api-webhook-retry-min-wait")
a.webhookRetryMaxWait = flags.MustDuration("api-webhook-retry-max-wait")
a.disableWebhook = flags.MustBool("api-disable-webhook")
// Port from env?
portEnvVar := flags.MustString("api-port-from-env")
if portEnvVar != "" {
val, ok := os.LookupEnv(portEnvVar)
if !ok {
return fmt.Errorf("environment variable '%s' does not exist", portEnvVar)
}
if val == "" {
return fmt.Errorf("environment variable '%s' is empty", portEnvVar)
}
port, err := strconv.Atoi(val)
if err != nil {
return fmt.Errorf("get int value of environment variable '%s': %w", portEnvVar, err)
}
a.port = port
}
// Get routes from modules.
mods, err := ctx.Modules(new(MultipartFormDataRouter))
if err != nil {
return fmt.Errorf("get multipart/form-data routers: %w", err)
}
routers := make([]MultipartFormDataRouter, len(mods))
for i, router := range mods {
routers[i] = router.(MultipartFormDataRouter)
}
for _, router := range routers {
routes, err := router.Routes()
if err != nil {
return fmt.Errorf("get routes: %w", err)
}
a.multipartFormDataRoutes = append(a.multipartFormDataRoutes, routes...)
}
// Get middlewares from modules.
mods, err = ctx.Modules(new(MiddlewareProvider))
if err != nil {
return fmt.Errorf("get middleware providers: %w", err)
}
middlewareProviders := make([]MiddlewareProvider, len(mods))
for i, middlewareProvider := range mods {
middlewareProviders[i] = middlewareProvider.(MiddlewareProvider)
}
for _, middlewareProvider := range middlewareProviders {
middlewares, err := middlewareProvider.Middlewares()
if err != nil {
return fmt.Errorf("get middlewares: %w", err)
}
a.externalMiddlewares = append(a.externalMiddlewares, middlewares...)
}
// Sort middlewares by priority.
sort.Slice(a.externalMiddlewares, func(i, j int) bool {
return a.externalMiddlewares[i].Priority > a.externalMiddlewares[j].Priority
})
// Get health checks from modules.
mods, err = ctx.Modules(new(HealthChecker))
if err != nil {
return fmt.Errorf("get health checkers: %w", err)
}
healthCheckers := make([]HealthChecker, len(mods))
for i, healthChecker := range mods {
healthCheckers[i] = healthChecker.(HealthChecker)
}
for _, healthChecker := range healthCheckers {
checks, err := healthChecker.Checks()
if err != nil {
return fmt.Errorf("get health checks: %w", err)
}
a.healthChecks = append(a.healthChecks, checks...)
}
loggerProvider, err := ctx.Module(new(gotenberg.LoggerProvider))
if err != nil {
return fmt.Errorf("get logger provider: %w", err)
}
logger, err := loggerProvider.(gotenberg.LoggerProvider).Logger(a)
if err != nil {
return fmt.Errorf("get logger: %w", err)
}
a.logger = logger
return nil
}
// Validate validates the module properties.
func (a API) Validate() error {
var err error
if a.port < 1 || a.port > 65535 {
err = multierr.Append(err,
errors.New("port must be more than 1 and less than 65535"),
)
}
if !strings.HasPrefix(a.rootPath, "/") {
err = multierr.Append(err,
errors.New("root path must start with /"),
)
}
if !strings.HasSuffix(a.rootPath, "/") {
err = multierr.Append(err,
errors.New("root path must end with /"),
)
}
if len(strings.TrimSpace(a.traceHeader)) == 0 {
err = multierr.Append(err,
errors.New("trace header must not be empty"),
)
}
if err != nil {
return err
}
routesMap := make(map[string]MultipartFormDataRoute, len(a.multipartFormDataRoutes))
for _, route := range a.multipartFormDataRoutes {
if route.Path == "" {
return errors.New("route with empty path cannot be registered")
}
if !strings.HasPrefix(route.Path, "/") {
return fmt.Errorf("route %s does not start with /", route.Path)
}
if route.Handler == nil {
return fmt.Errorf("route %s has a nil handler", route.Path)
}
if _, ok := routesMap[route.Path]; ok {
return fmt.Errorf("route %s is already registered", route.Path)
}
routesMap[route.Path] = route
}
for _, middleware := range a.externalMiddlewares {
if middleware.Handler == nil {
return errors.New("a middleware has a nil handler")
}
}
return nil
}
// Start starts the HTTP server.
func (a *API) Start() error {
a.srv = echo.New()
a.srv.HideBanner = true
a.srv.HidePort = true
a.srv.Server.ReadTimeout = a.readTimeout
a.srv.Server.WriteTimeout = a.writeTimeout
a.srv.HTTPErrorHandler = httpErrorHandler(a.traceHeader)
a.srv.Pre(
latencyMiddleware(),
rootPathMiddleware(a.rootPath),
traceMiddleware(a.traceHeader),
loggerMiddleware(a.logger, a.disableHealthCheckLogging),
)
for _, externalMiddleware := range a.externalMiddlewares {
if externalMiddleware.RunBeforeRouter {
a.srv.Pre(externalMiddleware.Handler)
continue
}
a.srv.Use(externalMiddleware.Handler)
}
hardTimeout := a.processTimeout + (time.Duration(5) * time.Second)
a.srv.GET(
fmt.Sprintf("%shealth", a.rootPath),
func() echo.HandlerFunc {
checks := append(a.healthChecks, health.WithTimeout(a.processTimeout))
checker := health.NewChecker(checks...)
return func(echoCtx echo.Context) error {
health.NewHandler(checker).ServeHTTP(echoCtx.Response().Writer, echoCtx.Request())
return nil
}
}(),
timeoutMiddleware(hardTimeout),
)
formsGroup := a.srv.Group(
fmt.Sprintf("%sforms", a.rootPath),
contextMiddleware(
contextMiddlewareConfig{
traceHeader: a.traceHeader,
timeout: struct {
process time.Duration
write time.Duration
}{
process: a.processTimeout,
write: a.writeTimeout,
},
webhook: struct {
allowList *regexp.Regexp
denyList *regexp.Regexp
errorAllowList *regexp.Regexp
errorDenyList *regexp.Regexp
maxRetry int
retryMinWait time.Duration
retryMaxWait time.Duration
disable bool
}{
allowList: a.webhookAllowList,
denyList: a.webhookDenyList,
errorAllowList: a.webhookErrorAllowList,
errorDenyList: a.webhookErrorDenyList,
maxRetry: a.webhookMaxRetry,
retryMinWait: a.webhookRetryMinWait,
retryMaxWait: a.webhookRetryMaxWait,
disable: a.disableWebhook,
},
},
),
timeoutMiddleware(hardTimeout),
)
// Add routes from other modules.
for _, route := range a.multipartFormDataRoutes {
formsGroup.POST(
route.Path,
func(route MultipartFormDataRoute) echo.HandlerFunc {
return func(c echo.Context) error {
ctx := c.Get("context").(*Context)
err := route.Handler(ctx)
if err != nil {
return fmt.Errorf("handle request: %w", err)
}
return nil
}
}(route),
)
}
// As the listen method is blocking, run it in a goroutine.
go func() {
err := a.srv.Start(fmt.Sprintf(":%d", a.port))
if !errors.Is(err, http.ErrServerClosed) {
a.logger.Fatal(err.Error())
}
}()
return nil
}
// StartupMessage returns a custom startup message.
func (a API) StartupMessage() string {
return fmt.Sprintf("server listening on port %d", a.port)
}
// Stop stops the HTTP server.
func (a API) Stop(ctx context.Context) error {
return a.srv.Shutdown(ctx)
}
// GraceDuration updates the expiration time of files and directories parsed by
// the gc.GarbageCollector.
func (a API) GraceDuration() time.Duration {
duration := a.readTimeout + a.processTimeout + a.writeTimeout
if a.disableWebhook {
return duration
}
for i := 0; i < a.webhookMaxRetry; i++ {
// Yep... Golang does not allow int * time.Duration.
duration += a.webhookRetryMaxWait
}
return duration
}
// Interface guards.
var (
_ gotenberg.Module = (*API)(nil)
_ gotenberg.Provisioner = (*API)(nil)
_ gotenberg.Validator = (*API)(nil)
_ gotenberg.App = (*API)(nil)
_ gc.GarbageCollectorGraceDurationModifier = (*API)(nil)
)

793
pkg/modules/api/api_test.go Normal file
View File

@@ -0,0 +1,793 @@
package api
import (
"bytes"
"context"
"errors"
"mime/multipart"
"net/http"
"net/http/httptest"
"os"
"reflect"
"testing"
"time"
"github.com/alexliesenfeld/health"
"github.com/gotenberg/gotenberg/v7/pkg/gotenberg"
"github.com/labstack/echo/v4"
"go.uber.org/zap"
)
type ProtoModule struct {
descriptor func() gotenberg.ModuleDescriptor
}
func (mod ProtoModule) Descriptor() gotenberg.ModuleDescriptor {
return mod.descriptor()
}
type ProtoValidator struct {
ProtoModule
validate func() error
}
func (mod ProtoValidator) Validate() error {
return mod.validate()
}
type ProtoMultipartFormDataRouter struct {
ProtoValidator
routes func() ([]MultipartFormDataRoute, error)
}
func (mod ProtoMultipartFormDataRouter) Routes() ([]MultipartFormDataRoute, error) {
return mod.routes()
}
type ProtoMiddlewareProvider struct {
ProtoValidator
middlewares func() ([]Middleware, error)
}
func (mod ProtoMiddlewareProvider) Middlewares() ([]Middleware, error) {
return mod.middlewares()
}
type ProtoHealthChecker struct {
ProtoValidator
checks func() ([]health.CheckerOption, error)
}
func (mod ProtoHealthChecker) Checks() ([]health.CheckerOption, error) {
return mod.checks()
}
type ProtoLoggerProvider struct {
ProtoModule
logger func(mod gotenberg.Module) (*zap.Logger, error)
}
func (factory ProtoLoggerProvider) Logger(mod gotenberg.Module) (*zap.Logger, error) {
return factory.logger(mod)
}
func TestAPI_Descriptor(t *testing.T) {
descriptor := API{}.Descriptor()
actual := reflect.TypeOf(descriptor.New())
expect := reflect.TypeOf(new(API))
if actual != expect {
t.Errorf("expected '%s' but got '%s'", expect, actual)
}
}
func TestAPI_Provision(t *testing.T) {
for i, tc := range []struct {
ctx *gotenberg.Context
setEnv func(i int)
expectPort int
expectMiddlewares []Middleware
expectErr bool
}{
{
ctx: func() *gotenberg.Context {
fs := new(API).Descriptor().FlagSet
err := fs.Parse([]string{"--api-port-from-env=FOO"})
if err != nil {
t.Fatalf("expected no error but got: %v", err)
}
return gotenberg.NewContext(
gotenberg.ParsedFlags{
FlagSet: fs,
},
nil,
)
}(),
expectErr: true,
},
{
ctx: func() *gotenberg.Context {
fs := new(API).Descriptor().FlagSet
err := fs.Parse([]string{"--api-port-from-env=PORT"})
if err != nil {
t.Fatalf("expected no error but got: %v", err)
}
return gotenberg.NewContext(
gotenberg.ParsedFlags{
FlagSet: fs,
},
nil,
)
}(),
setEnv: func(i int) {
err := os.Setenv("PORT", "")
if err != nil {
t.Fatalf("test %d: expected no error but got: %v", i, err)
}
},
expectErr: true,
},
{
ctx: func() *gotenberg.Context {
fs := new(API).Descriptor().FlagSet
err := fs.Parse([]string{"--api-port-from-env=PORT"})
if err != nil {
t.Fatalf("expected no error but got: %v", err)
}
return gotenberg.NewContext(
gotenberg.ParsedFlags{
FlagSet: fs,
},
nil,
)
}(),
setEnv: func(i int) {
err := os.Setenv("PORT", "foo")
if err != nil {
t.Fatalf("test %d: expected no error but got: %v", i, err)
}
},
expectErr: true,
},
{
ctx: func() *gotenberg.Context {
fs := new(API).Descriptor().FlagSet
err := fs.Parse([]string{"--api-port-from-env=PORT"})
if err != nil {
t.Fatalf("expected no error but got: %v", err)
}
return gotenberg.NewContext(
gotenberg.ParsedFlags{
FlagSet: fs,
},
nil,
)
}(),
setEnv: func(i int) {
err := os.Setenv("PORT", "1337")
if err != nil {
t.Fatalf("test %d: expected no error but got: %v", i, err)
}
},
expectPort: 1337,
expectErr: true,
},
{
ctx: func() *gotenberg.Context {
mod := struct{ ProtoMultipartFormDataRouter }{}
mod.descriptor = func() gotenberg.ModuleDescriptor {
return gotenberg.ModuleDescriptor{ID: "foo", New: func() gotenberg.Module { return mod }}
}
mod.validate = func() error {
return errors.New("foo")
}
mod.routes = func() ([]MultipartFormDataRoute, error) {
return nil, nil
}
return gotenberg.NewContext(
gotenberg.ParsedFlags{
FlagSet: new(API).Descriptor().FlagSet,
},
[]gotenberg.ModuleDescriptor{
mod.Descriptor(),
},
)
}(),
expectErr: true,
},
{
ctx: func() *gotenberg.Context {
mod := struct{ ProtoMiddlewareProvider }{}
mod.descriptor = func() gotenberg.ModuleDescriptor {
return gotenberg.ModuleDescriptor{ID: "foo", New: func() gotenberg.Module { return mod }}
}
mod.validate = func() error {
return errors.New("foo")
}
mod.middlewares = func() ([]Middleware, error) {
return nil, nil
}
return gotenberg.NewContext(
gotenberg.ParsedFlags{
FlagSet: new(API).Descriptor().FlagSet,
},
[]gotenberg.ModuleDescriptor{
mod.Descriptor(),
},
)
}(),
expectErr: true,
},
{
ctx: func() *gotenberg.Context {
mod := struct{ ProtoMiddlewareProvider }{}
mod.descriptor = func() gotenberg.ModuleDescriptor {
return gotenberg.ModuleDescriptor{ID: "foo", New: func() gotenberg.Module { return mod }}
}
mod.validate = func() error {
return nil
}
mod.middlewares = func() ([]Middleware, error) {
return nil, errors.New("foo")
}
return gotenberg.NewContext(
gotenberg.ParsedFlags{
FlagSet: new(API).Descriptor().FlagSet,
},
[]gotenberg.ModuleDescriptor{
mod.Descriptor(),
},
)
}(),
expectErr: true,
},
{
ctx: func() *gotenberg.Context {
mod := struct{ ProtoMultipartFormDataRouter }{}
mod.descriptor = func() gotenberg.ModuleDescriptor {
return gotenberg.ModuleDescriptor{ID: "foo", New: func() gotenberg.Module { return mod }}
}
mod.validate = func() error {
return nil
}
mod.routes = func() ([]MultipartFormDataRoute, error) {
return nil, errors.New("foo")
}
return gotenberg.NewContext(
gotenberg.ParsedFlags{
FlagSet: new(API).Descriptor().FlagSet,
},
[]gotenberg.ModuleDescriptor{
mod.Descriptor(),
},
)
}(),
expectErr: true,
},
{
ctx: func() *gotenberg.Context {
mod := struct{ ProtoHealthChecker }{}
mod.descriptor = func() gotenberg.ModuleDescriptor {
return gotenberg.ModuleDescriptor{ID: "foo", New: func() gotenberg.Module { return mod }}
}
mod.validate = func() error {
return errors.New("foo")
}
mod.checks = func() ([]health.CheckerOption, error) {
return nil, nil
}
return gotenberg.NewContext(
gotenberg.ParsedFlags{
FlagSet: new(API).Descriptor().FlagSet,
},
[]gotenberg.ModuleDescriptor{
mod.Descriptor(),
},
)
}(),
expectErr: true,
},
{
ctx: func() *gotenberg.Context {
mod := struct{ ProtoHealthChecker }{}
mod.descriptor = func() gotenberg.ModuleDescriptor {
return gotenberg.ModuleDescriptor{ID: "foo", New: func() gotenberg.Module { return mod }}
}
mod.validate = func() error {
return nil
}
mod.checks = func() ([]health.CheckerOption, error) {
return nil, errors.New("foo")
}
return gotenberg.NewContext(
gotenberg.ParsedFlags{
FlagSet: new(API).Descriptor().FlagSet,
},
[]gotenberg.ModuleDescriptor{
mod.Descriptor(),
},
)
}(),
expectErr: true,
},
{
ctx: func() *gotenberg.Context {
return gotenberg.NewContext(
gotenberg.ParsedFlags{
FlagSet: new(API).Descriptor().FlagSet,
},
[]gotenberg.ModuleDescriptor{},
)
}(),
expectErr: true,
},
{
ctx: func() *gotenberg.Context {
mod := struct{ ProtoLoggerProvider }{}
mod.descriptor = func() gotenberg.ModuleDescriptor {
return gotenberg.ModuleDescriptor{ID: "foo", New: func() gotenberg.Module { return mod }}
}
mod.logger = func(_ gotenberg.Module) (*zap.Logger, error) {
return nil, errors.New("foo")
}
return gotenberg.NewContext(
gotenberg.ParsedFlags{
FlagSet: new(API).Descriptor().FlagSet,
},
[]gotenberg.ModuleDescriptor{
mod.Descriptor(),
},
)
}(),
expectErr: true,
},
{
ctx: func() *gotenberg.Context {
mod1 := struct{ ProtoMultipartFormDataRouter }{}
mod1.descriptor = func() gotenberg.ModuleDescriptor {
return gotenberg.ModuleDescriptor{ID: "foo", New: func() gotenberg.Module { return mod1 }}
}
mod1.validate = func() error {
return nil
}
mod1.routes = func() ([]MultipartFormDataRoute, error) {
return []MultipartFormDataRoute{{}}, nil
}
mod2 := struct{ ProtoMiddlewareProvider }{}
mod2.descriptor = func() gotenberg.ModuleDescriptor {
return gotenberg.ModuleDescriptor{ID: "bar", New: func() gotenberg.Module { return mod2 }}
}
mod2.validate = func() error {
return nil
}
mod2.middlewares = func() ([]Middleware, error) {
return []Middleware{
{
Priority: VeryLowPriority,
},
{
Priority: LowPriority,
},
{
Priority: MediumPriority,
},
{
Priority: HighPriority,
},
{
Priority: VeryHighPriority,
},
}, nil
}
mod3 := struct{ ProtoHealthChecker }{}
mod3.descriptor = func() gotenberg.ModuleDescriptor {
return gotenberg.ModuleDescriptor{ID: "baz", New: func() gotenberg.Module { return mod3 }}
}
mod3.validate = func() error {
return nil
}
mod3.checks = func() ([]health.CheckerOption, error) {
return []health.CheckerOption{health.WithDisabledAutostart()}, nil
}
mod4 := struct{ ProtoLoggerProvider }{}
mod4.descriptor = func() gotenberg.ModuleDescriptor {
return gotenberg.ModuleDescriptor{ID: "qux", New: func() gotenberg.Module { return mod4 }}
}
mod4.logger = func(_ gotenberg.Module) (*zap.Logger, error) {
return zap.NewNop(), nil
}
return gotenberg.NewContext(
gotenberg.ParsedFlags{
FlagSet: new(API).Descriptor().FlagSet,
},
[]gotenberg.ModuleDescriptor{
mod1.Descriptor(),
mod2.Descriptor(),
mod3.Descriptor(),
mod4.Descriptor(),
},
)
}(),
expectMiddlewares: []Middleware{
{
Priority: VeryHighPriority,
},
{
Priority: HighPriority,
},
{
Priority: MediumPriority,
},
{
Priority: LowPriority,
},
{
Priority: VeryLowPriority,
},
},
},
} {
if tc.setEnv != nil {
tc.setEnv(i)
}
mod := new(API)
err := mod.Provision(tc.ctx)
if tc.expectPort != 0 && mod.port != tc.expectPort {
t.Errorf("expected port %d but got %d", tc.expectPort, mod.port)
}
if !reflect.DeepEqual(mod.externalMiddlewares, tc.expectMiddlewares) {
t.Errorf("expected %+v, but got: %+v", tc.expectMiddlewares, mod.externalMiddlewares)
}
if tc.expectErr && err == nil {
t.Errorf("test %d: expected error but got: %v", i, err)
}
if !tc.expectErr && err != nil {
t.Errorf("test %d: expected no error but got: %v", i, err)
}
}
}
func TestAPI_Validate(t *testing.T) {
for i, tc := range []struct {
port int
rootPath string
traceHeader string
routes []MultipartFormDataRoute
middlewares []Middleware
expectErr bool
}{
{
port: 0,
expectErr: true,
},
{
port: 65536,
rootPath: "foo",
expectErr: true,
},
{
port: 10,
rootPath: "/foo/",
traceHeader: "foo",
routes: []MultipartFormDataRoute{
{
Path: "",
},
},
expectErr: true,
},
{
port: 10,
rootPath: "/foo/",
traceHeader: "foo",
routes: []MultipartFormDataRoute{
{
Path: "foo",
},
},
expectErr: true,
},
{
port: 10,
rootPath: "/foo/",
traceHeader: "foo",
routes: []MultipartFormDataRoute{
{
Path: "/foo",
},
},
expectErr: true,
},
{
port: 10,
rootPath: "/foo/",
traceHeader: "foo",
routes: []MultipartFormDataRoute{
{
Path: "/foo",
Handler: func(_ *Context) error { return nil },
},
{
Path: "/foo",
Handler: func(_ *Context) error { return nil },
},
},
expectErr: true,
},
{
port: 10,
rootPath: "/foo/",
traceHeader: "foo",
middlewares: []Middleware{
{
Priority: HighPriority,
},
},
expectErr: true,
},
{
port: 10,
rootPath: "/foo/",
traceHeader: "foo",
routes: []MultipartFormDataRoute{
{
Path: "/foo",
Handler: func(_ *Context) error { return nil },
},
},
middlewares: []Middleware{
{
Priority: HighPriority,
Handler: func() echo.MiddlewareFunc {
return func(next echo.HandlerFunc) echo.HandlerFunc {
return func(c echo.Context) error {
return next(c)
}
}
}(),
},
},
},
} {
mod := API{
port: tc.port,
rootPath: tc.rootPath,
traceHeader: tc.traceHeader,
multipartFormDataRoutes: tc.routes,
externalMiddlewares: tc.middlewares,
}
err := mod.Validate()
if tc.expectErr && err == nil {
t.Errorf("test %d: expected error but got: %v", i, err)
}
if !tc.expectErr && err != nil {
t.Errorf("test %d: expected no error but got: %v", i, err)
}
}
}
func TestAPI_Start(t *testing.T) {
mod := new(API)
mod.port = 3000
mod.rootPath = "/"
mod.multipartFormDataRoutes = []MultipartFormDataRoute{
{
Path: "/foo",
Handler: func(ctx *Context) error {
ctx.outputPaths = []string{
"/tests/test/testdata/api/sample1.txt",
}
return nil
},
},
{
Path: "/bar",
Handler: func(_ *Context) error { return errors.New("foo") },
},
}
mod.externalMiddlewares = []Middleware{
{
RunBeforeRouter: true,
Handler: func() echo.MiddlewareFunc {
return func(next echo.HandlerFunc) echo.HandlerFunc {
return func(c echo.Context) error {
return next(c)
}
}
}(),
},
{
Handler: func() echo.MiddlewareFunc {
return func(next echo.HandlerFunc) echo.HandlerFunc {
return func(c echo.Context) error {
return next(c)
}
}
}(),
},
}
mod.logger = zap.NewNop()
err := mod.Start()
if err != nil {
t.Fatalf("expected no error but got: %v", err)
}
// health request.
recorder := httptest.NewRecorder()
healthRequest := httptest.NewRequest(http.MethodGet, "/health", nil)
mod.srv.ServeHTTP(recorder, healthRequest)
if recorder.Code != http.StatusOK {
t.Errorf("expected %d status code but got %d", http.StatusOK, recorder.Code)
}
// "multipart/form-data" request.
multipartRequest := func(URL string) *http.Request {
body := &bytes.Buffer{}
writer := multipart.NewWriter(body)
defer func() {
err := writer.Close()
if err != nil {
t.Fatalf("expected no error but got: %v", err)
}
}()
err := writer.WriteField("foo", "foo")
if err != nil {
t.Fatalf("expected no error but got: %v", err)
}
part, err := writer.CreateFormFile("foo.txt", "foo.txt")
if err != nil {
t.Fatalf("expected no error but got: %v", err)
}
_, err = part.Write([]byte("foo"))
if err != nil {
t.Fatalf("expected no error but got: %v", err)
}
req := httptest.NewRequest(http.MethodPost, URL, body)
req.Header.Set(echo.HeaderContentType, writer.FormDataContentType())
return req
}
recorder = httptest.NewRecorder()
mod.srv.ServeHTTP(recorder, multipartRequest("/forms/foo"))
if recorder.Code != http.StatusOK {
t.Errorf("expected %d status code but got %d", http.StatusOK, recorder.Code)
}
recorder = httptest.NewRecorder()
mod.srv.ServeHTTP(recorder, multipartRequest("/forms/bar"))
if recorder.Code != http.StatusInternalServerError {
t.Errorf("expected %d status code but got %d", http.StatusInternalServerError, recorder.Code)
}
err = mod.srv.Shutdown(context.TODO())
if err != nil {
t.Errorf("expected no error but got: %v", err)
}
}
func TestAPI_StartupMessage(t *testing.T) {
mod := API{
port: 3000,
}
actual := mod.StartupMessage()
expect := "server listening on port 3000"
if actual != expect {
t.Errorf("expected '%s' but got '%s'", expect, actual)
}
}
func TestAPI_Stop(t *testing.T) {
mod := API{
port: 3000,
multipartFormDataRoutes: []MultipartFormDataRoute{
{
Path: "/foo",
Handler: func(_ *Context) error { return nil },
},
},
logger: zap.NewNop(),
}
err := mod.Start()
if err != nil {
t.Fatalf("expected no error but got: %v", err)
}
err = mod.Stop(context.TODO())
if err != nil {
t.Errorf("expected no error but got: %v", err)
}
}
func TestAPI_GraceDuration(t *testing.T) {
for i, tc := range []struct {
mod API
expect time.Duration
}{
{
mod: API{
readTimeout: time.Duration(1) * time.Second,
processTimeout: time.Duration(1) * time.Second,
writeTimeout: time.Duration(1) * time.Second,
disableWebhook: true,
},
expect: time.Duration(3) * time.Second,
},
{
mod: API{
readTimeout: time.Duration(1) * time.Second,
processTimeout: time.Duration(1) * time.Second,
writeTimeout: time.Duration(1) * time.Second,
webhookMaxRetry: 5,
webhookRetryMaxWait: time.Duration(5) * time.Second,
},
expect: time.Duration(28) * time.Second,
},
} {
actual := tc.mod.GraceDuration()
if actual != tc.expect {
t.Errorf("test %d: expected '%s' but got '%s'", i, tc.expect, actual)
}
}
}
// Interface guards.
var (
_ gotenberg.Module = (*ProtoModule)(nil)
_ gotenberg.Validator = (*ProtoValidator)(nil)
_ gotenberg.Module = (*ProtoValidator)(nil)
_ MultipartFormDataRouter = (*ProtoMultipartFormDataRouter)(nil)
_ gotenberg.Module = (*ProtoMultipartFormDataRouter)(nil)
_ gotenberg.Validator = (*ProtoMultipartFormDataRouter)(nil)
_ MiddlewareProvider = (*ProtoMiddlewareProvider)(nil)
_ gotenberg.Module = (*ProtoMiddlewareProvider)(nil)
_ gotenberg.Validator = (*ProtoMiddlewareProvider)(nil)
_ HealthChecker = (*ProtoHealthChecker)(nil)
_ gotenberg.Module = (*ProtoHealthChecker)(nil)
_ gotenberg.Validator = (*ProtoHealthChecker)(nil)
_ gotenberg.LoggerProvider = (*ProtoLoggerProvider)(nil)
_ gotenberg.Module = (*ProtoLoggerProvider)(nil)
)

319
pkg/modules/api/context.go Normal file
View File

@@ -0,0 +1,319 @@
package api
import (
"compress/flate"
"context"
"errors"
"fmt"
"io"
"mime/multipart"
"net/http"
"os"
"path/filepath"
"strings"
"time"
"unicode"
"github.com/google/uuid"
"github.com/gotenberg/gotenberg/v7/pkg/gotenberg"
"github.com/labstack/echo/v4"
"github.com/mholt/archiver/v3"
"go.uber.org/zap"
"golang.org/x/text/runes"
"golang.org/x/text/transform"
"golang.org/x/text/unicode/norm"
)
var (
// ErrContextAlreadyClosed happens when the context has been canceled.
ErrContextAlreadyClosed = errors.New("context already closed")
// ErrOutOfBoundsOutputPath happens when an output path is not within
// context's working directory. It enforces having all the files in the
// same directory.
ErrOutOfBoundsOutputPath = errors.New("output path is not within context's working directory")
)
// Context is the request context for a "multipart/form-data" requests.
type Context struct {
dirPath string
values map[string][]string
files map[string]string
outputPaths []string
cancelled bool
logger *zap.Logger
echoCtx echo.Context
context.Context
}
// newContext returns a Context by parsing a "multipart/form-data" request.
func newContext(echoCtx echo.Context, logger *zap.Logger, timeout time.Duration) (*Context, context.CancelFunc, error) {
processCtx, processCancel := context.WithTimeout(context.Background(), timeout)
ctx := &Context{
outputPaths: make([]string, 0),
cancelled: false,
logger: logger,
echoCtx: echoCtx,
Context: processCtx,
}
// A custom cancel function which removes the context's working directory
// when called.
cancel := func() context.CancelFunc {
return func() {
if ctx.cancelled {
return
}
processCancel()
if ctx.dirPath == "" {
return
}
err := os.RemoveAll(ctx.dirPath)
if err != nil {
ctx.logger.Error(fmt.Sprintf("remove context's working directory: %s", err))
return
}
ctx.logger.Debug(fmt.Sprintf("'%s' removed", ctx.dirPath))
ctx.cancelled = true
}
}()
form, err := echoCtx.MultipartForm()
if err != nil {
if errors.Is(err, http.ErrNotMultipart) {
return nil, cancel, WrapError(
fmt.Errorf("get multipart form: %w", err),
NewSentinelHTTPError(http.StatusUnsupportedMediaType, "Invalid 'Content-Type' header value: want 'multipart/form-data'"),
)
}
if errors.Is(err, http.ErrMissingBoundary) {
return nil, cancel, WrapError(
fmt.Errorf("get multipart form: %w", err),
NewSentinelHTTPError(http.StatusUnsupportedMediaType, "Invalid 'Content-Type' header value: no boundary"),
)
}
if strings.Contains(err.Error(), io.EOF.Error()) {
return nil, cancel, WrapError(
fmt.Errorf("get multipart form: %w", err),
NewSentinelHTTPError(http.StatusBadRequest, "Malformed body: it does not match the 'Content-Type' header boundaries"),
)
}
return nil, cancel, fmt.Errorf("get multipart form: %w", err)
}
dirPath, err := gotenberg.MkdirAll()
if err != nil {
return nil, cancel, fmt.Errorf("create working directory: %w", err)
}
ctx.dirPath = dirPath
ctx.values = form.Value
ctx.files = make(map[string]string)
copyToDisk := func(fh *multipart.FileHeader) error {
// Avoid directory traversal and normalize filename.
// See https://github.com/thecodingmachine/gotenberg/issues/104.
// See https://github.com/thecodingmachine/gotenberg/issues/228.
t := transform.Chain(norm.NFD, runes.Remove(runes.In(unicode.Mn)), norm.NFC)
filename, _, err := transform.String(t, strings.ToLower(filepath.Base(fh.Filename)))
if err != nil {
return fmt.Errorf("transform filename: %w", err)
}
in, err := fh.Open()
if err != nil {
return fmt.Errorf("open multipart file: %w", err)
}
defer func() {
err := in.Close()
if err != nil {
logger.Error(fmt.Sprintf("close file header: %s", err))
}
}()
path := fmt.Sprintf("%s/%s", ctx.dirPath, filename)
out, err := os.Create(path)
if err != nil {
return fmt.Errorf("create local file: %w", err)
}
defer func() {
err := out.Close()
if err != nil {
logger.Error(fmt.Sprintf("close local file: %s", err))
}
}()
_, err = io.Copy(out, in)
if err != nil {
return fmt.Errorf("copy multipart file to local file: %w", err)
}
ctx.files[filename] = path
return nil
}
for _, files := range form.File {
for _, fh := range files {
err = copyToDisk(fh)
if err != nil {
return ctx, cancel, fmt.Errorf("copy to disk: %w", err)
}
}
}
ctx.Log().Debug(fmt.Sprintf("form data values: %+v", ctx.values))
ctx.Log().Debug(fmt.Sprintf("form data files: %+v", ctx.files))
return ctx, cancel, err
}
// Request returns the http.Request.
func (ctx Context) Request() *http.Request {
return ctx.echoCtx.Request()
}
// FormData return a FormData.
func (ctx Context) FormData() *FormData {
return &FormData{
values: ctx.values,
files: ctx.files,
errors: nil,
}
}
// GeneratePath generates a path within the context's working directory. It
// does not create a file.
func (ctx Context) GeneratePath(extension string) string {
return fmt.Sprintf("%s/%s%s", ctx.dirPath, uuid.New(), extension)
}
// AddOutputPaths adds the given paths. Those paths will be used later to build
// the output file.
func (ctx *Context) AddOutputPaths(paths ...string) error {
if ctx.cancelled {
return ErrContextAlreadyClosed
}
for _, path := range paths {
if !strings.HasPrefix(path, ctx.dirPath) {
return ErrOutOfBoundsOutputPath
}
ctx.outputPaths = append(ctx.outputPaths, path)
}
return nil
}
// Log returns the context zap.Logger.
func (ctx Context) Log() *zap.Logger {
return ctx.logger
}
// buildOutputFile builds the output file according to the output paths
// registered in the context. If many output paths, an archive is created.
func (ctx Context) buildOutputFile() (string, error) {
if ctx.cancelled {
return "", ErrContextAlreadyClosed
}
if len(ctx.outputPaths) == 0 {
return "", errors.New("no output path")
}
if len(ctx.outputPaths) == 1 {
ctx.logger.Debug(fmt.Sprintf("only one output file '%s', skip archive creation", ctx.outputPaths[0]))
return ctx.outputPaths[0], nil
}
z := archiver.Zip{
CompressionLevel: flate.DefaultCompression,
MkdirAll: true,
SelectiveCompression: true,
ContinueOnError: false,
OverwriteExisting: false,
ImplicitTopLevelFolder: false,
}
archivePath := ctx.GeneratePath(".zip")
err := z.Archive(ctx.outputPaths, archivePath)
if err != nil {
return "", fmt.Errorf("archive output files: %w", err)
}
ctx.logger.Debug(fmt.Sprintf("archive '%s' created", archivePath))
return archivePath, nil
}
// MockContext is a helper for tests.
//
// ctx := &api.MockContext{Context: &api.Context{}}
type MockContext struct {
*Context
}
// SetDirPath sets the context's working directory path.
//
// ctx := &api.MockContext{Context: &api.Context{}}
// ctx.SetDirPath("/foo")
func (ctx *MockContext) SetDirPath(path string) {
ctx.dirPath = path
}
// SetValues sets the values.
//
// ctx := &api.MockContext{Context: &api.Context{}}
// ctx.SetValues(map[string][]string{
// "url": {
// "foo",
// },
// })
func (ctx *MockContext) SetValues(values map[string][]string) {
ctx.values = values
}
// SetFiles sets the files.
//
// ctx := &api.MockContext{Context: &api.Context{}}
// ctx.SetFiles(map[string]string{
// "foo": "/foo",
// })
func (ctx *MockContext) SetFiles(files map[string]string) {
ctx.files = files
}
// SetCancelled sets if the context is cancelled or not.
//
// ctx := &api.MockContext{Context: &api.Context{}}
// ctx.SetCancelled(true)
func (ctx *MockContext) SetCancelled(cancelled bool) {
ctx.cancelled = cancelled
}
// OutputPaths returns the registered output paths.
// ctx := &api.MockContext{Context: &api.Context{}}
// outputPaths := ctx.OutputPaths()
func (ctx MockContext) OutputPaths() []string {
return ctx.outputPaths
}

373
pkg/modules/api/context_test.go Normal file
View File

@@ -0,0 +1,373 @@
package api
import (
"bytes"
"errors"
"mime/multipart"
"net/http"
"net/http/httptest"
"os"
"reflect"
"strings"
"testing"
"time"
"github.com/gotenberg/gotenberg/v7/pkg/gotenberg"
"github.com/labstack/echo/v4"
"go.uber.org/zap"
)
func TestNewContext(t *testing.T) {
for i, tc := range []struct {
request *http.Request
expectErr bool
expectHTTPErr bool
expectHTTPStatus int
}{
{
request: httptest.NewRequest(http.MethodPost, "/", nil),
expectErr: true,
expectHTTPErr: true,
expectHTTPStatus: http.StatusUnsupportedMediaType,
},
{
request: func() *http.Request {
req := httptest.NewRequest(http.MethodPost, "/", nil)
req.Header.Set(echo.HeaderContentType, echo.MIMEMultipartForm)
return req
}(),
expectErr: true,
expectHTTPErr: true,
expectHTTPStatus: http.StatusUnsupportedMediaType,
},
{
request: func() *http.Request {
body := &bytes.Buffer{}
writer := multipart.NewWriter(body)
defer func() {
err := writer.Close()
if err != nil {
t.Fatalf("expected no error but got: %v", err)
}
}()
err := writer.WriteField("foo", "foo")
if err != nil {
t.Fatalf("expected no error but got: %v", err)
}
req := httptest.NewRequest(http.MethodPost, "/", nil)
req.Header.Set(echo.HeaderContentType, writer.FormDataContentType())
return req
}(),
expectErr: true,
expectHTTPErr: true,
expectHTTPStatus: http.StatusBadRequest,
},
{
request: func() *http.Request {
body := &bytes.Buffer{}
writer := multipart.NewWriter(body)
defer func() {
err := writer.Close()
if err != nil {
t.Fatalf("expected no error but got: %v", err)
}
}()
err := writer.WriteField("foo", "foo")
if err != nil {
t.Fatalf("expected no error but got: %v", err)
}
part, err := writer.CreateFormFile("foo.txt", "foo.txt")
if err != nil {
t.Fatalf("expected no error but got: %v", err)
}
_, err = part.Write([]byte("foo"))
if err != nil {
t.Fatalf("expected no error but got: %v", err)
}
req := httptest.NewRequest(http.MethodPost, "/", body)
req.Header.Set(echo.HeaderContentType, writer.FormDataContentType())
return req
}(),
},
} {
handler := func(c echo.Context) error {
_, cancel, err := newContext(c, zap.NewNop(), time.Duration(10)*time.Second)
defer cancel()
// Context already cancelled.
defer cancel()
if err != nil {
return err
}
return nil
}
recorder := httptest.NewRecorder()
srv := echo.New()
srv.HideBanner = true
srv.HidePort = true
c := srv.NewContext(tc.request, recorder)
err := handler(c)
if tc.expectErr && err == nil {
t.Errorf("test %d: expected error but got: %v", i, err)
}
if !tc.expectErr && err != nil {
t.Errorf("test %d: expected no error but got: %v", i, err)
}
var httpErr HTTPError
isHTTPErr := errors.As(err, &httpErr)
if tc.expectHTTPErr && !isHTTPErr {
t.Errorf("test %d: expected HTTP error but got: %v", i, err)
}
if !tc.expectHTTPErr && isHTTPErr {
t.Errorf("test %d: expected no HTTP error but got one: %v", i, httpErr)
}
if err != nil && tc.expectHTTPErr && isHTTPErr {
status, _ := httpErr.HTTPError()
if status != tc.expectHTTPStatus {
t.Errorf("test %d: expected %d HTTP status code but got %d", i, tc.expectHTTPStatus, status)
}
}
}
}
func TestContext_Request(t *testing.T) {
request := httptest.NewRequest(http.MethodPost, "/", nil)
recorder := httptest.NewRecorder()
c := echo.New().NewContext(request, recorder)
ctx := Context{
echoCtx: c,
}
if !reflect.DeepEqual(ctx.Request(), c.Request()) {
t.Errorf("expected %v but got %v", ctx.Request(), c.Request())
}
}
func TestContext_FormData(t *testing.T) {
ctx := Context{
values: map[string][]string{
"foo": {"foo"},
},
files: map[string]string{
"foo.txt": "/foo.txt",
},
}
actual := ctx.FormData()
expect := &FormData{
values: ctx.values,
files: ctx.files,
}
if !reflect.DeepEqual(actual, expect) {
t.Errorf("expected %+v but got %+v", expect, actual)
}
}
func TestContext_GeneratePath(t *testing.T) {
ctx := Context{
dirPath: "/foo",
}
path := ctx.GeneratePath(".pdf")
if !strings.HasPrefix(path, ctx.dirPath) {
t.Errorf("expected '%s' to start with '%s'", path, ctx.dirPath)
}
}
func TestContext_AddOutputPaths(t *testing.T) {
for i, tc := range []struct {
ctx *Context
path string
expectCount int
expectErr bool
}{
{
ctx: &Context{cancelled: true},
expectErr: true,
},
{
ctx: &Context{dirPath: "/foo"},
path: "/bar/foo.txt",
expectErr: true,
},
{
ctx: &Context{dirPath: "/foo"},
path: "/foo/foo.txt",
expectCount: 1,
},
} {
err := tc.ctx.AddOutputPaths(tc.path)
if tc.expectErr && err == nil {
t.Errorf("test %d: expected error but got: %v", i, err)
}
if !tc.expectErr && err != nil {
t.Errorf("test %d: expected no error but got: %v", i, err)
}
if len(tc.ctx.outputPaths) != tc.expectCount {
t.Errorf("test %d: expected %d output paths but got %d", i, tc.expectCount, len(tc.ctx.outputPaths))
}
}
}
func TestContext_Log(t *testing.T) {
expect := zap.NewNop()
ctx := Context{logger: expect}
actual := ctx.Log()
if !reflect.DeepEqual(actual, expect) {
t.Errorf("expected %v but got %v", expect, actual)
}
}
func TestContext_buildOutputFile(t *testing.T) {
for i, tc := range []struct {
ctx *Context
expectErr bool
}{
{
ctx: &Context{cancelled: true},
expectErr: true,
},
{
ctx: &Context{},
expectErr: true,
},
{
ctx: &Context{outputPaths: []string{"foo.txt"}},
},
{
ctx: &Context{outputPaths: []string{"foo.txt", "foo.pdf"}},
expectErr: true,
},
{
ctx: &Context{
outputPaths: []string{
"/tests/test/testdata/api/sample1.txt",
"/tests/test/testdata/api/sample1.txt",
},
},
},
} {
dirPath, err := gotenberg.MkdirAll()
if err != nil {
t.Fatalf("%d: expected no erro but got: %v", i, err)
}
tc.ctx.dirPath = dirPath
tc.ctx.logger = zap.NewNop()
_, err = tc.ctx.buildOutputFile()
if tc.expectErr && err == nil {
t.Errorf("test %d: expected error but got: %v", i, err)
}
if !tc.expectErr && err != nil {
t.Errorf("test %d: expected no error but got: %v", i, err)
}
err = os.RemoveAll(dirPath)
if err != nil {
t.Fatalf("%d: expected no erro but got: %v", i, err)
}
}
}
func TestMockContext_SetDirPath(t *testing.T) {
mock := &MockContext{&Context{}}
mock.SetDirPath("/foo")
actual := mock.dirPath
expect := "/foo"
if actual != expect {
t.Errorf("expected '%s' but got '%s'", expect, actual)
}
}
func TestMockContext_SetValues(t *testing.T) {
mock := &MockContext{&Context{}}
mock.SetValues(map[string][]string{
"foo": {"foo"},
})
actual := mock.values
expect := map[string][]string{
"foo": {"foo"},
}
if !reflect.DeepEqual(actual, expect) {
t.Errorf("expected %+v but got: %+v", expect, actual)
}
}
func TestMockContext_SetFiles(t *testing.T) {
mock := &MockContext{&Context{}}
mock.SetFiles(map[string]string{
"foo": "/foo",
})
actual := mock.files
expect := map[string]string{
"foo": "/foo",
}
if !reflect.DeepEqual(actual, expect) {
t.Errorf("expected %+v but got: %+v", expect, actual)
}
}
func TestMockContext_SetCancelled(t *testing.T) {
mock := &MockContext{&Context{}}
mock.SetCancelled(true)
actual := mock.cancelled
if !actual {
t.Errorf("expected %t but got %t", true, actual)
}
}
func TestMockContext_OutputPaths(t *testing.T) {
mock := MockContext{
&Context{
outputPaths: []string{"/foo"},
},
}
actual := mock.OutputPaths()
expect := []string{"/foo"}
if !reflect.DeepEqual(actual, expect) {
t.Errorf("expected %+v but got: %+v", expect, actual)
}
}

3
pkg/modules/api/doc.go Normal file
View File

@@ -0,0 +1,3 @@
// Package api provides a module which is an HTTP server. Other modules may
// add multipart/form-data routes, middlewares, and health checks.
package api

77
pkg/modules/api/errors.go Normal file
View File

@@ -0,0 +1,77 @@
package api
// Credits: https://www.joeshaw.org/error-handling-in-go-http-applications.
// HTTPError is an interface allowing to retrieve the HTTP details of an error.
type HTTPError interface {
HTTPError() (int, string)
}
// SentinelHTTPError is the HTTP sidekick of an error.
type SentinelHTTPError struct {
status int
message string
}
// NewSentinelHTTPError creates a SentinelHTTPError. The message will be sent
// as the response's body if returned from an handler, so make sure to not leak
// sensible information.
func NewSentinelHTTPError(status int, message string) SentinelHTTPError {
return SentinelHTTPError{
status: status,
message: message,
}
}
// Error returns the message.
func (err SentinelHTTPError) Error() string {
return err.message
}
// HTTPError returns the status and message.
func (err SentinelHTTPError) HTTPError() (int, string) {
return err.status, err.message
}
// sentinelWrappedError contains both the error which will logged and the
// sidekick SentinelHTTPError.
type sentinelWrappedError struct {
error
sentinel SentinelHTTPError
}
func (w sentinelWrappedError) Is(err error) bool {
return w.sentinel == err
}
func (w sentinelWrappedError) HTTPError() (int, string) {
return w.sentinel.HTTPError()
}
// WrapError wraps the given error with a SentinelHTTPError. The wrapped error
// will be displayed in a log, while the SentinelHTTPError will be sent in the
// response.
//
// return api.WrapError(
// // This first error will be logged.
// fmt.Errorf("my action: %w", err),
// // The HTTP error will be sent as a response.
// api.NewSentinelHTTPError(
// http.StatusForbidden,
// "Hey, you did something wrong!"
// ),
// )
func WrapError(err error, sentinel SentinelHTTPError) error {
return sentinelWrappedError{
error: err,
sentinel: sentinel,
}
}
// Interface guards.
var (
_ error = (*SentinelHTTPError)(nil)
_ HTTPError = (*SentinelHTTPError)(nil)
_ error = (*sentinelWrappedError)(nil)
_ HTTPError = (*sentinelWrappedError)(nil)
)

108
pkg/modules/api/errors_test.go Normal file
View File

@@ -0,0 +1,108 @@
package api
import (
"errors"
"net/http"
"reflect"
"testing"
)
func TestNewSentinelHTTPError(t *testing.T) {
actual := NewSentinelHTTPError(http.StatusInternalServerError, "foo")
expect := SentinelHTTPError{
status: http.StatusInternalServerError,
message: "foo",
}
if !reflect.DeepEqual(actual, expect) {
t.Errorf("expected %v but got %v", expect, actual)
}
}
func TestSentinelHTTPError_Error(t *testing.T) {
err := SentinelHTTPError{
message: "foo",
}
actual := err.Error()
expect := "foo"
if actual != expect {
t.Errorf("expected '%s' but got '%s'", expect, actual)
}
}
func TestSentinelHTTPError_HTTPError(t *testing.T) {
actualStatus, actualMessage := SentinelHTTPError{
status: http.StatusInternalServerError,
message: "foo",
}.HTTPError()
expectStatus := http.StatusInternalServerError
expectMessage := "foo"
if actualStatus != expectStatus {
t.Errorf("expected %d but got %d", expectStatus, actualStatus)
}
if actualMessage != expectMessage {
t.Errorf("expected '%s' but got '%s'", expectMessage, actualMessage)
}
}
func TestSentinelWrappedError_Is(t *testing.T) {
errSentinel := SentinelHTTPError{}
err := sentinelWrappedError{
error: errors.New("foo"),
sentinel: errSentinel,
}
if !err.Is(errSentinel) {
t.Error("expected true")
}
}
func TestSentinelWrappedError_HTTPError(t *testing.T) {
expectStatus, expectMessage := SentinelHTTPError{
status: http.StatusInternalServerError,
message: "foo",
}.HTTPError()
actualStatus, actualMessage := sentinelWrappedError{
error: errors.New("foo"),
sentinel: SentinelHTTPError{
status: http.StatusInternalServerError,
message: "foo",
},
}.HTTPError()
if actualStatus != expectStatus {
t.Errorf("expected %d but got %d", expectStatus, actualStatus)
}
if actualMessage != expectMessage {
t.Errorf("expected '%s' but got '%s'", expectMessage, actualMessage)
}
}
func TestWrapError(t *testing.T) {
errFoo := errors.New("foo")
expect := sentinelWrappedError{
error: errFoo,
sentinel: SentinelHTTPError{
status: http.StatusInternalServerError,
message: "foo",
},
}
actual := WrapError(errFoo, SentinelHTTPError{
status: http.StatusInternalServerError,
message: "foo",
})
if !reflect.DeepEqual(actual, expect) {
t.Errorf("expected %v but got %v", expect, actual)
}
}

439
pkg/modules/api/formdata.go Normal file
View File

@@ -0,0 +1,439 @@
package api
import (
"fmt"
"io/ioutil"
"net/http"
"path/filepath"
"sort"
"strconv"
"strings"
"time"
"go.uber.org/multierr"
)
// FormData is a helper for validating and hydrating values from a
// "multipart/form-data" request.
//
// form := ctx.FormData()
type FormData struct {
values map[string][]string
files map[string]string
errors error
}
// Validate returns nil or an error related to the FormData values, with a
// SentinelHTTPError (status code 400, errors' details as message) wrapped
// inside.
//
// var foo string
//
// err := ctx.FormData().
// MandatoryString("foo", &foo, "bar").
// Validate()
func (form FormData) Validate() error {
if form.errors == nil {
return nil
}
return WrapError(
form.errors,
NewSentinelHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid form data: %s", form.errors)),
)
}
// String binds a form data value to a string variable.
//
// var foo string
//
// ctx.FormData().String("foo", &foo, "bar")
func (form *FormData) String(key string, target *string, defaultValue string) *FormData {
return form.mustValue(key, target, defaultValue)
}
// MandatoryString binds a form data value to a string variable. It populates
// an error if the value is empty or the "key" does not exist.
//
// var foo string
//
// ctx.FormData().MandatoryString("foo", &foo)
func (form *FormData) MandatoryString(key string, target *string) *FormData {
return form.mustMandatoryValue(key, target)
}
// Bool binds a form data value to a bool variable. It populates an error if
// the value is not bool.
//
// var foo bool
//
// ctx.FormData().Bool("foo", &foo, true)
func (form *FormData) Bool(key string, target *bool, defaultValue bool) *FormData {
return form.mustValue(key, target, defaultValue)
}
// MandatoryBool binds a form data value to a bool variable. It populates an
// error if the value is not bool, is empty, or the "key" does not exist.
//
// var foo bool
//
// ctx.FormData().MandatoryBool("foo", &foo)
func (form *FormData) MandatoryBool(key string, target *bool) *FormData {
return form.mustMandatoryValue(key, target)
}
// Int binds a form data value to an int variable. It populates an error if the
// value is not int.
//
// var foo int
//
// ctx.FormData().Int("foo", &foo, 2)
func (form *FormData) Int(key string, target *int, defaultValue int) *FormData {
return form.mustValue(key, target, defaultValue)
}
// MandatoryInt binds a form data value to an int variable. It populates an
// error if the value is not int, is empty, or the "key" does not exist.
//
// var foo int
//
// ctx.FormData().MandatoryInt("foo", &foo)
func (form *FormData) MandatoryInt(key string, target *int) *FormData {
return form.mustMandatoryValue(key, target)
}
// Float64 binds a form data value to a float64 variable. It populates an error
// if the value is not float64.
//
// var foo float64
//
// ctx.FormData().Float64("foo", &foo, 2.0)
func (form *FormData) Float64(key string, target *float64, defaultValue float64) *FormData {
return form.mustValue(key, target, defaultValue)
}
// MandatoryFloat64 binds a form data value to a float64 variable. It populates
// an error if the is not float64, is empty, or the "key" does not exist.
//
// var foo float64
//
// ctx.FormData().MandatoryFloat64("foo", &foo)
func (form *FormData) MandatoryFloat64(key string, target *float64) *FormData {
return form.mustMandatoryValue(key, target)
}
// Duration binds a form data value to a time.Duration variable. It populates
// an error if the form data value is not time.Duration.
//
// var foo time.Duration
//
// ctx.FormData().Duration("foo", &foo, time.Duration(2) * time.Second)
func (form *FormData) Duration(key string, target *time.Duration, defaultValue time.Duration) *FormData {
return form.mustValue(key, target, defaultValue)
}
// MandatoryDuration binds a form data value to a time.Duration variable. It
// populates an error if the value is not time.Duration, is empty, or the "key"
// does not exist.
//
// var foo time.Duration
//
// ctx.FormData().MandatoryDuration("foo", &foo)
func (form *FormData) MandatoryDuration(key string, target *time.Duration) *FormData {
return form.mustMandatoryValue(key, target)
}
// Custom helps to define a custom binding function for a form data value.
//
// var foo map[string]string
//
// ctx.FormData().Custom("foo", func(value string) error {
// if value == "" {
// foo = "bar"
//
// return nil
// }
//
// err := json.Unmarshal([]byte(value), &foo)
// if err != nil {
// return fmt.Errorf("unmarshal foo: %w", err)
// }
//
// return nil
// })
func (form *FormData) Custom(key string, assign func(value string) error) *FormData {
var value string
form.mustValue(key, &value, "")
err := assign(value)
if err != nil {
form.append(
fmt.Errorf("form value '%s' is invalid (got '%s', resulting to %w)", key, value, err),
)
}
return form
}
// MandatoryCustom helps to define a custom binding function for a form data
// value. It populates an error if the value is empty or the "key" does not
// exist.
//
// var foo map[string]string
//
// ctx.FormData().MandatoryCustom("foo", func(value string) error {
// err := json.Unmarshal([]byte(value), &foo)
// if err != nil {
// return fmt.Errorf("unmarshal foo: %w", err)
// }
//
// return nil
// })
func (form *FormData) MandatoryCustom(key string, assign func(value string) error) *FormData {
var value string
form.mustMandatoryValue(key, &value)
if value == "" {
return form
}
err := assign(value)
if err != nil {
form.append(
fmt.Errorf("form value '%s' is invalid (got '%s', resulting to %w)", key, value, err),
)
}
return form
}
// Path binds the absolute path of a form data file to a string variable.
//
// var path string
//
// ctx.FormData().Path("foo.txt", &path)
func (form *FormData) Path(filename string, target *string) *FormData {
return form.path(filename, target)
}
// MandatoryPath binds the absolute path ofa form data file to a string
// variable. It populates an error if the file does not exist.
//
// var path string
//
// ctx.FormData().MandatoryPath("foo.txt", &path)
func (form *FormData) MandatoryPath(filename string, target *string) *FormData {
return form.mandatoryPath(filename, target)
}
// Content binds the content of a form data file to a string variable.
//
// var content string
//
// ctx.FormData().Content("foo.txt", &content, "bar")
func (form *FormData) Content(filename string, target *string, defaultValue string) *FormData {
var path string
form.path(filename, &path)
if path == "" {
*target = defaultValue
return form
}
return form.readFile(path, filename, target)
}
// MandatoryContent binds the content of a form data file to a string variable.
// It populates an error if the file does not exist.
//
// var content string
//
// ctx.FormData().MandatoryContent("foo.txt", &content)
func (form *FormData) MandatoryContent(filename string, target *string) *FormData {
var path string
form.mandatoryPath(filename, &path)
if path == "" {
return form
}
return form.readFile(path, filename, target)
}
// Paths binds the absolute paths of form data files, according to a list of
// file extensions, to a string slice variable.
//
// var paths []string
//
// ctx.FormData().Paths([]string{".txt"}, &paths)
func (form *FormData) Paths(extensions []string, target *[]string) *FormData {
return form.paths(extensions, target)
}
// MandatoryPaths binds the absolute paths of form data files, according to a
// list of file extensions, to a string slice variable. It populates an error
// if there is no file for given file extensions.
//
// var paths []string
//
// ctx.FormData().MandatoryPaths([]string{".txt"}, &paths)
func (form *FormData) MandatoryPaths(extensions []string, target *[]string) *FormData {
form.paths(extensions, target)
if len(*target) > 0 {
return form
}
form.append(
fmt.Errorf("no form file found for extensions: %v", extensions),
)
return form
}
// paths binds the absolute paths of form data files, according to a list of
// file extensions, to a string slice variable.
func (form *FormData) paths(extensions []string, target *[]string) *FormData {
for filename, path := range form.files {
for _, ext := range extensions {
// See https://github.com/thecodingmachine/gotenberg/issues/228.
if strings.ToLower(filepath.Ext(filename)) == ext {
*target = append(*target, path)
}
}
}
// See https://github.com/thecodingmachine/gotenberg/issues/139.
sort.Strings(*target)
return form
}
// append adds an error to the list of errors.
func (form *FormData) append(err error) {
form.errors = multierr.Append(form.errors, err)
}
// mustValue binds the target interface with a form data value. If the value is
// empty or the "key" does not exist, it binds the default value. Currently,
// only the string, bool, int, float64 and time.Duration types are bindable.
func (form *FormData) mustValue(key string, target interface{}, defaultValue interface{}) *FormData {
val, ok := form.values[key]
if !ok || val[0] == "" {
switch t := (target).(type) {
case *string:
*t = defaultValue.(string)
case *bool:
*t = defaultValue.(bool)
case *int:
*t = defaultValue.(int)
case *float64:
*t = defaultValue.(float64)
case *time.Duration:
*t = defaultValue.(time.Duration)
default:
panic("target type not supported")
}
return form
}
return form.mustAssign(key, val[0], target)
}
// mustMandatoryValue binds the target interface with a form data value. It
// populates an error if the value is empty or the "key" does not exist.
// Currently, only the string, bool, int, float64 and time.Duration types are
// bindable.
func (form *FormData) mustMandatoryValue(key string, target interface{}) *FormData {
val, ok := form.values[key]
if !ok || val[0] == "" {
form.append(
fmt.Errorf("form value '%s' is required", key),
)
return form
}
form.mustAssign(key, val[0], target)
return form
}
// mustAssign parses the string value and tries to convert it to the target
// interface real type. Currently, only the string, bool, int, float64 and
// time.Duration types are bindable.
func (form *FormData) mustAssign(key, value string, target interface{}) *FormData {
var err error
switch t := (target).(type) {
case *string:
*t = value
case *bool:
*t, err = strconv.ParseBool(value)
case *int:
*t, err = strconv.Atoi(value)
case *float64:
*t, err = strconv.ParseFloat(value, 64)
case *time.Duration:
*t, err = time.ParseDuration(value)
default:
panic("target type not supported")
}
if err != nil {
form.append(
fmt.Errorf("form value '%s' is invalid (got '%s', resulting to %w)", key, value, err),
)
}
return form
}
// path binds the absolute path of a form data file to a string variable.
func (form *FormData) path(filename string, target *string) *FormData {
for name, path := range form.files {
if name == filename {
*target = path
return form
}
}
return form
}
// mandatoryPath binds the absolute path of a form data file to a string
// variable. It populates an error if the file does not exist.
func (form *FormData) mandatoryPath(filename string, target *string) *FormData {
form.path(filename, target)
if *target != "" {
return form
}
form.append(
fmt.Errorf("form file '%s' is required", filename),
)
return form
}
// readFile binds the content of a file to a string variable. It populates an
// error if it fails to read the file content.
func (form *FormData) readFile(path, filename string, target *string) *FormData {
b, err := ioutil.ReadFile(path)
if err != nil {
form.append(
fmt.Errorf("form file '%s' is invalid (%w)", filename, err),
)
return form
}
*target = string(b)
return form
}

1111
pkg/modules/api/formdata_test.go Normal file
View File

File diff suppressed because it is too large Load Diff

576
pkg/modules/api/middlewares.go Normal file
View File

@@ -0,0 +1,576 @@
package api
import (
"bufio"
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"os"
"path/filepath"
"regexp"
"strconv"
"strings"
"time"
"github.com/google/uuid"
"github.com/hashicorp/go-retryablehttp"
"github.com/labstack/echo/v4"
"go.uber.org/zap"
)
// httpErrorHandler is the centralized HTTP error handler. It parses the error,
// returns either a response as "text/plain; charset=UTF-8" or, if a webhook
// client exists in the echo.Context, sends a request to the webhook error URL
// with a JSON body containing the trace, the status and the error message.
func httpErrorHandler(traceHeader string) echo.HTTPErrorHandler {
return func(err error, c echo.Context) {
parseError := func(err error) (int, string) {
echoErr, ok := err.(*echo.HTTPError)
if ok {
return echoErr.Code, http.StatusText(echoErr.Code)
}
if errors.Is(err, context.DeadlineExceeded) {
return http.StatusServiceUnavailable, http.StatusText(http.StatusServiceUnavailable)
}
var httpErr HTTPError
if errors.As(err, &httpErr) {
return httpErr.HTTPError()
}
// Default 500 status code.
return http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)
}
status, message := parseError(err)
logger := c.Get("logger").(*zap.Logger)
clientOrNil := c.Get("webhookClient")
// No webhook client, meaning we can send the error as a response.
if clientOrNil == nil {
c.Response().Header().Add(echo.HeaderContentType, echo.MIMETextPlainCharsetUTF8)
err = c.String(status, message)
if err != nil {
logger.Error(fmt.Sprintf("send error response: %s", err.Error()))
}
return
}
// We have to send the error to the webhook.
client := clientOrNil.(*webhookClient)
body := struct {
Status int `json:"status"`
Message string `json:"message"`
}{
Status: status,
Message: message,
}
b, err := json.Marshal(body)
if err != nil {
logger.Error(fmt.Sprintf("marshal JSON: %s", err.Error()))
return
}
headers := map[string]string{
echo.HeaderContentType: echo.MIMEApplicationJSONCharsetUTF8,
traceHeader: c.Get("trace").(string),
}
err = client.send(bytes.NewReader(b), headers, true)
if err != nil {
logger.Error(fmt.Sprintf("send error response to webhook: %s", err.Error()))
}
}
}
// latencyMiddleware sets the start time in the echo.Context under "startTime".
// Its value will be used later to calculate a request latency.
//
// startTime := c.Get("startTime").(time.Time)
func latencyMiddleware() echo.MiddlewareFunc {
return func(next echo.HandlerFunc) echo.HandlerFunc {
return func(c echo.Context) error {
// First piece for calculating the latency.
startTime := time.Now()
c.Set("startTime", startTime)
// Call the next middleware in the chain.
return next(c)
}
}
}
// rootPathMiddleware sets the root path in the echo.Context under "rootPath".
// Its value may be used to skip a middleware execution based on a request
// URI.
//
// rootPath := c.Get("rootPath").(string)
// healthURI := fmt.Sprintf("%shealth", rootPath)
//
// // Skip the middleware if health check URI.
// if c.Request().RequestURI == healthURI {
// // Call the next middleware in the chain.
// return next(c)
// }
func rootPathMiddleware(rootPath string) echo.MiddlewareFunc {
return func(next echo.HandlerFunc) echo.HandlerFunc {
return func(c echo.Context) error {
c.Set("rootPath", rootPath)
// Call the next middleware in the chain.
return next(c)
}
}
}
// traceMiddleware sets the request identifier in the echo.Context under
// "trace". Its value is either retrieved from the trace header or generated if
// the header is not present / its value is empty.
//
// trace := c.Get("trace").(string)
func traceMiddleware(header string) echo.MiddlewareFunc {
return func(next echo.HandlerFunc) echo.HandlerFunc {
return func(c echo.Context) error {
// Get or create the request identifier.
trace := c.Request().Header.Get(header)
if trace == "" {
trace = uuid.New().String()
}
c.Set("trace", trace)
c.Response().Header().Add(header, trace)
// Call the next middleware in the chain.
return next(c)
}
}
}
// loggerMiddleware sets the logger in the echo.Context under "logger" and logs
// a request result (but does not log a webhook call result, which is the job
// of the webhookClient).
//
// logger := c.Get("logger").(*zap.Logger)
func loggerMiddleware(logger *zap.Logger, skipHealthRouteLogging bool) echo.MiddlewareFunc {
return func(next echo.HandlerFunc) echo.HandlerFunc {
return func(c echo.Context) error {
startTime := c.Get("startTime").(time.Time)
trace := c.Get("trace").(string)
// Create the request logger and add it to our locals.
reqLogger := logger.With(zap.String("trace", trace))
c.Set("logger", reqLogger)
// Call the next middleware in the chain.
err := next(c)
if err != nil {
c.Error(err)
}
if skipHealthRouteLogging {
rootPath := c.Get("rootPath").(string)
healthURI := fmt.Sprintf("%shealth", rootPath)
if c.Request().RequestURI == healthURI {
return nil
}
}
// Last piece for calculating the latency.
finishTime := time.Now()
// Now, let's log!
fields := make([]zap.Field, 12)
fields[0] = zap.String("remote_ip", c.RealIP())
fields[1] = zap.String("host", c.Request().Host)
fields[2] = zap.String("uri", c.Request().RequestURI)
fields[3] = zap.String("method", c.Request().Method)
fields[4] = zap.String("path", func() string {
path := c.Request().URL.Path
if path == "" {
path = "/"
}
return path
}())
fields[5] = zap.String("referer", c.Request().Referer())
fields[6] = zap.String("user_agent", c.Request().UserAgent())
fields[7] = zap.Int("status", c.Response().Status)
fields[8] = zap.Int64("latency", int64(finishTime.Sub(startTime)))
fields[9] = zap.String("latency_human", finishTime.Sub(startTime).String())
fields[10] = zap.Int64("bytes_in", c.Request().ContentLength)
fields[11] = zap.Int64("bytes_out", c.Response().Size)
if err != nil {
reqLogger.Error(err.Error(), fields...)
} else {
reqLogger.Info("request handled", fields...)
}
return nil
}
}
}
type contextMiddlewareConfig struct {
traceHeader string
timeout struct {
process time.Duration
write time.Duration
}
webhook struct {
allowList *regexp.Regexp
denyList *regexp.Regexp
errorAllowList *regexp.Regexp
errorDenyList *regexp.Regexp
maxRetry int
retryMinWait time.Duration
retryMaxWait time.Duration
disable bool
}
}
// contextMiddleware handles the result of a "multipart/form-data" request. If
// a webhook URL is present in the headers, exit early and process the result
// in a goroutine.
func contextMiddleware(cfg contextMiddlewareConfig) echo.MiddlewareFunc {
return func(next echo.HandlerFunc) echo.HandlerFunc {
return func(c echo.Context) error {
webhookURL := c.Request().Header.Get("Gotenberg-Webhook-Url")
logger := c.Get("logger").(*zap.Logger).With(zap.Bool("webhook", webhookURL != ""))
// We create a context with a timeout so that underlying processes are
// able to stop early and handle correctly a timeout scenario.
ctx, cancel, err := newContext(c, logger, cfg.timeout.process)
if err != nil {
cancel()
return fmt.Errorf("create request context: %w", err)
}
c.Set("context", ctx)
// Helper function for retrieving/creating the output filename.
outputFilename := func(outputPath string) string {
filename := c.Request().Header.Get("Gotenberg-Output-Filename")
if filename == "" {
return filepath.Base(outputPath)
}
return fmt.Sprintf("%s%s", filename, filepath.Ext(outputPath))
}
if webhookURL == "" {
defer cancel()
// No webhook URL, call the next middleware in the chain.
err := next(c)
if err != nil {
return err
}
// No error, let's build the output file.
outputPath, err := ctx.buildOutputFile()
if err != nil {
return fmt.Errorf("build output file: %w", err)
}
// Send the output file.
err = c.Attachment(outputPath, outputFilename(outputPath))
if err != nil {
return fmt.Errorf("send response: %w", err)
}
return nil
}
// Ok, we got a webhook URL.
if cfg.webhook.disable {
// The client requested the webhook feature, but it has been
// disabled. Let's tell the client about that.
cancel()
return WrapError(
errors.New("webhook feature requested but it is disabled"),
NewSentinelHTTPError(http.StatusForbidden, "Invalid 'Gotenberg-Webhook-Url' header: feature is disabled"),
)
}
// Do we have a webhook error URL in case of... error?
webhookErrorURL := c.Request().Header.Get("Gotenberg-Webhook-Error-Url")
if webhookErrorURL == "" {
cancel()
return WrapError(
errors.New("empty webhook error URL"),
NewSentinelHTTPError(http.StatusBadRequest, "Invalid 'Gotenberg-Webhook-Error-Url' header: empty value or header not provided"),
)
}
// Let's check if the webhook URLs are acceptable according to our
// allowed/denied lists.
filter := func(URL, header string, allowList, denyList *regexp.Regexp) error {
if !allowList.MatchString(URL) {
return WrapError(
fmt.Errorf("'%s' does not match the expression from the allowed list", URL),
NewSentinelHTTPError(
http.StatusForbidden,
fmt.Sprintf("Invalid '%s' header value: '%s' does not match the authorized URLs", header, URL),
),
)
}
if denyList.String() != "" && denyList.MatchString(URL) {
return WrapError(
fmt.Errorf("'%s' matches the expression from the denied list", URL),
NewSentinelHTTPError(
http.StatusForbidden,
fmt.Sprintf("Invalid '%s' header value: '%s' does not match the authorized URLs", header, URL),
),
)
}
return nil
}
err = filter(webhookURL, "Gotenberg-Webhook-Url", cfg.webhook.allowList, cfg.webhook.denyList)
if err != nil {
cancel()
return fmt.Errorf("filter webhook URL: %w", err)
}
err = filter(webhookErrorURL, "Gotenberg-Webhook-Error-Url", cfg.webhook.errorAllowList, cfg.webhook.errorDenyList)
if err != nil {
cancel()
return fmt.Errorf("filter webhook error URL: %w", err)
}
// Let's check the HTTP methods for calling the webhook URLs.
methodFromHeader := func(header string) (string, error) {
method := c.Request().Header.Get(header)
if method == "" {
return http.MethodPost, nil
}
method = strings.ToUpper(method)
switch method {
case http.MethodPost:
return method, nil
case http.MethodPatch:
return method, nil
case http.MethodPut:
return method, nil
}
return "", WrapError(
fmt.Errorf("webhook method '%s' is not '%s', '%s' or '%s'", method, http.MethodPost, http.MethodPatch, http.MethodPut),
NewSentinelHTTPError(
http.StatusBadRequest,
fmt.Sprintf("Invalid '%s' header value: expected '%s', '%s' or '%s', but got '%s'", header, http.MethodPost, http.MethodPatch, http.MethodPut, method),
),
)
}
webhookMethod, err := methodFromHeader("Gotenberg-Webhook-Method")
if err != nil {
cancel()
return fmt.Errorf("get method to use for webhook: %w", err)
}
webhookErrorMethod, err := methodFromHeader("Gotenberg-Webhook-Error-Method")
if err != nil {
cancel()
return fmt.Errorf("get method to use for webhook error: %w", err)
}
// What about extra HTTP headers?
var extraHTTPHeaders map[string]string
extraHTTPHeadersJSON := c.Request().Header.Get("Gotenberg-Webhook-Extra-Http-Headers")
if extraHTTPHeadersJSON != "" {
err = json.Unmarshal([]byte(extraHTTPHeadersJSON), &extraHTTPHeaders)
if err != nil {
cancel()
return WrapError(
fmt.Errorf("unmarshal webhook extra HTTP headers: %w", err),
NewSentinelHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid 'Gotenberg-Webhook-Extra-Http-Headers' header value: %s", err.Error())),
)
}
}
client := &webhookClient{
url: webhookURL,
method: webhookMethod,
errorURL: webhookErrorURL,
errorMethod: webhookErrorMethod,
extraHTTPHeaders: extraHTTPHeaders,
startTime: c.Get("startTime").(time.Time),
client: &retryablehttp.Client{
HTTPClient: &http.Client{
Timeout: cfg.timeout.write,
},
RetryMax: cfg.webhook.maxRetry,
RetryWaitMin: cfg.webhook.retryMinWait,
RetryWaitMax: cfg.webhook.retryMaxWait,
Logger: leveledLogger{
logger: logger,
},
CheckRetry: retryablehttp.DefaultRetryPolicy,
Backoff: retryablehttp.DefaultBackoff,
},
logger: logger,
}
c.Set("webhookClient", client)
// As a webhook URL has been given, we handle the request in a
// goroutine and return immediately.
go func() {
defer cancel()
// Call the next middleware in the chain.
err := next(c)
if err != nil {
// The process failed for whatever reason. Let's send the
// details to the webhook.
ctx.Log().Error(err.Error())
c.Error(err)
return
}
// No error, let's get build the output file.
outputPath, err := ctx.buildOutputFile()
if err != nil {
ctx.Log().Error(fmt.Sprintf("build output file: %s", err))
c.Error(err)
return
}
outputFile, err := os.Open(outputPath)
if err != nil {
ctx.Log().Error(fmt.Sprintf("open output file: %s", err))
c.Error(err)
return
}
defer func() {
err := outputFile.Close()
if err != nil {
ctx.Log().Error(fmt.Sprintf("close output file: %s", err))
}
}()
fileHeader := make([]byte, 512)
_, err = outputFile.Read(fileHeader)
if err != nil {
ctx.Log().Error(fmt.Sprintf("read header of output file: %s", err))
c.Error(err)
return
}
fileStat, err := outputFile.Stat()
if err != nil {
ctx.Log().Error(fmt.Sprintf("get stat from output file: %s", err))
c.Error(err)
return
}
_, err = outputFile.Seek(0, 0)
if err != nil {
ctx.Log().Error(fmt.Sprintf("reset output file reader: %s", err))
c.Error(err)
return
}
headers := map[string]string{
echo.HeaderContentDisposition: fmt.Sprintf("attachement; filename=%q", outputFilename(outputPath)),
echo.HeaderContentType: http.DetectContentType(fileHeader),
echo.HeaderContentLength: strconv.FormatInt(fileStat.Size(), 10),
cfg.traceHeader: c.Get("trace").(string),
}
// Send the output file to the webhook.
err = client.send(bufio.NewReader(outputFile), headers, false)
if err != nil {
ctx.Log().Error(fmt.Sprintf("send output file to webhook: %s", err))
c.Error(err)
}
}()
return c.NoContent(http.StatusNoContent)
}
}
}
// timeoutMiddleware manages hard timeout scenarios, i.e., when a route handler
// fails to timeout as expected.
func timeoutMiddleware(hardTimeout time.Duration) echo.MiddlewareFunc {
return func(next echo.HandlerFunc) echo.HandlerFunc {
return func(c echo.Context) error {
logger := c.Get("logger").(*zap.Logger)
// Define a hard timeout if the route handler fails to timeout as
// expected.
hardTimeoutCtx, hardTimeoutCancel := context.WithTimeout(
context.Background(),
hardTimeout,
)
defer hardTimeoutCancel()
errChan := make(chan error, 1)
go func() {
// In case of hard timeout, a panic may occur.
// This deferred function allows us to recover from such scenarios.
defer func() {
if r := recover(); r != nil {
logger.Debug(fmt.Sprintf("recovering from a panic (possible cause being a hard timeout): %s", r))
}
}()
// Call the next middleware in the chain.
errChan <- next(c)
}()
select {
case err := <-errChan:
return err
case <-hardTimeoutCtx.Done():
logger.Debug("hard timeout as the route handler did not timeout as expected")
return fmt.Errorf("hard timeout: %w", hardTimeoutCtx.Err())
}
}
}
}

1033
pkg/modules/api/middlewares_test.go Normal file
View File

File diff suppressed because it is too large Load Diff

139
pkg/modules/api/webhook.go Normal file
View File

@@ -0,0 +1,139 @@
package api
import (
"fmt"
"io"
"strconv"
"time"
"github.com/hashicorp/go-retryablehttp"
"github.com/labstack/echo/v4"
"go.uber.org/zap"
)
// webhookClient gathers all the data required to send a request to a webhook.
type webhookClient struct {
url string
method string
errorURL string
errorMethod string
extraHTTPHeaders map[string]string
startTime time.Time
client *retryablehttp.Client
logger *zap.Logger
}
// send call the webhook either to send the success response or the error response.
func (webhook webhookClient) send(body io.Reader, headers map[string]string, erroed bool) error {
URL := webhook.url
if erroed {
URL = webhook.errorURL
}
method := webhook.method
if erroed {
method = webhook.errorMethod
}
req, err := retryablehttp.NewRequest(method, URL, body)
if err != nil {
return fmt.Errorf("create '%s' request to '%s': %w", method, URL, err)
}
req.Header.Set("User-Agent", "Gotenberg")
// Extra HTTP headers are the custom headers from the user.
for key, value := range webhook.extraHTTPHeaders {
req.Header.Set(key, value)
}
// Middleware caller's headers > extra HTTP headers from the user.
contentLength, ok := headers[echo.HeaderContentLength]
if ok {
// Golang "http" package should automatically calculate the size of the
// body. But, when using a buffered file reader, it does not work.
// Worse, the "Content-Length" header is also removed. Therefore, in
// order to keep this valuable information, we have to trust the caller
// by reading the value of the "Content-Length" entry and set it as the
// content length of the request. It's kinda sub-optimal, but hey, at
// least it works.
bodySize, err := strconv.ParseInt(contentLength, 10, 64)
if err != nil {
return fmt.Errorf("parse content length entry: %w", err)
}
req.ContentLength = bodySize
}
for key, value := range headers {
req.Header.Set(key, value)
}
resp, err := webhook.client.Do(req)
if err != nil {
return fmt.Errorf("send '%s' request to '%s': %w", method, URL, err)
}
defer func() {
err := resp.Body.Close()
if err != nil {
webhook.logger.Error(fmt.Sprintf("close response body from '%s': %s", URL, err))
}
}()
// Last piece for calculating the latency.
finishTime := time.Now()
// Now let's log!
fields := make([]zap.Field, 5)
fields[0] = zap.String("webhook_url", URL)
fields[1] = zap.String("method", method)
fields[2] = zap.Int64("latency", int64(finishTime.Sub(webhook.startTime)))
fields[3] = zap.String("latency_human", finishTime.Sub(webhook.startTime).String())
fields[4] = zap.Int64("bytes_out", req.ContentLength)
if erroed {
webhook.logger.Warn("request to webhook with error details handled", fields...)
return nil
}
webhook.logger.Info("request to webhook handled", fields...)
return nil
}
// leveledLogger is wrapper around a zap.Logger which is used by the
// retryablehttp.Client.
type leveledLogger struct {
logger *zap.Logger
}
// Error logs a message at error level using the wrapped zap.Logger.
func (leveled leveledLogger) Error(msg string, keysAndValues ...interface{}) {
leveled.logger.Error(fmt.Sprintf("%s: %+v", msg, keysAndValues))
}
// Warn logs a message at warning level using the wrapped zap.Logger.
func (leveled leveledLogger) Warn(msg string, keysAndValues ...interface{}) {
leveled.logger.Warn(fmt.Sprintf("%s: %+v", msg, keysAndValues))
}
// Info logs a message at info level using the wrapped zap.Logger.
func (leveled leveledLogger) Info(msg string, keysAndValues ...interface{}) {
leveled.logger.Info(fmt.Sprintf("%s: %+v", msg, keysAndValues))
}
// Debug logs a message at debug level using the wrapped zap.Logger.
func (leveled leveledLogger) Debug(msg string, keysAndValues ...interface{}) {
leveled.logger.Debug(fmt.Sprintf("%s: %+v", msg, keysAndValues))
}
// Interface guards.
var (
_ retryablehttp.LeveledLogger = (*leveledLogger)(nil)
)

23
pkg/modules/api/webhook_test.go Normal file
View File

@@ -0,0 +1,23 @@
package api
import (
"testing"
"go.uber.org/zap"
)
func TestLeveledLogger_Error(t *testing.T) {
leveledLogger{logger: zap.NewNop()}.Error("foo")
}
func TestLeveledLogger_Warn(t *testing.T) {
leveledLogger{logger: zap.NewNop()}.Warn("foo")
}
func TestLeveledLogger_Info(t *testing.T) {
leveledLogger{logger: zap.NewNop()}.Info("foo")
}
func TestLeveledLogger_Debug(t *testing.T) {
leveledLogger{logger: zap.NewNop()}.Debug("foo")
}