feat: add 7.x source code

pkg/modules/gc/doc.go

@@ -0,0 +1,3 @@
+// Package gc provides a module for removing files and directories that have
+// expired.
+package gc

pkg/modules/gc/gc.go

@@ -0,0 +1,235 @@
+package gc
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"os/signal"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/gotenberg/gotenberg/v7/pkg/gotenberg"
+	"go.uber.org/zap"
+)
+
+func init() {
+	gotenberg.MustRegisterModule(GarbageCollector{})
+}
+
+// GarbageCollector is a module for removing files and directories that have
+// expired. It allows us to make sure that the application does not leak files
+// or directories when running.
+type GarbageCollector struct {
+	rootPath      string
+	graceDuration time.Duration
+	excludeSubstr []string
+
+	ticker *time.Ticker
+	done   chan bool
+
+	logger *zap.Logger
+}
+
+// GarbageCollectorGraceDurationModifier is a module interface which allows to
+// update the expiration time of files and directories parsed by the garbage
+// collector. For instance, if the grace duration is 30s, the garbage collector
+// will remove paths that have a modification time older than 30s. If there are
+// many GarbageCollectorGraceDurationModifier, only the longest grace duration
+// is selected.
+type GarbageCollectorGraceDurationModifier interface {
+	GraceDuration() time.Duration
+}
+
+// GarbageCollectorExcludeSubstrModifier is a module interface which adds the
+// given substrings to the exclude list of the garbage collector. If a path
+// contains one of those substrings, the garbage collector ignores it.
+type GarbageCollectorExcludeSubstrModifier interface {
+	ExcludeSubstr() []string
+}
+
+// Descriptor returns a GarbageCollector's module descriptor.
+func (gc GarbageCollector) Descriptor() gotenberg.ModuleDescriptor {
+	return gotenberg.ModuleDescriptor{
+		ID:  "gc",
+		New: func() gotenberg.Module { return new(GarbageCollector) },
+	}
+}
+
+// Provision sets the module properties.
+func (gc *GarbageCollector) Provision(ctx *gotenberg.Context) error {
+	gc.rootPath = gotenberg.TmpPath()
+
+	graceDurationModifiers, err := ctx.Modules(new(GarbageCollectorGraceDurationModifier))
+	if err != nil {
+		return fmt.Errorf("get grace duration modifiers: %w", err)
+	}
+
+	for _, graceDurationModifier := range graceDurationModifiers {
+		modifier := graceDurationModifier.(GarbageCollectorGraceDurationModifier)
+
+		if gc.graceDuration < modifier.GraceDuration() {
+			gc.graceDuration = modifier.GraceDuration()
+		}
+	}
+
+	excludeSubstrModifiers, err := ctx.Modules(new(GarbageCollectorExcludeSubstrModifier))
+	if err != nil {
+		return fmt.Errorf("get exclude substr modifiers: %w", err)
+	}
+
+	gc.excludeSubstr = strings.Split(os.Getenv("GC_EXCLUDE_SUBSTR"), ",")
+
+	for _, excludeSubstrModifier := range excludeSubstrModifiers {
+		modifier := excludeSubstrModifier.(GarbageCollectorExcludeSubstrModifier)
+
+		gc.excludeSubstr = append(gc.excludeSubstr, modifier.ExcludeSubstr()...)
+	}
+
+	loggerProvider, err := ctx.Module(new(gotenberg.LoggerProvider))
+	if err != nil {
+		return fmt.Errorf("get logger provider: %w", err)
+	}
+
+	logger, err := loggerProvider.(gotenberg.LoggerProvider).Logger(gc)
+	if err != nil {
+		return fmt.Errorf("get logger: %w", err)
+	}
+
+	gc.logger = logger
+
+	return nil
+}
+
+// Start starts the garbage collector.
+func (gc *GarbageCollector) Start() error {
+	gc.ticker = time.NewTicker(gc.graceDuration + time.Duration(1)*time.Second)
+	gc.done = make(chan bool, 1)
+
+	go func() {
+		for {
+			func() {
+				gcMu.RLock()
+				defer gcMu.RUnlock()
+
+				select {
+				case <-gc.done:
+					return
+				case <-gc.ticker.C:
+					gc.collect(false)
+				}
+			}()
+		}
+	}()
+
+	return nil
+}
+
+// collect parses the root path of the garbage collector and removes files or
+// directories that have expired. It ignores the expiration date if the "force"
+// argument is set to true.
+func (gc GarbageCollector) collect(force bool) {
+	expirationTime := time.Now().Add(-gc.graceDuration)
+
+	// To make sure that the next Walk method stays on
+	// the root level of the considered path, we have to
+	// return a filepath.SkipDir error if the current path
+	// is a directory.
+	skipDirOrNil := func(info os.FileInfo) error {
+		if info.IsDir() {
+			return filepath.SkipDir
+		}
+
+		return nil
+	}
+
+	removePath := func(path string) {
+		err := os.RemoveAll(path)
+		if err != nil {
+			gc.logger.Error(fmt.Sprintf("remove '%s': %s", path, err))
+		}
+
+		gc.logger.Debug(fmt.Sprintf("'%s' removed", path))
+	}
+
+	err := filepath.Walk(gc.rootPath, func(path string, info os.FileInfo, pathErr error) error {
+		if pathErr != nil {
+			// For whatever reasons, the Walk method failed
+			// to process the current path.
+			return pathErr
+		}
+
+		if path == gc.rootPath {
+			return nil
+		}
+
+		for _, substr := range gc.excludeSubstr {
+			if strings.Contains(info.Name(), substr) {
+				return skipDirOrNil(info)
+			}
+		}
+
+		if force {
+			removePath(path)
+
+			return skipDirOrNil(info)
+		}
+
+		if info.ModTime().Before(expirationTime) {
+			removePath(path)
+		}
+
+		return skipDirOrNil(info)
+	})
+
+	if err != nil {
+		gc.logger.Error(err.Error())
+	}
+}
+
+// StartupMessage returns an empty string.
+func (gc GarbageCollector) StartupMessage() string {
+	return ""
+}
+
+// Stop stops the garbage collector.
+func (gc *GarbageCollector) Stop(ctx context.Context) error {
+	_, ok := ctx.Deadline()
+	if !ok {
+		return errors.New("no context dead line")
+	}
+
+	// Block until the context is done so that other module may gracefully stop
+	// before we do a shutdown cleanup. We skip this step if we receive a
+	// SIGINT in the meantime.
+	gc.logger.Debug("wait for the end of grace duration")
+
+	quit := make(chan os.Signal, 1)
+	signal.Notify(quit, os.Interrupt)
+
+	select {
+	case <-quit:
+		return nil
+	case <-ctx.Done():
+		break
+	}
+
+	gc.ticker.Stop()
+	gc.done <- true
+
+	gc.logger.Debug("shutdown cleanup...")
+	gc.collect(true)
+
+	return nil
+}
+
+var gcMu sync.RWMutex
+
+// Interface guards.
+var (
+	_ gotenberg.Module      = (*GarbageCollector)(nil)
+	_ gotenberg.Provisioner = (*GarbageCollector)(nil)
+	_ gotenberg.App         = (*GarbageCollector)(nil)
+)

pkg/modules/gc/gc_test.go

@@ -0,0 +1,468 @@
+package gc
+
+import (
+	"context"
+	"errors"
+	"os"
+	"reflect"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/gotenberg/gotenberg/v7/pkg/gotenberg"
+	"go.uber.org/zap"
+)
+
+type ProtoModule struct {
+	descriptor func() gotenberg.ModuleDescriptor
+}
+
+func (mod ProtoModule) Descriptor() gotenberg.ModuleDescriptor {
+	return mod.descriptor()
+}
+
+type ProtoValidator struct {
+	ProtoModule
+	validate func() error
+}
+
+func (mod ProtoValidator) Validate() error {
+	return mod.validate()
+}
+
+type ProtoGarbageCollectorGraceDurationModifier struct {
+	ProtoValidator
+	graceDuration func() time.Duration
+}
+
+func (mod ProtoGarbageCollectorGraceDurationModifier) GraceDuration() time.Duration {
+	return mod.graceDuration()
+}
+
+type ProtoGarbageCollectorExcludeSubstrModifier struct {
+	ProtoValidator
+	excludeSubstr func() []string
+}
+
+func (mod ProtoGarbageCollectorExcludeSubstrModifier) ExcludeSubstr() []string {
+	return mod.excludeSubstr()
+}
+
+type ProtoLoggerProvider struct {
+	ProtoModule
+	logger func(mod gotenberg.Module) (*zap.Logger, error)
+}
+
+func (factory ProtoLoggerProvider) Logger(mod gotenberg.Module) (*zap.Logger, error) {
+	return factory.logger(mod)
+}
+
+func TestGarbageCollector_Descriptor(t *testing.T) {
+	descriptor := GarbageCollector{}.Descriptor()
+
+	actual := reflect.TypeOf(descriptor.New())
+	expect := reflect.TypeOf(new(GarbageCollector))
+
+	if actual != expect {
+		t.Errorf("expected '%s' but got '%s'", expect, actual)
+	}
+}
+
+func TestGarbageCollector_Provision(t *testing.T) {
+	for i, tc := range []struct {
+		ctx                 *gotenberg.Context
+		expectGraceDuration time.Duration
+		expectExcludeSubstr []string
+		expectErr           bool
+	}{
+		{
+			ctx: func() *gotenberg.Context {
+				mod := struct {
+					ProtoGarbageCollectorGraceDurationModifier
+				}{}
+				mod.descriptor = func() gotenberg.ModuleDescriptor {
+					return gotenberg.ModuleDescriptor{ID: "foo", New: func() gotenberg.Module { return mod }}
+				}
+				mod.validate = func() error { return errors.New("foo") }
+
+				return gotenberg.NewContext(gotenberg.ParsedFlags{}, []gotenberg.ModuleDescriptor{
+					mod.Descriptor(),
+				})
+			}(),
+			expectErr: true,
+		},
+		{
+			ctx: func() *gotenberg.Context {
+				mod := struct {
+					ProtoGarbageCollectorExcludeSubstrModifier
+				}{}
+				mod.descriptor = func() gotenberg.ModuleDescriptor {
+					return gotenberg.ModuleDescriptor{ID: "foo", New: func() gotenberg.Module { return mod }}
+				}
+				mod.validate = func() error { return errors.New("foo") }
+
+				return gotenberg.NewContext(gotenberg.ParsedFlags{}, []gotenberg.ModuleDescriptor{
+					mod.Descriptor(),
+				})
+			}(),
+			expectErr: true,
+		},
+		{
+			ctx:       gotenberg.NewContext(gotenberg.ParsedFlags{}, make([]gotenberg.ModuleDescriptor, 0)),
+			expectErr: true,
+		},
+		{
+			ctx: func() *gotenberg.Context {
+				mod := struct {
+					ProtoLoggerProvider
+				}{}
+				mod.descriptor = func() gotenberg.ModuleDescriptor {
+					return gotenberg.ModuleDescriptor{ID: "foo", New: func() gotenberg.Module { return mod }}
+				}
+				mod.logger = func(mod gotenberg.Module) (*zap.Logger, error) { return nil, errors.New("foo") }
+
+				return gotenberg.NewContext(gotenberg.ParsedFlags{}, []gotenberg.ModuleDescriptor{
+					mod.Descriptor(),
+				})
+			}(),
+			expectErr: true,
+		},
+		{
+			ctx: func() *gotenberg.Context {
+				mod := struct {
+					ProtoLoggerProvider
+				}{}
+				mod.descriptor = func() gotenberg.ModuleDescriptor {
+					return gotenberg.ModuleDescriptor{ID: "foo", New: func() gotenberg.Module { return mod }}
+				}
+				mod.logger = func(mod gotenberg.Module) (*zap.Logger, error) { return zap.NewNop(), nil }
+
+				return gotenberg.NewContext(gotenberg.ParsedFlags{}, []gotenberg.ModuleDescriptor{
+					mod.Descriptor(),
+				})
+			}(),
+		},
+		{
+			ctx: func() *gotenberg.Context {
+				mod1 := struct {
+					ProtoGarbageCollectorGraceDurationModifier
+				}{}
+				mod1.descriptor = func() gotenberg.ModuleDescriptor {
+					return gotenberg.ModuleDescriptor{ID: "foo", New: func() gotenberg.Module { return mod1 }}
+				}
+				mod1.graceDuration = func() time.Duration { return time.Duration(10) * time.Second }
+				mod1.validate = func() error { return nil }
+
+				mod2 := struct {
+					ProtoGarbageCollectorGraceDurationModifier
+				}{}
+				mod2.descriptor = func() gotenberg.ModuleDescriptor {
+					return gotenberg.ModuleDescriptor{ID: "bar", New: func() gotenberg.Module { return mod2 }}
+				}
+				mod2.graceDuration = func() time.Duration { return time.Duration(20) * time.Second }
+				mod2.validate = func() error { return nil }
+
+				mod3 := struct {
+					ProtoLoggerProvider
+				}{}
+				mod3.descriptor = func() gotenberg.ModuleDescriptor {
+					return gotenberg.ModuleDescriptor{ID: "baz", New: func() gotenberg.Module { return mod3 }}
+				}
+				mod3.logger = func(mod gotenberg.Module) (*zap.Logger, error) { return zap.NewNop(), nil }
+
+				return gotenberg.NewContext(gotenberg.ParsedFlags{}, []gotenberg.ModuleDescriptor{
+					mod1.Descriptor(),
+					mod2.Descriptor(),
+					mod3.Descriptor(),
+				})
+			}(),
+			expectGraceDuration: time.Duration(20) * time.Second,
+		},
+		{
+			ctx: func() *gotenberg.Context {
+				mod1 := struct {
+					ProtoGarbageCollectorExcludeSubstrModifier
+				}{}
+				mod1.descriptor = func() gotenberg.ModuleDescriptor {
+					return gotenberg.ModuleDescriptor{ID: "foo", New: func() gotenberg.Module { return mod1 }}
+				}
+				mod1.excludeSubstr = func() []string { return []string{"foo"} }
+				mod1.validate = func() error { return nil }
+
+				mod2 := struct {
+					ProtoGarbageCollectorExcludeSubstrModifier
+				}{}
+				mod2.descriptor = func() gotenberg.ModuleDescriptor {
+					return gotenberg.ModuleDescriptor{ID: "bar", New: func() gotenberg.Module { return mod2 }}
+				}
+				mod2.excludeSubstr = func() []string { return []string{"bar"} }
+				mod2.validate = func() error { return nil }
+
+				mod3 := struct {
+					ProtoLoggerProvider
+				}{}
+				mod3.descriptor = func() gotenberg.ModuleDescriptor {
+					return gotenberg.ModuleDescriptor{ID: "baz", New: func() gotenberg.Module { return mod3 }}
+				}
+				mod3.logger = func(mod gotenberg.Module) (*zap.Logger, error) { return zap.NewNop(), nil }
+
+				return gotenberg.NewContext(gotenberg.ParsedFlags{}, []gotenberg.ModuleDescriptor{
+					mod1.Descriptor(),
+					mod2.Descriptor(),
+					mod3.Descriptor(),
+				})
+			}(),
+			expectExcludeSubstr: func() []string {
+				expect := strings.Split(os.Getenv("GC_EXCLUDE_SUBSTR"), ",")
+				return append(expect, "foo", "bar")
+			}(),
+		},
+	} {
+		mod := new(GarbageCollector)
+		err := mod.Provision(tc.ctx)
+
+		if tc.expectErr && err == nil {
+			t.Errorf("test %d: expected error but got: %v", i, err)
+		}
+
+		if !tc.expectErr && err != nil {
+			t.Errorf("test %d: expected no error but got: %v", i, err)
+		}
+
+		if tc.expectGraceDuration != 0 && tc.expectGraceDuration != mod.graceDuration {
+			t.Errorf("test %d: expected grace duration of '%s' but got '%s'", i, tc.expectGraceDuration, mod.graceDuration)
+		}
+
+		if tc.expectExcludeSubstr != nil && !reflect.DeepEqual(tc.expectExcludeSubstr, mod.excludeSubstr) {
+			t.Errorf("test %d: expected exclude substr '%s' but got '%s'", i, tc.expectExcludeSubstr, mod.excludeSubstr)
+		}
+	}
+}
+
+func TestGarbageCollector_Start(t *testing.T) {
+	mod := new(GarbageCollector)
+	mod.logger = zap.NewNop()
+
+	path, err := gotenberg.MkdirAll()
+	if err != nil {
+		t.Fatalf("expected no error but got: %v", err)
+	}
+
+	mod.rootPath = path
+
+	err = mod.Start()
+	if err != nil {
+		t.Errorf("expected no error but got: %v", err)
+	}
+
+	time.Sleep(time.Duration(2) * time.Second)
+	mod.ticker.Stop()
+	mod.done <- true
+}
+
+func TestGarbageCollector_collect(t *testing.T) {
+	for i, tc := range []struct {
+		gc              *GarbageCollector
+		expectNotExists []string
+		expectExists    []string
+		force           bool
+	}{
+		{
+			gc: func() *GarbageCollector {
+				mod := new(GarbageCollector)
+				mod.logger = zap.NewNop()
+
+				path, err := gotenberg.MkdirAll()
+				if err != nil {
+					t.Fatalf("expected no error but got: %v", err)
+				}
+
+				mod.rootPath = path
+
+				err = os.WriteFile(path+"/foo", []byte{1}, 0755)
+				if err != nil {
+					t.Fatalf("expected no error but got: %v", err)
+				}
+
+				mod.excludeSubstr = []string{
+					"foo",
+				}
+
+				return mod
+			}(),
+			expectExists: []string{
+				"/foo",
+			},
+		},
+		{
+			gc: func() *GarbageCollector {
+				mod := new(GarbageCollector)
+				mod.logger = zap.NewNop()
+
+				path, err := gotenberg.MkdirAll()
+				if err != nil {
+					t.Fatalf("expected no error but got: %v", err)
+				}
+
+				mod.rootPath = path
+
+				err = os.WriteFile(path+"/foo", []byte{1}, 0755)
+				if err != nil {
+					t.Fatalf("expected no error but got: %v", err)
+				}
+
+				err = os.MkdirAll(path+"/bar", 0755)
+				if err != nil {
+					t.Fatalf("expected no error but got: %v", err)
+				}
+
+				return mod
+			}(),
+			expectNotExists: []string{
+				"/foo",
+				"/bar",
+			},
+			force: true,
+		},
+		{
+			gc: func() *GarbageCollector {
+				mod := new(GarbageCollector)
+				mod.logger = zap.NewNop()
+				mod.graceDuration = time.Duration(10) * time.Second
+
+				path, err := gotenberg.MkdirAll()
+				if err != nil {
+					t.Fatalf("expected no error but got: %v", err)
+				}
+
+				mod.rootPath = path
+
+				err = os.WriteFile(path+"/foo", []byte{1}, 0755)
+				if err != nil {
+					t.Fatalf("expected no error but got: %v", err)
+				}
+
+				newTime := time.Now().Add(-time.Duration(20) * time.Second)
+				err = os.Chtimes(path+"/foo", newTime, newTime)
+
+				if err != nil {
+					t.Fatalf("expected no error but got: %v", err)
+				}
+
+				err = os.WriteFile(path+"/bar", []byte{1}, 0755)
+				if err != nil {
+					t.Fatalf("expected no error but got: %v", err)
+				}
+
+				newTime = time.Now().Add(time.Duration(10) * time.Second)
+				err = os.Chtimes(path+"/bar", newTime, newTime)
+
+				if err != nil {
+					t.Fatalf("expected no error but got: %v", err)
+				}
+
+				return mod
+			}(),
+			expectNotExists: []string{
+				"/foo",
+			},
+			expectExists: []string{
+				"/bar",
+			},
+		},
+	} {
+		tc.gc.collect(tc.force)
+
+		for _, name := range tc.expectNotExists {
+			path := tc.gc.rootPath + name
+			_, err := os.Stat(path)
+			if !os.IsNotExist(err) {
+				t.Errorf("test %d: expected '%s' not to exist but got: %v", i, path, err)
+			}
+		}
+
+		for _, name := range tc.expectExists {
+			path := tc.gc.rootPath + name
+			_, err := os.Stat(path)
+			if os.IsNotExist(err) {
+				t.Errorf("test %d: expected '%s' to exist but got: %v", i, path, err)
+			}
+		}
+
+		err := os.RemoveAll(tc.gc.rootPath)
+		if err != nil {
+			t.Fatalf("test %d: expected no error but got: %v", i, err)
+		}
+	}
+}
+
+func TestGarbageCollector_StartupMessage(t *testing.T) {
+	actual := new(GarbageCollector).StartupMessage()
+	expect := ""
+
+	if actual != expect {
+		t.Errorf("expected '%s' but got '%s'", expect, actual)
+	}
+}
+
+func TestGarbageCollector_Stop(t *testing.T) {
+	for i, tc := range []struct {
+		timeout   time.Duration
+		expectErr bool
+	}{
+		{
+			expectErr: true,
+		},
+		{
+			timeout: time.Duration(1) * time.Nanosecond,
+		},
+	} {
+		func() {
+			mod := new(GarbageCollector)
+			mod.logger = zap.NewNop()
+
+			path, err := gotenberg.MkdirAll()
+			if err != nil {
+				t.Fatalf("test %d: expected no error but got: %v", i, err)
+			}
+
+			mod.rootPath = path
+
+			err = mod.Start()
+			if err != nil {
+				t.Fatalf("test %d: expected no error but got: %v", i, err)
+			}
+
+			if tc.timeout == 0 {
+				err = mod.Stop(context.TODO())
+			} else {
+				ctx, cancel := context.WithTimeout(context.Background(), tc.timeout)
+				defer cancel()
+
+				err = mod.Stop(ctx)
+			}
+
+			if tc.expectErr && err == nil {
+				t.Errorf("test %d: expected error but got: %v", i, err)
+			}
+
+			if !tc.expectErr && err != nil {
+				t.Errorf("test %d: expected no error but got: %v", i, err)
+			}
+		}()
+	}
+}
+
+// Interface guards.
+var (
+	_ gotenberg.Module                      = (*ProtoModule)(nil)
+	_ gotenberg.Validator                   = (*ProtoValidator)(nil)
+	_ GarbageCollectorGraceDurationModifier = (*ProtoGarbageCollectorGraceDurationModifier)(nil)
+	_ gotenberg.Module                      = (*ProtoGarbageCollectorGraceDurationModifier)(nil)
+	_ gotenberg.Validator                   = (*ProtoGarbageCollectorGraceDurationModifier)(nil)
+	_ GarbageCollectorExcludeSubstrModifier = (*ProtoGarbageCollectorExcludeSubstrModifier)(nil)
+	_ gotenberg.Module                      = (*ProtoGarbageCollectorExcludeSubstrModifier)(nil)
+	_ gotenberg.Validator                   = (*ProtoGarbageCollectorExcludeSubstrModifier)(nil)
+	_ gotenberg.LoggerProvider              = (*ProtoLoggerProvider)(nil)
+	_ gotenberg.Module                      = (*ProtoLoggerProvider)(nil)
+)