mirror of
https://github.com/gotenberg/gotenberg.git
synced 2026-10-17 01:33:17 +01:00
Compare commits
8 Commits
8944db131c
...
e29b7cb4f5
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e29b7cb4f5 | ||
|
|
23d59f3133 | ||
|
|
8a0de7d5d7 | ||
|
|
3c691cbebc | ||
|
|
0e83f737b4 | ||
|
|
57b048c611 | ||
|
|
d79e174c6f | ||
|
|
88ddaed09b |
15
Makefile
15
Makefile
@@ -212,11 +212,22 @@ lint-prettier: ## Lint non-Golang codebase
|
||||
|
||||
.PHONY: lint-todo
|
||||
lint-todo: ## Find TODOs in Golang codebase
|
||||
golangci-lint run --no-config --disable-all --enable godox
|
||||
golangci-lint run --no-config --default=none --enable godox
|
||||
|
||||
# TODO: restore a plain "go fix ./..." once both modernizers stop rewriting this
|
||||
# codebase into code that does not compile. Re-check by dropping a flag and
|
||||
# running "make fmt && make lint". Removing an analyzer upstream makes go fix
|
||||
# fail with "flag provided but not defined", so this cannot rot silently.
|
||||
# embedlit folds a post-literal field assignment into a literal that already
|
||||
# sets that key, producing a duplicate field name (cmd/gotenberg.go). Fixed in
|
||||
# Go 1.27.1, so drop that flag once the toolchain moves past 1.27.0.
|
||||
# See https://github.com/golang/go/issues/81101.
|
||||
# errorsastype rewrites errors.As to errors.AsType[T] without checking that T
|
||||
# satisfies error, which breaks on api.HttpError since it does not embed
|
||||
# error. Still broken as of Go 1.27.1.
|
||||
.PHONY: fmt
|
||||
fmt: ## Format Golang codebase and "optimize" the dependencies
|
||||
go fix ./...
|
||||
go fix -embedlit=false -errorsastype=false ./...
|
||||
golangci-lint fmt
|
||||
go mod tidy
|
||||
|
||||
|
||||
@@ -59,8 +59,9 @@ type ProcessSupervisor interface {
|
||||
// Healthy checks and returns the health status of the managed [Process].
|
||||
//
|
||||
// A non-started process is considered healthy (startup is deferred until
|
||||
// the first request). Returns false if the process is currently restarting
|
||||
// or is reported unhealthy by the underlying [Process].
|
||||
// the first request), as is one going through a planned restart, since it
|
||||
// keeps serving traffic. Returns false during an unplanned restart or when
|
||||
// the underlying [Process] reports unhealthy.
|
||||
Healthy() bool
|
||||
|
||||
// Run executes a provided task while managing the state of the [Process].
|
||||
@@ -103,6 +104,28 @@ const healthCheckCacheTTL = 2 * time.Second
|
||||
// this. See https://github.com/gotenberg/gotenberg/issues/1561.
|
||||
const healthFailureThreshold = 2
|
||||
|
||||
// Restart reasons, also reported as the gotenberg.process.start.reason span
|
||||
// attribute by [processSupervisor.tracedLaunch]. Only
|
||||
// [restartReasonMaxRequests] is a planned restart: it fires on a healthy
|
||||
// process that reached its conversion limit, so the node keeps serving
|
||||
// traffic throughout. The others signal a process that cannot serve.
|
||||
const (
|
||||
restartReasonFirstStart = "first_start"
|
||||
restartReasonUnhealthy = "unhealthy"
|
||||
restartReasonMaxRequests = "max_requests"
|
||||
)
|
||||
|
||||
// defaultEagerRestartTimeout bounds the restart triggered after the maximum
|
||||
// request limit. That restart runs on a background context, unlike the one from
|
||||
// ensureHealthy which inherits the request deadline, so without a deadline of
|
||||
// its own the drain loop in [processSupervisor.doRestartLocked] would wait
|
||||
// forever on a task that never completes. That would pin isRestarting and,
|
||||
// with it, the health reported by [processSupervisor.Healthy]. Sized well above
|
||||
// --api-timeout (30s by default) plus the engine start timeouts (20s by
|
||||
// default) so it never fires while tasks are merely slow. The eager restart is
|
||||
// opportunistic: on expiry it aborts, and the next task retries it.
|
||||
const defaultEagerRestartTimeout = 2 * time.Minute
|
||||
|
||||
type processSupervisor struct {
|
||||
logger *slog.Logger
|
||||
engine string
|
||||
@@ -118,11 +141,16 @@ type processSupervisor struct {
|
||||
// transient failure (such as a cold-start timeout) must not poison the
|
||||
// supervisor for the rest of the container's lifetime. See
|
||||
// https://github.com/gotenberg/gotenberg/issues/1538.
|
||||
firstStartMu sync.Mutex
|
||||
reqCounter atomic.Int64
|
||||
reqQueueSize atomic.Int64
|
||||
restartsCounter atomic.Int64
|
||||
isRestarting atomic.Bool
|
||||
firstStartMu sync.Mutex
|
||||
reqCounter atomic.Int64
|
||||
reqQueueSize atomic.Int64
|
||||
restartsCounter atomic.Int64
|
||||
isRestarting atomic.Bool
|
||||
// restartPlanned records whether the in-flight restart is a planned one
|
||||
// (see [restartReasonMaxRequests]). Written before isRestarting and never
|
||||
// cleared, so a reader that observed isRestarting always sees the matching
|
||||
// kind. See [processSupervisor.Healthy].
|
||||
restartPlanned atomic.Bool
|
||||
activeTasks atomic.Int64
|
||||
restartMutex sync.Mutex
|
||||
idleShutdownTimeout time.Duration
|
||||
@@ -135,6 +163,9 @@ type processSupervisor struct {
|
||||
consecutiveHealthFailures atomic.Int64 // reset to 0 on every successful probe
|
||||
idleMu sync.Mutex // protects idleStopChan
|
||||
idleStopChan chan struct{} // signal to stop the idle ticker goroutine
|
||||
// eagerRestartTimeout bounds the restart from maybeRestartAfterTask.
|
||||
// Defaults to [defaultEagerRestartTimeout]; only tests shorten it.
|
||||
eagerRestartTimeout time.Duration
|
||||
}
|
||||
|
||||
// NewProcessSupervisor initializes a new [ProcessSupervisor]. engine names the
|
||||
@@ -158,6 +189,7 @@ func NewProcessSupervisor(logger *slog.Logger, engine string, process Process, m
|
||||
maxQueueSize: maxQueueSize,
|
||||
maxConcurrency: maxConcurrency,
|
||||
idleShutdownTimeout: idleShutdownTimeout,
|
||||
eagerRestartTimeout: defaultEagerRestartTimeout,
|
||||
}
|
||||
b.reqCounter.Store(0)
|
||||
b.reqQueueSize.Store(0)
|
||||
@@ -212,12 +244,18 @@ func (s *processSupervisor) restart() error {
|
||||
s.logger.WarnContext(context.Background(), fmt.Sprintf("stop process before restart: %s", err))
|
||||
}
|
||||
|
||||
// Reset the counter on the attempt, not on its outcome. Leaving it at the
|
||||
// limit after a failed launch re-triggers maybeRestartAfterTask on every
|
||||
// subsequent task, producing back-to-back restarts. Recovering a process
|
||||
// that will not start is ensureHealthy's job: it restarts synchronously
|
||||
// before running a task, and reports the failure to the caller.
|
||||
s.reqCounter.Store(0)
|
||||
|
||||
err = s.Launch()
|
||||
if err != nil {
|
||||
return fmt.Errorf("restart process: %w", err)
|
||||
}
|
||||
|
||||
s.reqCounter.Store(0)
|
||||
s.restartsCounter.Add(1)
|
||||
s.logger.DebugContext(context.Background(), "process successfully restarted")
|
||||
|
||||
@@ -234,9 +272,17 @@ func (s *processSupervisor) Healthy() bool {
|
||||
}
|
||||
|
||||
if s.isRestarting.Load() {
|
||||
// A restarting process is not yet healthy. This gives load balancers
|
||||
// honest information so they can avoid routing traffic to this node.
|
||||
return false
|
||||
// A planned restart is routine maintenance: the process reached the
|
||||
// limit set by --chromium-restart-after (env CHROMIUM_RESTART_AFTER) or
|
||||
// --libreoffice-restart-after (env LIBREOFFICE_RESTART_AFTER) while
|
||||
// healthy. Tasks arriving during it are requeued by acquireSlot, not
|
||||
// rejected, so the node still serves traffic and must report healthy. A
|
||||
// probe sent between two conversions used to fail here.
|
||||
// See https://github.com/gotenberg/gotenberg/issues/1648.
|
||||
//
|
||||
// An unplanned restart keeps reporting unhealthy, which gives load
|
||||
// balancers honest information so they can avoid routing traffic here.
|
||||
return s.restartPlanned.Load()
|
||||
}
|
||||
|
||||
// Cache hit: a recent probe succeeded. Skip the CDP roundtrip so probe
|
||||
@@ -484,7 +530,7 @@ func (s *processSupervisor) ensureStarted(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
err := s.tracedLaunch(ctx, "first_start", func() error {
|
||||
err := s.tracedLaunch(ctx, restartReasonFirstStart, func() error {
|
||||
return s.runWithDeadline(ctx, s.Launch)
|
||||
})
|
||||
if err != nil {
|
||||
@@ -525,7 +571,7 @@ func (s *processSupervisor) ensureHealthy(ctx context.Context) error {
|
||||
|
||||
s.logger.DebugContext(context.Background(), "process is unhealthy, cannot handle task, restarting...")
|
||||
|
||||
if err := s.doRestart(ctx, "unhealthy"); err != nil {
|
||||
if err := s.doRestart(ctx, restartReasonUnhealthy); err != nil {
|
||||
return fmt.Errorf("process restart before task: %w", err)
|
||||
}
|
||||
|
||||
@@ -533,9 +579,10 @@ func (s *processSupervisor) ensureHealthy(ctx context.Context) error {
|
||||
}
|
||||
|
||||
// maybeRestartAfterTask checks if the maximum request limit has been reached
|
||||
// and, if so, triggers an asynchronous restart. If a restart is initiated, it
|
||||
// takes ownership of the caller's semaphore slot (the caller must not release
|
||||
// it). Returns true if ownership was taken.
|
||||
// and, if so, triggers an asynchronous restart bounded by
|
||||
// [defaultEagerRestartTimeout]. If a restart is initiated, it takes ownership
|
||||
// of the caller's semaphore slot (the caller must not release it). Returns true
|
||||
// if ownership was taken.
|
||||
func (s *processSupervisor) maybeRestartAfterTask(logger *slog.Logger) bool {
|
||||
if s.maxReqLimit <= 0 || s.reqCounter.Load() < s.maxReqLimit {
|
||||
return false
|
||||
@@ -548,7 +595,10 @@ func (s *processSupervisor) maybeRestartAfterTask(logger *slog.Logger) bool {
|
||||
s.logger.DebugContext(context.Background(), "max request limit reached, restarting eagerly...")
|
||||
|
||||
go func() {
|
||||
restartErr := s.doRestartLocked(context.Background(), "max_requests")
|
||||
ctx, cancel := context.WithTimeout(context.Background(), s.eagerRestartTimeout)
|
||||
defer cancel()
|
||||
|
||||
restartErr := s.doRestartLocked(ctx, restartReasonMaxRequests)
|
||||
s.restartMutex.Unlock()
|
||||
if restartErr != nil {
|
||||
s.logger.ErrorContext(context.Background(), fmt.Sprintf("process restart after task: %v", restartErr))
|
||||
@@ -571,6 +621,10 @@ func (s *processSupervisor) doRestart(ctx context.Context, reason string) error
|
||||
|
||||
// doRestartLocked performs the restart drain logic. The caller must hold restartMutex.
|
||||
func (s *processSupervisor) doRestartLocked(ctx context.Context, reason string) error {
|
||||
// Publish the kind before raising the flag. [processSupervisor.Healthy]
|
||||
// reads restartPlanned only after it observes isRestarting, so this
|
||||
// ordering keeps it from pairing a new restart with a stale kind.
|
||||
s.restartPlanned.Store(reason == restartReasonMaxRequests)
|
||||
s.isRestarting.Store(true)
|
||||
defer s.isRestarting.Store(false)
|
||||
|
||||
|
||||
@@ -160,13 +160,49 @@ func TestProcessSupervisor_restart(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestProcessSupervisor_restart_ResetsCounterOnFailedLaunch verifies that a
|
||||
// restart whose launch fails still clears the request counter. Leaving it at
|
||||
// the limit makes maybeRestartAfterTask re-fire on every subsequent task.
|
||||
func TestProcessSupervisor_restart_ResetsCounterOnFailedLaunch(t *testing.T) {
|
||||
logger := slog.New(slog.DiscardHandler)
|
||||
|
||||
const maxReqLimit = 5
|
||||
|
||||
process := &ProcessMock{
|
||||
StartMock: func(_ *slog.Logger) error { return errors.New("start error") },
|
||||
StopMock: func(_ *slog.Logger) error { return nil },
|
||||
HealthyMock: func(_ *slog.Logger) bool { return true },
|
||||
}
|
||||
|
||||
ps := NewProcessSupervisor(logger, "test", process, maxReqLimit, 0, 1, 0).(*processSupervisor)
|
||||
ps.reqCounter.Store(maxReqLimit)
|
||||
|
||||
err := ps.restart()
|
||||
if err == nil {
|
||||
t.Fatal("expected error but got none")
|
||||
}
|
||||
|
||||
if got := ps.reqCounter.Load(); got != 0 {
|
||||
t.Fatalf("expected the request counter to be reset but got %d", got)
|
||||
}
|
||||
|
||||
if got := ps.restartsCounter.Load(); got != 0 {
|
||||
t.Fatalf("expected the restarts counter to stay at 0 but got %d", got)
|
||||
}
|
||||
|
||||
if ps.maybeRestartAfterTask(logger) {
|
||||
t.Fatal("expected no further eager restart to be triggered")
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessSupervisor_Healthy(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
scenario string
|
||||
initiallyStarted bool
|
||||
initiallyRestarting bool
|
||||
processHealthy bool
|
||||
expectHealthy bool
|
||||
scenario string
|
||||
initiallyStarted bool
|
||||
initiallyRestarting bool
|
||||
initiallyRestartPlanned bool
|
||||
processHealthy bool
|
||||
expectHealthy bool
|
||||
}{
|
||||
{
|
||||
scenario: "non-started process is healthy",
|
||||
@@ -179,6 +215,13 @@ func TestProcessSupervisor_Healthy(t *testing.T) {
|
||||
initiallyRestarting: true,
|
||||
expectHealthy: false,
|
||||
},
|
||||
{
|
||||
scenario: "process going through a planned restart is healthy",
|
||||
initiallyStarted: true,
|
||||
initiallyRestarting: true,
|
||||
initiallyRestartPlanned: true,
|
||||
expectHealthy: true,
|
||||
},
|
||||
{
|
||||
scenario: "process reports as healthy",
|
||||
initiallyStarted: true,
|
||||
@@ -208,6 +251,9 @@ func TestProcessSupervisor_Healthy(t *testing.T) {
|
||||
if tc.initiallyRestarting {
|
||||
ps.isRestarting.Store(true)
|
||||
}
|
||||
if tc.initiallyRestartPlanned {
|
||||
ps.restartPlanned.Store(true)
|
||||
}
|
||||
|
||||
healthy := ps.Healthy()
|
||||
|
||||
@@ -257,6 +303,204 @@ func TestProcessSupervisor_Healthy_ConsecutiveFailures(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestProcessSupervisor_Healthy_PlannedRestart reproduces
|
||||
// https://github.com/gotenberg/gotenberg/issues/1648. It drives the real
|
||||
// Run() path until the maximum request limit triggers the eager restart, then
|
||||
// asserts the supervisor reports healthy while that restart is in flight.
|
||||
// Tasks arriving during it are requeued by acquireSlot, not rejected, so the
|
||||
// node still serves traffic.
|
||||
func TestProcessSupervisor_Healthy_PlannedRestart(t *testing.T) {
|
||||
logger := slog.New(slog.DiscardHandler)
|
||||
|
||||
const maxReqLimit = 10
|
||||
|
||||
restarting := make(chan struct{})
|
||||
release := make(chan struct{})
|
||||
|
||||
var (
|
||||
starts atomic.Int64
|
||||
signalOne sync.Once
|
||||
)
|
||||
process := &ProcessMock{
|
||||
StartMock: func(_ *slog.Logger) error {
|
||||
// Hold the restart open so the assertions below run inside the
|
||||
// window that used to report unhealthy.
|
||||
if starts.Add(1) > 1 {
|
||||
signalOne.Do(func() { close(restarting) })
|
||||
<-release
|
||||
}
|
||||
|
||||
return nil
|
||||
},
|
||||
StopMock: func(_ *slog.Logger) error { return nil },
|
||||
HealthyMock: func(_ *slog.Logger) bool { return true },
|
||||
}
|
||||
|
||||
ps := NewProcessSupervisor(logger, "test", process, maxReqLimit, 0, 1, 0).(*processSupervisor)
|
||||
|
||||
for i := range maxReqLimit {
|
||||
err := ps.Run(context.Background(), logger, func() error { return nil })
|
||||
if err != nil {
|
||||
t.Fatalf("task %d: expected no error but got: %v", i+1, err)
|
||||
}
|
||||
}
|
||||
|
||||
select {
|
||||
case <-restarting:
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatalf("expected an eager restart after %d tasks", maxReqLimit)
|
||||
}
|
||||
|
||||
if !ps.isRestarting.Load() {
|
||||
t.Fatal("expected the supervisor to be restarting")
|
||||
}
|
||||
|
||||
if !ps.restartPlanned.Load() {
|
||||
t.Fatal("expected the restart to be flagged as planned")
|
||||
}
|
||||
|
||||
if !ps.Healthy() {
|
||||
t.Fatal("expected a planned restart to report healthy")
|
||||
}
|
||||
|
||||
close(release)
|
||||
}
|
||||
|
||||
// TestProcessSupervisor_Healthy_UnplannedRestart verifies the counterpart of
|
||||
// [TestProcessSupervisor_Healthy_PlannedRestart]: a restart triggered by an
|
||||
// unhealthy process keeps reporting unhealthy, so load balancers get honest
|
||||
// information.
|
||||
func TestProcessSupervisor_Healthy_UnplannedRestart(t *testing.T) {
|
||||
logger := slog.New(slog.DiscardHandler)
|
||||
|
||||
restarting := make(chan struct{})
|
||||
release := make(chan struct{})
|
||||
|
||||
var signalOne sync.Once
|
||||
process := &ProcessMock{
|
||||
StartMock: func(_ *slog.Logger) error {
|
||||
signalOne.Do(func() { close(restarting) })
|
||||
<-release
|
||||
|
||||
return nil
|
||||
},
|
||||
StopMock: func(_ *slog.Logger) error { return nil },
|
||||
HealthyMock: func(_ *slog.Logger) bool { return false },
|
||||
}
|
||||
|
||||
ps := NewProcessSupervisor(logger, "test", process, 0, 0, 1, 0).(*processSupervisor)
|
||||
ps.firstStart.Store(true)
|
||||
|
||||
go func() {
|
||||
_ = ps.ensureHealthy(context.Background())
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-restarting:
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("expected an unhealthy restart to be triggered")
|
||||
}
|
||||
|
||||
if !ps.isRestarting.Load() {
|
||||
t.Fatal("expected the supervisor to be restarting")
|
||||
}
|
||||
|
||||
if ps.restartPlanned.Load() {
|
||||
t.Fatal("expected the restart not to be flagged as planned")
|
||||
}
|
||||
|
||||
if ps.Healthy() {
|
||||
t.Fatal("expected an unplanned restart to report unhealthy")
|
||||
}
|
||||
|
||||
close(release)
|
||||
}
|
||||
|
||||
// TestProcessSupervisor_doRestartLocked_DrainDeadline verifies that a drain
|
||||
// unable to acquire every slot gives up on the context deadline and clears
|
||||
// isRestarting. Without a deadline on the eager restart, a task that never
|
||||
// completes would pin the flag and, since a planned restart reports healthy,
|
||||
// leave the supervisor claiming health forever.
|
||||
func TestProcessSupervisor_doRestartLocked_DrainDeadline(t *testing.T) {
|
||||
logger := slog.New(slog.DiscardHandler)
|
||||
|
||||
var starts atomic.Int64
|
||||
process := &ProcessMock{
|
||||
StartMock: func(_ *slog.Logger) error {
|
||||
starts.Add(1)
|
||||
|
||||
return nil
|
||||
},
|
||||
StopMock: func(_ *slog.Logger) error { return nil },
|
||||
HealthyMock: func(_ *slog.Logger) bool { return true },
|
||||
}
|
||||
|
||||
// A concurrency of 2 makes the drain acquire one slot on top of the one the
|
||||
// triggering task hands over. Fill the semaphore so it never can, mimicking
|
||||
// a concurrent task that never completes.
|
||||
ps := NewProcessSupervisor(logger, "test", process, 1, 0, 2, 0).(*processSupervisor)
|
||||
ps.semaphore <- struct{}{}
|
||||
ps.semaphore <- struct{}{}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
|
||||
defer cancel()
|
||||
|
||||
err := ps.doRestartLocked(ctx, restartReasonMaxRequests)
|
||||
if err == nil {
|
||||
t.Fatal("expected the drain to fail on the context deadline")
|
||||
}
|
||||
|
||||
if ps.isRestarting.Load() {
|
||||
t.Fatal("expected isRestarting to be cleared after a failed drain")
|
||||
}
|
||||
|
||||
if starts.Load() != 0 {
|
||||
t.Fatalf("expected no restart attempt after a failed drain but got %d", starts.Load())
|
||||
}
|
||||
}
|
||||
|
||||
// TestProcessSupervisor_maybeRestartAfterTask_Bounded verifies that the eager
|
||||
// restart runs under a deadline. A concurrent task that never completes blocks
|
||||
// the drain, and without a bound the restart goroutine would wait forever with
|
||||
// isRestarting pinned, leaving Healthy() reporting a planned restart for good.
|
||||
func TestProcessSupervisor_maybeRestartAfterTask_Bounded(t *testing.T) {
|
||||
logger := slog.New(slog.DiscardHandler)
|
||||
|
||||
process := &ProcessMock{
|
||||
StartMock: func(_ *slog.Logger) error { return nil },
|
||||
StopMock: func(_ *slog.Logger) error { return nil },
|
||||
HealthyMock: func(_ *slog.Logger) bool { return true },
|
||||
}
|
||||
|
||||
ps := NewProcessSupervisor(logger, "test", process, 1, 0, 2, 0).(*processSupervisor)
|
||||
ps.eagerRestartTimeout = 100 * time.Millisecond
|
||||
ps.firstStart.Store(true)
|
||||
|
||||
// Wedge one slot so the drain, which needs one on top of the slot the
|
||||
// triggering task hands over, can never complete.
|
||||
ps.semaphore <- struct{}{}
|
||||
|
||||
err := ps.Run(context.Background(), logger, func() error { return nil })
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error but got: %v", err)
|
||||
}
|
||||
|
||||
waitFor := func(what string, want bool) {
|
||||
t.Helper()
|
||||
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for ps.isRestarting.Load() != want {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("timed out waiting for the eager restart to %s", what)
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
waitFor("start", true)
|
||||
waitFor("give up on its deadline", false)
|
||||
}
|
||||
|
||||
// TestProcessSupervisor_Healthy_CachesPositiveResult verifies that a
|
||||
// successful probe is cached for [healthCheckCacheTTL] so subsequent
|
||||
// supervisor.Healthy() calls do not re-issue the underlying process
|
||||
|
||||
@@ -361,7 +361,7 @@ func TestSofficeProxyEnv_OverridesExisting(t *testing.T) {
|
||||
// Old proxy values must be gone, not duplicated. Count exact-case keys.
|
||||
counts := map[string]int{}
|
||||
for _, kv := range out {
|
||||
key := strings.SplitN(kv, "=", 2)[0]
|
||||
key, _, _ := strings.Cut(kv, "=")
|
||||
counts[key]++
|
||||
}
|
||||
for _, key := range []string{"http_proxy", "HTTP_PROXY", "https_proxy", "HTTPS_PROXY", "no_proxy", "NO_PROXY"} {
|
||||
|
||||
@@ -180,7 +180,7 @@ func (engine *PdfCpu) listImages(ctx context.Context, inputPath string) ([]pdfcp
|
||||
// their second column is not a numeric object number.
|
||||
func parseImagesList(output string) []pdfcpuImage {
|
||||
var images []pdfcpuImage
|
||||
for _, line := range strings.Split(output, "\n") {
|
||||
for line := range strings.SplitSeq(output, "\n") {
|
||||
cols := strings.Split(line, "│")
|
||||
if len(cols) < 9 {
|
||||
continue
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
# TODO:
|
||||
# 1. Check if down for each module.
|
||||
# 2. Restarting modules do not make health check fail.
|
||||
|
||||
@health
|
||||
Feature: /health
|
||||
@@ -106,7 +105,19 @@ Feature: /health
|
||||
When I make a "HEAD" request to Gotenberg at the "/foo/health" endpoint
|
||||
Then the response status code should be 200
|
||||
|
||||
# A planned restart, the eager cycle after LIBREOFFICE_RESTART_AFTER
|
||||
# conversions, must not fail the health check: requests arriving during it
|
||||
# are requeued, not rejected. Setting the limit to 1 restarts LibreOffice
|
||||
# after every conversion, so each probe lands right on a restart.
|
||||
# See https://github.com/gotenberg/gotenberg/issues/1648.
|
||||
Scenario: GET /health (Planned LibreOffice Restart)
|
||||
Given I have a Gotenberg container with the following environment variable(s):
|
||||
| LIBREOFFICE_RESTART_AFTER | 1 |
|
||||
When I make 5 sequential "POST" requests to Gotenberg at the "/forms/libreoffice/convert" endpoint, probing "/health" after each, with the following form data and header(s):
|
||||
| files | testdata/page_1.docx | file |
|
||||
Then all sequential response status codes should be 200
|
||||
Then all probe response status codes should be 200
|
||||
|
||||
|
||||
# TODO:
|
||||
# 1. Check if down for each module.
|
||||
# 2. Restarting modules do not make health check fail.
|
||||
# 1. Check if down for each module.
|
||||
@@ -88,6 +88,8 @@ func findScenarioLine(filePath, name string) int {
|
||||
type scenario struct {
|
||||
resp *httptest.ResponseRecorder
|
||||
concurrentResps []*httptest.ResponseRecorder
|
||||
probeResps []*httptest.ResponseRecorder
|
||||
sequentialResps []*httptest.ResponseRecorder
|
||||
workdir string
|
||||
teststoreDir string
|
||||
gotenbergContainer testcontainers.Container
|
||||
@@ -99,6 +101,8 @@ type scenario struct {
|
||||
func (s *scenario) reset(ctx context.Context) error {
|
||||
s.resp = httptest.NewRecorder()
|
||||
s.concurrentResps = nil
|
||||
s.probeResps = nil
|
||||
s.sequentialResps = nil
|
||||
|
||||
err := os.RemoveAll(s.workdir)
|
||||
if err != nil {
|
||||
@@ -470,6 +474,130 @@ func (s *scenario) iMakeConcurrentRequestsToGotenberg(ctx context.Context, count
|
||||
return nil
|
||||
}
|
||||
|
||||
// iMakeSequentialRequestsToGotenbergProbing mirrors the client loop from
|
||||
// https://github.com/gotenberg/gotenberg/issues/1648: a conversion, then a
|
||||
// probe, repeated. It records both the conversion and the probe responses so a
|
||||
// scenario can assert that a planned process restart neither fails the probe
|
||||
// nor breaks the conversions. Requests are sequential on purpose, since the bug
|
||||
// only surfaces between two conversions.
|
||||
func (s *scenario) iMakeSequentialRequestsToGotenbergProbing(ctx context.Context, count int, method, endpoint, probeEndpoint string, dataTable *godog.Table) error {
|
||||
if s.gotenbergContainer == nil {
|
||||
return errors.New("no Gotenberg container")
|
||||
}
|
||||
|
||||
fields := make(map[string][]string)
|
||||
files := make(map[string][]string)
|
||||
headers := make(map[string]string)
|
||||
|
||||
for _, row := range dataTable.Rows {
|
||||
name := row.Cells[0].Value
|
||||
value := row.Cells[1].Value
|
||||
kind := row.Cells[2].Value
|
||||
|
||||
switch kind {
|
||||
case "field":
|
||||
fields[name] = append(fields[name], value)
|
||||
case "file":
|
||||
wd, err := os.Getwd()
|
||||
if err != nil {
|
||||
return fmt.Errorf("get current directory: %w", err)
|
||||
}
|
||||
value = fmt.Sprintf("%s/%s", wd, value)
|
||||
files[name] = append(files[name], value)
|
||||
case "header":
|
||||
headers[name] = value
|
||||
default:
|
||||
return fmt.Errorf("unexpected %q %q", kind, value)
|
||||
}
|
||||
}
|
||||
|
||||
base, err := containerHttpEndpoint(ctx, s.gotenbergContainer, "3000")
|
||||
if err != nil {
|
||||
return fmt.Errorf("get container HTTP endpoint: %w", err)
|
||||
}
|
||||
|
||||
record := func(resp *http.Response) (*httptest.ResponseRecorder, error) {
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, readErr := io.ReadAll(resp.Body)
|
||||
if readErr != nil {
|
||||
return nil, fmt.Errorf("read response body: %w", readErr)
|
||||
}
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
rec.Code = resp.StatusCode
|
||||
for key, values := range resp.Header {
|
||||
for _, v := range values {
|
||||
rec.Header().Add(key, v)
|
||||
}
|
||||
}
|
||||
_, writeErr := rec.Body.Write(body)
|
||||
if writeErr != nil {
|
||||
return nil, fmt.Errorf("write response body: %w", writeErr)
|
||||
}
|
||||
|
||||
return rec, nil
|
||||
}
|
||||
|
||||
s.probeResps = make([]*httptest.ResponseRecorder, 0, count)
|
||||
s.sequentialResps = make([]*httptest.ResponseRecorder, 0, count)
|
||||
|
||||
for i := range count {
|
||||
resp, reqErr := doFormDataRequest(method, fmt.Sprintf("%s%s", base, endpoint), fields, files, headers)
|
||||
if reqErr != nil {
|
||||
return fmt.Errorf("request %d: do request: %w", i+1, reqErr)
|
||||
}
|
||||
|
||||
rec, recErr := record(resp)
|
||||
if recErr != nil {
|
||||
return fmt.Errorf("request %d: %w", i+1, recErr)
|
||||
}
|
||||
s.resp = rec
|
||||
s.sequentialResps = append(s.sequentialResps, rec)
|
||||
|
||||
probeResp, probeErr := doRequest(http.MethodGet, fmt.Sprintf("%s%s", base, probeEndpoint), nil, nil)
|
||||
if probeErr != nil {
|
||||
return fmt.Errorf("probe %d: do request: %w", i+1, probeErr)
|
||||
}
|
||||
|
||||
probeRec, probeRecErr := record(probeResp)
|
||||
if probeRecErr != nil {
|
||||
return fmt.Errorf("probe %d: %w", i+1, probeRecErr)
|
||||
}
|
||||
s.probeResps = append(s.probeResps, probeRec)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *scenario) allProbeResponseStatusCodesShouldBe(expected int) error {
|
||||
if len(s.probeResps) == 0 {
|
||||
return errors.New("no probe responses recorded")
|
||||
}
|
||||
|
||||
for i, resp := range s.probeResps {
|
||||
if resp.Code != expected {
|
||||
return fmt.Errorf("probe %d: expected status %d, got %d %q", i+1, expected, resp.Code, resp.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *scenario) allSequentialResponseStatusCodesShouldBe(expected int) error {
|
||||
if len(s.sequentialResps) == 0 {
|
||||
return errors.New("no sequential responses recorded")
|
||||
}
|
||||
|
||||
for i, resp := range s.sequentialResps {
|
||||
if resp.Code != expected {
|
||||
return fmt.Errorf("sequential response %d: expected status %d, got %d %q", i+1, expected, resp.Code, resp.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *scenario) allConcurrentResponseStatusCodesShouldBe(expected int) error {
|
||||
if len(s.concurrentResps) == 0 {
|
||||
return errors.New("no concurrent responses recorded")
|
||||
@@ -1644,10 +1772,13 @@ func InitializeScenario(ctx *godog.ScenarioContext) {
|
||||
ctx.When(`^I make a "(GET|HEAD)" request to Gotenberg at the "([^"]*)" endpoint with the following header\(s\):$`, s.iMakeARequestToGotenbergWithTheFollowingHeaders)
|
||||
ctx.When(`^I make a "(POST)" request to Gotenberg at the "([^"]*)" endpoint with the following form data and header\(s\):$`, s.iMakeARequestToGotenbergWithTheFollowingFormDataAndHeaders)
|
||||
ctx.When(`^I make (\d+) concurrent "(POST)" requests to Gotenberg at the "([^"]*)" endpoint with the following form data and header\(s\):$`, s.iMakeConcurrentRequestsToGotenberg)
|
||||
ctx.When(`^I make (\d+) sequential "(POST)" requests to Gotenberg at the "([^"]*)" endpoint, probing "([^"]*)" after each, with the following form data and header\(s\):$`, s.iMakeSequentialRequestsToGotenbergProbing)
|
||||
ctx.When(`^I wait for the asynchronous request to the webhook$`, s.iWaitForTheAsynchronousRequestToWebhook)
|
||||
ctx.Then(`^the Gotenberg container (should|should NOT) log the following entries:$`, s.theGotenbergContainerShouldLogTheFollowingEntries)
|
||||
ctx.Then(`^the response status code should be (\d+)$`, s.theResponseStatusCodeShouldBe)
|
||||
ctx.Then(`^all concurrent response status codes should be (\d+)$`, s.allConcurrentResponseStatusCodesShouldBe)
|
||||
ctx.Then(`^all probe response status codes should be (\d+)$`, s.allProbeResponseStatusCodesShouldBe)
|
||||
ctx.Then(`^all sequential response status codes should be (\d+)$`, s.allSequentialResponseStatusCodesShouldBe)
|
||||
ctx.Then(`^all concurrent responses should have (\d+) PDF\(s\)$`, s.allConcurrentResponsesShouldHavePdfs)
|
||||
ctx.Then(`^the (response|webhook request|file request|server request) header "([^"]*)" should be "([^"]*)"$`, s.theHeaderValueShouldBe)
|
||||
ctx.Then(`^the webhook request header "([^"]*)" should carry trace id "([^"]*)"$`, s.theWebhookRequestHeaderShouldCarryTraceID)
|
||||
|
||||
Reference in New Issue
Block a user