adalal/gotenberg
1
0
Fork 0
mirror of https://github.com/gotenberg/gotenberg.git synced 2026-10-14 00:03:18 +01:00
Code Issues Packages Projects Releases Wiki Activity

fix(supervisor): report healthy during planned process restarts

Browse Source 
The eager restart fired after --chromium-restart-after or
--libreoffice-restart-after conversions made Healthy() report false,
so a client probing /health between two conversions got a 503 from an
otherwise serving node. Tasks arriving during that window are requeued
by acquireSlot, not rejected.

Track whether the in-flight restart is planned and keep reporting
healthy for those. Unplanned restarts still report unhealthy so load
balancers get honest information.

Closes #1648
This commit is contained in:
Julien Neuhart
2026-09-03 16:04:20 +02:00
parent 8944db131c
commit 88ddaed09b
4 changed files with 295 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
pkg/gotenberg/supervisor.go
View File

@@ -59,8 +59,9 @@ type ProcessSupervisor interface {
// Healthy checks and returns the health status of the managed [Process].
//
// A non-started process is considered healthy (startup is deferred until
// the first request). Returns false if the process is currently restarting
// or is reported unhealthy by the underlying [Process].
// the first request), as is one going through a planned restart, since it
// keeps serving traffic. Returns false during an unplanned restart or when
// the underlying [Process] reports unhealthy.
Healthy() bool
// Run executes a provided task while managing the state of the [Process].
@@ -103,6 +104,17 @@ const healthCheckCacheTTL = 2 * time.Second
// this. See https://github.com/gotenberg/gotenberg/issues/1561.
const healthFailureThreshold = 2
// Restart reasons, also reported as the gotenberg.process.start.reason span
// attribute by [processSupervisor.tracedLaunch]. Only
// [restartReasonMaxRequests] is a planned restart: it fires on a healthy
// process that reached its conversion limit, so the node keeps serving
// traffic throughout. The others signal a process that cannot serve.
const (
restartReasonFirstStart = "first_start"
restartReasonUnhealthy = "unhealthy"
restartReasonMaxRequests = "max_requests"
)
type processSupervisor struct {
logger *slog.Logger
engine string
@@ -118,11 +130,16 @@ type processSupervisor struct {
// transient failure (such as a cold-start timeout) must not poison the
// supervisor for the rest of the container's lifetime. See
// https://github.com/gotenberg/gotenberg/issues/1538.
firstStartMu sync.Mutex
reqCounter atomic.Int64
reqQueueSize atomic.Int64
restartsCounter atomic.Int64
isRestarting atomic.Bool
firstStartMu sync.Mutex
reqCounter atomic.Int64
reqQueueSize atomic.Int64
restartsCounter atomic.Int64
isRestarting atomic.Bool
// restartPlanned records whether the in-flight restart is a planned one
// (see [restartReasonMaxRequests]). Written before isRestarting and never
// cleared, so a reader that observed isRestarting always sees the matching
// kind. See [processSupervisor.Healthy].
restartPlanned atomic.Bool
activeTasks atomic.Int64
restartMutex sync.Mutex
idleShutdownTimeout time.Duration
@@ -234,9 +251,17 @@ func (s *processSupervisor) Healthy() bool {
}
if s.isRestarting.Load() {
// A restarting process is not yet healthy. This gives load balancers
// honest information so they can avoid routing traffic to this node.
return false
// A planned restart is routine maintenance: the process reached the
// limit set by --chromium-restart-after (env CHROMIUM_RESTART_AFTER) or
// --libreoffice-restart-after (env LIBREOFFICE_RESTART_AFTER) while
// healthy. Tasks arriving during it are requeued by acquireSlot, not
// rejected, so the node still serves traffic and must report healthy. A
// probe sent between two conversions used to fail here.
// See https://github.com/gotenberg/gotenberg/issues/1648.
//
// An unplanned restart keeps reporting unhealthy, which gives load
// balancers honest information so they can avoid routing traffic here.
return s.restartPlanned.Load()
}
// Cache hit: a recent probe succeeded. Skip the CDP roundtrip so probe
@@ -484,7 +509,7 @@ func (s *processSupervisor) ensureStarted(ctx context.Context) error {
return nil
}
err := s.tracedLaunch(ctx, "first_start", func() error {
err := s.tracedLaunch(ctx, restartReasonFirstStart, func() error {
return s.runWithDeadline(ctx, s.Launch)
})
if err != nil {
@@ -525,7 +550,7 @@ func (s *processSupervisor) ensureHealthy(ctx context.Context) error {
s.logger.DebugContext(context.Background(), "process is unhealthy, cannot handle task, restarting...")
if err := s.doRestart(ctx, "unhealthy"); err != nil {
if err := s.doRestart(ctx, restartReasonUnhealthy); err != nil {
return fmt.Errorf("process restart before task: %w", err)
}
@@ -548,7 +573,7 @@ func (s *processSupervisor) maybeRestartAfterTask(logger *slog.Logger) bool {
s.logger.DebugContext(context.Background(), "max request limit reached, restarting eagerly...")
go func() {
restartErr := s.doRestartLocked(context.Background(), "max_requests")
restartErr := s.doRestartLocked(context.Background(), restartReasonMaxRequests)
s.restartMutex.Unlock()
if restartErr != nil {
s.logger.ErrorContext(context.Background(), fmt.Sprintf("process restart after task: %v", restartErr))
@@ -571,6 +596,10 @@ func (s *processSupervisor) doRestart(ctx context.Context, reason string) error
// doRestartLocked performs the restart drain logic. The caller must hold restartMutex.
func (s *processSupervisor) doRestartLocked(ctx context.Context, reason string) error {
// Publish the kind before raising the flag. [processSupervisor.Healthy]
// reads restartPlanned only after it observes isRestarting, so this
// ordering keeps it from pairing a new restart with a stale kind.
s.restartPlanned.Store(reason == restartReasonMaxRequests)
s.isRestarting.Store(true)
defer s.isRestarting.Store(false)

134
pkg/gotenberg/supervisor_test.go
View File

@@ -162,11 +162,12 @@ func TestProcessSupervisor_restart(t *testing.T) {
func TestProcessSupervisor_Healthy(t *testing.T) {
for _, tc := range []struct {
scenario string
initiallyStarted bool
initiallyRestarting bool
processHealthy bool
expectHealthy bool
scenario string
initiallyStarted bool
initiallyRestarting bool
initiallyRestartPlanned bool
processHealthy bool
expectHealthy bool
}{
{
scenario: "non-started process is healthy",
@@ -179,6 +180,13 @@ func TestProcessSupervisor_Healthy(t *testing.T) {
initiallyRestarting: true,
expectHealthy: false,
},
{
scenario: "process going through a planned restart is healthy",
initiallyStarted: true,
initiallyRestarting: true,
initiallyRestartPlanned: true,
expectHealthy: true,
},
{
scenario: "process reports as healthy",
initiallyStarted: true,
@@ -208,6 +216,9 @@ func TestProcessSupervisor_Healthy(t *testing.T) {
if tc.initiallyRestarting {
ps.isRestarting.Store(true)
}
if tc.initiallyRestartPlanned {
ps.restartPlanned.Store(true)
}
healthy := ps.Healthy()
@@ -257,6 +268,119 @@ func TestProcessSupervisor_Healthy_ConsecutiveFailures(t *testing.T) {
}
}
// TestProcessSupervisor_Healthy_PlannedRestart reproduces
// https://github.com/gotenberg/gotenberg/issues/1648. It drives the real
// Run() path until the maximum request limit triggers the eager restart, then
// asserts the supervisor reports healthy while that restart is in flight.
// Tasks arriving during it are requeued by acquireSlot, not rejected, so the
// node still serves traffic.
func TestProcessSupervisor_Healthy_PlannedRestart(t *testing.T) {
logger := slog.New(slog.DiscardHandler)
const maxReqLimit = 10
restarting := make(chan struct{})
release := make(chan struct{})
var (
starts atomic.Int64
signalOne sync.Once
)
process := &ProcessMock{
StartMock: func(_ *slog.Logger) error {
// Hold the restart open so the assertions below run inside the
// window that used to report unhealthy.
if starts.Add(1) > 1 {
signalOne.Do(func() { close(restarting) })
<-release
}
return nil
},
StopMock: func(_ *slog.Logger) error { return nil },
HealthyMock: func(_ *slog.Logger) bool { return true },
}
ps := NewProcessSupervisor(logger, "test", process, maxReqLimit, 0, 1, 0).(*processSupervisor)
for i := range maxReqLimit {
err := ps.Run(context.Background(), logger, func() error { return nil })
if err != nil {
t.Fatalf("task %d: expected no error but got: %v", i+1, err)
}
}
select {
case <-restarting:
case <-time.After(10 * time.Second):
t.Fatalf("expected an eager restart after %d tasks", maxReqLimit)
}
if !ps.isRestarting.Load() {
t.Fatal("expected the supervisor to be restarting")
}
if !ps.restartPlanned.Load() {
t.Fatal("expected the restart to be flagged as planned")
}
if !ps.Healthy() {
t.Fatal("expected a planned restart to report healthy")
}
close(release)
}
// TestProcessSupervisor_Healthy_UnplannedRestart verifies the counterpart of
// [TestProcessSupervisor_Healthy_PlannedRestart]: a restart triggered by an
// unhealthy process keeps reporting unhealthy, so load balancers get honest
// information.
func TestProcessSupervisor_Healthy_UnplannedRestart(t *testing.T) {
logger := slog.New(slog.DiscardHandler)
restarting := make(chan struct{})
release := make(chan struct{})
var signalOne sync.Once
process := &ProcessMock{
StartMock: func(_ *slog.Logger) error {
signalOne.Do(func() { close(restarting) })
<-release
return nil
},
StopMock: func(_ *slog.Logger) error { return nil },
HealthyMock: func(_ *slog.Logger) bool { return false },
}
ps := NewProcessSupervisor(logger, "test", process, 0, 0, 1, 0).(*processSupervisor)
ps.firstStart.Store(true)
go func() {
_ = ps.ensureHealthy(context.Background())
}()
select {
case <-restarting:
case <-time.After(10 * time.Second):
t.Fatal("expected an unhealthy restart to be triggered")
}
if !ps.isRestarting.Load() {
t.Fatal("expected the supervisor to be restarting")
}
if ps.restartPlanned.Load() {
t.Fatal("expected the restart not to be flagged as planned")
}
if ps.Healthy() {
t.Fatal("expected an unplanned restart to report unhealthy")
}
close(release)
}
// TestProcessSupervisor_Healthy_CachesPositiveResult verifies that a
// successful probe is cached for [healthCheckCacheTTL] so subsequent
// supervisor.Healthy() calls do not re-issue the underlying process