feat: eagerly auto restart processes (#916)

* Fix auto-restart's initial number of requests
It would need 1 less requests on the first restart
The number of request was technically off by one after the first restart

* Supervisor now eagerly restarts processes
When the maxReqLimit is reached, restart the process in the background

* Update pkg/gotenberg/supervisor.go

Co-authored-by: Julien Neuhart <neuhart.julien@gmail.com>

---------

Co-authored-by: Julien Neuhart <neuhart.julien@gmail.com>

pkg/gotenberg/supervisor.go

@@ -187,10 +187,13 @@ func (s *processSupervisor) Run(ctx context.Context, logger *zap.Logger, task fu
 				logger.Debug("process lock acquired")
 				s.reqQueueSize.Add(-1)
 				s.reqCounter.Add(1)
+				releaseMutexChan := true
 
 				defer func() {
-					logger.Debug("process lock released")
-					<-s.mutexChan
+					if releaseMutexChan {
+						logger.Debug("process lock released")
+						<-s.mutexChan
+					}
 				}()
 
 				if !s.firstStart.Load() {
@@ -212,18 +215,26 @@ func (s *processSupervisor) Run(ctx context.Context, logger *zap.Logger, task fu
 					}
 				}
 
+				err := s.runWithDeadline(ctx, task)
+
 				if s.maxReqLimit > 0 && s.reqCounter.Load() >= s.maxReqLimit {
-					s.logger.Debug("max request limit reached, restarting...")
-					err := s.runWithDeadline(ctx, func() error {
-						return s.restart()
-					})
-					if err != nil {
-						return fmt.Errorf("process restart before task: %w", err)
-					}
+					s.logger.Debug("max request limit reached, restarting eagerly...")
+					releaseMutexChan = false
+
+					go func() {
+						err := s.runWithDeadline(context.Background(), func() error {
+							return s.restart()
+						})
+						if err != nil {
+							s.logger.Error(fmt.Sprintf("process restart after task: %v", err))
+						}
+						logger.Debug("process lock released")
+						<-s.mutexChan
+					}()
 				}
 
 				// Note: no error wrapping because it leaks on Chromium console exceptions output.
-				return s.runWithDeadline(ctx, task)
+				return err
 			case <-ctx.Done():
 				logger.Debug("failed to acquire process lock before deadline")
 				s.reqQueueSize.Add(-1)

pkg/gotenberg/supervisor_test.go

@@ -325,7 +325,7 @@ func TestProcessSupervisor_Run(t *testing.T) {
 			expectedStopCalls:    1,
 		},
 		{
-			scenario:             "cannot restart after reaching max request limit",
+			scenario:             "auto-restart after reaching max request limit",
 			startError:           errors.New("start error"),
 			initiallyStarted:     true,
 			isRestarting:         false,
@@ -451,6 +451,10 @@ func TestProcessSupervisor_Run(t *testing.T) {
 				return
 			}
 
+			// Making sure restarts are finished.
+			ps.mutexChan <- struct{}{}
+			<-ps.mutexChan
+
 			if startCalls.Load() != tc.expectedStartCalls {
 				t.Errorf("expected %d process.Start calls, got %d", tc.expectedStartCalls, startCalls.Load())
 			}