diff --git a/Makefile b/Makefile index e6cc3bd6..8407fe71 100644 --- a/Makefile +++ b/Makefile @@ -127,6 +127,9 @@ NO_CONCURRENCY=false # chromium-convert-html # chromium-convert-markdown # chromium-convert-url +# chromium-screenshot-html +# chromium-screenshot-markdown +# chromium-screenshot-url # debug # health # libreoffice @@ -162,7 +165,7 @@ NO_CONCURRENCY=false TAGS= .PHONY: test-integration -test-integration: ## Run integration tests +test-integration: ## Run integration tests (automatically retries failed scenarios up to 3 times) go test -timeout 40m -tags=integration -v github.com/gotenberg/gotenberg/v8/test/integration -args \ --gotenberg-docker-repository=$(DOCKER_REPOSITORY) \ --gotenberg-version=$(GOTENBERG_VERSION) \ diff --git a/test/integration/main_test.go b/test/integration/main_test.go index 67144b22..e96471ff 100644 --- a/test/integration/main_test.go +++ b/test/integration/main_test.go @@ -3,6 +3,7 @@ package integration import ( + "fmt" "os" "runtime" "testing" @@ -40,17 +41,42 @@ func TestMain(m *testing.M) { concurrency = 0 } - code := godog.TestSuite{ - Name: "integration", - ScenarioInitializer: scenario.InitializeScenario, - Options: &godog.Options{ - Format: "pretty", - Paths: []string{"features"}, - Output: colors.Colored(os.Stdout), - Concurrency: concurrency, - Tags: *tags, - }, - }.Run() + maxAttempts := 4 // 1 initial run + up to 3 retries + paths := []string{"features"} - os.Exit(code) + for attempt := 1; attempt <= maxAttempts; attempt++ { + // Reset the failure collector before each run. + scenario.ResetFailedScenarios() + + code := godog.TestSuite{ + Name: "integration", + ScenarioInitializer: scenario.InitializeScenario, + Options: &godog.Options{ + Format: "pretty", + Paths: paths, + Output: colors.Colored(os.Stdout), + Concurrency: concurrency, + Tags: *tags, + }, + }.Run() + + if code == 0 { + os.Exit(0) + } + + failedPaths := scenario.FailedScenarioPaths() + if len(failedPaths) == 0 || attempt == maxAttempts { + os.Exit(code) + } + + fmt.Fprintf(colors.Colored(os.Stdout), + "\n\n--- %d scenario(s) failed, retrying (%d retries left) ---\n\n", + len(failedPaths), maxAttempts-attempt, + ) + + // Next run: only the failed scenarios via file:line paths. + paths = failedPaths + } + + os.Exit(1) } diff --git a/test/integration/scenario/scenario.go b/test/integration/scenario/scenario.go index 390aeebb..4a66238d 100644 --- a/test/integration/scenario/scenario.go +++ b/test/integration/scenario/scenario.go @@ -23,6 +23,66 @@ import ( "github.com/testcontainers/testcontainers-go" ) +var ( + failedScenariosMu sync.Mutex + failedScenarios []string // file:line paths for re-run +) + +// ResetFailedScenarios clears the collected failures. +func ResetFailedScenarios() { + failedScenariosMu.Lock() + defer failedScenariosMu.Unlock() + failedScenarios = nil +} + +// FailedScenarioPaths returns file:line paths of failed scenarios. +func FailedScenarioPaths() []string { + failedScenariosMu.Lock() + defer failedScenariosMu.Unlock() + result := make([]string, len(failedScenarios)) + copy(result, failedScenarios) + return result +} + +func recordFailedScenario(sc *godog.Scenario) { + // When godog runs with file:line paths, the Uri may contain + // the line suffix (e.g., "features/root.feature:4"). Strip it + // to get the actual file path for reading. + filePath := sc.Uri + if idx := strings.LastIndex(filePath, ":"); idx > 0 { + if _, err := strconv.Atoi(filePath[idx+1:]); err == nil { + filePath = filePath[:idx] + } + } + + line := findScenarioLine(filePath, sc.Name) + if line <= 0 { + return + } + + path := fmt.Sprintf("%s:%d", filePath, line) + + failedScenariosMu.Lock() + defer failedScenariosMu.Unlock() + failedScenarios = append(failedScenarios, path) +} + +func findScenarioLine(filePath, name string) int { + data, err := os.ReadFile(filePath) + if err != nil { + return 0 + } + + target := "Scenario: " + name + for i, line := range strings.Split(string(data), "\n") { + if strings.TrimSpace(line) == target { + return i + 1 + } + } + + return 0 +} + type scenario struct { resp *httptest.ResponseRecorder concurrentResps []*httptest.ResponseRecorder @@ -1226,6 +1286,10 @@ func InitializeScenario(ctx *godog.ScenarioContext) { return ctx, nil }) ctx.After(func(ctx context.Context, sc *godog.Scenario, err error) (context.Context, error) { + if err != nil { + recordFailedScenario(sc) + } + errReset := s.reset(ctx) if errReset != nil { return ctx, fmt.Errorf("reset scenario: %w", errReset)