mirror of
https://github.com/gotenberg/gotenberg.git
synced 2026-10-17 17:53:17 +01:00
Compare commits
14 Commits
9c5acd7418
...
ac825a2c03
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ac825a2c03 | ||
|
|
2c9fa6b6ed | ||
|
|
ca8b45cd3a | ||
|
|
40cf48442f | ||
|
|
df3bac99ed | ||
|
|
17868b8c02 | ||
|
|
78284df590 | ||
|
|
8f415186d5 | ||
|
|
f675f78f77 | ||
|
|
83b01c2baa | ||
|
|
9a46fdd681 | ||
|
|
4de9b0f68b | ||
|
|
201e80b9d7 | ||
|
|
86a013b664 |
8
Makefile
8
Makefile
@@ -33,7 +33,10 @@ API_OIDC_ISSUER=
|
||||
API_OIDC_AUDIENCE=
|
||||
API_OIDC_JWKS_URL=
|
||||
API_DOWNLOAD_FROM_ALLOW_LIST=
|
||||
API_DOWNLOAD_FROM_DENY_LIST=^https?://(10\.|172\.(1[6-9]|2[0-9]|3[01])\.|192\.168\.|169\.254\.|0\.0\.0\.0|127\.|localhost|\[::1\]|\[fd)
|
||||
# Empty, like the flag default since 8.32.0. A textual deny-list cannot
|
||||
# enumerate every way to write a private address, so *_DENY_PRIVATE_IPS is the
|
||||
# control to reach for. Left false here so local testing can reach the host.
|
||||
API_DOWNLOAD_FROM_DENY_LIST=
|
||||
API_DOWNLOAD_FROM_DENY_PRIVATE_IPS=false
|
||||
API_DOWNLOAD_FROM_DENY_PUBLIC_IPS=false
|
||||
API_DOWNLOAD_FROM_ENABLE_ENVIRONMENT_PROXY=false
|
||||
@@ -114,7 +117,8 @@ OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317
|
||||
OTEL_EXPORTER_OTLP_INSECURE=true
|
||||
WEBHOOK_ENABLE_SYNC_MODE=false
|
||||
WEBHOOK_ALLOW_LIST=
|
||||
WEBHOOK_DENY_LIST=^https?://(10\.|172\.(1[6-9]|2[0-9]|3[01])\.|192\.168\.|169\.254\.|0\.0\.0\.0|127\.|localhost|\[::1\]|\[fd)
|
||||
# See the note on API_DOWNLOAD_FROM_DENY_LIST.
|
||||
WEBHOOK_DENY_LIST=
|
||||
WEBHOOK_DENY_PRIVATE_IPS=false
|
||||
WEBHOOK_DENY_PUBLIC_IPS=false
|
||||
WEBHOOK_ENABLE_ENVIRONMENT_PROXY=false
|
||||
|
||||
@@ -80,7 +80,7 @@ func Run() {
|
||||
// Override their values if the corresponding environment variables are
|
||||
// set.
|
||||
fs.VisitAll(func(f *flag.Flag) {
|
||||
envName := strings.ToUpper(strings.ReplaceAll(f.Name, "-", "_"))
|
||||
envName := gotenberg.EnvVarName(f.Name)
|
||||
val, ok := os.LookupEnv(envName)
|
||||
if !ok {
|
||||
return
|
||||
|
||||
635
pkg/gotenberg/allowlist.go
Normal file
635
pkg/gotenberg/allowlist.go
Normal file
@@ -0,0 +1,635 @@
|
||||
package gotenberg
|
||||
|
||||
import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
// AllowListRisk classifies why an allow-list pattern is dangerous. A URL that
|
||||
// matches an allow-list skips the private and public IP checks, so a pattern
|
||||
// that matches more than its author intended silently widens outbound access.
|
||||
// See [AuditAllowList].
|
||||
type AllowListRisk string
|
||||
|
||||
const (
|
||||
// AllowListRiskUnanchored marks a pattern with no leading "^". regexp2
|
||||
// searches rather than matches, so the pattern hits anywhere in the URL,
|
||||
// including the query string.
|
||||
AllowListRiskUnanchored AllowListRisk = "unanchored"
|
||||
|
||||
// AllowListRiskUnanchoredBranch marks an alternation whose later branches
|
||||
// have no leading "^". Anchoring only the first branch is a common slip.
|
||||
AllowListRiskUnanchoredBranch AllowListRisk = "unanchored-branch"
|
||||
|
||||
// AllowListRiskCatchAll marks a pattern with no literal prefix, such as
|
||||
// ".+", which matches every URL and disables filtering entirely.
|
||||
AllowListRiskCatchAll AllowListRisk = "catch-all"
|
||||
|
||||
// AllowListRiskOpenHost marks a pattern whose host is not terminated, so
|
||||
// it also matches attacker-chosen suffix hosts. For example
|
||||
// "^https://trusted\.example\.com" matches
|
||||
// "https://trusted.example.com.attacker.example/".
|
||||
AllowListRiskOpenHost AllowListRisk = "open-host"
|
||||
)
|
||||
|
||||
// AllowListFinding reports one risky entry of an allow-list.
|
||||
type AllowListFinding struct {
|
||||
// Index is the zero-based position of the pattern within the flag value.
|
||||
Index int
|
||||
|
||||
// Pattern is the operator's pattern, verbatim.
|
||||
Pattern string
|
||||
|
||||
// Risk is why the pattern is dangerous.
|
||||
Risk AllowListRisk
|
||||
}
|
||||
|
||||
// maxAuditedPatternLength bounds the patterns [AuditAllowList] inspects. A
|
||||
// pathological pattern is not worth walking, and reporting nothing is better
|
||||
// than reporting a partial verdict.
|
||||
const maxAuditedPatternLength = 4096
|
||||
|
||||
// AuditAllowList reports the entries of an allow-list that match more URLs
|
||||
// than their author is likely to intend. It is a lint over the pattern source,
|
||||
// not a parser: it recognizes the shapes that are dangerous in practice and
|
||||
// stays silent when it cannot be sure.
|
||||
//
|
||||
// Callers use the findings to warn operators. Never use them to reject a
|
||||
// configuration: existing deployments rely on loose patterns, and a pattern
|
||||
// this function does not flag is not thereby safe.
|
||||
func AuditAllowList(patterns []string) []AllowListFinding {
|
||||
var findings []AllowListFinding
|
||||
|
||||
for i, pattern := range patterns {
|
||||
if pattern == "" || len(pattern) > maxAuditedPatternLength {
|
||||
continue
|
||||
}
|
||||
|
||||
risk, ok := auditPattern(pattern)
|
||||
if ok {
|
||||
findings = append(findings, AllowListFinding{Index: i, Pattern: pattern, Risk: risk})
|
||||
}
|
||||
}
|
||||
|
||||
return findings
|
||||
}
|
||||
|
||||
// auditPattern classifies a single pattern, reporting the first risk found.
|
||||
func auditPattern(pattern string) (AllowListRisk, bool) {
|
||||
body := trimInlineFlags(pattern)
|
||||
|
||||
branches := splitTopLevelAlternation(body)
|
||||
for i, branch := range branches {
|
||||
branch = strings.TrimSpace(branch)
|
||||
|
||||
anchored := hasStartAnchor(branch)
|
||||
rest := strings.TrimPrefix(strings.TrimPrefix(branch, `\A`), "^")
|
||||
|
||||
// Catch-all first: a pattern that constrains nothing matches every URL
|
||||
// whether or not it is anchored, and saying so is more useful than
|
||||
// telling the operator to anchor it.
|
||||
if literalPrefix(rest) == "" {
|
||||
return AllowListRiskCatchAll, true
|
||||
}
|
||||
|
||||
if !anchored {
|
||||
if i == 0 {
|
||||
return AllowListRiskUnanchored, true
|
||||
}
|
||||
return AllowListRiskUnanchoredBranch, true
|
||||
}
|
||||
|
||||
// A lookaround invalidates the token walk, so skip the host check for
|
||||
// this branch rather than guess. The anchor and catch-all checks above
|
||||
// still applied.
|
||||
if containsLookaround(rest) {
|
||||
continue
|
||||
}
|
||||
|
||||
if hostIsOpen(rest) {
|
||||
return AllowListRiskOpenHost, true
|
||||
}
|
||||
}
|
||||
|
||||
return "", false
|
||||
}
|
||||
|
||||
// trimInlineFlags removes a leading inline flag group such as "(?i)" so that
|
||||
// the anchor check sees the pattern proper.
|
||||
func trimInlineFlags(pattern string) string {
|
||||
if !strings.HasPrefix(pattern, "(?") {
|
||||
return pattern
|
||||
}
|
||||
|
||||
end := strings.Index(pattern, ")")
|
||||
if end == -1 {
|
||||
return pattern
|
||||
}
|
||||
|
||||
// Only a flag group qualifies. "(?:", "(?=", "(?!" and "(?<" open a real
|
||||
// group and must stay.
|
||||
flags := pattern[2:end]
|
||||
if flags == "" || strings.ContainsAny(flags, ":=!<") {
|
||||
return pattern
|
||||
}
|
||||
for _, r := range flags {
|
||||
if !strings.ContainsRune("imsUx-", r) {
|
||||
return pattern
|
||||
}
|
||||
}
|
||||
|
||||
return pattern[end+1:]
|
||||
}
|
||||
|
||||
// hasStartAnchor reports whether branch begins with a start-of-input anchor.
|
||||
func hasStartAnchor(branch string) bool {
|
||||
return strings.HasPrefix(branch, "^") || strings.HasPrefix(branch, `\A`)
|
||||
}
|
||||
|
||||
// containsLookaround reports whether the pattern uses a lookaround, which the
|
||||
// token walk in [hostIsOpen] cannot reason about.
|
||||
func containsLookaround(s string) bool {
|
||||
return strings.Contains(s, "(?=") || strings.Contains(s, "(?!") || strings.Contains(s, "(?<")
|
||||
}
|
||||
|
||||
// splitTopLevelAlternation splits on "|" at paren depth zero, honoring escapes
|
||||
// and character classes.
|
||||
func splitTopLevelAlternation(s string) []string {
|
||||
var (
|
||||
parts []string
|
||||
current strings.Builder
|
||||
depth int
|
||||
inClass bool
|
||||
)
|
||||
|
||||
for i := 0; i < len(s); i++ {
|
||||
c := s[i]
|
||||
|
||||
switch {
|
||||
case c == '\\' && i+1 < len(s):
|
||||
current.WriteByte(c)
|
||||
current.WriteByte(s[i+1])
|
||||
i++
|
||||
continue
|
||||
case c == '[' && !inClass:
|
||||
inClass = true
|
||||
case c == ']' && inClass:
|
||||
inClass = false
|
||||
case c == '(' && !inClass:
|
||||
depth++
|
||||
case c == ')' && !inClass:
|
||||
depth--
|
||||
case c == '|' && !inClass && depth == 0:
|
||||
parts = append(parts, current.String())
|
||||
current.Reset()
|
||||
continue
|
||||
}
|
||||
|
||||
current.WriteByte(c)
|
||||
}
|
||||
|
||||
parts = append(parts, current.String())
|
||||
|
||||
return parts
|
||||
}
|
||||
|
||||
// literalPrefix returns the characters a matching URL must start with. It
|
||||
// stops at the first optional or non-literal token, and descends one level
|
||||
// into a leading mandatory group so that "^(https|http)://" is not mistaken
|
||||
// for a catch-all. An empty result means the pattern constrains nothing.
|
||||
func literalPrefix(s string) string {
|
||||
var prefix strings.Builder
|
||||
|
||||
for i := 0; i < len(s); {
|
||||
// A group: descend once when it is mandatory, otherwise stop.
|
||||
if s[i] == '(' {
|
||||
end := matchingParen(s, i)
|
||||
if end == -1 {
|
||||
break
|
||||
}
|
||||
if isQuantified(s, end+1) {
|
||||
break
|
||||
}
|
||||
|
||||
inner := trimInlineFlags(s[i+1 : end])
|
||||
branches := splitTopLevelAlternation(inner)
|
||||
|
||||
common := literalPrefix(branches[0])
|
||||
for _, b := range branches[1:] {
|
||||
common = commonPrefix(common, literalPrefix(b))
|
||||
}
|
||||
prefix.WriteString(common)
|
||||
|
||||
// Only the leading group is worth descending into.
|
||||
break
|
||||
}
|
||||
|
||||
var token string
|
||||
switch {
|
||||
case s[i] == '\\' && i+1 < len(s):
|
||||
token = s[i : i+2]
|
||||
case s[i] == '[':
|
||||
end := matchingBracket(s, i)
|
||||
if end == -1 {
|
||||
return prefix.String()
|
||||
}
|
||||
token = s[i : end+1]
|
||||
default:
|
||||
token = s[i : i+1]
|
||||
}
|
||||
|
||||
next := i + len(token)
|
||||
if isQuantified(s, next) {
|
||||
break
|
||||
}
|
||||
|
||||
// Only a plain literal or an escaped literal contributes.
|
||||
switch {
|
||||
case len(token) == 2 && token[0] == '\\' && !isEscapeClass(token[1]):
|
||||
prefix.WriteByte(token[1])
|
||||
case len(token) == 1 && !strings.ContainsAny(token, `.[]()^$*+?{}|`):
|
||||
prefix.WriteByte(token[0])
|
||||
default:
|
||||
return prefix.String()
|
||||
}
|
||||
|
||||
i = next
|
||||
}
|
||||
|
||||
return prefix.String()
|
||||
}
|
||||
|
||||
// hostIsOpen reports whether the authority part of the pattern can be left
|
||||
// without crossing a terminator, which means the pattern also matches
|
||||
// attacker-chosen suffix hosts or userinfo.
|
||||
//
|
||||
// It walks the tokens after "://" and classifies each one. A terminator ends
|
||||
// the authority, so the pattern is safe. A crosser can match "@", "?" or "#"
|
||||
// and therefore lets a matching URL escape the authority, so the pattern is
|
||||
// open. Reaching the end without a terminator is open too, which is the
|
||||
// classic "^https://trusted\.example\.com" case.
|
||||
func hostIsOpen(s string) bool {
|
||||
_, after, ok := strings.Cut(s, "://")
|
||||
if !ok {
|
||||
// No authority to reason about, for example "^file:///tmp/".
|
||||
return false
|
||||
}
|
||||
|
||||
rest := after
|
||||
|
||||
for i := 0; i < len(rest); {
|
||||
var token string
|
||||
switch {
|
||||
case rest[i] == '\\' && i+1 < len(rest):
|
||||
token = rest[i : i+2]
|
||||
case rest[i] == '[':
|
||||
end := matchingBracket(rest, i)
|
||||
if end == -1 {
|
||||
return true
|
||||
}
|
||||
token = rest[i : end+1]
|
||||
case rest[i] == '(':
|
||||
end := matchingParen(rest, i)
|
||||
if end == -1 {
|
||||
return true
|
||||
}
|
||||
token = rest[i : end+1]
|
||||
default:
|
||||
token = rest[i : i+1]
|
||||
}
|
||||
|
||||
next := i + len(token)
|
||||
optional := isOptionalQuantifier(rest, next)
|
||||
|
||||
switch classifyHostToken(token) {
|
||||
case hostTokenTerminator:
|
||||
// An optional terminator does not end anything, since the URL may
|
||||
// match without it.
|
||||
if !optional {
|
||||
return false
|
||||
}
|
||||
case hostTokenCrosser:
|
||||
return true
|
||||
case hostTokenNeutral:
|
||||
// Part of the host itself, so keep walking.
|
||||
}
|
||||
|
||||
i = next
|
||||
for i < len(rest) && isQuantifierByte(rest[i]) {
|
||||
if rest[i] == '{' {
|
||||
end := strings.IndexByte(rest[i:], '}')
|
||||
if end == -1 {
|
||||
return true
|
||||
}
|
||||
i += end + 1
|
||||
continue
|
||||
}
|
||||
i++
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
// hostTokenKind is how a token affects the walk in [hostIsOpen].
|
||||
type hostTokenKind int
|
||||
|
||||
const (
|
||||
hostTokenNeutral hostTokenKind = iota
|
||||
hostTokenTerminator
|
||||
hostTokenCrosser
|
||||
)
|
||||
|
||||
// hostTerminators are the characters that end the authority of a URL.
|
||||
const hostTerminators = "/:#?"
|
||||
|
||||
// crosserClassChars are the characters that, if a class can match them, let a
|
||||
// match escape the authority. "/" is deliberately absent: it ends the
|
||||
// authority rather than escaping it, so a class such as "[:/]" is safe.
|
||||
const crosserClassChars = "@?#"
|
||||
|
||||
// classifyHostToken classifies one token of the authority walk.
|
||||
func classifyHostToken(token string) hostTokenKind {
|
||||
switch {
|
||||
case token == ".":
|
||||
// The wildcard matches "@", "#" and "?", so a host built on it can be
|
||||
// left without ever reaching a terminator.
|
||||
return hostTokenCrosser
|
||||
|
||||
case token == "$":
|
||||
return hostTokenTerminator
|
||||
|
||||
case len(token) == 1 && strings.Contains(hostTerminators, token):
|
||||
return hostTokenTerminator
|
||||
|
||||
case len(token) == 2 && token[0] == '\\':
|
||||
switch token[1] {
|
||||
case 'S', 'D', 'W':
|
||||
return hostTokenCrosser
|
||||
case 'd', 'w', 's':
|
||||
return hostTokenNeutral
|
||||
case 'p', 'P':
|
||||
return hostTokenCrosser
|
||||
}
|
||||
if strings.Contains(hostTerminators, token[1:]) {
|
||||
return hostTokenTerminator
|
||||
}
|
||||
return hostTokenNeutral
|
||||
|
||||
case strings.HasPrefix(token, "["):
|
||||
inner := strings.TrimSuffix(strings.TrimPrefix(token, "["), "]")
|
||||
if strings.HasPrefix(inner, "^") {
|
||||
// A negated class almost always admits "@".
|
||||
return hostTokenCrosser
|
||||
}
|
||||
if classContainsAny(inner, crosserClassChars) {
|
||||
return hostTokenCrosser
|
||||
}
|
||||
if classOnlyTerminators(inner) {
|
||||
return hostTokenTerminator
|
||||
}
|
||||
return hostTokenNeutral
|
||||
|
||||
case strings.HasPrefix(token, "("):
|
||||
return classifyGroup(token)
|
||||
}
|
||||
|
||||
return hostTokenNeutral
|
||||
}
|
||||
|
||||
// classifyGroup classifies a parenthesized group. A group whose every branch
|
||||
// starts with a terminator ends the authority, which is what makes the
|
||||
// idiomatic "(:|/|$)" safe. A group containing a crosser is a crosser.
|
||||
func classifyGroup(token string) hostTokenKind {
|
||||
inner := trimInlineFlags(strings.TrimSuffix(strings.TrimPrefix(token, "("), ")"))
|
||||
inner = strings.TrimPrefix(inner, "?:")
|
||||
|
||||
branches := splitTopLevelAlternation(inner)
|
||||
|
||||
allTerminate := true
|
||||
for _, branch := range branches {
|
||||
if branch == "" {
|
||||
allTerminate = false
|
||||
continue
|
||||
}
|
||||
|
||||
kind := classifyHostToken(firstToken(branch))
|
||||
if kind == hostTokenCrosser {
|
||||
return hostTokenCrosser
|
||||
}
|
||||
if kind != hostTokenTerminator {
|
||||
allTerminate = false
|
||||
}
|
||||
|
||||
// A crosser anywhere inside the branch still escapes the authority.
|
||||
if branchHasCrosser(branch) {
|
||||
return hostTokenCrosser
|
||||
}
|
||||
}
|
||||
|
||||
if allTerminate {
|
||||
return hostTokenTerminator
|
||||
}
|
||||
|
||||
return hostTokenNeutral
|
||||
}
|
||||
|
||||
// branchHasCrosser reports whether any token of branch is a crosser.
|
||||
func branchHasCrosser(branch string) bool {
|
||||
for i := 0; i < len(branch); {
|
||||
token := tokenAt(branch, i)
|
||||
if token == "" {
|
||||
return true
|
||||
}
|
||||
if classifyHostToken(token) == hostTokenCrosser {
|
||||
return true
|
||||
}
|
||||
i += len(token)
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// firstToken returns the first regex token of s.
|
||||
func firstToken(s string) string {
|
||||
return tokenAt(s, 0)
|
||||
}
|
||||
|
||||
// tokenAt returns the regex token starting at index i, or "" if it is
|
||||
// malformed.
|
||||
func tokenAt(s string, i int) string {
|
||||
if i >= len(s) {
|
||||
return ""
|
||||
}
|
||||
|
||||
switch {
|
||||
case s[i] == '\\' && i+1 < len(s):
|
||||
return s[i : i+2]
|
||||
case s[i] == '[':
|
||||
end := matchingBracket(s, i)
|
||||
if end == -1 {
|
||||
return ""
|
||||
}
|
||||
return s[i : end+1]
|
||||
case s[i] == '(':
|
||||
end := matchingParen(s, i)
|
||||
if end == -1 {
|
||||
return ""
|
||||
}
|
||||
return s[i : end+1]
|
||||
}
|
||||
|
||||
return s[i : i+1]
|
||||
}
|
||||
|
||||
// classOnlyTerminators reports whether every character a class can match ends
|
||||
// the authority, which makes the class itself a terminator. A range is never
|
||||
// treated as one.
|
||||
func classOnlyTerminators(class string) bool {
|
||||
if class == "" {
|
||||
return false
|
||||
}
|
||||
|
||||
for i := 0; i < len(class); i++ {
|
||||
if class[i] == '\\' && i+1 < len(class) {
|
||||
if !strings.Contains(hostTerminators, class[i+1:i+2]) {
|
||||
return false
|
||||
}
|
||||
i++
|
||||
continue
|
||||
}
|
||||
|
||||
if i+2 < len(class) && class[i+1] == '-' {
|
||||
return false
|
||||
}
|
||||
|
||||
if !strings.Contains(hostTerminators, class[i:i+1]) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
// classContainsAny reports whether a character class body can match any of the
|
||||
// given characters, expanding simple ranges.
|
||||
func classContainsAny(class, chars string) bool {
|
||||
for i := 0; i < len(class); i++ {
|
||||
if class[i] == '\\' && i+1 < len(class) {
|
||||
// An escape class such as \S inside a class admits everything.
|
||||
if strings.ContainsRune("SDW", rune(class[i+1])) {
|
||||
return true
|
||||
}
|
||||
if strings.ContainsRune(chars, rune(class[i+1])) {
|
||||
return true
|
||||
}
|
||||
i++
|
||||
continue
|
||||
}
|
||||
|
||||
if i+2 < len(class) && class[i+1] == '-' {
|
||||
lo, hi := class[i], class[i+2]
|
||||
for _, c := range []byte(chars) {
|
||||
if c >= lo && c <= hi {
|
||||
return true
|
||||
}
|
||||
}
|
||||
i += 2
|
||||
continue
|
||||
}
|
||||
|
||||
if strings.ContainsRune(chars, rune(class[i])) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// matchingParen returns the index of the ")" closing the "(" at start.
|
||||
func matchingParen(s string, start int) int {
|
||||
depth := 0
|
||||
inClass := false
|
||||
|
||||
for i := start; i < len(s); i++ {
|
||||
switch {
|
||||
case s[i] == '\\' && i+1 < len(s):
|
||||
i++
|
||||
case s[i] == '[' && !inClass:
|
||||
inClass = true
|
||||
case s[i] == ']' && inClass:
|
||||
inClass = false
|
||||
case s[i] == '(' && !inClass:
|
||||
depth++
|
||||
case s[i] == ')' && !inClass:
|
||||
depth--
|
||||
if depth == 0 {
|
||||
return i
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return -1
|
||||
}
|
||||
|
||||
// matchingBracket returns the index of the "]" closing the "[" at start.
|
||||
func matchingBracket(s string, start int) int {
|
||||
for i := start + 1; i < len(s); i++ {
|
||||
switch {
|
||||
case s[i] == '\\' && i+1 < len(s):
|
||||
i++
|
||||
case s[i] == ']':
|
||||
// A "]" immediately after "[" or "[^" is a literal.
|
||||
if i == start+1 || (i == start+2 && s[start+1] == '^') {
|
||||
continue
|
||||
}
|
||||
return i
|
||||
}
|
||||
}
|
||||
|
||||
return -1
|
||||
}
|
||||
|
||||
// isQuantifierByte reports whether c opens a quantifier.
|
||||
func isQuantifierByte(c byte) bool {
|
||||
return c == '?' || c == '*' || c == '+' || c == '{'
|
||||
}
|
||||
|
||||
// isQuantified reports whether a quantifier starts at index i.
|
||||
func isQuantified(s string, i int) bool {
|
||||
return i < len(s) && isQuantifierByte(s[i])
|
||||
}
|
||||
|
||||
// isOptionalQuantifier reports whether the quantifier at index i lets the
|
||||
// preceding token match nothing.
|
||||
func isOptionalQuantifier(s string, i int) bool {
|
||||
if i >= len(s) {
|
||||
return false
|
||||
}
|
||||
|
||||
switch s[i] {
|
||||
case '?', '*':
|
||||
return true
|
||||
case '{':
|
||||
return strings.HasPrefix(s[i:], "{0")
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// isEscapeClass reports whether c after a backslash denotes a character class
|
||||
// rather than a literal.
|
||||
func isEscapeClass(c byte) bool {
|
||||
return strings.ContainsRune("dDwWsSbBAzZpP", rune(c))
|
||||
}
|
||||
|
||||
// commonPrefix returns the longest common prefix of a and b.
|
||||
func commonPrefix(a, b string) string {
|
||||
n := min(len(a), len(b))
|
||||
for i := range n {
|
||||
if a[i] != b[i] {
|
||||
return a[:i]
|
||||
}
|
||||
}
|
||||
|
||||
return a[:n]
|
||||
}
|
||||
179
pkg/gotenberg/allowlist_test.go
Normal file
179
pkg/gotenberg/allowlist_test.go
Normal file
@@ -0,0 +1,179 @@
|
||||
package gotenberg
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/dlclark/regexp2"
|
||||
)
|
||||
|
||||
func TestAuditAllowList(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
scenario string
|
||||
pattern string
|
||||
want AllowListRisk
|
||||
}{
|
||||
// Safe: the host is terminated before anything can leave it.
|
||||
{"idiomatic terminator group", `^https?://internal\.svc(:|/|$)`, ""},
|
||||
{"trailing slash", `^https://trusted\.example\.com/`, ""},
|
||||
{"optional port then terminator", `^https://example\.com(:[0-9]+)?(/|$)`, ""},
|
||||
{"positive class cannot leave authority", `^https://[a-z0-9.-]+\.s3\.amazonaws\.com/`, ""},
|
||||
{"leading mandatory group", `^(https|http)://a\.example\.com/`, ""},
|
||||
{"optional subdomain group", `^https://(www\.)?example\.com/`, ""},
|
||||
{"port terminator", `^https://example\.com:8443/`, ""},
|
||||
{"end anchor", `^https://example\.com$`, ""},
|
||||
{"alternation both anchored and terminated", `^https://a\.example/|^https://b\.example/`, ""},
|
||||
{"no authority to check", `^file:///tmp/`, ""},
|
||||
{"digit class in host", `^https://node\d+\.example\.com/`, ""},
|
||||
{"class of only terminators", `^https://example\.com[:/]`, ""},
|
||||
{"feature file pattern, fixed", `^https?://host\.docker\.internal(:[0-9]+)?/`, ""},
|
||||
|
||||
// Unanchored: regexp2 searches, so these match anywhere in the URL.
|
||||
{"no anchor", `trusted\.example\.com`, AllowListRiskUnanchored},
|
||||
{"no anchor with scheme", `https://trusted\.example\.com/`, AllowListRiskUnanchored},
|
||||
|
||||
// Only the first branch anchored.
|
||||
{"second branch unanchored", `^http://a\.example/|http://b\.example/`, AllowListRiskUnanchoredBranch},
|
||||
|
||||
// Catch-all: matches every URL.
|
||||
{"dot plus", `.+`, AllowListRiskCatchAll},
|
||||
{"dot star", `.*`, AllowListRiskCatchAll},
|
||||
{"anchored dot star", `^.*`, AllowListRiskCatchAll},
|
||||
{"anchored dot plus", `^.+`, AllowListRiskCatchAll},
|
||||
|
||||
// Open host: the reported vulnerability class.
|
||||
{"advisory pattern", `^http://trusted\.example\.com`, AllowListRiskOpenHost},
|
||||
{"gotenberg.dev internet-facing recipe", `^https?://[^/]+\.internal\.example\.com`, AllowListRiskOpenHost},
|
||||
{"gotenberg.dev strict whitelist recipe", `^https://(api|cdn|images)\.internal\.example\.com`, AllowListRiskOpenHost},
|
||||
{"gotenberg.dev hooks recipe", `^https?://hooks\.internal\.example\.com`, AllowListRiskOpenHost},
|
||||
{"feature file pattern", `^https?://host.docker.internal.*`, AllowListRiskOpenHost},
|
||||
{"scheme only", `^https?://`, AllowListRiskOpenHost},
|
||||
{"wildcard subdomain", `^https://.+\.example\.com/`, AllowListRiskOpenHost},
|
||||
{"escaped dot is not a terminator", `^https?://example\.com\.`, AllowListRiskOpenHost},
|
||||
{"negated class in host", `^https://[^.]+\.example\.com/`, AllowListRiskOpenHost},
|
||||
} {
|
||||
t.Run(tc.scenario, func(t *testing.T) {
|
||||
findings := AuditAllowList([]string{tc.pattern})
|
||||
|
||||
if tc.want == "" {
|
||||
if len(findings) != 0 {
|
||||
t.Fatalf("AuditAllowList(%q) = %+v, want no finding", tc.pattern, findings)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if len(findings) != 1 {
|
||||
t.Fatalf("AuditAllowList(%q) returned %d findings, want 1", tc.pattern, len(findings))
|
||||
}
|
||||
if findings[0].Risk != tc.want {
|
||||
t.Fatalf("AuditAllowList(%q) risk = %q, want %q", tc.pattern, findings[0].Risk, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestAuditAllowList_FlaggedPatternsAreActuallyExploitable proves the audit is
|
||||
// not merely syntactic: every pattern it flags as open-host really does admit
|
||||
// a host the operator did not intend.
|
||||
func TestAuditAllowList_FlaggedPatternsAreActuallyExploitable(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
pattern string
|
||||
attack string
|
||||
}{
|
||||
{`^http://trusted\.example\.com`, "http://trusted.example.com.attacker.example/"},
|
||||
{`^https?://[^/]+\.internal\.example\.com`, "http://a.internal.example.com.attacker.example/"},
|
||||
{`^https://(api|cdn|images)\.internal\.example\.com`, "https://api.internal.example.com.attacker.example/"},
|
||||
{`^https?://hooks\.internal\.example\.com`, "http://hooks.internal.example.com.attacker.example/"},
|
||||
{`^https?://host.docker.internal.*`, "http://host.docker.internal.attacker.example/"},
|
||||
{`^https://.+\.example\.com/`, "https://attacker.example/#x.example.com/"},
|
||||
{`^https?://example\.com\.`, "http://example.com.attacker.example/"},
|
||||
} {
|
||||
t.Run(tc.pattern, func(t *testing.T) {
|
||||
findings := AuditAllowList([]string{tc.pattern})
|
||||
if len(findings) == 0 {
|
||||
t.Fatalf("pattern %q was not flagged", tc.pattern)
|
||||
}
|
||||
|
||||
ok, err := regexp2.MustCompile(tc.pattern, 0).MatchString(tc.attack)
|
||||
if err != nil {
|
||||
t.Fatalf("match %q: %v", tc.attack, err)
|
||||
}
|
||||
if !ok {
|
||||
t.Fatalf("pattern %q does not match %q, so the finding is a false positive", tc.pattern, tc.attack)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestAuditAllowList_SafePatternsRejectTheAttacks is the converse: the shapes
|
||||
// the audit stays silent about really do reject the same attacks.
|
||||
func TestAuditAllowList_SafePatternsRejectTheAttacks(t *testing.T) {
|
||||
safe := []string{
|
||||
`^https?://internal\.svc(:|/|$)`,
|
||||
`^https://trusted\.example\.com/`,
|
||||
`^https://example\.com(:[0-9]+)?(/|$)`,
|
||||
`^https://[a-z0-9.-]+\.s3\.amazonaws\.com/`,
|
||||
}
|
||||
attacks := []string{
|
||||
"https://internal.svc.attacker.example/",
|
||||
"https://trusted.example.com.attacker.example/",
|
||||
"https://trusted.example.com@169.254.169.254/",
|
||||
"https://example.com.attacker.example/",
|
||||
"https://example.com@10.0.0.5/",
|
||||
"https://bucket.s3.amazonaws.com.attacker.example/",
|
||||
"https://bucket.s3.amazonaws.com@127.0.0.1/",
|
||||
}
|
||||
|
||||
for _, pattern := range safe {
|
||||
t.Run(pattern, func(t *testing.T) {
|
||||
if findings := AuditAllowList([]string{pattern}); len(findings) != 0 {
|
||||
t.Fatalf("safe pattern %q was flagged as %q", pattern, findings[0].Risk)
|
||||
}
|
||||
|
||||
re := regexp2.MustCompile(pattern, 0)
|
||||
for _, attack := range attacks {
|
||||
ok, err := re.MatchString(attack)
|
||||
if err != nil {
|
||||
t.Fatalf("match %q: %v", attack, err)
|
||||
}
|
||||
if ok {
|
||||
t.Fatalf("pattern %q matches attack %q but was not flagged", pattern, attack)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAuditAllowList_SkipsEmptyAndOversized(t *testing.T) {
|
||||
oversized := make([]byte, maxAuditedPatternLength+1)
|
||||
for i := range oversized {
|
||||
oversized[i] = 'a'
|
||||
}
|
||||
|
||||
findings := AuditAllowList([]string{"", string(oversized)})
|
||||
if len(findings) != 0 {
|
||||
t.Fatalf("AuditAllowList returned %+v, want no finding", findings)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAuditAllowList_ReportsIndex(t *testing.T) {
|
||||
findings := AuditAllowList([]string{
|
||||
`^https://ok\.example\.com/`,
|
||||
`^https://open\.example\.com`,
|
||||
})
|
||||
if len(findings) != 1 {
|
||||
t.Fatalf("got %d findings, want 1", len(findings))
|
||||
}
|
||||
if findings[0].Index != 1 {
|
||||
t.Fatalf("findings[0].Index = %d, want 1", findings[0].Index)
|
||||
}
|
||||
}
|
||||
|
||||
// TestAuditAllowList_ShippedChromiumDenyListIsNotAudited guards the rule that
|
||||
// deny-lists are never audited. The shipped Chromium deny-list uses a
|
||||
// lookaround and has no authority, so auditing it would produce noise.
|
||||
func TestAuditAllowList_LookaroundIsNotFlaggedForHost(t *testing.T) {
|
||||
findings := AuditAllowList([]string{`^file:(?!//\/tmp/).*`})
|
||||
if len(findings) != 0 {
|
||||
t.Fatalf("AuditAllowList returned %+v, want no finding", findings)
|
||||
}
|
||||
}
|
||||
@@ -1,11 +1,17 @@
|
||||
package gotenberg
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/dlclark/regexp2"
|
||||
"github.com/labstack/gommon/bytes"
|
||||
flag "github.com/spf13/pflag"
|
||||
|
||||
"github.com/gotenberg/gotenberg/v8/pkg/gotenberg/internal/log"
|
||||
)
|
||||
|
||||
// ParsedFlags wraps a [flag.FlagSet] so that retrieving the typed values is
|
||||
@@ -226,9 +232,14 @@ func (f *ParsedFlags) MustDeprecatedRegexp(deprecated string, newName string) *r
|
||||
// MustRegexpSlice returns a slice of compiled regular expressions from a
|
||||
// string-slice flag given by name. Empty strings are skipped.
|
||||
// It panics if an error occurs.
|
||||
//
|
||||
// Every allow-list and deny-list in Gotenberg is read through this method, so
|
||||
// it is also where allow-list patterns are audited. See [AuditAllowList].
|
||||
func (f *ParsedFlags) MustRegexpSlice(name string) []*regexp2.Regexp {
|
||||
vals := f.MustStringSlice(name)
|
||||
|
||||
f.warnRiskyAllowList(name, vals)
|
||||
|
||||
var regexps []*regexp2.Regexp
|
||||
for _, val := range vals {
|
||||
if val == "" {
|
||||
@@ -241,6 +252,114 @@ func (f *ParsedFlags) MustRegexpSlice(name string) []*regexp2.Regexp {
|
||||
return regexps
|
||||
}
|
||||
|
||||
// allowListFlagSuffix identifies the flags whose patterns grant an IP-check
|
||||
// bypass. Deny-lists are never audited: they always apply, cannot be bypassed,
|
||||
// and a loose deny-list is safe rather than dangerous.
|
||||
const allowListFlagSuffix = "-allow-list"
|
||||
|
||||
// warnRiskyAllowList logs one warning per allow-list entry that matches more
|
||||
// URLs than its author is likely to intend.
|
||||
//
|
||||
// It warns and never fails: operators depend on loose patterns today, and
|
||||
// rejecting them at startup would break running deployments.
|
||||
func (f *ParsedFlags) warnRiskyAllowList(name string, vals []string) {
|
||||
if !strings.HasSuffix(name, allowListFlagSuffix) {
|
||||
return
|
||||
}
|
||||
|
||||
findings := AuditAllowList(vals)
|
||||
if len(findings) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
// The logger is nil until the entry point initializes it, which happens
|
||||
// before any module is provisioned. Tests and embedders that call this
|
||||
// method directly get no logger, and must not panic for it.
|
||||
logger := log.Logger()
|
||||
if logger == nil {
|
||||
return
|
||||
}
|
||||
|
||||
for _, finding := range findings {
|
||||
// Provision has no context.Context to propagate, so the trace-aware
|
||||
// logging convention is satisfied with a background context.
|
||||
logger.WarnContext(
|
||||
context.Background(),
|
||||
f.allowListWarning(name, finding),
|
||||
slog.String("flag", "--"+name),
|
||||
slog.String("env", EnvVarName(name)),
|
||||
slog.Int("entry", finding.Index+1),
|
||||
slog.String("reason", string(finding.Risk)),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// allowListWarning builds the operator-facing message for a finding. It names
|
||||
// the flag and its environment variable, and, when they exist, the IP-check
|
||||
// flags the entry silently disables.
|
||||
func (f *ParsedFlags) allowListWarning(name string, finding AllowListFinding) string {
|
||||
var b strings.Builder
|
||||
|
||||
// Print the pattern raw rather than quoted: %q escapes every backslash, so
|
||||
// the operator would not recognize the value they set.
|
||||
fmt.Fprintf(&b, "--%s (%s) entry %d '%s' ", name, EnvVarName(name), finding.Index+1, finding.Pattern)
|
||||
|
||||
switch finding.Risk {
|
||||
case AllowListRiskUnanchored:
|
||||
b.WriteString("is not anchored with ^, so it matches anywhere in the URL and a URL such as http://attacker.example/?u=trusted.example.com passes. ")
|
||||
case AllowListRiskUnanchoredBranch:
|
||||
b.WriteString("has an alternation branch that is not anchored with ^, and that branch matches anywhere in the URL. ")
|
||||
case AllowListRiskCatchAll:
|
||||
b.WriteString("matches every URL. ")
|
||||
case AllowListRiskOpenHost:
|
||||
b.WriteString("does not terminate the host, so it also matches suffix hosts such as http://trusted.example.com.attacker.example/. ")
|
||||
}
|
||||
|
||||
b.WriteString(f.bypassSentence(name))
|
||||
|
||||
switch finding.Risk {
|
||||
case AllowListRiskUnanchored, AllowListRiskUnanchoredBranch:
|
||||
b.WriteString("Anchor every branch with ^ and end the host with /, :, or $.")
|
||||
case AllowListRiskCatchAll:
|
||||
b.WriteString("Restrict the entry to the hosts you trust, or unset the flag.")
|
||||
case AllowListRiskOpenHost:
|
||||
b.WriteString("End the host with /, :, $, or a group such as (:|/|$).")
|
||||
}
|
||||
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// bypassSentence names the IP-check flags an allow-list match skips, when the
|
||||
// module registers them.
|
||||
func (f *ParsedFlags) bypassSentence(name string) string {
|
||||
prefix := strings.TrimSuffix(name, allowListFlagSuffix)
|
||||
|
||||
private, public := prefix+"-deny-private-ips", prefix+"-deny-public-ips"
|
||||
if f.Lookup(private) == nil || f.Lookup(public) == nil {
|
||||
// A deprecated alias such as webhook-error-allow-list carries an extra
|
||||
// segment that the IP-check flags do not have.
|
||||
if i := strings.LastIndex(prefix, "-"); i != -1 {
|
||||
private, public = prefix[:i]+"-deny-private-ips", prefix[:i]+"-deny-public-ips"
|
||||
}
|
||||
}
|
||||
|
||||
if f.Lookup(private) == nil || f.Lookup(public) == nil {
|
||||
return "A URL that matches the allow-list skips the private and public IP checks. "
|
||||
}
|
||||
|
||||
return fmt.Sprintf(
|
||||
"A URL that matches the allow-list skips --%s (%s) and --%s (%s). ",
|
||||
private, EnvVarName(private), public, EnvVarName(public),
|
||||
)
|
||||
}
|
||||
|
||||
// EnvVarName returns the environment variable that overrides the flag given by
|
||||
// name. The entry point derives the same name when it applies environment
|
||||
// overrides, so operator-facing messages can name both without drifting.
|
||||
func EnvVarName(name string) string {
|
||||
return strings.ToUpper(strings.ReplaceAll(name, "-", "_"))
|
||||
}
|
||||
|
||||
// MustDeprecatedRegexpSlice returns the slice of compiled regular expressions
|
||||
// of a deprecated flag if it was explicitly set or the slice of the new flag.
|
||||
// It panics if an error occurs.
|
||||
|
||||
@@ -3,6 +3,7 @@ package gotenberg
|
||||
import (
|
||||
"reflect"
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -951,3 +952,105 @@ func TestParsedFlags_MustDeprecatedRegexpSlice(t *testing.T) {
|
||||
|
||||
_ = regexp2.None // Keep import alive.
|
||||
}
|
||||
|
||||
func TestParsedFlags_AllowListWarning(t *testing.T) {
|
||||
fs := flag.NewFlagSet("tests", flag.ContinueOnError)
|
||||
fs.StringSlice("chromium-allow-list", []string{}, "")
|
||||
fs.Bool("chromium-deny-private-ips", false, "")
|
||||
fs.Bool("chromium-deny-public-ips", false, "")
|
||||
fs.StringSlice("standalone-allow-list", []string{}, "")
|
||||
|
||||
parsedFlags := ParsedFlags{FlagSet: fs}
|
||||
|
||||
for _, tc := range []struct {
|
||||
scenario string
|
||||
name string
|
||||
finding AllowListFinding
|
||||
contains []string
|
||||
}{
|
||||
{
|
||||
scenario: "open host names both IP-check flags and their env vars",
|
||||
name: "chromium-allow-list",
|
||||
finding: AllowListFinding{Index: 0, Pattern: `^https://trusted\.example\.com`, Risk: AllowListRiskOpenHost},
|
||||
contains: []string{
|
||||
"--chromium-allow-list (CHROMIUM_ALLOW_LIST)",
|
||||
"entry 1",
|
||||
`^https://trusted\.example\.com`,
|
||||
"does not terminate the host",
|
||||
"--chromium-deny-private-ips (CHROMIUM_DENY_PRIVATE_IPS)",
|
||||
"--chromium-deny-public-ips (CHROMIUM_DENY_PUBLIC_IPS)",
|
||||
"End the host with",
|
||||
},
|
||||
},
|
||||
{
|
||||
scenario: "catch-all tells the operator to restrict or unset",
|
||||
name: "chromium-allow-list",
|
||||
finding: AllowListFinding{Index: 2, Pattern: ".+", Risk: AllowListRiskCatchAll},
|
||||
contains: []string{"entry 3", "matches every URL", "Restrict the entry"},
|
||||
},
|
||||
{
|
||||
scenario: "unanchored explains the search semantics",
|
||||
name: "chromium-allow-list",
|
||||
finding: AllowListFinding{Index: 0, Pattern: `trusted\.example\.com`, Risk: AllowListRiskUnanchored},
|
||||
contains: []string{"is not anchored with ^", "Anchor every branch with ^"},
|
||||
},
|
||||
{
|
||||
scenario: "module without IP-check flags falls back to a generic sentence",
|
||||
name: "standalone-allow-list",
|
||||
finding: AllowListFinding{Index: 0, Pattern: `^https://a\.example\.com`, Risk: AllowListRiskOpenHost},
|
||||
contains: []string{"skips the private and public IP checks"},
|
||||
},
|
||||
} {
|
||||
t.Run(tc.scenario, func(t *testing.T) {
|
||||
msg := parsedFlags.allowListWarning(tc.name, tc.finding)
|
||||
|
||||
for _, want := range tc.contains {
|
||||
if !strings.Contains(msg, want) {
|
||||
t.Fatalf("message %q does not contain %q", msg, want)
|
||||
}
|
||||
}
|
||||
if strings.Contains(msg, "—") {
|
||||
t.Fatalf("message must not contain an em dash: %q", msg)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParsedFlags_WarnRiskyAllowList_SkipsDenyLists(t *testing.T) {
|
||||
fs := flag.NewFlagSet("tests", flag.ContinueOnError)
|
||||
fs.StringSlice("chromium-deny-list", []string{}, "")
|
||||
|
||||
parsedFlags := ParsedFlags{FlagSet: fs}
|
||||
|
||||
// A deny-list is never audited: it always applies and cannot be bypassed,
|
||||
// so a loose one is safe. This must also not panic on a nil logger.
|
||||
parsedFlags.warnRiskyAllowList("chromium-deny-list", []string{".+", `^file:(?!//\/tmp/).*`})
|
||||
}
|
||||
|
||||
func TestParsedFlags_WarnRiskyAllowList_NilLoggerDoesNotPanic(t *testing.T) {
|
||||
fs := flag.NewFlagSet("tests", flag.ContinueOnError)
|
||||
fs.StringSlice("chromium-allow-list", []string{}, "")
|
||||
|
||||
parsedFlags := ParsedFlags{FlagSet: fs}
|
||||
|
||||
// Provision runs after the entry point initializes the logger, but tests
|
||||
// and embedders reach this path with no logger at all.
|
||||
parsedFlags.warnRiskyAllowList("chromium-allow-list", []string{".+"})
|
||||
}
|
||||
|
||||
func TestEnvVarName(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
want string
|
||||
}{
|
||||
{"chromium-allow-list", "CHROMIUM_ALLOW_LIST"},
|
||||
{"api-download-from-deny-private-ips", "API_DOWNLOAD_FROM_DENY_PRIVATE_IPS"},
|
||||
{"log-level", "LOG_LEVEL"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := EnvVarName(tc.name); got != tc.want {
|
||||
t.Fatalf("EnvVarName(%q) = %q, want %q", tc.name, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/dlclark/regexp2"
|
||||
"github.com/hashicorp/go-retryablehttp"
|
||||
"golang.org/x/net/http/httpproxy"
|
||||
)
|
||||
|
||||
@@ -78,6 +79,18 @@ var nonPublicIPv6Prefixes = []netip.Prefix{
|
||||
netip.MustParsePrefix("100::/64"),
|
||||
}
|
||||
|
||||
// nonPublicIPv4Prefixes lists IPv4 ranges that the [netip.Addr] helpers do
|
||||
// not classify but that must not be considered public:
|
||||
//
|
||||
// - 100.64.0.0/10 Carrier-grade NAT (RFC 6598). Routable inside provider
|
||||
// and cluster networks, and Alibaba Cloud serves instance metadata from
|
||||
// 100.100.100.200.
|
||||
// - 198.18.0.0/15 Benchmarking (RFC 2544). Never routed on the internet.
|
||||
var nonPublicIPv4Prefixes = []netip.Prefix{
|
||||
netip.MustParsePrefix("100.64.0.0/10"),
|
||||
netip.MustParsePrefix("198.18.0.0/15"),
|
||||
}
|
||||
|
||||
// IsPublicIP reports whether addr is reachable on the public internet. It
|
||||
// returns false for loopback, private (RFC1918), link-local, unspecified,
|
||||
// multicast, and unique-local addresses. IPv4-mapped IPv6 addresses are
|
||||
@@ -88,7 +101,8 @@ var nonPublicIPv6Prefixes = []netip.Prefix{
|
||||
// (6to4, Teredo, NAT64) are rejected wholesale rather than recursed into,
|
||||
// because a host that routes them implicitly trusts the IPv4 mapping and
|
||||
// the prefixes themselves are deprecated or translation-only. See
|
||||
// [nonPublicIPv6Prefixes] for the full list and rationale.
|
||||
// [nonPublicIPv6Prefixes] and [nonPublicIPv4Prefixes] for the full lists
|
||||
// and rationale.
|
||||
func IsPublicIP(addr netip.Addr) bool {
|
||||
if !addr.IsValid() {
|
||||
return false
|
||||
@@ -104,6 +118,13 @@ func IsPublicIP(addr netip.Addr) bool {
|
||||
addr.IsInterfaceLocalMulticast():
|
||||
return false
|
||||
}
|
||||
if addr.Is4() {
|
||||
for _, p := range nonPublicIPv4Prefixes {
|
||||
if p.Contains(addr) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
if addr.Is6() {
|
||||
for _, p := range nonPublicIPv6Prefixes {
|
||||
if p.Contains(addr) {
|
||||
@@ -249,7 +270,9 @@ func httpLikeScheme(scheme string) bool {
|
||||
//
|
||||
// The semantics:
|
||||
//
|
||||
// 1. The URL is parsed and its scheme and host lowercased.
|
||||
// 1. The URL is parsed, its scheme and host lowercased, and any userinfo
|
||||
// dropped from the form the regexes see. The request still carries the
|
||||
// credentials.
|
||||
// 2. allowList and denyList apply against the normalized form with OR
|
||||
// semantics. The deny-list always applies.
|
||||
// 3. For http, https, ws, and wss, the host is resolved and every
|
||||
@@ -274,7 +297,19 @@ func DecideOutbound(ctx context.Context, rawURL string, allowList, denyList []*r
|
||||
}
|
||||
parsed.Scheme = strings.ToLower(parsed.Scheme)
|
||||
parsed.Host = strings.ToLower(parsed.Host)
|
||||
normalized := parsed.String()
|
||||
|
||||
// Match on a credential-free form. [url.URL.String] re-emits userinfo
|
||||
// between "scheme://" and the host, so keeping it would let any
|
||||
// "^https?://<host>" pattern be shifted past its own anchor:
|
||||
// http://a@127.0.0.1/ escapes a deny-list anchored on 127\. and
|
||||
// http://trusted.example.com@10.0.0.1/ satisfies an allow-list anchored on
|
||||
// trusted\.example\.com. The host checks below already read
|
||||
// [url.URL.Hostname], which ignores userinfo, so only the regex layer was
|
||||
// affected. Dropping the credentials here also keeps them out of the error
|
||||
// strings below, which reach operator logs and any OTEL log exporter.
|
||||
matchable := *parsed
|
||||
matchable.User = nil
|
||||
normalized := matchable.String()
|
||||
|
||||
allowMatched := false
|
||||
if len(allowList) > 0 {
|
||||
@@ -436,6 +471,16 @@ func (rt *outboundRoundTripper) RoundTrip(req *http.Request) (*http.Response, er
|
||||
// gate this behind their module's opt-in flag. See
|
||||
// https://github.com/gotenberg/gotenberg/issues/1592.
|
||||
func NewOutboundHttpClient(timeout time.Duration, allowList, denyList []*regexp2.Regexp, enableEnvironmentProxy bool, opts ...DecideOption) *http.Client {
|
||||
// A negative timeout means the caller's budget is already spent, which
|
||||
// happens when it derives one from a deadline that has passed. [http.Client]
|
||||
// treats any non-positive Timeout as no deadline at all, so passing it
|
||||
// through would silently produce an unbounded client. Fail closed instead.
|
||||
// Zero keeps meaning unbounded: callers that own the connection lifetime
|
||||
// themselves pass it deliberately.
|
||||
if timeout < 0 {
|
||||
timeout = time.Nanosecond
|
||||
}
|
||||
|
||||
base := http.DefaultTransport.(*http.Transport).Clone()
|
||||
|
||||
var proxyFunc func(*url.URL) (*url.URL, error)
|
||||
@@ -479,6 +524,25 @@ func NewOutboundHttpClient(timeout time.Duration, allowList, denyList []*regexp2
|
||||
}
|
||||
}
|
||||
|
||||
// ClampedBackoff is a [retryablehttp.Backoff] that honors max on every path.
|
||||
//
|
||||
// [retryablehttp.DefaultBackoff] returns a Retry-After header from the remote
|
||||
// verbatim for 429 and 503, and returns it before applying its own max clamp.
|
||||
// A hostile origin therefore decides how long Gotenberg waits, and the wait is
|
||||
// not interruptible. Retry-After is still respected here, just never beyond
|
||||
// the ceiling the caller set.
|
||||
func ClampedBackoff(min, max time.Duration, attemptNum int, resp *http.Response) time.Duration {
|
||||
wait := retryablehttp.DefaultBackoff(min, max, attemptNum, resp)
|
||||
if wait > max {
|
||||
return max
|
||||
}
|
||||
if wait < 0 {
|
||||
return 0
|
||||
}
|
||||
|
||||
return wait
|
||||
}
|
||||
|
||||
// environmentProxyVariables are the variables golang.org/x/net/http/httpproxy
|
||||
// reads, in the casing precedence it applies.
|
||||
var environmentProxyVariables = []string{
|
||||
|
||||
@@ -3,7 +3,9 @@ package gotenberg
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/netip"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -38,6 +40,24 @@ func TestIsPublicIP(t *testing.T) {
|
||||
|
||||
// Link-local.
|
||||
{"169.254.169.254", false},
|
||||
{"169.254.170.2", false},
|
||||
|
||||
// Carrier-grade NAT (RFC 6598). Alibaba Cloud serves instance
|
||||
// metadata from 100.100.100.200.
|
||||
{"100.64.0.0", false},
|
||||
{"100.100.100.200", false},
|
||||
{"100.127.255.255", false},
|
||||
{"::ffff:100.100.100.200", false},
|
||||
|
||||
// Benchmarking (RFC 2544).
|
||||
{"198.18.0.1", false},
|
||||
{"198.19.255.255", false},
|
||||
|
||||
// Adjacent to the ranges above, and public.
|
||||
{"100.63.255.255", true},
|
||||
{"100.128.0.0", true},
|
||||
{"198.17.255.255", true},
|
||||
{"198.20.0.0", true},
|
||||
{"fe80::1", false},
|
||||
|
||||
// Unique-local.
|
||||
@@ -520,3 +540,178 @@ func TestDecideOutbound_Permissive_AllowsPrivate(t *testing.T) {
|
||||
t.Fatalf("decision.Pinned = %v, want [10.0.0.5]", decision.Pinned)
|
||||
}
|
||||
}
|
||||
|
||||
// privateIPsDenyList is the textual private-IP deny-list that shipped as the
|
||||
// default for api-download-from-deny-list and webhook-deny-list in v8.31.0 and
|
||||
// is still published as a migration recipe. Every alternative is anchored on
|
||||
// "://", so userinfo used to slide the private address past the anchor.
|
||||
const privateIPsDenyList = `^https?://(10\.|172\.(1[6-9]|2[0-9]|3[01])\.|192\.168\.|169\.254\.|0\.0\.0\.0|127\.|localhost|\[::1\]|\[fd)`
|
||||
|
||||
func TestDecideOutbound_UserinfoDoesNotEvadeDenyList(t *testing.T) {
|
||||
for _, rawURL := range []string{
|
||||
"http://127.0.0.1:9999/",
|
||||
"http://a@127.0.0.1:9999/",
|
||||
"http://@127.0.0.1:9999/",
|
||||
"http://:@127.0.0.1:9999/",
|
||||
"http://%61@127.0.0.1:9999/",
|
||||
"http://user:pass@127.0.0.1:9999/",
|
||||
"HTTP://A@127.0.0.1:9999/",
|
||||
"http://a@169.254.169.254/latest/meta-data/",
|
||||
// url.Parse takes the last "@" as the userinfo separator, so the host
|
||||
// here is the second literal.
|
||||
"http://a@127.0.0.1:9999@127.0.0.1:9999/",
|
||||
} {
|
||||
t.Run(rawURL, func(t *testing.T) {
|
||||
withStubResolver(t, func(host string) ([]netip.Addr, error) {
|
||||
t.Fatalf("unexpected DNS lookup for %q: the deny-list must reject before resolution", host)
|
||||
return nil, nil
|
||||
})
|
||||
|
||||
// Deny-list only, with the permissive IP defaults the modules ship.
|
||||
_, err := DecideOutbound(
|
||||
context.Background(),
|
||||
rawURL,
|
||||
nil,
|
||||
[]*regexp2.Regexp{regexp2.MustCompile(privateIPsDenyList, 0)},
|
||||
time.Now().Add(5*time.Second),
|
||||
)
|
||||
if !errors.Is(err, ErrFiltered) {
|
||||
t.Fatalf("userinfo must not evade the deny-list, got: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecideOutbound_UserinfoDoesNotSatisfyAllowList(t *testing.T) {
|
||||
// A host-terminated allow-list, the shape the documentation recommends.
|
||||
allowList := []*regexp2.Regexp{regexp2.MustCompile(`^https://trusted\.example\.com(:[0-9]+)?(/|$)`, 0)}
|
||||
|
||||
for _, rawURL := range []string{
|
||||
"https://trusted.example.com@169.254.169.254/latest/meta-data/",
|
||||
"https://trusted.example.com@10.0.0.5/",
|
||||
"https://trusted.example.com:443@10.0.0.5/",
|
||||
} {
|
||||
t.Run(rawURL, func(t *testing.T) {
|
||||
withStubResolver(t, func(host string) ([]netip.Addr, error) {
|
||||
return mustAddrs(t, "10.0.0.5"), nil
|
||||
})
|
||||
|
||||
decision, err := DecideOutbound(
|
||||
context.Background(),
|
||||
rawURL,
|
||||
allowList, nil,
|
||||
time.Now().Add(5*time.Second),
|
||||
WithDenyPrivateIPs(true),
|
||||
)
|
||||
if err == nil {
|
||||
t.Fatalf("userinfo must not satisfy the allow-list, got decision %+v", decision)
|
||||
}
|
||||
if decision.Bypass {
|
||||
t.Fatal("userinfo must never produce a bypass")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecideOutbound_UserinfoKeptOutOfErrorMessages(t *testing.T) {
|
||||
withStubResolver(t, func(host string) ([]netip.Addr, error) {
|
||||
t.Fatalf("unexpected DNS lookup for %q", host)
|
||||
return nil, nil
|
||||
})
|
||||
|
||||
_, err := DecideOutbound(
|
||||
context.Background(),
|
||||
"http://alice:hunter2@127.0.0.1:9999/",
|
||||
nil,
|
||||
[]*regexp2.Regexp{regexp2.MustCompile(privateIPsDenyList, 0)},
|
||||
time.Now().Add(5*time.Second),
|
||||
)
|
||||
if err == nil {
|
||||
t.Fatal("expected the URL to be filtered")
|
||||
}
|
||||
if strings.Contains(err.Error(), "hunter2") || strings.Contains(err.Error(), "alice") {
|
||||
t.Fatalf("error message must not leak URL credentials: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecideOutbound_LegitimateCredentialsStillReachTheHost(t *testing.T) {
|
||||
withStubResolver(t, func(host string) ([]netip.Addr, error) {
|
||||
if host != "example.com" {
|
||||
t.Fatalf("host = %q, want example.com: userinfo must not reach resolution", host)
|
||||
}
|
||||
return mustAddrs(t, "93.184.216.34"), nil
|
||||
})
|
||||
|
||||
// Stripping userinfo is a matching concern only. A credentialed URL that
|
||||
// breaks no rule must still be allowed through.
|
||||
decision, err := DecideOutbound(
|
||||
context.Background(),
|
||||
"https://alice:hunter2@example.com/report.pdf",
|
||||
[]*regexp2.Regexp{regexp2.MustCompile(`^https://example\.com(:[0-9]+)?(/|$)`, 0)},
|
||||
nil,
|
||||
time.Now().Add(5*time.Second),
|
||||
WithDenyPrivateIPs(true),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("credentialed URL matching the allow-list must pass, got: %v", err)
|
||||
}
|
||||
if !decision.Bypass {
|
||||
t.Fatalf("decision.Bypass = false, want true")
|
||||
}
|
||||
}
|
||||
|
||||
func TestClampedBackoff(t *testing.T) {
|
||||
const (
|
||||
min = 1 * time.Second
|
||||
max = 30 * time.Second
|
||||
)
|
||||
|
||||
retryAfter := func(status int, seconds string) *http.Response {
|
||||
return &http.Response{StatusCode: status, Header: http.Header{"Retry-After": []string{seconds}}}
|
||||
}
|
||||
|
||||
for _, tc := range []struct {
|
||||
scenario string
|
||||
resp *http.Response
|
||||
want time.Duration
|
||||
}{
|
||||
{"429 with an hour is clamped", retryAfter(http.StatusTooManyRequests, "3600"), max},
|
||||
{"429 with a day is clamped", retryAfter(http.StatusTooManyRequests, "86400"), max},
|
||||
{"503 with an hour is clamped", retryAfter(http.StatusServiceUnavailable, "3600"), max},
|
||||
{"429 under the ceiling is honored", retryAfter(http.StatusTooManyRequests, "5"), 5 * time.Second},
|
||||
{"no response falls back to exponential", nil, min},
|
||||
} {
|
||||
t.Run(tc.scenario, func(t *testing.T) {
|
||||
got := ClampedBackoff(min, max, 0, tc.resp)
|
||||
if got != tc.want {
|
||||
t.Fatalf("ClampedBackoff = %s, want %s", got, tc.want)
|
||||
}
|
||||
if got > max {
|
||||
t.Fatalf("ClampedBackoff = %s, which exceeds max %s", got, max)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A negative max means the caller's budget is spent. The backoff must not
|
||||
// return a negative duration, which would make the retry loop spin.
|
||||
func TestClampedBackoff_NegativeMaxIsNotNegative(t *testing.T) {
|
||||
got := ClampedBackoff(1*time.Second, -5*time.Second, 0, nil)
|
||||
if got < 0 {
|
||||
t.Fatalf("ClampedBackoff = %s, want a non-negative duration", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewOutboundHttpClient_NonPositiveTimeout(t *testing.T) {
|
||||
// Zero stays unbounded: the LibreOffice proxy owns its own lifetime and
|
||||
// passes it deliberately.
|
||||
if got := NewOutboundHttpClient(0, nil, nil, false).Timeout; got != 0 {
|
||||
t.Fatalf("timeout for 0 = %s, want 0", got)
|
||||
}
|
||||
|
||||
// Negative means an expired budget. http.Client reads any non-positive
|
||||
// Timeout as no deadline at all, so it must not be passed through.
|
||||
if got := NewOutboundHttpClient(-5*time.Second, nil, nil, false).Timeout; got <= 0 {
|
||||
t.Fatalf("timeout for a negative budget = %s, want a positive value so the client fails closed", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -208,7 +208,7 @@ func (a *Api) Descriptor() gotenberg.ModuleDescriptor {
|
||||
fs.String("api-oidc-issuer", "", "Set the OIDC issuer URL, e.g. https://tenant.example.com/ - the token 'iss' claim must match")
|
||||
fs.String("api-oidc-audience", "", "Set the expected OIDC audience - the token 'aud' claim must contain it")
|
||||
fs.String("api-oidc-jwks-url", "", "Set the OIDC JWKS URL - discovered from the issuer's well-known configuration when empty")
|
||||
fs.StringSlice("api-download-from-allow-list", []string{}, "Set the allowed URLs for the download from feature using regular expressions - supports multiple values")
|
||||
fs.StringSlice("api-download-from-allow-list", []string{}, `Set the allowed URLs for the download from feature using regular expressions - supports multiple values. A match bypasses --api-download-from-deny-private-ips (API_DOWNLOAD_FROM_DENY_PRIVATE_IPS) and --api-download-from-deny-public-ips (API_DOWNLOAD_FROM_DENY_PUBLIC_IPS), so terminate the host or the pattern also matches suffix hosts, for example ^https?://internal\.svc(:|/|$)`)
|
||||
fs.StringSlice("api-download-from-deny-list", []string{}, "Set the denied URLs for the download from feature using regular expressions - supports multiple values")
|
||||
fs.Bool("api-download-from-deny-private-ips", false, "Reject downloadFrom URLs whose host resolves to a non-public IP address (loopback, RFC1918, link-local, unique-local). Enable on deployments that accept untrusted downloadFrom sources to mitigate SSRF against internal services")
|
||||
fs.Bool("api-download-from-deny-public-ips", false, "Reject downloadFrom URLs whose host resolves to a public IP address. Enable on air-gapped or data-governed deployments to prevent downloads from reaching the public internet")
|
||||
@@ -383,12 +383,37 @@ func (a *Api) Provision(ctx *gotenberg.Context) error {
|
||||
// Logger.
|
||||
a.logger = gotenberg.Logger(a)
|
||||
|
||||
a.warnInsecureDebugRoute()
|
||||
|
||||
// File system.
|
||||
a.fs = gotenberg.NewFileSystem(new(gotenberg.OsMkdirAll))
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// warnInsecureDebugRoute logs a warning when the debug route is reachable
|
||||
// without authentication.
|
||||
//
|
||||
// The route reports the resolved configuration of every module, which is
|
||||
// useful to an operator and equally useful to anyone else who can reach it.
|
||||
// This warns rather than refuses: an operator may sit behind a gateway that
|
||||
// authenticates on Gotenberg's behalf, and failing startup would break them.
|
||||
func (a *Api) warnInsecureDebugRoute() {
|
||||
if !a.enableDebugRoute || a.basicAuthUsername != "" || a.oidcEnabled {
|
||||
return
|
||||
}
|
||||
if a.logger == nil {
|
||||
return
|
||||
}
|
||||
|
||||
a.logger.WarnContext(
|
||||
context.Background(),
|
||||
"--api-enable-debug-route (API_ENABLE_DEBUG_ROUTE) is enabled but no authentication is configured, so anyone who can reach Gotenberg can read its configuration. Set --api-enable-basic-auth (API_ENABLE_BASIC_AUTH) with GOTENBERG_API_BASIC_AUTH_USERNAME and GOTENBERG_API_BASIC_AUTH_PASSWORD, set --api-enable-oidc-auth (API_ENABLE_OIDC_AUTH), or disable the route.",
|
||||
slog.String("flag", "--api-enable-debug-route"),
|
||||
slog.String("env", "API_ENABLE_DEBUG_ROUTE"),
|
||||
)
|
||||
}
|
||||
|
||||
// Validate validates the module properties.
|
||||
func (a *Api) Validate() error {
|
||||
var err error
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
@@ -65,3 +67,68 @@ func TestApi_Validate_Auth(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestApi_warnInsecureDebugRoute(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
scenario string
|
||||
api Api
|
||||
expectWarn bool
|
||||
expectFields []string
|
||||
}{
|
||||
{
|
||||
scenario: "debug route on with no auth warns",
|
||||
api: Api{enableDebugRoute: true},
|
||||
expectWarn: true,
|
||||
expectFields: []string{"--api-enable-debug-route", "API_ENABLE_DEBUG_ROUTE", "--api-enable-basic-auth", "API_ENABLE_BASIC_AUTH", "--api-enable-oidc-auth", "API_ENABLE_OIDC_AUTH"},
|
||||
},
|
||||
{
|
||||
scenario: "debug route off is silent",
|
||||
api: Api{enableDebugRoute: false},
|
||||
expectWarn: false,
|
||||
},
|
||||
{
|
||||
scenario: "basic auth silences it",
|
||||
api: Api{enableDebugRoute: true, basicAuthUsername: "foo"},
|
||||
expectWarn: false,
|
||||
},
|
||||
{
|
||||
scenario: "oidc silences it",
|
||||
api: Api{enableDebugRoute: true, oidcEnabled: true},
|
||||
expectWarn: false,
|
||||
},
|
||||
} {
|
||||
t.Run(tc.scenario, func(t *testing.T) {
|
||||
buf := new(bytes.Buffer)
|
||||
tc.api.logger = slog.New(slog.NewJSONHandler(buf, &slog.HandlerOptions{Level: slog.LevelWarn}))
|
||||
|
||||
tc.api.warnInsecureDebugRoute()
|
||||
|
||||
logged := buf.String()
|
||||
if !tc.expectWarn {
|
||||
if logged != "" {
|
||||
t.Fatalf("expected no warning, got: %s", logged)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if logged == "" {
|
||||
t.Fatal("expected a warning, got none")
|
||||
}
|
||||
// Every flag named must carry its environment variable.
|
||||
for _, want := range tc.expectFields {
|
||||
if !strings.Contains(logged, want) {
|
||||
t.Fatalf("warning does not mention %q: %s", want, logged)
|
||||
}
|
||||
}
|
||||
if strings.Contains(logged, "—") {
|
||||
t.Fatalf("warning must not contain an em dash: %s", logged)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The warning reads a.logger, which is nil until Provision assigns it.
|
||||
func TestApi_warnInsecureDebugRoute_NilLoggerDoesNotPanic(t *testing.T) {
|
||||
api := Api{enableDebugRoute: true}
|
||||
api.warnInsecureDebugRoute()
|
||||
}
|
||||
|
||||
@@ -51,6 +51,24 @@ type Context struct {
|
||||
outputPaths []string
|
||||
cancelled bool
|
||||
|
||||
// fileOrder records the order files were received in, keyed by disk path.
|
||||
// It breaks ties when two uploads share an original filename, so that
|
||||
// de-duplicated files keep their upload order instead of being ordered by
|
||||
// the suffix uniqueFilename added.
|
||||
fileOrder map[string]int
|
||||
|
||||
// fileBase maps a disk path to the original filename as received, before
|
||||
// de-duplication. Sorting on it keeps a de-duplicated file next to its
|
||||
// twin rather than wherever its numbered name would land.
|
||||
fileBase map[string]string
|
||||
|
||||
// outputFilename is the sanitized Gotenberg-Output-Filename header,
|
||||
// snapshotted while the [echo.Context] is still live. Echo returns that
|
||||
// context to a pool as soon as the handler returns, and an asynchronous
|
||||
// conversion outlives it, so reading the header from the pooled store later
|
||||
// yields whichever request happens to own it by then.
|
||||
outputFilename string
|
||||
|
||||
logger *slog.Logger
|
||||
echoCtx echo.Context
|
||||
mkdirAll gotenberg.MkdirAll
|
||||
@@ -80,6 +98,48 @@ func (t *trackingReader) Read(p []byte) (int, error) {
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// errTooManyDownloadFromEntries is returned by [decodeDownloadFrom] when the
|
||||
// array holds more entries than the configured maximum.
|
||||
var errTooManyDownloadFromEntries = errors.New("too many downloadFrom entries")
|
||||
|
||||
// decodeDownloadFrom decodes the downloadFrom form field, refusing to
|
||||
// accumulate more than maxEntries. A maxEntries of 0 means no limit.
|
||||
//
|
||||
// It decodes element by element rather than calling [json.Unmarshal] on the
|
||||
// whole value. A compact array such as "[{},{},{}]" costs three bytes per
|
||||
// entry on the wire and expands to roughly seventy times that once
|
||||
// unmarshalled, so counting the entries afterwards is too late to bound the
|
||||
// allocation. Streaming keeps the cost proportional to maxEntries no matter
|
||||
// how long the array is.
|
||||
func decodeDownloadFrom(raw string, maxEntries int) ([]downloadFrom, error) {
|
||||
dec := json.NewDecoder(strings.NewReader(raw))
|
||||
|
||||
token, err := dec.Token()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if delim, ok := token.(json.Delim); !ok || delim != '[' {
|
||||
return nil, fmt.Errorf("expected a JSON array, got '%v'", token)
|
||||
}
|
||||
|
||||
var dls []downloadFrom
|
||||
for dec.More() {
|
||||
if maxEntries > 0 && len(dls) >= maxEntries {
|
||||
return nil, errTooManyDownloadFromEntries
|
||||
}
|
||||
|
||||
var dl downloadFrom
|
||||
err = dec.Decode(&dl)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
dls = append(dls, dl)
|
||||
}
|
||||
|
||||
return dls, nil
|
||||
}
|
||||
|
||||
type downloadFrom struct {
|
||||
// Url is the URL to download a file from.
|
||||
Url string `json:"url"`
|
||||
@@ -117,14 +177,18 @@ func newContext(echoCtx echo.Context, logger *slog.Logger, fs *gotenberg.FileSys
|
||||
return nil
|
||||
}
|
||||
|
||||
// Snapshot now, while echoCtx still belongs to this request.
|
||||
outputFilename, _ := echoCtx.Get("outputFilename").(string)
|
||||
|
||||
ctx := &Context{
|
||||
outputPaths: make([]string, 0),
|
||||
cancelled: false,
|
||||
logger: logger,
|
||||
echoCtx: echoCtx,
|
||||
mkdirAll: new(gotenberg.OsMkdirAll),
|
||||
pathRename: new(gotenberg.OsPathRename),
|
||||
Context: processCtx,
|
||||
outputPaths: make([]string, 0),
|
||||
cancelled: false,
|
||||
outputFilename: outputFilename,
|
||||
logger: logger,
|
||||
echoCtx: echoCtx,
|
||||
mkdirAll: new(gotenberg.OsMkdirAll),
|
||||
pathRename: new(gotenberg.OsPathRename),
|
||||
Context: processCtx,
|
||||
}
|
||||
|
||||
// A custom cancel function which removes the context's working directory
|
||||
@@ -213,8 +277,13 @@ func newContext(echoCtx echo.Context, logger *slog.Logger, fs *gotenberg.FileSys
|
||||
// any.
|
||||
raw, ok := ctx.values["downloadFrom"]
|
||||
if !downloadFromCfg.disable && ok {
|
||||
var dls []downloadFrom
|
||||
err = json.Unmarshal([]byte(raw[0]), &dls)
|
||||
dls, err := decodeDownloadFrom(raw[0], downloadFromCfg.maxEntries)
|
||||
if errors.Is(err, errTooManyDownloadFromEntries) {
|
||||
return nil, cancel, WrapError(
|
||||
fmt.Errorf("decode downloadFrom: %w", err),
|
||||
NewSentinelHttpError(http.StatusBadRequest, fmt.Sprintf("Invalid 'downloadFrom' form field value: too many entries, the maximum is %d", downloadFromCfg.maxEntries)),
|
||||
)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, cancel, WrapError(
|
||||
fmt.Errorf("unmarshal json: %w", err),
|
||||
@@ -222,16 +291,6 @@ func newContext(echoCtx echo.Context, logger *slog.Logger, fs *gotenberg.FileSys
|
||||
)
|
||||
}
|
||||
|
||||
// Reject oversized arrays at the trust boundary, before allocating
|
||||
// the results slice or spawning any goroutine, so a compact request
|
||||
// cannot inflate into an unbounded fan-out.
|
||||
if downloadFromCfg.maxEntries > 0 && len(dls) > downloadFromCfg.maxEntries {
|
||||
return nil, cancel, WrapError(
|
||||
fmt.Errorf("too many downloadFrom entries: got %d, max %d", len(dls), downloadFromCfg.maxEntries),
|
||||
NewSentinelHttpError(http.StatusBadRequest, fmt.Sprintf("Invalid 'downloadFrom' form field value: too many entries, the maximum is %d", downloadFromCfg.maxEntries)),
|
||||
)
|
||||
}
|
||||
|
||||
// Each goroutine writes to its own results slot. The main
|
||||
// goroutine merges into ctx.files, ctx.diskToOriginal, and
|
||||
// ctx.filesByField after eg.Wait() to avoid concurrent map
|
||||
@@ -280,7 +339,12 @@ func newContext(echoCtx echo.Context, logger *slog.Logger, fs *gotenberg.FileSys
|
||||
|
||||
logger.DebugContext(dlCtx, fmt.Sprintf("download file from '%s'", dl.Url))
|
||||
|
||||
req, err := retryablehttp.NewRequest(http.MethodGet, dl.Url, nil)
|
||||
// The request must carry dlCtx: retryablehttp.NewRequest builds
|
||||
// on context.Background(), and its wait between attempts is a
|
||||
// select on the request context, so a contextless request cannot
|
||||
// be interrupted by --api-timeout (env API_TIMEOUT) or by the
|
||||
// caller going away.
|
||||
req, err := retryablehttp.NewRequestWithContext(dlCtx, http.MethodGet, dl.Url, nil)
|
||||
if err != nil {
|
||||
dlSpan.RecordError(err)
|
||||
dlSpan.SetStatus(codes.Error, err.Error())
|
||||
@@ -303,14 +367,28 @@ func newContext(echoCtx echo.Context, logger *slog.Logger, fs *gotenberg.FileSys
|
||||
}
|
||||
}
|
||||
|
||||
// Entries are serialized by the concurrency limit above, so a
|
||||
// late one can start after the deadline has already passed.
|
||||
// Fail closed rather than derive a non-positive timeout, which
|
||||
// [http.Client] reads as no deadline at all.
|
||||
remaining := time.Until(deadline)
|
||||
if remaining <= 0 {
|
||||
dlSpan.RecordError(context.DeadlineExceeded)
|
||||
dlSpan.SetStatus(codes.Error, context.DeadlineExceeded.Error())
|
||||
dlSpan.End()
|
||||
return fmt.Errorf("download file from '%s': %w", dl.Url, context.DeadlineExceeded)
|
||||
}
|
||||
|
||||
client := &retryablehttp.Client{
|
||||
HTTPClient: gotenberg.NewOutboundHttpClient(time.Until(deadline), downloadFromCfg.allowList, downloadFromCfg.denyList, downloadFromCfg.enableEnvironmentProxy, ipOpts...),
|
||||
HTTPClient: gotenberg.NewOutboundHttpClient(remaining, downloadFromCfg.allowList, downloadFromCfg.denyList, downloadFromCfg.enableEnvironmentProxy, ipOpts...),
|
||||
RetryMax: downloadFromCfg.maxRetry,
|
||||
RetryWaitMin: time.Duration(1) * time.Second,
|
||||
RetryWaitMax: time.Until(deadline),
|
||||
RetryWaitMax: remaining,
|
||||
Logger: gotenberg.NewLeveledLogger(logger),
|
||||
CheckRetry: retryablehttp.DefaultRetryPolicy,
|
||||
Backoff: retryablehttp.DefaultBackoff,
|
||||
// Not DefaultBackoff: it hands a hostile origin control of
|
||||
// the wait via Retry-After.
|
||||
Backoff: gotenberg.ClampedBackoff,
|
||||
}
|
||||
|
||||
resp, err := client.Do(req)
|
||||
@@ -318,6 +396,17 @@ func newContext(echoCtx echo.Context, logger *slog.Logger, fs *gotenberg.FileSys
|
||||
dlSpan.RecordError(err)
|
||||
dlSpan.SetStatus(codes.Error, err.Error())
|
||||
dlSpan.End()
|
||||
|
||||
// A redirect target is filtered inside the client, so the
|
||||
// policy verdict surfaces here rather than from the
|
||||
// pre-flight above. Keep it out of the response: the first
|
||||
// hop answers a filtered URL with a generic 403, and a
|
||||
// later hop must not describe the allow-list, the deny-list
|
||||
// or the IP policy instead.
|
||||
if errors.Is(err, gotenberg.ErrFiltered) {
|
||||
return fmt.Errorf("download file from '%s': %w", dl.Url, err)
|
||||
}
|
||||
|
||||
return WrapError(
|
||||
fmt.Errorf("download file from to '%s': %w", dl.Url, err),
|
||||
NewSentinelHttpError(http.StatusBadRequest, fmt.Sprintf("Unable to download file from '%s': %s", dl.Url, err)),
|
||||
@@ -391,7 +480,7 @@ func newContext(echoCtx echo.Context, logger *slog.Logger, fs *gotenberg.FileSys
|
||||
// Use a UUID-based name on disk to avoid filesystem
|
||||
// NAME_MAX limits with long filenames.
|
||||
// See: https://github.com/gotenberg/gotenberg/issues/1500.
|
||||
safeName := uuid.New().String() + filepath.Ext(filename)
|
||||
safeName := uuid.New().String() + safeExt(filename)
|
||||
path := fmt.Sprintf("%s/%s", ctx.dirPath, safeName)
|
||||
|
||||
out, err := os.Create(path)
|
||||
@@ -445,18 +534,20 @@ func newContext(echoCtx echo.Context, logger *slog.Logger, fs *gotenberg.FileSys
|
||||
}
|
||||
|
||||
for _, r := range results {
|
||||
ctx.files[r.filename] = r.path
|
||||
ctx.diskToOriginal[r.path] = r.filename
|
||||
filename := ctx.uniqueFilename(r.filename)
|
||||
ctx.files[filename] = r.path
|
||||
ctx.diskToOriginal[r.path] = filename
|
||||
ctx.trackFileOrder(r.path, r.filename)
|
||||
if r.formField != "" {
|
||||
ctx.filesByField[r.formField] = append(ctx.filesByField[r.formField], r.path)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
copyToDisk := func(fh *multipart.FileHeader) error {
|
||||
copyToDisk := func(fh *multipart.FileHeader) (string, error) {
|
||||
in, err := fh.Open()
|
||||
if err != nil {
|
||||
return fmt.Errorf("open multipart file: %w", err)
|
||||
return "", fmt.Errorf("open multipart file: %w", err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
@@ -479,12 +570,12 @@ func newContext(echoCtx echo.Context, logger *slog.Logger, fs *gotenberg.FileSys
|
||||
// Use a UUID-based name on disk to avoid filesystem
|
||||
// NAME_MAX limits with long filenames.
|
||||
// See: https://github.com/gotenberg/gotenberg/issues/1500.
|
||||
safeName := uuid.New().String() + filepath.Ext(filename)
|
||||
safeName := uuid.New().String() + safeExt(filename)
|
||||
path := fmt.Sprintf("%s/%s", ctx.dirPath, safeName)
|
||||
|
||||
out, err := os.Create(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create local file: %w", err)
|
||||
return "", fmt.Errorf("create local file: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
err := out.Close()
|
||||
@@ -495,26 +586,28 @@ func newContext(echoCtx echo.Context, logger *slog.Logger, fs *gotenberg.FileSys
|
||||
|
||||
_, err = io.Copy(out, reader)
|
||||
if err != nil {
|
||||
return fmt.Errorf("copy multipart file to local file: %w", err)
|
||||
return "", fmt.Errorf("copy multipart file to local file: %w", err)
|
||||
}
|
||||
|
||||
base := filename
|
||||
filename = ctx.uniqueFilename(filename)
|
||||
ctx.files[filename] = path
|
||||
ctx.diskToOriginal[path] = filename
|
||||
ctx.trackFileOrder(path, base)
|
||||
|
||||
return nil
|
||||
return filename, nil
|
||||
}
|
||||
|
||||
// Then, copy the form files, if any.
|
||||
for fieldName, files := range form.File {
|
||||
for _, fh := range files {
|
||||
err = copyToDisk(fh)
|
||||
if err != nil {
|
||||
return ctx, cancel, fmt.Errorf("copy to disk: %w", err)
|
||||
filename, errCopy := copyToDisk(fh)
|
||||
if errCopy != nil {
|
||||
return ctx, cancel, fmt.Errorf("copy to disk: %w", errCopy)
|
||||
}
|
||||
// Track files by field name
|
||||
filename := sanitizeFilename(fh.Filename)
|
||||
filePath := ctx.files[filename]
|
||||
ctx.filesByField[fieldName] = append(ctx.filesByField[fieldName], filePath)
|
||||
// Track files by field name, under the name copyToDisk actually
|
||||
// stored, which may be a de-duplicated variant.
|
||||
ctx.filesByField[fieldName] = append(ctx.filesByField[fieldName], ctx.files[filename])
|
||||
}
|
||||
}
|
||||
|
||||
@@ -529,9 +622,9 @@ func newContext(echoCtx echo.Context, logger *slog.Logger, fs *gotenberg.FileSys
|
||||
if symlinkPath == diskPath {
|
||||
continue
|
||||
}
|
||||
err = os.Symlink(filepath.Base(diskPath), symlinkPath)
|
||||
if err != nil {
|
||||
logger.DebugContext(context.Background(), fmt.Sprintf("skip symlink for '%s': %s", originalName, err))
|
||||
errSymlink := os.Symlink(filepath.Base(diskPath), symlinkPath)
|
||||
if errSymlink != nil {
|
||||
logger.DebugContext(context.Background(), fmt.Sprintf("skip symlink for '%s': %s", originalName, errSymlink))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -540,7 +633,10 @@ func newContext(echoCtx echo.Context, logger *slog.Logger, fs *gotenberg.FileSys
|
||||
ctx.Log().DebugContext(ctx, fmt.Sprintf("form files by field: %+v", ctx.filesByField))
|
||||
ctx.Log().DebugContext(ctx, fmt.Sprintf("total bytes: %d", totalBytesRead.Load()))
|
||||
|
||||
return ctx, cancel, err
|
||||
// Explicitly nil: the best-effort symlink loop above must not decide the
|
||||
// outcome of the request. Its failure used to escape here as a bare 500,
|
||||
// non-deterministically, because ctx.files iterates in random order.
|
||||
return ctx, cancel, nil
|
||||
}
|
||||
|
||||
// Request returns the [http.Request].
|
||||
@@ -555,6 +651,8 @@ func (ctx *Context) FormData() *FormData {
|
||||
files: ctx.files,
|
||||
filesByField: ctx.filesByField,
|
||||
diskToOriginal: ctx.diskToOriginal,
|
||||
fileOrder: ctx.fileOrder,
|
||||
fileBase: ctx.fileBase,
|
||||
errors: nil,
|
||||
}
|
||||
}
|
||||
@@ -595,7 +693,7 @@ func (ctx *Context) GeneratePath(extension string) string {
|
||||
// limits but registers the given filename so that [Context.OriginalFilename]
|
||||
// can resolve it. It does not create a file.
|
||||
func (ctx *Context) GeneratePathFromFilename(filename string) string {
|
||||
safeName := uuid.New().String() + filepath.Ext(filename)
|
||||
safeName := uuid.New().String() + safeExt(filename)
|
||||
path := fmt.Sprintf("%s/%s", ctx.dirPath, safeName)
|
||||
ctx.diskToOriginal[path] = filename
|
||||
return path
|
||||
@@ -698,15 +796,72 @@ func (ctx *Context) BuildOutputFile() (string, error) {
|
||||
// OutputFilename returns the filename based on the given output path or the
|
||||
// "Gotenberg-Output-Filename" header's value.
|
||||
func (ctx *Context) OutputFilename(outputPath string) string {
|
||||
filename := ctx.echoCtx.Get("outputFilename").(string)
|
||||
|
||||
if filename == "" {
|
||||
if ctx.outputFilename == "" {
|
||||
return ctx.OriginalFilename(outputPath)
|
||||
}
|
||||
|
||||
filename := ctx.outputFilename
|
||||
|
||||
return fmt.Sprintf("%s%s", filename, filepath.Ext(outputPath))
|
||||
}
|
||||
|
||||
// maxDiskExtLength bounds the extension copied onto a UUID-based disk name.
|
||||
// The UUID stem is 36 characters, so a longer extension risks NAME_MAX, which
|
||||
// is 255 on ext4 and overlayfs. The untruncated name is kept in
|
||||
// [Context.diskToOriginal], which never reaches the filesystem.
|
||||
const maxDiskExtLength = 32
|
||||
|
||||
// safeExt returns the extension to append to a UUID-based disk name. It drops
|
||||
// an extension too long to be safe rather than let [os.Create] fail with
|
||||
// ENAMETOOLONG, which surfaced to the caller as a bare 500.
|
||||
func safeExt(filename string) string {
|
||||
ext := filepath.Ext(filename)
|
||||
if len(ext) > maxDiskExtLength {
|
||||
return ""
|
||||
}
|
||||
|
||||
return ext
|
||||
}
|
||||
|
||||
// uniqueFilename returns filename, or a numbered variant of it when the
|
||||
// request already carries a file by that name.
|
||||
//
|
||||
// Uploads are keyed by their sanitized original filename, so two files sharing
|
||||
// one name used to collide: the second overwrote the first and only one
|
||||
// reached the conversion, while both stayed on disk and counted against the
|
||||
// body limit. Sanitizing strips directories, so "a/doc.pdf" and "b/doc.pdf"
|
||||
// collide too.
|
||||
func (ctx *Context) uniqueFilename(filename string) string {
|
||||
_, exists := ctx.files[filename]
|
||||
if !exists {
|
||||
return filename
|
||||
}
|
||||
|
||||
ext := filepath.Ext(filename)
|
||||
stem := strings.TrimSuffix(filename, ext)
|
||||
|
||||
for i := 2; ; i++ {
|
||||
candidate := fmt.Sprintf("%s (%d)%s", stem, i, ext)
|
||||
_, exists = ctx.files[candidate]
|
||||
if !exists {
|
||||
return candidate
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// trackFileOrder records where a file arrived in the request and the filename
|
||||
// it arrived under, so [FormData.paths] can order it the way the caller sent
|
||||
// it.
|
||||
func (ctx *Context) trackFileOrder(path, base string) {
|
||||
if ctx.fileOrder == nil {
|
||||
ctx.fileOrder = make(map[string]int)
|
||||
ctx.fileBase = make(map[string]string)
|
||||
}
|
||||
|
||||
ctx.fileOrder[path] = len(ctx.fileOrder)
|
||||
ctx.fileBase[path] = base
|
||||
}
|
||||
|
||||
// sanitizeFilename strips path separators (including backslashes, which
|
||||
// [filepath.Base] ignores on Linux) and control characters from a
|
||||
// caller-supplied filename, then NFC-normalizes the result. This prevents a
|
||||
|
||||
@@ -11,11 +11,15 @@ import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"regexp"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/dlclark/regexp2"
|
||||
"github.com/labstack/echo/v4"
|
||||
|
||||
"github.com/gotenberg/gotenberg/v8/pkg/gotenberg"
|
||||
@@ -414,3 +418,487 @@ func TestContext_FileCount(t *testing.T) {
|
||||
t.Errorf("expected 3 files, got %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A hostile origin must not choose how long Gotenberg waits.
|
||||
// [retryablehttp.DefaultBackoff] returns a Retry-After header verbatim for 429
|
||||
// and 503, and the wait between attempts is a select on the request context.
|
||||
// Building the request without a context therefore pinned the goroutine, its
|
||||
// connection, and its working directory for the attacker's chosen duration,
|
||||
// well past --api-timeout (env API_TIMEOUT).
|
||||
func TestNewContext_DownloadFromHostileRetryAfterIsBounded(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Retry-After", "3600")
|
||||
w.WriteHeader(http.StatusTooManyRequests)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
payload, err := json.Marshal([]downloadFrom{{Url: server.URL + "/file"}})
|
||||
if err != nil {
|
||||
t.Fatalf("marshal downloadFrom payload: %v", err)
|
||||
}
|
||||
|
||||
body := new(bytes.Buffer)
|
||||
writer := multipart.NewWriter(body)
|
||||
err = writer.WriteField("downloadFrom", string(payload))
|
||||
if err != nil {
|
||||
t.Fatalf("write downloadFrom field: %v", err)
|
||||
}
|
||||
err = writer.Close()
|
||||
if err != nil {
|
||||
t.Fatalf("close multipart writer: %v", err)
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/forms/libreoffice/convert", body)
|
||||
req.Header.Set("Content-Type", writer.FormDataContentType())
|
||||
|
||||
echoCtx := echo.New().NewContext(req, httptest.NewRecorder())
|
||||
logger := slog.New(slog.DiscardHandler)
|
||||
fs := gotenberg.NewFileSystem(new(gotenberg.OsMkdirAll))
|
||||
|
||||
const timeout = 500 * time.Millisecond
|
||||
|
||||
start := time.Now()
|
||||
_, cancel, err := newContext(echoCtx, logger, fs, timeout, 0, downloadFromConfig{maxRetry: 2})
|
||||
elapsed := time.Since(start)
|
||||
if cancel != nil {
|
||||
defer cancel()
|
||||
}
|
||||
|
||||
if err == nil {
|
||||
t.Fatal("expected newContext to fail against an origin that only answers 429")
|
||||
}
|
||||
// Generous: the deadline is 500ms and Retry-After asks for an hour. Any
|
||||
// value in seconds means the remote is still in control.
|
||||
if elapsed > 10*time.Second {
|
||||
t.Fatalf("newContext took %s with Retry-After 3600; --api-timeout must bound it", elapsed)
|
||||
}
|
||||
}
|
||||
|
||||
// An entry that starts after the deadline has passed must fail closed. It used
|
||||
// to derive a negative client timeout, which [http.Client] reads as no
|
||||
// deadline at all, leaving the download unbounded.
|
||||
func TestNewContext_DownloadFromExpiredBudgetFailsClosed(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
<-r.Context().Done()
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
// Two entries, serialized by the concurrency limit, so the second one
|
||||
// starts once the first has burned the whole budget.
|
||||
payload, err := json.Marshal([]downloadFrom{
|
||||
{Url: server.URL + "/first"},
|
||||
{Url: server.URL + "/second"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("marshal downloadFrom payload: %v", err)
|
||||
}
|
||||
|
||||
body := new(bytes.Buffer)
|
||||
writer := multipart.NewWriter(body)
|
||||
err = writer.WriteField("downloadFrom", string(payload))
|
||||
if err != nil {
|
||||
t.Fatalf("write downloadFrom field: %v", err)
|
||||
}
|
||||
err = writer.Close()
|
||||
if err != nil {
|
||||
t.Fatalf("close multipart writer: %v", err)
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/forms/libreoffice/convert", body)
|
||||
req.Header.Set("Content-Type", writer.FormDataContentType())
|
||||
|
||||
echoCtx := echo.New().NewContext(req, httptest.NewRecorder())
|
||||
logger := slog.New(slog.DiscardHandler)
|
||||
fs := gotenberg.NewFileSystem(new(gotenberg.OsMkdirAll))
|
||||
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, cancel, err := newContext(echoCtx, logger, fs, 400*time.Millisecond, 0, downloadFromConfig{
|
||||
maxRetry: 0,
|
||||
maxConcurrency: 1,
|
||||
})
|
||||
if cancel != nil {
|
||||
cancel()
|
||||
}
|
||||
done <- err
|
||||
}()
|
||||
|
||||
select {
|
||||
case err := <-done:
|
||||
if err == nil {
|
||||
t.Fatal("expected newContext to fail against a stalling origin")
|
||||
}
|
||||
case <-time.After(15 * time.Second):
|
||||
t.Fatal("newContext never returned: an entry starting past the deadline built an unbounded client")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeDownloadFrom(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
scenario string
|
||||
raw string
|
||||
maxEntries int
|
||||
expectErr error
|
||||
expectLen int
|
||||
}{
|
||||
{"empty array", `[]`, 10, nil, 0},
|
||||
{"under the limit", `[{"url":"http://a"},{"url":"http://b"}]`, 10, nil, 2},
|
||||
{"exactly the limit", `[{"url":"http://a"},{"url":"http://b"}]`, 2, nil, 2},
|
||||
{"over the limit", `[{"url":"http://a"},{"url":"http://b"}]`, 1, errTooManyDownloadFromEntries, 0},
|
||||
{"no limit", `[{"url":"http://a"},{"url":"http://b"}]`, 0, nil, 2},
|
||||
{"not an array", `{"url":"http://a"}`, 10, nil, 0},
|
||||
{"malformed", `[{"url":`, 10, nil, 0},
|
||||
{"not json", `nope`, 10, nil, 0},
|
||||
} {
|
||||
t.Run(tc.scenario, func(t *testing.T) {
|
||||
dls, err := decodeDownloadFrom(tc.raw, tc.maxEntries)
|
||||
|
||||
if tc.expectErr != nil {
|
||||
if !errors.Is(err, tc.expectErr) {
|
||||
t.Fatalf("error = %v, want %v", err, tc.expectErr)
|
||||
}
|
||||
return
|
||||
}
|
||||
if tc.scenario == "not an array" || tc.scenario == "malformed" || tc.scenario == "not json" {
|
||||
if err == nil {
|
||||
t.Fatalf("expected an error for %q", tc.raw)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if len(dls) != tc.expectLen {
|
||||
t.Fatalf("decoded %d entries, want %d", len(dls), tc.expectLen)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A compact array costs three bytes per entry on the wire and expands by
|
||||
// roughly seventy times once unmarshalled. Decoding must stop at the limit
|
||||
// rather than materialize the whole array and count afterwards.
|
||||
func TestDecodeDownloadFrom_StopsBeforeMaterializingTheArray(t *testing.T) {
|
||||
const entries = 2_000_000
|
||||
|
||||
raw := "[" + strings.Repeat("{},", entries) + "{}]"
|
||||
|
||||
var before, after runtime.MemStats
|
||||
runtime.GC()
|
||||
runtime.ReadMemStats(&before)
|
||||
|
||||
_, err := decodeDownloadFrom(raw, 1000)
|
||||
|
||||
runtime.ReadMemStats(&after)
|
||||
|
||||
if !errors.Is(err, errTooManyDownloadFromEntries) {
|
||||
t.Fatalf("error = %v, want errTooManyDownloadFromEntries", err)
|
||||
}
|
||||
|
||||
// json.Unmarshal on the same input allocates hundreds of MiB. Bounded
|
||||
// decoding should stay in the low single-digit MiB, so this threshold is
|
||||
// deliberately loose and still fails loudly on a regression.
|
||||
allocated := after.TotalAlloc - before.TotalAlloc
|
||||
if allocated > 32<<20 {
|
||||
t.Fatalf("decoding allocated %d MiB for a %d-entry array, want the limit to bound it", allocated>>20, entries)
|
||||
}
|
||||
t.Logf("allocated %d KiB decoding a %d-entry array with a limit of 1000", allocated>>10, entries)
|
||||
}
|
||||
|
||||
// An asynchronous conversion outlives the [echo.Context]. Echo returns that
|
||||
// context to a sync.Pool as soon as the handler returns, and
|
||||
// outputFilenameMiddleware runs in srv.Pre on every request, including
|
||||
// /health, so a later request overwrites the store. Reading the output
|
||||
// filename from it after the fact returned another caller's value.
|
||||
func TestContext_OutputFilename_SurvivesEchoContextRecycling(t *testing.T) {
|
||||
body := new(bytes.Buffer)
|
||||
writer := multipart.NewWriter(body)
|
||||
err := writer.Close()
|
||||
if err != nil {
|
||||
t.Fatalf("close multipart writer: %v", err)
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/forms/libreoffice/convert", body)
|
||||
req.Header.Set("Content-Type", writer.FormDataContentType())
|
||||
|
||||
echoCtx := echo.New().NewContext(req, httptest.NewRecorder())
|
||||
// What outputFilenameMiddleware does for this request.
|
||||
echoCtx.Set("outputFilename", "victim")
|
||||
|
||||
logger := slog.New(slog.DiscardHandler)
|
||||
fs := gotenberg.NewFileSystem(new(gotenberg.OsMkdirAll))
|
||||
|
||||
ctx, cancel, err := newContext(echoCtx, logger, fs, 10*time.Second, 0, downloadFromConfig{disable: true})
|
||||
if err != nil {
|
||||
t.Fatalf("newContext returned error: %v", err)
|
||||
}
|
||||
defer cancel()
|
||||
|
||||
// Echo recycles the context and another request claims the store.
|
||||
echoCtx.Set("outputFilename", "attacker-controlled")
|
||||
|
||||
if got := ctx.OutputFilename("/tmp/out.pdf"); got != "victim.pdf" {
|
||||
t.Fatalf("OutputFilename = %q, want %q", got, "victim.pdf")
|
||||
}
|
||||
}
|
||||
|
||||
// A recycled context has a nil store, so the previous unguarded type assertion
|
||||
// could panic. The snapshot must tolerate an absent value.
|
||||
func TestContext_OutputFilename_NoHeader(t *testing.T) {
|
||||
body := new(bytes.Buffer)
|
||||
writer := multipart.NewWriter(body)
|
||||
err := writer.Close()
|
||||
if err != nil {
|
||||
t.Fatalf("close multipart writer: %v", err)
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/forms/libreoffice/convert", body)
|
||||
req.Header.Set("Content-Type", writer.FormDataContentType())
|
||||
|
||||
// No Set call at all: the store holds nothing for "outputFilename".
|
||||
echoCtx := echo.New().NewContext(req, httptest.NewRecorder())
|
||||
|
||||
logger := slog.New(slog.DiscardHandler)
|
||||
fs := gotenberg.NewFileSystem(new(gotenberg.OsMkdirAll))
|
||||
|
||||
ctx, cancel, err := newContext(echoCtx, logger, fs, 10*time.Second, 0, downloadFromConfig{disable: true})
|
||||
if err != nil {
|
||||
t.Fatalf("newContext returned error: %v", err)
|
||||
}
|
||||
defer cancel()
|
||||
|
||||
if got := ctx.OutputFilename("/tmp/out.pdf"); got != "out.pdf" {
|
||||
t.Fatalf("OutputFilename = %q, want the original filename %q", got, "out.pdf")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSafeExt(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
scenario string
|
||||
filename string
|
||||
want string
|
||||
}{
|
||||
{"ordinary extension", "report.pdf", ".pdf"},
|
||||
{"no extension", "report", ""},
|
||||
{"at the limit", "a." + strings.Repeat("x", maxDiskExtLength-1), "." + strings.Repeat("x", maxDiskExtLength-1)},
|
||||
{"over the limit is dropped", "a." + strings.Repeat("x", 300), ""},
|
||||
} {
|
||||
t.Run(tc.scenario, func(t *testing.T) {
|
||||
got := safeExt(tc.filename)
|
||||
if got != tc.want {
|
||||
t.Fatalf("safeExt(%q) = %q, want %q", tc.filename, got, tc.want)
|
||||
}
|
||||
// A UUID stem is 36 characters. The whole disk name must stay
|
||||
// under NAME_MAX.
|
||||
if len(got)+36 > 255 {
|
||||
t.Fatalf("disk name would be %d characters, over NAME_MAX", len(got)+36)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// An upload whose extension exceeds NAME_MAX used to fail os.Create and return
|
||||
// a bare 500. The extension is bounded, and the original name survives in
|
||||
// diskToOriginal.
|
||||
func TestNewContext_LongExtensionIsAccepted(t *testing.T) {
|
||||
filename := "invoice." + strings.Repeat("x", 300)
|
||||
|
||||
body := new(bytes.Buffer)
|
||||
writer := multipart.NewWriter(body)
|
||||
part, err := writer.CreateFormFile("files", filename)
|
||||
if err != nil {
|
||||
t.Fatalf("create multipart file: %v", err)
|
||||
}
|
||||
_, err = part.Write([]byte("%PDF-1.4"))
|
||||
if err != nil {
|
||||
t.Fatalf("write multipart file: %v", err)
|
||||
}
|
||||
err = writer.Close()
|
||||
if err != nil {
|
||||
t.Fatalf("close multipart writer: %v", err)
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/forms/libreoffice/convert", body)
|
||||
req.Header.Set("Content-Type", writer.FormDataContentType())
|
||||
|
||||
echoCtx := echo.New().NewContext(req, httptest.NewRecorder())
|
||||
logger := slog.New(slog.DiscardHandler)
|
||||
fs := gotenberg.NewFileSystem(new(gotenberg.OsMkdirAll))
|
||||
|
||||
ctx, cancel, err := newContext(echoCtx, logger, fs, 10*time.Second, 0, downloadFromConfig{disable: true})
|
||||
if cancel != nil {
|
||||
defer cancel()
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("newContext returned error for a long extension: %v", err)
|
||||
}
|
||||
if got := len(ctx.files); got != 1 {
|
||||
t.Fatalf("files = %d, want 1", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A filename that cannot become a symlink (too long, "..", "/") must not fail
|
||||
// the request. The symlink loop is best-effort, but its error escaped through
|
||||
// the shared err variable, and ctx.files iterates randomly, so byte-identical
|
||||
// requests gave different HTTP outcomes.
|
||||
func TestNewContext_UnsymlinkableFilenameStillSucceeds(t *testing.T) {
|
||||
for _, filename := range []string{
|
||||
strings.Repeat("a", 300) + ".txt",
|
||||
"..",
|
||||
"/",
|
||||
} {
|
||||
t.Run(filename[:min(len(filename), 12)], func(t *testing.T) {
|
||||
body := new(bytes.Buffer)
|
||||
writer := multipart.NewWriter(body)
|
||||
part, err := writer.CreateFormFile("files", filename)
|
||||
if err != nil {
|
||||
t.Fatalf("create multipart file: %v", err)
|
||||
}
|
||||
_, err = part.Write([]byte("%PDF-1.4"))
|
||||
if err != nil {
|
||||
t.Fatalf("write multipart file: %v", err)
|
||||
}
|
||||
err = writer.Close()
|
||||
if err != nil {
|
||||
t.Fatalf("close multipart writer: %v", err)
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/forms/libreoffice/convert", body)
|
||||
req.Header.Set("Content-Type", writer.FormDataContentType())
|
||||
|
||||
echoCtx := echo.New().NewContext(req, httptest.NewRecorder())
|
||||
logger := slog.New(slog.DiscardHandler)
|
||||
fs := gotenberg.NewFileSystem(new(gotenberg.OsMkdirAll))
|
||||
|
||||
_, cancel, err := newContext(echoCtx, logger, fs, 10*time.Second, 0, downloadFromConfig{disable: true})
|
||||
if cancel != nil {
|
||||
defer cancel()
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("newContext failed on a best-effort symlink for %q: %v", filename, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Two uploads sharing a filename must both reach the conversion. The second
|
||||
// used to overwrite the first in ctx.files, so one file was silently dropped
|
||||
// while both stayed on disk and counted against the body limit.
|
||||
func TestNewContext_DuplicateFilenamesAreBothKept(t *testing.T) {
|
||||
body := new(bytes.Buffer)
|
||||
writer := multipart.NewWriter(body)
|
||||
for _, content := range []string{"FIRST", "SECOND"} {
|
||||
part, err := writer.CreateFormFile("files", "doc.pdf")
|
||||
if err != nil {
|
||||
t.Fatalf("create multipart file: %v", err)
|
||||
}
|
||||
_, err = part.Write([]byte(content))
|
||||
if err != nil {
|
||||
t.Fatalf("write multipart file: %v", err)
|
||||
}
|
||||
}
|
||||
err := writer.Close()
|
||||
if err != nil {
|
||||
t.Fatalf("close multipart writer: %v", err)
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/forms/pdfengines/merge", body)
|
||||
req.Header.Set("Content-Type", writer.FormDataContentType())
|
||||
|
||||
echoCtx := echo.New().NewContext(req, httptest.NewRecorder())
|
||||
logger := slog.New(slog.DiscardHandler)
|
||||
fs := gotenberg.NewFileSystem(new(gotenberg.OsMkdirAll))
|
||||
|
||||
ctx, cancel, err := newContext(echoCtx, logger, fs, 10*time.Second, 0, downloadFromConfig{disable: true})
|
||||
if err != nil {
|
||||
t.Fatalf("newContext returned error: %v", err)
|
||||
}
|
||||
defer cancel()
|
||||
|
||||
if got := len(ctx.files); got != 2 {
|
||||
t.Fatalf("ctx.files = %d entries, want 2: a duplicate filename dropped a file", got)
|
||||
}
|
||||
if got := len(ctx.filesByField["files"]); got != 2 {
|
||||
t.Fatalf("filesByField[files] = %d entries, want 2", got)
|
||||
}
|
||||
|
||||
// The two maps must agree, and both files must be distinct on disk.
|
||||
seen := make(map[string]struct{})
|
||||
for _, path := range ctx.files {
|
||||
if _, ok := ctx.diskToOriginal[path]; !ok {
|
||||
t.Fatalf("path %q has no diskToOriginal entry", path)
|
||||
}
|
||||
seen[path] = struct{}{}
|
||||
}
|
||||
if len(seen) != 2 {
|
||||
t.Fatalf("distinct disk paths = %d, want 2", len(seen))
|
||||
}
|
||||
}
|
||||
|
||||
// A redirect target is filtered inside the HTTP client, so the policy verdict
|
||||
// surfaces from client.Do rather than from the pre-flight check. It used to be
|
||||
// interpolated into the response body, so a redirect described the allow-list,
|
||||
// the deny-list or the IP policy where the first hop returns a generic 403.
|
||||
func TestNewContext_DownloadFromRedirectVerdictStaysGeneric(t *testing.T) {
|
||||
private := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Disposition", `attachment; filename="secret.txt"`)
|
||||
_, _ = w.Write([]byte("internal"))
|
||||
}))
|
||||
defer private.Close()
|
||||
|
||||
redirector := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
http.Redirect(w, r, private.URL+"/secret", http.StatusFound)
|
||||
}))
|
||||
defer redirector.Close()
|
||||
|
||||
payload, err := json.Marshal([]downloadFrom{{Url: redirector.URL + "/start"}})
|
||||
if err != nil {
|
||||
t.Fatalf("marshal downloadFrom payload: %v", err)
|
||||
}
|
||||
|
||||
body := new(bytes.Buffer)
|
||||
writer := multipart.NewWriter(body)
|
||||
err = writer.WriteField("downloadFrom", string(payload))
|
||||
if err != nil {
|
||||
t.Fatalf("write downloadFrom field: %v", err)
|
||||
}
|
||||
err = writer.Close()
|
||||
if err != nil {
|
||||
t.Fatalf("close multipart writer: %v", err)
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/forms/libreoffice/convert", body)
|
||||
req.Header.Set("Content-Type", writer.FormDataContentType())
|
||||
|
||||
echoCtx := echo.New().NewContext(req, httptest.NewRecorder())
|
||||
logger := slog.New(slog.DiscardHandler)
|
||||
fs := gotenberg.NewFileSystem(new(gotenberg.OsMkdirAll))
|
||||
|
||||
// The first hop is allowed, the redirect target is denied by the deny-list.
|
||||
denyList := []*regexp2.Regexp{regexp2.MustCompile("^"+regexp.QuoteMeta(private.URL), 0)}
|
||||
|
||||
_, cancel, err := newContext(echoCtx, logger, fs, 10*time.Second, 0, downloadFromConfig{
|
||||
denyList: denyList,
|
||||
maxRetry: 0,
|
||||
})
|
||||
if cancel != nil {
|
||||
defer cancel()
|
||||
}
|
||||
if err == nil {
|
||||
t.Fatal("expected the redirect to a denied host to fail")
|
||||
}
|
||||
|
||||
status, message := ParseError(err)
|
||||
if status != http.StatusForbidden {
|
||||
t.Fatalf("status = %d, want %d: a filtered redirect must answer like a filtered first hop", status, http.StatusForbidden)
|
||||
}
|
||||
if message != http.StatusText(http.StatusForbidden) {
|
||||
t.Fatalf("message = %q, want the generic %q", message, http.StatusText(http.StatusForbidden))
|
||||
}
|
||||
// The response must not name the policy, the pattern, or the blocked host.
|
||||
for _, leak := range []string{"denied list", "allowed list", "non-public", "expression", private.URL} {
|
||||
if strings.Contains(message, leak) {
|
||||
t.Fatalf("response message %q leaks %q", message, leak)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -41,6 +41,8 @@ type FormData struct {
|
||||
files map[string]string
|
||||
filesByField map[string][]string
|
||||
diskToOriginal map[string]string
|
||||
fileOrder map[string]int
|
||||
fileBase map[string]string
|
||||
errors error
|
||||
}
|
||||
|
||||
@@ -582,24 +584,42 @@ func (form *FormData) paths(extensions []string, target *[]string) *FormData {
|
||||
}
|
||||
|
||||
// See https://github.com/gotenberg/gotenberg/issues/139.
|
||||
originals := make(gotenberg.AlphanumericSort, len(entries))
|
||||
for i, e := range entries {
|
||||
originals[i] = e.original
|
||||
}
|
||||
sort.Sort(originals)
|
||||
//
|
||||
// Sort on the filename as received rather than on the map key. The key
|
||||
// carries the suffix uniqueFilename adds when two uploads share a name,
|
||||
// and that suffix would otherwise decide the order: "doc (2).pdf" sorts
|
||||
// before "doc.pdf". Ordering on the received name keeps the pair adjacent,
|
||||
// and the arrival index breaks the tie, so duplicates merge in the order
|
||||
// the caller sent them. A file with a unique name is unaffected, since its
|
||||
// received name and its key are the same string.
|
||||
sort.SliceStable(entries, func(i, j int) bool {
|
||||
nameI := form.receivedName(entries[i].disk, entries[i].original)
|
||||
nameJ := form.receivedName(entries[j].disk, entries[j].original)
|
||||
if nameI != nameJ {
|
||||
return gotenberg.AlphanumericSort{nameI, nameJ}.Less(0, 1)
|
||||
}
|
||||
|
||||
return form.fileOrder[entries[i].disk] < form.fileOrder[entries[j].disk]
|
||||
})
|
||||
|
||||
// Build a lookup from original name to disk path.
|
||||
lookup := make(map[string]string, len(entries))
|
||||
for _, e := range entries {
|
||||
lookup[e.original] = e.disk
|
||||
}
|
||||
for _, o := range originals {
|
||||
*target = append(*target, lookup[o])
|
||||
*target = append(*target, e.disk)
|
||||
}
|
||||
|
||||
return form
|
||||
}
|
||||
|
||||
// receivedName returns the filename the file at disk arrived under, before
|
||||
// de-duplication, falling back to fallback.
|
||||
func (form *FormData) receivedName(disk, fallback string) string {
|
||||
base, ok := form.fileBase[disk]
|
||||
if ok {
|
||||
return base
|
||||
}
|
||||
|
||||
return fallback
|
||||
}
|
||||
|
||||
// append adds an error to the list of errors.
|
||||
func (form *FormData) append(err error) {
|
||||
form.errors = errors.Join(form.errors, err)
|
||||
|
||||
@@ -1895,3 +1895,81 @@ func TestFormData_Watermarks(t *testing.T) {
|
||||
t.Errorf("expected %+v, got %+v", want, got)
|
||||
}
|
||||
}
|
||||
|
||||
// De-duplicating a repeated filename must not change merge order. Files with
|
||||
// unique names keep exactly the order they had before de-duplication existed,
|
||||
// and two files sharing a name merge in the order the caller sent them.
|
||||
func TestFormData_paths_DuplicateFilenamesKeepUploadOrder(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
scenario string
|
||||
files map[string]string
|
||||
fileBase map[string]string
|
||||
order map[string]int
|
||||
want []string
|
||||
}{
|
||||
{
|
||||
scenario: "unique names sort exactly as before",
|
||||
files: map[string]string{"b.pdf": "/w/2", "a.pdf": "/w/1", "c.pdf": "/w/3"},
|
||||
fileBase: map[string]string{"/w/1": "a.pdf", "/w/2": "b.pdf", "/w/3": "c.pdf"},
|
||||
order: map[string]int{"/w/1": 0, "/w/2": 1, "/w/3": 2},
|
||||
want: []string{"/w/1", "/w/2", "/w/3"},
|
||||
},
|
||||
{
|
||||
scenario: "numeric prefixes still win",
|
||||
files: map[string]string{"10_x.pdf": "/w/3", "2_x.pdf": "/w/2", "1_x.pdf": "/w/1"},
|
||||
fileBase: map[string]string{"/w/1": "1_x.pdf", "/w/2": "2_x.pdf", "/w/3": "10_x.pdf"},
|
||||
order: map[string]int{"/w/1": 0, "/w/2": 1, "/w/3": 2},
|
||||
want: []string{"/w/1", "/w/2", "/w/3"},
|
||||
},
|
||||
{
|
||||
scenario: "duplicates merge in upload order, not suffix order",
|
||||
files: map[string]string{"doc.pdf": "/w/1", "doc (2).pdf": "/w/2"},
|
||||
fileBase: map[string]string{"/w/1": "doc.pdf", "/w/2": "doc.pdf"},
|
||||
order: map[string]int{"/w/1": 0, "/w/2": 1},
|
||||
want: []string{"/w/1", "/w/2"},
|
||||
},
|
||||
{
|
||||
scenario: "duplicates stay adjacent and in position",
|
||||
files: map[string]string{
|
||||
"a.pdf": "/w/1", "doc.pdf": "/w/2", "doc (2).pdf": "/w/3", "z.pdf": "/w/4",
|
||||
},
|
||||
fileBase: map[string]string{
|
||||
"/w/1": "a.pdf", "/w/2": "doc.pdf", "/w/3": "doc.pdf", "/w/4": "z.pdf",
|
||||
},
|
||||
order: map[string]int{"/w/1": 0, "/w/2": 1, "/w/3": 2, "/w/4": 3},
|
||||
want: []string{"/w/1", "/w/2", "/w/3", "/w/4"},
|
||||
},
|
||||
{
|
||||
scenario: "three copies keep their order",
|
||||
files: map[string]string{"r.pdf": "/w/1", "r (2).pdf": "/w/2", "r (3).pdf": "/w/3"},
|
||||
fileBase: map[string]string{"/w/1": "r.pdf", "/w/2": "r.pdf", "/w/3": "r.pdf"},
|
||||
order: map[string]int{"/w/1": 0, "/w/2": 1, "/w/3": 2},
|
||||
want: []string{"/w/1", "/w/2", "/w/3"},
|
||||
},
|
||||
} {
|
||||
t.Run(tc.scenario, func(t *testing.T) {
|
||||
form := &FormData{
|
||||
files: tc.files,
|
||||
filesByField: map[string][]string{},
|
||||
fileBase: tc.fileBase,
|
||||
fileOrder: tc.order,
|
||||
}
|
||||
|
||||
// Map iteration is randomised, so run it repeatedly: an unstable
|
||||
// comparator shows up as a differing result across runs.
|
||||
for range 50 {
|
||||
var got []string
|
||||
form.paths([]string{".pdf"}, &got)
|
||||
|
||||
if len(got) != len(tc.want) {
|
||||
t.Fatalf("paths() returned %d entries, want %d", len(got), len(tc.want))
|
||||
}
|
||||
for i := range got {
|
||||
if got[i] != tc.want[i] {
|
||||
t.Fatalf("paths() = %v, want %v", got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -475,7 +475,7 @@ func (mod *Chromium) Descriptor() gotenberg.ModuleDescriptor {
|
||||
fs.String("chromium-host-resolver-rules", "", "Set custom mappings to the host resolver")
|
||||
fs.String("chromium-proxy-server", "", "Set the outbound proxy server; this switch only affects HTTP and HTTPS requests")
|
||||
fs.Bool("chromium-enable-environment-proxy", false, "Route Chromium's outbound requests through the proxy defined by the standard HTTP_PROXY, HTTPS_PROXY, and NO_PROXY variables, including credentials. Use this instead of --chromium-proxy-server for authenticated proxies, and leave --chromium-proxy-server and --chromium-host-resolver-rules unset")
|
||||
fs.StringSlice("chromium-allow-list", []string{}, "Set the allowed URLs for Chromium using regular expressions - supports multiple values")
|
||||
fs.StringSlice("chromium-allow-list", []string{}, `Set the allowed URLs for Chromium using regular expressions - supports multiple values. A match bypasses --chromium-deny-private-ips (CHROMIUM_DENY_PRIVATE_IPS) and --chromium-deny-public-ips (CHROMIUM_DENY_PUBLIC_IPS), so terminate the host or the pattern also matches suffix hosts, for example ^https?://internal\.svc(:|/|$)`)
|
||||
fs.StringSlice("chromium-deny-list", []string{`^file:(?!//\/tmp/).*`}, "Set the denied URLs for Chromium using regular expressions - supports multiple values")
|
||||
fs.Bool("chromium-deny-private-ips", false, "Reject URLs whose host resolves to a non-public IP address (loopback, RFC1918, link-local, unique-local). Enable on deployments that accept untrusted form input to mitigate SSRF against internal services")
|
||||
fs.Bool("chromium-deny-public-ips", false, "Reject URLs whose host resolves to a public IP address. Enable on air-gapped or data-governed deployments to prevent outbound traffic from leaving a private network")
|
||||
|
||||
@@ -80,6 +80,37 @@ var dangerousTags = []string{
|
||||
"FilePermissions", // Writing this changes the file's permissions
|
||||
}
|
||||
|
||||
// controlOptions lists ExifTool command-line option names that collide with a
|
||||
// tag assignment. A metadata key of "csv" becomes the argv entry "-csv=value",
|
||||
// which exiftool reads as its own option rather than as a tag, so the value
|
||||
// becomes a filename exiftool opens. Only an unprefixed key can collide:
|
||||
// "-XMP:csv=value" is unambiguously a tag.
|
||||
//
|
||||
// See https://exiftool.org/exiftool_pod.html.
|
||||
var controlOptions = []string{
|
||||
"api", "argfile", "charset", "common_args", "config", "csv", "diff",
|
||||
"echo", "efile", "execute", "ext", "fileorder", "geotag", "geosync",
|
||||
"htmldump", "if", "json", "lang", "listitem", "o", "out", "p", "php",
|
||||
"require", "srcfile", "stay_open", "tagsfromfile", "textout", "use", "w",
|
||||
"wm", "xmlformat",
|
||||
}
|
||||
|
||||
// isControlOption reports whether an unprefixed key would reach exiftool as
|
||||
// one of its own options instead of as a tag assignment.
|
||||
func isControlOption(key string) bool {
|
||||
if strings.Contains(key, ":") {
|
||||
return false
|
||||
}
|
||||
|
||||
for _, option := range controlOptions {
|
||||
if strings.EqualFold(key, option) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// isDangerousTag reports whether key matches one of the [dangerousTags]
|
||||
// after case-insensitive comparison with any group prefix stripped.
|
||||
func isDangerousTag(key string) bool {
|
||||
@@ -114,6 +145,9 @@ func buildExifToolWriteArgs(metadata map[string]any) ([]string, error) {
|
||||
if !safeKeyPattern.MatchString(key) {
|
||||
return nil, fmt.Errorf("write PDF metadata with ExifTool: invalid metadata key %q: %w", key, gotenberg.ErrPdfEngineMetadataValueNotSupported)
|
||||
}
|
||||
if isControlOption(key) {
|
||||
return nil, fmt.Errorf("write PDF metadata with ExifTool: metadata key %q is an ExifTool option, prefix it with a group such as %q: %w", key, "XMP:"+key, gotenberg.ErrPdfEngineMetadataValueNotSupported)
|
||||
}
|
||||
|
||||
tag := key
|
||||
if key == "Trapped" {
|
||||
|
||||
@@ -2,6 +2,7 @@ package exiftool
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"testing"
|
||||
|
||||
@@ -211,3 +212,48 @@ func TestSafeKeyPattern(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A metadata key that collides with an ExifTool option becomes a bare argv
|
||||
// entry such as "-csv=/etc/passwd", which exiftool reads as its own option and
|
||||
// treats the value as a filename to open.
|
||||
func TestBuildExifToolWriteArgs_RejectsControlOptions(t *testing.T) {
|
||||
for _, key := range []string{"csv", "CSV", "json", "geotag", "config", "tagsFromFile", "execute", "stay_open", "o", "w", "if", "p"} {
|
||||
t.Run(key, func(t *testing.T) {
|
||||
_, err := buildExifToolWriteArgs(map[string]any{key: "/etc/passwd"})
|
||||
if err == nil {
|
||||
t.Fatalf("buildExifToolWriteArgs accepted the control option %q", key)
|
||||
}
|
||||
if !errors.Is(err, gotenberg.ErrPdfEngineMetadataValueNotSupported) {
|
||||
t.Fatalf("error %v does not wrap ErrPdfEngineMetadataValueNotSupported", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A group prefix makes the key unambiguous, so it must still be accepted.
|
||||
func TestBuildExifToolWriteArgs_AcceptsPrefixedOptionNames(t *testing.T) {
|
||||
for _, key := range []string{"XMP:csv", "XMP-dc:json", "IPTC:p"} {
|
||||
t.Run(key, func(t *testing.T) {
|
||||
args, err := buildExifToolWriteArgs(map[string]any{key: "value"})
|
||||
if err != nil {
|
||||
t.Fatalf("buildExifToolWriteArgs rejected the prefixed key %q: %v", key, err)
|
||||
}
|
||||
want := fmt.Sprintf("-%s=value", key)
|
||||
if len(args) != 1 || args[0] != want {
|
||||
t.Fatalf("args = %v, want [%s]", args, want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Ordinary tags must be unaffected.
|
||||
func TestBuildExifToolWriteArgs_AcceptsOrdinaryTags(t *testing.T) {
|
||||
for _, key := range []string{"Author", "Title", "Subject", "Keywords", "Producer", "Creator"} {
|
||||
t.Run(key, func(t *testing.T) {
|
||||
_, err := buildExifToolWriteArgs(map[string]any{key: "value"})
|
||||
if err != nil {
|
||||
t.Fatalf("buildExifToolWriteArgs rejected the ordinary tag %q: %v", key, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -353,7 +353,7 @@ func (a *Api) Descriptor() gotenberg.ModuleDescriptor {
|
||||
fs.Duration("libreoffice-idle-shutdown-timeout", 0, "Shutdown LibreOffice after being idle for the given duration. Set to 0 to disable this feature")
|
||||
fs.Bool("libreoffice-auto-start", false, "Automatically launch LibreOffice upon initialization if set to true; otherwise, LibreOffice will start at the time of the first conversion")
|
||||
fs.Duration("libreoffice-start-timeout", time.Duration(20)*time.Second, "Maximum duration to wait for LibreOffice to start or restart")
|
||||
fs.StringSlice("libreoffice-allow-list", []string{}, "Set the allowed URLs for LibreOffice outbound fetches (embedded images, linked content) using regular expressions - supports multiple values")
|
||||
fs.StringSlice("libreoffice-allow-list", []string{}, `Set the allowed URLs for LibreOffice outbound fetches (embedded images, linked content) using regular expressions - supports multiple values. A match bypasses --libreoffice-deny-private-ips (LIBREOFFICE_DENY_PRIVATE_IPS) and --libreoffice-deny-public-ips (LIBREOFFICE_DENY_PUBLIC_IPS), so terminate the host or the pattern also matches suffix hosts, for example ^https?://internal\.svc(:|/|$)`)
|
||||
fs.StringSlice("libreoffice-deny-list", []string{}, "Set the denied URLs for LibreOffice outbound fetches using regular expressions - supports multiple values")
|
||||
fs.Bool("libreoffice-deny-private-ips", false, "Reject LibreOffice outbound URLs whose host resolves to a non-public IP address (loopback, RFC1918, link-local, unique-local). Enable on deployments that accept untrusted documents to mitigate SSRF against internal services")
|
||||
fs.Bool("libreoffice-deny-public-ips", false, "Reject LibreOffice outbound URLs whose host resolves to a public IP address. Enable on air-gapped or data-governed deployments to prevent outbound traffic from leaving a private network")
|
||||
|
||||
@@ -120,6 +120,30 @@ func (engine *QPdf) spanAttrs(extra ...attribute.KeyValue) []attribute.KeyValue
|
||||
return append(attrs, extra...)
|
||||
}
|
||||
|
||||
// qpdfPageRange matches the page range syntax qpdf accepts, and nothing else.
|
||||
//
|
||||
// A term is a page number, "z" for the last page, or "rN" counting from the
|
||||
// end, optionally prefixed with "x" to exclude it. Terms combine into ranges
|
||||
// with "-", ranges join with ",", and the whole thing takes an optional ":odd"
|
||||
// or ":even".
|
||||
var qpdfPageRange = regexp.MustCompile(`^x?(?:z|r\d+|\d+)(?:-x?(?:z|r\d+|\d+))?(?:,x?(?:z|r\d+|\d+)(?:-x?(?:z|r\d+|\d+))?)*(?::odd|:even)?$`)
|
||||
|
||||
// validateSplitSpan returns span when it is a qpdf page range.
|
||||
//
|
||||
// qpdf reads the argument after "--pages ." as either a page range or another
|
||||
// source file, so a span carrying a path makes qpdf append that file's pages
|
||||
// to the output. Other engines in the split chain accept spellings qpdf does
|
||||
// not, such as pdfcpu's "2-end", so a span this rejects is not necessarily
|
||||
// invalid. Returning an error lets the chain move on to an engine that
|
||||
// understands it.
|
||||
func validateSplitSpan(span string) error {
|
||||
if qpdfPageRange.MatchString(span) {
|
||||
return nil
|
||||
}
|
||||
|
||||
return fmt.Errorf("split span '%s' is not a QPDF page range: %w", span, gotenberg.ErrPdfSplitModeNotSupported)
|
||||
}
|
||||
|
||||
// Split splits a given PDF file.
|
||||
func (engine *QPdf) Split(ctx context.Context, logger *slog.Logger, mode gotenberg.SplitMode, inputPath, outputDirPath string) ([]string, error) {
|
||||
ctx, span := gotenberg.Tracer().Start(ctx, "qpdf.Split",
|
||||
@@ -139,6 +163,12 @@ func (engine *QPdf) Split(ctx context.Context, logger *slog.Logger, mode gotenbe
|
||||
span.SetStatus(codes.Error, err.Error())
|
||||
return nil, err
|
||||
}
|
||||
err := validateSplitSpan(mode.Span)
|
||||
if err != nil {
|
||||
span.RecordError(err)
|
||||
span.SetStatus(codes.Error, err.Error())
|
||||
return nil, err
|
||||
}
|
||||
args = append(args, inputPath)
|
||||
args = append(args, engine.globalArgs...)
|
||||
args = append(args, "--pages", ".", mode.Span)
|
||||
|
||||
@@ -3,9 +3,12 @@ package qpdf
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/gotenberg/gotenberg/v8/pkg/gotenberg"
|
||||
)
|
||||
|
||||
func TestStripQpdfStringPrefix(t *testing.T) {
|
||||
@@ -270,3 +273,54 @@ func TestSetStreamSubtype(t *testing.T) {
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestValidateSplitSpan(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
span string
|
||||
valid bool
|
||||
}{
|
||||
// qpdf page ranges.
|
||||
{"1", true},
|
||||
{"12", true},
|
||||
{"1-5", true},
|
||||
{"2-z", true},
|
||||
{"z", true},
|
||||
{"r1", true},
|
||||
{"r3-r1", true},
|
||||
{"1,3,5-9", true},
|
||||
{"1-5,x3", true},
|
||||
{"1-z:odd", true},
|
||||
{"1-z:even", true},
|
||||
|
||||
// Other engines' spellings. Not valid here, so the chain moves on.
|
||||
{"2-end", false},
|
||||
{"2-", false},
|
||||
{"foo", false},
|
||||
|
||||
// A span qpdf would read as a source file.
|
||||
{"/tmp/secret.pdf", false},
|
||||
{"secret.pdf", false},
|
||||
{"./secret.pdf", false},
|
||||
{"../../etc/hosts", false},
|
||||
{"1,/tmp/secret.pdf", false},
|
||||
{"1 /tmp/secret.pdf", false},
|
||||
{"", false},
|
||||
{"--password=x", false},
|
||||
} {
|
||||
t.Run(tc.span, func(t *testing.T) {
|
||||
err := validateSplitSpan(tc.span)
|
||||
if tc.valid && err != nil {
|
||||
t.Fatalf("validateSplitSpan(%q) = %v, want nil", tc.span, err)
|
||||
}
|
||||
if !tc.valid {
|
||||
if err == nil {
|
||||
t.Fatalf("validateSplitSpan(%q) = nil, want an error", tc.span)
|
||||
}
|
||||
// The chain must be able to try the next engine.
|
||||
if !errors.Is(err, gotenberg.ErrPdfSplitModeNotSupported) {
|
||||
t.Fatalf("error %v does not wrap ErrPdfSplitModeNotSupported", err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -31,10 +31,33 @@ type client struct {
|
||||
extraHttpHeaders map[string]string
|
||||
startTime time.Time
|
||||
|
||||
// deliveryTimeout bounds one delivery including retries. See
|
||||
// [Webhook.deliveryTimeout].
|
||||
deliveryTimeout time.Duration
|
||||
|
||||
client *retryablehttp.Client
|
||||
logger *slog.Logger
|
||||
}
|
||||
|
||||
// deliveryContext returns the context one delivery runs on.
|
||||
//
|
||||
// It keeps the values of ctx, so trace propagation and logging correlation
|
||||
// survive, and replaces its cancellation with a fresh budget. Threading the
|
||||
// conversion context straight through does not work: a delivery starts after
|
||||
// the handler returned, so that deadline may already be spent and the callback
|
||||
// would fail without a single attempt.
|
||||
func (c client) deliveryContext(ctx context.Context) (context.Context, context.CancelFunc) {
|
||||
timeout := c.deliveryTimeout
|
||||
if timeout <= 0 {
|
||||
// An unset budget would expire the delivery before its first attempt.
|
||||
// [Webhook.deliveryTimeout] never returns a non-positive value, so this
|
||||
// only guards a caller that builds a client without one.
|
||||
timeout = minDeliveryTimeout
|
||||
}
|
||||
|
||||
return context.WithTimeout(context.WithoutCancel(ctx), timeout)
|
||||
}
|
||||
|
||||
// send call the webhook either to send the success response or the error response.
|
||||
func (c client) send(ctx context.Context, body io.Reader, headers map[string]string, errored bool) error {
|
||||
url := c.url
|
||||
@@ -57,6 +80,9 @@ func (c client) send(ctx context.Context, body io.Reader, headers map[string]str
|
||||
spanName = fmt.Sprintf("%s Webhook Error", method)
|
||||
}
|
||||
|
||||
ctx, cancel := c.deliveryContext(ctx)
|
||||
defer cancel()
|
||||
|
||||
tracer := gotenberg.Tracer()
|
||||
ctx, span := tracer.Start(ctx, spanName,
|
||||
trace.WithSpanKind(trace.SpanKindClient),
|
||||
@@ -64,7 +90,10 @@ func (c client) send(ctx context.Context, body io.Reader, headers map[string]str
|
||||
)
|
||||
defer span.End()
|
||||
|
||||
req, err := retryablehttp.NewRequest(method, url, body)
|
||||
// The request must carry ctx: retryablehttp.NewRequest builds on
|
||||
// [context.Background], and its wait between attempts is a select on the
|
||||
// request context, so a contextless request cannot be interrupted.
|
||||
req, err := retryablehttp.NewRequestWithContext(ctx, method, url, body)
|
||||
if err != nil {
|
||||
span.RecordError(err)
|
||||
span.SetStatus(codes.Error, err.Error())
|
||||
@@ -165,6 +194,9 @@ func (c client) sendEvent(ctx context.Context, correlationIdHeader, correlationI
|
||||
return
|
||||
}
|
||||
|
||||
ctx, cancel := c.deliveryContext(ctx)
|
||||
defer cancel()
|
||||
|
||||
tracer := gotenberg.Tracer()
|
||||
ctx, span := tracer.Start(ctx, "POST Webhook Event",
|
||||
trace.WithSpanKind(trace.SpanKindClient),
|
||||
@@ -172,7 +204,7 @@ func (c client) sendEvent(ctx context.Context, correlationIdHeader, correlationI
|
||||
)
|
||||
defer span.End()
|
||||
|
||||
req, err := retryablehttp.NewRequest(http.MethodPost, c.eventsUrl, b)
|
||||
req, err := retryablehttp.NewRequestWithContext(ctx, http.MethodPost, c.eventsUrl, b)
|
||||
if err != nil {
|
||||
span.RecordError(err)
|
||||
span.SetStatus(codes.Error, err.Error())
|
||||
|
||||
@@ -222,6 +222,7 @@ func webhookMiddleware(w *Webhook) api.Middleware {
|
||||
eventsUrl: webhookEventsUrl,
|
||||
extraHttpHeaders: extraHttpHeaders,
|
||||
startTime: startTime,
|
||||
deliveryTimeout: w.deliveryTimeout(),
|
||||
|
||||
client: &retryablehttp.Client{
|
||||
HTTPClient: gotenberg.NewOutboundHttpClient(w.clientTimeout, w.allowList, w.denyList, w.enableEnvironmentProxy, ipOpts...),
|
||||
@@ -230,7 +231,10 @@ func webhookMiddleware(w *Webhook) api.Middleware {
|
||||
RetryWaitMax: w.retryMaxWait,
|
||||
Logger: gotenberg.NewLeveledLogger(ctx.Log()),
|
||||
CheckRetry: retryablehttp.DefaultRetryPolicy,
|
||||
Backoff: retryablehttp.DefaultBackoff,
|
||||
// Not DefaultBackoff: it returns a remote Retry-After
|
||||
// verbatim, ignoring --webhook-retry-max-wait (env
|
||||
// WEBHOOK_RETRY_MAX_WAIT).
|
||||
Backoff: gotenberg.ClampedBackoff,
|
||||
},
|
||||
logger: ctx.Log(),
|
||||
}
|
||||
|
||||
@@ -42,7 +42,7 @@ func (w *Webhook) Descriptor() gotenberg.ModuleDescriptor {
|
||||
FlagSet: func() *flag.FlagSet {
|
||||
fs := flag.NewFlagSet("webhook", flag.ExitOnError)
|
||||
fs.Bool("webhook-enable-sync-mode", false, "Enable synchronous mode for the webhook feature")
|
||||
fs.StringSlice("webhook-allow-list", []string{}, "Set the allowed URLs for the webhook feature using regular expressions - supports multiple values")
|
||||
fs.StringSlice("webhook-allow-list", []string{}, `Set the allowed URLs for the webhook feature using regular expressions - supports multiple values. A match bypasses --webhook-deny-private-ips (WEBHOOK_DENY_PRIVATE_IPS) and --webhook-deny-public-ips (WEBHOOK_DENY_PUBLIC_IPS), so terminate the host or the pattern also matches suffix hosts, for example ^https?://internal\.svc(:|/|$)`)
|
||||
fs.StringSlice("webhook-deny-list", []string{}, "Set the denied URLs for the webhook feature using regular expressions - supports multiple values")
|
||||
fs.Bool("webhook-deny-private-ips", false, "Reject webhook URLs whose host resolves to a non-public IP address (loopback, RFC1918, link-local, unique-local). Enable on deployments that accept untrusted webhook destinations to mitigate SSRF against internal services")
|
||||
fs.Bool("webhook-deny-public-ips", false, "Reject webhook URLs whose host resolves to a public IP address. Enable on air-gapped or data-governed deployments to prevent callbacks from leaving a private network")
|
||||
@@ -50,7 +50,7 @@ func (w *Webhook) Descriptor() gotenberg.ModuleDescriptor {
|
||||
fs.Int("webhook-max-retry", 4, "Set the maximum number of retries for the webhook feature")
|
||||
|
||||
// Deprecated flags.
|
||||
fs.StringSlice("webhook-error-allow-list", []string{}, "Set the allowed URLs in case of an error for the webhook feature using regular expressions - supports multiple values")
|
||||
fs.StringSlice("webhook-error-allow-list", []string{}, `Set the allowed URLs in case of an error for the webhook feature using regular expressions - supports multiple values. A match bypasses --webhook-deny-private-ips (WEBHOOK_DENY_PRIVATE_IPS) and --webhook-deny-public-ips (WEBHOOK_DENY_PUBLIC_IPS), so terminate the host or the pattern also matches suffix hosts, for example ^https?://internal\.svc(:|/|$)`)
|
||||
fs.StringSlice("webhook-error-deny-list", []string{}, "Set the denied URLs in case of an error for the webhook feature using regular expressions - supports multiple values")
|
||||
err := fs.MarkDeprecated("webhook-error-allow-list", "use --webhook-allow-list instead")
|
||||
if err != nil {
|
||||
@@ -92,6 +92,27 @@ func (w *Webhook) Provision(ctx *gotenberg.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// minDeliveryTimeout is the floor for [Webhook.deliveryTimeout], so that a
|
||||
// deliberately tiny --webhook-client-timeout (env WEBHOOK_CLIENT_TIMEOUT) never
|
||||
// leaves a delivery with no budget at all.
|
||||
const minDeliveryTimeout = 1 * time.Second
|
||||
|
||||
// deliveryTimeout bounds one webhook delivery including its retries. It is the
|
||||
// worst case a correctly behaving remote produces: one client timeout per
|
||||
// attempt, plus the capped wait between attempts.
|
||||
//
|
||||
// A delivery runs after the handler returned, so it cannot borrow the
|
||||
// conversion deadline. Without this budget it would have none, because
|
||||
// [retryablehttp] builds its requests on [context.Background].
|
||||
func (w *Webhook) deliveryTimeout() time.Duration {
|
||||
timeout := w.clientTimeout*time.Duration(w.maxRetry+1) + w.retryMaxWait*time.Duration(w.maxRetry)
|
||||
if timeout < minDeliveryTimeout {
|
||||
return minDeliveryTimeout
|
||||
}
|
||||
|
||||
return timeout
|
||||
}
|
||||
|
||||
// Middlewares returns the middleware.
|
||||
func (w *Webhook) Middlewares() ([]api.Middleware, error) {
|
||||
if w.disable {
|
||||
|
||||
95
pkg/modules/webhook/webhook_test.go
Normal file
95
pkg/modules/webhook/webhook_test.go
Normal file
@@ -0,0 +1,95 @@
|
||||
package webhook
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestWebhook_deliveryTimeout(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
scenario string
|
||||
clientTimeout time.Duration
|
||||
maxRetry int
|
||||
retryMaxWait time.Duration
|
||||
want time.Duration
|
||||
}{
|
||||
{
|
||||
scenario: "shipped defaults",
|
||||
clientTimeout: 30 * time.Second,
|
||||
maxRetry: 4,
|
||||
retryMaxWait: 30 * time.Second,
|
||||
want: 270 * time.Second,
|
||||
},
|
||||
{
|
||||
scenario: "no retry is one client timeout",
|
||||
clientTimeout: 30 * time.Second,
|
||||
maxRetry: 0,
|
||||
retryMaxWait: 30 * time.Second,
|
||||
want: 30 * time.Second,
|
||||
},
|
||||
{
|
||||
scenario: "a zero client timeout still gets a budget",
|
||||
clientTimeout: 0,
|
||||
maxRetry: 0,
|
||||
retryMaxWait: 0,
|
||||
want: minDeliveryTimeout,
|
||||
},
|
||||
} {
|
||||
t.Run(tc.scenario, func(t *testing.T) {
|
||||
w := &Webhook{
|
||||
clientTimeout: tc.clientTimeout,
|
||||
maxRetry: tc.maxRetry,
|
||||
retryMaxWait: tc.retryMaxWait,
|
||||
}
|
||||
if got := w.deliveryTimeout(); got != tc.want {
|
||||
t.Fatalf("deliveryTimeout() = %s, want %s", got, tc.want)
|
||||
}
|
||||
if w.deliveryTimeout() <= 0 {
|
||||
t.Fatal("deliveryTimeout() must always be positive")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A delivery must not inherit the conversion deadline. It runs after the
|
||||
// handler returned, so that deadline is often already spent, which would fail
|
||||
// the callback without a single attempt.
|
||||
func TestClient_deliveryContext_IgnoresAnExpiredParentDeadline(t *testing.T) {
|
||||
expired, cancelExpired := context.WithTimeout(context.Background(), -1*time.Second)
|
||||
defer cancelExpired()
|
||||
|
||||
if expired.Err() == nil {
|
||||
t.Fatal("expected the parent context to be expired")
|
||||
}
|
||||
|
||||
c := client{deliveryTimeout: 30 * time.Second}
|
||||
ctx, cancel := c.deliveryContext(expired)
|
||||
defer cancel()
|
||||
|
||||
if ctx.Err() != nil {
|
||||
t.Fatalf("delivery context inherited the expired parent: %v", ctx.Err())
|
||||
}
|
||||
|
||||
deadline, ok := ctx.Deadline()
|
||||
if !ok {
|
||||
t.Fatal("delivery context has no deadline, so a delivery would be unbounded")
|
||||
}
|
||||
if remaining := time.Until(deadline); remaining <= 0 {
|
||||
t.Fatalf("delivery budget = %s, want a positive value", remaining)
|
||||
}
|
||||
}
|
||||
|
||||
// The delivery context must still be bounded, so a hostile remote cannot hold
|
||||
// the goroutine and its output file open indefinitely.
|
||||
func TestClient_deliveryContext_IsBounded(t *testing.T) {
|
||||
c := client{deliveryTimeout: 50 * time.Millisecond}
|
||||
ctx, cancel := c.deliveryContext(context.Background())
|
||||
defer cancel()
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("delivery context never expired")
|
||||
}
|
||||
}
|
||||
@@ -589,7 +589,7 @@ Feature: /forms/chromium/convert/url
|
||||
@chromium-ssrf
|
||||
Scenario: POST /forms/chromium/convert/url (Redirect to a non-allow-listed address is re-filtered)
|
||||
Given I have a Gotenberg container with the following environment variable(s):
|
||||
| CHROMIUM_ALLOW_LIST | ^https?://host.docker.internal.* |
|
||||
| CHROMIUM_ALLOW_LIST | ^https?://host\\.docker\\.internal(:[0-9]+)?/ |
|
||||
Given I have a static server
|
||||
When I make a "POST" request to Gotenberg at the "/forms/chromium/convert/url" endpoint with the following form data and header(s):
|
||||
| url | http://host.docker.internal:%d/redirect-to-private | field |
|
||||
|
||||
Reference in New Issue
Block a user