feat(pdfengines): add support for flattening annotations (#1105)

* initial changes

* Add tests

* Fix edge case when we need to regenerate appearances

* Fix comments

* Add missing comment

* Add missing comment

* Add missing comment

* Add flatten option to the merge route

* Add flatten option to the libreoffice convert route

* Add flatten option to the chromium convert route

* Revert "Add flatten option to the chromium convert route"

This reverts commit cdab8b4e6b.

* Ignore lint false positives

* Add missing tests

* Add flatten route tests

* Replace input instead of creating a new file

* create copy before flatten in tests

---------

Co-authored-by: Peter Chakalov <peter.chakalov@abraxa.com>
This commit is contained in:
Peter Chakalov
2025-01-28 15:38:27 +02:00
committed by GitHub
parent a9b44ee39e
commit b418f1eb05
23 changed files with 634 additions and 4 deletions

View File

@@ -14,6 +14,7 @@ import (
type multiPdfEngines struct {
mergeEngines []gotenberg.PdfEngine
splitEngines []gotenberg.PdfEngine
flattenEngines []gotenberg.PdfEngine
convertEngines []gotenberg.PdfEngine
readMetadataEngines []gotenberg.PdfEngine
writeMetadataEngines []gotenberg.PdfEngine
@@ -22,6 +23,7 @@ type multiPdfEngines struct {
func newMultiPdfEngines(
mergeEngines,
splitEngines,
flattenEngines,
convertEngines,
readMetadataEngines,
writeMetadataEngines []gotenberg.PdfEngine,
@@ -29,6 +31,7 @@ func newMultiPdfEngines(
return &multiPdfEngines{
mergeEngines: mergeEngines,
splitEngines: splitEngines,
flattenEngines: flattenEngines,
convertEngines: convertEngines,
readMetadataEngines: readMetadataEngines,
writeMetadataEngines: writeMetadataEngines,
@@ -98,6 +101,32 @@ func (multi *multiPdfEngines) Split(ctx context.Context, logger *zap.Logger, mod
return nil, fmt.Errorf("split PDF with multi PDF engines: %w", err)
}
// Flatten merges existing annotation appearances with page content, effectively deleting the original annotations.
// This process can flatten forms as well, as forms share a relationship with annotations.
// Note that this operation is irreversible.
func (multi *multiPdfEngines) Flatten(ctx context.Context, logger *zap.Logger, inputPath string) error {
var err error
errChan := make(chan error, 1)
for _, engine := range multi.flattenEngines {
go func(engine gotenberg.PdfEngine) {
errChan <- engine.Flatten(ctx, logger, inputPath)
}(engine)
select {
case mergeErr := <-errChan:
errored := multierr.AppendInto(&err, mergeErr)
if !errored {
return nil
}
case <-ctx.Done():
return ctx.Err()
}
}
return fmt.Errorf("flatten PDF with multi PDF engines: %w", err)
}
// Convert converts the given PDF to a specific PDF format. thanks to its
// children. If the context is done, it stops and returns an error.
func (multi *multiPdfEngines) Convert(ctx context.Context, logger *zap.Logger, formats gotenberg.PdfFormats, inputPath, outputPath string) error {

View File

@@ -194,6 +194,97 @@ func TestMultiPdfEngines_Split(t *testing.T) {
}
}
func TestMultiPdfEngines_Flatten(t *testing.T) {
for _, tc := range []struct {
scenario string
engine *multiPdfEngines
ctx context.Context
expectError bool
}{
{
scenario: "nominal behavior",
engine: &multiPdfEngines{
flattenEngines: []gotenberg.PdfEngine{
&gotenberg.PdfEngineMock{
FlattenMock: func(ctx context.Context, logger *zap.Logger, inputPath string) error {
return nil
},
},
},
},
ctx: context.Background(),
},
{
scenario: "at least one engine does not return an error",
engine: &multiPdfEngines{
flattenEngines: []gotenberg.PdfEngine{
&gotenberg.PdfEngineMock{
FlattenMock: func(ctx context.Context, logger *zap.Logger, inputPath string) error {
return errors.New("foo")
},
},
&gotenberg.PdfEngineMock{
FlattenMock: func(ctx context.Context, logger *zap.Logger, inputPath string) error {
return nil
},
},
},
},
ctx: context.Background(),
},
{
scenario: "all engines return an error",
engine: &multiPdfEngines{
flattenEngines: []gotenberg.PdfEngine{
&gotenberg.PdfEngineMock{
FlattenMock: func(ctx context.Context, logger *zap.Logger, inputPath string) error {
return errors.New("foo")
},
},
&gotenberg.PdfEngineMock{
FlattenMock: func(ctx context.Context, logger *zap.Logger, inputPath string) error {
return errors.New("foo")
},
},
},
},
ctx: context.Background(),
expectError: true,
},
{
scenario: "context expired",
engine: &multiPdfEngines{
flattenEngines: []gotenberg.PdfEngine{
&gotenberg.PdfEngineMock{
FlattenMock: func(ctx context.Context, logger *zap.Logger, inputPath string) error {
return nil
},
},
},
},
ctx: func() context.Context {
ctx, cancel := context.WithCancel(context.Background())
cancel()
return ctx
}(),
expectError: true,
},
} {
t.Run(tc.scenario, func(t *testing.T) {
err := tc.engine.Flatten(tc.ctx, zap.NewNop(), "")
if !tc.expectError && err != nil {
t.Fatalf("expected no error but got: %v", err)
}
if tc.expectError && err == nil {
t.Fatal("expected error but got none")
}
})
}
}
func TestMultiPdfEngines_Convert(t *testing.T) {
for _, tc := range []struct {
scenario string

View File

@@ -29,6 +29,7 @@ func init() {
type PdfEngines struct {
mergeNames []string
splitNames []string
flattenNames []string
convertNames []string
readMetadataNames []string
writeMetadataNames []string
@@ -44,6 +45,7 @@ func (mod *PdfEngines) Descriptor() gotenberg.ModuleDescriptor {
fs := flag.NewFlagSet("pdfengines", flag.ExitOnError)
fs.StringSlice("pdfengines-merge-engines", []string{"qpdf", "pdfcpu", "pdftk"}, "Set the PDF engines and their order for the merge feature - empty means all")
fs.StringSlice("pdfengines-split-engines", []string{"pdfcpu", "qpdf", "pdftk"}, "Set the PDF engines and their order for the split feature - empty means all")
fs.StringSlice("pdfengines-flatten-engines", []string{"qpdf"}, "Set the PDF engines and their order for the flatten feature - empty means all")
fs.StringSlice("pdfengines-convert-engines", []string{"libreoffice-pdfengine"}, "Set the PDF engines and their order for the convert feature - empty means all")
fs.StringSlice("pdfengines-read-metadata-engines", []string{"exiftool"}, "Set the PDF engines and their order for the read metadata feature - empty means all")
fs.StringSlice("pdfengines-write-metadata-engines", []string{"exiftool"}, "Set the PDF engines and their order for the write metadata feature - empty means all")
@@ -67,6 +69,7 @@ func (mod *PdfEngines) Provision(ctx *gotenberg.Context) error {
flags := ctx.ParsedFlags()
mergeNames := flags.MustStringSlice("pdfengines-merge-engines")
splitNames := flags.MustStringSlice("pdfengines-split-engines")
flattenNames := flags.MustStringSlice("pdfengines-flatten-engines")
convertNames := flags.MustStringSlice("pdfengines-convert-engines")
readMetadataNames := flags.MustStringSlice("pdfengines-read-metadata-engines")
writeMetadataNames := flags.MustStringSlice("pdfengines-write-metadata-engines")
@@ -106,6 +109,11 @@ func (mod *PdfEngines) Provision(ctx *gotenberg.Context) error {
mod.splitNames = splitNames
}
mod.flattenNames = defaultNames
if len(flattenNames) > 0 {
mod.flattenNames = flattenNames
}
mod.convertNames = defaultNames
if len(convertNames) > 0 {
mod.convertNames = convertNames
@@ -170,6 +178,7 @@ func (mod *PdfEngines) Validate() error {
findNonExistingEngines(mod.mergeNames)
findNonExistingEngines(mod.splitNames)
findNonExistingEngines(mod.flattenNames)
findNonExistingEngines(mod.convertNames)
findNonExistingEngines(mod.readMetadataNames)
findNonExistingEngines(mod.writeMetadataNames)
@@ -187,6 +196,7 @@ func (mod *PdfEngines) SystemMessages() []string {
return []string{
fmt.Sprintf("merge engines - %s", strings.Join(mod.mergeNames[:], " ")),
fmt.Sprintf("split engines - %s", strings.Join(mod.splitNames[:], " ")),
fmt.Sprintf("flatten engines - %s", strings.Join(mod.flattenNames[:], " ")),
fmt.Sprintf("convert engines - %s", strings.Join(mod.convertNames[:], " ")),
fmt.Sprintf("read metadata engines - %s", strings.Join(mod.readMetadataNames[:], " ")),
fmt.Sprintf("write metadata engines - %s", strings.Join(mod.writeMetadataNames[:], " ")),
@@ -212,6 +222,7 @@ func (mod *PdfEngines) PdfEngine() (gotenberg.PdfEngine, error) {
return newMultiPdfEngines(
engines(mod.mergeNames),
engines(mod.splitNames),
engines(mod.flattenNames),
engines(mod.convertNames),
engines(mod.readMetadataNames),
engines(mod.writeMetadataNames),
@@ -234,6 +245,7 @@ func (mod *PdfEngines) Routes() ([]api.Route, error) {
return []api.Route{
mergeRoute(engine),
splitRoute(engine),
flattenRoute(engine),
convertRoute(engine),
readMetadataRoute(engine),
writeMetadataRoute(engine),

View File

@@ -27,6 +27,7 @@ func TestPdfEngines_Provision(t *testing.T) {
ctx *gotenberg.Context
expectedMergePdfEngines []string
expectedSplitPdfEngines []string
expectedFlattenPdfEngines []string
expectedConvertPdfEngines []string
expectedReadMetadataPdfEngines []string
expectedWriteMetadataPdfEngines []string
@@ -68,6 +69,7 @@ func TestPdfEngines_Provision(t *testing.T) {
}(),
expectedMergePdfEngines: []string{"qpdf", "pdfcpu", "pdftk"},
expectedSplitPdfEngines: []string{"pdfcpu", "qpdf", "pdftk"},
expectedFlattenPdfEngines: []string{"qpdf"},
expectedConvertPdfEngines: []string{"libreoffice-pdfengine"},
expectedReadMetadataPdfEngines: []string{"exiftool"},
expectedWriteMetadataPdfEngines: []string{"exiftool"},
@@ -109,7 +111,7 @@ func TestPdfEngines_Provision(t *testing.T) {
}
fs := new(PdfEngines).Descriptor().FlagSet
err := fs.Parse([]string{"--pdfengines-merge-engines=b", "--pdfengines-split-engines=a", "--pdfengines-convert-engines=b", "--pdfengines-read-metadata-engines=a", "--pdfengines-write-metadata-engines=a"})
err := fs.Parse([]string{"--pdfengines-merge-engines=b", "--pdfengines-split-engines=a", "--pdfengines-flatten-engines=c", "--pdfengines-convert-engines=b", "--pdfengines-read-metadata-engines=a", "--pdfengines-write-metadata-engines=a"})
if err != nil {
t.Fatalf("expected no error but got: %v", err)
}
@@ -128,6 +130,7 @@ func TestPdfEngines_Provision(t *testing.T) {
expectedMergePdfEngines: []string{"b"},
expectedSplitPdfEngines: []string{"a"},
expectedFlattenPdfEngines: []string{"c"},
expectedConvertPdfEngines: []string{"b"},
expectedReadMetadataPdfEngines: []string{"a"},
expectedWriteMetadataPdfEngines: []string{"a"},
@@ -185,6 +188,10 @@ func TestPdfEngines_Provision(t *testing.T) {
t.Fatalf("expected %d merge names but got %d", len(tc.expectedMergePdfEngines), len(mod.mergeNames))
}
if len(tc.expectedFlattenPdfEngines) != len(mod.flattenNames) {
t.Fatalf("expected %d flatten names but got %d", len(tc.expectedFlattenPdfEngines), len(mod.flattenNames))
}
if len(tc.expectedConvertPdfEngines) != len(mod.convertNames) {
t.Fatalf("expected %d convert names but got %d", len(tc.expectedConvertPdfEngines), len(mod.convertNames))
}
@@ -317,14 +324,16 @@ func TestPdfEngines_SystemMessages(t *testing.T) {
mod.readMetadataNames = []string{"foo", "bar"}
mod.writeMetadataNames = []string{"foo", "bar"}
expectedMessages := 6
messages := mod.SystemMessages()
if len(messages) != 5 {
t.Errorf("expected one and only one message, but got %d", len(messages))
if len(messages) != expectedMessages {
t.Errorf("expected %d message(s), but got %d", expectedMessages, len(messages))
}
expect := []string{
fmt.Sprintf("merge engines - %s", strings.Join(mod.mergeNames[:], " ")),
fmt.Sprintf("split engines - %s", strings.Join(mod.splitNames[:], " ")),
fmt.Sprintf("flatten engines - %s", strings.Join(mod.flattenNames[:], " ")),
fmt.Sprintf("convert engines - %s", strings.Join(mod.convertNames[:], " ")),
fmt.Sprintf("read metadata engines - %s", strings.Join(mod.readMetadataNames[:], " ")),
fmt.Sprintf("write metadata engines - %s", strings.Join(mod.writeMetadataNames[:], " ")),
@@ -382,7 +391,7 @@ func TestPdfEngines_Routes(t *testing.T) {
}{
{
scenario: "routes not disabled",
expectRoutes: 5,
expectRoutes: 6,
disableRoutes: false,
},
{

View File

@@ -202,6 +202,21 @@ func SplitPdfStub(ctx *api.Context, engine gotenberg.PdfEngine, mode gotenberg.S
return outputPaths, nil
}
// FlattenStub merges annotation appearances with page content for each given PDF
// in the input paths, effectively deleting the original annotations. It generates
// new output paths for the flattened PDFs and returns them. If an error occurs
// during the flattening process, it returns the error.
func FlattenStub(ctx *api.Context, engine gotenberg.PdfEngine, inputPaths []string) error {
for _, inputPath := range inputPaths {
err := engine.Flatten(ctx, ctx.Log(), inputPath)
if err != nil {
return fmt.Errorf("flatten '%s': %w", inputPath, err)
}
}
return nil
}
// ConvertStub transforms a given PDF to the specified formats defined in
// [gotenberg.PdfFormats]. If no format, it does nothing and returns the input
// paths.
@@ -255,8 +270,10 @@ func mergeRoute(engine gotenberg.PdfEngine) api.Route {
metadata := FormDataPdfMetadata(form, false)
var inputPaths []string
var flatten bool
err := form.
MandatoryPaths([]string{".pdf"}, &inputPaths).
Bool("flatten", &flatten, false).
Validate()
if err != nil {
return fmt.Errorf("validate form data: %w", err)
@@ -278,6 +295,13 @@ func mergeRoute(engine gotenberg.PdfEngine) api.Route {
return fmt.Errorf("write metadata: %w", err)
}
if flatten {
err = FlattenStub(ctx, engine, outputPaths)
if err != nil {
return fmt.Errorf("flatten PDFs: %w", err)
}
}
err = ctx.AddOutputPaths(outputPaths...)
if err != nil {
return fmt.Errorf("add output paths: %w", err)
@@ -347,6 +371,40 @@ func splitRoute(engine gotenberg.PdfEngine) api.Route {
}
}
// flattenRoute returns an [api.Route] which can flatten PDFs.
func flattenRoute(engine gotenberg.PdfEngine) api.Route {
return api.Route{
Method: http.MethodPost,
Path: "/forms/pdfengines/flatten",
IsMultipart: true,
Handler: func(c echo.Context) error {
ctx := c.Get("context").(*api.Context)
form := ctx.FormData()
var inputPaths []string
err := form.
MandatoryPaths([]string{".pdf"}, &inputPaths).
Validate()
if err != nil {
return fmt.Errorf("validate form data: %w", err)
}
err = FlattenStub(ctx, engine, inputPaths)
if err != nil {
return fmt.Errorf("convert PDFs: %w", err)
}
err = ctx.AddOutputPaths(inputPaths...)
if err != nil {
return fmt.Errorf("add output paths: %w", err)
}
return nil
},
}
}
// convertRoute returns an [api.Route] which can convert PDFs to a specific ODF
// format.
func convertRoute(engine gotenberg.PdfEngine) api.Route {

View File

@@ -719,6 +719,39 @@ func TestMergeHandler(t *testing.T) {
expectHttpError: false,
expectOutputPathsCount: 0,
},
{
scenario: "PDF engine flatten error",
ctx: func() *api.ContextMock {
ctx := &api.ContextMock{Context: new(api.Context)}
ctx.SetFiles(map[string]string{
"file.pdf": "/file.pdf",
"file2.pdf": "/file2.pdf",
})
ctx.SetValues(map[string][]string{
"metadata": {
"{\"Creator\": \"foo\", \"Producer\": \"bar\" }",
},
"flatten": {
"true",
},
})
return ctx
}(),
engine: &gotenberg.PdfEngineMock{
MergeMock: func(ctx context.Context, logger *zap.Logger, inputPaths []string, outputPath string) error {
return nil
},
WriteMetadataMock: func(ctx context.Context, logger *zap.Logger, metadata map[string]interface{}, inputPath string) error {
return nil
},
FlattenMock: func(ctx context.Context, logger *zap.Logger, inputPath string) error {
return errors.New("foo")
},
},
expectError: true,
expectHttpError: false,
expectOutputPathsCount: 0,
},
{
scenario: "cannot add output paths",
ctx: func() *api.ContextMock {
@@ -754,6 +787,9 @@ func TestMergeHandler(t *testing.T) {
"metadata": {
"{\"Creator\": \"foo\", \"Producer\": \"bar\" }",
},
"flatten": {
"true",
},
})
return ctx
}(),
@@ -767,6 +803,9 @@ func TestMergeHandler(t *testing.T) {
WriteMetadataMock: func(ctx context.Context, logger *zap.Logger, metadata map[string]interface{}, inputPath string) error {
return nil
},
FlattenMock: func(ctx context.Context, logger *zap.Logger, inputPath string) error {
return nil
},
},
expectError: false,
expectHttpError: false,
@@ -1055,6 +1094,147 @@ func TestSplitHandler(t *testing.T) {
}
}
func TestFlattenHandler(t *testing.T) {
for _, tc := range []struct {
scenario string
ctx *api.ContextMock
engine gotenberg.PdfEngine
expectError bool
expectHttpError bool
expectHttpStatus int
expectOutputPathsCount int
expectOutputPaths []string
}{
{
scenario: "missing at least one mandatory file",
ctx: &api.ContextMock{Context: new(api.Context)},
expectError: true,
expectHttpError: true,
expectHttpStatus: http.StatusBadRequest,
expectOutputPathsCount: 0,
},
{
scenario: "error from PDF engine",
ctx: func() *api.ContextMock {
ctx := &api.ContextMock{Context: new(api.Context)}
ctx.SetFiles(map[string]string{
"file.pdf": "/file.pdf",
})
return ctx
}(),
engine: &gotenberg.PdfEngineMock{
FlattenMock: func(ctx context.Context, logger *zap.Logger, inputPath string) error {
return errors.New("foo")
},
},
expectError: true,
expectHttpError: false,
expectOutputPathsCount: 0,
},
{
scenario: "cannot add output paths",
ctx: func() *api.ContextMock {
ctx := &api.ContextMock{Context: new(api.Context)}
ctx.SetFiles(map[string]string{
"file.pdf": "/file.pdf",
})
ctx.SetCancelled(true)
return ctx
}(),
engine: &gotenberg.PdfEngineMock{
FlattenMock: func(ctx context.Context, logger *zap.Logger, inputPath string) error {
return nil
},
},
expectError: true,
expectHttpError: false,
expectOutputPathsCount: 0,
},
{
scenario: "success with single file",
ctx: func() *api.ContextMock {
ctx := &api.ContextMock{Context: new(api.Context)}
ctx.SetFiles(map[string]string{
"file.pdf": "/file.pdf",
})
return ctx
}(),
engine: &gotenberg.PdfEngineMock{
FlattenMock: func(ctx context.Context, logger *zap.Logger, inputPath string) error {
return nil
},
},
expectError: false,
expectHttpError: false,
expectOutputPathsCount: 1,
},
{
scenario: "success (many files)",
ctx: func() *api.ContextMock {
ctx := &api.ContextMock{Context: new(api.Context)}
ctx.SetFiles(map[string]string{
"file.pdf": "/file.pdf",
"file2.pdf": "/file2.pdf",
})
return ctx
}(),
engine: &gotenberg.PdfEngineMock{
FlattenMock: func(ctx context.Context, logger *zap.Logger, inputPath string) error {
return nil
},
},
expectError: false,
expectHttpError: false,
expectOutputPathsCount: 2,
expectOutputPaths: []string{"/file.pdf", "/file2.pdf"},
},
} {
t.Run(tc.scenario, func(t *testing.T) {
tc.ctx.SetLogger(zap.NewNop())
c := echo.New().NewContext(nil, nil)
c.Set("context", tc.ctx.Context)
err := flattenRoute(tc.engine).Handler(c)
if tc.expectError && err == nil {
t.Fatal("expected error but got none", err)
}
if !tc.expectError && err != nil {
t.Fatalf("expected no error but got: %v", err)
}
var httpErr api.HttpError
isHttpError := errors.As(err, &httpErr)
if tc.expectHttpError && !isHttpError {
t.Errorf("expected an HTTP error but got: %v", err)
}
if !tc.expectHttpError && isHttpError {
t.Errorf("expected no HTTP error but got one: %v", httpErr)
}
if err != nil && tc.expectHttpError && isHttpError {
status, _ := httpErr.HttpError()
if status != tc.expectHttpStatus {
t.Errorf("expected %d as HTTP status code but got %d", tc.expectHttpStatus, status)
}
}
if tc.expectOutputPathsCount != len(tc.ctx.OutputPaths()) {
t.Errorf("expected %d output paths but got %d", tc.expectOutputPathsCount, len(tc.ctx.OutputPaths()))
}
for _, path := range tc.expectOutputPaths {
if !slices.Contains(tc.ctx.OutputPaths(), path) {
t.Errorf("expected '%s' in output paths %v", path, tc.ctx.OutputPaths())
}
}
})
}
}
func TestConvertHandler(t *testing.T) {
for _, tc := range []struct {
scenario string