huge refactoring

This commit is contained in:
Julien Neuhart
2018-03-29 20:13:19 +02:00
parent 2f97a44bed
commit e847c86baa
24 changed files with 1039 additions and 783 deletions

59
app/app.go Normal file
View File

@@ -0,0 +1,59 @@
package app
import (
"fmt"
"net/http"
"time"
"github.com/gulien/gotenberg/app/config"
"github.com/gulien/gotenberg/app/handlers"
"github.com/gulien/gotenberg/app/handlers/converter/process"
"github.com/gulien/gotenberg/app/logger"
"github.com/gorilla/mux"
)
type App struct {
version string
config *config.AppConfig
Server *http.Server
}
func NewApp(version string) (*App, error) {
c, err := config.NewAppConfig()
if err != nil {
logger.Error(err)
return nil, &appConfigError{}
}
a := &App{}
a.version = version
a.config = c
// defines our application logging.
logger.SetLevel(a.config.Logs.Level)
logger.SetFormatter(a.config.Logs.Formatter)
// defines our application router.
r := mux.NewRouter()
r.Handle("/", handlers.GetHandlersChain())
a.Server = &http.Server{
Addr: fmt.Sprintf(":%s", a.config.Port),
// good practice to set timeouts to avoid Slowloris attacks.
WriteTimeout: time.Second * 15,
ReadTimeout: time.Second * 15,
IdleTimeout: time.Second * 60,
Handler: r,
}
return a, nil
}
func (a *App) Run() error {
process.Load(a.config.CommandsConfig)
logger.Infof("Gotenberg %s", a.version)
logger.Infof("Application is starting on %s", a.Server.Addr)
return a.Server.ListenAndServe()
}

170
app/config/config.go Normal file
View File

@@ -0,0 +1,170 @@
package config
import (
"io/ioutil"
"text/template"
"github.com/gulien/gotenberg/app/logger"
"github.com/satori/go.uuid"
"github.com/sirupsen/logrus"
"gopkg.in/yaml.v2"
)
type (
AppConfig struct {
Port string
Logs struct {
Level logrus.Level
Formatter logrus.Formatter
}
CommandsConfig *CommandsConfig
}
CommandsConfig struct {
HTML *CommandConfig
Office *CommandConfig
Merge *CommandConfig
}
CommandConfig struct {
Timeout int
Template *template.Template
}
)
func NewAppConfig() (*AppConfig, error) {
fileConfig, err := loadFileConfig()
if err != nil {
logger.Error(err)
return nil, &fileConfigError{}
}
c := &AppConfig{}
c.Port = fileConfig.Port
c.Logs.Level = getLoggingLevelFromFileConfig(fileConfig)
c.Logs.Formatter = getLoggingFormatterFromFileConfig(fileConfig)
if c.Logs.Level == 999 {
return nil, &wrongLoggingLevelError{}
}
if c.Logs.Formatter == nil {
return nil, &wrongLoggingFormatError{}
}
c.CommandsConfig = &CommandsConfig{}
c.CommandsConfig.HTML = &CommandConfig{}
c.CommandsConfig.Office = &CommandConfig{}
c.CommandsConfig.Merge = &CommandConfig{}
c.CommandsConfig.HTML.Timeout = fileConfig.Commands.HTML.Timeout
c.CommandsConfig.Office.Timeout = fileConfig.Commands.Office.Timeout
c.CommandsConfig.Merge.Timeout = fileConfig.Commands.Merge.Timeout
tmplHTML, err := getCommandTemplate(fileConfig.Commands.HTML.Template)
if err != nil {
logger.Error(err)
return nil, &wrongHTMLCommandTemplate{}
}
tmplOffice, err := getCommandTemplate(fileConfig.Commands.Office.Template)
if err != nil {
logger.Error(err)
return nil, &wrongOfficeCommandTemplate{}
}
tmplMerge, err := getCommandTemplate(fileConfig.Commands.Merge.Template)
if err != nil {
logger.Error(err)
return nil, &wrongMergeCommandTemplate{}
}
c.CommandsConfig.HTML.Template = tmplHTML
c.CommandsConfig.Office.Template = tmplOffice
c.CommandsConfig.Merge.Template = tmplMerge
return c, nil
}
type fileConfig struct {
Port string `yaml:"port"`
Logs struct {
Level string `yaml:"level"`
Format string `yaml:"format"`
} `yaml:"logs"`
Commands struct {
HTML struct {
Timeout int
Template string
} `yaml:"html"`
Office struct {
Timeout int `yaml:"timeout"`
Template string `yaml:"template"`
} `yaml:"office"`
Merge struct {
Timeout int `yaml:"timeout"`
Template string `yaml:"template"`
} `yaml:"merge"`
} `yaml:"commands"`
}
// configurationFilePath is our default configuration file to parse.
const configurationFilePath = "gotenberg.yml"
func loadFileConfig() (*fileConfig, error) {
c := &fileConfig{}
data, err := ioutil.ReadFile(configurationFilePath)
if err != nil {
logger.Error(err)
return nil, &readFileError{}
}
if err := yaml.Unmarshal(data, &c); err != nil {
logger.Error(err)
return nil, &unmarshalError{}
}
return c, nil
}
var levels = map[string]logrus.Level{
"DEBUG": logrus.DebugLevel,
"INFO": logrus.InfoLevel,
"WARN": logrus.WarnLevel,
"ERROR": logrus.ErrorLevel,
"FATAL": logrus.FatalLevel,
"PANIC": logrus.PanicLevel,
}
func getLoggingLevelFromFileConfig(c *fileConfig) logrus.Level {
l, ok := levels[c.Logs.Level]
if !ok {
return 999
}
return l
}
var formatters = map[string]logrus.Formatter{
"text": &logrus.TextFormatter{},
"json": &logrus.JSONFormatter{},
}
func getLoggingFormatterFromFileConfig(c *fileConfig) logrus.Formatter {
f, ok := formatters[c.Logs.Format]
if !ok {
return nil
}
return f
}
func getCommandTemplate(command string) (*template.Template, error) {
t, err := template.New(uuid.NewV4().String()).Parse(command)
if err != nil {
return nil, err
}
return t, nil
}

49
app/config/errors.go Normal file
View File

@@ -0,0 +1,49 @@
package config
type fileConfigError struct{}
func (e *fileConfigError) Error() string {
return "An error occured while trying to load the configuration file"
}
type wrongLoggingLevelError struct{}
func (e *wrongLoggingLevelError) Error() string {
return "Accepted values for logging level: DEBUG, INFO, WARN, ERROR, FATAL, PANIC"
}
type wrongLoggingFormatError struct{}
func (e *wrongLoggingFormatError) Error() string {
return "Accepted value for logging format: text, json"
}
type wrongHTMLCommandTemplate struct{}
func (e *wrongHTMLCommandTemplate) Error() string {
return "An error occured while trying to parse the HTML command's template"
}
type wrongOfficeCommandTemplate struct{}
func (e *wrongOfficeCommandTemplate) Error() string {
return "An error occured while trying to parse the Office command's template"
}
type wrongMergeCommandTemplate struct{}
func (e *wrongMergeCommandTemplate) Error() string {
return "An error occured while trying to parse the merge command's template"
}
type readFileError struct{}
func (e *readFileError) Error() string {
return "An error occured while trying to read the configuration file"
}
type unmarshalError struct{}
func (e *unmarshalError) Error() string {
return "An error occured while trying to decode the configuration file as YAML"
}

7
app/errors.go Normal file
View File

@@ -0,0 +1,7 @@
package app
type appConfigError struct{}
func (e *appConfigError) Error() string {
return "A fatal error occured while setting up the application"
}

View File

@@ -0,0 +1,51 @@
package context
import (
"context"
"net/http"
"github.com/gulien/gotenberg/app/handlers/converter"
ghttp "github.com/gulien/gotenberg/app/handlers/http"
)
type key uint32
const (
contentTypeKey key = iota
converterKey
)
func WithContentType(r *http.Request, contentType ghttp.ContentType) *http.Request {
ctx := r.Context()
ctx = context.WithValue(ctx, contentTypeKey, contentType)
r = r.WithContext(ctx)
return r
}
func GetContentType(r *http.Request) (ghttp.ContentType, error) {
ct, ok := r.Context().Value(contentTypeKey).(ghttp.ContentType)
if !ok {
return "", &contentTypeNotFoundError{}
}
return ct, nil
}
func WithConverter(r *http.Request, converter *converter.Converter) *http.Request {
ctx := r.Context()
ctx = context.WithValue(ctx, converterKey, converter)
r = r.WithContext(ctx)
return r
}
func GetConverter(r *http.Request) (*converter.Converter, error) {
c, ok := r.Context().Value(converterKey).(*converter.Converter)
if !ok {
return nil, &converterNotFoundError{}
}
return c, nil
}

View File

@@ -0,0 +1,13 @@
package context
type contentTypeNotFoundError struct{}
func (e *contentTypeNotFoundError) Error() string {
return "The 'Content-Type' was not found in request context"
}
type converterNotFoundError struct{}
func (e *converterNotFoundError) Error() string {
return "The converter was not found in request context"
}

View File

@@ -0,0 +1,118 @@
package converter
import (
"net/http"
"os"
gfile "github.com/gulien/gotenberg/app/handlers/converter/file"
"github.com/gulien/gotenberg/app/handlers/converter/process"
ghttp "github.com/gulien/gotenberg/app/handlers/http"
)
type Converter struct {
files []*gfile.File
resultFilesPaths []string
FinalFilePath string
}
func NewConverter(r *http.Request, contentType ghttp.ContentType) (*Converter, error) {
c := &Converter{}
switch contentType {
case ghttp.MultipartFormDataContentType:
err := r.ParseMultipartForm(32 << 20)
if err != nil {
return nil, err
}
formData := r.MultipartForm
files, ok := formData.File["files"]
if !ok {
// TODO
return nil, nil
}
for i := range files {
file, err := files[i].Open()
if err != nil {
return nil, err
}
defer file.Close()
f, err := gfile.NewFile(file)
if err != nil {
// todo err
return nil, err
}
c.files = append(c.files, f)
}
break
default:
f, err := gfile.NewFile(r.Body)
if err != nil {
// todo err
return nil, err
}
c.files = append(c.files, f)
}
return c, nil
}
func (c *Converter) Convert() error {
if len(c.files) == 0 {
return &noFileToConvertError{}
}
var filesPaths []string
for _, f := range c.files {
if f.Type != gfile.PDFType {
path, err := process.ExecConversion(f)
if err != nil {
return err
}
filesPaths = append(filesPaths, path)
} else {
filesPaths = append(filesPaths, f.Path)
}
}
path, err := process.ExecMerge(filesPaths)
if err != nil {
return err
}
c.resultFilesPaths = filesPaths
c.FinalFilePath = path
return nil
}
func (c *Converter) Clear() error {
for _, f := range c.files {
err := os.Remove(f.Path)
if err != nil {
return err
}
}
for _, path := range c.resultFilesPaths {
err := os.Remove(path)
if err != nil {
return err
}
}
if c.FinalFilePath != "" {
err := os.Remove(c.FinalFilePath)
if err != nil {
return err
}
}
return nil
}

View File

@@ -0,0 +1,7 @@
package converter
type noFileToConvertError struct{}
func (e *noFileToConvertError) Error() string {
return "There is no file to convert"
}

View File

@@ -0,0 +1,120 @@
package file
import (
"fmt"
"io"
"os"
ghttp "github.com/gulien/gotenberg/app/handlers/http"
"github.com/satori/go.uuid"
)
type (
File struct {
Type FileType
Path string
}
FileType string
FileExt string
)
const (
PDFType FileType = "PDF"
HTMLType FileType = "HTML"
OfficeType FileType = "Office"
PDFExt FileExt = ".pdf"
HTMLExt FileExt = ".html"
OfficeExt FileExt = ""
)
func NewFile(r io.Reader) (*File, error) {
f := &File{
Path: MakeFilePath(),
}
file, err := os.Create(f.Path)
if err != nil {
return nil, err
}
defer file.Close()
_, err = io.Copy(file, r)
if err != nil {
return nil, err
}
// resets the read pointer.
file.Seek(0, 0)
t, err := findFileType(file)
if err != nil {
return nil, err
}
f.Type = t
f, err = reworkFilePath(f)
if err != nil {
return nil, err
}
return f, nil
}
func MakeFilePath() string {
return fmt.Sprintf("./%s", uuid.NewV4().String())
}
var filesTypes = map[ghttp.ContentType]FileType{
ghttp.PDFContentType: PDFType,
ghttp.HTMLContentType: HTMLType,
ghttp.OctetStreamContentType: OfficeType,
ghttp.ZipContentType: OfficeType,
}
func findFileType(f *os.File) (FileType, error) {
ct, err := ghttp.SniffContentType(f)
if err != nil {
return "", err
}
t, ok := filesTypes[ct]
if !ok {
// TODO error
return "", nil
}
return t, nil
}
var filesExtensions = map[FileType]FileExt{
PDFType: PDFExt,
HTMLType: HTMLExt,
OfficeType: OfficeExt,
}
func reworkFilePath(f *File) (*File, error) {
ext, ok := filesExtensions[f.Type]
if !ok {
// TODO error
return nil, nil
}
if ext != OfficeExt {
newPath := fmt.Sprintf("./%s%s", MakeFilePath(), ext)
err := os.Rename(f.Path, newPath)
if err != nil {
return nil, err
}
f.Path = newPath
}
return f, nil
}

View File

@@ -0,0 +1,13 @@
package process
type impossibleConversionError struct{}
func (e *impossibleConversionError) Error() string {
return "Impossible conversion"
}
type commandTimeoutError struct{}
func (e *commandTimeoutError) Error() string {
return "The command has reached timeout"
}

View File

@@ -0,0 +1,119 @@
package process
import (
"bytes"
"fmt"
"os/exec"
"text/template"
"time"
"github.com/gulien/gotenberg/app/config"
gfile "github.com/gulien/gotenberg/app/handlers/converter/file"
)
var commandsConfig *config.CommandsConfig
func Load(config *config.CommandsConfig) {
commandsConfig = config
}
func Reset() {
commandsConfig = nil
}
type conversionData struct {
FilePath string
ResultFilePath string
}
func ExecConversion(file *gfile.File) (string, error) {
cmdData := &conversionData{
FilePath: file.Path,
ResultFilePath: fmt.Sprintf("%s%s", gfile.MakeFilePath(), gfile.PDFExt),
}
var (
cmdTemplate *template.Template
cmdTimeout int
)
switch file.Type {
case gfile.HTMLType:
cmdTemplate = commandsConfig.HTML.Template
cmdTimeout = commandsConfig.HTML.Timeout
break
case gfile.OfficeType:
cmdTemplate = commandsConfig.Office.Template
cmdTimeout = commandsConfig.Office.Timeout
break
default:
return "", &impossibleConversionError{}
}
var data bytes.Buffer
if err := cmdTemplate.Execute(&data, cmdData); err != nil {
return "", err
}
err := execCommand(data.String(), cmdTimeout)
if err != nil {
return "", err
}
return cmdData.ResultFilePath, nil
}
type mergeData struct {
FilesPaths []string
ResultFilePath string
}
func ExecMerge(filesPaths []string) (string, error) {
cmdData := &mergeData{
FilesPaths: filesPaths,
ResultFilePath: fmt.Sprintf("%s%s", gfile.MakeFilePath(), gfile.PDFExt),
}
cmdTemplate := commandsConfig.Merge.Template
cmdTimeout := commandsConfig.Merge.Timeout
var data bytes.Buffer
if err := cmdTemplate.Execute(&data, cmdData); err != nil {
return "", err
}
err := execCommand(data.String(), cmdTimeout)
if err != nil {
return "", err
}
return cmdData.ResultFilePath, nil
}
func execCommand(command string, timeout int) error {
// Wait for the process to finish or kill it after a timeout.
cmd := exec.Command("/bin/sh", "-c", command)
if err := cmd.Start(); err != nil {
return err
}
done := make(chan error, 1)
go func() {
done <- cmd.Wait()
}()
select {
case <-time.After(time.Duration(timeout) * time.Second):
if err := cmd.Process.Kill(); err != nil {
return err
}
return &commandTimeoutError{}
case err := <-done:
if err != nil {
return err
}
return nil
}
}

7
app/handlers/errors.go Normal file
View File

@@ -0,0 +1,7 @@
package handlers
type requestHasNoContentError struct{}
func (e *requestHasNoContentError) Error() string {
return "Request has not content"
}

126
app/handlers/handlers.go Normal file
View File

@@ -0,0 +1,126 @@
package handlers
import (
"fmt"
"io"
"net/http"
"os"
"github.com/gulien/gotenberg/app/handlers/context"
"github.com/gulien/gotenberg/app/handlers/converter"
ghttp "github.com/gulien/gotenberg/app/handlers/http"
"github.com/gulien/gotenberg/app/logger"
"github.com/justinas/alice"
)
func GetHandlersChain() http.Handler {
return alice.New(enforceContentLengthHandler, enforceContentTypeHandler, convertHandler, serveHandler).ThenFunc(clearHandler)
}
// enforeContentLengthHandler checks if the request has content.
func enforceContentLengthHandler(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.ContentLength == 0 {
e := &requestHasNoContentError{}
http.Error(w, e.Error(), http.StatusBadRequest)
logger.Error(e)
return
}
next.ServeHTTP(w, r)
})
}
// enforceContentTypeHandler checks if the "Content-Type" entry
// from the request's header matches one of the allowed content types.
func enforceContentTypeHandler(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
ct, err := ghttp.FindAuthorizedContentType(r.Header)
if err != nil {
http.Error(w, err.Error(), http.StatusUnsupportedMediaType)
logger.Error(err)
return
}
r = context.WithContentType(r, ct)
next.ServeHTTP(w, r)
})
}
// convertHandler is in charge of converting the file(s) from the request to PDF.
func convertHandler(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
ct, err := context.GetContentType(r)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
logger.Error(err)
return
}
c, err := converter.NewConverter(r, ct)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
logger.Error(err)
return
}
err = c.Convert()
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
logger.Error(err)
return
}
r = context.WithConverter(r, c)
next.ServeHTTP(w, r)
})
}
// serveHandler simply serves the created PDF.
func serveHandler(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
c, err := context.GetConverter(r)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
logger.Error(err)
}
reader, err := os.Open(c.FinalFilePath)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
logger.Error(err)
return
}
defer reader.Close()
resultFileInfo, err := reader.Stat()
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
logger.Error(err)
return
}
w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", resultFileInfo.Name()))
w.Header().Set("Content-Type", "application/pdf")
w.Header().Set("Content-Length", fmt.Sprintf("%d", resultFileInfo.Size()))
io.Copy(w, reader)
next.ServeHTTP(w, r)
})
}
// clearHandler removes all files created during the conversion.
func clearHandler(w http.ResponseWriter, r *http.Request) {
c, err := context.GetConverter(r)
if err != nil {
logger.Warn(err.Error())
}
if err := c.Clear(); err != nil {
logger.Warn(err.Error())
}
}

View File

@@ -0,0 +1,15 @@
package http
import "fmt"
type notAuthorizedContentTypeError struct{}
func (e *notAuthorizedContentTypeError) Error() string {
return fmt.Sprintf("Accepted values for 'Content-Type': %s, %s, %s, %s", PDFContentType, HTMLContentType, OctetStreamContentType, MultipartFormDataContentType)
}
type notAuthorizedFileContentTypeError struct{}
func (e *notAuthorizedFileContentTypeError) Error() string {
return fmt.Sprintf("Unable to detect a file 'Content-Type'")
}

60
app/handlers/http/http.go Normal file
View File

@@ -0,0 +1,60 @@
package http
import (
"net/http"
"os"
"strings"
)
type ContentType string
const (
PDFContentType ContentType = "application/pdf"
HTMLContentType ContentType = "text/html"
OctetStreamContentType ContentType = "application/octet-stream"
ZipContentType ContentType = "application/zip"
MultipartFormDataContentType ContentType = "multipart/form-data"
)
func FindAuthorizedContentType(h http.Header) (ContentType, error) {
ct := findContentType(h.Get("Content-Type"), HTMLContentType, OctetStreamContentType, MultipartFormDataContentType)
if ct == "" {
return "", &notAuthorizedContentTypeError{}
}
return ct, nil
}
func SniffContentType(f *os.File) (ContentType, error) {
// only the first 512 bytes are used to sniff the content type.
buffer := make([]byte, 512)
n, err := f.Read(buffer)
if err != nil {
return "", err
}
// resets the read pointer.
f.Seek(0, 0)
// using n if size of buffer < 512 bytes.
ct := findContentType(http.DetectContentType(buffer[:n]), PDFContentType, HTMLContentType, OctetStreamContentType, ZipContentType)
if ct == "" {
return "", &notAuthorizedFileContentTypeError{}
}
return ct, nil
}
func findContentType(contentType string, contentTypes ...ContentType) ContentType {
for _, ct := range contentTypes {
if i := strings.IndexRune(contentType, ';'); i != -1 {
contentType = contentType[0:i]
}
if contentType == string(ct) {
return ct
}
}
return ""
}

65
app/logger/logger.go Normal file
View File

@@ -0,0 +1,65 @@
// Package logger
package logger
import (
"os"
"github.com/sirupsen/logrus"
)
type logger struct {
logger *logrus.Logger
}
var log *logger = newLogger()
func newLogger() *logger {
l := &logger{
logger: logrus.New(),
}
l.logger.Out = os.Stdout
l.logger.Level = logrus.InfoLevel
return l
}
func SetLevel(level logrus.Level) {
log.logger.SetLevel(level)
}
func SetFormatter(formatter logrus.Formatter) {
log.logger.Formatter = formatter
}
func Debug(message string) {
log.logger.Debug(message)
}
func Debugf(format string, args ...interface{}) {
log.logger.Debugf(format, args)
}
func Info(message string) {
log.logger.Info(message)
}
func Infof(format string, args ...interface{}) {
log.logger.Infof(format, args)
}
func Warn(message string) {
log.logger.Warn(message)
}
func Error(err error) {
log.logger.Error(err.Error())
}
func Fatal(err error) {
log.logger.Fatal(err.Error())
}
func Panic(err error) {
log.logger.Panic(err.Error())
}