From 4140c6defaadc66ac6c653c70ece9729d5607980 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Sat, 2 Jul 2022 15:00:45 -0300 Subject: [PATCH 001/135] Update readme --- readme.md | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/readme.md b/readme.md index f75e8f0..5b59f42 100644 --- a/readme.md +++ b/readme.md @@ -107,4 +107,44 @@ func (app *Config) SomeHandler(w http.ResponseWriter, r *http.Request) { // keep going in the handler... } +``` + +### Uploading a File: + +To upload a file to a specific directory: + +```go +package main + +import ( + "fmt" + "github.com/tsawler/toolbox" + "log" + "net/http" +) + +func main() { + + // handle route + http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/html") + fmt.Fprint(w, "

Hello World!

") + }) + + http.HandleFunc("/upload", func(w http.ResponseWriter, r *http.Request) { + var t toolbox.Tools + + u, err := t.UploadOneFile(r, "./uploads") + + // the returned variable, u, will have the type toolbox.Uploaded file + w.Write([]byte(fmt.Sprintf("New file name: %s, Original file name: %s, size: %d", u.NewFileName, u.OriginalFileName, u.FileSize))) + }) + + // print a log message + log.Println("Starting server on port 8080") + + // start the server + http.ListenAndServe(":8080", nil) +} + ``` \ No newline at end of file From 419ff464ad6388512a37654489afedfb7ad96de2 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Sat, 2 Jul 2022 15:03:37 -0300 Subject: [PATCH 002/135] update readme --- readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme.md b/readme.md index 5b59f42..0148966 100644 --- a/readme.md +++ b/readme.md @@ -134,7 +134,7 @@ func main() { http.HandleFunc("/upload", func(w http.ResponseWriter, r *http.Request) { var t toolbox.Tools - u, err := t.UploadOneFile(r, "./uploads") + u, _ := t.UploadOneFile(r, "./uploads") // the returned variable, u, will have the type toolbox.Uploaded file w.Write([]byte(fmt.Sprintf("New file name: %s, Original file name: %s, size: %d", u.NewFileName, u.OriginalFileName, u.FileSize))) From f51b2ae54d69d62c3e8fcb4a79ebb8f32663ecb1 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Sun, 3 Jul 2022 14:27:33 -0300 Subject: [PATCH 003/135] simplify --- tools.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools.go b/tools.go index 5ff341e..d95c262 100644 --- a/tools.go +++ b/tools.go @@ -31,7 +31,7 @@ type JSONResponse struct { // ReadJSON tries to read the body of a request and converts it into JSON func (t *Tools) ReadJSON(w http.ResponseWriter, r *http.Request, data any) error { - maxBytes := 1048576 // one megabyte + maxBytes := 1024 * 1024 // one megabyte if t.MaxFileSize > 0 { maxBytes = t.MaxFileSize } From dcf55f2e1ad0a5f562ad272fc464713f1f8dc7e1 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Sun, 3 Jul 2022 14:33:29 -0300 Subject: [PATCH 004/135] restrict file types on upload --- tools.go | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/tools.go b/tools.go index d95c262..eeba933 100644 --- a/tools.go +++ b/tools.go @@ -19,7 +19,8 @@ const randomStringSource = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ // Tools is the type for this package. Create a variable of this type and you have access // to all the methods with the receiver type *Tools. type Tools struct { - MaxFileSize int + MaxFileSize int // maximum size of uploaded files in bytes + AllowedFileTypes []string // allowed file types for upload (e.g. image/jpeg) } // JSONResponse is the type used for sending JSON around @@ -168,6 +169,21 @@ func (t *Tools) UploadOneFile(r *http.Request, uploadDir string) (*UploadedFile, return nil, err } + allowed := false + if len(t.AllowedFileTypes) > 0 { + for _, x := range t.AllowedFileTypes { + if ext.Is(x) { + allowed = true + } + } + } else { + allowed = true + } + + if !allowed { + return nil, errors.New("the uploaded file type is not permitted") + } + _, err = infile.Seek(0, 0) if err != nil { fmt.Println(err) From e7bc7e5a77afa73192a07dad1da5c70c328bac95 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Sun, 3 Jul 2022 15:04:29 -0300 Subject: [PATCH 005/135] improvements for cross-platform file uploads --- tools.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools.go b/tools.go index eeba933..1c298d8 100644 --- a/tools.go +++ b/tools.go @@ -10,6 +10,7 @@ import ( "net/http" "os" "path" + "path/filepath" "github.com/gabriel-vasile/mimetype" ) @@ -196,7 +197,7 @@ func (t *Tools) UploadOneFile(r *http.Request, uploadDir string) (*UploadedFile, var outfile *os.File defer outfile.Close() - if outfile, err = os.Create(uploadDir + uploadedFile.NewFileName); nil != err { + if outfile, err = os.Create(filepath.Join(uploadDir, uploadedFile.NewFileName)); nil != err { return nil, err } else { fileSize, err := io.Copy(outfile, infile) From 9cb98bbd0472c089a7838865e362f195b0f12948 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Sun, 3 Jul 2022 15:10:05 -0300 Subject: [PATCH 006/135] update readme --- readme.md | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/readme.md b/readme.md index 0148966..cf0d57d 100644 --- a/readme.md +++ b/readme.md @@ -132,9 +132,23 @@ func main() { }) http.HandleFunc("/upload", func(w http.ResponseWriter, r *http.Request) { - var t toolbox.Tools + if r.Method != "POST" { + http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) + return + } - u, _ := t.UploadOneFile(r, "./uploads") + t := toolbox.Tools{ + MaxFileSize: 1024 * 1024 * 1024, + AllowedFileTypes: []string{"image/gif"}, + } + + _ = t.CreateDirIfNotExist("./uploads") + + u, err := t.UploadOneFile(r, "./uploads") + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } // the returned variable, u, will have the type toolbox.Uploaded file w.Write([]byte(fmt.Sprintf("New file name: %s, Original file name: %s, size: %d", u.NewFileName, u.OriginalFileName, u.FileSize))) @@ -146,5 +160,4 @@ func main() { // start the server http.ListenAndServe(":8080", nil) } - ``` \ No newline at end of file From 4c65a4199a0ce3246e02f99284b99ea578940f26 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Sun, 3 Jul 2022 15:11:03 -0300 Subject: [PATCH 007/135] update readme --- readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme.md b/readme.md index cf0d57d..1934d26 100644 --- a/readme.md +++ b/readme.md @@ -139,7 +139,7 @@ func main() { t := toolbox.Tools{ MaxFileSize: 1024 * 1024 * 1024, - AllowedFileTypes: []string{"image/gif"}, + AllowedFileTypes: []string{"image/gif", "image/png", "image/jpeg"}, } _ = t.CreateDirIfNotExist("./uploads") From 04b522f78a8e967a2f7dd5a592ad773f0051c3fa Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Sun, 3 Jul 2022 15:17:56 -0300 Subject: [PATCH 008/135] update test --- tools_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/tools_test.go b/tools_test.go index 406539b..e6070d7 100644 --- a/tools_test.go +++ b/tools_test.go @@ -219,6 +219,7 @@ func TestTools_UploadOneFile(t *testing.T) { request.Header.Add("Content-Type", writer.FormDataContentType()) var testTools Tools + testTools.AllowedFileTypes = []string{"image/png"} uploadedFile, err := testTools.UploadOneFile(request, "./testdata/uploads/") if err != nil { From 4691ec863d4b4afca2bf08b87ec50a1d250e71e4 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Sun, 3 Jul 2022 15:20:42 -0300 Subject: [PATCH 009/135] typo in readme --- readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme.md b/readme.md index 1934d26..2af23db 100644 --- a/readme.md +++ b/readme.md @@ -150,7 +150,7 @@ func main() { return } - // the returned variable, u, will have the type toolbox.Uploaded file + // the returned variable, u, will have the type toolbox.UploadedFile w.Write([]byte(fmt.Sprintf("New file name: %s, Original file name: %s, size: %d", u.NewFileName, u.OriginalFileName, u.FileSize))) }) From 57e69ae1743dada04f7c0c53bfd8e42ca8290f01 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Sun, 3 Jul 2022 15:25:58 -0300 Subject: [PATCH 010/135] Check for file size on upload --- tools.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tools.go b/tools.go index 1c298d8..43e3a01 100644 --- a/tools.go +++ b/tools.go @@ -149,8 +149,13 @@ type UploadedFile struct { // UploadOneFile uploads one file to a specified directory, and gives it a random name. // It returns the newly named file, the original file name, and potentially an error. func (t *Tools) UploadOneFile(r *http.Request, uploadDir string) (*UploadedFile, error) { + // make sure the file does not exceed our file size limit + if r.ContentLength > int64(t.MaxFileSize) { + return nil, errors.New("the uploaded file is too big") + } + // parse the form so we have access to the file - err := r.ParseMultipartForm(1024 * 1024 * 1024) + err := r.ParseMultipartForm(32 << 20) if err != nil { return nil, err } From b2006b074a397fe326301fa0ab97318ec88ec7f5 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Sun, 3 Jul 2022 15:29:20 -0300 Subject: [PATCH 011/135] Check for file size on upload --- tools.go | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/tools.go b/tools.go index 43e3a01..15d7ce9 100644 --- a/tools.go +++ b/tools.go @@ -149,15 +149,10 @@ type UploadedFile struct { // UploadOneFile uploads one file to a specified directory, and gives it a random name. // It returns the newly named file, the original file name, and potentially an error. func (t *Tools) UploadOneFile(r *http.Request, uploadDir string) (*UploadedFile, error) { - // make sure the file does not exceed our file size limit - if r.ContentLength > int64(t.MaxFileSize) { - return nil, errors.New("the uploaded file is too big") - } - // parse the form so we have access to the file - err := r.ParseMultipartForm(32 << 20) + err := r.ParseMultipartForm(int64(t.MaxFileSize)) if err != nil { - return nil, err + return nil, errors.New("the uploaded file is too big") } var uploadedFile UploadedFile From 30bed91b83cded583051e5cbc92d1bc43669c42f Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Mon, 4 Jul 2022 15:51:52 -0300 Subject: [PATCH 012/135] Allow for preserving original filename when uploading a file. --- tools.go | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/tools.go b/tools.go index 15d7ce9..6ca4131 100644 --- a/tools.go +++ b/tools.go @@ -148,7 +148,15 @@ type UploadedFile struct { // UploadOneFile uploads one file to a specified directory, and gives it a random name. // It returns the newly named file, the original file name, and potentially an error. -func (t *Tools) UploadOneFile(r *http.Request, uploadDir string) (*UploadedFile, error) { +// If the optional last parameter is set to true, then we will not rename the file, but +// will use the original file name. +func (t *Tools) UploadOneFile(r *http.Request, uploadDir string, rename ...bool) (*UploadedFile, error) { + // check to see if we are renaming the file with the optional last parameter + renameFile := false + if len(rename) > 0 { + renameFile = rename[0] + } + // parse the form so we have access to the file err := r.ParseMultipartForm(int64(t.MaxFileSize)) if err != nil { @@ -191,7 +199,11 @@ func (t *Tools) UploadOneFile(r *http.Request, uploadDir string) (*UploadedFile, return nil, err } - uploadedFile.NewFileName = t.RandomString(25) + ext.Extension() + if renameFile { + uploadedFile.NewFileName = t.RandomString(25) + ext.Extension() + } else { + uploadedFile.NewFileName = hdr.Filename + } uploadedFile.OriginalFileName = hdr.Filename var outfile *os.File From 4d34f68d469f88c9d8496609bad1ba4a94f281ad Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Mon, 4 Jul 2022 15:54:30 -0300 Subject: [PATCH 013/135] update the readme --- readme.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/readme.md b/readme.md index 2af23db..63b7d9a 100644 --- a/readme.md +++ b/readme.md @@ -144,6 +144,9 @@ func main() { _ = t.CreateDirIfNotExist("./uploads") + // upload the file. Note that if you don't want the file to be renamed, + // you can add an optional final parameter -- true will rename the file (the default) + // and false will preserve the original filename, etc. u, err := t.UploadOneFile(r, "./uploads", false) u, err := t.UploadOneFile(r, "./uploads") if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) From 3576f633254d894b3ac2b841c5220b80dc5a6f2e Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Mon, 4 Jul 2022 15:55:20 -0300 Subject: [PATCH 014/135] update the readme --- readme.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/readme.md b/readme.md index 63b7d9a..d942039 100644 --- a/readme.md +++ b/readme.md @@ -146,7 +146,8 @@ func main() { // upload the file. Note that if you don't want the file to be renamed, // you can add an optional final parameter -- true will rename the file (the default) - // and false will preserve the original filename, etc. u, err := t.UploadOneFile(r, "./uploads", false) + // and false will preserve the original filename, for example: + // u, err := t.UploadOneFile(r, "./uploads", false) u, err := t.UploadOneFile(r, "./uploads") if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) From e60d15706ad84b5336146ffd6beee68b86a99592 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Mon, 4 Jul 2022 17:23:50 -0300 Subject: [PATCH 015/135] improve reading json --- tools.go | 44 ++++++++++++++++++++++++++++++++++++++------ 1 file changed, 38 insertions(+), 6 deletions(-) diff --git a/tools.go b/tools.go index 6ca4131..2a9017b 100644 --- a/tools.go +++ b/tools.go @@ -11,6 +11,7 @@ import ( "os" "path" "path/filepath" + "strings" "github.com/gabriel-vasile/mimetype" ) @@ -20,6 +21,7 @@ const randomStringSource = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ // Tools is the type for this package. Create a variable of this type and you have access // to all the methods with the receiver type *Tools. type Tools struct { + MaxJSONSize int // maximum siz of JSON file we'll process MaxFileSize int // maximum size of uploaded files in bytes AllowedFileTypes []string // allowed file types for upload (e.g. image/jpeg) } @@ -34,21 +36,51 @@ type JSONResponse struct { // ReadJSON tries to read the body of a request and converts it into JSON func (t *Tools) ReadJSON(w http.ResponseWriter, r *http.Request, data any) error { maxBytes := 1024 * 1024 // one megabyte - if t.MaxFileSize > 0 { - maxBytes = t.MaxFileSize - } - r.Body = http.MaxBytesReader(w, r.Body, int64(maxBytes)) dec := json.NewDecoder(r.Body) + dec.DisallowUnknownFields() + err := dec.Decode(data) if err != nil { - return err + var syntaxError *json.SyntaxError + var unmarshalTypeError *json.UnmarshalTypeError + var invalidUnmarshalError *json.InvalidUnmarshalError + + switch { + case errors.As(err, &syntaxError): + return fmt.Errorf("body contains badly-formed JSON (at character %d)", syntaxError.Offset) + + case errors.Is(err, io.ErrUnexpectedEOF): + return errors.New("body contains badly-formed JSON") + + case errors.As(err, &unmarshalTypeError): + if unmarshalTypeError.Field != "" { + return fmt.Errorf("body contains incorrect JSON type for field %q", unmarshalTypeError.Field) + } + return fmt.Errorf("body contains incorrect JSON type (at character %d)", unmarshalTypeError.Offset) + + case errors.Is(err, io.EOF): + return errors.New("body must not be empty") + + case strings.HasPrefix(err.Error(), "json: unknown field "): + fieldName := strings.TrimPrefix(err.Error(), "json: unknown field ") + return fmt.Errorf("body contains unknown key %s", fieldName) + + case err.Error() == "http: request body too large": + return fmt.Errorf("body must not be larger than %d bytes", maxBytes) + + case errors.As(err, &invalidUnmarshalError): + panic(err) + + default: + return err + } } err = dec.Decode(&struct{}{}) if err != io.EOF { - return errors.New("body must have only a single json value") + return errors.New("body must only contain a single JSON value") } return nil From 5d6bdccfd2dcdacb3ae83609eccba778b259dea9 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Mon, 4 Jul 2022 17:24:48 -0300 Subject: [PATCH 016/135] improve reading json --- tools.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools.go b/tools.go index 2a9017b..f483ae5 100644 --- a/tools.go +++ b/tools.go @@ -36,6 +36,9 @@ type JSONResponse struct { // ReadJSON tries to read the body of a request and converts it into JSON func (t *Tools) ReadJSON(w http.ResponseWriter, r *http.Request, data any) error { maxBytes := 1024 * 1024 // one megabyte + if t.MaxJSONSize != 0 { + maxBytes = t.MaxJSONSize + } r.Body = http.MaxBytesReader(w, r.Body, int64(maxBytes)) dec := json.NewDecoder(r.Body) From f653cfeeba133da8d34b9a1868ee3809e11125f9 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Mon, 4 Jul 2022 17:41:40 -0300 Subject: [PATCH 017/135] fix default for renaming files --- tools.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/tools.go b/tools.go index f483ae5..01a3c75 100644 --- a/tools.go +++ b/tools.go @@ -12,8 +12,6 @@ import ( "path" "path/filepath" "strings" - - "github.com/gabriel-vasile/mimetype" ) const randomStringSource = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0987654321_+" @@ -187,7 +185,7 @@ type UploadedFile struct { // will use the original file name. func (t *Tools) UploadOneFile(r *http.Request, uploadDir string, rename ...bool) (*UploadedFile, error) { // check to see if we are renaming the file with the optional last parameter - renameFile := false + renameFile := true if len(rename) > 0 { renameFile = rename[0] } @@ -207,16 +205,17 @@ func (t *Tools) UploadOneFile(r *http.Request, uploadDir string, rename ...bool) } defer infile.Close() - ext, err := mimetype.DetectReader(infile) + buff := make([]byte, 512) + _, err = infile.Read(buff) if err != nil { - fmt.Println(err) return nil, err } allowed := false + filetype := http.DetectContentType(buff) if len(t.AllowedFileTypes) > 0 { for _, x := range t.AllowedFileTypes { - if ext.Is(x) { + if strings.ToLower(filetype) == strings.ToLower(x) { allowed = true } } @@ -235,7 +234,7 @@ func (t *Tools) UploadOneFile(r *http.Request, uploadDir string, rename ...bool) } if renameFile { - uploadedFile.NewFileName = t.RandomString(25) + ext.Extension() + uploadedFile.NewFileName = t.RandomString(25) + filepath.Ext(hdr.Filename) } else { uploadedFile.NewFileName = hdr.Filename } From a0d3c1b3f5b44df18b60e43dd332063264496fda Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Mon, 4 Jul 2022 17:42:23 -0300 Subject: [PATCH 018/135] fix default for renaming files --- tools.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools.go b/tools.go index 01a3c75..ee1b656 100644 --- a/tools.go +++ b/tools.go @@ -215,7 +215,7 @@ func (t *Tools) UploadOneFile(r *http.Request, uploadDir string, rename ...bool) filetype := http.DetectContentType(buff) if len(t.AllowedFileTypes) > 0 { for _, x := range t.AllowedFileTypes { - if strings.ToLower(filetype) == strings.ToLower(x) { + if strings.EqualFold(filetype, x) { allowed = true } } From 27cfe133876124f79dafd7ecfcdcab79ef660d20 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Mon, 4 Jul 2022 17:43:03 -0300 Subject: [PATCH 019/135] remove dependency --- go.mod | 4 ---- go.sum | 11 ----------- 2 files changed, 15 deletions(-) diff --git a/go.mod b/go.mod index 03d1e16..1016502 100644 --- a/go.mod +++ b/go.mod @@ -1,7 +1,3 @@ module github.com/tsawler/toolbox go 1.18 - -require github.com/gabriel-vasile/mimetype v1.4.0 - -require golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 // indirect diff --git a/go.sum b/go.sum index 9876ebe..e69de29 100644 --- a/go.sum +++ b/go.sum @@ -1,11 +0,0 @@ -github.com/gabriel-vasile/mimetype v1.4.0 h1:Cn9dkdYsMIu56tGho+fqzh7XmvY2YyGU0FnbhiOsEro= -github.com/gabriel-vasile/mimetype v1.4.0/go.mod h1:fA8fi6KUiG7MgQQ+mEWotXoEOvmxRtOJlERCzSmRvr8= -golang.org/x/net v0.0.0-20210505024714-0287a6fb4125 h1:Ugb8sMTWuWRC3+sz5WeN/4kejDx9BvIwnPUiJBjJE+8= -golang.org/x/net v0.0.0-20210505024714-0287a6fb4125/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 h1:HVyaeDAYux4pnY+D/SiwmLOR36ewZ4iGQIIrtnuCjFA= -golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= From 7832f999f0e38cb69c53a4fddb962dac3e2eb61a Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Mon, 4 Jul 2022 17:44:39 -0300 Subject: [PATCH 020/135] comments --- tools.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools.go b/tools.go index ee1b656..4a37ec7 100644 --- a/tools.go +++ b/tools.go @@ -172,7 +172,7 @@ func (t *Tools) DownloadStaticFile(w http.ResponseWriter, r *http.Request, p, fi http.ServeFile(w, r, fp) } -// UploadedFile is a struct used to +// UploadedFile is a struct used for the uploaded file type UploadedFile struct { NewFileName string OriginalFileName string From 13eb2a9f718e486e5eed269b89d847b815e94389 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Mon, 4 Jul 2022 20:35:24 -0300 Subject: [PATCH 021/135] any -> interface{}, for backwards compatibility --- tools.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tools.go b/tools.go index 4a37ec7..dc34770 100644 --- a/tools.go +++ b/tools.go @@ -26,13 +26,13 @@ type Tools struct { // JSONResponse is the type used for sending JSON around type JSONResponse struct { - Error bool `json:"error"` - Message string `json:"message"` - Data any `json:"data,omitempty"` + Error bool `json:"error"` + Message string `json:"message"` + Data interface{} `json:"data,omitempty"` } // ReadJSON tries to read the body of a request and converts it into JSON -func (t *Tools) ReadJSON(w http.ResponseWriter, r *http.Request, data any) error { +func (t *Tools) ReadJSON(w http.ResponseWriter, r *http.Request, data interface{}) error { maxBytes := 1024 * 1024 // one megabyte if t.MaxJSONSize != 0 { maxBytes = t.MaxJSONSize @@ -88,7 +88,7 @@ func (t *Tools) ReadJSON(w http.ResponseWriter, r *http.Request, data any) error } // WriteJSON takes a response status code and arbitrary data and writes a json response to the client -func (t *Tools) WriteJSON(w http.ResponseWriter, status int, data any, headers ...http.Header) error { +func (t *Tools) WriteJSON(w http.ResponseWriter, status int, data interface{}, headers ...http.Header) error { out, err := json.Marshal(data) if err != nil { return err @@ -139,7 +139,7 @@ func (t *Tools) RandomString(n int) string { // PushJSONToRemote posts arbitrary json to some url, and returns error, // if any, as well as the response status code -func (t *Tools) PushJSONToRemote(client *http.Client, uri string, data any) (int, error) { +func (t *Tools) PushJSONToRemote(client *http.Client, uri string, data interface{}) (int, error) { // create json we'll send jsonData, err := json.MarshalIndent(data, "", "\t") if err != nil { From 97ac1761d92d54017dae31faa5cb5a2e9335199c Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Mon, 4 Jul 2022 20:36:06 -0300 Subject: [PATCH 022/135] marshalindent -> marshal --- tools.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools.go b/tools.go index dc34770..b345630 100644 --- a/tools.go +++ b/tools.go @@ -141,7 +141,7 @@ func (t *Tools) RandomString(n int) string { // if any, as well as the response status code func (t *Tools) PushJSONToRemote(client *http.Client, uri string, data interface{}) (int, error) { // create json we'll send - jsonData, err := json.MarshalIndent(data, "", "\t") + jsonData, err := json.Marshal(data) if err != nil { return 0, err } From 0401123b311907f91cf33423a6a377ecdc883d16 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Mon, 4 Jul 2022 20:38:16 -0300 Subject: [PATCH 023/135] Update readme --- readme.md | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/readme.md b/readme.md index d942039..2f58973 100644 --- a/readme.md +++ b/readme.md @@ -111,7 +111,29 @@ func (app *Config) SomeHandler(w http.ResponseWriter, r *http.Request) { ### Uploading a File: -To upload a file to a specific directory: +To upload a file to a specific directory, with this for HTML: + +``` + + + + + + + Upload test + + + +
+ + +
+ + + +``` +And this for a Go application: ```go package main @@ -125,12 +147,10 @@ import ( func main() { - // handle route - http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "text/html") - fmt.Fprint(w, "

Hello World!

") - }) + // handle html route (http://localhost:8080/) + http.Handle("/", http.StripPrefix("/", http.FileServer(http.Dir(".")))) + // Post handler http.HandleFunc("/upload", func(w http.ResponseWriter, r *http.Request) { if r.Method != "POST" { http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) From fe3ec25d483f2241e1e65b84893974d914f61d9a Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Mon, 4 Jul 2022 20:38:47 -0300 Subject: [PATCH 024/135] Update readme --- readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme.md b/readme.md index 2f58973..8e77bb9 100644 --- a/readme.md +++ b/readme.md @@ -113,7 +113,7 @@ func (app *Config) SomeHandler(w http.ResponseWriter, r *http.Request) { To upload a file to a specific directory, with this for HTML: -``` +```html From 339334c0e5383850a8c91a4bac0a472a6ece8211 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Tue, 5 Jul 2022 11:43:47 -0300 Subject: [PATCH 025/135] Update tests; remove panic --- tools.go | 2 +- tools_test.go | 84 +++++++++++++++++++++++++-------------------------- 2 files changed, 43 insertions(+), 43 deletions(-) diff --git a/tools.go b/tools.go index b345630..3b01f2f 100644 --- a/tools.go +++ b/tools.go @@ -72,7 +72,7 @@ func (t *Tools) ReadJSON(w http.ResponseWriter, r *http.Request, data interface{ return fmt.Errorf("body must not be larger than %d bytes", maxBytes) case errors.As(err, &invalidUnmarshalError): - panic(err) + return fmt.Errorf("error unmarshalling json: %s", err.Error()) default: return err diff --git a/tools_test.go b/tools_test.go index e6070d7..1fe7a1c 100644 --- a/tools_test.go +++ b/tools_test.go @@ -54,56 +54,56 @@ func TestTools_PushJSONToRemote(t *testing.T) { } } -func TestTools_ReadJSON(t *testing.T) { - var testApp Tools - testApp.MaxFileSize = 1048576 * 2 +var jsonTests = []struct { + name string + json string + errorExpected bool + maxSize int +}{ + {name: "good json", json: `{"foo": "bar"}`, errorExpected: false, maxSize: 1024}, + {name: "badly formatted json", json: `{"foo":"}`, errorExpected: true, maxSize: 1024}, + {name: "incorrect type", json: `{"foo": 1}`, errorExpected: true, maxSize: 1024}, + {name: "two json files", json: `{"foo": "bar"}{"alpha": "beta"}`, errorExpected: true, maxSize: 1024}, + {name: "empty body", json: ``, errorExpected: true, maxSize: 1024}, + {name: "syntax error in json", json: `{"foo": 1"}`, errorExpected: true, maxSize: 1024}, + {name: "unknown field in json", json: `{"fooo": "bar"}`, errorExpected: true, maxSize: 1024}, + {name: "missing field name", json: `{jack: "bar"}`, errorExpected: true, maxSize: 1024}, + {name: "file too large", json: `{"foo": "bar"}`, errorExpected: true, maxSize: 5}, + {name: "not json", json: `Hello, world`, errorExpected: true, maxSize: 5}, +} - // create a sample JSON file and add it to body - sampleJSON := map[string]interface{}{ - "foo": "bar", - } - body, _ := json.Marshal(sampleJSON) +func Test_ReadJSON(t *testing.T) { + var testApp Tools - // declare a variable to read the decoded json into - var decodedJSON struct { - Foo string `json:"foo"` - } + for _, e := range jsonTests { + // set max file size + testApp.MaxJSONSize = e.maxSize - // create a request with the body - req, err := http.NewRequest("POST", "/", bytes.NewReader(body)) - if err != nil { - t.Log("Error", err) - } + // declare a variable to read the decoded json into + var decodedJSON struct { + Foo string `json:"foo"` + } - // create a test response recorder, which satisfies the requirements - // for a ResponseWriter - rr := httptest.NewRecorder() - defer req.Body.Close() + // create a request with the body + req, err := http.NewRequest("POST", "/", bytes.NewReader([]byte(e.json))) + if err != nil { + t.Log("Error", err) + } - // call readJSON and check for an error - err = testApp.ReadJSON(rr, req, &decodedJSON) - if err != nil { - t.Error("failed to decode json", err) - } + // create a test response recorder, which satisfies the requirements + // for a ResponseWriter + rr := httptest.NewRecorder() - // create json with two json entries - badJSON := ` - { - "foo": "bar" + // call readJSON and check for an error + err = testApp.ReadJSON(rr, req, &decodedJSON) + if e.errorExpected && err == nil { + t.Errorf("%s: error expected, but none received", e.name) } - { - "alpha": "beta" - }` - // create a request with the body - req, err = http.NewRequest("POST", "/", bytes.NewReader([]byte(badJSON))) - if err != nil { - t.Log("Error", err) - } - - err = testApp.ReadJSON(rr, req, &decodedJSON) - if err == nil { - t.Error("did not get an error with bad json") + if !e.errorExpected && err != nil { + t.Errorf("%s: error not expected, but one received: %s \n%s", e.name, err.Error(), e.json) + } + req.Body.Close() } } From bb970203b3083d5d64e5c64d64fb1e5a68ac8282 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Tue, 5 Jul 2022 12:39:35 -0300 Subject: [PATCH 026/135] add table test --- tools_test.go | 100 ++++++++++++++++++++++++++++++-------------------- 1 file changed, 61 insertions(+), 39 deletions(-) diff --git a/tools_test.go b/tools_test.go index 1fe7a1c..98d2eb7 100644 --- a/tools_test.go +++ b/tools_test.go @@ -13,6 +13,7 @@ import ( "net/http" "net/http/httptest" "os" + "sync" "testing" ) @@ -185,53 +186,74 @@ func TestTools_DownloadStaticFile(t *testing.T) { } } +var uploadTests = []struct { + name string + allowedTypes []string + errorExpected bool +}{ + {name: "allowed", allowedTypes: []string{"image/jpeg", "image/png"}, errorExpected: false}, + {name: "not allowed", allowedTypes: []string{"image/jpeg"}, errorExpected: true}, +} + func TestTools_UploadOneFile(t *testing.T) { - // set up a pipe to avoid buffering - pr, pw := io.Pipe() - writer := multipart.NewWriter(pw) - - go func() { - defer writer.Close() - // create the form data field 'fileupload' - part, err := writer.CreateFormFile("file", "./testdata/img.png") - if err != nil { + for _, e := range uploadTests { + // set up a pipe to avoid buffering + pr, pw := io.Pipe() + writer := multipart.NewWriter(pw) + wg := sync.WaitGroup{} + wg.Add(1) + go func() { + defer writer.Close() + defer wg.Done() + + // create the form data field 'fileupload' + part, err := writer.CreateFormFile("file", "./testdata/img.png") + if err != nil { + t.Error(err) + } + + f, err := os.Open("./testdata/img.png") + if err != nil { + t.Error(err) + } + defer f.Close() + img, _, err := image.Decode(f) + if err != nil { + t.Error("error decoding image", err) + } + + err = png.Encode(part, img) + if err != nil { + t.Error(err) + } + }() + + // read from the pipe which receives data + request := httptest.NewRequest("POST", "/", pr) + request.Header.Add("Content-Type", writer.FormDataContentType()) + + var testTools Tools + testTools.AllowedFileTypes = e.allowedTypes + + uploadedFile, err := testTools.UploadOneFile(request, "./testdata/uploads/") + if err != nil && !e.errorExpected { t.Error(err) } - f, err := os.Open("./testdata/img.png") - if err != nil { - t.Error(err) - } - defer f.Close() - img, _, err := image.Decode(f) - if err != nil { - t.Error("error decoding image", err) - } + if !e.errorExpected { + if _, err := os.Stat(fmt.Sprintf("./testdata/uploads/%s", uploadedFile.NewFileName)); os.IsNotExist(err) { + t.Errorf("%s: expected file to exist: %s", e.name, err.Error()) + } - err = png.Encode(part, img) - if err != nil { - t.Error(err) + // clean up + _ = os.Remove(fmt.Sprintf("./testdata/uploads/%s", uploadedFile.NewFileName)) } - }() - - // read from the pipe which receives data - request := httptest.NewRequest("POST", "/", pr) - request.Header.Add("Content-Type", writer.FormDataContentType()) - - var testTools Tools - testTools.AllowedFileTypes = []string{"image/png"} - uploadedFile, err := testTools.UploadOneFile(request, "./testdata/uploads/") - if err != nil { - t.Error(err) - } - - if _, err := os.Stat(fmt.Sprintf("./testdata/uploads/%s", uploadedFile.NewFileName)); os.IsNotExist(err) { - t.Error("Expected file to exist", err) + if !e.errorExpected && err != nil { + t.Errorf("%s: error expected, but none received", e.name) + } + wg.Wait() } - - // clean up - _ = os.Remove(fmt.Sprintf("./testdata/uploads/%s", uploadedFile.NewFileName)) } func TestTools_CreateDirIfNotExist(t *testing.T) { From 3dcfffa1cc7e53646107c311a3d56e9505359313 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Tue, 5 Jul 2022 12:41:34 -0300 Subject: [PATCH 027/135] add table test --- tools_test.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tools_test.go b/tools_test.go index 98d2eb7..68de077 100644 --- a/tools_test.go +++ b/tools_test.go @@ -189,9 +189,11 @@ func TestTools_DownloadStaticFile(t *testing.T) { var uploadTests = []struct { name string allowedTypes []string + renameFile bool errorExpected bool }{ - {name: "allowed", allowedTypes: []string{"image/jpeg", "image/png"}, errorExpected: false}, + {name: "allowed", allowedTypes: []string{"image/jpeg", "image/png"}, renameFile: false, errorExpected: false}, + {name: "allowed", allowedTypes: []string{"image/jpeg", "image/png"}, renameFile: true, errorExpected: false}, {name: "not allowed", allowedTypes: []string{"image/jpeg"}, errorExpected: true}, } @@ -235,7 +237,7 @@ func TestTools_UploadOneFile(t *testing.T) { var testTools Tools testTools.AllowedFileTypes = e.allowedTypes - uploadedFile, err := testTools.UploadOneFile(request, "./testdata/uploads/") + uploadedFile, err := testTools.UploadOneFile(request, "./testdata/uploads/", e.renameFile) if err != nil && !e.errorExpected { t.Error(err) } From 476b596e90def4a8a224fbd83189bed6d43ecdee Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Tue, 5 Jul 2022 12:43:09 -0300 Subject: [PATCH 028/135] tests --- tools_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools_test.go b/tools_test.go index 68de077..de15217 100644 --- a/tools_test.go +++ b/tools_test.go @@ -70,7 +70,7 @@ var jsonTests = []struct { {name: "unknown field in json", json: `{"fooo": "bar"}`, errorExpected: true, maxSize: 1024}, {name: "missing field name", json: `{jack: "bar"}`, errorExpected: true, maxSize: 1024}, {name: "file too large", json: `{"foo": "bar"}`, errorExpected: true, maxSize: 5}, - {name: "not json", json: `Hello, world`, errorExpected: true, maxSize: 5}, + {name: "not json", json: `Hello, world`, errorExpected: true, maxSize: 1024}, } func Test_ReadJSON(t *testing.T) { From 41d06259050cdfc2da92b3b291f0f97168b01c77 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Tue, 5 Jul 2022 12:44:02 -0300 Subject: [PATCH 029/135] tests --- tools_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools_test.go b/tools_test.go index de15217..33630a2 100644 --- a/tools_test.go +++ b/tools_test.go @@ -97,10 +97,13 @@ func Test_ReadJSON(t *testing.T) { // call readJSON and check for an error err = testApp.ReadJSON(rr, req, &decodedJSON) + + // if we expect an error, but do not get one, something went wrong if e.errorExpected && err == nil { t.Errorf("%s: error expected, but none received", e.name) } + // if we do not expect an error, but get one, something went wrong if !e.errorExpected && err != nil { t.Errorf("%s: error not expected, but one received: %s \n%s", e.name, err.Error(), e.json) } From ea3ae206f2e9271bb248a8ed38be9b26a4450cb1 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Tue, 5 Jul 2022 12:44:43 -0300 Subject: [PATCH 030/135] tests --- tools_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools_test.go b/tools_test.go index 33630a2..674e707 100644 --- a/tools_test.go +++ b/tools_test.go @@ -195,8 +195,8 @@ var uploadTests = []struct { renameFile bool errorExpected bool }{ - {name: "allowed", allowedTypes: []string{"image/jpeg", "image/png"}, renameFile: false, errorExpected: false}, - {name: "allowed", allowedTypes: []string{"image/jpeg", "image/png"}, renameFile: true, errorExpected: false}, + {name: "allowed no rename", allowedTypes: []string{"image/jpeg", "image/png"}, renameFile: false, errorExpected: false}, + {name: "allowed rename", allowedTypes: []string{"image/jpeg", "image/png"}, renameFile: true, errorExpected: false}, {name: "not allowed", allowedTypes: []string{"image/jpeg"}, errorExpected: true}, } From 8c0ae561581a9e4c5955da1a5731efe764fdcb79 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Tue, 5 Jul 2022 12:45:10 -0300 Subject: [PATCH 031/135] tests --- tools_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools_test.go b/tools_test.go index 674e707..40483c5 100644 --- a/tools_test.go +++ b/tools_test.go @@ -211,7 +211,7 @@ func TestTools_UploadOneFile(t *testing.T) { defer writer.Close() defer wg.Done() - // create the form data field 'fileupload' + // create the form data field 'file' part, err := writer.CreateFormFile("file", "./testdata/img.png") if err != nil { t.Error(err) From 65954c47302295a16213fdae7315577c38b6b6b6 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Tue, 5 Jul 2022 12:45:40 -0300 Subject: [PATCH 032/135] tests --- tools_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools_test.go b/tools_test.go index 40483c5..4e7a433 100644 --- a/tools_test.go +++ b/tools_test.go @@ -257,6 +257,8 @@ func TestTools_UploadOneFile(t *testing.T) { if !e.errorExpected && err != nil { t.Errorf("%s: error expected, but none received", e.name) } + + // we're running table tests, so have to use a waitgroup wg.Wait() } } From 4a4641fa5f83d0d7560e802db77c9dfffda9e112 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Tue, 5 Jul 2022 12:53:47 -0300 Subject: [PATCH 033/135] update readme --- readme.md | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/readme.md b/readme.md index 8e77bb9..7772ab2 100644 --- a/readme.md +++ b/readme.md @@ -121,15 +121,31 @@ To upload a file to a specific directory, with this for HTML: + Upload test +
+
+
+

Upload a file

+
-
- - -
+
+
+ + +
+ + + +
+ +
+
+
``` From eccebefdcbd2065d0dd43ec769b6a565b6271da9 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Wed, 6 Jul 2022 11:30:10 -0300 Subject: [PATCH 034/135] Permit more than one file in upload --- tools.go | 18 +++++++++++------- tools_test.go | 8 ++++---- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/tools.go b/tools.go index 3b01f2f..2701a10 100644 --- a/tools.go +++ b/tools.go @@ -179,17 +179,19 @@ type UploadedFile struct { FileSize int64 } -// UploadOneFile uploads one file to a specified directory, and gives it a random name. -// It returns the newly named file, the original file name, and potentially an error. -// If the optional last parameter is set to true, then we will not rename the file, but -// will use the original file name. -func (t *Tools) UploadOneFile(r *http.Request, uploadDir string, rename ...bool) (*UploadedFile, error) { - // check to see if we are renaming the file with the optional last parameter +// UploadFiles uploads one or more file to a specified directory, and gives the files a random name. +// It returns a slice containing the newly named files, the original file names, the size of the files, +// and potentially an error. If the optional last parameter is set to true, then we will not rename +// the files, but will use the original file names. +func (t *Tools) UploadFiles(r *http.Request, uploadDir string, rename ...bool) ([]*UploadedFile, error) { + // check to see if we are renaming the uploadedFiles with the optional last parameter renameFile := true if len(rename) > 0 { renameFile = rename[0] } + var uploadedFiles []*UploadedFile + // parse the form so we have access to the file err := r.ParseMultipartForm(int64(t.MaxFileSize)) if err != nil { @@ -252,10 +254,12 @@ func (t *Tools) UploadOneFile(r *http.Request, uploadDir string, rename ...bool) } uploadedFile.FileSize = fileSize } + + uploadedFiles = append(uploadedFiles, &uploadedFile) } } - return &uploadedFile, nil + return uploadedFiles, nil } // CreateDirIfNotExist creates a directory, and all necessary parent directories, if it does not exist. diff --git a/tools_test.go b/tools_test.go index 4e7a433..c2cef70 100644 --- a/tools_test.go +++ b/tools_test.go @@ -200,7 +200,7 @@ var uploadTests = []struct { {name: "not allowed", allowedTypes: []string{"image/jpeg"}, errorExpected: true}, } -func TestTools_UploadOneFile(t *testing.T) { +func TestTools_UploadFiles(t *testing.T) { for _, e := range uploadTests { // set up a pipe to avoid buffering pr, pw := io.Pipe() @@ -240,18 +240,18 @@ func TestTools_UploadOneFile(t *testing.T) { var testTools Tools testTools.AllowedFileTypes = e.allowedTypes - uploadedFile, err := testTools.UploadOneFile(request, "./testdata/uploads/", e.renameFile) + uploadedFiles, err := testTools.UploadFiles(request, "./testdata/uploads/", e.renameFile) if err != nil && !e.errorExpected { t.Error(err) } if !e.errorExpected { - if _, err := os.Stat(fmt.Sprintf("./testdata/uploads/%s", uploadedFile.NewFileName)); os.IsNotExist(err) { + if _, err := os.Stat(fmt.Sprintf("./testdata/uploads/%s", uploadedFiles[0].NewFileName)); os.IsNotExist(err) { t.Errorf("%s: expected file to exist: %s", e.name, err.Error()) } // clean up - _ = os.Remove(fmt.Sprintf("./testdata/uploads/%s", uploadedFile.NewFileName)) + _ = os.Remove(fmt.Sprintf("./testdata/uploads/%s", uploadedFiles[0].NewFileName)) } if !e.errorExpected && err != nil { From 2e4941c807cad938c34bb1cad0d47782ad6da1ac Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Wed, 6 Jul 2022 11:38:31 -0300 Subject: [PATCH 035/135] Update readme --- readme.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/readme.md b/readme.md index 7772ab2..b918a35 100644 --- a/readme.md +++ b/readme.md @@ -180,18 +180,18 @@ func main() { _ = t.CreateDirIfNotExist("./uploads") - // upload the file. Note that if you don't want the file to be renamed, - // you can add an optional final parameter -- true will rename the file (the default) - // and false will preserve the original filename, for example: - // u, err := t.UploadOneFile(r, "./uploads", false) - u, err := t.UploadOneFile(r, "./uploads") + // Upload the file(s). Note that if you don't want the files to be renamed, + // you can add an optional final parameter -- true will rename the files (the default) + // and false will preserve the original filenames, for example: + // files, err := t.UploadFiles(r, "./uploads", false) + files, err := t.UploadFiles(r, "./uploads") if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } - // the returned variable, u, will have the type toolbox.UploadedFile - w.Write([]byte(fmt.Sprintf("New file name: %s, Original file name: %s, size: %d", u.NewFileName, u.OriginalFileName, u.FileSize))) + // the returned variable, files, will bea slice of the type toolbox.UploadedFile + w.Write([]byte(fmt.Sprintf("New file name: %s, Original file name: %s, size: %d", files[0].NewFileName, files[0].OriginalFileName, files[0].FileSize))) }) // print a log message From 08e0d4aed5a166258bbbb1cff3f3ff586bde750b Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Wed, 6 Jul 2022 11:41:29 -0300 Subject: [PATCH 036/135] Typo in readme --- readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme.md b/readme.md index b918a35..2c6a51d 100644 --- a/readme.md +++ b/readme.md @@ -190,7 +190,7 @@ func main() { return } - // the returned variable, files, will bea slice of the type toolbox.UploadedFile + // the returned variable, files, will be a slice of the type toolbox.UploadedFile w.Write([]byte(fmt.Sprintf("New file name: %s, Original file name: %s, size: %d", files[0].NewFileName, files[0].OriginalFileName, files[0].FileSize))) }) From 9124c27b9a462e20df6964e2310a60673df0e9dd Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Wed, 6 Jul 2022 11:49:05 -0300 Subject: [PATCH 037/135] Update readme --- readme.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/readme.md b/readme.md index 2c6a51d..55d32ee 100644 --- a/readme.md +++ b/readme.md @@ -136,7 +136,7 @@ To upload a file to a specific directory, with this for HTML:
- +
@@ -191,8 +191,7 @@ func main() { } // the returned variable, files, will be a slice of the type toolbox.UploadedFile - w.Write([]byte(fmt.Sprintf("New file name: %s, Original file name: %s, size: %d", files[0].NewFileName, files[0].OriginalFileName, files[0].FileSize))) - }) + _, _ = w.Write([]byte(fmt.Sprintf("Uploaded %d file(s) to the uploads folder", len(files)))) }) // print a log message log.Println("Starting server on port 8080") From a61f0dc37276b3ade2a1f76660ffc1eec19391b0 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Wed, 6 Jul 2022 11:50:20 -0300 Subject: [PATCH 038/135] readme --- readme.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/readme.md b/readme.md index 55d32ee..47b3a8f 100644 --- a/readme.md +++ b/readme.md @@ -191,7 +191,8 @@ func main() { } // the returned variable, files, will be a slice of the type toolbox.UploadedFile - _, _ = w.Write([]byte(fmt.Sprintf("Uploaded %d file(s) to the uploads folder", len(files)))) }) + _, _ = w.Write([]byte(fmt.Sprintf("Uploaded %d file(s) to the uploads folder", len(files)))) + }) // print a log message log.Println("Starting server on port 8080") From 752c6c586333c98251bb7ee85e18466181fc0436 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Wed, 6 Jul 2022 11:58:48 -0300 Subject: [PATCH 039/135] Add convenience method --- tools.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tools.go b/tools.go index 2701a10..febb88e 100644 --- a/tools.go +++ b/tools.go @@ -179,6 +179,22 @@ type UploadedFile struct { FileSize int64 } +// UploadOneFile is just a convenience method that calls UploadFiles, but expects only one file to +// be in the upload. +func (t *Tools) UploadOneFile(r *http.Request, uploadDir string, rename ...bool) (*UploadedFile, error) { + renameFile := true + if len(rename) > 0 { + renameFile = rename[0] + } + + files, err := t.UploadFiles(r, uploadDir, renameFile) + if err != nil { + return nil, err + } + + return files[0], nil +} + // UploadFiles uploads one or more file to a specified directory, and gives the files a random name. // It returns a slice containing the newly named files, the original file names, the size of the files, // and potentially an error. If the optional last parameter is set to true, then we will not rename From 6736fcef3d73a10b69c1e0d41405a8d8e9a7cf35 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Wed, 6 Jul 2022 12:03:34 -0300 Subject: [PATCH 040/135] update tests --- tools_test.go | 51 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/tools_test.go b/tools_test.go index c2cef70..642ed7a 100644 --- a/tools_test.go +++ b/tools_test.go @@ -263,6 +263,57 @@ func TestTools_UploadFiles(t *testing.T) { } } +func TestTools_UploadOneFile(t *testing.T) { + // set up a pipe to avoid buffering + pr, pw := io.Pipe() + writer := multipart.NewWriter(pw) + + go func() { + defer writer.Close() + + // create the form data field 'file' + part, err := writer.CreateFormFile("file", "./testdata/img.png") + if err != nil { + t.Error(err) + } + + f, err := os.Open("./testdata/img.png") + if err != nil { + t.Error(err) + } + defer f.Close() + img, _, err := image.Decode(f) + if err != nil { + t.Error("error decoding image", err) + } + + err = png.Encode(part, img) + if err != nil { + t.Error(err) + } + }() + + // read from the pipe which receives data + request := httptest.NewRequest("POST", "/", pr) + request.Header.Add("Content-Type", writer.FormDataContentType()) + + var testTools Tools + testTools.AllowedFileTypes = []string{"image/png"} + + uploadedFiles, err := testTools.UploadOneFile(request, "./testdata/uploads/", true) + if err != nil { + t.Error(err) + } + + if _, err := os.Stat(fmt.Sprintf("./testdata/uploads/%s", uploadedFiles.NewFileName)); os.IsNotExist(err) { + t.Errorf("expected file to exist: %s", err.Error()) + } + + // clean up + _ = os.Remove(fmt.Sprintf("./testdata/uploads/%s", uploadedFiles.NewFileName)) + +} + func TestTools_CreateDirIfNotExist(t *testing.T) { var testTool Tools From 4f44d3e1337c2a12a54989a2903b39cd46f0cd73 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Wed, 6 Jul 2022 14:45:36 -0300 Subject: [PATCH 041/135] create the directory when uploading files --- tools.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tools.go b/tools.go index febb88e..9d51cad 100644 --- a/tools.go +++ b/tools.go @@ -208,8 +208,14 @@ func (t *Tools) UploadFiles(r *http.Request, uploadDir string, rename ...bool) ( var uploadedFiles []*UploadedFile + // create the upload directory if it does not exist + err := t.CreateDirIfNotExist(uploadDir) + if err != nil { + return nil, err + } + // parse the form so we have access to the file - err := r.ParseMultipartForm(int64(t.MaxFileSize)) + err = r.ParseMultipartForm(int64(t.MaxFileSize)) if err != nil { return nil, errors.New("the uploaded file is too big") } From 9821826a84a02bc34a4774e7b55a7282d297391b Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Wed, 6 Jul 2022 14:46:25 -0300 Subject: [PATCH 042/135] Update readme --- readme.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/readme.md b/readme.md index 47b3a8f..fcbf698 100644 --- a/readme.md +++ b/readme.md @@ -177,13 +177,13 @@ func main() { MaxFileSize: 1024 * 1024 * 1024, AllowedFileTypes: []string{"image/gif", "image/png", "image/jpeg"}, } - - _ = t.CreateDirIfNotExist("./uploads") + // Upload the file(s). Note that if you don't want the files to be renamed, // you can add an optional final parameter -- true will rename the files (the default) // and false will preserve the original filenames, for example: // files, err := t.UploadFiles(r, "./uploads", false) + // n.b.: if the "./uploads" directory does not exist, we attemp to create it. files, err := t.UploadFiles(r, "./uploads") if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) From ebcdae638d7c3e10fc3005056442a6528adf631a Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Wed, 6 Jul 2022 14:50:32 -0300 Subject: [PATCH 043/135] add TODO item --- tools.go | 1 + 1 file changed, 1 insertion(+) diff --git a/tools.go b/tools.go index 9d51cad..438f7d2 100644 --- a/tools.go +++ b/tools.go @@ -139,6 +139,7 @@ func (t *Tools) RandomString(n int) string { // PushJSONToRemote posts arbitrary json to some url, and returns error, // if any, as well as the response status code +// TODO: capture returned JSON and send it back to the client func (t *Tools) PushJSONToRemote(client *http.Client, uri string, data interface{}) (int, error) { // create json we'll send jsonData, err := json.Marshal(data) From 01b5473c1b31b74bfacab57a4cf2d2bfdd04a1b8 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Wed, 6 Jul 2022 14:55:48 -0300 Subject: [PATCH 044/135] Return response in PushJSONToRemote --- tools.go | 15 +++++++-------- tools_test.go | 2 +- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/tools.go b/tools.go index 438f7d2..c244cae 100644 --- a/tools.go +++ b/tools.go @@ -137,31 +137,30 @@ func (t *Tools) RandomString(n int) string { return string(s) } -// PushJSONToRemote posts arbitrary json to some url, and returns error, -// if any, as well as the response status code -// TODO: capture returned JSON and send it back to the client -func (t *Tools) PushJSONToRemote(client *http.Client, uri string, data interface{}) (int, error) { +// PushJSONToRemote posts arbitrary json to some url, and returns the response, the response +// status code, and error, if any +func (t *Tools) PushJSONToRemote(client *http.Client, uri string, data interface{}) (*http.Response, int, error) { // create json we'll send jsonData, err := json.Marshal(data) if err != nil { - return 0, err + return nil, 0, err } // build the request and set header request, err := http.NewRequest("POST", uri, bytes.NewBuffer(jsonData)) if err != nil { - return 0, err + return nil, 0, err } request.Header.Set("Content-Type", "application/json") // call the uri response, err := client.Do(request) if err != nil { - return 0, err + return nil, 0, err } defer response.Body.Close() - return response.StatusCode, nil + return response, response.StatusCode, nil } // DownloadStaticFile downloads a file, and tries to force the browser to avoid displaying it in diff --git a/tools_test.go b/tools_test.go index 642ed7a..ad49bca 100644 --- a/tools_test.go +++ b/tools_test.go @@ -49,7 +49,7 @@ func TestTools_PushJSONToRemote(t *testing.T) { Bar string `json:"bar"` } foo.Bar = "bar" - _, err := testApp.PushJSONToRemote(client, "http://example.com/some/path", foo) + _, _, err := testApp.PushJSONToRemote(client, "http://example.com/some/path", foo) if err != nil { t.Error("failed to call remote url", err) } From 834afde3a25d37bac3532d86cfb446c806d46610 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Wed, 6 Jul 2022 15:02:44 -0300 Subject: [PATCH 045/135] comments --- tools.go | 1 + 1 file changed, 1 insertion(+) diff --git a/tools.go b/tools.go index c244cae..0213bf9 100644 --- a/tools.go +++ b/tools.go @@ -115,6 +115,7 @@ func (t *Tools) WriteJSON(w http.ResponseWriter, status int, data interface{}, h func (t *Tools) ErrorJSON(w http.ResponseWriter, err error, status ...int) error { statusCode := http.StatusBadRequest + // if a custom response code is specified, use that instead of bad request if len(status) > 0 { statusCode = status[0] } From 73ac1ccd74fa29a64646e07d378f0e1d9cc0a905 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Wed, 6 Jul 2022 15:09:28 -0300 Subject: [PATCH 046/135] wrap for loop contents in func to take care of deferring things --- tools.go | 93 +++++++++++++++++++++++++++++--------------------------- 1 file changed, 49 insertions(+), 44 deletions(-) diff --git a/tools.go b/tools.go index 0213bf9..b797d3b 100644 --- a/tools.go +++ b/tools.go @@ -224,63 +224,68 @@ func (t *Tools) UploadFiles(r *http.Request, uploadDir string, rename ...bool) ( for _, fHeaders := range r.MultipartForm.File { for _, hdr := range fHeaders { - infile, err := hdr.Open() - if err != nil { - return nil, err - } - defer infile.Close() + uploadedFiles, err = func() ([]*UploadedFile, error) { + infile, err := hdr.Open() + if err != nil { + return nil, err + } + defer infile.Close() - buff := make([]byte, 512) - _, err = infile.Read(buff) - if err != nil { - return nil, err - } + buff := make([]byte, 512) + _, err = infile.Read(buff) + if err != nil { + return nil, err + } - allowed := false - filetype := http.DetectContentType(buff) - if len(t.AllowedFileTypes) > 0 { - for _, x := range t.AllowedFileTypes { - if strings.EqualFold(filetype, x) { - allowed = true + allowed := false + filetype := http.DetectContentType(buff) + if len(t.AllowedFileTypes) > 0 { + for _, x := range t.AllowedFileTypes { + if strings.EqualFold(filetype, x) { + allowed = true + } } + } else { + allowed = true } - } else { - allowed = true - } - if !allowed { - return nil, errors.New("the uploaded file type is not permitted") - } + if !allowed { + return nil, errors.New("the uploaded file type is not permitted") + } - _, err = infile.Seek(0, 0) - if err != nil { - fmt.Println(err) - return nil, err - } + _, err = infile.Seek(0, 0) + if err != nil { + fmt.Println(err) + return nil, err + } - if renameFile { - uploadedFile.NewFileName = t.RandomString(25) + filepath.Ext(hdr.Filename) - } else { - uploadedFile.NewFileName = hdr.Filename - } - uploadedFile.OriginalFileName = hdr.Filename + if renameFile { + uploadedFile.NewFileName = t.RandomString(25) + filepath.Ext(hdr.Filename) + } else { + uploadedFile.NewFileName = hdr.Filename + } + uploadedFile.OriginalFileName = hdr.Filename - var outfile *os.File - defer outfile.Close() + var outfile *os.File + defer outfile.Close() - if outfile, err = os.Create(filepath.Join(uploadDir, uploadedFile.NewFileName)); nil != err { - return nil, err - } else { - fileSize, err := io.Copy(outfile, infile) - if err != nil { + if outfile, err = os.Create(filepath.Join(uploadDir, uploadedFile.NewFileName)); nil != err { return nil, err + } else { + fileSize, err := io.Copy(outfile, infile) + if err != nil { + return nil, err + } + uploadedFile.FileSize = fileSize } - uploadedFile.FileSize = fileSize - } - uploadedFiles = append(uploadedFiles, &uploadedFile) + uploadedFiles = append(uploadedFiles, &uploadedFile) + return uploadedFiles, nil + }() + if err != nil { + return uploadedFiles, err + } } - } return uploadedFiles, nil } From 26f0e639eb0a0e0abb3bb9f4f7339e8422298f62 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Wed, 6 Jul 2022 16:40:48 -0300 Subject: [PATCH 047/135] typo --- readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme.md b/readme.md index fcbf698..994a8a2 100644 --- a/readme.md +++ b/readme.md @@ -183,7 +183,7 @@ func main() { // you can add an optional final parameter -- true will rename the files (the default) // and false will preserve the original filenames, for example: // files, err := t.UploadFiles(r, "./uploads", false) - // n.b.: if the "./uploads" directory does not exist, we attemp to create it. + // n.b.: if the "./uploads" directory does not exist, we attempt to create it. files, err := t.UploadFiles(r, "./uploads") if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) From f20f0500e398fe2e4405dd31328d2f2eb90ddf1f Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Wed, 6 Jul 2022 16:47:00 -0300 Subject: [PATCH 048/135] allow unknown fields in JSON if desired --- tools.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tools.go b/tools.go index b797d3b..cd8f6ac 100644 --- a/tools.go +++ b/tools.go @@ -19,9 +19,10 @@ const randomStringSource = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ // Tools is the type for this package. Create a variable of this type and you have access // to all the methods with the receiver type *Tools. type Tools struct { - MaxJSONSize int // maximum siz of JSON file we'll process - MaxFileSize int // maximum size of uploaded files in bytes - AllowedFileTypes []string // allowed file types for upload (e.g. image/jpeg) + MaxJSONSize int // maximum siz of JSON file we'll process + MaxFileSize int // maximum size of uploaded files in bytes + AllowedFileTypes []string // allowed file types for upload (e.g. image/jpeg) + AllowUnknownFields bool // if set to true, we allow unknown fields in JSON } // JSONResponse is the type used for sending JSON around @@ -40,7 +41,11 @@ func (t *Tools) ReadJSON(w http.ResponseWriter, r *http.Request, data interface{ r.Body = http.MaxBytesReader(w, r.Body, int64(maxBytes)) dec := json.NewDecoder(r.Body) - dec.DisallowUnknownFields() + + // should we allow unknown fields? + if !t.AllowUnknownFields { + dec.DisallowUnknownFields() + } err := dec.Decode(data) if err != nil { From 7e61c5350155f3a796b738e055eceb8fd4dc39cd Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Wed, 6 Jul 2022 16:50:18 -0300 Subject: [PATCH 049/135] allow unknown fields in JSON if desired --- tools_test.go | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/tools_test.go b/tools_test.go index ad49bca..5d25a16 100644 --- a/tools_test.go +++ b/tools_test.go @@ -60,17 +60,19 @@ var jsonTests = []struct { json string errorExpected bool maxSize int + allowUnknown bool }{ - {name: "good json", json: `{"foo": "bar"}`, errorExpected: false, maxSize: 1024}, - {name: "badly formatted json", json: `{"foo":"}`, errorExpected: true, maxSize: 1024}, - {name: "incorrect type", json: `{"foo": 1}`, errorExpected: true, maxSize: 1024}, - {name: "two json files", json: `{"foo": "bar"}{"alpha": "beta"}`, errorExpected: true, maxSize: 1024}, - {name: "empty body", json: ``, errorExpected: true, maxSize: 1024}, - {name: "syntax error in json", json: `{"foo": 1"}`, errorExpected: true, maxSize: 1024}, - {name: "unknown field in json", json: `{"fooo": "bar"}`, errorExpected: true, maxSize: 1024}, - {name: "missing field name", json: `{jack: "bar"}`, errorExpected: true, maxSize: 1024}, - {name: "file too large", json: `{"foo": "bar"}`, errorExpected: true, maxSize: 5}, - {name: "not json", json: `Hello, world`, errorExpected: true, maxSize: 1024}, + {name: "good json", json: `{"foo": "bar"}`, errorExpected: false, maxSize: 1024, allowUnknown: false}, + {name: "badly formatted json", json: `{"foo":"}`, errorExpected: true, maxSize: 1024, allowUnknown: false}, + {name: "incorrect type", json: `{"foo": 1}`, errorExpected: true, maxSize: 1024, allowUnknown: false}, + {name: "two json files", json: `{"foo": "bar"}{"alpha": "beta"}`, errorExpected: true, maxSize: 1024, allowUnknown: false}, + {name: "empty body", json: ``, errorExpected: true, maxSize: 1024, allowUnknown: false}, + {name: "syntax error in json", json: `{"foo": 1"}`, errorExpected: true, maxSize: 1024, allowUnknown: false}, + {name: "unknown field in json", json: `{"fooo": "bar"}`, errorExpected: true, maxSize: 1024, allowUnknown: false}, + {name: "allow unknown field in json", json: `{"fooo": "bar"}`, errorExpected: false, maxSize: 1024, allowUnknown: true}, + {name: "missing field name", json: `{jack: "bar"}`, errorExpected: true, maxSize: 1024, allowUnknown: false}, + {name: "file too large", json: `{"foo": "bar"}`, errorExpected: true, maxSize: 5, allowUnknown: false}, + {name: "not json", json: `Hello, world`, errorExpected: true, maxSize: 1024, allowUnknown: false}, } func Test_ReadJSON(t *testing.T) { @@ -79,6 +81,8 @@ func Test_ReadJSON(t *testing.T) { for _, e := range jsonTests { // set max file size testApp.MaxJSONSize = e.maxSize + // allow/disallow unknown fields + testApp.AllowUnknownFields = e.allowUnknown // declare a variable to read the decoded json into var decodedJSON struct { From 96aff435cd0bb68aebe9059a3ba0cfc994d5836e Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Wed, 6 Jul 2022 16:53:56 -0300 Subject: [PATCH 050/135] formatting --- tools_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/tools_test.go b/tools_test.go index 5d25a16..108d0f0 100644 --- a/tools_test.go +++ b/tools_test.go @@ -81,6 +81,7 @@ func Test_ReadJSON(t *testing.T) { for _, e := range jsonTests { // set max file size testApp.MaxJSONSize = e.maxSize + // allow/disallow unknown fields testApp.AllowUnknownFields = e.allowUnknown From 837375f3cb65a2cbd05da50d0484c63b40a357aa Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Wed, 6 Jul 2022 16:54:46 -0300 Subject: [PATCH 051/135] rename testApp -> testTools (which makes more sense) --- tools_test.go | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/tools_test.go b/tools_test.go index 108d0f0..05eeeab 100644 --- a/tools_test.go +++ b/tools_test.go @@ -44,12 +44,12 @@ func TestTools_PushJSONToRemote(t *testing.T) { } }) - var testApp Tools + var testTools Tools var foo struct { Bar string `json:"bar"` } foo.Bar = "bar" - _, _, err := testApp.PushJSONToRemote(client, "http://example.com/some/path", foo) + _, _, err := testTools.PushJSONToRemote(client, "http://example.com/some/path", foo) if err != nil { t.Error("failed to call remote url", err) } @@ -76,14 +76,14 @@ var jsonTests = []struct { } func Test_ReadJSON(t *testing.T) { - var testApp Tools + var testTools Tools for _, e := range jsonTests { // set max file size - testApp.MaxJSONSize = e.maxSize + testTools.MaxJSONSize = e.maxSize // allow/disallow unknown fields - testApp.AllowUnknownFields = e.allowUnknown + testTools.AllowUnknownFields = e.allowUnknown // declare a variable to read the decoded json into var decodedJSON struct { @@ -101,7 +101,7 @@ func Test_ReadJSON(t *testing.T) { rr := httptest.NewRecorder() // call readJSON and check for an error - err = testApp.ReadJSON(rr, req, &decodedJSON) + err = testTools.ReadJSON(rr, req, &decodedJSON) // if we expect an error, but do not get one, something went wrong if e.errorExpected && err == nil { @@ -117,7 +117,7 @@ func Test_ReadJSON(t *testing.T) { } func TestTools_WriteJSON(t *testing.T) { - var testApp Tools + var testTools Tools rr := httptest.NewRecorder() payload := JSONResponse{ @@ -127,17 +127,17 @@ func TestTools_WriteJSON(t *testing.T) { headers := make(http.Header) headers.Add("FOO", "BAR") - err := testApp.WriteJSON(rr, http.StatusOK, payload, headers) + err := testTools.WriteJSON(rr, http.StatusOK, payload, headers) if err != nil { t.Errorf("failed to write JSON: %v", err) } } func TestTools_ErrorJSON(t *testing.T) { - var testApp Tools + var testTools Tools rr := httptest.NewRecorder() - err := testApp.ErrorJSON(rr, errors.New("some error")) + err := testTools.ErrorJSON(rr, errors.New("some error")) if err != nil { t.Error(err) } @@ -154,16 +154,16 @@ func TestTools_ErrorJSON(t *testing.T) { } // test with status - err = testApp.ErrorJSON(rr, errors.New("another error"), http.StatusServiceUnavailable) + err = testTools.ErrorJSON(rr, errors.New("another error"), http.StatusServiceUnavailable) if err != nil { t.Error(err) } } func TestTools_RandomString(t *testing.T) { - var testApp Tools + var testTools Tools - s := testApp.RandomString(10) + s := testTools.RandomString(10) if len(s) != 10 { t.Error("wrong length random string returned") } @@ -173,9 +173,9 @@ func TestTools_DownloadStaticFile(t *testing.T) { rr := httptest.NewRecorder() req, _ := http.NewRequest("GET", "/", nil) - var testApp Tools + var testTools Tools - testApp.DownloadStaticFile(rr, req, "./testdata", "tgg.jpg", "gatsby.jpg") + testTools.DownloadStaticFile(rr, req, "./testdata", "tgg.jpg", "gatsby.jpg") res := rr.Result() defer res.Body.Close() From 85ac2adee141d431ab70e4ac37298fa6d4a8987f Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Wed, 6 Jul 2022 17:33:27 -0300 Subject: [PATCH 052/135] typo --- tools.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools.go b/tools.go index cd8f6ac..8916268 100644 --- a/tools.go +++ b/tools.go @@ -19,7 +19,7 @@ const randomStringSource = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ // Tools is the type for this package. Create a variable of this type and you have access // to all the methods with the receiver type *Tools. type Tools struct { - MaxJSONSize int // maximum siz of JSON file we'll process + MaxJSONSize int // maximum size of JSON file we'll process MaxFileSize int // maximum size of uploaded files in bytes AllowedFileTypes []string // allowed file types for upload (e.g. image/jpeg) AllowUnknownFields bool // if set to true, we allow unknown fields in JSON From ad6f0b982c3c43d3f74104135e03d16721e57095 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Wed, 6 Jul 2022 17:33:39 -0300 Subject: [PATCH 053/135] typo --- tools.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools.go b/tools.go index 8916268..2e6c92e 100644 --- a/tools.go +++ b/tools.go @@ -22,7 +22,7 @@ type Tools struct { MaxJSONSize int // maximum size of JSON file we'll process MaxFileSize int // maximum size of uploaded files in bytes AllowedFileTypes []string // allowed file types for upload (e.g. image/jpeg) - AllowUnknownFields bool // if set to true, we allow unknown fields in JSON + AllowUnknownFields bool // if set to true, allow unknown fields in JSON } // JSONResponse is the type used for sending JSON around From 15bbe6187efbe4c00a5cfd031a512207369c8a0b Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Thu, 7 Jul 2022 08:25:57 -0300 Subject: [PATCH 054/135] grammar --- tools.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools.go b/tools.go index 2e6c92e..81b0081 100644 --- a/tools.go +++ b/tools.go @@ -16,7 +16,7 @@ import ( const randomStringSource = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0987654321_+" -// Tools is the type for this package. Create a variable of this type and you have access +// Tools is the type for this package. Create a variable of this type, and you have access // to all the methods with the receiver type *Tools. type Tools struct { MaxJSONSize int // maximum size of JSON file we'll process From bb6366921750cb126db16071ce94611f2067484c Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Thu, 7 Jul 2022 08:52:18 -0300 Subject: [PATCH 055/135] improve readme example html --- readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme.md b/readme.md index 994a8a2..b78a7fd 100644 --- a/readme.md +++ b/readme.md @@ -135,7 +135,7 @@ To upload a file to a specific directory, with this for HTML:
- +
From 6b25226323ae125b02b31f0ddcbf3da6ea0542ac Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Thu, 7 Jul 2022 09:18:04 -0300 Subject: [PATCH 056/135] use fmt.Sprintf instead of concatenation --- tools.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools.go b/tools.go index 81b0081..9007429 100644 --- a/tools.go +++ b/tools.go @@ -265,7 +265,7 @@ func (t *Tools) UploadFiles(r *http.Request, uploadDir string, rename ...bool) ( } if renameFile { - uploadedFile.NewFileName = t.RandomString(25) + filepath.Ext(hdr.Filename) + uploadedFile.NewFileName = fmt.Sprintf("%s%s", t.RandomString(25), filepath.Ext(hdr.Filename)) } else { uploadedFile.NewFileName = hdr.Filename } From 2848fad77298a7136b964c3e2287194591141258 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Thu, 7 Jul 2022 10:12:35 -0300 Subject: [PATCH 057/135] Change order of parameters so that http client defaults to stdlib, but can be overridden. --- tools.go | 13 ++++++++++--- tools_test.go | 2 +- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/tools.go b/tools.go index 9007429..dd8f0e7 100644 --- a/tools.go +++ b/tools.go @@ -144,14 +144,21 @@ func (t *Tools) RandomString(n int) string { } // PushJSONToRemote posts arbitrary json to some url, and returns the response, the response -// status code, and error, if any -func (t *Tools) PushJSONToRemote(client *http.Client, uri string, data interface{}) (*http.Response, int, error) { +// status code, and error, if any. The final parameter, client, is optional, and will default +// to the standard http.Client. It exists to make testing possible without an active remote +// url. +func (t *Tools) PushJSONToRemote(uri string, data interface{}, client ...*http.Client) (*http.Response, int, error) { // create json we'll send jsonData, err := json.Marshal(data) if err != nil { return nil, 0, err } + httpClient := &http.Client{} + if len(client) > 0 { + httpClient = client[0] + } + // build the request and set header request, err := http.NewRequest("POST", uri, bytes.NewBuffer(jsonData)) if err != nil { @@ -160,7 +167,7 @@ func (t *Tools) PushJSONToRemote(client *http.Client, uri string, data interface request.Header.Set("Content-Type", "application/json") // call the uri - response, err := client.Do(request) + response, err := httpClient.Do(request) if err != nil { return nil, 0, err } diff --git a/tools_test.go b/tools_test.go index 05eeeab..3c02fdd 100644 --- a/tools_test.go +++ b/tools_test.go @@ -49,7 +49,7 @@ func TestTools_PushJSONToRemote(t *testing.T) { Bar string `json:"bar"` } foo.Bar = "bar" - _, _, err := testTools.PushJSONToRemote(client, "http://example.com/some/path", foo) + _, _, err := testTools.PushJSONToRemote("http://example.com/some/path", foo, client) if err != nil { t.Error("failed to call remote url", err) } From 9f5048020882e5339401310f9a590883a4c21f87 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Thu, 7 Jul 2022 10:18:12 -0300 Subject: [PATCH 058/135] Update readme with example to push JSON to remote --- readme.md | 173 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 173 insertions(+) diff --git a/readme.md b/readme.md index b78a7fd..73c26dd 100644 --- a/readme.md +++ b/readme.md @@ -200,4 +200,177 @@ func main() { // start the server http.ListenAndServe(":8080", nil) } +``` + +### Calling a Remote API + +To make a JSON post to a remote URI, with this html: + +```html + + + + + + + + + JSON functionality + + + +
+
+
+

JSON functionality

+
+ + + + +
+ + +
+ + + Push JSON + +
+

Response from server:

+
+ 
No response from server yet...
+
+ +
+
+
+ + + + +``` + +You can use this kind of Go code: + +```go +package main + +import ( + "github.com/tsawler/toolbox" + "log" + "net/http" +) + +func main() { + // create a default server mux + mux := http.NewServeMux() + + // register routes + mux.Handle("/", http.StripPrefix("/", http.FileServer(http.Dir(".")))) + mux.HandleFunc("/receive-post", receivePost) + mux.HandleFunc("/remote-service", remoteService) + + // print a log message + log.Println("Starting server on port 8081") + + // start the server + err := http.ListenAndServe(":8081", mux) + if err != nil { + log.Fatal(err) + } +} + +// RequestPayload describes the JSON that this service accepts as an HTTP Post request +type RequestPayload struct { + Action string `json:"action"` + Message string `json:"message"` +} + +// ResponsePayload is the structure used for sending a JSON response +type ResponsePayload struct { + Message string `json:"message"` + StatusCode int `json:"status_code,omitempty"` +} + +func receivePost(w http.ResponseWriter, r *http.Request) { + // get the posted json and decode it + var requestPayload RequestPayload + var t toolbox.Tools + + err := t.ReadJSON(w, r, &requestPayload) + if err != nil { + _ = t.ErrorJSON(w, err) + return + } + + // call remote service + _, statusCode, err := t.PushJSONToRemote("http://localhost:8081/remote-service", requestPayload) + if err != nil { + _ = t.ErrorJSON(w, err) + return + } + + // send response + payload := ResponsePayload{ + Message: "hit the service ok", + StatusCode: statusCode, + } + + err = t.WriteJSON(w, http.StatusAccepted, payload) + if err != nil { + log.Println(err) + } +} + +// remoteService just simulates calling some remote API +func remoteService(w http.ResponseWriter, r *http.Request) { + payload := ResponsePayload{ + Message: "OK", + } + var t toolbox.Tools + + _ = t.WriteJSON(w, http.StatusOK, payload) +} ``` \ No newline at end of file From 10c42861a061c6de8987cedc15bbf7505469b602 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Thu, 7 Jul 2022 10:19:10 -0300 Subject: [PATCH 059/135] Update readme with example to push JSON to remote --- readme.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/readme.md b/readme.md index 73c26dd..5bcec6a 100644 --- a/readme.md +++ b/readme.md @@ -345,7 +345,8 @@ func receivePost(w http.ResponseWriter, r *http.Request) { return } - // call remote service + // Call remote service. Note that we are ignoring the first return parameter, which is the + // entire response from the remote service, but you have access to it if you need it. _, statusCode, err := t.PushJSONToRemote("http://localhost:8081/remote-service", requestPayload) if err != nil { _ = t.ErrorJSON(w, err) From d4096f1cdcf67cfffcc7573acf7214e6d1bd4893 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Thu, 7 Jul 2022 11:31:56 -0300 Subject: [PATCH 060/135] Correct appending to slice when multiple files uploaded in one request --- tools.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tools.go b/tools.go index dd8f0e7..ebb8921 100644 --- a/tools.go +++ b/tools.go @@ -232,11 +232,11 @@ func (t *Tools) UploadFiles(r *http.Request, uploadDir string, rename ...bool) ( if err != nil { return nil, errors.New("the uploaded file is too big") } - var uploadedFile UploadedFile for _, fHeaders := range r.MultipartForm.File { for _, hdr := range fHeaders { - uploadedFiles, err = func() ([]*UploadedFile, error) { + uploadedFiles, err = func(uploadedFiles []*UploadedFile) ([]*UploadedFile, error) { + var uploadedFile UploadedFile infile, err := hdr.Open() if err != nil { return nil, err @@ -292,8 +292,9 @@ func (t *Tools) UploadFiles(r *http.Request, uploadDir string, rename ...bool) ( } uploadedFiles = append(uploadedFiles, &uploadedFile) + return uploadedFiles, nil - }() + }(uploadedFiles) if err != nil { return uploadedFiles, err } From ba876b24a28975d846a0f8feda5d42fa3ddc01f8 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Thu, 7 Jul 2022 13:06:24 -0300 Subject: [PATCH 061/135] Add slugify method & tests --- tools.go | 15 +++++++++++++++ tools_test.go | 28 ++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/tools.go b/tools.go index ebb8921..2db3fff 100644 --- a/tools.go +++ b/tools.go @@ -11,6 +11,7 @@ import ( "os" "path" "path/filepath" + "regexp" "strings" ) @@ -314,3 +315,17 @@ func (t *Tools) CreateDirIfNotExist(path string) error { } return nil } + +// Slugify is a (very) simple means of creating a slug from a provided string +func (t *Tools) Slugify(s string) (string, error) { + if s == "" { + return "", errors.New("empty string not permitted") + } + var re = regexp.MustCompile("[^a-z\\d]+") + slug := strings.Trim(re.ReplaceAllString(strings.ToLower(s), "-"), "-") + if len(slug) == 0 { + return "", errors.New("after removing characters, slug is zero length") + } + + return slug, nil +} diff --git a/tools_test.go b/tools_test.go index 3c02fdd..62d9ebd 100644 --- a/tools_test.go +++ b/tools_test.go @@ -334,3 +334,31 @@ func TestTools_CreateDirIfNotExist(t *testing.T) { _ = os.Remove("./testdata/myDir") } + +var slugTests = []struct { + name string + s string + expected string + errorExpected bool +}{ + {name: "valid string", s: "now is the time", expected: "now-is-the-time", errorExpected: false}, + {name: "empty string", s: "", expected: "", errorExpected: true}, + {name: "complex string", s: "Now is the time for all GOOD men! + Fish & such &^?123", expected: "now-is-the-time-for-all-good-men-fish-such-123", errorExpected: false}, + {name: "japanese string", s: "こんにちは世界", expected: "", errorExpected: true}, + {name: "japanese string plus roman characters", s: "こんにちは世界 hello world", expected: "hello-world", errorExpected: false}, +} + +func TestTools_Slugify(t *testing.T) { + var testTool Tools + + for _, e := range slugTests { + slug, err := testTool.Slugify(e.s) + if err != nil && !e.errorExpected { + t.Errorf("%s: error received when none expected: %s", e.name, err.Error()) + } + + if !e.errorExpected && slug != e.expected { + t.Errorf("%s: wrong slug returned; expected %s but got %s", e.name, e.expected, slug) + } + } +} From ecec4e431dbfe076b627e012dd8ca6ad0bceb114 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Thu, 7 Jul 2022 13:11:49 -0300 Subject: [PATCH 062/135] Update readme --- readme.md | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/readme.md b/readme.md index 5bcec6a..648bacd 100644 --- a/readme.md +++ b/readme.md @@ -17,6 +17,7 @@ The included tools are: - Get a random string of length n - Post JSON to a remote service - Create a directory, including all parent directories, if it does not already exist +- Create a URL safe slug from a string **Not for production -- used in a course.** @@ -374,4 +375,31 @@ func remoteService(w http.ResponseWriter, r *http.Request) { _ = t.WriteJSON(w, http.StatusOK, payload) } +``` + +### Create a slug from a string + +To slugify a string, we simply remove all non URL safe characters and return the +original string with a hyphen where spaces would be. Example: + +```go +package main + +import ( + "github.com/tsawler/toolbox" + "log" +) + +func main() { + toSlugify := "hello, world! These are unsafe chars: こんにちは世界*!&^%" + log.Println("To slugify:", toSlugify) + var tools toolbox.Tools + + slug, err := tools.Slugify(toSlugify) + if err != nil { + log.Println(err) + } + + log.Println("Slugified:", slug) +} ``` \ No newline at end of file From 8a014bcfea6ae3fac192c759dad05684ae842770 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Thu, 7 Jul 2022 13:13:56 -0300 Subject: [PATCH 063/135] use raw string in regex --- tools.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools.go b/tools.go index 2db3fff..31f5790 100644 --- a/tools.go +++ b/tools.go @@ -321,7 +321,7 @@ func (t *Tools) Slugify(s string) (string, error) { if s == "" { return "", errors.New("empty string not permitted") } - var re = regexp.MustCompile("[^a-z\\d]+") + var re = regexp.MustCompile(`[^a-z\d]+`) slug := strings.Trim(re.ReplaceAllString(strings.ToLower(s), "-"), "-") if len(slug) == 0 { return "", errors.New("after removing characters, slug is zero length") From 344743ddb2382c8e09491bc9d4909271d368b91c Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Thu, 7 Jul 2022 15:22:15 -0300 Subject: [PATCH 064/135] update readme --- readme.md | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/readme.md b/readme.md index 648bacd..9c4b22b 100644 --- a/readme.md +++ b/readme.md @@ -386,20 +386,27 @@ original string with a hyphen where spaces would be. Example: package main import ( + "fmt" "github.com/tsawler/toolbox" - "log" ) func main() { toSlugify := "hello, world! These are unsafe chars: こんにちは世界*!&^%" - log.Println("To slugify:", toSlugify) + fmt.Println("To slugify:", toSlugify) var tools toolbox.Tools slug, err := tools.Slugify(toSlugify) if err != nil { - log.Println(err) + fmt.Println(err) } - log.Println("Slugified:", slug) + fmt.Println("Slugified:", slug) } +``` + +Output from this is: + +``` +To slugify: hello, world! These are unsafe chars: こんにちは世界*!&^% +Slugified: hello-world-these-are-unsafe-chars ``` \ No newline at end of file From b9c772a328202f726ada0e5792aec000a54e3ed2 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Sun, 10 Jul 2022 10:36:42 -0300 Subject: [PATCH 065/135] update readme --- readme.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/readme.md b/readme.md index 9c4b22b..48df112 100644 --- a/readme.md +++ b/readme.md @@ -266,8 +266,6 @@ To make a JSON post to a remote URI, with this html: let serverResponse = document.getElementById("response"); pushBtn.addEventListener("click", function () { - console.log("clicked, json is", jsonPayload.value); - const payload = jsonPayload.value; const headers = new Headers(); From 1fd448fcb4f486747b43a130262b5d0a8ae66117 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Mon, 11 Jul 2022 07:48:33 -0300 Subject: [PATCH 066/135] add default max file size --- tools.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tools.go b/tools.go index 31f5790..b6a5168 100644 --- a/tools.go +++ b/tools.go @@ -228,6 +228,11 @@ func (t *Tools) UploadFiles(r *http.Request, uploadDir string, rename ...bool) ( return nil, err } + // sanity check on t.MaxFileSize + if t.MaxFileSize == 0 { + t.MaxFileSize = 1024 * 1024 * 5 // 5 megabytes + } + // parse the form so we have access to the file err = r.ParseMultipartForm(int64(t.MaxFileSize)) if err != nil { From 1e1e08ef7c983bf25b83e11f1fa5dd83f4697e20 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Mon, 11 Jul 2022 09:31:38 -0300 Subject: [PATCH 067/135] improve error message --- tools.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools.go b/tools.go index b6a5168..e3aaf10 100644 --- a/tools.go +++ b/tools.go @@ -236,7 +236,7 @@ func (t *Tools) UploadFiles(r *http.Request, uploadDir string, rename ...bool) ( // parse the form so we have access to the file err = r.ParseMultipartForm(int64(t.MaxFileSize)) if err != nil { - return nil, errors.New("the uploaded file is too big") + return nil, errors.New(fmt.Sprintf("the uploaded file is too big, and must be less than %d", t.MaxFileSize)) } for _, fHeaders := range r.MultipartForm.File { From 452bb46fd51c0727a12b6a9dee886055dce83529 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Mon, 11 Jul 2022 09:38:54 -0300 Subject: [PATCH 068/135] use fmt.Errorf --- tools.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools.go b/tools.go index e3aaf10..8880d51 100644 --- a/tools.go +++ b/tools.go @@ -236,7 +236,7 @@ func (t *Tools) UploadFiles(r *http.Request, uploadDir string, rename ...bool) ( // parse the form so we have access to the file err = r.ParseMultipartForm(int64(t.MaxFileSize)) if err != nil { - return nil, errors.New(fmt.Sprintf("the uploaded file is too big, and must be less than %d", t.MaxFileSize)) + return nil, fmt.Errorf("the uploaded file is too big, and must be less than %d", t.MaxFileSize) } for _, fHeaders := range r.MultipartForm.File { From fff4fb39b07feb69503f9a6a60bb333b7b59a804 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Mon, 11 Jul 2022 11:45:37 -0300 Subject: [PATCH 069/135] Update readme --- readme.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/readme.md b/readme.md index 48df112..6f43033 100644 --- a/readme.md +++ b/readme.md @@ -19,8 +19,6 @@ The included tools are: - Create a directory, including all parent directories, if it does not already exist - Create a URL safe slug from a string -**Not for production -- used in a course.** - ## Installation `go get -u github.com/tsawler/toolbox` From b43bf8d877162068178498a04a209176b858dbf3 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Tue, 12 Jul 2022 11:56:46 -0300 Subject: [PATCH 070/135] comment --- tools_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/tools_test.go b/tools_test.go index 62d9ebd..e5471cd 100644 --- a/tools_test.go +++ b/tools_test.go @@ -117,6 +117,7 @@ func Test_ReadJSON(t *testing.T) { } func TestTools_WriteJSON(t *testing.T) { + // create a variable of type toolbox.Tools, and just use the defaults. var testTools Tools rr := httptest.NewRecorder() From ec283759269e39c1744c83970f63564a9ac0b05b Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Wed, 13 Jul 2022 11:35:38 -0300 Subject: [PATCH 071/135] fix test --- tools_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools_test.go b/tools_test.go index 62d9ebd..5cb62a6 100644 --- a/tools_test.go +++ b/tools_test.go @@ -259,7 +259,7 @@ func TestTools_UploadFiles(t *testing.T) { _ = os.Remove(fmt.Sprintf("./testdata/uploads/%s", uploadedFiles[0].NewFileName)) } - if !e.errorExpected && err != nil { + if e.errorExpected && err == nil { t.Errorf("%s: error expected, but none received", e.name) } From c18a1d763dc653d0e240a3657650f360065a3d0a Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Wed, 13 Jul 2022 15:59:04 -0300 Subject: [PATCH 072/135] tests --- tools_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools_test.go b/tools_test.go index e5471cd..c41c4c8 100644 --- a/tools_test.go +++ b/tools_test.go @@ -260,7 +260,7 @@ func TestTools_UploadFiles(t *testing.T) { _ = os.Remove(fmt.Sprintf("./testdata/uploads/%s", uploadedFiles[0].NewFileName)) } - if !e.errorExpected && err != nil { + if e.errorExpected && err == nil { t.Errorf("%s: error expected, but none received", e.name) } From 1f2043da7ff028ab083a32dd68e86252bcba1d0e Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Thu, 14 Jul 2022 07:40:47 -0300 Subject: [PATCH 073/135] Readme --- readme.md | 1 + 1 file changed, 1 insertion(+) diff --git a/readme.md b/readme.md index 6f43033..0edf8e2 100644 --- a/readme.md +++ b/readme.md @@ -1,4 +1,5 @@ [![Version](https://img.shields.io/badge/goversion-1.18.x-blue.svg)](https://golang.org) +Built with GoLang [![License](http://img.shields.io/badge/license-mit-blue.svg?style=flat-square)](https://raw.githubusercontent.com/tsawler/goblender/master/LICENSE) [![Go Report Card](https://goreportcard.com/badge/github.com/tsawler/toolbox)](https://goreportcard.com/report/github.com/tsawler/toolbox) ![Tests](https://github.com/tsawler/toolbox/actions/workflows/tests.yml/badge.svg) From d892d0f7ee2409f21278f9ff07ac062b0d4f1319 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Fri, 22 Jul 2022 10:16:58 -0300 Subject: [PATCH 074/135] comments --- tools.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools.go b/tools.go index 8880d51..abda88d 100644 --- a/tools.go +++ b/tools.go @@ -18,7 +18,7 @@ import ( const randomStringSource = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0987654321_+" // Tools is the type for this package. Create a variable of this type, and you have access -// to all the methods with the receiver type *Tools. +// to all the exported methods with the receiver type *Tools. type Tools struct { MaxJSONSize int // maximum size of JSON file we'll process MaxFileSize int // maximum size of uploaded files in bytes From e47438b309ac3edb39a932e350c7a9953069c4a6 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Tue, 26 Jul 2022 19:08:34 -0300 Subject: [PATCH 075/135] add test --- tools_test.go | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tools_test.go b/tools_test.go index c41c4c8..7b82dc7 100644 --- a/tools_test.go +++ b/tools_test.go @@ -116,6 +116,32 @@ func Test_ReadJSON(t *testing.T) { } } +func TestTools_ReadJSONBadMarshal(t *testing.T) { + // set max file size + var testTools Tools + + // create a request with the body + req, err := http.NewRequest("POST", "/", bytes.NewReader([]byte(`{"foo": "bar"}`))) + if err != nil { + t.Log("Error", err) + } + + // create a test response recorder, which satisfies the requirements + // for a ResponseWriter + rr := httptest.NewRecorder() + + // call readJSON and check for an error + err = testTools.ReadJSON(rr, req, nil) + + // if we expect an error, but do not get one, something went wrong + if err == nil { + t.Error("error expected, but none received") + } + + req.Body.Close() + +} + func TestTools_WriteJSON(t *testing.T) { // create a variable of type toolbox.Tools, and just use the defaults. var testTools Tools From d5cdfda06e8dd890c2d4f9e097d782d9d3b094b1 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Wed, 27 Jul 2022 14:37:14 -0300 Subject: [PATCH 076/135] fix name --- tools_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools_test.go b/tools_test.go index 7b82dc7..fd1e19b 100644 --- a/tools_test.go +++ b/tools_test.go @@ -65,6 +65,7 @@ var jsonTests = []struct { {name: "good json", json: `{"foo": "bar"}`, errorExpected: false, maxSize: 1024, allowUnknown: false}, {name: "badly formatted json", json: `{"foo":"}`, errorExpected: true, maxSize: 1024, allowUnknown: false}, {name: "incorrect type", json: `{"foo": 1}`, errorExpected: true, maxSize: 1024, allowUnknown: false}, + {name: "incorrect type", json: `{1: 1}`, errorExpected: true, maxSize: 1024, allowUnknown: false}, {name: "two json files", json: `{"foo": "bar"}{"alpha": "beta"}`, errorExpected: true, maxSize: 1024, allowUnknown: false}, {name: "empty body", json: ``, errorExpected: true, maxSize: 1024, allowUnknown: false}, {name: "syntax error in json", json: `{"foo": 1"}`, errorExpected: true, maxSize: 1024, allowUnknown: false}, @@ -116,7 +117,7 @@ func Test_ReadJSON(t *testing.T) { } } -func TestTools_ReadJSONBadMarshal(t *testing.T) { +func TestTools_ReadJSONAndMarshal(t *testing.T) { // set max file size var testTools Tools From 91870671fddb33d410d4897cbb8316daca69bf85 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Thu, 28 Jul 2022 13:36:33 -0300 Subject: [PATCH 077/135] add comments --- tools.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools.go b/tools.go index abda88d..19f3b32 100644 --- a/tools.go +++ b/tools.go @@ -100,12 +100,14 @@ func (t *Tools) WriteJSON(w http.ResponseWriter, status int, data interface{}, h return err } + // if we have a value as the last parameter in the function call, then we are setting a custom header. if len(headers) > 0 { for key, value := range headers[0] { w.Header()[key] = value } } + // set the content type and send respones w.Header().Set("Content-Type", "application/json") w.WriteHeader(status) _, err = w.Write(out) From c137e5b368bc265d14948a1714cc3069dbe46fbd Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Thu, 28 Jul 2022 13:39:22 -0300 Subject: [PATCH 078/135] add comments --- tools.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools.go b/tools.go index 19f3b32..6d3c0cd 100644 --- a/tools.go +++ b/tools.go @@ -107,7 +107,7 @@ func (t *Tools) WriteJSON(w http.ResponseWriter, status int, data interface{}, h } } - // set the content type and send respones + // set the content type and send response w.Header().Set("Content-Type", "application/json") w.WriteHeader(status) _, err = w.Write(out) From 0674903ecd01b8389341bc551f94b3e88d8ee233 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Thu, 28 Jul 2022 13:42:20 -0300 Subject: [PATCH 079/135] add comments --- tools.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools.go b/tools.go index 6d3c0cd..0186ec9 100644 --- a/tools.go +++ b/tools.go @@ -36,6 +36,8 @@ type JSONResponse struct { // ReadJSON tries to read the body of a request and converts it into JSON func (t *Tools) ReadJSON(w http.ResponseWriter, r *http.Request, data interface{}) error { maxBytes := 1024 * 1024 // one megabyte + + // if MaxJSONSize is set, use that value instead of default if t.MaxJSONSize != 0 { maxBytes = t.MaxJSONSize } From 7312287a3affafa9fdcd6164672c84a4d952a315 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Thu, 28 Jul 2022 14:44:21 -0300 Subject: [PATCH 080/135] fix test for errorjson --- tools_test.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tools_test.go b/tools_test.go index fd1e19b..38b9a0d 100644 --- a/tools_test.go +++ b/tools_test.go @@ -165,7 +165,7 @@ func TestTools_ErrorJSON(t *testing.T) { var testTools Tools rr := httptest.NewRecorder() - err := testTools.ErrorJSON(rr, errors.New("some error")) + err := testTools.ErrorJSON(rr, errors.New("some error"), http.StatusServiceUnavailable) if err != nil { t.Error(err) } @@ -181,10 +181,8 @@ func TestTools_ErrorJSON(t *testing.T) { t.Error("error set to false in response from ErrorJSON, and should be set to true") } - // test with status - err = testTools.ErrorJSON(rr, errors.New("another error"), http.StatusServiceUnavailable) - if err != nil { - t.Error(err) + if rr.Code != http.StatusServiceUnavailable { + t.Errorf("wrong status code returned; expected 503, but got %d", rr.Code) } } From 8d90f3983242d1b90013be6649f7efed23863ec3 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Sat, 30 Jul 2022 10:41:55 -0300 Subject: [PATCH 081/135] add comments --- tools.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools.go b/tools.go index 0186ec9..e835ed1 100644 --- a/tools.go +++ b/tools.go @@ -50,6 +50,8 @@ func (t *Tools) ReadJSON(w http.ResponseWriter, r *http.Request, data interface{ dec.DisallowUnknownFields() } + // attempt to decode the data, and figure out what the error is, so as to send back a human readable + // response err := dec.Decode(data) if err != nil { var syntaxError *json.SyntaxError From fde2543f0308c450ea201d390c3eef2defaa43d0 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Thu, 4 Aug 2022 14:34:37 -0300 Subject: [PATCH 082/135] update comments --- tools_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools_test.go b/tools_test.go index 38b9a0d..a68bff9 100644 --- a/tools_test.go +++ b/tools_test.go @@ -131,10 +131,11 @@ func TestTools_ReadJSONAndMarshal(t *testing.T) { // for a ResponseWriter rr := httptest.NewRecorder() - // call readJSON and check for an error + // call readJSON and check for an error; since we are using nil for the final parameter, + // we should get an error err = testTools.ReadJSON(rr, req, nil) - // if we expect an error, but do not get one, something went wrong + // we expect an error, but did not get one, so something went wrong if err == nil { t.Error("error expected, but none received") } From 9d818f070afe6f5aa8a4a303c52ba91f727254f9 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Sun, 16 Oct 2022 15:45:19 -0300 Subject: [PATCH 083/135] grammar --- tools.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tools.go b/tools.go index e835ed1..e9ab059 100644 --- a/tools.go +++ b/tools.go @@ -26,14 +26,14 @@ type Tools struct { AllowUnknownFields bool // if set to true, allow unknown fields in JSON } -// JSONResponse is the type used for sending JSON around +// JSONResponse is the type used for sending JSON around. type JSONResponse struct { Error bool `json:"error"` Message string `json:"message"` Data interface{} `json:"data,omitempty"` } -// ReadJSON tries to read the body of a request and converts it into JSON +// ReadJSON tries to read the body of a request and converts it into JSON. func (t *Tools) ReadJSON(w http.ResponseWriter, r *http.Request, data interface{}) error { maxBytes := 1024 * 1024 // one megabyte @@ -97,7 +97,7 @@ func (t *Tools) ReadJSON(w http.ResponseWriter, r *http.Request, data interface{ return nil } -// WriteJSON takes a response status code and arbitrary data and writes a json response to the client +// WriteJSON takes a response status code and arbitrary data and writes a json response to the client. func (t *Tools) WriteJSON(w http.ResponseWriter, status int, data interface{}, headers ...http.Header) error { out, err := json.Marshal(data) if err != nil { @@ -123,7 +123,7 @@ func (t *Tools) WriteJSON(w http.ResponseWriter, status int, data interface{}, h } // ErrorJSON takes an error, and optionally a response status code, and generates and sends -// a json error response +// a json error response. func (t *Tools) ErrorJSON(w http.ResponseWriter, err error, status ...int) error { statusCode := http.StatusBadRequest @@ -327,7 +327,7 @@ func (t *Tools) CreateDirIfNotExist(path string) error { return nil } -// Slugify is a (very) simple means of creating a slug from a provided string +// Slugify is a (very) simple means of creating a slug from a provided string. func (t *Tools) Slugify(s string) (string, error) { if s == "" { return "", errors.New("empty string not permitted") From edc17403d61aaba5502ce01da06af5054e435baf Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Wed, 5 Apr 2023 12:49:26 -0300 Subject: [PATCH 084/135] Add WriteXML --- tools.go | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tools.go b/tools.go index e9ab059..88ae05b 100644 --- a/tools.go +++ b/tools.go @@ -4,6 +4,7 @@ import ( "bytes" "crypto/rand" "encoding/json" + "encoding/xml" "errors" "fmt" "io" @@ -340,3 +341,28 @@ func (t *Tools) Slugify(s string) (string, error) { return slug, nil } + +// WriteXML takes a response status code and arbitrary data and writes a json response to the client. +func (t *Tools) WriteXML(w http.ResponseWriter, status int, data interface{}, headers ...http.Header) error { + out, err := xml.Marshal(data) + if err != nil { + return err + } + + // if we have a value as the last parameter in the function call, then we are setting a custom header. + if len(headers) > 0 { + for key, value := range headers[0] { + w.Header()[key] = value + } + } + + // set the content type and send response + w.Header().Set("Content-Type", "application/xml") + w.WriteHeader(status) + _, err = w.Write(out) + if err != nil { + return err + } + + return nil +} From 571330e5d136c856c65c862e66b8c4653e46bb01 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Thu, 6 Apr 2023 07:30:12 -0300 Subject: [PATCH 085/135] update tests --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 9b44840..203f513 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -14,7 +14,7 @@ jobs: - uses: actions/checkout@v2 - name: Set up Go - uses: actions/setup-go@v2 + uses: actions/setup-go@v3 with: go-version: 1.18 From 670c7ee4c55236344cd84f94ac33a6a091c67b38 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Thu, 6 Apr 2023 07:31:33 -0300 Subject: [PATCH 086/135] update tests --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 203f513..80c7cb9 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -11,7 +11,7 @@ jobs: audit: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Go uses: actions/setup-go@v3 From 31331bfde339ccf2e14d9cc2ef21b4ebcc6ab63b Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Thu, 6 Apr 2023 07:32:02 -0300 Subject: [PATCH 087/135] update tests --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 80c7cb9..3aebe45 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -16,7 +16,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v3 with: - go-version: 1.18 + go-version: 1.19 - name: Verify dependencies run: go mod verify From decb1119e6cfca3e2576e52441d715a65465b573 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Thu, 6 Apr 2023 07:32:13 -0300 Subject: [PATCH 088/135] update tests --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 1016502..94e8601 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,3 @@ module github.com/tsawler/toolbox -go 1.18 +go 1.19 From 2a0f63bedeb43a9da8e50ab820a5ff70f1770201 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Thu, 6 Apr 2023 07:35:50 -0300 Subject: [PATCH 089/135] remove deprecated ioutil --- tools_test.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tools_test.go b/tools_test.go index a68bff9..161e2ec 100644 --- a/tools_test.go +++ b/tools_test.go @@ -8,7 +8,6 @@ import ( "image" "image/png" "io" - "io/ioutil" "mime/multipart" "net/http" "net/http/httptest" @@ -38,7 +37,7 @@ func TestTools_PushJSONToRemote(t *testing.T) { return &http.Response{ StatusCode: http.StatusOK, // Send response to be tested - Body: ioutil.NopCloser(bytes.NewBufferString(`OK`)), + Body: io.NopCloser(bytes.NewBufferString(`OK`)), // Must be set to non-nil value or it panics Header: make(http.Header), } @@ -215,7 +214,7 @@ func TestTools_DownloadStaticFile(t *testing.T) { t.Error("wrong content disposition of", res.Header["Content-Disposition"][0]) } - _, err := ioutil.ReadAll(res.Body) + _, err := io.ReadAll(res.Body) if err != nil { t.Error(err) } From cbeeba06558e42c22db130fbdedc35197954dc6a Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Thu, 6 Apr 2023 10:54:00 -0300 Subject: [PATCH 090/135] Add test for WriteXML --- tools.go | 9 ++++++++- tools_test.go | 18 ++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/tools.go b/tools.go index 88ae05b..b470e40 100644 --- a/tools.go +++ b/tools.go @@ -34,6 +34,13 @@ type JSONResponse struct { Data interface{} `json:"data,omitempty"` } +// XMLResponse is the type used for sending JSON around. +type XMLResponse struct { + Error bool `xml:"error"` + Message string `xml:"message"` + Data interface{} `xml:"data,omitempty"` +} + // ReadJSON tries to read the body of a request and converts it into JSON. func (t *Tools) ReadJSON(w http.ResponseWriter, r *http.Request, data interface{}) error { maxBytes := 1024 * 1024 // one megabyte @@ -342,7 +349,7 @@ func (t *Tools) Slugify(s string) (string, error) { return slug, nil } -// WriteXML takes a response status code and arbitrary data and writes a json response to the client. +// WriteXML takes a response status code and arbitrary data and writes an XML response to the client. func (t *Tools) WriteXML(w http.ResponseWriter, status int, data interface{}, headers ...http.Header) error { out, err := xml.Marshal(data) if err != nil { diff --git a/tools_test.go b/tools_test.go index 161e2ec..a600178 100644 --- a/tools_test.go +++ b/tools_test.go @@ -388,3 +388,21 @@ func TestTools_Slugify(t *testing.T) { } } } + +func TestTools_WriteXML(t *testing.T) { + // create a variable of type toolbox.Tools, and just use the defaults. + var testTools Tools + + rr := httptest.NewRecorder() + payload := XMLResponse{ + Error: false, + Message: "foo", + } + + headers := make(http.Header) + headers.Add("FOO", "BAR") + err := testTools.WriteXML(rr, http.StatusOK, payload, headers) + if err != nil { + t.Errorf("failed to write XML: %v", err) + } +} From ca77dd02736111ed0b4d7c296b19ce8622417f25 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Thu, 6 Apr 2023 10:56:17 -0300 Subject: [PATCH 091/135] Update readme --- readme.md | 1 + 1 file changed, 1 insertion(+) diff --git a/readme.md b/readme.md index 0edf8e2..f9060e7 100644 --- a/readme.md +++ b/readme.md @@ -13,6 +13,7 @@ The included tools are: - Read JSON - Write JSON - Produce a JSON encoded error response +- Write XML - Upload a file to a specified directory - Download a static file - Get a random string of length n From a5f8e364528ae1cc6803c82ca5efdeecddae17a6 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Thu, 6 Apr 2023 14:54:14 -0300 Subject: [PATCH 092/135] add ReadXML --- readme.md | 1 + tools.go | 35 +++++++++++++++++++++++++++++++---- tools_test.go | 41 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 73 insertions(+), 4 deletions(-) diff --git a/readme.md b/readme.md index f9060e7..a86fb93 100644 --- a/readme.md +++ b/readme.md @@ -14,6 +14,7 @@ The included tools are: - Write JSON - Produce a JSON encoded error response - Write XML +- Read XML - Upload a file to a specified directory - Download a static file - Get a random string of length n diff --git a/tools.go b/tools.go index b470e40..d55a3c7 100644 --- a/tools.go +++ b/tools.go @@ -22,6 +22,7 @@ const randomStringSource = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ // to all the exported methods with the receiver type *Tools. type Tools struct { MaxJSONSize int // maximum size of JSON file we'll process + MaxXMLSize int // maximum size of XML file we'll process MaxFileSize int // maximum size of uploaded files in bytes AllowedFileTypes []string // allowed file types for upload (e.g. image/jpeg) AllowUnknownFields bool // if set to true, allow unknown fields in JSON @@ -41,11 +42,11 @@ type XMLResponse struct { Data interface{} `xml:"data,omitempty"` } -// ReadJSON tries to read the body of a request and converts it into JSON. +// ReadJSON tries to read the body of a request and converts it from JSON to a variable. func (t *Tools) ReadJSON(w http.ResponseWriter, r *http.Request, data interface{}) error { maxBytes := 1024 * 1024 // one megabyte - // if MaxJSONSize is set, use that value instead of default + // if MaxJSONSize is set, use that value instead of default. if t.MaxJSONSize != 0 { maxBytes = t.MaxJSONSize } @@ -58,8 +59,8 @@ func (t *Tools) ReadJSON(w http.ResponseWriter, r *http.Request, data interface{ dec.DisallowUnknownFields() } - // attempt to decode the data, and figure out what the error is, so as to send back a human readable - // response + // attempt to decode the data, and figure out what the error is, if any, to send back a human-readable + // response. err := dec.Decode(data) if err != nil { var syntaxError *json.SyntaxError @@ -373,3 +374,29 @@ func (t *Tools) WriteXML(w http.ResponseWriter, status int, data interface{}, he return nil } + +// ReadXML tries to read the body of an XML request. +func (t *Tools) ReadXML(w http.ResponseWriter, r *http.Request, data interface{}) error { + maxBytes := 1024 * 1024 // one megabyte + + // if MaxJSONSize is set, use that value instead of default + if t.MaxXMLSize != 0 { + maxBytes = t.MaxXMLSize + } + r.Body = http.MaxBytesReader(w, r.Body, int64(maxBytes)) + + dec := xml.NewDecoder(r.Body) + + // attempt to decode the data + err := dec.Decode(data) + if err != nil { + return err + } + + err = dec.Decode(&struct{}{}) + if err != io.EOF { + return errors.New("body must only contain a single XML value") + } + + return nil +} diff --git a/tools_test.go b/tools_test.go index a600178..b314668 100644 --- a/tools_test.go +++ b/tools_test.go @@ -406,3 +406,44 @@ func TestTools_WriteXML(t *testing.T) { t.Errorf("failed to write XML: %v", err) } } + +func TestTools_ReadXML(t *testing.T) { + // create a variable of type toolbox.Tools, and just use the defaults. + var tools Tools + + xmlPayload := ` + + + John Smith + Jane Jones + Reminder + Buy some bread. + ` + + // create a request with the body + req, err := http.NewRequest("POST", "/", bytes.NewReader([]byte(xmlPayload))) + if err != nil { + t.Log("Error", err) + } + + // create a test response recorder, which satisfies the requirements + // for a ResponseWriter + rr := httptest.NewRecorder() + + // call ReadXML and check for an error. + var note struct { + To string `xml:"to"` + From string `xml:"from"` + Heading string `xml:"heading"` + Body string `xml:"body"` + } + + err = tools.ReadXML(rr, req, ¬e) + if err != nil { + t.Error("error reading XML:", err) + } + + if note.To != "John Smith" { + t.Errorf("wrong value in note; expected %s but got %s", "John Smith", note.To) + } +} From 6b9a47f41e6dc73f8f453274af7d33e025bedc93 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Thu, 6 Apr 2023 14:54:50 -0300 Subject: [PATCH 093/135] add ReadXML --- tools_test.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tools_test.go b/tools_test.go index b314668..f767a5d 100644 --- a/tools_test.go +++ b/tools_test.go @@ -82,33 +82,33 @@ func Test_ReadJSON(t *testing.T) { // set max file size testTools.MaxJSONSize = e.maxSize - // allow/disallow unknown fields + // allow/disallow unknown fields. testTools.AllowUnknownFields = e.allowUnknown - // declare a variable to read the decoded json into + // declare a variable to read the decoded json into. var decodedJSON struct { Foo string `json:"foo"` } - // create a request with the body + // create a request with the body. req, err := http.NewRequest("POST", "/", bytes.NewReader([]byte(e.json))) if err != nil { t.Log("Error", err) } // create a test response recorder, which satisfies the requirements - // for a ResponseWriter + // for a ResponseWriter. rr := httptest.NewRecorder() - // call readJSON and check for an error + // call ReadJSON and check for an error. err = testTools.ReadJSON(rr, req, &decodedJSON) - // if we expect an error, but do not get one, something went wrong + // if we expect an error, but do not get one, something went wrong. if e.errorExpected && err == nil { t.Errorf("%s: error expected, but none received", e.name) } - // if we do not expect an error, but get one, something went wrong + // if we do not expect an error, but get one, something went wrong. if !e.errorExpected && err != nil { t.Errorf("%s: error not expected, but one received: %s \n%s", e.name, err.Error(), e.json) } From 52d8c2a2486630b68116030c09495ea53eeaef67 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Fri, 7 Apr 2023 09:14:42 -0300 Subject: [PATCH 094/135] improve test coverage --- tools_test.go | 89 ++++++++++++++++++++++++++++++++------------------- 1 file changed, 56 insertions(+), 33 deletions(-) diff --git a/tools_test.go b/tools_test.go index f767a5d..057ae78 100644 --- a/tools_test.go +++ b/tools_test.go @@ -76,9 +76,9 @@ var jsonTests = []struct { } func Test_ReadJSON(t *testing.T) { - var testTools Tools for _, e := range jsonTests { + var testTools Tools // set max file size testTools.MaxJSONSize = e.maxSize @@ -407,43 +407,66 @@ func TestTools_WriteXML(t *testing.T) { } } -func TestTools_ReadXML(t *testing.T) { - // create a variable of type toolbox.Tools, and just use the defaults. - var tools Tools +var xmlTests = []struct { + name string + xml string + maxBytes int + errorExpected bool +}{ + { + name: "good xml", + xml: `John SmithJane Jones`, + errorExpected: false, + }, + { + name: "badly formatted xml", + xml: `John SmithJane Jones`, + errorExpected: true, + }, + { + name: "too big", + xml: `John SmithJane Jones`, + maxBytes: 10, + errorExpected: true, + }, + { + name: "double xml", + xml: `John SmithJane Jones + Luke SkywalkerR2D2`, + errorExpected: true, + }, +} - xmlPayload := ` - - - John Smith - Jane Jones - Reminder - Buy some bread. - ` +func TestTools_ReadXML(t *testing.T) { - // create a request with the body - req, err := http.NewRequest("POST", "/", bytes.NewReader([]byte(xmlPayload))) - if err != nil { - t.Log("Error", err) - } + for _, e := range xmlTests { + // create a variable of type toolbox.Tools, and just use the defaults. + var tools Tools - // create a test response recorder, which satisfies the requirements - // for a ResponseWriter - rr := httptest.NewRecorder() + if e.maxBytes != 0 { + tools.MaxXMLSize = e.maxBytes + } + // create a request with the body. + req, err := http.NewRequest("POST", "/", bytes.NewReader([]byte(e.xml))) + if err != nil { + t.Log("Error", err) + } - // call ReadXML and check for an error. - var note struct { - To string `xml:"to"` - From string `xml:"from"` - Heading string `xml:"heading"` - Body string `xml:"body"` - } + // create a test response recorder, which satisfies the requirements + // for a ResponseWriter. + rr := httptest.NewRecorder() - err = tools.ReadXML(rr, req, ¬e) - if err != nil { - t.Error("error reading XML:", err) - } + // call ReadXML and check for an error. + var note struct { + To string `xml:"to"` + From string `xml:"from"` + } - if note.To != "John Smith" { - t.Errorf("wrong value in note; expected %s but got %s", "John Smith", note.To) + err = tools.ReadXML(rr, req, ¬e) + if e.errorExpected && err == nil { + t.Errorf("%s: expected an error, but did not get one", e.name) + } else if !e.errorExpected && err != nil { + t.Errorf("%s: did not expect an error, but got one: %s", e.name, err) + } } } From a9ec456e13e885f06ef0446f134b537c145c7662 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Fri, 7 Apr 2023 10:23:13 -0300 Subject: [PATCH 095/135] improve tests --- tools_test.go | 49 ++++++++++++++++++++++++++++++++++++------------- 1 file changed, 36 insertions(+), 13 deletions(-) diff --git a/tools_test.go b/tools_test.go index 057ae78..13ad08f 100644 --- a/tools_test.go +++ b/tools_test.go @@ -75,7 +75,7 @@ var jsonTests = []struct { {name: "not json", json: `Hello, world`, errorExpected: true, maxSize: 1024, allowUnknown: false}, } -func Test_ReadJSON(t *testing.T) { +func TestTools_ReadJSON(t *testing.T) { for _, e := range jsonTests { var testTools Tools @@ -143,22 +143,45 @@ func TestTools_ReadJSONAndMarshal(t *testing.T) { } +var testWriteJSONData = []struct { + name string + payload any + errorExpected bool +}{ + { + name: "valid", + payload: JSONResponse{ + Error: false, + Message: "foo", + }, + errorExpected: false, + }, + { + name: "invalid", + payload: make(chan int), + errorExpected: true, + }, +} + func TestTools_WriteJSON(t *testing.T) { - // create a variable of type toolbox.Tools, and just use the defaults. - var testTools Tools - rr := httptest.NewRecorder() - payload := JSONResponse{ - Error: false, - Message: "foo", - } + for _, e := range testWriteJSONData { + // create a variable of type toolbox.Tools, and just use the defaults. + var testTools Tools - headers := make(http.Header) - headers.Add("FOO", "BAR") - err := testTools.WriteJSON(rr, http.StatusOK, payload, headers) - if err != nil { - t.Errorf("failed to write JSON: %v", err) + rr := httptest.NewRecorder() + + headers := make(http.Header) + headers.Add("FOO", "BAR") + err := testTools.WriteJSON(rr, http.StatusOK, e.payload, headers) + if err == nil && e.errorExpected { + t.Errorf("%s: expected error, but did not get one", e.name) + } + if err != nil && !e.errorExpected { + t.Errorf("%s: did not expect error, but got one: %v", e.name, err) + } } + } func TestTools_ErrorJSON(t *testing.T) { From f3d796b9e1b826c68839144fd54ce38ab40b9898 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Fri, 7 Apr 2023 10:44:43 -0300 Subject: [PATCH 096/135] correct check for max file size --- tools.go | 6 +++++- tools_test.go | 12 +++++++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/tools.go b/tools.go index d55a3c7..6b24160 100644 --- a/tools.go +++ b/tools.go @@ -251,7 +251,7 @@ func (t *Tools) UploadFiles(r *http.Request, uploadDir string, rename ...bool) ( // parse the form so we have access to the file err = r.ParseMultipartForm(int64(t.MaxFileSize)) if err != nil { - return nil, fmt.Errorf("the uploaded file is too big, and must be less than %d", t.MaxFileSize) + return nil, fmt.Errorf("error parsing form data") } for _, fHeaders := range r.MultipartForm.File { @@ -264,6 +264,10 @@ func (t *Tools) UploadFiles(r *http.Request, uploadDir string, rename ...bool) ( } defer infile.Close() + if hdr.Size > int64(t.MaxFileSize) { + return nil, fmt.Errorf("the uploaded file is too big, and must be less than %d", t.MaxFileSize) + } + buff := make([]byte, 512) _, err = infile.Read(buff) if err != nil { diff --git a/tools_test.go b/tools_test.go index 13ad08f..03c0e7c 100644 --- a/tools_test.go +++ b/tools_test.go @@ -248,10 +248,13 @@ var uploadTests = []struct { allowedTypes []string renameFile bool errorExpected bool + maxSize int }{ - {name: "allowed no rename", allowedTypes: []string{"image/jpeg", "image/png"}, renameFile: false, errorExpected: false}, - {name: "allowed rename", allowedTypes: []string{"image/jpeg", "image/png"}, renameFile: true, errorExpected: false}, - {name: "not allowed", allowedTypes: []string{"image/jpeg"}, errorExpected: true}, + {name: "allowed no rename", allowedTypes: []string{"image/jpeg", "image/png"}, renameFile: false, errorExpected: false, maxSize: 0}, + {name: "allowed rename", allowedTypes: []string{"image/jpeg", "image/png"}, renameFile: true, errorExpected: false, maxSize: 0}, + {name: "allowed no filetype specified", allowedTypes: []string{}, renameFile: true, errorExpected: false, maxSize: 0}, + {name: "not allowed", allowedTypes: []string{"image/jpeg"}, errorExpected: true, maxSize: 0}, + {name: "too big", allowedTypes: []string{"image/jpeg,", "image/png"}, errorExpected: true, maxSize: 10}, } func TestTools_UploadFiles(t *testing.T) { @@ -293,6 +296,9 @@ func TestTools_UploadFiles(t *testing.T) { var testTools Tools testTools.AllowedFileTypes = e.allowedTypes + if e.maxSize > 0 { + testTools.MaxFileSize = e.maxSize + } uploadedFiles, err := testTools.UploadFiles(request, "./testdata/uploads/", e.renameFile) if err != nil && !e.errorExpected { From e2ede8ba663f7b3fae207fcd8c5b6f97110cc3f2 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Fri, 7 Apr 2023 10:49:40 -0300 Subject: [PATCH 097/135] formatting --- tools_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/tools_test.go b/tools_test.go index 03c0e7c..817c8b8 100644 --- a/tools_test.go +++ b/tools_test.go @@ -164,7 +164,6 @@ var testWriteJSONData = []struct { } func TestTools_WriteJSON(t *testing.T) { - for _, e := range testWriteJSONData { // create a variable of type toolbox.Tools, and just use the defaults. var testTools Tools @@ -181,7 +180,6 @@ func TestTools_WriteJSON(t *testing.T) { t.Errorf("%s: did not expect error, but got one: %v", e.name, err) } } - } func TestTools_ErrorJSON(t *testing.T) { From 30b6070a9c97101fb5080f71b5eba077a8a2b063 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Fri, 7 Apr 2023 11:05:45 -0300 Subject: [PATCH 098/135] improve tests --- tools.go | 10 ++------- tools_test.go | 60 ++++++++++++++++++++++++++++++++++++--------------- 2 files changed, 45 insertions(+), 25 deletions(-) diff --git a/tools.go b/tools.go index 6b24160..7d87ba1 100644 --- a/tools.go +++ b/tools.go @@ -123,10 +123,7 @@ func (t *Tools) WriteJSON(w http.ResponseWriter, status int, data interface{}, h // set the content type and send response w.Header().Set("Content-Type", "application/json") w.WriteHeader(status) - _, err = w.Write(out) - if err != nil { - return err - } + _, _ = w.Write(out) return nil } @@ -371,10 +368,7 @@ func (t *Tools) WriteXML(w http.ResponseWriter, status int, data interface{}, he // set the content type and send response w.Header().Set("Content-Type", "application/xml") w.WriteHeader(status) - _, err = w.Write(out) - if err != nil { - return err - } + _, _ = w.Write(out) return nil } diff --git a/tools_test.go b/tools_test.go index 817c8b8..2700037 100644 --- a/tools_test.go +++ b/tools_test.go @@ -31,26 +31,52 @@ func NewTestClient(fn RoundTripFunc) *http.Client { } } +type testData struct { + Data any `json:"bar"` +} + +var pushTests = []struct { + name string + payload any + errorExpected bool +}{ + { + name: "valid", + payload: testData{ + Data: "bar", + }, + errorExpected: false, + }, + { + name: "invalid", + payload: make(chan int), + errorExpected: true, + }, +} + func TestTools_PushJSONToRemote(t *testing.T) { - client := NewTestClient(func(req *http.Request) *http.Response { - // Test request parameters - return &http.Response{ - StatusCode: http.StatusOK, - // Send response to be tested - Body: io.NopCloser(bytes.NewBufferString(`OK`)), - // Must be set to non-nil value or it panics - Header: make(http.Header), + for _, e := range pushTests { + client := NewTestClient(func(req *http.Request) *http.Response { + // Test request parameters + return &http.Response{ + StatusCode: http.StatusOK, + // Send response to be tested + Body: io.NopCloser(bytes.NewBufferString(`OK`)), + // Must be set to non-nil value or it panics + Header: make(http.Header), + } + }) + + var testTools Tools + + _, _, err := testTools.PushJSONToRemote("http://example.com/some/path", e.payload, client) + if err == nil && e.errorExpected { + t.Errorf("%s: error expected, but none received", e.name) } - }) - var testTools Tools - var foo struct { - Bar string `json:"bar"` - } - foo.Bar = "bar" - _, _, err := testTools.PushJSONToRemote("http://example.com/some/path", foo, client) - if err != nil { - t.Error("failed to call remote url", err) + if err != nil && !e.errorExpected { + t.Errorf("%s: no error expected, but one received: %v", e.name, err) + } } } From d45e47a0cd4fa1c71b1831447f317da96187fe20 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Fri, 7 Apr 2023 11:07:09 -0300 Subject: [PATCH 099/135] clean up --- tools_test.go | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tools_test.go b/tools_test.go index 2700037..74d3f1c 100644 --- a/tools_test.go +++ b/tools_test.go @@ -102,7 +102,6 @@ var jsonTests = []struct { } func TestTools_ReadJSON(t *testing.T) { - for _, e := range jsonTests { var testTools Tools // set max file size @@ -166,10 +165,9 @@ func TestTools_ReadJSONAndMarshal(t *testing.T) { } req.Body.Close() - } -var testWriteJSONData = []struct { +var writeJSONTests = []struct { name string payload any errorExpected bool @@ -190,7 +188,7 @@ var testWriteJSONData = []struct { } func TestTools_WriteJSON(t *testing.T) { - for _, e := range testWriteJSONData { + for _, e := range writeJSONTests { // create a variable of type toolbox.Tools, and just use the defaults. var testTools Tools @@ -395,7 +393,6 @@ func TestTools_UploadOneFile(t *testing.T) { // clean up _ = os.Remove(fmt.Sprintf("./testdata/uploads/%s", uploadedFiles.NewFileName)) - } func TestTools_CreateDirIfNotExist(t *testing.T) { @@ -491,7 +488,6 @@ var xmlTests = []struct { } func TestTools_ReadXML(t *testing.T) { - for _, e := range xmlTests { // create a variable of type toolbox.Tools, and just use the defaults. var tools Tools From 3174d5c650e0c9d1b05e0f352e0eecca1bba5f2c Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Fri, 7 Apr 2023 11:09:03 -0300 Subject: [PATCH 100/135] update badge --- readme.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/readme.md b/readme.md index a86fb93..4b0199f 100644 --- a/readme.md +++ b/readme.md @@ -1,4 +1,4 @@ -[![Version](https://img.shields.io/badge/goversion-1.18.x-blue.svg)](https://golang.org) +[![Version](https://img.shields.io/badge/goversion-1.19.x-blue.svg)](https://golang.org) Built with GoLang [![License](http://img.shields.io/badge/license-mit-blue.svg?style=flat-square)](https://raw.githubusercontent.com/tsawler/goblender/master/LICENSE) [![Go Report Card](https://goreportcard.com/badge/github.com/tsawler/toolbox)](https://goreportcard.com/report/github.com/tsawler/toolbox) @@ -26,6 +26,7 @@ The included tools are: `go get -u github.com/tsawler/toolbox` + ## Usage ```go From 50934f114af03723bdbf9181e0a9c5c51ab4755a Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Fri, 7 Apr 2023 11:24:42 -0300 Subject: [PATCH 101/135] improve tests --- tools_test.go | 108 ++++++++++++++++++++++++++++++-------------------- 1 file changed, 64 insertions(+), 44 deletions(-) diff --git a/tools_test.go b/tools_test.go index 74d3f1c..fc5873d 100644 --- a/tools_test.go +++ b/tools_test.go @@ -271,12 +271,14 @@ var uploadTests = []struct { renameFile bool errorExpected bool maxSize int + uploadDir string }{ - {name: "allowed no rename", allowedTypes: []string{"image/jpeg", "image/png"}, renameFile: false, errorExpected: false, maxSize: 0}, - {name: "allowed rename", allowedTypes: []string{"image/jpeg", "image/png"}, renameFile: true, errorExpected: false, maxSize: 0}, - {name: "allowed no filetype specified", allowedTypes: []string{}, renameFile: true, errorExpected: false, maxSize: 0}, - {name: "not allowed", allowedTypes: []string{"image/jpeg"}, errorExpected: true, maxSize: 0}, - {name: "too big", allowedTypes: []string{"image/jpeg,", "image/png"}, errorExpected: true, maxSize: 10}, + {name: "allowed no rename", allowedTypes: []string{"image/jpeg", "image/png"}, renameFile: false, errorExpected: false, maxSize: 0, uploadDir: ""}, + {name: "allowed rename", allowedTypes: []string{"image/jpeg", "image/png"}, renameFile: true, errorExpected: false, maxSize: 0, uploadDir: ""}, + {name: "allowed no filetype specified", allowedTypes: []string{}, renameFile: true, errorExpected: false, maxSize: 0, uploadDir: ""}, + {name: "not allowed", allowedTypes: []string{"image/jpeg"}, errorExpected: true, maxSize: 0, uploadDir: ""}, + {name: "too big", allowedTypes: []string{"image/jpeg,", "image/png"}, errorExpected: true, maxSize: 10, uploadDir: ""}, + {name: "invalid directory", allowedTypes: []string{"image/jpeg,", "image/png"}, errorExpected: true, maxSize: 0, uploadDir: "//"}, } func TestTools_UploadFiles(t *testing.T) { @@ -322,7 +324,12 @@ func TestTools_UploadFiles(t *testing.T) { testTools.MaxFileSize = e.maxSize } - uploadedFiles, err := testTools.UploadFiles(request, "./testdata/uploads/", e.renameFile) + var uploadDir = "./testdata/uploads/" + if e.uploadDir != "" { + uploadDir = e.uploadDir + } + + uploadedFiles, err := testTools.UploadFiles(request, uploadDir, e.renameFile) if err != nil && !e.errorExpected { t.Error(err) } @@ -345,54 +352,67 @@ func TestTools_UploadFiles(t *testing.T) { } } +var uploadOneTests = []struct { + name string + uploadDir string + errorExpected bool +}{ + {name: "valid", uploadDir: "./testdata/uploads/", errorExpected: false}, + {name: "invalid", uploadDir: "//", errorExpected: true}, +} + func TestTools_UploadOneFile(t *testing.T) { - // set up a pipe to avoid buffering - pr, pw := io.Pipe() - writer := multipart.NewWriter(pw) + for _, e := range uploadOneTests { + // set up a pipe to avoid buffering + pr, pw := io.Pipe() + writer := multipart.NewWriter(pw) - go func() { - defer writer.Close() + go func() { + defer writer.Close() - // create the form data field 'file' - part, err := writer.CreateFormFile("file", "./testdata/img.png") - if err != nil { - t.Error(err) - } + // create the form data field 'file' + part, err := writer.CreateFormFile("file", "./testdata/img.png") + if err != nil { + t.Error(err) + } - f, err := os.Open("./testdata/img.png") - if err != nil { - t.Error(err) - } - defer f.Close() - img, _, err := image.Decode(f) - if err != nil { - t.Error("error decoding image", err) - } + f, err := os.Open("./testdata/img.png") + if err != nil { + t.Error(err) + } + defer f.Close() + img, _, err := image.Decode(f) + if err != nil { + t.Error("error decoding image", err) + } - err = png.Encode(part, img) - if err != nil { - t.Error(err) - } - }() + err = png.Encode(part, img) + if err != nil { + t.Error(err) + } + }() + + // read from the pipe which receives data + request := httptest.NewRequest("POST", "/", pr) + request.Header.Add("Content-Type", writer.FormDataContentType()) - // read from the pipe which receives data - request := httptest.NewRequest("POST", "/", pr) - request.Header.Add("Content-Type", writer.FormDataContentType()) + var testTools Tools + testTools.AllowedFileTypes = []string{"image/png"} - var testTools Tools - testTools.AllowedFileTypes = []string{"image/png"} + uploadedFiles, err := testTools.UploadOneFile(request, e.uploadDir, true) + if e.errorExpected && err == nil { + t.Errorf("%s: error expected, but none received", e.name) + } - uploadedFiles, err := testTools.UploadOneFile(request, "./testdata/uploads/", true) - if err != nil { - t.Error(err) - } + if !e.errorExpected { + if _, err := os.Stat(fmt.Sprintf("./testdata/uploads/%s", uploadedFiles.NewFileName)); os.IsNotExist(err) { + t.Errorf("%s: expected file to exist: %s", e.name, err.Error()) + } - if _, err := os.Stat(fmt.Sprintf("./testdata/uploads/%s", uploadedFiles.NewFileName)); os.IsNotExist(err) { - t.Errorf("expected file to exist: %s", err.Error()) + // clean up + _ = os.Remove(fmt.Sprintf("./testdata/uploads/%s", uploadedFiles.NewFileName)) + } } - - // clean up - _ = os.Remove(fmt.Sprintf("./testdata/uploads/%s", uploadedFiles.NewFileName)) } func TestTools_CreateDirIfNotExist(t *testing.T) { From 2cd9f406480e81d617dc73db9ff670922ea0aba1 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Fri, 7 Apr 2023 14:50:40 -0300 Subject: [PATCH 102/135] add ErrorXML method --- tools.go | 17 +++++++++++++++++ tools_test.go | 26 ++++++++++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/tools.go b/tools.go index 7d87ba1..08a6b56 100644 --- a/tools.go +++ b/tools.go @@ -398,3 +398,20 @@ func (t *Tools) ReadXML(w http.ResponseWriter, r *http.Request, data interface{} return nil } + +// ErrorXML takes an error, and optionally a response status code, and generates and sends +// an XML error response. +func (t *Tools) ErrorXML(w http.ResponseWriter, err error, status ...int) error { + statusCode := http.StatusBadRequest + + // if a custom response code is specified, use that instead of bad request + if len(status) > 0 { + statusCode = status[0] + } + + var payload XMLResponse + payload.Error = true + payload.Message = err.Error() + + return t.WriteXML(w, statusCode, payload) +} diff --git a/tools_test.go b/tools_test.go index fc5873d..3364056 100644 --- a/tools_test.go +++ b/tools_test.go @@ -3,6 +3,7 @@ package toolbox import ( "bytes" "encoding/json" + "encoding/xml" "errors" "fmt" "image" @@ -539,3 +540,28 @@ func TestTools_ReadXML(t *testing.T) { } } } + +func TestTools_ErrorXML(t *testing.T) { + var testTools Tools + + rr := httptest.NewRecorder() + err := testTools.ErrorXML(rr, errors.New("some error"), http.StatusServiceUnavailable) + if err != nil { + t.Error(err) + } + + var requestPayload XMLResponse + decoder := xml.NewDecoder(rr.Body) + err = decoder.Decode(&requestPayload) + if err != nil { + t.Error("received error when decoding ErrorXML payload:", err) + } + + if !requestPayload.Error { + t.Error("error set to false in response from ErrorXML, and should be set to true") + } + + if rr.Code != http.StatusServiceUnavailable { + t.Errorf("wrong status code returned; expected 503, but got %d", rr.Code) + } +} From 6fbe8da37bb1b948706ffee7ee30392787f860d5 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Fri, 7 Apr 2023 14:51:07 -0300 Subject: [PATCH 103/135] update readme --- readme.md | 1 + 1 file changed, 1 insertion(+) diff --git a/readme.md b/readme.md index 4b0199f..955f5ab 100644 --- a/readme.md +++ b/readme.md @@ -15,6 +15,7 @@ The included tools are: - Produce a JSON encoded error response - Write XML - Read XML +- Produce an XML encoded error response - Upload a file to a specified directory - Download a static file - Get a random string of length n From 056d5867418205efc8402753466b6aa2df54bfdb Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Fri, 7 Apr 2023 14:53:50 -0300 Subject: [PATCH 104/135] comments --- tools.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools.go b/tools.go index 08a6b56..4ba72da 100644 --- a/tools.go +++ b/tools.go @@ -35,7 +35,7 @@ type JSONResponse struct { Data interface{} `json:"data,omitempty"` } -// XMLResponse is the type used for sending JSON around. +// XMLResponse is the type used for sending XML around. type XMLResponse struct { Error bool `xml:"error"` Message string `xml:"message"` From d8bbcd27c5c7f1485831298600d122f0e886fc10 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Fri, 7 Apr 2023 14:56:37 -0300 Subject: [PATCH 105/135] comments --- tools.go | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/tools.go b/tools.go index 4ba72da..090a819 100644 --- a/tools.go +++ b/tools.go @@ -106,7 +106,7 @@ func (t *Tools) ReadJSON(w http.ResponseWriter, r *http.Request, data interface{ return nil } -// WriteJSON takes a response status code and arbitrary data and writes a json response to the client. +// WriteJSON takes a response status code and arbitrary data and writes a JSON response to the client. func (t *Tools) WriteJSON(w http.ResponseWriter, status int, data interface{}, headers ...http.Header) error { out, err := json.Marshal(data) if err != nil { @@ -120,7 +120,7 @@ func (t *Tools) WriteJSON(w http.ResponseWriter, status int, data interface{}, h } } - // set the content type and send response + // set the content type and send response. w.Header().Set("Content-Type", "application/json") w.WriteHeader(status) _, _ = w.Write(out) @@ -129,11 +129,11 @@ func (t *Tools) WriteJSON(w http.ResponseWriter, status int, data interface{}, h } // ErrorJSON takes an error, and optionally a response status code, and generates and sends -// a json error response. +// a JSON error response. func (t *Tools) ErrorJSON(w http.ResponseWriter, err error, status ...int) error { statusCode := http.StatusBadRequest - // if a custom response code is specified, use that instead of bad request + // if a custom response code is specified, use that instead of bad request. if len(status) > 0 { statusCode = status[0] } @@ -145,7 +145,7 @@ func (t *Tools) ErrorJSON(w http.ResponseWriter, err error, status ...int) error return t.WriteJSON(w, statusCode, payload) } -// RandomString returns a random string of letters of length n +// RandomString returns a random string of letters of length n. func (t *Tools) RandomString(n int) string { s, r := make([]rune, n), []rune(randomStringSource) for i := range s { @@ -172,14 +172,14 @@ func (t *Tools) PushJSONToRemote(uri string, data interface{}, client ...*http.C httpClient = client[0] } - // build the request and set header + // build the request and set header. request, err := http.NewRequest("POST", uri, bytes.NewBuffer(jsonData)) if err != nil { return nil, 0, err } request.Header.Set("Content-Type", "application/json") - // call the uri + // call the uri. response, err := httpClient.Do(request) if err != nil { return nil, 0, err @@ -198,7 +198,7 @@ func (t *Tools) DownloadStaticFile(w http.ResponseWriter, r *http.Request, p, fi http.ServeFile(w, r, fp) } -// UploadedFile is a struct used for the uploaded file +// UploadedFile is the type used for the uploaded file. type UploadedFile struct { NewFileName string OriginalFileName string @@ -226,7 +226,7 @@ func (t *Tools) UploadOneFile(r *http.Request, uploadDir string, rename ...bool) // and potentially an error. If the optional last parameter is set to true, then we will not rename // the files, but will use the original file names. func (t *Tools) UploadFiles(r *http.Request, uploadDir string, rename ...bool) ([]*UploadedFile, error) { - // check to see if we are renaming the uploadedFiles with the optional last parameter + // check to see if we are renaming the uploadedFiles with the optional last parameter. renameFile := true if len(rename) > 0 { renameFile = rename[0] @@ -234,18 +234,18 @@ func (t *Tools) UploadFiles(r *http.Request, uploadDir string, rename ...bool) ( var uploadedFiles []*UploadedFile - // create the upload directory if it does not exist + // create the upload directory if it does not exist. err := t.CreateDirIfNotExist(uploadDir) if err != nil { return nil, err } - // sanity check on t.MaxFileSize + // sanity check on t.MaxFileSize. if t.MaxFileSize == 0 { - t.MaxFileSize = 1024 * 1024 * 5 // 5 megabytes + t.MaxFileSize = 1024 * 1024 * 5 // 5 megabytes. } - // parse the form so we have access to the file + // parse the form so we have access to the file. err = r.ParseMultipartForm(int64(t.MaxFileSize)) if err != nil { return nil, fmt.Errorf("error parsing form data") @@ -365,7 +365,7 @@ func (t *Tools) WriteXML(w http.ResponseWriter, status int, data interface{}, he } } - // set the content type and send response + // set the content type and send response. w.Header().Set("Content-Type", "application/xml") w.WriteHeader(status) _, _ = w.Write(out) @@ -373,11 +373,11 @@ func (t *Tools) WriteXML(w http.ResponseWriter, status int, data interface{}, he return nil } -// ReadXML tries to read the body of an XML request. +// ReadXML tries to read the body of an XML request into a variable. func (t *Tools) ReadXML(w http.ResponseWriter, r *http.Request, data interface{}) error { maxBytes := 1024 * 1024 // one megabyte - // if MaxJSONSize is set, use that value instead of default + // if MaxXMLSize is set, use that value instead of default. if t.MaxXMLSize != 0 { maxBytes = t.MaxXMLSize } @@ -385,7 +385,7 @@ func (t *Tools) ReadXML(w http.ResponseWriter, r *http.Request, data interface{} dec := xml.NewDecoder(r.Body) - // attempt to decode the data + // attempt to decode the data. err := dec.Decode(data) if err != nil { return err @@ -404,7 +404,7 @@ func (t *Tools) ReadXML(w http.ResponseWriter, r *http.Request, data interface{} func (t *Tools) ErrorXML(w http.ResponseWriter, err error, status ...int) error { statusCode := http.StatusBadRequest - // if a custom response code is specified, use that instead of bad request + // if a custom response code is specified, use that instead of bad request. if len(status) > 0 { statusCode = status[0] } From ef3600e60db23096828d7300e827c19a42e390fa Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Mon, 10 Apr 2023 10:01:44 -0300 Subject: [PATCH 106/135] add test case for ReadJSON --- tools_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools_test.go b/tools_test.go index 3364056..c84b136 100644 --- a/tools_test.go +++ b/tools_test.go @@ -96,6 +96,7 @@ var jsonTests = []struct { {name: "empty body", json: ``, errorExpected: true, maxSize: 1024, allowUnknown: false}, {name: "syntax error in json", json: `{"foo": 1"}`, errorExpected: true, maxSize: 1024, allowUnknown: false}, {name: "unknown field in json", json: `{"fooo": "bar"}`, errorExpected: true, maxSize: 1024, allowUnknown: false}, + {name: "incorrect type for field", json: `{"foo": 10.2}`, errorExpected: true, maxSize: 1024, allowUnknown: false}, {name: "allow unknown field in json", json: `{"fooo": "bar"}`, errorExpected: false, maxSize: 1024, allowUnknown: true}, {name: "missing field name", json: `{jack: "bar"}`, errorExpected: true, maxSize: 1024, allowUnknown: false}, {name: "file too large", json: `{"foo": "bar"}`, errorExpected: true, maxSize: 5, allowUnknown: false}, @@ -113,7 +114,8 @@ func TestTools_ReadJSON(t *testing.T) { // declare a variable to read the decoded json into. var decodedJSON struct { - Foo string `json:"foo"` + Foo string `json:"foo"` + Chan chan int `json:"chan"` } // create a request with the body. From 78e577ba7fff472a8ba07ce3a7a5f8ada250a13d Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Mon, 10 Apr 2023 11:03:25 -0300 Subject: [PATCH 107/135] improve tests --- tools_test.go | 46 ++++++++++++++++++++++++++++++++++------------ 1 file changed, 34 insertions(+), 12 deletions(-) diff --git a/tools_test.go b/tools_test.go index c84b136..a8e99b2 100644 --- a/tools_test.go +++ b/tools_test.go @@ -462,21 +462,43 @@ func TestTools_Slugify(t *testing.T) { } } +var writeXMLTests = []struct { + name string + payload any + errorExpected bool +}{ + { + name: "valid", + payload: XMLResponse{ + Error: false, + Message: "foo", + }, + errorExpected: false, + }, + { + name: "invalid", + payload: make(chan int), + errorExpected: true, + }, +} + func TestTools_WriteXML(t *testing.T) { - // create a variable of type toolbox.Tools, and just use the defaults. - var testTools Tools + for _, e := range writeXMLTests { + // create a variable of type toolbox.Tools, and just use the defaults. + var testTools Tools - rr := httptest.NewRecorder() - payload := XMLResponse{ - Error: false, - Message: "foo", - } + rr := httptest.NewRecorder() - headers := make(http.Header) - headers.Add("FOO", "BAR") - err := testTools.WriteXML(rr, http.StatusOK, payload, headers) - if err != nil { - t.Errorf("failed to write XML: %v", err) + headers := make(http.Header) + headers.Add("FOO", "BAR") + err := testTools.WriteXML(rr, http.StatusOK, e.payload, headers) + if err != nil && !e.errorExpected { + t.Errorf("%s, failed to write XML: %v", e.name, err) + } + + if err == nil && e.errorExpected { + t.Errorf("%s: error expected, but none received", e.name) + } } } From 6415005fa2bb83005e8dd454aa6ad19df03545ee Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Mon, 10 Apr 2023 15:30:52 -0300 Subject: [PATCH 108/135] Add check for Content-Type header; simplify error checking; improve tests. --- tools.go | 15 +++++++++++---- tools_test.go | 10 ++++++++-- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/tools.go b/tools.go index 090a819..e766948 100644 --- a/tools.go +++ b/tools.go @@ -44,6 +44,16 @@ type XMLResponse struct { // ReadJSON tries to read the body of a request and converts it from JSON to a variable. func (t *Tools) ReadJSON(w http.ResponseWriter, r *http.Request, data interface{}) error { + + // check content-type header; it should be application/json. If it's not specified, try to decode the body anyway. + if r.Header.Get("Content-Type") != "" { + contentType := r.Header.Get("Content-Type") + if strings.ToLower(contentType) != "application/json" { + return errors.New("the Content-Type header is not application/json") + } + } + + // set a sensible default for the maximum payload size. maxBytes := 1024 * 1024 // one megabyte // if MaxJSONSize is set, use that value instead of default. @@ -75,10 +85,7 @@ func (t *Tools) ReadJSON(w http.ResponseWriter, r *http.Request, data interface{ return errors.New("body contains badly-formed JSON") case errors.As(err, &unmarshalTypeError): - if unmarshalTypeError.Field != "" { - return fmt.Errorf("body contains incorrect JSON type for field %q", unmarshalTypeError.Field) - } - return fmt.Errorf("body contains incorrect JSON type (at character %d)", unmarshalTypeError.Offset) + return fmt.Errorf("body contains incorrect JSON type for field %q at offset %d", unmarshalTypeError.Field, unmarshalTypeError.Offset) case errors.Is(err, io.EOF): return errors.New("body must not be empty") diff --git a/tools_test.go b/tools_test.go index a8e99b2..48e29f8 100644 --- a/tools_test.go +++ b/tools_test.go @@ -87,6 +87,7 @@ var jsonTests = []struct { errorExpected bool maxSize int allowUnknown bool + contentType string }{ {name: "good json", json: `{"foo": "bar"}`, errorExpected: false, maxSize: 1024, allowUnknown: false}, {name: "badly formatted json", json: `{"foo":"}`, errorExpected: true, maxSize: 1024, allowUnknown: false}, @@ -101,6 +102,7 @@ var jsonTests = []struct { {name: "missing field name", json: `{jack: "bar"}`, errorExpected: true, maxSize: 1024, allowUnknown: false}, {name: "file too large", json: `{"foo": "bar"}`, errorExpected: true, maxSize: 5, allowUnknown: false}, {name: "not json", json: `Hello, world`, errorExpected: true, maxSize: 1024, allowUnknown: false}, + {name: "wrong header", json: `{"foo": "bar"}`, errorExpected: true, maxSize: 1024, allowUnknown: false, contentType: "application/xml"}, } func TestTools_ReadJSON(t *testing.T) { @@ -114,8 +116,7 @@ func TestTools_ReadJSON(t *testing.T) { // declare a variable to read the decoded json into. var decodedJSON struct { - Foo string `json:"foo"` - Chan chan int `json:"chan"` + Foo string `json:"foo"` } // create a request with the body. @@ -123,6 +124,11 @@ func TestTools_ReadJSON(t *testing.T) { if err != nil { t.Log("Error", err) } + if e.contentType != "" { + req.Header.Add("Content-Type", e.contentType) + } else { + req.Header.Add("Content-Type", "application/json") + } // create a test response recorder, which satisfies the requirements // for a ResponseWriter. From 0a5726227fb39c831408d87aa1ea303f92e99aa3 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Mon, 10 Apr 2023 17:32:45 -0300 Subject: [PATCH 109/135] Lint --- tools.go | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/tools.go b/tools.go index e766948..e84235c 100644 --- a/tools.go +++ b/tools.go @@ -45,7 +45,8 @@ type XMLResponse struct { // ReadJSON tries to read the body of a request and converts it from JSON to a variable. func (t *Tools) ReadJSON(w http.ResponseWriter, r *http.Request, data interface{}) error { - // check content-type header; it should be application/json. If it's not specified, try to decode the body anyway. + // Check content-type header; it should be application/json. If it's not specified, + // try to decode the body anyway. if r.Header.Get("Content-Type") != "" { contentType := r.Header.Get("Content-Type") if strings.ToLower(contentType) != "application/json" { @@ -53,10 +54,10 @@ func (t *Tools) ReadJSON(w http.ResponseWriter, r *http.Request, data interface{ } } - // set a sensible default for the maximum payload size. + // Set a sensible default for the maximum payload size. maxBytes := 1024 * 1024 // one megabyte - // if MaxJSONSize is set, use that value instead of default. + // If MaxJSONSize is set, use that value instead of default. if t.MaxJSONSize != 0 { maxBytes = t.MaxJSONSize } @@ -64,12 +65,12 @@ func (t *Tools) ReadJSON(w http.ResponseWriter, r *http.Request, data interface{ dec := json.NewDecoder(r.Body) - // should we allow unknown fields? + // Should we allow unknown fields? if !t.AllowUnknownFields { dec.DisallowUnknownFields() } - // attempt to decode the data, and figure out what the error is, if any, to send back a human-readable + // Attempt to decode the data, and figure out what the error is, if any, to send back a human-readable // response. err := dec.Decode(data) if err != nil { @@ -120,14 +121,14 @@ func (t *Tools) WriteJSON(w http.ResponseWriter, status int, data interface{}, h return err } - // if we have a value as the last parameter in the function call, then we are setting a custom header. + // If we have a value as the last parameter in the function call, then we are setting a custom header. if len(headers) > 0 { for key, value := range headers[0] { w.Header()[key] = value } } - // set the content type and send response. + // Set the content type and send response. w.Header().Set("Content-Type", "application/json") w.WriteHeader(status) _, _ = w.Write(out) @@ -140,7 +141,7 @@ func (t *Tools) WriteJSON(w http.ResponseWriter, status int, data interface{}, h func (t *Tools) ErrorJSON(w http.ResponseWriter, err error, status ...int) error { statusCode := http.StatusBadRequest - // if a custom response code is specified, use that instead of bad request. + // If a custom response code is specified, use that instead of bad request. if len(status) > 0 { statusCode = status[0] } @@ -179,14 +180,14 @@ func (t *Tools) PushJSONToRemote(uri string, data interface{}, client ...*http.C httpClient = client[0] } - // build the request and set header. + // Build the request and set header. request, err := http.NewRequest("POST", uri, bytes.NewBuffer(jsonData)) if err != nil { return nil, 0, err } request.Header.Set("Content-Type", "application/json") - // call the uri. + // Call the url. response, err := httpClient.Do(request) if err != nil { return nil, 0, err @@ -241,18 +242,18 @@ func (t *Tools) UploadFiles(r *http.Request, uploadDir string, rename ...bool) ( var uploadedFiles []*UploadedFile - // create the upload directory if it does not exist. + // Create the upload directory if it does not exist. err := t.CreateDirIfNotExist(uploadDir) if err != nil { return nil, err } - // sanity check on t.MaxFileSize. + // Sanity check on t.MaxFileSize. if t.MaxFileSize == 0 { t.MaxFileSize = 1024 * 1024 * 5 // 5 megabytes. } - // parse the form so we have access to the file. + // Parse the form, so we have access to the file. err = r.ParseMultipartForm(int64(t.MaxFileSize)) if err != nil { return nil, fmt.Errorf("error parsing form data") @@ -365,14 +366,15 @@ func (t *Tools) WriteXML(w http.ResponseWriter, status int, data interface{}, he return err } - // if we have a value as the last parameter in the function call, then we are setting a custom header. + // If we have a value as the last parameter in the function call, then we are setting a custom header. if len(headers) > 0 { for key, value := range headers[0] { w.Header()[key] = value } } - // set the content type and send response. + // Set the content type and send response. According to RFC 7303, text/xml and application/xml are to be + // treated as the same, so we'll just pick one. w.Header().Set("Content-Type", "application/xml") w.WriteHeader(status) _, _ = w.Write(out) @@ -384,7 +386,7 @@ func (t *Tools) WriteXML(w http.ResponseWriter, status int, data interface{}, he func (t *Tools) ReadXML(w http.ResponseWriter, r *http.Request, data interface{}) error { maxBytes := 1024 * 1024 // one megabyte - // if MaxXMLSize is set, use that value instead of default. + // If MaxXMLSize is set, use that value instead of default. if t.MaxXMLSize != 0 { maxBytes = t.MaxXMLSize } @@ -392,7 +394,7 @@ func (t *Tools) ReadXML(w http.ResponseWriter, r *http.Request, data interface{} dec := xml.NewDecoder(r.Body) - // attempt to decode the data. + // Attempt to decode the data. err := dec.Decode(data) if err != nil { return err @@ -411,7 +413,7 @@ func (t *Tools) ReadXML(w http.ResponseWriter, r *http.Request, data interface{} func (t *Tools) ErrorXML(w http.ResponseWriter, err error, status ...int) error { statusCode := http.StatusBadRequest - // if a custom response code is specified, use that instead of bad request. + // If a custom response code is specified, use that instead of bad request. if len(status) > 0 { statusCode = status[0] } From f28124829ffa707a81646d5b219b8eaf907ca406 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Mon, 10 Apr 2023 17:35:19 -0300 Subject: [PATCH 110/135] comments --- tools.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tools.go b/tools.go index e84235c..d5b9c6c 100644 --- a/tools.go +++ b/tools.go @@ -42,7 +42,8 @@ type XMLResponse struct { Data interface{} `xml:"data,omitempty"` } -// ReadJSON tries to read the body of a request and converts it from JSON to a variable. +// ReadJSON tries to read the body of a request and converts it from JSON to a variable. The third parameter, data, +// is expected to be a pointer, so that we can read data into it. func (t *Tools) ReadJSON(w http.ResponseWriter, r *http.Request, data interface{}) error { // Check content-type header; it should be application/json. If it's not specified, @@ -360,6 +361,7 @@ func (t *Tools) Slugify(s string) (string, error) { } // WriteXML takes a response status code and arbitrary data and writes an XML response to the client. +// The Content-Type header is set to application/xml. func (t *Tools) WriteXML(w http.ResponseWriter, status int, data interface{}, headers ...http.Header) error { out, err := xml.Marshal(data) if err != nil { @@ -382,7 +384,8 @@ func (t *Tools) WriteXML(w http.ResponseWriter, status int, data interface{}, he return nil } -// ReadXML tries to read the body of an XML request into a variable. +// ReadXML tries to read the body of an XML request into a variable. The third parameter, data, +// is expected to be a pointer, so that we can read data into it. func (t *Tools) ReadXML(w http.ResponseWriter, r *http.Request, data interface{}) error { maxBytes := 1024 * 1024 // one megabyte From 907638cd3556739b4c5a29ce713b301d2976897e Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Tue, 18 Apr 2023 12:49:03 -0300 Subject: [PATCH 111/135] badge --- .github/workflows/tests.yml | 6 +++++- readme.md | 2 ++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3aebe45..a4ff896 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -5,6 +5,7 @@ on: branches: [main] pull_request: branches: [main] + workflow_dispatch: jobs: @@ -37,4 +38,7 @@ jobs: run: golint ./... - name: Run tests - run: go test -race -vet=off ./... \ No newline at end of file + run: go test -race -vet=off ./... + + - name: Update coverage report + uses: ncruces/go-coverage-report@main \ No newline at end of file diff --git a/readme.md b/readme.md index 955f5ab..c968b35 100644 --- a/readme.md +++ b/readme.md @@ -4,6 +4,8 @@ [![Go Report Card](https://goreportcard.com/badge/github.com/tsawler/toolbox)](https://goreportcard.com/report/github.com/tsawler/toolbox) ![Tests](https://github.com/tsawler/toolbox/actions/workflows/tests.yml/badge.svg) +[![Go Coverage](https://github.com/tsawler/toolbox/wiki/coverage.svg)](https://raw.githack.com/wiki/tsawler/toolbox/coverage.html) + # Toolbox A simple example of how to create a reusable Go module with commonly used tools. From 0e097e2f8a8c08b5085c109026ca200be6a460bb Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Fri, 21 Apr 2023 15:29:20 -0300 Subject: [PATCH 112/135] Add comment. --- tools.go | 1 + 1 file changed, 1 insertion(+) diff --git a/tools.go b/tools.go index d5b9c6c..1d28ff0 100644 --- a/tools.go +++ b/tools.go @@ -147,6 +147,7 @@ func (t *Tools) ErrorJSON(w http.ResponseWriter, err error, status ...int) error statusCode = status[0] } + // Build the JSON payload. var payload JSONResponse payload.Error = true payload.Message = err.Error() From 57add5b861e7f920f032ec57cd4eaf12b2debe7b Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Wed, 26 Apr 2023 13:59:39 -0300 Subject: [PATCH 113/135] Add comment. --- tools.go | 1 + 1 file changed, 1 insertion(+) diff --git a/tools.go b/tools.go index 1d28ff0..074dcd1 100644 --- a/tools.go +++ b/tools.go @@ -16,6 +16,7 @@ import ( "strings" ) +// randomStringSource is the source for generating random strings. const randomStringSource = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0987654321_+" // Tools is the type for this package. Create a variable of this type, and you have access From 917e45aa10d3514657a533fefc3dec4517bc55f0 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Wed, 26 Apr 2023 14:02:58 -0300 Subject: [PATCH 114/135] Add DownloadLargeFile. --- tools.go | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/tools.go b/tools.go index 074dcd1..1ab63eb 100644 --- a/tools.go +++ b/tools.go @@ -209,6 +209,38 @@ func (t *Tools) DownloadStaticFile(w http.ResponseWriter, r *http.Request, p, fi http.ServeFile(w, r, fp) } +// DownloadLargeFile is a more efficient way of serving large files, since it creates a local file, sends an HTTP +// GET request to the URL, and writes the response body to the local file using io.Copy(). This function streams +// the data from the response body to the file, which is efficient for downloading large files. +func (t *Tools) DownloadLargeFile(filepath string, url string) error { + // Create the file to write to. + out, err := os.Create(filepath) + if err != nil { + return err + } + defer out.Close() + + // Get the data from the URL. + resp, err := http.Get(url) + if err != nil { + return err + } + defer resp.Body.Close() + + // Check server response. + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("bad status: %s", resp.Status) + } + + // Copy the data from the response body to the file + _, err = io.Copy(out, resp.Body) + if err != nil { + return err + } + + return nil +} + // UploadedFile is the type used for the uploaded file. type UploadedFile struct { NewFileName string From 252a33da88d84663742e56be61d7e2c2c6cf8064 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Wed, 26 Apr 2023 14:05:54 -0300 Subject: [PATCH 115/135] remove download method. --- tools.go | 32 -------------------------------- tools_test.go | 2 +- 2 files changed, 1 insertion(+), 33 deletions(-) diff --git a/tools.go b/tools.go index 1ab63eb..074dcd1 100644 --- a/tools.go +++ b/tools.go @@ -209,38 +209,6 @@ func (t *Tools) DownloadStaticFile(w http.ResponseWriter, r *http.Request, p, fi http.ServeFile(w, r, fp) } -// DownloadLargeFile is a more efficient way of serving large files, since it creates a local file, sends an HTTP -// GET request to the URL, and writes the response body to the local file using io.Copy(). This function streams -// the data from the response body to the file, which is efficient for downloading large files. -func (t *Tools) DownloadLargeFile(filepath string, url string) error { - // Create the file to write to. - out, err := os.Create(filepath) - if err != nil { - return err - } - defer out.Close() - - // Get the data from the URL. - resp, err := http.Get(url) - if err != nil { - return err - } - defer resp.Body.Close() - - // Check server response. - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("bad status: %s", resp.Status) - } - - // Copy the data from the response body to the file - _, err = io.Copy(out, resp.Body) - if err != nil { - return err - } - - return nil -} - // UploadedFile is the type used for the uploaded file. type UploadedFile struct { NewFileName string diff --git a/tools_test.go b/tools_test.go index 48e29f8..fc1060d 100644 --- a/tools_test.go +++ b/tools_test.go @@ -249,7 +249,7 @@ func TestTools_RandomString(t *testing.T) { } } -func TestTools_DownloadStaticFile(t *testing.T) { +func TestTools_DownloadLargeStaticFile(t *testing.T) { rr := httptest.NewRecorder() req, _ := http.NewRequest("GET", "/", nil) From ee458053b917637de16c2c9b4e438e5a78ed9bb8 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Thu, 27 Apr 2023 11:14:34 -0300 Subject: [PATCH 116/135] Improve tests --- tools_test.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tools_test.go b/tools_test.go index fc1060d..e0845a5 100644 --- a/tools_test.go +++ b/tools_test.go @@ -440,6 +440,16 @@ func TestTools_CreateDirIfNotExist(t *testing.T) { _ = os.Remove("./testdata/myDir") } +func TestTools_CreateDirIfNotExistInvalidDirectory(t *testing.T) { + var testTool Tools + + // we should not be able to create a directory at the root level (no permissions) + err := testTool.CreateDirIfNotExist("/mydir") + if err == nil { + t.Error(errors.New("able to create a directory where we should not be able to")) + } +} + var slugTests = []struct { name string s string From a226c53611f01de4b25450fbee1ba573844ee1fa Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Thu, 18 May 2023 11:28:42 -0300 Subject: [PATCH 117/135] Improve comments. --- tools.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools.go b/tools.go index 074dcd1..472dd0f 100644 --- a/tools.go +++ b/tools.go @@ -156,7 +156,7 @@ func (t *Tools) ErrorJSON(w http.ResponseWriter, err error, status ...int) error return t.WriteJSON(w, statusCode, payload) } -// RandomString returns a random string of letters of length n. +// RandomString returns a random string of letters of length n, using characters specified in randomStringSource. func (t *Tools) RandomString(n int) string { s, r := make([]rune, n), []rune(randomStringSource) for i := range s { @@ -200,7 +200,7 @@ func (t *Tools) PushJSONToRemote(uri string, data interface{}, client ...*http.C return response, response.StatusCode, nil } -// DownloadStaticFile downloads a file, and tries to force the browser to avoid displaying it in +// DownloadStaticFile downloads a file to the remote user, and tries to force the browser to avoid displaying it in // the browser window by setting content-disposition. It also allows specification of the display name. func (t *Tools) DownloadStaticFile(w http.ResponseWriter, r *http.Request, p, file, displayName string) { fp := path.Join(p, file) From a13c1a12a9a5a89527594abebe3fc0bba940a3cb Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Thu, 18 May 2023 11:34:56 -0300 Subject: [PATCH 118/135] Drop else and outdent block. --- tools.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tools.go b/tools.go index 472dd0f..f6ec75e 100644 --- a/tools.go +++ b/tools.go @@ -316,13 +316,12 @@ func (t *Tools) UploadFiles(r *http.Request, uploadDir string, rename ...bool) ( if outfile, err = os.Create(filepath.Join(uploadDir, uploadedFile.NewFileName)); nil != err { return nil, err - } else { - fileSize, err := io.Copy(outfile, infile) - if err != nil { - return nil, err - } - uploadedFile.FileSize = fileSize } + fileSize, err := io.Copy(outfile, infile) + if err != nil { + return nil, err + } + uploadedFile.FileSize = fileSize uploadedFiles = append(uploadedFiles, &uploadedFile) From f2548f172ed59a396c2b2ff526e6eacfd41a3e99 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Thu, 18 May 2023 11:45:21 -0300 Subject: [PATCH 119/135] Improve error message. --- tools.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools.go b/tools.go index f6ec75e..89d60c6 100644 --- a/tools.go +++ b/tools.go @@ -259,7 +259,7 @@ func (t *Tools) UploadFiles(r *http.Request, uploadDir string, rename ...bool) ( // Parse the form, so we have access to the file. err = r.ParseMultipartForm(int64(t.MaxFileSize)) if err != nil { - return nil, fmt.Errorf("error parsing form data") + return nil, fmt.Errorf("error parsing form data: %v", err) } for _, fHeaders := range r.MultipartForm.File { From bb0a19df87ed29a81ef99666e7bfeeee37933a78 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Wed, 14 Jun 2023 11:11:17 -0300 Subject: [PATCH 120/135] Add XML header to output. --- tools.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools.go b/tools.go index 89d60c6..8aa1245 100644 --- a/tools.go +++ b/tools.go @@ -380,7 +380,9 @@ func (t *Tools) WriteXML(w http.ResponseWriter, status int, data interface{}, he // treated as the same, so we'll just pick one. w.Header().Set("Content-Type", "application/xml") w.WriteHeader(status) - _, _ = w.Write(out) + // Add the XML header. + xmlOut := []byte(xml.Header + string(out)) + _, _ = w.Write(xmlOut) return nil } From 0e19b102c3c77e358c6814037ecabeba2ce12bac Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Wed, 14 Jun 2023 11:15:49 -0300 Subject: [PATCH 121/135] Formatting --- tools.go | 1 + 1 file changed, 1 insertion(+) diff --git a/tools.go b/tools.go index 8aa1245..4e3642a 100644 --- a/tools.go +++ b/tools.go @@ -380,6 +380,7 @@ func (t *Tools) WriteXML(w http.ResponseWriter, status int, data interface{}, he // treated as the same, so we'll just pick one. w.Header().Set("Content-Type", "application/xml") w.WriteHeader(status) + // Add the XML header. xmlOut := []byte(xml.Header + string(out)) _, _ = w.Write(xmlOut) From c9015c23e98220c9c41a6b6b0f76fdcc0d14392c Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Mon, 26 Jun 2023 11:14:08 -0300 Subject: [PATCH 122/135] correct path in badge --- readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme.md b/readme.md index c968b35..039f4ed 100644 --- a/readme.md +++ b/readme.md @@ -1,6 +1,6 @@ [![Version](https://img.shields.io/badge/goversion-1.19.x-blue.svg)](https://golang.org) Built with GoLang -[![License](http://img.shields.io/badge/license-mit-blue.svg?style=flat-square)](https://raw.githubusercontent.com/tsawler/goblender/master/LICENSE) +[![License](http://img.shields.io/badge/license-mit-blue.svg?style=flat-square)](https://raw.githubusercontent.com/tsawler/toolbox/master/LICENSE) [![Go Report Card](https://goreportcard.com/badge/github.com/tsawler/toolbox)](https://goreportcard.com/report/github.com/tsawler/toolbox) ![Tests](https://github.com/tsawler/toolbox/actions/workflows/tests.yml/badge.svg) From 5d40b141462cddbd09cdab80c6e028b71eb3d73c Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Thu, 17 Aug 2023 11:54:48 -0300 Subject: [PATCH 123/135] set larger default upload size --- tools.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tools.go b/tools.go index 4e3642a..43b5e4e 100644 --- a/tools.go +++ b/tools.go @@ -19,6 +19,9 @@ import ( // randomStringSource is the source for generating random strings. const randomStringSource = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0987654321_+" +// defaultMaxUpload is the default max upload size (10 mb) +const defaultMaxUpload = 10485760 + // Tools is the type for this package. Create a variable of this type, and you have access // to all the exported methods with the receiver type *Tools. type Tools struct { @@ -57,7 +60,7 @@ func (t *Tools) ReadJSON(w http.ResponseWriter, r *http.Request, data interface{ } // Set a sensible default for the maximum payload size. - maxBytes := 1024 * 1024 // one megabyte + maxBytes := defaultMaxUpload // If MaxJSONSize is set, use that value instead of default. if t.MaxJSONSize != 0 { @@ -253,7 +256,7 @@ func (t *Tools) UploadFiles(r *http.Request, uploadDir string, rename ...bool) ( // Sanity check on t.MaxFileSize. if t.MaxFileSize == 0 { - t.MaxFileSize = 1024 * 1024 * 5 // 5 megabytes. + t.MaxFileSize = defaultMaxUpload } // Parse the form, so we have access to the file. @@ -391,7 +394,7 @@ func (t *Tools) WriteXML(w http.ResponseWriter, status int, data interface{}, he // ReadXML tries to read the body of an XML request into a variable. The third parameter, data, // is expected to be a pointer, so that we can read data into it. func (t *Tools) ReadXML(w http.ResponseWriter, r *http.Request, data interface{}) error { - maxBytes := 1024 * 1024 // one megabyte + maxBytes := defaultMaxUpload // If MaxXMLSize is set, use that value instead of default. if t.MaxXMLSize != 0 { From 1d26ea1cee36b93bc0b42631f5cb0081ecc35ca7 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Thu, 23 Nov 2023 15:16:01 -0400 Subject: [PATCH 124/135] Add loggers. --- tools.go | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/tools.go b/tools.go index 43b5e4e..b89dcb3 100644 --- a/tools.go +++ b/tools.go @@ -8,6 +8,7 @@ import ( "errors" "fmt" "io" + "log" "net/http" "os" "path" @@ -25,11 +26,24 @@ const defaultMaxUpload = 10485760 // Tools is the type for this package. Create a variable of this type, and you have access // to all the exported methods with the receiver type *Tools. type Tools struct { - MaxJSONSize int // maximum size of JSON file we'll process - MaxXMLSize int // maximum size of XML file we'll process - MaxFileSize int // maximum size of uploaded files in bytes - AllowedFileTypes []string // allowed file types for upload (e.g. image/jpeg) - AllowUnknownFields bool // if set to true, allow unknown fields in JSON + MaxJSONSize int // maximum size of JSON file we'll process + MaxXMLSize int // maximum size of XML file we'll process + MaxFileSize int // maximum size of uploaded files in bytes + AllowedFileTypes []string // allowed file types for upload (e.g. image/jpeg) + AllowUnknownFields bool // if set to true, allow unknown fields in JSON + ErrorLog *log.Logger // the info log. + InfoLog *log.Logger // the error log. +} + +// New neturns a new toolbox with sensible defaults. +func New() Tools { + return Tools{ + MaxJSONSize: defaultMaxUpload, + MaxXMLSize: defaultMaxUpload, + MaxFileSize: defaultMaxUpload, + InfoLog: log.New(os.Stdout, "INFO\t", log.Ldate|log.Ltime), + ErrorLog: log.New(os.Stdout, "ERROR\t", log.Ldate|log.Ltime|log.Lshortfile), + } } // JSONResponse is the type used for sending JSON around. From 6e61f57b573283cd0324c16b9ffa3f9af677a5e7 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Thu, 23 Nov 2023 15:16:13 -0400 Subject: [PATCH 125/135] Add loggers. --- tools.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools.go b/tools.go index b89dcb3..ac165d9 100644 --- a/tools.go +++ b/tools.go @@ -35,7 +35,7 @@ type Tools struct { InfoLog *log.Logger // the error log. } -// New neturns a new toolbox with sensible defaults. +// New returns a new toolbox with sensible defaults. func New() Tools { return Tools{ MaxJSONSize: defaultMaxUpload, From 15b0b98ec1d7f21bd4bbf854987a4140e08f0537 Mon Sep 17 00:00:00 2001 From: Trevor Sawler Date: Thu, 23 Nov 2023 15:21:01 -0400 Subject: [PATCH 126/135] Update tests --- tools_test.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tools_test.go b/tools_test.go index e0845a5..cf3a069 100644 --- a/tools_test.go +++ b/tools_test.go @@ -55,6 +55,13 @@ var pushTests = []struct { }, } +func TestNew(t *testing.T) { + tools := New() + if tools.MaxXMLSize != defaultMaxUpload { + t.Error("wrong MaxXMLSize") + } +} + func TestTools_PushJSONToRemote(t *testing.T) { for _, e := range pushTests { client := NewTestClient(func(req *http.Request) *http.Response { From da425fd0c86e8a8121ccd21e81db936ec2c73d92 Mon Sep 17 00:00:00 2001 From: mstgnz Date: Sun, 24 Dec 2023 16:45:29 +0300 Subject: [PATCH 127/135] MaxJSONSize and MaxXMLSize must be greater than zero --- tools.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools.go b/tools.go index ac165d9..2dc9eb4 100644 --- a/tools.go +++ b/tools.go @@ -77,7 +77,7 @@ func (t *Tools) ReadJSON(w http.ResponseWriter, r *http.Request, data interface{ maxBytes := defaultMaxUpload // If MaxJSONSize is set, use that value instead of default. - if t.MaxJSONSize != 0 { + if t.MaxJSONSize > 0 { maxBytes = t.MaxJSONSize } r.Body = http.MaxBytesReader(w, r.Body, int64(maxBytes)) @@ -411,7 +411,7 @@ func (t *Tools) ReadXML(w http.ResponseWriter, r *http.Request, data interface{} maxBytes := defaultMaxUpload // If MaxXMLSize is set, use that value instead of default. - if t.MaxXMLSize != 0 { + if t.MaxXMLSize > 0 { maxBytes = t.MaxXMLSize } r.Body = http.MaxBytesReader(w, r.Body, int64(maxBytes)) From 2be358d05655ce50a20300b16d3494640b2a6a0c Mon Sep 17 00:00:00 2001 From: mstgnz Date: Fri, 29 Dec 2023 01:16:55 +0300 Subject: [PATCH 128/135] load sql --- load_sql.go | 78 +++++++++++++++++++++++++++++++++++ load_sql_test.go | 101 ++++++++++++++++++++++++++++++++++++++++++++++ testdata/test.sql | 8 ++++ tools.go | 15 +++++++ tools_test.go | 28 +++++++++++++ 5 files changed, 230 insertions(+) create mode 100644 load_sql.go create mode 100644 load_sql_test.go create mode 100644 testdata/test.sql diff --git a/load_sql.go b/load_sql.go new file mode 100644 index 0000000..8942a9c --- /dev/null +++ b/load_sql.go @@ -0,0 +1,78 @@ +package toolbox + +import ( + "bufio" + "errors" + "io/fs" + "os" + "strings" +) + +// LoadSQLQueries loads SQL queries from a file and populates the QUERY map. +// This tool aims to facilitate the use of the go language's database/sql standard library. +// Writing SQL queries directly in the code can make it messy, so writing SQL queries in .sql files +// and then calling them from the code helps prevent code clutter, +// allowing SQL queries to be centralized in one place for better organization. +func (t *Tools) LoadSQLQueries(fileName string) (map[string]string, error) { + query := make(map[string]string) + + file, err := os.Open(fileName) + if err != nil { + return query, err + } + defer func(file fs.File) { + _ = file.Close() + }(file) + + query, err = parseSQLQueries(file, query) + return query, err +} + +// parseSQLQueries reads the SQL queries from the provided file and populates the QUERY map. +func parseSQLQueries(file *os.File, query map[string]string) (map[string]string, error) { + scanner := bufio.NewScanner(file) + var key string + var queries []string + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if isSQLQuery(line) || len(key) > 0 { + if len(key) > 0 { + queries = append(queries, line) + if strings.HasSuffix(line, ";") { + query[key] = strings.Join(queries, " ") + key, queries = "", nil + } + } else { + key = extractKey(line) + } + } + } + if err := scanner.Err(); err != nil { + return query, errors.New("error reading file: " + err.Error()) + } + return query, nil +} + +// isSQLQuery checks if the given line is an SQL query or a comment. +func isSQLQuery(line string) bool { + var tools Tools + return tools.HasPrefixInList(line, []string{"-- ", "SELECT", "INSERT", "UPDATE", "DELETE"}) +} + +// extractKey extracts the key from the comment line. +func extractKey(line string) string { + if strings.HasPrefix(line, "-- ") { + return strings.Split(line, "-- ")[1] + } + return "" +} + +// HasPrefixInList is a prefix checker +func (t *Tools) HasPrefixInList(str string, prefixes []string) bool { + for _, prefix := range prefixes { + if strings.HasPrefix(str, prefix) { + return true + } + } + return false +} diff --git a/load_sql_test.go b/load_sql_test.go new file mode 100644 index 0000000..2b7710e --- /dev/null +++ b/load_sql_test.go @@ -0,0 +1,101 @@ +package toolbox + +import ( + "io/fs" + "os" + "strconv" + "strings" + "testing" +) + +var tools Tools + +func TestLoadSQLQueries(t *testing.T) { + tests := []struct { + fileName string + key string + value string + equal bool + err bool + }{ + {fileName: "./testdata/not.sql", key: "", value: "", equal: true, err: true}, + {fileName: "./testdata/not.sql", key: "not", value: "", equal: true, err: true}, + {fileName: "./testdata/not.sql", key: "not", value: "equal", equal: false, err: true}, + {fileName: "./testdata/test.sql", key: "TEST1", value: "WHERE ass.id=$1;", equal: true, err: false}, + {fileName: "./testdata/test.sql", key: "TEST1", value: "WHERE ass.id=$1", equal: false, err: false}, + {fileName: "./testdata/test.sql", key: "TEST2", value: "id = $1;", equal: true, err: false}, + } + for i, tt := range tests { + t.Run(strconv.Itoa(i), func(t *testing.T) { + query, err := tools.LoadSQLQueries(tt.fileName) + if (err != nil) != tt.err { + t.Errorf("LoadSQLQueries() error: %v, except: %v", err, tt.err) + } + if strings.HasSuffix(query[tt.key], tt.value) != tt.equal { + t.Errorf("LoadSQLQueries() error: %v, except: %v", err, tt.equal) + } + }) + } +} + +func TestParseSQLQueries(t *testing.T) { + file, err := os.Open("./testdata/test.sql") + defer func(file fs.File) { + _ = file.Close() + }(file) + if (err != nil) != false { + t.Errorf("File Open result: %v, expect: %v", false, true) + } + _, err = parseSQLQueries(file, make(map[string]string)) + if (err != nil) != false { + t.Errorf("parseSQLQueries() result: %v, expect: %v", false, true) + } +} + +func TestIsSQLQuery(t *testing.T) { + if result := isSQLQuery("-- "); result != true { + t.Errorf("isSQLQuery() result: %v, expect: %v", result, true) + } + if result := isSQLQuery("--"); result != false { + t.Errorf("isSQLQuery() result: %v, expect: %v", result, false) + } +} + +func TestExtractKey(t *testing.T) { + tests := []struct { + value string + expect string + }{ + {value: "-- ABC", expect: "ABC"}, + {value: "DEF", expect: ""}, + } + for i, tt := range tests { + t.Run(strconv.Itoa(i), func(t *testing.T) { + if result := extractKey(tt.value); result != tt.expect { + t.Errorf("extractKey() result: %v, expect: %v", result, tt.expect) + } + }) + } +} + +func TestHasPrefixInList(t *testing.T) { + type args struct { + key string + value []string + } + tests := []struct { + args args + expect bool + }{ + {args: args{key: "abc"}, expect: false}, + {args: args{key: "abc", value: []string{"abc", "def"}}, expect: true}, + {args: args{key: "xyz", value: []string{"abc", "def"}}, expect: false}, + } + for i, tt := range tests { + t.Run(strconv.Itoa(i), func(t *testing.T) { + if result := tools.HasPrefixInList(tt.args.key, tt.args.value); result != tt.expect { + t.Errorf("HasPrefixInList() result: %v, expect: %v", result, tt.expect) + } + }) + } +} diff --git a/testdata/test.sql b/testdata/test.sql new file mode 100644 index 0000000..42d69ed --- /dev/null +++ b/testdata/test.sql @@ -0,0 +1,8 @@ +-- TEST1 +SELECT * FROM assistant_services as ass + JOIN assistants as a ON a.id=ass.assistant_id + JOIN vehicles as v ON v.id=a.vehicle_id + WHERE ass.id=$1; + +-- TEST2 +UPDATE assistants SET provider_id = $1 WHERE id = $1; \ No newline at end of file diff --git a/tools.go b/tools.go index 2dc9eb4..a66d6aa 100644 --- a/tools.go +++ b/tools.go @@ -13,6 +13,7 @@ import ( "os" "path" "path/filepath" + "reflect" "regexp" "strings" ) @@ -448,3 +449,17 @@ func (t *Tools) ErrorXML(w http.ResponseWriter, err error, status ...int) error return t.WriteXML(w, statusCode, payload) } + +// InArray checks if a value exists in a slice. +func (t *Tools) InArray(val interface{}, array interface{}) bool { + arr := reflect.ValueOf(array) + if arr.Kind() != reflect.Slice { + return false + } + for i := 0; i < arr.Len(); i++ { + if reflect.DeepEqual(val, arr.Index(i).Interface()) { + return true + } + } + return false +} diff --git a/tools_test.go b/tools_test.go index cf3a069..cc70943 100644 --- a/tools_test.go +++ b/tools_test.go @@ -612,3 +612,31 @@ func TestTools_ErrorXML(t *testing.T) { t.Errorf("wrong status code returned; expected 503, but got %d", rr.Code) } } + +func TestTools_InArray(t *testing.T) { + var testTools Tools + if result := testTools.InArray(55, []int{23, 45, 46, 68}); result != false { + t.Errorf("InArray() result: %v, expect: %v", result, false) + } + if result := testTools.InArray(45, []int{23, 45, 46, 68}); result != true { + t.Errorf("InArray() result: %v, expect: %v", result, true) + } + if result := testTools.InArray(45, []string{"abc", "def"}); result != false { + t.Errorf("InArray() result: %v, expect: %v", result, false) + } + if result := testTools.InArray("def", []string{"abc", "def"}); result != true { + t.Errorf("InArray() result: %v, expect: %v", result, true) + } + type test struct { + name string + } + tests := []test{{name: "abc"}, {name: "def"}} + t1 := test{name: "def"} + t2 := test{name: "xyz"} + if result := testTools.InArray(t1, tests); result != true { + t.Errorf("InArray() result: %v, expect: %v", result, true) + } + if result := testTools.InArray(t2, tests); result != false { + t.Errorf("InArray() result: %v, expect: %v", result, false) + } +} From 5f7909946de9125403f01f9943426796945e75f2 Mon Sep 17 00:00:00 2001 From: mstgnz Date: Fri, 29 Dec 2023 01:46:51 +0300 Subject: [PATCH 129/135] test sql --- testdata/test.sql | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/testdata/test.sql b/testdata/test.sql index 42d69ed..da7ce85 100644 --- a/testdata/test.sql +++ b/testdata/test.sql @@ -5,4 +5,7 @@ SELECT * FROM assistant_services as ass WHERE ass.id=$1; -- TEST2 -UPDATE assistants SET provider_id = $1 WHERE id = $1; \ No newline at end of file +UPDATE assistants SET provider_id = $1 WHERE id = $1; + +-- CITIES +SELECT c.id, c.name, c.country_id FROM cities; \ No newline at end of file From 87d8aa0f271af8fc0b3056694578163e13f7b032 Mon Sep 17 00:00:00 2001 From: mstgnz Date: Fri, 29 Dec 2023 01:47:04 +0300 Subject: [PATCH 130/135] readme --- readme.md | 86 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/readme.md b/readme.md index 039f4ed..5b20c99 100644 --- a/readme.md +++ b/readme.md @@ -24,6 +24,8 @@ The included tools are: - Post JSON to a remote service - Create a directory, including all parent directories, if it does not already exist - Create a URL safe slug from a string +- InArray checks if a value exists in a slice +- Creating a QUERY map from a SQL file ## Installation @@ -412,4 +414,88 @@ Output from this is: ``` To slugify: hello, world! These are unsafe chars: こんにちは世界*!&^% Slugified: hello-world-these-are-unsafe-chars +``` + +### Value exists in a slice +It is a method we often use when writing code. Is the value we are looking for present in the array? The type of this array can be of any type. Example: + +```go +package main + +import( + "fmt" + "github.com/tsawler/toolbox" +) + +func main(){ + var tools toolbox.Tools + + tests := []test{{name: "abc"}, {name: "def"}} + t1 := test{name: "def"} + t2 := test{name: "xyz"} + + if tools.InArray(t1, tests) { + fmt.Println("This slice contains the key you are looking for.") + } + + if !tools.InArray(t2, tests) { + fmt.Println("This slice does not contain the key you are looking for.") + } +} +``` + +### Creating a QUERY map from a SQL file +This tool aims to facilitate the use of the go language's database/sql standard library. Writing SQL queries directly in the code can make it messy, so writing SQL queries in .sql files and then calling them from the code helps prevent code clutter, allowing SQL queries to be centralized in one place for better organization. Example: + +```go +package main + +import( + "database/sql" + "encoding/json" + "log" + "github.com/tsawler/toolbox" +) + +type City struct { + ID int `json:"id,omitempty"` + Name string `json:"name,omitempty"` + CountryID int `json:"country_id,omitempty"` +} + +var DB *sql.DB +var QUERY map[string]string + +func main(){ + var tools toolbox.Tools + + // Load Sql + QUERY = make(map[string]string) + if query, err := tools.LoadSQLQueries("./testdata/test.sql"); err != nil { + log.Fatalf("Load Sql Error: %v", err) + } else { + QUERY = query + } + + var cities []City + rows, err := DB.Query(QUERY["CITIES"]) + if err != nil { + log.Printf("GET Cities Error %v", err) + return + } + defer func(rows *sql.Rows) { + _ = rows.Close() + }(rows) + for rows.Next() { + var city City + _ = rows.Scan(&city.ID, &city.Name, &city.CountryID) + cities = append(cities, city) + } + + if marshal, err := json.MarshalIndent(cities, "", " "); err != nil { + return + }else{ + log.Println(string(marshal)) + } +} ``` \ No newline at end of file From 34cfd15c3718bcb774f266740e26908a8177bb5f Mon Sep 17 00:00:00 2001 From: mstgnz Date: Fri, 29 Dec 2023 23:15:40 +0300 Subject: [PATCH 131/135] some fix --- load_sql.go | 10 ++++------ load_sql_test.go | 6 +++--- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/load_sql.go b/load_sql.go index 8942a9c..e20abde 100644 --- a/load_sql.go +++ b/load_sql.go @@ -3,7 +3,6 @@ package toolbox import ( "bufio" "errors" - "io/fs" "os" "strings" ) @@ -20,7 +19,7 @@ func (t *Tools) LoadSQLQueries(fileName string) (map[string]string, error) { if err != nil { return query, err } - defer func(file fs.File) { + defer func(file *os.File) { _ = file.Close() }(file) @@ -55,8 +54,7 @@ func parseSQLQueries(file *os.File, query map[string]string) (map[string]string, // isSQLQuery checks if the given line is an SQL query or a comment. func isSQLQuery(line string) bool { - var tools Tools - return tools.HasPrefixInList(line, []string{"-- ", "SELECT", "INSERT", "UPDATE", "DELETE"}) + return hasPrefixInList(line, []string{"-- ", "SELECT", "INSERT", "UPDATE", "DELETE"}) } // extractKey extracts the key from the comment line. @@ -67,8 +65,8 @@ func extractKey(line string) string { return "" } -// HasPrefixInList is a prefix checker -func (t *Tools) HasPrefixInList(str string, prefixes []string) bool { +// hasPrefixInList is a prefix checker +func hasPrefixInList(str string, prefixes []string) bool { for _, prefix := range prefixes { if strings.HasPrefix(str, prefix) { return true diff --git a/load_sql_test.go b/load_sql_test.go index 2b7710e..6c451e6 100644 --- a/load_sql_test.go +++ b/load_sql_test.go @@ -93,9 +93,9 @@ func TestHasPrefixInList(t *testing.T) { } for i, tt := range tests { t.Run(strconv.Itoa(i), func(t *testing.T) { - if result := tools.HasPrefixInList(tt.args.key, tt.args.value); result != tt.expect { - t.Errorf("HasPrefixInList() result: %v, expect: %v", result, tt.expect) + if result := hasPrefixInList(tt.args.key, tt.args.value); result != tt.expect { + t.Errorf("hasPrefixInList() result: %v, expect: %v", result, tt.expect) } }) } -} +} \ No newline at end of file From 47835fe9e593918ca8b44735cb222c0bc89a2d03 Mon Sep 17 00:00:00 2001 From: mstgnz Date: Fri, 29 Dec 2023 23:31:18 +0300 Subject: [PATCH 132/135] some fix --- load_sql_test.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/load_sql_test.go b/load_sql_test.go index 6c451e6..4bb8417 100644 --- a/load_sql_test.go +++ b/load_sql_test.go @@ -1,7 +1,6 @@ package toolbox import ( - "io/fs" "os" "strconv" "strings" @@ -40,7 +39,7 @@ func TestLoadSQLQueries(t *testing.T) { func TestParseSQLQueries(t *testing.T) { file, err := os.Open("./testdata/test.sql") - defer func(file fs.File) { + defer func(file *os.File) { _ = file.Close() }(file) if (err != nil) != false { @@ -98,4 +97,4 @@ func TestHasPrefixInList(t *testing.T) { } }) } -} \ No newline at end of file +} From 991eb092a5f5ff40c4a3f065badaf6fd20ba3530 Mon Sep 17 00:00:00 2001 From: mstgnz Date: Fri, 5 Jan 2024 22:57:27 +0300 Subject: [PATCH 133/135] change func name --- tools.go | 4 ++-- tools_test.go | 24 ++++++++++++------------ 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/tools.go b/tools.go index a66d6aa..87fb192 100644 --- a/tools.go +++ b/tools.go @@ -450,8 +450,8 @@ func (t *Tools) ErrorXML(w http.ResponseWriter, err error, status ...int) error return t.WriteXML(w, statusCode, payload) } -// InArray checks if a value exists in a slice. -func (t *Tools) InArray(val interface{}, array interface{}) bool { +// ContainsElement checks if a value exists in a slice. +func (t *Tools) ContainsElement(val interface{}, array interface{}) bool { arr := reflect.ValueOf(array) if arr.Kind() != reflect.Slice { return false diff --git a/tools_test.go b/tools_test.go index cc70943..82be010 100644 --- a/tools_test.go +++ b/tools_test.go @@ -615,17 +615,17 @@ func TestTools_ErrorXML(t *testing.T) { func TestTools_InArray(t *testing.T) { var testTools Tools - if result := testTools.InArray(55, []int{23, 45, 46, 68}); result != false { - t.Errorf("InArray() result: %v, expect: %v", result, false) + if result := testTools.ContainsElement(55, []int{23, 45, 46, 68}); result != false { + t.Errorf("ContainsElement() result: %v, expect: %v", result, false) } - if result := testTools.InArray(45, []int{23, 45, 46, 68}); result != true { - t.Errorf("InArray() result: %v, expect: %v", result, true) + if result := testTools.ContainsElement(45, []int{23, 45, 46, 68}); result != true { + t.Errorf("ContainsElement() result: %v, expect: %v", result, true) } - if result := testTools.InArray(45, []string{"abc", "def"}); result != false { - t.Errorf("InArray() result: %v, expect: %v", result, false) + if result := testTools.ContainsElement(45, []string{"abc", "def"}); result != false { + t.Errorf("ContainsElement() result: %v, expect: %v", result, false) } - if result := testTools.InArray("def", []string{"abc", "def"}); result != true { - t.Errorf("InArray() result: %v, expect: %v", result, true) + if result := testTools.ContainsElement("def", []string{"abc", "def"}); result != true { + t.Errorf("ContainsElement() result: %v, expect: %v", result, true) } type test struct { name string @@ -633,10 +633,10 @@ func TestTools_InArray(t *testing.T) { tests := []test{{name: "abc"}, {name: "def"}} t1 := test{name: "def"} t2 := test{name: "xyz"} - if result := testTools.InArray(t1, tests); result != true { - t.Errorf("InArray() result: %v, expect: %v", result, true) + if result := testTools.ContainsElement(t1, tests); result != true { + t.Errorf("ContainsElement() result: %v, expect: %v", result, true) } - if result := testTools.InArray(t2, tests); result != false { - t.Errorf("InArray() result: %v, expect: %v", result, false) + if result := testTools.ContainsElement(t2, tests); result != false { + t.Errorf("ContainsElement() result: %v, expect: %v", result, false) } } From 9808e788789d7ea0663815b36689b6fc0d00d315 Mon Sep 17 00:00:00 2001 From: mstgnz Date: Sat, 6 Jan 2024 16:52:55 +0300 Subject: [PATCH 134/135] readme --- readme.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/readme.md b/readme.md index 5b20c99..f0abb7c 100644 --- a/readme.md +++ b/readme.md @@ -24,7 +24,7 @@ The included tools are: - Post JSON to a remote service - Create a directory, including all parent directories, if it does not already exist - Create a URL safe slug from a string -- InArray checks if a value exists in a slice +- ContainsElement checks if a value exists in a slice - Creating a QUERY map from a SQL file ## Installation @@ -417,7 +417,7 @@ Slugified: hello-world-these-are-unsafe-chars ``` ### Value exists in a slice -It is a method we often use when writing code. Is the value we are looking for present in the array? The type of this array can be of any type. Example: +It is a method we often use when writing code. Is the value we are looking for present in the slice? The type of this slice can be of any type. Example: ```go package main @@ -434,11 +434,11 @@ func main(){ t1 := test{name: "def"} t2 := test{name: "xyz"} - if tools.InArray(t1, tests) { + if tools.ContainsElement(t1, tests) { fmt.Println("This slice contains the key you are looking for.") } - if !tools.InArray(t2, tests) { + if !tools.ContainsElement(t2, tests) { fmt.Println("This slice does not contain the key you are looking for.") } } From 31ac0d93fc3d6ef5f5e6e5d7b29f6d1d1d98ba27 Mon Sep 17 00:00:00 2001 From: Mesut GENEZ Date: Wed, 5 Jun 2024 16:41:10 +0300 Subject: [PATCH 135/135] fix load sql --- load_sql.go | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/load_sql.go b/load_sql.go index e20abde..24271e8 100644 --- a/load_sql.go +++ b/load_sql.go @@ -2,7 +2,7 @@ package toolbox import ( "bufio" - "errors" + "fmt" "os" "strings" ) @@ -19,9 +19,9 @@ func (t *Tools) LoadSQLQueries(fileName string) (map[string]string, error) { if err != nil { return query, err } - defer func(file *os.File) { + defer func() { _ = file.Close() - }(file) + }() query, err = parseSQLQueries(file, query) return query, err @@ -31,15 +31,17 @@ func (t *Tools) LoadSQLQueries(fileName string) (map[string]string, error) { func parseSQLQueries(file *os.File, query map[string]string) (map[string]string, error) { scanner := bufio.NewScanner(file) var key string - var queries []string + var queryBuilder strings.Builder for scanner.Scan() { line := strings.TrimSpace(scanner.Text()) if isSQLQuery(line) || len(key) > 0 { if len(key) > 0 { - queries = append(queries, line) if strings.HasSuffix(line, ";") { - query[key] = strings.Join(queries, " ") - key, queries = "", nil + queryBuilder.WriteString(line) + query[key] = queryBuilder.String() + key, queryBuilder = "", strings.Builder{} + } else { + queryBuilder.WriteString(line + " ") } } else { key = extractKey(line) @@ -47,7 +49,7 @@ func parseSQLQueries(file *os.File, query map[string]string) (map[string]string, } } if err := scanner.Err(); err != nil { - return query, errors.New("error reading file: " + err.Error()) + return query, fmt.Errorf("error reading file: %w", err) } return query, nil }