Docker
/
linx-server
1
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity
Browse Source

Clean up upload handling 

* Bail out on files that are too large earlier if possible.
* Return 400 instead of 500 for empty files and files that are too large
  (when we can bail out early).
pull/156/head
mutantmonkey 6 years ago
parent
7a38b6e0df
commit
b46010db83
6 changed files with 84 additions and 66 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 3
      backends/localfs/localfs.go
  2. 3
      backends/s3/s3.go
  3. 1
      backends/storage.go
  4. 35
      pages.go
  5. 32
      server_test.go
  6. 76
      upload.go

3
backends/localfs/localfs.go
View File

@ -2,7 +2,6 @@ package localfs
import ( import (
"encoding/json" "encoding/json"
"errors"
"io" "io"
"io/ioutil" "io/ioutil"
"os" "os"
@ -122,7 +121,7 @@ func (b LocalfsBackend) Put(key string, r io.Reader, expiry time.Time, deleteKey
bytes, err := io.Copy(dst, r) bytes, err := io.Copy(dst, r)
if bytes == 0 { if bytes == 0 {
os.Remove(filePath) os.Remove(filePath)
return m, errors.New("Empty file")
return m, backends.FileEmptyError
} else if err != nil { } else if err != nil {
os.Remove(filePath) os.Remove(filePath)
return m, err return m, err

3
backends/s3/s3.go
View File

@ -1,7 +1,6 @@
package s3 package s3
import ( import (
"errors"
"io" "io"
"io/ioutil" "io/ioutil"
"os" "os"
@ -112,7 +111,7 @@ func (b S3Backend) Put(key string, r io.Reader, expiry time.Time, deleteKey stri
bytes, err := io.Copy(tmpDst, r) bytes, err := io.Copy(tmpDst, r)
if bytes == 0 { if bytes == 0 {
return m, errors.New("Empty file")
return m, backends.FileEmptyError
} else if err != nil { } else if err != nil {
return m, err return m, err
} }

1
backends/storage.go
View File

@ -24,3 +24,4 @@ type MetaStorageBackend interface {
} }
var NotFoundErr = errors.New("File not found.") var NotFoundErr = errors.New("File not found.")
var FileEmptyError = errors.New("Empty file")

35
pages.go
View File

@ -64,12 +64,10 @@ func oopsHandler(c web.C, w http.ResponseWriter, r *http.Request, rt RespType, m
w.WriteHeader(500) w.WriteHeader(500)
renderTemplate(Templates["oops.html"], pongo2.Context{"msg": msg}, r, w) renderTemplate(Templates["oops.html"], pongo2.Context{"msg": msg}, r, w)
return return
} else if rt == RespPLAIN { } else if rt == RespPLAIN {
w.WriteHeader(500) w.WriteHeader(500)
fmt.Fprintf(w, "%s", msg) fmt.Fprintf(w, "%s", msg)
return return
} else if rt == RespJSON { } else if rt == RespJSON {
js, _ := json.Marshal(map[string]string{ js, _ := json.Marshal(map[string]string{
"error": msg, "error": msg,
@ -79,7 +77,6 @@ func oopsHandler(c web.C, w http.ResponseWriter, r *http.Request, rt RespType, m
w.WriteHeader(500) w.WriteHeader(500)
w.Write(js) w.Write(js)
return return
} else if rt == RespAUTO { } else if rt == RespAUTO {
if strings.EqualFold("application/json", r.Header.Get("Accept")) { if strings.EqualFold("application/json", r.Header.Get("Accept")) {
oopsHandler(c, w, r, RespJSON, msg) oopsHandler(c, w, r, RespJSON, msg)
@ -89,11 +86,33 @@ func oopsHandler(c web.C, w http.ResponseWriter, r *http.Request, rt RespType, m
} }
} }
func badRequestHandler(c web.C, w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusBadRequest)
err := renderTemplate(Templates["400.html"], pongo2.Context{}, r, w)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
func badRequestHandler(c web.C, w http.ResponseWriter, r *http.Request, rt RespType, msg string) {
if rt == RespHTML {
w.WriteHeader(http.StatusBadRequest)
err := renderTemplate(Templates["400.html"], pongo2.Context{"msg": msg}, r, w)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
}
return
} else if rt == RespPLAIN {
w.WriteHeader(http.StatusBadRequest)
fmt.Fprintf(w, "%s", msg)
return
} else if rt == RespJSON {
js, _ := json.Marshal(map[string]string{
"error": msg,
})
w.Header().Set("Content-Type", "application/json; charset=UTF-8")
w.WriteHeader(http.StatusBadRequest)
w.Write(js)
return
} else if rt == RespAUTO {
if strings.EqualFold("application/json", r.Header.Get("Accept")) {
badRequestHandler(c, w, r, RespJSON, msg)
} else {
badRequestHandler(c, w, r, RespHTML, msg)
}
} }
} }

32
server_test.go
View File

@ -486,7 +486,6 @@ func TestPostJSONUploadMaxExpiry(t *testing.T) {
var myjson RespOkJSON var myjson RespOkJSON
err = json.Unmarshal([]byte(w.Body.String()), &myjson) err = json.Unmarshal([]byte(w.Body.String()), &myjson)
if err != nil { if err != nil {
fmt.Println(w.Body.String())
t.Fatal(err) t.Fatal(err)
} }
@ -643,13 +642,9 @@ func TestPostEmptyUpload(t *testing.T) {
mux.ServeHTTP(w, req) mux.ServeHTTP(w, req)
if w.Code != 500 {
if w.Code != 400 {
t.Log(w.Body.String()) t.Log(w.Body.String())
t.Fatalf("Status code is not 500, but %d", w.Code)
}
if !strings.Contains(w.Body.String(), "Empty file") {
t.Fatal("Response did not contain 'Empty file'")
t.Fatalf("Status code is not 400, but %d", w.Code)
} }
} }
@ -680,13 +675,9 @@ func TestPostTooLargeUpload(t *testing.T) {
mux.ServeHTTP(w, req) mux.ServeHTTP(w, req)
if w.Code != 500 {
if w.Code != 400 {
t.Log(w.Body.String()) t.Log(w.Body.String())
t.Fatalf("Status code is not 500, but %d", w.Code)
}
if !strings.Contains(w.Body.String(), "request body too large") {
t.Fatal("Response did not contain 'request body too large'")
t.Fatalf("Status code is not 400, but %d", w.Code)
} }
Config.maxSize = oldMaxSize Config.maxSize = oldMaxSize
@ -718,9 +709,9 @@ func TestPostEmptyJSONUpload(t *testing.T) {
mux.ServeHTTP(w, req) mux.ServeHTTP(w, req)
if w.Code != 500 {
if w.Code != 400 {
t.Log(w.Body.String()) t.Log(w.Body.String())
t.Fatalf("Status code is not 500, but %d", w.Code)
t.Fatalf("Status code is not 400, but %d", w.Code)
} }
var myjson RespErrJSON var myjson RespErrJSON
@ -729,7 +720,7 @@ func TestPostEmptyJSONUpload(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
if myjson.Error != "Could not upload file: Empty file" {
if myjson.Error != "Empty file" {
t.Fatal("Json 'error' was not 'Empty file' but " + myjson.Error) t.Fatal("Json 'error' was not 'Empty file' but " + myjson.Error)
} }
} }
@ -807,13 +798,8 @@ func TestPutEmptyUpload(t *testing.T) {
mux.ServeHTTP(w, req) mux.ServeHTTP(w, req)
if w.Code != 500 {
t.Log(w.Body.String())
t.Fatalf("Status code is not 500, but %d", w.Code)
}
if !strings.Contains(w.Body.String(), "Empty file") {
t.Fatal("Response did not contain 'Empty file'")
if w.Code != 400 {
t.Fatalf("Status code is not 400, but %d", w.Code)
} }
} }

76
upload.go
View File

@ -22,6 +22,7 @@ import (
"gopkg.in/h2non/filetype.v1" "gopkg.in/h2non/filetype.v1"
) )
var FileTooLargeError = errors.New("File too large.")
var fileBlacklist = map[string]bool{ var fileBlacklist = map[string]bool{
"favicon.ico": true, "favicon.ico": true,
"index.htm": true, "index.htm": true,
@ -34,10 +35,11 @@ var fileBlacklist = map[string]bool{
// Describes metadata directly from the user request // Describes metadata directly from the user request
type UploadRequest struct { type UploadRequest struct {
src io.Reader src io.Reader
size int64
filename string filename string
expiry time.Duration // Seconds until expiry, 0 = never expiry time.Duration // Seconds until expiry, 0 = never
deleteKey string // Empty string if not defined
randomBarename bool randomBarename bool
deletionKey string // Empty string if not defined
} }
// Metadata associated with a file as it would actually be stored // Metadata associated with a file as it would actually be stored
@ -48,7 +50,7 @@ type Upload struct {
func uploadPostHandler(c web.C, w http.ResponseWriter, r *http.Request) { func uploadPostHandler(c web.C, w http.ResponseWriter, r *http.Request) {
if !strictReferrerCheck(r, getSiteURL(r), []string{"Linx-Delete-Key", "Linx-Expiry", "Linx-Randomize", "X-Requested-With"}) { if !strictReferrerCheck(r, getSiteURL(r), []string{"Linx-Delete-Key", "Linx-Expiry", "Linx-Randomize", "X-Requested-With"}) {
badRequestHandler(c, w, r)
badRequestHandler(c, w, r, RespAUTO, "")
return return
} }
@ -65,38 +67,39 @@ func uploadPostHandler(c web.C, w http.ResponseWriter, r *http.Request) {
} }
defer file.Close() defer file.Close()
r.ParseForm()
if r.Form.Get("randomize") == "true" {
upReq.randomBarename = true
}
upReq.expiry = parseExpiry(r.Form.Get("expires"))
upReq.src = http.MaxBytesReader(w, file, Config.maxSize)
upReq.src = file
upReq.size = headers.Size
upReq.filename = headers.Filename upReq.filename = headers.Filename
} else { } else {
if r.FormValue("content") == "" {
oopsHandler(c, w, r, RespHTML, "Empty file")
if r.PostFormValue("content") == "" {
badRequestHandler(c, w, r, RespAUTO, "Empty file")
return return
} }
extension := r.FormValue("extension")
extension := r.PostFormValue("extension")
if extension == "" { if extension == "" {
extension = "txt" extension = "txt"
} }
content := r.FormValue("content")
if int64(len(content)) > Config.maxSize {
oopsHandler(c, w, r, RespJSON, "Content length exceeds max size")
return
}
content := r.PostFormValue("content")
upReq.src = strings.NewReader(content) upReq.src = strings.NewReader(content)
upReq.expiry = parseExpiry(r.FormValue("expires"))
upReq.filename = r.FormValue("filename") + "." + extension
upReq.size = int64(len(content))
upReq.filename = r.PostFormValue("filename") + "." + extension
}
upReq.expiry = parseExpiry(r.PostFormValue("expires"))
if r.PostFormValue("randomize") == "true" {
upReq.randomBarename = true
} }
upload, err := processUpload(upReq) upload, err := processUpload(upReq)
if strings.EqualFold("application/json", r.Header.Get("Accept")) { if strings.EqualFold("application/json", r.Header.Get("Accept")) {
if err != nil {
if err == FileTooLargeError || err == backends.FileEmptyError {
badRequestHandler(c, w, r, RespJSON, err.Error())
return
} else if err != nil {
oopsHandler(c, w, r, RespJSON, "Could not upload file: "+err.Error()) oopsHandler(c, w, r, RespJSON, "Could not upload file: "+err.Error())
return return
} }
@ -105,14 +108,16 @@ func uploadPostHandler(c web.C, w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json; charset=UTF-8") w.Header().Set("Content-Type", "application/json; charset=UTF-8")
w.Write(js) w.Write(js)
} else { } else {
if err != nil {
if err == FileTooLargeError || err == backends.FileEmptyError {
badRequestHandler(c, w, r, RespHTML, err.Error())
return
} else if err != nil {
oopsHandler(c, w, r, RespHTML, "Could not upload file: "+err.Error()) oopsHandler(c, w, r, RespHTML, "Could not upload file: "+err.Error())
return return
} }
http.Redirect(w, r, Config.sitePath+upload.Filename, 303) http.Redirect(w, r, Config.sitePath+upload.Filename, 303)
} }
} }
func uploadPutHandler(c web.C, w http.ResponseWriter, r *http.Request) { func uploadPutHandler(c web.C, w http.ResponseWriter, r *http.Request) {
@ -126,7 +131,10 @@ func uploadPutHandler(c web.C, w http.ResponseWriter, r *http.Request) {
upload, err := processUpload(upReq) upload, err := processUpload(upReq)
if strings.EqualFold("application/json", r.Header.Get("Accept")) { if strings.EqualFold("application/json", r.Header.Get("Accept")) {
if err != nil {
if err == FileTooLargeError || err == backends.FileEmptyError {
badRequestHandler(c, w, r, RespJSON, err.Error())
return
} else if err != nil {
oopsHandler(c, w, r, RespJSON, "Could not upload file: "+err.Error()) oopsHandler(c, w, r, RespJSON, "Could not upload file: "+err.Error())
return return
} }
@ -135,7 +143,10 @@ func uploadPutHandler(c web.C, w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json; charset=UTF-8") w.Header().Set("Content-Type", "application/json; charset=UTF-8")
w.Write(js) w.Write(js)
} else { } else {
if err != nil {
if err == FileTooLargeError || err == backends.FileEmptyError {
badRequestHandler(c, w, r, RespPLAIN, err.Error())
return
} else if err != nil {
oopsHandler(c, w, r, RespPLAIN, "Could not upload file: "+err.Error()) oopsHandler(c, w, r, RespPLAIN, "Could not upload file: "+err.Error())
return return
} }
@ -169,7 +180,7 @@ func uploadRemote(c web.C, w http.ResponseWriter, r *http.Request) {
upReq.filename = filepath.Base(grabUrl.Path) upReq.filename = filepath.Base(grabUrl.Path)
upReq.src = http.MaxBytesReader(w, resp.Body, Config.maxSize) upReq.src = http.MaxBytesReader(w, resp.Body, Config.maxSize)
upReq.deletionKey = r.FormValue("deletekey")
upReq.deleteKey = r.FormValue("deletekey")
upReq.randomBarename = r.FormValue("randomize") == "yes" upReq.randomBarename = r.FormValue("randomize") == "yes"
upReq.expiry = parseExpiry(r.FormValue("expiry")) upReq.expiry = parseExpiry(r.FormValue("expiry"))
@ -199,15 +210,18 @@ func uploadHeaderProcess(r *http.Request, upReq *UploadRequest) {
upReq.randomBarename = true upReq.randomBarename = true
} }
upReq.deletionKey = r.Header.Get("Linx-Delete-Key")
upReq.deleteKey = r.Header.Get("Linx-Delete-Key")
// Get seconds until expiry. Non-integer responses never expire. // Get seconds until expiry. Non-integer responses never expire.
expStr := r.Header.Get("Linx-Expiry") expStr := r.Header.Get("Linx-Expiry")
upReq.expiry = parseExpiry(expStr) upReq.expiry = parseExpiry(expStr)
} }
func processUpload(upReq UploadRequest) (upload Upload, err error) { func processUpload(upReq UploadRequest) (upload Upload, err error) {
if upReq.size > Config.maxSize {
return upload, FileTooLargeError
}
// Determine the appropriate filename, then write to disk // Determine the appropriate filename, then write to disk
barename, extension := barePlusExt(upReq.filename) barename, extension := barePlusExt(upReq.filename)
@ -221,7 +235,7 @@ func processUpload(upReq UploadRequest) (upload Upload, err error) {
header = make([]byte, 512) header = make([]byte, 512)
n, _ := upReq.src.Read(header) n, _ := upReq.src.Read(header)
if n == 0 { if n == 0 {
return upload, errors.New("Empty file")
return upload, backends.FileEmptyError
} }
header = header[:n] header = header[:n]
@ -243,7 +257,7 @@ func processUpload(upReq UploadRequest) (upload Upload, err error) {
if fileexists { if fileexists {
metad, merr := storageBackend.Head(upload.Filename) metad, merr := storageBackend.Head(upload.Filename)
if merr == nil { if merr == nil {
if upReq.deletionKey == metad.DeleteKey {
if upReq.deleteKey == metad.DeleteKey {
fileexists = false fileexists = false
} }
} }
@ -273,11 +287,11 @@ func processUpload(upReq UploadRequest) (upload Upload, err error) {
fileExpiry = time.Now().Add(upReq.expiry) fileExpiry = time.Now().Add(upReq.expiry)
} }
if upReq.deletionKey == "" {
upReq.deletionKey = uniuri.NewLen(30)
if upReq.deleteKey == "" {
upReq.deleteKey = uniuri.NewLen(30)
} }
upload.Metadata, err = storageBackend.Put(upload.Filename, io.MultiReader(bytes.NewReader(header), upReq.src), fileExpiry, upReq.deletionKey)
upload.Metadata, err = storageBackend.Put(upload.Filename, io.MultiReader(bytes.NewReader(header), upReq.src), fileExpiry, upReq.deleteKey)
if err != nil { if err != nil {
return upload, err return upload, err
} }

Write Preview
Loading…
Cancel
Save