Mirror
/
seaweedfs
mirror of https://github.com/seaweedfs/seaweedfs.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Fix/parse upload filename (#6241) 

* fix: parse filename in PUT + refactor

* fix: master iu public url with http

* fix: better parsing and handle disposition header

* fix: take mime type from file extension if not set
pull/6249/head
Kevin Bulteel 1 month ago
committed by GitHub
parent
8ebbf37e8e
commit
4a0dc883cf
No known key found for this signature in database GPG Key ID: B5690EEEBB952194
4 changed files with 123 additions and 134 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 2
      weed/operation/needle_parse_test.go
  2. 8
      weed/server/master_ui/master.html
  3. 15
      weed/server/master_ui/templates.go
  4. 98
      weed/storage/needle/needle_parse_upload.go

2
weed/operation/needle_parse_test.go
View File

@ -44,7 +44,7 @@ func TestCreateNeedleFromRequest(t *testing.T) {
{
mockClient.needleHandling = func(n *needle.Needle, originalSize int, err error) {
assert.Equal(t, nil, err, "upload: %v", err)
assert.Equal(t, "", string(n.Mime), "mime detection failed: %v", string(n.Mime))
assert.Equal(t, "text/plain; charset=utf-8", string(n.Mime), "mime detection failed: %v", string(n.Mime))
assert.Equal(t, true, n.IsCompressed(), "this should be compressed")
assert.Equal(t, true, util.IsGzippedContent(n.Data), "this should be gzip")
fmt.Printf("needle: %v, originalSize: %d\n", n, originalSize)

8
weed/server/master_ui/master.html
View File

@ -33,14 +33,14 @@
{{ with .RaftServer }}
<tr>
<th>Leader</th>
<td><a href="http://{{ .Leader }}">{{ .Leader }}</a></td>
<td><a href="{{ url .Leader }}">{{ .Leader }}</a></td>
</tr>
<tr>
<th>Other Masters</th>
<td class="col-sm-5">
<ul class="list-unstyled">
{{ range $k, $p := .Peers }}
<li><a href="http://{{ $p.Name }}/ui/index.html">{{ $p.Name }}</a></li>
<li><a href="{{ url $p.Name }}/ui/index.html">{{ $p.Name }}</a></li>
{{ end }}
</ul>
</td>
@ -88,9 +88,9 @@
<tr>
<td><code>{{ $dc.Id }}</code></td>
<td>{{ $rack.Id }}</td>
<td><a href="http://{{ $dn.Url }}/ui/index.html">{{ $dn.Url }}</a>
<td><a href="{{ url $dn.Url }}/ui/index.html">{{ $dn.Url }}</a>
{{ if ne $dn.PublicUrl $dn.Url }}
/ <a href="http://{{ $dn.PublicUrl }}/ui/index.html">{{ $dn.PublicUrl }}</a>
/ <a href="{{ url $dn.PublicUrl }}/ui/index.html">{{ $dn.PublicUrl }}</a>
{{ end }}
</td>
<td>{{ $dn.Volumes }}</td>

15
weed/server/master_ui/templates.go
View File

@ -3,6 +3,7 @@ package master_ui
import (
_ "embed"
"html/template"
"strings"
)
//go:embed master.html
@ -11,5 +12,17 @@ var masterHtml string
//go:embed masterNewRaft.html
var masterNewRaftHtml string
var StatusTpl = template.Must(template.New("status").Parse(masterHtml))
var templateFunctions = template.FuncMap{
"url": func(input string) string {
if !strings.HasPrefix(input, "http://") && !strings.HasPrefix(input, "https://") {
return "http://" + input
}
return input
},
}
var StatusTpl = template.Must(template.New("status").Funcs(templateFunctions).Parse(masterHtml))
var StatusNewRaftTpl = template.Must(template.New("status").Parse(masterNewRaftHtml))

98
weed/storage/needle/needle_parse_upload.go
View File

@ -43,19 +43,8 @@ func ParseUpload(r *http.Request, sizeLimit int64, bytesBuffer *bytes.Buffer) (p
}
}
if r.Method == http.MethodPost {
contentType := r.Header.Get("Content-Type")
// If content-type is explicitly set, upload the file without parsing form-data
if contentType != "" && !strings.Contains(contentType, "form-data") {
e = parseRawPost(r, sizeLimit, pu)
} else {
e = parseMultipart(r, sizeLimit, pu)
}
e = parseUpload(r, sizeLimit, pu)
} else {
e = parsePut(r, sizeLimit, pu)
}
if e != nil {
return
}
@ -108,28 +97,21 @@ func ParseUpload(r *http.Request, sizeLimit int64, bytesBuffer *bytes.Buffer) (p
return
}
func parsePut(r *http.Request, sizeLimit int64, pu *ParsedUpload) error {
pu.IsGzipped = r.Header.Get("Content-Encoding") == "gzip"
// pu.IsZstd = r.Header.Get("Content-Encoding") == "zstd"
pu.MimeType = r.Header.Get("Content-Type")
pu.FileName = ""
dataSize, err := pu.bytesBuffer.ReadFrom(io.LimitReader(r.Body, sizeLimit+1))
if err == io.EOF || dataSize == sizeLimit+1 {
io.Copy(io.Discard, r.Body)
}
pu.Data = pu.bytesBuffer.Bytes()
r.Body.Close()
return nil
}
func parseUpload(r *http.Request, sizeLimit int64, pu *ParsedUpload) (e error) {
func parseMultipart(r *http.Request, sizeLimit int64, pu *ParsedUpload) (e error) {
defer func() {
if e != nil && r.Body != nil {
io.Copy(io.Discard, r.Body)
r.Body.Close()
}
}()
contentType := r.Header.Get("Content-Type")
var dataSize int64
if r.Method == http.MethodPost && (contentType == "" || strings.Contains(contentType, "form-data")) {
form, fe := r.MultipartReader()
if fe != nil {
glog.V(0).Infoln("MultipartReader [ERROR]", fe)
e = fe
@ -149,7 +131,6 @@ func parseMultipart(r *http.Request, sizeLimit int64, pu *ParsedUpload) (e error
pu.FileName = path.Base(pu.FileName)
}
var dataSize int64
dataSize, e = pu.bytesBuffer.ReadFrom(io.LimitReader(part, sizeLimit+1))
if e != nil {
glog.V(0).Infoln("Reading Content [ERROR]", e)
@ -161,6 +142,8 @@ func parseMultipart(r *http.Request, sizeLimit int64, pu *ParsedUpload) (e error
}
pu.Data = pu.bytesBuffer.Bytes()
contentType = part.Header.Get("Content-Type")
// if the filename is empty string, do a search on the other multi-part items
for pu.FileName == "" {
part2, fe := form.NextPart()
@ -187,49 +170,38 @@ func parseMultipart(r *http.Request, sizeLimit int64, pu *ParsedUpload) (e error
// update
pu.Data = pu.bytesBuffer.Bytes()
pu.FileName = path.Base(fName)
contentType = part.Header.Get("Content-Type")
part = part2
break
}
}
pu.IsChunkedFile, _ = strconv.ParseBool(r.FormValue("cm"))
pu.IsGzipped = part.Header.Get("Content-Encoding") == "gzip"
// pu.IsZstd = part.Header.Get("Content-Encoding") == "zstd"
if !pu.IsChunkedFile {
} else {
disposition := r.Header.Get("Content-Disposition")
dotIndex := strings.LastIndex(pu.FileName, ".")
ext, mtype := "", ""
if dotIndex > 0 {
ext = strings.ToLower(pu.FileName[dotIndex:])
mtype = mime.TypeByExtension(ext)
}
contentType := part.Header.Get("Content-Type")
if contentType != "" && contentType != "application/octet-stream" && mtype != contentType {
pu.MimeType = contentType // only return mime type if not deducible
mtype = contentType
}
if strings.Contains(disposition, "name=") {
if !strings.HasPrefix(disposition, "inline") && !strings.HasPrefix(disposition, "attachment") {
disposition = "attachment; " + disposition
}
pu.IsGzipped = part.Header.Get("Content-Encoding") == "gzip"
// pu.IsZstd = part.Header.Get("Content-Encoding") == "zstd"
return
}
_, mediaTypeParams, err := mime.ParseMediaType(disposition)
func parseRawPost(r *http.Request, sizeLimit int64, pu *ParsedUpload) (e error) {
if err == nil {
dpFilename, hasFilename := mediaTypeParams["filename"]
dpName, hasName := mediaTypeParams["name"]
defer func() {
if e != nil && r.Body != nil {
io.Copy(io.Discard, r.Body)
r.Body.Close()
if hasFilename {
pu.FileName = dpFilename
} else if hasName {
pu.FileName = dpName
}
}()
pu.FileName = r.Header.Get("Content-Disposition")
if pu.FileName != "" && strings.Contains(pu.FileName, "filename=") {
parts := strings.Split(pu.FileName, "filename=")
parts = strings.Split(parts[1], "\"")
}
pu.FileName = parts[1]
} else {
pu.FileName = ""
}
@ -240,7 +212,6 @@ func parseRawPost(r *http.Request, sizeLimit int64, pu *ParsedUpload) (e error)
pu.FileName = path.Base(r.URL.Path)
}
var dataSize int64
dataSize, e = pu.bytesBuffer.ReadFrom(io.LimitReader(r.Body, sizeLimit+1))
if e != nil {
@ -251,27 +222,32 @@ func parseRawPost(r *http.Request, sizeLimit int64, pu *ParsedUpload) (e error)
e = fmt.Errorf("file over the limited %d bytes", sizeLimit)
return
}
pu.Data = pu.bytesBuffer.Bytes()
pu.MimeType = contentType
pu.IsGzipped = r.Header.Get("Content-Encoding") == "gzip"
// pu.IsZstd = r.Header.Get("Content-Encoding") == "zstd"
}
pu.IsChunkedFile, _ = strconv.ParseBool(r.FormValue("cm"))
if !pu.IsChunkedFile {
dotIndex := strings.LastIndex(pu.FileName, ".")
ext, mtype := "", ""
if dotIndex > 0 {
ext = strings.ToLower(pu.FileName[dotIndex:])
mtype = mime.TypeByExtension(ext)
}
contentType := r.Header.Get("Content-Type")
if contentType != "" && contentType != "application/octet-stream" && mtype != contentType {
pu.MimeType = contentType // only return mime type if not deducible
mtype = contentType
} else if mtype != "" && pu.MimeType == "" && mtype != "application/octet-stream" {
pu.MimeType = mtype
}
}
pu.IsGzipped = r.Header.Get("Content-Encoding") == "gzip"
// pu.IsZstd = r.Header.Get("Content-Encoding") == "zstd"
return
}
Write Preview
Loading…
Cancel
Save