Fix/parse upload filename (#6241)

* fix: parse filename in PUT + refactor

* fix: master iu public url with http

* fix: better parsing and handle disposition header

* fix: take mime type from file extension if not set

weed/operation/needle_parse_test.go

@@ -44,7 +44,7 @@ func TestCreateNeedleFromRequest(t *testing.T) {
 	{
 		mockClient.needleHandling = func(n *needle.Needle, originalSize int, err error) {
 			assert.Equal(t, nil, err, "upload: %v", err)
-			assert.Equal(t, "", string(n.Mime), "mime detection failed: %v", string(n.Mime))
+			assert.Equal(t, "text/plain; charset=utf-8", string(n.Mime), "mime detection failed: %v", string(n.Mime))
 			assert.Equal(t, true, n.IsCompressed(), "this should be compressed")
 			assert.Equal(t, true, util.IsGzippedContent(n.Data), "this should be gzip")
 			fmt.Printf("needle: %v, originalSize: %d\n", n, originalSize)

weed/server/master_ui/master.html

@@ -33,14 +33,14 @@
                 {{ with .RaftServer }}
                 <tr>
                     <th>Leader</th>
-                    <td><a href="http://{{ .Leader }}">{{ .Leader }}</a></td>
+                    <td><a href="{{ url .Leader }}">{{ .Leader }}</a></td>
                 </tr>
                 <tr>
                     <th>Other Masters</th>
                     <td class="col-sm-5">
                         <ul class="list-unstyled">
                             {{ range $k, $p := .Peers }}
-                            <li><a href="http://{{ $p.Name }}/ui/index.html">{{ $p.Name }}</a></li>
+                            <li><a href="{{ url $p.Name }}/ui/index.html">{{ $p.Name }}</a></li>
                             {{ end }}
                         </ul>
                     </td>
@@ -88,9 +88,9 @@
             <tr>
                 <td><code>{{ $dc.Id }}</code></td>
                 <td>{{ $rack.Id }}</td>
-                <td><a href="http://{{ $dn.Url }}/ui/index.html">{{ $dn.Url }}</a>
+                <td><a href="{{ url $dn.Url }}/ui/index.html">{{ $dn.Url }}</a>
                     {{ if ne $dn.PublicUrl $dn.Url }}
-                    / <a href="http://{{ $dn.PublicUrl }}/ui/index.html">{{ $dn.PublicUrl }}</a>
+                    / <a href="{{ url $dn.PublicUrl }}/ui/index.html">{{ $dn.PublicUrl }}</a>
                     {{ end }}
                 </td>
                 <td>{{ $dn.Volumes }}</td>

weed/server/master_ui/templates.go

@@ -3,6 +3,7 @@ package master_ui
 import (
 	_ "embed"
 	"html/template"
+	"strings"
 )
 
 //go:embed master.html
@@ -11,5 +12,17 @@ var masterHtml string
 //go:embed masterNewRaft.html
 var masterNewRaftHtml string
 
-var StatusTpl = template.Must(template.New("status").Parse(masterHtml))
+var templateFunctions = template.FuncMap{
+	"url": func(input string) string {
+
+		if !strings.HasPrefix(input, "http://") && !strings.HasPrefix(input, "https://") {
+			return "http://" + input
+		}
+
+		return input
+	},
+}
+
+var StatusTpl = template.Must(template.New("status").Funcs(templateFunctions).Parse(masterHtml))
+
 var StatusNewRaftTpl = template.Must(template.New("status").Parse(masterNewRaftHtml))

weed/storage/needle/needle_parse_upload.go

@@ -43,19 +43,8 @@ func ParseUpload(r *http.Request, sizeLimit int64, bytesBuffer *bytes.Buffer) (p
 		}
 	}
 
-	if r.Method == http.MethodPost {
-		contentType := r.Header.Get("Content-Type")
+	e = parseUpload(r, sizeLimit, pu)
 
-		// If content-type is explicitly set, upload the file without parsing form-data
-		if contentType != "" && !strings.Contains(contentType, "form-data") {
-			e = parseRawPost(r, sizeLimit, pu)
-		} else {
-			e = parseMultipart(r, sizeLimit, pu)
-		}
-
-	} else {
-		e = parsePut(r, sizeLimit, pu)
-	}
 	if e != nil {
 		return
 	}
@@ -108,170 +97,157 @@ func ParseUpload(r *http.Request, sizeLimit int64, bytesBuffer *bytes.Buffer) (p
 	return
 }
 
-func parsePut(r *http.Request, sizeLimit int64, pu *ParsedUpload) error {
-	pu.IsGzipped = r.Header.Get("Content-Encoding") == "gzip"
-	// pu.IsZstd = r.Header.Get("Content-Encoding") == "zstd"
-	pu.MimeType = r.Header.Get("Content-Type")
-	pu.FileName = ""
-	dataSize, err := pu.bytesBuffer.ReadFrom(io.LimitReader(r.Body, sizeLimit+1))
-	if err == io.EOF || dataSize == sizeLimit+1 {
-		io.Copy(io.Discard, r.Body)
-	}
-	pu.Data = pu.bytesBuffer.Bytes()
-	r.Body.Close()
-	return nil
-}
+func parseUpload(r *http.Request, sizeLimit int64, pu *ParsedUpload) (e error) {
 
-func parseMultipart(r *http.Request, sizeLimit int64, pu *ParsedUpload) (e error) {
 	defer func() {
 		if e != nil && r.Body != nil {
 			io.Copy(io.Discard, r.Body)
 			r.Body.Close()
 		}
 	}()
-	form, fe := r.MultipartReader()
-	if fe != nil {
-		glog.V(0).Infoln("MultipartReader [ERROR]", fe)
-		e = fe
-		return
-	}
 
-	// first multi-part item
-	part, fe := form.NextPart()
-	if fe != nil {
-		glog.V(0).Infoln("Reading Multi part [ERROR]", fe)
-		e = fe
-		return
-	}
+	contentType := r.Header.Get("Content-Type")
+	var dataSize int64
 
-	pu.FileName = part.FileName()
-	if pu.FileName != "" {
-		pu.FileName = path.Base(pu.FileName)
-	}
+	if r.Method == http.MethodPost && (contentType == "" || strings.Contains(contentType, "form-data")) {
+		form, fe := r.MultipartReader()
 
-	var dataSize int64
-	dataSize, e = pu.bytesBuffer.ReadFrom(io.LimitReader(part, sizeLimit+1))
-	if e != nil {
-		glog.V(0).Infoln("Reading Content [ERROR]", e)
-		return
-	}
-	if dataSize == sizeLimit+1 {
-		e = fmt.Errorf("file over the limited %d bytes", sizeLimit)
-		return
-	}
-	pu.Data = pu.bytesBuffer.Bytes()
+		if fe != nil {
+			glog.V(0).Infoln("MultipartReader [ERROR]", fe)
+			e = fe
+			return
+		}
 
-	// if the filename is empty string, do a search on the other multi-part items
-	for pu.FileName == "" {
-		part2, fe := form.NextPart()
+		// first multi-part item
+		part, fe := form.NextPart()
 		if fe != nil {
-			break // no more or on error, just safely break
+			glog.V(0).Infoln("Reading Multi part [ERROR]", fe)
+			e = fe
+			return
 		}
 
-		fName := part2.FileName()
+		pu.FileName = part.FileName()
+		if pu.FileName != "" {
+			pu.FileName = path.Base(pu.FileName)
+		}
 
-		// found the first <file type> multi-part has filename
-		if fName != "" {
-			pu.bytesBuffer.Reset()
-			dataSize2, fe2 := pu.bytesBuffer.ReadFrom(io.LimitReader(part2, sizeLimit+1))
-			if fe2 != nil {
-				glog.V(0).Infoln("Reading Content [ERROR]", fe2)
-				e = fe2
-				return
-			}
-			if dataSize2 == sizeLimit+1 {
-				e = fmt.Errorf("file over the limited %d bytes", sizeLimit)
-				return
+		dataSize, e = pu.bytesBuffer.ReadFrom(io.LimitReader(part, sizeLimit+1))
+		if e != nil {
+			glog.V(0).Infoln("Reading Content [ERROR]", e)
+			return
+		}
+		if dataSize == sizeLimit+1 {
+			e = fmt.Errorf("file over the limited %d bytes", sizeLimit)
+			return
+		}
+		pu.Data = pu.bytesBuffer.Bytes()
+
+		contentType = part.Header.Get("Content-Type")
+
+		// if the filename is empty string, do a search on the other multi-part items
+		for pu.FileName == "" {
+			part2, fe := form.NextPart()
+			if fe != nil {
+				break // no more or on error, just safely break
 			}
 
-			// update
-			pu.Data = pu.bytesBuffer.Bytes()
-			pu.FileName = path.Base(fName)
-			break
+			fName := part2.FileName()
+
+			// found the first <file type> multi-part has filename
+			if fName != "" {
+				pu.bytesBuffer.Reset()
+				dataSize2, fe2 := pu.bytesBuffer.ReadFrom(io.LimitReader(part2, sizeLimit+1))
+				if fe2 != nil {
+					glog.V(0).Infoln("Reading Content [ERROR]", fe2)
+					e = fe2
+					return
+				}
+				if dataSize2 == sizeLimit+1 {
+					e = fmt.Errorf("file over the limited %d bytes", sizeLimit)
+					return
+				}
+
+				// update
+				pu.Data = pu.bytesBuffer.Bytes()
+				pu.FileName = path.Base(fName)
+				contentType = part.Header.Get("Content-Type")
+				part = part2
+				break
+			}
 		}
-	}
 
-	pu.IsChunkedFile, _ = strconv.ParseBool(r.FormValue("cm"))
+		pu.IsGzipped = part.Header.Get("Content-Encoding") == "gzip"
+		// pu.IsZstd = part.Header.Get("Content-Encoding") == "zstd"
 
-	if !pu.IsChunkedFile {
+	} else {
+		disposition := r.Header.Get("Content-Disposition")
 
-		dotIndex := strings.LastIndex(pu.FileName, ".")
-		ext, mtype := "", ""
-		if dotIndex > 0 {
-			ext = strings.ToLower(pu.FileName[dotIndex:])
-			mtype = mime.TypeByExtension(ext)
-		}
-		contentType := part.Header.Get("Content-Type")
-		if contentType != "" && contentType != "application/octet-stream" && mtype != contentType {
-			pu.MimeType = contentType // only return mime type if not deducible
-			mtype = contentType
-		}
+		if strings.Contains(disposition, "name=") {
 
-	}
-	pu.IsGzipped = part.Header.Get("Content-Encoding") == "gzip"
-	// pu.IsZstd = part.Header.Get("Content-Encoding") == "zstd"
+			if !strings.HasPrefix(disposition, "inline") && !strings.HasPrefix(disposition, "attachment") {
+				disposition = "attachment; " + disposition
+			}
 
-	return
-}
+			_, mediaTypeParams, err := mime.ParseMediaType(disposition)
 
-func parseRawPost(r *http.Request, sizeLimit int64, pu *ParsedUpload) (e error) {
+			if err == nil {
+				dpFilename, hasFilename := mediaTypeParams["filename"]
+				dpName, hasName := mediaTypeParams["name"]
 
-	defer func() {
-		if e != nil && r.Body != nil {
-			io.Copy(io.Discard, r.Body)
-			r.Body.Close()
-		}
-	}()
+				if hasFilename {
+					pu.FileName = dpFilename
+				} else if hasName {
+					pu.FileName = dpName
+				}
 
-	pu.FileName = r.Header.Get("Content-Disposition")
+			}
 
-	if pu.FileName != "" && strings.Contains(pu.FileName, "filename=") {
-		parts := strings.Split(pu.FileName, "filename=")
-		parts = strings.Split(parts[1], "\"")
+		} else {
+			pu.FileName = ""
+		}
 
-		pu.FileName = parts[1]
-	} else {
-		pu.FileName = ""
-	}
+		if pu.FileName != "" {
+			pu.FileName = path.Base(pu.FileName)
+		} else {
+			pu.FileName = path.Base(r.URL.Path)
+		}
 
-	if pu.FileName != "" {
-		pu.FileName = path.Base(pu.FileName)
-	} else {
-		pu.FileName = path.Base(r.URL.Path)
-	}
+		dataSize, e = pu.bytesBuffer.ReadFrom(io.LimitReader(r.Body, sizeLimit+1))
 
-	var dataSize int64
-	dataSize, e = pu.bytesBuffer.ReadFrom(io.LimitReader(r.Body, sizeLimit+1))
+		if e != nil {
+			glog.V(0).Infoln("Reading Content [ERROR]", e)
+			return
+		}
+		if dataSize == sizeLimit+1 {
+			e = fmt.Errorf("file over the limited %d bytes", sizeLimit)
+			return
+		}
 
-	if e != nil {
-		glog.V(0).Infoln("Reading Content [ERROR]", e)
-		return
-	}
-	if dataSize == sizeLimit+1 {
-		e = fmt.Errorf("file over the limited %d bytes", sizeLimit)
-		return
+		pu.Data = pu.bytesBuffer.Bytes()
+		pu.MimeType = contentType
+		pu.IsGzipped = r.Header.Get("Content-Encoding") == "gzip"
+		// pu.IsZstd = r.Header.Get("Content-Encoding") == "zstd"
 	}
-	pu.Data = pu.bytesBuffer.Bytes()
+
 	pu.IsChunkedFile, _ = strconv.ParseBool(r.FormValue("cm"))
 
 	if !pu.IsChunkedFile {
 
 		dotIndex := strings.LastIndex(pu.FileName, ".")
 		ext, mtype := "", ""
+
 		if dotIndex > 0 {
 			ext = strings.ToLower(pu.FileName[dotIndex:])
 			mtype = mime.TypeByExtension(ext)
 		}
-		contentType := r.Header.Get("Content-Type")
 
 		if contentType != "" && contentType != "application/octet-stream" && mtype != contentType {
 			pu.MimeType = contentType // only return mime type if not deducible
-			mtype = contentType
+		} else if mtype != "" && pu.MimeType == "" && mtype != "application/octet-stream" {
+			pu.MimeType = mtype
 		}
 
 	}
-	pu.IsGzipped = r.Header.Get("Content-Encoding") == "gzip"
-	// pu.IsZstd = r.Header.Get("Content-Encoding") == "zstd"
 
 	return
 }