You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

341 lines
9.5 KiB

9 years ago
9 years ago
9 years ago
6 years ago
6 years ago
5 years ago
6 years ago
6 years ago
6 years ago
5 years ago
6 years ago
5 years ago
6 years ago
6 years ago
5 years ago
5 years ago
6 years ago
5 years ago
6 years ago
6 years ago
9 years ago
  1. package weed_server
  2. import (
  3. "bytes"
  4. "context"
  5. "errors"
  6. "fmt"
  7. "io"
  8. "mime"
  9. "mime/multipart"
  10. "net/http"
  11. "net/url"
  12. "path"
  13. "strconv"
  14. "strings"
  15. "time"
  16. "encoding/json"
  17. "github.com/chrislusf/seaweedfs/weed/glog"
  18. "github.com/chrislusf/seaweedfs/weed/images"
  19. "github.com/chrislusf/seaweedfs/weed/operation"
  20. "github.com/chrislusf/seaweedfs/weed/stats"
  21. "github.com/chrislusf/seaweedfs/weed/storage/needle"
  22. "github.com/chrislusf/seaweedfs/weed/util"
  23. )
  24. var fileNameEscaper = strings.NewReplacer("\\", "\\\\", "\"", "\\\"")
  25. func (vs *VolumeServer) OldGetOrHeadHandler(w http.ResponseWriter, r *http.Request) {
  26. stats.VolumeServerRequestCounter.WithLabelValues("get").Inc()
  27. start := time.Now()
  28. defer func() { stats.VolumeServerRequestHistogram.WithLabelValues("get").Observe(time.Since(start).Seconds()) }()
  29. n := new(needle.Needle)
  30. vid, fid, filename, ext, _ := parseURLPath(r.URL.Path)
  31. if !vs.oldMaybeCheckJwtAuthorization(r, vid, fid, false) {
  32. oldWriteJsonError(w, r, http.StatusUnauthorized, errors.New("wrong jwt"))
  33. return
  34. }
  35. volumeId, err := needle.NewVolumeId(vid)
  36. if err != nil {
  37. glog.V(2).Infoln("parsing error:", err, r.URL.Path)
  38. w.WriteHeader(http.StatusBadRequest)
  39. return
  40. }
  41. err = n.ParsePath(fid)
  42. if err != nil {
  43. glog.V(2).Infoln("parsing fid error:", err, r.URL.Path)
  44. w.WriteHeader(http.StatusBadRequest)
  45. return
  46. }
  47. // glog.V(4).Infoln("volume", volumeId, "reading", n)
  48. hasVolume := vs.store.HasVolume(volumeId)
  49. _, hasEcVolume := vs.store.FindEcVolume(volumeId)
  50. if !hasVolume && !hasEcVolume {
  51. if !vs.ReadRedirect {
  52. glog.V(2).Infoln("volume is not local:", err, r.URL.Path)
  53. w.WriteHeader(http.StatusNotFound)
  54. return
  55. }
  56. lookupResult, err := operation.Lookup(vs.GetMaster(), volumeId.String())
  57. glog.V(2).Infoln("volume", volumeId, "found on", lookupResult, "error", err)
  58. if err == nil && len(lookupResult.Locations) > 0 {
  59. u, _ := url.Parse(util.NormalizeUrl(lookupResult.Locations[0].PublicUrl))
  60. u.Path = fmt.Sprintf("%s/%s,%s", u.Path, vid, fid)
  61. arg := url.Values{}
  62. if c := r.FormValue("collection"); c != "" {
  63. arg.Set("collection", c)
  64. }
  65. u.RawQuery = arg.Encode()
  66. http.Redirect(w, r, u.String(), http.StatusMovedPermanently)
  67. } else {
  68. glog.V(2).Infoln("lookup error:", err, r.URL.Path)
  69. w.WriteHeader(http.StatusNotFound)
  70. }
  71. return
  72. }
  73. cookie := n.Cookie
  74. var count int
  75. if hasVolume {
  76. count, err = vs.store.ReadVolumeNeedle(volumeId, n)
  77. } else if hasEcVolume {
  78. count, err = vs.store.ReadEcShardNeedle(context.Background(), volumeId, n)
  79. }
  80. // glog.V(4).Infoln("read bytes", count, "error", err)
  81. if err != nil || count < 0 {
  82. glog.V(0).Infof("read %s isNormalVolume %v error: %v", r.URL.Path, hasVolume, err)
  83. w.WriteHeader(http.StatusNotFound)
  84. return
  85. }
  86. if n.Cookie != cookie {
  87. glog.V(0).Infof("request %s with cookie:%x expected:%x from %s agent %s", r.URL.Path, cookie, n.Cookie, r.RemoteAddr, r.UserAgent())
  88. w.WriteHeader(http.StatusNotFound)
  89. return
  90. }
  91. if n.LastModified != 0 {
  92. w.Header().Set("Last-Modified", time.Unix(int64(n.LastModified), 0).UTC().Format(http.TimeFormat))
  93. if r.Header.Get("If-Modified-Since") != "" {
  94. if t, parseError := time.Parse(http.TimeFormat, r.Header.Get("If-Modified-Since")); parseError == nil {
  95. if t.Unix() >= int64(n.LastModified) {
  96. w.WriteHeader(http.StatusNotModified)
  97. return
  98. }
  99. }
  100. }
  101. }
  102. if inm := r.Header.Get("If-None-Match"); inm == "\""+n.Etag()+"\"" {
  103. w.WriteHeader(http.StatusNotModified)
  104. return
  105. }
  106. if r.Header.Get("ETag-MD5") == "True" {
  107. oldSetEtag(w, n.MD5())
  108. } else {
  109. oldSetEtag(w, n.Etag())
  110. }
  111. if n.HasPairs() {
  112. pairMap := make(map[string]string)
  113. err = json.Unmarshal(n.Pairs, &pairMap)
  114. if err != nil {
  115. glog.V(0).Infoln("Unmarshal pairs error:", err)
  116. }
  117. for k, v := range pairMap {
  118. w.Header().Set(k, v)
  119. }
  120. }
  121. if vs.tryHandleChunkedFile(n, filename, w, r) {
  122. return
  123. }
  124. if n.NameSize > 0 && filename == "" {
  125. filename = string(n.Name)
  126. if ext == "" {
  127. ext = path.Ext(filename)
  128. }
  129. }
  130. mtype := ""
  131. if n.MimeSize > 0 {
  132. mt := string(n.Mime)
  133. if !strings.HasPrefix(mt, "application/octet-stream") {
  134. mtype = mt
  135. }
  136. }
  137. if ext != ".gz" {
  138. if n.IsGzipped() {
  139. if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
  140. w.Header().Set("Content-Encoding", "gzip")
  141. } else {
  142. if n.Data, err = util.UnGzipData(n.Data); err != nil {
  143. glog.V(0).Infoln("ungzip error:", err, r.URL.Path)
  144. }
  145. }
  146. }
  147. }
  148. rs := conditionallyResizeImages(bytes.NewReader(n.Data), ext, r)
  149. if e := writeResponseContent(filename, mtype, rs, w, r); e != nil {
  150. glog.V(2).Infoln("response write error:", e)
  151. }
  152. }
  153. func (vs *VolumeServer) tryHandleChunkedFile(n *needle.Needle, fileName string, w http.ResponseWriter, r *http.Request) (processed bool) {
  154. if !n.IsChunkedManifest() || r.URL.Query().Get("cm") == "false" {
  155. return false
  156. }
  157. chunkManifest, e := operation.LoadChunkManifest(n.Data, n.IsGzipped())
  158. if e != nil {
  159. glog.V(0).Infof("load chunked manifest (%s) error: %v", r.URL.Path, e)
  160. return false
  161. }
  162. if fileName == "" && chunkManifest.Name != "" {
  163. fileName = chunkManifest.Name
  164. }
  165. ext := path.Ext(fileName)
  166. mType := ""
  167. if chunkManifest.Mime != "" {
  168. mt := chunkManifest.Mime
  169. if !strings.HasPrefix(mt, "application/octet-stream") {
  170. mType = mt
  171. }
  172. }
  173. w.Header().Set("X-File-Store", "chunked")
  174. chunkedFileReader := &operation.ChunkedFileReader{
  175. Manifest: chunkManifest,
  176. Master: vs.GetMaster(),
  177. }
  178. defer chunkedFileReader.Close()
  179. rs := conditionallyResizeImages(chunkedFileReader, ext, r)
  180. if e := writeResponseContent(fileName, mType, rs, w, r); e != nil {
  181. glog.V(2).Infoln("response write error:", e)
  182. }
  183. return true
  184. }
  185. func conditionallyResizeImages(originalDataReaderSeeker io.ReadSeeker, ext string, r *http.Request) io.ReadSeeker {
  186. rs := originalDataReaderSeeker
  187. if len(ext) > 0 {
  188. ext = strings.ToLower(ext)
  189. }
  190. if ext == ".png" || ext == ".jpg" || ext == ".jpeg" || ext == ".gif" {
  191. width, height := 0, 0
  192. if r.FormValue("width") != "" {
  193. width, _ = strconv.Atoi(r.FormValue("width"))
  194. }
  195. if r.FormValue("height") != "" {
  196. height, _ = strconv.Atoi(r.FormValue("height"))
  197. }
  198. rs, _, _ = images.Resized(ext, originalDataReaderSeeker, width, height, r.FormValue("mode"))
  199. }
  200. return rs
  201. }
  202. func writeResponseContent(filename, mimeType string, rs io.ReadSeeker, w http.ResponseWriter, r *http.Request) error {
  203. totalSize, e := rs.Seek(0, 2)
  204. if mimeType == "" {
  205. if ext := path.Ext(filename); ext != "" {
  206. mimeType = mime.TypeByExtension(ext)
  207. }
  208. }
  209. if mimeType != "" {
  210. w.Header().Set("Content-Type", mimeType)
  211. }
  212. if filename != "" {
  213. contentDisposition := "inline"
  214. if r.FormValue("dl") != "" {
  215. if dl, _ := strconv.ParseBool(r.FormValue("dl")); dl {
  216. contentDisposition = "attachment"
  217. }
  218. }
  219. w.Header().Set("Content-Disposition", contentDisposition+`; filename="`+fileNameEscaper.Replace(filename)+`"`)
  220. }
  221. w.Header().Set("Accept-Ranges", "bytes")
  222. if r.Method == "HEAD" {
  223. w.Header().Set("Content-Length", strconv.FormatInt(totalSize, 10))
  224. return nil
  225. }
  226. rangeReq := r.Header.Get("Range")
  227. if rangeReq == "" {
  228. w.Header().Set("Content-Length", strconv.FormatInt(totalSize, 10))
  229. if _, e = rs.Seek(0, 0); e != nil {
  230. return e
  231. }
  232. _, e = io.Copy(w, rs)
  233. return e
  234. }
  235. //the rest is dealing with partial content request
  236. //mostly copy from src/pkg/net/http/fs.go
  237. ranges, err := parseRange(rangeReq, totalSize)
  238. if err != nil {
  239. http.Error(w, err.Error(), http.StatusRequestedRangeNotSatisfiable)
  240. return nil
  241. }
  242. if sumRangesSize(ranges) > totalSize {
  243. // The total number of bytes in all the ranges
  244. // is larger than the size of the file by
  245. // itself, so this is probably an attack, or a
  246. // dumb client. Ignore the range request.
  247. return nil
  248. }
  249. if len(ranges) == 0 {
  250. return nil
  251. }
  252. if len(ranges) == 1 {
  253. // RFC 2616, Section 14.16:
  254. // "When an HTTP message includes the content of a single
  255. // range (for example, a response to a request for a
  256. // single range, or to a request for a set of ranges
  257. // that overlap without any holes), this content is
  258. // transmitted with a Content-Range header, and a
  259. // Content-Length header showing the number of bytes
  260. // actually transferred.
  261. // ...
  262. // A response to a request for a single range MUST NOT
  263. // be sent using the multipart/byteranges media type."
  264. ra := ranges[0]
  265. w.Header().Set("Content-Length", strconv.FormatInt(ra.length, 10))
  266. w.Header().Set("Content-Range", ra.contentRange(totalSize))
  267. w.WriteHeader(http.StatusPartialContent)
  268. if _, e = rs.Seek(ra.start, 0); e != nil {
  269. return e
  270. }
  271. _, e = io.CopyN(w, rs, ra.length)
  272. return e
  273. }
  274. // process multiple ranges
  275. for _, ra := range ranges {
  276. if ra.start > totalSize {
  277. http.Error(w, "Out of Range", http.StatusRequestedRangeNotSatisfiable)
  278. return nil
  279. }
  280. }
  281. sendSize := rangesMIMESize(ranges, mimeType, totalSize)
  282. pr, pw := io.Pipe()
  283. mw := multipart.NewWriter(pw)
  284. w.Header().Set("Content-Type", "multipart/byteranges; boundary="+mw.Boundary())
  285. sendContent := pr
  286. defer pr.Close() // cause writing goroutine to fail and exit if CopyN doesn't finish.
  287. go func() {
  288. for _, ra := range ranges {
  289. part, e := mw.CreatePart(ra.mimeHeader(mimeType, totalSize))
  290. if e != nil {
  291. pw.CloseWithError(e)
  292. return
  293. }
  294. if _, e = rs.Seek(ra.start, 0); e != nil {
  295. pw.CloseWithError(e)
  296. return
  297. }
  298. if _, e = io.CopyN(part, rs, ra.length); e != nil {
  299. pw.CloseWithError(e)
  300. return
  301. }
  302. }
  303. mw.Close()
  304. pw.Close()
  305. }()
  306. if w.Header().Get("Content-Encoding") == "" {
  307. w.Header().Set("Content-Length", strconv.FormatInt(sendSize, 10))
  308. }
  309. w.WriteHeader(http.StatusPartialContent)
  310. _, e = io.CopyN(w, sendContent, sendSize)
  311. return e
  312. }