You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

345 lines
9.6 KiB

10 years ago
5 years ago
10 years ago
5 years ago
4 years ago
4 years ago
4 years ago
  1. package weed_server
  2. import (
  3. "bytes"
  4. "encoding/json"
  5. "errors"
  6. "fmt"
  7. "io"
  8. "io/fs"
  9. "mime/multipart"
  10. "net/http"
  11. "path/filepath"
  12. "strconv"
  13. "strings"
  14. "time"
  15. "google.golang.org/grpc"
  16. "github.com/chrislusf/seaweedfs/weed/glog"
  17. "github.com/chrislusf/seaweedfs/weed/operation"
  18. "github.com/chrislusf/seaweedfs/weed/stats"
  19. "github.com/chrislusf/seaweedfs/weed/storage/needle"
  20. "github.com/chrislusf/seaweedfs/weed/util"
  21. "github.com/gorilla/mux"
  22. )
  23. var serverStats *stats.ServerStats
  24. var startTime = time.Now()
  25. func init() {
  26. serverStats = stats.NewServerStats()
  27. go serverStats.Start()
  28. }
  29. // bodyAllowedForStatus is a copy of http.bodyAllowedForStatus non-exported function.
  30. func bodyAllowedForStatus(status int) bool {
  31. switch {
  32. case status >= 100 && status <= 199:
  33. return false
  34. case status == http.StatusNoContent:
  35. return false
  36. case status == http.StatusNotModified:
  37. return false
  38. }
  39. return true
  40. }
  41. func writeJson(w http.ResponseWriter, r *http.Request, httpStatus int, obj interface{}) (err error) {
  42. if !bodyAllowedForStatus(httpStatus) {
  43. return
  44. }
  45. var bytes []byte
  46. if obj != nil {
  47. if r.FormValue("pretty") != "" {
  48. bytes, err = json.MarshalIndent(obj, "", " ")
  49. } else {
  50. bytes, err = json.Marshal(obj)
  51. }
  52. }
  53. if err != nil {
  54. return
  55. }
  56. if httpStatus >= 400 {
  57. glog.V(0).Infof("response method:%s URL:%s with httpStatus:%d and JSON:%s",
  58. r.Method, r.URL.String(), httpStatus, string(bytes))
  59. }
  60. callback := r.FormValue("callback")
  61. if callback == "" {
  62. w.Header().Set("Content-Type", "application/json")
  63. w.WriteHeader(httpStatus)
  64. if httpStatus == http.StatusNotModified {
  65. return
  66. }
  67. _, err = w.Write(bytes)
  68. } else {
  69. w.Header().Set("Content-Type", "application/javascript")
  70. w.WriteHeader(httpStatus)
  71. if httpStatus == http.StatusNotModified {
  72. return
  73. }
  74. if _, err = w.Write([]uint8(callback)); err != nil {
  75. return
  76. }
  77. if _, err = w.Write([]uint8("(")); err != nil {
  78. return
  79. }
  80. fmt.Fprint(w, string(bytes))
  81. if _, err = w.Write([]uint8(")")); err != nil {
  82. return
  83. }
  84. }
  85. return
  86. }
  87. // wrapper for writeJson - just logs errors
  88. func writeJsonQuiet(w http.ResponseWriter, r *http.Request, httpStatus int, obj interface{}) {
  89. if err := writeJson(w, r, httpStatus, obj); err != nil {
  90. glog.V(0).Infof("error writing JSON status %d: %v", httpStatus, err)
  91. glog.V(1).Infof("JSON content: %+v", obj)
  92. }
  93. }
  94. func writeJsonError(w http.ResponseWriter, r *http.Request, httpStatus int, err error) {
  95. m := make(map[string]interface{})
  96. m["error"] = err.Error()
  97. writeJsonQuiet(w, r, httpStatus, m)
  98. }
  99. func debug(params ...interface{}) {
  100. glog.V(4).Infoln(params...)
  101. }
  102. func submitForClientHandler(w http.ResponseWriter, r *http.Request, masterFn operation.GetMasterFn, grpcDialOption grpc.DialOption) {
  103. m := make(map[string]interface{})
  104. if r.Method != "POST" {
  105. writeJsonError(w, r, http.StatusMethodNotAllowed, errors.New("Only submit via POST!"))
  106. return
  107. }
  108. debug("parsing upload file...")
  109. bytesBuffer := bufPool.Get().(*bytes.Buffer)
  110. defer bufPool.Put(bytesBuffer)
  111. pu, pe := needle.ParseUpload(r, 256*1024*1024, bytesBuffer)
  112. if pe != nil {
  113. writeJsonError(w, r, http.StatusBadRequest, pe)
  114. return
  115. }
  116. debug("assigning file id for", pu.FileName)
  117. r.ParseForm()
  118. count := uint64(1)
  119. if r.FormValue("count") != "" {
  120. count, pe = strconv.ParseUint(r.FormValue("count"), 10, 32)
  121. if pe != nil {
  122. writeJsonError(w, r, http.StatusBadRequest, pe)
  123. return
  124. }
  125. }
  126. ar := &operation.VolumeAssignRequest{
  127. Count: count,
  128. DataCenter: r.FormValue("dataCenter"),
  129. Rack: r.FormValue("rack"),
  130. Replication: r.FormValue("replication"),
  131. Collection: r.FormValue("collection"),
  132. Ttl: r.FormValue("ttl"),
  133. DiskType: r.FormValue("disk"),
  134. }
  135. assignResult, ae := operation.Assign(masterFn, grpcDialOption, ar)
  136. if ae != nil {
  137. writeJsonError(w, r, http.StatusInternalServerError, ae)
  138. return
  139. }
  140. url := "http://" + assignResult.Url + "/" + assignResult.Fid
  141. if pu.ModifiedTime != 0 {
  142. url = url + "?ts=" + strconv.FormatUint(pu.ModifiedTime, 10)
  143. }
  144. debug("upload file to store", url)
  145. uploadResult, err := operation.UploadData(url, pu.FileName, false, pu.Data, pu.IsGzipped, pu.MimeType, pu.PairMap, assignResult.Auth)
  146. if err != nil {
  147. writeJsonError(w, r, http.StatusInternalServerError, err)
  148. return
  149. }
  150. m["fileName"] = pu.FileName
  151. m["fid"] = assignResult.Fid
  152. m["fileUrl"] = assignResult.PublicUrl + "/" + assignResult.Fid
  153. m["size"] = pu.OriginalDataSize
  154. m["eTag"] = uploadResult.ETag
  155. writeJsonQuiet(w, r, http.StatusCreated, m)
  156. return
  157. }
  158. func parseURLPath(path string) (vid, fid, filename, ext string, isVolumeIdOnly bool) {
  159. switch strings.Count(path, "/") {
  160. case 3:
  161. parts := strings.Split(path, "/")
  162. vid, fid, filename = parts[1], parts[2], parts[3]
  163. ext = filepath.Ext(filename)
  164. case 2:
  165. parts := strings.Split(path, "/")
  166. vid, fid = parts[1], parts[2]
  167. dotIndex := strings.LastIndex(fid, ".")
  168. if dotIndex > 0 {
  169. ext = fid[dotIndex:]
  170. fid = fid[0:dotIndex]
  171. }
  172. default:
  173. sepIndex := strings.LastIndex(path, "/")
  174. commaIndex := strings.LastIndex(path[sepIndex:], ",")
  175. if commaIndex <= 0 {
  176. vid, isVolumeIdOnly = path[sepIndex+1:], true
  177. return
  178. }
  179. dotIndex := strings.LastIndex(path[sepIndex:], ".")
  180. vid = path[sepIndex+1 : commaIndex]
  181. fid = path[commaIndex+1:]
  182. ext = ""
  183. if dotIndex > 0 {
  184. fid = path[commaIndex+1 : dotIndex]
  185. ext = path[dotIndex:]
  186. }
  187. }
  188. return
  189. }
  190. func statsHealthHandler(w http.ResponseWriter, r *http.Request) {
  191. m := make(map[string]interface{})
  192. m["Version"] = util.Version()
  193. writeJsonQuiet(w, r, http.StatusOK, m)
  194. }
  195. func statsCounterHandler(w http.ResponseWriter, r *http.Request) {
  196. m := make(map[string]interface{})
  197. m["Version"] = util.Version()
  198. m["Counters"] = serverStats
  199. writeJsonQuiet(w, r, http.StatusOK, m)
  200. }
  201. func statsMemoryHandler(w http.ResponseWriter, r *http.Request) {
  202. m := make(map[string]interface{})
  203. m["Version"] = util.Version()
  204. m["Memory"] = stats.MemStat()
  205. writeJsonQuiet(w, r, http.StatusOK, m)
  206. }
  207. var StaticFS fs.FS
  208. func handleStaticResources(defaultMux *http.ServeMux) {
  209. defaultMux.Handle("/favicon.ico", http.FileServer(http.FS(StaticFS)))
  210. defaultMux.Handle("/seaweedfsstatic/", http.StripPrefix("/seaweedfsstatic", http.FileServer(http.FS(StaticFS))))
  211. }
  212. func handleStaticResources2(r *mux.Router) {
  213. r.Handle("/favicon.ico", http.FileServer(http.FS(StaticFS)))
  214. r.PathPrefix("/seaweedfsstatic/").Handler(http.StripPrefix("/seaweedfsstatic", http.FileServer(http.FS(StaticFS))))
  215. }
  216. func adjustHeaderContentDisposition(w http.ResponseWriter, r *http.Request, filename string) {
  217. if filename != "" {
  218. contentDisposition := "inline"
  219. if r.FormValue("dl") != "" {
  220. if dl, _ := strconv.ParseBool(r.FormValue("dl")); dl {
  221. contentDisposition = "attachment"
  222. }
  223. }
  224. w.Header().Set("Content-Disposition", contentDisposition+`; filename="`+fileNameEscaper.Replace(filename)+`"`)
  225. }
  226. }
  227. func processRangeRequest(r *http.Request, w http.ResponseWriter, totalSize int64, mimeType string, writeFn func(writer io.Writer, offset int64, size int64) error) {
  228. rangeReq := r.Header.Get("Range")
  229. if rangeReq == "" {
  230. w.Header().Set("Content-Length", strconv.FormatInt(totalSize, 10))
  231. if err := writeFn(w, 0, totalSize); err != nil {
  232. http.Error(w, err.Error(), http.StatusInternalServerError)
  233. return
  234. }
  235. return
  236. }
  237. //the rest is dealing with partial content request
  238. //mostly copy from src/pkg/net/http/fs.go
  239. ranges, err := parseRange(rangeReq, totalSize)
  240. if err != nil {
  241. http.Error(w, err.Error(), http.StatusRequestedRangeNotSatisfiable)
  242. return
  243. }
  244. if sumRangesSize(ranges) > totalSize {
  245. // The total number of bytes in all the ranges
  246. // is larger than the size of the file by
  247. // itself, so this is probably an attack, or a
  248. // dumb client. Ignore the range request.
  249. return
  250. }
  251. if len(ranges) == 0 {
  252. return
  253. }
  254. if len(ranges) == 1 {
  255. // RFC 2616, Section 14.16:
  256. // "When an HTTP message includes the content of a single
  257. // range (for example, a response to a request for a
  258. // single range, or to a request for a set of ranges
  259. // that overlap without any holes), this content is
  260. // transmitted with a Content-Range header, and a
  261. // Content-Length header showing the number of bytes
  262. // actually transferred.
  263. // ...
  264. // A response to a request for a single range MUST NOT
  265. // be sent using the multipart/byteranges media type."
  266. ra := ranges[0]
  267. w.Header().Set("Content-Length", strconv.FormatInt(ra.length, 10))
  268. w.Header().Set("Content-Range", ra.contentRange(totalSize))
  269. w.WriteHeader(http.StatusPartialContent)
  270. err = writeFn(w, ra.start, ra.length)
  271. if err != nil {
  272. http.Error(w, err.Error(), http.StatusInternalServerError)
  273. return
  274. }
  275. return
  276. }
  277. // process multiple ranges
  278. for _, ra := range ranges {
  279. if ra.start > totalSize {
  280. http.Error(w, "Out of Range", http.StatusRequestedRangeNotSatisfiable)
  281. return
  282. }
  283. }
  284. sendSize := rangesMIMESize(ranges, mimeType, totalSize)
  285. pr, pw := io.Pipe()
  286. mw := multipart.NewWriter(pw)
  287. w.Header().Set("Content-Type", "multipart/byteranges; boundary="+mw.Boundary())
  288. sendContent := pr
  289. defer pr.Close() // cause writing goroutine to fail and exit if CopyN doesn't finish.
  290. go func() {
  291. for _, ra := range ranges {
  292. part, e := mw.CreatePart(ra.mimeHeader(mimeType, totalSize))
  293. if e != nil {
  294. pw.CloseWithError(e)
  295. return
  296. }
  297. if e = writeFn(part, ra.start, ra.length); e != nil {
  298. pw.CloseWithError(e)
  299. return
  300. }
  301. }
  302. mw.Close()
  303. pw.Close()
  304. }()
  305. if w.Header().Get("Content-Encoding") == "" {
  306. w.Header().Set("Content-Length", strconv.FormatInt(sendSize, 10))
  307. }
  308. w.WriteHeader(http.StatusPartialContent)
  309. if _, err := io.CopyN(w, sendContent, sendSize); err != nil {
  310. http.Error(w, "Internal Error", http.StatusInternalServerError)
  311. return
  312. }
  313. }