You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

336 lines
10 KiB

9 years ago
9 years ago
9 years ago
9 years ago
9 years ago
9 years ago
9 years ago
9 years ago
9 years ago
9 years ago
9 years ago
9 years ago
9 years ago
9 years ago
  1. package weed_server
  2. import (
  3. "bytes"
  4. "crypto/md5"
  5. "encoding/base64"
  6. "encoding/json"
  7. "errors"
  8. "fmt"
  9. "io"
  10. "io/ioutil"
  11. "mime/multipart"
  12. "net/http"
  13. "net/textproto"
  14. "net/url"
  15. "strings"
  16. "github.com/chrislusf/seaweedfs/weed/glog"
  17. "github.com/chrislusf/seaweedfs/weed/operation"
  18. "github.com/chrislusf/seaweedfs/weed/storage"
  19. "github.com/chrislusf/seaweedfs/weed/util"
  20. "github.com/syndtr/goleveldb/leveldb"
  21. )
  22. type FilerPostResult struct {
  23. Name string `json:"name,omitempty"`
  24. Size uint32 `json:"size,omitempty"`
  25. Error string `json:"error,omitempty"`
  26. Fid string `json:"fid,omitempty"`
  27. Url string `json:"url,omitempty"`
  28. }
  29. var quoteEscaper = strings.NewReplacer("\\", "\\\\", `"`, "\\\"")
  30. func escapeQuotes(s string) string {
  31. return quoteEscaper.Replace(s)
  32. }
  33. func createFormFile(writer *multipart.Writer, fieldname, filename, mime string) (io.Writer, error) {
  34. h := make(textproto.MIMEHeader)
  35. h.Set("Content-Disposition",
  36. fmt.Sprintf(`form-data; name="%s"; filename="%s"`,
  37. escapeQuotes(fieldname), escapeQuotes(filename)))
  38. if len(mime) == 0 {
  39. mime = "application/octet-stream"
  40. }
  41. h.Set("Content-Type", mime)
  42. return writer.CreatePart(h)
  43. }
  44. func makeFormData(filename, mimeType string, content io.Reader) (formData io.Reader, contentType string, err error) {
  45. buf := new(bytes.Buffer)
  46. writer := multipart.NewWriter(buf)
  47. defer writer.Close()
  48. part, err := createFormFile(writer, "file", filename, mimeType)
  49. if err != nil {
  50. glog.V(0).Infoln(err)
  51. return
  52. }
  53. _, err = io.Copy(part, content)
  54. if err != nil {
  55. glog.V(0).Infoln(err)
  56. return
  57. }
  58. formData = buf
  59. contentType = writer.FormDataContentType()
  60. return
  61. }
  62. func (fs *FilerServer) queryFileInfoByPath(w http.ResponseWriter, r *http.Request, path string) (fileId, urlLocation string, err error) {
  63. if fileId, err = fs.filer.FindFile(path); err != nil && err != leveldb.ErrNotFound {
  64. glog.V(0).Infoln("failing to find path in filer store", path, err.Error())
  65. writeJsonError(w, r, http.StatusInternalServerError, err)
  66. return
  67. } else if fileId != "" && err == nil {
  68. urlLocation, err = operation.LookupFileId(fs.getMasterNode(), fileId)
  69. if err != nil {
  70. glog.V(1).Infoln("operation LookupFileId %s failed, err is %s", fileId, err.Error())
  71. w.WriteHeader(http.StatusNotFound)
  72. return
  73. }
  74. }
  75. return
  76. }
  77. func (fs *FilerServer) assignNewFileInfo(w http.ResponseWriter, r *http.Request, replication, collection string) (fileId, urlLocation string, err error) {
  78. assignResult, ae := operation.Assign(fs.getMasterNode(), 1, replication, collection, r.URL.Query().Get("ttl"))
  79. if ae != nil {
  80. glog.V(0).Infoln("failing to assign a file id", ae.Error())
  81. writeJsonError(w, r, http.StatusInternalServerError, ae)
  82. err = ae
  83. return
  84. }
  85. fileId = assignResult.Fid
  86. urlLocation = "http://" + assignResult.Url + "/" + assignResult.Fid
  87. return
  88. }
  89. func (fs *FilerServer) multipartUploadAnalyzer(w http.ResponseWriter, r *http.Request, replication, collection string) (fileId, urlLocation string, err error) {
  90. //Default handle way for http multipart
  91. if r.Method == "PUT" {
  92. buf, _ := ioutil.ReadAll(r.Body)
  93. r.Body = ioutil.NopCloser(bytes.NewBuffer(buf))
  94. fileName, _, _, _, _, _, _, pe := storage.ParseUpload(r)
  95. if pe != nil {
  96. glog.V(0).Infoln("failing to parse post body", pe.Error())
  97. writeJsonError(w, r, http.StatusInternalServerError, pe)
  98. err = pe
  99. return
  100. }
  101. //reconstruct http request body for following new request to volume server
  102. r.Body = ioutil.NopCloser(bytes.NewBuffer(buf))
  103. path := r.URL.Path
  104. if strings.HasSuffix(path, "/") {
  105. if fileName != "" {
  106. path += fileName
  107. }
  108. }
  109. fileId, urlLocation, err = fs.queryFileInfoByPath(w, r, path)
  110. } else {
  111. fileId, urlLocation, err = fs.assignNewFileInfo(w, r, replication, collection)
  112. }
  113. return
  114. }
  115. func multipartHttpBodyBuilder(w http.ResponseWriter, r *http.Request, fileName string) (err error) {
  116. body, contentType, te := makeFormData(fileName, r.Header.Get("Content-Type"), r.Body)
  117. if te != nil {
  118. glog.V(0).Infoln("S3 protocol to raw seaweed protocol failed", te.Error())
  119. writeJsonError(w, r, http.StatusInternalServerError, te)
  120. err = te
  121. return
  122. }
  123. if body != nil {
  124. switch v := body.(type) {
  125. case *bytes.Buffer:
  126. r.ContentLength = int64(v.Len())
  127. case *bytes.Reader:
  128. r.ContentLength = int64(v.Len())
  129. case *strings.Reader:
  130. r.ContentLength = int64(v.Len())
  131. }
  132. }
  133. r.Header.Set("Content-Type", contentType)
  134. rc, ok := body.(io.ReadCloser)
  135. if !ok && body != nil {
  136. rc = ioutil.NopCloser(body)
  137. }
  138. r.Body = rc
  139. return
  140. }
  141. func checkContentMD5(w http.ResponseWriter, r *http.Request) (err error) {
  142. if contentMD5 := r.Header.Get("Content-MD5"); contentMD5 != "" {
  143. buf, _ := ioutil.ReadAll(r.Body)
  144. //checkMD5
  145. sum := md5.Sum(buf)
  146. fileDataMD5 := base64.StdEncoding.EncodeToString(sum[0:len(sum)])
  147. if strings.ToLower(fileDataMD5) != strings.ToLower(contentMD5) {
  148. glog.V(0).Infof("fileDataMD5 [%s] is not equal to Content-MD5 [%s]", fileDataMD5, contentMD5)
  149. err = fmt.Errorf("MD5 check failed")
  150. writeJsonError(w, r, http.StatusNotAcceptable, err)
  151. return
  152. }
  153. //reconstruct http request body for following new request to volume server
  154. r.Body = ioutil.NopCloser(bytes.NewBuffer(buf))
  155. }
  156. return
  157. }
  158. func (fs *FilerServer) monolithicUploadAnalyzer(w http.ResponseWriter, r *http.Request, replication, collection string) (fileId, urlLocation string, err error) {
  159. /*
  160. Amazon S3 ref link:[http://docs.aws.amazon.com/AmazonS3/latest/API/Welcome.html]
  161. There is a long way to provide a completely compatibility against all Amazon S3 API, I just made
  162. a simple data stream adapter between S3 PUT API and seaweedfs's volume storage Write API
  163. 1. The request url format should be http://$host:$port/$bucketName/$objectName
  164. 2. bucketName will be mapped to seaweedfs's collection name
  165. 3. You could customize and make your enhancement.
  166. */
  167. lastPos := strings.LastIndex(r.URL.Path, "/")
  168. if lastPos == -1 || lastPos == 0 || lastPos == len(r.URL.Path)-1 {
  169. glog.V(0).Infoln("URL Path [%s] is invalid, could not retrieve file name", r.URL.Path)
  170. err = fmt.Errorf("URL Path is invalid")
  171. writeJsonError(w, r, http.StatusInternalServerError, err)
  172. return
  173. }
  174. if err = checkContentMD5(w, r); err != nil {
  175. return
  176. }
  177. fileName := r.URL.Path[lastPos+1:]
  178. if err = multipartHttpBodyBuilder(w, r, fileName); err != nil {
  179. return
  180. }
  181. secondPos := strings.Index(r.URL.Path[1:], "/") + 1
  182. collection = r.URL.Path[1:secondPos]
  183. path := r.URL.Path
  184. if fileId, urlLocation, err = fs.queryFileInfoByPath(w, r, path); err == nil && fileId == "" {
  185. fileId, urlLocation, err = fs.assignNewFileInfo(w, r, replication, collection)
  186. }
  187. return
  188. }
  189. func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) {
  190. query := r.URL.Query()
  191. replication := query.Get("replication")
  192. if replication == "" {
  193. replication = fs.defaultReplication
  194. }
  195. collection := query.Get("collection")
  196. if collection == "" {
  197. collection = fs.collection
  198. }
  199. var fileId, urlLocation string
  200. var err error
  201. if strings.HasPrefix(r.Header.Get("Content-Type"), "multipart/form-data; boundary=") {
  202. fileId, urlLocation, err = fs.multipartUploadAnalyzer(w, r, replication, collection)
  203. if err != nil {
  204. return
  205. }
  206. } else {
  207. fileId, urlLocation, err = fs.monolithicUploadAnalyzer(w, r, replication, collection)
  208. if err != nil {
  209. return
  210. }
  211. }
  212. u, _ := url.Parse(urlLocation)
  213. glog.V(4).Infoln("post to", u)
  214. request := &http.Request{
  215. Method: r.Method,
  216. URL: u,
  217. Proto: r.Proto,
  218. ProtoMajor: r.ProtoMajor,
  219. ProtoMinor: r.ProtoMinor,
  220. Header: r.Header,
  221. Body: r.Body,
  222. Host: r.Host,
  223. ContentLength: r.ContentLength,
  224. }
  225. resp, do_err := util.Do(request)
  226. if do_err != nil {
  227. glog.V(0).Infoln("failing to connect to volume server", r.RequestURI, do_err.Error())
  228. writeJsonError(w, r, http.StatusInternalServerError, do_err)
  229. return
  230. }
  231. defer resp.Body.Close()
  232. resp_body, ra_err := ioutil.ReadAll(resp.Body)
  233. if ra_err != nil {
  234. glog.V(0).Infoln("failing to upload to volume server", r.RequestURI, ra_err.Error())
  235. writeJsonError(w, r, http.StatusInternalServerError, ra_err)
  236. return
  237. }
  238. glog.V(4).Infoln("post result", string(resp_body))
  239. var ret operation.UploadResult
  240. unmarshal_err := json.Unmarshal(resp_body, &ret)
  241. if unmarshal_err != nil {
  242. glog.V(0).Infoln("failing to read upload resonse", r.RequestURI, string(resp_body))
  243. writeJsonError(w, r, http.StatusInternalServerError, unmarshal_err)
  244. return
  245. }
  246. if ret.Error != "" {
  247. glog.V(0).Infoln("failing to post to volume server", r.RequestURI, ret.Error)
  248. writeJsonError(w, r, http.StatusInternalServerError, errors.New(ret.Error))
  249. return
  250. }
  251. path := r.URL.Path
  252. if strings.HasSuffix(path, "/") {
  253. if ret.Name != "" {
  254. path += ret.Name
  255. } else {
  256. operation.DeleteFile(fs.getMasterNode(), fileId, fs.jwt(fileId)) //clean up
  257. glog.V(0).Infoln("Can not to write to folder", path, "without a file name!")
  258. writeJsonError(w, r, http.StatusInternalServerError,
  259. errors.New("Can not to write to folder "+path+" without a file name"))
  260. return
  261. }
  262. }
  263. // also delete the old fid unless PUT operation
  264. if r.Method != "PUT" {
  265. if oldFid, err := fs.filer.FindFile(path); err == nil {
  266. operation.DeleteFile(fs.getMasterNode(), oldFid, fs.jwt(oldFid))
  267. }
  268. }
  269. glog.V(4).Infoln("saving", path, "=>", fileId)
  270. if db_err := fs.filer.CreateFile(path, fileId); db_err != nil {
  271. operation.DeleteFile(fs.getMasterNode(), fileId, fs.jwt(fileId)) //clean up
  272. glog.V(0).Infof("failing to write %s to filer server : %v", path, db_err)
  273. writeJsonError(w, r, http.StatusInternalServerError, db_err)
  274. return
  275. }
  276. reply := FilerPostResult{
  277. Name: ret.Name,
  278. Size: ret.Size,
  279. Error: ret.Error,
  280. Fid: fileId,
  281. Url: urlLocation,
  282. }
  283. writeJsonQuiet(w, r, http.StatusCreated, reply)
  284. }
  285. // curl -X DELETE http://localhost:8888/path/to
  286. // curl -X DELETE http://localhost:8888/path/to?recursive=true
  287. func (fs *FilerServer) DeleteHandler(w http.ResponseWriter, r *http.Request) {
  288. var err error
  289. var fid string
  290. if strings.HasSuffix(r.URL.Path, "/") {
  291. isRecursive := r.FormValue("recursive") == "true"
  292. err = fs.filer.DeleteDirectory(r.URL.Path, isRecursive)
  293. } else {
  294. fid, err = fs.filer.DeleteFile(r.URL.Path)
  295. if err == nil && fid != "" {
  296. err = operation.DeleteFile(fs.getMasterNode(), fid, fs.jwt(fid))
  297. }
  298. }
  299. if err == nil {
  300. writeJsonQuiet(w, r, http.StatusAccepted, map[string]string{"error": ""})
  301. } else {
  302. glog.V(4).Infoln("deleting", r.URL.Path, ":", err.Error())
  303. writeJsonError(w, r, http.StatusInternalServerError, err)
  304. }
  305. }