You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

382 lines
8.3 KiB

9 years ago
10 years ago
5 years ago
5 years ago
4 years ago
  1. package util
  2. import (
  3. "compress/gzip"
  4. "encoding/json"
  5. "errors"
  6. "fmt"
  7. "io"
  8. "io/ioutil"
  9. "net/http"
  10. "net/url"
  11. "strings"
  12. "github.com/chrislusf/seaweedfs/weed/glog"
  13. )
  14. var (
  15. client *http.Client
  16. Transport *http.Transport
  17. )
  18. func init() {
  19. Transport = &http.Transport{
  20. MaxIdleConns: 1024,
  21. MaxIdleConnsPerHost: 1024,
  22. }
  23. client = &http.Client{
  24. Transport: Transport,
  25. }
  26. }
  27. func Post(url string, values url.Values) ([]byte, error) {
  28. r, err := client.PostForm(url, values)
  29. if err != nil {
  30. return nil, err
  31. }
  32. defer r.Body.Close()
  33. b, err := ioutil.ReadAll(r.Body)
  34. if r.StatusCode >= 400 {
  35. if err != nil {
  36. return nil, fmt.Errorf("%s: %d - %s", url, r.StatusCode, string(b))
  37. } else {
  38. return nil, fmt.Errorf("%s: %s", url, r.Status)
  39. }
  40. }
  41. if err != nil {
  42. return nil, err
  43. }
  44. return b, nil
  45. }
  46. // github.com/chrislusf/seaweedfs/unmaintained/repeated_vacuum/repeated_vacuum.go
  47. // may need increasing http.Client.Timeout
  48. func Get(url string) ([]byte, bool, error) {
  49. request, err := http.NewRequest("GET", url, nil)
  50. request.Header.Add("Accept-Encoding", "gzip")
  51. response, err := client.Do(request)
  52. if err != nil {
  53. return nil, true, err
  54. }
  55. defer response.Body.Close()
  56. var reader io.ReadCloser
  57. switch response.Header.Get("Content-Encoding") {
  58. case "gzip":
  59. reader, err = gzip.NewReader(response.Body)
  60. defer reader.Close()
  61. default:
  62. reader = response.Body
  63. }
  64. b, err := ioutil.ReadAll(reader)
  65. if response.StatusCode >= 400 {
  66. retryable := response.StatusCode >= 500
  67. return nil, retryable, fmt.Errorf("%s: %s", url, response.Status)
  68. }
  69. if err != nil {
  70. return nil, false, err
  71. }
  72. return b, false, nil
  73. }
  74. func Head(url string) (http.Header, error) {
  75. r, err := client.Head(url)
  76. if err != nil {
  77. return nil, err
  78. }
  79. defer CloseResponse(r)
  80. if r.StatusCode >= 400 {
  81. return nil, fmt.Errorf("%s: %s", url, r.Status)
  82. }
  83. return r.Header, nil
  84. }
  85. func Delete(url string, jwt string) error {
  86. req, err := http.NewRequest("DELETE", url, nil)
  87. if jwt != "" {
  88. req.Header.Set("Authorization", "BEARER "+string(jwt))
  89. }
  90. if err != nil {
  91. return err
  92. }
  93. resp, e := client.Do(req)
  94. if e != nil {
  95. return e
  96. }
  97. defer resp.Body.Close()
  98. body, err := ioutil.ReadAll(resp.Body)
  99. if err != nil {
  100. return err
  101. }
  102. switch resp.StatusCode {
  103. case http.StatusNotFound, http.StatusAccepted, http.StatusOK:
  104. return nil
  105. }
  106. m := make(map[string]interface{})
  107. if e := json.Unmarshal(body, &m); e == nil {
  108. if s, ok := m["error"].(string); ok {
  109. return errors.New(s)
  110. }
  111. }
  112. return errors.New(string(body))
  113. }
  114. func GetBufferStream(url string, values url.Values, allocatedBytes []byte, eachBuffer func([]byte)) error {
  115. r, err := client.PostForm(url, values)
  116. if err != nil {
  117. return err
  118. }
  119. defer CloseResponse(r)
  120. if r.StatusCode != 200 {
  121. return fmt.Errorf("%s: %s", url, r.Status)
  122. }
  123. for {
  124. n, err := r.Body.Read(allocatedBytes)
  125. if n > 0 {
  126. eachBuffer(allocatedBytes[:n])
  127. }
  128. if err != nil {
  129. if err == io.EOF {
  130. return nil
  131. }
  132. return err
  133. }
  134. }
  135. }
  136. func GetUrlStream(url string, values url.Values, readFn func(io.Reader) error) error {
  137. r, err := client.PostForm(url, values)
  138. if err != nil {
  139. return err
  140. }
  141. defer CloseResponse(r)
  142. if r.StatusCode != 200 {
  143. return fmt.Errorf("%s: %s", url, r.Status)
  144. }
  145. return readFn(r.Body)
  146. }
  147. func DownloadFile(fileUrl string) (filename string, header http.Header, resp *http.Response, e error) {
  148. response, err := client.Get(fileUrl)
  149. if err != nil {
  150. return "", nil, nil, err
  151. }
  152. header = response.Header
  153. contentDisposition := response.Header["Content-Disposition"]
  154. if len(contentDisposition) > 0 {
  155. idx := strings.Index(contentDisposition[0], "filename=")
  156. if idx != -1 {
  157. filename = contentDisposition[0][idx+len("filename="):]
  158. filename = strings.Trim(filename, "\"")
  159. }
  160. }
  161. resp = response
  162. return
  163. }
  164. func Do(req *http.Request) (resp *http.Response, err error) {
  165. return client.Do(req)
  166. }
  167. func NormalizeUrl(url string) string {
  168. if strings.HasPrefix(url, "http://") || strings.HasPrefix(url, "https://") {
  169. return url
  170. }
  171. return "http://" + url
  172. }
  173. func ReadUrl(fileUrl string, cipherKey []byte, isContentCompressed bool, isFullChunk bool, offset int64, size int, buf []byte) (int64, error) {
  174. if cipherKey != nil {
  175. var n int
  176. _, err := readEncryptedUrl(fileUrl, cipherKey, isContentCompressed, isFullChunk, offset, size, func(data []byte) {
  177. n = copy(buf, data)
  178. })
  179. return int64(n), err
  180. }
  181. req, err := http.NewRequest("GET", fileUrl, nil)
  182. if err != nil {
  183. return 0, err
  184. }
  185. if !isFullChunk {
  186. req.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", offset, offset+int64(size)-1))
  187. } else {
  188. req.Header.Set("Accept-Encoding", "gzip")
  189. }
  190. r, err := client.Do(req)
  191. if err != nil {
  192. return 0, err
  193. }
  194. defer r.Body.Close()
  195. if r.StatusCode >= 400 {
  196. return 0, fmt.Errorf("%s: %s", fileUrl, r.Status)
  197. }
  198. var reader io.ReadCloser
  199. contentEncoding := r.Header.Get("Content-Encoding")
  200. switch contentEncoding {
  201. case "gzip":
  202. reader, err = gzip.NewReader(r.Body)
  203. defer reader.Close()
  204. default:
  205. reader = r.Body
  206. }
  207. var (
  208. i, m int
  209. n int64
  210. )
  211. // refers to https://github.com/golang/go/blob/master/src/bytes/buffer.go#L199
  212. // commit id c170b14c2c1cfb2fd853a37add92a82fd6eb4318
  213. for {
  214. m, err = reader.Read(buf[i:])
  215. i += m
  216. n += int64(m)
  217. if err == io.EOF {
  218. return n, nil
  219. }
  220. if err != nil {
  221. return n, err
  222. }
  223. if n == int64(len(buf)) {
  224. break
  225. }
  226. }
  227. // drains the response body to avoid memory leak
  228. data, _ := ioutil.ReadAll(reader)
  229. if len(data) != 0 {
  230. glog.V(1).Infof("%s reader has remaining %d bytes", contentEncoding, len(data))
  231. }
  232. return n, err
  233. }
  234. func ReadUrlAsStream(fileUrl string, cipherKey []byte, isContentGzipped bool, isFullChunk bool, offset int64, size int, fn func(data []byte)) (retryable bool, err error) {
  235. if cipherKey != nil {
  236. return readEncryptedUrl(fileUrl, cipherKey, isContentGzipped, isFullChunk, offset, size, fn)
  237. }
  238. req, err := http.NewRequest("GET", fileUrl, nil)
  239. if err != nil {
  240. return false, err
  241. }
  242. if isFullChunk {
  243. req.Header.Add("Accept-Encoding", "gzip")
  244. } else {
  245. req.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", offset, offset+int64(size)-1))
  246. }
  247. r, err := client.Do(req)
  248. if err != nil {
  249. return true, err
  250. }
  251. defer CloseResponse(r)
  252. if r.StatusCode >= 400 {
  253. retryable = r.StatusCode >= 500
  254. return retryable, fmt.Errorf("%s: %s", fileUrl, r.Status)
  255. }
  256. var reader io.ReadCloser
  257. contentEncoding := r.Header.Get("Content-Encoding")
  258. switch contentEncoding {
  259. case "gzip":
  260. reader, err = gzip.NewReader(r.Body)
  261. defer reader.Close()
  262. default:
  263. reader = r.Body
  264. }
  265. var (
  266. m int
  267. )
  268. buf := make([]byte, 64*1024)
  269. for {
  270. m, err = reader.Read(buf)
  271. fn(buf[:m])
  272. if err == io.EOF {
  273. return false, nil
  274. }
  275. if err != nil {
  276. return false, err
  277. }
  278. }
  279. }
  280. func readEncryptedUrl(fileUrl string, cipherKey []byte, isContentCompressed bool, isFullChunk bool, offset int64, size int, fn func(data []byte)) (bool, error) {
  281. encryptedData, retryable, err := Get(fileUrl)
  282. if err != nil {
  283. return retryable, fmt.Errorf("fetch %s: %v", fileUrl, err)
  284. }
  285. decryptedData, err := Decrypt(encryptedData, CipherKey(cipherKey))
  286. if err != nil {
  287. return false, fmt.Errorf("decrypt %s: %v", fileUrl, err)
  288. }
  289. if isContentCompressed {
  290. decryptedData, err = DecompressData(decryptedData)
  291. if err != nil {
  292. glog.V(0).Infof("unzip decrypt %s: %v", fileUrl, err)
  293. }
  294. }
  295. if len(decryptedData) < int(offset)+size {
  296. return false, fmt.Errorf("read decrypted %s size %d [%d, %d)", fileUrl, len(decryptedData), offset, int(offset)+size)
  297. }
  298. if isFullChunk {
  299. fn(decryptedData)
  300. } else {
  301. fn(decryptedData[int(offset) : int(offset)+size])
  302. }
  303. return false, nil
  304. }
  305. func ReadUrlAsReaderCloser(fileUrl string, rangeHeader string) (io.ReadCloser, error) {
  306. req, err := http.NewRequest("GET", fileUrl, nil)
  307. if err != nil {
  308. return nil, err
  309. }
  310. if rangeHeader != "" {
  311. req.Header.Add("Range", rangeHeader)
  312. } else {
  313. req.Header.Add("Accept-Encoding", "gzip")
  314. }
  315. r, err := client.Do(req)
  316. if err != nil {
  317. return nil, err
  318. }
  319. if r.StatusCode >= 400 {
  320. return nil, fmt.Errorf("%s: %s", fileUrl, r.Status)
  321. }
  322. var reader io.ReadCloser
  323. contentEncoding := r.Header.Get("Content-Encoding")
  324. switch contentEncoding {
  325. case "gzip":
  326. reader, err = gzip.NewReader(r.Body)
  327. defer reader.Close()
  328. default:
  329. reader = r.Body
  330. }
  331. return reader, nil
  332. }
  333. func CloseResponse(resp *http.Response) {
  334. io.Copy(ioutil.Discard, resp.Body)
  335. resp.Body.Close()
  336. }
  337. func CloseRequest(req *http.Request) {
  338. io.Copy(ioutil.Discard, req.Body)
  339. req.Body.Close()
  340. }