You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

269 lines
5.4 KiB

9 years ago
7 years ago
10 years ago
  1. package util
  2. import (
  3. "bytes"
  4. "compress/gzip"
  5. "encoding/json"
  6. "errors"
  7. "fmt"
  8. "io"
  9. "io/ioutil"
  10. "net/http"
  11. "net/url"
  12. "strings"
  13. )
  14. var (
  15. client *http.Client
  16. Transport *http.Transport
  17. )
  18. func init() {
  19. Transport = &http.Transport{
  20. MaxIdleConnsPerHost: 1024,
  21. }
  22. client = &http.Client{
  23. Transport: Transport,
  24. }
  25. }
  26. func PostBytes(url string, body []byte) ([]byte, error) {
  27. r, err := client.Post(url, "application/octet-stream", bytes.NewReader(body))
  28. if err != nil {
  29. return nil, fmt.Errorf("Post to %s: %v", url, err)
  30. }
  31. defer r.Body.Close()
  32. if r.StatusCode >= 400 {
  33. return nil, fmt.Errorf("%s: %s", url, r.Status)
  34. }
  35. b, err := ioutil.ReadAll(r.Body)
  36. if err != nil {
  37. return nil, fmt.Errorf("Read response body: %v", err)
  38. }
  39. return b, nil
  40. }
  41. func Post(url string, values url.Values) ([]byte, error) {
  42. r, err := client.PostForm(url, values)
  43. if err != nil {
  44. return nil, err
  45. }
  46. defer r.Body.Close()
  47. b, err := ioutil.ReadAll(r.Body)
  48. if r.StatusCode >= 400 {
  49. if err != nil {
  50. return nil, fmt.Errorf("%s: %d - %s", url, r.StatusCode, string(b))
  51. } else {
  52. return nil, fmt.Errorf("%s: %s", url, r.Status)
  53. }
  54. }
  55. if err != nil {
  56. return nil, err
  57. }
  58. return b, nil
  59. }
  60. // github.com/chrislusf/seaweedfs/unmaintained/repeated_vacuum/repeated_vacuum.go
  61. // may need increasing http.Client.Timeout
  62. func Get(url string) ([]byte, error) {
  63. r, err := client.Get(url)
  64. if err != nil {
  65. return nil, err
  66. }
  67. defer r.Body.Close()
  68. b, err := ioutil.ReadAll(r.Body)
  69. if r.StatusCode >= 400 {
  70. return nil, fmt.Errorf("%s: %s", url, r.Status)
  71. }
  72. if err != nil {
  73. return nil, err
  74. }
  75. return b, nil
  76. }
  77. func Head(url string) (http.Header, error) {
  78. r, err := client.Head(url)
  79. if err != nil {
  80. return nil, err
  81. }
  82. defer r.Body.Close()
  83. if r.StatusCode >= 400 {
  84. return nil, fmt.Errorf("%s: %s", url, r.Status)
  85. }
  86. return r.Header, nil
  87. }
  88. func Delete(url string, jwt string) error {
  89. req, err := http.NewRequest("DELETE", url, nil)
  90. if jwt != "" {
  91. req.Header.Set("Authorization", "BEARER "+string(jwt))
  92. }
  93. if err != nil {
  94. return err
  95. }
  96. resp, e := client.Do(req)
  97. if e != nil {
  98. return e
  99. }
  100. defer resp.Body.Close()
  101. body, err := ioutil.ReadAll(resp.Body)
  102. if err != nil {
  103. return err
  104. }
  105. switch resp.StatusCode {
  106. case http.StatusNotFound, http.StatusAccepted, http.StatusOK:
  107. return nil
  108. }
  109. m := make(map[string]interface{})
  110. if e := json.Unmarshal(body, m); e == nil {
  111. if s, ok := m["error"].(string); ok {
  112. return errors.New(s)
  113. }
  114. }
  115. return errors.New(string(body))
  116. }
  117. func GetBufferStream(url string, values url.Values, allocatedBytes []byte, eachBuffer func([]byte)) error {
  118. r, err := client.PostForm(url, values)
  119. if err != nil {
  120. return err
  121. }
  122. defer r.Body.Close()
  123. if r.StatusCode != 200 {
  124. return fmt.Errorf("%s: %s", url, r.Status)
  125. }
  126. for {
  127. n, err := r.Body.Read(allocatedBytes)
  128. if n > 0 {
  129. eachBuffer(allocatedBytes[:n])
  130. }
  131. if err != nil {
  132. if err == io.EOF {
  133. return nil
  134. }
  135. return err
  136. }
  137. }
  138. }
  139. func GetUrlStream(url string, values url.Values, readFn func(io.Reader) error) error {
  140. r, err := client.PostForm(url, values)
  141. if err != nil {
  142. return err
  143. }
  144. defer r.Body.Close()
  145. if r.StatusCode != 200 {
  146. return fmt.Errorf("%s: %s", url, r.Status)
  147. }
  148. return readFn(r.Body)
  149. }
  150. func DownloadFile(fileUrl string) (filename string, header http.Header, rc io.ReadCloser, e error) {
  151. response, err := client.Get(fileUrl)
  152. if err != nil {
  153. return "", nil, nil, err
  154. }
  155. header = response.Header
  156. contentDisposition := response.Header["Content-Disposition"]
  157. if len(contentDisposition) > 0 {
  158. idx := strings.Index(contentDisposition[0], "filename=")
  159. if idx != -1 {
  160. filename = contentDisposition[0][idx+len("filename="):]
  161. filename = strings.Trim(filename, "\"")
  162. }
  163. }
  164. rc = response.Body
  165. return
  166. }
  167. func Do(req *http.Request) (resp *http.Response, err error) {
  168. return client.Do(req)
  169. }
  170. func NormalizeUrl(url string) string {
  171. if strings.HasPrefix(url, "http://") || strings.HasPrefix(url, "https://") {
  172. return url
  173. }
  174. return "http://" + url
  175. }
  176. func ReadUrl(fileUrl string, offset int64, size int, buf []byte, isReadRange bool) (n int64, e error) {
  177. req, _ := http.NewRequest("GET", fileUrl, nil)
  178. if isReadRange {
  179. req.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", offset, offset+int64(size)))
  180. } else {
  181. req.Header.Set("Accept-Encoding", "gzip")
  182. }
  183. r, err := client.Do(req)
  184. if err != nil {
  185. return 0, err
  186. }
  187. defer r.Body.Close()
  188. if r.StatusCode >= 400 {
  189. return 0, fmt.Errorf("%s: %s", fileUrl, r.Status)
  190. }
  191. var reader io.ReadCloser
  192. switch r.Header.Get("Content-Encoding") {
  193. case "gzip":
  194. reader, err = gzip.NewReader(r.Body)
  195. defer reader.Close()
  196. default:
  197. reader = r.Body
  198. }
  199. var i, m int
  200. for {
  201. m, err = reader.Read(buf[i:])
  202. if m == 0 {
  203. return
  204. }
  205. i += m
  206. n += int64(m)
  207. if err == io.EOF {
  208. return n, nil
  209. }
  210. if e != nil {
  211. return n, e
  212. }
  213. }
  214. }
  215. func ReadUrlAsStream(fileUrl string, offset int64, size int, fn func(data []byte)) (n int64, e error) {
  216. req, _ := http.NewRequest("GET", fileUrl, nil)
  217. req.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", offset, offset+int64(size)))
  218. r, err := client.Do(req)
  219. if err != nil {
  220. return 0, err
  221. }
  222. defer r.Body.Close()
  223. if r.StatusCode >= 400 {
  224. return 0, fmt.Errorf("%s: %s", fileUrl, r.Status)
  225. }
  226. var m int
  227. buf := make([]byte, 64*1024)
  228. for {
  229. m, err = r.Body.Read(buf)
  230. if m == 0 {
  231. return
  232. }
  233. fn(buf[:m])
  234. n += int64(m)
  235. if err == io.EOF {
  236. return n, nil
  237. }
  238. if e != nil {
  239. return n, e
  240. }
  241. }
  242. }