|
@ -67,14 +67,14 @@ func Post(url string, values url.Values) ([]byte, error) { |
|
|
|
|
|
|
|
|
// github.com/chrislusf/seaweedfs/unmaintained/repeated_vacuum/repeated_vacuum.go
|
|
|
// github.com/chrislusf/seaweedfs/unmaintained/repeated_vacuum/repeated_vacuum.go
|
|
|
// may need increasing http.Client.Timeout
|
|
|
// may need increasing http.Client.Timeout
|
|
|
func Get(url string) ([]byte, error) { |
|
|
|
|
|
|
|
|
func Get(url string) ([]byte, bool, error) { |
|
|
|
|
|
|
|
|
request, err := http.NewRequest("GET", url, nil) |
|
|
request, err := http.NewRequest("GET", url, nil) |
|
|
request.Header.Add("Accept-Encoding", "gzip") |
|
|
request.Header.Add("Accept-Encoding", "gzip") |
|
|
|
|
|
|
|
|
response, err := client.Do(request) |
|
|
response, err := client.Do(request) |
|
|
if err != nil { |
|
|
if err != nil { |
|
|
return nil, err |
|
|
|
|
|
|
|
|
return nil, true, err |
|
|
} |
|
|
} |
|
|
defer response.Body.Close() |
|
|
defer response.Body.Close() |
|
|
|
|
|
|
|
@ -89,12 +89,13 @@ func Get(url string) ([]byte, error) { |
|
|
|
|
|
|
|
|
b, err := ioutil.ReadAll(reader) |
|
|
b, err := ioutil.ReadAll(reader) |
|
|
if response.StatusCode >= 400 { |
|
|
if response.StatusCode >= 400 { |
|
|
return nil, fmt.Errorf("%s: %s", url, response.Status) |
|
|
|
|
|
|
|
|
retryable := response.StatusCode >= 500 |
|
|
|
|
|
return nil, retryable, fmt.Errorf("%s: %s", url, response.Status) |
|
|
} |
|
|
} |
|
|
if err != nil { |
|
|
if err != nil { |
|
|
return nil, err |
|
|
|
|
|
|
|
|
return nil, false, err |
|
|
} |
|
|
} |
|
|
return b, nil |
|
|
|
|
|
|
|
|
return b, false, nil |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
func Head(url string) (http.Header, error) { |
|
|
func Head(url string) (http.Header, error) { |
|
@ -207,7 +208,7 @@ func ReadUrl(fileUrl string, cipherKey []byte, isContentCompressed bool, isFullC |
|
|
|
|
|
|
|
|
if cipherKey != nil { |
|
|
if cipherKey != nil { |
|
|
var n int |
|
|
var n int |
|
|
err := readEncryptedUrl(fileUrl, cipherKey, isContentCompressed, isFullChunk, offset, size, func(data []byte) { |
|
|
|
|
|
|
|
|
_, err := readEncryptedUrl(fileUrl, cipherKey, isContentCompressed, isFullChunk, offset, size, func(data []byte) { |
|
|
n = copy(buf, data) |
|
|
n = copy(buf, data) |
|
|
}) |
|
|
}) |
|
|
return int64(n), err |
|
|
return int64(n), err |
|
@ -272,7 +273,7 @@ func ReadUrl(fileUrl string, cipherKey []byte, isContentCompressed bool, isFullC |
|
|
return n, err |
|
|
return n, err |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
func ReadUrlAsStream(fileUrl string, cipherKey []byte, isContentGzipped bool, isFullChunk bool, offset int64, size int, fn func(data []byte)) error { |
|
|
|
|
|
|
|
|
func ReadUrlAsStream(fileUrl string, cipherKey []byte, isContentGzipped bool, isFullChunk bool, offset int64, size int, fn func(data []byte)) (retryable bool, err error) { |
|
|
|
|
|
|
|
|
if cipherKey != nil { |
|
|
if cipherKey != nil { |
|
|
return readEncryptedUrl(fileUrl, cipherKey, isContentGzipped, isFullChunk, offset, size, fn) |
|
|
return readEncryptedUrl(fileUrl, cipherKey, isContentGzipped, isFullChunk, offset, size, fn) |
|
@ -280,7 +281,7 @@ func ReadUrlAsStream(fileUrl string, cipherKey []byte, isContentGzipped bool, is |
|
|
|
|
|
|
|
|
req, err := http.NewRequest("GET", fileUrl, nil) |
|
|
req, err := http.NewRequest("GET", fileUrl, nil) |
|
|
if err != nil { |
|
|
if err != nil { |
|
|
return err |
|
|
|
|
|
|
|
|
return false, err |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
if isFullChunk { |
|
|
if isFullChunk { |
|
@ -291,11 +292,12 @@ func ReadUrlAsStream(fileUrl string, cipherKey []byte, isContentGzipped bool, is |
|
|
|
|
|
|
|
|
r, err := client.Do(req) |
|
|
r, err := client.Do(req) |
|
|
if err != nil { |
|
|
if err != nil { |
|
|
return err |
|
|
|
|
|
|
|
|
return true, err |
|
|
} |
|
|
} |
|
|
defer CloseResponse(r) |
|
|
defer CloseResponse(r) |
|
|
if r.StatusCode >= 400 { |
|
|
if r.StatusCode >= 400 { |
|
|
return fmt.Errorf("%s: %s", fileUrl, r.Status) |
|
|
|
|
|
|
|
|
retryable = r.StatusCode >= 500 |
|
|
|
|
|
return retryable, fmt.Errorf("%s: %s", fileUrl, r.Status) |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
var reader io.ReadCloser |
|
|
var reader io.ReadCloser |
|
@ -317,23 +319,23 @@ func ReadUrlAsStream(fileUrl string, cipherKey []byte, isContentGzipped bool, is |
|
|
m, err = reader.Read(buf) |
|
|
m, err = reader.Read(buf) |
|
|
fn(buf[:m]) |
|
|
fn(buf[:m]) |
|
|
if err == io.EOF { |
|
|
if err == io.EOF { |
|
|
return nil |
|
|
|
|
|
|
|
|
return false, nil |
|
|
} |
|
|
} |
|
|
if err != nil { |
|
|
if err != nil { |
|
|
return err |
|
|
|
|
|
|
|
|
return false, err |
|
|
} |
|
|
} |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
func readEncryptedUrl(fileUrl string, cipherKey []byte, isContentCompressed bool, isFullChunk bool, offset int64, size int, fn func(data []byte)) error { |
|
|
|
|
|
encryptedData, err := Get(fileUrl) |
|
|
|
|
|
|
|
|
func readEncryptedUrl(fileUrl string, cipherKey []byte, isContentCompressed bool, isFullChunk bool, offset int64, size int, fn func(data []byte)) (bool, error) { |
|
|
|
|
|
encryptedData, retryable, err := Get(fileUrl) |
|
|
if err != nil { |
|
|
if err != nil { |
|
|
return fmt.Errorf("fetch %s: %v", fileUrl, err) |
|
|
|
|
|
|
|
|
return retryable, fmt.Errorf("fetch %s: %v", fileUrl, err) |
|
|
} |
|
|
} |
|
|
decryptedData, err := Decrypt(encryptedData, CipherKey(cipherKey)) |
|
|
decryptedData, err := Decrypt(encryptedData, CipherKey(cipherKey)) |
|
|
if err != nil { |
|
|
if err != nil { |
|
|
return fmt.Errorf("decrypt %s: %v", fileUrl, err) |
|
|
|
|
|
|
|
|
return false, fmt.Errorf("decrypt %s: %v", fileUrl, err) |
|
|
} |
|
|
} |
|
|
if isContentCompressed { |
|
|
if isContentCompressed { |
|
|
decryptedData, err = DecompressData(decryptedData) |
|
|
decryptedData, err = DecompressData(decryptedData) |
|
@ -342,14 +344,14 @@ func readEncryptedUrl(fileUrl string, cipherKey []byte, isContentCompressed bool |
|
|
} |
|
|
} |
|
|
} |
|
|
} |
|
|
if len(decryptedData) < int(offset)+size { |
|
|
if len(decryptedData) < int(offset)+size { |
|
|
return fmt.Errorf("read decrypted %s size %d [%d, %d)", fileUrl, len(decryptedData), offset, int(offset)+size) |
|
|
|
|
|
|
|
|
return false, fmt.Errorf("read decrypted %s size %d [%d, %d)", fileUrl, len(decryptedData), offset, int(offset)+size) |
|
|
} |
|
|
} |
|
|
if isFullChunk { |
|
|
if isFullChunk { |
|
|
fn(decryptedData) |
|
|
fn(decryptedData) |
|
|
} else { |
|
|
} else { |
|
|
fn(decryptedData[int(offset) : int(offset)+size]) |
|
|
fn(decryptedData[int(offset) : int(offset)+size]) |
|
|
} |
|
|
} |
|
|
return nil |
|
|
|
|
|
|
|
|
return false, nil |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
func ReadUrlAsReaderCloser(fileUrl string, rangeHeader string) (io.ReadCloser, error) { |
|
|
func ReadUrlAsReaderCloser(fileUrl string, rangeHeader string) (io.ReadCloser, error) { |
|
|