2011-12-25 05:30:57 +00:00
|
|
|
package util
|
|
|
|
|
|
|
|
import (
|
2018-12-07 09:57:55 +00:00
|
|
|
"compress/gzip"
|
2015-12-15 06:38:58 +00:00
|
|
|
"encoding/json"
|
|
|
|
"errors"
|
2014-03-12 17:30:57 +00:00
|
|
|
"fmt"
|
2015-05-26 07:58:41 +00:00
|
|
|
"io"
|
2013-09-02 06:58:21 +00:00
|
|
|
"io/ioutil"
|
2012-06-29 07:53:47 +00:00
|
|
|
"net/http"
|
|
|
|
"net/url"
|
2014-03-30 18:28:04 +00:00
|
|
|
"strings"
|
2020-01-09 02:07:07 +00:00
|
|
|
|
|
|
|
"github.com/chrislusf/seaweedfs/weed/glog"
|
2011-12-25 05:30:57 +00:00
|
|
|
)
|
|
|
|
|
2014-03-12 17:30:57 +00:00
|
|
|
var (
|
|
|
|
client *http.Client
|
|
|
|
Transport *http.Transport
|
|
|
|
)
|
|
|
|
|
|
|
|
func init() {
|
|
|
|
Transport = &http.Transport{
|
2021-02-12 11:47:15 +00:00
|
|
|
MaxIdleConns: 1024,
|
2014-03-12 17:30:57 +00:00
|
|
|
MaxIdleConnsPerHost: 1024,
|
|
|
|
}
|
2019-01-10 11:42:31 +00:00
|
|
|
client = &http.Client{
|
|
|
|
Transport: Transport,
|
|
|
|
}
|
2014-03-12 17:30:57 +00:00
|
|
|
}
|
|
|
|
|
2012-09-17 00:31:15 +00:00
|
|
|
func Post(url string, values url.Values) ([]byte, error) {
|
2014-03-12 17:30:57 +00:00
|
|
|
r, err := client.PostForm(url, values)
|
2012-06-29 07:53:47 +00:00
|
|
|
if err != nil {
|
2012-09-17 00:31:15 +00:00
|
|
|
return nil, err
|
2012-06-29 07:53:47 +00:00
|
|
|
}
|
|
|
|
defer r.Body.Close()
|
2017-07-28 08:11:23 +00:00
|
|
|
b, err := ioutil.ReadAll(r.Body)
|
2016-07-21 06:45:55 +00:00
|
|
|
if r.StatusCode >= 400 {
|
2017-07-28 08:11:23 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, fmt.Errorf("%s: %d - %s", url, r.StatusCode, string(b))
|
|
|
|
} else {
|
|
|
|
return nil, fmt.Errorf("%s: %s", url, r.Status)
|
|
|
|
}
|
2016-07-21 06:45:55 +00:00
|
|
|
}
|
2012-06-29 07:53:47 +00:00
|
|
|
if err != nil {
|
2012-09-17 00:31:15 +00:00
|
|
|
return nil, err
|
2012-06-29 07:53:47 +00:00
|
|
|
}
|
2012-09-17 00:31:15 +00:00
|
|
|
return b, nil
|
2011-12-25 05:30:57 +00:00
|
|
|
}
|
2014-02-15 01:10:49 +00:00
|
|
|
|
2019-01-10 11:42:31 +00:00
|
|
|
// github.com/chrislusf/seaweedfs/unmaintained/repeated_vacuum/repeated_vacuum.go
|
|
|
|
// may need increasing http.Client.Timeout
|
2020-10-13 07:29:46 +00:00
|
|
|
func Get(url string) ([]byte, bool, error) {
|
2020-08-01 20:20:52 +00:00
|
|
|
|
|
|
|
request, err := http.NewRequest("GET", url, nil)
|
|
|
|
request.Header.Add("Accept-Encoding", "gzip")
|
|
|
|
|
|
|
|
response, err := client.Do(request)
|
2014-02-15 01:10:49 +00:00
|
|
|
if err != nil {
|
2020-10-13 07:29:46 +00:00
|
|
|
return nil, true, err
|
2014-02-15 01:10:49 +00:00
|
|
|
}
|
2020-08-01 20:20:52 +00:00
|
|
|
defer response.Body.Close()
|
|
|
|
|
|
|
|
var reader io.ReadCloser
|
|
|
|
switch response.Header.Get("Content-Encoding") {
|
|
|
|
case "gzip":
|
|
|
|
reader, err = gzip.NewReader(response.Body)
|
|
|
|
defer reader.Close()
|
|
|
|
default:
|
|
|
|
reader = response.Body
|
|
|
|
}
|
|
|
|
|
|
|
|
b, err := ioutil.ReadAll(reader)
|
|
|
|
if response.StatusCode >= 400 {
|
2020-10-13 07:29:46 +00:00
|
|
|
retryable := response.StatusCode >= 500
|
|
|
|
return nil, retryable, fmt.Errorf("%s: %s", url, response.Status)
|
2014-03-12 17:30:57 +00:00
|
|
|
}
|
2014-02-15 01:10:49 +00:00
|
|
|
if err != nil {
|
2020-10-13 07:29:46 +00:00
|
|
|
return nil, false, err
|
2014-02-15 01:10:49 +00:00
|
|
|
}
|
2020-10-13 07:29:46 +00:00
|
|
|
return b, false, nil
|
2014-02-15 01:10:49 +00:00
|
|
|
}
|
2014-03-20 20:30:34 +00:00
|
|
|
|
2020-02-10 21:43:53 +00:00
|
|
|
func Head(url string) (http.Header, error) {
|
|
|
|
r, err := client.Head(url)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
2017-03-17 07:02:27 +00:00
|
|
|
}
|
2020-02-14 17:46:36 +00:00
|
|
|
defer CloseResponse(r)
|
2020-02-10 21:43:53 +00:00
|
|
|
if r.StatusCode >= 400 {
|
|
|
|
return nil, fmt.Errorf("%s: %s", url, r.Status)
|
2017-03-17 07:02:27 +00:00
|
|
|
}
|
2020-02-10 21:43:53 +00:00
|
|
|
return r.Header, nil
|
2017-03-17 07:02:27 +00:00
|
|
|
}
|
|
|
|
|
2019-02-16 20:23:35 +00:00
|
|
|
func Delete(url string, jwt string) error {
|
2014-03-20 20:30:34 +00:00
|
|
|
req, err := http.NewRequest("DELETE", url, nil)
|
2015-02-07 23:35:28 +00:00
|
|
|
if jwt != "" {
|
|
|
|
req.Header.Set("Authorization", "BEARER "+string(jwt))
|
|
|
|
}
|
2014-03-20 20:30:34 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
resp, e := client.Do(req)
|
|
|
|
if e != nil {
|
|
|
|
return e
|
|
|
|
}
|
|
|
|
defer resp.Body.Close()
|
2015-12-03 08:27:02 +00:00
|
|
|
body, err := ioutil.ReadAll(resp.Body)
|
|
|
|
if err != nil {
|
2014-03-20 20:30:34 +00:00
|
|
|
return err
|
|
|
|
}
|
2015-12-03 08:27:02 +00:00
|
|
|
switch resp.StatusCode {
|
|
|
|
case http.StatusNotFound, http.StatusAccepted, http.StatusOK:
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
m := make(map[string]interface{})
|
2020-04-17 17:04:41 +00:00
|
|
|
if e := json.Unmarshal(body, &m); e == nil {
|
2015-12-03 08:27:02 +00:00
|
|
|
if s, ok := m["error"].(string); ok {
|
|
|
|
return errors.New(s)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return errors.New(string(body))
|
2014-03-20 20:30:34 +00:00
|
|
|
}
|
2014-03-30 18:28:04 +00:00
|
|
|
|
2015-05-26 07:58:41 +00:00
|
|
|
func GetBufferStream(url string, values url.Values, allocatedBytes []byte, eachBuffer func([]byte)) error {
|
|
|
|
r, err := client.PostForm(url, values)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2020-02-14 17:46:36 +00:00
|
|
|
defer CloseResponse(r)
|
2015-05-26 07:58:41 +00:00
|
|
|
if r.StatusCode != 200 {
|
|
|
|
return fmt.Errorf("%s: %s", url, r.Status)
|
|
|
|
}
|
|
|
|
for {
|
|
|
|
n, err := r.Body.Read(allocatedBytes)
|
2018-09-17 07:27:40 +00:00
|
|
|
if n > 0 {
|
|
|
|
eachBuffer(allocatedBytes[:n])
|
2015-05-26 07:58:41 +00:00
|
|
|
}
|
|
|
|
if err != nil {
|
|
|
|
if err == io.EOF {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func GetUrlStream(url string, values url.Values, readFn func(io.Reader) error) error {
|
|
|
|
r, err := client.PostForm(url, values)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2020-02-14 17:46:36 +00:00
|
|
|
defer CloseResponse(r)
|
2015-05-26 07:58:41 +00:00
|
|
|
if r.StatusCode != 200 {
|
|
|
|
return fmt.Errorf("%s: %s", url, r.Status)
|
|
|
|
}
|
|
|
|
return readFn(r.Body)
|
|
|
|
}
|
|
|
|
|
2020-09-09 10:53:09 +00:00
|
|
|
func DownloadFile(fileUrl string) (filename string, header http.Header, resp *http.Response, e error) {
|
2014-03-30 18:28:04 +00:00
|
|
|
response, err := client.Get(fileUrl)
|
|
|
|
if err != nil {
|
2018-09-21 08:54:29 +00:00
|
|
|
return "", nil, nil, err
|
2014-03-30 18:28:04 +00:00
|
|
|
}
|
2018-09-21 08:54:29 +00:00
|
|
|
header = response.Header
|
2014-03-30 18:28:04 +00:00
|
|
|
contentDisposition := response.Header["Content-Disposition"]
|
|
|
|
if len(contentDisposition) > 0 {
|
2017-01-09 06:55:21 +00:00
|
|
|
idx := strings.Index(contentDisposition[0], "filename=")
|
|
|
|
if idx != -1 {
|
|
|
|
filename = contentDisposition[0][idx+len("filename="):]
|
2014-05-13 05:59:00 +00:00
|
|
|
filename = strings.Trim(filename, "\"")
|
2014-03-30 18:28:04 +00:00
|
|
|
}
|
|
|
|
}
|
2020-09-09 10:53:09 +00:00
|
|
|
resp = response
|
2014-03-30 18:28:04 +00:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
func Do(req *http.Request) (resp *http.Response, err error) {
|
|
|
|
return client.Do(req)
|
|
|
|
}
|
2015-04-16 17:02:53 +00:00
|
|
|
|
|
|
|
func NormalizeUrl(url string) string {
|
|
|
|
if strings.HasPrefix(url, "http://") || strings.HasPrefix(url, "https://") {
|
|
|
|
return url
|
|
|
|
}
|
|
|
|
return "http://" + url
|
|
|
|
}
|
2018-05-24 08:22:37 +00:00
|
|
|
|
2020-06-20 05:45:27 +00:00
|
|
|
func ReadUrl(fileUrl string, cipherKey []byte, isContentCompressed bool, isFullChunk bool, offset int64, size int, buf []byte) (int64, error) {
|
2020-03-06 08:49:47 +00:00
|
|
|
|
|
|
|
if cipherKey != nil {
|
|
|
|
var n int
|
2020-10-13 07:29:46 +00:00
|
|
|
_, err := readEncryptedUrl(fileUrl, cipherKey, isContentCompressed, isFullChunk, offset, size, func(data []byte) {
|
2020-03-06 08:49:47 +00:00
|
|
|
n = copy(buf, data)
|
|
|
|
})
|
|
|
|
return int64(n), err
|
|
|
|
}
|
2018-05-24 08:22:37 +00:00
|
|
|
|
2019-12-26 02:03:06 +00:00
|
|
|
req, err := http.NewRequest("GET", fileUrl, nil)
|
|
|
|
if err != nil {
|
|
|
|
return 0, err
|
|
|
|
}
|
2020-03-06 08:49:47 +00:00
|
|
|
if !isFullChunk {
|
2020-01-23 06:59:12 +00:00
|
|
|
req.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", offset, offset+int64(size)-1))
|
2018-12-07 09:57:55 +00:00
|
|
|
} else {
|
|
|
|
req.Header.Set("Accept-Encoding", "gzip")
|
|
|
|
}
|
2018-05-24 08:22:37 +00:00
|
|
|
|
|
|
|
r, err := client.Do(req)
|
|
|
|
if err != nil {
|
|
|
|
return 0, err
|
|
|
|
}
|
2018-12-07 09:57:55 +00:00
|
|
|
|
2018-05-24 08:22:37 +00:00
|
|
|
defer r.Body.Close()
|
|
|
|
if r.StatusCode >= 400 {
|
|
|
|
return 0, fmt.Errorf("%s: %s", fileUrl, r.Status)
|
|
|
|
}
|
|
|
|
|
2018-12-07 09:57:55 +00:00
|
|
|
var reader io.ReadCloser
|
2020-01-09 02:07:07 +00:00
|
|
|
contentEncoding := r.Header.Get("Content-Encoding")
|
|
|
|
switch contentEncoding {
|
2018-12-07 09:57:55 +00:00
|
|
|
case "gzip":
|
|
|
|
reader, err = gzip.NewReader(r.Body)
|
|
|
|
defer reader.Close()
|
|
|
|
default:
|
|
|
|
reader = r.Body
|
|
|
|
}
|
|
|
|
|
2019-12-26 02:03:06 +00:00
|
|
|
var (
|
|
|
|
i, m int
|
|
|
|
n int64
|
|
|
|
)
|
2018-05-24 08:22:37 +00:00
|
|
|
|
2019-12-26 02:03:06 +00:00
|
|
|
// refers to https://github.com/golang/go/blob/master/src/bytes/buffer.go#L199
|
|
|
|
// commit id c170b14c2c1cfb2fd853a37add92a82fd6eb4318
|
2018-05-24 08:22:37 +00:00
|
|
|
for {
|
2018-12-07 09:57:55 +00:00
|
|
|
m, err = reader.Read(buf[i:])
|
2018-05-24 08:22:37 +00:00
|
|
|
i += m
|
|
|
|
n += int64(m)
|
|
|
|
if err == io.EOF {
|
|
|
|
return n, nil
|
|
|
|
}
|
2019-12-26 02:03:06 +00:00
|
|
|
if err != nil {
|
|
|
|
return n, err
|
2018-05-24 08:22:37 +00:00
|
|
|
}
|
2019-12-26 03:28:43 +00:00
|
|
|
if n == int64(len(buf)) {
|
|
|
|
break
|
|
|
|
}
|
2018-05-24 08:22:37 +00:00
|
|
|
}
|
2019-12-26 03:28:43 +00:00
|
|
|
// drains the response body to avoid memory leak
|
2019-12-26 05:39:33 +00:00
|
|
|
data, _ := ioutil.ReadAll(reader)
|
2019-12-26 03:28:43 +00:00
|
|
|
if len(data) != 0 {
|
2020-01-09 02:07:07 +00:00
|
|
|
glog.V(1).Infof("%s reader has remaining %d bytes", contentEncoding, len(data))
|
2019-12-26 03:28:43 +00:00
|
|
|
}
|
|
|
|
return n, err
|
2018-05-24 08:22:37 +00:00
|
|
|
}
|
2018-05-28 12:39:12 +00:00
|
|
|
|
2020-10-13 07:29:46 +00:00
|
|
|
func ReadUrlAsStream(fileUrl string, cipherKey []byte, isContentGzipped bool, isFullChunk bool, offset int64, size int, fn func(data []byte)) (retryable bool, err error) {
|
2020-03-06 08:49:47 +00:00
|
|
|
|
|
|
|
if cipherKey != nil {
|
2020-03-29 07:54:39 +00:00
|
|
|
return readEncryptedUrl(fileUrl, cipherKey, isContentGzipped, isFullChunk, offset, size, fn)
|
2020-03-06 08:49:47 +00:00
|
|
|
}
|
2018-05-28 12:39:12 +00:00
|
|
|
|
2019-12-26 02:03:06 +00:00
|
|
|
req, err := http.NewRequest("GET", fileUrl, nil)
|
|
|
|
if err != nil {
|
2020-10-13 07:29:46 +00:00
|
|
|
return false, err
|
2020-03-06 08:49:47 +00:00
|
|
|
}
|
|
|
|
|
2020-08-01 20:20:52 +00:00
|
|
|
if isFullChunk {
|
|
|
|
req.Header.Add("Accept-Encoding", "gzip")
|
|
|
|
} else {
|
2020-03-06 08:49:47 +00:00
|
|
|
req.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", offset, offset+int64(size)-1))
|
2019-12-26 02:03:06 +00:00
|
|
|
}
|
2018-05-28 12:39:12 +00:00
|
|
|
|
|
|
|
r, err := client.Do(req)
|
|
|
|
if err != nil {
|
2020-10-13 07:29:46 +00:00
|
|
|
return true, err
|
2018-05-28 12:39:12 +00:00
|
|
|
}
|
2020-02-14 17:46:36 +00:00
|
|
|
defer CloseResponse(r)
|
2018-05-28 12:39:12 +00:00
|
|
|
if r.StatusCode >= 400 {
|
2020-10-13 07:29:46 +00:00
|
|
|
retryable = r.StatusCode >= 500
|
|
|
|
return retryable, fmt.Errorf("%s: %s", fileUrl, r.Status)
|
2018-05-28 12:39:12 +00:00
|
|
|
}
|
|
|
|
|
2020-08-01 20:20:52 +00:00
|
|
|
var reader io.ReadCloser
|
|
|
|
contentEncoding := r.Header.Get("Content-Encoding")
|
|
|
|
switch contentEncoding {
|
|
|
|
case "gzip":
|
|
|
|
reader, err = gzip.NewReader(r.Body)
|
|
|
|
defer reader.Close()
|
|
|
|
default:
|
|
|
|
reader = r.Body
|
|
|
|
}
|
|
|
|
|
2019-12-26 02:03:06 +00:00
|
|
|
var (
|
|
|
|
m int
|
|
|
|
)
|
2018-05-28 12:39:12 +00:00
|
|
|
buf := make([]byte, 64*1024)
|
|
|
|
|
|
|
|
for {
|
2020-08-01 20:20:52 +00:00
|
|
|
m, err = reader.Read(buf)
|
2018-05-28 12:39:12 +00:00
|
|
|
fn(buf[:m])
|
|
|
|
if err == io.EOF {
|
2020-10-13 07:29:46 +00:00
|
|
|
return false, nil
|
2018-05-28 12:39:12 +00:00
|
|
|
}
|
2019-12-26 02:03:06 +00:00
|
|
|
if err != nil {
|
2020-10-13 07:29:46 +00:00
|
|
|
return false, err
|
2018-05-28 12:39:12 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
2020-01-31 08:11:08 +00:00
|
|
|
|
2020-10-13 07:29:46 +00:00
|
|
|
func readEncryptedUrl(fileUrl string, cipherKey []byte, isContentCompressed bool, isFullChunk bool, offset int64, size int, fn func(data []byte)) (bool, error) {
|
|
|
|
encryptedData, retryable, err := Get(fileUrl)
|
2020-03-06 08:49:47 +00:00
|
|
|
if err != nil {
|
2020-10-13 07:29:46 +00:00
|
|
|
return retryable, fmt.Errorf("fetch %s: %v", fileUrl, err)
|
2020-03-06 08:49:47 +00:00
|
|
|
}
|
|
|
|
decryptedData, err := Decrypt(encryptedData, CipherKey(cipherKey))
|
|
|
|
if err != nil {
|
2020-10-13 07:29:46 +00:00
|
|
|
return false, fmt.Errorf("decrypt %s: %v", fileUrl, err)
|
2020-03-06 08:49:47 +00:00
|
|
|
}
|
2020-06-20 05:45:27 +00:00
|
|
|
if isContentCompressed {
|
2020-06-20 15:16:16 +00:00
|
|
|
decryptedData, err = DecompressData(decryptedData)
|
2020-03-09 04:39:33 +00:00
|
|
|
if err != nil {
|
2020-08-01 20:46:52 +00:00
|
|
|
glog.V(0).Infof("unzip decrypt %s: %v", fileUrl, err)
|
2020-03-09 04:39:33 +00:00
|
|
|
}
|
|
|
|
}
|
2020-03-06 08:49:47 +00:00
|
|
|
if len(decryptedData) < int(offset)+size {
|
2020-10-13 07:29:46 +00:00
|
|
|
return false, fmt.Errorf("read decrypted %s size %d [%d, %d)", fileUrl, len(decryptedData), offset, int(offset)+size)
|
2020-03-06 08:49:47 +00:00
|
|
|
}
|
2020-03-29 07:54:39 +00:00
|
|
|
if isFullChunk {
|
|
|
|
fn(decryptedData)
|
|
|
|
} else {
|
|
|
|
fn(decryptedData[int(offset) : int(offset)+size])
|
|
|
|
}
|
2020-10-13 07:29:46 +00:00
|
|
|
return false, nil
|
2020-03-06 08:49:47 +00:00
|
|
|
}
|
|
|
|
|
2020-01-31 08:11:08 +00:00
|
|
|
func ReadUrlAsReaderCloser(fileUrl string, rangeHeader string) (io.ReadCloser, error) {
|
|
|
|
|
|
|
|
req, err := http.NewRequest("GET", fileUrl, nil)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
if rangeHeader != "" {
|
|
|
|
req.Header.Add("Range", rangeHeader)
|
2020-08-01 20:20:52 +00:00
|
|
|
} else {
|
|
|
|
req.Header.Add("Accept-Encoding", "gzip")
|
2020-01-31 08:11:08 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
r, err := client.Do(req)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
if r.StatusCode >= 400 {
|
|
|
|
return nil, fmt.Errorf("%s: %s", fileUrl, r.Status)
|
|
|
|
}
|
|
|
|
|
2020-08-01 20:20:52 +00:00
|
|
|
var reader io.ReadCloser
|
|
|
|
contentEncoding := r.Header.Get("Content-Encoding")
|
|
|
|
switch contentEncoding {
|
|
|
|
case "gzip":
|
|
|
|
reader, err = gzip.NewReader(r.Body)
|
|
|
|
defer reader.Close()
|
|
|
|
default:
|
|
|
|
reader = r.Body
|
|
|
|
}
|
|
|
|
|
|
|
|
return reader, nil
|
2020-01-31 08:11:08 +00:00
|
|
|
}
|
2020-02-14 17:09:15 +00:00
|
|
|
|
2020-02-14 17:46:36 +00:00
|
|
|
func CloseResponse(resp *http.Response) {
|
2020-02-14 17:09:15 +00:00
|
|
|
io.Copy(ioutil.Discard, resp.Body)
|
|
|
|
resp.Body.Close()
|
|
|
|
}
|
2020-11-30 10:45:00 +00:00
|
|
|
|
|
|
|
func CloseRequest(req *http.Request) {
|
|
|
|
io.Copy(ioutil.Discard, req.Body)
|
|
|
|
req.Body.Close()
|
|
|
|
}
|