gitlab-com-seaweedfs/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-01-19 02:48:24 +00:00
Code Issues Projects Releases Wiki Activity

revert fasthttp changes

Browse source 
related to https://github.com/chrislusf/seaweedfs/issues/1907
This commit is contained in:
Chris Lu 2021-03-16 00:33:14 -07:00
parent 10164d0386
commit 4b1ed227d1
7 changed files with 6 additions and 122 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
weed/command/benchmark.go
View file

@ -305,7 +305,7 @@ func readFiles(fileIdLineChan chan string, s *stat) {
}
var bytes []byte
for _, url := range urls {
bytes, _, err = util.FastGet(url)
bytes, _, err = util.Get(url)
if err == nil {
break
}

2
weed/filer/filechunk_manifest.go
View file

@ -102,7 +102,7 @@ func retriedFetchChunkData(urlStrings []string, cipherKey []byte, isGzipped bool
for waitTime := time.Second; waitTime < util.RetryWaitTime; waitTime += waitTime / 2 {
for _, urlString := range urlStrings {
shouldRetry, err = util.FastReadUrlAsStream(urlString+"?readDeleted=true", cipherKey, isGzipped, isFullChunk, offset, size, func(data []byte) {
shouldRetry, err = util.ReadUrlAsStream(urlString+"?readDeleted=true", cipherKey, isGzipped, isFullChunk, offset, size, func(data []byte) {
buffer.Write(data)
})
if !shouldRetry {

2
weed/filer/read_write.go
View file

@ -35,7 +35,7 @@ func ReadContent(filerAddress string, dir, name string) ([]byte, error) {
target := fmt.Sprintf("http://%s%s/%s", filerAddress, dir, name)
data, _, err := util.FastGet(target)
data, _, err := util.Get(target)
return data, err
}

2
weed/filer/stream.go
View file

@ -185,7 +185,7 @@ func (c *ChunkStreamReader) fetchChunkToBuffer(chunkView *ChunkView) error {
var buffer bytes.Buffer
var shouldRetry bool
for _, urlString := range urlStrings {
shouldRetry, err = util.FastReadUrlAsStream(urlString, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.Offset, int(chunkView.Size), func(data []byte) {
shouldRetry, err = util.ReadUrlAsStream(urlString, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.Offset, int(chunkView.Size), func(data []byte) {
buffer.Write(data)
})
if !shouldRetry {

2
weed/replication/repl_util/replication_util.go
View file

@ -20,7 +20,7 @@ func CopyFromChunkViews(chunkViews []*filer.ChunkView, filerSource *source.Filer
var shouldRetry bool
for _, fileUrl := range fileUrls {
shouldRetry, err = util.FastReadUrlAsStream(fileUrl, nil, false, chunk.IsFullChunk(), chunk.Offset, int(chunk.Size), func(data []byte) {
shouldRetry, err = util.ReadUrlAsStream(fileUrl, nil, false, chunk.IsFullChunk(), chunk.Offset, int(chunk.Size), func(data []byte) {
writeErr = writeFunc(data)
})
if err != nil {

116
weed/util/fasthttp_util.go
View file

@ -1,116 +0,0 @@
package util
import (
"bytes"
"fmt"
"github.com/valyala/fasthttp"
"sync"
"time"
)
var (
fastClient = &fasthttp.Client{
NoDefaultUserAgentHeader: true, // Don't send: User-Agent: fasthttp
MaxConnsPerHost: 1024,
ReadBufferSize: 4096, // Make sure to set this big enough that your whole request can be read at once.
WriteBufferSize: 64 * 1024, // Same but for your response.
ReadTimeout: time.Second,
WriteTimeout: time.Second,
MaxIdleConnDuration: time.Minute,
DisableHeaderNamesNormalizing: true, // If you set the case on your headers correctly you can enable this.
DialDualStack: true,
}
// Put everything in pools to prevent garbage.
bytesPool = sync.Pool{
New: func() interface{} {
b := make([]byte, 0)
return &b
},
}
responsePool = sync.Pool{
New: func() interface{} {
return make(chan *fasthttp.Response)
},
}
)
func FastGet(url string) ([]byte, bool, error) {
req := fasthttp.AcquireRequest()
res := fasthttp.AcquireResponse()
defer fasthttp.ReleaseRequest(req)
defer fasthttp.ReleaseResponse(res)
req.SetRequestURIBytes([]byte(url))
req.Header.Add("Accept-Encoding", "gzip")
err := fastClient.Do(req, res)
if err != nil {
return nil, true, err
}
var data []byte
contentEncoding := res.Header.Peek("Content-Encoding")
if bytes.Compare(contentEncoding, []byte("gzip")) == 0 {
data, err = res.BodyGunzip()
} else {
data = res.Body()
}
out := make([]byte, len(data))
copy(out, data)
if res.StatusCode() >= 400 {
retryable := res.StatusCode() >= 500
return nil, retryable, fmt.Errorf("%s: %d", url, res.StatusCode())
}
if err != nil {
return nil, false, err
}
return out, false, nil
}
func FastReadUrlAsStream(fileUrl string, cipherKey []byte, isContentGzipped bool, isFullChunk bool, offset int64, size int, fn func(data []byte)) (retryable bool, err error) {
if cipherKey != nil {
return readEncryptedUrl(fileUrl, cipherKey, isContentGzipped, isFullChunk, offset, size, fn)
}
req := fasthttp.AcquireRequest()
res := fasthttp.AcquireResponse()
defer fasthttp.ReleaseRequest(req)
defer fasthttp.ReleaseResponse(res)
req.SetRequestURIBytes([]byte(fileUrl))
if isFullChunk {
req.Header.Add("Accept-Encoding", "gzip")
} else {
req.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", offset, offset+int64(size)-1))
}
if err = fastClient.Do(req, res); err != nil {
return true, err
}
if res.StatusCode() >= 400 {
retryable = res.StatusCode() >= 500
return retryable, fmt.Errorf("%s: %d", fileUrl, res.StatusCode())
}
contentEncoding := res.Header.Peek("Content-Encoding")
if bytes.Compare(contentEncoding, []byte("gzip")) == 0 {
bodyData, err := res.BodyGunzip()
if err != nil {
return false, err
}
fn(bodyData)
} else {
fn(res.Body())
}
return false, nil
}

2
weed/util/http_util.go
View file

@ -313,7 +313,7 @@ func ReadUrlAsStream(fileUrl string, cipherKey []byte, isContentGzipped bool, is
}
func readEncryptedUrl(fileUrl string, cipherKey []byte, isContentCompressed bool, isFullChunk bool, offset int64, size int, fn func(data []byte)) (bool, error) {
encryptedData, retryable, err := FastGet(fileUrl)
encryptedData, retryable, err := Get(fileUrl)
if err != nil {
return retryable, fmt.Errorf("fetch %s: %v", fileUrl, err)
}