gitlab-com-seaweedfs/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-01-19 02:48:24 +00:00
Code Issues Projects Releases Wiki Activity

tiered caching

Browse source 
1/4 for small less than 1MB files. 1/4 for 1~4MB files, 1/2 for bigger than 4MB files
This commit is contained in:
Chris Lu 2020-04-13 22:19:27 -07:00
parent f282ed444b
commit 2b5c4fbbf3
5 changed files with 41 additions and 21 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
weed/filesys/wfs.go
View file

@ -84,7 +84,7 @@ func NewSeaweedFileSystem(option *Option) *WFS {
},
}
if option.CacheSizeMB > 0 {
wfs.chunkCache = chunk_cache.NewChunkCache(256, option.CacheDir, option.CacheSizeMB, 4)
wfs.chunkCache = chunk_cache.NewChunkCache(256, option.CacheDir, option.CacheSizeMB)
util.OnInterrupt(func() {
wfs.chunkCache.Shutdown()
})

2
weed/server/webdav_server.go
View file

@ -99,7 +99,7 @@ type WebDavFile struct {
func NewWebDavFileSystem(option *WebDavOption) (webdav.FileSystem, error) {
chunkCache := chunk_cache.NewChunkCache(256, option.CacheDir, option.CacheSizeMB, 4)
chunkCache := chunk_cache.NewChunkCache(256, option.CacheDir, option.CacheSizeMB)
util.OnInterrupt(func() {
chunkCache.Shutdown()
})

37
weed/util/chunk_cache/chunk_cache.go
View file

@ -9,26 +9,26 @@ import (
const (
memCacheSizeLimit = 1024 * 1024
onDiskCacheSizeLimit0 = memCacheSizeLimit
onDiskCacheSizeLimit1 = 4 * memCacheSizeLimit
)
// a global cache for recently accessed file chunks
type ChunkCache struct {
memCache *ChunkCacheInMemory
diskCache *OnDiskCacheLayer
diskCaches []*OnDiskCacheLayer
sync.RWMutex
}
func NewChunkCache(maxEntries int64, dir string, diskSizeMB int64, segmentCount int) *ChunkCache {
volumeCount, volumeSize := int(diskSizeMB/30000), int64(30000)
if volumeCount < segmentCount {
volumeCount, volumeSize = segmentCount, diskSizeMB/int64(segmentCount)
}
func NewChunkCache(maxEntries int64, dir string, diskSizeMB int64) *ChunkCache {
c := &ChunkCache{
memCache: NewChunkCacheInMemory(maxEntries),
diskCache: NewOnDiskCacheLayer(dir, "cache", volumeCount, volumeSize),
}
c.diskCaches = make([]*OnDiskCacheLayer, 3)
c.diskCaches[0] = NewOnDiskCacheLayer(dir, "c0_1", diskSizeMB/4, 4)
c.diskCaches[1] = NewOnDiskCacheLayer(dir, "c1_4", diskSizeMB/4, 4)
c.diskCaches[2] = NewOnDiskCacheLayer(dir, "cache", diskSizeMB/2, 4)
return c
}
@ -58,7 +58,14 @@ func (c *ChunkCache) doGetChunk(fileId string, chunkSize uint64) (data []byte) {
return nil
}
return c.diskCache.getChunk(fid.Key)
for _, diskCache := range c.diskCaches {
data := diskCache.getChunk(fid.Key)
if len(data) != 0 {
return data
}
}
return nil
}
@ -84,7 +91,13 @@ func (c *ChunkCache) doSetChunk(fileId string, data []byte) {
return
}
c.diskCache.setChunk(fid.Key, data)
if len(data) < onDiskCacheSizeLimit0 {
c.diskCaches[0].setChunk(fid.Key, data)
} else if len(data) < onDiskCacheSizeLimit1 {
c.diskCaches[1].setChunk(fid.Key, data)
} else {
c.diskCaches[2].setChunk(fid.Key, data)
}
}
@ -94,5 +107,7 @@ func (c *ChunkCache) Shutdown() {
}
c.Lock()
defer c.Unlock()
c.diskCache.shutdown()
for _, diskCache := range c.diskCaches {
diskCache.shutdown()
}
}

7
weed/util/chunk_cache/chunk_cache_on_disk_test.go
View file

@ -14,10 +14,9 @@ func TestOnDisk(t *testing.T) {
tmpDir, _ := ioutil.TempDir("", "c")
defer os.RemoveAll(tmpDir)
totalDiskSizeMb := int64(6)
segmentCount := 2
totalDiskSizeMb := int64(32)
cache := NewChunkCache(0, tmpDir, totalDiskSizeMb, segmentCount)
cache := NewChunkCache(0, tmpDir, totalDiskSizeMb)
writeCount := 5
type test_data struct {
@ -46,7 +45,7 @@ func TestOnDisk(t *testing.T) {
cache.Shutdown()
cache = NewChunkCache(0, tmpDir, totalDiskSizeMb, segmentCount)
cache = NewChunkCache(0, tmpDir, totalDiskSizeMb)
for i := 0; i < writeCount; i++ {
data := cache.GetChunk(testData[i].fileId, testData[i].size)

8
weed/util/chunk_cache/on_disk_cache_layer.go
View file

@ -14,7 +14,13 @@ type OnDiskCacheLayer struct {
diskCaches []*ChunkCacheVolume
}
func NewOnDiskCacheLayer(dir, namePrefix string, volumeCount int, volumeSize int64) *OnDiskCacheLayer{
func NewOnDiskCacheLayer(dir, namePrefix string, diskSizeMB int64, segmentCount int) *OnDiskCacheLayer{
volumeCount, volumeSize := int(diskSizeMB/30000), int64(30000)
if volumeCount < segmentCount {
volumeCount, volumeSize = segmentCount, diskSizeMB/int64(segmentCount)
}
c := &OnDiskCacheLayer{}
for i := 0; i < volumeCount; i++ {
fileName := path.Join(dir, fmt.Sprintf("%s_%d", namePrefix, i))