mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2024-01-19 02:48:24 +00:00
1b6ab2f6af
boltdb is fairly slow to write, about 6 minutes for recreating index for 1553934 files. Boltdb loads 1,553,934 x 16 = 24,862,944bytes from disk, and generate the boltdb as large as 134,217,728 bytes in 6 minutes. To compare, for leveldb, it recreates index in leveldb as large as 27,188,148 bytes in 8 seconds. For in memory version, it loads the index in To test the memory consumption, the leveldb or boltdb index are created. And the server is restarted. Using the benchmark tool to read lots of files. There are 7 volumes in benchmark collection, each with about 1553K files. For leveldb, the memory starts at 142,884KB, and stays at 179,340KB. For boltdb, the memory starts at 73,756KB, and stays at 144,564KB. For in-memory, the memory starts at 368,152KB, and stays at 448,032KB.
117 lines
3.5 KiB
Go
117 lines
3.5 KiB
Go
package weed_server
|
|
|
|
import (
|
|
"math/rand"
|
|
"net/http"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/chrislusf/weed-fs/go/glog"
|
|
"github.com/chrislusf/weed-fs/go/security"
|
|
"github.com/chrislusf/weed-fs/go/storage"
|
|
)
|
|
|
|
type VolumeServer struct {
|
|
masterNode string
|
|
mnLock sync.RWMutex
|
|
pulseSeconds int
|
|
dataCenter string
|
|
rack string
|
|
store *storage.Store
|
|
guard *security.Guard
|
|
|
|
needleMapKind storage.NeedleMapType
|
|
FixJpgOrientation bool
|
|
}
|
|
|
|
func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string,
|
|
port int, publicUrl string,
|
|
folders []string, maxCounts []int,
|
|
needleMapKind storage.NeedleMapType,
|
|
masterNode string, pulseSeconds int,
|
|
dataCenter string, rack string,
|
|
whiteList []string,
|
|
fixJpgOrientation bool) *VolumeServer {
|
|
vs := &VolumeServer{
|
|
pulseSeconds: pulseSeconds,
|
|
dataCenter: dataCenter,
|
|
rack: rack,
|
|
needleMapKind: needleMapKind,
|
|
FixJpgOrientation: fixJpgOrientation,
|
|
}
|
|
vs.SetMasterNode(masterNode)
|
|
vs.store = storage.NewStore(port, ip, publicUrl, folders, maxCounts, vs.needleMapKind)
|
|
|
|
vs.guard = security.NewGuard(whiteList, "")
|
|
|
|
adminMux.HandleFunc("/ui/index.html", vs.uiStatusHandler)
|
|
adminMux.HandleFunc("/status", vs.guard.WhiteList(vs.statusHandler))
|
|
adminMux.HandleFunc("/admin/assign_volume", vs.guard.WhiteList(vs.assignVolumeHandler))
|
|
adminMux.HandleFunc("/admin/vacuum_volume_check", vs.guard.WhiteList(vs.vacuumVolumeCheckHandler))
|
|
adminMux.HandleFunc("/admin/vacuum_volume_compact", vs.guard.WhiteList(vs.vacuumVolumeCompactHandler))
|
|
adminMux.HandleFunc("/admin/vacuum_volume_commit", vs.guard.WhiteList(vs.vacuumVolumeCommitHandler))
|
|
adminMux.HandleFunc("/admin/delete_collection", vs.guard.WhiteList(vs.deleteCollectionHandler))
|
|
adminMux.HandleFunc("/stats/counter", vs.guard.WhiteList(statsCounterHandler))
|
|
adminMux.HandleFunc("/stats/memory", vs.guard.WhiteList(statsMemoryHandler))
|
|
adminMux.HandleFunc("/stats/disk", vs.guard.WhiteList(vs.statsDiskHandler))
|
|
adminMux.HandleFunc("/delete", vs.guard.WhiteList(vs.batchDeleteHandler))
|
|
adminMux.HandleFunc("/", vs.privateStoreHandler)
|
|
if publicMux != adminMux {
|
|
// separated admin and public port
|
|
publicMux.HandleFunc("/", vs.publicReadOnlyHandler)
|
|
}
|
|
|
|
go func() {
|
|
connected := true
|
|
|
|
vs.store.SetBootstrapMaster(vs.GetMasterNode())
|
|
vs.store.SetDataCenter(vs.dataCenter)
|
|
vs.store.SetRack(vs.rack)
|
|
for {
|
|
master, secretKey, err := vs.store.SendHeartbeatToMaster()
|
|
if err == nil {
|
|
if !connected {
|
|
connected = true
|
|
vs.SetMasterNode(master)
|
|
vs.guard.SecretKey = secretKey
|
|
glog.V(0).Infoln("Volume Server Connected with master at", master)
|
|
}
|
|
} else {
|
|
glog.V(0).Infof("Volume Server Failed to talk with master %+v: %v", vs, err)
|
|
if connected {
|
|
connected = false
|
|
}
|
|
}
|
|
if connected {
|
|
time.Sleep(time.Duration(float32(vs.pulseSeconds*1e3)*(1+rand.Float32())) * time.Millisecond)
|
|
} else {
|
|
time.Sleep(time.Duration(float32(vs.pulseSeconds*1e3)*0.25) * time.Millisecond)
|
|
}
|
|
}
|
|
}()
|
|
|
|
return vs
|
|
}
|
|
|
|
func (vs *VolumeServer) GetMasterNode() string {
|
|
vs.mnLock.RLock()
|
|
defer vs.mnLock.RUnlock()
|
|
return vs.masterNode
|
|
}
|
|
|
|
func (vs *VolumeServer) SetMasterNode(masterNode string) {
|
|
vs.mnLock.Lock()
|
|
defer vs.mnLock.Unlock()
|
|
vs.masterNode = masterNode
|
|
}
|
|
|
|
func (vs *VolumeServer) Shutdown() {
|
|
glog.V(0).Infoln("Shutting down volume server...")
|
|
vs.store.Close()
|
|
glog.V(0).Infoln("Shut down successfully!")
|
|
}
|
|
|
|
func (vs *VolumeServer) jwt(fileId string) security.EncodedJwt {
|
|
return security.GenJwt(vs.guard.SecretKey, fileId)
|
|
}
|