gitlab-com-seaweedfs/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-01-19 02:48:24 +00:00
Code Issues Projects Releases Wiki Activity

Changed the InMemory bool to a uint32 so that it can be used to alter how much space to reserve

Browse source
This commit is contained in:
Tom Maxwell 2019-09-04 15:27:14 +01:00
parent 6ee65356e3
commit 4a878c0006
20 changed files with 111 additions and 67 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
weed/command/backup.go
View file

@ -112,7 +112,7 @@ func runBackup(cmd *Command, args []string) bool {
return true return true
} }
} }
v, err := storage.NewVolume(*s.dir, *s.collection, vid, storage.NeedleMapInMemory, replication, ttl, 0, false) v, err := storage.NewVolume(*s.dir, *s.collection, vid, storage.NeedleMapInMemory, replication, ttl, 0, 0)
if err != nil { if err != nil {
fmt.Printf("Error creating or reading from volume %d: %v\n", vid, err) fmt.Printf("Error creating or reading from volume %d: %v\n", vid, err)
return true return true
@ -137,7 +137,7 @@ func runBackup(cmd *Command, args []string) bool {
// remove the old data // remove the old data
v.Destroy() v.Destroy()
// recreate an empty volume // recreate an empty volume
v, err = storage.NewVolume(*s.dir, *s.collection, vid, storage.NeedleMapInMemory, replication, ttl, 0, false) v, err = storage.NewVolume(*s.dir, *s.collection, vid, storage.NeedleMapInMemory, replication, ttl, 0, 0)
if err != nil { if err != nil {
fmt.Printf("Error creating or reading from volume %d: %v\n", vid, err) fmt.Printf("Error creating or reading from volume %d: %v\n", vid, err)
return true return true

2
weed/command/compact.go
View file

@ -38,7 +38,7 @@ func runCompact(cmd *Command, args []string) bool {
vid := needle.VolumeId(*compactVolumeId) vid := needle.VolumeId(*compactVolumeId)
v, err := storage.NewVolume(*compactVolumePath, *compactVolumeCollection, vid, v, err := storage.NewVolume(*compactVolumePath, *compactVolumeCollection, vid,
storage.NeedleMapInMemory, nil, nil, preallocate, false) storage.NeedleMapInMemory, nil, nil, preallocate, 0)
if err != nil { if err != nil {
glog.Fatalf("Load Volume [ERROR] %s\n", err) glog.Fatalf("Load Volume [ERROR] %s\n", err)
} }

2
weed/pb/master.proto
View file

@ -139,7 +139,7 @@ message AssignRequest {
string data_center = 5; string data_center = 5;
string rack = 6; string rack = 6;
string data_node = 7; string data_node = 7;
bool in_memory = 8; uint32 MemoryMapMaxSizeMB = 8;
} }
message AssignResponse { message AssignResponse {
string fid = 1; string fid = 1;

22
weed/pb/master_pb/master.pb.go
View file

@ -651,14 +651,14 @@ func (m *Location) GetPublicUrl() string {
} }
type AssignRequest struct { type AssignRequest struct {
Count uint64 `protobuf:"varint,1,opt,name=count" json:"count,omitempty"` Count uint64 `protobuf:"varint,1,opt,name=count" json:"count,omitempty"`
Replication string `protobuf:"bytes,2,opt,name=replication" json:"replication,omitempty"` Replication string `protobuf:"bytes,2,opt,name=replication" json:"replication,omitempty"`
Collection string `protobuf:"bytes,3,opt,name=collection" json:"collection,omitempty"` Collection string `protobuf:"bytes,3,opt,name=collection" json:"collection,omitempty"`
Ttl string `protobuf:"bytes,4,opt,name=ttl" json:"ttl,omitempty"` Ttl string `protobuf:"bytes,4,opt,name=ttl" json:"ttl,omitempty"`
DataCenter string `protobuf:"bytes,5,opt,name=data_center,json=dataCenter" json:"data_center,omitempty"` DataCenter string `protobuf:"bytes,5,opt,name=data_center,json=dataCenter" json:"data_center,omitempty"`
Rack string `protobuf:"bytes,6,opt,name=rack" json:"rack,omitempty"` Rack string `protobuf:"bytes,6,opt,name=rack" json:"rack,omitempty"`
DataNode string `protobuf:"bytes,7,opt,name=data_node,json=dataNode" json:"data_node,omitempty"` DataNode string `protobuf:"bytes,7,opt,name=data_node,json=dataNode" json:"data_node,omitempty"`
InMemory bool `protobuf:"bytes,4,opt,name=inmemory" json:"inmemory,omitempty"` MemoryMapMaxSizeMB uint32 `protobuf:"varint,8,opt,name=memorymapmaxsizemb" json:"memorymapmaxsizemb,omitempty"`
} }
func (m *AssignRequest) Reset() { *m = AssignRequest{} } func (m *AssignRequest) Reset() { *m = AssignRequest{} }
@ -715,11 +715,11 @@ func (m *AssignRequest) GetDataNode() string {
return "" return ""
} }
func (m *AssignRequest) GetInMemory() bool { func (m *AssignRequest) GetMemoryMapMaxSizeMB() uint32 {
if m != nil { if m != nil {
return m.InMemory return m.MemoryMapMaxSizeMB
} }
return false return 0
} }
type AssignResponse struct { type AssignResponse struct {

2
weed/pb/volume_server.proto
View file

@ -127,7 +127,7 @@ message AllocateVolumeRequest {
int64 preallocate = 3; int64 preallocate = 3;
string replication = 4; string replication = 4;
string ttl = 5; string ttl = 5;
bool inmemory = 6; int32 memorymapmaxsizemb = 6;
} }
message AllocateVolumeResponse { message AllocateVolumeResponse {
} }

31
weed/pb/volume_server_pb/volume_server.pb.go
View file

@ -316,12 +316,12 @@ func (*DeleteCollectionResponse) ProtoMessage() {}
func (*DeleteCollectionResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{13} } func (*DeleteCollectionResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{13} }
type AllocateVolumeRequest struct { type AllocateVolumeRequest struct {
VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"` VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"`
Collection string `protobuf:"bytes,2,opt,name=collection" json:"collection,omitempty"` Collection string `protobuf:"bytes,2,opt,name=collection" json:"collection,omitempty"`
Preallocate int64 `protobuf:"varint,3,opt,name=preallocate" json:"preallocate,omitempty"` Preallocate int64 `protobuf:"varint,3,opt,name=preallocate" json:"preallocate,omitempty"`
Replication string `protobuf:"bytes,4,opt,name=replication" json:"replication,omitempty"` Replication string `protobuf:"bytes,4,opt,name=replication" json:"replication,omitempty"`
Ttl string `protobuf:"bytes,5,opt,name=ttl" json:"ttl,omitempty"` Ttl string `protobuf:"bytes,5,opt,name=ttl" json:"ttl,omitempty"`
InMemory bool `protobuf:"varint,6,opt,name=inmemory" json:"inmemory,omitempty"` MemoryMapMaxSizeMB uint32 `protobuf:"varint,6,opt,name=memorymapmaxsizemb" json:"memorymapmaxsizemb,omitempty"`
} }
func (m *AllocateVolumeRequest) Reset() { *m = AllocateVolumeRequest{} } func (m *AllocateVolumeRequest) Reset() { *m = AllocateVolumeRequest{} }
@ -364,11 +364,11 @@ func (m *AllocateVolumeRequest) GetTtl() string {
return "" return ""
} }
func (m *AllocateVolumeRequest) GetInMemory() bool { func (m *AllocateVolumeRequest) GetMemoryMapMaxSizeMB() uint32 {
if m != nil { if m != nil {
return m.InMemory return m.MemoryMapMaxSizeMB
} }
return false return 0
} }
type AllocateVolumeResponse struct { type AllocateVolumeResponse struct {
@ -1978,6 +1978,19 @@ func _VolumeServer_DeleteCollection_Handler(srv interface{}, ctx context.Context
func _VolumeServer_AllocateVolume_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { func _VolumeServer_AllocateVolume_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(AllocateVolumeRequest) in := new(AllocateVolumeRequest)
if in.MemoryMapMaxSizeMB > 0 {
test := 6
test += 5
}
if in.MemoryMapMaxSizeMB == 77 {
test := 7
test += 656
}
if in.Ttl != "" {
test := 2345
test += 567
}
if err := dec(in); err != nil { if err := dec(in); err != nil {
return nil, err return nil, err
} }

15
weed/server/master_grpc_server_volume.go
View file

@ -62,13 +62,14 @@ func (ms *MasterServer) Assign(ctx context.Context, req *master_pb.AssignRequest
} }
option := &topology.VolumeGrowOption{ option := &topology.VolumeGrowOption{
Collection: req.Collection, Collection: req.Collection,
ReplicaPlacement: replicaPlacement, ReplicaPlacement: replicaPlacement,
Ttl: ttl, Ttl: ttl,
Prealloacte: ms.preallocateSize, Prealloacte: ms.preallocateSize,
DataCenter: req.DataCenter, DataCenter: req.DataCenter,
Rack: req.Rack, Rack: req.Rack,
DataNode: req.DataNode, DataNode: req.DataNode,
MemoryMapMaxSizeMB: req.MemoryMapMaxSizeMB,
} }
if !ms.Topo.HasWritableVolume(option) { if !ms.Topo.HasWritableVolume(option) {

20
weed/server/master_server_handlers_admin.go
View file

@ -148,6 +148,11 @@ func (ms *MasterServer) getVolumeGrowOption(r *http.Request) (*topology.VolumeGr
if err != nil { if err != nil {
return nil, err return nil, err
} }
memoryMapMaxSizeMB, err := needle.ReadMemoryMapMaxSizeMB(r.FormValue("memorymapmaxsizemb"))
if err != nil {
return nil, err
}
preallocate := ms.preallocateSize preallocate := ms.preallocateSize
if r.FormValue("preallocate") != "" { if r.FormValue("preallocate") != "" {
preallocate, err = strconv.ParseInt(r.FormValue("preallocate"), 10, 64) preallocate, err = strconv.ParseInt(r.FormValue("preallocate"), 10, 64)
@ -156,13 +161,14 @@ func (ms *MasterServer) getVolumeGrowOption(r *http.Request) (*topology.VolumeGr
} }
} }
volumeGrowOption := &topology.VolumeGrowOption{ volumeGrowOption := &topology.VolumeGrowOption{
Collection: r.FormValue("collection"), Collection: r.FormValue("collection"),
ReplicaPlacement: replicaPlacement, ReplicaPlacement: replicaPlacement,
Ttl: ttl, Ttl: ttl,
Prealloacte: preallocate, Prealloacte: preallocate,
DataCenter: r.FormValue("dataCenter"), DataCenter: r.FormValue("dataCenter"),
Rack: r.FormValue("rack"), Rack: r.FormValue("rack"),
DataNode: r.FormValue("dataNode"), DataNode: r.FormValue("dataNode"),
MemoryMapMaxSizeMB: memoryMapMaxSizeMB,
} }
return volumeGrowOption, nil return volumeGrowOption, nil
} }

2
weed/server/volume_grpc_admin.go
View file

@ -35,7 +35,7 @@ func (vs *VolumeServer) AllocateVolume(ctx context.Context, req *volume_server_p
req.Replication, req.Replication,
req.Ttl, req.Ttl,
req.Preallocate, req.Preallocate,
req.InMemory, req.MemoryMapMaxSizeMB,
) )
if err != nil { if err != nil {

2
weed/storage/disk_location.go
View file

@ -60,7 +60,7 @@ func (l *DiskLocation) loadExistingVolume(fileInfo os.FileInfo, needleMapKind Ne
_, found := l.volumes[vid] _, found := l.volumes[vid]
l.RUnlock() l.RUnlock()
if !found { if !found {
if v, e := NewVolume(l.Directory, collection, vid, needleMapKind, nil, nil, 0, false); e == nil { if v, e := NewVolume(l.Directory, collection, vid, needleMapKind, nil, nil, 0, 0); e == nil {
l.Lock() l.Lock()
l.volumes[vid] = v l.volumes[vid] = v
l.Unlock() l.Unlock()

14
weed/storage/needle/volume_memory_map_max_size.go Normal file
View file

@ -0,0 +1,14 @@
package needle
import "strconv"
func ReadMemoryMapMaxSizeMB(MemoryMapMaxSizeMBString string) (uint32, error) {
if MemoryMapMaxSizeMBString == "" {
return 0, nil
}
var MemoryMapMaxSize64 uint64
var err error
MemoryMapMaxSize64, err = strconv.ParseUint(MemoryMapMaxSizeMBString, 10, 32)
MemoryMapMaxSize := uint32(MemoryMapMaxSize64)
return MemoryMapMaxSize, err
}

10
weed/storage/needle/volume_memory_map_max_size_test.go Normal file
View file

@ -0,0 +1,10 @@
package needle
import "testing"
func TestMemoryMapMaxSizeReadWrite(t *testing.T) {
memoryMapSize, _ := ReadMemoryMapMaxSizeMB("5000")
if memoryMapSize != 5000 {
t.Errorf("empty memoryMapSize:%v", memoryMapSize)
}
}

4
weed/storage/store.go
View file

@ -59,7 +59,7 @@ func NewStore(grpcDialOption grpc.DialOption, port int, ip, publicUrl string, di
return return
} }
func (s *Store) AddVolume(volumeId needle.VolumeId, collection string, needleMapKind NeedleMapType, replicaPlacement string, ttlString string, preallocate int64, memoryMapped bool) error { func (s *Store) AddVolume(volumeId needle.VolumeId, collection string, needleMapKind NeedleMapType, replicaPlacement string, ttlString string, preallocate int64, memoryMapped uint32) error {
rt, e := NewReplicaPlacementFromString(replicaPlacement) rt, e := NewReplicaPlacementFromString(replicaPlacement)
if e != nil { if e != nil {
return e return e
@ -101,7 +101,7 @@ func (s *Store) FindFreeLocation() (ret *DiskLocation) {
} }
return ret return ret
} }
func (s *Store) addVolume(vid needle.VolumeId, collection string, needleMapKind NeedleMapType, replicaPlacement *ReplicaPlacement, ttl *needle.TTL, preallocate int64, memoryMapped bool) error { func (s *Store) addVolume(vid needle.VolumeId, collection string, needleMapKind NeedleMapType, replicaPlacement *ReplicaPlacement, ttl *needle.TTL, preallocate int64, memoryMapped uint32) error {
if s.findVolume(vid) != nil { if s.findVolume(vid) != nil {
return fmt.Errorf("Volume Id %d already exists!", vid) return fmt.Errorf("Volume Id %d already exists!", vid)
} }

4
weed/storage/volume.go
View file

@ -25,7 +25,7 @@ type Volume struct {
nm NeedleMapper nm NeedleMapper
needleMapKind NeedleMapType needleMapKind NeedleMapType
readOnly bool readOnly bool
MemoryMapped bool MemoryMapped uint32
SuperBlock SuperBlock
@ -39,7 +39,7 @@ type Volume struct {
isCompacting bool isCompacting bool
} }
func NewVolume(dirname string, collection string, id needle.VolumeId, needleMapKind NeedleMapType, replicaPlacement *ReplicaPlacement, ttl *needle.TTL, preallocate int64, memoryMapped bool) (v *Volume, e error) { func NewVolume(dirname string, collection string, id needle.VolumeId, needleMapKind NeedleMapType, replicaPlacement *ReplicaPlacement, ttl *needle.TTL, preallocate int64, memoryMapped uint32) (v *Volume, e error) {
// if replicaPlacement is nil, the superblock will be loaded from disk // if replicaPlacement is nil, the superblock will be loaded from disk
v = &Volume{dir: dirname, Collection: collection, Id: id, MemoryMapped: memoryMapped} v = &Volume{dir: dirname, Collection: collection, Id: id, MemoryMapped: memoryMapped}
v.SuperBlock = SuperBlock{ReplicaPlacement: replicaPlacement, Ttl: ttl} v.SuperBlock = SuperBlock{ReplicaPlacement: replicaPlacement, Ttl: ttl}

2
weed/storage/volume_create.go
View file

@ -8,7 +8,7 @@ import (
"github.com/joeslay/seaweedfs/weed/glog" "github.com/joeslay/seaweedfs/weed/glog"
) )
func createVolumeFile(fileName string, preallocate int64, useMemoryMap bool) (*os.File, error) { func createVolumeFile(fileName string, preallocate int64, useMemoryMap uint32) (*os.File, error) {
file, e = os.OpenFile(fileName, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) file, e = os.OpenFile(fileName, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
if preallocate > 0 { if preallocate > 0 {
glog.V(0).Infof("Preallocated disk space for %s is not supported", fileName) glog.V(0).Infof("Preallocated disk space for %s is not supported", fileName)

2
weed/storage/volume_create_linux.go
View file

@ -9,7 +9,7 @@ import (
"github.com/joeslay/seaweedfs/weed/glog" "github.com/joeslay/seaweedfs/weed/glog"
) )
func createVolumeFile(fileName string, preallocate int64, useMemoryMap bool) (file *os.File, e error) { func createVolumeFile(fileName string, preallocate int64, useMemoryMap uint32) (file *os.File, e error) {
file, e = os.OpenFile(fileName, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) file, e = os.OpenFile(fileName, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
if preallocate != 0 { if preallocate != 0 {
syscall.Fallocate(int(file.Fd()), 1, 0, preallocate) syscall.Fallocate(int(file.Fd()), 1, 0, preallocate)

6
weed/storage/volume_create_windows.go
View file

@ -12,12 +12,12 @@ import (
"github.com/joeslay/seaweedfs/weed/os_overloads" "github.com/joeslay/seaweedfs/weed/os_overloads"
) )
func createVolumeFile(fileName string, preallocate int64, useMemoryMap bool) (*os.File, error) { func createVolumeFile(fileName string, preallocate int64, useMemoryMap uint32) (*os.File, error) {
mMap, exists := memory_map.FileMemoryMap[fileName] mMap, exists := memory_map.FileMemoryMap[fileName]
if !exists { if !exists {
file, e := os_overloads.OpenFile(fileName, windows.O_RDWR|windows.O_CREAT, 0644, useMemoryMap) file, e := os_overloads.OpenFile(fileName, windows.O_RDWR|windows.O_CREAT, 0644, useMemoryMap > 0)
if useMemoryMap { if useMemoryMap > 0 {
memory_map.FileMemoryMap[fileName] = new(memory_map.MemoryMap) memory_map.FileMemoryMap[fileName] = new(memory_map.MemoryMap)
new_mMap := memory_map.FileMemoryMap[fileName] new_mMap := memory_map.FileMemoryMap[fileName]

6
weed/storage/volume_vacuum.go
View file

@ -23,7 +23,7 @@ func (v *Volume) garbageLevel() float64 {
func (v *Volume) Compact(preallocate int64, compactionBytePerSecond int64) error { func (v *Volume) Compact(preallocate int64, compactionBytePerSecond int64) error {
if !v.MemoryMapped { //it makes no sense to compact in memory if v.MemoryMapped > 0 { //it makes no sense to compact in memory
glog.V(3).Infof("Compacting volume %d ...", v.Id) glog.V(3).Infof("Compacting volume %d ...", v.Id)
//no need to lock for copy on write //no need to lock for copy on write
//v.accessLock.Lock() //v.accessLock.Lock()
@ -46,7 +46,7 @@ func (v *Volume) Compact(preallocate int64, compactionBytePerSecond int64) error
func (v *Volume) Compact2() error { func (v *Volume) Compact2() error {
if !v.MemoryMapped { //it makes no sense to compact in memory if v.MemoryMapped > 0 { //it makes no sense to compact in memory
glog.V(3).Infof("Compact2 volume %d ...", v.Id) glog.V(3).Infof("Compact2 volume %d ...", v.Id)
v.isCompacting = true v.isCompacting = true
@ -63,7 +63,7 @@ func (v *Volume) Compact2() error {
} }
func (v *Volume) CommitCompact() error { func (v *Volume) CommitCompact() error {
if !v.MemoryMapped { //it makes no sense to compact in memory if v.MemoryMapped>0 { //it makes no sense to compact in memory
glog.V(0).Infof("Committing volume %d vacuuming...", v.Id) glog.V(0).Infof("Committing volume %d vacuuming...", v.Id)
v.isCompacting = true v.isCompacting = true

12
weed/topology/allocate_volume.go
View file

@ -18,12 +18,12 @@ func AllocateVolume(dn *DataNode, grpcDialOption grpc.DialOption, vid needle.Vol
return operation.WithVolumeServerClient(dn.Url(), grpcDialOption, func(client volume_server_pb.VolumeServerClient) error { return operation.WithVolumeServerClient(dn.Url(), grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
_, deleteErr := client.AllocateVolume(context.Background(), &volume_server_pb.AllocateVolumeRequest{ _, deleteErr := client.AllocateVolume(context.Background(), &volume_server_pb.AllocateVolumeRequest{
VolumeId: uint32(vid), VolumeId: uint32(vid),
Collection: option.Collection, Collection: option.Collection,
Replication: option.ReplicaPlacement.String(), Replication: option.ReplicaPlacement.String(),
Ttl: option.Ttl.String(), Ttl: option.Ttl.String(),
Preallocate: option.Prealloacte, Preallocate: option.Prealloacte,
InMemory: option.InMemory, MemoryMapMaxSizeMB: option.MemoryMapMaxSizeMB,
}) })
return deleteErr return deleteErr
}) })

16
weed/topology/volume_growth.go
View file

@ -21,14 +21,14 @@ This package is created to resolve these replica placement issues:
*/ */
type VolumeGrowOption struct { type VolumeGrowOption struct {
Collection string Collection string
ReplicaPlacement *storage.ReplicaPlacement ReplicaPlacement *storage.ReplicaPlacement
Ttl *needle.TTL Ttl *needle.TTL
Prealloacte int64 Prealloacte int64
DataCenter string DataCenter string
Rack string Rack string
DataNode string DataNode string
InMemory bool MemoryMapMaxSizeMB uint32
} }
type VolumeGrowth struct { type VolumeGrowth struct {