gitlab-com-seaweedfs/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-01-19 02:48:24 +00:00
Code Issues Projects Releases Wiki Activity

Changed the InMemory bool to a uint32 so that it can be used to alter how much space to reserve

Browse source
This commit is contained in:
Tom Maxwell 2019-09-04 15:27:14 +01:00
parent 6ee65356e3
commit 4a878c0006
20 changed files with 111 additions and 67 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
weed/command/backup.go
View file

@ -112,7 +112,7 @@ func runBackup(cmd *Command, args []string) bool {
return true
}
}
v, err := storage.NewVolume(*s.dir, *s.collection, vid, storage.NeedleMapInMemory, replication, ttl, 0, false)
v, err := storage.NewVolume(*s.dir, *s.collection, vid, storage.NeedleMapInMemory, replication, ttl, 0, 0)
if err != nil {
fmt.Printf("Error creating or reading from volume %d: %v\n", vid, err)
return true
@ -137,7 +137,7 @@ func runBackup(cmd *Command, args []string) bool {
// remove the old data
v.Destroy()
// recreate an empty volume
v, err = storage.NewVolume(*s.dir, *s.collection, vid, storage.NeedleMapInMemory, replication, ttl, 0, false)
v, err = storage.NewVolume(*s.dir, *s.collection, vid, storage.NeedleMapInMemory, replication, ttl, 0, 0)
if err != nil {
fmt.Printf("Error creating or reading from volume %d: %v\n", vid, err)
return true

2
weed/command/compact.go
View file

@ -38,7 +38,7 @@ func runCompact(cmd *Command, args []string) bool {
vid := needle.VolumeId(*compactVolumeId)
v, err := storage.NewVolume(*compactVolumePath, *compactVolumeCollection, vid,
storage.NeedleMapInMemory, nil, nil, preallocate, false)
storage.NeedleMapInMemory, nil, nil, preallocate, 0)
if err != nil {
glog.Fatalf("Load Volume [ERROR] %s\n", err)
}

2
weed/pb/master.proto
View file

@ -139,7 +139,7 @@ message AssignRequest {
string data_center = 5;
string rack = 6;
string data_node = 7;
bool in_memory = 8;
uint32 MemoryMapMaxSizeMB = 8;
}
message AssignResponse {
string fid = 1;

22
weed/pb/master_pb/master.pb.go
View file

@ -651,14 +651,14 @@ func (m *Location) GetPublicUrl() string {
}
type AssignRequest struct {
Count uint64 `protobuf:"varint,1,opt,name=count" json:"count,omitempty"`
Replication string `protobuf:"bytes,2,opt,name=replication" json:"replication,omitempty"`
Collection string `protobuf:"bytes,3,opt,name=collection" json:"collection,omitempty"`
Ttl string `protobuf:"bytes,4,opt,name=ttl" json:"ttl,omitempty"`
DataCenter string `protobuf:"bytes,5,opt,name=data_center,json=dataCenter" json:"data_center,omitempty"`
Rack string `protobuf:"bytes,6,opt,name=rack" json:"rack,omitempty"`
DataNode string `protobuf:"bytes,7,opt,name=data_node,json=dataNode" json:"data_node,omitempty"`
InMemory bool `protobuf:"bytes,4,opt,name=inmemory" json:"inmemory,omitempty"`
Count uint64 `protobuf:"varint,1,opt,name=count" json:"count,omitempty"`
Replication string `protobuf:"bytes,2,opt,name=replication" json:"replication,omitempty"`
Collection string `protobuf:"bytes,3,opt,name=collection" json:"collection,omitempty"`
Ttl string `protobuf:"bytes,4,opt,name=ttl" json:"ttl,omitempty"`
DataCenter string `protobuf:"bytes,5,opt,name=data_center,json=dataCenter" json:"data_center,omitempty"`
Rack string `protobuf:"bytes,6,opt,name=rack" json:"rack,omitempty"`
DataNode string `protobuf:"bytes,7,opt,name=data_node,json=dataNode" json:"data_node,omitempty"`
MemoryMapMaxSizeMB uint32 `protobuf:"varint,8,opt,name=memorymapmaxsizemb" json:"memorymapmaxsizemb,omitempty"`
}
func (m *AssignRequest) Reset() { *m = AssignRequest{} }
@ -715,11 +715,11 @@ func (m *AssignRequest) GetDataNode() string {
return ""
}
func (m *AssignRequest) GetInMemory() bool {
func (m *AssignRequest) GetMemoryMapMaxSizeMB() uint32 {
if m != nil {
return m.InMemory
return m.MemoryMapMaxSizeMB
}
return false
return 0
}
type AssignResponse struct {

2
weed/pb/volume_server.proto
View file

@ -127,7 +127,7 @@ message AllocateVolumeRequest {
int64 preallocate = 3;
string replication = 4;
string ttl = 5;
bool inmemory = 6;
int32 memorymapmaxsizemb = 6;
}
message AllocateVolumeResponse {
}

31
weed/pb/volume_server_pb/volume_server.pb.go
View file

@ -316,12 +316,12 @@ func (*DeleteCollectionResponse) ProtoMessage() {}
func (*DeleteCollectionResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{13} }
type AllocateVolumeRequest struct {
VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"`
Collection string `protobuf:"bytes,2,opt,name=collection" json:"collection,omitempty"`
Preallocate int64 `protobuf:"varint,3,opt,name=preallocate" json:"preallocate,omitempty"`
Replication string `protobuf:"bytes,4,opt,name=replication" json:"replication,omitempty"`
Ttl string `protobuf:"bytes,5,opt,name=ttl" json:"ttl,omitempty"`
InMemory bool `protobuf:"varint,6,opt,name=inmemory" json:"inmemory,omitempty"`
VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"`
Collection string `protobuf:"bytes,2,opt,name=collection" json:"collection,omitempty"`
Preallocate int64 `protobuf:"varint,3,opt,name=preallocate" json:"preallocate,omitempty"`
Replication string `protobuf:"bytes,4,opt,name=replication" json:"replication,omitempty"`
Ttl string `protobuf:"bytes,5,opt,name=ttl" json:"ttl,omitempty"`
MemoryMapMaxSizeMB uint32 `protobuf:"varint,6,opt,name=memorymapmaxsizemb" json:"memorymapmaxsizemb,omitempty"`
}
func (m *AllocateVolumeRequest) Reset() { *m = AllocateVolumeRequest{} }
@ -364,11 +364,11 @@ func (m *AllocateVolumeRequest) GetTtl() string {
return ""
}
func (m *AllocateVolumeRequest) GetInMemory() bool {
func (m *AllocateVolumeRequest) GetMemoryMapMaxSizeMB() uint32 {
if m != nil {
return m.InMemory
return m.MemoryMapMaxSizeMB
}
return false
return 0
}
type AllocateVolumeResponse struct {
@ -1978,6 +1978,19 @@ func _VolumeServer_DeleteCollection_Handler(srv interface{}, ctx context.Context
func _VolumeServer_AllocateVolume_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(AllocateVolumeRequest)
if in.MemoryMapMaxSizeMB > 0 {
test := 6
test += 5
}
if in.MemoryMapMaxSizeMB == 77 {
test := 7
test += 656
}
if in.Ttl != "" {
test := 2345
test += 567
}
if err := dec(in); err != nil {
return nil, err
}

15
weed/server/master_grpc_server_volume.go
View file

@ -62,13 +62,14 @@ func (ms *MasterServer) Assign(ctx context.Context, req *master_pb.AssignRequest
}
option := &topology.VolumeGrowOption{
Collection: req.Collection,
ReplicaPlacement: replicaPlacement,
Ttl: ttl,
Prealloacte: ms.preallocateSize,
DataCenter: req.DataCenter,
Rack: req.Rack,
DataNode: req.DataNode,
Collection: req.Collection,
ReplicaPlacement: replicaPlacement,
Ttl: ttl,
Prealloacte: ms.preallocateSize,
DataCenter: req.DataCenter,
Rack: req.Rack,
DataNode: req.DataNode,
MemoryMapMaxSizeMB: req.MemoryMapMaxSizeMB,
}
if !ms.Topo.HasWritableVolume(option) {

20
weed/server/master_server_handlers_admin.go
View file

@ -148,6 +148,11 @@ func (ms *MasterServer) getVolumeGrowOption(r *http.Request) (*topology.VolumeGr
if err != nil {
return nil, err
}
memoryMapMaxSizeMB, err := needle.ReadMemoryMapMaxSizeMB(r.FormValue("memorymapmaxsizemb"))
if err != nil {
return nil, err
}
preallocate := ms.preallocateSize
if r.FormValue("preallocate") != "" {
preallocate, err = strconv.ParseInt(r.FormValue("preallocate"), 10, 64)
@ -156,13 +161,14 @@ func (ms *MasterServer) getVolumeGrowOption(r *http.Request) (*topology.VolumeGr
}
}
volumeGrowOption := &topology.VolumeGrowOption{
Collection: r.FormValue("collection"),
ReplicaPlacement: replicaPlacement,
Ttl: ttl,
Prealloacte: preallocate,
DataCenter: r.FormValue("dataCenter"),
Rack: r.FormValue("rack"),
DataNode: r.FormValue("dataNode"),
Collection: r.FormValue("collection"),
ReplicaPlacement: replicaPlacement,
Ttl: ttl,
Prealloacte: preallocate,
DataCenter: r.FormValue("dataCenter"),
Rack: r.FormValue("rack"),
DataNode: r.FormValue("dataNode"),
MemoryMapMaxSizeMB: memoryMapMaxSizeMB,
}
return volumeGrowOption, nil
}

2
weed/server/volume_grpc_admin.go
View file

@ -35,7 +35,7 @@ func (vs *VolumeServer) AllocateVolume(ctx context.Context, req *volume_server_p
req.Replication,
req.Ttl,
req.Preallocate,
req.InMemory,
req.MemoryMapMaxSizeMB,
)
if err != nil {

2
weed/storage/disk_location.go
View file

@ -60,7 +60,7 @@ func (l *DiskLocation) loadExistingVolume(fileInfo os.FileInfo, needleMapKind Ne
_, found := l.volumes[vid]
l.RUnlock()
if !found {
if v, e := NewVolume(l.Directory, collection, vid, needleMapKind, nil, nil, 0, false); e == nil {
if v, e := NewVolume(l.Directory, collection, vid, needleMapKind, nil, nil, 0, 0); e == nil {
l.Lock()
l.volumes[vid] = v
l.Unlock()

14
weed/storage/needle/volume_memory_map_max_size.go Normal file
View file

@ -0,0 +1,14 @@
package needle
import "strconv"
func ReadMemoryMapMaxSizeMB(MemoryMapMaxSizeMBString string) (uint32, error) {
if MemoryMapMaxSizeMBString == "" {
return 0, nil
}
var MemoryMapMaxSize64 uint64
var err error
MemoryMapMaxSize64, err = strconv.ParseUint(MemoryMapMaxSizeMBString, 10, 32)
MemoryMapMaxSize := uint32(MemoryMapMaxSize64)
return MemoryMapMaxSize, err
}

10
weed/storage/needle/volume_memory_map_max_size_test.go Normal file
View file

@ -0,0 +1,10 @@
package needle
import "testing"
func TestMemoryMapMaxSizeReadWrite(t *testing.T) {
memoryMapSize, _ := ReadMemoryMapMaxSizeMB("5000")
if memoryMapSize != 5000 {
t.Errorf("empty memoryMapSize:%v", memoryMapSize)
}
}

4
weed/storage/store.go
View file

@ -59,7 +59,7 @@ func NewStore(grpcDialOption grpc.DialOption, port int, ip, publicUrl string, di
return
}
func (s *Store) AddVolume(volumeId needle.VolumeId, collection string, needleMapKind NeedleMapType, replicaPlacement string, ttlString string, preallocate int64, memoryMapped bool) error {
func (s *Store) AddVolume(volumeId needle.VolumeId, collection string, needleMapKind NeedleMapType, replicaPlacement string, ttlString string, preallocate int64, memoryMapped uint32) error {
rt, e := NewReplicaPlacementFromString(replicaPlacement)
if e != nil {
return e
@ -101,7 +101,7 @@ func (s *Store) FindFreeLocation() (ret *DiskLocation) {
}
return ret
}
func (s *Store) addVolume(vid needle.VolumeId, collection string, needleMapKind NeedleMapType, replicaPlacement *ReplicaPlacement, ttl *needle.TTL, preallocate int64, memoryMapped bool) error {
func (s *Store) addVolume(vid needle.VolumeId, collection string, needleMapKind NeedleMapType, replicaPlacement *ReplicaPlacement, ttl *needle.TTL, preallocate int64, memoryMapped uint32) error {
if s.findVolume(vid) != nil {
return fmt.Errorf("Volume Id %d already exists!", vid)
}

4
weed/storage/volume.go
View file

@ -25,7 +25,7 @@ type Volume struct {
nm NeedleMapper
needleMapKind NeedleMapType
readOnly bool
MemoryMapped bool
MemoryMapped uint32
SuperBlock
@ -39,7 +39,7 @@ type Volume struct {
isCompacting bool
}
func NewVolume(dirname string, collection string, id needle.VolumeId, needleMapKind NeedleMapType, replicaPlacement *ReplicaPlacement, ttl *needle.TTL, preallocate int64, memoryMapped bool) (v *Volume, e error) {
func NewVolume(dirname string, collection string, id needle.VolumeId, needleMapKind NeedleMapType, replicaPlacement *ReplicaPlacement, ttl *needle.TTL, preallocate int64, memoryMapped uint32) (v *Volume, e error) {
// if replicaPlacement is nil, the superblock will be loaded from disk
v = &Volume{dir: dirname, Collection: collection, Id: id, MemoryMapped: memoryMapped}
v.SuperBlock = SuperBlock{ReplicaPlacement: replicaPlacement, Ttl: ttl}

2
weed/storage/volume_create.go
View file

@ -8,7 +8,7 @@ import (
"github.com/joeslay/seaweedfs/weed/glog"
)
func createVolumeFile(fileName string, preallocate int64, useMemoryMap bool) (*os.File, error) {
func createVolumeFile(fileName string, preallocate int64, useMemoryMap uint32) (*os.File, error) {
file, e = os.OpenFile(fileName, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
if preallocate > 0 {
glog.V(0).Infof("Preallocated disk space for %s is not supported", fileName)

2
weed/storage/volume_create_linux.go
View file

@ -9,7 +9,7 @@ import (
"github.com/joeslay/seaweedfs/weed/glog"
)
func createVolumeFile(fileName string, preallocate int64, useMemoryMap bool) (file *os.File, e error) {
func createVolumeFile(fileName string, preallocate int64, useMemoryMap uint32) (file *os.File, e error) {
file, e = os.OpenFile(fileName, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
if preallocate != 0 {
syscall.Fallocate(int(file.Fd()), 1, 0, preallocate)

6
weed/storage/volume_create_windows.go
View file

@ -12,12 +12,12 @@ import (
"github.com/joeslay/seaweedfs/weed/os_overloads"
)
func createVolumeFile(fileName string, preallocate int64, useMemoryMap bool) (*os.File, error) {
func createVolumeFile(fileName string, preallocate int64, useMemoryMap uint32) (*os.File, error) {
mMap, exists := memory_map.FileMemoryMap[fileName]
if !exists {
file, e := os_overloads.OpenFile(fileName, windows.O_RDWR|windows.O_CREAT, 0644, useMemoryMap)
if useMemoryMap {
file, e := os_overloads.OpenFile(fileName, windows.O_RDWR|windows.O_CREAT, 0644, useMemoryMap > 0)
if useMemoryMap > 0 {
memory_map.FileMemoryMap[fileName] = new(memory_map.MemoryMap)
new_mMap := memory_map.FileMemoryMap[fileName]

6
weed/storage/volume_vacuum.go
View file

@ -23,7 +23,7 @@ func (v *Volume) garbageLevel() float64 {
func (v *Volume) Compact(preallocate int64, compactionBytePerSecond int64) error {
if !v.MemoryMapped { //it makes no sense to compact in memory
if v.MemoryMapped > 0 { //it makes no sense to compact in memory
glog.V(3).Infof("Compacting volume %d ...", v.Id)
//no need to lock for copy on write
//v.accessLock.Lock()
@ -46,7 +46,7 @@ func (v *Volume) Compact(preallocate int64, compactionBytePerSecond int64) error
func (v *Volume) Compact2() error {
if !v.MemoryMapped { //it makes no sense to compact in memory
if v.MemoryMapped > 0 { //it makes no sense to compact in memory
glog.V(3).Infof("Compact2 volume %d ...", v.Id)
v.isCompacting = true
@ -63,7 +63,7 @@ func (v *Volume) Compact2() error {
}
func (v *Volume) CommitCompact() error {
if !v.MemoryMapped { //it makes no sense to compact in memory
if v.MemoryMapped>0 { //it makes no sense to compact in memory
glog.V(0).Infof("Committing volume %d vacuuming...", v.Id)
v.isCompacting = true

12
weed/topology/allocate_volume.go
View file

@ -18,12 +18,12 @@ func AllocateVolume(dn *DataNode, grpcDialOption grpc.DialOption, vid needle.Vol
return operation.WithVolumeServerClient(dn.Url(), grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
_, deleteErr := client.AllocateVolume(context.Background(), &volume_server_pb.AllocateVolumeRequest{
VolumeId: uint32(vid),
Collection: option.Collection,
Replication: option.ReplicaPlacement.String(),
Ttl: option.Ttl.String(),
Preallocate: option.Prealloacte,
InMemory: option.InMemory,
VolumeId: uint32(vid),
Collection: option.Collection,
Replication: option.ReplicaPlacement.String(),
Ttl: option.Ttl.String(),
Preallocate: option.Prealloacte,
MemoryMapMaxSizeMB: option.MemoryMapMaxSizeMB,
})
return deleteErr
})

16
weed/topology/volume_growth.go
View file

@ -21,14 +21,14 @@ This package is created to resolve these replica placement issues:
*/
type VolumeGrowOption struct {
Collection string
ReplicaPlacement *storage.ReplicaPlacement
Ttl *needle.TTL
Prealloacte int64
DataCenter string
Rack string
DataNode string
InMemory bool
Collection string
ReplicaPlacement *storage.ReplicaPlacement
Ttl *needle.TTL
Prealloacte int64
DataCenter string
Rack string
DataNode string
MemoryMapMaxSizeMB uint32
}
type VolumeGrowth struct {