mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2024-01-19 02:48:24 +00:00
add more for volume placement
fix possible nil volume cases
This commit is contained in:
parent
c72f10511a
commit
2dceb44ae4
29
weed-fs/src/pkg/replication/volume_growth.go
Normal file
29
weed-fs/src/pkg/replication/volume_growth.go
Normal file
|
@ -0,0 +1,29 @@
|
||||||
|
package replication
|
||||||
|
|
||||||
|
import (
|
||||||
|
"pkg/topology"
|
||||||
|
)
|
||||||
|
|
||||||
|
/*
|
||||||
|
This package is created to resolve these replica placement issues:
|
||||||
|
1. growth factor for each replica level, e.g., add 10 volumes for 1 copy, 20 volumes for 2 copies, 30 volumes for 3 copies
|
||||||
|
2. in time of tight storage, how to reduce replica level
|
||||||
|
3. optimizing for hot data on faster disk, cold data on cheaper storage,
|
||||||
|
4. volume allocation for each bucket
|
||||||
|
*/
|
||||||
|
|
||||||
|
type VolumeGrowth struct {
|
||||||
|
copy1factor int
|
||||||
|
copy2factor int
|
||||||
|
copy3factor int
|
||||||
|
copyAll int
|
||||||
|
}
|
||||||
|
|
||||||
|
func (vg *VolumeGrowth) GrowVolumeCopy(copyLevel int, topo topology.Topology) {
|
||||||
|
if copyLevel == 1 {
|
||||||
|
for i := 0; i <vg.copy1factor; i++ {
|
||||||
|
topo.RandomlyCreateOneVolume()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -1,21 +1,13 @@
|
||||||
package storage
|
package storage
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"syscall"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type StorageLimit struct {
|
type StorageLimit struct {
|
||||||
sizeLimit uint64
|
sizeLimit uint64
|
||||||
detectedLimit uint64
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewStorageLimit(desiredLimit uint64) *StorageLimit {
|
func NewStorageLimit(desiredLimit uint64) *StorageLimit {
|
||||||
s := syscall.Statfs_t{}
|
sl := &StorageLimit{sizeLimit: desiredLimit}
|
||||||
errNo := syscall.Statfs(".", &s)
|
|
||||||
detected := uint64(0)
|
|
||||||
if errNo==nil {
|
|
||||||
detected = s.Bavail*uint64(s.Bsize)
|
|
||||||
}
|
|
||||||
sl := &StorageLimit{sizeLimit: desiredLimit, detectedLimit: detected}
|
|
||||||
return sl
|
return sl
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,11 +0,0 @@
|
||||||
package storage
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestReadStorageLimit(t *testing.T) {
|
|
||||||
sl := NewStorageLimit(1000)
|
|
||||||
println("detected:",sl.detectedLimit)
|
|
||||||
}
|
|
||||||
|
|
|
@ -89,11 +89,23 @@ func (s *Store) Close() {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
func (s *Store) Write(i VolumeId, n *Needle) uint32 {
|
func (s *Store) Write(i VolumeId, n *Needle) uint32 {
|
||||||
return s.volumes[i].write(n)
|
v := s.volumes[i]
|
||||||
|
if v!=nil{
|
||||||
|
return v.write(n)
|
||||||
|
}
|
||||||
|
return 0
|
||||||
}
|
}
|
||||||
func (s *Store) Delete(i VolumeId, n *Needle) uint32 {
|
func (s *Store) Delete(i VolumeId, n *Needle) uint32 {
|
||||||
return s.volumes[i].delete(n)
|
v := s.volumes[i]
|
||||||
|
if v!=nil{
|
||||||
|
return v.delete(n)
|
||||||
|
}
|
||||||
|
return 0
|
||||||
}
|
}
|
||||||
func (s *Store) Read(i VolumeId, n *Needle) (int, error) {
|
func (s *Store) Read(i VolumeId, n *Needle) (int, error) {
|
||||||
return s.volumes[i].read(n)
|
v := s.volumes[i]
|
||||||
|
if v!=nil{
|
||||||
|
return v.read(n)
|
||||||
|
}
|
||||||
|
return 0, errors.New("Not Found")
|
||||||
}
|
}
|
||||||
|
|
|
@ -11,4 +11,19 @@ type Node struct {
|
||||||
Ip string
|
Ip string
|
||||||
Port int
|
Port int
|
||||||
PublicUrl string
|
PublicUrl string
|
||||||
|
|
||||||
|
//transient
|
||||||
|
allocation *Allocation
|
||||||
}
|
}
|
||||||
|
type Allocation struct {
|
||||||
|
count int
|
||||||
|
limit int
|
||||||
|
}
|
||||||
|
|
||||||
|
func (n *Node) GetAllocation() *Allocation{
|
||||||
|
if n.allocation == nil {
|
||||||
|
n.allocation = &Allocation{count:len(n.volumes), limit : n.volumeLimit}
|
||||||
|
}
|
||||||
|
return n.allocation
|
||||||
|
}
|
||||||
|
|
||||||
|
|
|
@ -1,9 +1,16 @@
|
||||||
package topology
|
package topology
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"pkg/storage"
|
||||||
)
|
)
|
||||||
|
|
||||||
type Topology struct {
|
type Topology struct {
|
||||||
datacenters map[DataCenterId]*DataCenter
|
datacenters map[DataCenterId]*DataCenter
|
||||||
}
|
}
|
||||||
|
//FIXME
|
||||||
|
func (t *Topology) RandomlyCreateOneVolume() storage.VolumeId{
|
||||||
|
return t.findMaxVolumeId()
|
||||||
|
}
|
||||||
|
func (t *Topology) findMaxVolumeId() storage.VolumeId{
|
||||||
|
return storage.VolumeId(0);
|
||||||
|
}
|
||||||
|
|
Loading…
Reference in a new issue