gitlab-com-seaweedfs/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-01-19 02:48:24 +00:00
Code Issues Projects Releases Wiki Activity

periodically compact volumes

Browse source
This commit is contained in:
Chris Lu 2012-11-23 17:03:27 -08:00
parent 5e020196f6
commit 94dbe60233
9 changed files with 201 additions and 71 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
weed-fs/src/cmd/weed/volume.go
View file

@ -57,6 +57,15 @@ func assignVolumeHandler(w http.ResponseWriter, r *http.Request) {
}
debug("assign volume =", r.FormValue("volume"), ", replicationType =", r.FormValue("replicationType"), ", error =", err)
}
func vacuumVolumeCheckHandler(w http.ResponseWriter, r *http.Request) {
err, ret := store.CheckCompactVolume(r.FormValue("volume"), r.FormValue("garbageThreshold"))
if err == nil {
writeJson(w, r, map[string]interface{}{"error": "", "result": ret})
} else {
writeJson(w, r, map[string]interface{}{"error": err.Error(), "result": false})
}
debug("checked compacting volume =", r.FormValue("volume"), "garbageThreshold =", r.FormValue("garbageThreshold"), "vacuum =", ret)
}
func vacuumVolumeCompactHandler(w http.ResponseWriter, r *http.Request) {
err := store.CompactVolume(r.FormValue("volume"))
if err == nil {
@ -67,9 +76,9 @@ func vacuumVolumeCompactHandler(w http.ResponseWriter, r *http.Request) {
debug("compacted volume =", r.FormValue("volume"), ", error =", err)
}
func vacuumVolumeCommitHandler(w http.ResponseWriter, r *http.Request) {
count, err := store.CommitCompactVolume(r.FormValue("volume"))
err := store.CommitCompactVolume(r.FormValue("volume"))
if err == nil {
writeJson(w, r, map[string]interface{}{"error": "", "size": count})
writeJson(w, r, map[string]interface{}{"error": ""})
} else {
writeJson(w, r, map[string]string{"error": err.Error()})
}
@ -302,6 +311,7 @@ func runVolume(cmd *Command, args []string) bool {
http.HandleFunc("/", storeHandler)
http.HandleFunc("/status", statusHandler)
http.HandleFunc("/admin/assign_volume", assignVolumeHandler)
http.HandleFunc("/admin/vacuum_volume_check", vacuumVolumeCheckHandler)
http.HandleFunc("/admin/vacuum_volume_compact", vacuumVolumeCompactHandler)
http.HandleFunc("/admin/vacuum_volume_commit", vacuumVolumeCommitHandler)

53
weed-fs/src/pkg/storage/compact_map.go
View file

@ -29,17 +29,25 @@ func NewCompactSection(start Key) CompactSection {
start: start,
}
}
func (cs *CompactSection) Set(key Key, offset uint32, size uint32) {
//return old entry size
func (cs *CompactSection) Set(key Key, offset uint32, size uint32) uint32 {
ret := uint32(0)
if key > cs.end {
cs.end = key
}
if i := cs.binarySearchValues(key); i >= 0 {
ret = cs.values[i].Size
//println("key", key, "old size", ret)
cs.values[i].Offset, cs.values[i].Size = offset, size
} else {
needOverflow := cs.counter >= batch
needOverflow = needOverflow || cs.counter > 0 && cs.values[cs.counter-1].Key > key
needOverflow := cs.counter >= batch
needOverflow = needOverflow || cs.counter > 0 && cs.values[cs.counter-1].Key > key
if needOverflow {
//println("start", cs.start, "counter", cs.counter, "key", key)
if oldValue := cs.overflow[key]; oldValue != nil {
ret = oldValue.Size
}
cs.overflow[key] = &NeedleValue{Key: key, Offset: offset, Size: size}
} else {
p := &cs.values[cs.counter]
@ -48,12 +56,23 @@ func (cs *CompactSection) Set(key Key, offset uint32, size uint32) {
cs.counter++
}
}
return ret
}
func (cs *CompactSection) Delete(key Key) {
//return old entry size
func (cs *CompactSection) Delete(key Key) uint32 {
ret := uint32(0)
if i := cs.binarySearchValues(key); i >= 0 {
cs.values[i].Size = 0
if cs.values[i].Size > 0 {
ret = cs.values[i].Size
cs.values[i].Size = 0
}
}
delete(cs.overflow, key)
if v := cs.overflow[key]; v != nil {
delete(cs.overflow, key)
ret = v.Size
}
return ret
}
func (cs *CompactSection) Get(key Key) (*NeedleValue, bool) {
if v, ok := cs.overflow[key]; ok {
@ -94,21 +113,21 @@ func NewCompactMap() CompactMap {
return CompactMap{}
}
func (cm *CompactMap) Set(key Key, offset uint32, size uint32) {
func (cm *CompactMap) Set(key Key, offset uint32, size uint32) uint32 {
x := cm.binarySearchCompactSection(key)
if x < 0 {
//println(x, "creating", len(cm.list), "section1, starting", key)
cm.list = append(cm.list, NewCompactSection(key))
x = len(cm.list) - 1
}
cm.list[x].Set(key, offset, size)
return cm.list[x].Set(key, offset, size)
}
func (cm *CompactMap) Delete(key Key) {
func (cm *CompactMap) Delete(key Key) uint32 {
x := cm.binarySearchCompactSection(key)
if x < 0 {
return
return uint32(0)
}
cm.list[x].Delete(key)
return cm.list[x].Delete(key)
}
func (cm *CompactMap) Get(key Key) (*NeedleValue, bool) {
x := cm.binarySearchCompactSection(key)
@ -123,7 +142,7 @@ func (cm *CompactMap) binarySearchCompactSection(key Key) int {
return -5
}
if cm.list[h].start <= key {
if cm.list[h].counter < batch || key <= cm.list[h].end{
if cm.list[h].counter < batch || key <= cm.list[h].end {
return h
} else {
return -4
@ -150,9 +169,9 @@ func (cm *CompactMap) Peek() {
println("[", v.Key, v.Offset, v.Size, "]")
}
}
for k, v := range cm.list[0].overflow {
if k < 100 {
println("o[", v.Key, v.Offset, v.Size, "]")
}
}
for k, v := range cm.list[0].overflow {
if k < 100 {
println("o[", v.Key, v.Offset, v.Size, "]")
}
}
}

27
weed-fs/src/pkg/storage/needle_map.go
View file

@ -11,9 +11,11 @@ type NeedleMap struct {
m CompactMap
//transient
bytes []byte
deletionCounter int
fileCounter int
bytes []byte
deletionCounter int
fileCounter int
deletionByteCounter uint32
}
func NewNeedleMap(file *os.File) *NeedleMap {
@ -43,13 +45,18 @@ func LoadNeedleMap(file *os.File) *NeedleMap {
offset := util.BytesToUint32(bytes[i+8 : i+12])
size := util.BytesToUint32(bytes[i+12 : i+16])
if offset > 0 {
nm.m.Set(Key(key), offset, size)
//log.Println("reading key", key, "offset", offset, "size", size)
oldSize := nm.m.Set(Key(key), offset, size)
//log.Println("reading key", key, "offset", offset, "size", size, "oldSize", oldSize)
nm.fileCounter++
if oldSize > 0 {
nm.deletionCounter++
nm.deletionByteCounter = nm.deletionByteCounter + oldSize
}
} else {
nm.m.Delete(Key(key))
//log.Println("removing key", key)
//log.Println("removing key", key)
nm.deletionCounter++
nm.deletionByteCounter = nm.deletionByteCounter + size
}
}
@ -59,11 +66,15 @@ func LoadNeedleMap(file *os.File) *NeedleMap {
}
func (nm *NeedleMap) Put(key uint64, offset uint32, size uint32) (int, error) {
nm.m.Set(Key(key), offset, size)
oldSize := nm.m.Set(Key(key), offset, size)
util.Uint64toBytes(nm.bytes[0:8], key)
util.Uint32toBytes(nm.bytes[8:12], offset)
util.Uint32toBytes(nm.bytes[12:16], size)
nm.fileCounter++
if oldSize > 0 {
nm.deletionCounter++
nm.deletionByteCounter = nm.deletionByteCounter + oldSize
}
return nm.indexFile.Write(nm.bytes)
}
func (nm *NeedleMap) Get(key uint64) (element *NeedleValue, ok bool) {
@ -71,7 +82,7 @@ func (nm *NeedleMap) Get(key uint64) (element *NeedleValue, ok bool) {
return
}
func (nm *NeedleMap) Delete(key uint64) {
nm.m.Delete(Key(key))
nm.deletionByteCounter = nm.deletionByteCounter + nm.m.Delete(Key(key))
util.Uint64toBytes(nm.bytes[0:8], key)
util.Uint32toBytes(nm.bytes[8:12], 0)
util.Uint32toBytes(nm.bytes[12:16], 0)

19
weed-fs/src/pkg/storage/store.go
View file

@ -69,6 +69,17 @@ func (s *Store) addVolume(vid VolumeId, replicationType ReplicationType) error {
return nil
}
func (s *Store) CheckCompactVolume(volumeIdString string, garbageThresholdString string) (error, bool) {
vid, err := NewVolumeId(volumeIdString)
if err != nil {
return errors.New("Volume Id " + volumeIdString + " is not a valid unsigned integer!"), false
}
garbageThreshold, e := strconv.ParseFloat(garbageThresholdString, 32)
if e != nil {
return errors.New("garbageThreshold " + garbageThresholdString + " is not a valid float number!"), false
}
return nil, garbageThreshold < s.volumes[vid].garbageLevel()
}
func (s *Store) CompactVolume(volumeIdString string) error {
vid, err := NewVolumeId(volumeIdString)
if err != nil {
@ -76,10 +87,10 @@ func (s *Store) CompactVolume(volumeIdString string) error {
}
return s.volumes[vid].compact()
}
func (s *Store) CommitCompactVolume(volumeIdString string) (int,error) {
func (s *Store) CommitCompactVolume(volumeIdString string) (error) {
vid, err := NewVolumeId(volumeIdString)
if err != nil {
return 0, errors.New("Volume Id " + volumeIdString + " is not a valid unsigned integer!")
return errors.New("Volume Id " + volumeIdString + " is not a valid unsigned integer!")
}
return s.volumes[vid].commitCompact()
}
@ -104,7 +115,7 @@ func (s *Store) Status() []*VolumeInfo {
var stats []*VolumeInfo
for k, v := range s.volumes {
s := new(VolumeInfo)
s.Id, s.Size, s.RepType, s.FileCount, s.DeleteCount = VolumeId(k), v.Size(), v.replicaType, v.nm.fileCounter, v.nm.deletionCounter
s.Id, s.Size, s.RepType, s.FileCount, s.DeleteCount, s.DeletedByteCount = VolumeId(k), v.Size(), v.replicaType, v.nm.fileCounter, v.nm.deletionCounter, v.nm.deletionByteCounter
stats = append(stats, s)
}
return stats
@ -113,7 +124,7 @@ func (s *Store) Join(mserver string) error {
stats := new([]*VolumeInfo)
for k, v := range s.volumes {
s := new(VolumeInfo)
s.Id, s.Size, s.RepType, s.FileCount, s.DeleteCount = VolumeId(k), v.Size(), v.replicaType, v.nm.fileCounter, v.nm.deletionCounter
s.Id, s.Size, s.RepType, s.FileCount, s.DeleteCount, s.DeletedByteCount = VolumeId(k), v.Size(), v.replicaType, v.nm.fileCounter, v.nm.deletionCounter, v.nm.deletionByteCounter
*stats = append(*stats, s)
}
bytes, _ := json.Marshal(stats)

21
weed-fs/src/pkg/storage/volume.go
View file

@ -55,13 +55,18 @@ func (v *Volume) load() error {
return nil
}
func (v *Volume) Size() int64 {
v.accessLock.Lock()
defer v.accessLock.Unlock()
stat, e := v.dataFile.Stat()
if e == nil {
return stat.Size()
}
fmt.Printf("Failed to read file size %s %s\n", v.dataFile.Name(), e.Error())
return -1
}
func (v *Volume) Close() {
v.accessLock.Lock()
defer v.accessLock.Unlock()
v.nm.Close()
v.dataFile.Close()
}
@ -115,6 +120,7 @@ func (v *Volume) delete(n *Needle) uint32 {
}
return 0
}
func (v *Volume) read(n *Needle) (int, error) {
v.accessLock.Lock()
defer v.accessLock.Unlock()
@ -126,6 +132,10 @@ func (v *Volume) read(n *Needle) (int, error) {
return -1, errors.New("Not Found")
}
func (v *Volume) garbageLevel() float64 {
return float64(v.nm.deletionByteCounter)/float64(v.Size())
}
func (v *Volume) compact() error {
v.accessLock.Lock()
defer v.accessLock.Unlock()
@ -133,21 +143,21 @@ func (v *Volume) compact() error {
filePath := path.Join(v.dir, v.Id.String())
return v.copyDataAndGenerateIndexFile(filePath+".dat", filePath+".cpd", filePath+".cpx")
}
func (v *Volume) commitCompact() (int, error) {
func (v *Volume) commitCompact() (error) {
v.accessLock.Lock()
defer v.accessLock.Unlock()
v.dataFile.Close()
var e error
if e = os.Rename(path.Join(v.dir, v.Id.String()+".cpd"), path.Join(v.dir, v.Id.String()+".dat")); e != nil {
return 0, e
return e
}
if e = os.Rename(path.Join(v.dir, v.Id.String()+".cpx"), path.Join(v.dir, v.Id.String()+".idx")); e != nil {
return 0, e
return e
}
if e = v.load(); e != nil {
return 0, e
return e
}
return 0, nil
return nil
}
func (v *Volume) copyDataAndGenerateIndexFile(srcName, dstName, idxName string) (err error) {
@ -183,7 +193,6 @@ func (v *Volume) copyDataAndGenerateIndexFile(srcName, dstName, idxName string)
nv, ok := v.nm.Get(n.Id)
//log.Println("file size is", n.Size, "rest", rest)
if !ok || nv.Offset*8 != old_offset {
log.Println("expected offset should be", nv.Offset*8, "skipping", (rest - 16), "key", n.Id, "volume offset", old_offset, "data_size", n.Size, "rest", rest)
src.Seek(int64(rest), 1)
} else {
if nv.Size > 0 {

1
weed-fs/src/pkg/storage/volume_info.go
View file

@ -9,4 +9,5 @@ type VolumeInfo struct {
RepType ReplicationType
FileCount int
DeleteCount int
DeletedByteCount uint32
}

1
weed-fs/src/pkg/topology/node.go
View file

@ -164,6 +164,7 @@ func (n *NodeImpl) CollectDeadNodeAndFullVolumes(freshThreshHold int64, volumeSi
}
for _, v := range dn.volumes {
if uint64(v.Size) >= volumeSizeLimit {
//fmt.Println("volume",v.Id,"size",v.Size,">",volumeSizeLimit)
n.GetTopology().chanFullVolumes <- &v
}
}

130
weed-fs/src/pkg/topology/topology_compact.go
View file

@ -3,45 +3,90 @@ package topology
import (
"encoding/json"
"errors"
"fmt"
"fmt"
"net/url"
"pkg/storage"
"pkg/util"
"time"
)
func batchVacuumVolumeCheck(vl *VolumeLayout, vid storage.VolumeId, locationlist *VolumeLocationList) bool {
ch := make(chan bool, locationlist.Length())
for index, dn := range locationlist.list {
go func(index int, url string, vid storage.VolumeId) {
//fmt.Println(index, "Check vacuuming", vid, "on", dn.Url())
if e, ret := vacuumVolume_Check(url, vid); e != nil {
//fmt.Println(index, "Error when checking vacuuming", vid, "on", url, e)
ch <- false
} else {
//fmt.Println(index, "Checked vacuuming", vid, "on", url)
ch <- ret
}
}(index, dn.Url(), vid)
}
isCheckSuccess := true
for _ = range locationlist.list {
select {
case canVacuum := <-ch:
isCheckSuccess = isCheckSuccess && canVacuum
case <-time.After(30 * time.Minute):
isCheckSuccess = false
break
}
}
return isCheckSuccess
}
func batchVacuumVolumeCompact(vl *VolumeLayout, vid storage.VolumeId, locationlist *VolumeLocationList) bool {
vl.removeFromWritable(vid)
ch := make(chan bool, locationlist.Length())
for index, dn := range locationlist.list {
go func(index int, url string, vid storage.VolumeId) {
fmt.Println(index, "Start vacuuming", vid, "on", dn.Url())
if e := vacuumVolume_Compact(url, vid); e != nil {
fmt.Println(index, "Error when vacuuming", vid, "on", url, e)
ch <- false
} else {
fmt.Println(index, "Complete vacuuming", vid, "on", url)
ch <- true
}
}(index, dn.Url(), vid)
}
isVacuumSuccess := true
for _ = range locationlist.list {
select {
case _ = <-ch:
case <-time.After(30 * time.Minute):
isVacuumSuccess = false
break
}
}
return isVacuumSuccess
}
func batchVacuumVolumeCommit(vl *VolumeLayout, vid storage.VolumeId, locationlist *VolumeLocationList) bool {
isCommitSuccess := true
for _, dn := range locationlist.list {
fmt.Println("Start Commiting vacuum", vid, "on", dn.Url())
if e := vacuumVolume_Commit(dn.Url(), vid); e != nil {
fmt.Println("Error when committing vacuum", vid, "on", dn.Url(), e)
isCommitSuccess = false
} else {
fmt.Println("Complete Commiting vacuum", vid, "on", dn.Url())
}
}
if isCommitSuccess {
vl.setVolumeWritable(vid)
}
return isCommitSuccess
}
func (t *Topology) Vacuum() int {
total_counter := 0
for _, vl := range t.replicaType2VolumeLayout {
if vl != nil {
for vid, locationlist := range vl.vid2location {
each_volume_counter := 0
vl.removeFromWritable(vid)
ch := make(chan int, locationlist.Length())
for _, dn := range locationlist.list {
go func(url string, vid storage.VolumeId) {
vacuumVolume_Compact(url, vid)
}(dn.Url(), vid)
}
for _ = range locationlist.list {
select {
case count := <-ch:
each_volume_counter += count
case <-time.After(30 * time.Minute):
each_volume_counter = 0
break
if batchVacuumVolumeCheck(vl, vid, locationlist) {
if batchVacuumVolumeCompact(vl, vid, locationlist) {
batchVacuumVolumeCommit(vl, vid, locationlist)
}
}
if each_volume_counter > 0 {
for _, dn := range locationlist.list {
if e := vacuumVolume_Commit(dn.Url(), vid); e != nil {
fmt.Println("Error when committing on", dn.Url(), e)
panic(e)
}
}
vl.setVolumeWritable(vid)
total_counter += each_volume_counter
}
}
}
}
@ -49,25 +94,42 @@ func (t *Topology) Vacuum() int {
}
type VacuumVolumeResult struct {
Bytes int
Error string
Result bool
Error string
}
func vacuumVolume_Compact(urlLocation string, vid storage.VolumeId) (error, int) {
func vacuumVolume_Check(urlLocation string, vid storage.VolumeId) (error, bool) {
values := make(url.Values)
values.Add("volume", vid.String())
values.Add("garbageThreshold", "0.3")
jsonBlob, err := util.Post("http://"+urlLocation+"/admin/vacuum_volume_check", values)
if err != nil {
return err, false
}
var ret VacuumVolumeResult
if err := json.Unmarshal(jsonBlob, &ret); err != nil {
return err, false
}
if ret.Error != "" {
return errors.New(ret.Error), false
}
return nil, ret.Result
}
func vacuumVolume_Compact(urlLocation string, vid storage.VolumeId) error {
values := make(url.Values)
values.Add("volume", vid.String())
jsonBlob, err := util.Post("http://"+urlLocation+"/admin/vacuum_volume_compact", values)
if err != nil {
return err, 0
return err
}
var ret VacuumVolumeResult
if err := json.Unmarshal(jsonBlob, &ret); err != nil {
return err, 0
return err
}
if ret.Error != "" {
return errors.New(ret.Error), 0
return errors.New(ret.Error)
}
return nil, ret.Bytes
return nil
}
func vacuumVolume_Commit(urlLocation string, vid storage.VolumeId) error {
values := make(url.Values)

6
weed-fs/src/pkg/topology/topology_event_handling.go
View file

@ -15,6 +15,12 @@ func (t *Topology) StartRefreshWritableVolumes() {
time.Sleep(time.Duration(float32(t.pulse*1e3)*(1+rand.Float32())) * time.Millisecond)
}
}()
go func() {
c := time.Tick(15 * time.Minute)
for _ = range c {
t.Vacuum()
}
}()
go func() {
for {
select {