From 59f6a13609e63d23ffd4410edce38223a5705232 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 26 Mar 2014 13:22:27 -0700 Subject: [PATCH] adding lots of different stats --- go/stats/disk.go | 11 ++--- go/stats/disk_windows.go | 9 ++-- go/stats/memory.go | 38 ++++++++++++++++ go/stats/memory_windows.go | 29 ++++++++++++ go/storage/store.go | 44 ++++++++++--------- go/weed/weed_server/common.go | 8 +++- go/weed/weed_server/volume_server.go | 1 + go/weed/weed_server/volume_server_handlers.go | 14 ++++++ 8 files changed, 123 insertions(+), 31 deletions(-) create mode 100644 go/stats/memory.go create mode 100644 go/stats/memory_windows.go diff --git a/go/stats/disk.go b/go/stats/disk.go index acbbd51b9..d0ffcd679 100644 --- a/go/stats/disk.go +++ b/go/stats/disk.go @@ -7,13 +7,14 @@ import ( ) type DiskStatus struct { - All uint64 `json:"all"` - Used uint64 `json:"used"` - Free uint64 `json:"free"` + Dir string + All uint64 + Used uint64 + Free uint64 } -func DiskUsage(path string) (disk *DiskStatus) { - disk = &DiskStatus{} +func NewDiskStatus(path string) (disk *DiskStatus) { + disk = &DiskStatus{Dir: path} fs := syscall.Statfs_t{} err := syscall.Statfs(path, &fs) if err != nil { diff --git a/go/stats/disk_windows.go b/go/stats/disk_windows.go index 7db7668d6..e3fd7a2ff 100644 --- a/go/stats/disk_windows.go +++ b/go/stats/disk_windows.go @@ -7,11 +7,12 @@ import ( ) type DiskStatus struct { - All uint64 `json:"all"` - Used uint64 `json:"used"` - Free uint64 `json:"free"` + Dir string + All uint64 + Used uint64 + Free uint64 } -func DiskUsage(path string) (disk *DiskStatus) { +func NewDiskStatus(path string) (disk *DiskStatus) { return } diff --git a/go/stats/memory.go b/go/stats/memory.go new file mode 100644 index 000000000..b899e1e4e --- /dev/null +++ b/go/stats/memory.go @@ -0,0 +1,38 @@ +// +build !windows + +package stats + +import ( + "runtime" + "syscall" +) + +type MemStatus struct { + Goroutines int + All uint64 + Used uint64 + Free uint64 + Self uint64 + Heap uint64 + Stack uint64 +} + +func MemStat() MemStatus { + mem := MemStatus{} + mem.Goroutines = runtime.NumGoroutine() + memStat := new(runtime.MemStats) + runtime.ReadMemStats(memStat) + mem.Self = memStat.Alloc + mem.Heap = memStat.HeapAlloc + mem.Stack = memStat.StackInuse + + //system memory usage + sysInfo := new(syscall.Sysinfo_t) + err := syscall.Sysinfo(sysInfo) + if err == nil { + mem.All = sysInfo.Totalram //* uint64(syscall.Getpagesize()) + mem.Free = sysInfo.Freeram //* uint64(syscall.Getpagesize()) + mem.Used = mem.All - mem.Free + } + return mem +} diff --git a/go/stats/memory_windows.go b/go/stats/memory_windows.go new file mode 100644 index 000000000..bb1f9cbbf --- /dev/null +++ b/go/stats/memory_windows.go @@ -0,0 +1,29 @@ +// +build windows + +package stats + +import ( + "runtime" +) + +type MemStatus struct { + Goroutines uint32 + All uint32 + Used uint32 + Free uint32 + Self uint64 + Heap uint64 + Stack uint64 +} + +func MemStat() MemStatus { + memStat := new(runtime.MemStats) + mem.Goroutines = runtime.NumGoroutine() + runtime.ReadMemStats(memStat) + mem := MemStatus{} + mem.Self = memStat.Alloc + mem.Heap = memStat.HeapAlloc + mem.Stack = memStat.StackInuse + + return mem +} diff --git a/go/storage/store.go b/go/storage/store.go index 2e1db77c4..3e9597e83 100644 --- a/go/storage/store.go +++ b/go/storage/store.go @@ -15,10 +15,14 @@ import ( ) type DiskLocation struct { - directory string - maxVolumeCount int + Directory string + MaxVolumeCount int volumes map[VolumeId]*Volume } + +func (mn *DiskLocation) reset() { +} + type MasterNodes struct { nodes []string lastNode int @@ -57,7 +61,7 @@ type Store struct { Port int Ip string PublicUrl string - locations []*DiskLocation + Locations []*DiskLocation dataCenter string //optional informaton, overwriting master setting if exists rack string //optional information, overwriting master setting if exists connected bool @@ -67,12 +71,12 @@ type Store struct { func NewStore(port int, ip, publicUrl string, dirnames []string, maxVolumeCounts []int) (s *Store) { s = &Store{Port: port, Ip: ip, PublicUrl: publicUrl} - s.locations = make([]*DiskLocation, 0) + s.Locations = make([]*DiskLocation, 0) for i := 0; i < len(dirnames); i++ { - location := &DiskLocation{directory: dirnames[i], maxVolumeCount: maxVolumeCounts[i]} + location := &DiskLocation{Directory: dirnames[i], MaxVolumeCount: maxVolumeCounts[i]} location.volumes = make(map[VolumeId]*Volume) location.loadExistingVolumes() - s.locations = append(s.locations, location) + s.Locations = append(s.Locations, location) } return } @@ -109,7 +113,7 @@ func (s *Store) AddVolume(volumeListString string, collection string, replicaPla return e } func (s *Store) DeleteCollection(collection string) (e error) { - for _, location := range s.locations { + for _, location := range s.Locations { for k, v := range location.volumes { if v.Collection == collection { e = v.Destroy() @@ -123,7 +127,7 @@ func (s *Store) DeleteCollection(collection string) (e error) { return } func (s *Store) findVolume(vid VolumeId) *Volume { - for _, location := range s.locations { + for _, location := range s.Locations { if v, found := location.volumes[vid]; found { return v } @@ -132,8 +136,8 @@ func (s *Store) findVolume(vid VolumeId) *Volume { } func (s *Store) findFreeLocation() (ret *DiskLocation) { max := 0 - for _, location := range s.locations { - currentFreeCount := location.maxVolumeCount - len(location.volumes) + for _, location := range s.Locations { + currentFreeCount := location.MaxVolumeCount - len(location.volumes) if currentFreeCount > max { max = currentFreeCount ret = location @@ -146,8 +150,8 @@ func (s *Store) addVolume(vid VolumeId, collection string, replicaPlacement *Rep return fmt.Errorf("Volume Id %s already exists!", vid) } if location := s.findFreeLocation(); location != nil { - glog.V(0).Infoln("In dir", location.directory, "adds volume =", vid, ", collection =", collection, ", replicaPlacement =", replicaPlacement) - if volume, err := NewVolume(location.directory, collection, vid, replicaPlacement); err == nil { + glog.V(0).Infoln("In dir", location.Directory, "adds volume =", vid, ", collection =", collection, ", replicaPlacement =", replicaPlacement) + if volume, err := NewVolume(location.Directory, collection, vid, replicaPlacement); err == nil { location.volumes[vid] = volume return nil } else { @@ -206,7 +210,7 @@ func (s *Store) FreezeVolume(volumeIdString string) error { return fmt.Errorf("volume id %s is not found during freeze!", vid) } func (l *DiskLocation) loadExistingVolumes() { - if dirs, err := ioutil.ReadDir(l.directory); err == nil { + if dirs, err := ioutil.ReadDir(l.Directory); err == nil { for _, dir := range dirs { name := dir.Name() if !dir.IsDir() && strings.HasSuffix(name, ".dat") { @@ -218,20 +222,20 @@ func (l *DiskLocation) loadExistingVolumes() { } if vid, err := NewVolumeId(base); err == nil { if l.volumes[vid] == nil { - if v, e := NewVolume(l.directory, collection, vid, nil); e == nil { + if v, e := NewVolume(l.Directory, collection, vid, nil); e == nil { l.volumes[vid] = v - glog.V(0).Infoln("data file", l.directory+"/"+name, "replicaPlacement =", v.ReplicaPlacement, "version =", v.Version(), "size =", v.Size()) + glog.V(0).Infoln("data file", l.Directory+"/"+name, "replicaPlacement =", v.ReplicaPlacement, "version =", v.Version(), "size =", v.Size()) } } } } } } - glog.V(0).Infoln("Store started on dir:", l.directory, "with", len(l.volumes), "volumes", "max", l.maxVolumeCount) + glog.V(0).Infoln("Store started on dir:", l.Directory, "with", len(l.volumes), "volumes", "max", l.MaxVolumeCount) } func (s *Store) Status() []*VolumeInfo { var stats []*VolumeInfo - for _, location := range s.locations { + for _, location := range s.Locations { for k, v := range location.volumes { s := &VolumeInfo{Id: VolumeId(k), Size: v.ContentSize(), Collection: v.Collection, @@ -268,8 +272,8 @@ func (s *Store) Join() error { } stats := new([]*VolumeInfo) maxVolumeCount := 0 - for _, location := range s.locations { - maxVolumeCount = maxVolumeCount + location.maxVolumeCount + for _, location := range s.Locations { + maxVolumeCount = maxVolumeCount + location.MaxVolumeCount for k, v := range location.volumes { s := &VolumeInfo{Id: VolumeId(k), Size: uint64(v.Size()), Collection: v.Collection, @@ -308,7 +312,7 @@ func (s *Store) Join() error { return nil } func (s *Store) Close() { - for _, location := range s.locations { + for _, location := range s.Locations { for _, v := range location.volumes { v.Close() } diff --git a/go/weed/weed_server/common.go b/go/weed/weed_server/common.go index cb7c001f3..1b11a5574 100644 --- a/go/weed/weed_server/common.go +++ b/go/weed/weed_server/common.go @@ -165,12 +165,16 @@ func parseURLPath(path string) (vid, fid, filename, ext string, isVolumeIdOnly b func statsCounterHandler(w http.ResponseWriter, r *http.Request) { m := make(map[string]interface{}) m["Version"] = util.VERSION - m["Statistics"] = serverStats + m["Counters"] = serverStats writeJsonQuiet(w, r, m) } + +type MemoryStatistics struct { +} + func statsMemoryHandler(w http.ResponseWriter, r *http.Request) { m := make(map[string]interface{}) m["Version"] = util.VERSION - m["Statistics"] = serverStats + m["Memory"] = stats.MemStat() writeJsonQuiet(w, r, m) } diff --git a/go/weed/weed_server/volume_server.go b/go/weed/weed_server/volume_server.go index 639a08c19..921f8826f 100644 --- a/go/weed/weed_server/volume_server.go +++ b/go/weed/weed_server/volume_server.go @@ -40,6 +40,7 @@ func NewVolumeServer(r *http.ServeMux, ip string, port int, publicUrl string, fo r.HandleFunc("/admin/delete_collection", secure(vs.whiteList, vs.deleteCollectionHandler)) r.HandleFunc("/stats/counter", secure(vs.whiteList, statsCounterHandler)) r.HandleFunc("/stats/memory", secure(vs.whiteList, statsMemoryHandler)) + r.HandleFunc("/stats/disk", secure(vs.whiteList, vs.statsDiskHandler)) r.HandleFunc("/", vs.storeHandler) go func() { diff --git a/go/weed/weed_server/volume_server_handlers.go b/go/weed/weed_server/volume_server_handlers.go index 96acb4901..aaa441004 100644 --- a/go/weed/weed_server/volume_server_handlers.go +++ b/go/weed/weed_server/volume_server_handlers.go @@ -9,6 +9,7 @@ import ( "code.google.com/p/weed-fs/go/util" "mime" "net/http" + "path/filepath" "strconv" "strings" "time" @@ -259,3 +260,16 @@ func (vs *VolumeServer) DeleteHandler(w http.ResponseWriter, r *http.Request) { m["size"] = uint32(count) writeJsonQuiet(w, r, m) } + +func (vs *VolumeServer) statsDiskHandler(w http.ResponseWriter, r *http.Request) { + m := make(map[string]interface{}) + m["Version"] = util.VERSION + ds := make([]*stats.DiskStatus, 0) + for _, loc := range vs.store.Locations { + if dir, e := filepath.Abs(loc.Directory); e == nil { + ds = append(ds, stats.NewDiskStatus(dir)) + } + } + m["DiskStatues"] = ds + writeJsonQuiet(w, r, m) +}