gitlab-com-seaweedfs/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-01-19 02:48:24 +00:00
Code Issues Projects Releases Wiki Activity

adding lots of different stats

Browse source
This commit is contained in:
Chris Lu 2014-03-26 13:22:27 -07:00
parent 39b774a131
commit 59f6a13609
8 changed files with 123 additions and 31 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
go/stats/disk.go
View file

@ -7,13 +7,14 @@ import (
)
type DiskStatus struct {
All uint64 `json:"all"`
Used uint64 `json:"used"`
Free uint64 `json:"free"`
Dir string
All uint64
Used uint64
Free uint64
}
func DiskUsage(path string) (disk *DiskStatus) {
disk = &DiskStatus{}
func NewDiskStatus(path string) (disk *DiskStatus) {
disk = &DiskStatus{Dir: path}
fs := syscall.Statfs_t{}
err := syscall.Statfs(path, &fs)
if err != nil {

9
go/stats/disk_windows.go
View file

@ -7,11 +7,12 @@ import (
)
type DiskStatus struct {
All uint64 `json:"all"`
Used uint64 `json:"used"`
Free uint64 `json:"free"`
Dir string
All uint64
Used uint64
Free uint64
}
func DiskUsage(path string) (disk *DiskStatus) {
func NewDiskStatus(path string) (disk *DiskStatus) {
return
}

38
go/stats/memory.go Normal file
View file

@ -0,0 +1,38 @@
// +build !windows
package stats
import (
"runtime"
"syscall"
)
type MemStatus struct {
Goroutines int
All uint64
Used uint64
Free uint64
Self uint64
Heap uint64
Stack uint64
}
func MemStat() MemStatus {
mem := MemStatus{}
mem.Goroutines = runtime.NumGoroutine()
memStat := new(runtime.MemStats)
runtime.ReadMemStats(memStat)
mem.Self = memStat.Alloc
mem.Heap = memStat.HeapAlloc
mem.Stack = memStat.StackInuse
//system memory usage
sysInfo := new(syscall.Sysinfo_t)
err := syscall.Sysinfo(sysInfo)
if err == nil {
mem.All = sysInfo.Totalram //* uint64(syscall.Getpagesize())
mem.Free = sysInfo.Freeram //* uint64(syscall.Getpagesize())
mem.Used = mem.All - mem.Free
}
return mem
}

29
go/stats/memory_windows.go Normal file
View file

@ -0,0 +1,29 @@
// +build windows
package stats
import (
"runtime"
)
type MemStatus struct {
Goroutines uint32
All uint32
Used uint32
Free uint32
Self uint64
Heap uint64
Stack uint64
}
func MemStat() MemStatus {
memStat := new(runtime.MemStats)
mem.Goroutines = runtime.NumGoroutine()
runtime.ReadMemStats(memStat)
mem := MemStatus{}
mem.Self = memStat.Alloc
mem.Heap = memStat.HeapAlloc
mem.Stack = memStat.StackInuse
return mem
}

44
go/storage/store.go
View file

@ -15,10 +15,14 @@ import (
)
type DiskLocation struct {
directory string
maxVolumeCount int
Directory string
MaxVolumeCount int
volumes map[VolumeId]*Volume
}
func (mn *DiskLocation) reset() {
}
type MasterNodes struct {
nodes []string
lastNode int
@ -57,7 +61,7 @@ type Store struct {
Port int
Ip string
PublicUrl string
locations []*DiskLocation
Locations []*DiskLocation
dataCenter string //optional informaton, overwriting master setting if exists
rack string //optional information, overwriting master setting if exists
connected bool
@ -67,12 +71,12 @@ type Store struct {
func NewStore(port int, ip, publicUrl string, dirnames []string, maxVolumeCounts []int) (s *Store) {
s = &Store{Port: port, Ip: ip, PublicUrl: publicUrl}
s.locations = make([]*DiskLocation, 0)
s.Locations = make([]*DiskLocation, 0)
for i := 0; i < len(dirnames); i++ {
location := &DiskLocation{directory: dirnames[i], maxVolumeCount: maxVolumeCounts[i]}
location := &DiskLocation{Directory: dirnames[i], MaxVolumeCount: maxVolumeCounts[i]}
location.volumes = make(map[VolumeId]*Volume)
location.loadExistingVolumes()
s.locations = append(s.locations, location)
s.Locations = append(s.Locations, location)
}
return
}
@ -109,7 +113,7 @@ func (s *Store) AddVolume(volumeListString string, collection string, replicaPla
return e
}
func (s *Store) DeleteCollection(collection string) (e error) {
for _, location := range s.locations {
for _, location := range s.Locations {
for k, v := range location.volumes {
if v.Collection == collection {
e = v.Destroy()
@ -123,7 +127,7 @@ func (s *Store) DeleteCollection(collection string) (e error) {
return
}
func (s *Store) findVolume(vid VolumeId) *Volume {
for _, location := range s.locations {
for _, location := range s.Locations {
if v, found := location.volumes[vid]; found {
return v
}
@ -132,8 +136,8 @@ func (s *Store) findVolume(vid VolumeId) *Volume {
}
func (s *Store) findFreeLocation() (ret *DiskLocation) {
max := 0
for _, location := range s.locations {
currentFreeCount := location.maxVolumeCount - len(location.volumes)
for _, location := range s.Locations {
currentFreeCount := location.MaxVolumeCount - len(location.volumes)
if currentFreeCount > max {
max = currentFreeCount
ret = location
@ -146,8 +150,8 @@ func (s *Store) addVolume(vid VolumeId, collection string, replicaPlacement *Rep
return fmt.Errorf("Volume Id %s already exists!", vid)
}
if location := s.findFreeLocation(); location != nil {
glog.V(0).Infoln("In dir", location.directory, "adds volume =", vid, ", collection =", collection, ", replicaPlacement =", replicaPlacement)
if volume, err := NewVolume(location.directory, collection, vid, replicaPlacement); err == nil {
glog.V(0).Infoln("In dir", location.Directory, "adds volume =", vid, ", collection =", collection, ", replicaPlacement =", replicaPlacement)
if volume, err := NewVolume(location.Directory, collection, vid, replicaPlacement); err == nil {
location.volumes[vid] = volume
return nil
} else {
@ -206,7 +210,7 @@ func (s *Store) FreezeVolume(volumeIdString string) error {
return fmt.Errorf("volume id %s is not found during freeze!", vid)
}
func (l *DiskLocation) loadExistingVolumes() {
if dirs, err := ioutil.ReadDir(l.directory); err == nil {
if dirs, err := ioutil.ReadDir(l.Directory); err == nil {
for _, dir := range dirs {
name := dir.Name()
if !dir.IsDir() && strings.HasSuffix(name, ".dat") {
@ -218,20 +222,20 @@ func (l *DiskLocation) loadExistingVolumes() {
}
if vid, err := NewVolumeId(base); err == nil {
if l.volumes[vid] == nil {
if v, e := NewVolume(l.directory, collection, vid, nil); e == nil {
if v, e := NewVolume(l.Directory, collection, vid, nil); e == nil {
l.volumes[vid] = v
glog.V(0).Infoln("data file", l.directory+"/"+name, "replicaPlacement =", v.ReplicaPlacement, "version =", v.Version(), "size =", v.Size())
glog.V(0).Infoln("data file", l.Directory+"/"+name, "replicaPlacement =", v.ReplicaPlacement, "version =", v.Version(), "size =", v.Size())
}
}
}
}
}
}
glog.V(0).Infoln("Store started on dir:", l.directory, "with", len(l.volumes), "volumes", "max", l.maxVolumeCount)
glog.V(0).Infoln("Store started on dir:", l.Directory, "with", len(l.volumes), "volumes", "max", l.MaxVolumeCount)
}
func (s *Store) Status() []*VolumeInfo {
var stats []*VolumeInfo
for _, location := range s.locations {
for _, location := range s.Locations {
for k, v := range location.volumes {
s := &VolumeInfo{Id: VolumeId(k), Size: v.ContentSize(),
Collection: v.Collection,
@ -268,8 +272,8 @@ func (s *Store) Join() error {
}
stats := new([]*VolumeInfo)
maxVolumeCount := 0
for _, location := range s.locations {
maxVolumeCount = maxVolumeCount + location.maxVolumeCount
for _, location := range s.Locations {
maxVolumeCount = maxVolumeCount + location.MaxVolumeCount
for k, v := range location.volumes {
s := &VolumeInfo{Id: VolumeId(k), Size: uint64(v.Size()),
Collection: v.Collection,
@ -308,7 +312,7 @@ func (s *Store) Join() error {
return nil
}
func (s *Store) Close() {
for _, location := range s.locations {
for _, location := range s.Locations {
for _, v := range location.volumes {
v.Close()
}

8
go/weed/weed_server/common.go
View file

@ -165,12 +165,16 @@ func parseURLPath(path string) (vid, fid, filename, ext string, isVolumeIdOnly b
func statsCounterHandler(w http.ResponseWriter, r *http.Request) {
m := make(map[string]interface{})
m["Version"] = util.VERSION
m["Statistics"] = serverStats
m["Counters"] = serverStats
writeJsonQuiet(w, r, m)
}
type MemoryStatistics struct {
}
func statsMemoryHandler(w http.ResponseWriter, r *http.Request) {
m := make(map[string]interface{})
m["Version"] = util.VERSION
m["Statistics"] = serverStats
m["Memory"] = stats.MemStat()
writeJsonQuiet(w, r, m)
}

1
go/weed/weed_server/volume_server.go
View file

@ -40,6 +40,7 @@ func NewVolumeServer(r *http.ServeMux, ip string, port int, publicUrl string, fo
r.HandleFunc("/admin/delete_collection", secure(vs.whiteList, vs.deleteCollectionHandler))
r.HandleFunc("/stats/counter", secure(vs.whiteList, statsCounterHandler))
r.HandleFunc("/stats/memory", secure(vs.whiteList, statsMemoryHandler))
r.HandleFunc("/stats/disk", secure(vs.whiteList, vs.statsDiskHandler))
r.HandleFunc("/", vs.storeHandler)
go func() {

14
go/weed/weed_server/volume_server_handlers.go
View file

@ -9,6 +9,7 @@ import (
"code.google.com/p/weed-fs/go/util"
"mime"
"net/http"
"path/filepath"
"strconv"
"strings"
"time"
@ -259,3 +260,16 @@ func (vs *VolumeServer) DeleteHandler(w http.ResponseWriter, r *http.Request) {
m["size"] = uint32(count)
writeJsonQuiet(w, r, m)
}
func (vs *VolumeServer) statsDiskHandler(w http.ResponseWriter, r *http.Request) {
m := make(map[string]interface{})
m["Version"] = util.VERSION
ds := make([]*stats.DiskStatus, 0)
for _, loc := range vs.store.Locations {
if dir, e := filepath.Abs(loc.Directory); e == nil {
ds = append(ds, stats.NewDiskStatus(dir))
}
}
m["DiskStatues"] = ds
writeJsonQuiet(w, r, m)
}