volume server: adds basic metrics

weed/command/server.go

@@ -149,6 +149,8 @@ func runServer(cmd *Command, args []string) bool {
 
 	filerOptions.metricsAddress = serverOptions.metricsAddress
 	filerOptions.metricsIntervalSec = serverOptions.metricsIntervalSec
+	serverOptions.v.metricsAddress = serverOptions.metricsAddress
+	serverOptions.v.metricsIntervalSec = serverOptions.metricsIntervalSec
 
 	filerAddress := fmt.Sprintf("%s:%d", *serverIp, *filerOptions.port)
 	s3Options.filer = &filerAddress

weed/command/volume.go

@@ -45,6 +45,8 @@ type VolumeServerOptions struct {
 	cpuProfile            *string
 	memProfile            *string
 	compactionMBPerSecond *int
+	metricsAddress        *string
+	metricsIntervalSec    *int
 }
 
 func init() {
@@ -66,6 +68,8 @@ func init() {
 	v.cpuProfile = cmdVolume.Flag.String("cpuprofile", "", "cpu profile output file")
 	v.memProfile = cmdVolume.Flag.String("memprofile", "", "memory profile output file")
 	v.compactionMBPerSecond = cmdVolume.Flag.Int("compactionMBps", 0, "limit background compaction or copying speed in mega bytes per second")
+	v.metricsAddress = cmdVolume.Flag.String("metrics.address", "", "Prometheus gateway address")
+	v.metricsIntervalSec = cmdVolume.Flag.Int("metrics.intervalSeconds", 15, "Prometheus push interval in seconds")
 }
 
 var cmdVolume = &Command{
@@ -161,6 +165,8 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
 		v.whiteList,
 		*v.fixJpgOrientation, *v.readRedirect,
 		*v.compactionMBPerSecond,
+		*v.metricsAddress,
+		*v.metricsIntervalSec,
 	)
 
 	listeningAddress := *v.bindIp + ":" + strconv.Itoa(*v.port)

weed/server/filer_server.go

@@ -88,23 +88,23 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption)
 		readonlyMux.HandleFunc("/", fs.readonlyFilerHandler)
 	}
 
-	startPushingMetric(option.MetricsAddress, option.MetricsIntervalSec)
+	startPushingMetric("filer", filerGather, option.MetricsAddress, option.MetricsIntervalSec)
 
 	return fs, nil
 }
 
-func startPushingMetric(addr string, intervalSeconds int) {
+func startPushingMetric(name string, gatherer *prometheus.Registry, addr string, intervalSeconds int) {
 	if intervalSeconds == 0 || addr == "" {
 		glog.V(0).Info("disable metrics reporting")
 		return
 	}
 	glog.V(0).Infof("push metrics to %s every %d seconds", addr, intervalSeconds)
-	go loopPushMetrics(addr, intervalSeconds)
+	go loopPushMetrics(name, gatherer, addr, intervalSeconds)
 }
 
-func loopPushMetrics(addr string, intervalSeconds int) {
+func loopPushMetrics(name string, gatherer *prometheus.Registry, addr string, intervalSeconds int) {
 
-	pusher := push.New(addr, "filer").Gatherer(prometheus.DefaultGatherer)
+	pusher := push.New(addr, name).Gatherer(gatherer)
 
 	for {
 		err := pusher.Push()

weed/server/metrics.go

@@ -3,6 +3,9 @@ package weed_server
 import "github.com/prometheus/client_golang/prometheus"
 
 var (
+	filerGather        = prometheus.NewRegistry()
+	volumeServerGather = prometheus.NewRegistry()
+
 	filerRequestCounter = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
 			Namespace: "SeaweedFS",
@@ -17,11 +20,33 @@ var (
 			Subsystem: "filer",
 			Name:      "request_seconds",
 			Help:      "Bucketed histogram of filer request processing time.",
-			Buckets:   prometheus.ExponentialBuckets(0.0005, 2, 18),
+			Buckets:   prometheus.ExponentialBuckets(0.0001, 2, 24),
+		}, []string{"type"})
+
+	volumeServerRequestCounter = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: "SeaweedFS",
+			Subsystem: "volumeServer",
+			Name:      "request_total",
+			Help:      "Counter of filer requests.",
+		}, []string{"type"})
+
+	volumeServerHistogram = prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Namespace: "SeaweedFS",
+			Subsystem: "volumeServer",
+			Name:      "request_seconds",
+			Help:      "Bucketed histogram of filer request processing time.",
+			Buckets:   prometheus.ExponentialBuckets(0.0001, 2, 24),
 		}, []string{"type"})
 )
 
 func init() {
-	prometheus.MustRegister(filerRequestCounter)
+
-	prometheus.MustRegister(filerRequestHistogram)
+	filerGather.MustRegister(filerRequestCounter)
+	filerGather.MustRegister(filerRequestHistogram)
+
+	volumeServerGather.MustRegister(volumeServerRequestCounter)
+	volumeServerGather.MustRegister(volumeServerHistogram)
+
 }

weed/server/volume_server.go

@@ -24,6 +24,8 @@ type VolumeServer struct {
 	FixJpgOrientation       bool
 	ReadRedirect            bool
 	compactionBytePerSecond int64
+	MetricsAddress          string
+	MetricsIntervalSec      int
 }
 
 func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string,
@@ -36,6 +38,8 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string,
 	fixJpgOrientation bool,
 	readRedirect bool,
 	compactionMBPerSecond int,
+	metricsAddress          string,
+	metricsIntervalSec      int,
 ) *VolumeServer {
 
 	v := viper.GetViper()
@@ -80,6 +84,7 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string,
 	}
 
 	go vs.heartbeat()
+	startPushingMetric("volumeServer", volumeServerGather, metricsAddress, metricsIntervalSec)
 
 	return vs
 }

weed/server/volume_server_handlers_read.go

@@ -26,6 +26,11 @@ import (
 var fileNameEscaper = strings.NewReplacer("\\", "\\\\", "\"", "\\\"")
 
 func (vs *VolumeServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) {
+
+	volumeServerRequestCounter.WithLabelValues("get").Inc()
+	start := time.Now()
+	defer func() { volumeServerHistogram.WithLabelValues("get").Observe(time.Since(start).Seconds()) }()
+
 	n := new(needle.Needle)
 	vid, fid, filename, ext, _ := parseURLPath(r.URL.Path)
 

weed/server/volume_server_handlers_write.go

@@ -15,6 +15,11 @@ import (
 )
 
 func (vs *VolumeServer) PostHandler(w http.ResponseWriter, r *http.Request) {
+
+	volumeServerRequestCounter.WithLabelValues("post").Inc()
+	start := time.Now()
+	defer func() { volumeServerHistogram.WithLabelValues("post").Observe(time.Since(start).Seconds()) }()
+
 	if e := r.ParseForm(); e != nil {
 		glog.V(0).Infoln("form parse error:", e)
 		writeJsonError(w, r, http.StatusBadRequest, e)
@@ -60,6 +65,11 @@ func (vs *VolumeServer) PostHandler(w http.ResponseWriter, r *http.Request) {
 }
 
 func (vs *VolumeServer) DeleteHandler(w http.ResponseWriter, r *http.Request) {
+
+	volumeServerRequestCounter.WithLabelValues("delete").Inc()
+	start := time.Now()
+	defer func() { volumeServerHistogram.WithLabelValues("delete").Observe(time.Since(start).Seconds()) }()
+
 	n := new(needle.Needle)
 	vid, fid, _, _, _ := parseURLPath(r.URL.Path)
 	volumeId, _ := needle.NewVolumeId(vid)