gitlab-com-seaweedfs/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-01-19 02:48:24 +00:00
Code Issues Projects Releases Wiki Activity

refactoring

Browse source
This commit is contained in:
Chris Lu 2019-06-15 12:21:44 -07:00
parent 5336008dcd
commit 8b43679ae3
8 changed files with 53 additions and 34 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
weed/command/filer.go
View file

@ -116,6 +116,7 @@ func (fo *FilerOptions) startFiler() {
DisableHttp: *fo.disableHttp, DisableHttp: *fo.disableHttp,
MetricsAddress: *fo.metricsAddress, MetricsAddress: *fo.metricsAddress,
MetricsIntervalSec: *fo.metricsIntervalSec, MetricsIntervalSec: *fo.metricsIntervalSec,
Port: *fo.port,
}) })
if nfs_err != nil { if nfs_err != nil {
glog.Fatalf("Filer startup error: %v", nfs_err) glog.Fatalf("Filer startup error: %v", nfs_err)

12
weed/server/filer_server.go
View file

@ -1,9 +1,11 @@
package weed_server package weed_server
import ( import (
"fmt"
"net/http" "net/http"
"os" "os"
"github.com/chrislusf/seaweedfs/weed/stats"
"github.com/chrislusf/seaweedfs/weed/util" "github.com/chrislusf/seaweedfs/weed/util"
"google.golang.org/grpc" "google.golang.org/grpc"
@ -38,6 +40,7 @@ type FilerOption struct {
DisableHttp bool DisableHttp bool
MetricsAddress string MetricsAddress string
MetricsIntervalSec int MetricsIntervalSec int
Port int
} }
type FilerServer struct { type FilerServer struct {
@ -85,8 +88,15 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption)
readonlyMux.HandleFunc("/", fs.readonlyFilerHandler) readonlyMux.HandleFunc("/", fs.readonlyFilerHandler)
} }
startPushingMetric("filer", filerGather, option.MetricsAddress, option.MetricsIntervalSec) stats.StartPushingMetric("filer", sourceName(option.Port), stats.FilerGather, option.MetricsAddress, option.MetricsIntervalSec)
return fs, nil return fs, nil
} }
func sourceName(port int) string {
hostname, err := os.Hostname()
if err != nil {
return "unknown"
}
return fmt.Sprintf("%s_%d", hostname, port)
}

9
weed/server/filer_server_handlers_read.go
View file

@ -15,14 +15,15 @@ import (
"github.com/chrislusf/seaweedfs/weed/filer2" "github.com/chrislusf/seaweedfs/weed/filer2"
"github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/stats"
"github.com/chrislusf/seaweedfs/weed/util" "github.com/chrislusf/seaweedfs/weed/util"
) )
func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request, isGetMethod bool) { func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request, isGetMethod bool) {
filerRequestCounter.WithLabelValues("get").Inc() stats.FilerRequestCounter.WithLabelValues("get").Inc()
start := time.Now() start := time.Now()
defer func() { filerRequestHistogram.WithLabelValues("get").Observe(time.Since(start).Seconds()) }() defer func() { stats.FilerRequestHistogram.WithLabelValues("get").Observe(time.Since(start).Seconds()) }()
path := r.URL.Path path := r.URL.Path
if strings.HasSuffix(path, "/") && len(path) > 1 { if strings.HasSuffix(path, "/") && len(path) > 1 {
@ -37,7 +38,7 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request,
} }
glog.V(1).Infof("Not found %s: %v", path, err) glog.V(1).Infof("Not found %s: %v", path, err)
filerRequestCounter.WithLabelValues("read.notfound").Inc() stats.FilerRequestCounter.WithLabelValues("read.notfound").Inc()
w.WriteHeader(http.StatusNotFound) w.WriteHeader(http.StatusNotFound)
return return
} }
@ -53,7 +54,7 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request,
if len(entry.Chunks) == 0 { if len(entry.Chunks) == 0 {
glog.V(1).Infof("no file chunks for %s, attr=%+v", path, entry.Attr) glog.V(1).Infof("no file chunks for %s, attr=%+v", path, entry.Attr)
filerRequestCounter.WithLabelValues("read.nocontent").Inc() stats.FilerRequestCounter.WithLabelValues("read.nocontent").Inc()
w.WriteHeader(http.StatusNoContent) w.WriteHeader(http.StatusNoContent)
return return
} }

9
weed/server/filer_server_handlers_write.go
View file

@ -20,6 +20,7 @@ import (
"github.com/chrislusf/seaweedfs/weed/operation" "github.com/chrislusf/seaweedfs/weed/operation"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/security" "github.com/chrislusf/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/stats"
"github.com/chrislusf/seaweedfs/weed/util" "github.com/chrislusf/seaweedfs/weed/util"
) )
@ -70,9 +71,9 @@ func (fs *FilerServer) assignNewFileInfo(w http.ResponseWriter, r *http.Request,
func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) { func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) {
filerRequestCounter.WithLabelValues("post").Inc() stats.FilerRequestCounter.WithLabelValues("post").Inc()
start := time.Now() start := time.Now()
defer func() { filerRequestHistogram.WithLabelValues("post").Observe(time.Since(start).Seconds()) }() defer func() { stats.FilerRequestHistogram.WithLabelValues("post").Observe(time.Since(start).Seconds()) }()
ctx := context.Background() ctx := context.Background()
@ -232,9 +233,9 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) {
// curl -X DELETE http://localhost:8888/path/to?recursive=true // curl -X DELETE http://localhost:8888/path/to?recursive=true
func (fs *FilerServer) DeleteHandler(w http.ResponseWriter, r *http.Request) { func (fs *FilerServer) DeleteHandler(w http.ResponseWriter, r *http.Request) {
filerRequestCounter.WithLabelValues("delete").Inc() stats.FilerRequestCounter.WithLabelValues("delete").Inc()
start := time.Now() start := time.Now()
defer func() { filerRequestHistogram.WithLabelValues("delete").Observe(time.Since(start).Seconds()) }() defer func() { stats.FilerRequestHistogram.WithLabelValues("delete").Observe(time.Since(start).Seconds()) }()
isRecursive := r.FormValue("recursive") == "true" isRecursive := r.FormValue("recursive") == "true"

8
weed/server/volume_server.go
View file

@ -1,9 +1,12 @@
package weed_server package weed_server
import ( import (
"google.golang.org/grpc" "fmt"
"net/http" "net/http"
"github.com/chrislusf/seaweedfs/weed/stats"
"google.golang.org/grpc"
"github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/security" "github.com/chrislusf/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/storage" "github.com/chrislusf/seaweedfs/weed/storage"
@ -84,7 +87,8 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string,
} }
go vs.heartbeat() go vs.heartbeat()
startPushingMetric("volumeServer", volumeServerGather, metricsAddress, metricsIntervalSec) hostAddress := fmt.Sprintf("%s:%d", ip, port)
stats.StartPushingMetric("volumeServer", hostAddress, stats.VolumeServerGather, metricsAddress, metricsIntervalSec)
return vs return vs
} }

5
weed/server/volume_server_handlers_read.go
View file

@ -19,6 +19,7 @@ import (
"github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/images" "github.com/chrislusf/seaweedfs/weed/images"
"github.com/chrislusf/seaweedfs/weed/operation" "github.com/chrislusf/seaweedfs/weed/operation"
"github.com/chrislusf/seaweedfs/weed/stats"
"github.com/chrislusf/seaweedfs/weed/storage/needle" "github.com/chrislusf/seaweedfs/weed/storage/needle"
"github.com/chrislusf/seaweedfs/weed/util" "github.com/chrislusf/seaweedfs/weed/util"
) )
@ -27,9 +28,9 @@ var fileNameEscaper = strings.NewReplacer("\\", "\\\\", "\"", "\\\"")
func (vs *VolumeServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) { func (vs *VolumeServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) {
volumeServerRequestCounter.WithLabelValues("get").Inc() stats.VolumeServerRequestCounter.WithLabelValues("get").Inc()
start := time.Now() start := time.Now()
defer func() { volumeServerRequestHistogram.WithLabelValues("get").Observe(time.Since(start).Seconds()) }() defer func() { stats.VolumeServerRequestHistogram.WithLabelValues("get").Observe(time.Since(start).Seconds()) }()
n := new(needle.Needle) n := new(needle.Needle)
vid, fid, filename, ext, _ := parseURLPath(r.URL.Path) vid, fid, filename, ext, _ := parseURLPath(r.URL.Path)

9
weed/server/volume_server_handlers_write.go
View file

@ -10,15 +10,16 @@ import (
"github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/operation" "github.com/chrislusf/seaweedfs/weed/operation"
"github.com/chrislusf/seaweedfs/weed/stats"
"github.com/chrislusf/seaweedfs/weed/storage/needle" "github.com/chrislusf/seaweedfs/weed/storage/needle"
"github.com/chrislusf/seaweedfs/weed/topology" "github.com/chrislusf/seaweedfs/weed/topology"
) )
func (vs *VolumeServer) PostHandler(w http.ResponseWriter, r *http.Request) { func (vs *VolumeServer) PostHandler(w http.ResponseWriter, r *http.Request) {
volumeServerRequestCounter.WithLabelValues("post").Inc() stats.VolumeServerRequestCounter.WithLabelValues("post").Inc()
start := time.Now() start := time.Now()
defer func() { volumeServerRequestHistogram.WithLabelValues("post").Observe(time.Since(start).Seconds()) }() defer func() { stats.VolumeServerRequestHistogram.WithLabelValues("post").Observe(time.Since(start).Seconds()) }()
if e := r.ParseForm(); e != nil { if e := r.ParseForm(); e != nil {
glog.V(0).Infoln("form parse error:", e) glog.V(0).Infoln("form parse error:", e)
@ -66,9 +67,9 @@ func (vs *VolumeServer) PostHandler(w http.ResponseWriter, r *http.Request) {
func (vs *VolumeServer) DeleteHandler(w http.ResponseWriter, r *http.Request) { func (vs *VolumeServer) DeleteHandler(w http.ResponseWriter, r *http.Request) {
volumeServerRequestCounter.WithLabelValues("delete").Inc() stats.VolumeServerRequestCounter.WithLabelValues("delete").Inc()
start := time.Now() start := time.Now()
defer func() { volumeServerRequestHistogram.WithLabelValues("delete").Observe(time.Since(start).Seconds()) }() defer func() { stats.VolumeServerRequestHistogram.WithLabelValues("delete").Observe(time.Since(start).Seconds()) }()
n := new(needle.Needle) n := new(needle.Needle)
vid, fid, _, _, _ := parseURLPath(r.URL.Path) vid, fid, _, _, _ := parseURLPath(r.URL.Path)

34
weed/server/metrics.go → weed/stats/metrics.go
View file

@ -1,4 +1,4 @@
package weed_server package stats
import ( import (
"time" "time"
@ -9,10 +9,10 @@ import (
) )
var ( var (
filerGather = prometheus.NewRegistry() FilerGather = prometheus.NewRegistry()
volumeServerGather = prometheus.NewRegistry() VolumeServerGather = prometheus.NewRegistry()
filerRequestCounter = prometheus.NewCounterVec( FilerRequestCounter = prometheus.NewCounterVec(
prometheus.CounterOpts{ prometheus.CounterOpts{
Namespace: "SeaweedFS", Namespace: "SeaweedFS",
Subsystem: "filer", Subsystem: "filer",
@ -20,7 +20,7 @@ var (
Help: "Counter of filer requests.", Help: "Counter of filer requests.",
}, []string{"type"}) }, []string{"type"})
filerRequestHistogram = prometheus.NewHistogramVec( FilerRequestHistogram = prometheus.NewHistogramVec(
prometheus.HistogramOpts{ prometheus.HistogramOpts{
Namespace: "SeaweedFS", Namespace: "SeaweedFS",
Subsystem: "filer", Subsystem: "filer",
@ -29,7 +29,7 @@ var (
Buckets: prometheus.ExponentialBuckets(0.0001, 2, 24), Buckets: prometheus.ExponentialBuckets(0.0001, 2, 24),
}, []string{"type"}) }, []string{"type"})
volumeServerRequestCounter = prometheus.NewCounterVec( VolumeServerRequestCounter = prometheus.NewCounterVec(
prometheus.CounterOpts{ prometheus.CounterOpts{
Namespace: "SeaweedFS", Namespace: "SeaweedFS",
Subsystem: "volumeServer", Subsystem: "volumeServer",
@ -37,7 +37,7 @@ var (
Help: "Counter of filer requests.", Help: "Counter of filer requests.",
}, []string{"type"}) }, []string{"type"})
volumeServerRequestHistogram = prometheus.NewHistogramVec( VolumeServerRequestHistogram = prometheus.NewHistogramVec(
prometheus.HistogramOpts{ prometheus.HistogramOpts{
Namespace: "SeaweedFS", Namespace: "SeaweedFS",
Subsystem: "volumeServer", Subsystem: "volumeServer",
@ -46,38 +46,38 @@ var (
Buckets: prometheus.ExponentialBuckets(0.0001, 2, 24), Buckets: prometheus.ExponentialBuckets(0.0001, 2, 24),
}, []string{"type"}) }, []string{"type"})
volumeServerVolumeCounter = prometheus.NewGauge( VolumeServerVolumeCounter = prometheus.NewGauge(
prometheus.GaugeOpts{ prometheus.GaugeOpts{
Namespace: "SeaweedFS", Namespace: "SeaweedFS",
Subsystem: "volumeServer", Subsystem: "volumeServer",
Name: "volumes", Name: "volumes",
Help: "Number of volumes.", Help: "Number of volumes.",
}) })
) )
func init() { func init() {
filerGather.MustRegister(filerRequestCounter) FilerGather.MustRegister(FilerRequestCounter)
filerGather.MustRegister(filerRequestHistogram) FilerGather.MustRegister(FilerRequestHistogram)
volumeServerGather.MustRegister(volumeServerRequestCounter) VolumeServerGather.MustRegister(VolumeServerRequestCounter)
volumeServerGather.MustRegister(volumeServerRequestHistogram) VolumeServerGather.MustRegister(VolumeServerRequestHistogram)
VolumeServerGather.MustRegister(VolumeServerVolumeCounter)
} }
func startPushingMetric(name string, gatherer *prometheus.Registry, addr string, intervalSeconds int) { func StartPushingMetric(name, instance string, gatherer *prometheus.Registry, addr string, intervalSeconds int) {
if intervalSeconds == 0 || addr == "" { if intervalSeconds == 0 || addr == "" {
glog.V(0).Info("disable metrics reporting") glog.V(0).Info("disable metrics reporting")
return return
} }
glog.V(0).Infof("push metrics to %s every %d seconds", addr, intervalSeconds) glog.V(0).Infof("push metrics to %s every %d seconds", addr, intervalSeconds)
go loopPushMetrics(name, gatherer, addr, intervalSeconds) go loopPushMetrics(name, instance, gatherer, addr, intervalSeconds)
} }
func loopPushMetrics(name string, gatherer *prometheus.Registry, addr string, intervalSeconds int) { func loopPushMetrics(name, instance string, gatherer *prometheus.Registry, addr string, intervalSeconds int) {
pusher := push.New(addr, name).Gatherer(gatherer) pusher := push.New(addr, name).Gatherer(gatherer).Grouping("instance", instance)
for { for {
err := pusher.Push() err := pusher.Push()