From 62f74f5e3cce0668478c179f81fd1a9c5916e455 Mon Sep 17 00:00:00 2001
From: zzq09494 <zzq09494@ly.com>
Date: Thu, 16 Jun 2022 13:21:25 +0800
Subject: [PATCH 1/3] add bucket label to s3 prometheus metrics

---
 weed/s3api/stats.go   | 6 ++++--
 weed/stats/metrics.go | 5 +++--
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/weed/s3api/stats.go b/weed/s3api/stats.go
index 973d8c0eb..003807a25 100644
--- a/weed/s3api/stats.go
+++ b/weed/s3api/stats.go
@@ -1,6 +1,7 @@
 package s3api
 
 import (
+	"github.com/chrislusf/seaweedfs/weed/s3api/s3_constants"
 	stats_collect "github.com/chrislusf/seaweedfs/weed/stats"
 	"net/http"
 	"strconv"
@@ -27,11 +28,12 @@ func (r *StatusRecorder) Flush() {
 
 func track(f http.HandlerFunc, action string) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
+		bucket, _ := s3_constants.GetBucketAndObject(r)
 		w.Header().Set("Server", "SeaweedFS S3")
 		recorder := NewStatusResponseWriter(w)
 		start := time.Now()
 		f(recorder, r)
-		stats_collect.S3RequestHistogram.WithLabelValues(action).Observe(time.Since(start).Seconds())
-		stats_collect.S3RequestCounter.WithLabelValues(action, strconv.Itoa(recorder.Status)).Inc()
+		stats_collect.S3RequestHistogram.WithLabelValues(action, bucket).Observe(time.Since(start).Seconds())
+		stats_collect.S3RequestCounter.WithLabelValues(action, strconv.Itoa(recorder.Status), bucket).Inc()
 	}
 }
diff --git a/weed/stats/metrics.go b/weed/stats/metrics.go
index 207b37c81..f0b810608 100644
--- a/weed/stats/metrics.go
+++ b/weed/stats/metrics.go
@@ -173,7 +173,8 @@ var (
 			Subsystem: "s3",
 			Name:      "request_total",
 			Help:      "Counter of s3 requests.",
-		}, []string{"type", "code"})
+		}, []string{"type", "code", "bucket"})
+
 	S3RequestHistogram = prometheus.NewHistogramVec(
 		prometheus.HistogramOpts{
 			Namespace: "SeaweedFS",
@@ -181,7 +182,7 @@ var (
 			Name:      "request_seconds",
 			Help:      "Bucketed histogram of s3 request processing time.",
 			Buckets:   prometheus.ExponentialBuckets(0.0001, 2, 24),
-		}, []string{"type"})
+		}, []string{"type", "bucket"})
 )
 
 func init() {

From 0a613876caa22542cac3bd99c19f44e64846a52f Mon Sep 17 00:00:00 2001
From: zzq09494 <zzq09494@ly.com>
Date: Thu, 16 Jun 2022 13:50:16 +0800
Subject: [PATCH 2/3] add  bucket label to the grafana dashboard

---
 other/metrics/grafana_seaweedfs.json | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/other/metrics/grafana_seaweedfs.json b/other/metrics/grafana_seaweedfs.json
index 3b9b222b4..d10fb7c51 100644
--- a/other/metrics/grafana_seaweedfs.json
+++ b/other/metrics/grafana_seaweedfs.json
@@ -539,11 +539,12 @@
           "step": 60
         },
         {
-          "expr": "histogram_quantile(0.90, sum(rate(SeaweedFS_s3_request_seconds_bucket[1m])) by (le, type))",
+          "expr": "histogram_quantile(0.90, sum(rate(SeaweedFS_s3_request_seconds_bucket[1m])) by (le, type, bucket))",
           "format": "time_series",
           "hide": false,
+          "interval": "",
           "intervalFactor": 2,
-          "legendFormat": "{{type}}",
+          "legendFormat": "{{bucket}} {{type}}",
           "refId": "B",
           "step": 60
         }
@@ -645,11 +646,12 @@
           "step": 60
         },
         {
-          "expr": "histogram_quantile(0.95, sum(rate(SeaweedFS_s3_request_seconds_bucket[1m])) by (le, type))",
+          "expr": "histogram_quantile(0.95, sum(rate(SeaweedFS_s3_request_seconds_bucket[1m])) by (le, type,bucket))",
           "format": "time_series",
           "hide": false,
+          "interval": "",
           "intervalFactor": 2,
-          "legendFormat": "{{type}}",
+          "legendFormat": "{{bucket}} {{type}}",
           "refId": "B",
           "step": 60
         }
@@ -751,11 +753,11 @@
           "step": 60
         },
         {
-          "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_s3_request_seconds_bucket[1m])) by (le, type))",
+          "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_s3_request_seconds_bucket[1m])) by (le, type, bucket))",
           "format": "time_series",
           "hide": false,
           "intervalFactor": 2,
-          "legendFormat": "{{type}}",
+          "legendFormat": "{{bucket}} {{type}}",
           "refId": "B",
           "step": 60
         }
@@ -864,7 +866,7 @@
           "expr": "rate(SeaweedFS_s3_request_total[1m])",
           "format": "time_series",
           "intervalFactor": 2,
-          "legendFormat": "{{type}}",
+          "legendFormat": "{{bucket}} {{type}}",
           "refId": "A",
           "step": 30
         }

From 6449114e5e88953b0fe22b50b46d91a7bed52523 Mon Sep 17 00:00:00 2001
From: zzq09494 <zzq09494@ly.com>
Date: Thu, 16 Jun 2022 13:52:36 +0800
Subject: [PATCH 3/3] format

---
 other/metrics/grafana_seaweedfs.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/other/metrics/grafana_seaweedfs.json b/other/metrics/grafana_seaweedfs.json
index d10fb7c51..88844b3c3 100644
--- a/other/metrics/grafana_seaweedfs.json
+++ b/other/metrics/grafana_seaweedfs.json
@@ -646,7 +646,7 @@
           "step": 60
         },
         {
-          "expr": "histogram_quantile(0.95, sum(rate(SeaweedFS_s3_request_seconds_bucket[1m])) by (le, type,bucket))",
+          "expr": "histogram_quantile(0.95, sum(rate(SeaweedFS_s3_request_seconds_bucket[1m])) by (le, type, bucket))",
           "format": "time_series",
           "hide": false,
           "interval": "",