Add bucket's traffic metrics (#6444)

* Add bucket's traffic metrics

* Add bucket traffic to dashboards

* Fix bucket metrics help messages

* Fix variable names

weed/s3api/s3api_object_handlers.go

@@ -135,7 +135,7 @@ func (s3a *S3ApiServer) HeadObjectHandler(w http.ResponseWriter, r *http.Request
 	s3a.proxyToFiler(w, r, destUrl, false, passThroughResponse)
 }
 
-func (s3a *S3ApiServer) proxyToFiler(w http.ResponseWriter, r *http.Request, destUrl string, isWrite bool, responseFn func(proxyResponse *http.Response, w http.ResponseWriter) (statusCode int)) {
+func (s3a *S3ApiServer) proxyToFiler(w http.ResponseWriter, r *http.Request, destUrl string, isWrite bool, responseFn func(proxyResponse *http.Response, w http.ResponseWriter) (statusCode int, bytesTransferred int64)) {
 
 	glog.V(3).Infof("s3 proxying %s to %s", r.Method, destUrl)
 	start := time.Now()
@@ -190,7 +190,7 @@ func (s3a *S3ApiServer) proxyToFiler(w http.ResponseWriter, r *http.Request, des
 	if r.Method == http.MethodDelete {
 		if resp.StatusCode == http.StatusNotFound {
 			// this is normal
-			responseStatusCode := responseFn(resp, w)
+			responseStatusCode, _ := responseFn(resp, w)
 			s3err.PostLog(r, responseStatusCode, s3err.ErrNone)
 			return
 		}
@@ -202,7 +202,7 @@ func (s3a *S3ApiServer) proxyToFiler(w http.ResponseWriter, r *http.Request, des
 
 	TimeToFirstByte(r.Method, start, r)
 	if resp.Header.Get(s3_constants.SeaweedFSIsDirectoryKey) == "true" {
-		responseStatusCode := responseFn(resp, w)
+		responseStatusCode, _ := responseFn(resp, w)
 		s3err.PostLog(r, responseStatusCode, s3err.ErrNone)
 		return
 	}
@@ -233,7 +233,9 @@ func (s3a *S3ApiServer) proxyToFiler(w http.ResponseWriter, r *http.Request, des
 
 	setUserMetadataKeyToLowercase(resp)
 
-	responseStatusCode := responseFn(resp, w)
+	responseStatusCode, bytesTransferred := responseFn(resp, w)
+	BucketTrafficSent(bytesTransferred, r)
+
 	s3err.PostLog(r, responseStatusCode, s3err.ErrNone)
 }
 
@@ -246,7 +248,7 @@ func setUserMetadataKeyToLowercase(resp *http.Response) {
 	}
 }
 
-func passThroughResponse(proxyResponse *http.Response, w http.ResponseWriter) (statusCode int) {
+func passThroughResponse(proxyResponse *http.Response, w http.ResponseWriter) (statusCode int, bytesTransferred int64) {
 	for k, v := range proxyResponse.Header {
 		w.Header()[k] = v
 	}
@@ -259,8 +261,9 @@ func passThroughResponse(proxyResponse *http.Response, w http.ResponseWriter) (s
 	w.WriteHeader(statusCode)
 	buf := mem.Allocate(128 * 1024)
 	defer mem.Free(buf)
-	if n, err := io.CopyBuffer(w, proxyResponse.Body, buf); err != nil {
-		glog.V(1).Infof("passthrough response read %d bytes: %v", n, err)
+	bytesTransferred, err := io.CopyBuffer(w, proxyResponse.Body, buf)
+	if err != nil {
+		glog.V(1).Infof("passthrough response read %d bytes: %v", bytesTransferred, err)
 	}
-	return statusCode
+	return statusCode, bytesTransferred
 }

weed/s3api/s3api_object_handlers_put.go

@@ -17,6 +17,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	weed_server "github.com/seaweedfs/seaweedfs/weed/server"
+	stats_collect "github.com/seaweedfs/seaweedfs/weed/stats"
 )
 
 func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
@@ -161,7 +162,7 @@ func (s3a *S3ApiServer) putToFiler(r *http.Request, uploadUrl string, dataReader
 		glog.Errorf("upload to filer error: %v", ret.Error)
 		return "", filerErrorToS3Error(ret.Error)
 	}
-
+	stats_collect.S3BucketTrafficReceivedBytesCounter.WithLabelValues(bucket).Add(float64(ret.Size))
 	return etag, s3err.ErrNone
 }
 

weed/s3api/stats.go

@@ -35,3 +35,8 @@ func TimeToFirstByte(action string, start time.Time, r *http.Request) {
 	stats_collect.S3TimeToFirstByteHistogram.WithLabelValues(action, bucket).Observe(float64(time.Since(start).Milliseconds()))
 	stats_collect.RecordBucketActiveTime(bucket)
 }
+
+func BucketTrafficSent(bytesTransferred int64, r *http.Request) {
+	bucket, _ := s3_constants.GetBucketAndObject(r)
+	stats_collect.S3BucketTrafficSentBytesCounter.WithLabelValues(bucket).Add(float64(bytesTransferred))
+}

weed/stats/metrics.go

@@ -319,6 +319,22 @@ var (
 			Name:      "in_flight_requests",
 			Help:      "Current number of in-flight requests being handled by s3.",
 		}, []string{"type"})
+
+	S3BucketTrafficReceivedBytesCounter = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: Namespace,
+			Subsystem: "s3",
+			Name:      "bucket_traffic_received_bytes_total",
+			Help:      "Total number of bytes received by an S3 bucket from clients.",
+		}, []string{"bucket"})
+
+	S3BucketTrafficSentBytesCounter = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: Namespace,
+			Subsystem: "s3",
+			Name:      "bucket_traffic_sent_bytes_total",
+			Help:      "Total number of bytes sent from an S3 bucket to clients.",
+		}, []string{"bucket"})
 )
 
 func init() {
@@ -362,6 +378,8 @@ func init() {
 	Gather.MustRegister(S3RequestHistogram)
 	Gather.MustRegister(S3InFlightRequestsGauge)
 	Gather.MustRegister(S3TimeToFirstByteHistogram)
+	Gather.MustRegister(S3BucketTrafficReceivedBytesCounter)
+	Gather.MustRegister(S3BucketTrafficSentBytesCounter)
 
 	go bucketMetricTTLControl()
 }