Add time to first byte metric for s3 (#4768)

* Add time to first byte metric for s3

* Change second to millisecond

weed/s3api/s3api_object_handlers.go

@@ -355,6 +355,7 @@ func (s3a *S3ApiServer) doDeleteEmptyDirectories(client filer_pb.SeaweedFilerCli
 func (s3a *S3ApiServer) proxyToFiler(w http.ResponseWriter, r *http.Request, destUrl string, isWrite bool, responseFn func(proxyResponse *http.Response, w http.ResponseWriter) (statusCode int)) {
 
 	glog.V(3).Infof("s3 proxying %s to %s", r.Method, destUrl)
+	start := time.Now()
 
 	proxyReq, err := http.NewRequest(r.Method, destUrl, r.Body)
 
@@ -404,12 +405,12 @@ func (s3a *S3ApiServer) proxyToFiler(w http.ResponseWriter, r *http.Request, des
 			return
 		}
 	}
-
 	if resp.StatusCode == http.StatusNotFound {
 		s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
 		return
 	}
 
+	TimeToFirstByte(r.Method, start, r)
 	if resp.Header.Get(s3_constants.X_SeaweedFS_Header_Directory_Key) == "true" {
 		responseStatusCode := responseFn(resp, w)
 		s3err.PostLog(r, responseStatusCode, s3err.ErrNone)

weed/s3api/stats.go

@@ -43,3 +43,8 @@ func track(f http.HandlerFunc, action string) http.HandlerFunc {
 		stats_collect.S3RequestCounter.WithLabelValues(action, strconv.Itoa(recorder.Status), bucket).Inc()
 	}
 }
+
+func TimeToFirstByte(action string, start time.Time, r *http.Request) {
+	bucket, _ := s3_constants.GetBucketAndObject(r)
+	stats_collect.S3TimeToFirstByteHistogram.WithLabelValues(action, bucket).Observe(float64(time.Since(start).Milliseconds()))
+}

weed/stats/metrics.go

@@ -226,6 +226,15 @@ var (
 			Help:      "Bucketed histogram of s3 request processing time.",
 			Buckets:   prometheus.ExponentialBuckets(0.0001, 2, 24),
 		}, []string{"type", "bucket"})
+	S3TimeToFirstByteHistogram = prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Namespace: Namespace,
+			Subsystem: "s3",
+			Name:      "time_to_first_byte_millisecond",
+			Help:      "Bucketed histogram of s3 time to first byte request processing time.",
+			Buckets:   prometheus.ExponentialBuckets(0.001, 2, 27),
+		}, []string{"type", "bucket"})
+
 )
 
 func init() {
@@ -258,6 +267,7 @@ func init() {
 
 	Gather.MustRegister(S3RequestCounter)
 	Gather.MustRegister(S3RequestHistogram)
+	Gather.MustRegister(S3TimeToFirstByteHistogram)
 }
 
 func LoopPushingMetric(name, instance, addr string, intervalSeconds int) {