From afcf5d13b0322e0397fc578af85cf712c91b6c3f Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Fri, 23 Dec 2022 15:28:30 +0300 Subject: [PATCH 1/4] [TrueCloudLab#5] Request metrics per user Signed-off-by: Denis Kirillov --- api/handler/util.go | 4 + api/{metrics/api.go => metrics.go} | 172 +++++++++++++++++- .../collector.go => metrics_collector.go} | 2 +- api/reqinfo.go | 26 ++- api/router.go | 171 ++++++++++------- 5 files changed, 297 insertions(+), 78 deletions(-) rename api/{metrics/api.go => metrics.go} (53%) rename api/{metrics/collector.go => metrics_collector.go} (99%) diff --git a/api/handler/util.go b/api/handler/util.go index 4516cf8..61af738 100644 --- a/api/handler/util.go +++ b/api/handler/util.go @@ -42,6 +42,10 @@ func transformToS3Error(err error) error { return errors.GetAPIError(errors.ErrInternalError) } +func (h *handler) ResolveBucket(ctx context.Context, bucket string) (*data.BucketInfo, error) { + return h.obj.GetBucketInfo(ctx, bucket) +} + func (h *handler) getBucketAndCheckOwner(r *http.Request, bucket string, header ...string) (*data.BucketInfo, error) { bktInfo, err := h.obj.GetBucketInfo(r.Context(), bucket) if err != nil { diff --git a/api/metrics/api.go b/api/metrics.go similarity index 53% rename from api/metrics/api.go rename to api/metrics.go index 92a5801..a456947 100644 --- a/api/metrics/api.go +++ b/api/metrics.go @@ -1,4 +1,4 @@ -package metrics +package api import ( "io" @@ -8,9 +8,70 @@ import ( "sync/atomic" "time" + "github.com/TrueCloudLab/frostfs-s3-gw/creds/accessbox" + "github.com/TrueCloudLab/frostfs-sdk-go/bearer" "github.com/prometheus/client_golang/prometheus" ) +type RequestType int + +const ( + HEADRequest RequestType = iota + PUTRequest RequestType = iota + LISTRequest RequestType = iota + GETRequest RequestType = iota + DELETERequest RequestType = iota +) + +func (t RequestType) String() string { + switch t { + case 0: + return "HEAD" + case 1: + return "PUT" + case 2: + return "LIST" + case 3: + return "GET" + case 4: + return "DELETE" + default: + return "Unknown" + } +} + +func RequestTypeFromAPI(api string) RequestType { + switch api { + case "headobject", "headbucket": + return HEADRequest + case "createmultipartupload", "uploadpartcopy", "uploadpart", "completemutipartupload", + "putobjectacl", "putobjecttagging", "copyobject", "putobjectretention", "putobjectlegalhold", + "putobject", "putbucketcors", "putbucketacl", "putbucketlifecycle", "putbucketencryption", + "putbucketpolicy", "putbucketobjectlockconfig", "putbuckettagging", "putbucketversioning", + "putbucketnotification", "createbucket", "postobject": + return PUTRequest + case "listmultipartuploads", "listobjectsv2M", "listobjectsv2", "listbucketversions", + "listobjectsv1", "listbuckets": + return LISTRequest + case "getobjectacl", "getobjecttagging", "getobjectretention", "getobjectlegalhold", + "getobjectattributes", "getobject", "getbucketlocation", "getbucketpolicy", + "getbucketlifecycle", "getbucketencryption", "getbucketcors", "getbucketacl", + "getbucketwebsite", "getbucketaccelerate", "getbucketrequestpayment", "getbucketlogging", + "getbucketreplication", "getbuckettagging", "selectobjectcontent", + "getbucketobjectlockconfiguration", "getbucketversioning", "getbucketnotification", + "listenbucketnotification": + return GETRequest + case "abortmultipartupload", "deleteobjecttagging", "deleteobject", "deletebucketcors", + "deletebucketwebsite", "deletebuckettagging", "deletemultipleobjects", "deletebucketpolicy", + "deletebucketlifecycle", "deletebucketencryption", "deletebucket": + return DELETERequest + default: + return RequestType(-1) + } +} + +type OperationList [5]int + type ( // HTTPAPIStats holds statistics information about // the API given in the requests. @@ -19,9 +80,25 @@ type ( sync.RWMutex } + UsersAPIStats struct { + users map[string]*userAPIStats + sync.RWMutex + } + + bucketKey struct { + name string + cid string + } + + userAPIStats struct { + buckets map[bucketKey]OperationList + user string + } + // HTTPStats holds statistics information about // HTTP requests made by all clients. HTTPStats struct { + usersS3Requests UsersAPIStats currentS3Requests HTTPAPIStats totalS3Requests HTTPAPIStats totalS3Errors HTTPAPIStats @@ -101,6 +178,21 @@ func collectHTTPMetrics(ch chan<- prometheus.Metric) { api, ) } + + for _, value := range httpStatsMetric.usersS3Requests.DumpMetrics() { + ch <- prometheus.MustNewConstMetric( + prometheus.NewDesc( + prometheus.BuildFQName("frostfs_s3", "user_requests", "count"), + "", + []string{"user", "bucket", "cid", "operation"}, nil), + prometheus.CounterValue, + float64(value.Requests), + value.User, + value.Bucket, + value.ContainerID, + value.Operation, + ) + } } // APIStats wraps http handler for api with basic statistics collection. @@ -169,6 +261,66 @@ func (stats *HTTPAPIStats) Load() map[string]int { return apiStats } +func (u *UsersAPIStats) Update(user, bucket, cnrID string, reqType RequestType) { + u.Lock() + defer u.Unlock() + + usersStat := u.users[user] + if usersStat == nil { + if u.users == nil { + u.users = make(map[string]*userAPIStats) + } + usersStat = &userAPIStats{ + buckets: make(map[bucketKey]OperationList, 1), + user: user, + } + u.users[user] = usersStat + } + + key := bucketKey{ + name: bucket, + cid: cnrID, + } + + bucketStat := usersStat.buckets[key] + bucketStat[reqType] += 1 + usersStat.buckets[key] = bucketStat +} + +type UserMetricsInfo struct { + User string + Bucket string + ContainerID string + Operation string + Requests int +} + +func (u *UsersAPIStats) DumpMetrics() []UserMetricsInfo { + u.Lock() + defer u.Unlock() + + result := make([]UserMetricsInfo, 0, len(u.users)) + for user, userStat := range u.users { + for key, operations := range userStat.buckets { + for op, val := range operations { + if val != 0 { + result = append(result, UserMetricsInfo{ + User: user, + Bucket: key.name, + ContainerID: key.cid, + Operation: RequestType(op).String(), + Requests: val, + }) + } + } + } + } + + u.users = make(map[string]*userAPIStats) + + return result +} + func (st *HTTPStats) getInputBytes() uint64 { return atomic.LoadUint64(&st.totalInputBytes) } @@ -178,26 +330,36 @@ func (st *HTTPStats) getOutputBytes() uint64 { } // Update statistics from http request and response data. -func (st *HTTPStats) updateStats(api string, w http.ResponseWriter, r *http.Request, durationSecs float64) { +func (st *HTTPStats) updateStats(apiOperation string, w http.ResponseWriter, r *http.Request, durationSecs float64) { var code int if res, ok := w.(*responseWrapper); ok { code = res.statusCode } + user := "anon" + if bd, ok := r.Context().Value(BoxData).(*accessbox.Box); ok && bd != nil && bd.Gate != nil && bd.Gate.BearerToken != nil { + user = bearer.ResolveIssuer(*bd.Gate.BearerToken).String() + } + + reqInfo := GetReqInfo(r.Context()) + cnrID := GetCID(r.Context()) + + st.usersS3Requests.Update(user, reqInfo.BucketName, cnrID, RequestTypeFromAPI(apiOperation)) + // A successful request has a 2xx response code successReq := code >= http.StatusOK && code < http.StatusMultipleChoices if !strings.HasSuffix(r.URL.Path, systemPath) { - st.totalS3Requests.Inc(api) + st.totalS3Requests.Inc(apiOperation) if !successReq && code != 0 { - st.totalS3Errors.Inc(api) + st.totalS3Errors.Inc(apiOperation) } } if r.Method == http.MethodGet { // Increment the prometheus http request response histogram with appropriate label - httpRequestsDuration.With(prometheus.Labels{"api": api}).Observe(durationSecs) + httpRequestsDuration.With(prometheus.Labels{"api": apiOperation}).Observe(durationSecs) } } diff --git a/api/metrics/collector.go b/api/metrics_collector.go similarity index 99% rename from api/metrics/collector.go rename to api/metrics_collector.go index 5972e9a..2b9d31a 100644 --- a/api/metrics/collector.go +++ b/api/metrics_collector.go @@ -1,4 +1,4 @@ -package metrics +package api import ( "github.com/TrueCloudLab/frostfs-s3-gw/internal/version" diff --git a/api/reqinfo.go b/api/reqinfo.go index b57b20a..98de8f7 100644 --- a/api/reqinfo.go +++ b/api/reqinfo.go @@ -2,6 +2,7 @@ package api import ( "context" + cid "github.com/TrueCloudLab/frostfs-sdk-go/container/id" "net" "net/http" "net/url" @@ -42,10 +43,13 @@ type ( } ) -// Key used for Get/SetReqInfo. +// Key used for Get/Set context values. type contextKeyType string -const ctxRequestInfo = contextKeyType("FrostFS-S3-GW") +const ( + ctxRequestInfo = contextKeyType("FrostFS-S3-GW") + ctxCID = contextKeyType("FrostFS-S3-GW-CID") +) var ( // De-facto standard header keys. @@ -202,3 +206,21 @@ func GetReqInfo(ctx context.Context) *ReqInfo { } return &ReqInfo{} } + +// SetCID sets CID in the context. +func SetCID(ctx context.Context, id cid.ID) context.Context { + if ctx == nil { + return nil + } + return context.WithValue(ctx, ctxCID, id.EncodeToString()) +} + +// GetCID returns CID if set. +func GetCID(ctx context.Context) string { + if ctx == nil { + return "" + } else if id, ok := ctx.Value(ctxCID).(string); ok { + return id + } + return "" +} diff --git a/api/router.go b/api/router.go index 7e49d67..7df0af9 100644 --- a/api/router.go +++ b/api/router.go @@ -6,7 +6,7 @@ import ( "sync" "github.com/TrueCloudLab/frostfs-s3-gw/api/auth" - "github.com/TrueCloudLab/frostfs-s3-gw/api/metrics" + "github.com/TrueCloudLab/frostfs-s3-gw/api/data" "github.com/google/uuid" "github.com/gorilla/mux" "go.uber.org/zap" @@ -82,6 +82,8 @@ type ( AbortMultipartUploadHandler(http.ResponseWriter, *http.Request) ListPartsHandler(w http.ResponseWriter, r *http.Request) ListMultipartUploadsHandler(http.ResponseWriter, *http.Request) + + ResolveBucket(ctx context.Context, bucket string) (*data.BucketInfo, error) } // mimeType represents various MIME types used in API responses. @@ -145,6 +147,32 @@ func appendCORS(handler Handler) mux.MiddlewareFunc { } } +func resolveBucket(log *zap.Logger, resolveBucket func(ctx context.Context, bucket string) (*data.BucketInfo, error)) mux.MiddlewareFunc { + return func(h http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + reqInfo := GetReqInfo(r.Context()) + + if reqInfo.BucketName != "" { + bktInfo, err := resolveBucket(r.Context(), reqInfo.BucketName) + if err != nil { + code := WriteErrorResponse(w, reqInfo, err) + log.Error("failed to resolve bucket", zap.Int("status", code), + zap.String("request_id", reqInfo.RequestID), zap.String("method", reqInfo.API), + zap.String("bucket", reqInfo.BucketName), zap.String("object", reqInfo.ObjectName), + zap.Error(err)) + return + } + + // todo: (@KirillovDenis) consider save bktInfo into ReqInfo + // (in order to optimize resolving bucket in further handlers) + r = r.WithContext(SetCID(r.Context(), bktInfo.CID)) + } + + h.ServeHTTP(w, r) + }) + } +} + func logErrorResponse(l *zap.Logger) mux.MiddlewareFunc { return func(h http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -192,6 +220,9 @@ func Attach(r *mux.Router, domains []string, m MaxClients, h Handler, center aut // -- prepare request setRequestID, + // -- resolve bucket to set cid in context + resolveBucket(log, h.ResolveBucket), + // -- logging error requests logErrorResponse(log), ) @@ -213,277 +244,277 @@ func Attach(r *mux.Router, domains []string, m MaxClients, h Handler, center aut // -- append CORS headers to a response for appendCORS(h), ) - bucket.Methods(http.MethodOptions).HandlerFunc(m.Handle(metrics.APIStats("preflight", h.Preflight))).Name("Options") + bucket.Methods(http.MethodOptions).HandlerFunc(m.Handle(APIStats("preflight", h.Preflight))).Name("Options") bucket.Methods(http.MethodHead).Path("/{object:.+}").HandlerFunc( - m.Handle(metrics.APIStats("headobject", h.HeadObjectHandler))).Name("HeadObject") + m.Handle(APIStats("headobject", h.HeadObjectHandler))).Name("HeadObject") // CopyObjectPart - bucket.Methods(http.MethodPut).Path("/{object:.+}").Headers(hdrAmzCopySource, "").HandlerFunc(m.Handle(metrics.APIStats("uploadpartcopy", h.UploadPartCopy))).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}"). + bucket.Methods(http.MethodPut).Path("/{object:.+}").Headers(hdrAmzCopySource, "").HandlerFunc(m.Handle(APIStats("uploadpartcopy", h.UploadPartCopy))).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}"). Name("UploadPartCopy") // PutObjectPart bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc( - m.Handle(metrics.APIStats("uploadpart", h.UploadPartHandler))).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}"). + m.Handle(APIStats("uploadpart", h.UploadPartHandler))).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}"). Name("UploadPart") // ListParts bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc( - m.Handle(metrics.APIStats("listobjectparts", h.ListPartsHandler))).Queries("uploadId", "{uploadId:.*}"). + m.Handle(APIStats("listobjectparts", h.ListPartsHandler))).Queries("uploadId", "{uploadId:.*}"). Name("ListObjectParts") // CompleteMultipartUpload bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc( - m.Handle(metrics.APIStats("completemutipartupload", h.CompleteMultipartUploadHandler))).Queries("uploadId", "{uploadId:.*}"). + m.Handle(APIStats("completemutipartupload", h.CompleteMultipartUploadHandler))).Queries("uploadId", "{uploadId:.*}"). Name("CompleteMultipartUpload") // CreateMultipartUpload bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc( - m.Handle(metrics.APIStats("createmultipartupload", h.CreateMultipartUploadHandler))).Queries("uploads", ""). + m.Handle(APIStats("createmultipartupload", h.CreateMultipartUploadHandler))).Queries("uploads", ""). Name("CreateMultipartUpload") // AbortMultipartUpload bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc( - m.Handle(metrics.APIStats("abortmultipartupload", h.AbortMultipartUploadHandler))).Queries("uploadId", "{uploadId:.*}"). + m.Handle(APIStats("abortmultipartupload", h.AbortMultipartUploadHandler))).Queries("uploadId", "{uploadId:.*}"). Name("AbortMultipartUpload") // ListMultipartUploads bucket.Methods(http.MethodGet).HandlerFunc( - m.Handle(metrics.APIStats("listmultipartuploads", h.ListMultipartUploadsHandler))).Queries("uploads", ""). + m.Handle(APIStats("listmultipartuploads", h.ListMultipartUploadsHandler))).Queries("uploads", ""). Name("ListMultipartUploads") // GetObjectACL -- this is a dummy call. bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc( - m.Handle(metrics.APIStats("getobjectacl", h.GetObjectACLHandler))).Queries("acl", ""). + m.Handle(APIStats("getobjectacl", h.GetObjectACLHandler))).Queries("acl", ""). Name("GetObjectACL") // PutObjectACL -- this is a dummy call. bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc( - m.Handle(metrics.APIStats("putobjectacl", h.PutObjectACLHandler))).Queries("acl", ""). + m.Handle(APIStats("putobjectacl", h.PutObjectACLHandler))).Queries("acl", ""). Name("PutObjectACL") // GetObjectTagging bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc( - m.Handle(metrics.APIStats("getobjecttagging", h.GetObjectTaggingHandler))).Queries("tagging", ""). + m.Handle(APIStats("getobjecttagging", h.GetObjectTaggingHandler))).Queries("tagging", ""). Name("GetObjectTagging") // PutObjectTagging bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc( - m.Handle(metrics.APIStats("putobjecttagging", h.PutObjectTaggingHandler))).Queries("tagging", ""). + m.Handle(APIStats("putobjecttagging", h.PutObjectTaggingHandler))).Queries("tagging", ""). Name("PutObjectTagging") // DeleteObjectTagging bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc( - m.Handle(metrics.APIStats("deleteobjecttagging", h.DeleteObjectTaggingHandler))).Queries("tagging", ""). + m.Handle(APIStats("deleteobjecttagging", h.DeleteObjectTaggingHandler))).Queries("tagging", ""). Name("DeleteObjectTagging") // SelectObjectContent bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc( - m.Handle(metrics.APIStats("selectobjectcontent", h.SelectObjectContentHandler))).Queries("select", "").Queries("select-type", "2"). + m.Handle(APIStats("selectobjectcontent", h.SelectObjectContentHandler))).Queries("select", "").Queries("select-type", "2"). Name("SelectObjectContent") // GetObjectRetention bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc( - m.Handle(metrics.APIStats("getobjectretention", h.GetObjectRetentionHandler))).Queries("retention", ""). + m.Handle(APIStats("getobjectretention", h.GetObjectRetentionHandler))).Queries("retention", ""). Name("GetObjectRetention") // GetObjectLegalHold bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc( - m.Handle(metrics.APIStats("getobjectlegalhold", h.GetObjectLegalHoldHandler))).Queries("legal-hold", ""). + m.Handle(APIStats("getobjectlegalhold", h.GetObjectLegalHoldHandler))).Queries("legal-hold", ""). Name("GetObjectLegalHold") // GetObjectAttributes bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc( - m.Handle(metrics.APIStats("getobjectattributes", h.GetObjectAttributesHandler))).Queries("attributes", ""). + m.Handle(APIStats("getobjectattributes", h.GetObjectAttributesHandler))).Queries("attributes", ""). Name("GetObjectAttributes") // GetObject bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc( - m.Handle(metrics.APIStats("getobject", h.GetObjectHandler))). + m.Handle(APIStats("getobject", h.GetObjectHandler))). Name("GetObject") // CopyObject - bucket.Methods(http.MethodPut).Path("/{object:.+}").Headers(hdrAmzCopySource, "").HandlerFunc(m.Handle(metrics.APIStats("copyobject", h.CopyObjectHandler))). + bucket.Methods(http.MethodPut).Path("/{object:.+}").Headers(hdrAmzCopySource, "").HandlerFunc(m.Handle(APIStats("copyobject", h.CopyObjectHandler))). Name("CopyObject") // PutObjectRetention bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc( - m.Handle(metrics.APIStats("putobjectretention", h.PutObjectRetentionHandler))).Queries("retention", ""). + m.Handle(APIStats("putobjectretention", h.PutObjectRetentionHandler))).Queries("retention", ""). Name("PutObjectRetention") // PutObjectLegalHold bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc( - m.Handle(metrics.APIStats("putobjectlegalhold", h.PutObjectLegalHoldHandler))).Queries("legal-hold", ""). + m.Handle(APIStats("putobjectlegalhold", h.PutObjectLegalHoldHandler))).Queries("legal-hold", ""). Name("PutObjectLegalHold") // PutObject bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc( - m.Handle(metrics.APIStats("putobject", h.PutObjectHandler))). + m.Handle(APIStats("putobject", h.PutObjectHandler))). Name("PutObject") // DeleteObject bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc( - m.Handle(metrics.APIStats("deleteobject", h.DeleteObjectHandler))). + m.Handle(APIStats("deleteobject", h.DeleteObjectHandler))). Name("DeleteObject") // Bucket operations // GetBucketLocation bucket.Methods(http.MethodGet).HandlerFunc( - m.Handle(metrics.APIStats("getbucketlocation", h.GetBucketLocationHandler))).Queries("location", ""). + m.Handle(APIStats("getbucketlocation", h.GetBucketLocationHandler))).Queries("location", ""). Name("GetBucketLocation") // GetBucketPolicy bucket.Methods(http.MethodGet).HandlerFunc( - m.Handle(metrics.APIStats("getbucketpolicy", h.GetBucketPolicyHandler))).Queries("policy", ""). + m.Handle(APIStats("getbucketpolicy", h.GetBucketPolicyHandler))).Queries("policy", ""). Name("GetBucketPolicy") // GetBucketLifecycle bucket.Methods(http.MethodGet).HandlerFunc( - m.Handle(metrics.APIStats("getbucketlifecycle", h.GetBucketLifecycleHandler))).Queries("lifecycle", ""). + m.Handle(APIStats("getbucketlifecycle", h.GetBucketLifecycleHandler))).Queries("lifecycle", ""). Name("GetBucketLifecycle") // GetBucketEncryption bucket.Methods(http.MethodGet).HandlerFunc( - m.Handle(metrics.APIStats("getbucketencryption", h.GetBucketEncryptionHandler))).Queries("encryption", ""). + m.Handle(APIStats("getbucketencryption", h.GetBucketEncryptionHandler))).Queries("encryption", ""). Name("GetBucketEncryption") bucket.Methods(http.MethodGet).HandlerFunc( - m.Handle(metrics.APIStats("getbucketcors", h.GetBucketCorsHandler))).Queries("cors", ""). + m.Handle(APIStats("getbucketcors", h.GetBucketCorsHandler))).Queries("cors", ""). Name("GetBucketCors") bucket.Methods(http.MethodPut).HandlerFunc( - m.Handle(metrics.APIStats("putbucketcors", h.PutBucketCorsHandler))).Queries("cors", ""). + m.Handle(APIStats("putbucketcors", h.PutBucketCorsHandler))).Queries("cors", ""). Name("PutBucketCors") bucket.Methods(http.MethodDelete).HandlerFunc( - m.Handle(metrics.APIStats("deletebucketcors", h.DeleteBucketCorsHandler))).Queries("cors", ""). + m.Handle(APIStats("deletebucketcors", h.DeleteBucketCorsHandler))).Queries("cors", ""). Name("DeleteBucketCors") // Dummy Bucket Calls // GetBucketACL -- this is a dummy call. bucket.Methods(http.MethodGet).HandlerFunc( - m.Handle(metrics.APIStats("getbucketacl", h.GetBucketACLHandler))).Queries("acl", ""). + m.Handle(APIStats("getbucketacl", h.GetBucketACLHandler))).Queries("acl", ""). Name("GetBucketACL") // PutBucketACL -- this is a dummy call. bucket.Methods(http.MethodPut).HandlerFunc( - m.Handle(metrics.APIStats("putbucketacl", h.PutBucketACLHandler))).Queries("acl", ""). + m.Handle(APIStats("putbucketacl", h.PutBucketACLHandler))).Queries("acl", ""). Name("PutBucketACL") // GetBucketWebsiteHandler -- this is a dummy call. bucket.Methods(http.MethodGet).HandlerFunc( - m.Handle(metrics.APIStats("getbucketwebsite", h.GetBucketWebsiteHandler))).Queries("website", ""). + m.Handle(APIStats("getbucketwebsite", h.GetBucketWebsiteHandler))).Queries("website", ""). Name("GetBucketWebsite") // GetBucketAccelerateHandler -- this is a dummy call. bucket.Methods(http.MethodGet).HandlerFunc( - m.Handle(metrics.APIStats("getbucketaccelerate", h.GetBucketAccelerateHandler))).Queries("accelerate", ""). + m.Handle(APIStats("getbucketaccelerate", h.GetBucketAccelerateHandler))).Queries("accelerate", ""). Name("GetBucketAccelerate") // GetBucketRequestPaymentHandler -- this is a dummy call. bucket.Methods(http.MethodGet).HandlerFunc( - m.Handle(metrics.APIStats("getbucketrequestpayment", h.GetBucketRequestPaymentHandler))).Queries("requestPayment", ""). + m.Handle(APIStats("getbucketrequestpayment", h.GetBucketRequestPaymentHandler))).Queries("requestPayment", ""). Name("GetBucketRequestPayment") // GetBucketLoggingHandler -- this is a dummy call. bucket.Methods(http.MethodGet).HandlerFunc( - m.Handle(metrics.APIStats("getbucketlogging", h.GetBucketLoggingHandler))).Queries("logging", ""). + m.Handle(APIStats("getbucketlogging", h.GetBucketLoggingHandler))).Queries("logging", ""). Name("GetBucketLogging") // GetBucketLifecycleHandler -- this is a dummy call. bucket.Methods(http.MethodGet).HandlerFunc( - m.Handle(metrics.APIStats("getbucketlifecycle", h.GetBucketLifecycleHandler))).Queries("lifecycle", ""). + m.Handle(APIStats("getbucketlifecycle", h.GetBucketLifecycleHandler))).Queries("lifecycle", ""). Name("GetBucketLifecycle") // GetBucketReplicationHandler -- this is a dummy call. bucket.Methods(http.MethodGet).HandlerFunc( - m.Handle(metrics.APIStats("getbucketreplication", h.GetBucketReplicationHandler))).Queries("replication", ""). + m.Handle(APIStats("getbucketreplication", h.GetBucketReplicationHandler))).Queries("replication", ""). Name("GetBucketReplication") // GetBucketTaggingHandler bucket.Methods(http.MethodGet).HandlerFunc( - m.Handle(metrics.APIStats("getbuckettagging", h.GetBucketTaggingHandler))).Queries("tagging", ""). + m.Handle(APIStats("getbuckettagging", h.GetBucketTaggingHandler))).Queries("tagging", ""). Name("GetBucketTagging") // DeleteBucketWebsiteHandler bucket.Methods(http.MethodDelete).HandlerFunc( - m.Handle(metrics.APIStats("deletebucketwebsite", h.DeleteBucketWebsiteHandler))).Queries("website", ""). + m.Handle(APIStats("deletebucketwebsite", h.DeleteBucketWebsiteHandler))).Queries("website", ""). Name("DeleteBucketWebsite") // DeleteBucketTaggingHandler bucket.Methods(http.MethodDelete).HandlerFunc( - m.Handle(metrics.APIStats("deletebuckettagging", h.DeleteBucketTaggingHandler))).Queries("tagging", ""). + m.Handle(APIStats("deletebuckettagging", h.DeleteBucketTaggingHandler))).Queries("tagging", ""). Name("DeleteBucketTagging") // GetBucketObjectLockConfig bucket.Methods(http.MethodGet).HandlerFunc( - m.Handle(metrics.APIStats("getbucketobjectlockconfiguration", h.GetBucketObjectLockConfigHandler))).Queries("object-lock", ""). + m.Handle(APIStats("getbucketobjectlockconfiguration", h.GetBucketObjectLockConfigHandler))).Queries("object-lock", ""). Name("GetBucketObjectLockConfig") // GetBucketVersioning bucket.Methods(http.MethodGet).HandlerFunc( - m.Handle(metrics.APIStats("getbucketversioning", h.GetBucketVersioningHandler))).Queries("versioning", ""). + m.Handle(APIStats("getbucketversioning", h.GetBucketVersioningHandler))).Queries("versioning", ""). Name("GetBucketVersioning") // GetBucketNotification bucket.Methods(http.MethodGet).HandlerFunc( - m.Handle(metrics.APIStats("getbucketnotification", h.GetBucketNotificationHandler))).Queries("notification", ""). + m.Handle(APIStats("getbucketnotification", h.GetBucketNotificationHandler))).Queries("notification", ""). Name("GetBucketNotification") // ListenBucketNotification - bucket.Methods(http.MethodGet).HandlerFunc(metrics.APIStats("listenbucketnotification", h.ListenBucketNotificationHandler)).Queries("events", "{events:.*}"). + bucket.Methods(http.MethodGet).HandlerFunc(APIStats("listenbucketnotification", h.ListenBucketNotificationHandler)).Queries("events", "{events:.*}"). Name("ListenBucketNotification") // ListObjectsV2M bucket.Methods(http.MethodGet).HandlerFunc( - m.Handle(metrics.APIStats("listobjectsv2M", h.ListObjectsV2MHandler))).Queries("list-type", "2", "metadata", "true"). + m.Handle(APIStats("listobjectsv2M", h.ListObjectsV2MHandler))).Queries("list-type", "2", "metadata", "true"). Name("ListObjectsV2M") // ListObjectsV2 bucket.Methods(http.MethodGet).HandlerFunc( - m.Handle(metrics.APIStats("listobjectsv2", h.ListObjectsV2Handler))).Queries("list-type", "2"). + m.Handle(APIStats("listobjectsv2", h.ListObjectsV2Handler))).Queries("list-type", "2"). Name("ListObjectsV2") // ListBucketVersions bucket.Methods(http.MethodGet).HandlerFunc( - m.Handle(metrics.APIStats("listbucketversions", h.ListBucketObjectVersionsHandler))).Queries("versions", ""). + m.Handle(APIStats("listbucketversions", h.ListBucketObjectVersionsHandler))).Queries("versions", ""). Name("ListBucketVersions") // ListObjectsV1 (Legacy) bucket.Methods(http.MethodGet).HandlerFunc( - m.Handle(metrics.APIStats("listobjectsv1", h.ListObjectsV1Handler))). + m.Handle(APIStats("listobjectsv1", h.ListObjectsV1Handler))). Name("ListObjectsV1") // PutBucketLifecycle bucket.Methods(http.MethodPut).HandlerFunc( - m.Handle(metrics.APIStats("putbucketlifecycle", h.PutBucketLifecycleHandler))).Queries("lifecycle", ""). + m.Handle(APIStats("putbucketlifecycle", h.PutBucketLifecycleHandler))).Queries("lifecycle", ""). Name("PutBucketLifecycle") // PutBucketEncryption bucket.Methods(http.MethodPut).HandlerFunc( - m.Handle(metrics.APIStats("putbucketencryption", h.PutBucketEncryptionHandler))).Queries("encryption", ""). + m.Handle(APIStats("putbucketencryption", h.PutBucketEncryptionHandler))).Queries("encryption", ""). Name("PutBucketEncryption") // PutBucketPolicy bucket.Methods(http.MethodPut).HandlerFunc( - m.Handle(metrics.APIStats("putbucketpolicy", h.PutBucketPolicyHandler))).Queries("policy", ""). + m.Handle(APIStats("putbucketpolicy", h.PutBucketPolicyHandler))).Queries("policy", ""). Name("PutBucketPolicy") // PutBucketObjectLockConfig bucket.Methods(http.MethodPut).HandlerFunc( - m.Handle(metrics.APIStats("putbucketobjectlockconfig", h.PutBucketObjectLockConfigHandler))).Queries("object-lock", ""). + m.Handle(APIStats("putbucketobjectlockconfig", h.PutBucketObjectLockConfigHandler))).Queries("object-lock", ""). Name("PutBucketObjectLockConfig") // PutBucketTaggingHandler bucket.Methods(http.MethodPut).HandlerFunc( - m.Handle(metrics.APIStats("putbuckettagging", h.PutBucketTaggingHandler))).Queries("tagging", ""). + m.Handle(APIStats("putbuckettagging", h.PutBucketTaggingHandler))).Queries("tagging", ""). Name("PutBucketTagging") // PutBucketVersioning bucket.Methods(http.MethodPut).HandlerFunc( - m.Handle(metrics.APIStats("putbucketversioning", h.PutBucketVersioningHandler))).Queries("versioning", ""). + m.Handle(APIStats("putbucketversioning", h.PutBucketVersioningHandler))).Queries("versioning", ""). Name("PutBucketVersioning") // PutBucketNotification bucket.Methods(http.MethodPut).HandlerFunc( - m.Handle(metrics.APIStats("putbucketnotification", h.PutBucketNotificationHandler))).Queries("notification", ""). + m.Handle(APIStats("putbucketnotification", h.PutBucketNotificationHandler))).Queries("notification", ""). Name("PutBucketNotification") // CreateBucket bucket.Methods(http.MethodPut).HandlerFunc( - m.Handle(metrics.APIStats("createbucket", h.CreateBucketHandler))). + m.Handle(APIStats("createbucket", h.CreateBucketHandler))). Name("CreateBucket") // HeadBucket bucket.Methods(http.MethodHead).HandlerFunc( - m.Handle(metrics.APIStats("headbucket", h.HeadBucketHandler))). + m.Handle(APIStats("headbucket", h.HeadBucketHandler))). Name("HeadBucket") // PostPolicy bucket.Methods(http.MethodPost).HeadersRegexp(hdrContentType, "multipart/form-data*").HandlerFunc( - m.Handle(metrics.APIStats("postobject", h.PostObject))). + m.Handle(APIStats("postobject", h.PostObject))). Name("PostObject") // DeleteMultipleObjects bucket.Methods(http.MethodPost).HandlerFunc( - m.Handle(metrics.APIStats("deletemultipleobjects", h.DeleteMultipleObjectsHandler))).Queries("delete", ""). + m.Handle(APIStats("deletemultipleobjects", h.DeleteMultipleObjectsHandler))).Queries("delete", ""). Name("DeleteMultipleObjects") // DeleteBucketPolicy bucket.Methods(http.MethodDelete).HandlerFunc( - m.Handle(metrics.APIStats("deletebucketpolicy", h.DeleteBucketPolicyHandler))).Queries("policy", ""). + m.Handle(APIStats("deletebucketpolicy", h.DeleteBucketPolicyHandler))).Queries("policy", ""). Name("DeleteBucketPolicy") // DeleteBucketLifecycle bucket.Methods(http.MethodDelete).HandlerFunc( - m.Handle(metrics.APIStats("deletebucketlifecycle", h.DeleteBucketLifecycleHandler))).Queries("lifecycle", ""). + m.Handle(APIStats("deletebucketlifecycle", h.DeleteBucketLifecycleHandler))).Queries("lifecycle", ""). Name("DeleteBucketLifecycle") // DeleteBucketEncryption bucket.Methods(http.MethodDelete).HandlerFunc( - m.Handle(metrics.APIStats("deletebucketencryption", h.DeleteBucketEncryptionHandler))).Queries("encryption", ""). + m.Handle(APIStats("deletebucketencryption", h.DeleteBucketEncryptionHandler))).Queries("encryption", ""). Name("DeleteBucketEncryption") // DeleteBucket bucket.Methods(http.MethodDelete).HandlerFunc( - m.Handle(metrics.APIStats("deletebucket", h.DeleteBucketHandler))). + m.Handle(APIStats("deletebucket", h.DeleteBucketHandler))). Name("DeleteBucket") } // Root operation // ListBuckets api.Methods(http.MethodGet).Path(SlashSeparator).HandlerFunc( - m.Handle(metrics.APIStats("listbuckets", h.ListBucketsHandler))). + m.Handle(APIStats("listbuckets", h.ListBucketsHandler))). Name("ListBuckets") // S3 browser with signature v4 adds '//' for ListBuckets request, so rather // than failing with UnknownAPIRequest we simply handle it for now. api.Methods(http.MethodGet).Path(SlashSeparator + SlashSeparator).HandlerFunc( - m.Handle(metrics.APIStats("listbuckets", h.ListBucketsHandler))). + m.Handle(APIStats("listbuckets", h.ListBucketsHandler))). Name("ListBuckets") // If none of the routes match, add default error handler routes - api.NotFoundHandler = metrics.APIStats("notfound", errorResponseHandler) - api.MethodNotAllowedHandler = metrics.APIStats("methodnotallowed", errorResponseHandler) + api.NotFoundHandler = APIStats("notfound", errorResponseHandler) + api.MethodNotAllowedHandler = APIStats("methodnotallowed", errorResponseHandler) } -- 2.45.2 From ab948e6fea125f3ec08121a8da41dae00e050b0b Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Mon, 26 Dec 2022 16:13:24 +0300 Subject: [PATCH 2/4] [TrueCloudLab#5] Add traffic metrics per user Signed-off-by: Denis Kirillov --- api/metrics.go | 143 +++++++++++++++++++++++++++++++-------- api/metrics_collector.go | 1 + api/router.go | 2 +- 3 files changed, 116 insertions(+), 30 deletions(-) diff --git a/api/metrics.go b/api/metrics.go index a456947..e6c1f4c 100644 --- a/api/metrics.go +++ b/api/metrics.go @@ -13,6 +13,25 @@ import ( "github.com/prometheus/client_golang/prometheus" ) +type TrafficType int + +const ( + UnknownTraffic TrafficType = iota + INTraffic TrafficType = iota + OUTTraffic TrafficType = iota +) + +func (t TrafficType) String() string { + switch t { + case 1: + return "IN" + case 2: + return "OUT" + default: + return "Unknown" + } +} + type RequestType int const ( @@ -90,11 +109,40 @@ type ( cid string } + bucketStat struct { + Operations OperationList + InTraffic uint64 + OutTraffic uint64 + } + userAPIStats struct { - buckets map[bucketKey]OperationList + buckets map[bucketKey]bucketStat user string } + UserBucketInfo struct { + User string + Bucket string + ContainerID string + } + + UserMetricsInfo struct { + UserBucketInfo + Operation RequestType + Requests int + } + + UserTrafficMetricsInfo struct { + UserBucketInfo + Type TrafficType + Value uint64 + } + + UserMetrics struct { + Requests []UserMetricsInfo + Traffic []UserTrafficMetricsInfo + } + // HTTPStats holds statistics information about // HTTP requests made by all clients. HTTPStats struct { @@ -178,8 +226,12 @@ func collectHTTPMetrics(ch chan<- prometheus.Metric) { api, ) } +} - for _, value := range httpStatsMetric.usersS3Requests.DumpMetrics() { +func collectUserMetrics(ch chan<- prometheus.Metric) { + userMetrics := httpStatsMetric.usersS3Requests.DumpMetrics() + + for _, value := range userMetrics.Requests { ch <- prometheus.MustNewConstMetric( prometheus.NewDesc( prometheus.BuildFQName("frostfs_s3", "user_requests", "count"), @@ -190,7 +242,22 @@ func collectHTTPMetrics(ch chan<- prometheus.Metric) { value.User, value.Bucket, value.ContainerID, - value.Operation, + value.Operation.String(), + ) + } + + for _, value := range userMetrics.Traffic { + ch <- prometheus.MustNewConstMetric( + prometheus.NewDesc( + prometheus.BuildFQName("frostfs_s3", "user_traffic", "bytes"), + "", + []string{"user", "bucket", "cid", "type"}, nil), + prometheus.CounterValue, + float64(value.Value), + value.User, + value.Bucket, + value.ContainerID, + value.Type.String(), ) } } @@ -218,7 +285,7 @@ func APIStats(api string, f http.HandlerFunc) http.HandlerFunc { // simply for the fact that it is not human readable. durationSecs := time.Since(statsWriter.startTime).Seconds() - httpStatsMetric.updateStats(api, statsWriter, r, durationSecs) + httpStatsMetric.updateStats(api, statsWriter, r, durationSecs, in.countBytes, out.countBytes) atomic.AddUint64(&httpStatsMetric.totalInputBytes, in.countBytes) atomic.AddUint64(&httpStatsMetric.totalOutputBytes, out.countBytes) @@ -261,7 +328,7 @@ func (stats *HTTPAPIStats) Load() map[string]int { return apiStats } -func (u *UsersAPIStats) Update(user, bucket, cnrID string, reqType RequestType) { +func (u *UsersAPIStats) Update(user, bucket, cnrID string, reqType RequestType, in, out uint64) { u.Lock() defer u.Unlock() @@ -271,7 +338,7 @@ func (u *UsersAPIStats) Update(user, bucket, cnrID string, reqType RequestType) u.users = make(map[string]*userAPIStats) } usersStat = &userAPIStats{ - buckets: make(map[bucketKey]OperationList, 1), + buckets: make(map[bucketKey]bucketStat, 1), user: user, } u.users[user] = usersStat @@ -282,34 +349,52 @@ func (u *UsersAPIStats) Update(user, bucket, cnrID string, reqType RequestType) cid: cnrID, } - bucketStat := usersStat.buckets[key] - bucketStat[reqType] += 1 - usersStat.buckets[key] = bucketStat + bktStat := usersStat.buckets[key] + bktStat.Operations[reqType] += 1 + bktStat.InTraffic += in + bktStat.OutTraffic += out + usersStat.buckets[key] = bktStat } -type UserMetricsInfo struct { - User string - Bucket string - ContainerID string - Operation string - Requests int -} - -func (u *UsersAPIStats) DumpMetrics() []UserMetricsInfo { +func (u *UsersAPIStats) DumpMetrics() UserMetrics { u.Lock() defer u.Unlock() - result := make([]UserMetricsInfo, 0, len(u.users)) + result := UserMetrics{ + Requests: make([]UserMetricsInfo, 0, len(u.users)), + Traffic: make([]UserTrafficMetricsInfo, 0, len(u.users)), + } + for user, userStat := range u.users { - for key, operations := range userStat.buckets { - for op, val := range operations { + for key, bktStat := range userStat.buckets { + userBktInfo := UserBucketInfo{ + User: user, + Bucket: key.name, + ContainerID: key.cid, + } + + if bktStat.InTraffic != 0 { + result.Traffic = append(result.Traffic, UserTrafficMetricsInfo{ + UserBucketInfo: userBktInfo, + Type: INTraffic, + Value: bktStat.InTraffic, + }) + } + + if bktStat.OutTraffic != 0 { + result.Traffic = append(result.Traffic, UserTrafficMetricsInfo{ + UserBucketInfo: userBktInfo, + Type: OUTTraffic, + Value: bktStat.OutTraffic, + }) + } + + for op, val := range bktStat.Operations { if val != 0 { - result = append(result, UserMetricsInfo{ - User: user, - Bucket: key.name, - ContainerID: key.cid, - Operation: RequestType(op).String(), - Requests: val, + result.Requests = append(result.Requests, UserMetricsInfo{ + UserBucketInfo: userBktInfo, + Operation: RequestType(op), + Requests: val, }) } } @@ -330,7 +415,7 @@ func (st *HTTPStats) getOutputBytes() uint64 { } // Update statistics from http request and response data. -func (st *HTTPStats) updateStats(apiOperation string, w http.ResponseWriter, r *http.Request, durationSecs float64) { +func (st *HTTPStats) updateStats(apiOperation string, w http.ResponseWriter, r *http.Request, durationSecs float64, in, out uint64) { var code int if res, ok := w.(*responseWrapper); ok { @@ -345,7 +430,7 @@ func (st *HTTPStats) updateStats(apiOperation string, w http.ResponseWriter, r * reqInfo := GetReqInfo(r.Context()) cnrID := GetCID(r.Context()) - st.usersS3Requests.Update(user, reqInfo.BucketName, cnrID, RequestTypeFromAPI(apiOperation)) + st.usersS3Requests.Update(user, reqInfo.BucketName, cnrID, RequestTypeFromAPI(apiOperation), in, out) // A successful request has a 2xx response code successReq := code >= http.StatusOK && code < http.StatusMultipleChoices diff --git a/api/metrics_collector.go b/api/metrics_collector.go index 2b9d31a..eac7c5b 100644 --- a/api/metrics_collector.go +++ b/api/metrics_collector.go @@ -64,5 +64,6 @@ func (s *stats) Collect(ch chan<- prometheus.Metric) { // connect collectors collectHTTPMetrics(ch) + collectUserMetrics(ch) collectNetworkMetrics(ch) } diff --git a/api/router.go b/api/router.go index 7df0af9..173604f 100644 --- a/api/router.go +++ b/api/router.go @@ -152,7 +152,7 @@ func resolveBucket(log *zap.Logger, resolveBucket func(ctx context.Context, buck return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { reqInfo := GetReqInfo(r.Context()) - if reqInfo.BucketName != "" { + if reqInfo.BucketName != "" && reqInfo.API != "CreateBucket" { bktInfo, err := resolveBucket(r.Context(), reqInfo.BucketName) if err != nil { code := WriteErrorResponse(w, reqInfo, err) -- 2.45.2 From b24d3bd8ca8662743bd269348b1f8212cb7682c4 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Tue, 27 Dec 2022 15:30:11 +0300 Subject: [PATCH 3/4] [TrueCloudLab#5] Refactor middlewares Signed-off-by: Denis Kirillov --- api/metrics.go | 141 ++++++++++++----------- api/reqinfo.go | 26 +---- api/router.go | 285 ++++++++++++++++++++++++++++++----------------- api/user_auth.go | 8 +- 4 files changed, 257 insertions(+), 203 deletions(-) diff --git a/api/metrics.go b/api/metrics.go index e6c1f4c..eee2fe0 100644 --- a/api/metrics.go +++ b/api/metrics.go @@ -1,6 +1,7 @@ package api import ( + "context" "io" "net/http" "strings" @@ -35,24 +36,25 @@ func (t TrafficType) String() string { type RequestType int const ( - HEADRequest RequestType = iota - PUTRequest RequestType = iota - LISTRequest RequestType = iota - GETRequest RequestType = iota - DELETERequest RequestType = iota + UNKNOWNRequest RequestType = iota + HEADRequest RequestType = iota + PUTRequest RequestType = iota + LISTRequest RequestType = iota + GETRequest RequestType = iota + DELETERequest RequestType = iota ) func (t RequestType) String() string { switch t { - case 0: - return "HEAD" case 1: - return "PUT" + return "HEAD" case 2: - return "LIST" + return "PUT" case 3: - return "GET" + return "LIST" case 4: + return "GET" + case 5: return "DELETE" default: return "Unknown" @@ -61,35 +63,34 @@ func (t RequestType) String() string { func RequestTypeFromAPI(api string) RequestType { switch api { - case "headobject", "headbucket": + case "Options", "HeadObject", "HeadBucket": return HEADRequest - case "createmultipartupload", "uploadpartcopy", "uploadpart", "completemutipartupload", - "putobjectacl", "putobjecttagging", "copyobject", "putobjectretention", "putobjectlegalhold", - "putobject", "putbucketcors", "putbucketacl", "putbucketlifecycle", "putbucketencryption", - "putbucketpolicy", "putbucketobjectlockconfig", "putbuckettagging", "putbucketversioning", - "putbucketnotification", "createbucket", "postobject": + case "CreateMultipartUpload", "UploadPartCopy", "UploadPart", "CompleteMultipartUpload", + "PutObjectACL", "PutObjectTagging", "CopyObject", "PutObjectRetention", "PutObjectLegalHold", + "PutObject", "PutBucketCors", "PutBucketACL", "PutBucketLifecycle", "PutBucketEncryption", + "PutBucketPolicy", "PutBucketObjectLockConfig", "PutBucketTagging", "PutBucketVersioning", + "PutBucketNotification", "CreateBucket", "PostObject": return PUTRequest - case "listmultipartuploads", "listobjectsv2M", "listobjectsv2", "listbucketversions", - "listobjectsv1", "listbuckets": + case "ListObjectParts", "ListMultipartUploads", "ListObjectsV2M", "ListObjectsV2", "ListBucketVersions", + "ListObjectsV1", "ListBuckets": return LISTRequest - case "getobjectacl", "getobjecttagging", "getobjectretention", "getobjectlegalhold", - "getobjectattributes", "getobject", "getbucketlocation", "getbucketpolicy", - "getbucketlifecycle", "getbucketencryption", "getbucketcors", "getbucketacl", - "getbucketwebsite", "getbucketaccelerate", "getbucketrequestpayment", "getbucketlogging", - "getbucketreplication", "getbuckettagging", "selectobjectcontent", - "getbucketobjectlockconfiguration", "getbucketversioning", "getbucketnotification", - "listenbucketnotification": + case "GetObjectACL", "GetObjectTagging", "SelectObjectContent", "GetObjectRetention", "getobjectlegalhold", + "GetObjectAttributes", "GetObject", "GetBucketLocation", "GetBucketPolicy", + "GetBucketLifecycle", "GetBucketEncryption", "GetBucketCors", "GetBucketACL", + "GetBucketWebsite", "GetBucketAccelerate", "GetBucketRequestPayment", "GetBucketLogging", + "GetBucketReplication", "GetBucketTagging", "GetBucketObjectLockConfig", + "GetBucketVersioning", "GetBucketNotification", "ListenBucketNotification": return GETRequest - case "abortmultipartupload", "deleteobjecttagging", "deleteobject", "deletebucketcors", - "deletebucketwebsite", "deletebuckettagging", "deletemultipleobjects", "deletebucketpolicy", - "deletebucketlifecycle", "deletebucketencryption", "deletebucket": + case "AbortMultipartUpload", "DeleteObjectTagging", "DeleteObject", "DeleteBucketCors", + "DeleteBucketWebsite", "DeleteBucketTagging", "DeleteMultipleObjects", "DeleteBucketPolicy", + "DeleteBucketLifecycle", "DeleteBucketEncryption", "DeleteBucket": return DELETERequest default: - return RequestType(-1) + return UNKNOWNRequest } } -type OperationList [5]int +type OperationList [6]int type ( // HTTPAPIStats holds statistics information about @@ -262,11 +263,16 @@ func collectUserMetrics(ch chan<- prometheus.Metric) { } } -// APIStats wraps http handler for api with basic statistics collection. -func APIStats(api string, f http.HandlerFunc) http.HandlerFunc { +// CIDResolveFunc is a func to resolve CID in Stats handler. +type CIDResolveFunc func(ctx context.Context, reqInfo *ReqInfo) (cnrID string) + +// Stats is a handler that update metrics. +func Stats(f http.HandlerFunc, resolveCID CIDResolveFunc) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - httpStatsMetric.currentS3Requests.Inc(api) - defer httpStatsMetric.currentS3Requests.Dec(api) + reqInfo := GetReqInfo(r.Context()) + + httpStatsMetric.currentS3Requests.Inc(reqInfo.API) + defer httpStatsMetric.currentS3Requests.Dec(reqInfo.API) in := &readCounter{ReadCloser: r.Body} out := &writeCounter{ResponseWriter: w} @@ -278,20 +284,45 @@ func APIStats(api string, f http.HandlerFunc) http.HandlerFunc { startTime: time.Now(), } - f.ServeHTTP(statsWriter, r) + f(statsWriter, r) // Time duration in secs since the call started. // We don't need to do nanosecond precision here - // simply for the fact that it is not human readable. + // simply for the fact that it is not human-readable. durationSecs := time.Since(statsWriter.startTime).Seconds() - httpStatsMetric.updateStats(api, statsWriter, r, durationSecs, in.countBytes, out.countBytes) + user := resolveUser(r.Context()) + cnrID := resolveCID(r.Context(), reqInfo) + httpStatsMetric.usersS3Requests.Update(user, reqInfo.BucketName, cnrID, RequestTypeFromAPI(reqInfo.API), in.countBytes, out.countBytes) + + code := statsWriter.statusCode + // A successful request has a 2xx response code + successReq := code >= http.StatusOK && code < http.StatusMultipleChoices + if !strings.HasSuffix(r.URL.Path, systemPath) { + httpStatsMetric.totalS3Requests.Inc(reqInfo.API) + if !successReq && code != 0 { + httpStatsMetric.totalS3Errors.Inc(reqInfo.API) + } + } + + if r.Method == http.MethodGet { + // Increment the prometheus http request response histogram with appropriate label + httpRequestsDuration.With(prometheus.Labels{"api": reqInfo.API}).Observe(durationSecs) + } atomic.AddUint64(&httpStatsMetric.totalInputBytes, in.countBytes) atomic.AddUint64(&httpStatsMetric.totalOutputBytes, out.countBytes) } } +func resolveUser(ctx context.Context) string { + user := "anon" + if bd, ok := ctx.Value(BoxData).(*accessbox.Box); ok && bd != nil && bd.Gate != nil && bd.Gate.BearerToken != nil { + user = bearer.ResolveIssuer(*bd.Gate.BearerToken).String() + } + return user +} + // Inc increments the api stats counter. func (stats *HTTPAPIStats) Inc(api string) { if stats == nil { @@ -350,7 +381,7 @@ func (u *UsersAPIStats) Update(user, bucket, cnrID string, reqType RequestType, } bktStat := usersStat.buckets[key] - bktStat.Operations[reqType] += 1 + bktStat.Operations[reqType]++ bktStat.InTraffic += in bktStat.OutTraffic += out usersStat.buckets[key] = bktStat @@ -414,40 +445,6 @@ func (st *HTTPStats) getOutputBytes() uint64 { return atomic.LoadUint64(&st.totalOutputBytes) } -// Update statistics from http request and response data. -func (st *HTTPStats) updateStats(apiOperation string, w http.ResponseWriter, r *http.Request, durationSecs float64, in, out uint64) { - var code int - - if res, ok := w.(*responseWrapper); ok { - code = res.statusCode - } - - user := "anon" - if bd, ok := r.Context().Value(BoxData).(*accessbox.Box); ok && bd != nil && bd.Gate != nil && bd.Gate.BearerToken != nil { - user = bearer.ResolveIssuer(*bd.Gate.BearerToken).String() - } - - reqInfo := GetReqInfo(r.Context()) - cnrID := GetCID(r.Context()) - - st.usersS3Requests.Update(user, reqInfo.BucketName, cnrID, RequestTypeFromAPI(apiOperation), in, out) - - // A successful request has a 2xx response code - successReq := code >= http.StatusOK && code < http.StatusMultipleChoices - - if !strings.HasSuffix(r.URL.Path, systemPath) { - st.totalS3Requests.Inc(apiOperation) - if !successReq && code != 0 { - st.totalS3Errors.Inc(apiOperation) - } - } - - if r.Method == http.MethodGet { - // Increment the prometheus http request response histogram with appropriate label - httpRequestsDuration.With(prometheus.Labels{"api": apiOperation}).Observe(durationSecs) - } -} - // WriteHeader -- writes http status code. func (w *responseWrapper) WriteHeader(code int) { w.Do(func() { diff --git a/api/reqinfo.go b/api/reqinfo.go index 98de8f7..b57b20a 100644 --- a/api/reqinfo.go +++ b/api/reqinfo.go @@ -2,7 +2,6 @@ package api import ( "context" - cid "github.com/TrueCloudLab/frostfs-sdk-go/container/id" "net" "net/http" "net/url" @@ -43,13 +42,10 @@ type ( } ) -// Key used for Get/Set context values. +// Key used for Get/SetReqInfo. type contextKeyType string -const ( - ctxRequestInfo = contextKeyType("FrostFS-S3-GW") - ctxCID = contextKeyType("FrostFS-S3-GW-CID") -) +const ctxRequestInfo = contextKeyType("FrostFS-S3-GW") var ( // De-facto standard header keys. @@ -206,21 +202,3 @@ func GetReqInfo(ctx context.Context) *ReqInfo { } return &ReqInfo{} } - -// SetCID sets CID in the context. -func SetCID(ctx context.Context, id cid.ID) context.Context { - if ctx == nil { - return nil - } - return context.WithValue(ctx, ctxCID, id.EncodeToString()) -} - -// GetCID returns CID if set. -func GetCID(ctx context.Context) string { - if ctx == nil { - return "" - } else if id, ok := ctx.Value(ctxCID).(string); ok { - return id - } - return "" -} diff --git a/api/router.go b/api/router.go index 173604f..793755f 100644 --- a/api/router.go +++ b/api/router.go @@ -108,7 +108,7 @@ const ( MimeXML mimeType = "application/xml" ) -var _ = logErrorResponse +var _ = logSuccessResponse func (lrw *logResponseWriter) WriteHeader(code int) { lrw.Do(func() { @@ -147,33 +147,37 @@ func appendCORS(handler Handler) mux.MiddlewareFunc { } } -func resolveBucket(log *zap.Logger, resolveBucket func(ctx context.Context, bucket string) (*data.BucketInfo, error)) mux.MiddlewareFunc { +// BucketResolveFunc is a func to resolve bucket info by name. +type BucketResolveFunc func(ctx context.Context, bucket string) (*data.BucketInfo, error) + +// metricsMiddleware wraps http handler for api with basic statistics collection. +func metricsMiddleware(log *zap.Logger, resolveBucket BucketResolveFunc) mux.MiddlewareFunc { return func(h http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - reqInfo := GetReqInfo(r.Context()) - - if reqInfo.BucketName != "" && reqInfo.API != "CreateBucket" { - bktInfo, err := resolveBucket(r.Context(), reqInfo.BucketName) - if err != nil { - code := WriteErrorResponse(w, reqInfo, err) - log.Error("failed to resolve bucket", zap.Int("status", code), - zap.String("request_id", reqInfo.RequestID), zap.String("method", reqInfo.API), - zap.String("bucket", reqInfo.BucketName), zap.String("object", reqInfo.ObjectName), - zap.Error(err)) - return - } - - // todo: (@KirillovDenis) consider save bktInfo into ReqInfo - // (in order to optimize resolving bucket in further handlers) - r = r.WithContext(SetCID(r.Context(), bktInfo.CID)) - } - - h.ServeHTTP(w, r) - }) + return Stats(h.ServeHTTP, resolveCID(log, resolveBucket)) } } -func logErrorResponse(l *zap.Logger) mux.MiddlewareFunc { +// resolveCID forms CIDResolveFunc using BucketResolveFunc. +func resolveCID(log *zap.Logger, resolveBucket BucketResolveFunc) CIDResolveFunc { + return func(ctx context.Context, reqInfo *ReqInfo) (cnrID string) { + if reqInfo.BucketName == "" || reqInfo.API == "CreateBucket" || reqInfo.API == "" { + return "" + } + + bktInfo, err := resolveBucket(ctx, reqInfo.BucketName) + if err != nil { + log.Debug("failed to resolve CID", + zap.String("request_id", reqInfo.RequestID), zap.String("method", reqInfo.API), + zap.String("bucket", reqInfo.BucketName), zap.String("object", reqInfo.ObjectName), + zap.Error(err)) + return "" + } + + return bktInfo.CID.EncodeToString() + } +} + +func logSuccessResponse(l *zap.Logger) mux.MiddlewareFunc { return func(h http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { lw := &logResponseWriter{ResponseWriter: w} @@ -211,6 +215,30 @@ func GetRequestID(v interface{}) string { } } +func setErrorAPI(apiName string, h http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ctx := SetReqInfo(r.Context(), &ReqInfo{API: apiName}) + h.ServeHTTP(w, r.WithContext(ctx)) + }) +} + +// attachErrorHandler set NotFoundHandler and MethodNotAllowedHandler for mux.Router. +func attachErrorHandler(api *mux.Router, log *zap.Logger, h Handler, center auth.Center) { + middlewares := []mux.MiddlewareFunc{ + AuthMiddleware(log, center), + metricsMiddleware(log, h.ResolveBucket), + } + + var errorHandler http.Handler = http.HandlerFunc(errorResponseHandler) + for i := len(middlewares) - 1; i >= 0; i-- { + errorHandler = middlewares[i](errorHandler) + } + + // If none of the routes match, add default error handler routes + api.NotFoundHandler = setErrorAPI("NotFound", errorHandler) + api.MethodNotAllowedHandler = setErrorAPI("MethodNotAllowed", errorHandler) +} + // Attach adds S3 API handlers from h to r for domains with m client limit using // center authentication and log logger. func Attach(r *mux.Router, domains []string, m MaxClients, h Handler, center auth.Center, log *zap.Logger) { @@ -220,15 +248,16 @@ func Attach(r *mux.Router, domains []string, m MaxClients, h Handler, center aut // -- prepare request setRequestID, - // -- resolve bucket to set cid in context - resolveBucket(log, h.ResolveBucket), + // Attach user authentication for all S3 routes. + AuthMiddleware(log, center), + + metricsMiddleware(log, h.ResolveBucket), // -- logging error requests - logErrorResponse(log), + logSuccessResponse(log), ) - // Attach user authentication for all S3 routes. - AttachUserAuth(api, center, log) + attachErrorHandler(api, log, h, center) buckets := make([]*mux.Router, 0, len(domains)+1) buckets = append(buckets, api.PathPrefix("/{bucket}").Subrouter()) @@ -244,277 +273,327 @@ func Attach(r *mux.Router, domains []string, m MaxClients, h Handler, center aut // -- append CORS headers to a response for appendCORS(h), ) - bucket.Methods(http.MethodOptions).HandlerFunc(m.Handle(APIStats("preflight", h.Preflight))).Name("Options") + bucket.Methods(http.MethodOptions).HandlerFunc( + m.Handle(h.Preflight)). + Name("Options") bucket.Methods(http.MethodHead).Path("/{object:.+}").HandlerFunc( - m.Handle(APIStats("headobject", h.HeadObjectHandler))).Name("HeadObject") + m.Handle(h.HeadObjectHandler)). + Name("HeadObject") // CopyObjectPart - bucket.Methods(http.MethodPut).Path("/{object:.+}").Headers(hdrAmzCopySource, "").HandlerFunc(m.Handle(APIStats("uploadpartcopy", h.UploadPartCopy))).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}"). + bucket.Methods(http.MethodPut).Path("/{object:.+}").Headers(hdrAmzCopySource, "").HandlerFunc( + m.Handle(h.UploadPartCopy)). + Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}"). Name("UploadPartCopy") // PutObjectPart bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc( - m.Handle(APIStats("uploadpart", h.UploadPartHandler))).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}"). + m.Handle(h.UploadPartHandler)). + Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}"). Name("UploadPart") // ListParts bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc( - m.Handle(APIStats("listobjectparts", h.ListPartsHandler))).Queries("uploadId", "{uploadId:.*}"). + m.Handle(h.ListPartsHandler)). + Queries("uploadId", "{uploadId:.*}"). Name("ListObjectParts") // CompleteMultipartUpload bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc( - m.Handle(APIStats("completemutipartupload", h.CompleteMultipartUploadHandler))).Queries("uploadId", "{uploadId:.*}"). + m.Handle(h.CompleteMultipartUploadHandler)). + Queries("uploadId", "{uploadId:.*}"). Name("CompleteMultipartUpload") // CreateMultipartUpload bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc( - m.Handle(APIStats("createmultipartupload", h.CreateMultipartUploadHandler))).Queries("uploads", ""). + m.Handle(h.CreateMultipartUploadHandler)). + Queries("uploads", ""). Name("CreateMultipartUpload") // AbortMultipartUpload bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc( - m.Handle(APIStats("abortmultipartupload", h.AbortMultipartUploadHandler))).Queries("uploadId", "{uploadId:.*}"). + m.Handle(h.AbortMultipartUploadHandler)). + Queries("uploadId", "{uploadId:.*}"). Name("AbortMultipartUpload") // ListMultipartUploads bucket.Methods(http.MethodGet).HandlerFunc( - m.Handle(APIStats("listmultipartuploads", h.ListMultipartUploadsHandler))).Queries("uploads", ""). + m.Handle(h.ListMultipartUploadsHandler)). + Queries("uploads", ""). Name("ListMultipartUploads") // GetObjectACL -- this is a dummy call. bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc( - m.Handle(APIStats("getobjectacl", h.GetObjectACLHandler))).Queries("acl", ""). + m.Handle(h.GetObjectACLHandler)). + Queries("acl", ""). Name("GetObjectACL") // PutObjectACL -- this is a dummy call. bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc( - m.Handle(APIStats("putobjectacl", h.PutObjectACLHandler))).Queries("acl", ""). + m.Handle(h.PutObjectACLHandler)). + Queries("acl", ""). Name("PutObjectACL") // GetObjectTagging bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc( - m.Handle(APIStats("getobjecttagging", h.GetObjectTaggingHandler))).Queries("tagging", ""). + m.Handle(h.GetObjectTaggingHandler)). + Queries("tagging", ""). Name("GetObjectTagging") // PutObjectTagging bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc( - m.Handle(APIStats("putobjecttagging", h.PutObjectTaggingHandler))).Queries("tagging", ""). + m.Handle(h.PutObjectTaggingHandler)). + Queries("tagging", ""). Name("PutObjectTagging") // DeleteObjectTagging bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc( - m.Handle(APIStats("deleteobjecttagging", h.DeleteObjectTaggingHandler))).Queries("tagging", ""). + m.Handle(h.DeleteObjectTaggingHandler)). + Queries("tagging", ""). Name("DeleteObjectTagging") // SelectObjectContent bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc( - m.Handle(APIStats("selectobjectcontent", h.SelectObjectContentHandler))).Queries("select", "").Queries("select-type", "2"). + m.Handle(h.SelectObjectContentHandler)). + Queries("select", "").Queries("select-type", "2"). Name("SelectObjectContent") // GetObjectRetention bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc( - m.Handle(APIStats("getobjectretention", h.GetObjectRetentionHandler))).Queries("retention", ""). + m.Handle(h.GetObjectRetentionHandler)). + Queries("retention", ""). Name("GetObjectRetention") // GetObjectLegalHold bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc( - m.Handle(APIStats("getobjectlegalhold", h.GetObjectLegalHoldHandler))).Queries("legal-hold", ""). + m.Handle(h.GetObjectLegalHoldHandler)). + Queries("legal-hold", ""). Name("GetObjectLegalHold") // GetObjectAttributes bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc( - m.Handle(APIStats("getobjectattributes", h.GetObjectAttributesHandler))).Queries("attributes", ""). + m.Handle(h.GetObjectAttributesHandler)). + Queries("attributes", ""). Name("GetObjectAttributes") // GetObject bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc( - m.Handle(APIStats("getobject", h.GetObjectHandler))). + m.Handle(h.GetObjectHandler)). Name("GetObject") // CopyObject - bucket.Methods(http.MethodPut).Path("/{object:.+}").Headers(hdrAmzCopySource, "").HandlerFunc(m.Handle(APIStats("copyobject", h.CopyObjectHandler))). + bucket.Methods(http.MethodPut).Path("/{object:.+}").Headers(hdrAmzCopySource, "").HandlerFunc( + m.Handle(h.CopyObjectHandler)). Name("CopyObject") // PutObjectRetention bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc( - m.Handle(APIStats("putobjectretention", h.PutObjectRetentionHandler))).Queries("retention", ""). + m.Handle(h.PutObjectRetentionHandler)). + Queries("retention", ""). Name("PutObjectRetention") // PutObjectLegalHold bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc( - m.Handle(APIStats("putobjectlegalhold", h.PutObjectLegalHoldHandler))).Queries("legal-hold", ""). + m.Handle(h.PutObjectLegalHoldHandler)). + Queries("legal-hold", ""). Name("PutObjectLegalHold") // PutObject bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc( - m.Handle(APIStats("putobject", h.PutObjectHandler))). + m.Handle(h.PutObjectHandler)). Name("PutObject") // DeleteObject bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc( - m.Handle(APIStats("deleteobject", h.DeleteObjectHandler))). + m.Handle(h.DeleteObjectHandler)). Name("DeleteObject") // Bucket operations // GetBucketLocation bucket.Methods(http.MethodGet).HandlerFunc( - m.Handle(APIStats("getbucketlocation", h.GetBucketLocationHandler))).Queries("location", ""). + m.Handle(h.GetBucketLocationHandler)). + Queries("location", ""). Name("GetBucketLocation") // GetBucketPolicy bucket.Methods(http.MethodGet).HandlerFunc( - m.Handle(APIStats("getbucketpolicy", h.GetBucketPolicyHandler))).Queries("policy", ""). + m.Handle(h.GetBucketPolicyHandler)). + Queries("policy", ""). Name("GetBucketPolicy") // GetBucketLifecycle bucket.Methods(http.MethodGet).HandlerFunc( - m.Handle(APIStats("getbucketlifecycle", h.GetBucketLifecycleHandler))).Queries("lifecycle", ""). + m.Handle(h.GetBucketLifecycleHandler)). + Queries("lifecycle", ""). Name("GetBucketLifecycle") // GetBucketEncryption bucket.Methods(http.MethodGet).HandlerFunc( - m.Handle(APIStats("getbucketencryption", h.GetBucketEncryptionHandler))).Queries("encryption", ""). + m.Handle(h.GetBucketEncryptionHandler)). + Queries("encryption", ""). Name("GetBucketEncryption") bucket.Methods(http.MethodGet).HandlerFunc( - m.Handle(APIStats("getbucketcors", h.GetBucketCorsHandler))).Queries("cors", ""). + m.Handle(h.GetBucketCorsHandler)). + Queries("cors", ""). Name("GetBucketCors") bucket.Methods(http.MethodPut).HandlerFunc( - m.Handle(APIStats("putbucketcors", h.PutBucketCorsHandler))).Queries("cors", ""). + m.Handle(h.PutBucketCorsHandler)). + Queries("cors", ""). Name("PutBucketCors") bucket.Methods(http.MethodDelete).HandlerFunc( - m.Handle(APIStats("deletebucketcors", h.DeleteBucketCorsHandler))).Queries("cors", ""). + m.Handle(h.DeleteBucketCorsHandler)). + Queries("cors", ""). Name("DeleteBucketCors") // Dummy Bucket Calls // GetBucketACL -- this is a dummy call. bucket.Methods(http.MethodGet).HandlerFunc( - m.Handle(APIStats("getbucketacl", h.GetBucketACLHandler))).Queries("acl", ""). + m.Handle(h.GetBucketACLHandler)). + Queries("acl", ""). Name("GetBucketACL") // PutBucketACL -- this is a dummy call. bucket.Methods(http.MethodPut).HandlerFunc( - m.Handle(APIStats("putbucketacl", h.PutBucketACLHandler))).Queries("acl", ""). + m.Handle(h.PutBucketACLHandler)). + Queries("acl", ""). Name("PutBucketACL") // GetBucketWebsiteHandler -- this is a dummy call. bucket.Methods(http.MethodGet).HandlerFunc( - m.Handle(APIStats("getbucketwebsite", h.GetBucketWebsiteHandler))).Queries("website", ""). + m.Handle(h.GetBucketWebsiteHandler)). + Queries("website", ""). Name("GetBucketWebsite") // GetBucketAccelerateHandler -- this is a dummy call. bucket.Methods(http.MethodGet).HandlerFunc( - m.Handle(APIStats("getbucketaccelerate", h.GetBucketAccelerateHandler))).Queries("accelerate", ""). + m.Handle(h.GetBucketAccelerateHandler)). + Queries("accelerate", ""). Name("GetBucketAccelerate") // GetBucketRequestPaymentHandler -- this is a dummy call. bucket.Methods(http.MethodGet).HandlerFunc( - m.Handle(APIStats("getbucketrequestpayment", h.GetBucketRequestPaymentHandler))).Queries("requestPayment", ""). + m.Handle(h.GetBucketRequestPaymentHandler)). + Queries("requestPayment", ""). Name("GetBucketRequestPayment") // GetBucketLoggingHandler -- this is a dummy call. bucket.Methods(http.MethodGet).HandlerFunc( - m.Handle(APIStats("getbucketlogging", h.GetBucketLoggingHandler))).Queries("logging", ""). + m.Handle(h.GetBucketLoggingHandler)). + Queries("logging", ""). Name("GetBucketLogging") - // GetBucketLifecycleHandler -- this is a dummy call. - bucket.Methods(http.MethodGet).HandlerFunc( - m.Handle(APIStats("getbucketlifecycle", h.GetBucketLifecycleHandler))).Queries("lifecycle", ""). - Name("GetBucketLifecycle") // GetBucketReplicationHandler -- this is a dummy call. bucket.Methods(http.MethodGet).HandlerFunc( - m.Handle(APIStats("getbucketreplication", h.GetBucketReplicationHandler))).Queries("replication", ""). + m.Handle(h.GetBucketReplicationHandler)). + Queries("replication", ""). Name("GetBucketReplication") // GetBucketTaggingHandler bucket.Methods(http.MethodGet).HandlerFunc( - m.Handle(APIStats("getbuckettagging", h.GetBucketTaggingHandler))).Queries("tagging", ""). + m.Handle(h.GetBucketTaggingHandler)). + Queries("tagging", ""). Name("GetBucketTagging") // DeleteBucketWebsiteHandler bucket.Methods(http.MethodDelete).HandlerFunc( - m.Handle(APIStats("deletebucketwebsite", h.DeleteBucketWebsiteHandler))).Queries("website", ""). + m.Handle(h.DeleteBucketWebsiteHandler)). + Queries("website", ""). Name("DeleteBucketWebsite") // DeleteBucketTaggingHandler bucket.Methods(http.MethodDelete).HandlerFunc( - m.Handle(APIStats("deletebuckettagging", h.DeleteBucketTaggingHandler))).Queries("tagging", ""). + m.Handle(h.DeleteBucketTaggingHandler)). + Queries("tagging", ""). Name("DeleteBucketTagging") // GetBucketObjectLockConfig bucket.Methods(http.MethodGet).HandlerFunc( - m.Handle(APIStats("getbucketobjectlockconfiguration", h.GetBucketObjectLockConfigHandler))).Queries("object-lock", ""). + m.Handle(h.GetBucketObjectLockConfigHandler)). + Queries("object-lock", ""). Name("GetBucketObjectLockConfig") // GetBucketVersioning bucket.Methods(http.MethodGet).HandlerFunc( - m.Handle(APIStats("getbucketversioning", h.GetBucketVersioningHandler))).Queries("versioning", ""). + m.Handle(h.GetBucketVersioningHandler)). + Queries("versioning", ""). Name("GetBucketVersioning") // GetBucketNotification bucket.Methods(http.MethodGet).HandlerFunc( - m.Handle(APIStats("getbucketnotification", h.GetBucketNotificationHandler))).Queries("notification", ""). + m.Handle(h.GetBucketNotificationHandler)). + Queries("notification", ""). Name("GetBucketNotification") // ListenBucketNotification - bucket.Methods(http.MethodGet).HandlerFunc(APIStats("listenbucketnotification", h.ListenBucketNotificationHandler)).Queries("events", "{events:.*}"). + bucket.Methods(http.MethodGet).HandlerFunc(h.ListenBucketNotificationHandler). + Queries("events", "{events:.*}"). Name("ListenBucketNotification") // ListObjectsV2M bucket.Methods(http.MethodGet).HandlerFunc( - m.Handle(APIStats("listobjectsv2M", h.ListObjectsV2MHandler))).Queries("list-type", "2", "metadata", "true"). + m.Handle(h.ListObjectsV2MHandler)). + Queries("list-type", "2", "metadata", "true"). Name("ListObjectsV2M") // ListObjectsV2 bucket.Methods(http.MethodGet).HandlerFunc( - m.Handle(APIStats("listobjectsv2", h.ListObjectsV2Handler))).Queries("list-type", "2"). + m.Handle(h.ListObjectsV2Handler)). + Queries("list-type", "2"). Name("ListObjectsV2") // ListBucketVersions bucket.Methods(http.MethodGet).HandlerFunc( - m.Handle(APIStats("listbucketversions", h.ListBucketObjectVersionsHandler))).Queries("versions", ""). + m.Handle(h.ListBucketObjectVersionsHandler)). + Queries("versions", ""). Name("ListBucketVersions") // ListObjectsV1 (Legacy) bucket.Methods(http.MethodGet).HandlerFunc( - m.Handle(APIStats("listobjectsv1", h.ListObjectsV1Handler))). + m.Handle(h.ListObjectsV1Handler)). Name("ListObjectsV1") // PutBucketLifecycle bucket.Methods(http.MethodPut).HandlerFunc( - m.Handle(APIStats("putbucketlifecycle", h.PutBucketLifecycleHandler))).Queries("lifecycle", ""). + m.Handle(h.PutBucketLifecycleHandler)). + Queries("lifecycle", ""). Name("PutBucketLifecycle") // PutBucketEncryption bucket.Methods(http.MethodPut).HandlerFunc( - m.Handle(APIStats("putbucketencryption", h.PutBucketEncryptionHandler))).Queries("encryption", ""). + m.Handle(h.PutBucketEncryptionHandler)). + Queries("encryption", ""). Name("PutBucketEncryption") // PutBucketPolicy bucket.Methods(http.MethodPut).HandlerFunc( - m.Handle(APIStats("putbucketpolicy", h.PutBucketPolicyHandler))).Queries("policy", ""). + m.Handle(h.PutBucketPolicyHandler)). + Queries("policy", ""). Name("PutBucketPolicy") // PutBucketObjectLockConfig bucket.Methods(http.MethodPut).HandlerFunc( - m.Handle(APIStats("putbucketobjectlockconfig", h.PutBucketObjectLockConfigHandler))).Queries("object-lock", ""). + m.Handle(h.PutBucketObjectLockConfigHandler)). + Queries("object-lock", ""). Name("PutBucketObjectLockConfig") // PutBucketTaggingHandler bucket.Methods(http.MethodPut).HandlerFunc( - m.Handle(APIStats("putbuckettagging", h.PutBucketTaggingHandler))).Queries("tagging", ""). + m.Handle(h.PutBucketTaggingHandler)). + Queries("tagging", ""). Name("PutBucketTagging") // PutBucketVersioning bucket.Methods(http.MethodPut).HandlerFunc( - m.Handle(APIStats("putbucketversioning", h.PutBucketVersioningHandler))).Queries("versioning", ""). + m.Handle(h.PutBucketVersioningHandler)). + Queries("versioning", ""). Name("PutBucketVersioning") // PutBucketNotification bucket.Methods(http.MethodPut).HandlerFunc( - m.Handle(APIStats("putbucketnotification", h.PutBucketNotificationHandler))).Queries("notification", ""). + m.Handle(h.PutBucketNotificationHandler)). + Queries("notification", ""). Name("PutBucketNotification") // CreateBucket bucket.Methods(http.MethodPut).HandlerFunc( - m.Handle(APIStats("createbucket", h.CreateBucketHandler))). + m.Handle(h.CreateBucketHandler)). Name("CreateBucket") // HeadBucket bucket.Methods(http.MethodHead).HandlerFunc( - m.Handle(APIStats("headbucket", h.HeadBucketHandler))). + m.Handle(h.HeadBucketHandler)). Name("HeadBucket") // PostPolicy bucket.Methods(http.MethodPost).HeadersRegexp(hdrContentType, "multipart/form-data*").HandlerFunc( - m.Handle(APIStats("postobject", h.PostObject))). + m.Handle(h.PostObject)). Name("PostObject") // DeleteMultipleObjects bucket.Methods(http.MethodPost).HandlerFunc( - m.Handle(APIStats("deletemultipleobjects", h.DeleteMultipleObjectsHandler))).Queries("delete", ""). + m.Handle(h.DeleteMultipleObjectsHandler)). + Queries("delete", ""). Name("DeleteMultipleObjects") // DeleteBucketPolicy bucket.Methods(http.MethodDelete).HandlerFunc( - m.Handle(APIStats("deletebucketpolicy", h.DeleteBucketPolicyHandler))).Queries("policy", ""). + m.Handle(h.DeleteBucketPolicyHandler)). + Queries("policy", ""). Name("DeleteBucketPolicy") // DeleteBucketLifecycle bucket.Methods(http.MethodDelete).HandlerFunc( - m.Handle(APIStats("deletebucketlifecycle", h.DeleteBucketLifecycleHandler))).Queries("lifecycle", ""). + m.Handle(h.DeleteBucketLifecycleHandler)). + Queries("lifecycle", ""). Name("DeleteBucketLifecycle") // DeleteBucketEncryption bucket.Methods(http.MethodDelete).HandlerFunc( - m.Handle(APIStats("deletebucketencryption", h.DeleteBucketEncryptionHandler))).Queries("encryption", ""). + m.Handle(h.DeleteBucketEncryptionHandler)). + Queries("encryption", ""). Name("DeleteBucketEncryption") // DeleteBucket bucket.Methods(http.MethodDelete).HandlerFunc( - m.Handle(APIStats("deletebucket", h.DeleteBucketHandler))). + m.Handle(h.DeleteBucketHandler)). Name("DeleteBucket") } // Root operation // ListBuckets api.Methods(http.MethodGet).Path(SlashSeparator).HandlerFunc( - m.Handle(APIStats("listbuckets", h.ListBucketsHandler))). + m.Handle(h.ListBucketsHandler)). Name("ListBuckets") // S3 browser with signature v4 adds '//' for ListBuckets request, so rather // than failing with UnknownAPIRequest we simply handle it for now. api.Methods(http.MethodGet).Path(SlashSeparator + SlashSeparator).HandlerFunc( - m.Handle(APIStats("listbuckets", h.ListBucketsHandler))). + m.Handle(h.ListBucketsHandler)). Name("ListBuckets") - - // If none of the routes match, add default error handler routes - api.NotFoundHandler = APIStats("notfound", errorResponseHandler) - api.MethodNotAllowedHandler = APIStats("methodnotallowed", errorResponseHandler) } diff --git a/api/user_auth.go b/api/user_auth.go index 911621e..938b469 100644 --- a/api/user_auth.go +++ b/api/user_auth.go @@ -19,9 +19,9 @@ var BoxData = KeyWrapper("__context_box_key") // ClientTime is an ID used to store client time.Time in a context. var ClientTime = KeyWrapper("__context_client_time") -// AttachUserAuth adds user authentication via center to router using log for logging. -func AttachUserAuth(router *mux.Router, center auth.Center, log *zap.Logger) { - router.Use(func(h http.Handler) http.Handler { +// AuthMiddleware adds user authentication via center to router using log for logging. +func AuthMiddleware(log *zap.Logger, center auth.Center) mux.MiddlewareFunc { + return func(h http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { var ctx context.Context box, err := center.Authenticate(r) @@ -46,5 +46,5 @@ func AttachUserAuth(router *mux.Router, center auth.Center, log *zap.Logger) { h.ServeHTTP(w, r.WithContext(ctx)) }) - }) + } } -- 2.45.2 From 875f8546aa63c6bfe16f2b52db514a61388048ad Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Wed, 25 Jan 2023 10:37:14 +0300 Subject: [PATCH 4/4] [TrueCloudLab#5] Update CHANGELOG.md Signed-off-by: Denis Kirillov --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0fe46c9..3f389a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,9 @@ This document outlines major changes between releases. ## [Unreleased] +### Added +- Billing metrics (TrueCloudLab#5) + ### Changed - Update neo-go to v0.101.0 (#14) - Update viper to v1.15.0 (#14) -- 2.45.2