From 0eb12ae9962a361567b191201b2c65aa08526f27 Mon Sep 17 00:00:00 2001 From: Roman Loginov Date: Thu, 28 Nov 2024 05:48:14 +0300 Subject: [PATCH 1/4] [#145] handler: Add spans to detail the trace Signed-off-by: Roman Loginov --- internal/handler/download.go | 21 ++++++++++++++++++++- internal/handler/head.go | 13 +++++++++++++ internal/handler/upload.go | 5 ++++- 3 files changed, 37 insertions(+), 2 deletions(-) diff --git a/internal/handler/download.go b/internal/handler/download.go index cd4e55a..f324460 100644 --- a/internal/handler/download.go +++ b/internal/handler/download.go @@ -13,6 +13,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/response" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -28,10 +29,22 @@ func (h *Handler) DownloadByAddressOrBucketName(c *fasthttp.RequestCtx) { switch { case isObjectID(oidURLParam): + ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(c), "handler.DownloadByNativeAddress") + defer span.End() + + utils.SetContextToRequest(ctx, c) h.byNativeAddress(c, h.receiveFile) case !isContainerRoot(oidURLParam) && (downloadQueryParam || !isDir(oidURLParam)): + ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(c), "handler.DownloadByS3Path") + defer span.End() + + utils.SetContextToRequest(ctx, c) h.byS3Path(c, h.receiveFile) default: + ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(c), "handler.BrowseIndex") + defer span.End() + + utils.SetContextToRequest(ctx, c) h.browseIndex(c) } } @@ -45,6 +58,10 @@ func (h *Handler) newRequest(ctx *fasthttp.RequestCtx, log *zap.Logger) *request // DownloadByAttribute handles attribute-based download requests. func (h *Handler) DownloadByAttribute(c *fasthttp.RequestCtx) { + ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(c), "handler.DownloadByAttribute") + defer span.End() + + utils.SetContextToRequest(ctx, c) h.byAttribute(c, h.receiveFile) } @@ -84,10 +101,12 @@ func (h *Handler) addObjectToZip(zw *zip.Writer, obj *object.Object) (io.Writer, // DownloadZipped handles zip by prefix requests. func (h *Handler) DownloadZipped(c *fasthttp.RequestCtx) { + ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(c), "handler.DownloadZipped") + defer span.End() + scid, _ := c.UserValue("cid").(string) prefix, _ := c.UserValue("prefix").(string) - ctx := utils.GetContextFromRequest(c) log := utils.GetReqLogOrDefault(ctx, h.log) prefix, err := url.QueryUnescape(prefix) diff --git a/internal/handler/head.go b/internal/handler/head.go index ccd6a91..09fe1e0 100644 --- a/internal/handler/head.go +++ b/internal/handler/head.go @@ -9,6 +9,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/valyala/fasthttp" @@ -107,13 +108,25 @@ func (h *Handler) HeadByAddressOrBucketName(c *fasthttp.RequestCtx) { err := id.DecodeString(test) if err != nil { + ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(c), "handler.HeadByS3Path") + defer span.End() + + utils.SetContextToRequest(ctx, c) h.byS3Path(c, h.headObject) } else { + ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(c), "handler.HeadByNativeAddress") + defer span.End() + + utils.SetContextToRequest(ctx, c) h.byNativeAddress(c, h.headObject) } } // HeadByAttribute handles attribute-based head requests. func (h *Handler) HeadByAttribute(c *fasthttp.RequestCtx) { + ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(c), "handler.HeadByAttribute") + defer span.End() + + utils.SetContextToRequest(ctx, c) h.byAttribute(c, h.headObject) } diff --git a/internal/handler/upload.go b/internal/handler/upload.go index 867025d..158018e 100644 --- a/internal/handler/upload.go +++ b/internal/handler/upload.go @@ -12,6 +12,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/response" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -44,6 +45,9 @@ func (pr *putResponse) encode(w io.Writer) error { // Upload handles multipart upload request. func (h *Handler) Upload(c *fasthttp.RequestCtx) { + ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(c), "handler.Upload") + defer span.End() + var ( file MultipartFile idObj oid.ID @@ -54,7 +58,6 @@ func (h *Handler) Upload(c *fasthttp.RequestCtx) { bodyStream := c.RequestBodyStream() drainBuf := make([]byte, drainBufSize) - ctx := utils.GetContextFromRequest(c) reqLog := utils.GetReqLogOrDefault(ctx, h.log) log := reqLog.With(zap.String("cid", scid)) -- 2.45.2 From 838d84181787a4b682cd6d18a2e137cb35495cad Mon Sep 17 00:00:00 2001 From: Roman Loginov Date: Thu, 28 Nov 2024 05:48:35 +0300 Subject: [PATCH 2/4] [#145] frostfs: Add spans to detail the trace Signed-off-by: Roman Loginov --- internal/service/frostfs/frostfs.go | 25 +++++++++++++++++++ .../service/frostfs/multi_object_reader.go | 7 ++++++ 2 files changed, 32 insertions(+) diff --git a/internal/service/frostfs/frostfs.go b/internal/service/frostfs/frostfs.go index c7e56a4..d19161e 100644 --- a/internal/service/frostfs/frostfs.go +++ b/internal/service/frostfs/frostfs.go @@ -9,6 +9,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -34,6 +35,9 @@ func NewFrostFS(p *pool.Pool) *FrostFS { // Container implements frostfs.FrostFS interface method. func (x *FrostFS) Container(ctx context.Context, containerPrm handler.PrmContainer) (*container.Container, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "frostfs.Container") + defer span.End() + prm := pool.PrmContainerGet{ ContainerID: containerPrm.ContainerID, } @@ -48,6 +52,9 @@ func (x *FrostFS) Container(ctx context.Context, containerPrm handler.PrmContain // CreateObject implements frostfs.FrostFS interface method. func (x *FrostFS) CreateObject(ctx context.Context, prm handler.PrmObjectCreate) (oid.ID, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "frostfs.CreateObject") + defer span.End() + var prmPut pool.PrmObjectPut prmPut.SetHeader(*prm.Object) prmPut.SetPayload(prm.Payload) @@ -82,6 +89,9 @@ func (x payloadReader) Read(p []byte) (int, error) { // HeadObject implements frostfs.FrostFS interface method. func (x *FrostFS) HeadObject(ctx context.Context, prm handler.PrmObjectHead) (*object.Object, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "frostfs.HeadObject") + defer span.End() + var prmHead pool.PrmObjectHead prmHead.SetAddress(prm.Address) @@ -99,6 +109,9 @@ func (x *FrostFS) HeadObject(ctx context.Context, prm handler.PrmObjectHead) (*o // GetObject implements frostfs.FrostFS interface method. func (x *FrostFS) GetObject(ctx context.Context, prm handler.PrmObjectGet) (*handler.Object, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "frostfs.GetObject") + defer span.End() + var prmGet pool.PrmObjectGet prmGet.SetAddress(prm.Address) @@ -119,6 +132,9 @@ func (x *FrostFS) GetObject(ctx context.Context, prm handler.PrmObjectGet) (*han // RangeObject implements frostfs.FrostFS interface method. func (x *FrostFS) RangeObject(ctx context.Context, prm handler.PrmObjectRange) (io.ReadCloser, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "frostfs.RangeObject") + defer span.End() + var prmRange pool.PrmObjectRange prmRange.SetAddress(prm.Address) prmRange.SetOffset(prm.PayloadRange[0]) @@ -138,6 +154,9 @@ func (x *FrostFS) RangeObject(ctx context.Context, prm handler.PrmObjectRange) ( // SearchObjects implements frostfs.FrostFS interface method. func (x *FrostFS) SearchObjects(ctx context.Context, prm handler.PrmObjectSearch) (handler.ResObjectSearch, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "frostfs.SearchObjects") + defer span.End() + var prmSearch pool.PrmObjectSearch prmSearch.SetContainerID(prm.Container) prmSearch.SetFilters(prm.Filters) @@ -156,6 +175,9 @@ func (x *FrostFS) SearchObjects(ctx context.Context, prm handler.PrmObjectSearch // GetEpochDurations implements frostfs.FrostFS interface method. func (x *FrostFS) GetEpochDurations(ctx context.Context) (*utils.EpochDurations, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "frostfs.GetEpochDurations") + defer span.End() + networkInfo, err := x.pool.NetworkInfo(ctx) if err != nil { return nil, err @@ -186,6 +208,9 @@ func NewResolverFrostFS(p *pool.Pool) *ResolverFrostFS { // SystemDNS implements resolver.FrostFS interface method. func (x *ResolverFrostFS) SystemDNS(ctx context.Context) (string, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "frostfs.SystemDNS") + defer span.End() + networkInfo, err := x.pool.NetworkInfo(ctx) if err != nil { return "", handleObjectError("read network info via client", err) diff --git a/internal/service/frostfs/multi_object_reader.go b/internal/service/frostfs/multi_object_reader.go index 93f1f60..b943474 100644 --- a/internal/service/frostfs/multi_object_reader.go +++ b/internal/service/frostfs/multi_object_reader.go @@ -9,6 +9,7 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" ) @@ -74,6 +75,9 @@ var ( ) func (x *FrostFS) InitMultiObjectReader(ctx context.Context, p handler.PrmInitMultiObjectReader) (io.Reader, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "frostfs.InitMultiObjectReader") + defer span.End() + combinedObj, err := x.GetObject(ctx, handler.PrmObjectGet{ PrmAuth: handler.PrmAuth{BearerToken: p.Bearer}, Address: p.Addr, @@ -215,6 +219,9 @@ func (x *MultiObjectReader) Read(p []byte) (n int, err error) { // InitFrostFSObjectPayloadReader initializes payload reader of the FrostFS object. // Zero range corresponds to full payload (panics if only offset is set). func (x *FrostFS) InitFrostFSObjectPayloadReader(ctx context.Context, p GetFrostFSParams) (io.ReadCloser, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "frostfs.InitFrostFSObjectPayloadReader") + defer span.End() + var prmAuth handler.PrmAuth if p.Off+p.Ln != 0 { -- 2.45.2 From 3953da7ba8943540f2a87ef593201519a5223ac9 Mon Sep 17 00:00:00 2001 From: Roman Loginov Date: Thu, 28 Nov 2024 05:48:46 +0300 Subject: [PATCH 3/4] [#145] tree: Add spans to detail the trace Signed-off-by: Roman Loginov --- tree/tree.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tree/tree.go b/tree/tree.go index 40209a5..3ff4b77 100644 --- a/tree/tree.go +++ b/tree/tree.go @@ -9,6 +9,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/api" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/api/layer" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" ) @@ -181,6 +182,9 @@ func (m *multiSystemNode) Old() []*treeNode { } func (c *Tree) GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*api.NodeVersion, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "tree.GetLatestVersion") + defer span.End() + nodes, err := c.GetVersions(ctx, cnrID, objectName) if err != nil { return nil, err @@ -211,6 +215,9 @@ func (c *Tree) GetVersions(ctx context.Context, cnrID *cid.ID, objectName string } func (c *Tree) CheckSettingsNodeExist(ctx context.Context, bktInfo *data.BucketInfo) error { + ctx, span := tracing.StartSpanFromContext(ctx, "tree.CheckSettingsNodeExist") + defer span.End() + _, err := c.getSystemNode(ctx, bktInfo, settingsFileName) if err != nil { return err @@ -299,6 +306,9 @@ func pathFromName(objectName string) []string { } func (c *Tree) GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix string, latestOnly bool) ([]NodeResponse, string, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "tree.GetSubTreeByPrefix") + defer span.End() + rootID, tailPrefix, err := c.determinePrefixNode(ctx, bktInfo, versionTree, prefix) if err != nil { return nil, "", err -- 2.45.2 From 8bdb3d1d934ca2dc043c13a78a4b172d82727018 Mon Sep 17 00:00:00 2001 From: Roman Loginov Date: Thu, 28 Nov 2024 11:21:53 +0300 Subject: [PATCH 4/4] [#145] Update frostfs-observability version The new version of frostfs-observability has improved the detail of tracing low-level rpc calls by adding send and receive events. Signed-off-by: Roman Loginov --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 3dd27b8..51a30d4 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module git.frostfs.info/TrueCloudLab/frostfs-http-gw go 1.22 require ( - git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88 + git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241125133852-37bd75821121 git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241206094944-81c423e7094d git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02 diff --git a/go.sum b/go.sum index 7f43c86..50def39 100644 --- a/go.sum +++ b/go.sum @@ -41,10 +41,10 @@ git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e/go.mod h1:F/fe1OoIDKr5Bz99q4sriuHDuf3aZefZy9ZsCqEtgxc= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= -git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88 h1:9bvBDLApbbO5sXBKdODpE9tzy3HV99nXxkDWNn22rdI= -git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88/go.mod h1:kbwB4v2o6RyOfCo9kEFeUDZIX3LKhmS0yXPrtvzkQ1g= git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241206094944-81c423e7094d h1:FpXI+mOrmJk3t2MKQFZuhLjCHDyDeo5rtP1WXl7gUWc= git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241206094944-81c423e7094d/go.mod h1:eoK7+KZQ9GJxbzIs6vTnoUJqFDppavInLRHaN4MYgZg= +git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241125133852-37bd75821121 h1:/Z8DfbLZXp7exUQWUKoG/9tbFdI9d5lV1qSReaYoG8I= +git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241125133852-37bd75821121/go.mod h1:kbwB4v2o6RyOfCo9kEFeUDZIX3LKhmS0yXPrtvzkQ1g= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 h1:/960fWeyn2AFHwQUwDsWB3sbP6lTEnFnMzLMM6tx6N8= -- 2.45.2