TrueCloudLab/frostfs-s3-lifecycler
12
0
Fork 4
Code Issues 1 Pull requests 1 Projects Releases 1 Packages Wiki Activity Actions

[#3] Update tree to handle split nodes
All checks were successful
/ DCO (pull_request) Successful in 1m43s
Details
/ Vulncheck (pull_request) Successful in 2m5s
Details
/ Builds (1.21) (pull_request) Successful in 2m20s
Details
/ Builds (1.22) (pull_request) Successful in 2m30s
Details
/ Lint (pull_request) Successful in 4m2s
Details
/ Tests (1.21) (pull_request) Successful in 3m38s
Details
/ Tests (1.22) (pull_request) Successful in 1m7s
Details

Browse source 
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
This commit is contained in:
Denis Kirillov 2024-07-22 13:29:26 +03:00
parent d8b5cd5fc2
commit 84ed2f55e3
3 changed files with 43 additions and 22 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
go.mod
View file

@ -5,12 +5,12 @@ go 1.21
require ( require (
git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e
git.frostfs.info/TrueCloudLab/frostfs-node v0.42.0-rc.5 git.frostfs.info/TrueCloudLab/frostfs-node v0.42.0-rc.5
git.frostfs.info/TrueCloudLab/frostfs-s3-gw v0.30.0-rc.4.0.20240709102501-0e1ab11a1bd7 git.frostfs.info/TrueCloudLab/frostfs-s3-gw v0.30.1-0.20240722101532-e6be3aa7c0c4
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240705093617-560cbbd1f1e4 git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240718141740-ce8270568d36
git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/hrw v1.2.1
git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240611102930-ac965e8d176a git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240611102930-ac965e8d176a
git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02 git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02
github.com/nspcc-dev/neo-go v0.106.0 github.com/nspcc-dev/neo-go v0.106.2
github.com/prometheus/client_golang v1.19.1 github.com/prometheus/client_golang v1.19.1
github.com/prometheus/client_model v0.6.1 github.com/prometheus/client_model v0.6.1
github.com/spf13/pflag v1.0.5 github.com/spf13/pflag v1.0.5
@ -24,7 +24,7 @@ require (
replace github.com/nspcc-dev/neo-go => git.frostfs.info/TrueCloudLab/neoneo-go v0.106.1-0.20240611123832-594f716b3d18 replace github.com/nspcc-dev/neo-go => git.frostfs.info/TrueCloudLab/neoneo-go v0.106.1-0.20240611123832-594f716b3d18
require ( require (
git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240530152826-2f6d3209e1d3 // indirect git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240716113920-f517e3949164 // indirect
git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 // indirect git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 // indirect
git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 // indirect git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 // indirect
git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 // indirect git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 // indirect

12
go.sum
View file

@ -1,5 +1,5 @@
git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240530152826-2f6d3209e1d3 h1:H5GvrVlowIMWfzqQkhY0p0myooJxQ1sMRVSFfXawwWg= git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240716113920-f517e3949164 h1:XxvwQKJT/f16qS3df5PBQPRYKkhy0/A7zH6644QpKD0=
git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240530152826-2f6d3209e1d3/go.mod h1:OBDSr+DqV1z4VDouoX3YMleNc4DPBVBWTG3WDT2PK1o= git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240716113920-f517e3949164/go.mod h1:OBDSr+DqV1z4VDouoX3YMleNc4DPBVBWTG3WDT2PK1o=
git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e h1:kcBqZBiFIUBATUqEuvVigtkJJWQ2Gug/eYXn967o3M4= git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e h1:kcBqZBiFIUBATUqEuvVigtkJJWQ2Gug/eYXn967o3M4=
git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e/go.mod h1:F/fe1OoIDKr5Bz99q4sriuHDuf3aZefZy9ZsCqEtgxc= git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e/go.mod h1:F/fe1OoIDKr5Bz99q4sriuHDuf3aZefZy9ZsCqEtgxc=
git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk=
@ -8,10 +8,10 @@ git.frostfs.info/TrueCloudLab/frostfs-node v0.42.0-rc.5 h1:lVWO3JtF3R4Irb+/xT5+w
git.frostfs.info/TrueCloudLab/frostfs-node v0.42.0-rc.5/go.mod h1:IZBD+sRxSxpXXIkg0rAK5yvkGHZUaHBqmcWFu2UmbmQ= git.frostfs.info/TrueCloudLab/frostfs-node v0.42.0-rc.5/go.mod h1:IZBD+sRxSxpXXIkg0rAK5yvkGHZUaHBqmcWFu2UmbmQ=
git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 h1:PaZ8GpnUoXxUoNsc1qp36bT2u7FU+neU4Jn9cl8AWqI= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 h1:PaZ8GpnUoXxUoNsc1qp36bT2u7FU+neU4Jn9cl8AWqI=
git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65/go.mod h1:6aAX80dvJ3r5fjN9CzzPglRptoiPgIC9KFGGsUA+1Hw= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65/go.mod h1:6aAX80dvJ3r5fjN9CzzPglRptoiPgIC9KFGGsUA+1Hw=
git.frostfs.info/TrueCloudLab/frostfs-s3-gw v0.30.0-rc.4.0.20240709102501-0e1ab11a1bd7 h1:MENM+TjT05VtHQoxu0AjeyTVWH48/piizYpHpATFhNI= git.frostfs.info/TrueCloudLab/frostfs-s3-gw v0.30.1-0.20240722101532-e6be3aa7c0c4 h1:caHHp72qqO70ET2DsKNQYVyqAfyabCgFY/A3L8tx31M=
git.frostfs.info/TrueCloudLab/frostfs-s3-gw v0.30.0-rc.4.0.20240709102501-0e1ab11a1bd7/go.mod h1:y5tKPMT0xgKwb6TEMWzBgdmRqFWTWXckX03u55/4Gyg= git.frostfs.info/TrueCloudLab/frostfs-s3-gw v0.30.1-0.20240722101532-e6be3aa7c0c4/go.mod h1:HJ5gKSAfyhIU/UjmgftfNEumR0FD22W8Z85KbaS8NX0=
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240705093617-560cbbd1f1e4 h1:izmHYpkz7cPr2Zpudxxh0wvrtAIxYywEG+uraghVSlo= git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240718141740-ce8270568d36 h1:MV/vKJWLQT34RRbXYvkNKFYGNjL5bRNuCQMXkbC7fLI=
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240705093617-560cbbd1f1e4/go.mod h1:4AObM67VUqkXQJlODTFThFnuMGEuK8h9DrAXHDZqvCU= git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240718141740-ce8270568d36/go.mod h1:vluJ/+yQMcq8ZIZZSA7Te+JKClr0lgtRErjICvb8wto=
git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc=
git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM=
git.frostfs.info/TrueCloudLab/neoneo-go v0.106.1-0.20240611123832-594f716b3d18 h1:JRjwcHaQajTbSCBCK3yZnqvyHvgWBaoThDGuT4kvIIc= git.frostfs.info/TrueCloudLab/neoneo-go v0.106.1-0.20240611123832-594f716b3d18 h1:JRjwcHaQajTbSCBCK3yZnqvyHvgWBaoThDGuT4kvIIc=

45
internal/frostfs/tree.go
View file

@ -18,16 +18,16 @@ type GetNodeByPathResponseInfoWrapper struct {
response *grpcService.GetNodeByPathResponse_Info response *grpcService.GetNodeByPathResponse_Info
} }
func (n GetNodeByPathResponseInfoWrapper) GetNodeID() uint64 { func (n GetNodeByPathResponseInfoWrapper) GetNodeID() []uint64 {
return n.response.GetNodeId() return []uint64{n.response.GetNodeId()}
} }
func (n GetNodeByPathResponseInfoWrapper) GetParentID() uint64 { func (n GetNodeByPathResponseInfoWrapper) GetParentID() []uint64 {
return n.response.GetParentId() return []uint64{n.response.GetParentId()}
} }
func (n GetNodeByPathResponseInfoWrapper) GetTimestamp() uint64 { func (n GetNodeByPathResponseInfoWrapper) GetTimestamp() []uint64 {
return n.response.GetTimestamp() return []uint64{n.response.GetTimestamp()}
} }
func (n GetNodeByPathResponseInfoWrapper) GetMeta() []tree.Meta { func (n GetNodeByPathResponseInfoWrapper) GetMeta() []tree.Meta {
@ -42,15 +42,21 @@ type GetSubTreeResponseBodyWrapper struct {
response *grpcService.GetSubTreeResponse_Body response *grpcService.GetSubTreeResponse_Body
} }
func (n GetSubTreeResponseBodyWrapper) GetNodeID() uint64 { func (n GetSubTreeResponseBodyWrapper) GetNodeID() []uint64 {
return n.response.GetNodeId() return n.response.GetNodeId()
} }
func (n GetSubTreeResponseBodyWrapper) GetParentID() uint64 { func (n GetSubTreeResponseBodyWrapper) GetParentID() []uint64 {
return n.response.GetParentId() resp := n.response.GetParentId()
if resp == nil {
// storage sends nil that should be interpreted as []uint64{0}
// due to protobuf compatibility, see 'GetSubTree' function
return []uint64{0}
}
return resp
} }
func (n GetSubTreeResponseBodyWrapper) GetTimestamp() uint64 { func (n GetSubTreeResponseBodyWrapper) GetTimestamp() []uint64 {
return n.response.GetTimestamp() return n.response.GetTimestamp()
} }
@ -95,13 +101,21 @@ func (w *TreePoolWrapper) GetNodes(ctx context.Context, prm *tree.GetNodesParams
return res, nil return res, nil
} }
func (w *TreePoolWrapper) GetSubTree(ctx context.Context, bktInfo *data.BucketInfo, treeID string, rootID uint64, depth uint32) ([]tree.NodeResponse, error) { func (w *TreePoolWrapper) GetSubTree(ctx context.Context, bktInfo *data.BucketInfo, treeID string, rootID []uint64, depth uint32) ([]tree.NodeResponse, error) {
poolPrm := treepool.GetSubTreeParams{ poolPrm := treepool.GetSubTreeParams{
CID: bktInfo.CID, CID: bktInfo.CID,
TreeID: treeID, TreeID: treeID,
RootID: rootID, RootID: rootID,
Depth: depth, Depth: depth,
BearerToken: getBearer(ctx, bktInfo), BearerToken: getBearer(ctx, bktInfo),
Order: treepool.AscendingOrder,
}
if len(rootID) == 1 && rootID[0] == 0 {
// storage node interprets 'nil' value as []uint64{0}
// gate wants to send 'nil' value instead of []uint64{0}, because
// it provides compatibility with previous tree service api where
// single uint64(0) value is dropped from signature
poolPrm.RootID = nil
} }
subTreeReader, err := w.p.GetSubTree(ctx, poolPrm) subTreeReader, err := w.p.GetSubTree(ctx, poolPrm)
@ -161,7 +175,7 @@ func (s *SubTreeStreamImpl) Next() (tree.NodeResponse, error) {
return s.Next() return s.Next()
} }
func (w *TreePoolWrapper) GetSubTreeStream(ctx context.Context, bktInfo *data.BucketInfo, treeID string, rootID uint64, depth uint32) (tree.SubTreeStream, error) { func (w *TreePoolWrapper) GetSubTreeStream(ctx context.Context, bktInfo *data.BucketInfo, treeID string, rootID []uint64, depth uint32) (tree.SubTreeStream, error) {
poolPrm := treepool.GetSubTreeParams{ poolPrm := treepool.GetSubTreeParams{
CID: bktInfo.CID, CID: bktInfo.CID,
TreeID: treeID, TreeID: treeID,
@ -170,6 +184,13 @@ func (w *TreePoolWrapper) GetSubTreeStream(ctx context.Context, bktInfo *data.Bu
BearerToken: getBearer(ctx, bktInfo), BearerToken: getBearer(ctx, bktInfo),
Order: treepool.AscendingOrder, Order: treepool.AscendingOrder,
} }
if len(rootID) == 1 && rootID[0] == 0 {
// storage node interprets 'nil' value as []uint64{0}
// gate wants to send 'nil' value instead of []uint64{0}, because
// it provides compatibility with previous tree service api where
// single uint64(0) value is dropped from signature
poolPrm.RootID = nil
}
subTreeReader, err := w.p.GetSubTree(ctx, poolPrm) subTreeReader, err := w.p.GetSubTree(ctx, poolPrm)
if err != nil { if err != nil {