feature/integrate_tree_service #8
3 changed files with 624 additions and 44 deletions
89
go.mod
89
go.mod
|
@ -37,6 +37,11 @@ require (
|
||||||
)
|
)
|
||||||
|
|
||||||
require (
|
require (
|
||||||
|
github.com/benbjohnson/clock v1.1.0 // indirect
|
||||||
|
cloud.google.com/go v0.110.0 // indirect
|
||||||
|
cloud.google.com/go/compute v1.19.1 // indirect
|
||||||
|
cloud.google.com/go/compute/metadata v0.2.3 // indirect
|
||||||
|
cloud.google.com/go/iam v0.13.0 // indirect
|
||||||
dario.cat/mergo v1.0.0 // indirect
|
dario.cat/mergo v1.0.0 // indirect
|
||||||
git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230802075510-964c3edb3f44 // indirect
|
git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230802075510-964c3edb3f44 // indirect
|
||||||
git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb // indirect
|
git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb // indirect
|
||||||
|
@ -44,23 +49,47 @@ require (
|
||||||
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230825064515-46a214d065f8
|
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230825064515-46a214d065f8
|
||||||
git.frostfs.info/TrueCloudLab/hrw v1.2.1 // indirect
|
git.frostfs.info/TrueCloudLab/hrw v1.2.1 // indirect
|
||||||
git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 // indirect
|
git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 // indirect
|
||||||
|
github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect
|
||||||
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
|
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
|
||||||
|
github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 // indirect
|
||||||
github.com/Microsoft/go-winio v0.6.1 // indirect
|
github.com/Microsoft/go-winio v0.6.1 // indirect
|
||||||
github.com/antlr4-go/antlr/v4 v4.13.0 // indirect
|
github.com/antlr4-go/antlr/v4 v4.13.0 // indirect
|
||||||
|
github.com/beorn7/perks v1.0.1 // indirect
|
||||||
|
github.com/bitly/go-simplejson v0.5.0 // indirect
|
||||||
|
github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b // indirect
|
||||||
|
github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0 // indirect
|
||||||
github.com/cenkalti/backoff/v4 v4.2.1 // indirect
|
github.com/cenkalti/backoff/v4 v4.2.1 // indirect
|
||||||
|
github.com/cespare/xxhash/v2 v2.2.0 // indirect
|
||||||
github.com/containerd/containerd v1.7.3 // indirect
|
github.com/containerd/containerd v1.7.3 // indirect
|
||||||
github.com/cpuguy83/dockercfg v0.3.1 // indirect
|
github.com/cpuguy83/dockercfg v0.3.1 // indirect
|
||||||
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
|
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
|
||||||
|
github.com/cyphar/filepath-securejoin v0.2.3 // indirect
|
||||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||||
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect
|
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect
|
||||||
github.com/docker/distribution v2.8.2+incompatible // indirect
|
github.com/docker/distribution v2.8.2+incompatible // indirect
|
||||||
github.com/docker/docker v24.0.5+incompatible // indirect
|
github.com/docker/docker v24.0.5+incompatible // indirect
|
||||||
github.com/docker/go-connections v0.4.0 // indirect
|
github.com/docker/go-connections v0.4.0 // indirect
|
||||||
github.com/docker/go-units v0.5.0 // indirect
|
github.com/docker/go-units v0.5.0 // indirect
|
||||||
|
github.com/felixge/httpsnoop v1.0.1 // indirect
|
||||||
github.com/gogo/protobuf v1.3.2 // indirect
|
github.com/gogo/protobuf v1.3.2 // indirect
|
||||||
|
github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
|
||||||
|
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
|
||||||
|
github.com/golang/protobuf v1.5.3 // indirect
|
||||||
|
github.com/google/go-cmp v0.5.9 // indirect
|
||||||
|
github.com/google/uuid v1.3.0
|
||||||
|
github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect
|
||||||
|
github.com/googleapis/gax-go/v2 v2.7.1 // indirect
|
||||||
github.com/gorilla/websocket v1.5.0 // indirect
|
github.com/gorilla/websocket v1.5.0 // indirect
|
||||||
github.com/hashicorp/golang-lru v0.6.0 // indirect
|
github.com/hashicorp/golang-lru v0.6.0 // indirect
|
||||||
|
github.com/hashicorp/golang-lru/v2 v2.0.5 // indirect
|
||||||
|
github.com/inconshreveable/mousetrap v1.0.1 // indirect
|
||||||
|
github.com/jmespath/go-jmespath v0.4.0 // indirect
|
||||||
|
github.com/kr/pretty v0.3.0 // indirect
|
||||||
|
github.com/kr/text v0.2.0 // indirect
|
||||||
|
github.com/kylelemons/godebug v1.1.0 // indirect
|
||||||
github.com/magiconair/properties v1.8.7 // indirect
|
github.com/magiconair/properties v1.8.7 // indirect
|
||||||
|
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
|
||||||
|
github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f // indirect
|
||||||
github.com/moby/patternmatcher v0.5.0 // indirect
|
github.com/moby/patternmatcher v0.5.0 // indirect
|
||||||
github.com/moby/sys/sequential v0.5.0 // indirect
|
github.com/moby/sys/sequential v0.5.0 // indirect
|
||||||
github.com/moby/term v0.5.0 // indirect
|
github.com/moby/term v0.5.0 // indirect
|
||||||
|
@ -70,63 +99,35 @@ require (
|
||||||
github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20230615193820-9185820289ce // indirect
|
github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20230615193820-9185820289ce // indirect
|
||||||
github.com/nspcc-dev/rfc6979 v0.2.0 // indirect
|
github.com/nspcc-dev/rfc6979 v0.2.0 // indirect
|
||||||
github.com/opencontainers/runc v1.1.5 // indirect
|
github.com/opencontainers/runc v1.1.5 // indirect
|
||||||
|
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
|
||||||
github.com/pkg/errors v0.9.1 // indirect
|
github.com/pkg/errors v0.9.1 // indirect
|
||||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||||
github.com/rogpeppe/go-internal v1.8.1 // indirect
|
|
||||||
github.com/russross/blackfriday/v2 v2.1.0 // indirect
|
|
||||||
github.com/twmb/murmur3 v1.1.8 // indirect
|
|
||||||
github.com/urfave/cli v1.22.12 // indirect
|
|
||||||
go.uber.org/atomic v1.10.0 // indirect
|
|
||||||
go.uber.org/multierr v1.11.0 // indirect
|
|
||||||
go.uber.org/zap v1.24.0 // indirect
|
|
||||||
golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc // indirect
|
|
||||||
golang.org/x/mod v0.9.0 // indirect
|
|
||||||
golang.org/x/sync v0.2.0 // indirect
|
|
||||||
golang.org/x/tools v0.7.0 // indirect
|
|
||||||
google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9 // indirect
|
|
||||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect
|
|
||||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
|
||||||
cloud.google.com/go v0.110.0 // indirect
|
|
||||||
cloud.google.com/go/compute v1.19.1 // indirect
|
|
||||||
cloud.google.com/go/compute/metadata v0.2.3 // indirect
|
|
||||||
cloud.google.com/go/iam v0.13.0 // indirect
|
|
||||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect
|
|
||||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 // indirect
|
|
||||||
github.com/beorn7/perks v1.0.1 // indirect
|
|
||||||
github.com/bitly/go-simplejson v0.5.0 // indirect
|
|
||||||
github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b // indirect
|
|
||||||
github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0 // indirect
|
|
||||||
github.com/cespare/xxhash/v2 v2.2.0 // indirect
|
|
||||||
github.com/cyphar/filepath-securejoin v0.2.3 // indirect
|
|
||||||
github.com/felixge/httpsnoop v1.0.1 // indirect
|
|
||||||
github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
|
|
||||||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
|
|
||||||
github.com/golang/protobuf v1.5.3 // indirect
|
|
||||||
github.com/google/go-cmp v0.5.9 // indirect
|
|
||||||
github.com/google/uuid v1.3.0
|
|
||||||
github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect
|
|
||||||
github.com/googleapis/gax-go/v2 v2.7.1 // indirect
|
|
||||||
github.com/hashicorp/golang-lru/v2 v2.0.5 // indirect
|
|
||||||
github.com/inconshreveable/mousetrap v1.0.1 // indirect
|
|
||||||
github.com/jmespath/go-jmespath v0.4.0 // indirect
|
|
||||||
github.com/kr/pretty v0.3.0 // indirect
|
|
||||||
github.com/kr/text v0.2.0 // indirect
|
|
||||||
github.com/kylelemons/godebug v1.1.0 // indirect
|
|
||||||
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
|
|
||||||
github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f // indirect
|
|
||||||
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
|
|
||||||
github.com/prometheus/client_golang v1.14.0 // indirect; updated to latest
|
github.com/prometheus/client_golang v1.14.0 // indirect; updated to latest
|
||||||
github.com/prometheus/client_model v0.3.0 // indirect
|
github.com/prometheus/client_model v0.3.0 // indirect
|
||||||
github.com/prometheus/common v0.37.0 // indirect
|
github.com/prometheus/common v0.37.0 // indirect
|
||||||
github.com/prometheus/procfs v0.8.0 // indirect
|
github.com/prometheus/procfs v0.8.0 // indirect
|
||||||
|
github.com/rogpeppe/go-internal v1.8.1 // indirect
|
||||||
|
github.com/russross/blackfriday/v2 v2.1.0 // indirect
|
||||||
github.com/spf13/pflag v1.0.5 // indirect
|
github.com/spf13/pflag v1.0.5 // indirect
|
||||||
|
github.com/twmb/murmur3 v1.1.8 // indirect
|
||||||
|
github.com/urfave/cli v1.22.12 // indirect
|
||||||
go.opencensus.io v0.24.0 // indirect
|
go.opencensus.io v0.24.0 // indirect
|
||||||
|
go.uber.org/atomic v1.10.0 // indirect
|
||||||
|
go.uber.org/multierr v1.11.0 // indirect
|
||||||
|
go.uber.org/zap v1.24.0
|
||||||
|
golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc // indirect
|
||||||
|
golang.org/x/mod v0.9.0 // indirect
|
||||||
golang.org/x/net v0.10.0 // indirect; updated for CVE-2022-27664, CVE-2022-41717
|
golang.org/x/net v0.10.0 // indirect; updated for CVE-2022-27664, CVE-2022-41717
|
||||||
|
golang.org/x/sync v0.2.0 // indirect
|
||||||
golang.org/x/sys v0.11.0 // indirect
|
golang.org/x/sys v0.11.0 // indirect
|
||||||
golang.org/x/text v0.9.0 // indirect
|
golang.org/x/text v0.9.0 // indirect
|
||||||
|
golang.org/x/tools v0.7.0 // indirect
|
||||||
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
|
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
|
||||||
google.golang.org/appengine v1.6.7 // indirect
|
google.golang.org/appengine v1.6.7 // indirect
|
||||||
google.golang.org/genproto v0.0.0-20230526161137-0005af68ea54 // indirect
|
google.golang.org/genproto v0.0.0-20230526161137-0005af68ea54 // indirect
|
||||||
google.golang.org/grpc v1.57.0 // indirect
|
google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9 // indirect
|
||||||
|
google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect
|
||||||
|
google.golang.org/grpc v1.57.0
|
||||||
google.golang.org/protobuf v1.30.0 // indirect
|
google.golang.org/protobuf v1.30.0 // indirect
|
||||||
|
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||||
)
|
)
|
||||||
|
|
347
registry/storage/driver/frostfs/tree/tree_client_in_memory.go
Normal file
347
registry/storage/driver/frostfs/tree/tree_client_in_memory.go
Normal file
|
@ -0,0 +1,347 @@
|
||||||
|
package tree
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||||
|
)
|
||||||
|
|
||||||
|
type nodeMeta struct {
|
||||||
|
key string
|
||||||
|
value []byte
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m nodeMeta) GetKey() string {
|
||||||
|
return m.key
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m nodeMeta) GetValue() []byte {
|
||||||
|
return m.value
|
||||||
|
}
|
||||||
|
|
||||||
|
type nodeResponse struct {
|
||||||
|
meta []nodeMeta
|
||||||
|
nodeID uint64
|
||||||
|
parentID uint64
|
||||||
|
timestamp uint64
|
||||||
|
}
|
||||||
|
|
||||||
|
func (n nodeResponse) GetNodeID() uint64 {
|
||||||
|
return n.nodeID
|
||||||
|
}
|
||||||
|
|
||||||
|
func (n nodeResponse) GetParentID() uint64 {
|
||||||
|
return n.parentID
|
||||||
|
}
|
||||||
|
|
||||||
|
func (n nodeResponse) GetTimestamp() uint64 {
|
||||||
|
return n.timestamp
|
||||||
|
}
|
||||||
|
|
||||||
|
func (n nodeResponse) GetMeta() []Meta {
|
||||||
|
res := make([]Meta, len(n.meta))
|
||||||
|
for i, value := range n.meta {
|
||||||
|
res[i] = value
|
||||||
|
}
|
||||||
|
return res
|
||||||
|
}
|
||||||
|
|
||||||
|
func (n nodeResponse) getValue(key string) string {
|
||||||
|
for _, value := range n.meta {
|
||||||
|
if value.key == key {
|
||||||
|
return string(value.value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
type ServiceClientMemory struct {
|
||||||
|
containers map[string]containerInfo
|
||||||
|
}
|
||||||
|
|
||||||
|
type containerInfo struct {
|
||||||
|
containerID cid.ID
|
||||||
|
trees map[string]memoryTree
|
||||||
|
}
|
||||||
|
|
||||||
|
type memoryTree struct {
|
||||||
|
idCounter uint64
|
||||||
|
treeData *treeNodeMemory
|
||||||
|
}
|
||||||
|
|
||||||
|
type treeNodeMemory struct {
|
||||||
|
data nodeResponse
|
||||||
|
parent *treeNodeMemory
|
||||||
|
children []*treeNodeMemory
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t *treeNodeMemory) getNode(nodeID uint64) *treeNodeMemory {
|
||||||
|
if t.data.nodeID == nodeID {
|
||||||
|
return t
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, child := range t.children {
|
||||||
|
if node := child.getNode(nodeID); node != nil {
|
||||||
|
return node
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t *memoryTree) getNodesByPath(path []string) []nodeResponse {
|
||||||
|
if len(path) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var res []nodeResponse
|
||||||
|
for _, child := range t.treeData.children {
|
||||||
|
res = child.listNodesByPath(res, path)
|
||||||
|
}
|
||||||
|
|
||||||
|
return res
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t *treeNodeMemory) listNodesByPath(res []nodeResponse, path []string) []nodeResponse {
|
||||||
|
if len(path) == 0 || t.data.getValue(FileNameKey) != path[0] {
|
||||||
|
return res
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(path) == 1 {
|
||||||
|
return append(res, t.data)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, ch := range t.children {
|
||||||
|
res = ch.listNodesByPath(res, path[1:])
|
||||||
|
}
|
||||||
|
|
||||||
|
return res
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t *memoryTree) createPathIfNotExist(parent *treeNodeMemory, path []string) *treeNodeMemory {
|
||||||
|
if len(path) == 0 {
|
||||||
|
return parent
|
||||||
|
}
|
||||||
|
|
||||||
|
var node *treeNodeMemory
|
||||||
|
for _, child := range parent.children {
|
||||||
|
if len(child.data.meta) == 1 && child.data.getValue(FileNameKey) == path[0] {
|
||||||
|
node = child
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if node == nil {
|
||||||
|
node = &treeNodeMemory{
|
||||||
|
data: nodeResponse{
|
||||||
|
meta: []nodeMeta{{key: FileNameKey, value: []byte(path[0])}},
|
||||||
|
nodeID: t.idCounter,
|
||||||
|
parentID: parent.data.nodeID,
|
||||||
|
timestamp: uint64(time.Now().UnixMicro()),
|
||||||
|
},
|
||||||
|
parent: parent,
|
||||||
|
}
|
||||||
|
t.idCounter++
|
||||||
|
parent.children = append(parent.children, node)
|
||||||
|
}
|
||||||
|
|
||||||
|
return t.createPathIfNotExist(node, path[1:])
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t *treeNodeMemory) removeChild(nodeID uint64) {
|
||||||
|
ind := -1
|
||||||
|
for i, ch := range t.children {
|
||||||
|
if ch.data.nodeID == nodeID {
|
||||||
|
ind = i
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if ind != -1 {
|
||||||
|
t.children = append(t.children[:ind], t.children[ind+1:]...)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t *treeNodeMemory) listNodes(res []NodeResponse, depth uint32) []NodeResponse {
|
||||||
|
res = append(res, t.data)
|
||||||
|
|
||||||
|
if depth == 0 {
|
||||||
|
return res
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, ch := range t.children {
|
||||||
|
res = ch.listNodes(res, depth-1)
|
||||||
|
}
|
||||||
|
return res
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewTreeServiceClientMemory() (*ServiceClientMemory, error) {
|
||||||
|
return &ServiceClientMemory{
|
||||||
|
containers: make(map[string]containerInfo),
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *ServiceClientMemory) GetNodes(_ context.Context, p *GetNodesParams) ([]NodeResponse, error) {
|
||||||
|
cnr, ok := c.containers[p.ContainerID.EncodeToString()]
|
||||||
|
if !ok {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
tr, ok := cnr.trees[p.TreeID]
|
||||||
|
if !ok {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
res := tr.getNodesByPath(p.Path)
|
||||||
|
sort.Slice(res, func(i, j int) bool {
|
||||||
|
return res[i].timestamp < res[j].timestamp
|
||||||
|
})
|
||||||
|
|
||||||
|
if p.LatestOnly && len(res) != 0 {
|
||||||
|
res = res[len(res)-1:]
|
||||||
|
}
|
||||||
|
|
||||||
|
res2 := make([]NodeResponse, len(res))
|
||||||
|
for i, n := range res {
|
||||||
|
res2[i] = n
|
||||||
|
}
|
||||||
|
|
||||||
|
return res2, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *ServiceClientMemory) GetSubTree(_ context.Context, containerID cid.ID, treeID string, rootID uint64, depth uint32) ([]NodeResponse, error) {
|
||||||
|
cnr, ok := c.containers[containerID.EncodeToString()]
|
||||||
|
if !ok {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
tr, ok := cnr.trees[treeID]
|
||||||
|
if !ok {
|
||||||
|
return nil, ErrNodeNotFound
|
||||||
|
}
|
||||||
|
|
||||||
|
sortNode(tr.treeData)
|
||||||
|
|
||||||
|
node := tr.treeData.getNode(rootID)
|
||||||
|
if node == nil {
|
||||||
|
return nil, ErrNodeNotFound
|
||||||
|
}
|
||||||
|
|
||||||
|
// we depth-1 in case of uint32 and 0 as mark to get all subtree leads to overflow and depth is getting quite big to walk all tree levels
|
||||||
|
return node.listNodes(nil, depth-1), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func newContainerInfo(containerID cid.ID, treeID string) containerInfo {
|
||||||
|
return containerInfo{
|
||||||
|
containerID: containerID,
|
||||||
|
trees: map[string]memoryTree{
|
||||||
|
treeID: {
|
||||||
|
idCounter: 1,
|
||||||
|
treeData: &treeNodeMemory{
|
||||||
|
data: nodeResponse{
|
||||||
|
timestamp: uint64(time.Now().UnixMicro()),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func newMemoryTree() memoryTree {
|
||||||
|
return memoryTree{
|
||||||
|
idCounter: 1,
|
||||||
|
treeData: &treeNodeMemory{
|
||||||
|
data: nodeResponse{
|
||||||
|
timestamp: uint64(time.Now().UnixMicro()),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *ServiceClientMemory) AddNodeByPath(_ context.Context, containerID cid.ID, treeID string, path []string, meta map[string]string) (uint64, error) {
|
||||||
|
cnr, ok := c.containers[containerID.EncodeToString()]
|
||||||
|
if !ok {
|
||||||
|
cnr = newContainerInfo(containerID, treeID)
|
||||||
|
c.containers[containerID.EncodeToString()] = cnr
|
||||||
|
}
|
||||||
|
|
||||||
|
tr, ok := cnr.trees[treeID]
|
||||||
|
if !ok {
|
||||||
|
tr = newMemoryTree()
|
||||||
|
cnr.trees[treeID] = tr
|
||||||
|
}
|
||||||
|
|
||||||
|
parentNode := tr.createPathIfNotExist(tr.treeData, path)
|
||||||
|
if parentNode == nil {
|
||||||
|
return 0, fmt.Errorf("create path '%s'", path)
|
||||||
|
}
|
||||||
|
|
||||||
|
newID := tr.idCounter
|
||||||
|
tr.idCounter++
|
||||||
|
|
||||||
|
tn := &treeNodeMemory{
|
||||||
|
data: nodeResponse{
|
||||||
|
meta: metaToNodeMeta(meta),
|
||||||
|
nodeID: newID,
|
||||||
|
parentID: parentNode.data.nodeID,
|
||||||
|
timestamp: uint64(time.Now().UnixMicro()),
|
||||||
|
},
|
||||||
|
parent: parentNode,
|
||||||
|
}
|
||||||
|
|
||||||
|
parentNode.children = append(parentNode.children, tn)
|
||||||
|
cnr.trees[treeID] = tr
|
||||||
|
|
||||||
|
return newID, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func sortNode(node *treeNodeMemory) {
|
||||||
|
if node == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
sortNodes(node.children)
|
||||||
|
|
||||||
|
for _, child := range node.children {
|
||||||
|
sortNode(child)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func sortNodes(list []*treeNodeMemory) {
|
||||||
|
sort.Slice(list, func(i, j int) bool {
|
||||||
|
return list[i].data.getValue(FileNameKey) < list[j].data.getValue(FileNameKey)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *ServiceClientMemory) RemoveNode(_ context.Context, containerID cid.ID, treeID string, nodeID uint64) error {
|
||||||
|
cnr, ok := c.containers[containerID.EncodeToString()]
|
||||||
|
if !ok {
|
||||||
|
return ErrNodeNotFound
|
||||||
|
}
|
||||||
|
|
||||||
|
tr, ok := cnr.trees[treeID]
|
||||||
|
if !ok {
|
||||||
|
return ErrNodeNotFound
|
||||||
|
}
|
||||||
|
|
||||||
|
node := tr.treeData.getNode(nodeID)
|
||||||
|
if node == nil {
|
||||||
|
return ErrNodeNotFound
|
||||||
|
}
|
||||||
|
|
||||||
|
node.parent.removeChild(nodeID)
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func metaToNodeMeta(m map[string]string) []nodeMeta {
|
||||||
|
result := make([]nodeMeta, 0, len(m))
|
||||||
|
|
||||||
|
for key, value := range m {
|
||||||
|
result = append(result, nodeMeta{key: key, value: []byte(value)})
|
||||||
|
}
|
||||||
|
|
||||||
|
return result
|
||||||
|
}
|
232
registry/storage/driver/frostfs/tree/tree_test.go
Normal file
232
registry/storage/driver/frostfs/tree/tree_test.go
Normal file
|
@ -0,0 +1,232 @@
|
||||||
|
package tree
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||||
|
oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test"
|
||||||
|
"github.com/google/uuid"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
"go.uber.org/zap/zaptest"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestGetObjectByPath(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
memCli, err := NewTreeServiceClientMemory()
|
||||||
|
require.NoError(t, err)
|
||||||
|
treeService := NewTree(memCli, zaptest.NewLogger(t))
|
||||||
|
|
||||||
|
cidTest := cidtest.ID()
|
||||||
|
oidTest1 := oidtest.ID()
|
||||||
|
oidTest2 := oidtest.ID()
|
||||||
|
testSize1 := uint64(10)
|
||||||
|
testSize2 := uint64(20)
|
||||||
|
|
||||||
|
nodeID1, err := treeService.AddObject(ctx, cidTest, "/a/b", oidTest1, testSize1)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
nodeID2, err := treeService.AddObject(ctx, cidTest, "/a/b/c/d", oidTest2, testSize2)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
node1, err := treeService.GetObjectByPath(ctx, cidTest, "/a/b")
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Equal(t, nodeID1, node1.ID)
|
||||||
|
require.Equal(t, oidTest1, node1.ObjID)
|
||||||
|
require.Equal(t, testSize1, node1.PayloadSize)
|
||||||
|
|
||||||
|
node2, err := treeService.GetObjectByPath(ctx, cidTest, "/a/b/c/d")
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Equal(t, nodeID2, node2.ID)
|
||||||
|
require.Equal(t, oidTest2, node2.ObjID)
|
||||||
|
require.Equal(t, testSize2, node2.PayloadSize)
|
||||||
|
|
||||||
|
_, err = treeService.GetObjectByPath(ctx, cidTest, "/g")
|
||||||
|
require.ErrorIs(t, err, ErrNodeNotFound)
|
||||||
|
|
||||||
|
_, err = treeService.GetObjectByPath(ctx, cidTest, "/a/b/c")
|
||||||
|
require.ErrorIs(t, err, ErrNodeNotFound)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGetObjectByPathDir(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
memCli, err := NewTreeServiceClientMemory()
|
||||||
|
require.NoError(t, err)
|
||||||
|
treeService := NewTree(memCli, zaptest.NewLogger(t))
|
||||||
|
|
||||||
|
cidTest := cidtest.ID()
|
||||||
|
oidTest1 := oidtest.ID()
|
||||||
|
oidTest2 := oidtest.ID()
|
||||||
|
testSize1 := uint64(10)
|
||||||
|
testSize2 := uint64(20)
|
||||||
|
|
||||||
|
nodeID1, err := treeService.AddObject(ctx, cidTest, "/a/b", oidTest1, testSize1)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
nodeID2, err := treeService.AddObject(ctx, cidTest, "/a/b/c/d", oidTest2, testSize2)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
node1, err := treeService.GetObjectByPath(ctx, cidTest, "/a/b")
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Equal(t, nodeID1, node1.ID)
|
||||||
|
require.Equal(t, oidTest1, node1.ObjID)
|
||||||
|
require.Equal(t, testSize1, node1.PayloadSize)
|
||||||
|
|
||||||
|
node2, err := treeService.GetObjectByPath(ctx, cidTest, "/a/b/c/d")
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Equal(t, nodeID2, node2.ID)
|
||||||
|
require.Equal(t, oidTest2, node2.ObjID)
|
||||||
|
require.Equal(t, testSize2, node2.PayloadSize)
|
||||||
|
|
||||||
|
_, err = treeService.GetObjectByPathDir(ctx, cidTest, "/g")
|
||||||
|
require.ErrorIs(t, err, ErrNodeNotFound)
|
||||||
|
|
||||||
|
_, err = treeService.GetObjectByPathDir(ctx, cidTest, "/a/b/c")
|
||||||
|
require.ErrorIs(t, err, ErrOnlyDirFound)
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGetListObjectByPrefix(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
memCli, err := NewTreeServiceClientMemory()
|
||||||
|
require.NoError(t, err)
|
||||||
|
treeService := NewTree(memCli, zaptest.NewLogger(t))
|
||||||
|
|
||||||
|
cidTest := cidtest.ID()
|
||||||
|
oidTest1 := oidtest.ID()
|
||||||
|
oidTest2 := oidtest.ID()
|
||||||
|
oidTest3 := oidtest.ID()
|
||||||
|
oidTest4 := oidtest.ID()
|
||||||
|
testSize1 := uint64(10)
|
||||||
|
testSize2 := uint64(20)
|
||||||
|
testSize3 := uint64(30)
|
||||||
|
testSize4 := uint64(40)
|
||||||
|
|
||||||
|
_, err = treeService.AddObject(ctx, cidTest, "/a/b", oidTest1, testSize1)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
nodeID2, err := treeService.AddObject(ctx, cidTest, "/a/b/c", oidTest2, testSize2)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
nodeID3, err := treeService.AddObject(ctx, cidTest, "/a/b/c/d", oidTest3, testSize3)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
nodeID4, err := treeService.AddObject(ctx, cidTest, "/a/b/c/d/e", oidTest4, testSize4)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
nodes, err := treeService.GetListObjectByPrefix(ctx, cidTest, "/a/b")
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Equal(t, nodeID2, nodes[0].ID)
|
||||||
|
require.Equal(t, nodeID3, nodes[1].ID)
|
||||||
|
require.Equal(t, nodeID4, nodes[2].ID)
|
||||||
|
require.Equal(t, testSize2, nodes[0].PayloadSize)
|
||||||
|
require.Equal(t, testSize3, nodes[1].PayloadSize)
|
||||||
|
require.Equal(t, testSize4, nodes[2].PayloadSize)
|
||||||
|
|
||||||
|
nodes, err = treeService.GetListObjectByPrefix(ctx, cidTest, "/g/s")
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Equal(t, 0, len(nodes))
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGetListOIDBySplitID(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
memCli, err := NewTreeServiceClientMemory()
|
||||||
|
require.NoError(t, err)
|
||||||
|
treeService := NewTree(memCli, zaptest.NewLogger(t))
|
||||||
|
|
||||||
|
cidTest := cidtest.ID()
|
||||||
|
oidTest1 := oidtest.ID()
|
||||||
|
oidTest2 := oidtest.ID()
|
||||||
|
oidTest3 := oidtest.ID()
|
||||||
|
|
||||||
|
splitID := object.NewSplitID()
|
||||||
|
splitID.SetUUID(uuid.New())
|
||||||
|
|
||||||
|
_, err = treeService.AddPHYObject(ctx, cidTest, "/a/b", oidTest1, splitID)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
_, err = treeService.AddPHYObject(ctx, cidTest, "/a/b", oidTest2, splitID)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
_, err = treeService.AddPHYObject(ctx, cidTest, "/a/b", oidTest3, splitID)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
ids, err := treeService.GetListOIDBySplitID(ctx, cidTest, "/a/b", splitID)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Equal(t, oidTest1, ids[0])
|
||||||
|
require.Equal(t, oidTest2, ids[1])
|
||||||
|
require.Equal(t, oidTest3, ids[2])
|
||||||
|
|
||||||
|
ids, err = treeService.GetListOIDBySplitID(ctx, cidTest, "/c/d", splitID)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Equal(t, 0, len(ids))
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDeleteObject(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
memCli, err := NewTreeServiceClientMemory()
|
||||||
|
require.NoError(t, err)
|
||||||
|
treeService := NewTree(memCli, zaptest.NewLogger(t))
|
||||||
|
|
||||||
|
cidTest := cidtest.ID()
|
||||||
|
oidTest1 := oidtest.ID()
|
||||||
|
testSize := uint64(10)
|
||||||
|
|
||||||
|
nodeID1, err := treeService.AddObject(ctx, cidTest, "/a/b", oidTest1, testSize)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
err = treeService.DeleteObject(ctx, cidTest, nodeID1)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
_, err = treeService.GetObjectByPath(ctx, cidTest, "/a/b")
|
||||||
|
require.ErrorIs(t, err, ErrNodeNotFound)
|
||||||
|
|
||||||
|
err = treeService.DeleteObject(ctx, cidTest, nodeID1+1)
|
||||||
|
require.ErrorIs(t, err, ErrNodeNotFound)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDeleteObjectsBySplitID(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
memCli, err := NewTreeServiceClientMemory()
|
||||||
|
require.NoError(t, err)
|
||||||
|
treeService := NewTree(memCli, zaptest.NewLogger(t))
|
||||||
|
|
||||||
|
cidTest := cidtest.ID()
|
||||||
|
oidTest1 := oidtest.ID()
|
||||||
|
oidTest2 := oidtest.ID()
|
||||||
|
oidTest3 := oidtest.ID()
|
||||||
|
|
||||||
|
splitID := object.NewSplitID()
|
||||||
|
splitID.SetUUID(uuid.New())
|
||||||
|
|
||||||
|
_, err = treeService.AddPHYObject(ctx, cidTest, "/a/b", oidTest1, splitID)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
_, err = treeService.AddPHYObject(ctx, cidTest, "/a/b", oidTest2, splitID)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
_, err = treeService.AddPHYObject(ctx, cidTest, "/a/b", oidTest3, splitID)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
ids, err := treeService.GetListOIDBySplitID(ctx, cidTest, "/a/b", splitID)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Equal(t, 3, len(ids))
|
||||||
|
|
||||||
|
err = treeService.DeleteObjectsBySplitID(ctx, cidTest, "/a/b", splitID)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
ids, err = treeService.GetListOIDBySplitID(ctx, cidTest, "/a/b", splitID)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Equal(t, 0, len(ids))
|
||||||
|
|
||||||
|
ids, err = treeService.GetListOIDBySplitID(ctx, cidTest, "/c/d", splitID)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Equal(t, 0, len(ids))
|
||||||
|
}
|
Loading…
Reference in a new issue