|
|
@ -45,7 +45,7 @@ type (
|
|
|
|
// ListObjectsInfo contains common fields of data for ListObjectsV1 and ListObjectsV2.
|
|
|
|
// ListObjectsInfo contains common fields of data for ListObjectsV1 and ListObjectsV2.
|
|
|
|
ListObjectsInfo struct {
|
|
|
|
ListObjectsInfo struct {
|
|
|
|
Prefixes []string
|
|
|
|
Prefixes []string
|
|
|
|
Objects []*data.NodeVersion
|
|
|
|
Objects []*data.ExtendedNodeVersion
|
|
|
|
IsTruncated bool
|
|
|
|
IsTruncated bool
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
@ -118,10 +118,10 @@ func (n *layer) ListObjectsV1(ctx context.Context, p *ListObjectsParamsV1) (*Lis
|
|
|
|
|
|
|
|
|
|
|
|
if next != nil {
|
|
|
|
if next != nil {
|
|
|
|
result.IsTruncated = true
|
|
|
|
result.IsTruncated = true
|
|
|
|
result.NextMarker = objects[len(objects)-1].FilePath
|
|
|
|
result.NextMarker = objects[len(objects)-1].Name()
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
result.Prefixes, result.Objects = triageObjects(objects, p.Prefix, p.Delimiter)
|
|
|
|
result.Prefixes, result.Objects = triageExtendedObjects(objects)
|
|
|
|
|
|
|
|
|
|
|
|
return &result, nil
|
|
|
|
return &result, nil
|
|
|
|
}
|
|
|
|
}
|
|
|
@ -149,10 +149,10 @@ func (n *layer) ListObjectsV2(ctx context.Context, p *ListObjectsParamsV2) (*Lis
|
|
|
|
|
|
|
|
|
|
|
|
if next != nil {
|
|
|
|
if next != nil {
|
|
|
|
result.IsTruncated = true
|
|
|
|
result.IsTruncated = true
|
|
|
|
result.NextContinuationToken = next.OID.EncodeToString()
|
|
|
|
result.NextContinuationToken = next.NodeVersion.OID.EncodeToString()
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
result.Prefixes, result.Objects = triageObjects(objects, p.Prefix, p.Delimiter)
|
|
|
|
result.Prefixes, result.Objects = triageExtendedObjects(objects)
|
|
|
|
|
|
|
|
|
|
|
|
return &result, nil
|
|
|
|
return &result, nil
|
|
|
|
}
|
|
|
|
}
|
|
|
@ -183,12 +183,12 @@ func (n *layer) ListObjectVersions(ctx context.Context, p *ListObjectVersionsPar
|
|
|
|
res.NextVersionIDMarker = objects[p.MaxKeys-1].NodeVersion.OID.EncodeToString()
|
|
|
|
res.NextVersionIDMarker = objects[p.MaxKeys-1].NodeVersion.OID.EncodeToString()
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
res.CommonPrefixes, objects = triageExtendedObjects(objects, p.Prefix, p.Delimiter)
|
|
|
|
res.CommonPrefixes, objects = triageExtendedObjects(objects)
|
|
|
|
res.Version, res.DeleteMarker = triageVersions(objects)
|
|
|
|
res.Version, res.DeleteMarker = triageVersions(objects)
|
|
|
|
return res, nil
|
|
|
|
return res, nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
func (n *layer) getLatestObjectsVersions(ctx context.Context, p commonLatestVersionsListingParams) (objects []*data.NodeVersion, next *data.NodeVersion, err error) {
|
|
|
|
func (n *layer) getLatestObjectsVersions(ctx context.Context, p commonLatestVersionsListingParams) (objects []*data.ExtendedNodeVersion, next *data.ExtendedNodeVersion, err error) {
|
|
|
|
if p.MaxKeys == 0 {
|
|
|
|
if p.MaxKeys == 0 {
|
|
|
|
return nil, nil, nil
|
|
|
|
return nil, nil, nil
|
|
|
|
}
|
|
|
|
}
|
|
|
@ -204,7 +204,7 @@ func (n *layer) getLatestObjectsVersions(ctx context.Context, p commonLatestVers
|
|
|
|
return nil, nil, fmt.Errorf("failed to init worker pool: %w", err)
|
|
|
|
return nil, nil, fmt.Errorf("failed to init worker pool: %w", err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
objects = make([]*data.NodeVersion, 0, p.MaxKeys+1)
|
|
|
|
objects = make([]*data.ExtendedNodeVersion, 0, p.MaxKeys+1)
|
|
|
|
objects = append(objects, session.Next...)
|
|
|
|
objects = append(objects, session.Next...)
|
|
|
|
for obj := range objOutCh {
|
|
|
|
for obj := range objOutCh {
|
|
|
|
objects = append(objects, obj)
|
|
|
|
objects = append(objects, obj)
|
|
|
@ -261,17 +261,15 @@ func handleGeneratedVersions(objOutCh <-chan *data.ExtendedNodeVersion, p common
|
|
|
|
allObjects := make([]*data.ExtendedNodeVersion, 0, p.MaxKeys)
|
|
|
|
allObjects := make([]*data.ExtendedNodeVersion, 0, p.MaxKeys)
|
|
|
|
for eoi := range objOutCh {
|
|
|
|
for eoi := range objOutCh {
|
|
|
|
name := eoi.NodeVersion.FilePath
|
|
|
|
name := eoi.NodeVersion.FilePath
|
|
|
|
|
|
|
|
if eoi.DirName != "" {
|
|
|
|
dirName := tryDirectoryName(eoi.NodeVersion, p.Prefix, p.Delimiter)
|
|
|
|
name = eoi.DirName
|
|
|
|
if dirName != "" {
|
|
|
|
|
|
|
|
name = dirName
|
|
|
|
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
if lastName != name {
|
|
|
|
if lastName != name {
|
|
|
|
formVersionsListRow(allObjects, listRowStartIndex, session)
|
|
|
|
formVersionsListRow(allObjects, listRowStartIndex, session)
|
|
|
|
listRowStartIndex = len(allObjects)
|
|
|
|
listRowStartIndex = len(allObjects)
|
|
|
|
allObjects = append(allObjects, eoi)
|
|
|
|
allObjects = append(allObjects, eoi)
|
|
|
|
} else if dirName == "" {
|
|
|
|
} else if eoi.DirName == "" {
|
|
|
|
allObjects = append(allObjects, eoi)
|
|
|
|
allObjects = append(allObjects, eoi)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
lastName = name
|
|
|
|
lastName = name
|
|
|
@ -292,7 +290,11 @@ func formVersionsListRow(objects []*data.ExtendedNodeVersion, rowStartIndex int,
|
|
|
|
return prevVersions[j].NodeVersion.Timestamp < prevVersions[i].NodeVersion.Timestamp // sort in reverse order to have last added first
|
|
|
|
return prevVersions[j].NodeVersion.Timestamp < prevVersions[i].NodeVersion.Timestamp // sort in reverse order to have last added first
|
|
|
|
})
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
|
|
objects[rowStartIndex].IsLatest = len(session.Next) == 0 || session.Next[0].FilePath != objects[rowStartIndex].NodeVersion.FilePath
|
|
|
|
prevVersions[0].IsLatest = len(session.Next) == 0 || session.Next[0].NodeVersion.FilePath != prevVersions[0].NodeVersion.FilePath
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
for _, version := range prevVersions[1:] {
|
|
|
|
|
|
|
|
version.IsLatest = false
|
|
|
|
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
func (n *layer) getListLatestVersionsSession(ctx context.Context, p commonLatestVersionsListingParams) (*data.ListSession, error) {
|
|
|
|
func (n *layer) getListLatestVersionsSession(ctx context.Context, p commonLatestVersionsListingParams) (*data.ListSession, error) {
|
|
|
@ -340,7 +342,7 @@ func (n *layer) initNewVersionsByPrefixSession(ctx context.Context, p commonVers
|
|
|
|
return session, nil
|
|
|
|
return session, nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
func (n *layer) putListLatestVersionsSession(ctx context.Context, p commonLatestVersionsListingParams, session *data.ListSession, allObjects []*data.NodeVersion) {
|
|
|
|
func (n *layer) putListLatestVersionsSession(ctx context.Context, p commonLatestVersionsListingParams, session *data.ListSession, allObjects []*data.ExtendedNodeVersion) {
|
|
|
|
if len(allObjects) <= p.MaxKeys {
|
|
|
|
if len(allObjects) <= p.MaxKeys {
|
|
|
|
return
|
|
|
|
return
|
|
|
|
}
|
|
|
|
}
|
|
|
@ -348,16 +350,16 @@ func (n *layer) putListLatestVersionsSession(ctx context.Context, p commonLatest
|
|
|
|
var cacheKey cache.ListSessionKey
|
|
|
|
var cacheKey cache.ListSessionKey
|
|
|
|
switch p.ListType {
|
|
|
|
switch p.ListType {
|
|
|
|
case ListObjectsV1Type:
|
|
|
|
case ListObjectsV1Type:
|
|
|
|
cacheKey = cache.CreateListSessionCacheKey(p.BktInfo.CID, p.Prefix, allObjects[p.MaxKeys-1].FilePath)
|
|
|
|
cacheKey = cache.CreateListSessionCacheKey(p.BktInfo.CID, p.Prefix, allObjects[p.MaxKeys-1].Name())
|
|
|
|
case ListObjectsV2Type:
|
|
|
|
case ListObjectsV2Type:
|
|
|
|
cacheKey = cache.CreateListSessionCacheKey(p.BktInfo.CID, p.Prefix, allObjects[p.MaxKeys].OID.EncodeToString())
|
|
|
|
cacheKey = cache.CreateListSessionCacheKey(p.BktInfo.CID, p.Prefix, allObjects[p.MaxKeys].NodeVersion.OID.EncodeToString())
|
|
|
|
default:
|
|
|
|
default:
|
|
|
|
// should never happen
|
|
|
|
// should never happen
|
|
|
|
panic("invalid list type")
|
|
|
|
panic("invalid list type")
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
session.Acquired.Store(false)
|
|
|
|
session.Acquired.Store(false)
|
|
|
|
session.Next = []*data.NodeVersion{allObjects[p.MaxKeys]}
|
|
|
|
session.Next = []*data.ExtendedNodeVersion{allObjects[p.MaxKeys]}
|
|
|
|
n.cache.PutListSession(n.BearerOwner(ctx), cacheKey, session)
|
|
|
|
n.cache.PutListSession(n.BearerOwner(ctx), cacheKey, session)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
@ -368,20 +370,19 @@ func (n *layer) putListAllVersionsSession(ctx context.Context, p commonVersionsL
|
|
|
|
|
|
|
|
|
|
|
|
session.Acquired.Store(false)
|
|
|
|
session.Acquired.Store(false)
|
|
|
|
|
|
|
|
|
|
|
|
session.Next = make([]*data.NodeVersion, len(allObjects)-p.MaxKeys+1)
|
|
|
|
session.Next = make([]*data.ExtendedNodeVersion, len(allObjects)-p.MaxKeys+1)
|
|
|
|
session.Next[0] = allObjects[p.MaxKeys-1].NodeVersion
|
|
|
|
session.Next[0] = allObjects[p.MaxKeys-1]
|
|
|
|
for i, node := range allObjects[p.MaxKeys:] {
|
|
|
|
for i, node := range allObjects[p.MaxKeys:] {
|
|
|
|
session.Next[i+1] = node.NodeVersion
|
|
|
|
session.Next[i+1] = node
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
cacheKey := cache.CreateListSessionCacheKey(p.BktInfo.CID, p.Prefix, session.Next[0].OID.EncodeToString())
|
|
|
|
cacheKey := cache.CreateListSessionCacheKey(p.BktInfo.CID, p.Prefix, session.Next[0].NodeVersion.OID.EncodeToString())
|
|
|
|
n.cache.PutListSession(n.BearerOwner(ctx), cacheKey, session)
|
|
|
|
n.cache.PutListSession(n.BearerOwner(ctx), cacheKey, session)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
func nodesGeneratorStream(ctx context.Context, p commonVersionsListingParams, stream *data.ListSession) (<-chan *data.NodeVersion, <-chan error) {
|
|
|
|
func nodesGeneratorStream(ctx context.Context, p commonVersionsListingParams, stream *data.ListSession) (<-chan *data.ExtendedNodeVersion, <-chan error) {
|
|
|
|
nodeCh := make(chan *data.NodeVersion, 1000)
|
|
|
|
nodeCh := make(chan *data.ExtendedNodeVersion, 1000)
|
|
|
|
errCh := make(chan error, 1)
|
|
|
|
errCh := make(chan error, 1)
|
|
|
|
//existed := make(map[string]struct{}, p.MaxKeys) // to squash the same directories
|
|
|
|
|
|
|
|
existed := stream.NamesMap
|
|
|
|
existed := stream.NamesMap
|
|
|
|
|
|
|
|
|
|
|
|
if len(stream.Next) != 0 {
|
|
|
|
if len(stream.Next) != 0 {
|
|
|
@ -407,14 +408,20 @@ func nodesGeneratorStream(ctx context.Context, p commonVersionsListingParams, st
|
|
|
|
break LOOP
|
|
|
|
break LOOP
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
if shouldSkip(node, p, existed) {
|
|
|
|
nodeExt := &data.ExtendedNodeVersion{
|
|
|
|
|
|
|
|
NodeVersion: node,
|
|
|
|
|
|
|
|
IsLatest: true,
|
|
|
|
|
|
|
|
DirName: tryDirectoryName(node, p.Prefix, p.Delimiter),
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if shouldSkip(nodeExt, p, existed) {
|
|
|
|
continue
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
select {
|
|
|
|
select {
|
|
|
|
case <-ctx.Done():
|
|
|
|
case <-ctx.Done():
|
|
|
|
break LOOP
|
|
|
|
break LOOP
|
|
|
|
case nodeCh <- node:
|
|
|
|
case nodeCh <- nodeExt:
|
|
|
|
generated++
|
|
|
|
generated++
|
|
|
|
|
|
|
|
|
|
|
|
if generated == limit { // we use maxKeys+1 to be able to know nextMarker/nextContinuationToken
|
|
|
|
if generated == limit { // we use maxKeys+1 to be able to know nextMarker/nextContinuationToken
|
|
|
@ -429,8 +436,8 @@ func nodesGeneratorStream(ctx context.Context, p commonVersionsListingParams, st
|
|
|
|
return nodeCh, errCh
|
|
|
|
return nodeCh, errCh
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
func nodesGeneratorVersions(ctx context.Context, p commonVersionsListingParams, stream *data.ListSession) (<-chan *data.NodeVersion, <-chan error) {
|
|
|
|
func nodesGeneratorVersions(ctx context.Context, p commonVersionsListingParams, stream *data.ListSession) (<-chan *data.ExtendedNodeVersion, <-chan error) {
|
|
|
|
nodeCh := make(chan *data.NodeVersion, 1000)
|
|
|
|
nodeCh := make(chan *data.ExtendedNodeVersion, 1000)
|
|
|
|
errCh := make(chan error, 1)
|
|
|
|
errCh := make(chan error, 1)
|
|
|
|
existed := stream.NamesMap
|
|
|
|
existed := stream.NamesMap
|
|
|
|
|
|
|
|
|
|
|
@ -442,13 +449,14 @@ func nodesGeneratorVersions(ctx context.Context, p commonVersionsListingParams,
|
|
|
|
ind int
|
|
|
|
ind int
|
|
|
|
err error
|
|
|
|
err error
|
|
|
|
lastName string
|
|
|
|
lastName string
|
|
|
|
|
|
|
|
node *data.NodeVersion
|
|
|
|
|
|
|
|
nodeExt *data.ExtendedNodeVersion
|
|
|
|
)
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
LOOP:
|
|
|
|
LOOP:
|
|
|
|
for err == nil {
|
|
|
|
for err == nil {
|
|
|
|
var node *data.NodeVersion
|
|
|
|
|
|
|
|
if ind < len(stream.Next) {
|
|
|
|
if ind < len(stream.Next) {
|
|
|
|
node = stream.Next[ind]
|
|
|
|
nodeExt = stream.Next[ind]
|
|
|
|
ind++
|
|
|
|
ind++
|
|
|
|
} else {
|
|
|
|
} else {
|
|
|
|
node, err = stream.Stream.Next(ctx)
|
|
|
|
node, err = stream.Stream.Next(ctx)
|
|
|
@ -458,21 +466,26 @@ func nodesGeneratorVersions(ctx context.Context, p commonVersionsListingParams,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
break LOOP
|
|
|
|
break LOOP
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
nodeExt = &data.ExtendedNodeVersion{
|
|
|
|
|
|
|
|
NodeVersion: node,
|
|
|
|
|
|
|
|
DirName: tryDirectoryName(node, p.Prefix, p.Delimiter),
|
|
|
|
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
if shouldSkipVersions(node, p, existed) {
|
|
|
|
if shouldSkipVersions(nodeExt, p, existed) {
|
|
|
|
continue
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
select {
|
|
|
|
select {
|
|
|
|
case <-ctx.Done():
|
|
|
|
case <-ctx.Done():
|
|
|
|
break LOOP
|
|
|
|
break LOOP
|
|
|
|
case nodeCh <- node:
|
|
|
|
case nodeCh <- nodeExt:
|
|
|
|
generated++
|
|
|
|
generated++
|
|
|
|
if generated > p.MaxKeys && node.FilePath != lastName {
|
|
|
|
if generated > p.MaxKeys && nodeExt.NodeVersion.FilePath != lastName {
|
|
|
|
break LOOP
|
|
|
|
break LOOP
|
|
|
|
}
|
|
|
|
}
|
|
|
|
lastName = node.FilePath
|
|
|
|
lastName = nodeExt.NodeVersion.FilePath
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
close(nodeCh)
|
|
|
|
close(nodeCh)
|
|
|
@ -482,13 +495,13 @@ func nodesGeneratorVersions(ctx context.Context, p commonVersionsListingParams,
|
|
|
|
return nodeCh, errCh
|
|
|
|
return nodeCh, errCh
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
func (n *layer) initWorkerPoolStream(ctx context.Context, size int, p commonVersionsListingParams, input <-chan *data.NodeVersion) (<-chan *data.NodeVersion, error) {
|
|
|
|
func (n *layer) initWorkerPoolStream(ctx context.Context, size int, p commonVersionsListingParams, input <-chan *data.ExtendedNodeVersion) (<-chan *data.ExtendedNodeVersion, error) {
|
|
|
|
reqLog := n.reqLogger(ctx)
|
|
|
|
reqLog := n.reqLogger(ctx)
|
|
|
|
pool, err := ants.NewPool(size, ants.WithLogger(&logWrapper{reqLog}))
|
|
|
|
pool, err := ants.NewPool(size, ants.WithLogger(&logWrapper{reqLog}))
|
|
|
|
if err != nil {
|
|
|
|
if err != nil {
|
|
|
|
return nil, fmt.Errorf("coudln't init go pool for listing: %w", err)
|
|
|
|
return nil, fmt.Errorf("coudln't init go pool for listing: %w", err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
objCh := make(chan *data.NodeVersion, size)
|
|
|
|
objCh := make(chan *data.ExtendedNodeVersion, size)
|
|
|
|
|
|
|
|
|
|
|
|
go func() {
|
|
|
|
go func() {
|
|
|
|
var wg sync.WaitGroup
|
|
|
|
var wg sync.WaitGroup
|
|
|
@ -501,10 +514,7 @@ func (n *layer) initWorkerPoolStream(ctx context.Context, size int, p commonVers
|
|
|
|
default:
|
|
|
|
default:
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
if dirName := tryDirectoryName(node, p.Prefix, p.Delimiter); dirName != "" || node.IsFilledExtra() { // todo think to not compute twice
|
|
|
|
if node.DirName != "" || node.NodeVersion.IsFilledExtra() {
|
|
|
|
if dirName != "" {
|
|
|
|
|
|
|
|
node.FilePath = dirName
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
select {
|
|
|
|
select {
|
|
|
|
case <-ctx.Done():
|
|
|
|
case <-ctx.Done():
|
|
|
|
case objCh <- node:
|
|
|
|
case objCh <- node:
|
|
|
@ -512,20 +522,20 @@ func (n *layer) initWorkerPoolStream(ctx context.Context, size int, p commonVers
|
|
|
|
} else {
|
|
|
|
} else {
|
|
|
|
// We have to make a copy of pointer to data.NodeVersion
|
|
|
|
// We have to make a copy of pointer to data.NodeVersion
|
|
|
|
// to get correct value in submitted task function.
|
|
|
|
// to get correct value in submitted task function.
|
|
|
|
func(node *data.NodeVersion) {
|
|
|
|
func(node *data.ExtendedNodeVersion) {
|
|
|
|
wg.Add(1)
|
|
|
|
wg.Add(1)
|
|
|
|
err = pool.Submit(func() {
|
|
|
|
err = pool.Submit(func() {
|
|
|
|
defer wg.Done()
|
|
|
|
defer wg.Done()
|
|
|
|
oi := n.objectInfoFromObjectsCacheOrFrostFS(ctx, p.BktInfo, node)
|
|
|
|
oi := n.objectInfoFromObjectsCacheOrFrostFS(ctx, p.BktInfo, node.NodeVersion)
|
|
|
|
if oi == nil {
|
|
|
|
if oi == nil {
|
|
|
|
// try to get object again
|
|
|
|
// try to get object again
|
|
|
|
if oi = n.objectInfoFromObjectsCacheOrFrostFS(ctx, p.BktInfo, node); oi == nil {
|
|
|
|
if oi = n.objectInfoFromObjectsCacheOrFrostFS(ctx, p.BktInfo, node.NodeVersion); oi == nil {
|
|
|
|
// do not process object which are definitely missing in object service
|
|
|
|
// do not process object which are definitely missing in object service
|
|
|
|
return
|
|
|
|
return
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
node.FillExtra(oi)
|
|
|
|
node.NodeVersion.FillExtra(oi)
|
|
|
|
|
|
|
|
|
|
|
|
select {
|
|
|
|
select {
|
|
|
|
case <-ctx.Done():
|
|
|
|
case <-ctx.Done():
|
|
|
@ -547,7 +557,7 @@ func (n *layer) initWorkerPoolStream(ctx context.Context, size int, p commonVers
|
|
|
|
return objCh, nil
|
|
|
|
return objCh, nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
func (n *layer) initWorkerPoolVersions(ctx context.Context, size int, p commonVersionsListingParams, input <-chan *data.NodeVersion) (<-chan *data.ExtendedNodeVersion, error) {
|
|
|
|
func (n *layer) initWorkerPoolVersions(ctx context.Context, size int, p commonVersionsListingParams, input <-chan *data.ExtendedNodeVersion) (<-chan *data.ExtendedNodeVersion, error) {
|
|
|
|
reqLog := n.reqLogger(ctx)
|
|
|
|
reqLog := n.reqLogger(ctx)
|
|
|
|
pool, err := ants.NewPool(size, ants.WithLogger(&logWrapper{reqLog}))
|
|
|
|
pool, err := ants.NewPool(size, ants.WithLogger(&logWrapper{reqLog}))
|
|
|
|
if err != nil {
|
|
|
|
if err != nil {
|
|
|
@ -566,33 +576,33 @@ func (n *layer) initWorkerPoolVersions(ctx context.Context, size int, p commonVe
|
|
|
|
default:
|
|
|
|
default:
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
if node.IsFilledExtra() {
|
|
|
|
if node.DirName != "" || node.NodeVersion.IsFilledExtra() {
|
|
|
|
select {
|
|
|
|
select {
|
|
|
|
case <-ctx.Done():
|
|
|
|
case <-ctx.Done():
|
|
|
|
case objCh <- &data.ExtendedNodeVersion{NodeVersion: node}:
|
|
|
|
case objCh <- node:
|
|
|
|
}
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
} else {
|
|
|
|
// We have to make a copy of pointer to data.NodeVersion
|
|
|
|
// We have to make a copy of pointer to data.NodeVersion
|
|
|
|
// to get correct value in submitted task function.
|
|
|
|
// to get correct value in submitted task function.
|
|
|
|
func(node *data.NodeVersion) {
|
|
|
|
func(node *data.ExtendedNodeVersion) {
|
|
|
|
wg.Add(1)
|
|
|
|
wg.Add(1)
|
|
|
|
err = pool.Submit(func() {
|
|
|
|
err = pool.Submit(func() {
|
|
|
|
defer wg.Done()
|
|
|
|
defer wg.Done()
|
|
|
|
|
|
|
|
|
|
|
|
oi := n.objectInfoFromObjectsCacheOrFrostFS(ctx, p.BktInfo, node)
|
|
|
|
oi := n.objectInfoFromObjectsCacheOrFrostFS(ctx, p.BktInfo, node.NodeVersion)
|
|
|
|
if oi == nil {
|
|
|
|
if oi == nil {
|
|
|
|
// try to get object again
|
|
|
|
// try to get object again
|
|
|
|
if oi = n.objectInfoFromObjectsCacheOrFrostFS(ctx, p.BktInfo, node); oi == nil {
|
|
|
|
if oi = n.objectInfoFromObjectsCacheOrFrostFS(ctx, p.BktInfo, node.NodeVersion); oi == nil {
|
|
|
|
// do not process object which are definitely missing in object service
|
|
|
|
// do not process object which are definitely missing in object service
|
|
|
|
return
|
|
|
|
return
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
node.FillExtra(oi)
|
|
|
|
node.NodeVersion.FillExtra(oi)
|
|
|
|
|
|
|
|
|
|
|
|
select {
|
|
|
|
select {
|
|
|
|
case <-ctx.Done():
|
|
|
|
case <-ctx.Done():
|
|
|
|
case objCh <- &data.ExtendedNodeVersion{NodeVersion: node}:
|
|
|
|
case objCh <- node:
|
|
|
|
}
|
|
|
|
}
|
|
|
|
})
|
|
|
|
})
|
|
|
|
if err != nil {
|
|
|
|
if err != nil {
|
|
|
@ -610,15 +620,16 @@ func (n *layer) initWorkerPoolVersions(ctx context.Context, size int, p commonVe
|
|
|
|
return objCh, nil
|
|
|
|
return objCh, nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
func shouldSkip(node *data.NodeVersion, p commonVersionsListingParams, existed map[string]struct{}) bool {
|
|
|
|
func shouldSkip(node *data.ExtendedNodeVersion, p commonVersionsListingParams, existed map[string]struct{}) bool {
|
|
|
|
if node.IsDeleteMarker {
|
|
|
|
if node.NodeVersion.IsDeleteMarker {
|
|
|
|
return true
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
filePath := node.FilePath
|
|
|
|
filePath := node.NodeVersion.FilePath
|
|
|
|
if dirName := tryDirectoryName(node, p.Prefix, p.Delimiter); len(dirName) != 0 {
|
|
|
|
if node.DirName != "" {
|
|
|
|
filePath = dirName
|
|
|
|
filePath = node.DirName
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
if _, ok := existed[filePath]; ok {
|
|
|
|
if _, ok := existed[filePath]; ok {
|
|
|
|
return true
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
}
|
|
|
@ -629,7 +640,7 @@ func shouldSkip(node *data.NodeVersion, p commonVersionsListingParams, existed m
|
|
|
|
|
|
|
|
|
|
|
|
if p.Bookmark != "" {
|
|
|
|
if p.Bookmark != "" {
|
|
|
|
if _, ok := existed[continuationToken]; !ok {
|
|
|
|
if _, ok := existed[continuationToken]; !ok {
|
|
|
|
if p.Bookmark != node.OID.EncodeToString() {
|
|
|
|
if p.Bookmark != node.NodeVersion.OID.EncodeToString() {
|
|
|
|
return true
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
existed[continuationToken] = struct{}{}
|
|
|
|
existed[continuationToken] = struct{}{}
|
|
|
@ -640,10 +651,10 @@ func shouldSkip(node *data.NodeVersion, p commonVersionsListingParams, existed m
|
|
|
|
return false
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
func shouldSkipVersions(node *data.NodeVersion, p commonVersionsListingParams, existed map[string]struct{}) bool {
|
|
|
|
func shouldSkipVersions(node *data.ExtendedNodeVersion, p commonVersionsListingParams, existed map[string]struct{}) bool {
|
|
|
|
filePath := node.FilePath
|
|
|
|
filePath := node.NodeVersion.FilePath
|
|
|
|
if dirName := tryDirectoryName(node, p.Prefix, p.Delimiter); len(dirName) != 0 {
|
|
|
|
if node.DirName != "" {
|
|
|
|
filePath = dirName
|
|
|
|
filePath = node.DirName
|
|
|
|
if _, ok := existed[filePath]; ok {
|
|
|
|
if _, ok := existed[filePath]; ok {
|
|
|
|
return true
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
}
|
|
|
@ -655,7 +666,7 @@ func shouldSkipVersions(node *data.NodeVersion, p commonVersionsListingParams, e
|
|
|
|
|
|
|
|
|
|
|
|
if p.Bookmark != "" {
|
|
|
|
if p.Bookmark != "" {
|
|
|
|
if _, ok := existed[continuationToken]; !ok {
|
|
|
|
if _, ok := existed[continuationToken]; !ok {
|
|
|
|
if p.Bookmark != node.OID.EncodeToString() {
|
|
|
|
if p.Bookmark != node.NodeVersion.OID.EncodeToString() {
|
|
|
|
return true
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
existed[continuationToken] = struct{}{}
|
|
|
|
existed[continuationToken] = struct{}{}
|
|
|
@ -667,23 +678,10 @@ func shouldSkipVersions(node *data.NodeVersion, p commonVersionsListingParams, e
|
|
|
|
return false
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
func triageObjects(allObjects []*data.NodeVersion, prefix, delimiter string) (prefixes []string, objects []*data.NodeVersion) {
|
|
|
|
func triageExtendedObjects(allObjects []*data.ExtendedNodeVersion) (prefixes []string, objects []*data.ExtendedNodeVersion) {
|
|
|
|
objects = make([]*data.NodeVersion, 0, len(allObjects))
|
|
|
|
|
|
|
|
for _, ov := range allObjects {
|
|
|
|
for _, ov := range allObjects {
|
|
|
|
if dirName := tryDirectoryName(ov, prefix, delimiter); dirName != "" {
|
|
|
|
if ov.DirName != "" {
|
|
|
|
prefixes = append(prefixes, dirName)
|
|
|
|
prefixes = append(prefixes, ov.DirName)
|
|
|
|
} else {
|
|
|
|
|
|
|
|
objects = append(objects, ov)
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
return
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
func triageExtendedObjects(allObjects []*data.ExtendedNodeVersion, prefix, delimiter string) (prefixes []string, objects []*data.ExtendedNodeVersion) {
|
|
|
|
|
|
|
|
for _, ov := range allObjects {
|
|
|
|
|
|
|
|
if dirName := tryDirectoryName(ov.NodeVersion, prefix, delimiter); dirName != "" {
|
|
|
|
|
|
|
|
prefixes = append(prefixes, dirName)
|
|
|
|
|
|
|
|
} else {
|
|
|
|
} else {
|
|
|
|
objects = append(objects, ov)
|
|
|
|
objects = append(objects, ov)
|
|
|
|
}
|
|
|
|
}
|
|
|
|