TrueCloudLab/neo-go
8
0
Fork 0
mirror of https://github.com/nspcc-dev/neo-go.git synced 2024-12-01 13:49:13 +00:00
Code Issues Projects Releases Packages Wiki Activity

*: migrate to SEARCH with strict equality comparator

Browse source 
Close #3670

Signed-off-by: Ekaterina Pavlova <ekt@morphbits.io>
This commit is contained in:
Ekaterina Pavlova 2024-11-21 17:56:55 +03:00
parent 4b6db2dc33
commit c1311871d6
2 changed files with 22 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
cli/util/upload_bin.go
View file

@ -38,7 +38,8 @@ const (
uploadBatchSize = 10000 uploadBatchSize = 10000
// Number of objects to search in a batch. // Number of objects to search in a batch.
// If it is larger than uploadBatchSize, it may lead to many duplicate uploads. // If it is larger than uploadBatchSize, it may lead to many duplicate uploads.
searchBatchSize = uploadBatchSize // We need to search with EQ filter to avoid partially-completed SEARCH responses.
searchBatchSize = 1
// Size of object ID. // Size of object ID.
oidSize = sha256.Size oidSize = sha256.Size
) )
@ -215,8 +216,12 @@ func fetchLatestMissingBlockIndex(ctx context.Context, p *pool.Pool, containerID
prm := client.PrmObjectSearch{} prm := client.PrmObjectSearch{}
filters := object.NewSearchFilters() filters := object.NewSearchFilters()
if endIndex == startIndex+1 {
filters.AddFilter(attributeKey, fmt.Sprintf("%d", startIndex), object.MatchStringEqual)
} else {
filters.AddFilter(attributeKey, fmt.Sprintf("%d", startIndex), object.MatchNumGE) filters.AddFilter(attributeKey, fmt.Sprintf("%d", startIndex), object.MatchNumGE)
filters.AddFilter(attributeKey, fmt.Sprintf("%d", endIndex), object.MatchNumLT) filters.AddFilter(attributeKey, fmt.Sprintf("%d", endIndex), object.MatchNumLT)
}
prm.SetFilters(filters) prm.SetFilters(filters)
var ( var (
objectIDs []oid.ID objectIDs []oid.ID
@ -528,8 +533,12 @@ func searchObjects(ctx context.Context, p *pool.Pool, containerID cid.ID, accoun
if len(additionalFilters) != 0 { if len(additionalFilters) != 0 {
filters = additionalFilters[0] filters = additionalFilters[0]
} }
if end == start+1 {
filters.AddFilter(blockAttributeKey, fmt.Sprintf("%d", start), object.MatchStringEqual)
} else {
filters.AddFilter(blockAttributeKey, fmt.Sprintf("%d", start), object.MatchNumGE) filters.AddFilter(blockAttributeKey, fmt.Sprintf("%d", start), object.MatchNumGE)
filters.AddFilter(blockAttributeKey, fmt.Sprintf("%d", end), object.MatchNumLT) filters.AddFilter(blockAttributeKey, fmt.Sprintf("%d", end), object.MatchNumLT)
}
prm.SetFilters(filters) prm.SetFilters(filters)
var objIDs []oid.ID var objIDs []oid.ID

6
pkg/services/blockfetcher/blockfetcher.go
View file

@ -179,6 +179,8 @@ func (bfs *Service) oidDownloader() {
var err error var err error
if bfs.cfg.SkipIndexFilesSearch { if bfs.cfg.SkipIndexFilesSearch {
//We need to search with EQ filter to avoid partially-completed SEARCH responses.
bfs.cfg.OIDBatchSize = 1
err = bfs.fetchOIDsBySearch() err = bfs.fetchOIDsBySearch()
} else { } else {
err = bfs.fetchOIDsFromIndexFiles() err = bfs.fetchOIDsFromIndexFiles()
@ -351,8 +353,12 @@ func (bfs *Service) fetchOIDsBySearch() error {
default: default:
prm := client.PrmObjectSearch{} prm := client.PrmObjectSearch{}
filters := object.NewSearchFilters() filters := object.NewSearchFilters()
if startIndex == startIndex+batchSize-1 {
filters.AddFilter(bfs.cfg.BlockAttribute, fmt.Sprintf("%d", startIndex), object.MatchStringEqual)
} else {
filters.AddFilter(bfs.cfg.BlockAttribute, fmt.Sprintf("%d", startIndex), object.MatchNumGE) filters.AddFilter(bfs.cfg.BlockAttribute, fmt.Sprintf("%d", startIndex), object.MatchNumGE)
filters.AddFilter(bfs.cfg.BlockAttribute, fmt.Sprintf("%d", startIndex+batchSize-1), object.MatchNumLE) filters.AddFilter(bfs.cfg.BlockAttribute, fmt.Sprintf("%d", startIndex+batchSize-1), object.MatchNumLE)
}
prm.SetFilters(filters) prm.SetFilters(filters)
ctx, cancel := context.WithTimeout(bfs.ctx, bfs.cfg.Timeout) ctx, cancel := context.WithTimeout(bfs.ctx, bfs.cfg.Timeout)
blockOids, err := bfs.objectSearch(ctx, prm) blockOids, err := bfs.objectSearch(ctx, prm)