[#19] Stop object iteration after all objects were processed

At the moment we don't need logic that swings back to beginning of registry when
all objects have been processed. So, for now we can stop iterating and return an
error when selector reaches the end of registry.

Signed-off-by: Vladimir Domnich <v.domnich@yadro.com>
This commit is contained in:
Vladimir Domnich 2022-09-22 20:57:21 +04:00 committed by Alex Vanin
parent bf4a11474e
commit b1ec6d562c
6 changed files with 21 additions and 19 deletions

View file

@ -231,7 +231,7 @@ func (c *Client) VerifyHash(containerID, objectID, expectedHash string) VerifyHa
if err != nil { if err != nil {
return VerifyHashResponse{Success: false, Error: err.Error()} return VerifyHashResponse{Success: false, Error: err.Error()}
} }
actualHash := hex.EncodeToString(hasher.Sum(make([]byte, 0, sha256.Size))) actualHash := hex.EncodeToString(hasher.Sum(nil))
if actualHash != expectedHash { if actualHash != expectedHash {
return VerifyHashResponse{Success: true, Error: "hash mismatch"} return VerifyHashResponse{Success: true, Error: "hash mismatch"}
} }

View file

@ -3,6 +3,7 @@ package registry
import ( import (
"encoding/binary" "encoding/binary"
"encoding/json" "encoding/json"
"errors"
"os" "os"
"time" "time"
@ -15,7 +16,7 @@ type ObjRegistry struct {
} }
const ( const (
// Indicates that an object was created, but its data wasn't verified yet // Indicates that an object was created, but its data wasn't verified yet.
statusCreated = "created" statusCreated = "created"
) )
@ -87,7 +88,7 @@ func (o *ObjRegistry) SetObjectStatus(id uint64, newStatus string) error {
objBytes := b.Get(encodeId(id)) objBytes := b.Get(encodeId(id))
if objBytes == nil { if objBytes == nil {
return nil return errors.New("object doesn't exist")
} }
obj := new(ObjectInfo) obj := new(ObjectInfo)
@ -112,20 +113,19 @@ func (o *ObjRegistry) GetObjectCountInStatus(status string) (int, error) {
return nil return nil
} }
c := b.Cursor() return b.ForEach(func(_, objBytes []byte) error {
for keyBytes, objBytes := c.First(); keyBytes != nil; keyBytes, objBytes = c.Next() {
if objBytes != nil { if objBytes != nil {
var obj ObjectInfo var obj ObjectInfo
if err := json.Unmarshal(objBytes, &obj); err != nil { if err := json.Unmarshal(objBytes, &obj); err != nil {
// Ignore malformed objects // Ignore malformed objects
continue return nil
} }
if obj.Status == status { if obj.Status == status {
objCount++ objCount++
} }
} }
} return nil
return nil })
}) })
return objCount, err return objCount, err
} }

View file

@ -2,6 +2,7 @@ package registry
import ( import (
"encoding/json" "encoding/json"
"errors"
"sync" "sync"
"go.etcd.io/bbolt" "go.etcd.io/bbolt"
@ -59,14 +60,10 @@ func (o *ObjSelector) NextObject() (*ObjectInfo, error) {
// Update the last key // Update the last key
if keyBytes != nil { if keyBytes != nil {
o.lastId = decodeId(keyBytes) o.lastId = decodeId(keyBytes)
} else { return nil
// Loopback to beginning so that we can revisit objects which were taken for verification
// but their status wasn't changed
// TODO: stop looping back to beginning too quickly
o.lastId = 0
} }
return nil return errors.New("no objects are available")
}) })
return foundObj, err return foundObj, err
} }

View file

@ -49,7 +49,7 @@ func (r *Registry) Exports() modules.Exports {
// Open creates a new instance of object registry that will store information about objects // Open creates a new instance of object registry that will store information about objects
// in the specified file. If repository instance for the file was previously created, then // in the specified file. If repository instance for the file was previously created, then
// Open will return the existing instance of repository, because bolt database allows only // Open will return the existing instance of repository, because bolt database allows only
// one write connection at a time // one write connection at a time.
func (r *Registry) Open(dbFilePath string) *ObjRegistry { func (r *Registry) Open(dbFilePath string) *ObjRegistry {
r.root.mu.Lock() r.root.mu.Lock()
defer r.root.mu.Unlock() defer r.root.mu.Unlock()

View file

@ -120,7 +120,7 @@ func (c *Client) VerifyHash(bucket, key, expectedHash string) VerifyHashResponse
if err != nil { if err != nil {
return VerifyHashResponse{Success: false, Error: err.Error()} return VerifyHashResponse{Success: false, Error: err.Error()}
} }
actualHash := hex.EncodeToString(hasher.Sum(make([]byte, 0, sha256.Size))) actualHash := hex.EncodeToString(hasher.Sum(nil))
if actualHash != expectedHash { if actualHash != expectedHash {
return VerifyHashResponse{Success: true, Error: "hash mismatch"} return VerifyHashResponse{Success: true, Error: "hash mismatch"}
} }

View file

@ -39,11 +39,16 @@ if (__ENV.S3_ENDPOINTS) {
// We will attempt to verify every object in "created" status. The scenario will execute // We will attempt to verify every object in "created" status. The scenario will execute
// as many scenarios as there are objects. Each object will have 3 retries to be verified // as many scenarios as there are objects. Each object will have 3 retries to be verified
const obj_count_to_verify = obj_registry.getObjectCountInStatus("created"); const obj_count_to_verify = obj_registry.getObjectCountInStatus("created");
// Execute at least one iteration (shared-iterations can't run 0 iterations)
const iterations = Math.max(1, obj_count_to_verify);
// Executor shared-iterations requires number of iterations to be larger than number of VUs
const vus = Math.min(__ENV.CLIENTS, iterations);
const scenarios = { const scenarios = {
verify: { verify: {
executor: 'shared-iterations', executor: 'shared-iterations',
vus: __ENV.CLIENTS, vus,
iterations: obj_count_to_verify, iterations,
maxDuration: `${time_limit}s`, maxDuration: `${time_limit}s`,
exec: 'obj_verify', exec: 'obj_verify',
gracefulStop: '5s', gracefulStop: '5s',
@ -51,7 +56,7 @@ const scenarios = {
}; };
export const options = { export const options = {
scenarios: scenarios, scenarios,
setupTimeout: '5s', setupTimeout: '5s',
}; };