From b1ec6d562c84a8f62f4540ecfe3265d5ffd1863f Mon Sep 17 00:00:00 2001 From: Vladimir Domnich Date: Thu, 22 Sep 2022 20:57:21 +0400 Subject: [PATCH] [#19] Stop object iteration after all objects were processed At the moment we don't need logic that swings back to beginning of registry when all objects have been processed. So, for now we can stop iterating and return an error when selector reaches the end of registry. Signed-off-by: Vladimir Domnich --- internal/native/client.go | 2 +- internal/registry/obj_registry.go | 14 +++++++------- internal/registry/obj_selector.go | 9 +++------ internal/registry/registry.go | 2 +- internal/s3/client.go | 2 +- scenarios/verify.js | 11 ++++++++--- 6 files changed, 21 insertions(+), 19 deletions(-) diff --git a/internal/native/client.go b/internal/native/client.go index ee5f34a..f606957 100644 --- a/internal/native/client.go +++ b/internal/native/client.go @@ -231,7 +231,7 @@ func (c *Client) VerifyHash(containerID, objectID, expectedHash string) VerifyHa if err != nil { return VerifyHashResponse{Success: false, Error: err.Error()} } - actualHash := hex.EncodeToString(hasher.Sum(make([]byte, 0, sha256.Size))) + actualHash := hex.EncodeToString(hasher.Sum(nil)) if actualHash != expectedHash { return VerifyHashResponse{Success: true, Error: "hash mismatch"} } diff --git a/internal/registry/obj_registry.go b/internal/registry/obj_registry.go index fb44bed..0caed21 100644 --- a/internal/registry/obj_registry.go +++ b/internal/registry/obj_registry.go @@ -3,6 +3,7 @@ package registry import ( "encoding/binary" "encoding/json" + "errors" "os" "time" @@ -15,7 +16,7 @@ type ObjRegistry struct { } const ( - // Indicates that an object was created, but its data wasn't verified yet + // Indicates that an object was created, but its data wasn't verified yet. statusCreated = "created" ) @@ -87,7 +88,7 @@ func (o *ObjRegistry) SetObjectStatus(id uint64, newStatus string) error { objBytes := b.Get(encodeId(id)) if objBytes == nil { - return nil + return errors.New("object doesn't exist") } obj := new(ObjectInfo) @@ -112,20 +113,19 @@ func (o *ObjRegistry) GetObjectCountInStatus(status string) (int, error) { return nil } - c := b.Cursor() - for keyBytes, objBytes := c.First(); keyBytes != nil; keyBytes, objBytes = c.Next() { + return b.ForEach(func(_, objBytes []byte) error { if objBytes != nil { var obj ObjectInfo if err := json.Unmarshal(objBytes, &obj); err != nil { // Ignore malformed objects - continue + return nil } if obj.Status == status { objCount++ } } - } - return nil + return nil + }) }) return objCount, err } diff --git a/internal/registry/obj_selector.go b/internal/registry/obj_selector.go index 613a75e..8efe5d6 100644 --- a/internal/registry/obj_selector.go +++ b/internal/registry/obj_selector.go @@ -2,6 +2,7 @@ package registry import ( "encoding/json" + "errors" "sync" "go.etcd.io/bbolt" @@ -59,14 +60,10 @@ func (o *ObjSelector) NextObject() (*ObjectInfo, error) { // Update the last key if keyBytes != nil { o.lastId = decodeId(keyBytes) - } else { - // Loopback to beginning so that we can revisit objects which were taken for verification - // but their status wasn't changed - // TODO: stop looping back to beginning too quickly - o.lastId = 0 + return nil } - return nil + return errors.New("no objects are available") }) return foundObj, err } diff --git a/internal/registry/registry.go b/internal/registry/registry.go index 67359ee..f1cb8f9 100644 --- a/internal/registry/registry.go +++ b/internal/registry/registry.go @@ -49,7 +49,7 @@ func (r *Registry) Exports() modules.Exports { // Open creates a new instance of object registry that will store information about objects // in the specified file. If repository instance for the file was previously created, then // Open will return the existing instance of repository, because bolt database allows only -// one write connection at a time +// one write connection at a time. func (r *Registry) Open(dbFilePath string) *ObjRegistry { r.root.mu.Lock() defer r.root.mu.Unlock() diff --git a/internal/s3/client.go b/internal/s3/client.go index 3c1ec33..747a9ed 100644 --- a/internal/s3/client.go +++ b/internal/s3/client.go @@ -120,7 +120,7 @@ func (c *Client) VerifyHash(bucket, key, expectedHash string) VerifyHashResponse if err != nil { return VerifyHashResponse{Success: false, Error: err.Error()} } - actualHash := hex.EncodeToString(hasher.Sum(make([]byte, 0, sha256.Size))) + actualHash := hex.EncodeToString(hasher.Sum(nil)) if actualHash != expectedHash { return VerifyHashResponse{Success: true, Error: "hash mismatch"} } diff --git a/scenarios/verify.js b/scenarios/verify.js index 9e8422a..6925dc5 100644 --- a/scenarios/verify.js +++ b/scenarios/verify.js @@ -39,11 +39,16 @@ if (__ENV.S3_ENDPOINTS) { // We will attempt to verify every object in "created" status. The scenario will execute // as many scenarios as there are objects. Each object will have 3 retries to be verified const obj_count_to_verify = obj_registry.getObjectCountInStatus("created"); +// Execute at least one iteration (shared-iterations can't run 0 iterations) +const iterations = Math.max(1, obj_count_to_verify); +// Executor shared-iterations requires number of iterations to be larger than number of VUs +const vus = Math.min(__ENV.CLIENTS, iterations); + const scenarios = { verify: { executor: 'shared-iterations', - vus: __ENV.CLIENTS, - iterations: obj_count_to_verify, + vus, + iterations, maxDuration: `${time_limit}s`, exec: 'obj_verify', gracefulStop: '5s', @@ -51,7 +56,7 @@ const scenarios = { }; export const options = { - scenarios: scenarios, + scenarios, setupTimeout: '5s', };