[#1658] node: Init object counter on meta's Init

Includes:
1. Renaming counter key to distinguish logical and physical objects
2. Version update dropping since changes could be done in a compatible way

Signed-off-by: Pavel Karpy <carpawell@nspcc.ru>
This commit is contained in:
Pavel Karpy 2022-08-25 18:58:18 +03:00 committed by fyrchik
parent cd6f8e051a
commit c7c1c257e1
4 changed files with 79 additions and 9 deletions

View file

@ -31,7 +31,7 @@ This file describes changes between the metabase versions.
- Keys and values - Keys and values
- `id` -> shard id as bytes - `id` -> shard id as bytes
- `version` -> metabase version as little-endian uint64 - `version` -> metabase version as little-endian uint64
- `counter` -> shard's object counter as little-endian uint64 - `phy_counter` -> shard's physical object counter as little-endian uint64
### Unique index buckets ### Unique index buckets
- Buckets containing objects of REGULAR type - Buckets containing objects of REGULAR type
@ -84,10 +84,6 @@ This file describes changes between the metabase versions.
- Value: list of object IDs - Value: list of object IDs
## Version 2
- Added shard's object counter to the info bucket
## Version 1 ## Version 1
- Metabase now stores generic storage id instead of blobovnicza ID. - Metabase now stores generic storage id instead of blobovnicza ID.

View file

@ -106,6 +106,11 @@ func (db *DB) init(reset bool) error {
} }
if !reset { if !reset {
err = syncCounter(tx)
if err != nil {
return fmt.Errorf("could not sync object counter: %w", err)
}
return nil return nil
} }

View file

@ -2,11 +2,14 @@ package meta
import ( import (
"encoding/binary" "encoding/binary"
"fmt"
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
"go.etcd.io/bbolt" "go.etcd.io/bbolt"
) )
var shardCounterKey = []byte("counter") var objectCounterKey = []byte("phy_counter")
// ObjectCounter returns object count that metabase has // ObjectCounter returns object count that metabase has
// tracked since it was opened and initialized. // tracked since it was opened and initialized.
@ -17,7 +20,7 @@ func (db *DB) ObjectCounter() (counter uint64, err error) {
err = db.boltDB.View(func(tx *bbolt.Tx) error { err = db.boltDB.View(func(tx *bbolt.Tx) error {
b := tx.Bucket(shardInfoBucket) b := tx.Bucket(shardInfoBucket)
if b != nil { if b != nil {
data := b.Get(shardCounterKey) data := b.Get(objectCounterKey)
if len(data) == 8 { if len(data) == 8 {
counter = binary.LittleEndian.Uint64(data) counter = binary.LittleEndian.Uint64(data)
} }
@ -39,7 +42,7 @@ func (db *DB) updateCounter(tx *bbolt.Tx, delta uint64, inc bool) error {
var counter uint64 var counter uint64
data := b.Get(shardCounterKey) data := b.Get(objectCounterKey)
if len(data) == 8 { if len(data) == 8 {
counter = binary.LittleEndian.Uint64(data) counter = binary.LittleEndian.Uint64(data)
} }
@ -55,5 +58,42 @@ func (db *DB) updateCounter(tx *bbolt.Tx, delta uint64, inc bool) error {
newCounter := make([]byte, 8) newCounter := make([]byte, 8)
binary.LittleEndian.PutUint64(newCounter, counter) binary.LittleEndian.PutUint64(newCounter, counter)
return b.Put(shardCounterKey, newCounter) return b.Put(objectCounterKey, newCounter)
}
// syncCounter updates object counter according to metabase state:
// it counts all the physically stored objects using internal indexes.
// Tx MUST be writable.
//
// Does nothing if counter not empty.
func syncCounter(tx *bbolt.Tx) error {
var counter uint64
b, err := tx.CreateBucketIfNotExists(shardInfoBucket)
if err != nil {
return fmt.Errorf("could not get shard info bucket: %w", err)
}
data := b.Get(objectCounterKey)
if len(data) == 8 {
return nil
}
err = iteratePhyObjects(tx, func(_ cid.ID, _ oid.ID) error {
counter++
return nil
})
if err != nil {
return fmt.Errorf("count not iterate objects: %w", err)
}
data = make([]byte, 8)
binary.LittleEndian.PutUint64(data, counter)
err = b.Put(objectCounterKey, data)
if err != nil {
return fmt.Errorf("could not update object counter: %w", err)
}
return nil
} }

View file

@ -155,3 +155,32 @@ func (db *DB) iterateCoveredByTombstones(tx *bbolt.Tx, tss map[string]oid.Addres
return err return err
} }
func iteratePhyObjects(tx *bbolt.Tx, f func(cid.ID, oid.ID) error) error {
var cid cid.ID
var oid oid.ID
return tx.ForEach(func(name []byte, b *bbolt.Bucket) error {
b58CID, postfix := parseContainerIDWithPostfix(&cid, name)
if len(b58CID) == 0 {
return nil
}
switch postfix {
case "",
storageGroupPostfix,
bucketNameSuffixLockers,
tombstonePostfix:
default:
return nil
}
return b.ForEach(func(k, v []byte) error {
if oid.DecodeString(string(k)) == nil {
return f(cid, oid)
}
return nil
})
})
}