forked from TrueCloudLab/frostfs-node
[#1658] node: Init object counter on meta's Init
Includes: 1. Renaming counter key to distinguish logical and physical objects 2. Version update dropping since changes could be done in a compatible way Signed-off-by: Pavel Karpy <carpawell@nspcc.ru>
This commit is contained in:
parent
cd6f8e051a
commit
c7c1c257e1
4 changed files with 79 additions and 9 deletions
|
@ -31,7 +31,7 @@ This file describes changes between the metabase versions.
|
|||
- Keys and values
|
||||
- `id` -> shard id as bytes
|
||||
- `version` -> metabase version as little-endian uint64
|
||||
- `counter` -> shard's object counter as little-endian uint64
|
||||
- `phy_counter` -> shard's physical object counter as little-endian uint64
|
||||
|
||||
### Unique index buckets
|
||||
- Buckets containing objects of REGULAR type
|
||||
|
@ -84,10 +84,6 @@ This file describes changes between the metabase versions.
|
|||
- Value: list of object IDs
|
||||
|
||||
|
||||
## Version 2
|
||||
|
||||
- Added shard's object counter to the info bucket
|
||||
|
||||
## Version 1
|
||||
|
||||
- Metabase now stores generic storage id instead of blobovnicza ID.
|
||||
|
|
|
@ -106,6 +106,11 @@ func (db *DB) init(reset bool) error {
|
|||
}
|
||||
|
||||
if !reset {
|
||||
err = syncCounter(tx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not sync object counter: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
|
@ -2,11 +2,14 @@ package meta
|
|||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
|
||||
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
|
||||
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
|
||||
"go.etcd.io/bbolt"
|
||||
)
|
||||
|
||||
var shardCounterKey = []byte("counter")
|
||||
var objectCounterKey = []byte("phy_counter")
|
||||
|
||||
// ObjectCounter returns object count that metabase has
|
||||
// tracked since it was opened and initialized.
|
||||
|
@ -17,7 +20,7 @@ func (db *DB) ObjectCounter() (counter uint64, err error) {
|
|||
err = db.boltDB.View(func(tx *bbolt.Tx) error {
|
||||
b := tx.Bucket(shardInfoBucket)
|
||||
if b != nil {
|
||||
data := b.Get(shardCounterKey)
|
||||
data := b.Get(objectCounterKey)
|
||||
if len(data) == 8 {
|
||||
counter = binary.LittleEndian.Uint64(data)
|
||||
}
|
||||
|
@ -39,7 +42,7 @@ func (db *DB) updateCounter(tx *bbolt.Tx, delta uint64, inc bool) error {
|
|||
|
||||
var counter uint64
|
||||
|
||||
data := b.Get(shardCounterKey)
|
||||
data := b.Get(objectCounterKey)
|
||||
if len(data) == 8 {
|
||||
counter = binary.LittleEndian.Uint64(data)
|
||||
}
|
||||
|
@ -55,5 +58,42 @@ func (db *DB) updateCounter(tx *bbolt.Tx, delta uint64, inc bool) error {
|
|||
newCounter := make([]byte, 8)
|
||||
binary.LittleEndian.PutUint64(newCounter, counter)
|
||||
|
||||
return b.Put(shardCounterKey, newCounter)
|
||||
return b.Put(objectCounterKey, newCounter)
|
||||
}
|
||||
|
||||
// syncCounter updates object counter according to metabase state:
|
||||
// it counts all the physically stored objects using internal indexes.
|
||||
// Tx MUST be writable.
|
||||
//
|
||||
// Does nothing if counter not empty.
|
||||
func syncCounter(tx *bbolt.Tx) error {
|
||||
var counter uint64
|
||||
|
||||
b, err := tx.CreateBucketIfNotExists(shardInfoBucket)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not get shard info bucket: %w", err)
|
||||
}
|
||||
|
||||
data := b.Get(objectCounterKey)
|
||||
if len(data) == 8 {
|
||||
return nil
|
||||
}
|
||||
|
||||
err = iteratePhyObjects(tx, func(_ cid.ID, _ oid.ID) error {
|
||||
counter++
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("count not iterate objects: %w", err)
|
||||
}
|
||||
|
||||
data = make([]byte, 8)
|
||||
binary.LittleEndian.PutUint64(data, counter)
|
||||
|
||||
err = b.Put(objectCounterKey, data)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not update object counter: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
|
@ -155,3 +155,32 @@ func (db *DB) iterateCoveredByTombstones(tx *bbolt.Tx, tss map[string]oid.Addres
|
|||
|
||||
return err
|
||||
}
|
||||
|
||||
func iteratePhyObjects(tx *bbolt.Tx, f func(cid.ID, oid.ID) error) error {
|
||||
var cid cid.ID
|
||||
var oid oid.ID
|
||||
|
||||
return tx.ForEach(func(name []byte, b *bbolt.Bucket) error {
|
||||
b58CID, postfix := parseContainerIDWithPostfix(&cid, name)
|
||||
if len(b58CID) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
switch postfix {
|
||||
case "",
|
||||
storageGroupPostfix,
|
||||
bucketNameSuffixLockers,
|
||||
tombstonePostfix:
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
|
||||
return b.ForEach(func(k, v []byte) error {
|
||||
if oid.DecodeString(string(k)) == nil {
|
||||
return f(cid, oid)
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
})
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue