forked from TrueCloudLab/frostfs-node
Compare commits
5 commits
be0df7f136
...
4afc85ddb1
Author | SHA1 | Date | |
---|---|---|---|
4afc85ddb1 | |||
04c095348e | |||
cfda9003a7 | |||
6ff0b0996b | |||
8319b59238 |
17 changed files with 155 additions and 38 deletions
|
@ -8,7 +8,7 @@ jobs:
|
|||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
go_versions: [ '1.21', '1.22' ]
|
||||
go_versions: [ '1.22', '1.23' ]
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
|
|
@ -16,7 +16,7 @@ jobs:
|
|||
- name: Set up Go
|
||||
uses: actions/setup-go@v3
|
||||
with:
|
||||
go-version: 1.22
|
||||
go-version: 1.23
|
||||
- name: Set up Python
|
||||
run: |
|
||||
apt update
|
||||
|
|
|
@ -11,7 +11,7 @@ jobs:
|
|||
- name: Set up Go
|
||||
uses: actions/setup-go@v3
|
||||
with:
|
||||
go-version: '1.22'
|
||||
go-version: '1.23'
|
||||
cache: true
|
||||
|
||||
- name: Install linters
|
||||
|
@ -25,7 +25,7 @@ jobs:
|
|||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
go_versions: [ '1.21', '1.22' ]
|
||||
go_versions: [ '1.22', '1.23' ]
|
||||
fail-fast: false
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
@ -48,7 +48,7 @@ jobs:
|
|||
- name: Set up Go
|
||||
uses: actions/setup-go@v3
|
||||
with:
|
||||
go-version: '1.21'
|
||||
go-version: '1.23'
|
||||
cache: true
|
||||
|
||||
- name: Run tests
|
||||
|
@ -63,7 +63,7 @@ jobs:
|
|||
- name: Set up Go
|
||||
uses: actions/setup-go@v3
|
||||
with:
|
||||
go-version: '1.22'
|
||||
go-version: '1.23'
|
||||
cache: true
|
||||
|
||||
- name: Install staticcheck
|
||||
|
@ -81,7 +81,7 @@ jobs:
|
|||
- name: Set up Go
|
||||
uses: actions/setup-go@v3
|
||||
with:
|
||||
go-version: '1.21'
|
||||
go-version: '1.23'
|
||||
cache: true
|
||||
|
||||
- name: Install gopls
|
||||
|
@ -99,7 +99,7 @@ jobs:
|
|||
- name: Set up Go
|
||||
uses: actions/setup-go@v3
|
||||
with:
|
||||
go-version: '1.22'
|
||||
go-version: '1.23'
|
||||
cache: true
|
||||
|
||||
- name: Install gofumpt
|
||||
|
|
|
@ -13,7 +13,7 @@ jobs:
|
|||
- name: Setup Go
|
||||
uses: actions/setup-go@v3
|
||||
with:
|
||||
go-version: '1.22'
|
||||
go-version: '1.23'
|
||||
|
||||
- name: Install govulncheck
|
||||
run: go install golang.org/x/vuln/cmd/govulncheck@latest
|
||||
|
|
|
@ -49,7 +49,7 @@ The latest version of frostfs-node works with frostfs-contract
|
|||
|
||||
# Building
|
||||
|
||||
To make all binaries you need Go 1.21+ and `make`:
|
||||
To make all binaries you need Go 1.22+ and `make`:
|
||||
```
|
||||
make all
|
||||
```
|
||||
|
|
2
go.mod
2
go.mod
|
@ -1,6 +1,6 @@
|
|||
module git.frostfs.info/TrueCloudLab/frostfs-node
|
||||
|
||||
go 1.21
|
||||
go 1.22
|
||||
|
||||
require (
|
||||
code.gitea.io/sdk/gitea v0.17.1
|
||||
|
|
|
@ -27,21 +27,21 @@ type objectDesc struct {
|
|||
storageID []byte
|
||||
}
|
||||
|
||||
func TestAll(t *testing.T, cons Constructor, min, max uint64) {
|
||||
func TestAll(t *testing.T, cons Constructor, minSize, maxSize uint64) {
|
||||
t.Run("get", func(t *testing.T) {
|
||||
TestGet(t, cons, min, max)
|
||||
TestGet(t, cons, minSize, maxSize)
|
||||
})
|
||||
t.Run("get range", func(t *testing.T) {
|
||||
TestGetRange(t, cons, min, max)
|
||||
TestGetRange(t, cons, minSize, maxSize)
|
||||
})
|
||||
t.Run("delete", func(t *testing.T) {
|
||||
TestDelete(t, cons, min, max)
|
||||
TestDelete(t, cons, minSize, maxSize)
|
||||
})
|
||||
t.Run("exists", func(t *testing.T) {
|
||||
TestExists(t, cons, min, max)
|
||||
TestExists(t, cons, minSize, maxSize)
|
||||
})
|
||||
t.Run("iterate", func(t *testing.T) {
|
||||
TestIterate(t, cons, min, max)
|
||||
TestIterate(t, cons, minSize, maxSize)
|
||||
})
|
||||
}
|
||||
|
||||
|
@ -51,12 +51,12 @@ func TestInfo(t *testing.T, cons Constructor, expectedType string, expectedPath
|
|||
require.Equal(t, expectedPath, s.Path())
|
||||
}
|
||||
|
||||
func prepare(t *testing.T, count int, s common.Storage, min, max uint64) []objectDesc {
|
||||
func prepare(t *testing.T, count int, s common.Storage, minSize, maxSize uint64) []objectDesc {
|
||||
objects := make([]objectDesc, count)
|
||||
|
||||
r := mrand.New(mrand.NewSource(0))
|
||||
for i := range objects {
|
||||
objects[i].obj = NewObject(min + uint64(r.Intn(int(max-min+1)))) // not too large
|
||||
objects[i].obj = NewObject(minSize + uint64(r.Intn(int(maxSize-minSize+1)))) // not too large
|
||||
objects[i].addr = objectCore.AddressOf(objects[i].obj)
|
||||
|
||||
raw, err := objects[i].obj.Marshal()
|
||||
|
|
|
@ -13,12 +13,12 @@ import (
|
|||
|
||||
// TestControl checks correctness of a read-only mode.
|
||||
// cons must return a storage which is NOT opened.
|
||||
func TestControl(t *testing.T, cons Constructor, min, max uint64) {
|
||||
func TestControl(t *testing.T, cons Constructor, minSize, maxSize uint64) {
|
||||
s := cons(t)
|
||||
require.NoError(t, s.Open(mode.ComponentReadWrite))
|
||||
require.NoError(t, s.Init())
|
||||
|
||||
objects := prepare(t, 10, s, min, max)
|
||||
objects := prepare(t, 10, s, minSize, maxSize)
|
||||
require.NoError(t, s.Close())
|
||||
|
||||
require.NoError(t, s.Open(mode.ComponentReadOnly))
|
||||
|
@ -34,7 +34,7 @@ func TestControl(t *testing.T, cons Constructor, min, max uint64) {
|
|||
|
||||
t.Run("put fails", func(t *testing.T) {
|
||||
var prm common.PutPrm
|
||||
prm.Object = NewObject(min + uint64(rand.Intn(int(max-min+1))))
|
||||
prm.Object = NewObject(minSize + uint64(rand.Intn(int(maxSize-minSize+1))))
|
||||
prm.Address = objectCore.AddressOf(prm.Object)
|
||||
|
||||
_, err := s.Put(context.Background(), prm)
|
||||
|
|
|
@ -11,13 +11,13 @@ import (
|
|||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestDelete(t *testing.T, cons Constructor, min, max uint64) {
|
||||
func TestDelete(t *testing.T, cons Constructor, minSize, maxSize uint64) {
|
||||
s := cons(t)
|
||||
require.NoError(t, s.Open(mode.ComponentReadWrite))
|
||||
require.NoError(t, s.Init())
|
||||
defer func() { require.NoError(t, s.Close()) }()
|
||||
|
||||
objects := prepare(t, 4, s, min, max)
|
||||
objects := prepare(t, 4, s, minSize, maxSize)
|
||||
|
||||
t.Run("delete non-existent", func(t *testing.T) {
|
||||
var prm common.DeletePrm
|
||||
|
|
|
@ -10,13 +10,13 @@ import (
|
|||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestExists(t *testing.T, cons Constructor, min, max uint64) {
|
||||
func TestExists(t *testing.T, cons Constructor, minSize, maxSize uint64) {
|
||||
s := cons(t)
|
||||
require.NoError(t, s.Open(mode.ComponentReadWrite))
|
||||
require.NoError(t, s.Init())
|
||||
defer func() { require.NoError(t, s.Close()) }()
|
||||
|
||||
objects := prepare(t, 1, s, min, max)
|
||||
objects := prepare(t, 1, s, minSize, maxSize)
|
||||
|
||||
t.Run("missing object", func(t *testing.T) {
|
||||
prm := common.ExistsPrm{Address: oidtest.Address()}
|
||||
|
|
|
@ -11,13 +11,13 @@ import (
|
|||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestGet(t *testing.T, cons Constructor, min, max uint64) {
|
||||
func TestGet(t *testing.T, cons Constructor, minSize, maxSize uint64) {
|
||||
s := cons(t)
|
||||
require.NoError(t, s.Open(mode.ComponentReadWrite))
|
||||
require.NoError(t, s.Init())
|
||||
defer func() { require.NoError(t, s.Close()) }()
|
||||
|
||||
objects := prepare(t, 2, s, min, max)
|
||||
objects := prepare(t, 2, s, minSize, maxSize)
|
||||
|
||||
t.Run("missing object", func(t *testing.T) {
|
||||
gPrm := common.GetPrm{Address: oidtest.Address()}
|
||||
|
|
|
@ -13,13 +13,13 @@ import (
|
|||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestGetRange(t *testing.T, cons Constructor, min, max uint64) {
|
||||
func TestGetRange(t *testing.T, cons Constructor, minSize, maxSize uint64) {
|
||||
s := cons(t)
|
||||
require.NoError(t, s.Open(mode.ComponentReadWrite))
|
||||
require.NoError(t, s.Init())
|
||||
defer func() { require.NoError(t, s.Close()) }()
|
||||
|
||||
objects := prepare(t, 1, s, min, max)
|
||||
objects := prepare(t, 1, s, minSize, maxSize)
|
||||
|
||||
t.Run("missing object", func(t *testing.T) {
|
||||
gPrm := common.GetRangePrm{Address: oidtest.Address()}
|
||||
|
|
|
@ -10,13 +10,13 @@ import (
|
|||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestIterate(t *testing.T, cons Constructor, min, max uint64) {
|
||||
func TestIterate(t *testing.T, cons Constructor, minSize, maxSize uint64) {
|
||||
s := cons(t)
|
||||
require.NoError(t, s.Open(mode.ComponentReadWrite))
|
||||
require.NoError(t, s.Init())
|
||||
defer func() { require.NoError(t, s.Close()) }()
|
||||
|
||||
objects := prepare(t, 10, s, min, max)
|
||||
objects := prepare(t, 10, s, minSize, maxSize)
|
||||
|
||||
// Delete random object to ensure it is not iterated over.
|
||||
const delID = 2
|
||||
|
|
|
@ -236,7 +236,7 @@ func (db *DB) inhumeTx(tx *bbolt.Tx, epoch uint64, prm InhumePrm, res *InhumeRes
|
|||
return err
|
||||
}
|
||||
} else if errors.As(err, &ecErr) {
|
||||
err = db.inhumeECInfo(tx, epoch, prm.tomb, res, garbageBKT, graveyardBKT, ecErr.ECInfo(), cnr, bkt, value, targetKey)
|
||||
err = db.inhumeECInfo(tx, epoch, prm.tomb, res, garbageBKT, graveyardBKT, ecErr.ECInfo(), cnr, bkt, value)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
@ -280,7 +280,7 @@ func (db *DB) inhumeTx(tx *bbolt.Tx, epoch uint64, prm InhumePrm, res *InhumeRes
|
|||
|
||||
func (db *DB) inhumeECInfo(tx *bbolt.Tx, epoch uint64, tomb *oid.Address, res *InhumeRes,
|
||||
garbageBKT *bbolt.Bucket, graveyardBKT *bbolt.Bucket,
|
||||
ecInfo *objectSDK.ECInfo, cnr cid.ID, targetBucket *bbolt.Bucket, value []byte, targetKey []byte,
|
||||
ecInfo *objectSDK.ECInfo, cnr cid.ID, targetBucket *bbolt.Bucket, value []byte,
|
||||
) error {
|
||||
for _, chunk := range ecInfo.Chunks {
|
||||
chunkBuf := make([]byte, addressKeySize)
|
||||
|
@ -296,11 +296,11 @@ func (db *DB) inhumeECInfo(tx *bbolt.Tx, epoch uint64, tomb *oid.Address, res *I
|
|||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = db.updateDeleteInfo(tx, garbageBKT, graveyardBKT, targetKey, cnr, chunkObj, res)
|
||||
chunkKey := addressKey(chunkAddr, chunkBuf)
|
||||
err = db.updateDeleteInfo(tx, garbageBKT, graveyardBKT, chunkKey, cnr, chunkObj, res)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
chunkKey := addressKey(chunkAddr, chunkBuf)
|
||||
if tomb != nil {
|
||||
_, err = db.markAsGC(graveyardBKT, garbageBKT, chunkKey)
|
||||
if err != nil {
|
||||
|
|
116
pkg/local_object_storage/metabase/inhume_ec_test.go
Normal file
116
pkg/local_object_storage/metabase/inhume_ec_test.go
Normal file
|
@ -0,0 +1,116 @@
|
|||
package meta
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
|
||||
apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
|
||||
cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
|
||||
objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestInhumeECObject(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db := New(
|
||||
WithPath(filepath.Join(t.TempDir(), "metabase")),
|
||||
WithPermissions(0o600),
|
||||
WithEpochState(epochState{uint64(12)}),
|
||||
)
|
||||
|
||||
require.NoError(t, db.Open(context.Background(), mode.ReadWrite))
|
||||
require.NoError(t, db.Init())
|
||||
defer func() { require.NoError(t, db.Close()) }()
|
||||
|
||||
cnr := cidtest.ID()
|
||||
ecChunk := oidtest.ID()
|
||||
ecChunk2 := oidtest.ID()
|
||||
ecParent := oidtest.ID()
|
||||
tombstoneID := oidtest.ID()
|
||||
|
||||
chunkObj := testutil.GenerateObjectWithCID(cnr)
|
||||
chunkObj.SetContainerID(cnr)
|
||||
chunkObj.SetID(ecChunk)
|
||||
chunkObj.SetPayload([]byte{0, 1, 2, 3, 4})
|
||||
chunkObj.SetPayloadSize(uint64(5))
|
||||
chunkObj.SetECHeader(objectSDK.NewECHeader(objectSDK.ECParentInfo{ID: ecParent}, 0, 3, []byte{}, 0))
|
||||
|
||||
chunkObj2 := testutil.GenerateObjectWithCID(cnr)
|
||||
chunkObj2.SetContainerID(cnr)
|
||||
chunkObj2.SetID(ecChunk2)
|
||||
chunkObj2.SetPayload([]byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9})
|
||||
chunkObj2.SetPayloadSize(uint64(10))
|
||||
chunkObj2.SetECHeader(objectSDK.NewECHeader(objectSDK.ECParentInfo{ID: ecParent}, 1, 3, []byte{}, 0))
|
||||
|
||||
// put object with EC
|
||||
|
||||
var prm PutPrm
|
||||
prm.SetObject(chunkObj)
|
||||
prm.SetStorageID([]byte("0/0"))
|
||||
_, err := db.Put(context.Background(), prm)
|
||||
require.NoError(t, err)
|
||||
|
||||
prm.SetObject(chunkObj2)
|
||||
_, err = db.Put(context.Background(), prm)
|
||||
require.NoError(t, err)
|
||||
|
||||
var ecChunkAddress oid.Address
|
||||
ecChunkAddress.SetContainer(cnr)
|
||||
ecChunkAddress.SetObject(ecChunk)
|
||||
|
||||
var ecParentAddress oid.Address
|
||||
ecParentAddress.SetContainer(cnr)
|
||||
ecParentAddress.SetObject(ecParent)
|
||||
|
||||
var chunkObjectAddress oid.Address
|
||||
chunkObjectAddress.SetContainer(cnr)
|
||||
chunkObjectAddress.SetObject(ecChunk)
|
||||
|
||||
var getPrm GetPrm
|
||||
|
||||
getPrm.SetAddress(ecChunkAddress)
|
||||
_, err = db.Get(context.Background(), getPrm)
|
||||
require.NoError(t, err)
|
||||
|
||||
var ecInfoError *objectSDK.ECInfoError
|
||||
getPrm.SetAddress(ecParentAddress)
|
||||
_, err = db.Get(context.Background(), getPrm)
|
||||
require.ErrorAs(t, err, &ecInfoError)
|
||||
require.True(t, len(ecInfoError.ECInfo().Chunks) == 2 &&
|
||||
ecInfoError.ECInfo().Chunks[0].Index == 0 &&
|
||||
ecInfoError.ECInfo().Chunks[0].Total == 3)
|
||||
|
||||
// inhume Chunk
|
||||
var inhumePrm InhumePrm
|
||||
var tombAddress oid.Address
|
||||
inhumePrm.SetAddresses(chunkObjectAddress)
|
||||
res, err := db.Inhume(context.Background(), inhumePrm)
|
||||
require.NoError(t, err)
|
||||
require.True(t, len(res.deletionDetails) == 1)
|
||||
require.True(t, res.deletionDetails[0].Size == 5)
|
||||
|
||||
// inhume EC parent (like Delete does)
|
||||
tombAddress.SetContainer(cnr)
|
||||
tombAddress.SetObject(tombstoneID)
|
||||
inhumePrm.SetAddresses(ecParentAddress)
|
||||
inhumePrm.SetTombstoneAddress(tombAddress)
|
||||
res, err = db.Inhume(context.Background(), inhumePrm)
|
||||
require.NoError(t, err)
|
||||
// Previously deleted chunk shouldn't be in the details, because it is marked as garbage
|
||||
require.True(t, len(res.deletionDetails) == 1)
|
||||
require.True(t, res.deletionDetails[0].Size == 10)
|
||||
|
||||
getPrm.SetAddress(ecParentAddress)
|
||||
_, err = db.Get(context.Background(), getPrm)
|
||||
require.ErrorAs(t, err, new(*apistatus.ObjectAlreadyRemoved))
|
||||
|
||||
getPrm.SetAddress(ecChunkAddress)
|
||||
_, err = db.Get(context.Background(), getPrm)
|
||||
require.ErrorAs(t, err, new(*apistatus.ObjectAlreadyRemoved))
|
||||
}
|
|
@ -641,8 +641,8 @@ func (c *Client) notaryTxValidationLimit() (uint32, error) {
|
|||
return 0, fmt.Errorf("can't get current blockchain height: %w", err)
|
||||
}
|
||||
|
||||
min := bc + c.notary.txValidTime
|
||||
rounded := (min/c.notary.roundTime + 1) * c.notary.roundTime
|
||||
minTime := bc + c.notary.txValidTime
|
||||
rounded := (minTime/c.notary.roundTime + 1) * c.notary.roundTime
|
||||
|
||||
return rounded, nil
|
||||
}
|
||||
|
|
|
@ -213,6 +213,7 @@ func (s putStreamMetric) CloseAndRecv(ctx context.Context) (*object.PutResponse,
|
|||
|
||||
return res, err
|
||||
}
|
||||
|
||||
func (s patchStreamMetric) Send(ctx context.Context, req *object.PatchRequest) error {
|
||||
s.metrics.AddPayloadSize("Patch", len(req.GetBody().GetPatch().GetChunk()))
|
||||
|
||||
|
|
Loading…
Reference in a new issue