Compare commits

...

5 commits

Author SHA1 Message Date
4afc85ddb1 [#1316] lint: Fix conflicts
Renamed parameters `min/max` to avoid conflicts with
predeclared identifiers.

Signed-off-by: Ekaterina Lebedeva <ekaterina.lebedeva@yadro.com>
2024-08-20 19:04:28 +03:00
04c095348e [#1316] go.mod: Bump go version to 1.22
Signed-off-by: Ekaterina Lebedeva <ekaterina.lebedeva@yadro.com>
2024-08-20 19:04:28 +03:00
cfda9003a7 [#1318] meta: Add test TestInhumeECObject
Signed-off-by: Anton Nikiforov <an.nikiforov@yadro.com>
2024-08-20 16:21:27 +03:00
6ff0b0996b [#1318] metrics: Fix container_size_bytes for EC
When node put chunk into EC container, `policer` may remove it as redundant.
This chunk marked as removed. When parent object removed and `gc` start iterating over chunk,
node count removing chunk twice.

Signed-off-by: Anton Nikiforov <an.nikiforov@yadro.com>
2024-08-20 15:33:28 +03:00
8319b59238 [#1318] Fix gofumpt issue
Signed-off-by: Anton Nikiforov <an.nikiforov@yadro.com>
2024-08-20 14:34:54 +03:00
17 changed files with 155 additions and 38 deletions

View file

@ -8,7 +8,7 @@ jobs:
runs-on: ubuntu-latest
strategy:
matrix:
go_versions: [ '1.21', '1.22' ]
go_versions: [ '1.22', '1.23' ]
steps:
- uses: actions/checkout@v3

View file

@ -16,7 +16,7 @@ jobs:
- name: Set up Go
uses: actions/setup-go@v3
with:
go-version: 1.22
go-version: 1.23
- name: Set up Python
run: |
apt update

View file

@ -11,7 +11,7 @@ jobs:
- name: Set up Go
uses: actions/setup-go@v3
with:
go-version: '1.22'
go-version: '1.23'
cache: true
- name: Install linters
@ -25,7 +25,7 @@ jobs:
runs-on: ubuntu-latest
strategy:
matrix:
go_versions: [ '1.21', '1.22' ]
go_versions: [ '1.22', '1.23' ]
fail-fast: false
steps:
- uses: actions/checkout@v3
@ -48,7 +48,7 @@ jobs:
- name: Set up Go
uses: actions/setup-go@v3
with:
go-version: '1.21'
go-version: '1.23'
cache: true
- name: Run tests
@ -63,7 +63,7 @@ jobs:
- name: Set up Go
uses: actions/setup-go@v3
with:
go-version: '1.22'
go-version: '1.23'
cache: true
- name: Install staticcheck
@ -81,7 +81,7 @@ jobs:
- name: Set up Go
uses: actions/setup-go@v3
with:
go-version: '1.21'
go-version: '1.23'
cache: true
- name: Install gopls
@ -99,7 +99,7 @@ jobs:
- name: Set up Go
uses: actions/setup-go@v3
with:
go-version: '1.22'
go-version: '1.23'
cache: true
- name: Install gofumpt

View file

@ -13,7 +13,7 @@ jobs:
- name: Setup Go
uses: actions/setup-go@v3
with:
go-version: '1.22'
go-version: '1.23'
- name: Install govulncheck
run: go install golang.org/x/vuln/cmd/govulncheck@latest

View file

@ -49,7 +49,7 @@ The latest version of frostfs-node works with frostfs-contract
# Building
To make all binaries you need Go 1.21+ and `make`:
To make all binaries you need Go 1.22+ and `make`:
```
make all
```

2
go.mod
View file

@ -1,6 +1,6 @@
module git.frostfs.info/TrueCloudLab/frostfs-node
go 1.21
go 1.22
require (
code.gitea.io/sdk/gitea v0.17.1

View file

@ -27,21 +27,21 @@ type objectDesc struct {
storageID []byte
}
func TestAll(t *testing.T, cons Constructor, min, max uint64) {
func TestAll(t *testing.T, cons Constructor, minSize, maxSize uint64) {
t.Run("get", func(t *testing.T) {
TestGet(t, cons, min, max)
TestGet(t, cons, minSize, maxSize)
})
t.Run("get range", func(t *testing.T) {
TestGetRange(t, cons, min, max)
TestGetRange(t, cons, minSize, maxSize)
})
t.Run("delete", func(t *testing.T) {
TestDelete(t, cons, min, max)
TestDelete(t, cons, minSize, maxSize)
})
t.Run("exists", func(t *testing.T) {
TestExists(t, cons, min, max)
TestExists(t, cons, minSize, maxSize)
})
t.Run("iterate", func(t *testing.T) {
TestIterate(t, cons, min, max)
TestIterate(t, cons, minSize, maxSize)
})
}
@ -51,12 +51,12 @@ func TestInfo(t *testing.T, cons Constructor, expectedType string, expectedPath
require.Equal(t, expectedPath, s.Path())
}
func prepare(t *testing.T, count int, s common.Storage, min, max uint64) []objectDesc {
func prepare(t *testing.T, count int, s common.Storage, minSize, maxSize uint64) []objectDesc {
objects := make([]objectDesc, count)
r := mrand.New(mrand.NewSource(0))
for i := range objects {
objects[i].obj = NewObject(min + uint64(r.Intn(int(max-min+1)))) // not too large
objects[i].obj = NewObject(minSize + uint64(r.Intn(int(maxSize-minSize+1)))) // not too large
objects[i].addr = objectCore.AddressOf(objects[i].obj)
raw, err := objects[i].obj.Marshal()

View file

@ -13,12 +13,12 @@ import (
// TestControl checks correctness of a read-only mode.
// cons must return a storage which is NOT opened.
func TestControl(t *testing.T, cons Constructor, min, max uint64) {
func TestControl(t *testing.T, cons Constructor, minSize, maxSize uint64) {
s := cons(t)
require.NoError(t, s.Open(mode.ComponentReadWrite))
require.NoError(t, s.Init())
objects := prepare(t, 10, s, min, max)
objects := prepare(t, 10, s, minSize, maxSize)
require.NoError(t, s.Close())
require.NoError(t, s.Open(mode.ComponentReadOnly))
@ -34,7 +34,7 @@ func TestControl(t *testing.T, cons Constructor, min, max uint64) {
t.Run("put fails", func(t *testing.T) {
var prm common.PutPrm
prm.Object = NewObject(min + uint64(rand.Intn(int(max-min+1))))
prm.Object = NewObject(minSize + uint64(rand.Intn(int(maxSize-minSize+1))))
prm.Address = objectCore.AddressOf(prm.Object)
_, err := s.Put(context.Background(), prm)

View file

@ -11,13 +11,13 @@ import (
"github.com/stretchr/testify/require"
)
func TestDelete(t *testing.T, cons Constructor, min, max uint64) {
func TestDelete(t *testing.T, cons Constructor, minSize, maxSize uint64) {
s := cons(t)
require.NoError(t, s.Open(mode.ComponentReadWrite))
require.NoError(t, s.Init())
defer func() { require.NoError(t, s.Close()) }()
objects := prepare(t, 4, s, min, max)
objects := prepare(t, 4, s, minSize, maxSize)
t.Run("delete non-existent", func(t *testing.T) {
var prm common.DeletePrm

View file

@ -10,13 +10,13 @@ import (
"github.com/stretchr/testify/require"
)
func TestExists(t *testing.T, cons Constructor, min, max uint64) {
func TestExists(t *testing.T, cons Constructor, minSize, maxSize uint64) {
s := cons(t)
require.NoError(t, s.Open(mode.ComponentReadWrite))
require.NoError(t, s.Init())
defer func() { require.NoError(t, s.Close()) }()
objects := prepare(t, 1, s, min, max)
objects := prepare(t, 1, s, minSize, maxSize)
t.Run("missing object", func(t *testing.T) {
prm := common.ExistsPrm{Address: oidtest.Address()}

View file

@ -11,13 +11,13 @@ import (
"github.com/stretchr/testify/require"
)
func TestGet(t *testing.T, cons Constructor, min, max uint64) {
func TestGet(t *testing.T, cons Constructor, minSize, maxSize uint64) {
s := cons(t)
require.NoError(t, s.Open(mode.ComponentReadWrite))
require.NoError(t, s.Init())
defer func() { require.NoError(t, s.Close()) }()
objects := prepare(t, 2, s, min, max)
objects := prepare(t, 2, s, minSize, maxSize)
t.Run("missing object", func(t *testing.T) {
gPrm := common.GetPrm{Address: oidtest.Address()}

View file

@ -13,13 +13,13 @@ import (
"github.com/stretchr/testify/require"
)
func TestGetRange(t *testing.T, cons Constructor, min, max uint64) {
func TestGetRange(t *testing.T, cons Constructor, minSize, maxSize uint64) {
s := cons(t)
require.NoError(t, s.Open(mode.ComponentReadWrite))
require.NoError(t, s.Init())
defer func() { require.NoError(t, s.Close()) }()
objects := prepare(t, 1, s, min, max)
objects := prepare(t, 1, s, minSize, maxSize)
t.Run("missing object", func(t *testing.T) {
gPrm := common.GetRangePrm{Address: oidtest.Address()}

View file

@ -10,13 +10,13 @@ import (
"github.com/stretchr/testify/require"
)
func TestIterate(t *testing.T, cons Constructor, min, max uint64) {
func TestIterate(t *testing.T, cons Constructor, minSize, maxSize uint64) {
s := cons(t)
require.NoError(t, s.Open(mode.ComponentReadWrite))
require.NoError(t, s.Init())
defer func() { require.NoError(t, s.Close()) }()
objects := prepare(t, 10, s, min, max)
objects := prepare(t, 10, s, minSize, maxSize)
// Delete random object to ensure it is not iterated over.
const delID = 2

View file

@ -236,7 +236,7 @@ func (db *DB) inhumeTx(tx *bbolt.Tx, epoch uint64, prm InhumePrm, res *InhumeRes
return err
}
} else if errors.As(err, &ecErr) {
err = db.inhumeECInfo(tx, epoch, prm.tomb, res, garbageBKT, graveyardBKT, ecErr.ECInfo(), cnr, bkt, value, targetKey)
err = db.inhumeECInfo(tx, epoch, prm.tomb, res, garbageBKT, graveyardBKT, ecErr.ECInfo(), cnr, bkt, value)
if err != nil {
return err
}
@ -280,7 +280,7 @@ func (db *DB) inhumeTx(tx *bbolt.Tx, epoch uint64, prm InhumePrm, res *InhumeRes
func (db *DB) inhumeECInfo(tx *bbolt.Tx, epoch uint64, tomb *oid.Address, res *InhumeRes,
garbageBKT *bbolt.Bucket, graveyardBKT *bbolt.Bucket,
ecInfo *objectSDK.ECInfo, cnr cid.ID, targetBucket *bbolt.Bucket, value []byte, targetKey []byte,
ecInfo *objectSDK.ECInfo, cnr cid.ID, targetBucket *bbolt.Bucket, value []byte,
) error {
for _, chunk := range ecInfo.Chunks {
chunkBuf := make([]byte, addressKeySize)
@ -296,11 +296,11 @@ func (db *DB) inhumeECInfo(tx *bbolt.Tx, epoch uint64, tomb *oid.Address, res *I
if err != nil {
return err
}
err = db.updateDeleteInfo(tx, garbageBKT, graveyardBKT, targetKey, cnr, chunkObj, res)
chunkKey := addressKey(chunkAddr, chunkBuf)
err = db.updateDeleteInfo(tx, garbageBKT, graveyardBKT, chunkKey, cnr, chunkObj, res)
if err != nil {
return err
}
chunkKey := addressKey(chunkAddr, chunkBuf)
if tomb != nil {
_, err = db.markAsGC(graveyardBKT, garbageBKT, chunkKey)
if err != nil {

View file

@ -0,0 +1,116 @@
package meta
import (
"context"
"path/filepath"
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test"
"github.com/stretchr/testify/require"
)
func TestInhumeECObject(t *testing.T) {
t.Parallel()
db := New(
WithPath(filepath.Join(t.TempDir(), "metabase")),
WithPermissions(0o600),
WithEpochState(epochState{uint64(12)}),
)
require.NoError(t, db.Open(context.Background(), mode.ReadWrite))
require.NoError(t, db.Init())
defer func() { require.NoError(t, db.Close()) }()
cnr := cidtest.ID()
ecChunk := oidtest.ID()
ecChunk2 := oidtest.ID()
ecParent := oidtest.ID()
tombstoneID := oidtest.ID()
chunkObj := testutil.GenerateObjectWithCID(cnr)
chunkObj.SetContainerID(cnr)
chunkObj.SetID(ecChunk)
chunkObj.SetPayload([]byte{0, 1, 2, 3, 4})
chunkObj.SetPayloadSize(uint64(5))
chunkObj.SetECHeader(objectSDK.NewECHeader(objectSDK.ECParentInfo{ID: ecParent}, 0, 3, []byte{}, 0))
chunkObj2 := testutil.GenerateObjectWithCID(cnr)
chunkObj2.SetContainerID(cnr)
chunkObj2.SetID(ecChunk2)
chunkObj2.SetPayload([]byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9})
chunkObj2.SetPayloadSize(uint64(10))
chunkObj2.SetECHeader(objectSDK.NewECHeader(objectSDK.ECParentInfo{ID: ecParent}, 1, 3, []byte{}, 0))
// put object with EC
var prm PutPrm
prm.SetObject(chunkObj)
prm.SetStorageID([]byte("0/0"))
_, err := db.Put(context.Background(), prm)
require.NoError(t, err)
prm.SetObject(chunkObj2)
_, err = db.Put(context.Background(), prm)
require.NoError(t, err)
var ecChunkAddress oid.Address
ecChunkAddress.SetContainer(cnr)
ecChunkAddress.SetObject(ecChunk)
var ecParentAddress oid.Address
ecParentAddress.SetContainer(cnr)
ecParentAddress.SetObject(ecParent)
var chunkObjectAddress oid.Address
chunkObjectAddress.SetContainer(cnr)
chunkObjectAddress.SetObject(ecChunk)
var getPrm GetPrm
getPrm.SetAddress(ecChunkAddress)
_, err = db.Get(context.Background(), getPrm)
require.NoError(t, err)
var ecInfoError *objectSDK.ECInfoError
getPrm.SetAddress(ecParentAddress)
_, err = db.Get(context.Background(), getPrm)
require.ErrorAs(t, err, &ecInfoError)
require.True(t, len(ecInfoError.ECInfo().Chunks) == 2 &&
ecInfoError.ECInfo().Chunks[0].Index == 0 &&
ecInfoError.ECInfo().Chunks[0].Total == 3)
// inhume Chunk
var inhumePrm InhumePrm
var tombAddress oid.Address
inhumePrm.SetAddresses(chunkObjectAddress)
res, err := db.Inhume(context.Background(), inhumePrm)
require.NoError(t, err)
require.True(t, len(res.deletionDetails) == 1)
require.True(t, res.deletionDetails[0].Size == 5)
// inhume EC parent (like Delete does)
tombAddress.SetContainer(cnr)
tombAddress.SetObject(tombstoneID)
inhumePrm.SetAddresses(ecParentAddress)
inhumePrm.SetTombstoneAddress(tombAddress)
res, err = db.Inhume(context.Background(), inhumePrm)
require.NoError(t, err)
// Previously deleted chunk shouldn't be in the details, because it is marked as garbage
require.True(t, len(res.deletionDetails) == 1)
require.True(t, res.deletionDetails[0].Size == 10)
getPrm.SetAddress(ecParentAddress)
_, err = db.Get(context.Background(), getPrm)
require.ErrorAs(t, err, new(*apistatus.ObjectAlreadyRemoved))
getPrm.SetAddress(ecChunkAddress)
_, err = db.Get(context.Background(), getPrm)
require.ErrorAs(t, err, new(*apistatus.ObjectAlreadyRemoved))
}

View file

@ -641,8 +641,8 @@ func (c *Client) notaryTxValidationLimit() (uint32, error) {
return 0, fmt.Errorf("can't get current blockchain height: %w", err)
}
min := bc + c.notary.txValidTime
rounded := (min/c.notary.roundTime + 1) * c.notary.roundTime
minTime := bc + c.notary.txValidTime
rounded := (minTime/c.notary.roundTime + 1) * c.notary.roundTime
return rounded, nil
}

View file

@ -213,6 +213,7 @@ func (s putStreamMetric) CloseAndRecv(ctx context.Context) (*object.PutResponse,
return res, err
}
func (s patchStreamMetric) Send(ctx context.Context, req *object.PatchRequest) error {
s.metrics.AddPayloadSize("Patch", len(req.GetBody().GetPatch().GetChunk()))