Revert 1.21 changes #995

Merged
fyrchik merged 7 commits from fyrchik/frostfs-node:revert-1.21 into master 2024-02-19 15:36:03 +00:00
33 changed files with 198 additions and 39 deletions

View file

@ -1,4 +1,4 @@
FROM golang:1.22 as builder FROM golang:1.21 as builder
ARG BUILD=now ARG BUILD=now
ARG VERSION=dev ARG VERSION=dev
ARG REPO=repository ARG REPO=repository

View file

@ -1,4 +1,4 @@
FROM golang:1.22 FROM golang:1.21
WORKDIR /tmp WORKDIR /tmp

View file

@ -1,4 +1,4 @@
FROM golang:1.22 as builder FROM golang:1.21 as builder
ARG BUILD=now ARG BUILD=now
ARG VERSION=dev ARG VERSION=dev
ARG REPO=repository ARG REPO=repository

View file

@ -1,4 +1,4 @@
FROM golang:1.22 as builder FROM golang:1.21 as builder
ARG BUILD=now ARG BUILD=now
ARG VERSION=dev ARG VERSION=dev
ARG REPO=repository ARG REPO=repository

View file

@ -1,4 +1,4 @@
FROM golang:1.22 as builder FROM golang:1.21 as builder
ARG BUILD=now ARG BUILD=now
ARG VERSION=dev ARG VERSION=dev
ARG REPO=repository ARG REPO=repository

View file

@ -8,7 +8,7 @@ jobs:
runs-on: ubuntu-latest runs-on: ubuntu-latest
strategy: strategy:
matrix: matrix:
go_versions: [ '1.21', '1.22' ] go_versions: [ '1.20', '1.21' ]
steps: steps:
- uses: actions/checkout@v3 - uses: actions/checkout@v3

View file

@ -13,7 +13,7 @@ jobs:
- name: Setup Go - name: Setup Go
uses: actions/setup-go@v3 uses: actions/setup-go@v3
with: with:
go-version: '1.22' go-version: '1.21'
- name: Run commit format checker - name: Run commit format checker
uses: https://git.frostfs.info/TrueCloudLab/dco-go@v3 uses: https://git.frostfs.info/TrueCloudLab/dco-go@v3

View file

@ -11,7 +11,7 @@ jobs:
- name: Set up Go - name: Set up Go
uses: actions/setup-go@v3 uses: actions/setup-go@v3
with: with:
go-version: '1.22' go-version: '1.21'
cache: true cache: true
- name: Install linters - name: Install linters
@ -25,7 +25,7 @@ jobs:
runs-on: ubuntu-latest runs-on: ubuntu-latest
strategy: strategy:
matrix: matrix:
go_versions: [ '1.21', '1.22' ] go_versions: [ '1.20', '1.21' ]
fail-fast: false fail-fast: false
steps: steps:
- uses: actions/checkout@v3 - uses: actions/checkout@v3
@ -63,7 +63,7 @@ jobs:
- name: Set up Go - name: Set up Go
uses: actions/setup-go@v3 uses: actions/setup-go@v3
with: with:
go-version: '1.22' go-version: '1.21'
cache: true cache: true
- name: Install staticcheck - name: Install staticcheck

View file

@ -13,7 +13,7 @@ jobs:
- name: Setup Go - name: Setup Go
uses: actions/setup-go@v3 uses: actions/setup-go@v3
with: with:
go-version: '1.22' go-version: '1.21'
- name: Install govulncheck - name: Install govulncheck
run: go install golang.org/x/vuln/cmd/govulncheck@latest run: go install golang.org/x/vuln/cmd/govulncheck@latest

View file

@ -7,7 +7,7 @@ VERSION ?= $(shell git describe --tags --dirty --match "v*" --always --abbrev=8
HUB_IMAGE ?= truecloudlab/frostfs HUB_IMAGE ?= truecloudlab/frostfs
HUB_TAG ?= "$(shell echo ${VERSION} | sed 's/^v//')" HUB_TAG ?= "$(shell echo ${VERSION} | sed 's/^v//')"
GO_VERSION ?= 1.22 GO_VERSION ?= 1.21
LINT_VERSION ?= 1.55.2 LINT_VERSION ?= 1.55.2
TRUECLOUDLAB_LINT_VERSION ?= 0.0.3 TRUECLOUDLAB_LINT_VERSION ?= 0.0.3
PROTOC_VERSION ?= 25.0 PROTOC_VERSION ?= 25.0

View file

@ -49,7 +49,7 @@ The latest version of frostfs-node works with frostfs-contract
# Building # Building
To make all binaries you need Go 1.21+ and `make`: To make all binaries you need Go 1.20+ and `make`:
``` ```
make all make all
``` ```

View file

@ -5,7 +5,6 @@ import (
"errors" "errors"
"fmt" "fmt"
"os" "os"
"slices"
"sort" "sort"
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants"
@ -447,7 +446,7 @@ func getCIDFilterFunc(cmd *cobra.Command) (func([]byte) bool, error) {
var id cid.ID var id cid.ID
id.SetSHA256(v) id.SetSHA256(v)
idStr := id.EncodeToString() idStr := id.EncodeToString()
_, found := slices.BinarySearch(rawIDs, idStr) n := sort.Search(len(rawIDs), func(i int) bool { return rawIDs[i] >= idStr })
return found return n < len(rawIDs) && rawIDs[n] == idStr
}, nil }, nil
} }

View file

@ -100,7 +100,10 @@ func registerCandidates(c *helper.InitializeContext) error {
// Register candidates in batches in order to overcome the signers amount limit. // Register candidates in batches in order to overcome the signers amount limit.
// See: https://github.com/nspcc-dev/neo-go/blob/master/pkg/core/transaction/transaction.go#L27 // See: https://github.com/nspcc-dev/neo-go/blob/master/pkg/core/transaction/transaction.go#L27
for i := 0; i < need; i += registerBatchSize { for i := 0; i < need; i += registerBatchSize {
start, end := i, min(i+registerBatchSize, need) start, end := i, i+registerBatchSize
if end > need {
end = need
}
// This check is sound because transactions are accepted/rejected atomically. // This check is sound because transactions are accepted/rejected atomically.
if have >= end { if have >= end {
continue continue

View file

@ -3,7 +3,7 @@ package control
import ( import (
"bytes" "bytes"
"fmt" "fmt"
"slices" "sort"
"strings" "strings"
rawclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" rawclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client"
@ -177,6 +177,9 @@ func getShardIDListFromIDFlag(cmd *cobra.Command, withAllFlag bool) [][]byte {
res = append(res, raw) res = append(res, raw)
} }
slices.SortFunc(res, bytes.Compare) sort.Slice(res, func(i, j int) bool {
return bytes.Compare(res[i], res[j]) < 0
})
return res return res
} }

2
go.mod
View file

@ -1,6 +1,6 @@
module git.frostfs.info/TrueCloudLab/frostfs-node module git.frostfs.info/TrueCloudLab/frostfs-node
go 1.21 go 1.20
require ( require (
code.gitea.io/sdk/gitea v0.17.1 code.gitea.io/sdk/gitea v0.17.1

BIN
go.sum

Binary file not shown.

View file

@ -3,13 +3,13 @@ package frostfs
import ( import (
"bytes" "bytes"
"encoding/hex" "encoding/hex"
"slices"
"git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event"
frostfsEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/frostfs" frostfsEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/frostfs"
"go.uber.org/zap" "go.uber.org/zap"
"golang.org/x/exp/slices"
) )
func (np *Processor) handleDeposit(ev event.Event) { func (np *Processor) handleDeposit(ev event.Event) {

View file

@ -82,7 +82,10 @@ func (c *cleanupTable) touch(keyString string, now uint64, binNodeInfo []byte) b
result := !ok || access.removeFlag || !bytes.Equal(access.binNodeInfo, binNodeInfo) result := !ok || access.removeFlag || !bytes.Equal(access.binNodeInfo, binNodeInfo)
access.removeFlag = false // reset remove flag on each touch access.removeFlag = false // reset remove flag on each touch
access.epoch = max(access.epoch, now) if now > access.epoch {
access.epoch = now
}
access.binNodeInfo = binNodeInfo // update binary node info access.binNodeInfo = binNodeInfo // update binary node info
c.lastAccess[keyString] = access c.lastAccess[keyString] = access

View file

@ -56,6 +56,14 @@ func (b *Blobovnicza) iterateBounds(useObjLimitBound bool, f func(uint64, uint64
return nil return nil
} }
func max(a, b uint64) uint64 {
if a > b {
return a
}
return b
}
// IterationElement represents a unit of elements through which Iterate operation passes. // IterationElement represents a unit of elements through which Iterate operation passes.
type IterationElement struct { type IterationElement struct {
addr oid.Address addr oid.Address

View file

@ -134,7 +134,10 @@ func (e *StorageEngine) ListWithCursor(ctx context.Context, prm ListWithCursorPr
continue continue
} }
count := min(prm.count-uint32(len(result)), batchSize) count := prm.count - uint32(len(result))
if count > batchSize {
count = batchSize
}
var shardPrm shard.ListWithCursorPrm var shardPrm shard.ListWithCursorPrm
shardPrm.WithCount(count) shardPrm.WithCount(count)

View file

@ -3,6 +3,7 @@ package engine
import ( import (
"context" "context"
"path/filepath" "path/filepath"
"sort"
"testing" "testing"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object"
@ -17,6 +18,13 @@ import (
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
func sortAddresses(addrWithType []object.AddressWithType) []object.AddressWithType {
sort.Slice(addrWithType, func(i, j int) bool {
return addrWithType[i].Address.EncodeToString() < addrWithType[j].Address.EncodeToString()
})
return addrWithType
}
func TestListWithCursor(t *testing.T) { func TestListWithCursor(t *testing.T) {
t.Parallel() t.Parallel()
@ -90,6 +98,7 @@ func TestListWithCursor(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
expected = append(expected, object.AddressWithType{Type: objectSDK.TypeRegular, Address: object.AddressOf(obj)}) expected = append(expected, object.AddressWithType{Type: objectSDK.TypeRegular, Address: object.AddressOf(obj)})
} }
expected = sortAddresses(expected)
var prm ListWithCursorPrm var prm ListWithCursorPrm
prm.count = tt.batchSize prm.count = tt.batchSize
@ -104,7 +113,8 @@ func TestListWithCursor(t *testing.T) {
prm.cursor = res.Cursor() prm.cursor = res.Cursor()
} }
require.ElementsMatch(t, expected, got) got = sortAddresses(got)
require.Equal(t, expected, got)
}) })
} }
} }

View file

@ -2,10 +2,10 @@ package testutil
import ( import (
"encoding/binary" "encoding/binary"
"slices"
"testing" "testing"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"golang.org/x/exp/slices"
) )
func TestOverwriteObjGenerator(t *testing.T) { func TestOverwriteObjGenerator(t *testing.T) {

View file

@ -3,6 +3,7 @@ package meta_test
import ( import (
"context" "context"
"math/rand" "math/rand"
"sort"
"testing" "testing"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object"
@ -124,9 +125,18 @@ func TestDB_ContainersCount(t *testing.T) {
} }
} }
sort.Slice(expected, func(i, j int) bool {
return expected[i].EncodeToString() < expected[j].EncodeToString()
})
got, err := db.Containers(context.Background()) got, err := db.Containers(context.Background())
require.NoError(t, err) require.NoError(t, err)
require.ElementsMatch(t, expected, got)
sort.Slice(got, func(i, j int) bool {
return got[i].EncodeToString() < got[j].EncodeToString()
})
require.Equal(t, expected, got)
} }
func TestDB_ContainerSize(t *testing.T) { func TestDB_ContainerSize(t *testing.T) {

View file

@ -3,6 +3,7 @@ package meta_test
import ( import (
"context" "context"
"errors" "errors"
"sort"
"testing" "testing"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object"
@ -127,6 +128,8 @@ func TestLisObjectsWithCursor(t *testing.T) {
expected = append(expected, object.AddressWithType{Address: object.AddressOf(child), Type: objectSDK.TypeRegular}) expected = append(expected, object.AddressWithType{Address: object.AddressOf(child), Type: objectSDK.TypeRegular})
} }
expected = sortAddresses(expected)
t.Run("success with various count", func(t *testing.T) { t.Run("success with various count", func(t *testing.T) {
for countPerReq := 1; countPerReq <= total; countPerReq++ { for countPerReq := 1; countPerReq <= total; countPerReq++ {
got := make([]object.AddressWithType, 0, total) got := make([]object.AddressWithType, 0, total)
@ -148,7 +151,9 @@ func TestLisObjectsWithCursor(t *testing.T) {
_, _, err = metaListWithCursor(db, uint32(countPerReq), cursor) _, _, err = metaListWithCursor(db, uint32(countPerReq), cursor)
require.ErrorIs(t, err, meta.ErrEndOfListing, "count:%d", countPerReq, cursor) require.ErrorIs(t, err, meta.ErrEndOfListing, "count:%d", countPerReq, cursor)
require.ElementsMatch(t, expected, got, "count:%d", countPerReq)
got = sortAddresses(got)
require.Equal(t, expected, got, "count:%d", countPerReq)
} }
}) })
@ -211,6 +216,13 @@ func TestAddObjectDuringListingWithCursor(t *testing.T) {
} }
} }
func sortAddresses(addrWithType []object.AddressWithType) []object.AddressWithType {
sort.Slice(addrWithType, func(i, j int) bool {
return addrWithType[i].Address.EncodeToString() < addrWithType[j].Address.EncodeToString()
})
return addrWithType
}
func metaListWithCursor(db *meta.DB, count uint32, cursor *meta.Cursor) ([]object.AddressWithType, *meta.Cursor, error) { func metaListWithCursor(db *meta.DB, count uint32, cursor *meta.Cursor) ([]object.AddressWithType, *meta.Cursor, error) {
var listPrm meta.ListPrm var listPrm meta.ListPrm
listPrm.SetCount(count) listPrm.SetCount(count)

View file

@ -2,7 +2,6 @@ package pilorama
import ( import (
"encoding/binary" "encoding/binary"
"slices"
"sort" "sort"
"sync" "sync"
"time" "time"
@ -51,7 +50,7 @@ func (b *batch) run() {
sort.Slice(b.operations, func(i, j int) bool { sort.Slice(b.operations, func(i, j int) bool {
return b.operations[i].Time < b.operations[j].Time return b.operations[i].Time < b.operations[j].Time
}) })
b.operations = slices.CompactFunc(b.operations, func(x, y *Move) bool { return x.Time == y.Time }) b.operations = removeDuplicatesInPlace(b.operations)
// Our main use-case is addition of new items. In this case, // Our main use-case is addition of new items. In this case,
// we do not need to perform undo()/redo(), just do(). // we do not need to perform undo()/redo(), just do().
@ -116,3 +115,15 @@ func (b *batch) run() {
b.results[i] <- err b.results[i] <- err
} }
} }
func removeDuplicatesInPlace(a []*Move) []*Move {
equalCount := 0
for i := 1; i < len(a); i++ {
if a[i].Time == a[i-1].Time {
equalCount++
} else {
a[i-equalCount] = a[i]
}
}
return a[:len(a)-equalCount]
}

View file

@ -0,0 +1,70 @@
package pilorama
import (
"testing"
"github.com/stretchr/testify/require"
)
func Test_removeDuplicatesInPlace(t *testing.T) {
testCases := []struct {
before []int
after []int
}{
{
before: []int{},
after: []int{},
},
{
before: []int{1},
after: []int{1},
},
{
before: []int{1, 2},
after: []int{1, 2},
},
{
before: []int{1, 2, 3},
after: []int{1, 2, 3},
},
{
before: []int{1, 1, 2},
after: []int{1, 2},
},
{
before: []int{1, 2, 2},
after: []int{1, 2},
},
{
before: []int{1, 2, 2, 3},
after: []int{1, 2, 3},
},
{
before: []int{1, 1, 1},
after: []int{1},
},
{
before: []int{1, 1, 2, 2},
after: []int{1, 2},
},
{
before: []int{1, 1, 1, 2, 3, 3, 3},
after: []int{1, 2, 3},
},
}
for _, tc := range testCases {
ops := make([]*Move, len(tc.before))
for i := range ops {
ops[i] = &Move{Meta: Meta{Time: Timestamp(tc.before[i])}}
}
expected := make([]*Move, len(tc.after))
for i := range expected {
expected[i] = &Move{Meta: Meta{Time: Timestamp(tc.after[i])}}
}
actual := removeDuplicatesInPlace(ops)
require.Equal(t, expected, actual, "%d", tc.before)
}
}

View file

@ -343,8 +343,16 @@ func (s *Shard) removeGarbage(pctx context.Context) (result gcRunResult) {
} }
func (s *Shard) getExpiredObjectsParameters() (workerCount, batchSize int) { func (s *Shard) getExpiredObjectsParameters() (workerCount, batchSize int) {
workerCount = max(minExpiredWorkers, s.gc.gcCfg.expiredCollectorWorkerCount) workerCount = minExpiredWorkers
batchSize = max(minExpiredBatchSize, s.gc.gcCfg.expiredCollectorBatchSize) batchSize = minExpiredBatchSize
if s.gc.gcCfg.expiredCollectorBatchSize > batchSize {
batchSize = s.gc.gcCfg.expiredCollectorBatchSize
}
if s.gc.gcCfg.expiredCollectorWorkerCount > workerCount {
workerCount = s.gc.gcCfg.expiredCollectorWorkerCount
}
return return
} }

View file

@ -162,7 +162,11 @@ func (c *Client) DepositNotary(amount fixedn.Fixed8, delta uint32) (res util.Uin
return util.Uint256{}, fmt.Errorf("can't get previous expiration value: %w", err) return util.Uint256{}, fmt.Errorf("can't get previous expiration value: %w", err)
} }
till := max(int64(bc+delta), currentTill) till := int64(bc + delta)
if till < currentTill {
till = currentTill
}
return c.depositNotary(amount, till) return c.depositNotary(amount, till)
} }

View file

@ -343,7 +343,10 @@ func PayloadRange(ctx context.Context, prm PayloadRangePrm) (*PayloadRangeRes, e
return nil, new(apistatus.ObjectOutOfRange) return nil, new(apistatus.ObjectOutOfRange)
} }
ln := min(prm.ln, maxInitialBufferSize) ln := prm.ln
if ln > maxInitialBufferSize {
ln = maxInitialBufferSize
}
w := bytes.NewBuffer(make([]byte, ln)) w := bytes.NewBuffer(make([]byte, ln))
_, err = io.CopyN(w, rdr, int64(prm.ln)) _, err = io.CopyN(w, rdr, int64(prm.ln))

View file

@ -164,7 +164,10 @@ func (s *searchStreamMsgSizeCtrl) Send(resp *object.SearchResponse) error {
newResp.SetBody(body) newResp.SetBody(body)
} }
cut := min(s.addrAmount, ln) cut := s.addrAmount
if cut > ln {
cut = ln
}
body.SetIDList(ids[:cut]) body.SetIDList(ids[:cut])
newResp.SetMetaHeader(resp.GetMetaHeader()) newResp.SetMetaHeader(resp.GetMetaHeader())

View file

@ -388,7 +388,10 @@ func (it *sliceKeySpaceIterator) Next(_ context.Context, size uint32) ([]objectc
if it.cur >= len(it.objs) { if it.cur >= len(it.objs) {
return nil, engine.ErrEndOfListing return nil, engine.ErrEndOfListing
} }
end := min(it.cur+int(size), len(it.objs)) end := it.cur + int(size)
if end > len(it.objs) {
end = len(it.objs)
}
ret := it.objs[it.cur:end] ret := it.objs[it.cur:end]
it.cur = end it.cur = end
return ret, nil return ret, nil

View file

@ -4,7 +4,7 @@ import (
"context" "context"
"errors" "errors"
"path" "path"
"slices" "sort"
"testing" "testing"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama"
@ -177,7 +177,9 @@ func TestGetSubTreeOrderAsc(t *testing.T) {
require.True(t, found, "unknown node") require.True(t, found, "unknown node")
} }
require.True(t, slices.IsSorted(paths)) require.True(t, sort.SliceIsSorted(paths, func(i, j int) bool {
return paths[i] < paths[j]
}))
} }
var ( var (

View file

@ -167,7 +167,9 @@ func mergeOperationStreams(streams []chan *pilorama.Move, merged chan<- *piloram
merged <- ms[minTimeMoveIndex] merged <- ms[minTimeMoveIndex]
height := ms[minTimeMoveIndex].Time height := ms[minTimeMoveIndex].Time
if ms[minTimeMoveIndex] = <-streams[minTimeMoveIndex]; ms[minTimeMoveIndex] == nil { if ms[minTimeMoveIndex] = <-streams[minTimeMoveIndex]; ms[minTimeMoveIndex] == nil {
minStreamedLastHeight = min(minStreamedLastHeight, height) if minStreamedLastHeight > height {
minStreamedLastHeight = height
}
} }
} }
@ -201,7 +203,9 @@ func (s *Service) applyOperationStream(ctx context.Context, cid cid.ID, treeID s
errGroup.Go(func() error { errGroup.Go(func() error {
if err := s.forest.TreeApply(ctx, cid, treeID, m, true); err != nil { if err := s.forest.TreeApply(ctx, cid, treeID, m, true); err != nil {
heightMtx.Lock() heightMtx.Lock()
unappliedOperationHeight = min(unappliedOperationHeight, m.Time) if m.Time < unappliedOperationHeight {
unappliedOperationHeight = m.Time
}
heightMtx.Unlock() heightMtx.Unlock()
return err return err
} }