frostfs-node/pkg/local_object_storage/engine/error_test.go

package engine

import (
	"fmt"
	"os"
	"path/filepath"
	"strconv"
	"testing"

	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object"
	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor"
	meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase"
	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama"
	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard"
	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger"
	apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
	cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
	objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
	"github.com/stretchr/testify/require"
	"go.uber.org/zap/zaptest"
)

const errSmallSize = 256

func newEngineWithErrorThreshold(t testing.TB, dir string, errThreshold uint32) (*StorageEngine, string, [2]*shard.ID) {
	if dir == "" {
		var err error

		dir, err = os.MkdirTemp("", "*")
		require.NoError(t, err)
		t.Cleanup(func() { _ = os.RemoveAll(dir) })
	}

	e := New(
		WithLogger(&logger.Logger{Logger: zaptest.NewLogger(t)}),
		WithShardPoolSize(1),
		WithErrorThreshold(errThreshold))

	var ids [2]*shard.ID
	var err error

	for i := range ids {
		ids[i], err = e.AddShard(
			shard.WithLogger(&logger.Logger{Logger: zaptest.NewLogger(t)}),
			shard.WithBlobStorOptions(
				blobstor.WithStorages(newStorages(filepath.Join(dir, strconv.Itoa(i)), errSmallSize))),
			shard.WithMetaBaseOptions(
				meta.WithPath(filepath.Join(dir, fmt.Sprintf("%d.metabase", i))),
				meta.WithPermissions(0700),
				meta.WithEpochState(epochState{}),
			),
			shard.WithPiloramaOptions(
				pilorama.WithPath(filepath.Join(dir, fmt.Sprintf("%d.pilorama", i))),
				pilorama.WithPerm(0700)))
		require.NoError(t, err)
	}
	require.NoError(t, e.Open())
	require.NoError(t, e.Init())

	return e, dir, ids
}

func TestErrorReporting(t *testing.T) {
	t.Run("ignore errors by default", func(t *testing.T) {
		e, dir, id := newEngineWithErrorThreshold(t, "", 0)

		obj := generateObjectWithCID(t, cidtest.ID())
		obj.SetPayload(make([]byte, errSmallSize))

		var prm shard.PutPrm
		prm.SetObject(obj)
		e.mtx.RLock()
		_, err := e.shards[id[0].String()].Shard.Put(prm)
		e.mtx.RUnlock()
		require.NoError(t, err)

		_, err = e.Get(GetPrm{addr: object.AddressOf(obj)})
		require.NoError(t, err)

		checkShardState(t, e, id[0], 0, mode.ReadWrite)
		checkShardState(t, e, id[1], 0, mode.ReadWrite)

		corruptSubDir(t, filepath.Join(dir, "0"))

		for i := uint32(1); i < 3; i++ {
			_, err = e.Get(GetPrm{addr: object.AddressOf(obj)})
			require.Error(t, err)
			checkShardState(t, e, id[0], i, mode.ReadWrite)
			checkShardState(t, e, id[1], 0, mode.ReadWrite)
		}
	})
	t.Run("with error threshold", func(t *testing.T) {
		const errThreshold = 3

		e, dir, id := newEngineWithErrorThreshold(t, "", errThreshold)

		obj := generateObjectWithCID(t, cidtest.ID())
		obj.SetPayload(make([]byte, errSmallSize))

		var prm shard.PutPrm
		prm.SetObject(obj)
		e.mtx.RLock()
		_, err := e.shards[id[0].String()].Put(prm)
		e.mtx.RUnlock()
		require.NoError(t, err)

		_, err = e.Get(GetPrm{addr: object.AddressOf(obj)})
		require.NoError(t, err)

		checkShardState(t, e, id[0], 0, mode.ReadWrite)
		checkShardState(t, e, id[1], 0, mode.ReadWrite)

		corruptSubDir(t, filepath.Join(dir, "0"))

		for i := uint32(1); i < errThreshold; i++ {
			_, err = e.Get(GetPrm{addr: object.AddressOf(obj)})
			require.Error(t, err)
			checkShardState(t, e, id[0], i, mode.ReadWrite)
			checkShardState(t, e, id[1], 0, mode.ReadWrite)
		}

		for i := uint32(0); i < 2; i++ {
			_, err = e.Get(GetPrm{addr: object.AddressOf(obj)})
			require.Error(t, err)
			checkShardState(t, e, id[0], errThreshold+i, mode.DegradedReadOnly)
			checkShardState(t, e, id[1], 0, mode.ReadWrite)
		}

		require.NoError(t, e.SetShardMode(id[0], mode.ReadWrite, false))
		checkShardState(t, e, id[0], errThreshold+1, mode.ReadWrite)

		require.NoError(t, e.SetShardMode(id[0], mode.ReadWrite, true))
		checkShardState(t, e, id[0], 0, mode.ReadWrite)
	})
}

// Issue #1186.
func TestBlobstorFailback(t *testing.T) {
	dir, err := os.MkdirTemp("", "*")
	require.NoError(t, err)
	t.Cleanup(func() { require.NoError(t, os.RemoveAll(dir)) })

	e, _, id := newEngineWithErrorThreshold(t, dir, 1)

	objs := make([]*objectSDK.Object, 0, 2)
	for _, size := range []int{15, errSmallSize + 1} {
		obj := generateObjectWithCID(t, cidtest.ID())
		obj.SetPayload(make([]byte, size))

		var prm shard.PutPrm
		prm.SetObject(obj)
		e.mtx.RLock()
		_, err = e.shards[id[0].String()].Shard.Put(prm)
		e.mtx.RUnlock()
		require.NoError(t, err)
		objs = append(objs, obj)
	}

	for i := range objs {
		addr := object.AddressOf(objs[i])
		_, err = e.Get(GetPrm{addr: addr})
		require.NoError(t, err)
		_, err = e.GetRange(RngPrm{addr: addr})
		require.NoError(t, err)
	}

	checkShardState(t, e, id[0], 0, mode.ReadWrite)
	require.NoError(t, e.Close())

	p1 := e.shards[id[0].String()].Shard.DumpInfo().BlobStorInfo.SubStorages[1].Path
	p2 := e.shards[id[1].String()].Shard.DumpInfo().BlobStorInfo.SubStorages[1].Path
	tmp := filepath.Join(dir, "tmp")
	require.NoError(t, os.Rename(p1, tmp))
	require.NoError(t, os.Rename(p2, p1))
	require.NoError(t, os.Rename(tmp, p2))

	e, _, id = newEngineWithErrorThreshold(t, dir, 1)

	for i := range objs {
		addr := object.AddressOf(objs[i])
		getRes, err := e.Get(GetPrm{addr: addr})
		require.NoError(t, err)
		require.Equal(t, objs[i], getRes.Object())

		rngRes, err := e.GetRange(RngPrm{addr: addr, off: 1, ln: 10})
		require.NoError(t, err)
		require.Equal(t, objs[i].Payload()[1:11], rngRes.Object().Payload())

		_, err = e.GetRange(RngPrm{addr: addr, off: errSmallSize + 10, ln: 1})
		require.ErrorAs(t, err, &apistatus.ObjectOutOfRange{})
	}

	checkShardState(t, e, id[0], 1, mode.DegradedReadOnly)
	checkShardState(t, e, id[1], 0, mode.ReadWrite)
}

func checkShardState(t *testing.T, e *StorageEngine, id *shard.ID, errCount uint32, mode mode.Mode) {
	e.mtx.RLock()
	sh := e.shards[id.String()]
	e.mtx.RUnlock()

	require.Equal(t, errCount, sh.errorCount.Load())
	require.Equal(t, mode, sh.GetMode())
}

// corruptSubDir makes random directory except "blobovnicza" in blobstor FSTree unreadable.
func corruptSubDir(t *testing.T, dir string) {
	de, err := os.ReadDir(dir)
	require.NoError(t, err)

	// FIXME(@cthulhu-rider): copy-paste of unexported const from blobstor package, see #1407
	const dirBlobovnicza = "blobovnicza"

	for i := range de {
		if de[i].IsDir() && de[i].Name() != dirBlobovnicza {
			require.NoError(t, os.Chmod(filepath.Join(dir, de[i].Name()), 0))
			return
		}
	}
}