Range failed on complex object #617

New issue

Closed

opened 2023-08-16 12:50:08 +00:00 by dkirillov · 2 comments

dkirillov commented

2023-08-16 12:50:08 +00:00

Member

Any range on complex object failed with object not found error

Expected Behavior

Range successfully performed.

Current Behavior

object not found error

Steps to Reproduce (for bugs)

Test creates container with REP 2 policy, put object and try range using different nodes
Sample output:

max obj size 67108864, obj size 67109198
container  9VXYTw2dP4No2xwq4k3thTWVyJC1C2RarRqdgHN9xvs3
object  5vBt8vehYeQmvtSfcDqCnduEGsADYPFrU711eDbrgoYj
s01.frostfs.devenv:8080 <nil>
s02.frostfs.devenv:8080 rpc error: code = Unknown desc = status: code = 2049 message = object not found
s03.frostfs.devenv:8080 rpc error: code = Unknown desc = status: code = 2049 message = object not found
s04.frostfs.devenv:8080 rpc error: code = Unknown desc = status: code = 2049 message = object not found

Exact node on which I get error differ from run to run. Also it doesn't matter what exactly range I want to get (error occurs on even first 20 bytes)

func TestPool(t *testing.T) {
	ctx := context.Background()
	devenvKey := "1dd37fba80fec4e6a6f13fd708d8dcb3b29def768017052f6c930fa1c5d90bbb"
	key, err := keys.NewPrivateKeyFromHex(devenvKey)
	require.NoError(t, err)
	var devenvOwner user.ID
	user.IDFromKey(&devenvOwner, key.PrivateKey.PublicKey)

	var prm InitParameters
	prm.SetKey(&key.PrivateKey)
	prm.SetLogger(zaptest.NewLogger(t))
	prm.SetNodeDialTimeout(3 * time.Second)
	prm.SetClientRebalanceInterval(5 * time.Second)
	prm.AddNode(NewNodeParam(1, "s01.frostfs.devenv:8080", 1))
	clientPool, err := NewPool(prm)
	require.NoError(t, err)
	err = clientPool.Dial(ctx)
	require.NoError(t, err)

	ni, err := clientPool.NetworkInfo(ctx)
	require.NoError(t, err)
	objSize := ni.MaxObjectSize() + 334
	fmt.Printf("max obj size %d, obj size %d\n", ni.MaxObjectSize(), objSize)

	var pp netmap.PlacementPolicy
	_ = pp.DecodeString("REP 2")
	var cnr container.Container
	cnr.Init()
	cnr.SetBasicACL(acl.PublicRWExtended)
	cnr.SetPlacementPolicy(pp)
	cnr.SetOwner(devenvOwner)
	var prmPutCnr PrmContainerPut
	prmPutCnr.SetContainer(cnr)

	cnrID, err := clientPool.PutContainer(ctx, prmPutCnr)
	require.NoError(t, err)
	fmt.Println("container ", cnrID)

	obj := object.New()
	obj.SetContainerID(cnrID)
	obj.SetPayload(make([]byte, objSize))
	obj.SetOwnerID(&devenvOwner)
	var prmPutObj PrmObjectPut
	prmPutObj.SetHeader(*obj)

	objID, err := clientPool.PutObject(ctx, prmPutObj)
	require.NoError(t, err)
	fmt.Println("object ", objID)

	addressses := []string{
		"s01.frostfs.devenv:8080",
		"s02.frostfs.devenv:8080",
		"s03.frostfs.devenv:8080",
		"s04.frostfs.devenv:8080",
	}
	clients := make([]sdkClient.Client, 4)

	for i, addr := range addressses {
		var prm sdkClient.PrmInit
		prm.SetDefaultPrivateKey(key.PrivateKey)
		prm.ResolveFrostFSFailures()
		var prmDial sdkClient.PrmDial
		prmDial.SetServerURI(addr)
		var cl sdkClient.Client
		cl.Init(prm)
		err = cl.Dial(ctx, prmDial)
		require.NoError(t, err)

		clients[i] = cl

		var prmrange sdkClient.PrmObjectRange
		prmrange.FromContainer(cnrID)
		prmrange.ByID(objID)
		prmrange.SetLength(ni.MaxObjectSize() + 1)

		rr, err := cl.ObjectRangeInit(ctx, prmrange)
		if err == nil {
			_, err = io.ReadAll(rr)
		}
		fmt.Println(addr, err)
	}
}

Regression

Probably yes

Your Environment

Version used: c3e23a1448
Server setup and configuration: devenv

Any range on complex object failed with `object not found` error ## Expected Behavior Range successfully performed. ## Current Behavior `object not found` error ## Steps to Reproduce (for bugs) Test creates container with `REP 2` policy, put object and try range using different nodes Sample output: ``` max obj size 67108864, obj size 67109198 container 9VXYTw2dP4No2xwq4k3thTWVyJC1C2RarRqdgHN9xvs3 object 5vBt8vehYeQmvtSfcDqCnduEGsADYPFrU711eDbrgoYj s01.frostfs.devenv:8080 <nil> s02.frostfs.devenv:8080 rpc error: code = Unknown desc = status: code = 2049 message = object not found s03.frostfs.devenv:8080 rpc error: code = Unknown desc = status: code = 2049 message = object not found s04.frostfs.devenv:8080 rpc error: code = Unknown desc = status: code = 2049 message = object not found ``` Exact node on which I get error differ from run to run. Also it doesn't matter what exactly range I want to get (error occurs on even first 20 bytes) ```golang func TestPool(t *testing.T) { ctx := context.Background() devenvKey := "1dd37fba80fec4e6a6f13fd708d8dcb3b29def768017052f6c930fa1c5d90bbb" key, err := keys.NewPrivateKeyFromHex(devenvKey) require.NoError(t, err) var devenvOwner user.ID user.IDFromKey(&devenvOwner, key.PrivateKey.PublicKey) var prm InitParameters prm.SetKey(&key.PrivateKey) prm.SetLogger(zaptest.NewLogger(t)) prm.SetNodeDialTimeout(3 * time.Second) prm.SetClientRebalanceInterval(5 * time.Second) prm.AddNode(NewNodeParam(1, "s01.frostfs.devenv:8080", 1)) clientPool, err := NewPool(prm) require.NoError(t, err) err = clientPool.Dial(ctx) require.NoError(t, err) ni, err := clientPool.NetworkInfo(ctx) require.NoError(t, err) objSize := ni.MaxObjectSize() + 334 fmt.Printf("max obj size %d, obj size %d\n", ni.MaxObjectSize(), objSize) var pp netmap.PlacementPolicy _ = pp.DecodeString("REP 2") var cnr container.Container cnr.Init() cnr.SetBasicACL(acl.PublicRWExtended) cnr.SetPlacementPolicy(pp) cnr.SetOwner(devenvOwner) var prmPutCnr PrmContainerPut prmPutCnr.SetContainer(cnr) cnrID, err := clientPool.PutContainer(ctx, prmPutCnr) require.NoError(t, err) fmt.Println("container ", cnrID) obj := object.New() obj.SetContainerID(cnrID) obj.SetPayload(make([]byte, objSize)) obj.SetOwnerID(&devenvOwner) var prmPutObj PrmObjectPut prmPutObj.SetHeader(*obj) objID, err := clientPool.PutObject(ctx, prmPutObj) require.NoError(t, err) fmt.Println("object ", objID) addressses := []string{ "s01.frostfs.devenv:8080", "s02.frostfs.devenv:8080", "s03.frostfs.devenv:8080", "s04.frostfs.devenv:8080", } clients := make([]sdkClient.Client, 4) for i, addr := range addressses { var prm sdkClient.PrmInit prm.SetDefaultPrivateKey(key.PrivateKey) prm.ResolveFrostFSFailures() var prmDial sdkClient.PrmDial prmDial.SetServerURI(addr) var cl sdkClient.Client cl.Init(prm) err = cl.Dial(ctx, prmDial) require.NoError(t, err) clients[i] = cl var prmrange sdkClient.PrmObjectRange prmrange.FromContainer(cnrID) prmrange.ByID(objID) prmrange.SetLength(ni.MaxObjectSize() + 1) rr, err := cl.ObjectRangeInit(ctx, prmrange) if err == nil { _, err = io.ReadAll(rr) } fmt.Println(addr, err) } } ``` ## Regression Probably yes ## Your Environment  * Version used: https://git.frostfs.info/TrueCloudLab/frostfs-node/commit/c3e23a14489b97aab71392e436a435e99f6d7361 * Server setup and configuration: devenv

dkirillov added the

bug

frostfs-node

triage

labels 2023-08-16 12:50:08 +00:00

ale64bit was assigned by fyrchik

2023-08-17 07:32:03 +00:00

ale64bit commented

2023-08-17 08:08:40 +00:00

Member

I ran that test several times on master (both node and sdk) and can't reproduce:

~/src/frostfs-sdk-go$ go test -count 1 -timeout 30s -v -run ^TestPool$ git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool
=== RUN   TestPool
max obj size 67108864, obj size 67109198
container  4UKqyXQBahVx5ihgCyUiFXzci1LuETgK6a4eYeYUjnNc
object  Fud8xXFC5gEhCtSDayAzWx9X7e9pqRk4bE2L4sPFi4wG
s01.frostfs.devenv:8080 <nil>
s02.frostfs.devenv:8080 <nil>
s03.frostfs.devenv:8080 <nil>
s04.frostfs.devenv:8080 <nil>
--- PASS: TestPool (9.43s)
PASS
ok  	git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool	9.444s

Could you retry on master and confirm it still happens for you?

EDIT: apologies, it does happen on master. I'll try to bisect it.

I ran that test several times on master (both node and sdk) and can't reproduce: ```sh ~/src/frostfs-sdk-go$ go test -count 1 -timeout 30s -v -run ^TestPool$ git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool === RUN TestPool max obj size 67108864, obj size 67109198 container 4UKqyXQBahVx5ihgCyUiFXzci1LuETgK6a4eYeYUjnNc object Fud8xXFC5gEhCtSDayAzWx9X7e9pqRk4bE2L4sPFi4wG s01.frostfs.devenv:8080 <nil> s02.frostfs.devenv:8080 <nil> s03.frostfs.devenv:8080 <nil> s04.frostfs.devenv:8080 <nil> --- PASS: TestPool (9.43s) PASS ok git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool 9.444s ``` Could you retry on master and confirm it still happens for you? EDIT: apologies, it does happen on master. I'll try to bisect it.

ale64bit commented

2023-08-17 08:09:28 +00:00

Member

I also tried an engine-specific test but that also didn't repro the issue:

func TestGetRange(t *testing.T) {
	s1 := testNewShard(t, 1)
	s2 := testNewShard(t, 2)

	e := testNewEngine(t).setInitializedShards(t, s1, s2).engine
	defer e.Close()

	obj := testutil.GenerateObjectWithSize((64 << 20) + 334)
	addr := testutil.AddressFromObject(t, obj)

	{
		prm := PutPrm{}
		prm.WithObject(obj)
		require.NoError(t, e.Put(context.Background(), prm))
	}

	{
		r := object.NewRange()
		r.SetOffset(0)
		r.SetLength(20)
		prm := RngPrm{}
		prm.WithAddress(addr)
		prm.WithPayloadRange(r)
		_, err := e.GetRange(context.Background(), prm)
		require.NoError(t, err)
	}
}

I also tried an engine-specific test but that also didn't repro the issue: ```go func TestGetRange(t *testing.T) { s1 := testNewShard(t, 1) s2 := testNewShard(t, 2) e := testNewEngine(t).setInitializedShards(t, s1, s2).engine defer e.Close() obj := testutil.GenerateObjectWithSize((64 << 20) + 334) addr := testutil.AddressFromObject(t, obj) { prm := PutPrm{} prm.WithObject(obj) require.NoError(t, e.Put(context.Background(), prm)) } { r := object.NewRange() r.SetOffset(0) r.SetLength(20) prm := RngPrm{} prm.WithAddress(addr) prm.WithPayloadRange(r) _, err := e.GetRange(context.Background(), prm) require.NoError(t, err) } } ```