TestEvacuateShard is flaky #382

Closed
opened 2023-05-22 08:57:20 +00:00 by fyrchik · 4 comments
Owner
--- FAIL: TestEvacuateShard (0.26s)
    logger.go:130: 2023-05-22T11:54:35.856+0300 DEBUG   opening...      {"shard_id": "PTE71R4Y4wrdasvh8pairb"}
    logger.go:130: 2023-05-22T11:54:35.871+0300 DEBUG   created directory for Metabase  {"shard_id": "PTE71R4Y4wrdasvh8pairb", "path": "/run/user/1000/TestEvacuateShard3572005266/001/1.metabase"}
    logger.go:130: 2023-05-22T11:54:35.864+0300 DEBUG   opening...      {"shard_id": "VxBm545PypAuhU9Ca7FYqC"}
    logger.go:130: 2023-05-22T11:54:35.873+0300 DEBUG   created directory for Metabase  {"shard_id": "VxBm545PypAuhU9Ca7FYqC", "path": "/run/user/1000/TestEvacuateShard3572005266/001/0.metabase"}
    logger.go:130: 2023-05-22T11:54:35.864+0300 DEBUG   opening...      {"shard_id": "61fea8kdkosKmQaoCwiSDQ"}
    logger.go:130: 2023-05-22T11:54:35.875+0300 DEBUG   created directory for Metabase  {"shard_id": "61fea8kdkosKmQaoCwiSDQ", "path": "/run/user/1000/TestEvacuateShard3572005266/001/2.metabase"}
    logger.go:130: 2023-05-22T11:54:35.900+0300 DEBUG   opened boltDB instance for Metabase   {"shard_id": "PTE71R4Y4wrdasvh8pairb"}
    logger.go:130: 2023-05-22T11:54:35.900+0300 DEBUG   checking metabase version       {"shard_id": "PTE71R4Y4wrdasvh8pairb"}
    logger.go:130: 2023-05-22T11:54:35.906+0300 DEBUG   opened boltDB instance for Metabase   {"shard_id": "61fea8kdkosKmQaoCwiSDQ"}
    logger.go:130: 2023-05-22T11:54:35.906+0300 DEBUG   checking metabase version       {"shard_id": "61fea8kdkosKmQaoCwiSDQ"}
    logger.go:130: 2023-05-22T11:54:35.907+0300 DEBUG   opened boltDB instance for Metabase   {"shard_id": "VxBm545PypAuhU9Ca7FYqC"}
    logger.go:130: 2023-05-22T11:54:35.907+0300 DEBUG   checking metabase version       {"shard_id": "VxBm545PypAuhU9Ca7FYqC"}
    logger.go:130: 2023-05-22T11:54:35.913+0300 DEBUG   initializing... {"shard_id": "VxBm545PypAuhU9Ca7FYqC"}
    logger.go:130: 2023-05-22T11:54:35.914+0300 DEBUG   initializing... {"shard_id": "PTE71R4Y4wrdasvh8pairb"}
    logger.go:130: 2023-05-22T11:54:35.914+0300 DEBUG   initializing... {"shard_id": "61fea8kdkosKmQaoCwiSDQ"}
    logger.go:130: 2023-05-22T11:54:35.916+0300 INFO    local object storage operation  {"shard_id": "VxBm545PypAuhU9Ca7FYqC", "address": "28Y1RXGnfhtdZyd4S3MWtmpTKj4dn9EpGVmyJ5rayg85/4krvMcXaR1vka6GYSL2gPPEzu62zFtWRigWjxVTSrovr", "op": "PUT", "type": "fstree", "storage_id": ""}
    logger.go:130: 2023-05-22T11:54:35.929+0300 INFO    local object storage operation  {"shard_id": "VxBm545PypAuhU9Ca7FYqC", "address": "28Y1RXGnfhtdZyd4S3MWtmpTKj4dn9EpGVmyJ5rayg85/4krvMcXaR1vka6GYSL2gPPEzu62zFtWRigWjxVTSrovr", "op": "metabase PUT"}
    logger.go:130: 2023-05-22T11:54:35.929+0300 INFO    local object storage operation  {"shard_id": "PTE71R4Y4wrdasvh8pairb", "address": "EKWcXeXbr8q7no2fsm5c2PaX9oU6DaFHLet3396qfdGC/6yGQ9gXM9DDkByWJZv26FxV7gSef8cYsUtPfqYrsYCmB", "op": "PUT", "type": "fstree", "storage_id": ""}
    logger.go:130: 2023-05-22T11:54:35.943+0300 INFO    local object storage operation  {"shard_id": "PTE71R4Y4wrdasvh8pairb", "address": "EKWcXeXbr8q7no2fsm5c2PaX9oU6DaFHLet3396qfdGC/6yGQ9gXM9DDkByWJZv26FxV7gSef8cYsUtPfqYrsYCmB", "op": "metabase PUT"}
    logger.go:130: 2023-05-22T11:54:35.944+0300 INFO    local object storage operation  {"shard_id": "61fea8kdkosKmQaoCwiSDQ", "address": "FVbcYaWh2f5NXaor2dtDhEZrAKhovr8HQNWR7Mm4TUMJ/CBBj3dpaTrDAnJ3x41e2MSZiFkdjsRtK37xJNuZA58ag", "op": "PUT", "type": "fstree", "storage_id": ""}
    logger.go:130: 2023-05-22T11:54:35.956+0300 INFO    local object storage operation  {"shard_id": "61fea8kdkosKmQaoCwiSDQ", "address": "FVbcYaWh2f5NXaor2dtDhEZrAKhovr8HQNWR7Mm4TUMJ/CBBj3dpaTrDAnJ3x41e2MSZiFkdjsRtK37xJNuZA58ag", "op": "metabase PUT"}
    logger.go:130: 2023-05-22T11:54:35.958+0300 INFO    local object storage operation  {"shard_id": "VxBm545PypAuhU9Ca7FYqC", "address": "BXUnEhBZratySCBTXSnuZ5VFVtxU8RzXkHZSyBr2eUok/jr1RtwsagXM4a6sHCbBhp8zk7xRpFkaJFmSuLkceQ9D", "op": "PUT", "type": "fstree", "storage_id": ""}
    logger.go:130: 2023-05-22T11:54:35.971+0300 INFO    local object storage operation  {"shard_id": "VxBm545PypAuhU9Ca7FYqC", "address": "BXUnEhBZratySCBTXSnuZ5VFVtxU8RzXkHZSyBr2eUok/jr1RtwsagXM4a6sHCbBhp8zk7xRpFkaJFmSuLkceQ9D", "op": "metabase PUT"}
    logger.go:130: 2023-05-22T11:54:35.972+0300 INFO    local object storage operation  {"shard_id": "61fea8kdkosKmQaoCwiSDQ", "address": "BYjTCAWsZqoKGv1gr2F4p6WM4KAuPFJkMwrPPxTtPZXu/EnyYYEsD66dj1BqHXaa9diVaSAoprYJac9rHDnsN39Dh", "op": "PUT", "type": "fstree", "storage_id": ""}
    logger.go:130: 2023-05-22T11:54:35.995+0300 INFO    local object storage operation  {"shard_id": "61fea8kdkosKmQaoCwiSDQ", "address": "BYjTCAWsZqoKGv1gr2F4p6WM4KAuPFJkMwrPPxTtPZXu/EnyYYEsD66dj1BqHXaa9diVaSAoprYJac9rHDnsN39Dh", "op": "metabase PUT"}
    logger.go:130: 2023-05-22T11:54:35.997+0300 INFO    local object storage operation  {"shard_id": "PTE71R4Y4wrdasvh8pairb", "address": "GyAGZLkmEF5xnWACmVePVh4vkpffAS3BZXWUrAp1vBRH/AjNezC38g9TfHURjKwdxhcWi1WfPwD4cq2BPgv8RFh6G", "op": "PUT", "type": "fstree", "storage_id": ""}
    logger.go:130: 2023-05-22T11:54:36.009+0300 INFO    local object storage operation  {"shard_id": "PTE71R4Y4wrdasvh8pairb", "address": "GyAGZLkmEF5xnWACmVePVh4vkpffAS3BZXWUrAp1vBRH/AjNezC38g9TfHURjKwdxhcWi1WfPwD4cq2BPgv8RFh6G", "op": "metabase PUT"}
    logger.go:130: 2023-05-22T11:54:36.010+0300 INFO    local object storage operation  {"shard_id": "PTE71R4Y4wrdasvh8pairb", "address": "22c8TETMng4CzWuyGGadUwhhFnNR6wJRn66JnD5sJY9i/RLXD3Lz4Rnu1dqQ5tf2dVuaaYRZE5CYuKQn4z9mN7Q5", "op": "PUT", "type": "fstree", "storage_id": ""}
    logger.go:130: 2023-05-22T11:54:36.021+0300 INFO    local object storage operation  {"shard_id": "PTE71R4Y4wrdasvh8pairb", "address": "22c8TETMng4CzWuyGGadUwhhFnNR6wJRn66JnD5sJY9i/RLXD3Lz4Rnu1dqQ5tf2dVuaaYRZE5CYuKQn4z9mN7Q5", "op": "metabase PUT"}
    logger.go:130: 2023-05-22T11:54:36.023+0300 INFO    local object storage operation  {"shard_id": "61fea8kdkosKmQaoCwiSDQ", "address": "E3kNgc25wGz9RNsNpp8sZ8PSjyBMCjBK7vFJYNmXFCBd/J9TXRUFQkwucuoRTBFMkJfFP6oYZuhm6XXijB7grQ8fa", "op": "PUT", "type": "fstree", "storage_id": ""}
    logger.go:130: 2023-05-22T11:54:36.034+0300 INFO    local object storage operation  {"shard_id": "61fea8kdkosKmQaoCwiSDQ", "address": "E3kNgc25wGz9RNsNpp8sZ8PSjyBMCjBK7vFJYNmXFCBd/J9TXRUFQkwucuoRTBFMkJfFP6oYZuhm6XXijB7grQ8fa", "op": "metabase PUT"}
    logger.go:130: 2023-05-22T11:54:36.039+0300 INFO    setting shard mode      {"shard_id": "61fea8kdkosKmQaoCwiSDQ", "old_mode": "READ_WRITE", "new_mode": "READ_ONLY"}
    logger.go:130: 2023-05-22T11:54:36.039+0300 DEBUG   closing...      {"shard_id": "61fea8kdkosKmQaoCwiSDQ"}
    logger.go:130: 2023-05-22T11:54:36.039+0300 DEBUG   opening...      {"shard_id": "61fea8kdkosKmQaoCwiSDQ"}
    logger.go:130: 2023-05-22T11:54:36.039+0300 DEBUG   initializing... {"shard_id": "61fea8kdkosKmQaoCwiSDQ"}
    logger.go:130: 2023-05-22T11:54:36.039+0300 DEBUG   created directory for Metabase  {"shard_id": "61fea8kdkosKmQaoCwiSDQ", "path": "/run/user/1000/TestEvacuateShard3572005266/001/2.metabase"}
    logger.go:130: 2023-05-22T11:54:36.039+0300 DEBUG   opened boltDB instance for Metabase   {"shard_id": "61fea8kdkosKmQaoCwiSDQ"}
    logger.go:130: 2023-05-22T11:54:36.039+0300 DEBUG   checking metabase version       {"shard_id": "61fea8kdkosKmQaoCwiSDQ"}
    logger.go:130: 2023-05-22T11:54:36.039+0300 INFO    shard mode set successfully     {"shard_id": "61fea8kdkosKmQaoCwiSDQ", "mode": "READ_ONLY"}
    logger.go:130: 2023-05-22T11:54:36.039+0300 INFO    started shards evacuation       {"shard_ids": ["61fea8kdkosKmQaoCwiSDQ"]}
    logger.go:130: 2023-05-22T11:54:36.048+0300 INFO    local object storage operation  {"shard_id": "PTE71R4Y4wrdasvh8pairb", "address": "BYjTCAWsZqoKGv1gr2F4p6WM4KAuPFJkMwrPPxTtPZXu/EnyYYEsD66dj1BqHXaa9diVaSAoprYJac9rHDnsN39Dh", "op": "PUT", "type": "fstree", "storage_id": ""}
    logger.go:130: 2023-05-22T11:54:36.064+0300 INFO    local object storage operation  {"shard_id": "PTE71R4Y4wrdasvh8pairb", "address": "BYjTCAWsZqoKGv1gr2F4p6WM4KAuPFJkMwrPPxTtPZXu/EnyYYEsD66dj1BqHXaa9diVaSAoprYJac9rHDnsN39Dh", "op": "metabase PUT"}
    logger.go:130: 2023-05-22T11:54:36.064+0300 DEBUG   object is moved to another shard      {"from": "61fea8kdkosKmQaoCwiSDQ", "to": "PTE71R4Y4wrdasvh8pairb", "addr": "BYjTCAWsZqoKGv1gr2F4p6WM4KAuPFJkMwrPPxTtPZXu/EnyYYEsD66dj1BqHXaa9diVaSAoprYJac9rHDnsN39Dh"}
    logger.go:130: 2023-05-22T11:54:36.065+0300 INFO    local object storage operation  {"shard_id": "PTE71R4Y4wrdasvh8pairb", "address": "E3kNgc25wGz9RNsNpp8sZ8PSjyBMCjBK7vFJYNmXFCBd/J9TXRUFQkwucuoRTBFMkJfFP6oYZuhm6XXijB7grQ8fa", "op": "PUT", "type": "fstree", "storage_id": ""}
    logger.go:130: 2023-05-22T11:54:36.077+0300 INFO    local object storage operation  {"shard_id": "PTE71R4Y4wrdasvh8pairb", "address": "E3kNgc25wGz9RNsNpp8sZ8PSjyBMCjBK7vFJYNmXFCBd/J9TXRUFQkwucuoRTBFMkJfFP6oYZuhm6XXijB7grQ8fa", "op": "metabase PUT"}
    logger.go:130: 2023-05-22T11:54:36.077+0300 DEBUG   object is moved to another shard      {"from": "61fea8kdkosKmQaoCwiSDQ", "to": "PTE71R4Y4wrdasvh8pairb", "addr": "E3kNgc25wGz9RNsNpp8sZ8PSjyBMCjBK7vFJYNmXFCBd/J9TXRUFQkwucuoRTBFMkJfFP6oYZuhm6XXijB7grQ8fa"}
    logger.go:130: 2023-05-22T11:54:36.078+0300 INFO    local object storage operation  {"shard_id": "PTE71R4Y4wrdasvh8pairb", "address": "FVbcYaWh2f5NXaor2dtDhEZrAKhovr8HQNWR7Mm4TUMJ/CBBj3dpaTrDAnJ3x41e2MSZiFkdjsRtK37xJNuZA58ag", "op": "PUT", "type": "fstree", "storage_id": ""}
    logger.go:130: 2023-05-22T11:54:36.089+0300 INFO    local object storage operation  {"shard_id": "PTE71R4Y4wrdasvh8pairb", "address": "FVbcYaWh2f5NXaor2dtDhEZrAKhovr8HQNWR7Mm4TUMJ/CBBj3dpaTrDAnJ3x41e2MSZiFkdjsRtK37xJNuZA58ag", "op": "metabase PUT"}
    logger.go:130: 2023-05-22T11:54:36.089+0300 DEBUG   object is moved to another shard      {"from": "61fea8kdkosKmQaoCwiSDQ", "to": "PTE71R4Y4wrdasvh8pairb", "addr": "FVbcYaWh2f5NXaor2dtDhEZrAKhovr8HQNWR7Mm4TUMJ/CBBj3dpaTrDAnJ3x41e2MSZiFkdjsRtK37xJNuZA58ag"}
    logger.go:130: 2023-05-22T11:54:36.090+0300 INFO    shards evacuation finished successfully{"shard_ids": ["61fea8kdkosKmQaoCwiSDQ"]}
    logger.go:130: 2023-05-22T11:54:36.096+0300 INFO    started shards evacuation       {"shard_ids": ["61fea8kdkosKmQaoCwiSDQ"]}
    logger.go:130: 2023-05-22T11:54:36.098+0300 INFO    local object storage operation  {"shard_id": "VxBm545PypAuhU9Ca7FYqC", "address": "E3kNgc25wGz9RNsNpp8sZ8PSjyBMCjBK7vFJYNmXFCBd/J9TXRUFQkwucuoRTBFMkJfFP6oYZuhm6XXijB7grQ8fa", "op": "PUT", "type": "fstree", "storage_id": ""}
    logger.go:130: 2023-05-22T11:54:36.109+0300 INFO    local object storage operation  {"shard_id": "VxBm545PypAuhU9Ca7FYqC", "address": "E3kNgc25wGz9RNsNpp8sZ8PSjyBMCjBK7vFJYNmXFCBd/J9TXRUFQkwucuoRTBFMkJfFP6oYZuhm6XXijB7grQ8fa", "op": "metabase PUT"}
    logger.go:130: 2023-05-22T11:54:36.109+0300 DEBUG   object is moved to another shard      {"from": "61fea8kdkosKmQaoCwiSDQ", "to": "VxBm545PypAuhU9Ca7FYqC", "addr": "E3kNgc25wGz9RNsNpp8sZ8PSjyBMCjBK7vFJYNmXFCBd/J9TXRUFQkwucuoRTBFMkJfFP6oYZuhm6XXijB7grQ8fa"}
    logger.go:130: 2023-05-22T11:54:36.111+0300 INFO    shards evacuation finished successfully{"shard_ids": ["61fea8kdkosKmQaoCwiSDQ"]}
    evacuate_test.go:126:
                Error Trace:    /repo/frostfs/node/pkg/local_object_storage/engine/evacuate_test.go:126
                Error:          Not equal:
                                expected: 0x0
                                actual  : 0x1
                Test:           TestEvacuateShard
FAIL
``` --- FAIL: TestEvacuateShard (0.26s) logger.go:130: 2023-05-22T11:54:35.856+0300 DEBUG opening... {"shard_id": "PTE71R4Y4wrdasvh8pairb"} logger.go:130: 2023-05-22T11:54:35.871+0300 DEBUG created directory for Metabase {"shard_id": "PTE71R4Y4wrdasvh8pairb", "path": "/run/user/1000/TestEvacuateShard3572005266/001/1.metabase"} logger.go:130: 2023-05-22T11:54:35.864+0300 DEBUG opening... {"shard_id": "VxBm545PypAuhU9Ca7FYqC"} logger.go:130: 2023-05-22T11:54:35.873+0300 DEBUG created directory for Metabase {"shard_id": "VxBm545PypAuhU9Ca7FYqC", "path": "/run/user/1000/TestEvacuateShard3572005266/001/0.metabase"} logger.go:130: 2023-05-22T11:54:35.864+0300 DEBUG opening... {"shard_id": "61fea8kdkosKmQaoCwiSDQ"} logger.go:130: 2023-05-22T11:54:35.875+0300 DEBUG created directory for Metabase {"shard_id": "61fea8kdkosKmQaoCwiSDQ", "path": "/run/user/1000/TestEvacuateShard3572005266/001/2.metabase"} logger.go:130: 2023-05-22T11:54:35.900+0300 DEBUG opened boltDB instance for Metabase {"shard_id": "PTE71R4Y4wrdasvh8pairb"} logger.go:130: 2023-05-22T11:54:35.900+0300 DEBUG checking metabase version {"shard_id": "PTE71R4Y4wrdasvh8pairb"} logger.go:130: 2023-05-22T11:54:35.906+0300 DEBUG opened boltDB instance for Metabase {"shard_id": "61fea8kdkosKmQaoCwiSDQ"} logger.go:130: 2023-05-22T11:54:35.906+0300 DEBUG checking metabase version {"shard_id": "61fea8kdkosKmQaoCwiSDQ"} logger.go:130: 2023-05-22T11:54:35.907+0300 DEBUG opened boltDB instance for Metabase {"shard_id": "VxBm545PypAuhU9Ca7FYqC"} logger.go:130: 2023-05-22T11:54:35.907+0300 DEBUG checking metabase version {"shard_id": "VxBm545PypAuhU9Ca7FYqC"} logger.go:130: 2023-05-22T11:54:35.913+0300 DEBUG initializing... {"shard_id": "VxBm545PypAuhU9Ca7FYqC"} logger.go:130: 2023-05-22T11:54:35.914+0300 DEBUG initializing... {"shard_id": "PTE71R4Y4wrdasvh8pairb"} logger.go:130: 2023-05-22T11:54:35.914+0300 DEBUG initializing... {"shard_id": "61fea8kdkosKmQaoCwiSDQ"} logger.go:130: 2023-05-22T11:54:35.916+0300 INFO local object storage operation {"shard_id": "VxBm545PypAuhU9Ca7FYqC", "address": "28Y1RXGnfhtdZyd4S3MWtmpTKj4dn9EpGVmyJ5rayg85/4krvMcXaR1vka6GYSL2gPPEzu62zFtWRigWjxVTSrovr", "op": "PUT", "type": "fstree", "storage_id": ""} logger.go:130: 2023-05-22T11:54:35.929+0300 INFO local object storage operation {"shard_id": "VxBm545PypAuhU9Ca7FYqC", "address": "28Y1RXGnfhtdZyd4S3MWtmpTKj4dn9EpGVmyJ5rayg85/4krvMcXaR1vka6GYSL2gPPEzu62zFtWRigWjxVTSrovr", "op": "metabase PUT"} logger.go:130: 2023-05-22T11:54:35.929+0300 INFO local object storage operation {"shard_id": "PTE71R4Y4wrdasvh8pairb", "address": "EKWcXeXbr8q7no2fsm5c2PaX9oU6DaFHLet3396qfdGC/6yGQ9gXM9DDkByWJZv26FxV7gSef8cYsUtPfqYrsYCmB", "op": "PUT", "type": "fstree", "storage_id": ""} logger.go:130: 2023-05-22T11:54:35.943+0300 INFO local object storage operation {"shard_id": "PTE71R4Y4wrdasvh8pairb", "address": "EKWcXeXbr8q7no2fsm5c2PaX9oU6DaFHLet3396qfdGC/6yGQ9gXM9DDkByWJZv26FxV7gSef8cYsUtPfqYrsYCmB", "op": "metabase PUT"} logger.go:130: 2023-05-22T11:54:35.944+0300 INFO local object storage operation {"shard_id": "61fea8kdkosKmQaoCwiSDQ", "address": "FVbcYaWh2f5NXaor2dtDhEZrAKhovr8HQNWR7Mm4TUMJ/CBBj3dpaTrDAnJ3x41e2MSZiFkdjsRtK37xJNuZA58ag", "op": "PUT", "type": "fstree", "storage_id": ""} logger.go:130: 2023-05-22T11:54:35.956+0300 INFO local object storage operation {"shard_id": "61fea8kdkosKmQaoCwiSDQ", "address": "FVbcYaWh2f5NXaor2dtDhEZrAKhovr8HQNWR7Mm4TUMJ/CBBj3dpaTrDAnJ3x41e2MSZiFkdjsRtK37xJNuZA58ag", "op": "metabase PUT"} logger.go:130: 2023-05-22T11:54:35.958+0300 INFO local object storage operation {"shard_id": "VxBm545PypAuhU9Ca7FYqC", "address": "BXUnEhBZratySCBTXSnuZ5VFVtxU8RzXkHZSyBr2eUok/jr1RtwsagXM4a6sHCbBhp8zk7xRpFkaJFmSuLkceQ9D", "op": "PUT", "type": "fstree", "storage_id": ""} logger.go:130: 2023-05-22T11:54:35.971+0300 INFO local object storage operation {"shard_id": "VxBm545PypAuhU9Ca7FYqC", "address": "BXUnEhBZratySCBTXSnuZ5VFVtxU8RzXkHZSyBr2eUok/jr1RtwsagXM4a6sHCbBhp8zk7xRpFkaJFmSuLkceQ9D", "op": "metabase PUT"} logger.go:130: 2023-05-22T11:54:35.972+0300 INFO local object storage operation {"shard_id": "61fea8kdkosKmQaoCwiSDQ", "address": "BYjTCAWsZqoKGv1gr2F4p6WM4KAuPFJkMwrPPxTtPZXu/EnyYYEsD66dj1BqHXaa9diVaSAoprYJac9rHDnsN39Dh", "op": "PUT", "type": "fstree", "storage_id": ""} logger.go:130: 2023-05-22T11:54:35.995+0300 INFO local object storage operation {"shard_id": "61fea8kdkosKmQaoCwiSDQ", "address": "BYjTCAWsZqoKGv1gr2F4p6WM4KAuPFJkMwrPPxTtPZXu/EnyYYEsD66dj1BqHXaa9diVaSAoprYJac9rHDnsN39Dh", "op": "metabase PUT"} logger.go:130: 2023-05-22T11:54:35.997+0300 INFO local object storage operation {"shard_id": "PTE71R4Y4wrdasvh8pairb", "address": "GyAGZLkmEF5xnWACmVePVh4vkpffAS3BZXWUrAp1vBRH/AjNezC38g9TfHURjKwdxhcWi1WfPwD4cq2BPgv8RFh6G", "op": "PUT", "type": "fstree", "storage_id": ""} logger.go:130: 2023-05-22T11:54:36.009+0300 INFO local object storage operation {"shard_id": "PTE71R4Y4wrdasvh8pairb", "address": "GyAGZLkmEF5xnWACmVePVh4vkpffAS3BZXWUrAp1vBRH/AjNezC38g9TfHURjKwdxhcWi1WfPwD4cq2BPgv8RFh6G", "op": "metabase PUT"} logger.go:130: 2023-05-22T11:54:36.010+0300 INFO local object storage operation {"shard_id": "PTE71R4Y4wrdasvh8pairb", "address": "22c8TETMng4CzWuyGGadUwhhFnNR6wJRn66JnD5sJY9i/RLXD3Lz4Rnu1dqQ5tf2dVuaaYRZE5CYuKQn4z9mN7Q5", "op": "PUT", "type": "fstree", "storage_id": ""} logger.go:130: 2023-05-22T11:54:36.021+0300 INFO local object storage operation {"shard_id": "PTE71R4Y4wrdasvh8pairb", "address": "22c8TETMng4CzWuyGGadUwhhFnNR6wJRn66JnD5sJY9i/RLXD3Lz4Rnu1dqQ5tf2dVuaaYRZE5CYuKQn4z9mN7Q5", "op": "metabase PUT"} logger.go:130: 2023-05-22T11:54:36.023+0300 INFO local object storage operation {"shard_id": "61fea8kdkosKmQaoCwiSDQ", "address": "E3kNgc25wGz9RNsNpp8sZ8PSjyBMCjBK7vFJYNmXFCBd/J9TXRUFQkwucuoRTBFMkJfFP6oYZuhm6XXijB7grQ8fa", "op": "PUT", "type": "fstree", "storage_id": ""} logger.go:130: 2023-05-22T11:54:36.034+0300 INFO local object storage operation {"shard_id": "61fea8kdkosKmQaoCwiSDQ", "address": "E3kNgc25wGz9RNsNpp8sZ8PSjyBMCjBK7vFJYNmXFCBd/J9TXRUFQkwucuoRTBFMkJfFP6oYZuhm6XXijB7grQ8fa", "op": "metabase PUT"} logger.go:130: 2023-05-22T11:54:36.039+0300 INFO setting shard mode {"shard_id": "61fea8kdkosKmQaoCwiSDQ", "old_mode": "READ_WRITE", "new_mode": "READ_ONLY"} logger.go:130: 2023-05-22T11:54:36.039+0300 DEBUG closing... {"shard_id": "61fea8kdkosKmQaoCwiSDQ"} logger.go:130: 2023-05-22T11:54:36.039+0300 DEBUG opening... {"shard_id": "61fea8kdkosKmQaoCwiSDQ"} logger.go:130: 2023-05-22T11:54:36.039+0300 DEBUG initializing... {"shard_id": "61fea8kdkosKmQaoCwiSDQ"} logger.go:130: 2023-05-22T11:54:36.039+0300 DEBUG created directory for Metabase {"shard_id": "61fea8kdkosKmQaoCwiSDQ", "path": "/run/user/1000/TestEvacuateShard3572005266/001/2.metabase"} logger.go:130: 2023-05-22T11:54:36.039+0300 DEBUG opened boltDB instance for Metabase {"shard_id": "61fea8kdkosKmQaoCwiSDQ"} logger.go:130: 2023-05-22T11:54:36.039+0300 DEBUG checking metabase version {"shard_id": "61fea8kdkosKmQaoCwiSDQ"} logger.go:130: 2023-05-22T11:54:36.039+0300 INFO shard mode set successfully {"shard_id": "61fea8kdkosKmQaoCwiSDQ", "mode": "READ_ONLY"} logger.go:130: 2023-05-22T11:54:36.039+0300 INFO started shards evacuation {"shard_ids": ["61fea8kdkosKmQaoCwiSDQ"]} logger.go:130: 2023-05-22T11:54:36.048+0300 INFO local object storage operation {"shard_id": "PTE71R4Y4wrdasvh8pairb", "address": "BYjTCAWsZqoKGv1gr2F4p6WM4KAuPFJkMwrPPxTtPZXu/EnyYYEsD66dj1BqHXaa9diVaSAoprYJac9rHDnsN39Dh", "op": "PUT", "type": "fstree", "storage_id": ""} logger.go:130: 2023-05-22T11:54:36.064+0300 INFO local object storage operation {"shard_id": "PTE71R4Y4wrdasvh8pairb", "address": "BYjTCAWsZqoKGv1gr2F4p6WM4KAuPFJkMwrPPxTtPZXu/EnyYYEsD66dj1BqHXaa9diVaSAoprYJac9rHDnsN39Dh", "op": "metabase PUT"} logger.go:130: 2023-05-22T11:54:36.064+0300 DEBUG object is moved to another shard {"from": "61fea8kdkosKmQaoCwiSDQ", "to": "PTE71R4Y4wrdasvh8pairb", "addr": "BYjTCAWsZqoKGv1gr2F4p6WM4KAuPFJkMwrPPxTtPZXu/EnyYYEsD66dj1BqHXaa9diVaSAoprYJac9rHDnsN39Dh"} logger.go:130: 2023-05-22T11:54:36.065+0300 INFO local object storage operation {"shard_id": "PTE71R4Y4wrdasvh8pairb", "address": "E3kNgc25wGz9RNsNpp8sZ8PSjyBMCjBK7vFJYNmXFCBd/J9TXRUFQkwucuoRTBFMkJfFP6oYZuhm6XXijB7grQ8fa", "op": "PUT", "type": "fstree", "storage_id": ""} logger.go:130: 2023-05-22T11:54:36.077+0300 INFO local object storage operation {"shard_id": "PTE71R4Y4wrdasvh8pairb", "address": "E3kNgc25wGz9RNsNpp8sZ8PSjyBMCjBK7vFJYNmXFCBd/J9TXRUFQkwucuoRTBFMkJfFP6oYZuhm6XXijB7grQ8fa", "op": "metabase PUT"} logger.go:130: 2023-05-22T11:54:36.077+0300 DEBUG object is moved to another shard {"from": "61fea8kdkosKmQaoCwiSDQ", "to": "PTE71R4Y4wrdasvh8pairb", "addr": "E3kNgc25wGz9RNsNpp8sZ8PSjyBMCjBK7vFJYNmXFCBd/J9TXRUFQkwucuoRTBFMkJfFP6oYZuhm6XXijB7grQ8fa"} logger.go:130: 2023-05-22T11:54:36.078+0300 INFO local object storage operation {"shard_id": "PTE71R4Y4wrdasvh8pairb", "address": "FVbcYaWh2f5NXaor2dtDhEZrAKhovr8HQNWR7Mm4TUMJ/CBBj3dpaTrDAnJ3x41e2MSZiFkdjsRtK37xJNuZA58ag", "op": "PUT", "type": "fstree", "storage_id": ""} logger.go:130: 2023-05-22T11:54:36.089+0300 INFO local object storage operation {"shard_id": "PTE71R4Y4wrdasvh8pairb", "address": "FVbcYaWh2f5NXaor2dtDhEZrAKhovr8HQNWR7Mm4TUMJ/CBBj3dpaTrDAnJ3x41e2MSZiFkdjsRtK37xJNuZA58ag", "op": "metabase PUT"} logger.go:130: 2023-05-22T11:54:36.089+0300 DEBUG object is moved to another shard {"from": "61fea8kdkosKmQaoCwiSDQ", "to": "PTE71R4Y4wrdasvh8pairb", "addr": "FVbcYaWh2f5NXaor2dtDhEZrAKhovr8HQNWR7Mm4TUMJ/CBBj3dpaTrDAnJ3x41e2MSZiFkdjsRtK37xJNuZA58ag"} logger.go:130: 2023-05-22T11:54:36.090+0300 INFO shards evacuation finished successfully{"shard_ids": ["61fea8kdkosKmQaoCwiSDQ"]} logger.go:130: 2023-05-22T11:54:36.096+0300 INFO started shards evacuation {"shard_ids": ["61fea8kdkosKmQaoCwiSDQ"]} logger.go:130: 2023-05-22T11:54:36.098+0300 INFO local object storage operation {"shard_id": "VxBm545PypAuhU9Ca7FYqC", "address": "E3kNgc25wGz9RNsNpp8sZ8PSjyBMCjBK7vFJYNmXFCBd/J9TXRUFQkwucuoRTBFMkJfFP6oYZuhm6XXijB7grQ8fa", "op": "PUT", "type": "fstree", "storage_id": ""} logger.go:130: 2023-05-22T11:54:36.109+0300 INFO local object storage operation {"shard_id": "VxBm545PypAuhU9Ca7FYqC", "address": "E3kNgc25wGz9RNsNpp8sZ8PSjyBMCjBK7vFJYNmXFCBd/J9TXRUFQkwucuoRTBFMkJfFP6oYZuhm6XXijB7grQ8fa", "op": "metabase PUT"} logger.go:130: 2023-05-22T11:54:36.109+0300 DEBUG object is moved to another shard {"from": "61fea8kdkosKmQaoCwiSDQ", "to": "VxBm545PypAuhU9Ca7FYqC", "addr": "E3kNgc25wGz9RNsNpp8sZ8PSjyBMCjBK7vFJYNmXFCBd/J9TXRUFQkwucuoRTBFMkJfFP6oYZuhm6XXijB7grQ8fa"} logger.go:130: 2023-05-22T11:54:36.111+0300 INFO shards evacuation finished successfully{"shard_ids": ["61fea8kdkosKmQaoCwiSDQ"]} evacuate_test.go:126: Error Trace: /repo/frostfs/node/pkg/local_object_storage/engine/evacuate_test.go:126 Error: Not equal: expected: 0x0 actual : 0x1 Test: TestEvacuateShard FAIL ```
fyrchik added the
bug
triage
labels 2023-05-22 08:57:20 +00:00
fyrchik added the
frostfs-node
label 2023-05-22 09:27:07 +00:00
fyrchik added this to the v0.37.0 milestone 2023-05-22 09:27:26 +00:00
dstepanov-yadro self-assigned this 2023-05-25 15:06:54 +00:00

@fyrchik on which version were there problems? Do you know commit id?

@fyrchik on which version were there problems? Do you know commit id?

I think we can close it. Can't reproduce. Test is valid.

I think we can close it. Can't reproduce. Test is valid.
fyrchik reopened this issue 2023-07-11 07:31:55 +00:00
Author
Owner

Reproduced on CI here #495

Reproduced on CI here https://git.frostfs.info/TrueCloudLab/frostfs-node/pulls/495

New logs:

94
?   	git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/storagetest	[no test files]
95
--- FAIL: TestEvacuateShard (0.32s)
96
    logger.go:130: 2023-07-10T13:09:14.391Z	DEBUG	opening...	{"shard_id": "8xc2a7SH8M4XX3byZeZmzJ"}
97
    logger.go:130: 2023-07-10T13:09:14.392Z	DEBUG	created directory for Metabase	{"shard_id": "8xc2a7SH8M4XX3byZeZmzJ", "path": "/tmp/TestEvacuateShard3284936385/001/2.metabase"}
98
    logger.go:130: 2023-07-10T13:09:14.392Z	DEBUG	opened boltDB instance for Metabase	{"shard_id": "8xc2a7SH8M4XX3byZeZmzJ"}
99
    logger.go:130: 2023-07-10T13:09:14.392Z	DEBUG	checking metabase version	{"shard_id": "8xc2a7SH8M4XX3byZeZmzJ"}
100
    logger.go:130: 2023-07-10T13:09:14.392Z	DEBUG	opening...	{"shard_id": "CfB4de9Puv2cUVtyuft8JZ"}
101
    logger.go:130: 2023-07-10T13:09:14.392Z	DEBUG	created directory for Metabase	{"shard_id": "CfB4de9Puv2cUVtyuft8JZ", "path": "/tmp/TestEvacuateShard3284936385/001/1.metabase"}
102
    logger.go:130: 2023-07-10T13:09:14.392Z	DEBUG	opened boltDB instance for Metabase	{"shard_id": "CfB4de9Puv2cUVtyuft8JZ"}
103
    logger.go:130: 2023-07-10T13:09:14.392Z	DEBUG	checking metabase version	{"shard_id": "CfB4de9Puv2cUVtyuft8JZ"}
104
    logger.go:130: 2023-07-10T13:09:14.392Z	DEBUG	opening...	{"shard_id": "6tRaTRHDUWw7Ngo924ZBNR"}
105
    logger.go:130: 2023-07-10T13:09:14.392Z	DEBUG	created directory for Metabase	{"shard_id": "6tRaTRHDUWw7Ngo924ZBNR", "path": "/tmp/TestEvacuateShard3284936385/001/0.metabase"}
106
    logger.go:130: 2023-07-10T13:09:14.392Z	DEBUG	opened boltDB instance for Metabase	{"shard_id": "6tRaTRHDUWw7Ngo924ZBNR"}
107
    logger.go:130: 2023-07-10T13:09:14.392Z	DEBUG	checking metabase version	{"shard_id": "6tRaTRHDUWw7Ngo924ZBNR"}
108
    logger.go:130: 2023-07-10T13:09:14.393Z	DEBUG	initializing...	{"shard_id": "CfB4de9Puv2cUVtyuft8JZ"}
109
    logger.go:130: 2023-07-10T13:09:14.393Z	DEBUG	initializing...	{"shard_id": "6tRaTRHDUWw7Ngo924ZBNR"}
110
    logger.go:130: 2023-07-10T13:09:14.394Z	DEBUG	initializing...	{"shard_id": "8xc2a7SH8M4XX3byZeZmzJ"}
111
    logger.go:130: 2023-07-10T13:09:14.396Z	INFO	local object storage operation	{"shard_id": "6tRaTRHDUWw7Ngo924ZBNR", "address": "GAemMRbozspJWvrYGugq3r4SWzdF7ee1N7dMvDZPYp3w/AZrnTvBg961G7W5GKsL8CoawGh5M7YCbDsw612MdUXjT", "op": "PUT", "type": "fstree", "storage_id": ""}
112
    logger.go:130: 2023-07-10T13:09:14.409Z	INFO	local object storage operation	{"shard_id": "6tRaTRHDUWw7Ngo924ZBNR", "address": "GAemMRbozspJWvrYGugq3r4SWzdF7ee1N7dMvDZPYp3w/AZrnTvBg961G7W5GKsL8CoawGh5M7YCbDsw612MdUXjT", "op": "metabase PUT"}
113
    logger.go:130: 2023-07-10T13:09:14.410Z	INFO	local object storage operation	{"shard_id": "CfB4de9Puv2cUVtyuft8JZ", "address": "Bn1Fv1sUin97Mn4uyq4Mg8wKXQtF3KArxK9PN5eZAYKp/8RuCMYuScswcMu3HTSYNELXnXyWuLamKbcBmRmTm8QyW", "op": "PUT", "type": "fstree", "storage_id": ""}
114
    logger.go:130: 2023-07-10T13:09:14.423Z	INFO	local object storage operation	{"shard_id": "CfB4de9Puv2cUVtyuft8JZ", "address": "Bn1Fv1sUin97Mn4uyq4Mg8wKXQtF3KArxK9PN5eZAYKp/8RuCMYuScswcMu3HTSYNELXnXyWuLamKbcBmRmTm8QyW", "op": "metabase PUT"}
115
    logger.go:130: 2023-07-10T13:09:14.424Z	INFO	local object storage operation	{"shard_id": "8xc2a7SH8M4XX3byZeZmzJ", "address": "4RMiBySomWHCXkDtdairHtktfzy72byfoFVJvUdcGWN8/GhGSF4FhkpaK8z3A5bYJnHB3FXjgvw3Z5uLoxkADq9qh", "op": "PUT", "type": "fstree", "storage_id": ""}
116
    logger.go:130: 2023-07-10T13:09:14.437Z	INFO	local object storage operation	{"shard_id": "8xc2a7SH8M4XX3byZeZmzJ", "address": "4RMiBySomWHCXkDtdairHtktfzy72byfoFVJvUdcGWN8/GhGSF4FhkpaK8z3A5bYJnHB3FXjgvw3Z5uLoxkADq9qh", "op": "metabase PUT"}
117
    logger.go:130: 2023-07-10T13:09:14.440Z	INFO	local object storage operation	{"shard_id": "CfB4de9Puv2cUVtyuft8JZ", "address": "33rHfBr9Kp3B95ZJnLgAsZCqJX7aQwYRTpf4KxFeCBjV/Aub8Ut9YeMzEwjE4cschsXCRcQBBJb2xWW7Fz1j6hPGc", "op": "PUT", "type": "fstree", "storage_id": ""}
118
    logger.go:130: 2023-07-10T13:09:14.452Z	INFO	local object storage operation	{"shard_id": "CfB4de9Puv2cUVtyuft8JZ", "address": "33rHfBr9Kp3B95ZJnLgAsZCqJX7aQwYRTpf4KxFeCBjV/Aub8Ut9YeMzEwjE4cschsXCRcQBBJb2xWW7Fz1j6hPGc", "op": "metabase PUT"}
119
    logger.go:130: 2023-07-10T13:09:14.455Z	INFO	local object storage operation	{"shard_id": "6tRaTRHDUWw7Ngo924ZBNR", "address": "FcqzmjgmxEpdiQ8jTZh1dBmGPNKeM5KM315d78qQNwzn/AMw5ccTWX6ahsnJytTMmbDiUeK38L8LgF3w8AsjWf23b", "op": "PUT", "type": "fstree", "storage_id": ""}
120
    logger.go:130: 2023-07-10T13:09:14.468Z	INFO	local object storage operation	{"shard_id": "6tRaTRHDUWw7Ngo924ZBNR", "address": "FcqzmjgmxEpdiQ8jTZh1dBmGPNKeM5KM315d78qQNwzn/AMw5ccTWX6ahsnJytTMmbDiUeK38L8LgF3w8AsjWf23b", "op": "metabase PUT"}
121
    logger.go:130: 2023-07-10T13:09:14.470Z	INFO	local object storage operation	{"shard_id": "6tRaTRHDUWw7Ngo924ZBNR", "address": "9zbjpF9CmjCJryA7PzWSUDmJVhvTof6TwMmVitUu5G9M/3icHinczc9VrVJuff5cu4NKA4VvkwPi27YAi4dySe526", "op": "PUT", "type": "fstree", "storage_id": ""}
122
    logger.go:130: 2023-07-10T13:09:14.483Z	INFO	local object storage operation	{"shard_id": "6tRaTRHDUWw7Ngo924ZBNR", "address": "9zbjpF9CmjCJryA7PzWSUDmJVhvTof6TwMmVitUu5G9M/3icHinczc9VrVJuff5cu4NKA4VvkwPi27YAi4dySe526", "op": "metabase PUT"}
123
    logger.go:130: 2023-07-10T13:09:14.518Z	INFO	local object storage operation	{"shard_id": "8xc2a7SH8M4XX3byZeZmzJ", "address": "9BaNbX3v9nB9eb2J43gTa7JZRkQqKzR5VwxaBVq5JLdh/DCq4fk4E8wFKnnVXwBecMDiSVGQ5tgx926BRdCPNNWBM", "op": "PUT", "type": "fstree", "storage_id": ""}
124
    logger.go:130: 2023-07-10T13:09:14.531Z	INFO	local object storage operation	{"shard_id": "8xc2a7SH8M4XX3byZeZmzJ", "address": "9BaNbX3v9nB9eb2J43gTa7JZRkQqKzR5VwxaBVq5JLdh/DCq4fk4E8wFKnnVXwBecMDiSVGQ5tgx926BRdCPNNWBM", "op": "metabase PUT"}
125
    logger.go:130: 2023-07-10T13:09:14.542Z	INFO	local object storage operation	{"shard_id": "CfB4de9Puv2cUVtyuft8JZ", "address": "8VkYw1t941XUkULBSZHbe6QttRPXVnqd7XuR6H3Ae7ob/DJrAjB6WPMBRvf8DDhW3F4sci7QYvVi8fpt6kdpA3mYa", "op": "PUT", "type": "fstree", "storage_id": ""}
126
    logger.go:130: 2023-07-10T13:09:14.555Z	INFO	local object storage operation	{"shard_id": "CfB4de9Puv2cUVtyuft8JZ", "address": "8VkYw1t941XUkULBSZHbe6QttRPXVnqd7XuR6H3Ae7ob/DJrAjB6WPMBRvf8DDhW3F4sci7QYvVi8fpt6kdpA3mYa", "op": "metabase PUT"}
127
    logger.go:130: 2023-07-10T13:09:14.590Z	INFO	local object storage operation	{"shard_id": "8xc2a7SH8M4XX3byZeZmzJ", "address": "HvpDfjA7tNMZtQZ4wPfZeMbZvnFH4uP49AdDmbgxHsVK/AJs3EXYUVZmuBaAU2UMGCQWHhgTfjdkFcVNk4G3YPr3z", "op": "PUT", "type": "fstree", "storage_id": ""}
128
    logger.go:130: 2023-07-10T13:09:14.605Z	INFO	local object storage operation	{"shard_id": "8xc2a7SH8M4XX3byZeZmzJ", "address": "HvpDfjA7tNMZtQZ4wPfZeMbZvnFH4uP49AdDmbgxHsVK/AJs3EXYUVZmuBaAU2UMGCQWHhgTfjdkFcVNk4G3YPr3z", "op": "metabase PUT"}
129
    logger.go:130: 2023-07-10T13:09:14.613Z	INFO	setting shard mode	{"shard_id": "8xc2a7SH8M4XX3byZeZmzJ", "old_mode": "READ_WRITE", "new_mode": "READ_ONLY"}
130
    logger.go:130: 2023-07-10T13:09:14.613Z	DEBUG	closing...	{"shard_id": "8xc2a7SH8M4XX3byZeZmzJ"}
131
    logger.go:130: 2023-07-10T13:09:14.613Z	DEBUG	opening...	{"shard_id": "8xc2a7SH8M4XX3byZeZmzJ"}
132
    logger.go:130: 2023-07-10T13:09:14.613Z	DEBUG	initializing...	{"shard_id": "8xc2a7SH8M4XX3byZeZmzJ"}
133
    logger.go:130: 2023-07-10T13:09:14.613Z	DEBUG	created directory for Metabase	{"shard_id": "8xc2a7SH8M4XX3byZeZmzJ", "path": "/tmp/TestEvacuateShard3284936385/001/2.metabase"}
134
    logger.go:130: 2023-07-10T13:09:14.613Z	DEBUG	opened boltDB instance for Metabase	{"shard_id": "8xc2a7SH8M4XX3byZeZmzJ"}
135
    logger.go:130: 2023-07-10T13:09:14.613Z	DEBUG	checking metabase version	{"shard_id": "8xc2a7SH8M4XX3byZeZmzJ"}
136
    logger.go:130: 2023-07-10T13:09:14.613Z	INFO	shard mode set successfully	{"shard_id": "8xc2a7SH8M4XX3byZeZmzJ", "mode": "READ_ONLY"}
137
    logger.go:130: 2023-07-10T13:09:14.613Z	INFO	started shards evacuation	{"shard_ids": ["8xc2a7SH8M4XX3byZeZmzJ"], "operation": "evacuation"}
138
    logger.go:130: 2023-07-10T13:09:14.616Z	INFO	local object storage operation	{"shard_id": "6tRaTRHDUWw7Ngo924ZBNR", "address": "4RMiBySomWHCXkDtdairHtktfzy72byfoFVJvUdcGWN8/GhGSF4FhkpaK8z3A5bYJnHB3FXjgvw3Z5uLoxkADq9qh", "op": "PUT", "type": "fstree", "storage_id": ""}
139
    logger.go:130: 2023-07-10T13:09:14.631Z	INFO	local object storage operation	{"shard_id": "6tRaTRHDUWw7Ngo924ZBNR", "address": "4RMiBySomWHCXkDtdairHtktfzy72byfoFVJvUdcGWN8/GhGSF4FhkpaK8z3A5bYJnHB3FXjgvw3Z5uLoxkADq9qh", "op": "metabase PUT"}
140
    logger.go:130: 2023-07-10T13:09:14.631Z	DEBUG	object is moved to another shard	{"from": "8xc2a7SH8M4XX3byZeZmzJ", "to": "6tRaTRHDUWw7Ngo924ZBNR", "addr": "4RMiBySomWHCXkDtdairHtktfzy72byfoFVJvUdcGWN8/GhGSF4FhkpaK8z3A5bYJnHB3FXjgvw3Z5uLoxkADq9qh", "operation": "evacuation"}
141
    logger.go:130: 2023-07-10T13:09:14.633Z	INFO	local object storage operation	{"shard_id": "6tRaTRHDUWw7Ngo924ZBNR", "address": "9BaNbX3v9nB9eb2J43gTa7JZRkQqKzR5VwxaBVq5JLdh/DCq4fk4E8wFKnnVXwBecMDiSVGQ5tgx926BRdCPNNWBM", "op": "PUT", "type": "fstree", "storage_id": ""}
142
    logger.go:130: 2023-07-10T13:09:14.654Z	INFO	local object storage operation	{"shard_id": "6tRaTRHDUWw7Ngo924ZBNR", "address": "9BaNbX3v9nB9eb2J43gTa7JZRkQqKzR5VwxaBVq5JLdh/DCq4fk4E8wFKnnVXwBecMDiSVGQ5tgx926BRdCPNNWBM", "op": "metabase PUT"}
143
    logger.go:130: 2023-07-10T13:09:14.654Z	DEBUG	object is moved to another shard	{"from": "8xc2a7SH8M4XX3byZeZmzJ", "to": "6tRaTRHDUWw7Ngo924ZBNR", "addr": "9BaNbX3v9nB9eb2J43gTa7JZRkQqKzR5VwxaBVq5JLdh/DCq4fk4E8wFKnnVXwBecMDiSVGQ5tgx926BRdCPNNWBM", "operation": "evacuation"}
144
    logger.go:130: 2023-07-10T13:09:14.656Z	INFO	local object storage operation	{"shard_id": "6tRaTRHDUWw7Ngo924ZBNR", "address": "HvpDfjA7tNMZtQZ4wPfZeMbZvnFH4uP49AdDmbgxHsVK/AJs3EXYUVZmuBaAU2UMGCQWHhgTfjdkFcVNk4G3YPr3z", "op": "PUT", "type": "fstree", "storage_id": ""}
145
    logger.go:130: 2023-07-10T13:09:14.669Z	INFO	local object storage operation	{"shard_id": "6tRaTRHDUWw7Ngo924ZBNR", "address": "HvpDfjA7tNMZtQZ4wPfZeMbZvnFH4uP49AdDmbgxHsVK/AJs3EXYUVZmuBaAU2UMGCQWHhgTfjdkFcVNk4G3YPr3z", "op": "metabase PUT"}
146
    logger.go:130: 2023-07-10T13:09:14.669Z	DEBUG	object is moved to another shard	{"from": "8xc2a7SH8M4XX3byZeZmzJ", "to": "6tRaTRHDUWw7Ngo924ZBNR", "addr": "HvpDfjA7tNMZtQZ4wPfZeMbZvnFH4uP49AdDmbgxHsVK/AJs3EXYUVZmuBaAU2UMGCQWHhgTfjdkFcVNk4G3YPr3z", "operation": "evacuation"}
147
    logger.go:130: 2023-07-10T13:09:14.669Z	INFO	shards evacuation finished successfully	{"shard_ids": ["8xc2a7SH8M4XX3byZeZmzJ"], "operation": "evacuation", "total": 3, "evacuated": 3, "failed": 0}
148
    logger.go:130: 2023-07-10T13:09:14.677Z	INFO	started shards evacuation	{"shard_ids": ["8xc2a7SH8M4XX3byZeZmzJ"], "operation": "evacuation"}
149
    logger.go:130: 2023-07-10T13:09:14.683Z	INFO	local object storage operation	{"shard_id": "CfB4de9Puv2cUVtyuft8JZ", "address": "9BaNbX3v9nB9eb2J43gTa7JZRkQqKzR5VwxaBVq5JLdh/DCq4fk4E8wFKnnVXwBecMDiSVGQ5tgx926BRdCPNNWBM", "op": "PUT", "type": "fstree", "storage_id": ""}
150
    logger.go:130: 2023-07-10T13:09:14.696Z	INFO	local object storage operation	{"shard_id": "CfB4de9Puv2cUVtyuft8JZ", "address": "9BaNbX3v9nB9eb2J43gTa7JZRkQqKzR5VwxaBVq5JLdh/DCq4fk4E8wFKnnVXwBecMDiSVGQ5tgx926BRdCPNNWBM", "op": "metabase PUT"}
151
    logger.go:130: 2023-07-10T13:09:14.696Z	DEBUG	object is moved to another shard	{"from": "8xc2a7SH8M4XX3byZeZmzJ", "to": "CfB4de9Puv2cUVtyuft8JZ", "addr": "9BaNbX3v9nB9eb2J43gTa7JZRkQqKzR5VwxaBVq5JLdh/DCq4fk4E8wFKnnVXwBecMDiSVGQ5tgx926BRdCPNNWBM", "operation": "evacuation"}
152
    logger.go:130: 2023-07-10T13:09:14.701Z	INFO	shards evacuation finished successfully	{"shard_ids": ["8xc2a7SH8M4XX3byZeZmzJ"], "operation": "evacuation", "total": 3, "evacuated": 1, "failed": 0}
153
    evacuate_test.go:126: 
154
        	Error Trace:	/workspace/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine/evacuate_test.go:126
155
        	Error:      	Not equal: 
156
        	            	expected: 0x0
157
        	            	actual  : 0x1
158
        	Test:       	TestEvacuateShard
159
FAIL
New logs: ``` 94 ? git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/storagetest [no test files] 95 --- FAIL: TestEvacuateShard (0.32s) 96 logger.go:130: 2023-07-10T13:09:14.391Z DEBUG opening... {"shard_id": "8xc2a7SH8M4XX3byZeZmzJ"} 97 logger.go:130: 2023-07-10T13:09:14.392Z DEBUG created directory for Metabase {"shard_id": "8xc2a7SH8M4XX3byZeZmzJ", "path": "/tmp/TestEvacuateShard3284936385/001/2.metabase"} 98 logger.go:130: 2023-07-10T13:09:14.392Z DEBUG opened boltDB instance for Metabase {"shard_id": "8xc2a7SH8M4XX3byZeZmzJ"} 99 logger.go:130: 2023-07-10T13:09:14.392Z DEBUG checking metabase version {"shard_id": "8xc2a7SH8M4XX3byZeZmzJ"} 100 logger.go:130: 2023-07-10T13:09:14.392Z DEBUG opening... {"shard_id": "CfB4de9Puv2cUVtyuft8JZ"} 101 logger.go:130: 2023-07-10T13:09:14.392Z DEBUG created directory for Metabase {"shard_id": "CfB4de9Puv2cUVtyuft8JZ", "path": "/tmp/TestEvacuateShard3284936385/001/1.metabase"} 102 logger.go:130: 2023-07-10T13:09:14.392Z DEBUG opened boltDB instance for Metabase {"shard_id": "CfB4de9Puv2cUVtyuft8JZ"} 103 logger.go:130: 2023-07-10T13:09:14.392Z DEBUG checking metabase version {"shard_id": "CfB4de9Puv2cUVtyuft8JZ"} 104 logger.go:130: 2023-07-10T13:09:14.392Z DEBUG opening... {"shard_id": "6tRaTRHDUWw7Ngo924ZBNR"} 105 logger.go:130: 2023-07-10T13:09:14.392Z DEBUG created directory for Metabase {"shard_id": "6tRaTRHDUWw7Ngo924ZBNR", "path": "/tmp/TestEvacuateShard3284936385/001/0.metabase"} 106 logger.go:130: 2023-07-10T13:09:14.392Z DEBUG opened boltDB instance for Metabase {"shard_id": "6tRaTRHDUWw7Ngo924ZBNR"} 107 logger.go:130: 2023-07-10T13:09:14.392Z DEBUG checking metabase version {"shard_id": "6tRaTRHDUWw7Ngo924ZBNR"} 108 logger.go:130: 2023-07-10T13:09:14.393Z DEBUG initializing... {"shard_id": "CfB4de9Puv2cUVtyuft8JZ"} 109 logger.go:130: 2023-07-10T13:09:14.393Z DEBUG initializing... {"shard_id": "6tRaTRHDUWw7Ngo924ZBNR"} 110 logger.go:130: 2023-07-10T13:09:14.394Z DEBUG initializing... {"shard_id": "8xc2a7SH8M4XX3byZeZmzJ"} 111 logger.go:130: 2023-07-10T13:09:14.396Z INFO local object storage operation {"shard_id": "6tRaTRHDUWw7Ngo924ZBNR", "address": "GAemMRbozspJWvrYGugq3r4SWzdF7ee1N7dMvDZPYp3w/AZrnTvBg961G7W5GKsL8CoawGh5M7YCbDsw612MdUXjT", "op": "PUT", "type": "fstree", "storage_id": ""} 112 logger.go:130: 2023-07-10T13:09:14.409Z INFO local object storage operation {"shard_id": "6tRaTRHDUWw7Ngo924ZBNR", "address": "GAemMRbozspJWvrYGugq3r4SWzdF7ee1N7dMvDZPYp3w/AZrnTvBg961G7W5GKsL8CoawGh5M7YCbDsw612MdUXjT", "op": "metabase PUT"} 113 logger.go:130: 2023-07-10T13:09:14.410Z INFO local object storage operation {"shard_id": "CfB4de9Puv2cUVtyuft8JZ", "address": "Bn1Fv1sUin97Mn4uyq4Mg8wKXQtF3KArxK9PN5eZAYKp/8RuCMYuScswcMu3HTSYNELXnXyWuLamKbcBmRmTm8QyW", "op": "PUT", "type": "fstree", "storage_id": ""} 114 logger.go:130: 2023-07-10T13:09:14.423Z INFO local object storage operation {"shard_id": "CfB4de9Puv2cUVtyuft8JZ", "address": "Bn1Fv1sUin97Mn4uyq4Mg8wKXQtF3KArxK9PN5eZAYKp/8RuCMYuScswcMu3HTSYNELXnXyWuLamKbcBmRmTm8QyW", "op": "metabase PUT"} 115 logger.go:130: 2023-07-10T13:09:14.424Z INFO local object storage operation {"shard_id": "8xc2a7SH8M4XX3byZeZmzJ", "address": "4RMiBySomWHCXkDtdairHtktfzy72byfoFVJvUdcGWN8/GhGSF4FhkpaK8z3A5bYJnHB3FXjgvw3Z5uLoxkADq9qh", "op": "PUT", "type": "fstree", "storage_id": ""} 116 logger.go:130: 2023-07-10T13:09:14.437Z INFO local object storage operation {"shard_id": "8xc2a7SH8M4XX3byZeZmzJ", "address": "4RMiBySomWHCXkDtdairHtktfzy72byfoFVJvUdcGWN8/GhGSF4FhkpaK8z3A5bYJnHB3FXjgvw3Z5uLoxkADq9qh", "op": "metabase PUT"} 117 logger.go:130: 2023-07-10T13:09:14.440Z INFO local object storage operation {"shard_id": "CfB4de9Puv2cUVtyuft8JZ", "address": "33rHfBr9Kp3B95ZJnLgAsZCqJX7aQwYRTpf4KxFeCBjV/Aub8Ut9YeMzEwjE4cschsXCRcQBBJb2xWW7Fz1j6hPGc", "op": "PUT", "type": "fstree", "storage_id": ""} 118 logger.go:130: 2023-07-10T13:09:14.452Z INFO local object storage operation {"shard_id": "CfB4de9Puv2cUVtyuft8JZ", "address": "33rHfBr9Kp3B95ZJnLgAsZCqJX7aQwYRTpf4KxFeCBjV/Aub8Ut9YeMzEwjE4cschsXCRcQBBJb2xWW7Fz1j6hPGc", "op": "metabase PUT"} 119 logger.go:130: 2023-07-10T13:09:14.455Z INFO local object storage operation {"shard_id": "6tRaTRHDUWw7Ngo924ZBNR", "address": "FcqzmjgmxEpdiQ8jTZh1dBmGPNKeM5KM315d78qQNwzn/AMw5ccTWX6ahsnJytTMmbDiUeK38L8LgF3w8AsjWf23b", "op": "PUT", "type": "fstree", "storage_id": ""} 120 logger.go:130: 2023-07-10T13:09:14.468Z INFO local object storage operation {"shard_id": "6tRaTRHDUWw7Ngo924ZBNR", "address": "FcqzmjgmxEpdiQ8jTZh1dBmGPNKeM5KM315d78qQNwzn/AMw5ccTWX6ahsnJytTMmbDiUeK38L8LgF3w8AsjWf23b", "op": "metabase PUT"} 121 logger.go:130: 2023-07-10T13:09:14.470Z INFO local object storage operation {"shard_id": "6tRaTRHDUWw7Ngo924ZBNR", "address": "9zbjpF9CmjCJryA7PzWSUDmJVhvTof6TwMmVitUu5G9M/3icHinczc9VrVJuff5cu4NKA4VvkwPi27YAi4dySe526", "op": "PUT", "type": "fstree", "storage_id": ""} 122 logger.go:130: 2023-07-10T13:09:14.483Z INFO local object storage operation {"shard_id": "6tRaTRHDUWw7Ngo924ZBNR", "address": "9zbjpF9CmjCJryA7PzWSUDmJVhvTof6TwMmVitUu5G9M/3icHinczc9VrVJuff5cu4NKA4VvkwPi27YAi4dySe526", "op": "metabase PUT"} 123 logger.go:130: 2023-07-10T13:09:14.518Z INFO local object storage operation {"shard_id": "8xc2a7SH8M4XX3byZeZmzJ", "address": "9BaNbX3v9nB9eb2J43gTa7JZRkQqKzR5VwxaBVq5JLdh/DCq4fk4E8wFKnnVXwBecMDiSVGQ5tgx926BRdCPNNWBM", "op": "PUT", "type": "fstree", "storage_id": ""} 124 logger.go:130: 2023-07-10T13:09:14.531Z INFO local object storage operation {"shard_id": "8xc2a7SH8M4XX3byZeZmzJ", "address": "9BaNbX3v9nB9eb2J43gTa7JZRkQqKzR5VwxaBVq5JLdh/DCq4fk4E8wFKnnVXwBecMDiSVGQ5tgx926BRdCPNNWBM", "op": "metabase PUT"} 125 logger.go:130: 2023-07-10T13:09:14.542Z INFO local object storage operation {"shard_id": "CfB4de9Puv2cUVtyuft8JZ", "address": "8VkYw1t941XUkULBSZHbe6QttRPXVnqd7XuR6H3Ae7ob/DJrAjB6WPMBRvf8DDhW3F4sci7QYvVi8fpt6kdpA3mYa", "op": "PUT", "type": "fstree", "storage_id": ""} 126 logger.go:130: 2023-07-10T13:09:14.555Z INFO local object storage operation {"shard_id": "CfB4de9Puv2cUVtyuft8JZ", "address": "8VkYw1t941XUkULBSZHbe6QttRPXVnqd7XuR6H3Ae7ob/DJrAjB6WPMBRvf8DDhW3F4sci7QYvVi8fpt6kdpA3mYa", "op": "metabase PUT"} 127 logger.go:130: 2023-07-10T13:09:14.590Z INFO local object storage operation {"shard_id": "8xc2a7SH8M4XX3byZeZmzJ", "address": "HvpDfjA7tNMZtQZ4wPfZeMbZvnFH4uP49AdDmbgxHsVK/AJs3EXYUVZmuBaAU2UMGCQWHhgTfjdkFcVNk4G3YPr3z", "op": "PUT", "type": "fstree", "storage_id": ""} 128 logger.go:130: 2023-07-10T13:09:14.605Z INFO local object storage operation {"shard_id": "8xc2a7SH8M4XX3byZeZmzJ", "address": "HvpDfjA7tNMZtQZ4wPfZeMbZvnFH4uP49AdDmbgxHsVK/AJs3EXYUVZmuBaAU2UMGCQWHhgTfjdkFcVNk4G3YPr3z", "op": "metabase PUT"} 129 logger.go:130: 2023-07-10T13:09:14.613Z INFO setting shard mode {"shard_id": "8xc2a7SH8M4XX3byZeZmzJ", "old_mode": "READ_WRITE", "new_mode": "READ_ONLY"} 130 logger.go:130: 2023-07-10T13:09:14.613Z DEBUG closing... {"shard_id": "8xc2a7SH8M4XX3byZeZmzJ"} 131 logger.go:130: 2023-07-10T13:09:14.613Z DEBUG opening... {"shard_id": "8xc2a7SH8M4XX3byZeZmzJ"} 132 logger.go:130: 2023-07-10T13:09:14.613Z DEBUG initializing... {"shard_id": "8xc2a7SH8M4XX3byZeZmzJ"} 133 logger.go:130: 2023-07-10T13:09:14.613Z DEBUG created directory for Metabase {"shard_id": "8xc2a7SH8M4XX3byZeZmzJ", "path": "/tmp/TestEvacuateShard3284936385/001/2.metabase"} 134 logger.go:130: 2023-07-10T13:09:14.613Z DEBUG opened boltDB instance for Metabase {"shard_id": "8xc2a7SH8M4XX3byZeZmzJ"} 135 logger.go:130: 2023-07-10T13:09:14.613Z DEBUG checking metabase version {"shard_id": "8xc2a7SH8M4XX3byZeZmzJ"} 136 logger.go:130: 2023-07-10T13:09:14.613Z INFO shard mode set successfully {"shard_id": "8xc2a7SH8M4XX3byZeZmzJ", "mode": "READ_ONLY"} 137 logger.go:130: 2023-07-10T13:09:14.613Z INFO started shards evacuation {"shard_ids": ["8xc2a7SH8M4XX3byZeZmzJ"], "operation": "evacuation"} 138 logger.go:130: 2023-07-10T13:09:14.616Z INFO local object storage operation {"shard_id": "6tRaTRHDUWw7Ngo924ZBNR", "address": "4RMiBySomWHCXkDtdairHtktfzy72byfoFVJvUdcGWN8/GhGSF4FhkpaK8z3A5bYJnHB3FXjgvw3Z5uLoxkADq9qh", "op": "PUT", "type": "fstree", "storage_id": ""} 139 logger.go:130: 2023-07-10T13:09:14.631Z INFO local object storage operation {"shard_id": "6tRaTRHDUWw7Ngo924ZBNR", "address": "4RMiBySomWHCXkDtdairHtktfzy72byfoFVJvUdcGWN8/GhGSF4FhkpaK8z3A5bYJnHB3FXjgvw3Z5uLoxkADq9qh", "op": "metabase PUT"} 140 logger.go:130: 2023-07-10T13:09:14.631Z DEBUG object is moved to another shard {"from": "8xc2a7SH8M4XX3byZeZmzJ", "to": "6tRaTRHDUWw7Ngo924ZBNR", "addr": "4RMiBySomWHCXkDtdairHtktfzy72byfoFVJvUdcGWN8/GhGSF4FhkpaK8z3A5bYJnHB3FXjgvw3Z5uLoxkADq9qh", "operation": "evacuation"} 141 logger.go:130: 2023-07-10T13:09:14.633Z INFO local object storage operation {"shard_id": "6tRaTRHDUWw7Ngo924ZBNR", "address": "9BaNbX3v9nB9eb2J43gTa7JZRkQqKzR5VwxaBVq5JLdh/DCq4fk4E8wFKnnVXwBecMDiSVGQ5tgx926BRdCPNNWBM", "op": "PUT", "type": "fstree", "storage_id": ""} 142 logger.go:130: 2023-07-10T13:09:14.654Z INFO local object storage operation {"shard_id": "6tRaTRHDUWw7Ngo924ZBNR", "address": "9BaNbX3v9nB9eb2J43gTa7JZRkQqKzR5VwxaBVq5JLdh/DCq4fk4E8wFKnnVXwBecMDiSVGQ5tgx926BRdCPNNWBM", "op": "metabase PUT"} 143 logger.go:130: 2023-07-10T13:09:14.654Z DEBUG object is moved to another shard {"from": "8xc2a7SH8M4XX3byZeZmzJ", "to": "6tRaTRHDUWw7Ngo924ZBNR", "addr": "9BaNbX3v9nB9eb2J43gTa7JZRkQqKzR5VwxaBVq5JLdh/DCq4fk4E8wFKnnVXwBecMDiSVGQ5tgx926BRdCPNNWBM", "operation": "evacuation"} 144 logger.go:130: 2023-07-10T13:09:14.656Z INFO local object storage operation {"shard_id": "6tRaTRHDUWw7Ngo924ZBNR", "address": "HvpDfjA7tNMZtQZ4wPfZeMbZvnFH4uP49AdDmbgxHsVK/AJs3EXYUVZmuBaAU2UMGCQWHhgTfjdkFcVNk4G3YPr3z", "op": "PUT", "type": "fstree", "storage_id": ""} 145 logger.go:130: 2023-07-10T13:09:14.669Z INFO local object storage operation {"shard_id": "6tRaTRHDUWw7Ngo924ZBNR", "address": "HvpDfjA7tNMZtQZ4wPfZeMbZvnFH4uP49AdDmbgxHsVK/AJs3EXYUVZmuBaAU2UMGCQWHhgTfjdkFcVNk4G3YPr3z", "op": "metabase PUT"} 146 logger.go:130: 2023-07-10T13:09:14.669Z DEBUG object is moved to another shard {"from": "8xc2a7SH8M4XX3byZeZmzJ", "to": "6tRaTRHDUWw7Ngo924ZBNR", "addr": "HvpDfjA7tNMZtQZ4wPfZeMbZvnFH4uP49AdDmbgxHsVK/AJs3EXYUVZmuBaAU2UMGCQWHhgTfjdkFcVNk4G3YPr3z", "operation": "evacuation"} 147 logger.go:130: 2023-07-10T13:09:14.669Z INFO shards evacuation finished successfully {"shard_ids": ["8xc2a7SH8M4XX3byZeZmzJ"], "operation": "evacuation", "total": 3, "evacuated": 3, "failed": 0} 148 logger.go:130: 2023-07-10T13:09:14.677Z INFO started shards evacuation {"shard_ids": ["8xc2a7SH8M4XX3byZeZmzJ"], "operation": "evacuation"} 149 logger.go:130: 2023-07-10T13:09:14.683Z INFO local object storage operation {"shard_id": "CfB4de9Puv2cUVtyuft8JZ", "address": "9BaNbX3v9nB9eb2J43gTa7JZRkQqKzR5VwxaBVq5JLdh/DCq4fk4E8wFKnnVXwBecMDiSVGQ5tgx926BRdCPNNWBM", "op": "PUT", "type": "fstree", "storage_id": ""} 150 logger.go:130: 2023-07-10T13:09:14.696Z INFO local object storage operation {"shard_id": "CfB4de9Puv2cUVtyuft8JZ", "address": "9BaNbX3v9nB9eb2J43gTa7JZRkQqKzR5VwxaBVq5JLdh/DCq4fk4E8wFKnnVXwBecMDiSVGQ5tgx926BRdCPNNWBM", "op": "metabase PUT"} 151 logger.go:130: 2023-07-10T13:09:14.696Z DEBUG object is moved to another shard {"from": "8xc2a7SH8M4XX3byZeZmzJ", "to": "CfB4de9Puv2cUVtyuft8JZ", "addr": "9BaNbX3v9nB9eb2J43gTa7JZRkQqKzR5VwxaBVq5JLdh/DCq4fk4E8wFKnnVXwBecMDiSVGQ5tgx926BRdCPNNWBM", "operation": "evacuation"} 152 logger.go:130: 2023-07-10T13:09:14.701Z INFO shards evacuation finished successfully {"shard_ids": ["8xc2a7SH8M4XX3byZeZmzJ"], "operation": "evacuation", "total": 3, "evacuated": 1, "failed": 0} 153 evacuate_test.go:126: 154 Error Trace: /workspace/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine/evacuate_test.go:126 155 Error: Not equal: 156 expected: 0x0 157 actual : 0x1 158 Test: TestEvacuateShard 159 FAIL ```
Sign in to join this conversation.
No milestone
No project
No assignees
2 participants
Notifications
Due date
The due date is invalid or out of range. Please use the format "yyyy-mm-dd".

No due date set.

Dependencies

No dependencies set.

Reference: TrueCloudLab/frostfs-node#382
No description provided.