policer: Panics when iterate over `detached` shard #1173

New issue

Closed

opened 2024-06-11 14:49:16 +00:00 by acid-ant · 0 comments

acid-ant commented

2024-06-11 14:49:16 +00:00

Member

Steps to reproduce:

Run dev-env
Detach shard
Send SIGHUP

2024-06-11T13:30:06.295Z        info    blobovniczatree/rebuild.go:66   collecting blobovniczas to rebuild completed successfully       {"blobovniczas_to_rebuild": 0}
2024-06-11T13:30:06.295Z        info    blobstor/rebuild.go:40  storages rebuild completed      {"shard_id": "E8PZdHLD5EmRYYo14hh6Qr", "success": true, "total_files_removed"
: 0, "total_objects_moved": 0}
2024-06-11T13:30:06.295Z        info    shard/rebuilder.go:53   blobstore rebuild completed successfully        {"shard_id": "E8PZdHLD5EmRYYo14hh6Qr"}
panic: runtime error: invalid memory address or nil pointer dereference [recovered]
        panic: runtime error: invalid memory address or nil pointer dereference [recovered]
        panic: runtime error: invalid memory address or nil pointer dereference
[signal SIGSEGV: segmentation violation code=0x1 addr=0x1c0 pc=0x96bc2d]
goroutine 25 [running]:
go.opentelemetry.io/otel/sdk/trace.(*recordingSpan).End.func1()
        go.opentelemetry.io/otel/sdk@v1.22.0/trace/span.go:405 +0x25
go.opentelemetry.io/otel/sdk/trace.(*recordingSpan).End(0xc001841c80, {0x0, 0x0, 0xc0002c5726?})
        go.opentelemetry.io/otel/sdk@v1.22.0/trace/span.go:443 +0xa3b
panic({0x1190ee0?, 0x2008360?})
        runtime/panic.go:914 +0x21f
go.opentelemetry.io/otel/sdk/trace.(*recordingSpan).End.func1()
        go.opentelemetry.io/otel/sdk@v1.22.0/trace/span.go:405 +0x25
go.opentelemetry.io/otel/sdk/trace.(*recordingSpan).End(0xc001841e00, {0x0, 0x0, 0x10ffd60?})
        go.opentelemetry.io/otel/sdk@v1.22.0/trace/span.go:443 +0xa3b
panic({0x1190ee0?, 0x2008360?})
        runtime/panic.go:914 +0x21f

go.opentelemetry.io/otel/sdk/trace.(*recordingSpan).End(0xc001841c80, {0x0, 0x0, 0xc0002c5726?})
        go.opentelemetry.io/otel/sdk@v1.22.0/trace/span.go:443 +0xa3b
panic({0x1190ee0?, 0x2008360?})
        runtime/panic.go:914 +0x21f
go.opentelemetry.io/otel/sdk/trace.(*recordingSpan).End.func1()
        go.opentelemetry.io/otel/sdk@v1.22.0/trace/span.go:405 +0x25
go.opentelemetry.io/otel/sdk/trace.(*recordingSpan).End(0xc001841e00, {0x0, 0x0, 0x10ffd60?})
        go.opentelemetry.io/otel/sdk@v1.22.0/trace/span.go:443 +0xa3b
panic({0x1190ee0?, 0x2008360?})
        runtime/panic.go:914 +0x21f
go.etcd.io/bbolt.(*DB).beginTx(0x0)
        go.etcd.io/bbolt@v1.3.9/db.go:730 +0x2d
go.etcd.io/bbolt.(*DB).Begin(0x7f870a152e90?, 0xb8?)
        go.etcd.io/bbolt@v1.3.9/db.go:723 +0x25
go.etcd.io/bbolt.(*DB).View(0x15d9bb0?, 0xc0002c5948)
        go.etcd.io/bbolt@v1.3.9/db.go:901 +0x30
git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase.(*DB).ListWithCursor(0xc000327780, {0x15d9bb0, 0xc0002999f0}, {0x1?, 0x0?})
        git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase/list.go:94 +0x61c
git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard.(*Shard).ListWithCursor(0xc0003c7960, {0x15d9bb0, 0xc0002999f0}, {0x16?, 0x0?})
        git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/list.go:157 +0x41c
git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine.(*StorageEngine).ListWithCursor(0xc000328cf0, {0x15d9bb0, 0xc0002999f0}, {0xd28006fe?, 0x0?})
        git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine/list.go:143 +0x345
main.(*keySpaceIterator).Next(0xc00157b010, {0x15d9bb0?, 0xc0002999f0?}, 0x1842360?)
        git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/keyspaceiterator.go:21 +0x50
git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/policer.(*Policer).shardPolicyWorker(0xc001555b00, {0x15d9bb0?, 0xc0002999f0})
        git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/policer/process.go:29 +0x9d
git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/policer.(*Policer).Run(0xc001555b00, {0x15d9bb0?, 0xc0002999f0?})
        git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/policer/process.go:16 +0x25
main.addPolicer.func7({0x15d9bb0?, 0xc0002999f0?})
        git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/object.go:293 +0x2a
main.startWorker.func1({{0x0?, 0x0?}, 0xc00157b0d0?})
        git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/worker.go:28 +0x31
created by main.startWorker in goroutine 1
        git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/worker.go:27 +0x129
2024-06-11T13:30:07.512Z        info    frostfs-node/main.go:76 initializing profiler service...

Steps to reproduce: - Run dev-env - Detach shard - Send SIGHUP ``` 2024-06-11T13:30:06.295Z info blobovniczatree/rebuild.go:66 collecting blobovniczas to rebuild completed successfully {"blobovniczas_to_rebuild": 0} 2024-06-11T13:30:06.295Z info blobstor/rebuild.go:40 storages rebuild completed {"shard_id": "E8PZdHLD5EmRYYo14hh6Qr", "success": true, "total_files_removed" : 0, "total_objects_moved": 0} 2024-06-11T13:30:06.295Z info shard/rebuilder.go:53 blobstore rebuild completed successfully {"shard_id": "E8PZdHLD5EmRYYo14hh6Qr"} panic: runtime error: invalid memory address or nil pointer dereference [recovered] panic: runtime error: invalid memory address or nil pointer dereference [recovered] panic: runtime error: invalid memory address or nil pointer dereference [signal SIGSEGV: segmentation violation code=0x1 addr=0x1c0 pc=0x96bc2d] goroutine 25 [running]: go.opentelemetry.io/otel/sdk/trace.(*recordingSpan).End.func1() go.opentelemetry.io/otel/sdk@v1.22.0/trace/span.go:405 +0x25 go.opentelemetry.io/otel/sdk/trace.(*recordingSpan).End(0xc001841c80, {0x0, 0x0, 0xc0002c5726?}) go.opentelemetry.io/otel/sdk@v1.22.0/trace/span.go:443 +0xa3b panic({0x1190ee0?, 0x2008360?}) runtime/panic.go:914 +0x21f go.opentelemetry.io/otel/sdk/trace.(*recordingSpan).End.func1() go.opentelemetry.io/otel/sdk@v1.22.0/trace/span.go:405 +0x25 go.opentelemetry.io/otel/sdk/trace.(*recordingSpan).End(0xc001841e00, {0x0, 0x0, 0x10ffd60?}) go.opentelemetry.io/otel/sdk@v1.22.0/trace/span.go:443 +0xa3b panic({0x1190ee0?, 0x2008360?}) runtime/panic.go:914 +0x21f go.opentelemetry.io/otel/sdk/trace.(*recordingSpan).End(0xc001841c80, {0x0, 0x0, 0xc0002c5726?}) go.opentelemetry.io/otel/sdk@v1.22.0/trace/span.go:443 +0xa3b panic({0x1190ee0?, 0x2008360?}) runtime/panic.go:914 +0x21f go.opentelemetry.io/otel/sdk/trace.(*recordingSpan).End.func1() go.opentelemetry.io/otel/sdk@v1.22.0/trace/span.go:405 +0x25 go.opentelemetry.io/otel/sdk/trace.(*recordingSpan).End(0xc001841e00, {0x0, 0x0, 0x10ffd60?}) go.opentelemetry.io/otel/sdk@v1.22.0/trace/span.go:443 +0xa3b panic({0x1190ee0?, 0x2008360?}) runtime/panic.go:914 +0x21f go.etcd.io/bbolt.(*DB).beginTx(0x0) go.etcd.io/bbolt@v1.3.9/db.go:730 +0x2d go.etcd.io/bbolt.(*DB).Begin(0x7f870a152e90?, 0xb8?) go.etcd.io/bbolt@v1.3.9/db.go:723 +0x25 go.etcd.io/bbolt.(*DB).View(0x15d9bb0?, 0xc0002c5948) go.etcd.io/bbolt@v1.3.9/db.go:901 +0x30 git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase.(*DB).ListWithCursor(0xc000327780, {0x15d9bb0, 0xc0002999f0}, {0x1?, 0x0?}) git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase/list.go:94 +0x61c git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard.(*Shard).ListWithCursor(0xc0003c7960, {0x15d9bb0, 0xc0002999f0}, {0x16?, 0x0?}) git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/list.go:157 +0x41c git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine.(*StorageEngine).ListWithCursor(0xc000328cf0, {0x15d9bb0, 0xc0002999f0}, {0xd28006fe?, 0x0?}) git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine/list.go:143 +0x345 main.(*keySpaceIterator).Next(0xc00157b010, {0x15d9bb0?, 0xc0002999f0?}, 0x1842360?) git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/keyspaceiterator.go:21 +0x50 git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/policer.(*Policer).shardPolicyWorker(0xc001555b00, {0x15d9bb0?, 0xc0002999f0}) git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/policer/process.go:29 +0x9d git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/policer.(*Policer).Run(0xc001555b00, {0x15d9bb0?, 0xc0002999f0?}) git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/policer/process.go:16 +0x25 main.addPolicer.func7({0x15d9bb0?, 0xc0002999f0?}) git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/object.go:293 +0x2a main.startWorker.func1({{0x0?, 0x0?}, 0xc00157b0d0?}) git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/worker.go:28 +0x31 created by main.startWorker in goroutine 1 git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/worker.go:27 +0x129 2024-06-11T13:30:07.512Z info frostfs-node/main.go:76 initializing profiler service... ```