From 8466894fdff20a6ac1b8b9b9b972f8c0cb436862 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 14 Apr 2023 09:38:29 +0300 Subject: [PATCH] [#250] control: remove `DumpShard` and `RestoreShard` RPC We have `Evacuate` with a cleaner interface. Also, remove them from CLI and engine. Signed-off-by: Evgenii Stratonikov --- .../modules/control/evacuate_shard.go | 6 +- cmd/frostfs-cli/modules/control/shards.go | 4 - .../modules/control/shards_dump.go | 66 --- .../modules/control/shards_restore.go | 66 --- pkg/local_object_storage/engine/dump.go | 19 - pkg/local_object_storage/engine/evacuate.go | 5 +- .../engine/evacuate_test.go | 2 +- pkg/local_object_storage/engine/restore.go | 32 -- pkg/local_object_storage/shard/dump.go | 129 ------ pkg/local_object_storage/shard/dump_test.go | 412 ------------------ pkg/local_object_storage/shard/restore.go | 145 ------ pkg/services/control/convert.go | 36 -- pkg/services/control/rpc.go | 28 -- pkg/services/control/server/dump.go | 37 -- pkg/services/control/server/restore.go | 37 -- pkg/services/control/service.go | 58 --- pkg/services/control/service.pb.go | Bin 121755 -> 100124 bytes pkg/services/control/service.proto | 75 ---- pkg/services/control/service_frostfs.pb.go | Bin 57948 -> 47352 bytes pkg/services/control/service_grpc.pb.go | Bin 20760 -> 17524 bytes 20 files changed, 9 insertions(+), 1148 deletions(-) delete mode 100644 cmd/frostfs-cli/modules/control/shards_dump.go delete mode 100644 cmd/frostfs-cli/modules/control/shards_restore.go delete mode 100644 pkg/local_object_storage/engine/dump.go delete mode 100644 pkg/local_object_storage/engine/restore.go delete mode 100644 pkg/local_object_storage/shard/dump.go delete mode 100644 pkg/local_object_storage/shard/dump_test.go delete mode 100644 pkg/local_object_storage/shard/restore.go delete mode 100644 pkg/services/control/server/dump.go delete mode 100644 pkg/services/control/server/restore.go diff --git a/cmd/frostfs-cli/modules/control/evacuate_shard.go b/cmd/frostfs-cli/modules/control/evacuate_shard.go index 02ee88ce..b72ff630 100644 --- a/cmd/frostfs-cli/modules/control/evacuate_shard.go +++ b/cmd/frostfs-cli/modules/control/evacuate_shard.go @@ -8,6 +8,8 @@ import ( "github.com/spf13/cobra" ) +const ignoreErrorsFlag = "no-errors" + var evacuateShardCmd = &cobra.Command{ Use: "evacuate", Short: "Evacuate objects from shard", @@ -20,7 +22,7 @@ func evacuateShard(cmd *cobra.Command, _ []string) { req := &control.EvacuateShardRequest{Body: new(control.EvacuateShardRequest_Body)} req.Body.Shard_ID = getShardIDList(cmd) - req.Body.IgnoreErrors, _ = cmd.Flags().GetBool(dumpIgnoreErrorsFlag) + req.Body.IgnoreErrors, _ = cmd.Flags().GetBool(ignoreErrorsFlag) signRequest(cmd, pk, req) @@ -47,7 +49,7 @@ func initControlEvacuateShardCmd() { flags := evacuateShardCmd.Flags() flags.StringSlice(shardIDFlag, nil, "List of shard IDs in base58 encoding") flags.Bool(shardAllFlag, false, "Process all shards") - flags.Bool(dumpIgnoreErrorsFlag, false, "Skip invalid/unreadable objects") + flags.Bool(ignoreErrorsFlag, false, "Skip invalid/unreadable objects") evacuateShardCmd.MarkFlagsMutuallyExclusive(shardIDFlag, shardAllFlag) } diff --git a/cmd/frostfs-cli/modules/control/shards.go b/cmd/frostfs-cli/modules/control/shards.go index 9d3eb5c0..8e7ecff8 100644 --- a/cmd/frostfs-cli/modules/control/shards.go +++ b/cmd/frostfs-cli/modules/control/shards.go @@ -13,16 +13,12 @@ var shardsCmd = &cobra.Command{ func initControlShardsCmd() { shardsCmd.AddCommand(listShardsCmd) shardsCmd.AddCommand(setShardModeCmd) - shardsCmd.AddCommand(dumpShardCmd) - shardsCmd.AddCommand(restoreShardCmd) shardsCmd.AddCommand(evacuateShardCmd) shardsCmd.AddCommand(flushCacheCmd) shardsCmd.AddCommand(doctorCmd) initControlShardsListCmd() initControlSetShardModeCmd() - initControlDumpShardCmd() - initControlRestoreShardCmd() initControlEvacuateShardCmd() initControlFlushCacheCmd() initControlDoctorCmd() diff --git a/cmd/frostfs-cli/modules/control/shards_dump.go b/cmd/frostfs-cli/modules/control/shards_dump.go deleted file mode 100644 index c0d0aca9..00000000 --- a/cmd/frostfs-cli/modules/control/shards_dump.go +++ /dev/null @@ -1,66 +0,0 @@ -package control - -import ( - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" - commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" - "github.com/spf13/cobra" -) - -const ( - dumpFilepathFlag = "path" - dumpIgnoreErrorsFlag = "no-errors" -) - -var dumpShardCmd = &cobra.Command{ - Use: "dump", - Short: "Dump objects from shard", - Long: "Dump objects from shard to a file", - Run: dumpShard, -} - -func dumpShard(cmd *cobra.Command, _ []string) { - pk := key.Get(cmd) - - body := new(control.DumpShardRequest_Body) - body.SetShardID(getShardID(cmd)) - - p, _ := cmd.Flags().GetString(dumpFilepathFlag) - body.SetFilepath(p) - - ignore, _ := cmd.Flags().GetBool(dumpIgnoreErrorsFlag) - body.SetIgnoreErrors(ignore) - - req := new(control.DumpShardRequest) - req.SetBody(body) - - signRequest(cmd, pk, req) - - cli := getClient(cmd, pk) - - var resp *control.DumpShardResponse - var err error - err = cli.ExecRaw(func(client *client.Client) error { - resp, err = control.DumpShard(client, req) - return err - }) - commonCmd.ExitOnErr(cmd, "rpc error: %w", err) - - verifyResponse(cmd, resp.GetSignature(), resp.GetBody()) - - cmd.Println("Shard has been dumped successfully.") -} - -func initControlDumpShardCmd() { - initControlFlags(dumpShardCmd) - - flags := dumpShardCmd.Flags() - flags.String(shardIDFlag, "", "Shard ID in base58 encoding") - flags.String(dumpFilepathFlag, "", "File to write objects to") - flags.Bool(dumpIgnoreErrorsFlag, false, "Skip invalid/unreadable objects") - - _ = dumpShardCmd.MarkFlagRequired(shardIDFlag) - _ = dumpShardCmd.MarkFlagRequired(dumpFilepathFlag) - _ = dumpShardCmd.MarkFlagRequired(controlRPC) -} diff --git a/cmd/frostfs-cli/modules/control/shards_restore.go b/cmd/frostfs-cli/modules/control/shards_restore.go deleted file mode 100644 index edf97a73..00000000 --- a/cmd/frostfs-cli/modules/control/shards_restore.go +++ /dev/null @@ -1,66 +0,0 @@ -package control - -import ( - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" - commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" - "github.com/spf13/cobra" -) - -const ( - restoreFilepathFlag = "path" - restoreIgnoreErrorsFlag = "no-errors" -) - -var restoreShardCmd = &cobra.Command{ - Use: "restore", - Short: "Restore objects from shard", - Long: "Restore objects from shard to a file", - Run: restoreShard, -} - -func restoreShard(cmd *cobra.Command, _ []string) { - pk := key.Get(cmd) - - body := new(control.RestoreShardRequest_Body) - body.SetShardID(getShardID(cmd)) - - p, _ := cmd.Flags().GetString(restoreFilepathFlag) - body.SetFilepath(p) - - ignore, _ := cmd.Flags().GetBool(restoreIgnoreErrorsFlag) - body.SetIgnoreErrors(ignore) - - req := new(control.RestoreShardRequest) - req.SetBody(body) - - signRequest(cmd, pk, req) - - cli := getClient(cmd, pk) - - var resp *control.RestoreShardResponse - var err error - err = cli.ExecRaw(func(client *client.Client) error { - resp, err = control.RestoreShard(client, req) - return err - }) - commonCmd.ExitOnErr(cmd, "rpc error: %w", err) - - verifyResponse(cmd, resp.GetSignature(), resp.GetBody()) - - cmd.Println("Shard has been restored successfully.") -} - -func initControlRestoreShardCmd() { - initControlFlags(restoreShardCmd) - - flags := restoreShardCmd.Flags() - flags.String(shardIDFlag, "", "Shard ID in base58 encoding") - flags.String(restoreFilepathFlag, "", "File to read objects from") - flags.Bool(restoreIgnoreErrorsFlag, false, "Skip invalid/unreadable objects") - - _ = restoreShardCmd.MarkFlagRequired(shardIDFlag) - _ = restoreShardCmd.MarkFlagRequired(restoreFilepathFlag) - _ = restoreShardCmd.MarkFlagRequired(controlRPC) -} diff --git a/pkg/local_object_storage/engine/dump.go b/pkg/local_object_storage/engine/dump.go deleted file mode 100644 index f5cf8c32..00000000 --- a/pkg/local_object_storage/engine/dump.go +++ /dev/null @@ -1,19 +0,0 @@ -package engine - -import "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" - -// DumpShard dumps objects from the shard with provided identifier. -// -// Returns an error if shard is not read-only. -func (e *StorageEngine) DumpShard(id *shard.ID, prm shard.DumpPrm) error { - e.mtx.RLock() - defer e.mtx.RUnlock() - - sh, ok := e.shards[id.String()] - if !ok { - return errShardNotFound - } - - _, err := sh.Dump(prm) - return err -} diff --git a/pkg/local_object_storage/engine/evacuate.go b/pkg/local_object_storage/engine/evacuate.go index 2ec2c2b3..e212784a 100644 --- a/pkg/local_object_storage/engine/evacuate.go +++ b/pkg/local_object_storage/engine/evacuate.go @@ -9,6 +9,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -16,6 +17,8 @@ import ( "go.uber.org/zap" ) +var ErrMustBeReadOnly = logicerr.New("shard must be in read-only mode") + // EvacuateShardPrm represents parameters for the EvacuateShard operation. type EvacuateShardPrm struct { shardID []*shard.ID @@ -135,7 +138,7 @@ func (e *StorageEngine) getActualShards(shardIDs []string, handlerDefined bool) } if !sh.GetMode().ReadOnly() { - return nil, nil, shard.ErrMustBeReadOnly + return nil, nil, ErrMustBeReadOnly } } diff --git a/pkg/local_object_storage/engine/evacuate_test.go b/pkg/local_object_storage/engine/evacuate_test.go index 291bc2b7..fc9da5e3 100644 --- a/pkg/local_object_storage/engine/evacuate_test.go +++ b/pkg/local_object_storage/engine/evacuate_test.go @@ -103,7 +103,7 @@ func TestEvacuateShard(t *testing.T) { t.Run("must be read-only", func(t *testing.T) { res, err := e.Evacuate(context.Background(), prm) - require.ErrorIs(t, err, shard.ErrMustBeReadOnly) + require.ErrorIs(t, err, ErrMustBeReadOnly) require.Equal(t, 0, res.Count()) }) diff --git a/pkg/local_object_storage/engine/restore.go b/pkg/local_object_storage/engine/restore.go deleted file mode 100644 index 7cc2eaf6..00000000 --- a/pkg/local_object_storage/engine/restore.go +++ /dev/null @@ -1,32 +0,0 @@ -package engine - -import ( - "context" - - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/trace" -) - -// RestoreShard restores objects from dump to the shard with provided identifier. -// -// Returns an error if shard is not read-only. -func (e *StorageEngine) RestoreShard(ctx context.Context, id *shard.ID, prm shard.RestorePrm) error { - ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.RestoreShard", - trace.WithAttributes( - attribute.String("shard_id", id.String()), - )) - defer span.End() - - e.mtx.RLock() - defer e.mtx.RUnlock() - - sh, ok := e.shards[id.String()] - if !ok { - return errShardNotFound - } - - _, err := sh.Restore(ctx, prm) - return err -} diff --git a/pkg/local_object_storage/shard/dump.go b/pkg/local_object_storage/shard/dump.go deleted file mode 100644 index 8d9fe0f7..00000000 --- a/pkg/local_object_storage/shard/dump.go +++ /dev/null @@ -1,129 +0,0 @@ -package shard - -import ( - "encoding/binary" - "io" - "os" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" -) - -var dumpMagic = []byte("NEOF") - -// DumpPrm groups the parameters of Dump operation. -type DumpPrm struct { - path string - stream io.Writer - ignoreErrors bool -} - -// WithPath is an Dump option to set the destination path. -func (p *DumpPrm) WithPath(path string) { - p.path = path -} - -// WithStream is an Dump option to set the destination stream. -// It takes priority over `path` option. -func (p *DumpPrm) WithStream(r io.Writer) { - p.stream = r -} - -// WithIgnoreErrors is an Dump option to allow ignore all errors during iteration. -// This includes invalid blobovniczas as well as corrupted objects. -func (p *DumpPrm) WithIgnoreErrors(ignore bool) { - p.ignoreErrors = ignore -} - -// DumpRes groups the result fields of Dump operation. -type DumpRes struct { - count int -} - -// Count return amount of object written. -func (r DumpRes) Count() int { - return r.count -} - -var ErrMustBeReadOnly = logicerr.New("shard must be in read-only mode") - -// Dump dumps all objects from the shard to a file or stream. -// -// Returns any error encountered. -func (s *Shard) Dump(prm DumpPrm) (DumpRes, error) { - s.m.RLock() - defer s.m.RUnlock() - - if !s.info.Mode.ReadOnly() { - return DumpRes{}, ErrMustBeReadOnly - } - - w := prm.stream - if w == nil { - f, err := os.OpenFile(prm.path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0640) - if err != nil { - return DumpRes{}, err - } - defer f.Close() - - w = f - } - - _, err := w.Write(dumpMagic) - if err != nil { - return DumpRes{}, err - } - - var count int - - if s.hasWriteCache() { - var iterPrm writecache.IterationPrm - - iterPrm.WithIgnoreErrors(prm.ignoreErrors) - iterPrm.WithHandler(func(data []byte) error { - var size [4]byte - binary.LittleEndian.PutUint32(size[:], uint32(len(data))) - if _, err := w.Write(size[:]); err != nil { - return err - } - - if _, err := w.Write(data); err != nil { - return err - } - - count++ - return nil - }) - - err := s.writeCache.Iterate(iterPrm) - if err != nil { - return DumpRes{}, err - } - } - - var pi common.IteratePrm - pi.IgnoreErrors = prm.ignoreErrors - pi.Handler = func(elem common.IterationElement) error { - data := elem.ObjectData - - var size [4]byte - binary.LittleEndian.PutUint32(size[:], uint32(len(data))) - if _, err := w.Write(size[:]); err != nil { - return err - } - - if _, err := w.Write(data); err != nil { - return err - } - - count++ - return nil - } - - if _, err := s.blobStor.Iterate(pi); err != nil { - return DumpRes{}, err - } - - return DumpRes{count: count}, nil -} diff --git a/pkg/local_object_storage/shard/dump_test.go b/pkg/local_object_storage/shard/dump_test.go deleted file mode 100644 index 92171720..00000000 --- a/pkg/local_object_storage/shard/dump_test.go +++ /dev/null @@ -1,412 +0,0 @@ -package shard_test - -import ( - "bytes" - "context" - "io" - "math/rand" - "os" - "path/filepath" - "testing" - "time" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" - objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" - oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - objecttest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" - "github.com/klauspost/compress/zstd" - "github.com/stretchr/testify/require" - "go.uber.org/zap/zaptest" -) - -func TestDump(t *testing.T) { - t.Run("without write-cache", func(t *testing.T) { - testDump(t, 10, false) - }) - t.Run("with write-cache", func(t *testing.T) { - // Put a bit more objects to write-cache to facilitate race-conditions. - testDump(t, 100, true) - }) -} - -func testDump(t *testing.T, objCount int, hasWriteCache bool) { - const ( - wcSmallObjectSize = 1024 // 1 KiB, goes to write-cache memory - wcBigObjectSize = 4 * 1024 // 4 KiB, goes to write-cache FSTree - bsSmallObjectSize = 10 * 1024 // 10 KiB, goes to blobovnicza DB - bsBigObjectSize = 1024*1024 + 1 // > 1 MiB, goes to blobovnicza FSTree - ) - - var sh *shard.Shard - if !hasWriteCache { - sh = newShard(t, false) - } else { - sh = newCustomShard(t, t.TempDir(), true, - []writecache.Option{ - writecache.WithSmallObjectSize(wcSmallObjectSize), - writecache.WithMaxObjectSize(wcBigObjectSize), - writecache.WithLogger(&logger.Logger{Logger: zaptest.NewLogger(t)}), - }, - nil) - } - defer releaseShard(sh, t) - - out := filepath.Join(t.TempDir(), "dump") - var prm shard.DumpPrm - prm.WithPath(out) - - t.Run("must be read-only", func(t *testing.T) { - _, err := sh.Dump(prm) - require.ErrorIs(t, err, shard.ErrMustBeReadOnly) - }) - - require.NoError(t, sh.SetMode(mode.ReadOnly)) - outEmpty := out + ".empty" - var dumpPrm shard.DumpPrm - dumpPrm.WithPath(outEmpty) - - res, err := sh.Dump(dumpPrm) - require.NoError(t, err) - require.Equal(t, 0, res.Count()) - require.NoError(t, sh.SetMode(mode.ReadWrite)) - - // Approximate object header size. - const headerSize = 400 - - objects := make([]*objectSDK.Object, objCount) - for i := 0; i < objCount; i++ { - cnr := cidtest.ID() - var size int - switch i % 6 { - case 0, 1: - size = wcSmallObjectSize - headerSize - case 2, 3: - size = bsSmallObjectSize - headerSize - case 4: - size = wcBigObjectSize - headerSize - default: - size = bsBigObjectSize - headerSize - } - data := make([]byte, size) - rand.Read(data) - obj := testutil.GenerateObjectWithCIDWithPayload(cnr, data) - objects[i] = obj - - var prm shard.PutPrm - prm.SetObject(objects[i]) - _, err := sh.Put(context.Background(), prm) - require.NoError(t, err) - } - - require.NoError(t, sh.SetMode(mode.ReadOnly)) - - t.Run("invalid path", func(t *testing.T) { - var dumpPrm shard.DumpPrm - dumpPrm.WithPath("\x00") - - _, err := sh.Dump(dumpPrm) - require.Error(t, err) - }) - - res, err = sh.Dump(prm) - require.NoError(t, err) - require.Equal(t, objCount, res.Count()) - - t.Run("restore", func(t *testing.T) { - sh := newShard(t, false) - defer releaseShard(sh, t) - - t.Run("empty dump", func(t *testing.T) { - var restorePrm shard.RestorePrm - restorePrm.WithPath(outEmpty) - res, err := sh.Restore(context.Background(), restorePrm) - require.NoError(t, err) - require.Equal(t, 0, res.Count()) - }) - - t.Run("invalid path", func(t *testing.T) { - _, err := sh.Restore(context.Background(), *new(shard.RestorePrm)) - require.ErrorIs(t, err, os.ErrNotExist) - }) - - t.Run("invalid file", func(t *testing.T) { - t.Run("invalid magic", func(t *testing.T) { - out := out + ".wrongmagic" - require.NoError(t, os.WriteFile(out, []byte{0, 0, 0, 0}, os.ModePerm)) - - var restorePrm shard.RestorePrm - restorePrm.WithPath(out) - - _, err := sh.Restore(context.Background(), restorePrm) - require.ErrorIs(t, err, shard.ErrInvalidMagic) - }) - - fileData, err := os.ReadFile(out) - require.NoError(t, err) - - t.Run("incomplete size", func(t *testing.T) { - out := out + ".wrongsize" - fileData := append(fileData, 1) - require.NoError(t, os.WriteFile(out, fileData, os.ModePerm)) - - var restorePrm shard.RestorePrm - restorePrm.WithPath(out) - - _, err := sh.Restore(context.Background(), restorePrm) - require.ErrorIs(t, err, io.ErrUnexpectedEOF) - }) - t.Run("incomplete object data", func(t *testing.T) { - out := out + ".wrongsize" - fileData := append(fileData, 1, 0, 0, 0) - require.NoError(t, os.WriteFile(out, fileData, os.ModePerm)) - - var restorePrm shard.RestorePrm - restorePrm.WithPath(out) - - _, err := sh.Restore(context.Background(), restorePrm) - require.ErrorIs(t, err, io.EOF) - }) - t.Run("invalid object", func(t *testing.T) { - out := out + ".wrongobj" - fileData := append(fileData, 1, 0, 0, 0, 0xFF, 4, 0, 0, 0, 1, 2, 3, 4) - require.NoError(t, os.WriteFile(out, fileData, os.ModePerm)) - - var restorePrm shard.RestorePrm - restorePrm.WithPath(out) - - _, err := sh.Restore(context.Background(), restorePrm) - require.Error(t, err) - - t.Run("skip errors", func(t *testing.T) { - sh := newCustomShard(t, filepath.Join(t.TempDir(), "ignore"), false, nil, nil) - t.Cleanup(func() { require.NoError(t, sh.Close()) }) - - var restorePrm shard.RestorePrm - restorePrm.WithPath(out) - restorePrm.WithIgnoreErrors(true) - - res, err := sh.Restore(context.Background(), restorePrm) - require.NoError(t, err) - require.Equal(t, objCount, res.Count()) - require.Equal(t, 2, res.FailCount()) - }) - }) - }) - - var prm shard.RestorePrm - prm.WithPath(out) - t.Run("must allow write", func(t *testing.T) { - require.NoError(t, sh.SetMode(mode.ReadOnly)) - - _, err := sh.Restore(context.Background(), prm) - require.ErrorIs(t, err, shard.ErrReadOnlyMode) - }) - - require.NoError(t, sh.SetMode(mode.ReadWrite)) - - checkRestore(t, sh, prm, objects) - }) -} - -func TestStream(t *testing.T) { - sh1 := newCustomShard(t, filepath.Join(t.TempDir(), "shard1"), false, nil, nil) - defer releaseShard(sh1, t) - - sh2 := newCustomShard(t, filepath.Join(t.TempDir(), "shard2"), false, nil, nil) - defer releaseShard(sh2, t) - - const objCount = 5 - objects := make([]*objectSDK.Object, objCount) - for i := 0; i < objCount; i++ { - cnr := cidtest.ID() - obj := testutil.GenerateObjectWithCID(cnr) - objects[i] = obj - - var prm shard.PutPrm - prm.SetObject(objects[i]) - _, err := sh1.Put(context.Background(), prm) - require.NoError(t, err) - } - - require.NoError(t, sh1.SetMode(mode.ReadOnly)) - - r, w := io.Pipe() - finish := make(chan struct{}) - - go func() { - var dumpPrm shard.DumpPrm - dumpPrm.WithStream(w) - - res, err := sh1.Dump(dumpPrm) - require.NoError(t, err) - require.Equal(t, objCount, res.Count()) - require.NoError(t, w.Close()) - close(finish) - }() - - var restorePrm shard.RestorePrm - restorePrm.WithStream(r) - - checkRestore(t, sh2, restorePrm, objects) - require.Eventually(t, func() bool { - select { - case <-finish: - return true - default: - return false - } - }, time.Second, time.Millisecond) -} - -func checkRestore(t *testing.T, sh *shard.Shard, prm shard.RestorePrm, objects []*objectSDK.Object) { - res, err := sh.Restore(context.Background(), prm) - require.NoError(t, err) - require.Equal(t, len(objects), res.Count()) - - var getPrm shard.GetPrm - - for i := range objects { - getPrm.SetAddress(object.AddressOf(objects[i])) - res, err := sh.Get(context.Background(), getPrm) - require.NoError(t, err) - require.Equal(t, objects[i], res.Object()) - } -} - -func TestDumpIgnoreErrors(t *testing.T) { - const ( - wcSmallObjectSize = 512 // goes to write-cache memory - wcBigObjectSize = wcSmallObjectSize << 1 // goes to write-cache FSTree - bsSmallObjectSize = wcSmallObjectSize << 2 // goes to blobovnicza DB - - objCount = 10 - headerSize = 400 - ) - - dir := t.TempDir() - bsPath := filepath.Join(dir, "blob") - bsOpts := func(sw uint64) []blobstor.Option { - return []blobstor.Option{ - blobstor.WithCompressObjects(true), - blobstor.WithStorages([]blobstor.SubStorage{ - { - Storage: blobovniczatree.NewBlobovniczaTree( - blobovniczatree.WithRootPath(filepath.Join(bsPath, "blobovnicza")), - blobovniczatree.WithBlobovniczaShallowDepth(1), - blobovniczatree.WithBlobovniczaShallowWidth(sw), - blobovniczatree.WithOpenedCacheSize(1)), - Policy: func(_ *objectSDK.Object, data []byte) bool { - return len(data) < bsSmallObjectSize - }, - }, - { - Storage: fstree.New( - fstree.WithPath(bsPath), - fstree.WithDepth(1)), - }, - }), - } - } - wcPath := filepath.Join(dir, "writecache") - wcOpts := []writecache.Option{ - writecache.WithPath(wcPath), - writecache.WithSmallObjectSize(wcSmallObjectSize), - writecache.WithMaxObjectSize(wcBigObjectSize), - } - sh := newCustomShard(t, dir, true, wcOpts, bsOpts(2)) - - objects := make([]*objectSDK.Object, objCount) - for i := 0; i < objCount; i++ { - size := (wcSmallObjectSize << (i % 4)) - headerSize - obj := testutil.GenerateObjectWithCIDWithPayload(cidtest.ID(), make([]byte, size)) - objects[i] = obj - - var prm shard.PutPrm - prm.SetObject(objects[i]) - _, err := sh.Put(context.Background(), prm) - require.NoError(t, err) - } - - releaseShard(sh, t) - - b := bytes.NewBuffer(nil) - badObject := make([]byte, 1000) - enc, err := zstd.NewWriter(b) - require.NoError(t, err) - corruptedData := enc.EncodeAll(badObject, nil) - for i := 4; i < len(corruptedData); i++ { - corruptedData[i] ^= 0xFF - } - - // There are 3 different types of errors to consider. - // To setup envirionment we use implementation details so this test must be updated - // if any of them are changed. - { - // 1. Invalid object in fs tree. - // 1.1. Invalid compressed data. - addr := cidtest.ID().EncodeToString() + "." + objecttest.ID().EncodeToString() - dirName := filepath.Join(bsPath, addr[:2]) - require.NoError(t, os.MkdirAll(dirName, os.ModePerm)) - require.NoError(t, os.WriteFile(filepath.Join(dirName, addr[2:]), corruptedData, os.ModePerm)) - - // 1.2. Unreadable file. - addr = cidtest.ID().EncodeToString() + "." + objecttest.ID().EncodeToString() - dirName = filepath.Join(bsPath, addr[:2]) - require.NoError(t, os.MkdirAll(dirName, os.ModePerm)) - - fname := filepath.Join(dirName, addr[2:]) - require.NoError(t, os.WriteFile(fname, []byte{}, 0)) - - // 1.3. Unreadable dir. - require.NoError(t, os.MkdirAll(filepath.Join(bsPath, "ZZ"), 0)) - } - - sh = newCustomShard(t, dir, true, wcOpts, bsOpts(3)) - require.NoError(t, sh.SetMode(mode.ReadOnly)) - - { - // 2. Invalid object in blobovnicza. - // 2.1. Invalid blobovnicza. - bTree := filepath.Join(bsPath, "blobovnicza") - data := make([]byte, 1024) - rand.Read(data) - require.NoError(t, os.WriteFile(filepath.Join(bTree, "0", "2"), data, 0)) - - // 2.2. Invalid object in valid blobovnicza. - var prm blobovnicza.PutPrm - prm.SetAddress(oid.Address{}) - prm.SetMarshaledObject(corruptedData) - b := blobovnicza.New(blobovnicza.WithPath(filepath.Join(bTree, "1", "2"))) - require.NoError(t, b.Open()) - _, err := b.Put(prm) - require.NoError(t, err) - require.NoError(t, b.Close()) - } - - { - // 3. Invalid object in write-cache. Note that because shard is read-only - // the object won't be flushed. - addr := cidtest.ID().EncodeToString() + "." + objecttest.ID().EncodeToString() - dir := filepath.Join(wcPath, addr[:1]) - require.NoError(t, os.MkdirAll(dir, os.ModePerm)) - require.NoError(t, os.WriteFile(filepath.Join(dir, addr[1:]), nil, 0)) - } - - out := filepath.Join(t.TempDir(), "out.dump") - var dumpPrm shard.DumpPrm - dumpPrm.WithPath(out) - dumpPrm.WithIgnoreErrors(true) - res, err := sh.Dump(dumpPrm) - require.NoError(t, err) - require.Equal(t, objCount, res.Count()) -} diff --git a/pkg/local_object_storage/shard/restore.go b/pkg/local_object_storage/shard/restore.go deleted file mode 100644 index 2cb64a51..00000000 --- a/pkg/local_object_storage/shard/restore.go +++ /dev/null @@ -1,145 +0,0 @@ -package shard - -import ( - "bytes" - "context" - "encoding/binary" - "errors" - "io" - "os" - - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/trace" -) - -// ErrInvalidMagic is returned when dump format is invalid. -var ErrInvalidMagic = logicerr.New("invalid magic") - -// RestorePrm groups the parameters of Restore operation. -type RestorePrm struct { - path string - stream io.Reader - ignoreErrors bool -} - -// WithPath is a Restore option to set the destination path. -func (p *RestorePrm) WithPath(path string) { - p.path = path -} - -// WithStream is a Restore option to set the stream to read objects from. -// It takes priority over `WithPath` option. -func (p *RestorePrm) WithStream(r io.Reader) { - p.stream = r -} - -// WithIgnoreErrors is a Restore option which allows to ignore errors encountered during restore. -// Corrupted objects will not be processed. -func (p *RestorePrm) WithIgnoreErrors(ignore bool) { - p.ignoreErrors = ignore -} - -// RestoreRes groups the result fields of Restore operation. -type RestoreRes struct { - count int - failed int -} - -// Count return amount of object written. -func (r RestoreRes) Count() int { - return r.count -} - -// FailCount return amount of object skipped. -func (r RestoreRes) FailCount() int { - return r.failed -} - -// Restore restores objects from the dump prepared by Dump. -// -// Returns any error encountered. -func (s *Shard) Restore(ctx context.Context, prm RestorePrm) (RestoreRes, error) { - ctx, span := tracing.StartSpanFromContext(ctx, "Shard.Restore", - trace.WithAttributes( - attribute.String("shard_id", s.ID().String()), - attribute.String("path", prm.path), - attribute.Bool("ignore_errors", prm.ignoreErrors), - )) - defer span.End() - - s.m.RLock() - defer s.m.RUnlock() - - if s.info.Mode.ReadOnly() { - return RestoreRes{}, ErrReadOnlyMode - } - - r := prm.stream - if r == nil { - f, err := os.OpenFile(prm.path, os.O_RDONLY, os.ModeExclusive) - if err != nil { - return RestoreRes{}, err - } - defer f.Close() - - r = f - } - - var m [4]byte - _, _ = io.ReadFull(r, m[:]) - if !bytes.Equal(m[:], dumpMagic) { - return RestoreRes{}, ErrInvalidMagic - } - - var putPrm PutPrm - - var count, failCount int - var data []byte - var size [4]byte - for { - // If there are less than 4 bytes left, `Read` returns nil error instead of - // io.ErrUnexpectedEOF, thus `ReadFull` is used. - _, err := io.ReadFull(r, size[:]) - if err != nil { - if errors.Is(err, io.EOF) { - break - } - return RestoreRes{}, err - } - - sz := binary.LittleEndian.Uint32(size[:]) - if uint32(cap(data)) < sz { - data = make([]byte, sz) - } else { - data = data[:sz] - } - - _, err = r.Read(data) - if err != nil { - return RestoreRes{}, err - } - - obj := object.New() - err = obj.Unmarshal(data) - if err != nil { - if prm.ignoreErrors { - failCount++ - continue - } - return RestoreRes{}, err - } - - putPrm.SetObject(obj) - _, err = s.Put(ctx, putPrm) - if err != nil && !IsErrObjectExpired(err) && !IsErrRemoved(err) { - return RestoreRes{}, err - } - - count++ - } - - return RestoreRes{count: count, failed: failCount}, nil -} diff --git a/pkg/services/control/convert.go b/pkg/services/control/convert.go index f7582dd6..84bde31d 100644 --- a/pkg/services/control/convert.go +++ b/pkg/services/control/convert.go @@ -111,42 +111,6 @@ func (w *setShardModeResponseWrapper) FromGRPCMessage(m grpc.Message) error { return nil } -type dumpShardResponseWrapper struct { - *DumpShardResponse -} - -func (w *dumpShardResponseWrapper) ToGRPCMessage() grpc.Message { - return w.DumpShardResponse -} - -func (w *dumpShardResponseWrapper) FromGRPCMessage(m grpc.Message) error { - r, ok := m.(*DumpShardResponse) - if !ok { - return message.NewUnexpectedMessageType(m, (*DumpShardResponse)(nil)) - } - - w.DumpShardResponse = r - return nil -} - -type restoreShardResponseWrapper struct { - *RestoreShardResponse -} - -func (w *restoreShardResponseWrapper) ToGRPCMessage() grpc.Message { - return w.RestoreShardResponse -} - -func (w *restoreShardResponseWrapper) FromGRPCMessage(m grpc.Message) error { - r, ok := m.(*RestoreShardResponse) - if !ok { - return message.NewUnexpectedMessageType(m, (*RestoreShardResponse)(nil)) - } - - w.RestoreShardResponse = r - return nil -} - type synchronizeTreeResponseWrapper struct { *SynchronizeTreeResponse } diff --git a/pkg/services/control/rpc.go b/pkg/services/control/rpc.go index 2676ea7a..625f485c 100644 --- a/pkg/services/control/rpc.go +++ b/pkg/services/control/rpc.go @@ -13,8 +13,6 @@ const ( rpcDropObjects = "DropObjects" rpcListShards = "ListShards" rpcSetShardMode = "SetShardMode" - rpcDumpShard = "DumpShard" - rpcRestoreShard = "RestoreShard" rpcSynchronizeTree = "SynchronizeTree" rpcEvacuateShard = "EvacuateShard" rpcFlushCache = "FlushCache" @@ -128,32 +126,6 @@ func SetShardMode( return wResp.m, nil } -// DumpShard executes ControlService.DumpShard RPC. -func DumpShard(cli *client.Client, req *DumpShardRequest, opts ...client.CallOption) (*DumpShardResponse, error) { - wResp := &dumpShardResponseWrapper{new(DumpShardResponse)} - wReq := &requestWrapper{m: req} - - err := client.SendUnary(cli, common.CallMethodInfoUnary(serviceName, rpcDumpShard), wReq, wResp, opts...) - if err != nil { - return nil, err - } - - return wResp.DumpShardResponse, nil -} - -// RestoreShard executes ControlService.DumpShard RPC. -func RestoreShard(cli *client.Client, req *RestoreShardRequest, opts ...client.CallOption) (*RestoreShardResponse, error) { - wResp := &restoreShardResponseWrapper{new(RestoreShardResponse)} - wReq := &requestWrapper{m: req} - - err := client.SendUnary(cli, common.CallMethodInfoUnary(serviceName, rpcRestoreShard), wReq, wResp, opts...) - if err != nil { - return nil, err - } - - return wResp.RestoreShardResponse, nil -} - // SynchronizeTree executes ControlService.SynchronizeTree RPC. func SynchronizeTree(cli *client.Client, req *SynchronizeTreeRequest, opts ...client.CallOption) (*SynchronizeTreeResponse, error) { wResp := &synchronizeTreeResponseWrapper{new(SynchronizeTreeResponse)} diff --git a/pkg/services/control/server/dump.go b/pkg/services/control/server/dump.go deleted file mode 100644 index 28be02aa..00000000 --- a/pkg/services/control/server/dump.go +++ /dev/null @@ -1,37 +0,0 @@ -package control - -import ( - "context" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" -) - -func (s *Server) DumpShard(_ context.Context, req *control.DumpShardRequest) (*control.DumpShardResponse, error) { - err := s.isValidRequest(req) - if err != nil { - return nil, status.Error(codes.PermissionDenied, err.Error()) - } - - shardID := shard.NewIDFromBytes(req.GetBody().GetShard_ID()) - - var prm shard.DumpPrm - prm.WithPath(req.GetBody().GetFilepath()) - prm.WithIgnoreErrors(req.GetBody().GetIgnoreErrors()) - - err = s.s.DumpShard(shardID, prm) - if err != nil { - return nil, status.Error(codes.Internal, err.Error()) - } - - resp := new(control.DumpShardResponse) - resp.SetBody(new(control.DumpShardResponse_Body)) - - err = SignMessage(s.key, resp) - if err != nil { - return nil, status.Error(codes.Internal, err.Error()) - } - return resp, nil -} diff --git a/pkg/services/control/server/restore.go b/pkg/services/control/server/restore.go deleted file mode 100644 index dba186f5..00000000 --- a/pkg/services/control/server/restore.go +++ /dev/null @@ -1,37 +0,0 @@ -package control - -import ( - "context" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" -) - -func (s *Server) RestoreShard(ctx context.Context, req *control.RestoreShardRequest) (*control.RestoreShardResponse, error) { - err := s.isValidRequest(req) - if err != nil { - return nil, status.Error(codes.PermissionDenied, err.Error()) - } - - shardID := shard.NewIDFromBytes(req.GetBody().GetShard_ID()) - - var prm shard.RestorePrm - prm.WithPath(req.GetBody().GetFilepath()) - prm.WithIgnoreErrors(req.GetBody().GetIgnoreErrors()) - - err = s.s.RestoreShard(ctx, shardID, prm) - if err != nil { - return nil, status.Error(codes.Internal, err.Error()) - } - - resp := new(control.RestoreShardResponse) - resp.SetBody(new(control.RestoreShardResponse_Body)) - - err = SignMessage(s.key, resp) - if err != nil { - return nil, status.Error(codes.Internal, err.Error()) - } - return resp, nil -} diff --git a/pkg/services/control/service.go b/pkg/services/control/service.go index dd349dc5..ef0c0a8d 100644 --- a/pkg/services/control/service.go +++ b/pkg/services/control/service.go @@ -127,64 +127,6 @@ func (x *SetShardModeResponse) SetBody(v *SetShardModeResponse_Body) { } } -// SetShardID sets shard ID for the dump shard request. -func (x *DumpShardRequest_Body) SetShardID(id []byte) { - x.Shard_ID = id -} - -// SetFilepath sets filepath for the dump shard request. -func (x *DumpShardRequest_Body) SetFilepath(p string) { - x.Filepath = p -} - -// SetIgnoreErrors sets ignore errors flag for the dump shard request. -func (x *DumpShardRequest_Body) SetIgnoreErrors(ignore bool) { - x.IgnoreErrors = ignore -} - -// SetBody sets request body. -func (x *DumpShardRequest) SetBody(v *DumpShardRequest_Body) { - if x != nil { - x.Body = v - } -} - -// SetBody sets response body. -func (x *DumpShardResponse) SetBody(v *DumpShardResponse_Body) { - if x != nil { - x.Body = v - } -} - -// SetShardID sets shard ID for the restore shard request. -func (x *RestoreShardRequest_Body) SetShardID(id []byte) { - x.Shard_ID = id -} - -// SetFilepath sets filepath for the restore shard request. -func (x *RestoreShardRequest_Body) SetFilepath(p string) { - x.Filepath = p -} - -// SetIgnoreErrors sets ignore errors flag for the restore shard request. -func (x *RestoreShardRequest_Body) SetIgnoreErrors(ignore bool) { - x.IgnoreErrors = ignore -} - -// SetBody sets request body. -func (x *RestoreShardRequest) SetBody(v *RestoreShardRequest_Body) { - if x != nil { - x.Body = v - } -} - -// SetBody sets response body. -func (x *RestoreShardResponse) SetBody(v *RestoreShardResponse_Body) { - if x != nil { - x.Body = v - } -} - // SetBody sets list shards request body. func (x *SynchronizeTreeRequest) SetBody(v *SynchronizeTreeRequest_Body) { if x != nil { diff --git a/pkg/services/control/service.pb.go b/pkg/services/control/service.pb.go index ca3e2770e1425432e663a12d4763d7420d57f637..d713bb38d3c269ac81a6e90c3b94f3573b98c833 100644 GIT binary patch delta 5421 zcmbVQdt8-ew*IY+`vPoW10slmkffp{`^rUAAmu26mvp=vhzcquYTm$7DK9Asf=gK` zDyYLgCh8A~V6^L`(oXF7kKe>UH{-nG`d*0Y|q z-jA=g(Y&kDbfhAk%ko0J$&)h;{QIX0_~%){T>lp%rwqyDJu~#@*~VyIIIV=EhPLqP zmSCB_olIQ5tPfjrSCXAuXZ27f-b1;GK5QS+i9Z{j4?kUb>Wn*BE1Pisad2M_$hGm2 zJQGg}n9XOigS_AI5) zS@Sq%<=Y1KpHs*&4+6NPteF?hy;(yPz*#Tt;=p+}&RrA2gF0KhL2Hg0xKFWCv~b!w z3uk8ymBXE&c`UAnTvrdJ5&2xbK$%WmsT({{UZ(VwmubOn(lyBm>oCCb8E1vV=J}Z=RfLv=L zD=)16o;N(NGz0kbHh-RWFWy_uolR^iE8zLTUD&q6%rQHE`cEFYxrOb$0=y4i z+ic*!RTX%#K|E`#l7@2CV@Am_QD2_lD`e{dGw-=K%e(iDJmNh&3V3zjV6Ho8yHAH~hyQvSHU#rx#j z*`0WIL*dpzft+~C%3FRI!ZFFATaTN0(}V6@cD$6Wuif-sJF&-y%MU=UGp7$UBB9|t z^0+_S(@fgBe|;>s9(%a{`Nw>||N2e7b|R2RH)&9E%*ntG{ayD8dB=P4eC3AfAwS-I z;RSERmyrg3aImng$@EzCCbMe6ES%Pb1Dmbf^k53~jJ)vh4b^a(gG1MT%MIs3+HJiX zuCsyv`lfOg&K3V*=DPpfHjb?RF^f_f1OceR6rv-x+G) z+7Ak~mTCS$IsT42oxR!CQb&05^^lIe^XF`w{_SACKGx*j{f}|PqdrlFA#(js7$U_# z-r?h)D$5YAx@!a>f6c>g#(NX)R~mTsWz{-A@fUwk4dP|Px1qdhb%3NB$glA#QK*5B zd~W3Q&+^+k%p@`8aC)I`s2u7{K*}^#=9n}Nf7T)dO~c^QG~(I zSGFvsRI!&(w2TTRi;|ofL5Wf~gJR`3p%f&Q0n}fnFQ#r%l&zW6WKxDSEyLD#{qU0< zPc{kihnI`F6d`BxsAtVE>LvC(tX9o|uU3DGluH9BO#ax1vJ9Slxsyj-m0`Gq4=1Pm zemI2%dI}T#eC;JksR=RomFmebuL>orR1JrD@e&-BKAd{U=flY*QJLf?U9u=p1spsL zv8Ja(b!-+zQj8R5QMgGtv**jTEQJ5NEF3<{2Aj{@Z2p=>!(`@S+9Ib$z#r0$kop8D zgk)2ie3MB9jrIZbDalJCVMoq>rOcot(%)j`~xI zl#Qcl3fHLrCra+54wSmc`7wyGI1Mp&?}vIGm`s6UO(wfsOam-`Po_EI_c$Q6WMJdL z(-bEIGoaj*LS5wPv9Rrd1SDq=#Y&-f~LmRX9-5 zi{nty{-Y_PaZDghBw0QlI6Qcq!i)|r$=Alyu$p85c6U4l8nw#F3yDa#_XPME*@n@k zWt6FiWs+&B)L9yGQS#;qDEaHNftxKF8w&>fom zByp2ra%qf?`x!SPc{UA6j_#r3pb`1+0c{zO}2E$8Ct5pJk2U6 z`U2RCMbuLg13{;t1VjLac9EuB%B*>uOp0k4lW3f@Mv_%6XehG1-Wg6~tVlU}DDt?b zn6Rpw4pq0{nHrhx4n=8uzT(D!XAmWt$L8l$LZ>2qN5?%Te_lp|)rt6@+Jsm9Q z_Y|-IV{NKvR+=F}6H$*wY(=VAX0(NgXNMMYvR3I%`S5c!U-zT++WLI-cUV5CcQT)R z)ma}ZowR2_p7-ZKN_IUNUf+#ic0fn%^LdtVt+g!Vkr9d!F&`Po#hBIOf1`8DRm z(mv|p(%_kqq=n>p7?{04J-g-UanQeAjI#z6P^u9!`wXE%3YEwZ6zzHdg8QlvKkpR) z$35LKhE4^5M399@a1>w)T1sXqEkr!6!Fq0ottjZFLWG%Jh|m#|HeNDPaa8}=n3)G` zNbpcs7}w4QM?l{QDa=D|2j{?bbZ3gzs#Ai++&MVb)*A}jCIRDi1&yg0J+U{Y80Qoh zkVQdn*PwU1Rc-(szeyS~wiLnD=DB(Xwf{%RD~o`8bgUk~7Ub6ukc;8Yup8P?T^y#GVw_j+fMI)+#Vb64-C`QApv;mz z&!LSLX6pd%U5uy@a8HHekd8eKyV}JVx+-FSx5fm$tL`i5#U+qpI3ae&tY6IGI4aPy zS98vy5Y{q_jROJdm!g`w({`$<{#i;H{>qu|GGEQgP<5mA`PXsPeHpOqSwarEJC71n z#cltpf{BJv5dJGXo_)xm@oh?fR%H0S52;Vqn{rLiJZA z&;p&JWxE^Bf7v`ejc%=3ZKSJFExske`0zes&>=LBznuWc_<0`ficBDmFD zh9Pva1`$5-5~ME47u?gOcs95PNct;ipEGLV(H3D+zaKDI*C4lmmoURl>;Md-)=)p0 zq?TXpM}ph2YjC7U#%2)j>KaNkI8xOjT#oF-VfM8+$1bB+Q8)FTE-TlP1JN8_OJn8T zHBk8XN~HJMS_)$O8&*kQhZF_5)E|tBDjXlQ9_KleWN$Sfp0u74 z(RcOL)D`cyCP~|hNs+RFLh&-{FBR($efoAxK<5Tbm4@vUrT(*%ku}iTv;jSmx*jLZ z6C93z=H!8re+fAP1TZmj{be}lzY&PKoEjprQo51ieO#`G_?V<=BiRiuw;bJpV)&I) zqK^ysUE2=`2JM8$Ddox}X@Fd=0Ps7pGPsiEgL29+xKhNji86d#sRQIjEq14G!s&L| zw23D8I8qX1{z~xOyiMI)o2e`6D+4#vNQ(ovRbpa{vjb?e9NJ87iwjd2a@QkrO9k~a z+cmNKVHu`WP)fVD=b`v`1tpkqp}Ka2UArk7S($3J){1Oa_AX_5 zyNX7sVy9KpWVPN|O`{FYWcj9=rgr#BRa}VB>OX%gxcS`{G8>#4`(wqj4Uv?g^6lzQ zhiZ#h`Dh3A^>w!4=v3_yE8S`+wL{OP%U0E3P7JOl6SAK!6sa-gQVk__D0&oGx@z(# zgc2+|eux_44zLikPAQ?je(K6Z%FYyxM%^Uw6>_#$!zsZ#z?`yI$nNLTQFYlLrQ5M8 zu%Y!8>gv~CZO5bA{dcHp;C{jZF1fQ2-L+x|^>2^X@qXl8#oUrX_p zHgekamhQfboan!!`w{q@T{OtxcIu0!U3jH(BmDiSXzXr!(%?qVDYcxv=;AfI(S>f0 zF1fIq(hPVn*S7z>hte!M;_hT!lvGCpEjmW-6kW8nj@-Tuy?@-PN){ux>rnewcL8ri zVwI=rF)@w1v8<}6VFr&DSfqSkPlLs)#0X@N!P6EMVhE7;_F*QhuEs|cHJ<*mmwY9E z4>S>zO>H{b8pqQXD*_9YnfpM1TERtAxephOr!DB`_Jec|y%ZiT>S0!o4&GFceSk+6 zHNdW01BF?23Ql!926*gpy#cdxYy)+EMC^9x;-&`ji&#>+upqBEe|~|Vulu3pBPu<} z(FiYL0dlaBw6ASa`wpFx^Ka4!CAuc9UObKd19fg3`4)wcdQ{)M%o6XVvoi1<>|Xd2 zAWS?#=VfN2z6_B*UU=F3J1mP%(G@v*jIyQM=YT-l_~JM=-uSfwuiRfwU7-)<-U(OjaSX&co`he|l%Q#b*1&GM`BcmhlT Z(s+eFl~orgyAfDzBMipIqhHY1^gjtKF$n+w delta 8981 zcmcIJYj~99vCk~o>~6A~Z0`4DLjoc9y=8O1K_HMot^|TuLLdo9$pr`qDiE}4t56cA zWLiaq(^IV|61vr(cpzy#fJGsx9(%AQ1O-$qkBL1Nss+xx-?zKjSbR?V2I>U`- z^uD$L<5sV&US2I5+g2K8Y&IlGH~zRLOeQ~EQ{hJbMfNfo+)*1OFLu@b$R+Q~r3N?J zM$K871nOekz?bH}ErV~@)wqF=#3xKBv@Y~Mw3l6VLys4j+;&yEc3eE)SLV}KAMS>V zNhqJ-(`RX%wxP3qee>OHUc=^hq8w(&xy92xWjOj)EEG1^P9if#g*oxwm1mvH(aM_1 zWNdi{&i?(Xh2W$+S#6t4$JWRDYkZyfT{Yp(=33 zOGBK{;rc*q*95AwUhBj;-^mr2zPYXDe?BAOOp}omB|3% zn4i_V6gpfV;k;$IzG^aQpQXXRt7x$TpIzFFJNtsMVb(Z)bMF<*I~glwFzqb@9N)I;GFf%Xn4yH1MmF=U)^v8N5A(g$inf{;pls>3UdOz z?I)J~UV-;)7GzqCYEq-uicj#JErL`_`IVr;uM1{i+8KYW*R6tVK5MYQ)btw#c0H^_ zuN{rpKCHvhRU^E=iZ2f7@V)9pjQ&84sjF2u&(H*BTs|CWU$f@80_~5r;GOq;a9`^v zntpf{SG2#RFq?4vy-0go_%T-riO-~ z@o^QN>xslxKR+A@_r{^eS1^qSezoEfGZJ`~k0rPxtyQQWY_gC1XodpE_BUfiybiZV zE3qL`$pUaVXawocc&=!7I8yZ;1u^?AV{0K2Cuv7_fJ zT>8r^d`RAl8`nD4K7%cG%}*Kt|MYS*Zcp-~(M=AcerD9tA2)$F7WVGN#jjk!r703t zX+K9h6!@RN67*Ue_?eE?9miR}78F{0-@ZZ*+->=2?17Gr1iJCVqVbO~| z*pR8j-TRgHhMzw+1#6EpT_6_aD)F7CW}_j~2fJU?NhTY8;Q>5z;wqoHCfCQ<*A8Ld zv!VD!ng)lT&5x>T>|AZ=Y&UeSZZ@=U?CjXcO-M86zO~IAjKJcT;$vDIo0N|A?VatG zsfINh+S_ubI>8v#--)-L^TvJs3*9R3x~}Mn-JZRx@QvTL$P(N-x@ZD}eTR}JcJ|lb zs)dOy9v-qyl<(b#AC(!LJQjx~eX*#0Gd|hX18d|C+Ay=dX-lG^w7Ju%!;D13hR*f( zwXH(GoG7&uiX(jyGPkTZf9Q7Ok2gM!&#u$TsKMi0IlKKPL@=FpG_k?sizi6V^+Zh( zeOZtiFV(9@99ST`}A0w| zozEoVz?;4_I$w0ZV@izr%}sk!e+xjCz2|5r|3fIa%w3F&0b@mRjjDH%uf^{T``eEn z>s4@P={dRqjSIb{f>E_7i1gdJV4dxW!m1Mw?I{pVPJ$`%%)miqW{SbM6^AP##S!~~ z!7zowYQ*ZNn(T(RJOQk@<6+Ul{BUTQd~4bnEvEifD}uu#iw`@Rpb`h(QsMby751W2 zOF&^Y2_8}}Y&|85vS5Wj>QBq%zWeRXf<$PpGwzW00dy`H{OETo(3<{r z2G@LHy@w@*g*_0?TDRTtrY=3y;WJ-`-Bsa^+i_Iu560dwP*G+O1Y+NLe_<@@3W5|I zyX^me*-A&t{UNZw!V8W|HlYg76QJ)TP9D)QrOJscWWB81%ns;Ap|0X*MdkL3e!AH!b-+ah!e1c z!n=A-xYzbiyDm=h+8n3X{`tdF8%^8d$)XKq-+F z8I4pK2Z`8y!;_x!1PjekL9jvRDbX9yU`OAV#|LNBwds+2t$X7W+9I3Ir!D@c!|axm?l0`X!qoB}EIlo@nXp@Hdm z_Iw;_+2>a5-NQfk35UALG z1z7yIV7A@R^-3EX5PP>i;*&XW?(Lhb25f*{3y^fT7; zn*ffbyw8IrH>Cu^Ys` zCP zUE{WTQ%Ns`fJM^7SzBqqmm`BSS&Eg{d?AZAhPuQmrS55*5MyEBF9f|E$}E@tHBVC9 zC=|~VvPnTT76ef=IJD50256wNi+3x9dd!WN3h|F{t+ zS!hEvWJ|U-uGo8tt7C2qm}spA0>w$67zo+>5hoWzC4;pyfT9f$OTUbftu`beBHcK& zv=nTRg)$i!+8+Z(+G2oE!OG9VX>dJ)3o6?yPBab=BiE!oAD8aA=>;okT_mjKvJ&si zB3nOK-)s<8N{X|{77|0tW~0;@9sX<<5Uc21*!b~u8jN!(gN+w5aruY7G$)?r-xdk! z^h7-8+D|V;v9+S`BHc>iP>KN1;&UI*qnPQe>657(yT$}A zr;SQB?#@wfs!D<+x;~u^AAK{O&L#81;By~qu<)}`JTdBteOIUA_RoyH>D(RPOaQgR zzPRqsb4Wi!7RQPY{y9}bWLk0XqiJLa=ISuO-j$@nO@A(-KZHS=#AItCch#XUr_z~3 zD5N_xcvigh*Kj=dmv}VX@DddhyFc`z%O)<81|8?RPZG?N4b3umtYG_4TC#8*uzJs> za0s2d?Wr)W6fvF4Jy0c-FtDGn`0Kd6RUmjqQ(g)LQh^62AgiyX?i7%U z%zV1!8izZi5avQ868j%Utf`JbHf2+B0OSUK$0R0=QlfI=j`Km#Nbe>>aiDYR5{%uZ zHi=dmCE|jl*?@y#g|iiMsI2_Q29_Y%+_aJhx?Flb7W|#9m#x#iHEKGHEx^;Rvv8_@N zOU>Ia9g3+o6oM5dGv%j1vXtpY9jFC|4kMT6`C7=6tAae#ipezR?%hM4v503ZJ#9*a zF#1U-x4{^boRDuP!fa8^q#_b-V$4t|=p?^iF|mWClwrJ?ukIDi*_pTWv{?!j>WO8t zAv0+3_m{KjPBvfkL+}P%MawtPsUFHdy8D0eL&X zTd5-2=5$BMOz(n&`EuaXrRnC(`~$LDgTD1z#$p zM{~NE3z0?x|D~rJOvW6;%J#O-_3dlZzWsK=OEW`yAuhc@rhz;NPnM3D?s{ZHM|<0b zW^qP-YfK}#5M_}Lm?sXnzfpF)#F#iDzp55Dh0>fn@SXG!PsUuZa=Q7>=b4jVh?=+= z@;a8@%Y_uM_2xkteLo+($%{|&lba{4$%7O{Ru1{qao7HJ9wc#@($C~N{#72BrkIU6 zw7-&fh53*yhGoAhzV-*w#!N@@VLlZWa%LaP=OJhLY%W((1z=^{Jv5B10VRvbdR2xQliagX3ew;PC*uI1_XM>{W|p!r~Iya51>v zoANQMLa7@@1@5MdD6B* zP%8zs&E+2@T`K}3Ki><;R4OQD<7KgzqG?w#+bx@yVUn9h7{d?P!Zqdm1T4Eqyd96umtY68ULX_$iEZvjobTLq{q&(Q0Qw;uLE}4jrgw(Rz8Ou-a&F7H5z( zlRQeGko{0n0*e%8GwtL5YEfWC_zOoz(dIHpR`GKJLnPOS2#4n?tX6tvHdM0Dug`{M zV!L1t`?_Kt&q2SK152G;GDS>GrKD2k!X22oWYLjq4!X^yU=_z+DTNBiq+6x1MB}{K zM8$<{jIDDaSm}Ry0&f`PbR5=e8yAEXtiYR(#9{1Se^FgIB+2lZq zEQ1UeRi+HPpS7aa1)$=99Liy%iP)mgEBHxUQwBcLRxO7b%LwIAMKKki4!m1Qpmn7X zC*J#shzp2xC}F-^DWHxrFh^P(=KNN`%C#i63gRVZ9G8JEX5R{02Wcty?KX~vP_y;h zk?5!wEZS7JI1Y?)O7{<;>o_0;jInS975=t;2iL(eh;wq~h!1jc&HRo7u`H>^BKzN* z<(3BzltY%&H|A_N#2?EcS!3bJjpg8qzGCMsG4)2O%>ObUK* z8(qjbwX2p3MTSM*crWH&F;)v^8e7C3ay3*cs)f*S=QnFt-xD(qTPPz#Iz74D@rjV= zyLL;HT)HxB@}2n$p-?RwAtRIe7W0t&(n3h(g4a;bW5nMUf>mRdot%*^6bPrPI-Xo? zM)~|Rbx^3VndA*CTgr{9a1ks~*c?i+hTe315ttb2jeCM)GSAiK+!>C#G1_b&#MT6Pyd?Xqf%)N+8^vISK=%&2ms? z8fE3W>LFEUlNDZSGv;KPXuO_l<{#?8$6G$W)S8`RqcJ|Lod8&~1zj-ZFM;5@gS}|` z5(wgwEa>`QS^{^pZZ5vq0B7m(RZu%X)L!eO#sC9)6hgYwB>rW~T%Yj}n^*a9OI zyH;LFA9LxmRdAYSw=!sGD|b8J`*_>bCU1SoxP~wOK|60Nckr`Wbb}||^L;o)hdN}C zKW$yh+Yi=3IqhzR_i4%od8MY;+TdL>b@KLXJKuWGMqU*x_;i zX7TPn{t);B8MeSh>f6TLmN$X;h^M6!5Ag*TwnLel5lvkX4bG2B0WwM2!%*f3Dirff zfe@_bkqMxfVfdEQG~CD+vrWa92v3QQ8*yU`+{2$70R#c|*DY zhg)V&YC&R2#^m>+QjEr%SJW(FMA5l^vVE4qWP^I?$+cOE2-Wc`4^5}uq@zZ9W- zv%^YpCT5_uo2{B=GO>UhF!^@NBZPv<1#Wo|VTZQU5C&tX)MmX6%Nar1CNp+gOrB>p z9n4lfv-l|n S5|h6|5VJBtgAKE3Pp<*`gBO+vHC0F_TZV5YlSoMi&5I>-uz{G_bZcl9}AKQqkt+iQ1fJe717DD9PDaf8&Qo)0m=hS zLAD%b5D$tVqkswOq19qV}N)npHq^QSDaHLyKLY v@i4E{TK1?Q2N<#)$l{wHnmpl!xXHj-9m0rr)_@tKk6^|*Yi>U3w4M