From 4e15452853fea599058a6724bb92bd924099b980 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Mon, 22 Jan 2024 11:09:11 +0300 Subject: [PATCH] [#165] Fix lint errors Signed-off-by: Denis Kirillov --- api/data/tree.go | 2 +- api/handler/delete_test.go | 2 +- api/handler/object_list_test.go | 57 ------------------- api/layer/listing.go | 69 +---------------------- internal/frostfs/services/pool_wrapper.go | 6 +- internal/logs/logs.go | 1 + pkg/service/tree/tree.go | 14 ++--- pkg/service/tree/tree_client_in_memory.go | 2 +- 8 files changed, 13 insertions(+), 140 deletions(-) diff --git a/api/data/tree.go b/api/data/tree.go index efc600d..277f32d 100644 --- a/api/data/tree.go +++ b/api/data/tree.go @@ -81,7 +81,7 @@ func (v *BaseNodeVersion) GetETag(md5Enabled bool) string { return v.ETag } -// IsFilledExtra returns true is node was created by version of gate v0.29.x and later +// IsFilledExtra returns true is node was created by version of gate v0.29.x and later. func (v BaseNodeVersion) IsFilledExtra() bool { return v.Created != nil && v.Owner != nil } diff --git a/api/handler/delete_test.go b/api/handler/delete_test.go index e97f585..28af78e 100644 --- a/api/handler/delete_test.go +++ b/api/handler/delete_test.go @@ -434,7 +434,7 @@ func createBucketAndObject(tc *handlerContext, bktName, objName string) (*data.B return bktInfo, objInfo } -func createVersionedBucketAndObject(t *testing.T, tc *handlerContext, bktName, objName string) (*data.BucketInfo, *data.ObjectInfo) { +func createVersionedBucketAndObject(_ *testing.T, tc *handlerContext, bktName, objName string) (*data.BucketInfo, *data.ObjectInfo) { bktInfo := createVersionedBucket(tc, bktName) objInfo := createTestObject(tc, bktInfo, objName, encryption.Params{}) diff --git a/api/handler/object_list_test.go b/api/handler/object_list_test.go index 5ef31c9..6e1feb8 100644 --- a/api/handler/object_list_test.go +++ b/api/handler/object_list_test.go @@ -6,12 +6,10 @@ import ( "net/url" "sort" "strconv" - "strings" "testing" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data" - "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption" "github.com/stretchr/testify/require" ) @@ -563,61 +561,6 @@ func checkVersionsNames(t *testing.T, versions *ListObjectsVersionsResponse, nam } } -func prepareObjects(hc *handlerContext, bktInfo *data.BucketInfo, prefix string, size int) []string { - treeID := "version" - parentID := uint64(0) - if prefix != "" { - for _, filename := range strings.Split(prefix, "/") { - nodeID, err := hc.treeMock.AddNode(hc.Context(), bktInfo, treeID, parentID, map[string]string{ - "FileName": filename, - }) - require.NoError(hc.t, err) - parentID = nodeID - } - prefix += "/" - } - - objects := make([]string, size) - - for i := range objects { - filename := "object" + strconv.Itoa(i) - filepath := prefix + filename - - prm := layer.PrmObjectCreate{ - Container: bktInfo.CID, - Filepath: filepath, - Payload: nil, - } - - id, err := hc.tp.CreateObject(hc.Context(), prm) - require.NoError(hc.t, err) - - newVersion := &data.NodeVersion{ - BaseNodeVersion: data.BaseNodeVersion{ - OID: id, - ETag: "12345678", - FilePath: filepath, - }, - IsUnversioned: true, - IsCombined: false, - } - - _, err = hc.treeMock.AddNodeBase(hc.Context(), bktInfo, treeID, parentID, map[string]string{ - "OID": newVersion.OID.EncodeToString(), - "FileName": filename, - "IsUnversioned": "true", - }, false) - require.NoError(hc.t, err) - objects[i] = filepath - } - - hc.treeMock.Sort() - - sort.Strings(objects) - - return objects -} - func listObjectsV2(hc *handlerContext, bktName, prefix, delimiter, startAfter, continuationToken string, maxKeys int) *ListObjectsV2Response { return listObjectsV2Ext(hc, bktName, prefix, delimiter, startAfter, continuationToken, "", maxKeys) } diff --git a/api/layer/listing.go b/api/layer/listing.go index 6e7984f..89ca9d9 100644 --- a/api/layer/listing.go +++ b/api/layer/listing.go @@ -199,7 +199,7 @@ func (n *layer) getLatestObjectsVersions(ctx context.Context, p commonLatestVers } generator, errorCh := nodesGeneratorStream(ctx, p.commonVersionsListingParams, session) - objOutCh, err := n.initWorkerPoolStream(ctx, 2, p.commonVersionsListingParams, generator) + objOutCh, err := n.initWorkerPool(ctx, 2, p.commonVersionsListingParams, generator) if err != nil { return nil, nil, fmt.Errorf("failed to init worker pool: %w", err) } @@ -234,7 +234,7 @@ func (n *layer) getAllObjectsVersions(ctx context.Context, p commonVersionsListi } generator, errorCh := nodesGeneratorVersions(ctx, p, session) - objOutCh, err := n.initWorkerPoolVersions(ctx, 2, p, generator) + objOutCh, err := n.initWorkerPool(ctx, 2, p, generator) if err != nil { return nil, false, err } @@ -312,7 +312,6 @@ func (n *layer) getListVersionsSession(ctx context.Context, p commonVersionsList session := n.cache.GetListSession(owner, cacheKey) if session == nil { return n.initNewVersionsByPrefixSession(ctx, p, latestOnly) - } if session.Acquired.Swap(true) { @@ -495,7 +494,7 @@ func nodesGeneratorVersions(ctx context.Context, p commonVersionsListingParams, return nodeCh, errCh } -func (n *layer) initWorkerPoolStream(ctx context.Context, size int, p commonVersionsListingParams, input <-chan *data.ExtendedNodeVersion) (<-chan *data.ExtendedNodeVersion, error) { +func (n *layer) initWorkerPool(ctx context.Context, size int, p commonVersionsListingParams, input <-chan *data.ExtendedNodeVersion) (<-chan *data.ExtendedNodeVersion, error) { reqLog := n.reqLogger(ctx) pool, err := ants.NewPool(size, ants.WithLogger(&logWrapper{reqLog})) if err != nil { @@ -503,68 +502,6 @@ func (n *layer) initWorkerPoolStream(ctx context.Context, size int, p commonVers } objCh := make(chan *data.ExtendedNodeVersion, size) - go func() { - var wg sync.WaitGroup - - LOOP: - for node := range input { - select { - case <-ctx.Done(): - break LOOP - default: - } - - if node.DirName != "" || node.NodeVersion.IsFilledExtra() { - select { - case <-ctx.Done(): - case objCh <- node: - } - } else { - // We have to make a copy of pointer to data.NodeVersion - // to get correct value in submitted task function. - func(node *data.ExtendedNodeVersion) { - wg.Add(1) - err = pool.Submit(func() { - defer wg.Done() - oi := n.objectInfoFromObjectsCacheOrFrostFS(ctx, p.BktInfo, node.NodeVersion) - if oi == nil { - // try to get object again - if oi = n.objectInfoFromObjectsCacheOrFrostFS(ctx, p.BktInfo, node.NodeVersion); oi == nil { - // do not process object which are definitely missing in object service - return - } - } - - node.NodeVersion.FillExtra(oi) - - select { - case <-ctx.Done(): - case objCh <- node: - } - }) - if err != nil { - wg.Done() - reqLog.Warn(logs.FailedToSubmitTaskToPool, zap.Error(err)) - } - }(node) - } - } - wg.Wait() - close(objCh) - pool.Release() - }() - - return objCh, nil -} - -func (n *layer) initWorkerPoolVersions(ctx context.Context, size int, p commonVersionsListingParams, input <-chan *data.ExtendedNodeVersion) (<-chan *data.ExtendedNodeVersion, error) { - reqLog := n.reqLogger(ctx) - pool, err := ants.NewPool(size, ants.WithLogger(&logWrapper{reqLog})) - if err != nil { - return nil, fmt.Errorf("coudln't init go pool for listing: %w", err) - } - objCh := make(chan *data.ExtendedNodeVersion) - go func() { var wg sync.WaitGroup diff --git a/internal/frostfs/services/pool_wrapper.go b/internal/frostfs/services/pool_wrapper.go index 480d753..e8722a8 100644 --- a/internal/frostfs/services/pool_wrapper.go +++ b/internal/frostfs/services/pool_wrapper.go @@ -151,11 +151,9 @@ func (s *SubTreeStreamImpl) Next() (tree.NodeResponse, error) { s.ln, err = s.r.Read(s.buffer) if err != nil { if err != io.EOF { - err = handleError(err) - return nil, fmt.Errorf("sub tree stream impl pool wrap: %w", err) - } else { - s.eof = true + return nil, fmt.Errorf("sub tree stream impl pool wrap: %w", handleError(err)) } + s.eof = true } if s.ln > 0 { s.index = 0 diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 2e4954c..510a3ba 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -138,4 +138,5 @@ const ( ControlAPIGetPolicy = "get policy request" ControlAPIListPolicies = "list policies request" PolicyValidationFailed = "policy validation failed" + ParseTreeNode = "parse tree node" ) diff --git a/pkg/service/tree/tree.go b/pkg/service/tree/tree.go index e36d9ef..ab3d86f 100644 --- a/pkg/service/tree/tree.go +++ b/pkg/service/tree/tree.go @@ -675,7 +675,7 @@ func (s *VersionsByPrefixStreamImpl) Next(context.Context) (*data.NodeVersion, e return nil, io.EOF } - for true { + for { if s.innerStream == nil { node, err := s.getNodeFromMainStream() if err != nil { @@ -707,12 +707,10 @@ func (s *VersionsByPrefixStreamImpl) Next(context.Context) (*data.NodeVersion, e } return nodeVersion, nil } - - panic("unreachable code") } func (s *VersionsByPrefixStreamImpl) getNodeFromMainStream() (NodeResponse, error) { - for true { + for { node, err := s.mainStream.Next() if err != nil { if errors.Is(err, ErrNodeNotFound) { @@ -725,8 +723,6 @@ func (s *VersionsByPrefixStreamImpl) getNodeFromMainStream() (NodeResponse, erro return node, nil } } - - panic("unreachable code") } func (s *VersionsByPrefixStreamImpl) initInnerStream(node NodeResponse) (err error) { @@ -747,7 +743,7 @@ func (s *VersionsByPrefixStreamImpl) initInnerStream(node NodeResponse) (err err } func (s *VersionsByPrefixStreamImpl) getNodeVersionFromInnerStream() (*data.NodeVersion, error) { - for true { + for { node, err := s.innerStream.Next() if err != nil { return nil, fmt.Errorf("inner stream: %w", err) @@ -782,14 +778,12 @@ func (s *VersionsByPrefixStreamImpl) getNodeVersionFromInnerStream() (*data.Node return nodeVersion, nil } - - panic("unreachable code") } func (s *VersionsByPrefixStreamImpl) parseNodeResponse(node NodeResponse) (res *data.NodeVersion, skip bool, err error) { trNode, fileName, err := parseTreeNode(node) if err != nil { - s.log.Debug("parse tree node", zap.Error(err)) + s.log.Debug(logs.ParseTreeNode, zap.Error(err)) return nil, true, nil } diff --git a/pkg/service/tree/tree_client_in_memory.go b/pkg/service/tree/tree_client_in_memory.go index dc8fe2b..1429c0b 100644 --- a/pkg/service/tree/tree_client_in_memory.go +++ b/pkg/service/tree/tree_client_in_memory.go @@ -336,7 +336,7 @@ func (c *ServiceClientMemory) AddNodeBase(_ context.Context, bktInfo *data.Bucke parentNode.children = append(parentNode.children, tn) if needSort { - //sortNodes(parentNode.children) + sortNodes(parentNode.children) } cnr.trees[treeID] = tr