From fbd4a83602eb6622b7175fe7933583f6ca7ee2ad Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Thu, 22 Oct 2020 03:19:16 +0300 Subject: [PATCH] [#25] Refactoring over api/layer closes #25 Signed-off-by: Evgeniy Kulikov --- api/layer/container.go | 37 ++++++++++++++++++++++++++++++------- api/layer/layer.go | 9 +++------ api/layer/object.go | 24 ++++++++++++------------ api/layer/sdk.go | 14 ++++++++++++++ api/layer/util.go | 4 ++-- api/layer/writer.go | 13 +++++++++---- api/layer/writer_test.go | 37 +++++++++++++++++++++++++++++++++++++ go.mod | 2 +- go.sum | 3 +++ 9 files changed, 111 insertions(+), 32 deletions(-) create mode 100644 api/layer/sdk.go diff --git a/api/layer/container.go b/api/layer/container.go index c4ea524b..7d8026c9 100644 --- a/api/layer/container.go +++ b/api/layer/container.go @@ -2,6 +2,7 @@ package layer import ( "context" + "strconv" "time" "github.com/nspcc-dev/neofs-api-go/pkg/client" @@ -28,7 +29,15 @@ type ( ) func (n *layer) containerInfo(ctx context.Context, cid *container.ID) (*BucketInfo, error) { - rid := api.GetRequestID(ctx) + var ( + rid = api.GetRequestID(ctx) + + info = &BucketInfo{ + CID: cid, + Name: cid.String(), + } + ) + bearer, err := auth.GetBearerToken(ctx) if err != nil { n.log.Error("could not receive bearer token", @@ -60,13 +69,27 @@ func (n *layer) containerInfo(ctx context.Context, cid *container.ID) (*BucketIn return nil, err } - _ = res + for _, attr := range res.GetAttributes() { + switch key, val := attr.GetKey(), attr.GetValue(); key { + case ContainerName: + info.Name = val + case LocallyCreationTime: + unix, err := strconv.ParseInt(attr.GetValue(), 10, 64) + if err != nil { + n.log.Error("could not parse container creation time", + zap.Stringer("cid", cid), + zap.String("request_id", rid), + zap.String("created_at", val), + zap.Error(err)) - return &BucketInfo{ - CID: cid, - Name: cid.String(), // should be fetched from container.Attributes - Created: time.Time{}, // should be fetched from container.Attributes - }, nil + continue + } + + info.Created = time.Unix(unix, 0) + } + } + + return info, nil } func (n *layer) containerList(ctx context.Context) ([]BucketInfo, error) { diff --git a/api/layer/layer.go b/api/layer/layer.go index a8073aef..42c4c32b 100644 --- a/api/layer/layer.go +++ b/api/layer/layer.go @@ -86,9 +86,6 @@ type ( } ) -// AWS3NameHeader key in the object NeoFS. -const AWS3NameHeader = "filename" - // NewGatewayLayer creates instance of layer. It checks credentials // and establishes gRPC connection with node. func NewLayer(p *Params) (Client, error) { @@ -248,7 +245,7 @@ func (n *layer) GetObject(ctx context.Context, p *GetObjectParams) error { if err = cid.Parse(p.Bucket); err != nil { return err - } else if oid, err = n.objectFindID(ctx, &findParams{cid: cid, key: p.Object}); err != nil { + } else if oid, err = n.objectFindID(ctx, &findParams{cid: cid, val: p.Object}); err != nil { return err } @@ -280,7 +277,7 @@ func (n *layer) GetObjectInfo(ctx context.Context, bucketName, filename string) if err = cid.Parse(bucketName); err != nil { return nil, err - } else if oid, err = n.objectFindID(ctx, &findParams{cid: cid, key: filename}); err != nil { + } else if oid, err = n.objectFindID(ctx, &findParams{cid: cid, val: filename}); err != nil { return nil, err } @@ -356,7 +353,7 @@ func (n *layer) DeleteObject(ctx context.Context, bucket, filename string) error Err: err, Object: filename, } - } else if ids, err = n.objectSearch(ctx, &findParams{cid: cid, key: filename}); err != nil { + } else if ids, err = n.objectSearch(ctx, &findParams{cid: cid, val: filename}); err != nil { return &api.DeleteError{ Err: err, Object: filename, diff --git a/api/layer/object.go b/api/layer/object.go index 0d46ebd0..a2c70ad1 100644 --- a/api/layer/object.go +++ b/api/layer/object.go @@ -21,7 +21,7 @@ import ( type ( findParams struct { - key string + val string cid *container.ID } @@ -65,14 +65,10 @@ func (n *layer) objectSearch(ctx context.Context, p *findParams) ([]*object.ID, filter.AddNonLeafFilter() sop := new(client.SearchObjectParams) + sop.WithContainerID(p.cid) - if p.cid != nil { - filter.AddFilter(object.HdrSysNameCID, p.cid.String(), object.MatchStringEqual) - sop.WithContainerID(p.cid) - } - - if p.key != "" { - filter.AddFilter(AWS3NameHeader, p.key, object.MatchStringEqual) + if p.val != "" { + filter.AddFilter(ObjectName, p.val, object.MatchStringEqual) } sop.WithSearchFilters(filter) @@ -138,9 +134,13 @@ func (n *layer) objectPut(ctx context.Context, p *PutObjectParams) (*ObjectInfo, return nil, err } else if own, err = GetOwnerID(brt); err != nil { return nil, err - } else if err = cid.Parse(p.Bucket); err != nil { + } + + _ = own + + if bkt, err := n.GetBucketInfo(ctx, p.Bucket); err != nil { return nil, err - } else if _, err = n.objectFindID(ctx, &findParams{cid: cid, key: p.Object}); err == nil { + } else if _, err = n.objectFindID(ctx, &findParams{cid: bkt.CID, val: p.Object}); err == nil { return nil, &api.ObjectAlreadyExists{ Bucket: p.Bucket, Object: p.Object, @@ -155,7 +155,7 @@ func (n *layer) objectPut(ctx context.Context, p *PutObjectParams) (*ObjectInfo, attributes := make([]*object.Attribute, 0, len(p.Header)+1) filename := object.NewAttribute() - filename.SetKey(AWS3NameHeader) + filename.SetKey(ObjectName) filename.SetValue(p.Object) attributes = append(attributes, filename) @@ -172,7 +172,7 @@ func (n *layer) objectPut(ctx context.Context, p *PutObjectParams) (*ObjectInfo, r := io.TeeReader(p.Reader, b) raw := object.NewRaw() - raw.SetOwnerID(own) + raw.SetOwnerID(tkn.OwnerID()) // should be replaced with BearerToken.GetOwnerID() raw.SetContainerID(cid) raw.SetAttributes(attributes...) diff --git a/api/layer/sdk.go b/api/layer/sdk.go new file mode 100644 index 00000000..784e368a --- /dev/null +++ b/api/layer/sdk.go @@ -0,0 +1,14 @@ +package layer + +const ( + // TODO should be replaced with well-known types from SDK + + // ObjectName human readable name of the root object. + ObjectName = "filename" + + // ContainerName human readable name of the container (`bucket`). + ContainerName = "dirname" + + // LocallyCreationTime human readable creation time of the object / container. + LocallyCreationTime = "created_at" +) diff --git a/api/layer/util.go b/api/layer/util.go index e504e83b..d0de56ce 100644 --- a/api/layer/util.go +++ b/api/layer/util.go @@ -62,7 +62,7 @@ func objectInfoFromMeta(meta *object.Object) *ObjectInfo { aws3name := meta.GetID().String() userHeaders := userHeaders(meta.GetAttributes()) - if name, ok := userHeaders[AWS3NameHeader]; ok { + if name, ok := userHeaders[ObjectName]; ok { aws3name = name delete(userHeaders, name) } @@ -83,7 +83,7 @@ func nameFromObject(o *object.Object) (string, string) { var name = o.GetID().String() for _, attr := range o.GetAttributes() { - if attr.GetKey() == AWS3NameHeader { + if attr.GetKey() == ObjectName { name = attr.GetValue() break diff --git a/api/layer/writer.go b/api/layer/writer.go index 98f24b89..833a794e 100644 --- a/api/layer/writer.go +++ b/api/layer/writer.go @@ -33,11 +33,16 @@ func (w *offsetWriter) Write(p []byte) (int, error) { length -= offset - left := w.length - w.written - if left-length < 0 || length-left < length { - length = left - } else { + // Writer should write enough and stop writing + // 1. When passed zero length, it should write all bytes except offset + // 2. When the written buffer is almost filled (left < length), + // should write some bytes to fill up the buffer + // 3. When the written buffer is filled, should stop to write + + if left := w.length - w.written; left == 0 && w.length != 0 { return 0, nil + } else if left > 0 && left < length { + length = left } n, err := w.Writer.Write(p[offset : offset+length]) diff --git a/api/layer/writer_test.go b/api/layer/writer_test.go index 9c820f0c..1a242f64 100644 --- a/api/layer/writer_test.go +++ b/api/layer/writer_test.go @@ -20,6 +20,7 @@ func TestOffsetWriter(t *testing.T) { b := testBuffer(t) k := 64 d := len(b) / k + s := int64(len(b)) t.Run("1024 / 100 / 100 bytes success", func(t *testing.T) { w := new(bytes.Buffer) @@ -74,4 +75,40 @@ func TestOffsetWriter(t *testing.T) { require.Equal(t, l, wo.written) require.Equal(t, b[o:o+l], w.Bytes()) }) + + t.Run("should read all data when empty length passed", func(t *testing.T) { + w := new(bytes.Buffer) + o := int64(0) + l := int64(0) + + wt := newWriter(w, o, l) + for i := 0; i < k; i++ { + _, err := wt.Write(b[i*d : (i+1)*d]) + require.NoError(t, err) + } + + wo := wt.(*offsetWriter) + + require.Equal(t, o, wo.skipped) + require.Equal(t, s, wo.written) + require.Equal(t, b, w.Bytes()) + }) + + t.Run("should read all data when empty length passed", func(t *testing.T) { + w := new(bytes.Buffer) + o := int64(0) + l := s + 1 + + wt := newWriter(w, o, l) + for i := 0; i < k; i++ { + _, err := wt.Write(b[i*d : (i+1)*d]) + require.NoError(t, err) + } + + wo := wt.(*offsetWriter) + + require.Equal(t, o, wo.skipped) + require.Equal(t, s, wo.written) + require.Equal(t, b, w.Bytes()) + }) } diff --git a/go.mod b/go.mod index b182151c..c441df37 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/google/uuid v1.1.2 github.com/gorilla/mux v1.7.4 github.com/mitchellh/mapstructure v1.3.3 // indirect - github.com/nspcc-dev/neofs-api-go v1.3.1-0.20201007071636-fa18f5ede719 + github.com/nspcc-dev/neofs-api-go v1.3.1-0.20201020152448-c8f46f7d9762 github.com/nspcc-dev/neofs-authmate v0.0.0 github.com/nspcc-dev/neofs-crypto v0.3.0 github.com/pelletier/go-toml v1.8.0 // indirect diff --git a/go.sum b/go.sum index c62c233a..a9db83cc 100644 --- a/go.sum +++ b/go.sum @@ -174,6 +174,8 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= @@ -490,6 +492,7 @@ github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljT github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1 h1:+mkCCcOFKPnCmVYVcURKps1Xe+3zP90gSYGNfRkjoIY= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/urfave/cli/v2 v2.2.0 h1:JTTnM6wKzdA0Jqodd966MVj4vWbbquZykeX1sKbe2C4= github.com/urfave/cli/v2 v2.2.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=