dkirillov/frostfs-s3-gw
1
0
Fork 0
forked from TrueCloudLab/frostfs-s3-gw
Code Issues Pull requests Projects Releases Packages Wiki Activity

[#25] Refactoring over api/layer

Browse source 
closes #25

Signed-off-by: Evgeniy Kulikov <kim@nspcc.ru>
This commit is contained in:
Evgeniy Kulikov 2020-10-22 03:19:16 +03:00
parent 14517d682c
commit fbd4a83602
9 changed files with 111 additions and 32 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

37
api/layer/container.go
View file

@ -2,6 +2,7 @@ package layer
import (
"context"
"strconv"
"time"
"github.com/nspcc-dev/neofs-api-go/pkg/client"
@ -28,7 +29,15 @@ type (
)
func (n *layer) containerInfo(ctx context.Context, cid *container.ID) (*BucketInfo, error) {
rid := api.GetRequestID(ctx)
var (
rid = api.GetRequestID(ctx)
info = &BucketInfo{
CID: cid,
Name: cid.String(),
}
)
bearer, err := auth.GetBearerToken(ctx)
if err != nil {
n.log.Error("could not receive bearer token",
@ -60,13 +69,27 @@ func (n *layer) containerInfo(ctx context.Context, cid *container.ID) (*BucketIn
return nil, err
}
_ = res
for _, attr := range res.GetAttributes() {
switch key, val := attr.GetKey(), attr.GetValue(); key {
case ContainerName:
info.Name = val
case LocallyCreationTime:
unix, err := strconv.ParseInt(attr.GetValue(), 10, 64)
if err != nil {
n.log.Error("could not parse container creation time",
zap.Stringer("cid", cid),
zap.String("request_id", rid),
zap.String("created_at", val),
zap.Error(err))
return &BucketInfo{
CID: cid,
Name: cid.String(), // should be fetched from container.Attributes
Created: time.Time{}, // should be fetched from container.Attributes
}, nil
continue
}
info.Created = time.Unix(unix, 0)
}
}
return info, nil
}
func (n *layer) containerList(ctx context.Context) ([]BucketInfo, error) {

9
api/layer/layer.go
View file

@ -86,9 +86,6 @@ type (
}
)
// AWS3NameHeader key in the object NeoFS.
const AWS3NameHeader = "filename"
// NewGatewayLayer creates instance of layer. It checks credentials
// and establishes gRPC connection with node.
func NewLayer(p *Params) (Client, error) {
@ -248,7 +245,7 @@ func (n *layer) GetObject(ctx context.Context, p *GetObjectParams) error {
if err = cid.Parse(p.Bucket); err != nil {
return err
} else if oid, err = n.objectFindID(ctx, &findParams{cid: cid, key: p.Object}); err != nil {
} else if oid, err = n.objectFindID(ctx, &findParams{cid: cid, val: p.Object}); err != nil {
return err
}
@ -280,7 +277,7 @@ func (n *layer) GetObjectInfo(ctx context.Context, bucketName, filename string)
if err = cid.Parse(bucketName); err != nil {
return nil, err
} else if oid, err = n.objectFindID(ctx, &findParams{cid: cid, key: filename}); err != nil {
} else if oid, err = n.objectFindID(ctx, &findParams{cid: cid, val: filename}); err != nil {
return nil, err
}
@ -356,7 +353,7 @@ func (n *layer) DeleteObject(ctx context.Context, bucket, filename string) error
Err: err,
Object: filename,
}
} else if ids, err = n.objectSearch(ctx, &findParams{cid: cid, key: filename}); err != nil {
} else if ids, err = n.objectSearch(ctx, &findParams{cid: cid, val: filename}); err != nil {
return &api.DeleteError{
Err: err,
Object: filename,

22
api/layer/object.go
View file

@ -21,7 +21,7 @@ import (
type (
findParams struct {
key string
val string
cid *container.ID
}
@ -65,14 +65,10 @@ func (n *layer) objectSearch(ctx context.Context, p *findParams) ([]*object.ID,
filter.AddNonLeafFilter()
sop := new(client.SearchObjectParams)
if p.cid != nil {
filter.AddFilter(object.HdrSysNameCID, p.cid.String(), object.MatchStringEqual)
sop.WithContainerID(p.cid)
}
if p.key != "" {
filter.AddFilter(AWS3NameHeader, p.key, object.MatchStringEqual)
if p.val != "" {
filter.AddFilter(ObjectName, p.val, object.MatchStringEqual)
}
sop.WithSearchFilters(filter)
@ -138,9 +134,13 @@ func (n *layer) objectPut(ctx context.Context, p *PutObjectParams) (*ObjectInfo,
return nil, err
} else if own, err = GetOwnerID(brt); err != nil {
return nil, err
} else if err = cid.Parse(p.Bucket); err != nil {
}
_ = own
if bkt, err := n.GetBucketInfo(ctx, p.Bucket); err != nil {
return nil, err
} else if _, err = n.objectFindID(ctx, &findParams{cid: cid, key: p.Object}); err == nil {
} else if _, err = n.objectFindID(ctx, &findParams{cid: bkt.CID, val: p.Object}); err == nil {
return nil, &api.ObjectAlreadyExists{
Bucket: p.Bucket,
Object: p.Object,
@ -155,7 +155,7 @@ func (n *layer) objectPut(ctx context.Context, p *PutObjectParams) (*ObjectInfo,
attributes := make([]*object.Attribute, 0, len(p.Header)+1)
filename := object.NewAttribute()
filename.SetKey(AWS3NameHeader)
filename.SetKey(ObjectName)
filename.SetValue(p.Object)
attributes = append(attributes, filename)
@ -172,7 +172,7 @@ func (n *layer) objectPut(ctx context.Context, p *PutObjectParams) (*ObjectInfo,
r := io.TeeReader(p.Reader, b)
raw := object.NewRaw()
raw.SetOwnerID(own)
raw.SetOwnerID(tkn.OwnerID()) // should be replaced with BearerToken.GetOwnerID()
raw.SetContainerID(cid)
raw.SetAttributes(attributes...)

14
api/layer/sdk.go Normal file
View file

@ -0,0 +1,14 @@
package layer
const (
// TODO should be replaced with well-known types from SDK
// ObjectName human readable name of the root object.
ObjectName = "filename"
// ContainerName human readable name of the container (`bucket`).
ContainerName = "dirname"
// LocallyCreationTime human readable creation time of the object / container.
LocallyCreationTime = "created_at"
)

4
api/layer/util.go
View file

@ -62,7 +62,7 @@ func objectInfoFromMeta(meta *object.Object) *ObjectInfo {
aws3name := meta.GetID().String()
userHeaders := userHeaders(meta.GetAttributes())
if name, ok := userHeaders[AWS3NameHeader]; ok {
if name, ok := userHeaders[ObjectName]; ok {
aws3name = name
delete(userHeaders, name)
}
@ -83,7 +83,7 @@ func nameFromObject(o *object.Object) (string, string) {
var name = o.GetID().String()
for _, attr := range o.GetAttributes() {
if attr.GetKey() == AWS3NameHeader {
if attr.GetKey() == ObjectName {
name = attr.GetValue()
break

13
api/layer/writer.go
View file

@ -33,11 +33,16 @@ func (w *offsetWriter) Write(p []byte) (int, error) {
length -= offset
left := w.length - w.written
if left-length < 0 || length-left < length {
length = left
} else {
// Writer should write enough and stop writing
// 1. When passed zero length, it should write all bytes except offset
// 2. When the written buffer is almost filled (left < length),
// should write some bytes to fill up the buffer
// 3. When the written buffer is filled, should stop to write
if left := w.length - w.written; left == 0 && w.length != 0 {
return 0, nil
} else if left > 0 && left < length {
length = left
}
n, err := w.Writer.Write(p[offset : offset+length])

37
api/layer/writer_test.go
View file

@ -20,6 +20,7 @@ func TestOffsetWriter(t *testing.T) {
b := testBuffer(t)
k := 64
d := len(b) / k
s := int64(len(b))
t.Run("1024 / 100 / 100 bytes success", func(t *testing.T) {
w := new(bytes.Buffer)
@ -74,4 +75,40 @@ func TestOffsetWriter(t *testing.T) {
require.Equal(t, l, wo.written)
require.Equal(t, b[o:o+l], w.Bytes())
})
t.Run("should read all data when empty length passed", func(t *testing.T) {
w := new(bytes.Buffer)
o := int64(0)
l := int64(0)
wt := newWriter(w, o, l)
for i := 0; i < k; i++ {
_, err := wt.Write(b[i*d : (i+1)*d])
require.NoError(t, err)
}
wo := wt.(*offsetWriter)
require.Equal(t, o, wo.skipped)
require.Equal(t, s, wo.written)
require.Equal(t, b, w.Bytes())
})
t.Run("should read all data when empty length passed", func(t *testing.T) {
w := new(bytes.Buffer)
o := int64(0)
l := s + 1
wt := newWriter(w, o, l)
for i := 0; i < k; i++ {
_, err := wt.Write(b[i*d : (i+1)*d])
require.NoError(t, err)
}
wo := wt.(*offsetWriter)
require.Equal(t, o, wo.skipped)
require.Equal(t, s, wo.written)
require.Equal(t, b, w.Bytes())
})
}

2
go.mod
View file

@ -8,7 +8,7 @@ require (
github.com/google/uuid v1.1.2
github.com/gorilla/mux v1.7.4
github.com/mitchellh/mapstructure v1.3.3 // indirect
github.com/nspcc-dev/neofs-api-go v1.3.1-0.20201007071636-fa18f5ede719
github.com/nspcc-dev/neofs-api-go v1.3.1-0.20201020152448-c8f46f7d9762
github.com/nspcc-dev/neofs-authmate v0.0.0
github.com/nspcc-dev/neofs-crypto v0.3.0
github.com/pelletier/go-toml v1.8.0 // indirect

3
go.sum
View file

@ -174,6 +174,8 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq
github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM=
github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
@ -490,6 +492,7 @@ github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljT
github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
github.com/urfave/cli v1.22.1 h1:+mkCCcOFKPnCmVYVcURKps1Xe+3zP90gSYGNfRkjoIY=
github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
github.com/urfave/cli/v2 v2.2.0 h1:JTTnM6wKzdA0Jqodd966MVj4vWbbquZykeX1sKbe2C4=
github.com/urfave/cli/v2 v2.2.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ=
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8=
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=