Add test examples from different object storage clients
Signed-off-by: Alex Vanin <a.vanin@yadro.com>
This commit is contained in:
parent
9d40228cec
commit
99091ad436
6 changed files with 416 additions and 1 deletions
99
compare/aws_sdk_test.go
Normal file
99
compare/aws_sdk_test.go
Normal file
|
|
@ -0,0 +1,99 @@
|
|||
package compare
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/config"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/davecgh/go-spew/spew"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
var (
|
||||
accessKeyID = "9TtxuFepf2sibaAM"
|
||||
secretKeyID = "SfxKtYws2We0uzsiXT011aWGMV4iTKlD"
|
||||
bucketName = "test-bucket-5"
|
||||
keyName = "foo"
|
||||
)
|
||||
|
||||
// 1. Initialize client
|
||||
// 2. Create bucket / container
|
||||
// 3. Put object
|
||||
// 4. Get object
|
||||
|
||||
// 1. Опции с публичными полями вместо структур с сеттерами
|
||||
// 2. Credentials
|
||||
// 3. Создавать контейнеры принимая аргументы с опциями контейнера, а не с самим контейнером, включая строковый Placement Policy
|
||||
// 4. Сделать pool умнее путём раскладывания объектов в параллель / по плейсменту
|
||||
// 5. Подумать о том можно ли как-то объеденить сахарные сценарии в pool и более прямые сценарии в SDK Client
|
||||
|
||||
func TestAWSSDK(t *testing.T) {
|
||||
// 1. Initialize client
|
||||
|
||||
payloadReader := strings.NewReader("Hello World") // reader seeker
|
||||
_ = payloadReader
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
cred := credentials.NewStaticCredentialsProvider(accessKeyID, secretKeyID, "")
|
||||
cli := s3.New(s3.Options{
|
||||
Credentials: cred,
|
||||
EndpointResolver: s3.EndpointResolverFromURL("http://127.0.0.1:9000"),
|
||||
Retryer: aws.NopRetryer{},
|
||||
UsePathStyle: true,
|
||||
})
|
||||
|
||||
cfg, err := config.LoadDefaultConfig(ctx)
|
||||
require.NoError(t, err)
|
||||
cli = s3.NewFromConfig(cfg, func(options *s3.Options) {
|
||||
options.Retryer = aws.NopRetryer{}
|
||||
options.UsePathStyle = true
|
||||
options.EndpointResolver = s3.EndpointResolverFromURL("http://127.0.0.1:9000")
|
||||
})
|
||||
|
||||
// 2. Create bucket / container
|
||||
|
||||
respCreateBucket, err := cli.CreateBucket(ctx, &s3.CreateBucketInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
ACL: "public-read-write",
|
||||
CreateBucketConfiguration: &types.CreateBucketConfiguration{
|
||||
LocationConstraint: "default",
|
||||
},
|
||||
ObjectLockEnabledForBucket: false,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
_ = respCreateBucket
|
||||
|
||||
// 3. Put object
|
||||
|
||||
bbuf := bytes.NewBufferString("lalala")
|
||||
respPutObject, err := cli.PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(keyName),
|
||||
Body: bbuf,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
spew.Dump(respPutObject)
|
||||
|
||||
// 4. Get object
|
||||
|
||||
objectReader, err := cli.GetObject(ctx, &s3.GetObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(keyName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
spew.Dump(objectReader.ContentLength)
|
||||
|
||||
buf, err := io.ReadAll(objectReader.Body)
|
||||
require.NoError(t, err)
|
||||
spew.Dump(buf)
|
||||
require.NoError(t, objectReader.Body.Close())
|
||||
}
|
||||
134
compare/frostfs_client_test.go
Normal file
134
compare/frostfs_client_test.go
Normal file
|
|
@ -0,0 +1,134 @@
|
|||
package compare
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
|
||||
apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||
"github.com/davecgh/go-spew/spew"
|
||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestFrostFSClient(t *testing.T) {
|
||||
// 1. Initialize client
|
||||
|
||||
payload := []byte("Hello World")
|
||||
|
||||
ctx := context.Background()
|
||||
k, err := keys.NewPrivateKeyFromWIF("L32sMMkUcEfsWFdjoFNKAPtj7y6cfkcpHwG2pmTap68AmUt86ZTo")
|
||||
require.NoError(t, err)
|
||||
|
||||
var prmInit client.PrmInit
|
||||
prmInit.SetDefaultPrivateKey(k.PrivateKey)
|
||||
|
||||
var prmDial client.PrmDial
|
||||
prmDial.SetServerURI("localhost:8080")
|
||||
|
||||
var cli client.Client
|
||||
cli.Init(prmInit)
|
||||
|
||||
err = cli.Dial(ctx, prmDial)
|
||||
require.NoError(t, err)
|
||||
|
||||
// 2. Create bucket / container
|
||||
|
||||
var pp netmap.PlacementPolicy
|
||||
err = pp.DecodeString("REP 1")
|
||||
require.NoError(t, err)
|
||||
|
||||
var owner user.ID
|
||||
user.IDFromKey(&owner, k.PrivateKey.PublicKey)
|
||||
|
||||
var cnr container.Container
|
||||
cnr.Init()
|
||||
cnr.SetOwner(owner)
|
||||
cnr.SetBasicACL(acl.PublicRWExtended)
|
||||
cnr.SetPlacementPolicy(pp)
|
||||
|
||||
var prmContainerPut client.PrmContainerPut
|
||||
prmContainerPut.SetContainer(cnr)
|
||||
cnrPut, err := cli.ContainerPut(ctx, prmContainerPut)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Wait for container to persist.
|
||||
var ok bool
|
||||
for i := 0; i < 100; i++ {
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
var prmContainerGet client.PrmContainerGet
|
||||
prmContainerGet.SetContainer(cnrPut.ID())
|
||||
cnrGet, err := cli.ContainerGet(ctx, prmContainerGet)
|
||||
require.NoError(t, err)
|
||||
if apistatus.IsSuccessful(cnrGet.Status()) {
|
||||
ok = true
|
||||
break
|
||||
}
|
||||
}
|
||||
require.True(t, ok)
|
||||
|
||||
// 3. Put object
|
||||
|
||||
var sha checksum.Checksum
|
||||
checksum.Calculate(&sha, checksum.SHA256, payload)
|
||||
|
||||
var tz checksum.Checksum
|
||||
checksum.Calculate(&sha, checksum.TZ, payload)
|
||||
|
||||
var putHeader object.Object
|
||||
putHeader.SetContainerID(cnrPut.ID())
|
||||
putHeader.SetOwnerID(&owner)
|
||||
putHeader.SetPayloadChecksum(sha)
|
||||
putHeader.SetPayloadHomomorphicHash(tz)
|
||||
putHeader.SetPayloadSize(uint64(len(payload)))
|
||||
|
||||
err = object.CalculateAndSetID(&putHeader)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = object.CalculateAndSetSignature(k.PrivateKey, &putHeader)
|
||||
require.NoError(t, err)
|
||||
|
||||
objectWriter, err := cli.ObjectPutInit(ctx, client.PrmObjectPutInit{})
|
||||
require.NoError(t, err)
|
||||
|
||||
ok = objectWriter.WriteHeader(putHeader)
|
||||
require.True(t, ok)
|
||||
|
||||
ok = objectWriter.WritePayloadChunk(payload)
|
||||
require.NoError(t, err)
|
||||
|
||||
objectPutResponse, err := objectWriter.Close()
|
||||
require.NoError(t, err)
|
||||
require.True(t, apistatus.IsSuccessful(objectPutResponse.Status()))
|
||||
|
||||
// 4. Get object
|
||||
|
||||
var prmObjectGet client.PrmObjectGet
|
||||
prmObjectGet.ByID(objectPutResponse.StoredObjectID())
|
||||
prmObjectGet.FromContainer(cnrPut.ID())
|
||||
|
||||
var getHeader object.Object
|
||||
|
||||
objectReader, err := cli.ObjectGetInit(ctx, prmObjectGet)
|
||||
require.NoError(t, err)
|
||||
|
||||
ok = objectReader.ReadHeader(&getHeader)
|
||||
require.True(t, ok)
|
||||
spew.Dump(getHeader)
|
||||
|
||||
buf, err := io.ReadAll(objectReader)
|
||||
require.NoError(t, err)
|
||||
spew.Dump(buf)
|
||||
|
||||
st, err := objectReader.Close()
|
||||
require.NoError(t, err)
|
||||
spew.Dump(st)
|
||||
}
|
||||
92
compare/frostfs_pool_test.go
Normal file
92
compare/frostfs_pool_test.go
Normal file
|
|
@ -0,0 +1,92 @@
|
|||
package compare
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||
"github.com/davecgh/go-spew/spew"
|
||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestFrostFSPool(t *testing.T) {
|
||||
// 1. Initialize client
|
||||
|
||||
payloadReader := bytes.NewBufferString("Hello World")
|
||||
|
||||
ctx := context.Background()
|
||||
k, err := keys.NewPrivateKeyFromWIF("L32sMMkUcEfsWFdjoFNKAPtj7y6cfkcpHwG2pmTap68AmUt86ZTo")
|
||||
require.NoError(t, err)
|
||||
|
||||
var prmInit pool.InitParameters
|
||||
prmInit.SetKey(&k.PrivateKey)
|
||||
prmInit.AddNode(pool.NewNodeParam(1, "localhost:8080", 1))
|
||||
|
||||
cli, err := pool.NewPool(prmInit)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = cli.Dial(ctx)
|
||||
require.NoError(t, err)
|
||||
|
||||
// 2. Create bucket / container
|
||||
|
||||
var pp netmap.PlacementPolicy
|
||||
err = pp.DecodeString("REP 1")
|
||||
require.NoError(t, err)
|
||||
|
||||
var owner user.ID
|
||||
user.IDFromKey(&owner, k.PrivateKey.PublicKey)
|
||||
|
||||
var cnr container.Container
|
||||
cnr.Init()
|
||||
cnr.SetOwner(owner)
|
||||
cnr.SetBasicACL(acl.PublicRWExtended)
|
||||
cnr.SetPlacementPolicy(pp)
|
||||
|
||||
var prmContainerPut pool.PrmContainerPut
|
||||
prmContainerPut.SetContainer(cnr)
|
||||
cnrID, err := cli.PutContainer(ctx, prmContainerPut)
|
||||
require.NoError(t, err)
|
||||
|
||||
// 3. Put object
|
||||
|
||||
var putHeader object.Object
|
||||
putHeader.SetOwnerID(&owner)
|
||||
putHeader.SetContainerID(cnrID)
|
||||
|
||||
var prmObjectPut pool.PrmObjectPut
|
||||
prmObjectPut.SetHeader(putHeader)
|
||||
prmObjectPut.SetPayload(payloadReader)
|
||||
|
||||
objectID, err := cli.PutObject(ctx, prmObjectPut)
|
||||
require.NoError(t, err)
|
||||
|
||||
// 4. Get object
|
||||
|
||||
var addr oid.Address
|
||||
addr.SetContainer(cnrID)
|
||||
addr.SetObject(objectID)
|
||||
|
||||
var prmObjectGet pool.PrmObjectGet
|
||||
prmObjectGet.SetAddress(addr)
|
||||
|
||||
objectReader, err := cli.GetObject(ctx, prmObjectGet)
|
||||
require.NoError(t, err)
|
||||
|
||||
spew.Dump(objectReader.Header)
|
||||
|
||||
buf, err := io.ReadAll(objectReader.Payload)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, objectReader.Payload.Close())
|
||||
|
||||
spew.Dump(buf)
|
||||
}
|
||||
60
compare/minio_go_test.go
Normal file
60
compare/minio_go_test.go
Normal file
|
|
@ -0,0 +1,60 @@
|
|||
package compare
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
"github.com/davecgh/go-spew/spew"
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestMinioGo(t *testing.T) {
|
||||
// 1. Initialize client
|
||||
|
||||
payloadReader := bytes.NewBufferString("Hello World")
|
||||
ctx := context.Background()
|
||||
|
||||
cli, err := minio.New("127.0.0.1:9000", &minio.Options{ // do not pass scheme
|
||||
Creds: credentials.NewStaticV4(accessKeyID, secretKeyID, ""),
|
||||
Region: "",
|
||||
BucketLookup: minio.BucketLookupPath,
|
||||
})
|
||||
|
||||
core, err := minio.NewCore("http://127.0.0.1:9000", &minio.Options{
|
||||
Creds: credentials.NewStaticV4(accessKeyID, secretKeyID, ""),
|
||||
BucketLookup: minio.BucketLookupPath,
|
||||
})
|
||||
_ = core
|
||||
|
||||
// 2. Create bucket / container
|
||||
|
||||
err = cli.MakeBucket(ctx, bucketName, minio.MakeBucketOptions{
|
||||
Region: "",
|
||||
ObjectLocking: false,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// 3. Put object
|
||||
|
||||
respPutObject, err := cli.PutObject(ctx, bucketName, keyName, payloadReader, -1, minio.PutObjectOptions{})
|
||||
require.NoError(t, err)
|
||||
spew.Dump(respPutObject)
|
||||
|
||||
// 4. Get object
|
||||
|
||||
objectReader, err := cli.GetObject(ctx, bucketName, keyName, minio.GetObjectOptions{})
|
||||
require.NoError(t, err)
|
||||
|
||||
header, err := objectReader.Stat()
|
||||
require.NoError(t, err)
|
||||
spew.Dump(header)
|
||||
|
||||
buf, err := io.ReadAll(objectReader)
|
||||
require.NoError(t, err)
|
||||
spew.Dump(buf)
|
||||
require.NoError(t, objectReader.Close())
|
||||
}
|
||||
32
go.mod
32
go.mod
|
|
@ -9,8 +9,14 @@ require (
|
|||
git.frostfs.info/TrueCloudLab/hrw v1.2.1
|
||||
git.frostfs.info/TrueCloudLab/tzhash v1.8.0
|
||||
github.com/antlr4-go/antlr/v4 v4.13.0
|
||||
github.com/aws/aws-sdk-go-v2 v1.18.1
|
||||
github.com/aws/aws-sdk-go-v2/config v1.18.27
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.13.26
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.35.0
|
||||
github.com/davecgh/go-spew v1.1.1
|
||||
github.com/google/uuid v1.3.0
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.2
|
||||
github.com/minio/minio-go/v7 v7.0.57
|
||||
github.com/mr-tron/base58 v1.2.0
|
||||
github.com/nspcc-dev/neo-go v0.101.1
|
||||
github.com/stretchr/testify v1.8.3
|
||||
|
|
@ -21,16 +27,39 @@ require (
|
|||
|
||||
require (
|
||||
git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.34 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.28 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.35 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.26 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.29 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.28 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.12.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.19.2 // indirect
|
||||
github.com/aws/smithy-go v1.13.5 // indirect
|
||||
github.com/benbjohnson/clock v1.1.0 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/golang/protobuf v1.5.3 // indirect
|
||||
github.com/gorilla/websocket v1.5.0 // indirect
|
||||
github.com/hashicorp/golang-lru v0.6.0 // indirect
|
||||
github.com/json-iterator/go v1.1.12 // indirect
|
||||
github.com/klauspost/compress v1.16.5 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.4 // indirect
|
||||
github.com/minio/md5-simd v1.1.2 // indirect
|
||||
github.com/minio/sha256-simd v1.0.1 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
github.com/nspcc-dev/go-ordered-json v0.0.0-20220111165707-25110be27d22 // indirect
|
||||
github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20221202075445-cb5c18dc73eb // indirect
|
||||
github.com/nspcc-dev/rfc6979 v0.2.0 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/rs/xid v1.5.0 // indirect
|
||||
github.com/sirupsen/logrus v1.9.2 // indirect
|
||||
github.com/twmb/murmur3 v1.1.8 // indirect
|
||||
go.uber.org/atomic v1.9.0 // indirect
|
||||
go.uber.org/goleak v1.2.1 // indirect
|
||||
|
|
@ -42,5 +71,6 @@ require (
|
|||
golang.org/x/sys v0.8.0 // indirect
|
||||
golang.org/x/text v0.9.0 // indirect
|
||||
google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect
|
||||
gopkg.in/ini.v1 v1.67.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
)
|
||||
|
|
|
|||
BIN
go.sum
BIN
go.sum
Binary file not shown.
Loading…
Reference in a new issue