diff --git a/compare/aws_sdk_test.go b/compare/aws_sdk_test.go new file mode 100644 index 0000000..96ea93f --- /dev/null +++ b/compare/aws_sdk_test.go @@ -0,0 +1,99 @@ +package compare + +import ( + "bytes" + "context" + "io" + "strings" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/davecgh/go-spew/spew" + "github.com/stretchr/testify/require" +) + +var ( + accessKeyID = "9TtxuFepf2sibaAM" + secretKeyID = "SfxKtYws2We0uzsiXT011aWGMV4iTKlD" + bucketName = "test-bucket-5" + keyName = "foo" +) + +// 1. Initialize client +// 2. Create bucket / container +// 3. Put object +// 4. Get object + +// 1. Опции с публичными полями вместо структур с сеттерами +// 2. Credentials +// 3. Создавать контейнеры принимая аргументы с опциями контейнера, а не с самим контейнером, включая строковый Placement Policy +// 4. Сделать pool умнее путём раскладывания объектов в параллель / по плейсменту +// 5. Подумать о том можно ли как-то объеденить сахарные сценарии в pool и более прямые сценарии в SDK Client + +func TestAWSSDK(t *testing.T) { + // 1. Initialize client + + payloadReader := strings.NewReader("Hello World") // reader seeker + _ = payloadReader + + ctx := context.Background() + + cred := credentials.NewStaticCredentialsProvider(accessKeyID, secretKeyID, "") + cli := s3.New(s3.Options{ + Credentials: cred, + EndpointResolver: s3.EndpointResolverFromURL("http://127.0.0.1:9000"), + Retryer: aws.NopRetryer{}, + UsePathStyle: true, + }) + + cfg, err := config.LoadDefaultConfig(ctx) + require.NoError(t, err) + cli = s3.NewFromConfig(cfg, func(options *s3.Options) { + options.Retryer = aws.NopRetryer{} + options.UsePathStyle = true + options.EndpointResolver = s3.EndpointResolverFromURL("http://127.0.0.1:9000") + }) + + // 2. Create bucket / container + + respCreateBucket, err := cli.CreateBucket(ctx, &s3.CreateBucketInput{ + Bucket: aws.String(bucketName), + ACL: "public-read-write", + CreateBucketConfiguration: &types.CreateBucketConfiguration{ + LocationConstraint: "default", + }, + ObjectLockEnabledForBucket: false, + }) + require.NoError(t, err) + _ = respCreateBucket + + // 3. Put object + + bbuf := bytes.NewBufferString("lalala") + respPutObject, err := cli.PutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String(bucketName), + Key: aws.String(keyName), + Body: bbuf, + }) + require.NoError(t, err) + spew.Dump(respPutObject) + + // 4. Get object + + objectReader, err := cli.GetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String(bucketName), + Key: aws.String(keyName), + }) + require.NoError(t, err) + + spew.Dump(objectReader.ContentLength) + + buf, err := io.ReadAll(objectReader.Body) + require.NoError(t, err) + spew.Dump(buf) + require.NoError(t, objectReader.Body.Close()) +} diff --git a/compare/frostfs_client_test.go b/compare/frostfs_client_test.go new file mode 100644 index 0000000..e306272 --- /dev/null +++ b/compare/frostfs_client_test.go @@ -0,0 +1,134 @@ +package compare + +import ( + "context" + "io" + "testing" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" + apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" + "github.com/davecgh/go-spew/spew" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" + "github.com/stretchr/testify/require" +) + +func TestFrostFSClient(t *testing.T) { + // 1. Initialize client + + payload := []byte("Hello World") + + ctx := context.Background() + k, err := keys.NewPrivateKeyFromWIF("L32sMMkUcEfsWFdjoFNKAPtj7y6cfkcpHwG2pmTap68AmUt86ZTo") + require.NoError(t, err) + + var prmInit client.PrmInit + prmInit.SetDefaultPrivateKey(k.PrivateKey) + + var prmDial client.PrmDial + prmDial.SetServerURI("localhost:8080") + + var cli client.Client + cli.Init(prmInit) + + err = cli.Dial(ctx, prmDial) + require.NoError(t, err) + + // 2. Create bucket / container + + var pp netmap.PlacementPolicy + err = pp.DecodeString("REP 1") + require.NoError(t, err) + + var owner user.ID + user.IDFromKey(&owner, k.PrivateKey.PublicKey) + + var cnr container.Container + cnr.Init() + cnr.SetOwner(owner) + cnr.SetBasicACL(acl.PublicRWExtended) + cnr.SetPlacementPolicy(pp) + + var prmContainerPut client.PrmContainerPut + prmContainerPut.SetContainer(cnr) + cnrPut, err := cli.ContainerPut(ctx, prmContainerPut) + require.NoError(t, err) + + // Wait for container to persist. + var ok bool + for i := 0; i < 100; i++ { + time.Sleep(200 * time.Millisecond) + var prmContainerGet client.PrmContainerGet + prmContainerGet.SetContainer(cnrPut.ID()) + cnrGet, err := cli.ContainerGet(ctx, prmContainerGet) + require.NoError(t, err) + if apistatus.IsSuccessful(cnrGet.Status()) { + ok = true + break + } + } + require.True(t, ok) + + // 3. Put object + + var sha checksum.Checksum + checksum.Calculate(&sha, checksum.SHA256, payload) + + var tz checksum.Checksum + checksum.Calculate(&sha, checksum.TZ, payload) + + var putHeader object.Object + putHeader.SetContainerID(cnrPut.ID()) + putHeader.SetOwnerID(&owner) + putHeader.SetPayloadChecksum(sha) + putHeader.SetPayloadHomomorphicHash(tz) + putHeader.SetPayloadSize(uint64(len(payload))) + + err = object.CalculateAndSetID(&putHeader) + require.NoError(t, err) + + err = object.CalculateAndSetSignature(k.PrivateKey, &putHeader) + require.NoError(t, err) + + objectWriter, err := cli.ObjectPutInit(ctx, client.PrmObjectPutInit{}) + require.NoError(t, err) + + ok = objectWriter.WriteHeader(putHeader) + require.True(t, ok) + + ok = objectWriter.WritePayloadChunk(payload) + require.NoError(t, err) + + objectPutResponse, err := objectWriter.Close() + require.NoError(t, err) + require.True(t, apistatus.IsSuccessful(objectPutResponse.Status())) + + // 4. Get object + + var prmObjectGet client.PrmObjectGet + prmObjectGet.ByID(objectPutResponse.StoredObjectID()) + prmObjectGet.FromContainer(cnrPut.ID()) + + var getHeader object.Object + + objectReader, err := cli.ObjectGetInit(ctx, prmObjectGet) + require.NoError(t, err) + + ok = objectReader.ReadHeader(&getHeader) + require.True(t, ok) + spew.Dump(getHeader) + + buf, err := io.ReadAll(objectReader) + require.NoError(t, err) + spew.Dump(buf) + + st, err := objectReader.Close() + require.NoError(t, err) + spew.Dump(st) +} diff --git a/compare/frostfs_pool_test.go b/compare/frostfs_pool_test.go new file mode 100644 index 0000000..1285fb1 --- /dev/null +++ b/compare/frostfs_pool_test.go @@ -0,0 +1,92 @@ +package compare + +import ( + "bytes" + "context" + "io" + "testing" + + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" + "github.com/davecgh/go-spew/spew" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" + "github.com/stretchr/testify/require" +) + +func TestFrostFSPool(t *testing.T) { + // 1. Initialize client + + payloadReader := bytes.NewBufferString("Hello World") + + ctx := context.Background() + k, err := keys.NewPrivateKeyFromWIF("L32sMMkUcEfsWFdjoFNKAPtj7y6cfkcpHwG2pmTap68AmUt86ZTo") + require.NoError(t, err) + + var prmInit pool.InitParameters + prmInit.SetKey(&k.PrivateKey) + prmInit.AddNode(pool.NewNodeParam(1, "localhost:8080", 1)) + + cli, err := pool.NewPool(prmInit) + require.NoError(t, err) + + err = cli.Dial(ctx) + require.NoError(t, err) + + // 2. Create bucket / container + + var pp netmap.PlacementPolicy + err = pp.DecodeString("REP 1") + require.NoError(t, err) + + var owner user.ID + user.IDFromKey(&owner, k.PrivateKey.PublicKey) + + var cnr container.Container + cnr.Init() + cnr.SetOwner(owner) + cnr.SetBasicACL(acl.PublicRWExtended) + cnr.SetPlacementPolicy(pp) + + var prmContainerPut pool.PrmContainerPut + prmContainerPut.SetContainer(cnr) + cnrID, err := cli.PutContainer(ctx, prmContainerPut) + require.NoError(t, err) + + // 3. Put object + + var putHeader object.Object + putHeader.SetOwnerID(&owner) + putHeader.SetContainerID(cnrID) + + var prmObjectPut pool.PrmObjectPut + prmObjectPut.SetHeader(putHeader) + prmObjectPut.SetPayload(payloadReader) + + objectID, err := cli.PutObject(ctx, prmObjectPut) + require.NoError(t, err) + + // 4. Get object + + var addr oid.Address + addr.SetContainer(cnrID) + addr.SetObject(objectID) + + var prmObjectGet pool.PrmObjectGet + prmObjectGet.SetAddress(addr) + + objectReader, err := cli.GetObject(ctx, prmObjectGet) + require.NoError(t, err) + + spew.Dump(objectReader.Header) + + buf, err := io.ReadAll(objectReader.Payload) + require.NoError(t, err) + require.NoError(t, objectReader.Payload.Close()) + + spew.Dump(buf) +} diff --git a/compare/minio_go_test.go b/compare/minio_go_test.go new file mode 100644 index 0000000..5dcac79 --- /dev/null +++ b/compare/minio_go_test.go @@ -0,0 +1,60 @@ +package compare + +import ( + "bytes" + "context" + "io" + "testing" + + "github.com/davecgh/go-spew/spew" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/stretchr/testify/require" +) + +func TestMinioGo(t *testing.T) { + // 1. Initialize client + + payloadReader := bytes.NewBufferString("Hello World") + ctx := context.Background() + + cli, err := minio.New("127.0.0.1:9000", &minio.Options{ // do not pass scheme + Creds: credentials.NewStaticV4(accessKeyID, secretKeyID, ""), + Region: "", + BucketLookup: minio.BucketLookupPath, + }) + + core, err := minio.NewCore("http://127.0.0.1:9000", &minio.Options{ + Creds: credentials.NewStaticV4(accessKeyID, secretKeyID, ""), + BucketLookup: minio.BucketLookupPath, + }) + _ = core + + // 2. Create bucket / container + + err = cli.MakeBucket(ctx, bucketName, minio.MakeBucketOptions{ + Region: "", + ObjectLocking: false, + }) + require.NoError(t, err) + + // 3. Put object + + respPutObject, err := cli.PutObject(ctx, bucketName, keyName, payloadReader, -1, minio.PutObjectOptions{}) + require.NoError(t, err) + spew.Dump(respPutObject) + + // 4. Get object + + objectReader, err := cli.GetObject(ctx, bucketName, keyName, minio.GetObjectOptions{}) + require.NoError(t, err) + + header, err := objectReader.Stat() + require.NoError(t, err) + spew.Dump(header) + + buf, err := io.ReadAll(objectReader) + require.NoError(t, err) + spew.Dump(buf) + require.NoError(t, objectReader.Close()) +} diff --git a/go.mod b/go.mod index 89f6fe3..5602a33 100644 --- a/go.mod +++ b/go.mod @@ -9,8 +9,14 @@ require ( git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/tzhash v1.8.0 github.com/antlr4-go/antlr/v4 v4.13.0 + github.com/aws/aws-sdk-go-v2 v1.18.1 + github.com/aws/aws-sdk-go-v2/config v1.18.27 + github.com/aws/aws-sdk-go-v2/credentials v1.13.26 + github.com/aws/aws-sdk-go-v2/service/s3 v1.35.0 + github.com/davecgh/go-spew v1.1.1 github.com/google/uuid v1.3.0 github.com/hashicorp/golang-lru/v2 v2.0.2 + github.com/minio/minio-go/v7 v7.0.57 github.com/mr-tron/base58 v1.2.0 github.com/nspcc-dev/neo-go v0.101.1 github.com/stretchr/testify v1.8.3 @@ -21,16 +27,39 @@ require ( require ( git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.4 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.34 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.28 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.3.35 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.26 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.29 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.28 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.3 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.12.12 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.12 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.19.2 // indirect + github.com/aws/smithy-go v1.13.5 // indirect github.com/benbjohnson/clock v1.1.0 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/gorilla/websocket v1.5.0 // indirect github.com/hashicorp/golang-lru v0.6.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.16.5 // indirect + github.com/klauspost/cpuid/v2 v2.2.4 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/minio/sha256-simd v1.0.1 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect github.com/nspcc-dev/go-ordered-json v0.0.0-20220111165707-25110be27d22 // indirect github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20221202075445-cb5c18dc73eb // indirect github.com/nspcc-dev/rfc6979 v0.2.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/rs/xid v1.5.0 // indirect + github.com/sirupsen/logrus v1.9.2 // indirect github.com/twmb/murmur3 v1.1.8 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/goleak v1.2.1 // indirect @@ -42,5 +71,6 @@ require ( golang.org/x/sys v0.8.0 // indirect golang.org/x/text v0.9.0 // indirect google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index a409e2f..499e96d 100644 Binary files a/go.sum and b/go.sum differ