diff --git a/.gitignore b/.gitignore index 8e89bbf..e2f53be 100644 --- a/.gitignore +++ b/.gitignore @@ -1,24 +1,9 @@ -**/*.swp -cover.out -*~ -minio -!*/ -site/ -**/*.test -**/*.sublime-workspace -/.idea/ -/Minio.iml -**/access.log -vendor/**/*.js -vendor/**/*.json -release +# IDE +.idea +.vscode + +# Vendoring +vendor + +# tempfiles .DS_Store -*.syso -coverage.txt -.vscode/ -*.tar.bz2 -parts/ -prime/ -stage/ -.sia_temp/ -config.json diff --git a/.golangci.yml b/.golangci.yml deleted file mode 100644 index 891e90f..0000000 --- a/.golangci.yml +++ /dev/null @@ -1,27 +0,0 @@ -linters-settings: - golint: - min-confidence: 0 - - misspell: - locale: US - -linters: - disable-all: true - enable: - - typecheck - - goimports - - misspell - - govet - - golint - - ineffassign - - gosimple - - deadcode - - structcheck - -issues: - exclude-use-default: false - exclude: - - should have a package comment - - error strings should not be capitalized or end with punctuation or a newline -service: - golangci-lint-version: 1.20.0 # use the fixed version to not introduce new linters unexpectedly diff --git a/.mailmap b/.mailmap deleted file mode 100644 index 4adf1be..0000000 --- a/.mailmap +++ /dev/null @@ -1,18 +0,0 @@ -# Generate CONTRIBUTORS.md: contributors.sh - -# Tip for finding duplicates (besides scanning the output of CONTRIBUTORS.md for name -# duplicates that aren't also email duplicates): scan the output of: -# git log --format='%aE - %aN' | sort -uf -# -# For explanation on this file format: man git-shortlog - -Anand Babu (AB) Periasamy Anand Babu (AB) Periasamy -Anand Babu (AB) Periasamy -Anis Elleuch -Frederick F. Kautz IV -Harshavardhana -Harshavardhana -Harshavardhana -Krishna Srinivas -Matthew Farrellee -Nate Rosenblum \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index a05293f..5e091ab 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,36 +1,30 @@ -FROM golang:1.13-alpine +FROM golang:1 as builder -LABEL maintainer="MinIO Inc " +COPY . /src -ENV GOPATH /go -ENV CGO_ENABLED 0 -ENV GO111MODULE on +WORKDIR /src -RUN \ - apk add --no-cache git && \ - git clone https://github.com/minio/minio && cd minio && \ - go install -v -ldflags "$(go run buildscripts/gen-ldflags.go)" +ARG VERSION=dev +# https://github.com/golang/go/wiki/Modules#how-do-i-use-vendoring-with-modules-is-vendoring-going-away +# go build -mod=vendor +# The -gcflags "all=-N -l" flag helps us get a better debug experience +RUN set -x \ + && export BUILD=$(date -u +%s%N) \ + && export REPO=$(go list -m) \ + && export LDFLAGS="-X ${REPO}/misc.Version=${VERSION} -X ${REPO}/misc.Build=${BUILD}" \ + && export GOGC=off \ + && export CGO_ENABLED=0 \ + && [ -d "./vendor" ] || go mod vendor \ + && go build -v -mod=vendor -trimpath -gcflags "all=-N -l" -ldflags "${LDFLAGS}" -o /go/bin/neofs-s3 ./main.go + +# Executable image FROM alpine:3.10 -ENV MINIO_UPDATE off -ENV MINIO_ACCESS_KEY_FILE=access_key \ - MINIO_SECRET_KEY_FILE=secret_key \ - MINIO_KMS_MASTER_KEY_FILE=kms_master_key \ - MINIO_SSE_MASTER_KEY_FILE=sse_master_key +WORKDIR / -EXPOSE 9000 +COPY --from=builder /go/bin/neofs-s3 /usr/bin/neofs-s3 +COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ -COPY --from=0 /go/bin/minio /usr/bin/minio -COPY --from=0 /go/minio/CREDITS /third_party/ -COPY --from=0 /go/minio/dockerscripts/docker-entrypoint.sh /usr/bin/ - -RUN \ - apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' && \ - echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf - -ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"] - -VOLUME ["/data"] - -CMD ["minio"] +# Run delve +CMD ["/usr/bin/neofs-s3"] diff --git a/Dockerfile.arm.release b/Dockerfile.arm.release deleted file mode 100644 index 3e1f641..0000000 --- a/Dockerfile.arm.release +++ /dev/null @@ -1,41 +0,0 @@ -FROM golang:1.13-alpine as builder - -WORKDIR /home - -ENV GOPATH /go -ENV CGO_ENABLED 0 -ENV GO111MODULE on - -RUN \ - apk add --no-cache git 'curl>7.61.0' && \ - git clone https://github.com/minio/minio && \ - curl -L https://github.com/balena-io/qemu/releases/download/v3.0.0%2Bresin/qemu-3.0.0+resin-arm.tar.gz | tar zxvf - -C . && mv qemu-3.0.0+resin-arm/qemu-arm-static . - -FROM arm32v7/alpine:3.10 - -LABEL maintainer="MinIO Inc " - -COPY dockerscripts/docker-entrypoint.sh /usr/bin/ -COPY CREDITS /third_party/ -COPY --from=builder /home/qemu-arm-static /usr/bin/qemu-arm-static - -ENV MINIO_UPDATE off -ENV MINIO_ACCESS_KEY_FILE=access_key \ - MINIO_SECRET_KEY_FILE=secret_key \ - MINIO_KMS_MASTER_KEY_FILE=kms_master_key \ - MINIO_SSE_MASTER_KEY_FILE=sse_master_key - -RUN \ - apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' && \ - echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \ - curl https://dl.min.io/server/minio/release/linux-arm/minio > /usr/bin/minio && \ - chmod +x /usr/bin/minio && \ - chmod +x /usr/bin/docker-entrypoint.sh - -EXPOSE 9000 - -ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"] - -VOLUME ["/data"] - -CMD ["minio"] diff --git a/Dockerfile.arm64.release b/Dockerfile.arm64.release deleted file mode 100644 index 7118868..0000000 --- a/Dockerfile.arm64.release +++ /dev/null @@ -1,41 +0,0 @@ -FROM golang:1.13-alpine as builder - -WORKDIR /home - -ENV GOPATH /go -ENV CGO_ENABLED 0 -ENV GO111MODULE on - -RUN \ - apk add --no-cache git 'curl>7.61.0' && \ - git clone https://github.com/minio/minio && \ - curl -L https://github.com/balena-io/qemu/releases/download/v3.0.0%2Bresin/qemu-3.0.0+resin-arm.tar.gz | tar zxvf - -C . && mv qemu-3.0.0+resin-arm/qemu-arm-static . - -FROM arm64v8/alpine:3.10 - -LABEL maintainer="MinIO Inc " - -COPY dockerscripts/docker-entrypoint.sh /usr/bin/ -COPY CREDITS /third_party/ -COPY --from=builder /home/qemu-arm-static /usr/bin/qemu-arm-static - -ENV MINIO_UPDATE off -ENV MINIO_ACCESS_KEY_FILE=access_key \ - MINIO_SECRET_KEY_FILE=secret_key \ - MINIO_KMS_MASTER_KEY_FILE=kms_master_key \ - MINIO_SSE_MASTER_KEY_FILE=sse_master_key - -RUN \ - apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' && \ - echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \ - curl https://dl.min.io/server/minio/release/linux-arm64/minio > /usr/bin/minio && \ - chmod +x /usr/bin/minio && \ - chmod +x /usr/bin/docker-entrypoint.sh - -EXPOSE 9000 - -ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"] - -VOLUME ["/data"] - -CMD ["minio"] diff --git a/Dockerfile.dev b/Dockerfile.dev deleted file mode 100644 index 6790efb..0000000 --- a/Dockerfile.dev +++ /dev/null @@ -1,27 +0,0 @@ -FROM alpine:3.10 - -LABEL maintainer="MinIO Inc " - -COPY dockerscripts/docker-entrypoint.sh /usr/bin/ -COPY minio /usr/bin/ -COPY CREDITS /third_party/ - -ENV MINIO_UPDATE off -ENV MINIO_ACCESS_KEY_FILE=access_key \ - MINIO_SECRET_KEY_FILE=secret_key \ - MINIO_KMS_MASTER_KEY_FILE=kms_master_key \ - MINIO_SSE_MASTER_KEY_FILE=sse_master_key - -RUN \ - apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' && \ - echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \ - chmod +x /usr/bin/minio && \ - chmod +x /usr/bin/docker-entrypoint.sh - -EXPOSE 9000 - -ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"] - -VOLUME ["/data"] - -CMD ["minio"] diff --git a/Dockerfile.dev.browser b/Dockerfile.dev.browser deleted file mode 100644 index da38e59..0000000 --- a/Dockerfile.dev.browser +++ /dev/null @@ -1,13 +0,0 @@ -FROM ubuntu - -LABEL maintainer="MinIO Inc " - -RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends --no-install-suggests \ - git golang make npm && \ - apt-get clean && rm -rf /var/lib/apt/lists/* - -ENV PATH=$PATH:/root/go/bin - -RUN go get github.com/go-bindata/go-bindata/go-bindata && \ - go get github.com/elazarl/go-bindata-assetfs/go-bindata-assetfs - diff --git a/Dockerfile.mint b/Dockerfile.mint deleted file mode 100644 index 6daffba..0000000 --- a/Dockerfile.mint +++ /dev/null @@ -1,17 +0,0 @@ -FROM ubuntu:18.04 - -ENV DEBIAN_FRONTEND noninteractive -ENV LANG C.UTF-8 -ENV GOROOT /usr/local/go -ENV GOPATH /usr/local/gopath -ENV PATH $GOPATH/bin:$GOROOT/bin:$PATH -ENV MINT_ROOT_DIR /mint -COPY mint /mint - -RUN apt-get --yes update && apt-get --yes upgrade && \ - apt-get --yes --quiet install wget jq curl git dnsmasq && \ - cd /mint && /mint/release.sh - -WORKDIR /mint - -ENTRYPOINT ["/mint/entrypoint.sh"] diff --git a/Dockerfile.release b/Dockerfile.release deleted file mode 100644 index 1dd41bf..0000000 --- a/Dockerfile.release +++ /dev/null @@ -1,37 +0,0 @@ -FROM golang:1.13-alpine - -ENV GOPATH /go -ENV CGO_ENABLED 0 -ENV GO111MODULE on - -RUN \ - apk add --no-cache git && \ - git clone https://github.com/minio/minio - -FROM alpine:3.10 - -LABEL maintainer="MinIO Inc " - -COPY dockerscripts/docker-entrypoint.sh /usr/bin/ -COPY CREDITS /third_party/ - -ENV MINIO_UPDATE off -ENV MINIO_ACCESS_KEY_FILE=access_key \ - MINIO_SECRET_KEY_FILE=secret_key \ - MINIO_KMS_MASTER_KEY_FILE=kms_master_key \ - MINIO_SSE_MASTER_KEY_FILE=sse_master_key - -RUN \ - apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' && \ - echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \ - curl https://dl.min.io/server/minio/release/linux-amd64/minio > /usr/bin/minio && \ - chmod +x /usr/bin/minio && \ - chmod +x /usr/bin/docker-entrypoint.sh - -EXPOSE 9000 - -ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"] - -VOLUME ["/data"] - -CMD ["minio"] diff --git a/Makefile b/Makefile index 4f6f3f6..887150b 100644 --- a/Makefile +++ b/Makefile @@ -1,79 +1,39 @@ -PWD := $(shell pwd) -GOPATH := $(shell go env GOPATH) -LDFLAGS := $(shell go run buildscripts/gen-ldflags.go) +VERSION ?= "$(shell git describe --tags 2>/dev/null | sed 's/^v//')" +BUILD_VERSION ?= "$(shell git describe --abbrev=0 --tags | sed 's/^v//')" -GOARCH := $(shell go env GOARCH) -GOOS := $(shell go env GOOS) +.PHONY: help format deps -VERSION ?= $(shell git describe --tags) -TAG ?= "minio/minio:$(VERSION)" +# Show this help prompt +help: + @echo ' Usage:' + @echo '' + @echo ' make ' + @echo '' + @echo ' Targets:' + @echo '' + @awk '/^#/{ comment = substr($$0,3) } comment && /^[a-zA-Z][a-zA-Z0-9_-]+ ?:/{ print " ", $$1, comment }' $(MAKEFILE_LIST) | column -t -s ':' | grep -v 'IGNORE' | sort | uniq -all: build +# Reformat code +format: + @[ ! -z `which goimports` ] || (echo "install goimports" && exit 2) + @for f in `find . -type f -name '*.go' -not -path './vendor/*' -not -name '*.pb.go' -prune`; do \ + echo "⇒ Processing $$f"; \ + goimports -w $$f; \ + done -checks: - @echo "Checking dependencies" - @(env bash $(PWD)/buildscripts/checkdeps.sh) +# Make sure that all files added to commit +deps: + @printf "⇒ Ensure vendor: " + @go mod tidy -v && echo OK || (echo fail && exit 2) + @printf "⇒ Download requirements: " + @go mod download && echo OK || (echo fail && exit 2) + @printf "⇒ Store vendor localy: " + @go mod vendor && echo OK || (echo fail && exit 2) -getdeps: - @mkdir -p ${GOPATH}/bin - @which golangci-lint 1>/dev/null || (echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOPATH)/bin v1.27.0) - -crosscompile: - @(env bash $(PWD)/buildscripts/cross-compile.sh) - -verifiers: getdeps fmt lint - -fmt: - @echo "Running $@ check" - @GO111MODULE=on gofmt -d cmd/ - @GO111MODULE=on gofmt -d pkg/ - -lint: - @echo "Running $@ check" - @GO111MODULE=on ${GOPATH}/bin/golangci-lint cache clean - @GO111MODULE=on ${GOPATH}/bin/golangci-lint run --timeout=5m --config ./.golangci.yml - -# Builds minio, runs the verifiers then runs the tests. -check: test -test: verifiers build - @echo "Running unit tests" - @GO111MODULE=on CGO_ENABLED=0 go test -tags kqueue ./... 1>/dev/null - -test-race: verifiers build - @echo "Running unit tests under -race" - @(env bash $(PWD)/buildscripts/race.sh) - -# Verify minio binary -verify: - @echo "Verifying build with race" - @GO111MODULE=on CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null - @(env bash $(PWD)/buildscripts/verify-build.sh) - -# Verify healing of disks with minio binary -verify-healing: - @echo "Verify healing build with race" - @GO111MODULE=on CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null - @(env bash $(PWD)/buildscripts/verify-healing.sh) - -# Builds minio locally. -build: checks - @echo "Building minio binary to './minio'" - @GO111MODULE=on CGO_ENABLED=0 go build -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null - -docker: build - @docker build -t $(TAG) . -f Dockerfile.dev - -# Builds minio and installs it to $GOPATH/bin. -install: build - @echo "Installing minio binary to '$(GOPATH)/bin/minio'" - @mkdir -p $(GOPATH)/bin && cp -f $(PWD)/minio $(GOPATH)/bin/minio - @echo "Installation successful. To learn more, try \"minio --help\"." - -clean: - @echo "Cleaning up all the generated files" - @find . -name '*.test' | xargs rm -fv - @find . -name '*~' | xargs rm -fv - @rm -rvf minio - @rm -rvf build - @rm -rvf release - @rm -rvf .verify* +# Build current docker image +image-build: + @echo "⇒ Build docker-image" + @docker build \ + --build-arg VERSION=$(VERSION) \ + -f Dockerfile \ + -t nspccdev/neofs-s3-gate:$(VERSION) . \ No newline at end of file diff --git a/cmd/web-handlers.go b/cmd/web-handlers.go index 93af710..ab768ee 100644 --- a/cmd/web-handlers.go +++ b/cmd/web-handlers.go @@ -37,7 +37,6 @@ import ( "github.com/klauspost/compress/zip" miniogopolicy "github.com/minio/minio-go/v6/pkg/policy" "github.com/minio/minio-go/v6/pkg/s3utils" - "github.com/minio/minio/browser" "github.com/minio/minio/cmd/config/etcd/dns" "github.com/minio/minio/cmd/config/identity/openid" "github.com/minio/minio/cmd/crypto" @@ -108,7 +107,7 @@ func (web *webAPIHandlers) ServerInfo(r *http.Request, args *WebGenericArgs, rep reply.MinioPlatform = platform reply.MinioRuntime = goruntime - reply.UIVersion = browser.UIVersion + reply.UIVersion = "browser.UIVersion" return nil } @@ -131,7 +130,7 @@ func (web *webAPIHandlers) StorageInfo(r *http.Request, args *WebGenericArgs, re } dataUsageInfo, _ := loadDataUsageFromBackend(ctx, objectAPI) reply.Used = dataUsageInfo.ObjectsTotalSize - reply.UIVersion = browser.UIVersion + reply.UIVersion = "browser.UIVersion" return nil } @@ -181,7 +180,7 @@ func (web *webAPIHandlers) MakeBucket(r *http.Request, args *MakeBucketArgs, rep return toJSONError(ctx, err) } - reply.UIVersion = browser.UIVersion + reply.UIVersion = "browser.UIVersion" return nil } return toJSONError(ctx, err) @@ -193,7 +192,7 @@ func (web *webAPIHandlers) MakeBucket(r *http.Request, args *MakeBucketArgs, rep return toJSONError(ctx, err, args.BucketName) } - reply.UIVersion = browser.UIVersion + reply.UIVersion = "browser.UIVersion" return nil } @@ -231,7 +230,7 @@ func (web *webAPIHandlers) DeleteBucket(r *http.Request, args *RemoveBucketArgs, return toJSONError(ctx, errInvalidBucketName) } - reply.UIVersion = browser.UIVersion + reply.UIVersion = "browser.UIVersion" if isRemoteCallRequired(ctx, args.BucketName, objectAPI) { sr, err := globalDNSConfig.Get(args.BucketName) @@ -351,7 +350,7 @@ func (web *webAPIHandlers) ListBuckets(r *http.Request, args *WebGenericArgs, re } } - reply.UIVersion = browser.UIVersion + reply.UIVersion = "browser.UIVersion" return nil } @@ -384,7 +383,7 @@ type WebObjectInfo struct { // ListObjects - list objects api. func (web *webAPIHandlers) ListObjects(r *http.Request, args *ListObjectsArgs, reply *ListObjectsRep) error { ctx := newWebContext(r, args, "WebListObjects") - reply.UIVersion = browser.UIVersion + reply.UIVersion = "browser.UIVersion" objectAPI := web.ObjectAPI() if objectAPI == nil { return toJSONError(ctx, errServerNotInitialized) @@ -622,7 +621,7 @@ func (web *webAPIHandlers) RemoveObject(r *http.Request, args *RemoveObjectArgs, return toJSONError(ctx, errInvalidBucketName) } - reply.UIVersion = browser.UIVersion + reply.UIVersion = "browser.UIVersion" if isRemoteCallRequired(ctx, args.BucketName, objectAPI) { sr, err := globalDNSConfig.Get(args.BucketName) if err != nil { @@ -813,7 +812,7 @@ func (web *webAPIHandlers) Login(r *http.Request, args *LoginArgs, reply *LoginR } reply.Token = token - reply.UIVersion = browser.UIVersion + reply.UIVersion = "browser.UIVersion" return nil } @@ -839,7 +838,7 @@ func (web webAPIHandlers) GenerateAuth(r *http.Request, args *WebGenericArgs, re } reply.AccessKey = cred.AccessKey reply.SecretKey = cred.SecretKey - reply.UIVersion = browser.UIVersion + reply.UIVersion = "browser.UIVersion" return nil } @@ -898,7 +897,7 @@ func (web *webAPIHandlers) SetAuth(r *http.Request, args *SetAuthArgs, reply *Se return toJSONError(ctx, err) } - reply.UIVersion = browser.UIVersion + reply.UIVersion = "browser.UIVersion" return nil } @@ -937,7 +936,7 @@ func (web *webAPIHandlers) CreateURLToken(r *http.Request, args *WebGenericArgs, reply.Token = token } - reply.UIVersion = browser.UIVersion + reply.UIVersion = "browser.UIVersion" return nil } @@ -1672,7 +1671,7 @@ func (web *webAPIHandlers) GetBucketPolicy(r *http.Request, args *GetBucketPolic } } - reply.UIVersion = browser.UIVersion + reply.UIVersion = "browser.UIVersion" reply.Policy = miniogopolicy.GetPolicy(policyInfo.Statements, args.BucketName, args.Prefix) return nil @@ -1764,7 +1763,7 @@ func (web *webAPIHandlers) ListAllBucketPolicies(r *http.Request, args *ListAllB } } - reply.UIVersion = browser.UIVersion + reply.UIVersion = "browser.UIVersion" for prefix, policy := range miniogopolicy.GetPolicies(policyInfo.Statements, args.BucketName, "") { bucketName, objectPrefix := path2BucketObject(prefix) objectPrefix = strings.TrimSuffix(objectPrefix, "*") @@ -1789,7 +1788,7 @@ type SetBucketPolicyWebArgs struct { func (web *webAPIHandlers) SetBucketPolicy(r *http.Request, args *SetBucketPolicyWebArgs, reply *WebGenericRep) error { ctx := newWebContext(r, args, "WebSetBucketPolicy") objectAPI := web.ObjectAPI() - reply.UIVersion = browser.UIVersion + reply.UIVersion = "browser.UIVersion" if objectAPI == nil { return toJSONError(ctx, errServerNotInitialized) @@ -1983,7 +1982,7 @@ func (web *webAPIHandlers) PresignedGet(r *http.Request, args *PresignedGetArgs, return toJSONError(ctx, errPresignedNotAllowed) } - reply.UIVersion = browser.UIVersion + reply.UIVersion = "browser.UIVersion" reply.URL = presignedGet(args.HostName, args.BucketName, args.ObjectName, args.Expiry, creds, region) return nil } @@ -2040,7 +2039,7 @@ func (web *webAPIHandlers) GetDiscoveryDoc(r *http.Request, args *WebGenericArgs reply.DiscoveryDoc = globalOpenIDConfig.DiscoveryDoc reply.ClientID = globalOpenIDConfig.ClientID } - reply.UIVersion = browser.UIVersion + reply.UIVersion = "browser.UIVersion" return nil } @@ -2099,7 +2098,7 @@ func (web *webAPIHandlers) LoginSTS(r *http.Request, args *LoginSTSArgs, reply * } reply.Token = a.Result.Credentials.SessionToken - reply.UIVersion = browser.UIVersion + reply.UIVersion = "browser.UIVersion" return nil } diff --git a/cmd/web-router.go b/cmd/web-router.go index 6e94246..02bd822 100644 --- a/cmd/web-router.go +++ b/cmd/web-router.go @@ -17,15 +17,16 @@ package cmd import ( + "errors" "fmt" "net/http" + "os" assetfs "github.com/elazarl/go-bindata-assetfs" "github.com/gorilla/handlers" "github.com/gorilla/mux" jsonrpc "github.com/gorilla/rpc/v2" "github.com/gorilla/rpc/v2/json2" - "github.com/minio/minio/browser" ) // webAPI container for Web API. @@ -48,9 +49,9 @@ const assetPrefix = "production" func assetFS() *assetfs.AssetFS { return &assetfs.AssetFS{ - Asset: browser.Asset, - AssetDir: browser.AssetDir, - AssetInfo: browser.AssetInfo, + Asset: func(path string) ([]byte, error) { return nil, errors.New("no ui") }, + AssetDir: func(path string) ([]string, error) { return nil, errors.New("no ui") }, + AssetInfo: func(path string) (os.FileInfo, error) { return nil, errors.New("no ui") }, Prefix: assetPrefix, } } diff --git a/go.mod b/go.mod index a5a67a9..164aebc 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,6 @@ require ( github.com/Shopify/sarama v1.24.1 github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect github.com/alecthomas/participle v0.2.1 - github.com/aws/aws-sdk-go v1.20.21 github.com/bcicen/jstream v0.0.0-20190220045926-16c1f8af81c2 github.com/beevik/ntp v0.2.0 github.com/cespare/xxhash/v2 v2.1.1 @@ -88,7 +87,6 @@ require ( github.com/rs/cors v1.7.0 github.com/secure-io/sio-go v0.3.0 github.com/shirou/gopsutil v2.20.3-0.20200314133625-53cec6b37e6a+incompatible - github.com/sirupsen/logrus v1.5.0 github.com/smartystreets/assertions v0.0.0-20190401211740-f487f9de1cd3 // indirect github.com/spf13/viper v1.7.0 github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94 diff --git a/go.sum b/go.sum index 6b7a866..d5e4457 100644 --- a/go.sum +++ b/go.sum @@ -52,8 +52,6 @@ github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQh github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/awalterschulze/gographviz v0.0.0-20181013152038-b2885df04310 h1:t+qxRrRtwNiUYA+Xh2jSXhoG2grnMCMKX4Fg6lx9X1U= github.com/awalterschulze/gographviz v0.0.0-20181013152038-b2885df04310/go.mod h1:GEV5wmg4YquNw7v1kkyoX9etIk8yVmXj+AkDHuuETHs= -github.com/aws/aws-sdk-go v1.20.21 h1:22vHWL9rur+SRTYPHAXlxJMFIA9OSYsYDIAHFDhQ7Z0= -github.com/aws/aws-sdk-go v1.20.21/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/bcicen/jstream v0.0.0-20190220045926-16c1f8af81c2 h1:M+TYzBcNIRyzPRg66ndEqUMd7oWDmhvdQmaPC6EZNwM= github.com/bcicen/jstream v0.0.0-20190220045926-16c1f8af81c2/go.mod h1:RDu/qcrnpEdJC/p8tx34+YBFqqX71lB7dOX9QE+ZC4M= github.com/beevik/ntp v0.2.0 h1:sGsd+kAXzT0bfVfzJfce04g+dSRfrs+tbQW8lweuYgw= @@ -284,8 +282,6 @@ github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf/go.mod h github.com/jcmturner/gofork v0.0.0-20180107083740-2aebee971930/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= github.com/jcmturner/gofork v0.0.0-20190328161633-dc7c13fece03 h1:FUwcHNlEqkqLjLBdCp5PRlCFijNjvcYANOZXzCfXwCM= github.com/jcmturner/gofork v0.0.0-20190328161633-dc7c13fece03/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= -github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= -github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= diff --git a/main.go b/main.go index dbbe3cc..e66a243 100644 --- a/main.go +++ b/main.go @@ -25,14 +25,14 @@ package main // import "github.com/minio/minio" import ( - "os" - - minio "github.com/minio/minio/cmd" + "fmt" + "github.com/minio/minio/misc" // Import gateway - _ "github.com/minio/minio/cmd/gateway" + // _ "github.com/minio/minio/cmd/gateway" ) func main() { - minio.Main(os.Args) + fmt.Println(misc.Build) + fmt.Println(misc.Version) } diff --git a/mint/.gitignore b/mint/.gitignore deleted file mode 100644 index ef999ea..0000000 --- a/mint/.gitignore +++ /dev/null @@ -1,17 +0,0 @@ -*.test -*.jar -src/* -temp -__pycache__/ -log/* -minio.test -bin/* -node_modules -# exception to the rule -!log/.gitkeep -!bin/.gitkeep -*.class -*~ -run/core/minio-dotnet/bin/* -run/core/minio-dotnet/obj/* -run/core/minio-dotnet/out/* diff --git a/mint/README.md b/mint/README.md deleted file mode 100644 index 4a33eec..0000000 --- a/mint/README.md +++ /dev/null @@ -1,119 +0,0 @@ -# Mint [![Slack](https://slack.minio.io/slack?type=svg)](https://slack.minio.io) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/mint.svg?maxAge=604800)](https://hub.docker.com/r/minio/mint/) - -Mint is a testing framework for Minio object server, available as a docker image. It runs correctness, benchmarking and stress tests. Following are the SDKs/tools used in correctness tests. - -- awscli -- aws-sdk-go -- aws-sdk-php -- aws-sdk-ruby -- aws-sdk-java -- mc -- minio-go -- minio-java -- minio-js -- minio-py -- minio-dotnet -- s3cmd - -## Running Mint - -Mint is run by `docker run` command which requires Docker to be installed. For Docker installation follow the steps [here](https://docs.docker.com/engine/installation/linux/docker-ce/ubuntu/). - -To run Mint with Minio Play server as test target, - -```sh -$ docker run -e SERVER_ENDPOINT=play.minio.io:9000 -e ACCESS_KEY=Q3AM3UQ867SPQQA43P2F \ - -e SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG -e ENABLE_HTTPS=1 minio/mint -``` - -After the tests are run, output is stored in `/mint/log` directory inside the container. To get these logs, use `docker cp` command. For example -```sh -docker cp :/mint/log /tmp/logs -``` - -### Mint environment variables - -Below environment variables are required to be passed to the docker container. Supported environment variables: - -| Environment variable | Description | Example | -|:--- |:--- |:--- | -| `SERVER_ENDPOINT` | Endpoint of Minio server in the format `HOST:PORT`; for virtual style `IP:PORT` | `play.minio.io:9000` | -| `ACCESS_KEY` | Access key of access `SERVER_ENDPOINT` | `Q3AM3UQ867SPQQA43P2F` | -| `SECRET_KEY` | Secret Key of access `SERVER_ENDPOINT` | `zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG` | -| `ENABLE_HTTPS` | (Optional) Set `1` to indicate to use HTTPS to access `SERVER_ENDPOINT`. Defaults to `0` (HTTP) | `1` | -| `MINT_MODE` | (Optional) Set mode indicating what category of tests to be run by values `core`, `full`. Defaults to `core` | `full` | -| `DOMAIN` | (Optional) Value of MINIO_DOMAIN environment variable used in Minio server | `myminio.com` | -| `ENABLE_VIRTUAL_STYLE` | (Optional) Set `1` to indicate virtual style access . Defaults to `0` (Path style) | `1` | - - -### Test virtual style access against Minio server - -To test Minio server virtual style access with Mint, follow these steps: - -- Set a domain in your Minio server using environment variable MINIO_DOMAIN. For example `export MINIO_DOMAIN=myminio.com`. -- Start Minio server. -- Execute Mint against Minio server (with `MINIO_DOMAIN` set to `myminio.com`) using this command -```sh -$ docker run -e "SERVER_ENDPOINT=192.168.86.133:9000" -e "DOMAIN=minio.com" \ - -e "ACCESS_KEY=minio" -e "SECRET_KEY=minio123" -e "ENABLE_HTTPS=0" \ - -e "ENABLE_VIRTUAL_STYLE=1" minio/mint -``` - -### Mint log format - -All test logs are stored in `/mint/log/log.json` as multiple JSON document. Below is the JSON format for every entry in the log file. - -| JSON field | Type | Description | Example | -|:--- |:--- |:--- |:--- | -| `name` | _string_ | Testing tool/SDK name | `"aws-sdk-php"` | -| `function` | _string_ | Test function name | `"getBucketLocation ( array $params = [] )"` | -| `args` | _object_ | (Optional) Key/Value map of arguments passed to test function | `{"Bucket":"aws-sdk-php-bucket-20341"}` | -| `duration` | _int_ | Time taken in milliseconds to run the test | `384` | -| `status` | _string_ | one of `PASS`, `FAIL` or `NA` | `"PASS"` | -| `alert` | _string_ | (Optional) Alert message indicating test failure | `"I/O error on create file"` | -| `message` | _string_ | (Optional) Any log message | `"validating checksum of downloaded object"` | -| `error` | _string_ | Detailed error message including stack trace on status `FAIL` | `"Error executing \"CompleteMultipartUpload\" on ...` | - -## For Developers - -### Running Mint development code - -After making changes to Mint source code a local docker image can be built/run by - -```sh -$ docker build -t minio/mint . -f Dockerfile.mint -$ docker run -e SERVER_ENDPOINT=play.minio.io:9000 -e ACCESS_KEY=Q3AM3UQ867SPQQA43P2F \ - -e SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG \ - -e ENABLE_HTTPS=1 -e MINT_MODE=full minio/mint:latest -``` - - -### Adding tests with new tool/SDK - -Below are the steps need to be followed - -- Create new app directory under [build](https://github.com/minio/mint/tree/master/build) and [run/core](https://github.com/minio/mint/tree/master/run/core) directories. -- Create `install.sh` which does installation of required tool/SDK under app directory. -- Any build and install time dependencies should be added to [install-packages.list](https://github.com/minio/mint/blob/master/install-packages.list). -- Build time dependencies should be added to [remove-packages.list](https://github.com/minio/mint/blob/master/remove-packages.list) for removal to have clean Mint docker image. -- Add `run.sh` in app directory under `run/core` which execute actual tests. - -#### Test data -Tests may use pre-created data set to perform various object operations on Minio server. Below data files are available under `/mint/data` directory. - -| File name | Size | -|:--- |:--- | -| datafile-0-b | 0B | -| datafile-1-b | 1B | -| datafile-1-kB |1KiB | -| datafile-10-kB |10KiB | -| datafile-33-kB |33KiB | -| datafile-100-kB |100KiB | -| datafile-1-MB |1MiB | -| datafile-1.03-MB |1.03MiB | -| datafile-5-MB |5MiB | -| datafile-6-MB |6MiB | -| datafile-10-MB |10MiB | -| datafile-11-MB |11MiB | -| datafile-65-MB |65MiB | -| datafile-129-MB |129MiB | diff --git a/mint/build/aws-sdk-go/install.sh b/mint/build/aws-sdk-go/install.sh deleted file mode 100755 index d731628..0000000 --- a/mint/build/aws-sdk-go/install.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash -e -# -# Mint (C) 2017 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -test_run_dir="$MINT_RUN_CORE_DIR/aws-sdk-go" -GO111MODULE=on go build -o "$test_run_dir/aws-sdk-go" "$test_run_dir/quick-tests.go" diff --git a/mint/build/aws-sdk-java/build.xml b/mint/build/aws-sdk-java/build.xml deleted file mode 100644 index 8a1f521..0000000 --- a/mint/build/aws-sdk-java/build.xml +++ /dev/null @@ -1,58 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/mint/build/aws-sdk-java/install.sh b/mint/build/aws-sdk-java/install.sh deleted file mode 100755 index c3206d4..0000000 --- a/mint/build/aws-sdk-java/install.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -e -# -# Mint (C) 2017 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -test_run_dir="$MINT_RUN_CORE_DIR/aws-sdk-java" - -cd "$(dirname "$(realpath "$0")")" - -ant init-ivy && \ - ant resolve && \ - ant compile && \ - ant jar - -cp build/jar/FunctionalTests.jar "$test_run_dir/" - -rm -rf lib/ build/ - diff --git a/mint/build/aws-sdk-java/ivy.xml b/mint/build/aws-sdk-java/ivy.xml deleted file mode 100644 index e61fd0d..0000000 --- a/mint/build/aws-sdk-java/ivy.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - diff --git a/mint/build/aws-sdk-java/src/FunctionalTests.java b/mint/build/aws-sdk-java/src/FunctionalTests.java deleted file mode 100644 index 2005a78..0000000 --- a/mint/build/aws-sdk-java/src/FunctionalTests.java +++ /dev/null @@ -1,632 +0,0 @@ -/* -* Mint, (C) 2018 Minio, Inc. -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - -package io.minio.awssdk.tests; - -import java.io.*; -import java.security.NoSuchAlgorithmException; -import java.security.SecureRandom; - -import java.security.*; -import java.util.*; - -import java.nio.file.*; -import java.math.BigInteger; - -import javax.crypto.KeyGenerator; -import javax.crypto.SecretKey; -import javax.crypto.spec.SecretKeySpec; - -import com.amazonaws.auth.AWSCredentials; -import com.amazonaws.auth.BasicAWSCredentials; -import com.amazonaws.services.s3.AmazonS3ClientBuilder; -import com.amazonaws.AmazonClientException; -import com.amazonaws.AmazonServiceException; -import com.amazonaws.auth.profile.ProfileCredentialsProvider; -import com.amazonaws.auth.AWSStaticCredentialsProvider; -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.AmazonS3Client; -import com.amazonaws.services.s3.model.CreateBucketRequest; -import com.amazonaws.services.s3.model.ObjectListing; -import com.amazonaws.services.s3.model.S3ObjectSummary; -import com.amazonaws.services.s3.model.SSECustomerKey; - -// Main Testing class -public class FunctionalTests { - - private static final String PASS = "PASS"; - private static final String FAILED = "FAIL"; - private static final String IGNORED = "NA"; - - private static String accessKey; - private static String secretKey; - private static String region; - private static String endpoint; - private static boolean enableHTTPS; - - private static final Random random = new Random(new SecureRandom().nextLong()); - private static String bucketName = getRandomName(); - private static boolean mintEnv = false; - - private static String file1Kb; - private static String file1Mb; - private static String file6Mb; - - private static SSECustomerKey sseKey1; - private static SSECustomerKey sseKey2; - private static SSECustomerKey sseKey3; - - private static AmazonS3 s3Client; - private static S3TestUtils s3TestUtils; - - public static String getRandomName() { - return "aws-java-sdk-test-" + new BigInteger(32, random).toString(32); - } - - /** - * Prints a success log entry in JSON format. - */ - public static void mintSuccessLog(String function, String args, long startTime) { - if (mintEnv) { - System.out.println( - new MintLogger(function, args, System.currentTimeMillis() - startTime, PASS, null, null, null)); - } - } - - /** - * Prints a failure log entry in JSON format. - */ - public static void mintFailedLog(String function, String args, long startTime, String message, String error) { - if (mintEnv) { - System.out.println(new MintLogger(function, args, System.currentTimeMillis() - startTime, FAILED, null, - message, error)); - } - } - - /** - * Prints a ignore log entry in JSON format. - */ - public static void mintIgnoredLog(String function, String args, long startTime) { - if (mintEnv) { - System.out.println( - new MintLogger(function, args, System.currentTimeMillis() - startTime, IGNORED, null, null, null)); - } - } - - public static void initTests() throws IOException { - // Create encryption key. - byte[] rawKey1 = "32byteslongsecretkeymustgenerate".getBytes(); - SecretKey secretKey1 = new SecretKeySpec(rawKey1, 0, rawKey1.length, "AES"); - sseKey1 = new SSECustomerKey(secretKey1); - - // Create new encryption key for target so it is saved using sse-c - byte[] rawKey2 = "xxbytescopysecretkeymustprovided".getBytes(); - SecretKey secretKey2 = new SecretKeySpec(rawKey2, 0, rawKey2.length, "AES"); - sseKey2 = new SSECustomerKey(secretKey2); - - // Create new encryption key for target so it is saved using sse-c - byte[] rawKey3 = "32byteslongsecretkeymustgenerat1".getBytes(); - SecretKey secretKey3 = new SecretKeySpec(rawKey3, 0, rawKey3.length, "AES"); - sseKey3 = new SSECustomerKey(secretKey3); - - // Create bucket - s3Client.createBucket(new CreateBucketRequest(bucketName)); - } - - public static void teardown() throws IOException { - - // Remove all objects under the test bucket & the bucket itself - // TODO: use multi delete API instead - ObjectListing objectListing = s3Client.listObjects(bucketName); - while (true) { - for (Iterator iterator = objectListing.getObjectSummaries().iterator(); iterator.hasNext();) { - S3ObjectSummary summary = (S3ObjectSummary) iterator.next(); - s3Client.deleteObject(bucketName, summary.getKey()); - } - // more objectListing to retrieve? - if (objectListing.isTruncated()) { - objectListing = s3Client.listNextBatchOfObjects(objectListing); - } else { - break; - } - } - ; - s3Client.deleteBucket(bucketName); - } - - // Test regular object upload using encryption - public static void uploadObjectEncryption_test1() throws Exception { - if (!mintEnv) { - System.out.println( - "Test: uploadObject(String bucketName, String objectName, String f, SSECustomerKey sseKey)"); - } - - if (!enableHTTPS) { - return; - } - - long startTime = System.currentTimeMillis(); - String file1KbMD5 = Utils.getFileMD5(file1Kb); - String objectName = "testobject"; - try { - s3TestUtils.uploadObject(bucketName, objectName, file1Kb, sseKey1); - s3TestUtils.downloadObject(bucketName, objectName, sseKey1, file1KbMD5); - mintSuccessLog("uploadObject(String bucketName, String objectName, String f, SSECustomerKey sseKey)", - "bucketName: " + bucketName + ", objectName: " + objectName + ", String: " + file1Kb - + ", SSECustomerKey: " + sseKey1, - startTime); - } catch (Exception e) { - mintFailedLog("uploadObject(String bucketName, String objectName, String f, SSECustomerKey sseKey)", - "bucketName: " + bucketName + ", objectName: " + objectName + ", String: " + file1Kb - + ", SSECustomerKey: " + sseKey1, - startTime, null, e.toString() + " >>> " + Arrays.toString(e.getStackTrace())); - throw e; - } - } - - // Test downloading an object with a wrong encryption key - public static void downloadObjectEncryption_test1() throws Exception { - if (!mintEnv) { - System.out.println("Test: downloadObject(String bucketName, String objectName, SSECustomerKey sseKey)"); - } - - if (!enableHTTPS) { - return; - } - - long startTime = System.currentTimeMillis(); - - String file1KbMD5 = Utils.getFileMD5(file1Kb); - String objectName = "testobject"; - - try { - s3TestUtils.uploadObject(bucketName, "testobject", file1Kb, sseKey1); - s3TestUtils.downloadObject(bucketName, objectName, sseKey2); - Exception ex = new Exception("downloadObject did not throw an S3 Access denied exception"); - mintFailedLog("downloadObject(String bucketName, String objectName, SSECustomerKey sseKey)", - "bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey2, - startTime, null, ex.toString() + " >>> " + Arrays.toString(ex.getStackTrace())); - throw ex; - } catch (Exception e) { - if (!e.getMessage().contains("Access Denied")) { - Exception ex = new Exception( - "downloadObject did not throw S3 Access denied Exception but it did throw: " + e.getMessage()); - mintFailedLog("downloadObject(String bucketName, String objectName, SSECustomerKey sseKey)", - "bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey2, - startTime, null, ex.toString() + " >>> " + Arrays.toString(ex.getStackTrace())); - throw ex; - } - mintSuccessLog("downloadObject(String bucketName, String objectName, SSECustomerKey sseKey)", - "bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey2, - startTime); - } - } - - // Test copying object with a new different encryption key - public static void copyObjectEncryption_test1() throws Exception { - if (!mintEnv) { - System.out.println("Test: copyObject(String bucketName, String objectName, SSECustomerKey sseKey, " - + "String destBucketName, String dstObjectName, SSECustomerKey sseKey2, boolean replaceDirective)"); - } - - if (!enableHTTPS) { - return; - } - - long startTime = System.currentTimeMillis(); - String file1KbMD5 = Utils.getFileMD5(file1Kb); - String objectName = "testobject"; - String dstObjectName = "dir/newobject"; - - try { - s3TestUtils.uploadObject(bucketName, objectName, file1Kb, sseKey1); - s3TestUtils.copyObject(bucketName, objectName, sseKey1, bucketName, dstObjectName, sseKey2, false); - s3TestUtils.downloadObject(bucketName, dstObjectName, sseKey2, file1KbMD5); - } catch (Exception e) { - mintFailedLog("copyObject(String bucketName, String objectName, SSECustomerKey sseKey, " - + "String destBucketName, String dstObjectName, SSECustomerKey sseKey2, boolean replaceDirective)", - "bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1 - + "DstbucketName: " + bucketName + ", DstObjectName: " + dstObjectName - + ", SSECustomerKey: " + sseKey2 + ", replaceDirective: " + false, - startTime, null, e.toString() + " >>> " + Arrays.toString(e.getStackTrace())); - throw e; - } - mintSuccessLog("copyObject(String bucketName, String objectName, SSECustomerKey sseKey, " - + "String destBucketName, String dstObjectName, SSECustomerKey sseKey2, boolean replaceDirective)", - "bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1 - + "DstbucketName: " + bucketName + ", DstObjectName: " + dstObjectName + ", SSECustomerKey: " - + sseKey2 + ", replaceDirective: " + false, - startTime); - } - - // Test copying object with wrong source encryption key - public static void copyObjectEncryption_test2() throws Exception { - if (!mintEnv) { - System.out.println("Test: copyObject(String bucketName, String objectName, SSECustomerKey sseKey, " - + "String destBucketName, String dstObjectName, SSECustomerKey sseKey2, boolean replaceDirective)"); - } - - if (!enableHTTPS) { - return; - } - - String objectName = "testobject"; - String dstObjectName = "dir/newobject"; - - long startTime = System.currentTimeMillis(); - - try { - s3TestUtils.copyObject(bucketName, objectName, sseKey3, bucketName, dstObjectName, sseKey2, false); - Exception ex = new Exception("copyObject did not throw an S3 Access denied exception"); - mintFailedLog("copyObject(String bucketName, String objectName, SSECustomerKey sseKey, " - + "String destBucketName, String dstObjectName, SSECustomerKey sseKey2, boolean replaceDirective)", - "bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey3 - + "DstbucketName: " + bucketName + ", DstObjectName: " + dstObjectName - + ", SSECustomerKey: " + sseKey2 + ", replaceDirective: " + false, - startTime, null, ex.toString() + " >>> " + Arrays.toString(ex.getStackTrace())); - throw ex; - } catch (Exception e) { - if (!e.getMessage().contains("Access Denied")) { - Exception ex = new Exception( - "copyObject did not throw S3 Access denied Exception but it did throw: " + e.getMessage()); - mintFailedLog("copyObject(String bucketName, String objectName, SSECustomerKey sseKey, " - + "String destBucketName, String dstObjectName, SSECustomerKey sseKey2, boolean replaceDirective)", - "bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey3 - + "DstbucketName: " + bucketName + ", DstObjectName: " + dstObjectName - + ", SSECustomerKey: " + sseKey2 + ", replaceDirective: " + false, - startTime, null, ex.toString() + " >>> " + Arrays.toString(ex.getStackTrace())); - throw ex; - } - mintSuccessLog("copyObject(String bucketName, String objectName, SSECustomerKey sseKey, " - + "String destBucketName, String dstObjectName, SSECustomerKey sseKey2, boolean replaceDirective)", - "bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey3 - + "DstbucketName: " + bucketName + ", DstObjectName: " + dstObjectName - + ", SSECustomerKey: " + sseKey2 + ", replaceDirective: " + false, - startTime); - } - } - - // Test copying multipart object - public static void copyObjectEncryption_test3() throws Exception { - if (!mintEnv) { - System.out.println("Test: copyObject(String bucketName, String objectName, SSECustomerKey sseKey, " - + "String destBucketName, String dstObjectName, SSECustomerKey sseKey2, boolean replaceDirective)"); - } - - if (!enableHTTPS) { - return; - } - - long startTime = System.currentTimeMillis(); - String file6MbMD5 = Utils.getFileMD5(file6Mb); - String objectName = "testobject"; - String dstObjectName = "dir/newobject"; - - try { - s3TestUtils.uploadMultipartObject(bucketName, objectName, file6Mb, sseKey1); - s3TestUtils.copyObject(bucketName, objectName, sseKey1, bucketName, dstObjectName, sseKey2, false); - s3TestUtils.downloadObject(bucketName, dstObjectName, sseKey2, file6MbMD5); - } catch (Exception e) { - mintFailedLog("copyObject(String bucketName, String objectName, SSECustomerKey sseKey, " - + "String destBucketName, String dstObjectName, SSECustomerKey sseKey2, boolean replaceDirective)", - "bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1 - + "DstbucketName: " + bucketName + ", DstObjectName: " + dstObjectName - + ", SSECustomerKey: " + sseKey2 + ", replaceDirective: " + false, - startTime, null, e.toString() + " >>> " + Arrays.toString(e.getStackTrace())); - throw e; - } - mintSuccessLog("copyObject(String bucketName, String objectName, SSECustomerKey sseKey, " - + "String destBucketName, String dstObjectName, SSECustomerKey sseKey2, boolean replaceDirective)", - "bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1 - + "DstbucketName: " + bucketName + ", DstObjectName: " + dstObjectName + ", SSECustomerKey: " - + sseKey2 + ", replaceDirective: " + false, - startTime); - } - - // Test downloading encrypted object with Get Range, 0 -> 1024 - public static void downloadGetRangeEncryption_test1() throws Exception { - if (!mintEnv) { - System.out.println("Test: downloadObjectGetRange(String bucketName, String objectName, " - + "SSECustomerKey sseKey, String expectedMD5, int start, int length)"); - } - - if (!enableHTTPS) { - return; - } - - long startTime = System.currentTimeMillis(); - - String objectName = "testobject"; - String range1MD5 = Utils.getFileMD5(file1Kb); - int start = 0; - int length = 1024; - try { - s3TestUtils.uploadObject(bucketName, objectName, file1Kb, sseKey1); - s3TestUtils.downloadObject(bucketName, objectName, sseKey1, range1MD5, start, length); - } catch (Exception e) { - mintFailedLog( - "downloadObjectGetRange(String bucketName, String objectName, " - + "SSECustomerKey sseKey, String expectedMD5, int start, int length)", - "bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1 - + ", expectedMD5: " + range1MD5 + ", start: " + start + ", length: " + length, - startTime, null, e.toString() + " >>> " + Arrays.toString(e.getStackTrace())); - throw e; - } - mintSuccessLog( - "downloadObjectGetRange(String bucketName, String objectName, " - + "SSECustomerKey sseKey, String expectedMD5, int start, int length)", - "bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1 - + ", expectedMD5: " + range1MD5 + ", start: " + start + ", length: " + length, - startTime); - } - - // Test downloading encrypted object with Get Range, 0 -> 1 - public static void downloadGetRangeEncryption_test2() throws Exception { - if (!mintEnv) { - System.out.println("Test: downloadObjectGetRange(String bucketName, String objectName, " - + "SSECustomerKey sseKey, String expectedMD5, int start, int length)"); - } - - if (!enableHTTPS) { - return; - } - - long startTime = System.currentTimeMillis(); - - String objectName = "testobject"; - int start = 0; - int length = 1; - String range1MD5 = Utils.getFileMD5(file1Kb, start, length); - try { - s3TestUtils.uploadObject(bucketName, objectName, file1Kb, sseKey1); - s3TestUtils.downloadObject(bucketName, objectName, sseKey1, range1MD5, start, length); - } catch (Exception e) { - mintFailedLog( - "downloadObjectGetRange(String bucketName, String objectName, " - + "SSECustomerKey sseKey, String expectedMD5, int start, int length)", - "bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1 - + ", expectedMD5: " + range1MD5 + ", start: " + start + ", length: " + length, - startTime, null, e.toString() + " >>> " + Arrays.toString(e.getStackTrace())); - throw e; - } - mintSuccessLog( - "downloadObjectGetRange(String bucketName, String objectName, " - + "SSECustomerKey sseKey, String expectedMD5, int start, int length)", - "bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1 - + ", expectedMD5: " + range1MD5 + ", start: " + start + ", length: " + length, - startTime); - } - - // Test downloading encrypted object with Get Range, 0 -> 1024-1 - public static void downloadGetRangeEncryption_test3() throws Exception { - if (!mintEnv) { - System.out.println("Test: downloadObjectGetRange(String bucketName, String objectName, " - + "SSECustomerKey sseKey, String expectedMD5, int start, int length)"); - } - - if (!enableHTTPS) { - return; - } - - long startTime = System.currentTimeMillis(); - - String objectName = "testobject"; - int start = 0; - int length = 1023; - String range1MD5 = Utils.getFileMD5(file1Kb, start, length); - try { - s3TestUtils.uploadObject(bucketName, objectName, file1Kb, sseKey1); - s3TestUtils.downloadObject(bucketName, objectName, sseKey1, range1MD5, start, length); - } catch (Exception e) { - mintFailedLog( - "downloadObjectGetRange(String bucketName, String objectName, " - + "SSECustomerKey sseKey, String expectedMD5, int start, int length)", - "bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1 - + ", expectedMD5: " + range1MD5 + ", start: " + start + ", length: " + length, - startTime, null, e.toString() + " >>> " + Arrays.toString(e.getStackTrace())); - throw e; - } - mintSuccessLog( - "downloadObjectGetRange(String bucketName, String objectName, " - + "SSECustomerKey sseKey, String expectedMD5, int start, int length)", - "bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1 - + ", expectedMD5: " + range1MD5 + ", start: " + start + ", length: " + length, - startTime); - } - - // Test downloading encrypted object with Get Range, 1 -> 1024-1 - public static void downloadGetRangeEncryption_test4() throws Exception { - if (!mintEnv) { - System.out.println("Test: downloadObjectGetRange(String bucketName, String objectName, " - + "SSECustomerKey sseKey, String expectedMD5, int start, int length)"); - } - - if (!enableHTTPS) { - return; - } - - long startTime = System.currentTimeMillis(); - - String objectName = "testobject"; - int start = 1; - int length = 1023; - String range1MD5 = Utils.getFileMD5(file1Kb, start, length); - try { - s3TestUtils.uploadObject(bucketName, objectName, file1Kb, sseKey1); - s3TestUtils.downloadObject(bucketName, objectName, sseKey1, range1MD5, start, length); - } catch (Exception e) { - mintFailedLog( - "downloadObjectGetRange(String bucketName, String objectName, " - + "SSECustomerKey sseKey, String expectedMD5, int start, int length)", - "bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1 - + ", expectedMD5: " + range1MD5 + ", start: " + start + ", length: " + length, - startTime, null, e.toString() + " >>> " + Arrays.toString(e.getStackTrace())); - throw e; - } - mintSuccessLog( - "downloadObjectGetRange(String bucketName, String objectName, " - + "SSECustomerKey sseKey, String expectedMD5, int start, int length)", - "bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1 - + ", expectedMD5: " + range1MD5 + ", start: " + start + ", length: " + length, - startTime); - } - - // Test downloading encrypted object with Get Range, 64*1024 -> 64*1024 - public static void downloadGetRangeEncryption_test5() throws Exception { - if (!mintEnv) { - System.out.println("Test: downloadObjectGetRange(String bucketName, String objectName, " - + "SSECustomerKey sseKey, String expectedMD5, int start, int length)"); - } - - if (!enableHTTPS) { - return; - } - - long startTime = System.currentTimeMillis(); - - String objectName = "testobject"; - int start = 64 * 1024; - int length = 64 * 1024; - String range1MD5 = Utils.getFileMD5(file1Mb, start, length); - try { - s3TestUtils.uploadObject(bucketName, objectName, file1Mb, sseKey1); - s3TestUtils.downloadObject(bucketName, objectName, sseKey1, range1MD5, start, length); - } catch (Exception e) { - mintFailedLog( - "downloadObjectGetRange(String bucketName, String objectName, " - + "SSECustomerKey sseKey, String expectedMD5, int start, int length)", - "bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1 - + ", expectedMD5: " + range1MD5 + ", start: " + start + ", length: " + length, - startTime, null, e.toString() + " >>> " + Arrays.toString(e.getStackTrace())); - throw e; - } - mintSuccessLog( - "downloadObjectGetRange(String bucketName, String objectName, " - + "SSECustomerKey sseKey, String expectedMD5, int start, int length)", - "bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1 - + ", expectedMD5: " + range1MD5 + ", start: " + start + ", length: " + length, - startTime); - } - - // Test downloading encrypted object with Get Range, 64*1024 -> - // 1024*1024-64*1024 - public static void downloadGetRangeEncryption_test6() throws Exception { - if (!mintEnv) { - System.out.println("Test: downloadObjectGetRange(String bucketName, String objectName, " - + "SSECustomerKey sseKey, String expectedMD5, int start, int length)"); - } - - if (!enableHTTPS) { - return; - } - - long startTime = System.currentTimeMillis(); - - String objectName = "testobject"; - int start = 64 * 1024; - int length = 1024 * 1024 - 64 * 1024; - String range1MD5 = Utils.getFileMD5(file1Mb, start, length); - try { - s3TestUtils.uploadObject(bucketName, objectName, file1Mb, sseKey1); - s3TestUtils.downloadObject(bucketName, objectName, sseKey1, range1MD5, start, length); - } catch (Exception e) { - mintFailedLog( - "downloadObjectGetRange(String bucketName, String objectName, " - + "SSECustomerKey sseKey, String expectedMD5, int start, int length)", - "bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1 - + ", expectedMD5: " + range1MD5 + ", start: " + start + ", length: " + length, - startTime, null, e.toString() + " >>> " + Arrays.toString(e.getStackTrace())); - throw e; - } - mintSuccessLog( - "downloadObjectGetRange(String bucketName, String objectName, " - + "SSECustomerKey sseKey, String expectedMD5, int start, int length)", - "bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1 - + ", expectedMD5: " + range1MD5 + ", start: " + start + ", length: " + length, - startTime); - } - - // Run tests - public static void runTests() throws Exception { - - uploadObjectEncryption_test1(); - - downloadObjectEncryption_test1(); - - copyObjectEncryption_test1(); - copyObjectEncryption_test2(); - copyObjectEncryption_test3(); - - downloadGetRangeEncryption_test1(); - downloadGetRangeEncryption_test2(); - downloadGetRangeEncryption_test3(); - downloadGetRangeEncryption_test4(); - downloadGetRangeEncryption_test5(); - downloadGetRangeEncryption_test6(); - } - - public static void main(String[] args) throws Exception, IOException, NoSuchAlgorithmException { - - endpoint = System.getenv("SERVER_ENDPOINT"); - accessKey = System.getenv("ACCESS_KEY"); - secretKey = System.getenv("SECRET_KEY"); - enableHTTPS = System.getenv("ENABLE_HTTPS").equals("1"); - - region = "us-east-1"; - - if (enableHTTPS) { - endpoint = "https://" + endpoint; - } else { - endpoint = "http://" + endpoint; - } - - String dataDir = System.getenv("MINT_DATA_DIR"); - if (dataDir != null && !dataDir.equals("")) { - mintEnv = true; - file1Kb = Paths.get(dataDir, "datafile-1-kB").toString(); - file1Mb = Paths.get(dataDir, "datafile-1-MB").toString(); - file6Mb = Paths.get(dataDir, "datafile-6-MB").toString(); - } - - String mintMode = null; - if (mintEnv) { - mintMode = System.getenv("MINT_MODE"); - } - - AWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey); - AmazonS3ClientBuilder.EndpointConfiguration endpointConfiguration = new AmazonS3ClientBuilder.EndpointConfiguration( - endpoint, region); - - AmazonS3ClientBuilder clientBuilder = AmazonS3ClientBuilder.standard(); - clientBuilder.setCredentials(new AWSStaticCredentialsProvider(credentials)); - clientBuilder.setEndpointConfiguration(endpointConfiguration); - clientBuilder.setPathStyleAccessEnabled(true); - - s3Client = clientBuilder.build(); - s3TestUtils = new S3TestUtils(s3Client); - - try { - initTests(); - FunctionalTests.runTests(); - } catch (Exception e) { - e.printStackTrace(); - System.exit(-1); - } finally { - teardown(); - } - } -} diff --git a/mint/build/aws-sdk-java/src/LimitedInputStream.java b/mint/build/aws-sdk-java/src/LimitedInputStream.java deleted file mode 100644 index 310a174..0000000 --- a/mint/build/aws-sdk-java/src/LimitedInputStream.java +++ /dev/null @@ -1,60 +0,0 @@ -/* -* Mint, (C) 2018 Minio, Inc. -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - -package io.minio.awssdk.tests; - -import java.io.*; - -// LimitedInputStream wraps a regular InputStream, calling -// read() will skip some bytes as configured and will also -// return only data with configured length - -class LimitedInputStream extends InputStream { - - private int skip; - private int length; - private InputStream is; - - LimitedInputStream(InputStream is, int skip, int length) { - this.is = is; - this.skip = skip; - this.length = length; - } - - @Override - public int read() throws IOException { - int r; - while (skip > 0) { - r = is.read(); - if (r < 0) { - throw new IOException("stream ended before being able to skip all bytes"); - } - skip--; - } - if (length == 0) { - return -1; - } - r = is.read(); - if (r < 0) { - throw new IOException("stream ended before being able to read all bytes"); - } - length--; - return r; - } -} - - diff --git a/mint/build/aws-sdk-java/src/MintLogger.java b/mint/build/aws-sdk-java/src/MintLogger.java deleted file mode 100755 index b6319f0..0000000 --- a/mint/build/aws-sdk-java/src/MintLogger.java +++ /dev/null @@ -1,151 +0,0 @@ -/* - * Minio Java SDK for Amazon S3 Compatible Cloud Storage, - * (C) 2015, 2016, 2017, 2018 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.minio.awssdk.tests; - -import com.fasterxml.jackson.annotation.JsonAutoDetect; -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonInclude.Include; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; - -@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY) -public class MintLogger { - - @JsonProperty("name") - private String name; - - @JsonProperty("function") - private String function; - - @JsonProperty("args") - private String args; - - @JsonProperty("duration") - private long duration; - - @JsonProperty("status") - private String status; - - @JsonProperty("alert") - private String alert; - - @JsonProperty("message") - private String message; - - @JsonProperty("error") - private String error; - - /** - * Constructor. - **/ - public MintLogger(String function, - String args, - long duration, - String status, - String alert, - String message, - String error) { - this.name = "aws-sdk-java"; - this.function = function; - this.duration = duration; - this.args = args; - this.status = status; - this.alert = alert; - this.message = message; - this.error = error; - } - - /** - * Return JSON Log Entry. - **/ - @JsonIgnore - public String toString() { - - try { - return new ObjectMapper().setSerializationInclusion(Include.NON_NULL).writeValueAsString(this); - } catch (JsonProcessingException e) { - e.printStackTrace(); - } - return ""; - } - - /** - * Return Alert. - **/ - @JsonIgnore - public String alert() { - return alert; - } - - /** - * Return Error. - **/ - @JsonIgnore - public String error() { - return error; - } - - /** - * Return Message. - **/ - @JsonIgnore - public String message() { - return message; - } - - /** - * Return args. - **/ - @JsonIgnore - public String args() { - return args; - } - - /** - * Return status. - **/ - @JsonIgnore - public String status() { - return status; - } - - /** - * Return name. - **/ - @JsonIgnore - public String name() { - return name; - } - - /** - * Return function. - **/ - @JsonIgnore - public String function() { - return function; - } - - /** - * Return duration. - **/ - @JsonIgnore - public long duration() { - return duration; - } -} diff --git a/mint/build/aws-sdk-java/src/S3TestUtils.java b/mint/build/aws-sdk-java/src/S3TestUtils.java deleted file mode 100644 index 236caf9..0000000 --- a/mint/build/aws-sdk-java/src/S3TestUtils.java +++ /dev/null @@ -1,187 +0,0 @@ -/* -* Mint, (C) 2018 Minio, Inc. -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - -package io.minio.awssdk.tests; - -import java.io.*; -import java.util.*; -import java.nio.channels.Channels; - -import com.amazonaws.services.s3.model.GetObjectMetadataRequest; -import com.amazonaws.services.s3.model.GetObjectRequest; -import com.amazonaws.services.s3.model.ObjectMetadata; -import com.amazonaws.services.s3.model.PutObjectRequest; -import com.amazonaws.services.s3.model.CopyObjectRequest; -import com.amazonaws.services.s3.model.S3Object; -import com.amazonaws.services.s3.model.S3ObjectInputStream; -import com.amazonaws.services.s3.model.SSECustomerKey; - -import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest; -import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest; -import com.amazonaws.services.s3.model.InitiateMultipartUploadResult; -import com.amazonaws.services.s3.model.PartETag; -import com.amazonaws.services.s3.model.UploadPartRequest; - -import com.amazonaws.services.s3.model.MetadataDirective; - -import com.amazonaws.services.s3.AmazonS3; - -class S3TestUtils { - - private AmazonS3 s3Client; - - S3TestUtils(AmazonS3 s3Client) { - this.s3Client = s3Client; - } - - void uploadMultipartObject(String bucketName, String keyName, - String filePath, SSECustomerKey sseKey) throws IOException { - - File file = new File(filePath); - - List partETags = new ArrayList(); - - // Step 1: Initialize. - InitiateMultipartUploadRequest initRequest = new - InitiateMultipartUploadRequest(bucketName, keyName); - - if (sseKey != null) { - initRequest.setSSECustomerKey(sseKey); - } - - InitiateMultipartUploadResult initResponse = - s3Client.initiateMultipartUpload(initRequest); - - long contentLength = file.length(); - long partSize = 5242880; // Set part size to 5 MB. - - // Step 2: Upload parts. - long filePosition = 0; - for (int i = 1; filePosition < contentLength; i++) { - // Last part can be less than 5 MB. Adjust part size. - partSize = Math.min(partSize, (contentLength - filePosition)); - - // Create request to upload a part. - UploadPartRequest uploadRequest = new UploadPartRequest() - .withBucketName(bucketName).withKey(keyName) - .withUploadId(initResponse.getUploadId()).withPartNumber(i) - .withFileOffset(filePosition) - .withFile(file) - .withPartSize(partSize); - - if (sseKey != null) { - uploadRequest.withSSECustomerKey(sseKey); - } - - // Upload part and add response to our list. - partETags.add(s3Client.uploadPart(uploadRequest).getPartETag()); - - filePosition += partSize; - } - - // Step 3: Complete. - CompleteMultipartUploadRequest compRequest = new - CompleteMultipartUploadRequest( - bucketName, - keyName, - initResponse.getUploadId(), - partETags); - - s3Client.completeMultipartUpload(compRequest); - } - - void uploadObject(String bucketName, String keyName, - String filePath, SSECustomerKey sseKey) throws IOException { - - File f = new File(filePath); - PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, keyName, f); - if (sseKey != null) { - putObjectRequest.withSSECustomerKey(sseKey); - } - s3Client.putObject(putObjectRequest); - } - - void downloadObject(String bucketName, String keyName, SSECustomerKey sseKey) - throws Exception, IOException { - downloadObject(bucketName, keyName, sseKey, "", -1, -1); - } - - void downloadObject(String bucketName, String keyName, SSECustomerKey sseKey, - String expectedMD5) - throws Exception, IOException { - downloadObject(bucketName, keyName, sseKey, expectedMD5, -1, -1); - } - - void downloadObject(String bucketName, String keyName, SSECustomerKey sseKey, - String expectedMD5, int start, int length) throws Exception, IOException { - GetObjectRequest getObjectRequest = new GetObjectRequest(bucketName, keyName) - .withSSECustomerKey(sseKey); - - if (start >= 0 && length >= 0) { - getObjectRequest.setRange(start, start+length-1); - } - - S3Object s3Object = s3Client.getObject(getObjectRequest); - - int size = 0; - int c; - - S3ObjectInputStream input = s3Object.getObjectContent(); - - ByteArrayOutputStream output = new ByteArrayOutputStream(); - String data = ""; - while ((c = input.read()) != -1) { - output.write((byte) c); - size++; - } - - if (length >= 0 && size != length) { - throw new Exception("downloaded object has unexpected size, expected: " + length + ", received: " + size); - } - - String calculatedMD5 = Utils.getBufferMD5(output.toByteArray()); - - if (!expectedMD5.equals("") && !calculatedMD5.equals(expectedMD5)) { - throw new Exception("downloaded object has unexpected md5sum, expected: " + expectedMD5 + ", found: " + calculatedMD5); - - } - } - - void copyObject(String bucketName, String keyName, SSECustomerKey sseKey, - String targetBucketName, String targetKeyName, SSECustomerKey newSseKey, - boolean replace) { - CopyObjectRequest copyRequest = new CopyObjectRequest(bucketName, keyName, targetBucketName, targetKeyName); - if (sseKey != null) { - copyRequest.withSourceSSECustomerKey(sseKey); - } - if (newSseKey != null) { - copyRequest.withDestinationSSECustomerKey(newSseKey); - } - if (replace) { - copyRequest.withMetadataDirective(MetadataDirective.COPY); - } - s3Client.copyObject(copyRequest); - } - - long retrieveObjectMetadata(String bucketName, String keyName, SSECustomerKey sseKey) { - GetObjectMetadataRequest getMetadataRequest = new GetObjectMetadataRequest(bucketName, keyName) - .withSSECustomerKey(sseKey); - ObjectMetadata objectMetadata = s3Client.getObjectMetadata(getMetadataRequest); - return objectMetadata.getContentLength(); - } - -} diff --git a/mint/build/aws-sdk-java/src/Utils.java b/mint/build/aws-sdk-java/src/Utils.java deleted file mode 100644 index d6c9712..0000000 --- a/mint/build/aws-sdk-java/src/Utils.java +++ /dev/null @@ -1,76 +0,0 @@ -/* -* Mint, (C) 2018 Minio, Inc. -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - - -package io.minio.awssdk.tests; - -import java.io.*; -import java.nio.channels.*; -import java.security.*; - -class Utils { - - public static byte[] createChecksum(InputStream is, int skip, int length) throws Exception { - int numRead; - byte[] buffer = new byte[1024]; - - MessageDigest complete = MessageDigest.getInstance("MD5"); - - if (skip > -1 && length > -1) { - is = new LimitedInputStream(is, skip, length); - } - - do { - numRead = is.read(buffer); - if (numRead > 0) { - complete.update(buffer, 0, numRead); - } - } while (numRead != -1); - - return complete.digest(); - } - - public static String getInputStreamMD5(InputStream is) throws Exception { - return getInputStreamMD5(is, -1, -1); - } - - public static String getInputStreamMD5(InputStream is, int start, int length) throws Exception { - byte[] b = createChecksum(is, start, length); - String result = ""; - - for (int i=0; i < b.length; i++) { - result += Integer.toString( ( b[i] & 0xff ) + 0x100, 16).substring( 1 ); - } - return result; - } - - public static String getFileMD5(String filePath) throws Exception { - return getFileMD5(filePath, -1, -1); - } - - public static String getFileMD5(String filePath, int start, int length) throws Exception { - File f = new File(filePath); - InputStream is = new FileInputStream(f); - return getInputStreamMD5(is, start, length); - } - - public static String getBufferMD5(byte[] data) throws Exception { - ByteArrayInputStream bis = new ByteArrayInputStream(data); - return getInputStreamMD5(bis); - } -} - diff --git a/mint/build/aws-sdk-php/install.sh b/mint/build/aws-sdk-php/install.sh deleted file mode 100755 index 1363fa0..0000000 --- a/mint/build/aws-sdk-php/install.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -e -# -# Mint (C) 2017 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -test_run_dir="$MINT_RUN_CORE_DIR/aws-sdk-php" -$WGET --output-document=- https://getcomposer.org/installer | php -- --install-dir="$test_run_dir" -php "$test_run_dir/composer.phar" --working-dir="$test_run_dir" install diff --git a/mint/build/aws-sdk-ruby/install.sh b/mint/build/aws-sdk-ruby/install.sh deleted file mode 100755 index 7c7055f..0000000 --- a/mint/build/aws-sdk-ruby/install.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -e -# -# Mint (C) 20172-2020 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -gem install --no-rdoc --no-ri aws-sdk-resources aws-sdk multipart_body diff --git a/mint/build/awscli/install.sh b/mint/build/awscli/install.sh deleted file mode 100755 index cf7d9bd..0000000 --- a/mint/build/awscli/install.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -e -# -# Mint (C) 2017-2020 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -pip3 install awscli --upgrade diff --git a/mint/build/healthcheck/install.sh b/mint/build/healthcheck/install.sh deleted file mode 100755 index 71ffe48..0000000 --- a/mint/build/healthcheck/install.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash -e -# -# Mint (C) 2019 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -test_run_dir="$MINT_RUN_CORE_DIR/healthcheck" -GO111MODULE=on go build -o "$test_run_dir/healthcheck" "$test_run_dir/healthcheck.go" diff --git a/mint/build/mc/install.sh b/mint/build/mc/install.sh deleted file mode 100755 index a41aaeb..0000000 --- a/mint/build/mc/install.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -e -# -# Mint (C) 2017 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -MC_VERSION=$(curl --retry 10 -Ls -o /dev/null -w "%{url_effective}" https://github.com/minio/mc/releases/latest | sed "s/https:\/\/github.com\/minio\/mc\/releases\/tag\///") -if [ -z "$MC_VERSION" ]; then - echo "unable to get mc version from github" - exit 1 -fi - -test_run_dir="$MINT_RUN_CORE_DIR/mc" -$WGET --output-document="${test_run_dir}/mc" "https://dl.minio.io/client/mc/release/linux-amd64/mc.${MC_VERSION}" -chmod a+x "${test_run_dir}/mc" - -git clone --quiet https://github.com/minio/mc.git "$test_run_dir/mc.git" -(cd "$test_run_dir/mc.git"; git checkout --quiet "tags/${MC_VERSION}") -cp -a "${test_run_dir}/mc.git/functional-tests.sh" "$test_run_dir/" -rm -fr "$test_run_dir/mc.git" diff --git a/mint/build/minio-dotnet/install.sh b/mint/build/minio-dotnet/install.sh deleted file mode 100755 index fc00e85..0000000 --- a/mint/build/minio-dotnet/install.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash -# -# Mint (C) 2017-2020 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -set -e - -MINIO_DOTNET_SDK_PATH="$MINT_RUN_CORE_DIR/minio-dotnet" - -MINIO_DOTNET_SDK_VERSION=$(curl --retry 10 -Ls -o /dev/null -w "%{url_effective}" https://github.com/minio/minio-dotnet/releases/latest | sed "s/https:\/\/github.com\/minio\/minio-dotnet\/releases\/tag\///") -if [ -z "$MINIO_DOTNET_SDK_VERSION" ]; then - echo "unable to get minio-dotnet version from github" - exit 1 -fi - -out_dir="$MINIO_DOTNET_SDK_PATH/out" -if [ -z "$out_dir" ]; then - mkdir "$out_dir" -fi - -temp_dir="$MINIO_DOTNET_SDK_PATH/temp" -git clone --quiet https://github.com/minio/minio-dotnet.git "${temp_dir}/minio-dotnet.git/" -(cd "${temp_dir}/minio-dotnet.git"; git checkout --quiet "tags/${MINIO_DOTNET_SDK_VERSION}") - -cp -a "${temp_dir}/minio-dotnet.git/Minio.Functional.Tests/"* "${MINIO_DOTNET_SDK_PATH}/" -rm -fr "${temp_dir}" - -cd "$MINIO_DOTNET_SDK_PATH" -dotnet restore /p:Configuration=Mint -dotnet publish --runtime ubuntu.18.04-x64 --output out /p:Configuration=Mint diff --git a/mint/build/minio-go/install.sh b/mint/build/minio-go/install.sh deleted file mode 100755 index 5e30051..0000000 --- a/mint/build/minio-go/install.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash -e -# -# Mint (C) 2017 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -MINIO_GO_VERSION=$(curl --retry 10 -Ls -o /dev/null -w "%{url_effective}" https://github.com/minio/minio-go/releases/latest | sed "s/https:\/\/github.com\/minio\/minio-go\/releases\/tag\///") -if [ -z "$MINIO_GO_VERSION" ]; then - echo "unable to get minio-go version from github" - exit 1 -fi - -test_run_dir="$MINT_RUN_CORE_DIR/minio-go" -(git clone https://github.com/minio/minio-go && cd minio-go && git checkout --quiet "tags/$MINIO_GO_VERSION") -GO111MODULE=on CGO_ENABLED=0 go build -o "$test_run_dir/minio-go" "minio-go/functional_tests.go" -rm -rf minio-go diff --git a/mint/build/minio-java/install.sh b/mint/build/minio-java/install.sh deleted file mode 100755 index 399934d..0000000 --- a/mint/build/minio-java/install.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -e -# -# Mint (C) 2017 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -MINIO_JAVA_VERSION=$(curl --retry 10 -s "https://repo1.maven.org/maven2/io/minio/minio/maven-metadata.xml" | sed -n "//{s/<.[^>]*>//g;p;q}" | sed "s/ *//g") -if [ -z "$MINIO_JAVA_VERSION" ]; then - echo "unable to get latest minio-java version from maven" - exit 1 -fi - -test_run_dir="$MINT_RUN_CORE_DIR/minio-java" -git clone --quiet https://github.com/minio/minio-java.git "$test_run_dir/minio-java.git" -(cd "$test_run_dir/minio-java.git"; git checkout --quiet "tags/${MINIO_JAVA_VERSION}") -$WGET --output-document="$test_run_dir/minio-${MINIO_JAVA_VERSION}-all.jar" "https://repo1.maven.org/maven2/io/minio/minio/${MINIO_JAVA_VERSION}/minio-${MINIO_JAVA_VERSION}-all.jar" -javac -cp "$test_run_dir/minio-${MINIO_JAVA_VERSION}-all.jar" "${test_run_dir}/minio-java.git/functional"/*.java -cp -a "${test_run_dir}/minio-java.git/functional"/*.class "$test_run_dir/" -rm -fr "$test_run_dir/minio-java.git" diff --git a/mint/build/minio-js/install.sh b/mint/build/minio-js/install.sh deleted file mode 100755 index 0b9c4f4..0000000 --- a/mint/build/minio-js/install.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -e -# -# Mint (C) 2017 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -MINIO_JS_VERSION=$(curl --retry 10 -Ls -o /dev/null -w "%{url_effective}" https://github.com/minio/minio-js/releases/latest | sed "s/https:\/\/github.com\/minio\/minio-js\/releases\/tag\///") -if [ -z "$MINIO_JS_VERSION" ]; then - echo "unable to get minio-js version from github" - exit 1 -fi - -test_run_dir="$MINT_RUN_CORE_DIR/minio-js" -mkdir "${test_run_dir}/test" -$WGET --output-document="${test_run_dir}/test/functional-tests.js" "https://raw.githubusercontent.com/minio/minio-js/${MINIO_JS_VERSION}/src/test/functional/functional-tests.js" -npm --prefix "$test_run_dir" install --save "minio@$MINIO_JS_VERSION" -npm --prefix "$test_run_dir" install diff --git a/mint/build/minio-py/install.sh b/mint/build/minio-py/install.sh deleted file mode 100755 index 39bd990..0000000 --- a/mint/build/minio-py/install.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash -e -# -# Mint (C) 2017-2020 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -MINIO_PY_VERSION=$(curl --retry 10 -Ls -o /dev/null -w "%{url_effective}" https://github.com/minio/minio-py/releases/latest | sed "s/https:\/\/github.com\/minio\/minio-py\/releases\/tag\///") -if [ -z "$MINIO_PY_VERSION" ]; then - echo "unable to get minio-py version from github" - exit 1 -fi - -test_run_dir="$MINT_RUN_CORE_DIR/minio-py" -pip3 install --user faker -pip3 install minio=="$MINIO_PY_VERSION" -$WGET --output-document="$test_run_dir/tests.py" "https://raw.githubusercontent.com/minio/minio-py/${MINIO_PY_VERSION}/tests/functional/tests.py" diff --git a/mint/build/s3cmd/install.sh b/mint/build/s3cmd/install.sh deleted file mode 100755 index 6865930..0000000 --- a/mint/build/s3cmd/install.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash -e -# -# Mint (C) 2017 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Always install the latest. -python -m pip install s3cmd diff --git a/mint/build/s3select/install.sh b/mint/build/s3select/install.sh deleted file mode 100755 index aae2af2..0000000 --- a/mint/build/s3select/install.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -e -# -# Mint (C) 2020 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -python -m pip install minio diff --git a/mint/build/security/install.sh b/mint/build/security/install.sh deleted file mode 100755 index a517bfb..0000000 --- a/mint/build/security/install.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash -e -# -# Mint (C) 2018 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -test_run_dir="$MINT_RUN_CORE_DIR/security" -GO111MODULE=on go build -o "$test_run_dir/tls-tests" "$test_run_dir/tls-tests.go" diff --git a/mint/create-data-files.sh b/mint/create-data-files.sh deleted file mode 100755 index ba0ce43..0000000 --- a/mint/create-data-files.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash -e -# -# Mint (C) 2017 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -MINT_DATA_DIR="$MINT_ROOT_DIR/data" - -declare -A data_file_map -data_file_map["datafile-0-b"]="0" -data_file_map["datafile-1-b"]="1" -data_file_map["datafile-1-kB"]="1K" -data_file_map["datafile-10-kB"]="10K" -data_file_map["datafile-33-kB"]="33K" -data_file_map["datafile-100-kB"]="100K" -data_file_map["datafile-1.03-MB"]="1056K" -data_file_map["datafile-1-MB"]="1M" -data_file_map["datafile-5-MB"]="5M" -data_file_map["datafile-5243880-b"]="5243880" -data_file_map["datafile-6-MB"]="6M" -data_file_map["datafile-10-MB"]="10M" -data_file_map["datafile-11-MB"]="11M" -data_file_map["datafile-65-MB"]="65M" -data_file_map["datafile-129-MB"]="129M" - -mkdir -p "$MINT_DATA_DIR" -for filename in "${!data_file_map[@]}"; do - echo "creating $MINT_DATA_DIR/$filename" - if ! shred -n 1 -s "${data_file_map[$filename]}" - 1>"$MINT_DATA_DIR/$filename" 2>/dev/null; then - echo "unable to create data file $MINT_DATA_DIR/$filename" - exit 1 - fi -done diff --git a/mint/entrypoint.sh b/mint/entrypoint.sh deleted file mode 100755 index 5e15f6c..0000000 --- a/mint/entrypoint.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -# -# Mint (C) 2017 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -./mint.sh "$@" & - -# Get the pid to be used for kill command if required -main_pid="$!" -trap 'echo -e "\nAborting Mint..."; kill $main_pid' SIGINT SIGTERM -# use -n here to catch mint.sh exit code, notify to ci -wait -n diff --git a/mint/install-packages.list b/mint/install-packages.list deleted file mode 100644 index 600e36f..0000000 --- a/mint/install-packages.list +++ /dev/null @@ -1,18 +0,0 @@ -git -python3-pip -nodejs -openjdk-8-jdk -openjdk-8-jre -dirmngr -apt-transport-https -dotnet-sdk-2.1 -ca-certificates-mono -nuget -libunwind8 -ruby -ruby-dev -ruby-bundler -php -php-curl -php-xml -ant diff --git a/mint/mint.sh b/mint/mint.sh deleted file mode 100755 index 56d44b1..0000000 --- a/mint/mint.sh +++ /dev/null @@ -1,190 +0,0 @@ -#!/bin/bash -# -# Mint (C) 2017, 2018 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -CONTAINER_ID=$(grep -o -e '[0-f]\{12,\}' /proc/1/cpuset | awk '{print substr($1, 1, 12)}') -MINT_DATA_DIR=${MINT_DATA_DIR:-/mint/data} -MINT_MODE=${MINT_MODE:-core} -SERVER_REGION=${SERVER_REGION:-us-east-1} -ENABLE_HTTPS=${ENABLE_HTTPS:-0} -ENABLE_VIRTUAL_STYLE=${ENABLE_VIRTUAL_STYLE:-0} -GO111MODULE=on - -if [ -z "$SERVER_ENDPOINT" ]; then - SERVER_ENDPOINT="play.minio.io:9000" - ACCESS_KEY="Q3AM3UQ867SPQQA43P2F" - SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" - ENABLE_HTTPS=1 -fi - -if [ "$ENABLE_VIRTUAL_STYLE" -eq 1 ]; then - SERVER_IP="${SERVER_ENDPOINT%%:*}" - SERVER_PORT="${SERVER_ENDPOINT/*:/}" - # Check if SERVER_IP is actually IPv4 address - octets=("${SERVER_IP//./ }") - if [ "${#octets[@]}" -ne 4 ]; then - echo "$SERVER_IP must be an IP address" - exit 1 - fi - for octet in "${octets[@]}"; do - if [ "$octet" -lt 0 ] 2>/dev/null || [ "$octet" -gt 255 ] 2>/dev/null; then - echo "$SERVER_IP must be an IP address" - exit 1 - fi - done -fi - -ROOT_DIR="$PWD" -TESTS_DIR="$ROOT_DIR/run/core" - -BASE_LOG_DIR="$ROOT_DIR/log" -LOG_FILE="log.json" -ERROR_FILE="error.log" -mkdir -p "$BASE_LOG_DIR" - -function humanize_time() -{ - time="$1" - days=$(( time / 60 / 60 / 24 )) - hours=$(( time / 60 / 60 % 24 )) - minutes=$(( time / 60 % 60 )) - seconds=$(( time % 60 )) - - (( days > 0 )) && echo -n "$days days " - (( hours > 0 )) && echo -n "$hours hours " - (( minutes > 0 )) && echo -n "$minutes minutes " - (( days > 0 || hours > 0 || minutes > 0 )) && echo -n "and " - echo "$seconds seconds" -} - -function run_test() -{ - if [ ! -d "$1" ]; then - return 1 - fi - - start=$(date +%s) - - mkdir -p "$BASE_LOG_DIR/$sdk_name" - - (cd "$sdk_dir" && ./run.sh "$BASE_LOG_DIR/$LOG_FILE" "$BASE_LOG_DIR/$sdk_name/$ERROR_FILE") - rv=$? - end=$(date +%s) - duration=$(humanize_time $(( end - start ))) - - if [ "$rv" -eq 0 ]; then - echo "done in $duration" - else - echo "FAILED in $duration" - entry=$(tail -n 1 "$BASE_LOG_DIR/$LOG_FILE") - status=$(jq -e -r .status <<<"$entry") - jq_rv=$? - if [ "$jq_rv" -ne 0 ]; then - echo "$entry" - fi - ## Show error.log when status is empty or not "FAIL". - ## This may happen when test run failed without providing logs. - if [ "$jq_rv" -ne 0 ] || [ -z "$status" ] || ([ "$status" != "FAIL" ] && [ "$status" != "fail" ]); then - cat "$BASE_LOG_DIR/$sdk_name/$ERROR_FILE" - else - jq . <<<"$entry" - fi - fi - return $rv -} - -function trust_s3_endpoint_tls_cert() -{ - # Download the public certificate from the server - openssl s_client -showcerts -connect "$SERVER_ENDPOINT" /dev/null | \ - openssl x509 -outform PEM -out /usr/local/share/ca-certificates/s3_server_cert.crt || \ - exit 1 - - # Load the certificate in the system - update-ca-certificates --fresh >/dev/null - - # Ask different SDKs/tools to load system certificates - export REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt - export NODE_EXTRA_CA_CERTS=/etc/ssl/certs/ca-certificates.crt - export SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt -} - - -function main() -{ - export MINT_DATA_DIR - export MINT_MODE - export SERVER_ENDPOINT - export SERVER_IP - export SERVER_PORT - - export ACCESS_KEY - export SECRET_KEY - export ENABLE_HTTPS - export SERVER_REGION - export ENABLE_VIRTUAL_STYLE - export GO111MODULE - - echo "Running with" - echo "SERVER_ENDPOINT: $SERVER_ENDPOINT" - echo "ACCESS_KEY: $ACCESS_KEY" - echo "SECRET_KEY: ***REDACTED***" - echo "ENABLE_HTTPS: $ENABLE_HTTPS" - echo "SERVER_REGION: $SERVER_REGION" - echo "MINT_DATA_DIR: $MINT_DATA_DIR" - echo "MINT_MODE: $MINT_MODE" - echo "ENABLE_VIRTUAL_STYLE: $ENABLE_VIRTUAL_STYLE" - echo - echo "To get logs, run 'docker cp ${CONTAINER_ID}:/mint/log /tmp/mint-logs'" - echo - - [ "$ENABLE_HTTPS" == "1" ] && trust_s3_endpoint_tls_cert - - declare -a run_list - sdks=( "$@" ) - - if [ "$#" -eq 0 ]; then - sdks=( $(ls "$TESTS_DIR") ) - fi - - for sdk in "${sdks[@]}"; do - run_list=( "${run_list[@]}" "$TESTS_DIR/$sdk" ) - done - - count="${#run_list[@]}" - i=0 - for sdk_dir in "${run_list[@]}"; do - sdk_name=$(basename "$sdk_dir") - (( i++ )) - if [ ! -d "$sdk_dir" ]; then - echo "Test $sdk_name not found. Exiting Mint." - exit 1 - fi - echo -n "($i/$count) Running $sdk_name tests ... " - if ! run_test "$sdk_dir"; then - (( i-- )) - fi - done - - ## Report when all tests in run_list are run - if [ $i -eq "$count" ]; then - echo -e "\nAll tests ran successfully" - else - echo -e "\nExecuted $i out of $count tests successfully." - exit 1 - fi -} -main "$@" diff --git a/mint/postinstall.sh b/mint/postinstall.sh deleted file mode 100755 index 72e357d..0000000 --- a/mint/postinstall.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -e -# -# Mint (C) 2017 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -export APT="apt --quiet --yes" - -# remove all packages listed in remove-packages.list -xargs --arg-file="${MINT_ROOT_DIR}/remove-packages.list" apt --quiet --yes purge -${APT} autoremove - -# remove unwanted files -rm -fr "$GOROOT" "$GOPATH/src" /var/lib/apt/lists/* - -# flush to disk -sync diff --git a/mint/preinstall.sh b/mint/preinstall.sh deleted file mode 100755 index 44d7a0b..0000000 --- a/mint/preinstall.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/bin/bash -e -# -# Mint (C) 2017-2020 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -export APT="apt --quiet --yes" -export WGET="wget --quiet --no-check-certificate" - -# install nodejs source list -if ! $WGET --output-document=- https://deb.nodesource.com/setup_13.x | bash -; then - echo "unable to set nodejs repository" - exit 1 -fi - -$APT install apt-transport-https - -if ! $WGET --output-document=packages-microsoft-prod.deb https://packages.microsoft.com/config/ubuntu/18.04/packages-microsoft-prod.deb | bash -; then - echo "unable to download dotnet packages" - exit 1 -fi - -dpkg -i packages-microsoft-prod.deb -rm -f packages-microsoft-prod.deb - -$APT update -$APT install gnupg ca-certificates - -# download and install golang -GO_VERSION="1.13.10" -GO_INSTALL_PATH="/usr/local" -download_url="https://storage.googleapis.com/golang/go${GO_VERSION}.linux-amd64.tar.gz" -if ! $WGET --output-document=- "$download_url" | tar -C "${GO_INSTALL_PATH}" -zxf -; then - echo "unable to install go$GO_VERSION" - exit 1 -fi - -xargs --arg-file="${MINT_ROOT_DIR}/install-packages.list" apt --quiet --yes install - -nuget update -self - -# set python 3.6 as default -update-alternatives --install /usr/bin/python python /usr/bin/python3.6 1 - -sync diff --git a/mint/release.sh b/mint/release.sh deleted file mode 100755 index 8ee2f7e..0000000 --- a/mint/release.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash -e -# -# Minio Cloud Storage, (C) 2017 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -export MINT_ROOT_DIR=${MINT_ROOT_DIR:-/mint} -export MINT_RUN_CORE_DIR="$MINT_ROOT_DIR/run/core" -export MINT_RUN_SECURITY_DIR="$MINT_ROOT_DIR/run/security" -export WGET="wget --quiet --no-check-certificate" - -"${MINT_ROOT_DIR}"/create-data-files.sh -"${MINT_ROOT_DIR}"/preinstall.sh - -# install mint app packages -for pkg in "$MINT_ROOT_DIR/build"/*/install.sh; do - echo "Running $pkg" - $pkg -done - -"${MINT_ROOT_DIR}"/postinstall.sh diff --git a/mint/remove-packages.list b/mint/remove-packages.list deleted file mode 100644 index 0206fbc..0000000 --- a/mint/remove-packages.list +++ /dev/null @@ -1,9 +0,0 @@ -wget -git -python3-pip -ruby-dev -ruby-bundler -openjdk-8-jdk -ant -dotnet -nuget diff --git a/mint/run/core/aws-sdk-go/quick-tests.go b/mint/run/core/aws-sdk-go/quick-tests.go deleted file mode 100644 index e9f4a88..0000000 --- a/mint/run/core/aws-sdk-go/quick-tests.go +++ /dev/null @@ -1,1066 +0,0 @@ -/* -* -* Mint, (C) 2017 Minio, Inc. -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software - -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -* - */ - -package main - -import ( - "bytes" - "encoding/json" - "encoding/xml" - "errors" - "fmt" - "math/rand" - "net/http" - "os" - "reflect" - "strings" - "time" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/awserr" - "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/s3" - log "github.com/sirupsen/logrus" -) - -const letterBytes = "abcdefghijklmnopqrstuvwxyz01234569" -const ( - letterIdxBits = 6 // 6 bits to represent a letter index - letterIdxMask = 1<= 0; { - if remain == 0 { - cache, remain = src.Int63(), letterIdxMax - } - if idx := int(cache & letterIdxMask); idx < len(letterBytes) { - b[i] = letterBytes[idx] - i-- - } - cache >>= letterIdxBits - remain-- - } - return prefix + string(b[0:30-len(prefix)]) -} - -func isObjectTaggingImplemented(s3Client *s3.S3) bool { - bucket := randString(60, rand.NewSource(time.Now().UnixNano()), "aws-sdk-go-test-") - object := randString(60, rand.NewSource(time.Now().UnixNano()), "") - startTime := time.Now() - function := "isObjectTaggingImplemented" - args := map[string]interface{}{ - "bucketName": bucket, - "objectName": object, - } - defer cleanup(s3Client, bucket, object, function, args, startTime, true) - - _, err := s3Client.CreateBucket(&s3.CreateBucketInput{ - Bucket: aws.String(bucket), - }) - if err != nil { - failureLog(function, args, startTime, "", "AWS SDK Go CreateBucket Failed", err).Fatal() - return false - } - - _, err = s3Client.PutObject(&s3.PutObjectInput{ - Body: aws.ReadSeekCloser(strings.NewReader("testfile")), - Bucket: aws.String(bucket), - Key: aws.String(object), - }) - - if err != nil { - failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go PUT expected to success but got %v", err), err).Fatal() - return false - } - - _, err = s3Client.GetObjectTagging(&s3.GetObjectTaggingInput{ - Bucket: aws.String(bucket), - Key: aws.String(object), - }) - if err != nil { - if awsErr, ok := err.(awserr.Error); ok { - if awsErr.Code() == "NotImplemented" { - return false - } - } - } - return true -} - -func cleanup(s3Client *s3.S3, bucket string, object string, function string, - args map[string]interface{}, startTime time.Time, deleteBucket bool) { - - // Deleting the object, just in case it was created. Will not check for errors. - s3Client.DeleteObject(&s3.DeleteObjectInput{ - Bucket: aws.String(bucket), - Key: aws.String(object), - }) - - if deleteBucket { - _, err := s3Client.DeleteBucket(&s3.DeleteBucketInput{ - Bucket: aws.String(bucket), - }) - if err != nil { - failureLog(function, args, startTime, "", "AWS SDK Go DeleteBucket Failed", err).Fatal() - return - } - } - -} - -func testPresignedPutInvalidHash(s3Client *s3.S3) { - startTime := time.Now() - function := "PresignedPut" - bucket := randString(60, rand.NewSource(time.Now().UnixNano()), "aws-sdk-go-test-") - object := "presignedTest" - expiry := 1 * time.Minute - args := map[string]interface{}{ - "bucketName": bucket, - "objectName": object, - "expiry": expiry, - } - - _, err := s3Client.CreateBucket(&s3.CreateBucketInput{ - Bucket: aws.String(bucket), - }) - if err != nil { - failureLog(function, args, startTime, "", "AWS SDK Go CreateBucket Failed", err).Fatal() - return - } - defer cleanup(s3Client, bucket, object, function, args, startTime, true) - - req, _ := s3Client.PutObjectRequest(&s3.PutObjectInput{ - Bucket: aws.String(bucket), - Key: aws.String(object), - ContentType: aws.String("application/octet-stream"), - }) - - req.HTTPRequest.Header.Set("X-Amz-Content-Sha256", "invalid-sha256") - url, err := req.Presign(expiry) - if err != nil { - failureLog(function, args, startTime, "", "AWS SDK Go presigned Put request creation failed", err).Fatal() - return - } - - rreq, err := http.NewRequest("PUT", url, bytes.NewReader([]byte(""))) - if err != nil { - failureLog(function, args, startTime, "", "AWS SDK Go presigned PUT request failed", err).Fatal() - return - } - - rreq.Header.Add("X-Amz-Content-Sha256", "invalid-sha256") - rreq.Header.Add("Content-Type", "application/octet-stream") - - resp, err := http.DefaultClient.Do(rreq) - if err != nil { - failureLog(function, args, startTime, "", "AWS SDK Go presigned put request failed", err).Fatal() - return - } - defer resp.Body.Close() - - dec := xml.NewDecoder(resp.Body) - errResp := errorResponse{} - err = dec.Decode(&errResp) - if err != nil { - failureLog(function, args, startTime, "", "AWS SDK Go unmarshalling xml failed", err).Fatal() - return - } - - if errResp.Code != "XAmzContentSHA256Mismatch" { - failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go presigned PUT expected to fail with XAmzContentSHA256Mismatch but got %v", errResp.Code), errors.New("AWS S3 error code mismatch")).Fatal() - return - } - - successLogger(function, args, startTime).Info() -} - -func testListObjects(s3Client *s3.S3) { - startTime := time.Now() - function := "testListObjects" - bucket := randString(60, rand.NewSource(time.Now().UnixNano()), "aws-sdk-go-test-") - object1 := "testObject1" - object2 := "testObject2" - expiry := 1 * time.Minute - args := map[string]interface{}{ - "bucketName": bucket, - "objectName1": object1, - "objectName2": object2, - "expiry": expiry, - } - - getKeys := func(objects []*s3.Object) []string { - var rv []string - for _, obj := range objects { - rv = append(rv, *obj.Key) - } - return rv - } - _, err := s3Client.CreateBucket(&s3.CreateBucketInput{ - Bucket: aws.String(bucket), - }) - if err != nil { - failureLog(function, args, startTime, "", "AWS SDK Go CreateBucket Failed", err).Fatal() - return - } - defer cleanup(s3Client, bucket, object1, function, args, startTime, true) - defer cleanup(s3Client, bucket, object2, function, args, startTime, false) - - listInput := &s3.ListObjectsV2Input{ - Bucket: aws.String(bucket), - MaxKeys: aws.Int64(1000), - Prefix: aws.String(""), - } - result, err := s3Client.ListObjectsV2(listInput) - if err != nil { - failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go listobjects expected to success but got %v", err), err).Fatal() - return - } - if *result.KeyCount != 0 { - failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go listobjects with prefix '' expected 0 key but got %v, %v", result.KeyCount, getKeys(result.Contents)), errors.New("AWS S3 key count mismatch")).Fatal() - return - } - putInput1 := &s3.PutObjectInput{ - Body: aws.ReadSeekCloser(strings.NewReader("filetoupload")), - Bucket: aws.String(bucket), - Key: aws.String(object1), - } - _, err = s3Client.PutObject(putInput1) - if err != nil { - failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go PUT expected to success but got %v", err), err).Fatal() - return - } - putInput2 := &s3.PutObjectInput{ - Body: aws.ReadSeekCloser(strings.NewReader("filetoupload")), - Bucket: aws.String(bucket), - Key: aws.String(object2), - } - _, err = s3Client.PutObject(putInput2) - if err != nil { - failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go PUT expected to success but got %v", err), err).Fatal() - return - } - result, err = s3Client.ListObjectsV2(listInput) - if err != nil { - failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go listobjects expected to success but got %v", err), err).Fatal() - return - } - if *result.KeyCount != 2 { - failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go listobjects with prefix '' expected 2 key but got %v, %v", *result.KeyCount, getKeys(result.Contents)), errors.New("AWS S3 key count mismatch")).Fatal() - return - } - - successLogger(function, args, startTime).Info() -} - -func testSelectObject(s3Client *s3.S3) { - startTime := time.Now() - function := "testSelectObject" - bucket := randString(60, rand.NewSource(time.Now().UnixNano()), "aws-sdk-go-test-") - object1 := "object1.csv" - object2 := "object2.csv" - args := map[string]interface{}{ - "bucketName": bucket, - "objectName1": object1, - "objectName2": object2, - } - - _, err := s3Client.CreateBucket(&s3.CreateBucketInput{ - Bucket: aws.String(bucket), - }) - if err != nil { - failureLog(function, args, startTime, "", "AWS SDK Go CreateBucket Failed", err).Fatal() - return - } - - // Test comma field separator - inputCsv1 := `year,gender,ethnicity,firstname,count,rank -2011,FEMALE,ASIAN AND PACIFIC ISLANDER,SOPHIA,119,1 -2011,FEMALE,ASIAN AND PACIFIC ISLANDER,CHLOE,106,2 -2011,FEMALE,ASIAN AND PACIFIC ISLANDER,EMILY,93,3 -2011,FEMALE,ASIAN AND PACIFIC ISLANDER,OLIVIA,89,4 -2011,FEMALE,ASIAN AND PACIFIC ISLANDER,EMMA,75,5 -2011,FEMALE,ASIAN AND PACIFIC ISLANDER,ISABELLA,67,6 -2011,FEMALE,ASIAN AND PACIFIC ISLANDER,TIFFANY,54,7 -2011,FEMALE,ASIAN AND PACIFIC ISLANDER,ASHLEY,52,8 -2011,FEMALE,ASIAN AND PACIFIC ISLANDER,FIONA,48,9 -2011,FEMALE,ASIAN AND PACIFIC ISLANDER,ANGELA,47,10 -` - - outputCSV1 := `2011 -2011 -2011 -2011 -2011 -2011 -2011 -2011 -2011 -2011 -` - - putInput1 := &s3.PutObjectInput{ - Body: aws.ReadSeekCloser(strings.NewReader(inputCsv1)), - Bucket: aws.String(bucket), - Key: aws.String(object1), - } - _, err = s3Client.PutObject(putInput1) - if err != nil { - failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go Select object failed %v", err), err).Fatal() - return - } - - defer cleanup(s3Client, bucket, object1, function, args, startTime, true) - - params := &s3.SelectObjectContentInput{ - Bucket: &bucket, - Key: &object1, - ExpressionType: aws.String(s3.ExpressionTypeSql), - Expression: aws.String("SELECT s._1 FROM S3Object s"), - RequestProgress: &s3.RequestProgress{}, - InputSerialization: &s3.InputSerialization{ - CompressionType: aws.String("NONE"), - CSV: &s3.CSVInput{ - FileHeaderInfo: aws.String(s3.FileHeaderInfoIgnore), - FieldDelimiter: aws.String(","), - RecordDelimiter: aws.String("\n"), - }, - }, - OutputSerialization: &s3.OutputSerialization{ - CSV: &s3.CSVOutput{}, - }, - } - - resp, err := s3Client.SelectObjectContent(params) - if err != nil { - failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go Select object failed %v", err), err).Fatal() - return - } - defer resp.EventStream.Close() - - payload := "" - for event := range resp.EventStream.Events() { - switch v := event.(type) { - case *s3.RecordsEvent: - // s3.RecordsEvent.Records is a byte slice of select records - payload = string(v.Payload) - } - } - - if err := resp.EventStream.Err(); err != nil { - failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go Select object failed %v", err), err).Fatal() - return - } - - if payload != outputCSV1 { - failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go Select object output mismatch %v", payload), errors.New("AWS S3 select object mismatch")).Fatal() - return - } - - // Test unicode field separator - inputCsv2 := `"year"╦"gender"╦"ethnicity"╦"firstname"╦"count"╦"rank" -"2011"╦"FEMALE"╦"ASIAN AND PACIFIC ISLANDER"╦"SOPHIA"╦"119"╦"1" -"2011"╦"FEMALE"╦"ASIAN AND PACIFIC ISLANDER"╦"CHLOE"╦"106"╦"2" -"2011"╦"FEMALE"╦"ASIAN AND PACIFIC ISLANDER"╦"EMILY"╦"93"╦"3" -"2011"╦"FEMALE"╦"ASIAN AND PACIFIC ISLANDER"╦"OLIVIA"╦"89"╦"4" -"2011"╦"FEMALE"╦"ASIAN AND PACIFIC ISLANDER"╦"EMMA"╦"75"╦"5" -"2011"╦"FEMALE"╦"ASIAN AND PACIFIC ISLANDER"╦"ISABELLA"╦"67"╦"6" -"2011"╦"FEMALE"╦"ASIAN AND PACIFIC ISLANDER"╦"TIFFANY"╦"54"╦"7" -"2011"╦"FEMALE"╦"ASIAN AND PACIFIC ISLANDER"╦"ASHLEY"╦"52"╦"8" -"2011"╦"FEMALE"╦"ASIAN AND PACIFIC ISLANDER"╦"FIONA"╦"48"╦"9" -"2011"╦"FEMALE"╦"ASIAN AND PACIFIC ISLANDER"╦"ANGELA"╦"47"╦"10" -` - - outputCSV2 := `2011 -2011 -2011 -2011 -2011 -2011 -2011 -2011 -2011 -2011 -` - - putInput2 := &s3.PutObjectInput{ - Body: aws.ReadSeekCloser(strings.NewReader(inputCsv2)), - Bucket: aws.String(bucket), - Key: aws.String(object2), - } - _, err = s3Client.PutObject(putInput2) - if err != nil { - failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go Select object upload failed: %v", err), err).Fatal() - return - } - - defer cleanup(s3Client, bucket, object2, function, args, startTime, false) - - params2 := &s3.SelectObjectContentInput{ - Bucket: &bucket, - Key: &object2, - ExpressionType: aws.String(s3.ExpressionTypeSql), - Expression: aws.String("SELECT s._1 FROM S3Object s"), - RequestProgress: &s3.RequestProgress{}, - InputSerialization: &s3.InputSerialization{ - CompressionType: aws.String("NONE"), - CSV: &s3.CSVInput{ - FileHeaderInfo: aws.String(s3.FileHeaderInfoIgnore), - FieldDelimiter: aws.String("╦"), - RecordDelimiter: aws.String("\n"), - }, - }, - OutputSerialization: &s3.OutputSerialization{ - CSV: &s3.CSVOutput{}, - }, - } - - resp, err = s3Client.SelectObjectContent(params2) - if err != nil { - failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go Select object failed for unicode separator %v", err), err).Fatal() - return - } - defer resp.EventStream.Close() - - for event := range resp.EventStream.Events() { - switch v := event.(type) { - case *s3.RecordsEvent: - // s3.RecordsEvent.Records is a byte slice of select records - payload = string(v.Payload) - } - } - - if err := resp.EventStream.Err(); err != nil { - failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go Select object failed for unicode separator %v", err), err).Fatal() - return - } - - if payload != outputCSV2 { - failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go Select object output mismatch %v", payload), errors.New("AWS S3 select object mismatch")).Fatal() - return - } - - successLogger(function, args, startTime).Info() -} - -func testObjectTagging(s3Client *s3.S3) { - startTime := time.Now() - function := "testObjectTagging" - bucket := randString(60, rand.NewSource(time.Now().UnixNano()), "aws-sdk-go-test-") - object := randString(60, rand.NewSource(time.Now().UnixNano()), "") - args := map[string]interface{}{ - "bucketName": bucket, - "objectName": object, - } - - _, err := s3Client.CreateBucket(&s3.CreateBucketInput{ - Bucket: aws.String(bucket), - }) - if err != nil { - failureLog(function, args, startTime, "", "AWS SDK Go CreateBucket Failed", err).Fatal() - return - } - defer cleanup(s3Client, bucket, object, function, args, startTime, true) - - taginput := "Tag1=Value1" - tagInputSet := []*s3.Tag{ - { - Key: aws.String("Tag1"), - Value: aws.String("Value1"), - }, - } - _, err = s3Client.PutObject(&s3.PutObjectInput{ - Body: aws.ReadSeekCloser(strings.NewReader("testfile")), - Bucket: aws.String(bucket), - Key: aws.String(object), - Tagging: &taginput, - }) - - if err != nil { - failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go PUT expected to success but got %v", err), err).Fatal() - return - } - - tagop, err := s3Client.GetObjectTagging(&s3.GetObjectTaggingInput{ - Bucket: aws.String(bucket), - Key: aws.String(object), - }) - if err != nil { - if awsErr, ok := err.(awserr.Error); ok { - failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go PUTObjectTagging expected to success but got %v", awsErr.Code()), err).Fatal() - return - } - } - if !reflect.DeepEqual(tagop.TagSet, tagInputSet) { - failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go PUTObject Tag input did not match with GetObjectTagging output %v", nil), nil).Fatal() - return - } - - taginputSet1 := []*s3.Tag{ - { - Key: aws.String("Key4"), - Value: aws.String("Value4"), - }, - } - _, err = s3Client.PutObjectTagging(&s3.PutObjectTaggingInput{ - Bucket: aws.String(bucket), - Key: aws.String(object), - Tagging: &s3.Tagging{ - TagSet: taginputSet1, - }, - }) - if err != nil { - if awsErr, ok := err.(awserr.Error); ok { - failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go PUTObjectTagging expected to success but got %v", awsErr.Code()), err).Fatal() - return - } - } - - tagop, err = s3Client.GetObjectTagging(&s3.GetObjectTaggingInput{ - Bucket: aws.String(bucket), - Key: aws.String(object), - }) - if err != nil { - if awsErr, ok := err.(awserr.Error); ok { - failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go PUTObjectTagging expected to success but got %v", awsErr.Code()), err).Fatal() - return - } - } - if !reflect.DeepEqual(tagop.TagSet, taginputSet1) { - failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go PUTObjectTagging input did not match with GetObjectTagging output %v", nil), nil).Fatal() - return - } - successLogger(function, args, startTime).Info() -} - -func testObjectTaggingErrors(s3Client *s3.S3) { - startTime := time.Now() - function := "testObjectTaggingErrors" - bucket := randString(60, rand.NewSource(time.Now().UnixNano()), "aws-sdk-go-test-") - object := randString(60, rand.NewSource(time.Now().UnixNano()), "") - args := map[string]interface{}{ - "bucketName": bucket, - "objectName": object, - } - - _, err := s3Client.CreateBucket(&s3.CreateBucketInput{ - Bucket: aws.String(bucket), - }) - if err != nil { - failureLog(function, args, startTime, "", "AWS SDK Go CreateBucket Failed", err).Fatal() - return - } - defer cleanup(s3Client, bucket, object, function, args, startTime, true) - - _, err = s3Client.PutObject(&s3.PutObjectInput{ - Body: aws.ReadSeekCloser(strings.NewReader("testfile")), - Bucket: aws.String(bucket), - Key: aws.String(object), - }) - - if err != nil { - failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go PUT expected to success but got %v", err), err).Fatal() - return - } - - // case 1 : Too many tags > 10 - input := &s3.PutObjectTaggingInput{ - Bucket: aws.String(bucket), - Key: aws.String(object), - Tagging: &s3.Tagging{ - TagSet: []*s3.Tag{ - { - Key: aws.String("Key1"), - Value: aws.String("Value3"), - }, - { - Key: aws.String("Key2"), - Value: aws.String("Value4"), - }, - { - Key: aws.String("Key3"), - Value: aws.String("Value3"), - }, - { - Key: aws.String("Key4"), - Value: aws.String("Value3"), - }, - { - Key: aws.String("Key5"), - Value: aws.String("Value3"), - }, - { - Key: aws.String("Key6"), - Value: aws.String("Value3"), - }, - { - Key: aws.String("Key7"), - Value: aws.String("Value3"), - }, - { - Key: aws.String("Key8"), - Value: aws.String("Value3"), - }, - { - Key: aws.String("Key9"), - Value: aws.String("Value3"), - }, - { - Key: aws.String("Key10"), - Value: aws.String("Value3"), - }, - { - Key: aws.String("Key11"), - Value: aws.String("Value3"), - }, - }, - }, - } - - _, err = s3Client.PutObjectTagging(input) - if err == nil { - failureLog(function, args, startTime, "", "AWS SDK Go PUT expected to fail but succeeded", err).Fatal() - return - } - - if aerr, ok := err.(awserr.Error); ok { - if aerr.Code() != "BadRequest" && aerr.Message() != "BadRequest: Object tags cannot be greater than 10" { - failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go PUT expected to fail but got %v", err), err).Fatal() - return - } - } - - // case 2 : Duplicate Tag Keys - input = &s3.PutObjectTaggingInput{ - Bucket: aws.String(bucket), - Key: aws.String(object), - Tagging: &s3.Tagging{ - TagSet: []*s3.Tag{ - { - Key: aws.String("Key1"), - Value: aws.String("Value3"), - }, - { - Key: aws.String("Key1"), - Value: aws.String("Value4"), - }, - }, - }, - } - - _, err = s3Client.PutObjectTagging(input) - if err == nil { - failureLog(function, args, startTime, "", "AWS SDK Go PUT expected to fail but succeeded", err).Fatal() - return - } - - if aerr, ok := err.(awserr.Error); ok { - if aerr.Code() != "InvalidTag" && aerr.Message() != "InvalidTag: Cannot provide multiple Tags with the same key" { - failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go PUT expected to fail but got %v", err), err).Fatal() - return - } - } - - // case 3 : Too long Tag Key - input = &s3.PutObjectTaggingInput{ - Bucket: aws.String(bucket), - Key: aws.String(object), - Tagging: &s3.Tagging{ - TagSet: []*s3.Tag{ - { - Key: aws.String("Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1"), - Value: aws.String("Value3"), - }, - { - Key: aws.String("Key1"), - Value: aws.String("Value4"), - }, - }, - }, - } - - _, err = s3Client.PutObjectTagging(input) - if err == nil { - failureLog(function, args, startTime, "", "AWS SDK Go PUT expected to fail but succeeded", err).Fatal() - return - } - - if aerr, ok := err.(awserr.Error); ok { - if aerr.Code() != "InvalidTag" && aerr.Message() != "InvalidTag: The TagKey you have provided is invalid" { - failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go PUT expected to fail but got %v", err), err).Fatal() - return - } - } - - // case 4 : Too long Tag value - input = &s3.PutObjectTaggingInput{ - Bucket: aws.String(bucket), - Key: aws.String(object), - Tagging: &s3.Tagging{ - TagSet: []*s3.Tag{ - { - Key: aws.String("Key1"), - Value: aws.String("Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1"), - }, - { - Key: aws.String("Key1"), - Value: aws.String("Value4"), - }, - }, - }, - } - - _, err = s3Client.PutObjectTagging(input) - if err == nil { - failureLog(function, args, startTime, "", "AWS SDK Go PUT expected to fail but succeeded", err).Fatal() - return - } - - if aerr, ok := err.(awserr.Error); ok { - if aerr.Code() != "InvalidTag" && aerr.Message() != "InvalidTag: The TagValue you have provided is invalid" { - failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go PUT expected to fail but got %v", err), err).Fatal() - return - } - } - - successLogger(function, args, startTime).Info() -} - -// Tests bucket re-create errors. -func testCreateBucketError(s3Client *s3.S3) { - region := s3Client.Config.Region - // Amazon S3 returns error in all AWS Regions except in the North Virginia Region. - // More details in https://docs.aws.amazon.com/sdk-for-go/api/service/s3/#S3.CreateBucket - s3Client.Config.Region = aws.String("us-west-1") - - // initialize logging params - startTime := time.Now() - function := "testMakeBucketError" - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "aws-sdk-go-test-") - args := map[string]interface{}{ - "bucketName": bucketName, - } - _, err := s3Client.CreateBucket(&s3.CreateBucketInput{ - Bucket: aws.String(bucketName), - }) - if err != nil { - failureLog(function, args, startTime, "", "AWS SDK Go CreateBucket Failed", err).Fatal() - return - } - defer cleanup(s3Client, bucketName, "", function, args, startTime, true) - - _, errCreating := s3Client.CreateBucket(&s3.CreateBucketInput{ - Bucket: aws.String(bucketName), - }) - if errCreating == nil { - failureLog(function, args, startTime, "", "AWS SDK Go CreateBucket Should Return Error for Existing bucket", err).Fatal() - return - } - // Verify valid error response from server. - if errCreating.(s3.RequestFailure).Code() != "BucketAlreadyExists" && - errCreating.(s3.RequestFailure).Code() != "BucketAlreadyOwnedByYou" { - failureLog(function, args, startTime, "", "Invalid error returned by server", err).Fatal() - return - } - - // Restore region in s3Client - s3Client.Config.Region = region - successLogger(function, args, startTime).Info() -} - -func testListMultipartUploads(s3Client *s3.S3) { - startTime := time.Now() - function := "testListMultipartUploads" - bucket := randString(60, rand.NewSource(time.Now().UnixNano()), "aws-sdk-go-test-") - object := randString(60, rand.NewSource(time.Now().UnixNano()), "") - args := map[string]interface{}{ - "bucketName": bucket, - "objectName": object, - } - _, errCreating := s3Client.CreateBucket(&s3.CreateBucketInput{ - Bucket: aws.String(bucket), - }) - if errCreating != nil { - failureLog(function, args, startTime, "", "AWS SDK Go CreateBucket Failed", errCreating).Fatal() - return - } - defer cleanup(s3Client, bucket, object, function, args, startTime, true) - - multipartUpload, err := s3Client.CreateMultipartUpload(&s3.CreateMultipartUploadInput{ - Bucket: aws.String(bucket), - Key: aws.String(object), - }) - if err != nil { - failureLog(function, args, startTime, "", "AWS SDK Go createMultipartupload API failed", err).Fatal() - return - } - parts := make(map[*int64]*string) - for i := 0; i < 5; i++ { - result, errUpload := s3Client.UploadPart(&s3.UploadPartInput{ - Bucket: aws.String(bucket), - Key: aws.String(object), - UploadId: multipartUpload.UploadId, - PartNumber: aws.Int64(int64(i + 1)), - Body: aws.ReadSeekCloser(strings.NewReader("fileToUpload")), - }) - if errUpload != nil { - _, _ = s3Client.AbortMultipartUpload(&s3.AbortMultipartUploadInput{ - Bucket: aws.String(bucket), - Key: aws.String(object), - UploadId: multipartUpload.UploadId, - }) - failureLog(function, args, startTime, "", "AWS SDK Go uploadPart API failed for", errUpload).Fatal() - return - } - parts[aws.Int64(int64(i+1))] = result.ETag - } - - listParts, errParts := s3Client.ListParts(&s3.ListPartsInput{ - Bucket: aws.String(bucket), - Key: aws.String(object), - UploadId: multipartUpload.UploadId, - }) - if errParts != nil { - failureLog(function, args, startTime, "", "AWS SDK Go ListPartsInput API failed for", err).Fatal() - return - } - - if len(parts) != len(listParts.Parts) { - failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go ListParts.Parts len mismatch want: %v got: %v", len(parts), len(listParts.Parts)), err).Fatal() - return - } - - for _, part := range listParts.Parts { - if tag, ok := parts[part.PartNumber]; ok { - if tag != part.ETag { - failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go ListParts.Parts output mismatch want: %v got: %v", tag, part.ETag), err).Fatal() - return - } - } - } - - successLogger(function, args, startTime).Info() -} - -func testSSECopyObject(s3Client *s3.S3) { - // initialize logging params - startTime := time.Now() - function := "testSSECopyObjectSourceEncrypted" - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "aws-sdk-go-test-") - object := randString(60, rand.NewSource(time.Now().UnixNano()), "") - args := map[string]interface{}{ - "bucketName": bucketName, - "objectName": object, - } - _, err := s3Client.CreateBucket(&s3.CreateBucketInput{ - Bucket: aws.String(bucketName), - }) - if err != nil { - failureLog(function, args, startTime, "", "AWS SDK Go CreateBucket Failed", err).Fatal() - return - } - defer cleanup(s3Client, bucketName, object+"-enc", function, args, startTime, true) - defer cleanup(s3Client, bucketName, object+"-noenc", function, args, startTime, false) - var wrongSuccess = errors.New("Succeeded instead of failing. ") - - // create encrypted object - sseCustomerKey := aws.String("32byteslongsecretkeymustbegiven2") - _, errPutEnc := s3Client.PutObject(&s3.PutObjectInput{ - Body: aws.ReadSeekCloser(strings.NewReader("fileToUpload")), - Bucket: aws.String(bucketName), - Key: aws.String(object + "-enc"), - SSECustomerAlgorithm: aws.String("AES256"), - SSECustomerKey: sseCustomerKey, - }) - if errPutEnc != nil { - failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go PUT expected to succeed but got %v", errPutEnc), errPutEnc).Fatal() - return - } - - // copy the encrypted object - _, errCopyEnc := s3Client.CopyObject(&s3.CopyObjectInput{ - SSECustomerAlgorithm: aws.String("AES256"), - SSECustomerKey: sseCustomerKey, - CopySource: aws.String(bucketName + "/" + object + "-enc"), - Bucket: aws.String(bucketName), - Key: aws.String(object + "-copy"), - }) - if errCopyEnc == nil { - failureLog(function, args, startTime, "", "AWS SDK Go CopyObject expected to fail, but it succeeds ", wrongSuccess).Fatal() - return - } - var invalidSSECustomerAlgorithm = "Requests specifying Server Side Encryption with Customer provided keys must provide a valid encryption algorithm" - if !strings.Contains(errCopyEnc.Error(), invalidSSECustomerAlgorithm) { - failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go CopyObject expected error %v got %v", invalidSSECustomerAlgorithm, errCopyEnc), errCopyEnc).Fatal() - return - } - - // create non-encrypted object - _, errPut := s3Client.PutObject(&s3.PutObjectInput{ - Body: aws.ReadSeekCloser(strings.NewReader("fileToUpload")), - Bucket: aws.String(bucketName), - Key: aws.String(object + "-noenc"), - }) - if errPut != nil { - failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go PUT expected to succeed but got %v", errPut), errPut).Fatal() - return - } - - // copy the non-encrypted object - _, errCopy := s3Client.CopyObject(&s3.CopyObjectInput{ - CopySourceSSECustomerAlgorithm: aws.String("AES256"), - CopySourceSSECustomerKey: sseCustomerKey, - SSECustomerAlgorithm: aws.String("AES256"), - SSECustomerKey: sseCustomerKey, - CopySource: aws.String(bucketName + "/" + object + "-noenc"), - Bucket: aws.String(bucketName), - Key: aws.String(object + "-copy"), - }) - if errCopy == nil { - failureLog(function, args, startTime, "", "AWS SDK Go CopyObject expected to fail, but it succeeds ", wrongSuccess).Fatal() - return - } - var invalidEncryptionParameters = "The encryption parameters are not applicable to this object." - if !strings.Contains(errCopy.Error(), invalidEncryptionParameters) { - failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go CopyObject expected error %v got %v", invalidEncryptionParameters, errCopy), errCopy).Fatal() - return - } - - successLogger(function, args, startTime).Info() -} - -func main() { - endpoint := os.Getenv("SERVER_ENDPOINT") - accessKey := os.Getenv("ACCESS_KEY") - secretKey := os.Getenv("SECRET_KEY") - secure := os.Getenv("ENABLE_HTTPS") - sdkEndpoint := "http://" + endpoint - if secure == "1" { - sdkEndpoint = "https://" + endpoint - } - - creds := credentials.NewStaticCredentials(accessKey, secretKey, "") - newSession := session.New() - s3Config := &aws.Config{ - Credentials: creds, - Endpoint: aws.String(sdkEndpoint), - Region: aws.String("us-east-1"), - S3ForcePathStyle: aws.Bool(true), - } - - // Create an S3 service object in the default region. - s3Client := s3.New(newSession, s3Config) - - // Output to stdout instead of the default stderr - log.SetOutput(os.Stdout) - // create custom formatter - mintFormatter := mintJSONFormatter{} - // set custom formatter - log.SetFormatter(&mintFormatter) - // log Info or above -- success cases are Info level, failures are Fatal level - log.SetLevel(log.InfoLevel) - // execute tests - testPresignedPutInvalidHash(s3Client) - testListObjects(s3Client) - testSelectObject(s3Client) - testCreateBucketError(s3Client) - testListMultipartUploads(s3Client) - if secure == "1" { - testSSECopyObject(s3Client) - } - if isObjectTaggingImplemented(s3Client) { - testObjectTagging(s3Client) - testObjectTaggingErrors(s3Client) - } -} diff --git a/mint/run/core/aws-sdk-go/run.sh b/mint/run/core/aws-sdk-go/run.sh deleted file mode 100755 index 813b3ee..0000000 --- a/mint/run/core/aws-sdk-go/run.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -# -# Mint (C) 2017 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# handle command line arguments -if [ $# -ne 2 ]; then - echo "usage: run.sh " - exit 1 -fi - -output_log_file="$1" -error_log_file="$2" - -# run tests -/mint/run/core/aws-sdk-go/aws-sdk-go 1>>"$output_log_file" 2>"$error_log_file" diff --git a/mint/run/core/aws-sdk-java/run.sh b/mint/run/core/aws-sdk-java/run.sh deleted file mode 100755 index 9a9ad38..0000000 --- a/mint/run/core/aws-sdk-java/run.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -# -# Mint (C) 2018 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# handle command line arguments -if [ $# -ne 2 ]; then - echo "usage: run.sh " - exit 1 -fi - -output_log_file="$1" -error_log_file="$2" - -# run tests -cd /mint/run/core/aws-sdk-java/ || exit 1 - -java -jar FunctionalTests.jar 1>>"$output_log_file" 2>"$error_log_file" diff --git a/mint/run/core/aws-sdk-php/README.md b/mint/run/core/aws-sdk-php/README.md deleted file mode 100644 index d08162d..0000000 --- a/mint/run/core/aws-sdk-php/README.md +++ /dev/null @@ -1,19 +0,0 @@ -## `aws-sdk-php` tests -This directory serves as the location for Mint tests using `aws-sdk-php`. Top level `mint.sh` calls `run.sh` to execute tests. - -## Adding new tests -New tests is added into `quick-tests.php` as new functions. - -## Running tests manually -- Set environment variables `MINT_DATA_DIR`, `MINT_MODE`, `SERVER_ENDPOINT`, `ACCESS_KEY`, `SECRET_KEY`, `SERVER_REGION` and `ENABLE_HTTPS` -- Call `run.sh` with output log file and error log file. for example -```bash -export MINT_DATA_DIR=~/my-mint-dir -export MINT_MODE=core -export SERVER_ENDPOINT="play.minio.io:9000" -export ACCESS_KEY="Q3AM3UQ867SPQQA43P2F" -export SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" -export ENABLE_HTTPS=1 -export SERVER_REGION=us-east-1 -./run.sh /tmp/output.log /tmp/error.log -``` diff --git a/mint/run/core/aws-sdk-php/composer.json b/mint/run/core/aws-sdk-php/composer.json deleted file mode 100644 index cf64a72..0000000 --- a/mint/run/core/aws-sdk-php/composer.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "require": { - "aws/aws-sdk-php": "^3.30", - "guzzlehttp/psr7": "^1.4" - } -} diff --git a/mint/run/core/aws-sdk-php/quick-tests.php b/mint/run/core/aws-sdk-php/quick-tests.php deleted file mode 100644 index cf00cb8..0000000 --- a/mint/run/core/aws-sdk-php/quick-tests.php +++ /dev/null @@ -1,1124 +0,0 @@ - 'val-1']; - -/** - * ClientConfig abstracts configuration details to connect to a - * S3-like service - */ -class ClientConfig { - public $creds; - public $endpoint; - public $region; - - function __construct(string $access_key, string $secret_key, string $host, string $secure, string $region) { - $this->creds = new Aws\Credentials\Credentials($access_key, $secret_key); - - if ($secure == "1") { - $this->endpoint = "https://" . $host; - } else { - $this->endpoint = "http://" . $host; - } - - $this->region = $region; - } -} - - /** - * randomName returns a name prefixed by aws-sdk-php using uniqid() - * from standard library - * - * @return string - */ -function randomName():string { - return uniqid("aws-sdk-php-"); -} - - /** - * getStatusCode returns HTTP status code of the given result. - * - * @param $result - AWS\S3 result object - * - * @return string - HTTP status code. E.g, "400" for Bad Request. - */ -function getStatusCode($result):string { - return $result->toArray()['@metadata']['statusCode']; -} - - /** - * runExceptionalTests executes a collection of tests that will throw - * a known exception. - * - * @param $s3Client AWS\S3\S3Client object - * - * @param $apiCall Name of the S3Client API method to call - * - * @param $exceptionMatcher Name of Aws\S3\Exception\S3Exception - * method to fetch exception details - * - * @param $exceptionParamMap Associative array of exception names to - * API parameters. E.g, - * $apiCall = 'headBucket' - * $exceptionMatcher = 'getStatusCode' - * $exceptionParamMap = [ - * // Non existent bucket - * '404' => ['Bucket' => $bucket['Name'] . '--'], - * - * // Non existent bucket - * '404' => ['Bucket' => $bucket['Name'] . '-non-existent'], - * ]; - * - * @return string - HTTP status code. E.g, "404" for Non existent bucket. - */ -function runExceptionalTests($s3Client, $apiCall, $exceptionMatcher, $exceptionParamMap) { - foreach($exceptionParamMap as $exn => $params) { - $exceptionCaught = false; - try { - $result = $s3Client->$apiCall($params); - } catch(Aws\S3\Exception\S3Exception $e) { - $exceptionCaught = true; - switch ($e->$exceptionMatcher()) { - case $exn: - // This is expected - continue; - default: - throw $e; - } - } - finally { - if (!$exceptionCaught) { - $message = sprintf("Expected %s to fail with %s", $apiCall, $exn); - throw new Exception($message); - } - } - } -} - - /** - * testListBuckets tests ListBuckets S3 API - * - * @param $s3Client AWS\S3\S3Client object - * - * @return void - */ -function testListBuckets(S3Client $s3Client) { - $buckets = $s3Client->listBuckets(); - $debugger = $GLOBALS['debugger']; - foreach ($buckets['Buckets'] as $bucket){ - $debugger->out($bucket['Name'] . "\n"); - } -} - - /** - * testBucketExists tests HEAD Bucket S3 API - * - * @param $s3Client AWS\S3\S3Client object - * - * @return void - */ -function testBucketExists(S3Client $s3Client) { - // List all buckets - $buckets = $s3Client->listBuckets(); - // All HEAD on existing buckets must return success - foreach($buckets['Buckets'] as $bucket) { - $result = $s3Client->headBucket(['Bucket' => $bucket['Name']]); - if (getStatusCode($result) != HTTP_OK) - throw new Exception('headBucket API failed for ' . $bucket['Name']); - } - - // Run failure tests - $params = [ - // Non existent bucket - '404' => ['Bucket' => $bucket['Name'] . '--'], - - // Non existent bucket - '404' => ['Bucket' => $bucket['Name'] . '-non-existent'], - ]; - runExceptionalTests($s3Client, 'headBucket', 'getStatusCode', $params); -} - - - /** - * testHeadObject tests HeadObject S3 API - * - * @param $s3Client AWS\S3\S3Client object - * - * @param $objects Associative array of buckets and objects - * - * @return void - */ -function testHeadObject($s3Client, $objects) { - foreach($objects as $bucket => $object) { - $result = $s3Client->headObject(['Bucket' => $bucket, 'Key' => $object]); - if (getStatusCode($result) != HTTP_OK) - throw new Exception('headObject API failed for ' . - $bucket . '/' . $object); - if (strtolower($result['Metadata']) != strtolower(TEST_METADATA)) { - throw new Exception("headObject API Metadata didn't match for " . - $bucket . '/' . $object); - } - } - - // Run failure tests - $params = [ - '404' => ['Bucket' => $bucket, 'Key' => $object . '-non-existent'] - ]; - runExceptionalTests($s3Client, 'headObject', 'getStatusCode', $params); -} - - /** - * testListObjects tests ListObjectsV1 and V2 S3 APIs - * - * @param $s3Client AWS\S3\S3Client object - * - * @param $params associative array containing bucket and object names - * - * @return void - */ -function testListObjects($s3Client, $params) { - $bucket = $params['Bucket']; - $object = $params['Object']; - $debugger = $GLOBALS['debugger']; - try { - for ($i = 0; $i < 5; $i++) { - $copyKey = $object . '-copy-' . strval($i); - $result = $s3Client->copyObject([ - 'Bucket' => $bucket, - 'Key' => $copyKey, - 'CopySource' => $bucket . '/' . $object, - ]); - if (getStatusCode($result) != HTTP_OK) - throw new Exception("copyObject API failed for " . $bucket . '/' . $object); - } - - $paginator = $s3Client->getPaginator('ListObjects', ['Bucket' => $bucket]); - foreach ($paginator->search('Contents[].Key') as $key) { - $debugger->out('key = ' . $key . "\n"); - } - - $paginator = $s3Client->getPaginator('ListObjectsV2', ['Bucket' => $bucket]); - foreach ($paginator->search('Contents[].Key') as $key) { - $debugger->out('key = ' . $key . "\n"); - } - - $prefix = 'obj'; - $result = $s3Client->listObjects(['Bucket' => $bucket, 'Prefix' => $prefix]); - if (getStatusCode($result) != HTTP_OK || $result['Prefix'] != $prefix) - throw new Exception("listObject API failed for " . $bucket . '/' . $object); - - $maxKeys = 1; - $result = $s3Client->listObjects(['Bucket' => $bucket, 'MaxKeys' => $maxKeys]); - if (getStatusCode($result) != HTTP_OK || count($result['Contents']) != $maxKeys) - throw new Exception("listObject API failed for " . $bucket . '/' . $object); - - $params = [ - 'InvalidArgument' => ['Bucket' => $bucket, 'MaxKeys' => -1], - 'NoSuchBucket' => ['Bucket' => $bucket . '-non-existent'] - ]; - runExceptionalTests($s3Client, 'listObjects', 'getAwsErrorCode', $params); - - } finally { - $s3Client->deleteObjects([ - 'Bucket' => $bucket, - 'Delete' => [ - 'Objects' => array_map(function($a, $b) { - return ['Key' => $a . '-copy-' . strval($b)]; - }, array_fill(0, 5, $object), range(0,4)) - ], - ]); - } -} - - /** - * testListMultipartUploads tests ListMultipartUploads, ListParts and - * UploadPartCopy S3 APIs - * - * @param $s3Client AWS\S3\S3Client object - * - * @param $params associative array containing bucket and object names - * - * @return void - */ -function testListMultipartUploads($s3Client, $params) { - $bucket = $params['Bucket']; - $object = $params['Object']; - $debugger = $GLOBALS['debugger']; - - $data_dir = $GLOBALS['MINT_DATA_DIR']; - // Initiate multipart upload - $result = $s3Client->createMultipartUpload([ - 'Bucket' => $bucket, - 'Key' => $object . '-copy', - ]); - if (getStatusCode($result) != HTTP_OK) - throw new Exception('createMultipartupload API failed for ' . - $bucket . '/' . $object); - - // upload 5 parts - $uploadId = $result['UploadId']; - $parts = []; - try { - for ($i = 0; $i < 5; $i++) { - $result = $s3Client->uploadPartCopy([ - 'Bucket' => $bucket, - 'Key' => $object . '-copy', - 'UploadId' => $uploadId, - 'PartNumber' => $i+1, - 'CopySource' => $bucket . '/' . $object, - ]); - if (getStatusCode($result) != HTTP_OK) { - throw new Exception('uploadPart API failed for ' . - $bucket . '/' . $object); - } - array_push($parts, [ - 'ETag' => $result['ETag'], - 'PartNumber' => $i+1, - ]); - } - - // ListMultipartUploads and ListParts may return empty - // responses in the case of minio gateway gcs and minio server - // FS mode. So, the following tests don't make assumptions on - // result response. - $paginator = $s3Client->getPaginator('ListMultipartUploads', - ['Bucket' => $bucket]); - foreach ($paginator->search('Uploads[].{Key: Key, UploadId: UploadId}') as $keyHash) { - $debugger->out('key = ' . $keyHash['Key'] . ' uploadId = ' . $keyHash['UploadId'] . "\n"); - } - - $paginator = $s3Client->getPaginator('ListParts', [ - 'Bucket' => $bucket, - 'Key' => $object . '-copy', - 'UploadId' => $uploadId, - ]); - foreach ($paginator->search('Parts[].{PartNumber: PartNumber, ETag: ETag}') as $partsHash) { - $debugger->out('partNumber = ' . $partsHash['PartNumber'] . ' ETag = ' . $partsHash['ETag'] . "\n"); - } - - }finally { - $s3Client->abortMultipartUpload([ - 'Bucket' => $bucket, - 'Key' => $object . '-copy', - 'UploadId' => $uploadId - ]); - } -} - - /** - * initSetup creates buckets and objects necessary for the functional - * tests to run - * - * @param $s3Client AWS\S3\S3Client object - * - * @param $objects Associative array of buckets and objects - * - * @return void - */ -function initSetup(S3Client $s3Client, $objects) { - $MINT_DATA_DIR = $GLOBALS['MINT_DATA_DIR']; - foreach($objects as $bucket => $object) { - $s3Client->createBucket(['Bucket' => $bucket]); - $stream = NULL; - try { - if (!file_exists($MINT_DATA_DIR . '/' . FILE_1_KB)) - throw new Exception('File not found ' . $MINT_DATA_DIR . '/' . FILE_1_KB); - - $stream = Psr7\stream_for(fopen($MINT_DATA_DIR . '/' . FILE_1_KB, 'r')); - $result = $s3Client->putObject([ - 'Bucket' => $bucket, - 'Key' => $object, - 'Body' => $stream, - 'Metadata' => TEST_METADATA, - ]); - if (getStatusCode($result) != HTTP_OK) - throw new Exception("putObject API failed for " . $bucket . '/' . $object); - } - - finally { - // close data file - if (!is_null($stream)) - $stream->close(); - } - } - - // Create an empty bucket for bucket policy + delete tests - $result = $s3Client->createBucket(['Bucket' => $GLOBALS['emptyBucket']]); - if (getStatusCode($result) != HTTP_OK) - throw new Exception("createBucket API failed for " . $bucket); - -} - - - /** - * testGetPutObject tests GET/PUT object S3 API - * - * @param $s3Client AWS\S3\S3Client object - * - * @param $params associative array containing bucket and object names - * - * @return void - */ -function testGetPutObject($s3Client, $params) { - $bucket = $params['Bucket']; - $object = $params['Object']; - - // Upload a 10KB file - $MINT_DATA_DIR = $GLOBALS['MINT_DATA_DIR']; - try { - $stream = Psr7\stream_for(fopen($MINT_DATA_DIR . '/' . FILE_1_KB, 'r')); - $result = $s3Client->putObject([ - 'Bucket' => $bucket, - 'Key' => $object, - 'Body' => $stream, - ]); - } - finally { - $stream->close(); - } - - if (getStatusCode($result) != HTTP_OK) - throw new Exception("putObject API failed for " . $bucket . '/' . $object); - - // Download the same object and verify size - $result = $s3Client->getObject([ - 'Bucket' => $bucket, - 'Key' => $object, - ]); - if (getStatusCode($result) != HTTP_OK) - throw new Exception("getObject API failed for " . $bucket . '/' . $object); - - $body = $result['Body']; - $bodyLen = 0; - while (!$body->eof()) { - $bodyLen += strlen($body->read(4096)); - } - - if ($bodyLen != 1 * 1024) { - throw new Exception("Object downloaded has different content length than uploaded object " - . $bucket . '/' . $object); - } -} - - /** - * testMultipartUploadFailure tests MultipartUpload failures - * - * @param $s3Client AWS\S3\S3Client object - * - * @param $params associative array containing bucket and object names - * - * @return void - */ -function testMultipartUploadFailure($s3Client, $params) { - $bucket = $params['Bucket']; - $object = $params['Object']; - - $MINT_DATA_DIR = $GLOBALS['MINT_DATA_DIR']; - // Initiate multipart upload - $result = $s3Client->createMultipartUpload([ - 'Bucket' => $bucket, - 'Key' => $object, - ]); - if (getStatusCode($result) != HTTP_OK) - throw new Exception('createMultipartupload API failed for ' . - $bucket . '/' . $object); - - // upload 2 parts - $uploadId = $result['UploadId']; - $parts = []; - try { - for ($i = 0; $i < 2; $i++) { - $stream = Psr7\stream_for(fopen($MINT_DATA_DIR . '/' . FILE_5_MB, 'r')); - $limitedStream = new Psr7\LimitStream($stream, 4 * 1024 * 1024, 0); - $result = $s3Client->uploadPart([ - 'Bucket' => $bucket, - 'Key' => $object, - 'UploadId' => $uploadId, - 'ContentLength' => 4 * 1024 * 1024, - 'Body' => $limitedStream, - 'PartNumber' => $i+1, - ]); - if (getStatusCode($result) != HTTP_OK) { - throw new Exception('uploadPart API failed for ' . - $bucket . '/' . $object); - } - array_push($parts, [ - 'ETag' => $result['ETag'], - 'PartNumber' => $i+1, - ]); - - $limitedStream->close(); - $limitedStream = NULL; - } - } - finally { - if (!is_null($limitedStream)) - $limitedStream->close(); - } - - $params = [ - 'EntityTooSmall' => [ - 'Bucket' => $bucket, - 'Key' => $object, - 'UploadId' => $uploadId, - 'MultipartUpload' => [ - 'Parts' => $parts, - ], - ], - 'NoSuchUpload' => [ - 'Bucket' => $bucket, - 'Key' => $object, - 'UploadId' => 'non-existent', - 'MultipartUpload' => [ - 'Parts' => $parts, - ], - ], - ]; - runExceptionalTests($s3Client, 'completeMultipartUpload', 'getAwsErrorCode', $params); -} - - /** - * testMultipartUpload tests MultipartUpload S3 APIs - * - * @param $s3Client AWS\S3\S3Client object - * - * @param $params associative array containing bucket and object names - * - * @return void - */ -function testMultipartUpload($s3Client, $params) { - $bucket = $params['Bucket']; - $object = $params['Object']; - - $MINT_DATA_DIR = $GLOBALS['MINT_DATA_DIR']; - // Initiate multipart upload - $result = $s3Client->createMultipartUpload([ - 'Bucket' => $bucket, - 'Key' => $object, - ]); - if (getStatusCode($result) != HTTP_OK) - throw new Exception('createMultipartupload API failed for ' . - $bucket . '/' . $object); - - // upload 2 parts - $uploadId = $result['UploadId']; - $parts = []; - try { - for ($i = 0; $i < 2; $i++) { - $stream = Psr7\stream_for(fopen($MINT_DATA_DIR . '/' . FILE_5_MB, 'r')); - $result = $s3Client->uploadPart([ - 'Bucket' => $bucket, - 'Key' => $object, - 'UploadId' => $uploadId, - 'ContentLength' => 5 * 1024 * 1024, - 'Body' => $stream, - 'PartNumber' => $i+1, - ]); - if (getStatusCode($result) != HTTP_OK) { - throw new Exception('uploadPart API failed for ' . - $bucket . '/' . $object); - } - array_push($parts, [ - 'ETag' => $result['ETag'], - 'PartNumber' => $i+1, - ]); - - $stream->close(); - $stream = NULL; - } - } - finally { - if (!is_null($stream)) - $stream->close(); - } - - // complete multipart upload - $result = $s3Client->completeMultipartUpload([ - 'Bucket' => $bucket, - 'Key' => $object, - 'UploadId' => $uploadId, - 'MultipartUpload' => [ - 'Parts' => $parts, - ], - ]); - if (getStatusCode($result) != HTTP_OK) { - throw new Exception('completeMultipartupload API failed for ' . - $bucket . '/' . $object); - } -} - - /** - * testAbortMultipartUpload tests aborting of a multipart upload - * - * @param $s3Client AWS\S3\S3Client object - * - * @param $params associative array containing bucket and object names - * - * @return void - */ -function testAbortMultipartUpload($s3Client, $params) { - $bucket = $params['Bucket']; - $object = $params['Object']; - - $MINT_DATA_DIR = $GLOBALS['MINT_DATA_DIR']; - // Initiate multipart upload - $result = $s3Client->createMultipartUpload([ - 'Bucket' => $bucket, - 'Key' => $object, - ]); - if (getStatusCode($result) != HTTP_OK) - throw new Exception('createMultipartupload API failed for ' . - $bucket . '/' . $object); - - // Abort multipart upload - $uploadId = $result['UploadId']; - $result = $s3Client->abortMultipartUpload([ - 'Bucket' => $bucket, - 'Key' => $object, - 'UploadId' => $uploadId, - ]); - if (getStatusCode($result) != HTTP_NOCONTENT) - throw new Exception('abortMultipartupload API failed for ' . - $bucket . '/' . $object); - - //Run failure tests - $params = [ - // Upload doesn't exist - 'NoSuchUpload' => [ - 'Bucket' => $bucket, - 'Key' => $object, - 'UploadId' => 'non-existent', - ], - ]; - runExceptionalTests($s3Client, 'abortMultipartUpload', 'getAwsErrorCode', $params); -} - - /** - * testGetBucketLocation tests GET bucket location S3 API - * - * @param $s3Client AWS\S3\S3Client object - * - * @param $params associative array containing bucket name - * - * @return void - */ -function testGetBucketLocation($s3Client, $params) { - $bucket = $params['Bucket']; - - // Valid test - $result = $s3Client->getBucketLocation(['Bucket' => $bucket]); - if (getStatusCode($result) != HTTP_OK) - throw new Exception('getBucketLocation API failed for ' . - $bucket); - - // Run failure tests. - $params = [ - // Non existent bucket - 'NoSuchBucket' => ['Bucket' => $bucket . '--'], - - // Bucket not found - 'NoSuchBucket' => ['Bucket' => $bucket . '-non-existent'], - ]; - runExceptionalTests($s3Client, 'getBucketLocation', 'getAwsErrorCode', $params); -} - - /** - * testCopyObject tests copy object S3 API - * - * @param $s3Client AWS\S3\S3Client object - * - * @param $params associative array containing bucket and object name - * - * @return void - */ -function testCopyObject($s3Client, $params) { - $bucket = $params['Bucket']; - $object = $params['Object']; - - $result = $s3Client->copyObject([ - 'Bucket' => $bucket, - 'Key' => $object . '-copy', - 'CopySource' => $bucket . '/' . $object, - ]); - if (getStatusCode($result) != HTTP_OK) - throw new Exception('copyObject API failed for ' . - $bucket); - - $s3Client->deleteObject([ - 'Bucket' => $bucket, - 'Key' => $object . '-copy', - ]); - - // Run failure tests - $params = [ - // Invalid copy source format - 'InvalidArgument' => [ - 'Bucket' => $bucket, - 'Key' => $object . '-copy', - 'CopySource' => $bucket . $object - ], - - // Missing source object - 'NoSuchKey' => [ - 'Bucket' => $bucket, - 'Key' => $object . '-copy', - 'CopySource' => $bucket . '/' . $object . '-non-existent' - ], - ]; - runExceptionalTests($s3Client, 'copyObject', 'getAwsErrorCode', $params); -} - - /** - * testDeleteObjects tests Delete Objects S3 API - * - * @param $s3Client AWS\S3\S3Client object - * - * @param $params associative array containing bucket and object names - * - * @return void - */ -function testDeleteObjects($s3Client, $params) { - $bucket = $params['Bucket']; - $object = $params['Object']; - - $copies = []; - for ($i = 0; $i < 3; $i++) { - $copyKey = $object . '-copy' . strval($i); - $result = $s3Client->copyObject([ - 'Bucket' => $bucket, - 'Key' => $copyKey, - 'CopySource' => $bucket . '/' . $object, - ]); - if (getstatuscode($result) != HTTP_OK) - throw new Exception('copyobject API failed for ' . - $bucket); - array_push($copies, ['Key' => $copyKey]); - } - - $result = $s3Client->deleteObjects([ - 'Bucket' => $bucket, - 'Delete' => [ - 'Objects' => $copies, - ], - ]); - if (getstatuscode($result) != HTTP_OK) - throw new Exception('deleteObjects api failed for ' . - $bucket); -} - - /** - * testAnonDeleteObjects tests Delete Objects S3 API for anonymous requests. - * The test case checks this scenario: - * http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html#multiobjectdeleteapi-examples - * - * @param $s3Client AWS\S3\S3Client object - * - * @param $params associative array containing bucket and object names - * - * @return void - */ -function testAnonDeleteObjects($s3Client, $params) { - $bucket = $params['Bucket']; - $object = $params['Object']; - - // Create anonymous config object - $anonConfig = new ClientConfig("", "", $GLOBALS['endpoint'], $GLOBALS['secure'], $GLOBALS['region']); - - // Create anonymous S3 client - $anonymousClient = new S3Client([ - 'credentials' => false, - 'endpoint' => $anonConfig->endpoint, - 'use_path_style_endpoint' => true, - 'region' => $anonConfig->region, - 'version' => '2006-03-01' - ]); - - $copies = []; - for ($i = 0; $i < 3; $i++) { - $copyKey = $object . '-copy' . strval($i); - $result = $s3Client->copyObject([ - 'Bucket' => $bucket, - 'Key' => $copyKey, - 'CopySource' => $bucket . '/' . $object, - ]); - if (getstatuscode($result) != HTTP_OK) - throw new Exception('copyobject API failed for ' . - $bucket); - array_push($copies, ['Key' => $copyKey]); - } - - // Try anonymous delete. - $result = $anonymousClient->deleteObjects([ - 'Bucket' => $bucket, - 'Delete' => [ - 'Objects' => $copies, - ], - ]); - // Response code should be 200 - if (getstatuscode($result) != HTTP_OK) - throw new Exception('deleteObjects returned incorrect response ' . - getStatusCode($result)); - - // Each object should have error code AccessDenied - for ($i = 0; $i < 3; $i++) { - if ($result["Errors"][$i]["Code"] != "AccessDenied") - throw new Exception('Incorrect response deleteObjects anonymous - call for ' .$bucket); - } - - // Delete objects after the test passed - $result = $s3Client->deleteObjects([ - 'Bucket' => $bucket, - 'Delete' => [ - 'Objects' => $copies, - ], - ]); - - if (getstatuscode($result) != HTTP_OK) - throw new Exception('deleteObjects api failed for ' . - $bucket); - - // Each object should have empty code in case of successful delete - for ($i = 0; $i < 3; $i++) { - if ($result["Errors"][$i]["Code"] != "") - throw new Exception('Incorrect response deleteObjects anonymous - call for ' .$bucket); - } -} - -// Check if the policy statements are equal -function are_statements_equal($expected, $got) { - $expected = json_decode($expected, TRUE); - $got = json_decode($got, TRUE); - - function are_actions_equal($action1, $action2) { - return ( - is_array($action1) - && is_array($action2) - && count($action1) == count($action2) - && array_diff($action1, $action2) === array_diff($action2, $action1) - ); - } - - foreach ($expected['Statement'] as $index => $value) { - if (!are_actions_equal($value['Action'], $got['Statement'][$index]['Action'])) - return FALSE; - } - - return TRUE; - -} - /** - * testBucketPolicy tests GET/PUT Bucket policy S3 APIs - * - * @param $s3Client AWS\S3\S3Client object - * - * @param $params associative array containing bucket and object names - * - * @return void - */ -function testBucketPolicy($s3Client, $params) { - $bucket = $params['Bucket']; - - $downloadPolicy = sprintf('{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:GetBucketLocation","s3:ListBucket","s3:GetObject"],"Resource":["arn:aws:s3:::%s","arn:aws:s3:::%s/*"]}]}', $bucket, $bucket); - - $result = $s3Client->putBucketPolicy([ - 'Bucket' => $bucket, - 'Policy' => $downloadPolicy - ]); - if (getstatuscode($result) != HTTP_NOCONTENT) - throw new Exception('putBucketPolicy API failed for ' . - $bucket); - $result = $s3Client->getBucketPolicy(['Bucket' => $bucket]); - if (getstatuscode($result) != HTTP_OK) - throw new Exception('putBucketPolicy API failed for ' . - $bucket); - - if ($result['Policy'] != $downloadPolicy) - if (!are_statements_equal($result['Policy'], $downloadPolicy)) - throw new Exception('bucket policy we got is not we set'); - - - // Delete the bucket, make the bucket (again) and check if policy is none - // Ref: https://github.com/minio/minio/issues/4714 - $result = $s3Client->deleteBucket(['Bucket' => $bucket]); - if (getstatuscode($result) != HTTP_NOCONTENT) - throw new Exception('deleteBucket API failed for ' . - $bucket); - - try { - $s3Client->getBucketPolicy(['Bucket' => $bucket]); - } catch (AWSException $e) { - switch ($e->getAwsErrorCode()) { - case 'NoSuchBucket': - break; - } - } - - // Sleep is needed for Minio Gateway for Azure, ref: - // https://docs.microsoft.com/en-us/rest/api/storageservices/Delete-Container#remarks - sleep(40); - - $s3Client->createBucket(['Bucket' => $bucket]); - - $params = [ - '404' => ['Bucket' => $bucket] - ]; - runExceptionalTests($s3Client, 'getBucketPolicy', 'getStatusCode', $params); - - try { - $MINT_DATA_DIR = $GLOBALS['MINT_DATA_DIR']; - // Create an object to test anonymous GET object - $object = 'test-anon'; - if (!file_exists($MINT_DATA_DIR . '/' . FILE_1_KB)) - throw new Exception('File not found ' . $MINT_DATA_DIR . '/' . FILE_1_KB); - - $stream = Psr7\stream_for(fopen($MINT_DATA_DIR . '/' . FILE_1_KB, 'r')); - $result = $s3Client->putObject([ - 'Bucket' => $bucket, - 'Key' => $object, - 'Body' => $stream, - ]); - if (getstatuscode($result) != HTTP_OK) - throw new Exception('createBucket API failed for ' . - $bucket); - - $anonConfig = new ClientConfig("", "", $GLOBALS['endpoint'], $GLOBALS['secure'], $GLOBALS['region']); - $anonymousClient = new S3Client([ - 'credentials' => false, - 'endpoint' => $anonConfig->endpoint, - 'use_path_style_endpoint' => true, - 'region' => $anonConfig->region, - 'version' => '2006-03-01' - ]); - runExceptionalTests($anonymousClient, 'getObject', 'getStatusCode', [ - '403' => [ - 'Bucket' => $bucket, - 'Key' => $object, - ] - ]); - - } finally { - // close data file - if (!is_null($stream)) - $stream->close(); - $s3Client->deleteObject(['Bucket' => $bucket, 'Key' => $object]); - } - -} - - /** - * cleanupSetup removes all buckets and objects created during the - * functional test - * - * @param $s3Client AWS\S3\S3Client object - * - * @param $objects Associative array of buckets to objects - * - * @return void - */ -function cleanupSetup($s3Client, $objects) { - // Delete all objects - foreach ($objects as $bucket => $object) { - $s3Client->deleteObject(['Bucket' => $bucket, 'Key' => $object]); - } - - // Delete the buckets incl. emptyBucket - $allBuckets = array_keys($objects); - array_push($allBuckets, $GLOBALS['emptyBucket']); - foreach ($allBuckets as $bucket) { - try { - // Delete the bucket - $s3Client->deleteBucket(['Bucket' => $bucket]); - - // Wait until the bucket is removed from object store - $s3Client->waitUntil('BucketNotExists', ['Bucket' => $bucket]); - } catch (Exception $e) { - // Ignore exceptions thrown during cleanup - } - } -} - - - /** - * runTest helper function to wrap a test function and log - * success or failure accordingly. - * - * @param myfunc name of test function to be run - * - * @param fnSignature function signature of the main S3 SDK API - * - * @param args parameters to be passed to test function - * - * @return void - */ -function runTest($s3Client, $myfunc, $fnSignature, $args = []) { - try { - $start_time = microtime(true); - $status = "PASS"; - $error = ""; - $message = ""; - $myfunc($s3Client, $args); - } catch (AwsException $e) { - $errorCode = $e->getAwsErrorCode(); - // $fnSignature holds the specific API that is being - // tested. It is possible that functions used to create the - // test setup may not be implemented. - if ($errorCode != "NotImplemented") { - $status = "FAIL"; - $error = $e->getMessage(); - } else { - $status = "NA"; - $error = $e->getMessage(); - $alert = sprintf("%s or a related API is NOT IMPLEMENTED, see \"error\" for exact details.", $fnSignature); - } - - } catch (Exception $e) { - // This exception handler handles high-level custom exceptions. - $status = "FAIL"; - $error = $e->getMessage(); - } finally { - $end_time = microtime(true); - $json_log = [ - "name" => "aws-sdk-php", - "function" => $fnSignature, - "args" => $args, - "duration" => sprintf("%d", ($end_time - $start_time) * 1000), // elapsed time in ms - "status" => $status, - ]; - if ($error !== "") { - $json_log["error"] = $error; - } - if ($message !== "") { - $json_log["message"] = $message; - } - print_r(json_encode($json_log)."\n"); - - // Exit on first failure. - switch ($status) { - case "FAIL": - exit(1); - } - } -} - -// Get client configuration from environment variables -$GLOBALS['access_key'] = getenv("ACCESS_KEY"); -$GLOBALS['secret_key'] = getenv("SECRET_KEY"); -$GLOBALS['endpoint'] = getenv("SERVER_ENDPOINT"); -$GLOBALS['region'] = getenv("SERVER_REGION"); -$GLOBALS['secure'] = getenv("ENABLE_HTTPS"); - -/** - * @global string $GLOBALS['MINT_DATA_DIR'] - * @name $MINT_DATA_DIR - */ -$GLOBALS['MINT_DATA_DIR'] = '/mint/data'; -$GLOBALS['MINT_DATA_DIR'] = getenv("MINT_DATA_DIR"); - - -// Useful for debugging test failures; Set $debugmode it to true when required -$debugmode = false; - -interface Debugger { - public function out($data); -} - -class EchoDebugger implements Debugger { - public function out($data) { - echo $data; - } -} - -class NullDebugger implements Debugger { - public function out($data) { - // Do nothing - } -} - -if($debugmode) - $debugger = new EchoDebugger(); -else - $debugger = new NullDebugger(); - -// Make $debugger global -$GLOBALS['debugger'] = $debugger; - -// Create config object -$config = new ClientConfig($GLOBALS['access_key'], $GLOBALS['secret_key'], - $GLOBALS['endpoint'], $GLOBALS['secure'], - $GLOBALS['region']); - -// Create a S3Client -$s3Client = new S3Client([ - 'credentials' => $config->creds, - 'endpoint' => $config->endpoint, - 'use_path_style_endpoint' => true, - 'region' => $config->region, - 'version' => '2006-03-01' -]); - -// Used by initSetup -$emptyBucket = randomName(); -$objects = [ - randomName() => 'obj1', - randomName() => 'obj2', -]; - -try { - initSetup($s3Client, $objects); - $firstBucket = array_keys($objects)[0]; - $firstObject = $objects[$firstBucket]; - $testParams = ['Bucket' => $firstBucket, 'Object' => $firstObject]; - runTest($s3Client, 'testGetBucketLocation', "getBucketLocation ( array \$params = [] )", ['Bucket' => $firstBucket]); - runTest($s3Client, 'testListBuckets', "listBuckets ( array \$params = [] )"); - runTest($s3Client, 'testListObjects', "listObjects ( array \$params = [] )", $testParams); - runTest($s3Client, 'testListMultipartUploads', "listMultipartUploads ( array \$params = [] )", $testParams); - runTest($s3Client, 'testBucketExists', "headBucket ( array \$params = [] )", array_keys($objects)); - runTest($s3Client, 'testHeadObject', "headObject ( array \$params = [] )", $objects); - runTest($s3Client, 'testGetPutObject', "getObject ( array \$params = [] )", $testParams); - runTest($s3Client, 'testCopyObject', "copyObject ( array \$params = [] )", $testParams); - runTest($s3Client, 'testDeleteObjects', "deleteObjects (array \$params = [] )", $testParams); - runTest($s3Client, 'testAnonDeleteObjects', "anonDeleteObjects ( array \$params = [] )", $testParams); - runTest($s3Client, 'testMultipartUpload', "createMultipartUpload ( array \$params = [] )", $testParams); - runTest($s3Client, 'testMultipartUploadFailure', "uploadPart ( array \$params = [] )", $testParams); - runTest($s3Client, 'testAbortMultipartUpload', "abortMultipartupload ( array \$params = [] )", $testParams); - runTest($s3Client, 'testBucketPolicy', "getBucketPolicy ( array \$params = [] )", ['Bucket' => $emptyBucket]); -} -finally { - cleanupSetup($s3Client, $objects); -} - -?> diff --git a/mint/run/core/aws-sdk-php/run.sh b/mint/run/core/aws-sdk-php/run.sh deleted file mode 100755 index e3a37a3..0000000 --- a/mint/run/core/aws-sdk-php/run.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -# -# Mint, (C) 2017 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# handle command line arguments -if [ $# -ne 2 ]; then - echo "usage: run.sh " - exit 1 -fi - -output_log_file="$1" -error_log_file="$2" - -# run tests -php ./quick-tests.php 1>>"$output_log_file" 2>"$error_log_file" diff --git a/mint/run/core/aws-sdk-ruby/README.md b/mint/run/core/aws-sdk-ruby/README.md deleted file mode 100644 index 7c66ada..0000000 --- a/mint/run/core/aws-sdk-ruby/README.md +++ /dev/null @@ -1,19 +0,0 @@ -## `aws-sdk-ruby` tests -This directory serves as the location for Mint tests using `aws-sdk-ruby`. Top level `mint.sh` calls `run.sh` to execute tests. - -## Adding new tests -New tests is added into `aws-stub-test.rb` as new functions. - -## Running tests manually -- Set environment variables `MINT_DATA_DIR`, `MINT_MODE`, `SERVER_ENDPOINT`, `ACCESS_KEY`, `SECRET_KEY`, `SERVER_REGION` and `ENABLE_HTTPS` -- Call `run.sh` with output log file and error log file. for example -```bash -export MINT_DATA_DIR=~/my-mint-dir -export MINT_MODE=core -export SERVER_ENDPOINT="play.minio.io:9000" -export ACCESS_KEY="Q3AM3UQ867SPQQA43P2F" -export SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" -export ENABLE_HTTPS=1 -export SERVER_REGION=us-east-1 -./run.sh /tmp/output.log /tmp/error.log -``` diff --git a/mint/run/core/aws-sdk-ruby/aws-stub-tests.rb b/mint/run/core/aws-sdk-ruby/aws-stub-tests.rb deleted file mode 100755 index 185004d..0000000 --- a/mint/run/core/aws-sdk-ruby/aws-stub-tests.rb +++ /dev/null @@ -1,855 +0,0 @@ -#!/usr/bin/env ruby -# -# Mint (C) 2017 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -require 'aws-sdk' -require 'securerandom' -require 'net/http' -require 'multipart_body' - -class AwsSdkRubyTest - # Set variables necessary to create an s3 client instance. - # Get them from the environment variables - - # Region information, eg. "us-east-1" - region = ENV['SERVER_REGION'] ||= 'SERVER_REGION is not set' - # Minio server, eg. "play.minio.io:9000" - endpoint = ENV['SERVER_ENDPOINT'] ||= 'SERVER_ENDPOINT is not set' - access_key_id = ENV['ACCESS_KEY'] ||= 'ACCESS_KEY is not set' - secret_access_key = ENV['SECRET_KEY'] ||= 'SECRET_KEY is not set' - enable_https = ENV['ENABLE_HTTPS'] - endpoint = enable_https == '1' ? 'https://' + endpoint : 'http://' + endpoint - # Create s3 client instances, "s3Resource" and "s3Client" - @@s3 = Aws::S3::Resource.new(region: region, - endpoint: endpoint, - access_key_id: access_key_id, - secret_access_key: secret_access_key, - force_path_style: true) - - def initialize_log_output(meth, alert = nil) - # Initialize and return log content in log_output hash table - - # Collect args in args_arr - args_arr = method(meth).parameters.flatten.map(&:to_s) - .reject { |x| x == 'req' || x == 'opt' } - # Create and return log output content - { name: 'aws-sdk-ruby', - function: "#{meth}(#{args_arr.join(',')})", # method name and arguments - args: args_arr, # array of arg names. This'll be replaced with a - # a arg/value pairs insdie the caller method - duration: 0, # test runtime duration in seconds - alert: alert, - message: nil, - error: nil } - end - - def get_random_bucket_name() - bucket_name = "aws-sdk-ruby-bucket-"+SecureRandom.hex(6) - return bucket_name - end - - def calculate_duration(t2, t1) - # Durations are in miliseconds, with precision of 2 decimal places - ((t2 - t1) * 1000).round(2) - end - - def print_log(log_output, start_time) - # Calculate duration in miliseconds - log_output[:duration] = calculate_duration(Time.now, start_time) - # Get rid of the log_output fields if nil - puts log_output.delete_if{|k, value| value == nil}.to_json - # Exit at the first failure - exit 1 if log_output[:status] == 'FAIL' - end - - def cleanUp(buckets, log_output) - # Removes objects and bucket if bucket exists - bucket_name = '' - buckets.each do |b| - bucket_name = b - if bucketExistsWrapper(b, log_output) - removeObjectsWrapper(b, log_output) - removeBucketWrapper(b, log_output) - end - end - rescue => e - raise "Failed to clean-up bucket '#{bucket_name}', #{e}" - end - - # - # API commands/methods - # - def makeBucket(bucket_name) - # Creates a bucket, "bucket_name" - # on S3 client , "s3". - # Returns bucket_name if already exists - @@s3.bucket(bucket_name).exists? ? @@s3.bucket(bucket_name) : @@s3.create_bucket(bucket: bucket_name) - rescue => e - raise e - end - - def makeBucketWrapper(bucket_name, log_output) - makeBucket(bucket_name) - rescue => e - log_output[:function] = "makeBucket(bucket_name)" - log_output[:args] = {'bucket_name': bucket_name} - raise e - end - - def removeBucket(bucket_name) - # Deletes/removes bucket, "bucket_name" on S3 client, "s3" - @@s3.bucket(bucket_name).delete - rescue => e - raise e - end - - def removeBucketWrapper(bucket_name, log_output) - removeBucket(bucket_name) - rescue => e - log_output[:function] = "removeBucket(bucket_name)" - log_output[:args] = {'bucket_name': bucket_name} - raise e - end - - def putObject(bucket_name, file) - # Creates "file" (full path) in bucket, "bucket_name", - # on S3 client, "s3" - file_name = File.basename(file) - @@s3.bucket(bucket_name).object(file_name).upload_file(file) - rescue => e - raise e - end - - def putObjectWrapper(bucket_name, file, log_output) - putObject(bucket_name, file) - rescue => e - log_output[:function] = "putObject(bucket_name, file)" - log_output[:args] = {'bucket_name': bucket_name, - 'file': file} - raise e - end - - def getObject(bucket_name, file, destination) - # Gets/Downloads file, "file", - # from bucket, "bucket_name", of S3 client, "s3" - file_name = File.basename(file) - dest = File.join(destination, file_name) - @@s3.bucket(bucket_name).object(file_name).get(response_target: dest) - rescue => e - raise e - end - - def getObjectWrapper(bucket_name, file, destination, log_output) - getObject(bucket_name, file, destination) - rescue => e - log_output[:function] = "getObject(bucket_name, file)" - log_output[:args] = {'bucket_name': bucket_name, - 'file': file, - 'destination': destination} - raise e - end - - def copyObject(source_bucket_name, target_bucket_name, source_file_name, target_file_name = '') - # Copies file, "file_name", from source bucket, - # "source_bucket_name", to target bucket, - # "target_bucket_name", on S3 client, "s3" - target_file_name = source_file_name if target_file_name.empty? - source = @@s3.bucket(source_bucket_name) - target = @@s3.bucket(target_bucket_name) - source_obj = source.object(source_file_name) - target_obj = target.object(target_file_name) - source_obj.copy_to(target_obj) - rescue => e - raise e - end - - def copyObjectWrapper(source_bucket_name, target_bucket_name, source_file_name, target_file_name = '', log_output) - copyObject(source_bucket_name, target_bucket_name, source_file_name, target_file_name) - rescue => e - log_output[:function] = 'copyObject(source_bucket_name, target_bucket_name, source_file_name, target_file_name = '')' - log_output[:args] = {'source_bucket_name': source_bucket_name, - 'target_bucket_name': target_bucket_name, - 'source_file_name': source_file_name, - 'target_file_name': target_file_name} - raise e - end - - def removeObject(bucket_name, file) - # Deletes file in bucket, - # "bucket_name", on S3 client, "s3". - # If file, "file_name" does not exist, - # it quietly returns without any error message - @@s3.bucket(bucket_name).object(file).delete - rescue => e - raise e - end - - def removeObjectWrapper(bucket_name, file_name, log_output) - removeObject(bucket_name, file_name) - rescue => e - log_output[:function] = "removeObject(bucket_name, file_name)" - log_output[:args] = {'bucket_name': bucket_name, - 'file_name': file_name} - raise e - end - - def removeObjects(bucket_name) - # Deletes all files in bucket, "bucket_name" - # on S3 client, "s3" - file_name = '' - @@s3.bucket(bucket_name).objects.each do |obj| - file_name = obj.key - obj.delete - end - rescue => e - raise "File name: '#{file_name}', #{e}" - end - - def removeObjectsWrapper(bucket_name, log_output) - removeObjects(bucket_name) - rescue => e - log_output[:function] = 'removeObjects(bucket_name)' - log_output[:args] = {'bucket_name': bucket_name} - raise e - end - - def listBuckets - # Returns an array of bucket names on S3 client, "s3" - bucket_name_list = [] - @@s3.buckets.each do |b| - bucket_name_list.push(b.name) - end - return bucket_name_list - rescue => e - raise e - end - - def listBucketsWrapper(log_output) - listBuckets - rescue => e - log_output[:function] = 'listBuckets' - log_output[:args] = {} - raise e - end - - def listObjects(bucket_name) - # Returns an array of object/file names - # in bucket, "bucket_name", on S3 client, "s3" - object_list = [] - @@s3.bucket(bucket_name).objects.each do |obj| - object_list.push(obj.key) - end - return object_list - rescue => e - raise e - end - - def listObjectsWrapper(bucket_name, log_output) - listObjects(bucket_name) - rescue => e - log_output[:function] = 'listObjects(bucket_name)' - log_output[:args] = {'bucket_name': bucket_name} - raise e - end - - def statObject(bucket_name, file_name) - return @@s3.bucket(bucket_name).object(file_name).exists? - rescue => e - raise e - end - - def statObjectWrapper(bucket_name, file_name, log_output) - statObject(bucket_name, file_name) - rescue => e - log_output[:function] = 'statObject(bucket_name, file_name)' - log_output[:args] = {'bucket_name': bucket_name, - 'file_name': file_name} - raise e - end - - def bucketExists?(bucket_name) - # Returns true if bucket, "bucket_name", exists, - # false otherwise - return @@s3.bucket(bucket_name).exists? - rescue => e - raise e - end - - def bucketExistsWrapper(bucket_name, log_output) - bucketExists?(bucket_name) - rescue => e - log_output[:function] = 'bucketExists?(bucket_name)' - log_output[:args] = {'bucket_name': bucket_name} - raise e - end - - def presignedGet(bucket_name, file_name) - # Returns download/get url - obj = @@s3.bucket(bucket_name).object(file_name) - return obj.presigned_url(:get, expires_in: 600) - rescue => e - raise e - end - - def presignedGetWrapper(bucket_name, file_name, log_output) - presignedGet(bucket_name, file_name) - rescue => e - log_output[:function] = 'presignedGet(bucket_name, file_name)' - log_output[:args] = {'bucket_name': bucket_name, - 'file_name': file_name} - raise e - end - - def presignedPut(bucket_name, file_name) - # Returns put url - obj = @@s3.bucket(bucket_name).object(file_name) - return obj.presigned_url(:put, expires_in: 600) - rescue => e - raise e - end - - def presignedPutWrapper(bucket_name, file_name, log_output) - presignedPut(bucket_name, file_name) - rescue => e - log_output[:function] = 'presignedPut(bucket_name, file_name)' - log_output[:args] = {'bucket_name': bucket_name, - 'file_name': file_name} - raise e - end - - def presignedPost(bucket_name, file_name, expires_in_sec, max_byte_size) - # Returns upload/post url - obj = @@s3.bucket(bucket_name).object(file_name) - return obj.presigned_post(expires: Time.now + expires_in_sec, - content_length_range: 1..max_byte_size) - rescue => e - raise e - end - - def presignedPostWrapper(bucket_name, file_name, expires_in_sec, max_byte_size, log_output) - presignedPost(bucket_name, file_name, expires_in_sec, max_byte_size) - rescue => e - log_output[:function] = 'presignedPost(bucket_name, file_name, expires_in_sec, max_byte_size)' - log_output[:args] = {'bucket_name': bucket_name, - 'file_name': file_name, - 'expires_in_sec': expires_in_sec, - 'max_byte_size': max_byte_size} - raise e - end - - # To be addressed. S3 API 'get_bucket_policy' does not work! - # def getBucketPolicy(bucket_name) - # # Returns bucket policy - # return @@s3.bucket(bucket_name).get_bucket_policy - # rescue => e - # raise e - # end - - # - # Test case methods - # - def listBucketsTest() - # Tests listBuckets api command by creating - # new buckets from bucket_name_list - - # get random bucket names and create list - bucket_name1 = get_random_bucket_name() - bucket_name2 = get_random_bucket_name() - bucket_name_list = [bucket_name1, bucket_name2] - # Initialize hash table, 'log_output' - log_output = initialize_log_output('listBuckets') - # Prepare arg/value hash table and set it in log_output - arg_value_hash = {} - log_output[:args].each { |x| arg_value_hash[:"#{x}"] = eval x.to_s } - log_output[:args] = arg_value_hash - - begin - start_time = Time.now - prev_total_buckets = listBucketsWrapper(log_output).length - new_buckets = bucket_name_list.length - bucket_name_list.each do |b| - makeBucketWrapper(b, log_output) - end - new_total_buckets = prev_total_buckets + new_buckets - if new_total_buckets >= prev_total_buckets + new_buckets - log_output[:status] = 'PASS' - else - log_output[:error] = 'Could not find expected number of buckets' - log_output[:status] = 'FAIL' - end - cleanUp(bucket_name_list, log_output) - rescue => log_output[:error] - log_output[:status] = 'FAIL' - end - - print_log(log_output, start_time) - end - - def makeBucketTest() - # Tests makeBucket api command. - - # get random bucket name - bucket_name = get_random_bucket_name() - # Initialize hash table, 'log_output' - log_output = initialize_log_output('makeBucket') - # Prepare arg/value hash table and set it in log_output - arg_value_hash = {} - log_output[:args].each { |x| arg_value_hash[:"#{x}"] = eval x.to_s } - log_output[:args] = arg_value_hash - - begin - start_time = Time.now - makeBucketWrapper(bucket_name, log_output) - - if bucketExistsWrapper(bucket_name, log_output) - log_output[:status] = 'PASS' - else - log_output[:error] = 'Bucket expected to be created does not exist' - log_output[:status] = 'FAIL' - end - cleanUp([bucket_name], log_output) - rescue => log_output[:error] - log_output[:status] = 'FAIL' - end - - print_log(log_output, start_time) - end - - def bucketExistsNegativeTest() - # Tests bucketExists api command. - - # get random bucket name - bucket_name = get_random_bucket_name() - # Initialize hash table, 'log_output' - log_output = initialize_log_output('bucketExists?') - # Prepare arg/value hash table and set it in log_output - arg_value_hash = {} - log_output[:args].each { |x| arg_value_hash[:"#{x}"] = eval x.to_s } - log_output[:args] = arg_value_hash - - begin - start_time = Time.now - if !bucketExistsWrapper(bucket_name, log_output) - log_output[:status] = 'PASS' - else - log_output[:error] = "Failed to return 'false' for a non-existing bucket" - log_output[:status] = 'FAIL' - end - cleanUp([bucket_name], log_output) - rescue => log_output[:error] - log_output[:status] = 'FAIL' - end - - print_log(log_output, start_time) - end - - def removeBucketTest() - # Tests removeBucket api command. - - # get a random bucket name - bucket_name = get_random_bucket_name() - # Initialize hash table, 'log_output' - log_output = initialize_log_output('removeBucket') - # Prepare arg/value hash table and set it in log_output - arg_value_hash = {} - log_output[:args].each { |x| arg_value_hash[:"#{x}"] = eval x.to_s } - log_output[:args] = arg_value_hash - - begin - start_time = Time.now - makeBucketWrapper(bucket_name, log_output) - removeBucketWrapper(bucket_name, log_output) - if !bucketExistsWrapper(bucket_name, log_output) - log_output[:status] = 'PASS' - else - log_output[:error] = 'Bucket expected to be removed still exists' - log_output[:status] = 'FAIL' - end - cleanUp([bucket_name], log_output) - rescue => log_output[:error] - log_output[:status] = 'FAIL' - end - - print_log(log_output, start_time) - end - - def putObjectTest(file) - # Tests putObject api command by uploading a file - - # get random bucket name - bucket_name = get_random_bucket_name() - # Initialize hash table, 'log_output' - log_output = initialize_log_output('putObject') - # Prepare arg/value hash table and set it in log_output - arg_value_hash = {} - log_output[:args].each { |x| arg_value_hash[:"#{x}"] = eval x.to_s } - log_output[:args] = arg_value_hash - - begin - start_time = Time.now - makeBucketWrapper(bucket_name, log_output) - putObjectWrapper(bucket_name, file, log_output) - if statObjectWrapper(bucket_name, File.basename(file), log_output) - log_output[:status] = 'PASS' - else - log_output[:error] = "Status for the created object returned 'false'" - log_output[:status] = 'FAIL' - end - cleanUp([bucket_name], log_output) - rescue => log_output[:error] - log_output[:status] = 'FAIL' - end - - print_log(log_output, start_time) - end - - def removeObjectTest(file) - # Tests removeObject api command by uploading and removing a file - - # get random bucket name - bucket_name = get_random_bucket_name() - # Initialize hash table, 'log_output' - log_output = initialize_log_output('removeObject') - # Prepare arg/value hash table and set it in log_output - arg_value_hash = {} - log_output[:args].each { |x| arg_value_hash[:"#{x}"] = eval x.to_s } - log_output[:args] = arg_value_hash - - begin - start_time = Time.now - makeBucketWrapper(bucket_name, log_output) - putObjectWrapper(bucket_name, file, log_output) - removeObjectWrapper(bucket_name, File.basename(file), log_output) - if !statObjectWrapper(bucket_name, File.basename(file), log_output) - log_output[:status] = 'PASS' - else - log_output[:error] = "Status for the removed object returned 'true'" - log_output[:status] = 'FAIL' - end - cleanUp([bucket_name], log_output) - rescue => log_output[:error] - log_output[:status] = 'FAIL' - end - - print_log(log_output, start_time) - end - - def getObjectTest(file, destination) - # Tests getObject api command - - # get random bucket name - bucket_name = get_random_bucket_name() - # Initialize hash table, 'log_output' - log_output = initialize_log_output('getObject') - # Prepare arg/value hash table and set it in log_output - arg_value_hash = {} - log_output[:args].each { |x| arg_value_hash[:"#{x}"] = eval x.to_s } - log_output[:args] = arg_value_hash - - begin - start_time = Time.now - makeBucketWrapper(bucket_name, log_output) - putObjectWrapper(bucket_name, file, log_output) - getObjectWrapper(bucket_name, file, destination, log_output) - if system("ls -l #{destination} > /dev/null") - log_output[:status] = 'PASS' - else - log_output[:error] = "Downloaded object does not exist at #{destination}" - log_output[:status] = 'FAIL' - end - cleanUp([bucket_name], log_output) - rescue => log_output[:error] - log_output[:status] = 'FAIL' - end - - print_log(log_output, start_time) - end - - def listObjectsTest(file_list) - # Tests listObjects api command - - # get random bucket name - bucket_name = get_random_bucket_name() - # Initialize hash table, 'log_output' - log_output = initialize_log_output('listObjects') - # Prepare arg/value hash table and set it in log_output - arg_value_hash = {} - log_output[:args].each { |x| arg_value_hash[:"#{x}"] = eval x.to_s } - log_output[:args] = arg_value_hash - - begin - start_time = Time.now - makeBucketWrapper(bucket_name, log_output) - # Put all objects into the bucket - file_list.each do |f| - putObjectWrapper(bucket_name, f, log_output) - end - # Total number of files uploaded - expected_no = file_list.length - # Actual number is what api returns - actual_no = listObjectsWrapper(bucket_name, log_output).length - # Compare expected and actual values - if expected_no == actual_no - log_output[:status] = 'PASS' - else - log_output[:error] = 'Expected and actual number of listed files/objects do not match!' - log_output[:status] = 'FAIL' - end - cleanUp([bucket_name], log_output) - rescue => log_output[:error] - log_output[:status] = 'FAIL' - end - - print_log(log_output, start_time) - end - - def copyObjectTest(data_dir, source_file_name, target_file_name = '') - # Tests copyObject api command - - # get random bucket names - source_bucket_name = get_random_bucket_name() - target_bucket_name = get_random_bucket_name() - # Initialize hash table, 'log_output' - log_output = initialize_log_output('copyObject') - # Prepare arg/value hash table and set it in log_output - arg_value_hash = {} - log_output[:args].each { |x| arg_value_hash[:"#{x}"] = eval x.to_s } - log_output[:args] = arg_value_hash - - begin - start_time = Time.now - target_file_name = source_file_name if target_file_name.empty? - makeBucketWrapper(source_bucket_name, log_output) - makeBucketWrapper(target_bucket_name, log_output) - putObjectWrapper(source_bucket_name, - File.join(data_dir, source_file_name), log_output) - copyObjectWrapper(source_bucket_name, target_bucket_name, - source_file_name, target_file_name, log_output) - # Check if copy worked fine - if statObjectWrapper(target_bucket_name, target_file_name, log_output) - log_output[:status] = 'PASS' - else - log_output[:error] = 'Copied file could not be found in the expected location' - log_output[:status] = 'FAIL' - end - cleanUp([source_bucket_name, target_bucket_name], log_output) - rescue => log_output[:error] - log_output[:status] = 'FAIL' - end - - print_log(log_output, start_time) - end - - def presignedGetObjectTest(data_dir, file_name) - # Tests presignedGetObject api command - - # get random bucket name - bucket_name = get_random_bucket_name() - # Initialize hash table, 'log_output' - log_output = initialize_log_output('presignedGet') - # Prepare arg/value hash table and set it in log_output - arg_value_hash = {} - log_output[:args].each { |x| arg_value_hash[:"#{x}"] = eval x.to_s } - log_output[:args] = arg_value_hash - - begin - start_time = Time.now - makeBucketWrapper(bucket_name, log_output) - file = File.join(data_dir, file_name) - # Get check sum value without the file name - cksum_orig = `cksum #{file}`.split[0..1] - putObjectWrapper(bucket_name, file, log_output) - get_url = presignedGetWrapper(bucket_name, file_name, log_output) - # Download the file using the URL - # generated by presignedGet api command - `wget -O /tmp/#{file_name}, '#{get_url}' > /dev/null 2>&1` - # Get check sum value for the downloaded file - # Split to get rid of the file name - cksum_new = `cksum /tmp/#{file_name}`.split[0..1] - - # Check if check sum values for the orig file - # and the downloaded file match - if cksum_orig == cksum_new - log_output[:status] = 'PASS' - else - log_output[:error] = 'Check sum values do NOT match' - log_output[:status] = 'FAIL' - end - cleanUp([bucket_name], log_output) - rescue => log_output[:error] - log_output[:status] = 'FAIL' - end - - print_log(log_output, start_time) - end - - def presignedPutObjectTest(data_dir, file_name) - # Tests presignedPutObject api command - - # get random bucket name - bucket_name = get_random_bucket_name() - # Initialize hash table, 'log_output' - log_output = initialize_log_output('presignedPut') - # Prepare arg/value hash table and set it in log_output - arg_value_hash = {} - log_output[:args].each { |x| arg_value_hash[:"#{x}"] = eval x.to_s } - log_output[:args] = arg_value_hash - - begin - start_time = Time.now - makeBucketWrapper(bucket_name, log_output) - file = File.join(data_dir, file_name) - - # Get check sum value and - # split to get rid of the file name - cksum_orig = `cksum #{file}`.split[0..1] - - # Generate presigned Put URL and parse it - uri = URI.parse(presignedPutWrapper(bucket_name, file_name, log_output)) - request = Net::HTTP::Put.new(uri.request_uri, 'x-amz-acl' => 'public-read') - request.body = IO.read(File.join(data_dir, file_name)) - - http = Net::HTTP.new(uri.host, uri.port) - http.use_ssl = true if ENV['ENABLE_HTTPS'] == '1' - - http.request(request) - - if statObjectWrapper(bucket_name, file_name, log_output) - getObjectWrapper(bucket_name, file_name, '/tmp', log_output) - cksum_new = `cksum /tmp/#{file_name}`.split[0..1] - # Check if check sum values of the orig file - # and the downloaded file match - if cksum_orig == cksum_new - log_output[:status] = 'PASS' - else - log_output[:error] = 'Check sum values do NOT match' - log_output[:status] = 'FAIL' - end - else - log_output[:error] = 'Expected to be created object does NOT exist' - log_output[:status] = 'FAIL' - end - cleanUp([bucket_name], log_output) - rescue => log_output[:error] - log_output[:status] = 'FAIL' - end - - print_log(log_output, start_time) - end - - def presignedPostObjectTest(data_dir, file_name, - expires_in_sec, max_byte_size) - # Tests presignedPostObject api command - - # get random bucket name - bucket_name = get_random_bucket_name() - # Initialize hash table, 'log_output' - log_output = initialize_log_output('presignedPost') - # Prepare arg/value hash table and set it in log_output - arg_value_hash = {} - log_output[:args].each { |x| arg_value_hash[:"#{x}"] = eval x.to_s } - log_output[:args] = arg_value_hash - - begin - start_time = Time.now - makeBucketWrapper(bucket_name, log_output) - file = File.join(data_dir, file_name) - - # Get check sum value and split it - # into parts to get rid of the file name - cksum_orig = `cksum #{file}`.split[0..1] - # Create the presigned POST url - post = presignedPostWrapper(bucket_name, file_name, - expires_in_sec, max_byte_size, log_output) - - # Prepare multi parts array for POST command request - file_part = Part.new name: 'file', - body: IO.read(File.join(data_dir, file_name)), - filename: file_name, - content_type: 'application/octet-stream' - parts = [file_part] - # Add POST fields into parts array - post.fields.each do |field, value| - parts.push(Part.new(field, value)) - end - boundary = "---------------------------#{rand(10_000_000_000_000_000)}" - body_parts = MultipartBody.new parts, boundary - - # Parse presigned Post URL - uri = URI.parse(post.url) - - # Create the HTTP objects - http = Net::HTTP.new(uri.host, uri.port) - http.use_ssl = true if ENV['ENABLE_HTTPS'] == '1' - request = Net::HTTP::Post.new(uri.request_uri) - request.body = body_parts.to_s - request.content_type = "multipart/form-data; boundary=#{boundary}" - # Send the request - log_output[:error] = http.request(request) - - if statObjectWrapper(bucket_name, file_name, log_output) - getObjectWrapper(bucket_name, file_name, '/tmp', log_output) - cksum_new = `cksum /tmp/#{file_name}`.split[0..1] - # Check if check sum values of the orig file - # and the downloaded file match - if cksum_orig == cksum_new - log_output[:status] = 'PASS' - # FIXME: HTTP No Content error, status code=204 is returned as error - log_output[:error] = nil - else - log_output[:error] = 'Check sum values do NOT match' - log_output[:status] = 'FAIL' - end - else - log_output[:error] = 'Expected to be created object does NOT exist' - log_output[:status] = 'FAIL' - end - cleanUp([bucket_name], log_output) - rescue => log_output[:error] - log_output[:status] = 'FAIL' - end - - print_log(log_output, start_time) - end -end - -# MAIN CODE - -# Create test Class instance and call the tests -aws = AwsSdkRubyTest.new -file_name1 = 'datafile-1-kB' -file_new_name = 'datafile-1-kB-copy' -file_name_list = ['datafile-1-kB', 'datafile-1-b', 'datafile-6-MB'] -# Add data_dir in front of each file name in file_name_list -# The location where the bucket and file -# objects are going to be created. -data_dir = ENV['MINT_DATA_DIR'] ||= 'MINT_DATA_DIR is not set' -file_list = file_name_list.map { |f| File.join(data_dir, f) } -destination = '/tmp' - -aws.listBucketsTest() -aws.listObjectsTest(file_list) -aws.makeBucketTest() -aws.bucketExistsNegativeTest() -aws.removeBucketTest() -aws.putObjectTest(File.join(data_dir, file_name1)) -aws.removeObjectTest(File.join(data_dir, file_name1)) -aws.getObjectTest(File.join(data_dir, file_name1), destination) -aws.copyObjectTest(data_dir, file_name1) -aws.copyObjectTest(data_dir, file_name1, file_new_name) -aws.presignedGetObjectTest(data_dir, file_name1) -aws.presignedPutObjectTest(data_dir, file_name1) -aws.presignedPostObjectTest(data_dir, file_name1, 60, 3*1024*1024) diff --git a/mint/run/core/aws-sdk-ruby/run.sh b/mint/run/core/aws-sdk-ruby/run.sh deleted file mode 100755 index 1e59a71..0000000 --- a/mint/run/core/aws-sdk-ruby/run.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -# -# Mint (C) 2017 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# handle command line arguments -if [ $# -ne 2 ]; then - echo "usage: run.sh " - exit 1 -fi - -output_log_file="$1" -error_log_file="$2" - -# run tests -chmod a+x aws-stub-tests.rb -ruby aws-stub-tests.rb 1>>"$output_log_file" 2>"$error_log_file" diff --git a/mint/run/core/awscli/README.md b/mint/run/core/awscli/README.md deleted file mode 100644 index b395f9a..0000000 --- a/mint/run/core/awscli/README.md +++ /dev/null @@ -1,19 +0,0 @@ -## `awscli` tests -This directory serves as the location for Mint tests using `awscli`. Top level `mint.sh` calls `run.sh` to execute tests. - -## Adding new tests -New tests is added into `test.sh` as new functions. - -## Running tests manually -- Set environment variables `MINT_DATA_DIR`, `MINT_MODE`, `SERVER_ENDPOINT`, `ACCESS_KEY`, `SECRET_KEY`, `SERVER_REGION` and `ENABLE_HTTPS` -- Call `run.sh` with output log file and error log file. for example -```bash -export MINT_DATA_DIR=~/my-mint-dir -export MINT_MODE=core -export SERVER_ENDPOINT="play.minio.io:9000" -export ACCESS_KEY="Q3AM3UQ867SPQQA43P2F" -export SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" -export ENABLE_HTTPS=1 -export SERVER_REGION=us-east-1 -./run.sh /tmp/output.log /tmp/error.log -``` diff --git a/mint/run/core/awscli/run.sh b/mint/run/core/awscli/run.sh deleted file mode 100755 index 98bd61b..0000000 --- a/mint/run/core/awscli/run.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/bash -# -# Mint (C) 2017 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# handle command line arguments -if [ $# -ne 2 ]; then - echo "usage: run.sh " - exit 1 -fi - -output_log_file="$1" -error_log_file="$2" - -# configure awscli -aws configure set aws_access_key_id "$ACCESS_KEY" -aws configure set aws_secret_access_key "$SECRET_KEY" -aws configure set default.region "$SERVER_REGION" - -# run tests for virtual style if provided -if [ "$ENABLE_VIRTUAL_STYLE" -eq 1 ]; then - # Setup endpoint scheme - endpoint="http://$DOMAIN:$SERVER_PORT" - if [ "$ENABLE_HTTPS" -eq 1 ]; then - endpoint="https://$DOMAIN:$SERVER_PORT" - fi - dnsmasq --address="/$DOMAIN/$SERVER_IP" --user=root - echo -e "nameserver 127.0.0.1\n$(cat /etc/resolv.conf)" > /etc/resolv.conf - aws configure set default.s3.addressing_style virtual - ./test.sh "$endpoint" 1>>"$output_log_file" 2>"$error_log_file" - aws configure set default.s3.addressing_style path -fi - -endpoint="http://$SERVER_ENDPOINT" -if [ "$ENABLE_HTTPS" -eq 1 ]; then - endpoint="https://$SERVER_ENDPOINT" -fi -# run path style tests -./test.sh "$endpoint" 1>>"$output_log_file" 2>"$error_log_file" diff --git a/mint/run/core/awscli/test.sh b/mint/run/core/awscli/test.sh deleted file mode 100755 index 096f4e5..0000000 --- a/mint/run/core/awscli/test.sh +++ /dev/null @@ -1,1726 +0,0 @@ -#!/bin/bash -# -# Mint (C) 2017-2020 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -HASH_1_KB=$(md5sum "${MINT_DATA_DIR}/datafile-1-kB" | awk '{print $1}') -HASH_65_MB=$(md5sum "${MINT_DATA_DIR}/datafile-65-MB" | awk '{print $1}') - -_init() { - AWS="aws --endpoint-url $1" -} - -function get_time() { - date +%s%N -} - -function get_duration() { - start_time=$1 - end_time=$(get_time) - - echo $(( (end_time - start_time) / 1000000 )) -} - -function log_success() { - function=$(python -c 'import sys,json; print(json.dumps(sys.stdin.read()))' <<<"$2") - printf '{"name": "awscli", "duration": %d, "function": %s, "status": "PASS"}\n' "$1" "$function" -} - -function log_failure() { - function=$(python -c 'import sys,json; print(json.dumps(sys.stdin.read()))' <<<"$2") - err=$(echo "$3" | tr -d '\n') - printf '{"name": "awscli", "duration": %d, "function": %s, "status": "FAIL", "error": "%s"}\n' "$1" "$function" "$err" -} - -function log_alert() { - function=$(python -c 'import sys,json; print(json.dumps(sys.stdin.read()))' <<<"$2") - err=$(echo "$4" | tr -d '\n') - printf '{"name": "awscli", "duration": %d, "function": %s, "status": "FAIL", "alert": "%s", "error": "%s"}\n' "$1" "$function" "$3" "$err" -} - -function make_bucket() { - # Make bucket - bucket_name="awscli-mint-test-bucket-$RANDOM" - function="${AWS} s3api create-bucket --bucket ${bucket_name}" - - # execute the test - out=$($function 2>&1) - rv=$? - - # if command is successful print bucket_name or print error - if [ $rv -eq 0 ]; then - echo "${bucket_name}" - else - echo "${out}" - fi - - return $rv -} - -function delete_bucket() { - # Delete bucket - function="${AWS} s3 rb s3://${1} --force" - out=$($function 2>&1) - rv=$? - - # echo the output - echo "${out}" - - return $rv -} - -# Tests creating, stat and delete on a bucket. -function test_create_bucket() { - # log start time - start_time=$(get_time) - - function="make_bucket" - bucket_name=$(make_bucket) - rv=$? - # save the ref to function being tested, so it can be logged - test_function=${function} - - # if make_bucket is successful stat the bucket - if [ $rv -eq 0 ]; then - function="${AWS} s3api head-bucket --bucket ${bucket_name}" - out=$($function 2>&1) - rv=$? - else - # if make bucket failes, $bucket_name has the error output - out="${bucket_name}" - fi - - # if stat bucket is successful remove the bucket - if [ $rv -eq 0 ]; then - function="delete_bucket" - out=$(delete_bucket "${bucket_name}") - rv=$? - else - # if make bucket failes, $bucket_name has the error output - out="${bucket_name}" - fi - - if [ $rv -eq 0 ]; then - log_success "$(get_duration "$start_time")" "${test_function}" - else - # clean up and log error - ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 - log_failure "$(get_duration "$start_time")" "${function}" "${out}" - fi - - return $rv -} - -# Tests creating and deleting an object. -function test_upload_object() { - # log start time - start_time=$(get_time) - - function="make_bucket" - bucket_name=$(make_bucket) - rv=$? - - # if make bucket succeeds upload a file - if [ $rv -eq 0 ]; then - function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB" - out=$($function 2>&1) - rv=$? - else - # if make bucket fails, $bucket_name has the error output - out="${bucket_name}" - fi - - # if upload succeeds download the file - if [ $rv -eq 0 ]; then - function="${AWS} s3api get-object --bucket ${bucket_name} --key datafile-1-kB /tmp/datafile-1-kB" - # save the ref to function being tested, so it can be logged - test_function=${function} - out=$($function 2>&1) - rv=$? - # calculate the md5 hash of downloaded file - hash2=$(md5sum /tmp/datafile-1-kB | awk '{print $1}') - fi - - # if download succeeds, verify downloaded file - if [ $rv -eq 0 ]; then - if [ "$HASH_1_KB" == "$hash2" ]; then - function="delete_bucket" - out=$(delete_bucket "$bucket_name") - rv=$? - # remove download file - rm -f /tmp/datafile-1-kB - else - rv=1 - out="Checksum verification failed for uploaded object" - fi - fi - - if [ $rv -eq 0 ]; then - log_success "$(get_duration "$start_time")" "${test_function}" - else - # clean up and log error - ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 - log_failure "$(get_duration "$start_time")" "${function}" "${out}" - fi - - return $rv -} - -# Test lookup a directory prefix. -function test_lookup_object_prefix() { - # log start time - start_time=$(get_time) - - function="make_bucket" - bucket_name=$(make_bucket) - rv=$? - - # if make bucket succeeds create a directory. - if [ $rv -eq 0 ]; then - function="${AWS} s3api put-object --bucket ${bucket_name} --key prefix/directory/" - # save the ref to function being tested, so it can be logged - test_function=${function} - - out=$($function 2>&1) - - rv=$? - else - # if make_bucket fails, $bucket_name has the error output - out="${bucket_name}" - fi - - if [ $rv -eq 0 ]; then - ## Attempt an overwrite of the prefix again and should succeed as well. - function="${AWS} s3api put-object --bucket ${bucket_name} --key prefix/directory/" - # save the ref to function being tested, so it can be logged - test_function=${function} - out=$($function 2>&1) - rv=$? - fi - - # if upload succeeds lookup for the prefix. - if [ $rv -eq 0 ]; then - function="${AWS} s3api head-object --bucket ${bucket_name} --key prefix/directory/" - # save the ref to function being tested, so it can be logged - test_function=${function} - out=$($function 2>&1) - rv=$? - fi - - # if directory create succeeds, upload the object. - if [ $rv -eq 0 ]; then - function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key prefix/directory/datafile-1-kB" - # save the ref to function being tested, so it can be logged - test_function=${function} - out=$($function 2>&1) - rv=$? - fi - - # Attempt a delete on prefix shouldn't delete the directory since we have an object inside it. - if [ $rv -eq 0 ]; then - function="${AWS} s3api delete-object --bucket ${bucket_name} --key prefix/directory/" - # save the ref to function being tested, so it can be logged - test_function=${function} - out=$($function 2>&1) - rv=$? - fi - - # if upload succeeds lookup for the object should succeed. - if [ $rv -eq 0 ]; then - function="${AWS} s3api head-object --bucket ${bucket_name} --key prefix/directory/datafile-1-kB" - # save the ref to function being tested, so it can be logged - test_function=${function} - out=$($function 2>&1) - rv=$? - fi - - # delete bucket - if [ $rv -eq 0 ]; then - function="delete_bucket" - out=$(delete_bucket "$bucket_name") - rv=$? - fi - - if [ $rv -ne 0 ]; then - # clean up and log error - ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 - log_failure "$(get_duration "$start_time")" "${function}" "${out}" - else - log_success "$(get_duration "$start_time")" "${test_function}" - fi - - return $rv -} - -# Tests listing objects for both v1 and v2 API. -function test_list_objects() { - # log start time - start_time=$(get_time) - - function="make_bucket" - bucket_name=$(make_bucket) - rv=$? - - # if make bucket succeeds upload a file - if [ $rv -eq 0 ]; then - function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB" - out=$($function 2>&1) - rv=$? - else - # if make bucket fails, $bucket_name has the error output - out="${bucket_name}" - fi - - # if upload objects succeeds, list objects with existing prefix - if [ $rv -eq 0 ]; then - function="${AWS} s3api list-objects --bucket ${bucket_name} --prefix datafile-1-kB" - test_function=${function} - out=$($function) - rv=$? - key_name=$(echo "$out" | jq -r .Contents[].Key) - if [ $rv -eq 0 ] && [ "$key_name" != "datafile-1-kB" ]; then - rv=1 - # since rv is 0, command passed, but didn't return expected value. In this case set the output - out="list-objects with existing prefix failed" - fi - fi - - # if upload objects succeeds, list objects without existing prefix - if [ $rv -eq 0 ]; then - function="${AWS} s3api list-objects --bucket ${bucket_name} --prefix linux" - out=$($function) - rv=$? - key_name=$(echo "$out" | jq -r .Contents[].Key) - if [ $rv -eq 0 ] && [ "$key_name" != "" ]; then - rv=1 - out="list-objects without existing prefix failed" - fi - fi - - # if upload objects succeeds, list objectsv2 with existing prefix - if [ $rv -eq 0 ]; then - function="${AWS} s3api list-objects-v2 --bucket ${bucket_name} --prefix datafile-1-kB" - out=$($function) - rv=$? - key_name=$(echo "$out" | jq -r .Contents[].Key) - if [ $rv -eq 0 ] && [ "$key_name" != "datafile-1-kB" ]; then - rv=1 - out="list-objects-v2 with existing prefix failed" - fi - fi - - # if upload objects succeeds, list objectsv2 without existing prefix - if [ $rv -eq 0 ]; then - function="${AWS} s3api list-objects-v2 --bucket ${bucket_name} --prefix linux" - out=$($function) - rv=$? - key_name=$(echo "$out" | jq -r .Contents[].Key) - if [ $rv -eq 0 ] && [ "$key_name" != "" ]; then - rv=1 - out="list-objects-v2 without existing prefix failed" - fi - fi - - if [ $rv -eq 0 ]; then - function="delete_bucket" - out=$(delete_bucket "$bucket_name") - rv=$? - # remove download file - rm -f /tmp/datafile-1-kB - fi - - if [ $rv -eq 0 ]; then - log_success "$(get_duration "$start_time")" "${test_function}" - else - # clean up and log error - ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 - rm -f /tmp/datafile-1-kB - log_failure "$(get_duration "$start_time")" "${function}" "${out}" - fi - - return $rv -} - -# Tests multipart API with 0 byte part. -function test_multipart_upload_0byte() { - # log start time - start_time=$(get_time) - - function="make_bucket" - bucket_name=$(make_bucket) - object_name=${bucket_name}"-object" - rv=$? - - # if make bucket succeeds upload a file - if [ $rv -eq 0 ]; then - function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-0-b --bucket ${bucket_name} --key datafile-0-b" - out=$($function 2>&1) - rv=$? - else - # if make bucket fails, $bucket_name has the error output - out="${bucket_name}" - fi - - if [ $rv -eq 0 ]; then - # create multipart - function="${AWS} s3api create-multipart-upload --bucket ${bucket_name} --key ${object_name}" - test_function=${function} - out=$($function) - rv=$? - upload_id=$(echo "$out" | jq -r .UploadId) - fi - - if [ $rv -eq 0 ]; then - # Capture etag for part-number 1 - function="${AWS} s3api upload-part --bucket ${bucket_name} --key ${object_name} --body ${MINT_DATA_DIR}/datafile-0-b --upload-id ${upload_id} --part-number 1" - out=$($function) - rv=$? - etag1=$(echo "$out" | jq -r .ETag) - fi - - if [ $rv -eq 0 ]; then - # Create a multipart struct file for completing multipart transaction - echo "{ - \"Parts\": [ - { - \"ETag\": ${etag1}, - \"PartNumber\": 1 - } - ] - }" >> /tmp/multipart - fi - - if [ $rv -eq 0 ]; then - # Use saved etags to complete the multipart transaction - function="${AWS} s3api complete-multipart-upload --multipart-upload file:///tmp/multipart --bucket ${bucket_name} --key ${object_name} --upload-id ${upload_id}" - out=$($function) - rv=$? - etag=$(echo "$out" | jq -r .ETag | sed -e 's/^"//' -e 's/"$//') - if [ "${etag}" == "" ]; then - rv=1 - out="complete-multipart-upload failed" - fi - fi - - if [ $rv -eq 0 ]; then - function="${AWS} s3api get-object --bucket ${bucket_name} --key ${object_name} /tmp/datafile-0-b" - test_function=${function} - out=$($function 2>&1) - rv=$? - fi - - if [ $rv -eq 0 ]; then - ret_etag=$(echo "$out" | jq -r .ETag | sed -e 's/^"//' -e 's/"$//') - # match etag - if [ "$etag" != "$ret_etag" ]; then - rv=1 - out="Etag mismatch for multipart 0 byte object" - fi - rm -f /tmp/datafile-0-b - fi - - if [ $rv -eq 0 ]; then - function="delete_bucket" - out=$(delete_bucket "$bucket_name") - rv=$? - # remove temp file - rm -f /tmp/multipart - fi - - if [ $rv -eq 0 ]; then - log_success "$(get_duration "$start_time")" "${test_function}" - else - # clean up and log error - ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 - rm -f /tmp/multipart - log_failure "$(get_duration "$start_time")" "${function}" "${out}" - fi - - return $rv -} - -# Tests multipart API by making each individual calls. -function test_multipart_upload() { - # log start time - start_time=$(get_time) - - function="make_bucket" - bucket_name=$(make_bucket) - object_name=${bucket_name}"-object" - rv=$? - - # if make bucket succeeds upload a file - if [ $rv -eq 0 ]; then - function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB" - out=$($function 2>&1) - rv=$? - else - # if make bucket fails, $bucket_name has the error output - out="${bucket_name}" - fi - - if [ $rv -eq 0 ]; then - # create multipart - function="${AWS} s3api create-multipart-upload --bucket ${bucket_name} --key ${object_name}" - test_function=${function} - out=$($function) - rv=$? - upload_id=$(echo "$out" | jq -r .UploadId) - fi - - if [ $rv -eq 0 ]; then - # Capture etag for part-number 1 - function="${AWS} s3api upload-part --bucket ${bucket_name} --key ${object_name} --body ${MINT_DATA_DIR}/datafile-5-MB --upload-id ${upload_id} --part-number 1" - out=$($function) - rv=$? - etag1=$(echo "$out" | jq -r .ETag) - fi - - if [ $rv -eq 0 ]; then - # Capture etag for part-number 2 - function="${AWS} s3api upload-part --bucket ${bucket_name} --key ${object_name} --body ${MINT_DATA_DIR}/datafile-1-kB --upload-id ${upload_id} --part-number 2" - out=$($function) - rv=$? - etag2=$(echo "$out" | jq -r .ETag) - # Create a multipart struct file for completing multipart transaction - echo "{ - \"Parts\": [ - { - \"ETag\": ${etag1}, - \"PartNumber\": 1 - }, - { - \"ETag\": ${etag2}, - \"PartNumber\": 2 - } - ] - }" >> /tmp/multipart - fi - - if [ $rv -eq 0 ]; then - # Use saved etags to complete the multipart transaction - function="${AWS} s3api complete-multipart-upload --multipart-upload file:///tmp/multipart --bucket ${bucket_name} --key ${object_name} --upload-id ${upload_id}" - out=$($function) - rv=$? - finalETag=$(echo "$out" | jq -r .ETag | sed -e 's/^"//' -e 's/"$//') - if [ "${finalETag}" == "" ]; then - rv=1 - out="complete-multipart-upload failed" - fi - fi - - if [ $rv -eq 0 ]; then - function="delete_bucket" - out=$(delete_bucket "$bucket_name") - rv=$? - # remove temp file - rm -f /tmp/multipart - fi - - if [ $rv -eq 0 ]; then - log_success "$(get_duration "$start_time")" "${test_function}" - else - # clean up and log error - ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 - rm -f /tmp/multipart - log_failure "$(get_duration "$start_time")" "${function}" "${out}" - fi - - return $rv -} - -# List number of objects based on the maxKey -# value set. -function test_max_key_list() { - # log start time - start_time=$(get_time) - - function="make_bucket" - bucket_name=$(make_bucket) - rv=$? - - # if make bucket succeeds upload a file - if [ $rv -eq 0 ]; then - function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-b --bucket ${bucket_name} --key datafile-1-b" - out=$($function 2>&1) - rv=$? - else - # if make bucket fails, $bucket_name has the error output - out="${bucket_name}" - fi - - # copy object server side - if [ $rv -eq 0 ]; then - function="${AWS} s3api copy-object --bucket ${bucket_name} --key datafile-1-b-copy --copy-source ${bucket_name}/datafile-1-b" - out=$($function) - rv=$? - fi - - if [ $rv -eq 0 ]; then - function="${AWS} s3api list-objects-v2 --bucket ${bucket_name} --max-keys 1" - test_function=${function} - out=$($function 2>&1) - rv=$? - if [ $rv -eq 0 ]; then - out=$(echo "$out" | jq '.KeyCount') - rv=$? - fi - fi - - if [ $rv -eq 0 ]; then - function="delete_bucket" - out=$(delete_bucket "$bucket_name") - rv=$? - # The command passed, but the delete_bucket failed - out="delete_bucket for test_max_key_list failed" - fi - - if [ $rv -eq 0 ]; then - log_success "$(get_duration "$start_time")" "${test_function}" - else - # clean up and log error - ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 - log_failure "$(get_duration "$start_time")" "${function}" "${out}" - fi - - return $rv -} - -# Copy object tests for server side copy -# of the object, validates returned md5sum. -function test_copy_object() { - # log start time - start_time=$(get_time) - - function="make_bucket" - bucket_name=$(make_bucket) - rv=$? - - # if make bucket succeeds upload a file - if [ $rv -eq 0 ]; then - function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB" - out=$($function 2>&1) - rv=$? - else - # if make bucket fails, $bucket_name has the error output - out="${bucket_name}" - fi - - # copy object server side - if [ $rv -eq 0 ]; then - function="${AWS} s3api copy-object --bucket ${bucket_name} --key datafile-1-kB-copy --copy-source ${bucket_name}/datafile-1-kB" - test_function=${function} - out=$($function) - rv=$? - hash2=$(echo "$out" | jq -r .CopyObjectResult.ETag | sed -e 's/^"//' -e 's/"$//') - if [ $rv -eq 0 ] && [ "$HASH_1_KB" != "$hash2" ]; then - # Verification failed - rv=1 - out="Hash mismatch expected $HASH_1_KB, got $hash2" - fi - fi - - ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 - if [ $rv -eq 0 ]; then - log_success "$(get_duration "$start_time")" "${test_function}" - else - log_failure "$(get_duration "$start_time")" "${function}" "${out}" - fi - - return $rv -} - -# Copy object tests for server side copy -# of the object, validates returned md5sum. -# validates change in storage class as well -function test_copy_object_storage_class() { - # log start time - start_time=$(get_time) - - function="make_bucket" - bucket_name=$(make_bucket) - rv=$? - - # if make bucket succeeds upload a file - if [ $rv -eq 0 ]; then - function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB" - out=$($function 2>&1) - rv=$? - else - # if make bucket fails, $bucket_name has the error output - out="${bucket_name}" - fi - - # copy object server side - if [ $rv -eq 0 ]; then - function="${AWS} s3api copy-object --bucket ${bucket_name} --storage-class REDUCED_REDUNDANCY --key datafile-1-kB-copy --copy-source ${bucket_name}/datafile-1-kB" - test_function=${function} - out=$($function 2>&1) - rv=$? - # if this functionality is not implemented return right away. - if [ $rv -ne 0 ]; then - if echo "$out" | grep -q "NotImplemented"; then - ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 - return 0 - fi - fi - hash2=$(echo "$out" | jq -r .CopyObjectResult.ETag | sed -e 's/^"//' -e 's/"$//') - if [ $rv -eq 0 ] && [ "$HASH_1_KB" != "$hash2" ]; then - # Verification failed - rv=1 - out="Hash mismatch expected $HASH_1_KB, got $hash2" - fi - # if copy succeeds stat the object - if [ $rv -eq 0 ]; then - function="${AWS} s3api head-object --bucket ${bucket_name} --key datafile-1-kB-copy" - # save the ref to function being tested, so it can be logged - test_function=${function} - out=$($function 2>&1) - storageClass=$(echo "$out" | jq -r .StorageClass) - rv=$? - fi - # if head-object succeeds, verify metadata has storage class - if [ $rv -eq 0 ]; then - if [ "${storageClass}" == "null" ]; then - rv=1 - out="StorageClass was not applied" - elif [ "${storageClass}" == "STANDARD" ]; then - rv=1 - out="StorageClass was applied incorrectly" - fi - fi - fi - - ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 - if [ $rv -eq 0 ]; then - log_success "$(get_duration "$start_time")" "${test_function}" - else - log_failure "$(get_duration "$start_time")" "${function}" "${out}" - fi - - return $rv -} - -# Copy object tests for server side copy -# to itself by changing storage class -function test_copy_object_storage_class_same() { - # log start time - start_time=$(get_time) - - function="make_bucket" - bucket_name=$(make_bucket) - rv=$? - - # if make bucket succeeds upload a file - if [ $rv -eq 0 ]; then - function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB" - out=$($function 2>&1) - rv=$? - else - # if make bucket fails, $bucket_name has the error output - out="${bucket_name}" - fi - - # copy object server side - if [ $rv -eq 0 ]; then - function="${AWS} s3api copy-object --bucket ${bucket_name} --storage-class REDUCED_REDUNDANCY --key datafile-1-kB --copy-source ${bucket_name}/datafile-1-kB" - test_function=${function} - out=$($function 2>&1) - rv=$? - # if this functionality is not implemented return right away. - if [ $rv -ne 0 ]; then - if echo "$out" | grep -q "NotImplemented"; then - ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 - return 0 - fi - fi - hash2=$(echo "$out" | jq -r .CopyObjectResult.ETag | sed -e 's/^"//' -e 's/"$//') - if [ $rv -eq 0 ] && [ "$HASH_1_KB" != "$hash2" ]; then - # Verification failed - rv=1 - out="Hash mismatch expected $HASH_1_KB, got $hash2" - fi - # if copy succeeds stat the object - if [ $rv -eq 0 ]; then - function="${AWS} s3api head-object --bucket ${bucket_name} --key datafile-1-kB" - # save the ref to function being tested, so it can be logged - test_function=${function} - out=$($function 2>&1) - storageClass=$(echo "$out" | jq -r .StorageClass) - rv=$? - fi - # if head-object succeeds, verify metadata has storage class - if [ $rv -eq 0 ]; then - if [ "${storageClass}" == "null" ]; then - rv=1 - out="StorageClass was not applied" - elif [ "${storageClass}" == "STANDARD" ]; then - rv=1 - out="StorageClass was applied incorrectly" - fi - fi - fi - - ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 - if [ $rv -eq 0 ]; then - log_success "$(get_duration "$start_time")" "${test_function}" - else - log_failure "$(get_duration "$start_time")" "${function}" "${out}" - fi - - return $rv -} - -# Tests for presigned URL success case, presigned URL -# is correct and accessible - we calculate md5sum of -# the object and validate it against a local files md5sum. -function test_presigned_object() { - # log start time - start_time=$(get_time) - - function="make_bucket" - bucket_name=$(make_bucket) - rv=$? - - # if make bucket succeeds upload a file - if [ $rv -eq 0 ]; then - function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB" - out=$($function 2>&1) - rv=$? - else - # if make bucket fails, $bucket_name has the error output - out="${bucket_name}" - fi - - if [ $rv -eq 0 ]; then - function="${AWS} s3 presign s3://${bucket_name}/datafile-1-kB" - test_function=${function} - url=$($function) - rv=$? - curl -sS -X GET "${url}" > /tmp/datafile-1-kB - hash2=$(md5sum /tmp/datafile-1-kB | awk '{print $1}') - if [ "$HASH_1_KB" == "$hash2" ]; then - function="delete_bucket" - out=$(delete_bucket "$bucket_name") - rv=$? - # remove download file - rm -f /tmp/datafile-1-kB - else - rv=1 - out="Checksum verification failed for downloaded object" - fi - fi - - if [ $rv -eq 0 ]; then - log_success "$(get_duration "$start_time")" "${test_function}" - else - # clean up and log error - ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 - log_failure "$(get_duration "$start_time")" "${function}" "${out}" - fi - - return $rv -} - -# Tests creating and deleting an object - 10MiB -function test_upload_object_10() { - # log start time - start_time=$(get_time) - - function="make_bucket" - bucket_name=$(make_bucket) - rv=$? - - # if make bucket succeeds upload a file - if [ $rv -eq 0 ]; then - function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-10-MB --bucket ${bucket_name} --key datafile-10-MB" - out=$($function 2>&1) - rv=$? - else - # if make bucket fails, $bucket_name has the error output - out="${bucket_name}" - fi - - if [ $rv -eq 0 ]; then - log_success "$(get_duration "$start_time")" "${test_function}" - else - # clean up and log error - ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 - log_failure "$(get_duration "$start_time")" "${function}" "${out}" - fi - - return $rv -} - -# Tests multipart API by making each individual calls with 10MiB part size. -function test_multipart_upload_10() { - # log start time - start_time=$(get_time) - - function="make_bucket" - bucket_name=$(make_bucket) - object_name=${bucket_name}"-object" - rv=$? - - if [ $rv -eq 0 ]; then - # create multipart - function="${AWS} s3api create-multipart-upload --bucket ${bucket_name} --key ${object_name}" - test_function=${function} - out=$($function) - rv=$? - upload_id=$(echo "$out" | jq -r .UploadId) - fi - - if [ $rv -eq 0 ]; then - # Capture etag for part-number 1 - function="${AWS} s3api upload-part --bucket ${bucket_name} --key ${object_name} --body ${MINT_DATA_DIR}/datafile-10-MB --upload-id ${upload_id} --part-number 1" - out=$($function) - rv=$? - etag1=$(echo "$out" | jq -r .ETag) - fi - - if [ $rv -eq 0 ]; then - # Capture etag for part-number 2 - function="${AWS} s3api upload-part --bucket ${bucket_name} --key ${object_name} --body ${MINT_DATA_DIR}/datafile-10-MB --upload-id ${upload_id} --part-number 2" - out=$($function) - rv=$? - etag2=$(echo "$out" | jq -r .ETag) - # Create a multipart struct file for completing multipart transaction - echo "{ - \"Parts\": [ - { - \"ETag\": ${etag1}, - \"PartNumber\": 1 - }, - { - \"ETag\": ${etag2}, - \"PartNumber\": 2 - } - ] - }" >> /tmp/multipart - fi - - if [ $rv -eq 0 ]; then - # Use saved etags to complete the multipart transaction - function="${AWS} s3api complete-multipart-upload --multipart-upload file:///tmp/multipart --bucket ${bucket_name} --key ${object_name} --upload-id ${upload_id}" - out=$($function) - rv=$? - finalETag=$(echo "$out" | jq -r .ETag | sed -e 's/^"//' -e 's/"$//') - if [ "${finalETag}" == "" ]; then - rv=1 - out="complete-multipart-upload failed" - fi - fi - - if [ $rv -eq 0 ]; then - function="delete_bucket" - out=$(delete_bucket "$bucket_name") - rv=$? - # remove temp file - rm -f /tmp/multipart - fi - - if [ $rv -eq 0 ]; then - log_success "$(get_duration "$start_time")" "${test_function}" - else - # clean up and log error - ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 - rm -f /tmp/multipart - log_failure "$(get_duration "$start_time")" "${function}" "${out}" - fi - - return $rv -} - -# Tests `aws s3 cp` by uploading a local file. -function test_aws_s3_cp() { - file_name="${MINT_DATA_DIR}/datafile-65-MB" - - # log start time - start_time=$(get_time) - - function="make_bucket" - bucket_name=$(make_bucket) - rv=$? - - # if make bucket succeeds upload a file using cp - if [ $rv -eq 0 ]; then - function="${AWS} s3 cp $file_name s3://${bucket_name}/$(basename "$file_name")" - test_function=${function} - out=$($function 2>&1) - rv=$? - else - # if make bucket fails, $bucket_name has the error output - out="${bucket_name}" - fi - - if [ $rv -eq 0 ]; then - function="${AWS} s3 rm s3://${bucket_name}/$(basename "$file_name")" - out=$($function 2>&1) - rv=$? - fi - - if [ $rv -eq 0 ]; then - function="${AWS} s3 rb s3://${bucket_name}/" - out=$($function 2>&1) - rv=$? - fi - - if [ $rv -eq 0 ]; then - log_success "$(get_duration "$start_time")" "${test_function}" - else - # clean up and log error - ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 - log_failure "$(get_duration "$start_time")" "${function}" "${out}" - fi - - return $rv -} - -# Tests `aws s3 sync` by mirroring all the -# local content to remove bucket. -function test_aws_s3_sync() { - # log start time - start_time=$(get_time) - - function="make_bucket" - bucket_name=$(make_bucket) - rv=$? - - # if make bucket succeeds sync all the files in a directory - if [ $rv -eq 0 ]; then - function="${AWS} s3 sync --no-progress $MINT_DATA_DIR s3://${bucket_name}/" - test_function=${function} - out=$($function 2>&1) - rv=$? - else - # if make bucket fails, $bucket_name has the error output - out="${bucket_name}" - fi - - # remove files recusively - if [ $rv -eq 0 ]; then - function="${AWS} s3 rm --recursive s3://${bucket_name}/" - out=$($function 2>&1) - rv=$? - fi - - # delete bucket - if [ $rv -eq 0 ]; then - function="delete_bucket" - out=$(delete_bucket "$bucket_name") - rv=$? - fi - - if [ $rv -eq 0 ]; then - log_success "$(get_duration "$start_time")" "${test_function}" - else - # clean up and log error - ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 - log_failure "$(get_duration "$start_time")" "${function}" "${out}" - fi - - return $rv -} - -# list objects negative test - tests for following conditions. -# v1 API with max-keys=-1 and max-keys=0 -# v2 API with max-keys=-1 and max-keys=0 -function test_list_objects_error() { - # log start time - start_time=$(get_time) - - function="make_bucket" - bucket_name=$(make_bucket) - rv=$? - - # if make bucket succeeds upload a file - if [ $rv -eq 0 ]; then - function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB" - out=$($function 2>&1) - rv=$? - else - # if make bucket fails, $bucket_name has the error output - out="${bucket_name}" - fi - - if [ $rv -eq 0 ]; then - # Server replies an error for v1 with max-key=-1 - function="${AWS} s3api list-objects --bucket ${bucket_name} --prefix datafile-1-kB --max-keys=-1" - test_function=${function} - out=$($function 2>&1) - rv=$? - if [ $rv -ne 255 ]; then - rv=1 - else - rv=0 - fi - fi - - if [ $rv -eq 0 ]; then - # Server replies an error for v2 with max-keys=-1 - function="${AWS} s3api list-objects-v2 --bucket ${bucket_name} --prefix datafile-1-kB --max-keys=-1" - test_function=${function} - out=$($function 2>&1) - rv=$? - if [ $rv -ne 255 ]; then - rv=1 - else - rv=0 - fi - fi - - if [ $rv -eq 0 ]; then - # Server returns success with no keys when max-keys=0 - function="${AWS} s3api list-objects-v2 --bucket ${bucket_name} --prefix datafile-1-kB --max-keys=0" - out=$($function 2>&1) - rv=$? - if [ $rv -eq 0 ]; then - function="delete_bucket" - out=$(delete_bucket "$bucket_name") - rv=$? - fi - fi - - if [ $rv -eq 0 ]; then - log_success "$(get_duration "$start_time")" "${test_function}" - else - # clean up and log error - ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 - log_failure "$(get_duration "$start_time")" "${function}" "${out}" - fi - - return $rv -} - -# put object negative test - tests for following conditions. -# - invalid object name. -# - invalid Content-Md5 -# - invalid Content-Length -function test_put_object_error() { - # log start time - start_time=$(get_time) - - function="make_bucket" - bucket_name=$(make_bucket) - rv=$? - - # if make bucket succeeds upload an object without content-md5. - if [ $rv -eq 0 ]; then - function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB --content-md5 invalid" - test_function=${function} - out=$($function 2>&1) - rv=$? - if [ $rv -ne 255 ]; then - rv=1 - else - rv=0 - fi - fi - - # upload an object without content-length. - if [ $rv -eq 0 ]; then - function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB --content-length -1" - test_function=${function} - out=$($function 2>&1) - rv=$? - if [ $rv -ne 255 ]; then - rv=1 - else - rv=0 - fi - fi - - if [ $rv -eq 0 ]; then - function="delete_bucket" - out=$(delete_bucket "$bucket_name") - rv=$? - fi - - if [ $rv -eq 0 ]; then - log_success "$(get_duration "$start_time")" "${test_function}" - else - # clean up and log error - ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 - log_failure "$(get_duration "$start_time")" "${function}" "${out}" - fi - - return $rv -} -# tests server side encryption headers for get and put calls -function test_serverside_encryption() { - #skip server side encryption tests if HTTPS disabled. - if [ "$ENABLE_HTTPS" != "1" ]; then - return 0 - fi - # log start time - start_time=$(get_time) - - function="make_bucket" - bucket_name=$(make_bucket) - rv=$? - - # put object with server side encryption headers - if [ $rv -eq 0 ]; then - function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB --sse-customer-algorithm AES256 --sse-customer-key MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ= --sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg==" - test_function=${function} - out=$($function 2>&1) - rv=$? - fi - # now get encrypted object from server - if [ $rv -eq 0 ]; then - etag1=$(echo "$out" | jq -r .ETag) - sse_customer_key1=$(echo "$out" | jq -r .SSECustomerKeyMD5) - sse_customer_algo1=$(echo "$out" | jq -r .SSECustomerAlgorithm) - - function="${AWS} s3api get-object --bucket ${bucket_name} --key datafile-1-kB --sse-customer-algorithm AES256 --sse-customer-key MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ= --sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg== /tmp/datafile-1-kB" - test_function=${function} - out=$($function 2>&1) - rv=$? - fi - if [ $rv -eq 0 ]; then - etag2=$(echo "$out" | jq -r .ETag) - sse_customer_key2=$(echo "$out" | jq -r .SSECustomerKeyMD5) - sse_customer_algo2=$(echo "$out" | jq -r .SSECustomerAlgorithm) - hash2=$(md5sum /tmp/datafile-1-kB | awk '{print $1}') - # match downloaded object's hash to original - if [ "$HASH_1_KB" == "$hash2" ]; then - function="delete_bucket" - out=$(delete_bucket "$bucket_name") - rv=$? - # remove download file - rm -f /tmp/datafile-1-kB - else - rv=1 - out="Checksum verification failed for downloaded object" - fi - # match etag and SSE headers - if [ "$etag1" != "$etag2" ]; then - rv=1 - out="Etag mismatch for object encrypted with server side encryption" - fi - if [ "$sse_customer_algo1" != "$sse_customer_algo2" ]; then - rv=1 - out="sse customer algorithm mismatch" - fi - if [ "$sse_customer_key1" != "$sse_customer_key2" ]; then - rv=1 - out="sse customer key mismatch" - fi - fi - - if [ $rv -eq 0 ]; then - log_success "$(get_duration "$start_time")" "${test_function}" - else - # clean up and log error - ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 - log_failure "$(get_duration "$start_time")" "${function}" "${out}" - fi - - return $rv -} - -# tests server side encryption headers for multipart put -function test_serverside_encryption_multipart() { - #skip server side encryption tests if HTTPS disabled. - if [ "$ENABLE_HTTPS" != "1" ]; then - return 0 - fi - # log start time - start_time=$(get_time) - - function="make_bucket" - bucket_name=$(make_bucket) - rv=$? - - # put object with server side encryption headers - if [ $rv -eq 0 ]; then - function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-65-MB --bucket ${bucket_name} --key datafile-65-MB --sse-customer-algorithm AES256 --sse-customer-key MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ= --sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg==" - test_function=${function} - out=$($function 2>&1) - rv=$? - fi - # now get encrypted object from server - if [ $rv -eq 0 ]; then - etag1=$(echo "$out" | jq -r .ETag) - sse_customer_key1=$(echo "$out" | jq -r .SSECustomerKeyMD5) - sse_customer_algo1=$(echo "$out" | jq -r .SSECustomerAlgorithm) - - function="${AWS} s3api get-object --bucket ${bucket_name} --key datafile-65-MB --sse-customer-algorithm AES256 --sse-customer-key MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ= --sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg== /tmp/datafile-65-MB" - test_function=${function} - out=$($function 2>&1) - rv=$? - fi - if [ $rv -eq 0 ]; then - etag2=$(echo "$out" | jq -r .ETag) - sse_customer_key2=$(echo "$out" | jq -r .SSECustomerKeyMD5) - sse_customer_algo2=$(echo "$out" | jq -r .SSECustomerAlgorithm) - hash2=$(md5sum /tmp/datafile-65-MB | awk '{print $1}') - # match downloaded object's hash to original - if [ "$HASH_65_MB" == "$hash2" ]; then - function="delete_bucket" - out=$(delete_bucket "$bucket_name") - rv=$? - # remove download file - rm -f /tmp/datafile-65-MB - else - rv=1 - out="Checksum verification failed for downloaded object" - fi - # match etag and SSE headers - if [ "$etag1" != "$etag2" ]; then - rv=1 - out="Etag mismatch for object encrypted with server side encryption" - fi - if [ "$sse_customer_algo1" != "$sse_customer_algo2" ]; then - rv=1 - out="sse customer algorithm mismatch" - fi - if [ "$sse_customer_key1" != "$sse_customer_key2" ]; then - rv=1 - out="sse customer key mismatch" - fi - fi - - if [ $rv -eq 0 ]; then - log_success "$(get_duration "$start_time")" "${test_function}" - else - # clean up and log error - ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 - log_failure "$(get_duration "$start_time")" "${function}" "${out}" - fi - - return $rv -} - -# tests encrypted copy from multipart encrypted object to -# single part encrypted object. This test in particular checks if copy -# succeeds for the case where encryption overhead for individually -# encrypted parts vs encryption overhead for the original datastream -# differs. -function test_serverside_encryption_multipart_copy() { - #skip server side encryption tests if HTTPS disabled. - if [ "$ENABLE_HTTPS" != "1" ]; then - return 0 - fi - # log start time - start_time=$(get_time) - - function="make_bucket" - bucket_name=$(make_bucket) - object_name=${bucket_name}"-object" - rv=$? - - if [ $rv -eq 0 ]; then - # create multipart - function="${AWS} s3api create-multipart-upload --bucket ${bucket_name} --key ${object_name} --sse-customer-algorithm AES256 --sse-customer-key MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ= --sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg==" - out=$($function) - rv=$? - upload_id=$(echo "$out" | jq -r .UploadId) - fi - - if [ $rv -eq 0 ]; then - # Capture etag for part-number 1 - function="${AWS} s3api upload-part --bucket ${bucket_name} --key ${object_name} --body ${MINT_DATA_DIR}/datafile-5243880-b --upload-id ${upload_id} --part-number 1 --sse-customer-algorithm AES256 --sse-customer-key MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ= --sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg==" - out=$($function) - rv=$? - etag1=$(echo "$out" | jq -r .ETag) - fi - - if [ $rv -eq 0 ]; then - # Capture etag for part-number 2 - function="${AWS} s3api upload-part --bucket ${bucket_name} --key ${object_name} --body ${MINT_DATA_DIR}/datafile-5243880-b --upload-id ${upload_id} --part-number 2 --sse-customer-algorithm AES256 --sse-customer-key MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ= --sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg==" - out=$($function) - rv=$? - etag2=$(echo "$out" | jq -r .ETag) - # Create a multipart struct file for completing multipart transaction - echo "{ - \"Parts\": [ - { - \"ETag\": ${etag1}, - \"PartNumber\": 1 - }, - { - \"ETag\": ${etag2}, - \"PartNumber\": 2 - } - ] - }" >> /tmp/multipart - fi - - if [ $rv -eq 0 ]; then - # Use saved etags to complete the multipart transaction - function="${AWS} s3api complete-multipart-upload --multipart-upload file:///tmp/multipart --bucket ${bucket_name} --key ${object_name} --upload-id ${upload_id}" - out=$($function) - rv=$? - finalETag=$(echo "$out" | jq -r .ETag | sed -e 's/^"//' -e 's/"$//') - if [ "${finalETag}" == "" ]; then - rv=1 - out="complete-multipart-upload failed" - fi - fi - - # copy object server side - if [ $rv -eq 0 ]; then - function="${AWS} s3api copy-object --bucket ${bucket_name} --key ${object_name}-copy --copy-source ${bucket_name}/${object_name} --copy-source-sse-customer-algorithm AES256 --copy-source-sse-customer-key MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ= --copy-source-sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg== --sse-customer-algorithm AES256 --sse-customer-key MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ= --sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg==" - test_function=${function} - out=$($function) - rv=$? - if [ $rv -ne 255 ]; then - rv=1 - else - rv=0 - fi - fi - - if [ $rv -eq 0 ]; then - log_success "$(get_duration "$start_time")" "${test_function}" - else - # clean up and log error - ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 - rm -f /tmp/multipart - log_failure "$(get_duration "$start_time")" "${function}" "${out}" - fi - - return $rv -} -# tests server side encryption headers for range get calls -function test_serverside_encryption_get_range() { - #skip server side encryption tests if HTTPS disabled. - if [ "$ENABLE_HTTPS" != "1" ]; then - return 0 - fi - # log start time - start_time=$(get_time) - - function="make_bucket" - bucket_name=$(make_bucket) - rv=$? - # put object with server side encryption headers - if [ $rv -eq 0 ]; then - function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-10-kB --bucket ${bucket_name} --key datafile-10-kB --sse-customer-algorithm AES256 --sse-customer-key MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ= --sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg==" - test_function=${function} - out=$($function 2>&1) - rv=$? - fi - # now get encrypted object from server for range 500-999 - if [ $rv -eq 0 ]; then - etag1=$(echo "$out" | jq -r .ETag) - sse_customer_key1=$(echo "$out" | jq -r .SSECustomerKeyMD5) - sse_customer_algo1=$(echo "$out" | jq -r .SSECustomerAlgorithm) - function="${AWS} s3api get-object --bucket ${bucket_name} --key datafile-10-kB --range bytes=500-999 --sse-customer-algorithm AES256 --sse-customer-key MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ= --sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg== /tmp/datafile-10-kB" - test_function=${function} - out=$($function 2>&1) - rv=$? - fi - if [ $rv -eq 0 ]; then - cnt=$(stat -c%s /tmp/datafile-10-kB) - if [ "$cnt" -ne 500 ]; then - rv=1 - fi - fi - if [ $rv -eq 0 ]; then - log_success "$(get_duration "$start_time")" "${test_function}" - else - # clean up and log error - ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 - log_failure "$(get_duration "$start_time")" "${function}" "${out}" - fi - return $rv -} - -# tests server side encryption error for get and put calls -function test_serverside_encryption_error() { - #skip server side encryption tests if HTTPS disabled. - if [ "$ENABLE_HTTPS" != "1" ]; then - return 0 - fi - # log start time - start_time=$(get_time) - - function="make_bucket" - bucket_name=$(make_bucket) - rv=$? - - # put object with server side encryption headers with MD5Sum mismatch for sse-customer-key-md5 header - if [ $rv -eq 0 ]; then - function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB --sse-customer-algorithm AES256 --sse-customer-key MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ= --sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg" - test_function=${function} - out=$($function 2>&1) - rv=$? - fi - - if [ $rv -ne 255 ]; then - rv=1 - else - rv=0 - fi - # put object with missing server side encryption header sse-customer-algorithm - if [ $rv -eq 0 ]; then - function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB --sse-customer-key MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ= --sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg==" - test_function=${function} - out=$($function 2>&1) - rv=$? - fi - - if [ $rv -ne 255 ]; then - rv=1 - else - rv=0 - fi - - # put object with server side encryption headers successfully - if [ $rv -eq 0 ]; then - function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB --sse-customer-algorithm AES256 --sse-customer-key MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ= --sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg==" - test_function=${function} - out=$($function 2>&1) - rv=$? - fi - - # now test get on encrypted object with nonmatching sse-customer-key and sse-customer-md5 headers - if [ $rv -eq 0 ]; then - function="${AWS} s3api get-object --bucket ${bucket_name} --key datafile-1-kB --sse-customer-algorithm AES256 --sse-customer-key MzJieXRlc --sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg== /tmp/datafile-1-kB" - test_function=${function} - out=$($function 2>&1) - rv=$? - fi - if [ $rv -ne 255 ]; then - rv=1 - else - rv=0 - fi - # delete bucket - if [ $rv -eq 0 ]; then - function="delete_bucket" - out=$(delete_bucket "$bucket_name") - rv=$? - fi - if [ $rv -eq 0 ]; then - log_success "$(get_duration "$start_time")" "${test_function}" - else - # clean up and log error - ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 - log_failure "$(get_duration "$start_time")" "${function}" "${out}" - fi - - return $rv -} - -# WORM bucket tests. -function test_worm_bucket() { - # log start time - start_time=$(get_time) - - # Make bucket - bucket_name="awscli-mint-test-bucket-$RANDOM" - function="${AWS} s3api create-bucket --bucket ${bucket_name} --object-lock-enabled-for-bucket" - - # execute the test - out=$($function 2>&1) - rv=$? - - if [ $rv -ne 0 ]; then - # if this functionality is not implemented return right away. - if echo "$out" | grep -q "NotImplemented"; then - ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 - return 0 - fi - fi - - # if make bucket succeeds set object lock configuration - if [ $rv -eq 0 ]; then - function="${AWS} s3api put-object-lock-configuration --bucket ${bucket_name} --object-lock-configuration ObjectLockEnabled=Enabled" - out=$($function 2>&1) - rv=$? - if [ $rv -ne 0 ]; then - # if this functionality is not implemented return right away. - if echo "$out" | grep -q "NotImplemented"; then - ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 - return 0 - fi - fi - else - # if make bucket fails, $bucket_name has the error output - out="${bucket_name}" - fi - - # if setting object lock configuration succeeds, upload a file first time - if [ $rv -eq 0 ]; then - function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB" - out=$($function 2>&1) - rv=$? - else - # if make bucket fails, $bucket_name has the error output - out="${bucket_name}" - fi - - # second time upload of same file should fail due to WORM setting - if [ $rv -eq 0 ]; then - function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB" - out=$($function 2>&1) - rv=$? - else - out="First time object upload failed" - fi - - if [ $rv -eq 0 ]; then - log_success "$(get_duration "$start_time")" "${test_function}" - else - # cleanup is not possible due to one day validity of object lock configurataion - log_failure "$(get_duration "$start_time")" "${function}" "${out}" - fi - - return $rv -} - -# Tests creating and deleting an object with legal hold. -function test_legal_hold() { - # log start time - start_time=$(get_time) - - # Make bucket - bucket_name="awscli-mint-test-bucket-$RANDOM" - function="${AWS} s3api create-bucket --bucket ${bucket_name} --object-lock-enabled-for-bucket" - - # execute the test - out=$($function 2>&1) - rv=$? - - if [ $rv -ne 0 ]; then - # if this functionality is not implemented return right away. - if echo "$out" | grep -q "NotImplemented"; then - ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 - return 0 - fi - fi - - # if make bucket succeeds upload a file - if [ $rv -eq 0 ]; then - function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB --object-lock-legal-hold-status ON" - out=$($function 2>&1) - errcnt=$(echo "$out" | sed -n '/Bucket is missing ObjectLockConfiguration/p' | wc -l) - # skip test for gateways - if [ "$errcnt" -eq 1 ]; then - return 0 - fi - rv=$? - else - # if make bucket fails, $bucket_name has the error output - out="${bucket_name}" - fi - - # if upload succeeds stat the file - if [ $rv -eq 0 ]; then - function="${AWS} s3api head-object --bucket ${bucket_name} --key datafile-1-kB" - # save the ref to function being tested, so it can be logged - test_function=${function} - out=$($function 2>&1) - lhold=$(echo "$out" | jq -r .ObjectLockLegalHoldStatus) - rv=$? - fi - - # if head-object succeeds, verify metadata has legal hold status - if [ $rv -eq 0 ]; then - if [ "${lhold}" == "" ]; then - rv=1 - out="Legal hold was not applied" - fi - if [ "${lhold}" == "OFF" ]; then - rv=1 - out="Legal hold was not applied" - fi - fi - if [ $rv -eq 0 ]; then - function="${AWS} s3api put-object-legal-hold --bucket ${bucket_name} --key datafile-1-kB --legal-hold Status=OFF" - out=$($function 2>&1) - rv=$? - else - # if make bucket fails, $bucket_name has the error output - out="${bucket_name}" - fi - # if upload succeeds download the file - if [ $rv -eq 0 ]; then - function="${AWS} s3api get-object-legal-hold --bucket ${bucket_name} --key datafile-1-kB" - # save the ref to function being tested, so it can be logged - test_function=${function} - out=$($function 2>&1) - lhold=$(echo "$out" | jq -r .LegalHold.Status) - rv=$? - fi - - # if head-object succeeds, verify metadata has legal hold status - if [ $rv -eq 0 ]; then - if [ "${lhold}" == "" ]; then - rv=1 - out="Legal hold was not applied" - fi - if [ "${lhold}" == "ON" ]; then - rv=1 - out="Legal hold status not turned off" - fi - fi - # Attempt a delete on prefix shouldn't delete the directory since we have an object inside it. - if [ $rv -eq 0 ]; then - function="${AWS} s3api delete-object --bucket ${bucket_name} --key datafile-1-kB" - # save the ref to function being tested, so it can be logged - test_function=${function} - out=$($function 2>&1) - rv=$? - fi - if [ $rv -eq 0 ]; then - log_success "$(get_duration "$start_time")" "${test_function}" - else - # clean up and log error - ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 - log_failure "$(get_duration "$start_time")" "${function}" "${out}" - fi - - return $rv -} - -# main handler for all the tests. -main() { - # Success tests - test_create_bucket && \ - test_upload_object && \ - test_lookup_object_prefix && \ - test_list_objects && \ - test_multipart_upload_0byte && \ - test_multipart_upload && \ - test_max_key_list && \ - test_copy_object && \ - test_copy_object_storage_class && \ - test_copy_object_storage_class_same && \ - test_presigned_object && \ - test_upload_object_10 && \ - test_multipart_upload_10 && \ - test_serverside_encryption && \ - test_serverside_encryption_get_range && \ - test_serverside_encryption_multipart && \ - test_serverside_encryption_multipart_copy && \ - # Success cli ops. - test_aws_s3_cp && \ - test_aws_s3_sync && \ - # Error tests - test_list_objects_error && \ - test_put_object_error && \ - test_serverside_encryption_error && \ - test_worm_bucket && \ - test_legal_hold - - return $? -} - -_init "$@" && main diff --git a/mint/run/core/healthcheck/healthcheck.go b/mint/run/core/healthcheck/healthcheck.go deleted file mode 100644 index d142fa6..0000000 --- a/mint/run/core/healthcheck/healthcheck.go +++ /dev/null @@ -1,219 +0,0 @@ -/* -* -* Mint, (C) 2019 Minio, Inc. -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software - -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -* - */ - -package main - -import ( - "crypto/tls" - "encoding/json" - "fmt" - "net/http" - "net/url" - "os" - "time" - - jwtgo "github.com/dgrijalva/jwt-go" - log "github.com/sirupsen/logrus" -) - -const ( - pass = "PASS" // Indicate that a test passed - fail = "FAIL" // Indicate that a test failed - livenessPath = "/minio/health/live" - readinessPath = "/minio/health/ready" - prometheusPath = "/minio/prometheus/metrics" - timeout = time.Duration(30 * time.Second) -) - -type mintJSONFormatter struct { -} - -func (f *mintJSONFormatter) Format(entry *log.Entry) ([]byte, error) { - data := make(log.Fields, len(entry.Data)) - for k, v := range entry.Data { - switch v := v.(type) { - case error: - // Otherwise errors are ignored by `encoding/json` - // https://github.com/sirupsen/logrus/issues/137 - data[k] = v.Error() - default: - data[k] = v - } - } - - serialized, err := json.Marshal(data) - if err != nil { - return nil, fmt.Errorf("Failed to marshal fields to JSON, %w", err) - } - return append(serialized, '\n'), nil -} - -// log successful test runs -func successLogger(function string, args map[string]interface{}, startTime time.Time) *log.Entry { - // calculate the test case duration - duration := time.Since(startTime) - // log with the fields as per mint - fields := log.Fields{"name": "healthcheck", "function": function, "args": args, "duration": duration.Nanoseconds() / 1000000, "status": pass} - return log.WithFields(fields) -} - -// log failed test runs -func failureLog(function string, args map[string]interface{}, startTime time.Time, alert string, message string, err error) *log.Entry { - // calculate the test case duration - duration := time.Since(startTime) - var fields log.Fields - // log with the fields as per mint - if err != nil { - fields = log.Fields{"name": "healthcheck", "function": function, "args": args, - "duration": duration.Nanoseconds() / 1000000, "status": fail, "alert": alert, "message": message, "error": err} - } else { - fields = log.Fields{"name": "healthcheck", "function": function, "args": args, - "duration": duration.Nanoseconds() / 1000000, "status": fail, "alert": alert, "message": message} - } - return log.WithFields(fields) -} - -func testLivenessEndpoint(endpoint string) { - startTime := time.Now() - function := "testLivenessEndpoint" - - u, err := url.Parse(fmt.Sprintf("%s%s", endpoint, livenessPath)) - if err != nil { - // Could not parse URL successfully - failureLog(function, nil, startTime, "", "URL Parsing for Healthcheck Liveness handler failed", err).Fatal() - } - - tr := &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: u.Scheme == "https"}, - } - client := &http.Client{Transport: tr, Timeout: timeout} - resp, err := client.Get(u.String()) - if err != nil { - // GET request errored - failureLog(function, nil, startTime, "", "GET request failed", err).Fatal() - } - if resp.StatusCode != http.StatusOK { - // Status not 200 OK - failureLog(function, nil, startTime, "", fmt.Sprintf("GET /minio/health/live returned %s", resp.Status), err).Fatal() - } - - defer resp.Body.Close() - defer successLogger(function, nil, startTime).Info() -} - -func testReadinessEndpoint(endpoint string) { - startTime := time.Now() - function := "testReadinessEndpoint" - - u, err := url.Parse(fmt.Sprintf("%s%s", endpoint, readinessPath)) - if err != nil { - // Could not parse URL successfully - failureLog(function, nil, startTime, "", "URL Parsing for Healthcheck Readiness handler failed", err).Fatal() - } - - tr := &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: u.Scheme == "https"}, - } - client := &http.Client{Transport: tr, Timeout: timeout} - resp, err := client.Get(u.String()) - if err != nil { - // GET request errored - failureLog(function, nil, startTime, "", "GET request to Readiness endpoint failed", err).Fatal() - } - if resp.StatusCode != http.StatusOK { - // Status not 200 OK - failureLog(function, nil, startTime, "", "GET /minio/health/ready returned non OK status", err).Fatal() - } - - defer resp.Body.Close() - defer successLogger(function, nil, startTime).Info() -} - -const ( - defaultPrometheusJWTExpiry = 100 * 365 * 24 * time.Hour -) - -func testPrometheusEndpoint(endpoint string) { - startTime := time.Now() - function := "testPrometheusEndpoint" - - u, err := url.Parse(fmt.Sprintf("%s%s", endpoint, prometheusPath)) - if err != nil { - // Could not parse URL successfully - failureLog(function, nil, startTime, "", "URL Parsing for Healthcheck Prometheus handler failed", err).Fatal() - } - - jwt := jwtgo.NewWithClaims(jwtgo.SigningMethodHS512, jwtgo.StandardClaims{ - ExpiresAt: time.Now().UTC().Add(defaultPrometheusJWTExpiry).Unix(), - Subject: os.Getenv("ACCESS_KEY"), - Issuer: "prometheus", - }) - - token, err := jwt.SignedString([]byte(os.Getenv("SECRET_KEY"))) - if err != nil { - failureLog(function, nil, startTime, "", "jwt generation failed", err).Fatal() - } - - tr := &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: u.Scheme == "https"}, - } - client := &http.Client{Transport: tr, Timeout: timeout} - - req, err := http.NewRequest(http.MethodGet, u.String(), nil) - if err != nil { - failureLog(function, nil, startTime, "", "Initializing GET request to Prometheus endpoint failed", err).Fatal() - } - req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token)) - - resp, err := client.Do(req) - if err != nil { - // GET request errored - failureLog(function, nil, startTime, "", "GET request to Prometheus endpoint failed", err).Fatal() - } - - if resp.StatusCode != http.StatusOK { - // Status not 200 OK - failureLog(function, nil, startTime, "", "GET /minio/prometheus/metrics returned non OK status", err).Fatal() - } - - defer resp.Body.Close() - defer successLogger(function, nil, startTime).Info() -} - -func main() { - endpoint := os.Getenv("SERVER_ENDPOINT") - secure := os.Getenv("ENABLE_HTTPS") - endpoint = "http://" + endpoint - if secure == "1" { - endpoint = "https://" + endpoint - } - - // Output to stdout instead of the default stderr - log.SetOutput(os.Stdout) - // create custom formatter - mintFormatter := mintJSONFormatter{} - // set custom formatter - log.SetFormatter(&mintFormatter) - // log Info or above -- success cases are Info level, failures are Fatal level - log.SetLevel(log.InfoLevel) - // execute tests - testLivenessEndpoint(endpoint) - testReadinessEndpoint(endpoint) - testPrometheusEndpoint(endpoint) -} diff --git a/mint/run/core/healthcheck/run.sh b/mint/run/core/healthcheck/run.sh deleted file mode 100755 index 727c5a8..0000000 --- a/mint/run/core/healthcheck/run.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -# -# Mint (C) 2019 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# handle command line arguments -if [ $# -ne 2 ]; then - echo "usage: run.sh " - exit 1 -fi - -output_log_file="$1" -error_log_file="$2" - -# run tests -/mint/run/core/healthcheck/healthcheck 1>>"$output_log_file" 2>"$error_log_file" diff --git a/mint/run/core/mc/README.md b/mint/run/core/mc/README.md deleted file mode 100644 index 9a550e0..0000000 --- a/mint/run/core/mc/README.md +++ /dev/null @@ -1,19 +0,0 @@ -## `mc` tests -This directory serves as the location for Mint tests using `mc`. Top level `mint.sh` calls `run.sh` to execute tests. - -## Adding new tests -New tests is added into `test.sh` as new functions. - -## Running tests manually -- Set environment variables `MINT_DATA_DIR`, `MINT_MODE`, `SERVER_ENDPOINT`, `ACCESS_KEY`, `SECRET_KEY`, `SERVER_REGION` and `ENABLE_HTTPS` -- Call `run.sh` with output log file and error log file. for example -```bash -export MINT_DATA_DIR=~/my-mint-dir -export MINT_MODE=core -export SERVER_ENDPOINT="play.minio.io:9000" -export ACCESS_KEY="Q3AM3UQ867SPQQA43P2F" -export SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" -export ENABLE_HTTPS=1 -export SERVER_REGION=us-east-1 -./run.sh /tmp/output.log /tmp/error.log -``` diff --git a/mint/run/core/mc/run.sh b/mint/run/core/mc/run.sh deleted file mode 100755 index b13c5e2..0000000 --- a/mint/run/core/mc/run.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash -# -# Minio Cloud Storage, (C) 2017 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# handle command line arguments -if [ $# -ne 2 ]; then - echo "usage: run.sh " - exit 1 -fi - -output_log_file="$1" -error_log_file="$2" - -./functional-tests.sh 1>>"$output_log_file" 2>"$error_log_file" diff --git a/mint/run/core/minio-dotnet/run.sh b/mint/run/core/minio-dotnet/run.sh deleted file mode 100755 index 473a8e1..0000000 --- a/mint/run/core/minio-dotnet/run.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -# -# Mint (C) 2017 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# handle command line arguments -if [ $# -ne 2 ]; then - echo "usage: run.sh " - exit 1 -fi - -output_log_file="$1" -error_log_file="$2" -/mint/run/core/minio-dotnet/out/Minio.Functional.Tests 1>>"$output_log_file" 2>"$error_log_file" diff --git a/mint/run/core/minio-go/README.md b/mint/run/core/minio-go/README.md deleted file mode 100644 index 6f1723b..0000000 --- a/mint/run/core/minio-go/README.md +++ /dev/null @@ -1,19 +0,0 @@ -## `minio-go` tests -This directory serves as the location for Mint tests using `minio-go`. Top level `mint.sh` calls `run.sh` to execute tests. - -## Adding new tests -New tests are added in functional tests of minio-go. Please check https://github.com/minio/minio-go - -## Running tests manually -- Set environment variables `MINT_DATA_DIR`, `MINT_MODE`, `SERVER_ENDPOINT`, `ACCESS_KEY`, `SECRET_KEY`, `SERVER_REGION` and `ENABLE_HTTPS` -- Call `run.sh` with output log file and error log file. for example -```bash -export MINT_DATA_DIR=~/my-mint-dir -export MINT_MODE=core -export SERVER_ENDPOINT="play.minio.io:9000" -export ACCESS_KEY="Q3AM3UQ867SPQQA43P2F" -export SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" -export ENABLE_HTTPS=1 -export SERVER_REGION=us-east-1 -./run.sh /tmp/output.log /tmp/error.log -``` diff --git a/mint/run/core/minio-go/run.sh b/mint/run/core/minio-go/run.sh deleted file mode 100755 index 5a3255b..0000000 --- a/mint/run/core/minio-go/run.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -# -# Mint (C) 2017 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# handle command line arguments -if [ $# -ne 2 ]; then - echo "usage: run.sh " - exit 1 -fi - -output_log_file="$1" -error_log_file="$2" - -# run tests -/mint/run/core/minio-go/minio-go 1>>"$output_log_file" 2>"$error_log_file" diff --git a/mint/run/core/minio-java/README.md b/mint/run/core/minio-java/README.md deleted file mode 100644 index 8e73d7d..0000000 --- a/mint/run/core/minio-java/README.md +++ /dev/null @@ -1,19 +0,0 @@ -## `minio-java` tests -This directory serves as the location for Mint tests using `minio-java`. Top level `mint.sh` calls `run.sh` to execute tests. - -## Adding new tests -New tests is added in functional tests of minio-java. Please check https://github.com/minio/minio-java - -## Running tests manually -- Set environment variables `MINT_DATA_DIR`, `MINT_MODE`, `SERVER_ENDPOINT`, `ACCESS_KEY`, `SECRET_KEY`, `SERVER_REGION` and `ENABLE_HTTPS` -- Call `run.sh` with output log file and error log file. for example -```bash -export MINT_DATA_DIR=~/my-mint-dir -export MINT_MODE=core -export SERVER_ENDPOINT="play.minio.io:9000" -export ACCESS_KEY="Q3AM3UQ867SPQQA43P2F" -export SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" -export ENABLE_HTTPS=1 -export SERVER_REGION=us-east-1 -./run.sh /tmp/output.log /tmp/error.log -``` diff --git a/mint/run/core/minio-java/run.sh b/mint/run/core/minio-java/run.sh deleted file mode 100755 index 6a2fc14..0000000 --- a/mint/run/core/minio-java/run.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash -# -# Mint (C) 2017 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# handle command line arguments -if [ $# -ne 2 ]; then - echo "usage: run.sh " - exit 1 -fi - -output_log_file="$1" -error_log_file="$2" - -# run tests -endpoint="http://$SERVER_ENDPOINT" -if [ "$ENABLE_HTTPS" -eq 1 ]; then - endpoint="https://$SERVER_ENDPOINT" -fi - -java -Xmx4096m -Xms256m -cp "/mint/run/core/minio-java/*:." FunctionalTest \ - "$endpoint" "$ACCESS_KEY" "$SECRET_KEY" "$SERVER_REGION" 1>>"$output_log_file" 2>"$error_log_file" diff --git a/mint/run/core/minio-js/README.md b/mint/run/core/minio-js/README.md deleted file mode 100644 index 1144a61..0000000 --- a/mint/run/core/minio-js/README.md +++ /dev/null @@ -1,19 +0,0 @@ -## `minio-js` tests -This directory serves as the location for Mint tests using `minio-js`. Top level `mint.sh` calls `run.sh` to execute tests. - -## Adding new tests -New tests is added in functional tests of minio-js. Please check https://github.com/minio/minio-js - -## Running tests manually -- Set environment variables `MINT_DATA_DIR`, `MINT_MODE`, `SERVER_ENDPOINT`, `ACCESS_KEY`, `SECRET_KEY`, `SERVER_REGION` and `ENABLE_HTTPS` -- Call `run.sh` with output log file and error log file. for example -```bash -export MINT_DATA_DIR=~/my-mint-dir -export MINT_MODE=core -export SERVER_ENDPOINT="play.minio.io:9000" -export ACCESS_KEY="Q3AM3UQ867SPQQA43P2F" -export SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" -export ENABLE_HTTPS=1 -export SERVER_REGION=us-east-1 -./run.sh /tmp/output.log /tmp/error.log -``` diff --git a/mint/run/core/minio-js/minioreporter.js b/mint/run/core/minio-js/minioreporter.js deleted file mode 100644 index 3e77934..0000000 --- a/mint/run/core/minio-js/minioreporter.js +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Minio Reporter for JSON formatted logging, (C) 2017 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -var mocha = require('mocha'); -module.exports = minioreporter; - -function minioreporter(runner) { - mocha.reporters.Base.call(this, runner); - var self = this; - - runner.on('pass', function (test) { - GenerateJsonEntry(test) - }); - - runner.on('fail', function (test, err) { - GenerateJsonEntry(test, err) - }); - -} - -/** - * Convert test result into a JSON object and print on the console. - * - * @api private - * @param test, err - */ - -function GenerateJsonEntry (test, err) { - var res = test.title.split("_") - var jsonEntry = {}; - - jsonEntry.name = "minio-js" - - if (res.length > 0 && res[0].length) { - jsonEntry.function = res[0] - } - - if (res.length > 1 && res[1].length) { - jsonEntry.args = res[1] - } - - jsonEntry.duration = test.duration - - if (res.length > 2 && res[2].length) { - jsonEntry.alert = res[2] - } - - if (err != null ) { - jsonEntry.status = "FAIL" - jsonEntry.error = err.stack.replace(/\n/g, " ").replace(/ +(?= )/g,'') - } else { - jsonEntry.status = "PASS" - } - - process.stdout.write(JSON.stringify(jsonEntry) + "\n") -} diff --git a/mint/run/core/minio-js/package.json b/mint/run/core/minio-js/package.json deleted file mode 100644 index 84d581c..0000000 --- a/mint/run/core/minio-js/package.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "name": "bin", - "version": "1.0.0", - "main": "functional_test.js", - "scripts": { - "test": "echo \"Error: no test specified\" && exit 1" - }, - "keywords": [], - "author": "", - "license": "ISC", - "description": "", - "dependencies": { - "app-module-path": "*", - "async": "*", - "block-stream2": "*", - "concat-stream": "*", - "es6-error": "*", - "json-stream": "*", - "lodash": "*", - "mime-types": "*", - "mkdirp": "*", - "moment": "*", - "source-map-support": "*", - "through2": "*", - "xml": "*", - "xml2js": "*" - }, - "devDependencies": { - "browserify": "*", - "chai": "*", - "gulp": "*", - "gulp-babel": "*", - "gulp-jscs": "*", - "jshint":"2.*", - "gulp-jshint": "*", - "gulp-mocha": "*", - "gulp-notify": "*", - "gulp-sourcemaps": "*", - "jshint-stylish": "*", - "mocha": "*", - "mocha-steps": "*", - "nock": "*", - "rewire": "*", - "superagent": "*" - }, - "scripts": { - "test": "mocha" - } -} diff --git a/mint/run/core/minio-js/run.sh b/mint/run/core/minio-js/run.sh deleted file mode 100755 index c6a0c58..0000000 --- a/mint/run/core/minio-js/run.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -# -# Minio Cloud Storage, (C) 2017 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# handle command line arguments -if [ $# -ne 2 ]; then - echo "usage: run.sh " - exit 1 -fi - -output_log_file="$1" -error_log_file="$2" - -# run tests -./node_modules/mocha/bin/mocha -R minioreporter -b --exit 1>>"$output_log_file" 2>"$error_log_file" diff --git a/mint/run/core/minio-py/README.md b/mint/run/core/minio-py/README.md deleted file mode 100644 index 36c65cc..0000000 --- a/mint/run/core/minio-py/README.md +++ /dev/null @@ -1,19 +0,0 @@ -## `minio-py` tests -This directory serves as the location for Mint tests using `minio-py`. Top level `mint.sh` calls `run.sh` to execute tests. - -## Adding new tests -New tests is added in functional tests of minio-py. Please check https://github.com/minio/minio-py - -## Running tests manually -- Set environment variables `MINT_DATA_DIR`, `MINT_MODE`, `SERVER_ENDPOINT`, `ACCESS_KEY`, `SECRET_KEY`, `SERVER_REGION` and `ENABLE_HTTPS` -- Call `run.sh` with output log file and error log file. for example -```bash -export MINT_DATA_DIR=~/my-mint-dir -export MINT_MODE=core -export SERVER_ENDPOINT="play.minio.io:9000" -export ACCESS_KEY="Q3AM3UQ867SPQQA43P2F" -export SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" -export ENABLE_HTTPS=1 -export SERVER_REGION=us-east-1 -./run.sh /tmp/output.log /tmp/error.log -``` diff --git a/mint/run/core/minio-py/run.sh b/mint/run/core/minio-py/run.sh deleted file mode 100755 index 2538783..0000000 --- a/mint/run/core/minio-py/run.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -# -# Mint (C) 2017 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# handle command line arguments -if [ $# -ne 2 ]; then - echo "usage: run.sh " - exit 1 -fi - -output_log_file="$1" -error_log_file="$2" - -# run tests -python "/mint/run/core/minio-py/tests.py" 1>>"$output_log_file" 2>"$error_log_file" diff --git a/mint/run/core/s3cmd/.gitignore b/mint/run/core/s3cmd/.gitignore deleted file mode 100644 index 518a1c6..0000000 --- a/mint/run/core/s3cmd/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -*~ -*.log diff --git a/mint/run/core/s3cmd/README.md b/mint/run/core/s3cmd/README.md deleted file mode 100644 index a961c02..0000000 --- a/mint/run/core/s3cmd/README.md +++ /dev/null @@ -1,19 +0,0 @@ -## `s3cmd` tests -This directory serves as the location for Mint tests using `s3cmd`. Top level `mint.sh` calls `run.sh` to execute tests. - -## Adding new tests -New tests is added into `test.sh` as new functions. - -## Running tests manually -- Set environment variables `MINT_DATA_DIR`, `MINT_MODE`, `SERVER_ENDPOINT`, `ACCESS_KEY`, `SECRET_KEY`, `SERVER_REGION` and `ENABLE_HTTPS` -- Call `run.sh` with output log file and error log file. for example -```bash -export MINT_DATA_DIR=~/my-mint-dir -export MINT_MODE=core -export SERVER_ENDPOINT="play.minio.io:9000" -export ACCESS_KEY="Q3AM3UQ867SPQQA43P2F" -export SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" -export ENABLE_HTTPS=1 -export SERVER_REGION=us-east-1 -./run.sh /tmp/output.log /tmp/error.log -``` diff --git a/mint/run/core/s3cmd/run.sh b/mint/run/core/s3cmd/run.sh deleted file mode 100755 index e6cdc3f..0000000 --- a/mint/run/core/s3cmd/run.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -# -# Mint (C) 2017 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# handle command line arguments -if [ $# -ne 2 ]; then - echo "usage: run.sh " - exit 1 -fi - -output_log_file="$1" -error_log_file="$2" - -# run tests -./test.sh 1>>"$output_log_file" 2>"$error_log_file" diff --git a/mint/run/core/s3cmd/test.sh b/mint/run/core/s3cmd/test.sh deleted file mode 100755 index a33d505..0000000 --- a/mint/run/core/s3cmd/test.sh +++ /dev/null @@ -1,406 +0,0 @@ -#!/bin/bash -# -# Mint (C) 2017-2020 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - - -if [ -n "$MINT_MODE" ]; then - if [ -z "${MINT_DATA_DIR+x}" ]; then - echo "MINT_DATA_DIR not defined" - exit 1 - fi - if [ -z "${SERVER_ENDPOINT+x}" ]; then - echo "SERVER_ENDPOINT not defined" - exit 1 - fi - if [ -z "${ACCESS_KEY+x}" ]; then - echo "ACCESS_KEY not defined" - exit 1 - fi - if [ -z "${SECRET_KEY+x}" ]; then - echo "SECRET_KEY not defined" - exit 1 - fi -fi - -if [ -z "${SERVER_ENDPOINT+x}" ]; then - SERVER_ENDPOINT="play.minio.io:9000" - ACCESS_KEY="Q3AM3UQ867SPQQA43P2F" - SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" - ENABLE_HTTPS=1 - SERVER_REGION="us-east-1" -fi - -WORK_DIR="$PWD" -DATA_DIR="$MINT_DATA_DIR" -if [ -z "$MINT_MODE" ]; then - WORK_DIR="$PWD/.run-$RANDOM" - DATA_DIR="$WORK_DIR/data" -fi - -FILE_1_MB="$DATA_DIR/datafile-1-MB" -FILE_65_MB="$DATA_DIR/datafile-65-MB" -declare FILE_1_MB_MD5SUM -declare FILE_65_MB_MD5SUM - -BUCKET_NAME="s3cmd-test-bucket-$RANDOM" -S3CMD=$(which s3cmd) -declare -a S3CMD_CMD - -function get_md5sum() -{ - filename="$FILE_1_MB" - out=$(md5sum "$filename" 2>/dev/null) - rv=$? - if [ "$rv" -eq 0 ]; then - awk '{ print $1 }' <<< "$out" - fi - - return "$rv" -} - -function get_time() -{ - date +%s%N -} - -function get_duration() -{ - start_time=$1 - end_time=$(get_time) - - echo $(( (end_time - start_time) / 1000000 )) -} - -function log_success() -{ - if [ -n "$MINT_MODE" ]; then - printf '{"name": "s3cmd", "duration": "%d", "function": "%s", "status": "PASS"}\n' "$(get_duration "$1")" "$2" - fi -} - -function show() -{ - if [ -z "$MINT_MODE" ]; then - func_name="$1" - echo "Running $func_name()" - fi -} - -function fail() -{ - rv="$1" - shift - - if [ "$rv" -ne 0 ]; then - echo "$@" - fi - - return "$rv" -} - -function assert() -{ - expected_rv="$1" - shift - start_time="$1" - shift - func_name="$1" - shift - - err=$("$@" 2>&1) - rv=$? - if [ "$rv" -ne 0 ] && [ "$expected_rv" -eq 0 ]; then - if [ -n "$MINT_MODE" ]; then - err=$(printf '%s' "$err" | python -c 'import sys,json; print(json.dumps(sys.stdin.read()))') - ## err is already JSON string, no need to double quote - printf '{"name": "s3cmd", "duration": "%d", "function": "%s", "status": "FAIL", "error": %s}\n' "$(get_duration "$start_time")" "$func_name" "$err" - else - echo "s3cmd: $func_name: $err" - fi - - exit "$rv" - fi - - return 0 -} - -function assert_success() { - assert 0 "$@" -} - -function assert_failure() { - assert 1 "$@" -} - -function s3cmd_cmd() -{ - cmd=( "${S3CMD_CMD[@]}" "$@" ) - "${cmd[@]}" - rv=$? - return "$rv" -} - -function check_md5sum() -{ - expected_checksum="$1" - shift - filename="$*" - - checksum="$(get_md5sum "$filename")" - rv=$? - if [ "$rv" -ne 0 ]; then - echo "unable to get md5sum for $filename" - return "$rv" - fi - - if [ "$checksum" != "$expected_checksum" ]; then - echo "$filename: md5sum mismatch" - return 1 - fi - - return 0 -} - -function test_make_bucket() -{ - show "${FUNCNAME[0]}" - - start_time=$(get_time) - bucket_name="s3cmd-test-bucket-$RANDOM" - assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd mb "s3://${bucket_name}" - assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rb "s3://${bucket_name}" - - log_success "$start_time" "${FUNCNAME[0]}" -} - -function test_make_bucket_error() { - show "${FUNCNAME[0]}" - - start_time=$(get_time) - bucket_name="S3CMD-test%bucket%$RANDOM" - assert_failure "$start_time" "${FUNCNAME[0]}" s3cmd_cmd mb "s3://${bucket_name}" - - log_success "$start_time" "${FUNCNAME[0]}" -} - -function setup() -{ - start_time=$(get_time) - assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd mb "s3://${BUCKET_NAME}" -} - -function teardown() -{ - start_time=$(get_time) - assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rm --force --recursive "s3://${BUCKET_NAME}" - assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rb --force "s3://${BUCKET_NAME}" -} - -function test_put_object() -{ - show "${FUNCNAME[0]}" - - start_time=$(get_time) - object_name="s3cmd-test-object-$RANDOM" - assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd put "${FILE_1_MB}" "s3://${BUCKET_NAME}/${object_name}" - assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rm "s3://${BUCKET_NAME}/${object_name}" - - log_success "$start_time" "${FUNCNAME[0]}" -} - -function test_put_object_error() -{ - show "${FUNCNAME[0]}" - start_time=$(get_time) - - object_long_name=$(printf "s3cmd-test-object-%01100d" 1) - assert_failure "$start_time" "${FUNCNAME[0]}" s3cmd_cmd put "${FILE_1_MB}" "s3://${BUCKET_NAME}/${object_long_name}" - - log_success "$start_time" "${FUNCNAME[0]}" -} - -function test_put_object_multipart() -{ - show "${FUNCNAME[0]}" - - start_time=$(get_time) - object_name="s3cmd-test-object-$RANDOM" - assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd put "${FILE_65_MB}" "s3://${BUCKET_NAME}/${object_name}" - assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rm "s3://${BUCKET_NAME}/${object_name}" - - log_success "$start_time" "${FUNCNAME[0]}" -} - -function test_get_object() -{ - show "${FUNCNAME[0]}" - - start_time=$(get_time) - object_name="s3cmd-test-object-$RANDOM" - assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd put "${FILE_1_MB}" "s3://${BUCKET_NAME}/${object_name}" - assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd get "s3://${BUCKET_NAME}/${object_name}" "${object_name}.downloaded" - assert_success "$start_time" "${FUNCNAME[0]}" check_md5sum "$FILE_1_MB_MD5SUM" "${object_name}.downloaded" - assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rm "s3://${BUCKET_NAME}/${object_name}" - assert_success "$start_time" "${FUNCNAME[0]}" rm -f "${object_name}.downloaded" - - log_success "$start_time" "${FUNCNAME[0]}" -} - -function test_get_object_error() -{ - show "${FUNCNAME[0]}" - - start_time=$(get_time) - object_name="s3cmd-test-object-$RANDOM" - assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd put "${FILE_1_MB}" "s3://${BUCKET_NAME}/${object_name}" - assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rm "s3://${BUCKET_NAME}/${object_name}" - assert_failure "$start_time" "${FUNCNAME[0]}" s3cmd_cmd get "s3://${BUCKET_NAME}/${object_name}" "${object_name}.downloaded" - assert_success "$start_time" "${FUNCNAME[0]}" rm -f "${object_name}.downloaded" - - log_success "$start_time" "${FUNCNAME[0]}" -} - -function test_get_object_multipart() -{ - show "${FUNCNAME[0]}" - - start_time=$(get_time) - object_name="s3cmd-test-object-$RANDOM" - assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd put "${FILE_65_MB}" "s3://${BUCKET_NAME}/${object_name}" - assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd get "s3://${BUCKET_NAME}/${object_name}" "${object_name}.downloaded" - assert_success "$start_time" "${FUNCNAME[0]}" check_md5sum "$FILE_65_MB_MD5SUM" "${object_name}.downloaded" - assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rm "s3://${BUCKET_NAME}/${object_name}" - assert_success "$start_time" "${FUNCNAME[0]}" rm -f "${object_name}.downloaded" - - log_success "$start_time" "${FUNCNAME[0]}" -} - -function test_sync_list_objects() -{ - show "${FUNCNAME[0]}" - - start_time=$(get_time) - bucket_name="s3cmd-test-bucket-$RANDOM" - object_name="s3cmd-test-object-$RANDOM" - assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd mb "s3://${bucket_name}" - assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd sync "$DATA_DIR/" "s3://${bucket_name}" - - diff -bB <(ls "$DATA_DIR") <("${S3CMD_CMD[@]}" ls "s3://${bucket_name}" | awk '{print $4}' | sed "s/s3:*..${bucket_name}.//g") >/dev/null 2>&1 - assert_success "$start_time" "${FUNCNAME[0]}" fail $? "sync and list differs" - assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rb --force --recursive "s3://${bucket_name}" - - log_success "$start_time" "${FUNCNAME[0]}" -} - -function run_test() -{ - test_make_bucket - test_make_bucket_error - - setup - - test_put_object - test_put_object_error - test_put_object_multipart - test_get_object - test_get_object_multipart - test_sync_list_objects - - teardown -} - -function __init__() -{ - set -e - - S3CMD_CONFIG_DIR="/tmp/.s3cmd-$RANDOM" - mkdir -p $S3CMD_CONFIG_DIR - S3CMD_CONFIG_FILE="$S3CMD_CONFIG_DIR/s3cfg" - - # configure s3cmd - cat > $S3CMD_CONFIG_FILE <"$FILE_1_MB" - fi - - if [ ! -e "$FILE_65_MB" ]; then - shred -n 1 -s 65MB - >"$FILE_65_MB" - fi - - set -E - set -o pipefail - - FILE_1_MB_MD5SUM="$(get_md5sum "$FILE_1_MB")" - rv=$? - if [ $rv -ne 0 ]; then - echo "unable to get md5sum of $FILE_1_MB" - exit 1 - fi - - FILE_65_MB_MD5SUM="$(get_md5sum "$FILE_65_MB")" - rv=$? - if [ $rv -ne 0 ]; then - echo "unable to get md5sum of $FILE_65_MB" - exit 1 - fi - - set +e -} - -function main() -{ - ( run_test ) - rv=$? - - rm -fr "$S3CMD_CONFIG_FILE" - if [ -z "$MINT_MODE" ]; then - rm -fr "$WORK_DIR" "$DATA_DIR" - fi - - exit "$rv" -} - -__init__ "$@" -main "$@" diff --git a/mint/run/core/s3select/README.md b/mint/run/core/s3select/README.md deleted file mode 100644 index 73a69a2..0000000 --- a/mint/run/core/s3select/README.md +++ /dev/null @@ -1,21 +0,0 @@ - -## `s3select` tests -This directory serves as the location for Mint tests for s3select features. Top level `mint.sh` calls `run.sh` to execute tests. - -## Adding new tests -New tests are added into `s3select/tests.py` as new functions. - -## Running tests manually -- Set environment variables `MINT_DATA_DIR`, `MINT_MODE`, `SERVER_ENDPOINT`, `ACCESS_KEY`, `SECRET_KEY`, `SERVER_REGION` and `ENABLE_HTTPS` -- Call `run.sh` with output log file and error log file. for example - -```bash -export MINT_DATA_DIR=~/my-mint-dir -export MINT_MODE=core -export SERVER_ENDPOINT="play.min.io" -export ACCESS_KEY="Q3AM3UQ867SPQQA43P2F" -export SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" -export ENABLE_HTTPS=1 -export SERVER_REGION=us-east-1 -./run.sh /tmp/output.log /tmp/error.log -``` diff --git a/mint/run/core/s3select/csv.py b/mint/run/core/s3select/csv.py deleted file mode 100644 index faee965..0000000 --- a/mint/run/core/s3select/csv.py +++ /dev/null @@ -1,172 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# MinIO Python Library for Amazon S3 Compatible Cloud Storage, -# (C) 2015-2020 MinIO, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import io - -from minio import Minio -from minio.select.options import (SelectObjectOptions, CSVInput, - RequestProgress, InputSerialization, - OutputSerialization, CSVOutput, JsonOutput) - -from utils import * - -def test_sql_api(test_name, client, bucket_name, input_data, sql_opts, expected_output): - """ Test if the passed SQL request has the output equal to the passed execpted one""" - object_name = generate_object_name() - got_output = b'' - try: - bytes_content = io.BytesIO(input_data) - client.put_object(bucket_name, object_name, io.BytesIO(input_data), len(input_data)) - data = client.select_object_content(bucket_name, object_name, sql_opts) - # Get the records - records = io.BytesIO() - for d in data.stream(10*1024): - records.write(d.encode('utf-8')) - got_output = records.getvalue() - except Exception as select_err: - if not isinstance(expected_output, Exception): - raise ValueError('Test {} unexpectedly failed with: {}'.format(test_name, select_err)) - else: - if isinstance(expected_output, Exception): - raise ValueError('Test {}: expected an exception, got {}'.format(test_name, got_output)) - if got_output != expected_output: - raise ValueError('Test {}: data mismatch. Expected : {}, Received {}'.format(test_name, expected_output, got_output)) - finally: - client.remove_object(bucket_name, object_name) - - -def test_csv_input_custom_quote_char(client, log_output): - # Get a unique bucket_name and object_name - log_output.args['bucket_name'] = bucket_name = generate_bucket_name() - - tests = [ - # Invalid quote character, should fail - ('""', '"', b'col1,col2,col3\n', Exception()), - # UTF-8 quote character - ('ع', '"', 'عcol1ع,عcol2ع,عcol3ع\n'.encode(), b'{"_1":"col1","_2":"col2","_3":"col3"}\n'), - # Only one field is quoted - ('"', '"', b'"col1",col2,col3\n', b'{"_1":"col1","_2":"col2","_3":"col3"}\n'), - ('"', '"', b'"col1,col2,col3"\n', b'{"_1":"col1,col2,col3"}\n'), - ('\'', '"', b'"col1",col2,col3\n', b'{"_1":"\\"col1\\"","_2":"col2","_3":"col3"}\n'), - ('', '"', b'"col1",col2,col3\n', b'{"_1":"\\"col1\\"","_2":"col2","_3":"col3"}\n'), - ('', '"', b'"col1",col2,col3\n', b'{"_1":"\\"col1\\"","_2":"col2","_3":"col3"}\n'), - ('', '"', b'"col1","col2","col3"\n', b'{"_1":"\\"col1\\"","_2":"\\"col2\\"","_3":"\\"col3\\""}\n'), - ('"', '"', b'""""""\n', b'{"_1":"\\"\\""}\n'), - ('"', '"', b'A",B\n', b'{"_1":"A\\"","_2":"B"}\n'), - ('"', '"', b'A"",B\n', b'{"_1":"A\\"\\"","_2":"B"}\n'), - ('"', '\\', b'A\\B,C\n', b'{"_1":"A\\\\B","_2":"C"}\n'), - ('"', '"', b'"A""B","CD"\n', b'{"_1":"A\\"B","_2":"CD"}\n'), - ('"', '\\', b'"A\\B","CD"\n', b'{"_1":"AB","_2":"CD"}\n'), - ('"', '\\', b'"A\\,","CD"\n', b'{"_1":"A,","_2":"CD"}\n'), - ('"', '\\', b'"A\\"B","CD"\n', b'{"_1":"A\\"B","_2":"CD"}\n'), - ('"', '\\', b'"A\\""\n', b'{"_1":"A\\""}\n'), - ('"', '\\', b'"A\\"\\"B"\n', b'{"_1":"A\\"\\"B"}\n'), - ('"', '\\', b'"A\\"","\\"B"\n', b'{"_1":"A\\"","_2":"\\"B"}\n'), - ] - - client.make_bucket(bucket_name) - - try: - for idx, (quote_char, escape_char, data, expected_output) in enumerate(tests): - sql_opts = SelectObjectOptions( - expression="select * from s3object", - input_serialization=InputSerialization( - compression_type="NONE", - csv=CSVInput(FileHeaderInfo="NONE", - RecordDelimiter="\n", - FieldDelimiter=",", - QuoteCharacter=quote_char, - QuoteEscapeCharacter=escape_char, - Comments="#", - AllowQuotedRecordDelimiter="FALSE",), - ), - output_serialization=OutputSerialization( - json = JsonOutput( - RecordDelimiter="\n", - ) - ), - request_progress=RequestProgress( - enabled="False" - ) - ) - - test_sql_api(f'test_{idx}', client, bucket_name, data, sql_opts, expected_output) - finally: - client.remove_bucket(bucket_name) - - # Test passes - print(log_output.json_report()) - -def test_csv_output_custom_quote_char(client, log_output): - # Get a unique bucket_name and object_name - log_output.args['bucket_name'] = bucket_name = generate_bucket_name() - - tests = [ - # UTF-8 quote character - ("''", "''", b'col1,col2,col3\n', Exception()), - ("'", "'", b'col1,col2,col3\n', b"'col1','col2','col3'\n"), - ("", '"', b'col1,col2,col3\n', b'\x00col1\x00,\x00col2\x00,\x00col3\x00\n'), - ('"', '"', b'col1,col2,col3\n', b'"col1","col2","col3"\n'), - ('"', '"', b'col"1,col2,col3\n', b'"col""1","col2","col3"\n'), - ('"', '"', b'""""\n', b'""""\n'), - ('"', '"', b'\n', b''), - ("'", "\\", b'col1,col2,col3\n', b"'col1','col2','col3'\n"), - ("'", "\\", b'col""1,col2,col3\n', b"'col\"\"1','col2','col3'\n"), - ("'", "\\", b'col\'1,col2,col3\n', b"'col\\'1','col2','col3'\n"), - ("'", "\\", b'"col\'1","col2","col3"\n', b"'col\\'1','col2','col3'\n"), - ("'", "\\", b'col\'\n', b"'col\\''\n"), - # Two consecutive escaped quotes - ("'", "\\", b'"a"""""\n', b"'a\"\"'\n"), - ] - - client.make_bucket(bucket_name) - - try: - for idx, (quote_char, escape_char, input_data, expected_output) in enumerate(tests): - sql_opts = SelectObjectOptions( - expression="select * from s3object", - input_serialization=InputSerialization( - compression_type="NONE", - csv=CSVInput(FileHeaderInfo="NONE", - RecordDelimiter="\n", - FieldDelimiter=",", - QuoteCharacter='"', - QuoteEscapeCharacter='"', - Comments="#", - AllowQuotedRecordDelimiter="FALSE",), - ), - output_serialization=OutputSerialization( - csv=CSVOutput(QuoteFields="ALWAYS", - RecordDelimiter="\n", - FieldDelimiter=",", - QuoteCharacter=quote_char, - QuoteEscapeCharacter=escape_char,) - ), - request_progress=RequestProgress( - enabled="False" - ) - ) - - test_sql_api(f'test_{idx}', client, bucket_name, input_data, sql_opts, expected_output) - finally: - client.remove_bucket(bucket_name) - - # Test passes - print(log_output.json_report()) - - diff --git a/mint/run/core/s3select/run.sh b/mint/run/core/s3select/run.sh deleted file mode 100755 index 0574ea8..0000000 --- a/mint/run/core/s3select/run.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -# -# Mint (C) 2020 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# handle command line arguments -if [ $# -ne 2 ]; then - echo "usage: run.sh " - exit 1 -fi - -output_log_file="$1" -error_log_file="$2" - -# run path style tests -python "./tests.py" 1>>"$output_log_file" 2>"$error_log_file" diff --git a/mint/run/core/s3select/sql_ops.py b/mint/run/core/s3select/sql_ops.py deleted file mode 100644 index 83931ad..0000000 --- a/mint/run/core/s3select/sql_ops.py +++ /dev/null @@ -1,399 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# MinIO Python Library for Amazon S3 Compatible Cloud Storage, -# (C) 2020 MinIO, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import io -from datetime import datetime - -from minio import Minio -from minio.select.options import (SelectObjectOptions, CSVInput, JSONInput, - RequestProgress, InputSerialization, - OutputSerialization, CSVOutput, JsonOutput) - -from utils import * - -def test_sql_expressions_custom_input_output(client, input_bytes, sql_input, sql_output, tests, log_output): - bucket_name = generate_bucket_name() - object_name = generate_object_name() - - log_output.args['total_tests'] = 0 - log_output.args['total_success'] = 0 - - client.make_bucket(bucket_name) - try: - content = io.BytesIO(bytes(input_bytes, 'utf-8')) - client.put_object(bucket_name, object_name, content, len(input_bytes)) - - for idx, (test_name, select_expression, expected_output) in enumerate(tests): - if select_expression == '': - continue - try: - log_output.args['total_tests'] += 1 - options = SelectObjectOptions( - expression=select_expression, - input_serialization=sql_input, - output_serialization=sql_output, - request_progress=RequestProgress( - enabled="False" - ) - ) - - data = client.select_object_content(bucket_name, object_name, options) - - # Get the records - records = io.BytesIO() - for d in data.stream(10*1024): - records.write(d.encode('utf-8')) - got_output = records.getvalue() - - if got_output != expected_output: - if type(expected_output) == datetime: - # Attempt to parse the date which will throw an exception for any issue - datetime.strptime(got_output.decode("utf-8").strip(), '%Y-%m-%dT%H:%M:%S.%f%z') - else: - raise ValueError('Test {}: data mismatch. Expected : {}. Received: {}.'.format(idx+1, expected_output, got_output)) - - log_output.args['total_success'] += 1 - except Exception as err: - continue ## TODO, raise instead - # raise Exception(err) - finally: - client.remove_object(bucket_name, object_name) - client.remove_bucket(bucket_name) - - -def test_sql_expressions(client, input_json_bytes, tests, log_output): - input_serialization = InputSerialization( - compression_type="NONE", - json=JSONInput(Type="DOCUMENT"), - ) - - output_serialization=OutputSerialization( - csv=CSVOutput(QuoteFields="ASNEEDED") - ) - - test_sql_expressions_custom_input_output(client, input_json_bytes, - input_serialization, output_serialization, tests, log_output) - - -def test_sql_operators(client, log_output): - - json_testfile = """{"id": 1, "name": "John", "age": 3} -{"id": 2, "name": "Elliot", "age": 4} -{"id": 3, "name": "Yves", "age": 5} -{"id": 4, "name": null, "age": 0} -""" - - tests = [ - # Logical operators - ("AND", "select * from S3Object s where s.id = 1 AND s.name = 'John'", b'1,John,3\n'), - ("NOT", "select * from S3Object s where NOT s.id = 1", b'2,Elliot,4\n3,Yves,5\n4,,0\n'), - ("OR", "select * from S3Object s where s.id = 1 OR s.id = 3", b'1,John,3\n3,Yves,5\n'), - # Comparison Operators - ("<", "select * from S3Object s where s.age < 4", b'1,John,3\n4,,0\n'), - (">", "select * from S3Object s where s.age > 4", b'3,Yves,5\n'), - ("<=", "select * from S3Object s where s.age <= 4", b'1,John,3\n2,Elliot,4\n4,,0\n'), - (">=", "select * from S3Object s where s.age >= 4", b'2,Elliot,4\n3,Yves,5\n'), - ("=", "select * from S3Object s where s.age = 4", b'2,Elliot,4\n'), - ("<>", "select * from S3Object s where s.age <> 4", b'1,John,3\n3,Yves,5\n4,,0\n'), - ("!=", "select * from S3Object s where s.age != 4", b'1,John,3\n3,Yves,5\n4,,0\n'), - ("BETWEEN", "select * from S3Object s where s.age BETWEEN 4 AND 5", b'2,Elliot,4\n3,Yves,5\n'), - ("IN", "select * from S3Object s where s.age IN (3,5)", b'1,John,3\n3,Yves,5\n'), - # Pattern Matching Operators - ("LIKE_", "select * from S3Object s where s.name LIKE '_ves'", b'3,Yves,5\n'), - ("LIKE%", "select * from S3Object s where s.name LIKE 'Ell%t'", b'2,Elliot,4\n'), - # Unitary Operators - ("NULL", "select * from S3Object s where s.name IS NULL", b'4,,0\n'), - ("NOT_NULL", "select * from S3Object s where s.age IS NOT NULL", b'1,John,3\n2,Elliot,4\n3,Yves,5\n4,,0\n'), - # Math Operators - ("+", "select * from S3Object s where s.age = 1+3 ", b'2,Elliot,4\n'), - ("-", "select * from S3Object s where s.age = 5-1 ", b'2,Elliot,4\n'), - ("*", "select * from S3Object s where s.age = 2*2 ", b'2,Elliot,4\n'), - ("%", "select * from S3Object s where s.age = 10%6 ", b'2,Elliot,4\n'), - ] - - try: - test_sql_expressions(client, json_testfile, tests, log_output) - except Exception as select_err: - raise select_err - # raise ValueError('Test {} unexpectedly failed with: {}'.format(test_name, select_err)) - # pass - - # Test passes - print(log_output.json_report()) - - -def test_sql_operators_precedence(client, log_output): - - json_testfile = """{"id": 1, "name": "Eric"}""" - - tests = [ - ("-_1", "select -3*3 from S3Object", b'-9\n'), - ("*", "select 10-3*2 from S3Object", b'4\n'), - ("/", "select 13-10/5 from S3Object", b'11\n'), - ("%", "select 13-10%5 from S3Object", b'13\n'), - ("+", "select 1+1*3 from S3Object", b'4\n'), - ("-_2", "select 1-1*3 from S3Object", b'-2\n'), - ("=", "select * from S3Object as s where s.id = 13-12", b'1,Eric\n'), - ("<>", "select * from S3Object as s where s.id <> 1-1", b'1,Eric\n'), - ("NOT", "select * from S3Object where false OR NOT false", b'1,Eric\n'), - ("AND", "select * from S3Object where true AND true OR false ", b'1,Eric\n'), - ("OR", "select * from S3Object where false OR NOT false", b'1,Eric\n'), - ("IN", "select * from S3Object as s where s.id <> -1 AND s.id IN (1,2,3)", b'1,Eric\n'), - ("BETWEEN", "select * from S3Object as s where s.id <> -1 AND s.id BETWEEN -1 AND 3", b'1,Eric\n'), - ("LIKE", "select * from S3Object as s where s.id <> -1 AND s.name LIKE 'E%'", b'1,Eric\n'), - ] - - try: - test_sql_expressions(client, json_testfile, tests, log_output) - except Exception as select_err: - raise select_err - # raise ValueError('Test {} unexpectedly failed with: {}'.format(test_name, select_err)) - # pass - - # Test passes - print(log_output.json_report()) - - - -def test_sql_functions_agg_cond_conv(client, log_output): - - json_testfile = """{"id": 1, "name": "John", "age": 3} -{"id": 2, "name": "Elliot", "age": 4} -{"id": 3, "name": "Yves", "age": 5} -{"id": 4, "name": "Christine", "age": null} -{"id": 5, "name": "Eric", "age": 0} -""" - tests = [ - # Aggregate functions - ("COUNT", "select count(*) from S3Object s", b'5\n'), - ("AVG", "select avg(s.age) from S3Object s", b'3\n'), - ("MAX", "select max(s.age) from S3Object s", b'5\n'), - ("MIN", "select min(s.age) from S3Object s", b'0\n'), - ("SUM", "select sum(s.age) from S3Object s", b'12\n'), - # Conditional functions - ("COALESCE", "SELECT COALESCE(s.age, 99) FROM S3Object s", b'3\n4\n5\n99\n0\n'), - ("NULLIF", "SELECT NULLIF(s.age, 0) FROM S3Object s", b'3\n4\n5\n\n\n'), - ## Conversion functions - ("CAST", "SELECT CAST(s.age AS FLOAT) FROM S3Object s", b'3.0\n4.0\n5.0\n\n0.0\n'), - - ] - - try: - test_sql_expressions(client, json_testfile, tests, log_output) - except Exception as select_err: - raise select_err - # raise ValueError('Test {} unexpectedly failed with: {}'.format(test_name, select_err)) - # pass - - # Test passes - print(log_output.json_report()) - - -def test_sql_functions_date(client, log_output): - - json_testfile = """ -{"id": 1, "name": "John", "datez": "2017-01-02T03:04:05.006+07:30"} -""" - - tests = [ - # DATE_ADD - ("DATE_ADD_1", "select DATE_ADD(year, 5, TO_TIMESTAMP(s.datez)) from S3Object as s", b'2022-01-02T03:04:05.006+07:30\n'), - ("DATE_ADD_2", "select DATE_ADD(month, 1, TO_TIMESTAMP(s.datez)) from S3Object as s", b'2017-02-02T03:04:05.006+07:30\n'), - ("DATE_ADD_3", "select DATE_ADD(day, -1, TO_TIMESTAMP(s.datez)) from S3Object as s", b'2017-01-01T03:04:05.006+07:30\n'), - ("DATE_ADD_4", "select DATE_ADD(hour, 1, TO_TIMESTAMP(s.datez)) from S3Object as s", b'2017-01-02T04:04:05.006+07:30\n'), - ("DATE_ADD_5", "select DATE_ADD(minute, 5, TO_TIMESTAMP(s.datez)) from S3Object as s", b'2017-01-02T03:09:05.006+07:30\n'), - ("DATE_ADD_6", "select DATE_ADD(second, 5, TO_TIMESTAMP(s.datez)) from S3Object as s", b'2017-01-02T03:04:10.006+07:30\n'), - # DATE_DIFF - ("DATE_DIFF_1", "select DATE_DIFF(year, TO_TIMESTAMP(s.datez), TO_TIMESTAMP('2011-01-01T')) from S3Object as s", b'-6\n'), - ("DATE_DIFF_2", "select DATE_DIFF(month, TO_TIMESTAMP(s.datez), TO_TIMESTAMP('2011T')) from S3Object as s", b'-72\n'), - ("DATE_DIFF_3", "select DATE_DIFF(day, TO_TIMESTAMP(s.datez), TO_TIMESTAMP('2010-01-02T')) from S3Object as s", b'-2556\n'), - # EXTRACT - ("EXTRACT_1", "select EXTRACT(year FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'2017\n'), - ("EXTRACT_2", "select EXTRACT(month FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'1\n'), - ("EXTRACT_3", "select EXTRACT(hour FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'3\n'), - ("EXTRACT_4", "select EXTRACT(minute FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'4\n'), - ("EXTRACT_5", "select EXTRACT(timezone_hour FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'7\n'), - ("EXTRACT_6", "select EXTRACT(timezone_minute FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'30\n'), - # TO_STRING - ("TO_STRING_1", "select TO_STRING(TO_TIMESTAMP(s.datez), 'MMMM d, y') from S3Object as s", b'"January 2, 2017"\n'), - ("TO_STRING_2", "select TO_STRING(TO_TIMESTAMP(s.datez), 'MMM d, yyyy') from S3Object as s", b'"Jan 2, 2017"\n'), - ("TO_STRING_3", "select TO_STRING(TO_TIMESTAMP(s.datez), 'M-d-yy') from S3Object as s", b'1-2-17\n'), - ("TO_STRING_4", "select TO_STRING(TO_TIMESTAMP(s.datez), 'MM-d-y') from S3Object as s", b'01-2-2017\n'), - ("TO_STRING_5", "select TO_STRING(TO_TIMESTAMP(s.datez), 'MMMM d, y h:m a') from S3Object as s", b'"January 2, 2017 3:4 AM"\n'), - ("TO_STRING_6", "select TO_STRING(TO_TIMESTAMP(s.datez), 'y-MM-dd''T''H:m:ssX') from S3Object as s", b'2017-01-02T3:4:05+0730\n'), - ("TO_STRING_7", "select TO_STRING(TO_TIMESTAMP(s.datez), 'y-MM-dd''T''H:m:ssX') from S3Object as s", b'2017-01-02T3:4:05+0730\n'), - ("TO_STRING_8", "select TO_STRING(TO_TIMESTAMP(s.datez), 'y-MM-dd''T''H:m:ssXXXX') from S3Object as s", b'2017-01-02T3:4:05+0730\n'), - ("TO_STRING_9", "select TO_STRING(TO_TIMESTAMP(s.datez), 'y-MM-dd''T''H:m:ssXXXXX') from S3Object as s", b'2017-01-02T3:4:05+07:30\n'), - ("TO_TIMESTAMP", "select TO_TIMESTAMP(s.datez) from S3Object as s", b'2017-01-02T03:04:05.006+07:30\n'), - ("UTCNOW", "select UTCNOW() from S3Object", datetime(1,1,1)), - - ] - - try: - test_sql_expressions(client, json_testfile, tests, log_output) - except Exception as select_err: - raise select_err - # raise ValueError('Test {} unexpectedly failed with: {}'.format(test_name, select_err)) - # pass - - # Test passes - print(log_output.json_report()) - -def test_sql_functions_string(client, log_output): - - json_testfile = """ -{"id": 1, "name": "John"} -{"id": 2, "name": " \tfoobar\t "} -{"id": 3, "name": "1112211foobar22211122"} -""" - - tests = [ - # CHAR_LENGTH - ("CHAR_LENGTH", "select CHAR_LENGTH(s.name) from S3Object as s", b'4\n24\n21\n'), - ("CHARACTER_LENGTH", "select CHARACTER_LENGTH(s.name) from S3Object as s", b'4\n24\n21\n'), - # LOWER - ("LOWER", "select LOWER(s.name) from S3Object as s where s.id= 1", b'john\n'), - # SUBSTRING - ("SUBSTRING_1", "select SUBSTRING(s.name FROM 2) from S3Object as s where s.id = 1", b'ohn\n'), - ("SUBSTRING_2", "select SUBSTRING(s.name FROM 2 FOR 2) from S3Object as s where s.id = 1", b'oh\n'), - ("SUBSTRING_3", "select SUBSTRING(s.name FROM -1 FOR 2) from S3Object as s where s.id = 1", b'\n'), - # TRIM - ("TRIM_1", "select TRIM(s.name) from S3Object as s where s.id = 2", b'\tfoobar\t\n'), - ("TRIM_2", "select TRIM(LEADING FROM s.name) from S3Object as s where s.id = 2", b'\tfoobar\t \n'), - ("TRIM_3", "select TRIM(TRAILING FROM s.name) from S3Object as s where s.id = 2", b' \tfoobar\t\n'), - ("TRIM_4", "select TRIM(BOTH FROM s.name) from S3Object as s where s.id = 2", b'\tfoobar\t\n'), - ("TRIM_5", "select TRIM(BOTH '12' FROM s.name) from S3Object as s where s.id = 3", b'foobar\n'), - # UPPER - ("UPPER", "select UPPER(s.name) from S3Object as s where s.id= 1", b'JOHN\n'), - ] - - try: - test_sql_expressions(client, json_testfile, tests, log_output) - except Exception as select_err: - raise select_err - # raise ValueError('Test {} unexpectedly failed with: {}'.format(test_name, select_err)) - # pass - - # Test passes - print(log_output.json_report()) - - -def test_sql_datatypes(client, log_output): - json_testfile = """ -{"name": "John"} -""" - tests = [ - ("bool", "select CAST('true' AS BOOL) from S3Object", b'true\n'), - ("int", "select CAST('13' AS INT) from S3Object", b'13\n'), - ("integer", "select CAST('13' AS INTEGER) from S3Object", b'13\n'), - ("string", "select CAST(true AS STRING) from S3Object", b'true\n'), - ("float", "select CAST('13.3' AS FLOAT) from S3Object", b'13.3\n'), - ("decimal", "select CAST('14.3' AS FLOAT) from S3Object", b'14.3\n'), - ("numeric", "select CAST('14.3' AS FLOAT) from S3Object", b'14.3\n'), - ("timestamp", "select CAST('2007-04-05T14:30Z' AS TIMESTAMP) from S3Object", b'2007-04-05T14:30Z\n'), - ] - - try: - test_sql_expressions(client, json_testfile, tests, log_output) - except Exception as select_err: - raise select_err - # raise ValueError('Test {} unexpectedly failed with: {}'.format(test_name, select_err)) - # pass - - # Test passes - print(log_output.json_report()) - - -def test_sql_select(client, log_output): - - json_testfile = """{"id": 1, "created": "June 27", "modified": "July 6" } -{"id": 2, "Created": "June 28", "Modified": "July 7", "Cast": "Random Date" }""" - tests = [ - ("select_1", "select * from S3Object", b'1,June 27,July 6\n2,June 28,July 7,Random Date\n'), - ("select_2", "select * from S3Object s", b'1,June 27,July 6\n2,June 28,July 7,Random Date\n'), - ("select_3", "select * from S3Object as s", b'1,June 27,July 6\n2,June 28,July 7,Random Date\n'), - ("select_4", "select s.line from S3Object as s", b'\n\n'), - ("select_5", 'select s."Created" from S3Object as s', b'\nJune 28\n'), - ("select_5", 'select s."Cast" from S3Object as s', b'\nRandom Date\n'), - ("where", 'select s.created from S3Object as s', b'June 27\nJune 28\n'), - ("limit", 'select * from S3Object as s LIMIT 1', b'1,June 27,July 6\n'), - ] - - try: - test_sql_expressions(client, json_testfile, tests, log_output) - except Exception as select_err: - raise select_err - # raise ValueError('Test {} unexpectedly failed with: {}'.format(test_name, select_err)) - # pass - - # Test passes - print(log_output.json_report()) - -def test_sql_select_json(client, log_output): - json_testcontent = """{ "Rules": [ {"id": "1"}, {"expr": "y > x"}, {"id": "2", "expr": "z = DEBUG"} ]} -{ "created": "June 27", "modified": "July 6" } -""" - tests = [ - ("select_1", "SELECT id FROM S3Object[*].Rules[*].id", b'{"id":"1"}\n{}\n{"id":"2"}\n{}\n'), - ("select_2", "SELECT id FROM S3Object[*].Rules[*].id WHERE id IS NOT MISSING", b'{"id":"1"}\n{"id":"2"}\n'), - ("select_3", "SELECT d.created, d.modified FROM S3Object[*] d", b'{}\n{"created":"June 27","modified":"July 6"}\n'), - ("select_4", "SELECT _1.created, _1.modified FROM S3Object[*]", b'{}\n{"created":"June 27","modified":"July 6"}\n'), - ("select_5", "Select s.rules[1].expr from S3Object s", b'{"expr":"y > x"}\n{}\n'), - ] - - input_serialization = InputSerialization(json=JSONInput(Type="DOCUMENT")) - output_serialization = OutputSerialization(json=JsonOutput()) - try: - test_sql_expressions_custom_input_output(client, json_testcontent, - input_serialization, output_serialization, tests, log_output) - except Exception as select_err: - raise select_err - # raise ValueError('Test {} unexpectedly failed with: {}'.format(test_name, select_err)) - # pass - - # Test passes - print(log_output.json_report()) - - -def test_sql_select_csv_no_header(client, log_output): - json_testcontent = """val1,val2,val3 -val4,val5,val6 -""" - tests = [ - ("select_1", "SELECT s._2 FROM S3Object as s", b'val2\nval5\n'), - ] - - input_serialization=InputSerialization( - csv=CSVInput( - FileHeaderInfo="NONE", - AllowQuotedRecordDelimiter="FALSE", - ), - ) - - output_serialization=OutputSerialization(csv=CSVOutput()) - try: - test_sql_expressions_custom_input_output(client, json_testcontent, - input_serialization, output_serialization, tests, log_output) - except Exception as select_err: - raise select_err - # raise ValueError('Test {} unexpectedly failed with: {}'.format(test_name, select_err)) - # pass - - # Test passes - print(log_output.json_report()) - - diff --git a/mint/run/core/s3select/tests.py b/mint/run/core/s3select/tests.py deleted file mode 100644 index 3e8c26d..0000000 --- a/mint/run/core/s3select/tests.py +++ /dev/null @@ -1,86 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# MinIO Python Library for Amazon S3 Compatible Cloud Storage, -# (C) 2015-2020 MinIO, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -from sys import exit -from minio import Minio - -from utils import LogOutput -from sql_ops import * -from csv import * - -def main(): - """ - Functional testing for S3 select. - """ - - try: - access_key = os.getenv('ACCESS_KEY', 'Q3AM3UQ867SPQQA43P2F') - secret_key = os.getenv('SECRET_KEY', - 'zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG') - server_endpoint = os.getenv('SERVER_ENDPOINT', 'play.min.io') - secure = os.getenv('ENABLE_HTTPS', '1') == '1' - if server_endpoint == 'play.min.io': - access_key = 'Q3AM3UQ867SPQQA43P2F' - secret_key = 'zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG' - secure = True - - client = Minio(server_endpoint, access_key, secret_key, secure=False) - - log_output = LogOutput(client.select_object_content, 'test_csv_input_quote_char') - test_csv_input_custom_quote_char(client, log_output) - - log_output = LogOutput(client.select_object_content, 'test_csv_output_quote_char') - test_csv_output_custom_quote_char(client, log_output) - - log_output = LogOutput(client.select_object_content, 'test_sql_operators') - test_sql_operators(client, log_output) - - log_output = LogOutput(client.select_object_content, 'test_sql_operators_precedence') - test_sql_operators_precedence(client, log_output) - - log_output = LogOutput(client.select_object_content, 'test_sql_functions_agg_cond_conv') - test_sql_functions_agg_cond_conv(client, log_output) - - log_output = LogOutput(client.select_object_content, 'test_sql_functions_date') - test_sql_functions_date(client, log_output) - - log_output = LogOutput(client.select_object_content, 'test_sql_functions_string') - test_sql_functions_string(client, log_output) - - log_output = LogOutput(client.select_object_content, 'test_sql_datatypes') - test_sql_datatypes(client, log_output) - - log_output = LogOutput(client.select_object_content, 'test_sql_select') - test_sql_select(client, log_output) - - log_output = LogOutput(client.select_object_content, 'test_sql_select_json') - test_sql_select_json(client, log_output) - - log_output = LogOutput(client.select_object_content, 'test_sql_select_csv') - test_sql_select_csv_no_header(client, log_output) - - except Exception as err: - print(log_output.json_report(err)) - exit(1) - -if __name__ == "__main__": - # Execute only if run as a script - main() - - - diff --git a/mint/run/core/s3select/utils.py b/mint/run/core/s3select/utils.py deleted file mode 100644 index 1baabf9..0000000 --- a/mint/run/core/s3select/utils.py +++ /dev/null @@ -1,106 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# MinIO Python Library for Amazon S3 Compatible Cloud Storage, -# (C) 2015-2020 MinIO, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import uuid -import inspect -import json -import time -import traceback - -class LogOutput(object): - """ - LogOutput is the class for log output. It is required standard for all - SDK tests controlled by mint. - Here are its attributes: - 'name': name of the SDK under test, e.g. 's3select' - 'function': name of the method/api under test with its signature - The following python code can be used to - pull args information of a and to - put together with the method name: - .__name__+'('+', '.join(args_list)+')' - e.g. 'remove_object(bucket_name, object_name)' - 'args': method/api arguments with their values, in - dictionary form: {'arg1': val1, 'arg2': val2, ...} - 'duration': duration of the whole test in milliseconds, - defaults to 0 - 'alert': any extra information user is needed to be alerted about, - like whether this is a Blocker/Gateway/Server related - issue, etc., defaults to None - 'message': descriptive error message, defaults to None - 'error': stack-trace/exception message(only in case of failure), - actual low level exception/error thrown by the program, - defaults to None - 'status': exit status, possible values are 'PASS', 'FAIL', 'NA', - defaults to 'PASS' - """ - - PASS = 'PASS' - FAIL = 'FAIL' - NA = 'NA' - - def __init__(self, meth, test_name): - self.__args_list = inspect.getargspec(meth).args[1:] - self.__name = 's3select:'+test_name - self.__function = meth.__name__+'('+', '.join(self.__args_list)+')' - self.__args = {} - self.__duration = 0 - self.__alert = '' - self.__message = None - self.__error = None - self.__status = self.PASS - self.__start_time = time.time() - - @property - def name(self): return self.__name - - @property - def function(self): return self.__function - - @property - def args(self): return self.__args - - @name.setter - def name(self, val): self.__name = val - - @function.setter - def function(self, val): self.__function = val - - @args.setter - def args(self, val): self.__args = val - - def json_report(self, err_msg='', alert='', status=''): - self.__args = {k: v for k, v in self.__args.items() if v and v != ''} - entry = {'name': self.__name, - 'function': self.__function, - 'args': self.__args, - 'duration': int(round((time.time() - self.__start_time)*1000)), - 'alert': str(alert), - 'message': str(err_msg), - 'error': traceback.format_exc() if err_msg and err_msg != '' else '', - 'status': status if status and status != '' else - self.FAIL if err_msg and err_msg != '' else self.PASS - } - return json.dumps({k: v for k, v in entry.items() if v and v != ''}) - - -def generate_bucket_name(): - return "s3select-test-" + str(uuid.uuid4()) - -def generate_object_name(): - return str(uuid.uuid4()) - - diff --git a/mint/run/core/security/run.sh b/mint/run/core/security/run.sh deleted file mode 100755 index 9e7c217..0000000 --- a/mint/run/core/security/run.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -# -# Mint (C) 2018 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# handle command line arguments -if [ $# -ne 2 ]; then - echo "usage: run.sh " - exit 1 -fi - -output_log_file="$1" -error_log_file="$2" - -# run tests -/mint/run/core/security/tls-tests 1>>"$output_log_file" 2>"$error_log_file" diff --git a/mint/run/core/security/tls-tests.go b/mint/run/core/security/tls-tests.go deleted file mode 100644 index adbd87b..0000000 --- a/mint/run/core/security/tls-tests.go +++ /dev/null @@ -1,272 +0,0 @@ -// Mint, (C) 2018 Minio, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software - -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "crypto/tls" - "encoding/json" - "fmt" - "os" - "time" - - log "github.com/sirupsen/logrus" -) - -const testName = "TLS-tests" - -const ( - // PASS indicate that a test passed - PASS = "PASS" - // FAIL indicate that a test failed - FAIL = "FAIL" - // NA indicates that a test is not applicable - NA = "NA" -) - -func main() { - log.SetOutput(os.Stdout) - log.SetFormatter(&mintJSONFormatter{}) - log.SetLevel(log.InfoLevel) - - endpoint := os.Getenv("SERVER_ENDPOINT") - secure := os.Getenv("ENABLE_HTTPS") - if secure != "1" { - log.WithFields(log.Fields{"name:": testName, "status": NA, "message": "TLS is not enabled"}).Info() - return - } - - testTLSVersions(endpoint) - testTLSCiphers(endpoint) - testTLSEllipticCurves(endpoint) -} - -// Tests whether the endpoint accepts TLS1.0 or TLS1.1 connections - fail if so. -// Tests whether the endpoint accepts TLS1.2 connections - fail if not. -func testTLSVersions(endpoint string) { - const function = "TLSVersions" - startTime := time.Now() - - // Tests whether the endpoint accepts TLS1.0 or TLS1.1 connections - args := map[string]interface{}{ - "MinVersion": "tls.VersionTLS10", - "MaxVersion": "tls.VersionTLS11", - } - _, err := tls.Dial("tcp", endpoint, &tls.Config{ - MinVersion: tls.VersionTLS10, - MaxVersion: tls.VersionTLS11, - }) - if err == nil { - failureLog(function, args, startTime, "", "Endpoint accepts insecure connection", err).Error() - return - } - - // Tests whether the endpoint accepts TLS1.2 connections - args = map[string]interface{}{ - "MinVersion": "tls.VersionTLS12", - } - _, err = tls.Dial("tcp", endpoint, &tls.Config{ - MinVersion: tls.VersionTLS12, - }) - if err != nil { - failureLog(function, args, startTime, "", "Endpoint rejects secure connection", err).Error() - return - } - successLog(function, args, startTime) -} - -// Tests whether the endpoint accepts SSL3.0, TLS1.0 or TLS1.1 connections - fail if so. -// Tests whether the endpoint accepts TLS1.2 connections - fail if not. -func testTLSCiphers(endpoint string) { - const function = "TLSCiphers" - startTime := time.Now() - - // Tests whether the endpoint accepts insecure ciphers - args := map[string]interface{}{ - "MinVersion": "tls.VersionTLS12", - "CipherSuites": unsupportedCipherSuites, - } - _, err := tls.Dial("tcp", endpoint, &tls.Config{ - MinVersion: tls.VersionTLS12, - CipherSuites: unsupportedCipherSuites, - }) - if err == nil { - failureLog(function, args, startTime, "", "Endpoint accepts insecure cipher suites", err).Error() - return - } - - // Tests whether the endpoint accepts at least one secure cipher - args = map[string]interface{}{ - "MinVersion": "tls.VersionTLS12", - "CipherSuites": supportedCipherSuites, - } - _, err = tls.Dial("tcp", endpoint, &tls.Config{ - MinVersion: tls.VersionTLS12, - CipherSuites: supportedCipherSuites, - }) - if err != nil { - failureLog(function, args, startTime, "", "Endpoint rejects all secure cipher suites", err).Error() - return - } - - // Tests whether the endpoint accepts at least one default cipher - args = map[string]interface{}{ - "MinVersion": "tls.VersionTLS12", - "CipherSuites": nil, - } - _, err = tls.Dial("tcp", endpoint, &tls.Config{ - MinVersion: tls.VersionTLS12, - CipherSuites: nil, // default value - }) - if err != nil { - failureLog(function, args, startTime, "", "Endpoint rejects default cipher suites", err).Error() - return - } - successLog(function, args, startTime) -} - -// Tests whether the endpoint accepts the P-384 or P-521 elliptic curve - fail if so. -// Tests whether the endpoint accepts Curve25519 or P-256 - fail if not. -func testTLSEllipticCurves(endpoint string) { - const function = "TLSEllipticCurves" - startTime := time.Now() - - // Tests whether the endpoint accepts curves using non-constant time implementations. - args := map[string]interface{}{ - "CurvePreferences": unsupportedCurves, - } - _, err := tls.Dial("tcp", endpoint, &tls.Config{ - MinVersion: tls.VersionTLS12, - CurvePreferences: unsupportedCurves, - CipherSuites: supportedCipherSuites, - }) - if err == nil { - failureLog(function, args, startTime, "", "Endpoint accepts insecure elliptic curves", err).Error() - return - } - - // Tests whether the endpoint accepts curves using constant time implementations. - args = map[string]interface{}{ - "CurvePreferences": unsupportedCurves, - } - _, err = tls.Dial("tcp", endpoint, &tls.Config{ - MinVersion: tls.VersionTLS12, - CurvePreferences: supportedCurves, - CipherSuites: supportedCipherSuites, - }) - if err != nil { - failureLog(function, args, startTime, "", "Endpoint does not accept secure elliptic curves", err).Error() - return - } - successLog(function, args, startTime) -} - -func successLog(function string, args map[string]interface{}, startTime time.Time) *log.Entry { - duration := time.Since(startTime).Nanoseconds() / 1000000 - return log.WithFields(log.Fields{ - "name": testName, - "function": function, - "args": args, - "duration": duration, - "status": PASS, - }) -} - -func failureLog(function string, args map[string]interface{}, startTime time.Time, alert string, message string, err error) *log.Entry { - duration := time.Since(startTime).Nanoseconds() / 1000000 - fields := log.Fields{ - "name": testName, - "function": function, - "args": args, - "duration": duration, - "status": FAIL, - "alert": alert, - "message": message, - } - if err != nil { - fields["error"] = err - } - return log.WithFields(fields) -} - -type mintJSONFormatter struct { -} - -func (f *mintJSONFormatter) Format(entry *log.Entry) ([]byte, error) { - data := make(log.Fields, len(entry.Data)) - for k, v := range entry.Data { - switch v := v.(type) { - case error: - // Otherwise errors are ignored by `encoding/json` - // https://github.com/sirupsen/logrus/issues/137 - data[k] = v.Error() - default: - data[k] = v - } - } - - serialized, err := json.Marshal(data) - if err != nil { - return nil, fmt.Errorf("Failed to marshal fields to JSON, %w", err) - } - return append(serialized, '\n'), nil -} - -// Secure Go implementations of modern TLS ciphers -// The following ciphers are excluded because: -// - RC4 ciphers: RC4 is broken -// - 3DES ciphers: Because of the 64 bit blocksize of DES (Sweet32) -// - CBC-SHA256 ciphers: No countermeasures against Lucky13 timing attack -// - CBC-SHA ciphers: Legacy ciphers (SHA-1) and non-constant time -// implementation of CBC. -// (CBC-SHA ciphers can be enabled again if required) -// - RSA key exchange ciphers: Disabled because of dangerous PKCS1-v1.5 RSA -// padding scheme. See Bleichenbacher attacks. -var supportedCipherSuites = []uint16{ - tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, - tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305, - tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, - tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, - tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, - tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, -} - -// Supported elliptic curves: Implementations are constant-time. -var supportedCurves = []tls.CurveID{tls.X25519, tls.CurveP256} - -var unsupportedCipherSuites = []uint16{ - tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA, // Go stack contains (some) countermeasures against timing attacks (Lucky13) - tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256, // No countermeasures against timing attacks - tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA, // Go stack contains (some) countermeasures against timing attacks (Lucky13) - tls.TLS_ECDHE_ECDSA_WITH_RC4_128_SHA, // Broken cipher - tls.TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA, // Sweet32 - tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, // Go stack contains (some) countermeasures against timing attacks (Lucky13) - tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256, // No countermeasures against timing attacks - tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, // Go stack contains (some) countermeasures against timing attacks (Lucky13) - tls.TLS_ECDHE_RSA_WITH_RC4_128_SHA, // Broken cipher - - // all RSA-PKCS1-v1.5 ciphers are disabled - danger of Bleichenbacher attack variants - tls.TLS_RSA_WITH_3DES_EDE_CBC_SHA, // Sweet32 - tls.TLS_RSA_WITH_AES_128_CBC_SHA, // Go stack contains (some) countermeasures against timing attacks (Lucky13) - tls.TLS_RSA_WITH_AES_128_CBC_SHA256, // No countermeasures against timing attacks - tls.TLS_RSA_WITH_AES_256_CBC_SHA, // Go stack contains (some) countermeasures against timing attacks (Lucky13) - tls.TLS_RSA_WITH_RC4_128_SHA, // Broken cipher - - tls.TLS_RSA_WITH_AES_128_GCM_SHA256, // Disabled because of RSA-PKCS1-v1.5 - AES-GCM is considered secure. - tls.TLS_RSA_WITH_AES_256_GCM_SHA384, // Disabled because of RSA-PKCS1-v1.5 - AES-GCM is considered secure. -} - -// Unsupported elliptic curves: Implementations are not constant-time. -var unsupportedCurves = []tls.CurveID{tls.CurveP384, tls.CurveP521} diff --git a/misc/build.go b/misc/build.go new file mode 100644 index 0000000..3d82ab9 --- /dev/null +++ b/misc/build.go @@ -0,0 +1,6 @@ +package misc + +var ( + Build = "now" + Version = "dev" +)