From 8d97dcbf10071232e39e32c4f0da8d7e5128c95b Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Fri, 3 Jul 2020 18:08:57 +0300 Subject: [PATCH] Cleanup + Makefile + Dockerfile --- .gitignore | 31 +- .golangci.yml | 27 - .mailmap | 18 - Dockerfile | 50 +- Dockerfile.arm.release | 41 - Dockerfile.arm64.release | 41 - Dockerfile.dev | 27 - Dockerfile.dev.browser | 13 - Dockerfile.mint | 17 - Dockerfile.release | 37 - Makefile | 108 +- cmd/web-handlers.go | 37 +- cmd/web-router.go | 9 +- go.mod | 2 - go.sum | 4 - main.go | 10 +- mint/.gitignore | 17 - mint/README.md | 119 -- mint/build/aws-sdk-go/install.sh | 19 - mint/build/aws-sdk-java/build.xml | 58 - mint/build/aws-sdk-java/install.sh | 30 - mint/build/aws-sdk-java/ivy.xml | 6 - .../aws-sdk-java/src/FunctionalTests.java | 632 ------ .../aws-sdk-java/src/LimitedInputStream.java | 60 - mint/build/aws-sdk-java/src/MintLogger.java | 151 -- mint/build/aws-sdk-java/src/S3TestUtils.java | 187 -- mint/build/aws-sdk-java/src/Utils.java | 76 - mint/build/aws-sdk-php/install.sh | 20 - mint/build/aws-sdk-ruby/install.sh | 18 - mint/build/awscli/install.sh | 18 - mint/build/healthcheck/install.sh | 19 - mint/build/mc/install.sh | 31 - mint/build/minio-dotnet/install.sh | 42 - mint/build/minio-go/install.sh | 27 - mint/build/minio-java/install.sh | 30 - mint/build/minio-js/install.sh | 28 - mint/build/minio-py/install.sh | 27 - mint/build/s3cmd/install.sh | 19 - mint/build/s3select/install.sh | 18 - mint/build/security/install.sh | 19 - mint/create-data-files.sh | 44 - mint/entrypoint.sh | 24 - mint/install-packages.list | 18 - mint/mint.sh | 190 -- mint/postinstall.sh | 28 - mint/preinstall.sh | 56 - mint/release.sh | 32 - mint/remove-packages.list | 9 - mint/run/core/aws-sdk-go/quick-tests.go | 1066 ---------- mint/run/core/aws-sdk-go/run.sh | 28 - mint/run/core/aws-sdk-java/run.sh | 30 - mint/run/core/aws-sdk-php/README.md | 19 - mint/run/core/aws-sdk-php/composer.json | 6 - mint/run/core/aws-sdk-php/quick-tests.php | 1124 ----------- mint/run/core/aws-sdk-php/run.sh | 28 - mint/run/core/aws-sdk-ruby/README.md | 19 - mint/run/core/aws-sdk-ruby/aws-stub-tests.rb | 855 -------- mint/run/core/aws-sdk-ruby/run.sh | 29 - mint/run/core/awscli/README.md | 19 - mint/run/core/awscli/run.sh | 51 - mint/run/core/awscli/test.sh | 1726 ----------------- mint/run/core/healthcheck/healthcheck.go | 219 --- mint/run/core/healthcheck/run.sh | 28 - mint/run/core/mc/README.md | 19 - mint/run/core/mc/run.sh | 27 - mint/run/core/minio-dotnet/run.sh | 26 - mint/run/core/minio-go/README.md | 19 - mint/run/core/minio-go/run.sh | 28 - mint/run/core/minio-java/README.md | 19 - mint/run/core/minio-java/run.sh | 34 - mint/run/core/minio-js/README.md | 19 - mint/run/core/minio-js/minioreporter.js | 69 - mint/run/core/minio-js/package.json | 49 - mint/run/core/minio-js/run.sh | 28 - mint/run/core/minio-py/README.md | 19 - mint/run/core/minio-py/run.sh | 28 - mint/run/core/s3cmd/.gitignore | 2 - mint/run/core/s3cmd/README.md | 19 - mint/run/core/s3cmd/run.sh | 28 - mint/run/core/s3cmd/test.sh | 406 ---- mint/run/core/s3select/README.md | 21 - mint/run/core/s3select/csv.py | 172 -- mint/run/core/s3select/run.sh | 28 - mint/run/core/s3select/sql_ops.py | 399 ---- mint/run/core/s3select/tests.py | 86 - mint/run/core/s3select/utils.py | 106 - mint/run/core/security/run.sh | 28 - mint/run/core/security/tls-tests.go | 272 --- misc/build.go | 6 + 89 files changed, 98 insertions(+), 9650 deletions(-) delete mode 100644 .golangci.yml delete mode 100644 .mailmap delete mode 100644 Dockerfile.arm.release delete mode 100644 Dockerfile.arm64.release delete mode 100644 Dockerfile.dev delete mode 100644 Dockerfile.dev.browser delete mode 100644 Dockerfile.mint delete mode 100644 Dockerfile.release delete mode 100644 mint/.gitignore delete mode 100644 mint/README.md delete mode 100755 mint/build/aws-sdk-go/install.sh delete mode 100644 mint/build/aws-sdk-java/build.xml delete mode 100755 mint/build/aws-sdk-java/install.sh delete mode 100644 mint/build/aws-sdk-java/ivy.xml delete mode 100644 mint/build/aws-sdk-java/src/FunctionalTests.java delete mode 100644 mint/build/aws-sdk-java/src/LimitedInputStream.java delete mode 100755 mint/build/aws-sdk-java/src/MintLogger.java delete mode 100644 mint/build/aws-sdk-java/src/S3TestUtils.java delete mode 100644 mint/build/aws-sdk-java/src/Utils.java delete mode 100755 mint/build/aws-sdk-php/install.sh delete mode 100755 mint/build/aws-sdk-ruby/install.sh delete mode 100755 mint/build/awscli/install.sh delete mode 100755 mint/build/healthcheck/install.sh delete mode 100755 mint/build/mc/install.sh delete mode 100755 mint/build/minio-dotnet/install.sh delete mode 100755 mint/build/minio-go/install.sh delete mode 100755 mint/build/minio-java/install.sh delete mode 100755 mint/build/minio-js/install.sh delete mode 100755 mint/build/minio-py/install.sh delete mode 100755 mint/build/s3cmd/install.sh delete mode 100755 mint/build/s3select/install.sh delete mode 100755 mint/build/security/install.sh delete mode 100755 mint/create-data-files.sh delete mode 100755 mint/entrypoint.sh delete mode 100644 mint/install-packages.list delete mode 100755 mint/mint.sh delete mode 100755 mint/postinstall.sh delete mode 100755 mint/preinstall.sh delete mode 100755 mint/release.sh delete mode 100644 mint/remove-packages.list delete mode 100644 mint/run/core/aws-sdk-go/quick-tests.go delete mode 100755 mint/run/core/aws-sdk-go/run.sh delete mode 100755 mint/run/core/aws-sdk-java/run.sh delete mode 100644 mint/run/core/aws-sdk-php/README.md delete mode 100644 mint/run/core/aws-sdk-php/composer.json delete mode 100644 mint/run/core/aws-sdk-php/quick-tests.php delete mode 100755 mint/run/core/aws-sdk-php/run.sh delete mode 100644 mint/run/core/aws-sdk-ruby/README.md delete mode 100755 mint/run/core/aws-sdk-ruby/aws-stub-tests.rb delete mode 100755 mint/run/core/aws-sdk-ruby/run.sh delete mode 100644 mint/run/core/awscli/README.md delete mode 100755 mint/run/core/awscli/run.sh delete mode 100755 mint/run/core/awscli/test.sh delete mode 100644 mint/run/core/healthcheck/healthcheck.go delete mode 100755 mint/run/core/healthcheck/run.sh delete mode 100644 mint/run/core/mc/README.md delete mode 100755 mint/run/core/mc/run.sh delete mode 100755 mint/run/core/minio-dotnet/run.sh delete mode 100644 mint/run/core/minio-go/README.md delete mode 100755 mint/run/core/minio-go/run.sh delete mode 100644 mint/run/core/minio-java/README.md delete mode 100755 mint/run/core/minio-java/run.sh delete mode 100644 mint/run/core/minio-js/README.md delete mode 100644 mint/run/core/minio-js/minioreporter.js delete mode 100644 mint/run/core/minio-js/package.json delete mode 100755 mint/run/core/minio-js/run.sh delete mode 100644 mint/run/core/minio-py/README.md delete mode 100755 mint/run/core/minio-py/run.sh delete mode 100644 mint/run/core/s3cmd/.gitignore delete mode 100644 mint/run/core/s3cmd/README.md delete mode 100755 mint/run/core/s3cmd/run.sh delete mode 100755 mint/run/core/s3cmd/test.sh delete mode 100644 mint/run/core/s3select/README.md delete mode 100644 mint/run/core/s3select/csv.py delete mode 100755 mint/run/core/s3select/run.sh delete mode 100644 mint/run/core/s3select/sql_ops.py delete mode 100644 mint/run/core/s3select/tests.py delete mode 100644 mint/run/core/s3select/utils.py delete mode 100755 mint/run/core/security/run.sh delete mode 100644 mint/run/core/security/tls-tests.go create mode 100644 misc/build.go diff --git a/.gitignore b/.gitignore index 8e89bbf3a..e2f53be22 100644 --- a/.gitignore +++ b/.gitignore @@ -1,24 +1,9 @@ -**/*.swp -cover.out -*~ -minio -!*/ -site/ -**/*.test -**/*.sublime-workspace -/.idea/ -/Minio.iml -**/access.log -vendor/**/*.js -vendor/**/*.json -release +# IDE +.idea +.vscode + +# Vendoring +vendor + +# tempfiles .DS_Store -*.syso -coverage.txt -.vscode/ -*.tar.bz2 -parts/ -prime/ -stage/ -.sia_temp/ -config.json diff --git a/.golangci.yml b/.golangci.yml deleted file mode 100644 index 891e90fcd..000000000 --- a/.golangci.yml +++ /dev/null @@ -1,27 +0,0 @@ -linters-settings: - golint: - min-confidence: 0 - - misspell: - locale: US - -linters: - disable-all: true - enable: - - typecheck - - goimports - - misspell - - govet - - golint - - ineffassign - - gosimple - - deadcode - - structcheck - -issues: - exclude-use-default: false - exclude: - - should have a package comment - - error strings should not be capitalized or end with punctuation or a newline -service: - golangci-lint-version: 1.20.0 # use the fixed version to not introduce new linters unexpectedly diff --git a/.mailmap b/.mailmap deleted file mode 100644 index 4adf1be93..000000000 --- a/.mailmap +++ /dev/null @@ -1,18 +0,0 @@ -# Generate CONTRIBUTORS.md: contributors.sh - -# Tip for finding duplicates (besides scanning the output of CONTRIBUTORS.md for name -# duplicates that aren't also email duplicates): scan the output of: -# git log --format='%aE - %aN' | sort -uf -# -# For explanation on this file format: man git-shortlog - -Anand Babu (AB) Periasamy Anand Babu (AB) Periasamy -Anand Babu (AB) Periasamy -Anis Elleuch -Frederick F. Kautz IV -Harshavardhana -Harshavardhana -Harshavardhana -Krishna Srinivas -Matthew Farrellee -Nate Rosenblum \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index a05293f0c..5e091abaa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,36 +1,30 @@ -FROM golang:1.13-alpine +FROM golang:1 as builder -LABEL maintainer="MinIO Inc " +COPY . /src -ENV GOPATH /go -ENV CGO_ENABLED 0 -ENV GO111MODULE on +WORKDIR /src -RUN \ - apk add --no-cache git && \ - git clone https://github.com/minio/minio && cd minio && \ - go install -v -ldflags "$(go run buildscripts/gen-ldflags.go)" +ARG VERSION=dev +# https://github.com/golang/go/wiki/Modules#how-do-i-use-vendoring-with-modules-is-vendoring-going-away +# go build -mod=vendor +# The -gcflags "all=-N -l" flag helps us get a better debug experience +RUN set -x \ + && export BUILD=$(date -u +%s%N) \ + && export REPO=$(go list -m) \ + && export LDFLAGS="-X ${REPO}/misc.Version=${VERSION} -X ${REPO}/misc.Build=${BUILD}" \ + && export GOGC=off \ + && export CGO_ENABLED=0 \ + && [ -d "./vendor" ] || go mod vendor \ + && go build -v -mod=vendor -trimpath -gcflags "all=-N -l" -ldflags "${LDFLAGS}" -o /go/bin/neofs-s3 ./main.go + +# Executable image FROM alpine:3.10 -ENV MINIO_UPDATE off -ENV MINIO_ACCESS_KEY_FILE=access_key \ - MINIO_SECRET_KEY_FILE=secret_key \ - MINIO_KMS_MASTER_KEY_FILE=kms_master_key \ - MINIO_SSE_MASTER_KEY_FILE=sse_master_key +WORKDIR / -EXPOSE 9000 +COPY --from=builder /go/bin/neofs-s3 /usr/bin/neofs-s3 +COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ -COPY --from=0 /go/bin/minio /usr/bin/minio -COPY --from=0 /go/minio/CREDITS /third_party/ -COPY --from=0 /go/minio/dockerscripts/docker-entrypoint.sh /usr/bin/ - -RUN \ - apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' && \ - echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf - -ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"] - -VOLUME ["/data"] - -CMD ["minio"] +# Run delve +CMD ["/usr/bin/neofs-s3"] diff --git a/Dockerfile.arm.release b/Dockerfile.arm.release deleted file mode 100644 index 3e1f64192..000000000 --- a/Dockerfile.arm.release +++ /dev/null @@ -1,41 +0,0 @@ -FROM golang:1.13-alpine as builder - -WORKDIR /home - -ENV GOPATH /go -ENV CGO_ENABLED 0 -ENV GO111MODULE on - -RUN \ - apk add --no-cache git 'curl>7.61.0' && \ - git clone https://github.com/minio/minio && \ - curl -L https://github.com/balena-io/qemu/releases/download/v3.0.0%2Bresin/qemu-3.0.0+resin-arm.tar.gz | tar zxvf - -C . && mv qemu-3.0.0+resin-arm/qemu-arm-static . - -FROM arm32v7/alpine:3.10 - -LABEL maintainer="MinIO Inc " - -COPY dockerscripts/docker-entrypoint.sh /usr/bin/ -COPY CREDITS /third_party/ -COPY --from=builder /home/qemu-arm-static /usr/bin/qemu-arm-static - -ENV MINIO_UPDATE off -ENV MINIO_ACCESS_KEY_FILE=access_key \ - MINIO_SECRET_KEY_FILE=secret_key \ - MINIO_KMS_MASTER_KEY_FILE=kms_master_key \ - MINIO_SSE_MASTER_KEY_FILE=sse_master_key - -RUN \ - apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' && \ - echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \ - curl https://dl.min.io/server/minio/release/linux-arm/minio > /usr/bin/minio && \ - chmod +x /usr/bin/minio && \ - chmod +x /usr/bin/docker-entrypoint.sh - -EXPOSE 9000 - -ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"] - -VOLUME ["/data"] - -CMD ["minio"] diff --git a/Dockerfile.arm64.release b/Dockerfile.arm64.release deleted file mode 100644 index 711886821..000000000 --- a/Dockerfile.arm64.release +++ /dev/null @@ -1,41 +0,0 @@ -FROM golang:1.13-alpine as builder - -WORKDIR /home - -ENV GOPATH /go -ENV CGO_ENABLED 0 -ENV GO111MODULE on - -RUN \ - apk add --no-cache git 'curl>7.61.0' && \ - git clone https://github.com/minio/minio && \ - curl -L https://github.com/balena-io/qemu/releases/download/v3.0.0%2Bresin/qemu-3.0.0+resin-arm.tar.gz | tar zxvf - -C . && mv qemu-3.0.0+resin-arm/qemu-arm-static . - -FROM arm64v8/alpine:3.10 - -LABEL maintainer="MinIO Inc " - -COPY dockerscripts/docker-entrypoint.sh /usr/bin/ -COPY CREDITS /third_party/ -COPY --from=builder /home/qemu-arm-static /usr/bin/qemu-arm-static - -ENV MINIO_UPDATE off -ENV MINIO_ACCESS_KEY_FILE=access_key \ - MINIO_SECRET_KEY_FILE=secret_key \ - MINIO_KMS_MASTER_KEY_FILE=kms_master_key \ - MINIO_SSE_MASTER_KEY_FILE=sse_master_key - -RUN \ - apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' && \ - echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \ - curl https://dl.min.io/server/minio/release/linux-arm64/minio > /usr/bin/minio && \ - chmod +x /usr/bin/minio && \ - chmod +x /usr/bin/docker-entrypoint.sh - -EXPOSE 9000 - -ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"] - -VOLUME ["/data"] - -CMD ["minio"] diff --git a/Dockerfile.dev b/Dockerfile.dev deleted file mode 100644 index 6790efbfb..000000000 --- a/Dockerfile.dev +++ /dev/null @@ -1,27 +0,0 @@ -FROM alpine:3.10 - -LABEL maintainer="MinIO Inc " - -COPY dockerscripts/docker-entrypoint.sh /usr/bin/ -COPY minio /usr/bin/ -COPY CREDITS /third_party/ - -ENV MINIO_UPDATE off -ENV MINIO_ACCESS_KEY_FILE=access_key \ - MINIO_SECRET_KEY_FILE=secret_key \ - MINIO_KMS_MASTER_KEY_FILE=kms_master_key \ - MINIO_SSE_MASTER_KEY_FILE=sse_master_key - -RUN \ - apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' && \ - echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \ - chmod +x /usr/bin/minio && \ - chmod +x /usr/bin/docker-entrypoint.sh - -EXPOSE 9000 - -ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"] - -VOLUME ["/data"] - -CMD ["minio"] diff --git a/Dockerfile.dev.browser b/Dockerfile.dev.browser deleted file mode 100644 index da38e5986..000000000 --- a/Dockerfile.dev.browser +++ /dev/null @@ -1,13 +0,0 @@ -FROM ubuntu - -LABEL maintainer="MinIO Inc " - -RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends --no-install-suggests \ - git golang make npm && \ - apt-get clean && rm -rf /var/lib/apt/lists/* - -ENV PATH=$PATH:/root/go/bin - -RUN go get github.com/go-bindata/go-bindata/go-bindata && \ - go get github.com/elazarl/go-bindata-assetfs/go-bindata-assetfs - diff --git a/Dockerfile.mint b/Dockerfile.mint deleted file mode 100644 index 6daffbaaf..000000000 --- a/Dockerfile.mint +++ /dev/null @@ -1,17 +0,0 @@ -FROM ubuntu:18.04 - -ENV DEBIAN_FRONTEND noninteractive -ENV LANG C.UTF-8 -ENV GOROOT /usr/local/go -ENV GOPATH /usr/local/gopath -ENV PATH $GOPATH/bin:$GOROOT/bin:$PATH -ENV MINT_ROOT_DIR /mint -COPY mint /mint - -RUN apt-get --yes update && apt-get --yes upgrade && \ - apt-get --yes --quiet install wget jq curl git dnsmasq && \ - cd /mint && /mint/release.sh - -WORKDIR /mint - -ENTRYPOINT ["/mint/entrypoint.sh"] diff --git a/Dockerfile.release b/Dockerfile.release deleted file mode 100644 index 1dd41bf99..000000000 --- a/Dockerfile.release +++ /dev/null @@ -1,37 +0,0 @@ -FROM golang:1.13-alpine - -ENV GOPATH /go -ENV CGO_ENABLED 0 -ENV GO111MODULE on - -RUN \ - apk add --no-cache git && \ - git clone https://github.com/minio/minio - -FROM alpine:3.10 - -LABEL maintainer="MinIO Inc " - -COPY dockerscripts/docker-entrypoint.sh /usr/bin/ -COPY CREDITS /third_party/ - -ENV MINIO_UPDATE off -ENV MINIO_ACCESS_KEY_FILE=access_key \ - MINIO_SECRET_KEY_FILE=secret_key \ - MINIO_KMS_MASTER_KEY_FILE=kms_master_key \ - MINIO_SSE_MASTER_KEY_FILE=sse_master_key - -RUN \ - apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' && \ - echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \ - curl https://dl.min.io/server/minio/release/linux-amd64/minio > /usr/bin/minio && \ - chmod +x /usr/bin/minio && \ - chmod +x /usr/bin/docker-entrypoint.sh - -EXPOSE 9000 - -ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"] - -VOLUME ["/data"] - -CMD ["minio"] diff --git a/Makefile b/Makefile index 4f6f3f6c3..887150b9e 100644 --- a/Makefile +++ b/Makefile @@ -1,79 +1,39 @@ -PWD := $(shell pwd) -GOPATH := $(shell go env GOPATH) -LDFLAGS := $(shell go run buildscripts/gen-ldflags.go) +VERSION ?= "$(shell git describe --tags 2>/dev/null | sed 's/^v//')" +BUILD_VERSION ?= "$(shell git describe --abbrev=0 --tags | sed 's/^v//')" -GOARCH := $(shell go env GOARCH) -GOOS := $(shell go env GOOS) +.PHONY: help format deps -VERSION ?= $(shell git describe --tags) -TAG ?= "minio/minio:$(VERSION)" +# Show this help prompt +help: + @echo ' Usage:' + @echo '' + @echo ' make ' + @echo '' + @echo ' Targets:' + @echo '' + @awk '/^#/{ comment = substr($$0,3) } comment && /^[a-zA-Z][a-zA-Z0-9_-]+ ?:/{ print " ", $$1, comment }' $(MAKEFILE_LIST) | column -t -s ':' | grep -v 'IGNORE' | sort | uniq -all: build +# Reformat code +format: + @[ ! -z `which goimports` ] || (echo "install goimports" && exit 2) + @for f in `find . -type f -name '*.go' -not -path './vendor/*' -not -name '*.pb.go' -prune`; do \ + echo "⇒ Processing $$f"; \ + goimports -w $$f; \ + done -checks: - @echo "Checking dependencies" - @(env bash $(PWD)/buildscripts/checkdeps.sh) +# Make sure that all files added to commit +deps: + @printf "⇒ Ensure vendor: " + @go mod tidy -v && echo OK || (echo fail && exit 2) + @printf "⇒ Download requirements: " + @go mod download && echo OK || (echo fail && exit 2) + @printf "⇒ Store vendor localy: " + @go mod vendor && echo OK || (echo fail && exit 2) -getdeps: - @mkdir -p ${GOPATH}/bin - @which golangci-lint 1>/dev/null || (echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOPATH)/bin v1.27.0) - -crosscompile: - @(env bash $(PWD)/buildscripts/cross-compile.sh) - -verifiers: getdeps fmt lint - -fmt: - @echo "Running $@ check" - @GO111MODULE=on gofmt -d cmd/ - @GO111MODULE=on gofmt -d pkg/ - -lint: - @echo "Running $@ check" - @GO111MODULE=on ${GOPATH}/bin/golangci-lint cache clean - @GO111MODULE=on ${GOPATH}/bin/golangci-lint run --timeout=5m --config ./.golangci.yml - -# Builds minio, runs the verifiers then runs the tests. -check: test -test: verifiers build - @echo "Running unit tests" - @GO111MODULE=on CGO_ENABLED=0 go test -tags kqueue ./... 1>/dev/null - -test-race: verifiers build - @echo "Running unit tests under -race" - @(env bash $(PWD)/buildscripts/race.sh) - -# Verify minio binary -verify: - @echo "Verifying build with race" - @GO111MODULE=on CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null - @(env bash $(PWD)/buildscripts/verify-build.sh) - -# Verify healing of disks with minio binary -verify-healing: - @echo "Verify healing build with race" - @GO111MODULE=on CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null - @(env bash $(PWD)/buildscripts/verify-healing.sh) - -# Builds minio locally. -build: checks - @echo "Building minio binary to './minio'" - @GO111MODULE=on CGO_ENABLED=0 go build -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null - -docker: build - @docker build -t $(TAG) . -f Dockerfile.dev - -# Builds minio and installs it to $GOPATH/bin. -install: build - @echo "Installing minio binary to '$(GOPATH)/bin/minio'" - @mkdir -p $(GOPATH)/bin && cp -f $(PWD)/minio $(GOPATH)/bin/minio - @echo "Installation successful. To learn more, try \"minio --help\"." - -clean: - @echo "Cleaning up all the generated files" - @find . -name '*.test' | xargs rm -fv - @find . -name '*~' | xargs rm -fv - @rm -rvf minio - @rm -rvf build - @rm -rvf release - @rm -rvf .verify* +# Build current docker image +image-build: + @echo "⇒ Build docker-image" + @docker build \ + --build-arg VERSION=$(VERSION) \ + -f Dockerfile \ + -t nspccdev/neofs-s3-gate:$(VERSION) . \ No newline at end of file diff --git a/cmd/web-handlers.go b/cmd/web-handlers.go index 93af71097..ab768ee6d 100644 --- a/cmd/web-handlers.go +++ b/cmd/web-handlers.go @@ -37,7 +37,6 @@ import ( "github.com/klauspost/compress/zip" miniogopolicy "github.com/minio/minio-go/v6/pkg/policy" "github.com/minio/minio-go/v6/pkg/s3utils" - "github.com/minio/minio/browser" "github.com/minio/minio/cmd/config/etcd/dns" "github.com/minio/minio/cmd/config/identity/openid" "github.com/minio/minio/cmd/crypto" @@ -108,7 +107,7 @@ func (web *webAPIHandlers) ServerInfo(r *http.Request, args *WebGenericArgs, rep reply.MinioPlatform = platform reply.MinioRuntime = goruntime - reply.UIVersion = browser.UIVersion + reply.UIVersion = "browser.UIVersion" return nil } @@ -131,7 +130,7 @@ func (web *webAPIHandlers) StorageInfo(r *http.Request, args *WebGenericArgs, re } dataUsageInfo, _ := loadDataUsageFromBackend(ctx, objectAPI) reply.Used = dataUsageInfo.ObjectsTotalSize - reply.UIVersion = browser.UIVersion + reply.UIVersion = "browser.UIVersion" return nil } @@ -181,7 +180,7 @@ func (web *webAPIHandlers) MakeBucket(r *http.Request, args *MakeBucketArgs, rep return toJSONError(ctx, err) } - reply.UIVersion = browser.UIVersion + reply.UIVersion = "browser.UIVersion" return nil } return toJSONError(ctx, err) @@ -193,7 +192,7 @@ func (web *webAPIHandlers) MakeBucket(r *http.Request, args *MakeBucketArgs, rep return toJSONError(ctx, err, args.BucketName) } - reply.UIVersion = browser.UIVersion + reply.UIVersion = "browser.UIVersion" return nil } @@ -231,7 +230,7 @@ func (web *webAPIHandlers) DeleteBucket(r *http.Request, args *RemoveBucketArgs, return toJSONError(ctx, errInvalidBucketName) } - reply.UIVersion = browser.UIVersion + reply.UIVersion = "browser.UIVersion" if isRemoteCallRequired(ctx, args.BucketName, objectAPI) { sr, err := globalDNSConfig.Get(args.BucketName) @@ -351,7 +350,7 @@ func (web *webAPIHandlers) ListBuckets(r *http.Request, args *WebGenericArgs, re } } - reply.UIVersion = browser.UIVersion + reply.UIVersion = "browser.UIVersion" return nil } @@ -384,7 +383,7 @@ type WebObjectInfo struct { // ListObjects - list objects api. func (web *webAPIHandlers) ListObjects(r *http.Request, args *ListObjectsArgs, reply *ListObjectsRep) error { ctx := newWebContext(r, args, "WebListObjects") - reply.UIVersion = browser.UIVersion + reply.UIVersion = "browser.UIVersion" objectAPI := web.ObjectAPI() if objectAPI == nil { return toJSONError(ctx, errServerNotInitialized) @@ -622,7 +621,7 @@ func (web *webAPIHandlers) RemoveObject(r *http.Request, args *RemoveObjectArgs, return toJSONError(ctx, errInvalidBucketName) } - reply.UIVersion = browser.UIVersion + reply.UIVersion = "browser.UIVersion" if isRemoteCallRequired(ctx, args.BucketName, objectAPI) { sr, err := globalDNSConfig.Get(args.BucketName) if err != nil { @@ -813,7 +812,7 @@ func (web *webAPIHandlers) Login(r *http.Request, args *LoginArgs, reply *LoginR } reply.Token = token - reply.UIVersion = browser.UIVersion + reply.UIVersion = "browser.UIVersion" return nil } @@ -839,7 +838,7 @@ func (web webAPIHandlers) GenerateAuth(r *http.Request, args *WebGenericArgs, re } reply.AccessKey = cred.AccessKey reply.SecretKey = cred.SecretKey - reply.UIVersion = browser.UIVersion + reply.UIVersion = "browser.UIVersion" return nil } @@ -898,7 +897,7 @@ func (web *webAPIHandlers) SetAuth(r *http.Request, args *SetAuthArgs, reply *Se return toJSONError(ctx, err) } - reply.UIVersion = browser.UIVersion + reply.UIVersion = "browser.UIVersion" return nil } @@ -937,7 +936,7 @@ func (web *webAPIHandlers) CreateURLToken(r *http.Request, args *WebGenericArgs, reply.Token = token } - reply.UIVersion = browser.UIVersion + reply.UIVersion = "browser.UIVersion" return nil } @@ -1672,7 +1671,7 @@ func (web *webAPIHandlers) GetBucketPolicy(r *http.Request, args *GetBucketPolic } } - reply.UIVersion = browser.UIVersion + reply.UIVersion = "browser.UIVersion" reply.Policy = miniogopolicy.GetPolicy(policyInfo.Statements, args.BucketName, args.Prefix) return nil @@ -1764,7 +1763,7 @@ func (web *webAPIHandlers) ListAllBucketPolicies(r *http.Request, args *ListAllB } } - reply.UIVersion = browser.UIVersion + reply.UIVersion = "browser.UIVersion" for prefix, policy := range miniogopolicy.GetPolicies(policyInfo.Statements, args.BucketName, "") { bucketName, objectPrefix := path2BucketObject(prefix) objectPrefix = strings.TrimSuffix(objectPrefix, "*") @@ -1789,7 +1788,7 @@ type SetBucketPolicyWebArgs struct { func (web *webAPIHandlers) SetBucketPolicy(r *http.Request, args *SetBucketPolicyWebArgs, reply *WebGenericRep) error { ctx := newWebContext(r, args, "WebSetBucketPolicy") objectAPI := web.ObjectAPI() - reply.UIVersion = browser.UIVersion + reply.UIVersion = "browser.UIVersion" if objectAPI == nil { return toJSONError(ctx, errServerNotInitialized) @@ -1983,7 +1982,7 @@ func (web *webAPIHandlers) PresignedGet(r *http.Request, args *PresignedGetArgs, return toJSONError(ctx, errPresignedNotAllowed) } - reply.UIVersion = browser.UIVersion + reply.UIVersion = "browser.UIVersion" reply.URL = presignedGet(args.HostName, args.BucketName, args.ObjectName, args.Expiry, creds, region) return nil } @@ -2040,7 +2039,7 @@ func (web *webAPIHandlers) GetDiscoveryDoc(r *http.Request, args *WebGenericArgs reply.DiscoveryDoc = globalOpenIDConfig.DiscoveryDoc reply.ClientID = globalOpenIDConfig.ClientID } - reply.UIVersion = browser.UIVersion + reply.UIVersion = "browser.UIVersion" return nil } @@ -2099,7 +2098,7 @@ func (web *webAPIHandlers) LoginSTS(r *http.Request, args *LoginSTSArgs, reply * } reply.Token = a.Result.Credentials.SessionToken - reply.UIVersion = browser.UIVersion + reply.UIVersion = "browser.UIVersion" return nil } diff --git a/cmd/web-router.go b/cmd/web-router.go index 6e94246c4..02bd8221a 100644 --- a/cmd/web-router.go +++ b/cmd/web-router.go @@ -17,15 +17,16 @@ package cmd import ( + "errors" "fmt" "net/http" + "os" assetfs "github.com/elazarl/go-bindata-assetfs" "github.com/gorilla/handlers" "github.com/gorilla/mux" jsonrpc "github.com/gorilla/rpc/v2" "github.com/gorilla/rpc/v2/json2" - "github.com/minio/minio/browser" ) // webAPI container for Web API. @@ -48,9 +49,9 @@ const assetPrefix = "production" func assetFS() *assetfs.AssetFS { return &assetfs.AssetFS{ - Asset: browser.Asset, - AssetDir: browser.AssetDir, - AssetInfo: browser.AssetInfo, + Asset: func(path string) ([]byte, error) { return nil, errors.New("no ui") }, + AssetDir: func(path string) ([]string, error) { return nil, errors.New("no ui") }, + AssetInfo: func(path string) (os.FileInfo, error) { return nil, errors.New("no ui") }, Prefix: assetPrefix, } } diff --git a/go.mod b/go.mod index a5a67a9f0..164aebc68 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,6 @@ require ( github.com/Shopify/sarama v1.24.1 github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect github.com/alecthomas/participle v0.2.1 - github.com/aws/aws-sdk-go v1.20.21 github.com/bcicen/jstream v0.0.0-20190220045926-16c1f8af81c2 github.com/beevik/ntp v0.2.0 github.com/cespare/xxhash/v2 v2.1.1 @@ -88,7 +87,6 @@ require ( github.com/rs/cors v1.7.0 github.com/secure-io/sio-go v0.3.0 github.com/shirou/gopsutil v2.20.3-0.20200314133625-53cec6b37e6a+incompatible - github.com/sirupsen/logrus v1.5.0 github.com/smartystreets/assertions v0.0.0-20190401211740-f487f9de1cd3 // indirect github.com/spf13/viper v1.7.0 github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94 diff --git a/go.sum b/go.sum index 6b7a8662c..d5e445742 100644 --- a/go.sum +++ b/go.sum @@ -52,8 +52,6 @@ github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQh github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/awalterschulze/gographviz v0.0.0-20181013152038-b2885df04310 h1:t+qxRrRtwNiUYA+Xh2jSXhoG2grnMCMKX4Fg6lx9X1U= github.com/awalterschulze/gographviz v0.0.0-20181013152038-b2885df04310/go.mod h1:GEV5wmg4YquNw7v1kkyoX9etIk8yVmXj+AkDHuuETHs= -github.com/aws/aws-sdk-go v1.20.21 h1:22vHWL9rur+SRTYPHAXlxJMFIA9OSYsYDIAHFDhQ7Z0= -github.com/aws/aws-sdk-go v1.20.21/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/bcicen/jstream v0.0.0-20190220045926-16c1f8af81c2 h1:M+TYzBcNIRyzPRg66ndEqUMd7oWDmhvdQmaPC6EZNwM= github.com/bcicen/jstream v0.0.0-20190220045926-16c1f8af81c2/go.mod h1:RDu/qcrnpEdJC/p8tx34+YBFqqX71lB7dOX9QE+ZC4M= github.com/beevik/ntp v0.2.0 h1:sGsd+kAXzT0bfVfzJfce04g+dSRfrs+tbQW8lweuYgw= @@ -284,8 +282,6 @@ github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf/go.mod h github.com/jcmturner/gofork v0.0.0-20180107083740-2aebee971930/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= github.com/jcmturner/gofork v0.0.0-20190328161633-dc7c13fece03 h1:FUwcHNlEqkqLjLBdCp5PRlCFijNjvcYANOZXzCfXwCM= github.com/jcmturner/gofork v0.0.0-20190328161633-dc7c13fece03/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= -github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= -github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= diff --git a/main.go b/main.go index dbbe3ccb7..e66a243c5 100644 --- a/main.go +++ b/main.go @@ -25,14 +25,14 @@ package main // import "github.com/minio/minio" import ( - "os" - - minio "github.com/minio/minio/cmd" + "fmt" + "github.com/minio/minio/misc" // Import gateway - _ "github.com/minio/minio/cmd/gateway" + // _ "github.com/minio/minio/cmd/gateway" ) func main() { - minio.Main(os.Args) + fmt.Println(misc.Build) + fmt.Println(misc.Version) } diff --git a/mint/.gitignore b/mint/.gitignore deleted file mode 100644 index ef999ea53..000000000 --- a/mint/.gitignore +++ /dev/null @@ -1,17 +0,0 @@ -*.test -*.jar -src/* -temp -__pycache__/ -log/* -minio.test -bin/* -node_modules -# exception to the rule -!log/.gitkeep -!bin/.gitkeep -*.class -*~ -run/core/minio-dotnet/bin/* -run/core/minio-dotnet/obj/* -run/core/minio-dotnet/out/* diff --git a/mint/README.md b/mint/README.md deleted file mode 100644 index 4a33eec70..000000000 --- a/mint/README.md +++ /dev/null @@ -1,119 +0,0 @@ -# Mint [![Slack](https://slack.minio.io/slack?type=svg)](https://slack.minio.io) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/mint.svg?maxAge=604800)](https://hub.docker.com/r/minio/mint/) - -Mint is a testing framework for Minio object server, available as a docker image. It runs correctness, benchmarking and stress tests. Following are the SDKs/tools used in correctness tests. - -- awscli -- aws-sdk-go -- aws-sdk-php -- aws-sdk-ruby -- aws-sdk-java -- mc -- minio-go -- minio-java -- minio-js -- minio-py -- minio-dotnet -- s3cmd - -## Running Mint - -Mint is run by `docker run` command which requires Docker to be installed. For Docker installation follow the steps [here](https://docs.docker.com/engine/installation/linux/docker-ce/ubuntu/). - -To run Mint with Minio Play server as test target, - -```sh -$ docker run -e SERVER_ENDPOINT=play.minio.io:9000 -e ACCESS_KEY=Q3AM3UQ867SPQQA43P2F \ - -e SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG -e ENABLE_HTTPS=1 minio/mint -``` - -After the tests are run, output is stored in `/mint/log` directory inside the container. To get these logs, use `docker cp` command. For example -```sh -docker cp :/mint/log /tmp/logs -``` - -### Mint environment variables - -Below environment variables are required to be passed to the docker container. Supported environment variables: - -| Environment variable | Description | Example | -|:--- |:--- |:--- | -| `SERVER_ENDPOINT` | Endpoint of Minio server in the format `HOST:PORT`; for virtual style `IP:PORT` | `play.minio.io:9000` | -| `ACCESS_KEY` | Access key of access `SERVER_ENDPOINT` | `Q3AM3UQ867SPQQA43P2F` | -| `SECRET_KEY` | Secret Key of access `SERVER_ENDPOINT` | `zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG` | -| `ENABLE_HTTPS` | (Optional) Set `1` to indicate to use HTTPS to access `SERVER_ENDPOINT`. Defaults to `0` (HTTP) | `1` | -| `MINT_MODE` | (Optional) Set mode indicating what category of tests to be run by values `core`, `full`. Defaults to `core` | `full` | -| `DOMAIN` | (Optional) Value of MINIO_DOMAIN environment variable used in Minio server | `myminio.com` | -| `ENABLE_VIRTUAL_STYLE` | (Optional) Set `1` to indicate virtual style access . Defaults to `0` (Path style) | `1` | - - -### Test virtual style access against Minio server - -To test Minio server virtual style access with Mint, follow these steps: - -- Set a domain in your Minio server using environment variable MINIO_DOMAIN. For example `export MINIO_DOMAIN=myminio.com`. -- Start Minio server. -- Execute Mint against Minio server (with `MINIO_DOMAIN` set to `myminio.com`) using this command -```sh -$ docker run -e "SERVER_ENDPOINT=192.168.86.133:9000" -e "DOMAIN=minio.com" \ - -e "ACCESS_KEY=minio" -e "SECRET_KEY=minio123" -e "ENABLE_HTTPS=0" \ - -e "ENABLE_VIRTUAL_STYLE=1" minio/mint -``` - -### Mint log format - -All test logs are stored in `/mint/log/log.json` as multiple JSON document. Below is the JSON format for every entry in the log file. - -| JSON field | Type | Description | Example | -|:--- |:--- |:--- |:--- | -| `name` | _string_ | Testing tool/SDK name | `"aws-sdk-php"` | -| `function` | _string_ | Test function name | `"getBucketLocation ( array $params = [] )"` | -| `args` | _object_ | (Optional) Key/Value map of arguments passed to test function | `{"Bucket":"aws-sdk-php-bucket-20341"}` | -| `duration` | _int_ | Time taken in milliseconds to run the test | `384` | -| `status` | _string_ | one of `PASS`, `FAIL` or `NA` | `"PASS"` | -| `alert` | _string_ | (Optional) Alert message indicating test failure | `"I/O error on create file"` | -| `message` | _string_ | (Optional) Any log message | `"validating checksum of downloaded object"` | -| `error` | _string_ | Detailed error message including stack trace on status `FAIL` | `"Error executing \"CompleteMultipartUpload\" on ...` | - -## For Developers - -### Running Mint development code - -After making changes to Mint source code a local docker image can be built/run by - -```sh -$ docker build -t minio/mint . -f Dockerfile.mint -$ docker run -e SERVER_ENDPOINT=play.minio.io:9000 -e ACCESS_KEY=Q3AM3UQ867SPQQA43P2F \ - -e SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG \ - -e ENABLE_HTTPS=1 -e MINT_MODE=full minio/mint:latest -``` - - -### Adding tests with new tool/SDK - -Below are the steps need to be followed - -- Create new app directory under [build](https://github.com/minio/mint/tree/master/build) and [run/core](https://github.com/minio/mint/tree/master/run/core) directories. -- Create `install.sh` which does installation of required tool/SDK under app directory. -- Any build and install time dependencies should be added to [install-packages.list](https://github.com/minio/mint/blob/master/install-packages.list). -- Build time dependencies should be added to [remove-packages.list](https://github.com/minio/mint/blob/master/remove-packages.list) for removal to have clean Mint docker image. -- Add `run.sh` in app directory under `run/core` which execute actual tests. - -#### Test data -Tests may use pre-created data set to perform various object operations on Minio server. Below data files are available under `/mint/data` directory. - -| File name | Size | -|:--- |:--- | -| datafile-0-b | 0B | -| datafile-1-b | 1B | -| datafile-1-kB |1KiB | -| datafile-10-kB |10KiB | -| datafile-33-kB |33KiB | -| datafile-100-kB |100KiB | -| datafile-1-MB |1MiB | -| datafile-1.03-MB |1.03MiB | -| datafile-5-MB |5MiB | -| datafile-6-MB |6MiB | -| datafile-10-MB |10MiB | -| datafile-11-MB |11MiB | -| datafile-65-MB |65MiB | -| datafile-129-MB |129MiB | diff --git a/mint/build/aws-sdk-go/install.sh b/mint/build/aws-sdk-go/install.sh deleted file mode 100755 index d73162840..000000000 --- a/mint/build/aws-sdk-go/install.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash -e -# -# Mint (C) 2017 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -test_run_dir="$MINT_RUN_CORE_DIR/aws-sdk-go" -GO111MODULE=on go build -o "$test_run_dir/aws-sdk-go" "$test_run_dir/quick-tests.go" diff --git a/mint/build/aws-sdk-java/build.xml b/mint/build/aws-sdk-java/build.xml deleted file mode 100644 index 8a1f52146..000000000 --- a/mint/build/aws-sdk-java/build.xml +++ /dev/null @@ -1,58 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/mint/build/aws-sdk-java/install.sh b/mint/build/aws-sdk-java/install.sh deleted file mode 100755 index c3206d42a..000000000 --- a/mint/build/aws-sdk-java/install.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -e -# -# Mint (C) 2017 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -test_run_dir="$MINT_RUN_CORE_DIR/aws-sdk-java" - -cd "$(dirname "$(realpath "$0")")" - -ant init-ivy && \ - ant resolve && \ - ant compile && \ - ant jar - -cp build/jar/FunctionalTests.jar "$test_run_dir/" - -rm -rf lib/ build/ - diff --git a/mint/build/aws-sdk-java/ivy.xml b/mint/build/aws-sdk-java/ivy.xml deleted file mode 100644 index e61fd0d33..000000000 --- a/mint/build/aws-sdk-java/ivy.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - diff --git a/mint/build/aws-sdk-java/src/FunctionalTests.java b/mint/build/aws-sdk-java/src/FunctionalTests.java deleted file mode 100644 index 2005a7807..000000000 --- a/mint/build/aws-sdk-java/src/FunctionalTests.java +++ /dev/null @@ -1,632 +0,0 @@ -/* -* Mint, (C) 2018 Minio, Inc. -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - -package io.minio.awssdk.tests; - -import java.io.*; -import java.security.NoSuchAlgorithmException; -import java.security.SecureRandom; - -import java.security.*; -import java.util.*; - -import java.nio.file.*; -import java.math.BigInteger; - -import javax.crypto.KeyGenerator; -import javax.crypto.SecretKey; -import javax.crypto.spec.SecretKeySpec; - -import com.amazonaws.auth.AWSCredentials; -import com.amazonaws.auth.BasicAWSCredentials; -import com.amazonaws.services.s3.AmazonS3ClientBuilder; -import com.amazonaws.AmazonClientException; -import com.amazonaws.AmazonServiceException; -import com.amazonaws.auth.profile.ProfileCredentialsProvider; -import com.amazonaws.auth.AWSStaticCredentialsProvider; -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.AmazonS3Client; -import com.amazonaws.services.s3.model.CreateBucketRequest; -import com.amazonaws.services.s3.model.ObjectListing; -import com.amazonaws.services.s3.model.S3ObjectSummary; -import com.amazonaws.services.s3.model.SSECustomerKey; - -// Main Testing class -public class FunctionalTests { - - private static final String PASS = "PASS"; - private static final String FAILED = "FAIL"; - private static final String IGNORED = "NA"; - - private static String accessKey; - private static String secretKey; - private static String region; - private static String endpoint; - private static boolean enableHTTPS; - - private static final Random random = new Random(new SecureRandom().nextLong()); - private static String bucketName = getRandomName(); - private static boolean mintEnv = false; - - private static String file1Kb; - private static String file1Mb; - private static String file6Mb; - - private static SSECustomerKey sseKey1; - private static SSECustomerKey sseKey2; - private static SSECustomerKey sseKey3; - - private static AmazonS3 s3Client; - private static S3TestUtils s3TestUtils; - - public static String getRandomName() { - return "aws-java-sdk-test-" + new BigInteger(32, random).toString(32); - } - - /** - * Prints a success log entry in JSON format. - */ - public static void mintSuccessLog(String function, String args, long startTime) { - if (mintEnv) { - System.out.println( - new MintLogger(function, args, System.currentTimeMillis() - startTime, PASS, null, null, null)); - } - } - - /** - * Prints a failure log entry in JSON format. - */ - public static void mintFailedLog(String function, String args, long startTime, String message, String error) { - if (mintEnv) { - System.out.println(new MintLogger(function, args, System.currentTimeMillis() - startTime, FAILED, null, - message, error)); - } - } - - /** - * Prints a ignore log entry in JSON format. - */ - public static void mintIgnoredLog(String function, String args, long startTime) { - if (mintEnv) { - System.out.println( - new MintLogger(function, args, System.currentTimeMillis() - startTime, IGNORED, null, null, null)); - } - } - - public static void initTests() throws IOException { - // Create encryption key. - byte[] rawKey1 = "32byteslongsecretkeymustgenerate".getBytes(); - SecretKey secretKey1 = new SecretKeySpec(rawKey1, 0, rawKey1.length, "AES"); - sseKey1 = new SSECustomerKey(secretKey1); - - // Create new encryption key for target so it is saved using sse-c - byte[] rawKey2 = "xxbytescopysecretkeymustprovided".getBytes(); - SecretKey secretKey2 = new SecretKeySpec(rawKey2, 0, rawKey2.length, "AES"); - sseKey2 = new SSECustomerKey(secretKey2); - - // Create new encryption key for target so it is saved using sse-c - byte[] rawKey3 = "32byteslongsecretkeymustgenerat1".getBytes(); - SecretKey secretKey3 = new SecretKeySpec(rawKey3, 0, rawKey3.length, "AES"); - sseKey3 = new SSECustomerKey(secretKey3); - - // Create bucket - s3Client.createBucket(new CreateBucketRequest(bucketName)); - } - - public static void teardown() throws IOException { - - // Remove all objects under the test bucket & the bucket itself - // TODO: use multi delete API instead - ObjectListing objectListing = s3Client.listObjects(bucketName); - while (true) { - for (Iterator iterator = objectListing.getObjectSummaries().iterator(); iterator.hasNext();) { - S3ObjectSummary summary = (S3ObjectSummary) iterator.next(); - s3Client.deleteObject(bucketName, summary.getKey()); - } - // more objectListing to retrieve? - if (objectListing.isTruncated()) { - objectListing = s3Client.listNextBatchOfObjects(objectListing); - } else { - break; - } - } - ; - s3Client.deleteBucket(bucketName); - } - - // Test regular object upload using encryption - public static void uploadObjectEncryption_test1() throws Exception { - if (!mintEnv) { - System.out.println( - "Test: uploadObject(String bucketName, String objectName, String f, SSECustomerKey sseKey)"); - } - - if (!enableHTTPS) { - return; - } - - long startTime = System.currentTimeMillis(); - String file1KbMD5 = Utils.getFileMD5(file1Kb); - String objectName = "testobject"; - try { - s3TestUtils.uploadObject(bucketName, objectName, file1Kb, sseKey1); - s3TestUtils.downloadObject(bucketName, objectName, sseKey1, file1KbMD5); - mintSuccessLog("uploadObject(String bucketName, String objectName, String f, SSECustomerKey sseKey)", - "bucketName: " + bucketName + ", objectName: " + objectName + ", String: " + file1Kb - + ", SSECustomerKey: " + sseKey1, - startTime); - } catch (Exception e) { - mintFailedLog("uploadObject(String bucketName, String objectName, String f, SSECustomerKey sseKey)", - "bucketName: " + bucketName + ", objectName: " + objectName + ", String: " + file1Kb - + ", SSECustomerKey: " + sseKey1, - startTime, null, e.toString() + " >>> " + Arrays.toString(e.getStackTrace())); - throw e; - } - } - - // Test downloading an object with a wrong encryption key - public static void downloadObjectEncryption_test1() throws Exception { - if (!mintEnv) { - System.out.println("Test: downloadObject(String bucketName, String objectName, SSECustomerKey sseKey)"); - } - - if (!enableHTTPS) { - return; - } - - long startTime = System.currentTimeMillis(); - - String file1KbMD5 = Utils.getFileMD5(file1Kb); - String objectName = "testobject"; - - try { - s3TestUtils.uploadObject(bucketName, "testobject", file1Kb, sseKey1); - s3TestUtils.downloadObject(bucketName, objectName, sseKey2); - Exception ex = new Exception("downloadObject did not throw an S3 Access denied exception"); - mintFailedLog("downloadObject(String bucketName, String objectName, SSECustomerKey sseKey)", - "bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey2, - startTime, null, ex.toString() + " >>> " + Arrays.toString(ex.getStackTrace())); - throw ex; - } catch (Exception e) { - if (!e.getMessage().contains("Access Denied")) { - Exception ex = new Exception( - "downloadObject did not throw S3 Access denied Exception but it did throw: " + e.getMessage()); - mintFailedLog("downloadObject(String bucketName, String objectName, SSECustomerKey sseKey)", - "bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey2, - startTime, null, ex.toString() + " >>> " + Arrays.toString(ex.getStackTrace())); - throw ex; - } - mintSuccessLog("downloadObject(String bucketName, String objectName, SSECustomerKey sseKey)", - "bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey2, - startTime); - } - } - - // Test copying object with a new different encryption key - public static void copyObjectEncryption_test1() throws Exception { - if (!mintEnv) { - System.out.println("Test: copyObject(String bucketName, String objectName, SSECustomerKey sseKey, " - + "String destBucketName, String dstObjectName, SSECustomerKey sseKey2, boolean replaceDirective)"); - } - - if (!enableHTTPS) { - return; - } - - long startTime = System.currentTimeMillis(); - String file1KbMD5 = Utils.getFileMD5(file1Kb); - String objectName = "testobject"; - String dstObjectName = "dir/newobject"; - - try { - s3TestUtils.uploadObject(bucketName, objectName, file1Kb, sseKey1); - s3TestUtils.copyObject(bucketName, objectName, sseKey1, bucketName, dstObjectName, sseKey2, false); - s3TestUtils.downloadObject(bucketName, dstObjectName, sseKey2, file1KbMD5); - } catch (Exception e) { - mintFailedLog("copyObject(String bucketName, String objectName, SSECustomerKey sseKey, " - + "String destBucketName, String dstObjectName, SSECustomerKey sseKey2, boolean replaceDirective)", - "bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1 - + "DstbucketName: " + bucketName + ", DstObjectName: " + dstObjectName - + ", SSECustomerKey: " + sseKey2 + ", replaceDirective: " + false, - startTime, null, e.toString() + " >>> " + Arrays.toString(e.getStackTrace())); - throw e; - } - mintSuccessLog("copyObject(String bucketName, String objectName, SSECustomerKey sseKey, " - + "String destBucketName, String dstObjectName, SSECustomerKey sseKey2, boolean replaceDirective)", - "bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1 - + "DstbucketName: " + bucketName + ", DstObjectName: " + dstObjectName + ", SSECustomerKey: " - + sseKey2 + ", replaceDirective: " + false, - startTime); - } - - // Test copying object with wrong source encryption key - public static void copyObjectEncryption_test2() throws Exception { - if (!mintEnv) { - System.out.println("Test: copyObject(String bucketName, String objectName, SSECustomerKey sseKey, " - + "String destBucketName, String dstObjectName, SSECustomerKey sseKey2, boolean replaceDirective)"); - } - - if (!enableHTTPS) { - return; - } - - String objectName = "testobject"; - String dstObjectName = "dir/newobject"; - - long startTime = System.currentTimeMillis(); - - try { - s3TestUtils.copyObject(bucketName, objectName, sseKey3, bucketName, dstObjectName, sseKey2, false); - Exception ex = new Exception("copyObject did not throw an S3 Access denied exception"); - mintFailedLog("copyObject(String bucketName, String objectName, SSECustomerKey sseKey, " - + "String destBucketName, String dstObjectName, SSECustomerKey sseKey2, boolean replaceDirective)", - "bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey3 - + "DstbucketName: " + bucketName + ", DstObjectName: " + dstObjectName - + ", SSECustomerKey: " + sseKey2 + ", replaceDirective: " + false, - startTime, null, ex.toString() + " >>> " + Arrays.toString(ex.getStackTrace())); - throw ex; - } catch (Exception e) { - if (!e.getMessage().contains("Access Denied")) { - Exception ex = new Exception( - "copyObject did not throw S3 Access denied Exception but it did throw: " + e.getMessage()); - mintFailedLog("copyObject(String bucketName, String objectName, SSECustomerKey sseKey, " - + "String destBucketName, String dstObjectName, SSECustomerKey sseKey2, boolean replaceDirective)", - "bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey3 - + "DstbucketName: " + bucketName + ", DstObjectName: " + dstObjectName - + ", SSECustomerKey: " + sseKey2 + ", replaceDirective: " + false, - startTime, null, ex.toString() + " >>> " + Arrays.toString(ex.getStackTrace())); - throw ex; - } - mintSuccessLog("copyObject(String bucketName, String objectName, SSECustomerKey sseKey, " - + "String destBucketName, String dstObjectName, SSECustomerKey sseKey2, boolean replaceDirective)", - "bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey3 - + "DstbucketName: " + bucketName + ", DstObjectName: " + dstObjectName - + ", SSECustomerKey: " + sseKey2 + ", replaceDirective: " + false, - startTime); - } - } - - // Test copying multipart object - public static void copyObjectEncryption_test3() throws Exception { - if (!mintEnv) { - System.out.println("Test: copyObject(String bucketName, String objectName, SSECustomerKey sseKey, " - + "String destBucketName, String dstObjectName, SSECustomerKey sseKey2, boolean replaceDirective)"); - } - - if (!enableHTTPS) { - return; - } - - long startTime = System.currentTimeMillis(); - String file6MbMD5 = Utils.getFileMD5(file6Mb); - String objectName = "testobject"; - String dstObjectName = "dir/newobject"; - - try { - s3TestUtils.uploadMultipartObject(bucketName, objectName, file6Mb, sseKey1); - s3TestUtils.copyObject(bucketName, objectName, sseKey1, bucketName, dstObjectName, sseKey2, false); - s3TestUtils.downloadObject(bucketName, dstObjectName, sseKey2, file6MbMD5); - } catch (Exception e) { - mintFailedLog("copyObject(String bucketName, String objectName, SSECustomerKey sseKey, " - + "String destBucketName, String dstObjectName, SSECustomerKey sseKey2, boolean replaceDirective)", - "bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1 - + "DstbucketName: " + bucketName + ", DstObjectName: " + dstObjectName - + ", SSECustomerKey: " + sseKey2 + ", replaceDirective: " + false, - startTime, null, e.toString() + " >>> " + Arrays.toString(e.getStackTrace())); - throw e; - } - mintSuccessLog("copyObject(String bucketName, String objectName, SSECustomerKey sseKey, " - + "String destBucketName, String dstObjectName, SSECustomerKey sseKey2, boolean replaceDirective)", - "bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1 - + "DstbucketName: " + bucketName + ", DstObjectName: " + dstObjectName + ", SSECustomerKey: " - + sseKey2 + ", replaceDirective: " + false, - startTime); - } - - // Test downloading encrypted object with Get Range, 0 -> 1024 - public static void downloadGetRangeEncryption_test1() throws Exception { - if (!mintEnv) { - System.out.println("Test: downloadObjectGetRange(String bucketName, String objectName, " - + "SSECustomerKey sseKey, String expectedMD5, int start, int length)"); - } - - if (!enableHTTPS) { - return; - } - - long startTime = System.currentTimeMillis(); - - String objectName = "testobject"; - String range1MD5 = Utils.getFileMD5(file1Kb); - int start = 0; - int length = 1024; - try { - s3TestUtils.uploadObject(bucketName, objectName, file1Kb, sseKey1); - s3TestUtils.downloadObject(bucketName, objectName, sseKey1, range1MD5, start, length); - } catch (Exception e) { - mintFailedLog( - "downloadObjectGetRange(String bucketName, String objectName, " - + "SSECustomerKey sseKey, String expectedMD5, int start, int length)", - "bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1 - + ", expectedMD5: " + range1MD5 + ", start: " + start + ", length: " + length, - startTime, null, e.toString() + " >>> " + Arrays.toString(e.getStackTrace())); - throw e; - } - mintSuccessLog( - "downloadObjectGetRange(String bucketName, String objectName, " - + "SSECustomerKey sseKey, String expectedMD5, int start, int length)", - "bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1 - + ", expectedMD5: " + range1MD5 + ", start: " + start + ", length: " + length, - startTime); - } - - // Test downloading encrypted object with Get Range, 0 -> 1 - public static void downloadGetRangeEncryption_test2() throws Exception { - if (!mintEnv) { - System.out.println("Test: downloadObjectGetRange(String bucketName, String objectName, " - + "SSECustomerKey sseKey, String expectedMD5, int start, int length)"); - } - - if (!enableHTTPS) { - return; - } - - long startTime = System.currentTimeMillis(); - - String objectName = "testobject"; - int start = 0; - int length = 1; - String range1MD5 = Utils.getFileMD5(file1Kb, start, length); - try { - s3TestUtils.uploadObject(bucketName, objectName, file1Kb, sseKey1); - s3TestUtils.downloadObject(bucketName, objectName, sseKey1, range1MD5, start, length); - } catch (Exception e) { - mintFailedLog( - "downloadObjectGetRange(String bucketName, String objectName, " - + "SSECustomerKey sseKey, String expectedMD5, int start, int length)", - "bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1 - + ", expectedMD5: " + range1MD5 + ", start: " + start + ", length: " + length, - startTime, null, e.toString() + " >>> " + Arrays.toString(e.getStackTrace())); - throw e; - } - mintSuccessLog( - "downloadObjectGetRange(String bucketName, String objectName, " - + "SSECustomerKey sseKey, String expectedMD5, int start, int length)", - "bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1 - + ", expectedMD5: " + range1MD5 + ", start: " + start + ", length: " + length, - startTime); - } - - // Test downloading encrypted object with Get Range, 0 -> 1024-1 - public static void downloadGetRangeEncryption_test3() throws Exception { - if (!mintEnv) { - System.out.println("Test: downloadObjectGetRange(String bucketName, String objectName, " - + "SSECustomerKey sseKey, String expectedMD5, int start, int length)"); - } - - if (!enableHTTPS) { - return; - } - - long startTime = System.currentTimeMillis(); - - String objectName = "testobject"; - int start = 0; - int length = 1023; - String range1MD5 = Utils.getFileMD5(file1Kb, start, length); - try { - s3TestUtils.uploadObject(bucketName, objectName, file1Kb, sseKey1); - s3TestUtils.downloadObject(bucketName, objectName, sseKey1, range1MD5, start, length); - } catch (Exception e) { - mintFailedLog( - "downloadObjectGetRange(String bucketName, String objectName, " - + "SSECustomerKey sseKey, String expectedMD5, int start, int length)", - "bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1 - + ", expectedMD5: " + range1MD5 + ", start: " + start + ", length: " + length, - startTime, null, e.toString() + " >>> " + Arrays.toString(e.getStackTrace())); - throw e; - } - mintSuccessLog( - "downloadObjectGetRange(String bucketName, String objectName, " - + "SSECustomerKey sseKey, String expectedMD5, int start, int length)", - "bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1 - + ", expectedMD5: " + range1MD5 + ", start: " + start + ", length: " + length, - startTime); - } - - // Test downloading encrypted object with Get Range, 1 -> 1024-1 - public static void downloadGetRangeEncryption_test4() throws Exception { - if (!mintEnv) { - System.out.println("Test: downloadObjectGetRange(String bucketName, String objectName, " - + "SSECustomerKey sseKey, String expectedMD5, int start, int length)"); - } - - if (!enableHTTPS) { - return; - } - - long startTime = System.currentTimeMillis(); - - String objectName = "testobject"; - int start = 1; - int length = 1023; - String range1MD5 = Utils.getFileMD5(file1Kb, start, length); - try { - s3TestUtils.uploadObject(bucketName, objectName, file1Kb, sseKey1); - s3TestUtils.downloadObject(bucketName, objectName, sseKey1, range1MD5, start, length); - } catch (Exception e) { - mintFailedLog( - "downloadObjectGetRange(String bucketName, String objectName, " - + "SSECustomerKey sseKey, String expectedMD5, int start, int length)", - "bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1 - + ", expectedMD5: " + range1MD5 + ", start: " + start + ", length: " + length, - startTime, null, e.toString() + " >>> " + Arrays.toString(e.getStackTrace())); - throw e; - } - mintSuccessLog( - "downloadObjectGetRange(String bucketName, String objectName, " - + "SSECustomerKey sseKey, String expectedMD5, int start, int length)", - "bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1 - + ", expectedMD5: " + range1MD5 + ", start: " + start + ", length: " + length, - startTime); - } - - // Test downloading encrypted object with Get Range, 64*1024 -> 64*1024 - public static void downloadGetRangeEncryption_test5() throws Exception { - if (!mintEnv) { - System.out.println("Test: downloadObjectGetRange(String bucketName, String objectName, " - + "SSECustomerKey sseKey, String expectedMD5, int start, int length)"); - } - - if (!enableHTTPS) { - return; - } - - long startTime = System.currentTimeMillis(); - - String objectName = "testobject"; - int start = 64 * 1024; - int length = 64 * 1024; - String range1MD5 = Utils.getFileMD5(file1Mb, start, length); - try { - s3TestUtils.uploadObject(bucketName, objectName, file1Mb, sseKey1); - s3TestUtils.downloadObject(bucketName, objectName, sseKey1, range1MD5, start, length); - } catch (Exception e) { - mintFailedLog( - "downloadObjectGetRange(String bucketName, String objectName, " - + "SSECustomerKey sseKey, String expectedMD5, int start, int length)", - "bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1 - + ", expectedMD5: " + range1MD5 + ", start: " + start + ", length: " + length, - startTime, null, e.toString() + " >>> " + Arrays.toString(e.getStackTrace())); - throw e; - } - mintSuccessLog( - "downloadObjectGetRange(String bucketName, String objectName, " - + "SSECustomerKey sseKey, String expectedMD5, int start, int length)", - "bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1 - + ", expectedMD5: " + range1MD5 + ", start: " + start + ", length: " + length, - startTime); - } - - // Test downloading encrypted object with Get Range, 64*1024 -> - // 1024*1024-64*1024 - public static void downloadGetRangeEncryption_test6() throws Exception { - if (!mintEnv) { - System.out.println("Test: downloadObjectGetRange(String bucketName, String objectName, " - + "SSECustomerKey sseKey, String expectedMD5, int start, int length)"); - } - - if (!enableHTTPS) { - return; - } - - long startTime = System.currentTimeMillis(); - - String objectName = "testobject"; - int start = 64 * 1024; - int length = 1024 * 1024 - 64 * 1024; - String range1MD5 = Utils.getFileMD5(file1Mb, start, length); - try { - s3TestUtils.uploadObject(bucketName, objectName, file1Mb, sseKey1); - s3TestUtils.downloadObject(bucketName, objectName, sseKey1, range1MD5, start, length); - } catch (Exception e) { - mintFailedLog( - "downloadObjectGetRange(String bucketName, String objectName, " - + "SSECustomerKey sseKey, String expectedMD5, int start, int length)", - "bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1 - + ", expectedMD5: " + range1MD5 + ", start: " + start + ", length: " + length, - startTime, null, e.toString() + " >>> " + Arrays.toString(e.getStackTrace())); - throw e; - } - mintSuccessLog( - "downloadObjectGetRange(String bucketName, String objectName, " - + "SSECustomerKey sseKey, String expectedMD5, int start, int length)", - "bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1 - + ", expectedMD5: " + range1MD5 + ", start: " + start + ", length: " + length, - startTime); - } - - // Run tests - public static void runTests() throws Exception { - - uploadObjectEncryption_test1(); - - downloadObjectEncryption_test1(); - - copyObjectEncryption_test1(); - copyObjectEncryption_test2(); - copyObjectEncryption_test3(); - - downloadGetRangeEncryption_test1(); - downloadGetRangeEncryption_test2(); - downloadGetRangeEncryption_test3(); - downloadGetRangeEncryption_test4(); - downloadGetRangeEncryption_test5(); - downloadGetRangeEncryption_test6(); - } - - public static void main(String[] args) throws Exception, IOException, NoSuchAlgorithmException { - - endpoint = System.getenv("SERVER_ENDPOINT"); - accessKey = System.getenv("ACCESS_KEY"); - secretKey = System.getenv("SECRET_KEY"); - enableHTTPS = System.getenv("ENABLE_HTTPS").equals("1"); - - region = "us-east-1"; - - if (enableHTTPS) { - endpoint = "https://" + endpoint; - } else { - endpoint = "http://" + endpoint; - } - - String dataDir = System.getenv("MINT_DATA_DIR"); - if (dataDir != null && !dataDir.equals("")) { - mintEnv = true; - file1Kb = Paths.get(dataDir, "datafile-1-kB").toString(); - file1Mb = Paths.get(dataDir, "datafile-1-MB").toString(); - file6Mb = Paths.get(dataDir, "datafile-6-MB").toString(); - } - - String mintMode = null; - if (mintEnv) { - mintMode = System.getenv("MINT_MODE"); - } - - AWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey); - AmazonS3ClientBuilder.EndpointConfiguration endpointConfiguration = new AmazonS3ClientBuilder.EndpointConfiguration( - endpoint, region); - - AmazonS3ClientBuilder clientBuilder = AmazonS3ClientBuilder.standard(); - clientBuilder.setCredentials(new AWSStaticCredentialsProvider(credentials)); - clientBuilder.setEndpointConfiguration(endpointConfiguration); - clientBuilder.setPathStyleAccessEnabled(true); - - s3Client = clientBuilder.build(); - s3TestUtils = new S3TestUtils(s3Client); - - try { - initTests(); - FunctionalTests.runTests(); - } catch (Exception e) { - e.printStackTrace(); - System.exit(-1); - } finally { - teardown(); - } - } -} diff --git a/mint/build/aws-sdk-java/src/LimitedInputStream.java b/mint/build/aws-sdk-java/src/LimitedInputStream.java deleted file mode 100644 index 310a174e5..000000000 --- a/mint/build/aws-sdk-java/src/LimitedInputStream.java +++ /dev/null @@ -1,60 +0,0 @@ -/* -* Mint, (C) 2018 Minio, Inc. -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - -package io.minio.awssdk.tests; - -import java.io.*; - -// LimitedInputStream wraps a regular InputStream, calling -// read() will skip some bytes as configured and will also -// return only data with configured length - -class LimitedInputStream extends InputStream { - - private int skip; - private int length; - private InputStream is; - - LimitedInputStream(InputStream is, int skip, int length) { - this.is = is; - this.skip = skip; - this.length = length; - } - - @Override - public int read() throws IOException { - int r; - while (skip > 0) { - r = is.read(); - if (r < 0) { - throw new IOException("stream ended before being able to skip all bytes"); - } - skip--; - } - if (length == 0) { - return -1; - } - r = is.read(); - if (r < 0) { - throw new IOException("stream ended before being able to read all bytes"); - } - length--; - return r; - } -} - - diff --git a/mint/build/aws-sdk-java/src/MintLogger.java b/mint/build/aws-sdk-java/src/MintLogger.java deleted file mode 100755 index b6319f00a..000000000 --- a/mint/build/aws-sdk-java/src/MintLogger.java +++ /dev/null @@ -1,151 +0,0 @@ -/* - * Minio Java SDK for Amazon S3 Compatible Cloud Storage, - * (C) 2015, 2016, 2017, 2018 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.minio.awssdk.tests; - -import com.fasterxml.jackson.annotation.JsonAutoDetect; -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonInclude.Include; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; - -@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY) -public class MintLogger { - - @JsonProperty("name") - private String name; - - @JsonProperty("function") - private String function; - - @JsonProperty("args") - private String args; - - @JsonProperty("duration") - private long duration; - - @JsonProperty("status") - private String status; - - @JsonProperty("alert") - private String alert; - - @JsonProperty("message") - private String message; - - @JsonProperty("error") - private String error; - - /** - * Constructor. - **/ - public MintLogger(String function, - String args, - long duration, - String status, - String alert, - String message, - String error) { - this.name = "aws-sdk-java"; - this.function = function; - this.duration = duration; - this.args = args; - this.status = status; - this.alert = alert; - this.message = message; - this.error = error; - } - - /** - * Return JSON Log Entry. - **/ - @JsonIgnore - public String toString() { - - try { - return new ObjectMapper().setSerializationInclusion(Include.NON_NULL).writeValueAsString(this); - } catch (JsonProcessingException e) { - e.printStackTrace(); - } - return ""; - } - - /** - * Return Alert. - **/ - @JsonIgnore - public String alert() { - return alert; - } - - /** - * Return Error. - **/ - @JsonIgnore - public String error() { - return error; - } - - /** - * Return Message. - **/ - @JsonIgnore - public String message() { - return message; - } - - /** - * Return args. - **/ - @JsonIgnore - public String args() { - return args; - } - - /** - * Return status. - **/ - @JsonIgnore - public String status() { - return status; - } - - /** - * Return name. - **/ - @JsonIgnore - public String name() { - return name; - } - - /** - * Return function. - **/ - @JsonIgnore - public String function() { - return function; - } - - /** - * Return duration. - **/ - @JsonIgnore - public long duration() { - return duration; - } -} diff --git a/mint/build/aws-sdk-java/src/S3TestUtils.java b/mint/build/aws-sdk-java/src/S3TestUtils.java deleted file mode 100644 index 236caf938..000000000 --- a/mint/build/aws-sdk-java/src/S3TestUtils.java +++ /dev/null @@ -1,187 +0,0 @@ -/* -* Mint, (C) 2018 Minio, Inc. -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - -package io.minio.awssdk.tests; - -import java.io.*; -import java.util.*; -import java.nio.channels.Channels; - -import com.amazonaws.services.s3.model.GetObjectMetadataRequest; -import com.amazonaws.services.s3.model.GetObjectRequest; -import com.amazonaws.services.s3.model.ObjectMetadata; -import com.amazonaws.services.s3.model.PutObjectRequest; -import com.amazonaws.services.s3.model.CopyObjectRequest; -import com.amazonaws.services.s3.model.S3Object; -import com.amazonaws.services.s3.model.S3ObjectInputStream; -import com.amazonaws.services.s3.model.SSECustomerKey; - -import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest; -import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest; -import com.amazonaws.services.s3.model.InitiateMultipartUploadResult; -import com.amazonaws.services.s3.model.PartETag; -import com.amazonaws.services.s3.model.UploadPartRequest; - -import com.amazonaws.services.s3.model.MetadataDirective; - -import com.amazonaws.services.s3.AmazonS3; - -class S3TestUtils { - - private AmazonS3 s3Client; - - S3TestUtils(AmazonS3 s3Client) { - this.s3Client = s3Client; - } - - void uploadMultipartObject(String bucketName, String keyName, - String filePath, SSECustomerKey sseKey) throws IOException { - - File file = new File(filePath); - - List partETags = new ArrayList(); - - // Step 1: Initialize. - InitiateMultipartUploadRequest initRequest = new - InitiateMultipartUploadRequest(bucketName, keyName); - - if (sseKey != null) { - initRequest.setSSECustomerKey(sseKey); - } - - InitiateMultipartUploadResult initResponse = - s3Client.initiateMultipartUpload(initRequest); - - long contentLength = file.length(); - long partSize = 5242880; // Set part size to 5 MB. - - // Step 2: Upload parts. - long filePosition = 0; - for (int i = 1; filePosition < contentLength; i++) { - // Last part can be less than 5 MB. Adjust part size. - partSize = Math.min(partSize, (contentLength - filePosition)); - - // Create request to upload a part. - UploadPartRequest uploadRequest = new UploadPartRequest() - .withBucketName(bucketName).withKey(keyName) - .withUploadId(initResponse.getUploadId()).withPartNumber(i) - .withFileOffset(filePosition) - .withFile(file) - .withPartSize(partSize); - - if (sseKey != null) { - uploadRequest.withSSECustomerKey(sseKey); - } - - // Upload part and add response to our list. - partETags.add(s3Client.uploadPart(uploadRequest).getPartETag()); - - filePosition += partSize; - } - - // Step 3: Complete. - CompleteMultipartUploadRequest compRequest = new - CompleteMultipartUploadRequest( - bucketName, - keyName, - initResponse.getUploadId(), - partETags); - - s3Client.completeMultipartUpload(compRequest); - } - - void uploadObject(String bucketName, String keyName, - String filePath, SSECustomerKey sseKey) throws IOException { - - File f = new File(filePath); - PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, keyName, f); - if (sseKey != null) { - putObjectRequest.withSSECustomerKey(sseKey); - } - s3Client.putObject(putObjectRequest); - } - - void downloadObject(String bucketName, String keyName, SSECustomerKey sseKey) - throws Exception, IOException { - downloadObject(bucketName, keyName, sseKey, "", -1, -1); - } - - void downloadObject(String bucketName, String keyName, SSECustomerKey sseKey, - String expectedMD5) - throws Exception, IOException { - downloadObject(bucketName, keyName, sseKey, expectedMD5, -1, -1); - } - - void downloadObject(String bucketName, String keyName, SSECustomerKey sseKey, - String expectedMD5, int start, int length) throws Exception, IOException { - GetObjectRequest getObjectRequest = new GetObjectRequest(bucketName, keyName) - .withSSECustomerKey(sseKey); - - if (start >= 0 && length >= 0) { - getObjectRequest.setRange(start, start+length-1); - } - - S3Object s3Object = s3Client.getObject(getObjectRequest); - - int size = 0; - int c; - - S3ObjectInputStream input = s3Object.getObjectContent(); - - ByteArrayOutputStream output = new ByteArrayOutputStream(); - String data = ""; - while ((c = input.read()) != -1) { - output.write((byte) c); - size++; - } - - if (length >= 0 && size != length) { - throw new Exception("downloaded object has unexpected size, expected: " + length + ", received: " + size); - } - - String calculatedMD5 = Utils.getBufferMD5(output.toByteArray()); - - if (!expectedMD5.equals("") && !calculatedMD5.equals(expectedMD5)) { - throw new Exception("downloaded object has unexpected md5sum, expected: " + expectedMD5 + ", found: " + calculatedMD5); - - } - } - - void copyObject(String bucketName, String keyName, SSECustomerKey sseKey, - String targetBucketName, String targetKeyName, SSECustomerKey newSseKey, - boolean replace) { - CopyObjectRequest copyRequest = new CopyObjectRequest(bucketName, keyName, targetBucketName, targetKeyName); - if (sseKey != null) { - copyRequest.withSourceSSECustomerKey(sseKey); - } - if (newSseKey != null) { - copyRequest.withDestinationSSECustomerKey(newSseKey); - } - if (replace) { - copyRequest.withMetadataDirective(MetadataDirective.COPY); - } - s3Client.copyObject(copyRequest); - } - - long retrieveObjectMetadata(String bucketName, String keyName, SSECustomerKey sseKey) { - GetObjectMetadataRequest getMetadataRequest = new GetObjectMetadataRequest(bucketName, keyName) - .withSSECustomerKey(sseKey); - ObjectMetadata objectMetadata = s3Client.getObjectMetadata(getMetadataRequest); - return objectMetadata.getContentLength(); - } - -} diff --git a/mint/build/aws-sdk-java/src/Utils.java b/mint/build/aws-sdk-java/src/Utils.java deleted file mode 100644 index d6c971216..000000000 --- a/mint/build/aws-sdk-java/src/Utils.java +++ /dev/null @@ -1,76 +0,0 @@ -/* -* Mint, (C) 2018 Minio, Inc. -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - - -package io.minio.awssdk.tests; - -import java.io.*; -import java.nio.channels.*; -import java.security.*; - -class Utils { - - public static byte[] createChecksum(InputStream is, int skip, int length) throws Exception { - int numRead; - byte[] buffer = new byte[1024]; - - MessageDigest complete = MessageDigest.getInstance("MD5"); - - if (skip > -1 && length > -1) { - is = new LimitedInputStream(is, skip, length); - } - - do { - numRead = is.read(buffer); - if (numRead > 0) { - complete.update(buffer, 0, numRead); - } - } while (numRead != -1); - - return complete.digest(); - } - - public static String getInputStreamMD5(InputStream is) throws Exception { - return getInputStreamMD5(is, -1, -1); - } - - public static String getInputStreamMD5(InputStream is, int start, int length) throws Exception { - byte[] b = createChecksum(is, start, length); - String result = ""; - - for (int i=0; i < b.length; i++) { - result += Integer.toString( ( b[i] & 0xff ) + 0x100, 16).substring( 1 ); - } - return result; - } - - public static String getFileMD5(String filePath) throws Exception { - return getFileMD5(filePath, -1, -1); - } - - public static String getFileMD5(String filePath, int start, int length) throws Exception { - File f = new File(filePath); - InputStream is = new FileInputStream(f); - return getInputStreamMD5(is, start, length); - } - - public static String getBufferMD5(byte[] data) throws Exception { - ByteArrayInputStream bis = new ByteArrayInputStream(data); - return getInputStreamMD5(bis); - } -} - diff --git a/mint/build/aws-sdk-php/install.sh b/mint/build/aws-sdk-php/install.sh deleted file mode 100755 index 1363fa0dd..000000000 --- a/mint/build/aws-sdk-php/install.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -e -# -# Mint (C) 2017 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -test_run_dir="$MINT_RUN_CORE_DIR/aws-sdk-php" -$WGET --output-document=- https://getcomposer.org/installer | php -- --install-dir="$test_run_dir" -php "$test_run_dir/composer.phar" --working-dir="$test_run_dir" install diff --git a/mint/build/aws-sdk-ruby/install.sh b/mint/build/aws-sdk-ruby/install.sh deleted file mode 100755 index 7c7055f5a..000000000 --- a/mint/build/aws-sdk-ruby/install.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -e -# -# Mint (C) 20172-2020 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -gem install --no-rdoc --no-ri aws-sdk-resources aws-sdk multipart_body diff --git a/mint/build/awscli/install.sh b/mint/build/awscli/install.sh deleted file mode 100755 index cf7d9bd6a..000000000 --- a/mint/build/awscli/install.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -e -# -# Mint (C) 2017-2020 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -pip3 install awscli --upgrade diff --git a/mint/build/healthcheck/install.sh b/mint/build/healthcheck/install.sh deleted file mode 100755 index 71ffe4823..000000000 --- a/mint/build/healthcheck/install.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash -e -# -# Mint (C) 2019 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -test_run_dir="$MINT_RUN_CORE_DIR/healthcheck" -GO111MODULE=on go build -o "$test_run_dir/healthcheck" "$test_run_dir/healthcheck.go" diff --git a/mint/build/mc/install.sh b/mint/build/mc/install.sh deleted file mode 100755 index a41aaebdd..000000000 --- a/mint/build/mc/install.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -e -# -# Mint (C) 2017 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -MC_VERSION=$(curl --retry 10 -Ls -o /dev/null -w "%{url_effective}" https://github.com/minio/mc/releases/latest | sed "s/https:\/\/github.com\/minio\/mc\/releases\/tag\///") -if [ -z "$MC_VERSION" ]; then - echo "unable to get mc version from github" - exit 1 -fi - -test_run_dir="$MINT_RUN_CORE_DIR/mc" -$WGET --output-document="${test_run_dir}/mc" "https://dl.minio.io/client/mc/release/linux-amd64/mc.${MC_VERSION}" -chmod a+x "${test_run_dir}/mc" - -git clone --quiet https://github.com/minio/mc.git "$test_run_dir/mc.git" -(cd "$test_run_dir/mc.git"; git checkout --quiet "tags/${MC_VERSION}") -cp -a "${test_run_dir}/mc.git/functional-tests.sh" "$test_run_dir/" -rm -fr "$test_run_dir/mc.git" diff --git a/mint/build/minio-dotnet/install.sh b/mint/build/minio-dotnet/install.sh deleted file mode 100755 index fc00e8592..000000000 --- a/mint/build/minio-dotnet/install.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash -# -# Mint (C) 2017-2020 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -set -e - -MINIO_DOTNET_SDK_PATH="$MINT_RUN_CORE_DIR/minio-dotnet" - -MINIO_DOTNET_SDK_VERSION=$(curl --retry 10 -Ls -o /dev/null -w "%{url_effective}" https://github.com/minio/minio-dotnet/releases/latest | sed "s/https:\/\/github.com\/minio\/minio-dotnet\/releases\/tag\///") -if [ -z "$MINIO_DOTNET_SDK_VERSION" ]; then - echo "unable to get minio-dotnet version from github" - exit 1 -fi - -out_dir="$MINIO_DOTNET_SDK_PATH/out" -if [ -z "$out_dir" ]; then - mkdir "$out_dir" -fi - -temp_dir="$MINIO_DOTNET_SDK_PATH/temp" -git clone --quiet https://github.com/minio/minio-dotnet.git "${temp_dir}/minio-dotnet.git/" -(cd "${temp_dir}/minio-dotnet.git"; git checkout --quiet "tags/${MINIO_DOTNET_SDK_VERSION}") - -cp -a "${temp_dir}/minio-dotnet.git/Minio.Functional.Tests/"* "${MINIO_DOTNET_SDK_PATH}/" -rm -fr "${temp_dir}" - -cd "$MINIO_DOTNET_SDK_PATH" -dotnet restore /p:Configuration=Mint -dotnet publish --runtime ubuntu.18.04-x64 --output out /p:Configuration=Mint diff --git a/mint/build/minio-go/install.sh b/mint/build/minio-go/install.sh deleted file mode 100755 index 5e3005199..000000000 --- a/mint/build/minio-go/install.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash -e -# -# Mint (C) 2017 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -MINIO_GO_VERSION=$(curl --retry 10 -Ls -o /dev/null -w "%{url_effective}" https://github.com/minio/minio-go/releases/latest | sed "s/https:\/\/github.com\/minio\/minio-go\/releases\/tag\///") -if [ -z "$MINIO_GO_VERSION" ]; then - echo "unable to get minio-go version from github" - exit 1 -fi - -test_run_dir="$MINT_RUN_CORE_DIR/minio-go" -(git clone https://github.com/minio/minio-go && cd minio-go && git checkout --quiet "tags/$MINIO_GO_VERSION") -GO111MODULE=on CGO_ENABLED=0 go build -o "$test_run_dir/minio-go" "minio-go/functional_tests.go" -rm -rf minio-go diff --git a/mint/build/minio-java/install.sh b/mint/build/minio-java/install.sh deleted file mode 100755 index 399934d8c..000000000 --- a/mint/build/minio-java/install.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -e -# -# Mint (C) 2017 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -MINIO_JAVA_VERSION=$(curl --retry 10 -s "https://repo1.maven.org/maven2/io/minio/minio/maven-metadata.xml" | sed -n "//{s/<.[^>]*>//g;p;q}" | sed "s/ *//g") -if [ -z "$MINIO_JAVA_VERSION" ]; then - echo "unable to get latest minio-java version from maven" - exit 1 -fi - -test_run_dir="$MINT_RUN_CORE_DIR/minio-java" -git clone --quiet https://github.com/minio/minio-java.git "$test_run_dir/minio-java.git" -(cd "$test_run_dir/minio-java.git"; git checkout --quiet "tags/${MINIO_JAVA_VERSION}") -$WGET --output-document="$test_run_dir/minio-${MINIO_JAVA_VERSION}-all.jar" "https://repo1.maven.org/maven2/io/minio/minio/${MINIO_JAVA_VERSION}/minio-${MINIO_JAVA_VERSION}-all.jar" -javac -cp "$test_run_dir/minio-${MINIO_JAVA_VERSION}-all.jar" "${test_run_dir}/minio-java.git/functional"/*.java -cp -a "${test_run_dir}/minio-java.git/functional"/*.class "$test_run_dir/" -rm -fr "$test_run_dir/minio-java.git" diff --git a/mint/build/minio-js/install.sh b/mint/build/minio-js/install.sh deleted file mode 100755 index 0b9c4f452..000000000 --- a/mint/build/minio-js/install.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -e -# -# Mint (C) 2017 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -MINIO_JS_VERSION=$(curl --retry 10 -Ls -o /dev/null -w "%{url_effective}" https://github.com/minio/minio-js/releases/latest | sed "s/https:\/\/github.com\/minio\/minio-js\/releases\/tag\///") -if [ -z "$MINIO_JS_VERSION" ]; then - echo "unable to get minio-js version from github" - exit 1 -fi - -test_run_dir="$MINT_RUN_CORE_DIR/minio-js" -mkdir "${test_run_dir}/test" -$WGET --output-document="${test_run_dir}/test/functional-tests.js" "https://raw.githubusercontent.com/minio/minio-js/${MINIO_JS_VERSION}/src/test/functional/functional-tests.js" -npm --prefix "$test_run_dir" install --save "minio@$MINIO_JS_VERSION" -npm --prefix "$test_run_dir" install diff --git a/mint/build/minio-py/install.sh b/mint/build/minio-py/install.sh deleted file mode 100755 index 39bd990e7..000000000 --- a/mint/build/minio-py/install.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash -e -# -# Mint (C) 2017-2020 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -MINIO_PY_VERSION=$(curl --retry 10 -Ls -o /dev/null -w "%{url_effective}" https://github.com/minio/minio-py/releases/latest | sed "s/https:\/\/github.com\/minio\/minio-py\/releases\/tag\///") -if [ -z "$MINIO_PY_VERSION" ]; then - echo "unable to get minio-py version from github" - exit 1 -fi - -test_run_dir="$MINT_RUN_CORE_DIR/minio-py" -pip3 install --user faker -pip3 install minio=="$MINIO_PY_VERSION" -$WGET --output-document="$test_run_dir/tests.py" "https://raw.githubusercontent.com/minio/minio-py/${MINIO_PY_VERSION}/tests/functional/tests.py" diff --git a/mint/build/s3cmd/install.sh b/mint/build/s3cmd/install.sh deleted file mode 100755 index 6865930c4..000000000 --- a/mint/build/s3cmd/install.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash -e -# -# Mint (C) 2017 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Always install the latest. -python -m pip install s3cmd diff --git a/mint/build/s3select/install.sh b/mint/build/s3select/install.sh deleted file mode 100755 index aae2af2f6..000000000 --- a/mint/build/s3select/install.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -e -# -# Mint (C) 2020 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -python -m pip install minio diff --git a/mint/build/security/install.sh b/mint/build/security/install.sh deleted file mode 100755 index a517bfb18..000000000 --- a/mint/build/security/install.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash -e -# -# Mint (C) 2018 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -test_run_dir="$MINT_RUN_CORE_DIR/security" -GO111MODULE=on go build -o "$test_run_dir/tls-tests" "$test_run_dir/tls-tests.go" diff --git a/mint/create-data-files.sh b/mint/create-data-files.sh deleted file mode 100755 index ba0ce43c3..000000000 --- a/mint/create-data-files.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash -e -# -# Mint (C) 2017 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -MINT_DATA_DIR="$MINT_ROOT_DIR/data" - -declare -A data_file_map -data_file_map["datafile-0-b"]="0" -data_file_map["datafile-1-b"]="1" -data_file_map["datafile-1-kB"]="1K" -data_file_map["datafile-10-kB"]="10K" -data_file_map["datafile-33-kB"]="33K" -data_file_map["datafile-100-kB"]="100K" -data_file_map["datafile-1.03-MB"]="1056K" -data_file_map["datafile-1-MB"]="1M" -data_file_map["datafile-5-MB"]="5M" -data_file_map["datafile-5243880-b"]="5243880" -data_file_map["datafile-6-MB"]="6M" -data_file_map["datafile-10-MB"]="10M" -data_file_map["datafile-11-MB"]="11M" -data_file_map["datafile-65-MB"]="65M" -data_file_map["datafile-129-MB"]="129M" - -mkdir -p "$MINT_DATA_DIR" -for filename in "${!data_file_map[@]}"; do - echo "creating $MINT_DATA_DIR/$filename" - if ! shred -n 1 -s "${data_file_map[$filename]}" - 1>"$MINT_DATA_DIR/$filename" 2>/dev/null; then - echo "unable to create data file $MINT_DATA_DIR/$filename" - exit 1 - fi -done diff --git a/mint/entrypoint.sh b/mint/entrypoint.sh deleted file mode 100755 index 5e15f6cb7..000000000 --- a/mint/entrypoint.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -# -# Mint (C) 2017 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -./mint.sh "$@" & - -# Get the pid to be used for kill command if required -main_pid="$!" -trap 'echo -e "\nAborting Mint..."; kill $main_pid' SIGINT SIGTERM -# use -n here to catch mint.sh exit code, notify to ci -wait -n diff --git a/mint/install-packages.list b/mint/install-packages.list deleted file mode 100644 index 600e36faf..000000000 --- a/mint/install-packages.list +++ /dev/null @@ -1,18 +0,0 @@ -git -python3-pip -nodejs -openjdk-8-jdk -openjdk-8-jre -dirmngr -apt-transport-https -dotnet-sdk-2.1 -ca-certificates-mono -nuget -libunwind8 -ruby -ruby-dev -ruby-bundler -php -php-curl -php-xml -ant diff --git a/mint/mint.sh b/mint/mint.sh deleted file mode 100755 index 56d44b1cf..000000000 --- a/mint/mint.sh +++ /dev/null @@ -1,190 +0,0 @@ -#!/bin/bash -# -# Mint (C) 2017, 2018 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -CONTAINER_ID=$(grep -o -e '[0-f]\{12,\}' /proc/1/cpuset | awk '{print substr($1, 1, 12)}') -MINT_DATA_DIR=${MINT_DATA_DIR:-/mint/data} -MINT_MODE=${MINT_MODE:-core} -SERVER_REGION=${SERVER_REGION:-us-east-1} -ENABLE_HTTPS=${ENABLE_HTTPS:-0} -ENABLE_VIRTUAL_STYLE=${ENABLE_VIRTUAL_STYLE:-0} -GO111MODULE=on - -if [ -z "$SERVER_ENDPOINT" ]; then - SERVER_ENDPOINT="play.minio.io:9000" - ACCESS_KEY="Q3AM3UQ867SPQQA43P2F" - SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" - ENABLE_HTTPS=1 -fi - -if [ "$ENABLE_VIRTUAL_STYLE" -eq 1 ]; then - SERVER_IP="${SERVER_ENDPOINT%%:*}" - SERVER_PORT="${SERVER_ENDPOINT/*:/}" - # Check if SERVER_IP is actually IPv4 address - octets=("${SERVER_IP//./ }") - if [ "${#octets[@]}" -ne 4 ]; then - echo "$SERVER_IP must be an IP address" - exit 1 - fi - for octet in "${octets[@]}"; do - if [ "$octet" -lt 0 ] 2>/dev/null || [ "$octet" -gt 255 ] 2>/dev/null; then - echo "$SERVER_IP must be an IP address" - exit 1 - fi - done -fi - -ROOT_DIR="$PWD" -TESTS_DIR="$ROOT_DIR/run/core" - -BASE_LOG_DIR="$ROOT_DIR/log" -LOG_FILE="log.json" -ERROR_FILE="error.log" -mkdir -p "$BASE_LOG_DIR" - -function humanize_time() -{ - time="$1" - days=$(( time / 60 / 60 / 24 )) - hours=$(( time / 60 / 60 % 24 )) - minutes=$(( time / 60 % 60 )) - seconds=$(( time % 60 )) - - (( days > 0 )) && echo -n "$days days " - (( hours > 0 )) && echo -n "$hours hours " - (( minutes > 0 )) && echo -n "$minutes minutes " - (( days > 0 || hours > 0 || minutes > 0 )) && echo -n "and " - echo "$seconds seconds" -} - -function run_test() -{ - if [ ! -d "$1" ]; then - return 1 - fi - - start=$(date +%s) - - mkdir -p "$BASE_LOG_DIR/$sdk_name" - - (cd "$sdk_dir" && ./run.sh "$BASE_LOG_DIR/$LOG_FILE" "$BASE_LOG_DIR/$sdk_name/$ERROR_FILE") - rv=$? - end=$(date +%s) - duration=$(humanize_time $(( end - start ))) - - if [ "$rv" -eq 0 ]; then - echo "done in $duration" - else - echo "FAILED in $duration" - entry=$(tail -n 1 "$BASE_LOG_DIR/$LOG_FILE") - status=$(jq -e -r .status <<<"$entry") - jq_rv=$? - if [ "$jq_rv" -ne 0 ]; then - echo "$entry" - fi - ## Show error.log when status is empty or not "FAIL". - ## This may happen when test run failed without providing logs. - if [ "$jq_rv" -ne 0 ] || [ -z "$status" ] || ([ "$status" != "FAIL" ] && [ "$status" != "fail" ]); then - cat "$BASE_LOG_DIR/$sdk_name/$ERROR_FILE" - else - jq . <<<"$entry" - fi - fi - return $rv -} - -function trust_s3_endpoint_tls_cert() -{ - # Download the public certificate from the server - openssl s_client -showcerts -connect "$SERVER_ENDPOINT" /dev/null | \ - openssl x509 -outform PEM -out /usr/local/share/ca-certificates/s3_server_cert.crt || \ - exit 1 - - # Load the certificate in the system - update-ca-certificates --fresh >/dev/null - - # Ask different SDKs/tools to load system certificates - export REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt - export NODE_EXTRA_CA_CERTS=/etc/ssl/certs/ca-certificates.crt - export SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt -} - - -function main() -{ - export MINT_DATA_DIR - export MINT_MODE - export SERVER_ENDPOINT - export SERVER_IP - export SERVER_PORT - - export ACCESS_KEY - export SECRET_KEY - export ENABLE_HTTPS - export SERVER_REGION - export ENABLE_VIRTUAL_STYLE - export GO111MODULE - - echo "Running with" - echo "SERVER_ENDPOINT: $SERVER_ENDPOINT" - echo "ACCESS_KEY: $ACCESS_KEY" - echo "SECRET_KEY: ***REDACTED***" - echo "ENABLE_HTTPS: $ENABLE_HTTPS" - echo "SERVER_REGION: $SERVER_REGION" - echo "MINT_DATA_DIR: $MINT_DATA_DIR" - echo "MINT_MODE: $MINT_MODE" - echo "ENABLE_VIRTUAL_STYLE: $ENABLE_VIRTUAL_STYLE" - echo - echo "To get logs, run 'docker cp ${CONTAINER_ID}:/mint/log /tmp/mint-logs'" - echo - - [ "$ENABLE_HTTPS" == "1" ] && trust_s3_endpoint_tls_cert - - declare -a run_list - sdks=( "$@" ) - - if [ "$#" -eq 0 ]; then - sdks=( $(ls "$TESTS_DIR") ) - fi - - for sdk in "${sdks[@]}"; do - run_list=( "${run_list[@]}" "$TESTS_DIR/$sdk" ) - done - - count="${#run_list[@]}" - i=0 - for sdk_dir in "${run_list[@]}"; do - sdk_name=$(basename "$sdk_dir") - (( i++ )) - if [ ! -d "$sdk_dir" ]; then - echo "Test $sdk_name not found. Exiting Mint." - exit 1 - fi - echo -n "($i/$count) Running $sdk_name tests ... " - if ! run_test "$sdk_dir"; then - (( i-- )) - fi - done - - ## Report when all tests in run_list are run - if [ $i -eq "$count" ]; then - echo -e "\nAll tests ran successfully" - else - echo -e "\nExecuted $i out of $count tests successfully." - exit 1 - fi -} -main "$@" diff --git a/mint/postinstall.sh b/mint/postinstall.sh deleted file mode 100755 index 72e357dc5..000000000 --- a/mint/postinstall.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -e -# -# Mint (C) 2017 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -export APT="apt --quiet --yes" - -# remove all packages listed in remove-packages.list -xargs --arg-file="${MINT_ROOT_DIR}/remove-packages.list" apt --quiet --yes purge -${APT} autoremove - -# remove unwanted files -rm -fr "$GOROOT" "$GOPATH/src" /var/lib/apt/lists/* - -# flush to disk -sync diff --git a/mint/preinstall.sh b/mint/preinstall.sh deleted file mode 100755 index 44d7a0b8f..000000000 --- a/mint/preinstall.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/bin/bash -e -# -# Mint (C) 2017-2020 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -export APT="apt --quiet --yes" -export WGET="wget --quiet --no-check-certificate" - -# install nodejs source list -if ! $WGET --output-document=- https://deb.nodesource.com/setup_13.x | bash -; then - echo "unable to set nodejs repository" - exit 1 -fi - -$APT install apt-transport-https - -if ! $WGET --output-document=packages-microsoft-prod.deb https://packages.microsoft.com/config/ubuntu/18.04/packages-microsoft-prod.deb | bash -; then - echo "unable to download dotnet packages" - exit 1 -fi - -dpkg -i packages-microsoft-prod.deb -rm -f packages-microsoft-prod.deb - -$APT update -$APT install gnupg ca-certificates - -# download and install golang -GO_VERSION="1.13.10" -GO_INSTALL_PATH="/usr/local" -download_url="https://storage.googleapis.com/golang/go${GO_VERSION}.linux-amd64.tar.gz" -if ! $WGET --output-document=- "$download_url" | tar -C "${GO_INSTALL_PATH}" -zxf -; then - echo "unable to install go$GO_VERSION" - exit 1 -fi - -xargs --arg-file="${MINT_ROOT_DIR}/install-packages.list" apt --quiet --yes install - -nuget update -self - -# set python 3.6 as default -update-alternatives --install /usr/bin/python python /usr/bin/python3.6 1 - -sync diff --git a/mint/release.sh b/mint/release.sh deleted file mode 100755 index 8ee2f7ec4..000000000 --- a/mint/release.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash -e -# -# Minio Cloud Storage, (C) 2017 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -export MINT_ROOT_DIR=${MINT_ROOT_DIR:-/mint} -export MINT_RUN_CORE_DIR="$MINT_ROOT_DIR/run/core" -export MINT_RUN_SECURITY_DIR="$MINT_ROOT_DIR/run/security" -export WGET="wget --quiet --no-check-certificate" - -"${MINT_ROOT_DIR}"/create-data-files.sh -"${MINT_ROOT_DIR}"/preinstall.sh - -# install mint app packages -for pkg in "$MINT_ROOT_DIR/build"/*/install.sh; do - echo "Running $pkg" - $pkg -done - -"${MINT_ROOT_DIR}"/postinstall.sh diff --git a/mint/remove-packages.list b/mint/remove-packages.list deleted file mode 100644 index 0206fbc6d..000000000 --- a/mint/remove-packages.list +++ /dev/null @@ -1,9 +0,0 @@ -wget -git -python3-pip -ruby-dev -ruby-bundler -openjdk-8-jdk -ant -dotnet -nuget diff --git a/mint/run/core/aws-sdk-go/quick-tests.go b/mint/run/core/aws-sdk-go/quick-tests.go deleted file mode 100644 index e9f4a887d..000000000 --- a/mint/run/core/aws-sdk-go/quick-tests.go +++ /dev/null @@ -1,1066 +0,0 @@ -/* -* -* Mint, (C) 2017 Minio, Inc. -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software - -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -* - */ - -package main - -import ( - "bytes" - "encoding/json" - "encoding/xml" - "errors" - "fmt" - "math/rand" - "net/http" - "os" - "reflect" - "strings" - "time" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/awserr" - "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/s3" - log "github.com/sirupsen/logrus" -) - -const letterBytes = "abcdefghijklmnopqrstuvwxyz01234569" -const ( - letterIdxBits = 6 // 6 bits to represent a letter index - letterIdxMask = 1<= 0; { - if remain == 0 { - cache, remain = src.Int63(), letterIdxMax - } - if idx := int(cache & letterIdxMask); idx < len(letterBytes) { - b[i] = letterBytes[idx] - i-- - } - cache >>= letterIdxBits - remain-- - } - return prefix + string(b[0:30-len(prefix)]) -} - -func isObjectTaggingImplemented(s3Client *s3.S3) bool { - bucket := randString(60, rand.NewSource(time.Now().UnixNano()), "aws-sdk-go-test-") - object := randString(60, rand.NewSource(time.Now().UnixNano()), "") - startTime := time.Now() - function := "isObjectTaggingImplemented" - args := map[string]interface{}{ - "bucketName": bucket, - "objectName": object, - } - defer cleanup(s3Client, bucket, object, function, args, startTime, true) - - _, err := s3Client.CreateBucket(&s3.CreateBucketInput{ - Bucket: aws.String(bucket), - }) - if err != nil { - failureLog(function, args, startTime, "", "AWS SDK Go CreateBucket Failed", err).Fatal() - return false - } - - _, err = s3Client.PutObject(&s3.PutObjectInput{ - Body: aws.ReadSeekCloser(strings.NewReader("testfile")), - Bucket: aws.String(bucket), - Key: aws.String(object), - }) - - if err != nil { - failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go PUT expected to success but got %v", err), err).Fatal() - return false - } - - _, err = s3Client.GetObjectTagging(&s3.GetObjectTaggingInput{ - Bucket: aws.String(bucket), - Key: aws.String(object), - }) - if err != nil { - if awsErr, ok := err.(awserr.Error); ok { - if awsErr.Code() == "NotImplemented" { - return false - } - } - } - return true -} - -func cleanup(s3Client *s3.S3, bucket string, object string, function string, - args map[string]interface{}, startTime time.Time, deleteBucket bool) { - - // Deleting the object, just in case it was created. Will not check for errors. - s3Client.DeleteObject(&s3.DeleteObjectInput{ - Bucket: aws.String(bucket), - Key: aws.String(object), - }) - - if deleteBucket { - _, err := s3Client.DeleteBucket(&s3.DeleteBucketInput{ - Bucket: aws.String(bucket), - }) - if err != nil { - failureLog(function, args, startTime, "", "AWS SDK Go DeleteBucket Failed", err).Fatal() - return - } - } - -} - -func testPresignedPutInvalidHash(s3Client *s3.S3) { - startTime := time.Now() - function := "PresignedPut" - bucket := randString(60, rand.NewSource(time.Now().UnixNano()), "aws-sdk-go-test-") - object := "presignedTest" - expiry := 1 * time.Minute - args := map[string]interface{}{ - "bucketName": bucket, - "objectName": object, - "expiry": expiry, - } - - _, err := s3Client.CreateBucket(&s3.CreateBucketInput{ - Bucket: aws.String(bucket), - }) - if err != nil { - failureLog(function, args, startTime, "", "AWS SDK Go CreateBucket Failed", err).Fatal() - return - } - defer cleanup(s3Client, bucket, object, function, args, startTime, true) - - req, _ := s3Client.PutObjectRequest(&s3.PutObjectInput{ - Bucket: aws.String(bucket), - Key: aws.String(object), - ContentType: aws.String("application/octet-stream"), - }) - - req.HTTPRequest.Header.Set("X-Amz-Content-Sha256", "invalid-sha256") - url, err := req.Presign(expiry) - if err != nil { - failureLog(function, args, startTime, "", "AWS SDK Go presigned Put request creation failed", err).Fatal() - return - } - - rreq, err := http.NewRequest("PUT", url, bytes.NewReader([]byte(""))) - if err != nil { - failureLog(function, args, startTime, "", "AWS SDK Go presigned PUT request failed", err).Fatal() - return - } - - rreq.Header.Add("X-Amz-Content-Sha256", "invalid-sha256") - rreq.Header.Add("Content-Type", "application/octet-stream") - - resp, err := http.DefaultClient.Do(rreq) - if err != nil { - failureLog(function, args, startTime, "", "AWS SDK Go presigned put request failed", err).Fatal() - return - } - defer resp.Body.Close() - - dec := xml.NewDecoder(resp.Body) - errResp := errorResponse{} - err = dec.Decode(&errResp) - if err != nil { - failureLog(function, args, startTime, "", "AWS SDK Go unmarshalling xml failed", err).Fatal() - return - } - - if errResp.Code != "XAmzContentSHA256Mismatch" { - failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go presigned PUT expected to fail with XAmzContentSHA256Mismatch but got %v", errResp.Code), errors.New("AWS S3 error code mismatch")).Fatal() - return - } - - successLogger(function, args, startTime).Info() -} - -func testListObjects(s3Client *s3.S3) { - startTime := time.Now() - function := "testListObjects" - bucket := randString(60, rand.NewSource(time.Now().UnixNano()), "aws-sdk-go-test-") - object1 := "testObject1" - object2 := "testObject2" - expiry := 1 * time.Minute - args := map[string]interface{}{ - "bucketName": bucket, - "objectName1": object1, - "objectName2": object2, - "expiry": expiry, - } - - getKeys := func(objects []*s3.Object) []string { - var rv []string - for _, obj := range objects { - rv = append(rv, *obj.Key) - } - return rv - } - _, err := s3Client.CreateBucket(&s3.CreateBucketInput{ - Bucket: aws.String(bucket), - }) - if err != nil { - failureLog(function, args, startTime, "", "AWS SDK Go CreateBucket Failed", err).Fatal() - return - } - defer cleanup(s3Client, bucket, object1, function, args, startTime, true) - defer cleanup(s3Client, bucket, object2, function, args, startTime, false) - - listInput := &s3.ListObjectsV2Input{ - Bucket: aws.String(bucket), - MaxKeys: aws.Int64(1000), - Prefix: aws.String(""), - } - result, err := s3Client.ListObjectsV2(listInput) - if err != nil { - failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go listobjects expected to success but got %v", err), err).Fatal() - return - } - if *result.KeyCount != 0 { - failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go listobjects with prefix '' expected 0 key but got %v, %v", result.KeyCount, getKeys(result.Contents)), errors.New("AWS S3 key count mismatch")).Fatal() - return - } - putInput1 := &s3.PutObjectInput{ - Body: aws.ReadSeekCloser(strings.NewReader("filetoupload")), - Bucket: aws.String(bucket), - Key: aws.String(object1), - } - _, err = s3Client.PutObject(putInput1) - if err != nil { - failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go PUT expected to success but got %v", err), err).Fatal() - return - } - putInput2 := &s3.PutObjectInput{ - Body: aws.ReadSeekCloser(strings.NewReader("filetoupload")), - Bucket: aws.String(bucket), - Key: aws.String(object2), - } - _, err = s3Client.PutObject(putInput2) - if err != nil { - failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go PUT expected to success but got %v", err), err).Fatal() - return - } - result, err = s3Client.ListObjectsV2(listInput) - if err != nil { - failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go listobjects expected to success but got %v", err), err).Fatal() - return - } - if *result.KeyCount != 2 { - failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go listobjects with prefix '' expected 2 key but got %v, %v", *result.KeyCount, getKeys(result.Contents)), errors.New("AWS S3 key count mismatch")).Fatal() - return - } - - successLogger(function, args, startTime).Info() -} - -func testSelectObject(s3Client *s3.S3) { - startTime := time.Now() - function := "testSelectObject" - bucket := randString(60, rand.NewSource(time.Now().UnixNano()), "aws-sdk-go-test-") - object1 := "object1.csv" - object2 := "object2.csv" - args := map[string]interface{}{ - "bucketName": bucket, - "objectName1": object1, - "objectName2": object2, - } - - _, err := s3Client.CreateBucket(&s3.CreateBucketInput{ - Bucket: aws.String(bucket), - }) - if err != nil { - failureLog(function, args, startTime, "", "AWS SDK Go CreateBucket Failed", err).Fatal() - return - } - - // Test comma field separator - inputCsv1 := `year,gender,ethnicity,firstname,count,rank -2011,FEMALE,ASIAN AND PACIFIC ISLANDER,SOPHIA,119,1 -2011,FEMALE,ASIAN AND PACIFIC ISLANDER,CHLOE,106,2 -2011,FEMALE,ASIAN AND PACIFIC ISLANDER,EMILY,93,3 -2011,FEMALE,ASIAN AND PACIFIC ISLANDER,OLIVIA,89,4 -2011,FEMALE,ASIAN AND PACIFIC ISLANDER,EMMA,75,5 -2011,FEMALE,ASIAN AND PACIFIC ISLANDER,ISABELLA,67,6 -2011,FEMALE,ASIAN AND PACIFIC ISLANDER,TIFFANY,54,7 -2011,FEMALE,ASIAN AND PACIFIC ISLANDER,ASHLEY,52,8 -2011,FEMALE,ASIAN AND PACIFIC ISLANDER,FIONA,48,9 -2011,FEMALE,ASIAN AND PACIFIC ISLANDER,ANGELA,47,10 -` - - outputCSV1 := `2011 -2011 -2011 -2011 -2011 -2011 -2011 -2011 -2011 -2011 -` - - putInput1 := &s3.PutObjectInput{ - Body: aws.ReadSeekCloser(strings.NewReader(inputCsv1)), - Bucket: aws.String(bucket), - Key: aws.String(object1), - } - _, err = s3Client.PutObject(putInput1) - if err != nil { - failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go Select object failed %v", err), err).Fatal() - return - } - - defer cleanup(s3Client, bucket, object1, function, args, startTime, true) - - params := &s3.SelectObjectContentInput{ - Bucket: &bucket, - Key: &object1, - ExpressionType: aws.String(s3.ExpressionTypeSql), - Expression: aws.String("SELECT s._1 FROM S3Object s"), - RequestProgress: &s3.RequestProgress{}, - InputSerialization: &s3.InputSerialization{ - CompressionType: aws.String("NONE"), - CSV: &s3.CSVInput{ - FileHeaderInfo: aws.String(s3.FileHeaderInfoIgnore), - FieldDelimiter: aws.String(","), - RecordDelimiter: aws.String("\n"), - }, - }, - OutputSerialization: &s3.OutputSerialization{ - CSV: &s3.CSVOutput{}, - }, - } - - resp, err := s3Client.SelectObjectContent(params) - if err != nil { - failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go Select object failed %v", err), err).Fatal() - return - } - defer resp.EventStream.Close() - - payload := "" - for event := range resp.EventStream.Events() { - switch v := event.(type) { - case *s3.RecordsEvent: - // s3.RecordsEvent.Records is a byte slice of select records - payload = string(v.Payload) - } - } - - if err := resp.EventStream.Err(); err != nil { - failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go Select object failed %v", err), err).Fatal() - return - } - - if payload != outputCSV1 { - failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go Select object output mismatch %v", payload), errors.New("AWS S3 select object mismatch")).Fatal() - return - } - - // Test unicode field separator - inputCsv2 := `"year"╦"gender"╦"ethnicity"╦"firstname"╦"count"╦"rank" -"2011"╦"FEMALE"╦"ASIAN AND PACIFIC ISLANDER"╦"SOPHIA"╦"119"╦"1" -"2011"╦"FEMALE"╦"ASIAN AND PACIFIC ISLANDER"╦"CHLOE"╦"106"╦"2" -"2011"╦"FEMALE"╦"ASIAN AND PACIFIC ISLANDER"╦"EMILY"╦"93"╦"3" -"2011"╦"FEMALE"╦"ASIAN AND PACIFIC ISLANDER"╦"OLIVIA"╦"89"╦"4" -"2011"╦"FEMALE"╦"ASIAN AND PACIFIC ISLANDER"╦"EMMA"╦"75"╦"5" -"2011"╦"FEMALE"╦"ASIAN AND PACIFIC ISLANDER"╦"ISABELLA"╦"67"╦"6" -"2011"╦"FEMALE"╦"ASIAN AND PACIFIC ISLANDER"╦"TIFFANY"╦"54"╦"7" -"2011"╦"FEMALE"╦"ASIAN AND PACIFIC ISLANDER"╦"ASHLEY"╦"52"╦"8" -"2011"╦"FEMALE"╦"ASIAN AND PACIFIC ISLANDER"╦"FIONA"╦"48"╦"9" -"2011"╦"FEMALE"╦"ASIAN AND PACIFIC ISLANDER"╦"ANGELA"╦"47"╦"10" -` - - outputCSV2 := `2011 -2011 -2011 -2011 -2011 -2011 -2011 -2011 -2011 -2011 -` - - putInput2 := &s3.PutObjectInput{ - Body: aws.ReadSeekCloser(strings.NewReader(inputCsv2)), - Bucket: aws.String(bucket), - Key: aws.String(object2), - } - _, err = s3Client.PutObject(putInput2) - if err != nil { - failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go Select object upload failed: %v", err), err).Fatal() - return - } - - defer cleanup(s3Client, bucket, object2, function, args, startTime, false) - - params2 := &s3.SelectObjectContentInput{ - Bucket: &bucket, - Key: &object2, - ExpressionType: aws.String(s3.ExpressionTypeSql), - Expression: aws.String("SELECT s._1 FROM S3Object s"), - RequestProgress: &s3.RequestProgress{}, - InputSerialization: &s3.InputSerialization{ - CompressionType: aws.String("NONE"), - CSV: &s3.CSVInput{ - FileHeaderInfo: aws.String(s3.FileHeaderInfoIgnore), - FieldDelimiter: aws.String("╦"), - RecordDelimiter: aws.String("\n"), - }, - }, - OutputSerialization: &s3.OutputSerialization{ - CSV: &s3.CSVOutput{}, - }, - } - - resp, err = s3Client.SelectObjectContent(params2) - if err != nil { - failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go Select object failed for unicode separator %v", err), err).Fatal() - return - } - defer resp.EventStream.Close() - - for event := range resp.EventStream.Events() { - switch v := event.(type) { - case *s3.RecordsEvent: - // s3.RecordsEvent.Records is a byte slice of select records - payload = string(v.Payload) - } - } - - if err := resp.EventStream.Err(); err != nil { - failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go Select object failed for unicode separator %v", err), err).Fatal() - return - } - - if payload != outputCSV2 { - failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go Select object output mismatch %v", payload), errors.New("AWS S3 select object mismatch")).Fatal() - return - } - - successLogger(function, args, startTime).Info() -} - -func testObjectTagging(s3Client *s3.S3) { - startTime := time.Now() - function := "testObjectTagging" - bucket := randString(60, rand.NewSource(time.Now().UnixNano()), "aws-sdk-go-test-") - object := randString(60, rand.NewSource(time.Now().UnixNano()), "") - args := map[string]interface{}{ - "bucketName": bucket, - "objectName": object, - } - - _, err := s3Client.CreateBucket(&s3.CreateBucketInput{ - Bucket: aws.String(bucket), - }) - if err != nil { - failureLog(function, args, startTime, "", "AWS SDK Go CreateBucket Failed", err).Fatal() - return - } - defer cleanup(s3Client, bucket, object, function, args, startTime, true) - - taginput := "Tag1=Value1" - tagInputSet := []*s3.Tag{ - { - Key: aws.String("Tag1"), - Value: aws.String("Value1"), - }, - } - _, err = s3Client.PutObject(&s3.PutObjectInput{ - Body: aws.ReadSeekCloser(strings.NewReader("testfile")), - Bucket: aws.String(bucket), - Key: aws.String(object), - Tagging: &taginput, - }) - - if err != nil { - failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go PUT expected to success but got %v", err), err).Fatal() - return - } - - tagop, err := s3Client.GetObjectTagging(&s3.GetObjectTaggingInput{ - Bucket: aws.String(bucket), - Key: aws.String(object), - }) - if err != nil { - if awsErr, ok := err.(awserr.Error); ok { - failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go PUTObjectTagging expected to success but got %v", awsErr.Code()), err).Fatal() - return - } - } - if !reflect.DeepEqual(tagop.TagSet, tagInputSet) { - failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go PUTObject Tag input did not match with GetObjectTagging output %v", nil), nil).Fatal() - return - } - - taginputSet1 := []*s3.Tag{ - { - Key: aws.String("Key4"), - Value: aws.String("Value4"), - }, - } - _, err = s3Client.PutObjectTagging(&s3.PutObjectTaggingInput{ - Bucket: aws.String(bucket), - Key: aws.String(object), - Tagging: &s3.Tagging{ - TagSet: taginputSet1, - }, - }) - if err != nil { - if awsErr, ok := err.(awserr.Error); ok { - failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go PUTObjectTagging expected to success but got %v", awsErr.Code()), err).Fatal() - return - } - } - - tagop, err = s3Client.GetObjectTagging(&s3.GetObjectTaggingInput{ - Bucket: aws.String(bucket), - Key: aws.String(object), - }) - if err != nil { - if awsErr, ok := err.(awserr.Error); ok { - failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go PUTObjectTagging expected to success but got %v", awsErr.Code()), err).Fatal() - return - } - } - if !reflect.DeepEqual(tagop.TagSet, taginputSet1) { - failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go PUTObjectTagging input did not match with GetObjectTagging output %v", nil), nil).Fatal() - return - } - successLogger(function, args, startTime).Info() -} - -func testObjectTaggingErrors(s3Client *s3.S3) { - startTime := time.Now() - function := "testObjectTaggingErrors" - bucket := randString(60, rand.NewSource(time.Now().UnixNano()), "aws-sdk-go-test-") - object := randString(60, rand.NewSource(time.Now().UnixNano()), "") - args := map[string]interface{}{ - "bucketName": bucket, - "objectName": object, - } - - _, err := s3Client.CreateBucket(&s3.CreateBucketInput{ - Bucket: aws.String(bucket), - }) - if err != nil { - failureLog(function, args, startTime, "", "AWS SDK Go CreateBucket Failed", err).Fatal() - return - } - defer cleanup(s3Client, bucket, object, function, args, startTime, true) - - _, err = s3Client.PutObject(&s3.PutObjectInput{ - Body: aws.ReadSeekCloser(strings.NewReader("testfile")), - Bucket: aws.String(bucket), - Key: aws.String(object), - }) - - if err != nil { - failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go PUT expected to success but got %v", err), err).Fatal() - return - } - - // case 1 : Too many tags > 10 - input := &s3.PutObjectTaggingInput{ - Bucket: aws.String(bucket), - Key: aws.String(object), - Tagging: &s3.Tagging{ - TagSet: []*s3.Tag{ - { - Key: aws.String("Key1"), - Value: aws.String("Value3"), - }, - { - Key: aws.String("Key2"), - Value: aws.String("Value4"), - }, - { - Key: aws.String("Key3"), - Value: aws.String("Value3"), - }, - { - Key: aws.String("Key4"), - Value: aws.String("Value3"), - }, - { - Key: aws.String("Key5"), - Value: aws.String("Value3"), - }, - { - Key: aws.String("Key6"), - Value: aws.String("Value3"), - }, - { - Key: aws.String("Key7"), - Value: aws.String("Value3"), - }, - { - Key: aws.String("Key8"), - Value: aws.String("Value3"), - }, - { - Key: aws.String("Key9"), - Value: aws.String("Value3"), - }, - { - Key: aws.String("Key10"), - Value: aws.String("Value3"), - }, - { - Key: aws.String("Key11"), - Value: aws.String("Value3"), - }, - }, - }, - } - - _, err = s3Client.PutObjectTagging(input) - if err == nil { - failureLog(function, args, startTime, "", "AWS SDK Go PUT expected to fail but succeeded", err).Fatal() - return - } - - if aerr, ok := err.(awserr.Error); ok { - if aerr.Code() != "BadRequest" && aerr.Message() != "BadRequest: Object tags cannot be greater than 10" { - failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go PUT expected to fail but got %v", err), err).Fatal() - return - } - } - - // case 2 : Duplicate Tag Keys - input = &s3.PutObjectTaggingInput{ - Bucket: aws.String(bucket), - Key: aws.String(object), - Tagging: &s3.Tagging{ - TagSet: []*s3.Tag{ - { - Key: aws.String("Key1"), - Value: aws.String("Value3"), - }, - { - Key: aws.String("Key1"), - Value: aws.String("Value4"), - }, - }, - }, - } - - _, err = s3Client.PutObjectTagging(input) - if err == nil { - failureLog(function, args, startTime, "", "AWS SDK Go PUT expected to fail but succeeded", err).Fatal() - return - } - - if aerr, ok := err.(awserr.Error); ok { - if aerr.Code() != "InvalidTag" && aerr.Message() != "InvalidTag: Cannot provide multiple Tags with the same key" { - failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go PUT expected to fail but got %v", err), err).Fatal() - return - } - } - - // case 3 : Too long Tag Key - input = &s3.PutObjectTaggingInput{ - Bucket: aws.String(bucket), - Key: aws.String(object), - Tagging: &s3.Tagging{ - TagSet: []*s3.Tag{ - { - Key: aws.String("Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1"), - Value: aws.String("Value3"), - }, - { - Key: aws.String("Key1"), - Value: aws.String("Value4"), - }, - }, - }, - } - - _, err = s3Client.PutObjectTagging(input) - if err == nil { - failureLog(function, args, startTime, "", "AWS SDK Go PUT expected to fail but succeeded", err).Fatal() - return - } - - if aerr, ok := err.(awserr.Error); ok { - if aerr.Code() != "InvalidTag" && aerr.Message() != "InvalidTag: The TagKey you have provided is invalid" { - failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go PUT expected to fail but got %v", err), err).Fatal() - return - } - } - - // case 4 : Too long Tag value - input = &s3.PutObjectTaggingInput{ - Bucket: aws.String(bucket), - Key: aws.String(object), - Tagging: &s3.Tagging{ - TagSet: []*s3.Tag{ - { - Key: aws.String("Key1"), - Value: aws.String("Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1Key1"), - }, - { - Key: aws.String("Key1"), - Value: aws.String("Value4"), - }, - }, - }, - } - - _, err = s3Client.PutObjectTagging(input) - if err == nil { - failureLog(function, args, startTime, "", "AWS SDK Go PUT expected to fail but succeeded", err).Fatal() - return - } - - if aerr, ok := err.(awserr.Error); ok { - if aerr.Code() != "InvalidTag" && aerr.Message() != "InvalidTag: The TagValue you have provided is invalid" { - failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go PUT expected to fail but got %v", err), err).Fatal() - return - } - } - - successLogger(function, args, startTime).Info() -} - -// Tests bucket re-create errors. -func testCreateBucketError(s3Client *s3.S3) { - region := s3Client.Config.Region - // Amazon S3 returns error in all AWS Regions except in the North Virginia Region. - // More details in https://docs.aws.amazon.com/sdk-for-go/api/service/s3/#S3.CreateBucket - s3Client.Config.Region = aws.String("us-west-1") - - // initialize logging params - startTime := time.Now() - function := "testMakeBucketError" - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "aws-sdk-go-test-") - args := map[string]interface{}{ - "bucketName": bucketName, - } - _, err := s3Client.CreateBucket(&s3.CreateBucketInput{ - Bucket: aws.String(bucketName), - }) - if err != nil { - failureLog(function, args, startTime, "", "AWS SDK Go CreateBucket Failed", err).Fatal() - return - } - defer cleanup(s3Client, bucketName, "", function, args, startTime, true) - - _, errCreating := s3Client.CreateBucket(&s3.CreateBucketInput{ - Bucket: aws.String(bucketName), - }) - if errCreating == nil { - failureLog(function, args, startTime, "", "AWS SDK Go CreateBucket Should Return Error for Existing bucket", err).Fatal() - return - } - // Verify valid error response from server. - if errCreating.(s3.RequestFailure).Code() != "BucketAlreadyExists" && - errCreating.(s3.RequestFailure).Code() != "BucketAlreadyOwnedByYou" { - failureLog(function, args, startTime, "", "Invalid error returned by server", err).Fatal() - return - } - - // Restore region in s3Client - s3Client.Config.Region = region - successLogger(function, args, startTime).Info() -} - -func testListMultipartUploads(s3Client *s3.S3) { - startTime := time.Now() - function := "testListMultipartUploads" - bucket := randString(60, rand.NewSource(time.Now().UnixNano()), "aws-sdk-go-test-") - object := randString(60, rand.NewSource(time.Now().UnixNano()), "") - args := map[string]interface{}{ - "bucketName": bucket, - "objectName": object, - } - _, errCreating := s3Client.CreateBucket(&s3.CreateBucketInput{ - Bucket: aws.String(bucket), - }) - if errCreating != nil { - failureLog(function, args, startTime, "", "AWS SDK Go CreateBucket Failed", errCreating).Fatal() - return - } - defer cleanup(s3Client, bucket, object, function, args, startTime, true) - - multipartUpload, err := s3Client.CreateMultipartUpload(&s3.CreateMultipartUploadInput{ - Bucket: aws.String(bucket), - Key: aws.String(object), - }) - if err != nil { - failureLog(function, args, startTime, "", "AWS SDK Go createMultipartupload API failed", err).Fatal() - return - } - parts := make(map[*int64]*string) - for i := 0; i < 5; i++ { - result, errUpload := s3Client.UploadPart(&s3.UploadPartInput{ - Bucket: aws.String(bucket), - Key: aws.String(object), - UploadId: multipartUpload.UploadId, - PartNumber: aws.Int64(int64(i + 1)), - Body: aws.ReadSeekCloser(strings.NewReader("fileToUpload")), - }) - if errUpload != nil { - _, _ = s3Client.AbortMultipartUpload(&s3.AbortMultipartUploadInput{ - Bucket: aws.String(bucket), - Key: aws.String(object), - UploadId: multipartUpload.UploadId, - }) - failureLog(function, args, startTime, "", "AWS SDK Go uploadPart API failed for", errUpload).Fatal() - return - } - parts[aws.Int64(int64(i+1))] = result.ETag - } - - listParts, errParts := s3Client.ListParts(&s3.ListPartsInput{ - Bucket: aws.String(bucket), - Key: aws.String(object), - UploadId: multipartUpload.UploadId, - }) - if errParts != nil { - failureLog(function, args, startTime, "", "AWS SDK Go ListPartsInput API failed for", err).Fatal() - return - } - - if len(parts) != len(listParts.Parts) { - failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go ListParts.Parts len mismatch want: %v got: %v", len(parts), len(listParts.Parts)), err).Fatal() - return - } - - for _, part := range listParts.Parts { - if tag, ok := parts[part.PartNumber]; ok { - if tag != part.ETag { - failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go ListParts.Parts output mismatch want: %v got: %v", tag, part.ETag), err).Fatal() - return - } - } - } - - successLogger(function, args, startTime).Info() -} - -func testSSECopyObject(s3Client *s3.S3) { - // initialize logging params - startTime := time.Now() - function := "testSSECopyObjectSourceEncrypted" - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "aws-sdk-go-test-") - object := randString(60, rand.NewSource(time.Now().UnixNano()), "") - args := map[string]interface{}{ - "bucketName": bucketName, - "objectName": object, - } - _, err := s3Client.CreateBucket(&s3.CreateBucketInput{ - Bucket: aws.String(bucketName), - }) - if err != nil { - failureLog(function, args, startTime, "", "AWS SDK Go CreateBucket Failed", err).Fatal() - return - } - defer cleanup(s3Client, bucketName, object+"-enc", function, args, startTime, true) - defer cleanup(s3Client, bucketName, object+"-noenc", function, args, startTime, false) - var wrongSuccess = errors.New("Succeeded instead of failing. ") - - // create encrypted object - sseCustomerKey := aws.String("32byteslongsecretkeymustbegiven2") - _, errPutEnc := s3Client.PutObject(&s3.PutObjectInput{ - Body: aws.ReadSeekCloser(strings.NewReader("fileToUpload")), - Bucket: aws.String(bucketName), - Key: aws.String(object + "-enc"), - SSECustomerAlgorithm: aws.String("AES256"), - SSECustomerKey: sseCustomerKey, - }) - if errPutEnc != nil { - failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go PUT expected to succeed but got %v", errPutEnc), errPutEnc).Fatal() - return - } - - // copy the encrypted object - _, errCopyEnc := s3Client.CopyObject(&s3.CopyObjectInput{ - SSECustomerAlgorithm: aws.String("AES256"), - SSECustomerKey: sseCustomerKey, - CopySource: aws.String(bucketName + "/" + object + "-enc"), - Bucket: aws.String(bucketName), - Key: aws.String(object + "-copy"), - }) - if errCopyEnc == nil { - failureLog(function, args, startTime, "", "AWS SDK Go CopyObject expected to fail, but it succeeds ", wrongSuccess).Fatal() - return - } - var invalidSSECustomerAlgorithm = "Requests specifying Server Side Encryption with Customer provided keys must provide a valid encryption algorithm" - if !strings.Contains(errCopyEnc.Error(), invalidSSECustomerAlgorithm) { - failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go CopyObject expected error %v got %v", invalidSSECustomerAlgorithm, errCopyEnc), errCopyEnc).Fatal() - return - } - - // create non-encrypted object - _, errPut := s3Client.PutObject(&s3.PutObjectInput{ - Body: aws.ReadSeekCloser(strings.NewReader("fileToUpload")), - Bucket: aws.String(bucketName), - Key: aws.String(object + "-noenc"), - }) - if errPut != nil { - failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go PUT expected to succeed but got %v", errPut), errPut).Fatal() - return - } - - // copy the non-encrypted object - _, errCopy := s3Client.CopyObject(&s3.CopyObjectInput{ - CopySourceSSECustomerAlgorithm: aws.String("AES256"), - CopySourceSSECustomerKey: sseCustomerKey, - SSECustomerAlgorithm: aws.String("AES256"), - SSECustomerKey: sseCustomerKey, - CopySource: aws.String(bucketName + "/" + object + "-noenc"), - Bucket: aws.String(bucketName), - Key: aws.String(object + "-copy"), - }) - if errCopy == nil { - failureLog(function, args, startTime, "", "AWS SDK Go CopyObject expected to fail, but it succeeds ", wrongSuccess).Fatal() - return - } - var invalidEncryptionParameters = "The encryption parameters are not applicable to this object." - if !strings.Contains(errCopy.Error(), invalidEncryptionParameters) { - failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go CopyObject expected error %v got %v", invalidEncryptionParameters, errCopy), errCopy).Fatal() - return - } - - successLogger(function, args, startTime).Info() -} - -func main() { - endpoint := os.Getenv("SERVER_ENDPOINT") - accessKey := os.Getenv("ACCESS_KEY") - secretKey := os.Getenv("SECRET_KEY") - secure := os.Getenv("ENABLE_HTTPS") - sdkEndpoint := "http://" + endpoint - if secure == "1" { - sdkEndpoint = "https://" + endpoint - } - - creds := credentials.NewStaticCredentials(accessKey, secretKey, "") - newSession := session.New() - s3Config := &aws.Config{ - Credentials: creds, - Endpoint: aws.String(sdkEndpoint), - Region: aws.String("us-east-1"), - S3ForcePathStyle: aws.Bool(true), - } - - // Create an S3 service object in the default region. - s3Client := s3.New(newSession, s3Config) - - // Output to stdout instead of the default stderr - log.SetOutput(os.Stdout) - // create custom formatter - mintFormatter := mintJSONFormatter{} - // set custom formatter - log.SetFormatter(&mintFormatter) - // log Info or above -- success cases are Info level, failures are Fatal level - log.SetLevel(log.InfoLevel) - // execute tests - testPresignedPutInvalidHash(s3Client) - testListObjects(s3Client) - testSelectObject(s3Client) - testCreateBucketError(s3Client) - testListMultipartUploads(s3Client) - if secure == "1" { - testSSECopyObject(s3Client) - } - if isObjectTaggingImplemented(s3Client) { - testObjectTagging(s3Client) - testObjectTaggingErrors(s3Client) - } -} diff --git a/mint/run/core/aws-sdk-go/run.sh b/mint/run/core/aws-sdk-go/run.sh deleted file mode 100755 index 813b3ee6a..000000000 --- a/mint/run/core/aws-sdk-go/run.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -# -# Mint (C) 2017 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# handle command line arguments -if [ $# -ne 2 ]; then - echo "usage: run.sh " - exit 1 -fi - -output_log_file="$1" -error_log_file="$2" - -# run tests -/mint/run/core/aws-sdk-go/aws-sdk-go 1>>"$output_log_file" 2>"$error_log_file" diff --git a/mint/run/core/aws-sdk-java/run.sh b/mint/run/core/aws-sdk-java/run.sh deleted file mode 100755 index 9a9ad38b0..000000000 --- a/mint/run/core/aws-sdk-java/run.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -# -# Mint (C) 2018 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# handle command line arguments -if [ $# -ne 2 ]; then - echo "usage: run.sh " - exit 1 -fi - -output_log_file="$1" -error_log_file="$2" - -# run tests -cd /mint/run/core/aws-sdk-java/ || exit 1 - -java -jar FunctionalTests.jar 1>>"$output_log_file" 2>"$error_log_file" diff --git a/mint/run/core/aws-sdk-php/README.md b/mint/run/core/aws-sdk-php/README.md deleted file mode 100644 index d08162d08..000000000 --- a/mint/run/core/aws-sdk-php/README.md +++ /dev/null @@ -1,19 +0,0 @@ -## `aws-sdk-php` tests -This directory serves as the location for Mint tests using `aws-sdk-php`. Top level `mint.sh` calls `run.sh` to execute tests. - -## Adding new tests -New tests is added into `quick-tests.php` as new functions. - -## Running tests manually -- Set environment variables `MINT_DATA_DIR`, `MINT_MODE`, `SERVER_ENDPOINT`, `ACCESS_KEY`, `SECRET_KEY`, `SERVER_REGION` and `ENABLE_HTTPS` -- Call `run.sh` with output log file and error log file. for example -```bash -export MINT_DATA_DIR=~/my-mint-dir -export MINT_MODE=core -export SERVER_ENDPOINT="play.minio.io:9000" -export ACCESS_KEY="Q3AM3UQ867SPQQA43P2F" -export SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" -export ENABLE_HTTPS=1 -export SERVER_REGION=us-east-1 -./run.sh /tmp/output.log /tmp/error.log -``` diff --git a/mint/run/core/aws-sdk-php/composer.json b/mint/run/core/aws-sdk-php/composer.json deleted file mode 100644 index cf64a7204..000000000 --- a/mint/run/core/aws-sdk-php/composer.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "require": { - "aws/aws-sdk-php": "^3.30", - "guzzlehttp/psr7": "^1.4" - } -} diff --git a/mint/run/core/aws-sdk-php/quick-tests.php b/mint/run/core/aws-sdk-php/quick-tests.php deleted file mode 100644 index cf00cb8be..000000000 --- a/mint/run/core/aws-sdk-php/quick-tests.php +++ /dev/null @@ -1,1124 +0,0 @@ - 'val-1']; - -/** - * ClientConfig abstracts configuration details to connect to a - * S3-like service - */ -class ClientConfig { - public $creds; - public $endpoint; - public $region; - - function __construct(string $access_key, string $secret_key, string $host, string $secure, string $region) { - $this->creds = new Aws\Credentials\Credentials($access_key, $secret_key); - - if ($secure == "1") { - $this->endpoint = "https://" . $host; - } else { - $this->endpoint = "http://" . $host; - } - - $this->region = $region; - } -} - - /** - * randomName returns a name prefixed by aws-sdk-php using uniqid() - * from standard library - * - * @return string - */ -function randomName():string { - return uniqid("aws-sdk-php-"); -} - - /** - * getStatusCode returns HTTP status code of the given result. - * - * @param $result - AWS\S3 result object - * - * @return string - HTTP status code. E.g, "400" for Bad Request. - */ -function getStatusCode($result):string { - return $result->toArray()['@metadata']['statusCode']; -} - - /** - * runExceptionalTests executes a collection of tests that will throw - * a known exception. - * - * @param $s3Client AWS\S3\S3Client object - * - * @param $apiCall Name of the S3Client API method to call - * - * @param $exceptionMatcher Name of Aws\S3\Exception\S3Exception - * method to fetch exception details - * - * @param $exceptionParamMap Associative array of exception names to - * API parameters. E.g, - * $apiCall = 'headBucket' - * $exceptionMatcher = 'getStatusCode' - * $exceptionParamMap = [ - * // Non existent bucket - * '404' => ['Bucket' => $bucket['Name'] . '--'], - * - * // Non existent bucket - * '404' => ['Bucket' => $bucket['Name'] . '-non-existent'], - * ]; - * - * @return string - HTTP status code. E.g, "404" for Non existent bucket. - */ -function runExceptionalTests($s3Client, $apiCall, $exceptionMatcher, $exceptionParamMap) { - foreach($exceptionParamMap as $exn => $params) { - $exceptionCaught = false; - try { - $result = $s3Client->$apiCall($params); - } catch(Aws\S3\Exception\S3Exception $e) { - $exceptionCaught = true; - switch ($e->$exceptionMatcher()) { - case $exn: - // This is expected - continue; - default: - throw $e; - } - } - finally { - if (!$exceptionCaught) { - $message = sprintf("Expected %s to fail with %s", $apiCall, $exn); - throw new Exception($message); - } - } - } -} - - /** - * testListBuckets tests ListBuckets S3 API - * - * @param $s3Client AWS\S3\S3Client object - * - * @return void - */ -function testListBuckets(S3Client $s3Client) { - $buckets = $s3Client->listBuckets(); - $debugger = $GLOBALS['debugger']; - foreach ($buckets['Buckets'] as $bucket){ - $debugger->out($bucket['Name'] . "\n"); - } -} - - /** - * testBucketExists tests HEAD Bucket S3 API - * - * @param $s3Client AWS\S3\S3Client object - * - * @return void - */ -function testBucketExists(S3Client $s3Client) { - // List all buckets - $buckets = $s3Client->listBuckets(); - // All HEAD on existing buckets must return success - foreach($buckets['Buckets'] as $bucket) { - $result = $s3Client->headBucket(['Bucket' => $bucket['Name']]); - if (getStatusCode($result) != HTTP_OK) - throw new Exception('headBucket API failed for ' . $bucket['Name']); - } - - // Run failure tests - $params = [ - // Non existent bucket - '404' => ['Bucket' => $bucket['Name'] . '--'], - - // Non existent bucket - '404' => ['Bucket' => $bucket['Name'] . '-non-existent'], - ]; - runExceptionalTests($s3Client, 'headBucket', 'getStatusCode', $params); -} - - - /** - * testHeadObject tests HeadObject S3 API - * - * @param $s3Client AWS\S3\S3Client object - * - * @param $objects Associative array of buckets and objects - * - * @return void - */ -function testHeadObject($s3Client, $objects) { - foreach($objects as $bucket => $object) { - $result = $s3Client->headObject(['Bucket' => $bucket, 'Key' => $object]); - if (getStatusCode($result) != HTTP_OK) - throw new Exception('headObject API failed for ' . - $bucket . '/' . $object); - if (strtolower($result['Metadata']) != strtolower(TEST_METADATA)) { - throw new Exception("headObject API Metadata didn't match for " . - $bucket . '/' . $object); - } - } - - // Run failure tests - $params = [ - '404' => ['Bucket' => $bucket, 'Key' => $object . '-non-existent'] - ]; - runExceptionalTests($s3Client, 'headObject', 'getStatusCode', $params); -} - - /** - * testListObjects tests ListObjectsV1 and V2 S3 APIs - * - * @param $s3Client AWS\S3\S3Client object - * - * @param $params associative array containing bucket and object names - * - * @return void - */ -function testListObjects($s3Client, $params) { - $bucket = $params['Bucket']; - $object = $params['Object']; - $debugger = $GLOBALS['debugger']; - try { - for ($i = 0; $i < 5; $i++) { - $copyKey = $object . '-copy-' . strval($i); - $result = $s3Client->copyObject([ - 'Bucket' => $bucket, - 'Key' => $copyKey, - 'CopySource' => $bucket . '/' . $object, - ]); - if (getStatusCode($result) != HTTP_OK) - throw new Exception("copyObject API failed for " . $bucket . '/' . $object); - } - - $paginator = $s3Client->getPaginator('ListObjects', ['Bucket' => $bucket]); - foreach ($paginator->search('Contents[].Key') as $key) { - $debugger->out('key = ' . $key . "\n"); - } - - $paginator = $s3Client->getPaginator('ListObjectsV2', ['Bucket' => $bucket]); - foreach ($paginator->search('Contents[].Key') as $key) { - $debugger->out('key = ' . $key . "\n"); - } - - $prefix = 'obj'; - $result = $s3Client->listObjects(['Bucket' => $bucket, 'Prefix' => $prefix]); - if (getStatusCode($result) != HTTP_OK || $result['Prefix'] != $prefix) - throw new Exception("listObject API failed for " . $bucket . '/' . $object); - - $maxKeys = 1; - $result = $s3Client->listObjects(['Bucket' => $bucket, 'MaxKeys' => $maxKeys]); - if (getStatusCode($result) != HTTP_OK || count($result['Contents']) != $maxKeys) - throw new Exception("listObject API failed for " . $bucket . '/' . $object); - - $params = [ - 'InvalidArgument' => ['Bucket' => $bucket, 'MaxKeys' => -1], - 'NoSuchBucket' => ['Bucket' => $bucket . '-non-existent'] - ]; - runExceptionalTests($s3Client, 'listObjects', 'getAwsErrorCode', $params); - - } finally { - $s3Client->deleteObjects([ - 'Bucket' => $bucket, - 'Delete' => [ - 'Objects' => array_map(function($a, $b) { - return ['Key' => $a . '-copy-' . strval($b)]; - }, array_fill(0, 5, $object), range(0,4)) - ], - ]); - } -} - - /** - * testListMultipartUploads tests ListMultipartUploads, ListParts and - * UploadPartCopy S3 APIs - * - * @param $s3Client AWS\S3\S3Client object - * - * @param $params associative array containing bucket and object names - * - * @return void - */ -function testListMultipartUploads($s3Client, $params) { - $bucket = $params['Bucket']; - $object = $params['Object']; - $debugger = $GLOBALS['debugger']; - - $data_dir = $GLOBALS['MINT_DATA_DIR']; - // Initiate multipart upload - $result = $s3Client->createMultipartUpload([ - 'Bucket' => $bucket, - 'Key' => $object . '-copy', - ]); - if (getStatusCode($result) != HTTP_OK) - throw new Exception('createMultipartupload API failed for ' . - $bucket . '/' . $object); - - // upload 5 parts - $uploadId = $result['UploadId']; - $parts = []; - try { - for ($i = 0; $i < 5; $i++) { - $result = $s3Client->uploadPartCopy([ - 'Bucket' => $bucket, - 'Key' => $object . '-copy', - 'UploadId' => $uploadId, - 'PartNumber' => $i+1, - 'CopySource' => $bucket . '/' . $object, - ]); - if (getStatusCode($result) != HTTP_OK) { - throw new Exception('uploadPart API failed for ' . - $bucket . '/' . $object); - } - array_push($parts, [ - 'ETag' => $result['ETag'], - 'PartNumber' => $i+1, - ]); - } - - // ListMultipartUploads and ListParts may return empty - // responses in the case of minio gateway gcs and minio server - // FS mode. So, the following tests don't make assumptions on - // result response. - $paginator = $s3Client->getPaginator('ListMultipartUploads', - ['Bucket' => $bucket]); - foreach ($paginator->search('Uploads[].{Key: Key, UploadId: UploadId}') as $keyHash) { - $debugger->out('key = ' . $keyHash['Key'] . ' uploadId = ' . $keyHash['UploadId'] . "\n"); - } - - $paginator = $s3Client->getPaginator('ListParts', [ - 'Bucket' => $bucket, - 'Key' => $object . '-copy', - 'UploadId' => $uploadId, - ]); - foreach ($paginator->search('Parts[].{PartNumber: PartNumber, ETag: ETag}') as $partsHash) { - $debugger->out('partNumber = ' . $partsHash['PartNumber'] . ' ETag = ' . $partsHash['ETag'] . "\n"); - } - - }finally { - $s3Client->abortMultipartUpload([ - 'Bucket' => $bucket, - 'Key' => $object . '-copy', - 'UploadId' => $uploadId - ]); - } -} - - /** - * initSetup creates buckets and objects necessary for the functional - * tests to run - * - * @param $s3Client AWS\S3\S3Client object - * - * @param $objects Associative array of buckets and objects - * - * @return void - */ -function initSetup(S3Client $s3Client, $objects) { - $MINT_DATA_DIR = $GLOBALS['MINT_DATA_DIR']; - foreach($objects as $bucket => $object) { - $s3Client->createBucket(['Bucket' => $bucket]); - $stream = NULL; - try { - if (!file_exists($MINT_DATA_DIR . '/' . FILE_1_KB)) - throw new Exception('File not found ' . $MINT_DATA_DIR . '/' . FILE_1_KB); - - $stream = Psr7\stream_for(fopen($MINT_DATA_DIR . '/' . FILE_1_KB, 'r')); - $result = $s3Client->putObject([ - 'Bucket' => $bucket, - 'Key' => $object, - 'Body' => $stream, - 'Metadata' => TEST_METADATA, - ]); - if (getStatusCode($result) != HTTP_OK) - throw new Exception("putObject API failed for " . $bucket . '/' . $object); - } - - finally { - // close data file - if (!is_null($stream)) - $stream->close(); - } - } - - // Create an empty bucket for bucket policy + delete tests - $result = $s3Client->createBucket(['Bucket' => $GLOBALS['emptyBucket']]); - if (getStatusCode($result) != HTTP_OK) - throw new Exception("createBucket API failed for " . $bucket); - -} - - - /** - * testGetPutObject tests GET/PUT object S3 API - * - * @param $s3Client AWS\S3\S3Client object - * - * @param $params associative array containing bucket and object names - * - * @return void - */ -function testGetPutObject($s3Client, $params) { - $bucket = $params['Bucket']; - $object = $params['Object']; - - // Upload a 10KB file - $MINT_DATA_DIR = $GLOBALS['MINT_DATA_DIR']; - try { - $stream = Psr7\stream_for(fopen($MINT_DATA_DIR . '/' . FILE_1_KB, 'r')); - $result = $s3Client->putObject([ - 'Bucket' => $bucket, - 'Key' => $object, - 'Body' => $stream, - ]); - } - finally { - $stream->close(); - } - - if (getStatusCode($result) != HTTP_OK) - throw new Exception("putObject API failed for " . $bucket . '/' . $object); - - // Download the same object and verify size - $result = $s3Client->getObject([ - 'Bucket' => $bucket, - 'Key' => $object, - ]); - if (getStatusCode($result) != HTTP_OK) - throw new Exception("getObject API failed for " . $bucket . '/' . $object); - - $body = $result['Body']; - $bodyLen = 0; - while (!$body->eof()) { - $bodyLen += strlen($body->read(4096)); - } - - if ($bodyLen != 1 * 1024) { - throw new Exception("Object downloaded has different content length than uploaded object " - . $bucket . '/' . $object); - } -} - - /** - * testMultipartUploadFailure tests MultipartUpload failures - * - * @param $s3Client AWS\S3\S3Client object - * - * @param $params associative array containing bucket and object names - * - * @return void - */ -function testMultipartUploadFailure($s3Client, $params) { - $bucket = $params['Bucket']; - $object = $params['Object']; - - $MINT_DATA_DIR = $GLOBALS['MINT_DATA_DIR']; - // Initiate multipart upload - $result = $s3Client->createMultipartUpload([ - 'Bucket' => $bucket, - 'Key' => $object, - ]); - if (getStatusCode($result) != HTTP_OK) - throw new Exception('createMultipartupload API failed for ' . - $bucket . '/' . $object); - - // upload 2 parts - $uploadId = $result['UploadId']; - $parts = []; - try { - for ($i = 0; $i < 2; $i++) { - $stream = Psr7\stream_for(fopen($MINT_DATA_DIR . '/' . FILE_5_MB, 'r')); - $limitedStream = new Psr7\LimitStream($stream, 4 * 1024 * 1024, 0); - $result = $s3Client->uploadPart([ - 'Bucket' => $bucket, - 'Key' => $object, - 'UploadId' => $uploadId, - 'ContentLength' => 4 * 1024 * 1024, - 'Body' => $limitedStream, - 'PartNumber' => $i+1, - ]); - if (getStatusCode($result) != HTTP_OK) { - throw new Exception('uploadPart API failed for ' . - $bucket . '/' . $object); - } - array_push($parts, [ - 'ETag' => $result['ETag'], - 'PartNumber' => $i+1, - ]); - - $limitedStream->close(); - $limitedStream = NULL; - } - } - finally { - if (!is_null($limitedStream)) - $limitedStream->close(); - } - - $params = [ - 'EntityTooSmall' => [ - 'Bucket' => $bucket, - 'Key' => $object, - 'UploadId' => $uploadId, - 'MultipartUpload' => [ - 'Parts' => $parts, - ], - ], - 'NoSuchUpload' => [ - 'Bucket' => $bucket, - 'Key' => $object, - 'UploadId' => 'non-existent', - 'MultipartUpload' => [ - 'Parts' => $parts, - ], - ], - ]; - runExceptionalTests($s3Client, 'completeMultipartUpload', 'getAwsErrorCode', $params); -} - - /** - * testMultipartUpload tests MultipartUpload S3 APIs - * - * @param $s3Client AWS\S3\S3Client object - * - * @param $params associative array containing bucket and object names - * - * @return void - */ -function testMultipartUpload($s3Client, $params) { - $bucket = $params['Bucket']; - $object = $params['Object']; - - $MINT_DATA_DIR = $GLOBALS['MINT_DATA_DIR']; - // Initiate multipart upload - $result = $s3Client->createMultipartUpload([ - 'Bucket' => $bucket, - 'Key' => $object, - ]); - if (getStatusCode($result) != HTTP_OK) - throw new Exception('createMultipartupload API failed for ' . - $bucket . '/' . $object); - - // upload 2 parts - $uploadId = $result['UploadId']; - $parts = []; - try { - for ($i = 0; $i < 2; $i++) { - $stream = Psr7\stream_for(fopen($MINT_DATA_DIR . '/' . FILE_5_MB, 'r')); - $result = $s3Client->uploadPart([ - 'Bucket' => $bucket, - 'Key' => $object, - 'UploadId' => $uploadId, - 'ContentLength' => 5 * 1024 * 1024, - 'Body' => $stream, - 'PartNumber' => $i+1, - ]); - if (getStatusCode($result) != HTTP_OK) { - throw new Exception('uploadPart API failed for ' . - $bucket . '/' . $object); - } - array_push($parts, [ - 'ETag' => $result['ETag'], - 'PartNumber' => $i+1, - ]); - - $stream->close(); - $stream = NULL; - } - } - finally { - if (!is_null($stream)) - $stream->close(); - } - - // complete multipart upload - $result = $s3Client->completeMultipartUpload([ - 'Bucket' => $bucket, - 'Key' => $object, - 'UploadId' => $uploadId, - 'MultipartUpload' => [ - 'Parts' => $parts, - ], - ]); - if (getStatusCode($result) != HTTP_OK) { - throw new Exception('completeMultipartupload API failed for ' . - $bucket . '/' . $object); - } -} - - /** - * testAbortMultipartUpload tests aborting of a multipart upload - * - * @param $s3Client AWS\S3\S3Client object - * - * @param $params associative array containing bucket and object names - * - * @return void - */ -function testAbortMultipartUpload($s3Client, $params) { - $bucket = $params['Bucket']; - $object = $params['Object']; - - $MINT_DATA_DIR = $GLOBALS['MINT_DATA_DIR']; - // Initiate multipart upload - $result = $s3Client->createMultipartUpload([ - 'Bucket' => $bucket, - 'Key' => $object, - ]); - if (getStatusCode($result) != HTTP_OK) - throw new Exception('createMultipartupload API failed for ' . - $bucket . '/' . $object); - - // Abort multipart upload - $uploadId = $result['UploadId']; - $result = $s3Client->abortMultipartUpload([ - 'Bucket' => $bucket, - 'Key' => $object, - 'UploadId' => $uploadId, - ]); - if (getStatusCode($result) != HTTP_NOCONTENT) - throw new Exception('abortMultipartupload API failed for ' . - $bucket . '/' . $object); - - //Run failure tests - $params = [ - // Upload doesn't exist - 'NoSuchUpload' => [ - 'Bucket' => $bucket, - 'Key' => $object, - 'UploadId' => 'non-existent', - ], - ]; - runExceptionalTests($s3Client, 'abortMultipartUpload', 'getAwsErrorCode', $params); -} - - /** - * testGetBucketLocation tests GET bucket location S3 API - * - * @param $s3Client AWS\S3\S3Client object - * - * @param $params associative array containing bucket name - * - * @return void - */ -function testGetBucketLocation($s3Client, $params) { - $bucket = $params['Bucket']; - - // Valid test - $result = $s3Client->getBucketLocation(['Bucket' => $bucket]); - if (getStatusCode($result) != HTTP_OK) - throw new Exception('getBucketLocation API failed for ' . - $bucket); - - // Run failure tests. - $params = [ - // Non existent bucket - 'NoSuchBucket' => ['Bucket' => $bucket . '--'], - - // Bucket not found - 'NoSuchBucket' => ['Bucket' => $bucket . '-non-existent'], - ]; - runExceptionalTests($s3Client, 'getBucketLocation', 'getAwsErrorCode', $params); -} - - /** - * testCopyObject tests copy object S3 API - * - * @param $s3Client AWS\S3\S3Client object - * - * @param $params associative array containing bucket and object name - * - * @return void - */ -function testCopyObject($s3Client, $params) { - $bucket = $params['Bucket']; - $object = $params['Object']; - - $result = $s3Client->copyObject([ - 'Bucket' => $bucket, - 'Key' => $object . '-copy', - 'CopySource' => $bucket . '/' . $object, - ]); - if (getStatusCode($result) != HTTP_OK) - throw new Exception('copyObject API failed for ' . - $bucket); - - $s3Client->deleteObject([ - 'Bucket' => $bucket, - 'Key' => $object . '-copy', - ]); - - // Run failure tests - $params = [ - // Invalid copy source format - 'InvalidArgument' => [ - 'Bucket' => $bucket, - 'Key' => $object . '-copy', - 'CopySource' => $bucket . $object - ], - - // Missing source object - 'NoSuchKey' => [ - 'Bucket' => $bucket, - 'Key' => $object . '-copy', - 'CopySource' => $bucket . '/' . $object . '-non-existent' - ], - ]; - runExceptionalTests($s3Client, 'copyObject', 'getAwsErrorCode', $params); -} - - /** - * testDeleteObjects tests Delete Objects S3 API - * - * @param $s3Client AWS\S3\S3Client object - * - * @param $params associative array containing bucket and object names - * - * @return void - */ -function testDeleteObjects($s3Client, $params) { - $bucket = $params['Bucket']; - $object = $params['Object']; - - $copies = []; - for ($i = 0; $i < 3; $i++) { - $copyKey = $object . '-copy' . strval($i); - $result = $s3Client->copyObject([ - 'Bucket' => $bucket, - 'Key' => $copyKey, - 'CopySource' => $bucket . '/' . $object, - ]); - if (getstatuscode($result) != HTTP_OK) - throw new Exception('copyobject API failed for ' . - $bucket); - array_push($copies, ['Key' => $copyKey]); - } - - $result = $s3Client->deleteObjects([ - 'Bucket' => $bucket, - 'Delete' => [ - 'Objects' => $copies, - ], - ]); - if (getstatuscode($result) != HTTP_OK) - throw new Exception('deleteObjects api failed for ' . - $bucket); -} - - /** - * testAnonDeleteObjects tests Delete Objects S3 API for anonymous requests. - * The test case checks this scenario: - * http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html#multiobjectdeleteapi-examples - * - * @param $s3Client AWS\S3\S3Client object - * - * @param $params associative array containing bucket and object names - * - * @return void - */ -function testAnonDeleteObjects($s3Client, $params) { - $bucket = $params['Bucket']; - $object = $params['Object']; - - // Create anonymous config object - $anonConfig = new ClientConfig("", "", $GLOBALS['endpoint'], $GLOBALS['secure'], $GLOBALS['region']); - - // Create anonymous S3 client - $anonymousClient = new S3Client([ - 'credentials' => false, - 'endpoint' => $anonConfig->endpoint, - 'use_path_style_endpoint' => true, - 'region' => $anonConfig->region, - 'version' => '2006-03-01' - ]); - - $copies = []; - for ($i = 0; $i < 3; $i++) { - $copyKey = $object . '-copy' . strval($i); - $result = $s3Client->copyObject([ - 'Bucket' => $bucket, - 'Key' => $copyKey, - 'CopySource' => $bucket . '/' . $object, - ]); - if (getstatuscode($result) != HTTP_OK) - throw new Exception('copyobject API failed for ' . - $bucket); - array_push($copies, ['Key' => $copyKey]); - } - - // Try anonymous delete. - $result = $anonymousClient->deleteObjects([ - 'Bucket' => $bucket, - 'Delete' => [ - 'Objects' => $copies, - ], - ]); - // Response code should be 200 - if (getstatuscode($result) != HTTP_OK) - throw new Exception('deleteObjects returned incorrect response ' . - getStatusCode($result)); - - // Each object should have error code AccessDenied - for ($i = 0; $i < 3; $i++) { - if ($result["Errors"][$i]["Code"] != "AccessDenied") - throw new Exception('Incorrect response deleteObjects anonymous - call for ' .$bucket); - } - - // Delete objects after the test passed - $result = $s3Client->deleteObjects([ - 'Bucket' => $bucket, - 'Delete' => [ - 'Objects' => $copies, - ], - ]); - - if (getstatuscode($result) != HTTP_OK) - throw new Exception('deleteObjects api failed for ' . - $bucket); - - // Each object should have empty code in case of successful delete - for ($i = 0; $i < 3; $i++) { - if ($result["Errors"][$i]["Code"] != "") - throw new Exception('Incorrect response deleteObjects anonymous - call for ' .$bucket); - } -} - -// Check if the policy statements are equal -function are_statements_equal($expected, $got) { - $expected = json_decode($expected, TRUE); - $got = json_decode($got, TRUE); - - function are_actions_equal($action1, $action2) { - return ( - is_array($action1) - && is_array($action2) - && count($action1) == count($action2) - && array_diff($action1, $action2) === array_diff($action2, $action1) - ); - } - - foreach ($expected['Statement'] as $index => $value) { - if (!are_actions_equal($value['Action'], $got['Statement'][$index]['Action'])) - return FALSE; - } - - return TRUE; - -} - /** - * testBucketPolicy tests GET/PUT Bucket policy S3 APIs - * - * @param $s3Client AWS\S3\S3Client object - * - * @param $params associative array containing bucket and object names - * - * @return void - */ -function testBucketPolicy($s3Client, $params) { - $bucket = $params['Bucket']; - - $downloadPolicy = sprintf('{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:GetBucketLocation","s3:ListBucket","s3:GetObject"],"Resource":["arn:aws:s3:::%s","arn:aws:s3:::%s/*"]}]}', $bucket, $bucket); - - $result = $s3Client->putBucketPolicy([ - 'Bucket' => $bucket, - 'Policy' => $downloadPolicy - ]); - if (getstatuscode($result) != HTTP_NOCONTENT) - throw new Exception('putBucketPolicy API failed for ' . - $bucket); - $result = $s3Client->getBucketPolicy(['Bucket' => $bucket]); - if (getstatuscode($result) != HTTP_OK) - throw new Exception('putBucketPolicy API failed for ' . - $bucket); - - if ($result['Policy'] != $downloadPolicy) - if (!are_statements_equal($result['Policy'], $downloadPolicy)) - throw new Exception('bucket policy we got is not we set'); - - - // Delete the bucket, make the bucket (again) and check if policy is none - // Ref: https://github.com/minio/minio/issues/4714 - $result = $s3Client->deleteBucket(['Bucket' => $bucket]); - if (getstatuscode($result) != HTTP_NOCONTENT) - throw new Exception('deleteBucket API failed for ' . - $bucket); - - try { - $s3Client->getBucketPolicy(['Bucket' => $bucket]); - } catch (AWSException $e) { - switch ($e->getAwsErrorCode()) { - case 'NoSuchBucket': - break; - } - } - - // Sleep is needed for Minio Gateway for Azure, ref: - // https://docs.microsoft.com/en-us/rest/api/storageservices/Delete-Container#remarks - sleep(40); - - $s3Client->createBucket(['Bucket' => $bucket]); - - $params = [ - '404' => ['Bucket' => $bucket] - ]; - runExceptionalTests($s3Client, 'getBucketPolicy', 'getStatusCode', $params); - - try { - $MINT_DATA_DIR = $GLOBALS['MINT_DATA_DIR']; - // Create an object to test anonymous GET object - $object = 'test-anon'; - if (!file_exists($MINT_DATA_DIR . '/' . FILE_1_KB)) - throw new Exception('File not found ' . $MINT_DATA_DIR . '/' . FILE_1_KB); - - $stream = Psr7\stream_for(fopen($MINT_DATA_DIR . '/' . FILE_1_KB, 'r')); - $result = $s3Client->putObject([ - 'Bucket' => $bucket, - 'Key' => $object, - 'Body' => $stream, - ]); - if (getstatuscode($result) != HTTP_OK) - throw new Exception('createBucket API failed for ' . - $bucket); - - $anonConfig = new ClientConfig("", "", $GLOBALS['endpoint'], $GLOBALS['secure'], $GLOBALS['region']); - $anonymousClient = new S3Client([ - 'credentials' => false, - 'endpoint' => $anonConfig->endpoint, - 'use_path_style_endpoint' => true, - 'region' => $anonConfig->region, - 'version' => '2006-03-01' - ]); - runExceptionalTests($anonymousClient, 'getObject', 'getStatusCode', [ - '403' => [ - 'Bucket' => $bucket, - 'Key' => $object, - ] - ]); - - } finally { - // close data file - if (!is_null($stream)) - $stream->close(); - $s3Client->deleteObject(['Bucket' => $bucket, 'Key' => $object]); - } - -} - - /** - * cleanupSetup removes all buckets and objects created during the - * functional test - * - * @param $s3Client AWS\S3\S3Client object - * - * @param $objects Associative array of buckets to objects - * - * @return void - */ -function cleanupSetup($s3Client, $objects) { - // Delete all objects - foreach ($objects as $bucket => $object) { - $s3Client->deleteObject(['Bucket' => $bucket, 'Key' => $object]); - } - - // Delete the buckets incl. emptyBucket - $allBuckets = array_keys($objects); - array_push($allBuckets, $GLOBALS['emptyBucket']); - foreach ($allBuckets as $bucket) { - try { - // Delete the bucket - $s3Client->deleteBucket(['Bucket' => $bucket]); - - // Wait until the bucket is removed from object store - $s3Client->waitUntil('BucketNotExists', ['Bucket' => $bucket]); - } catch (Exception $e) { - // Ignore exceptions thrown during cleanup - } - } -} - - - /** - * runTest helper function to wrap a test function and log - * success or failure accordingly. - * - * @param myfunc name of test function to be run - * - * @param fnSignature function signature of the main S3 SDK API - * - * @param args parameters to be passed to test function - * - * @return void - */ -function runTest($s3Client, $myfunc, $fnSignature, $args = []) { - try { - $start_time = microtime(true); - $status = "PASS"; - $error = ""; - $message = ""; - $myfunc($s3Client, $args); - } catch (AwsException $e) { - $errorCode = $e->getAwsErrorCode(); - // $fnSignature holds the specific API that is being - // tested. It is possible that functions used to create the - // test setup may not be implemented. - if ($errorCode != "NotImplemented") { - $status = "FAIL"; - $error = $e->getMessage(); - } else { - $status = "NA"; - $error = $e->getMessage(); - $alert = sprintf("%s or a related API is NOT IMPLEMENTED, see \"error\" for exact details.", $fnSignature); - } - - } catch (Exception $e) { - // This exception handler handles high-level custom exceptions. - $status = "FAIL"; - $error = $e->getMessage(); - } finally { - $end_time = microtime(true); - $json_log = [ - "name" => "aws-sdk-php", - "function" => $fnSignature, - "args" => $args, - "duration" => sprintf("%d", ($end_time - $start_time) * 1000), // elapsed time in ms - "status" => $status, - ]; - if ($error !== "") { - $json_log["error"] = $error; - } - if ($message !== "") { - $json_log["message"] = $message; - } - print_r(json_encode($json_log)."\n"); - - // Exit on first failure. - switch ($status) { - case "FAIL": - exit(1); - } - } -} - -// Get client configuration from environment variables -$GLOBALS['access_key'] = getenv("ACCESS_KEY"); -$GLOBALS['secret_key'] = getenv("SECRET_KEY"); -$GLOBALS['endpoint'] = getenv("SERVER_ENDPOINT"); -$GLOBALS['region'] = getenv("SERVER_REGION"); -$GLOBALS['secure'] = getenv("ENABLE_HTTPS"); - -/** - * @global string $GLOBALS['MINT_DATA_DIR'] - * @name $MINT_DATA_DIR - */ -$GLOBALS['MINT_DATA_DIR'] = '/mint/data'; -$GLOBALS['MINT_DATA_DIR'] = getenv("MINT_DATA_DIR"); - - -// Useful for debugging test failures; Set $debugmode it to true when required -$debugmode = false; - -interface Debugger { - public function out($data); -} - -class EchoDebugger implements Debugger { - public function out($data) { - echo $data; - } -} - -class NullDebugger implements Debugger { - public function out($data) { - // Do nothing - } -} - -if($debugmode) - $debugger = new EchoDebugger(); -else - $debugger = new NullDebugger(); - -// Make $debugger global -$GLOBALS['debugger'] = $debugger; - -// Create config object -$config = new ClientConfig($GLOBALS['access_key'], $GLOBALS['secret_key'], - $GLOBALS['endpoint'], $GLOBALS['secure'], - $GLOBALS['region']); - -// Create a S3Client -$s3Client = new S3Client([ - 'credentials' => $config->creds, - 'endpoint' => $config->endpoint, - 'use_path_style_endpoint' => true, - 'region' => $config->region, - 'version' => '2006-03-01' -]); - -// Used by initSetup -$emptyBucket = randomName(); -$objects = [ - randomName() => 'obj1', - randomName() => 'obj2', -]; - -try { - initSetup($s3Client, $objects); - $firstBucket = array_keys($objects)[0]; - $firstObject = $objects[$firstBucket]; - $testParams = ['Bucket' => $firstBucket, 'Object' => $firstObject]; - runTest($s3Client, 'testGetBucketLocation', "getBucketLocation ( array \$params = [] )", ['Bucket' => $firstBucket]); - runTest($s3Client, 'testListBuckets', "listBuckets ( array \$params = [] )"); - runTest($s3Client, 'testListObjects', "listObjects ( array \$params = [] )", $testParams); - runTest($s3Client, 'testListMultipartUploads', "listMultipartUploads ( array \$params = [] )", $testParams); - runTest($s3Client, 'testBucketExists', "headBucket ( array \$params = [] )", array_keys($objects)); - runTest($s3Client, 'testHeadObject', "headObject ( array \$params = [] )", $objects); - runTest($s3Client, 'testGetPutObject', "getObject ( array \$params = [] )", $testParams); - runTest($s3Client, 'testCopyObject', "copyObject ( array \$params = [] )", $testParams); - runTest($s3Client, 'testDeleteObjects', "deleteObjects (array \$params = [] )", $testParams); - runTest($s3Client, 'testAnonDeleteObjects', "anonDeleteObjects ( array \$params = [] )", $testParams); - runTest($s3Client, 'testMultipartUpload', "createMultipartUpload ( array \$params = [] )", $testParams); - runTest($s3Client, 'testMultipartUploadFailure', "uploadPart ( array \$params = [] )", $testParams); - runTest($s3Client, 'testAbortMultipartUpload', "abortMultipartupload ( array \$params = [] )", $testParams); - runTest($s3Client, 'testBucketPolicy', "getBucketPolicy ( array \$params = [] )", ['Bucket' => $emptyBucket]); -} -finally { - cleanupSetup($s3Client, $objects); -} - -?> diff --git a/mint/run/core/aws-sdk-php/run.sh b/mint/run/core/aws-sdk-php/run.sh deleted file mode 100755 index e3a37a37c..000000000 --- a/mint/run/core/aws-sdk-php/run.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -# -# Mint, (C) 2017 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# handle command line arguments -if [ $# -ne 2 ]; then - echo "usage: run.sh " - exit 1 -fi - -output_log_file="$1" -error_log_file="$2" - -# run tests -php ./quick-tests.php 1>>"$output_log_file" 2>"$error_log_file" diff --git a/mint/run/core/aws-sdk-ruby/README.md b/mint/run/core/aws-sdk-ruby/README.md deleted file mode 100644 index 7c66adae4..000000000 --- a/mint/run/core/aws-sdk-ruby/README.md +++ /dev/null @@ -1,19 +0,0 @@ -## `aws-sdk-ruby` tests -This directory serves as the location for Mint tests using `aws-sdk-ruby`. Top level `mint.sh` calls `run.sh` to execute tests. - -## Adding new tests -New tests is added into `aws-stub-test.rb` as new functions. - -## Running tests manually -- Set environment variables `MINT_DATA_DIR`, `MINT_MODE`, `SERVER_ENDPOINT`, `ACCESS_KEY`, `SECRET_KEY`, `SERVER_REGION` and `ENABLE_HTTPS` -- Call `run.sh` with output log file and error log file. for example -```bash -export MINT_DATA_DIR=~/my-mint-dir -export MINT_MODE=core -export SERVER_ENDPOINT="play.minio.io:9000" -export ACCESS_KEY="Q3AM3UQ867SPQQA43P2F" -export SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" -export ENABLE_HTTPS=1 -export SERVER_REGION=us-east-1 -./run.sh /tmp/output.log /tmp/error.log -``` diff --git a/mint/run/core/aws-sdk-ruby/aws-stub-tests.rb b/mint/run/core/aws-sdk-ruby/aws-stub-tests.rb deleted file mode 100755 index 185004ded..000000000 --- a/mint/run/core/aws-sdk-ruby/aws-stub-tests.rb +++ /dev/null @@ -1,855 +0,0 @@ -#!/usr/bin/env ruby -# -# Mint (C) 2017 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -require 'aws-sdk' -require 'securerandom' -require 'net/http' -require 'multipart_body' - -class AwsSdkRubyTest - # Set variables necessary to create an s3 client instance. - # Get them from the environment variables - - # Region information, eg. "us-east-1" - region = ENV['SERVER_REGION'] ||= 'SERVER_REGION is not set' - # Minio server, eg. "play.minio.io:9000" - endpoint = ENV['SERVER_ENDPOINT'] ||= 'SERVER_ENDPOINT is not set' - access_key_id = ENV['ACCESS_KEY'] ||= 'ACCESS_KEY is not set' - secret_access_key = ENV['SECRET_KEY'] ||= 'SECRET_KEY is not set' - enable_https = ENV['ENABLE_HTTPS'] - endpoint = enable_https == '1' ? 'https://' + endpoint : 'http://' + endpoint - # Create s3 client instances, "s3Resource" and "s3Client" - @@s3 = Aws::S3::Resource.new(region: region, - endpoint: endpoint, - access_key_id: access_key_id, - secret_access_key: secret_access_key, - force_path_style: true) - - def initialize_log_output(meth, alert = nil) - # Initialize and return log content in log_output hash table - - # Collect args in args_arr - args_arr = method(meth).parameters.flatten.map(&:to_s) - .reject { |x| x == 'req' || x == 'opt' } - # Create and return log output content - { name: 'aws-sdk-ruby', - function: "#{meth}(#{args_arr.join(',')})", # method name and arguments - args: args_arr, # array of arg names. This'll be replaced with a - # a arg/value pairs insdie the caller method - duration: 0, # test runtime duration in seconds - alert: alert, - message: nil, - error: nil } - end - - def get_random_bucket_name() - bucket_name = "aws-sdk-ruby-bucket-"+SecureRandom.hex(6) - return bucket_name - end - - def calculate_duration(t2, t1) - # Durations are in miliseconds, with precision of 2 decimal places - ((t2 - t1) * 1000).round(2) - end - - def print_log(log_output, start_time) - # Calculate duration in miliseconds - log_output[:duration] = calculate_duration(Time.now, start_time) - # Get rid of the log_output fields if nil - puts log_output.delete_if{|k, value| value == nil}.to_json - # Exit at the first failure - exit 1 if log_output[:status] == 'FAIL' - end - - def cleanUp(buckets, log_output) - # Removes objects and bucket if bucket exists - bucket_name = '' - buckets.each do |b| - bucket_name = b - if bucketExistsWrapper(b, log_output) - removeObjectsWrapper(b, log_output) - removeBucketWrapper(b, log_output) - end - end - rescue => e - raise "Failed to clean-up bucket '#{bucket_name}', #{e}" - end - - # - # API commands/methods - # - def makeBucket(bucket_name) - # Creates a bucket, "bucket_name" - # on S3 client , "s3". - # Returns bucket_name if already exists - @@s3.bucket(bucket_name).exists? ? @@s3.bucket(bucket_name) : @@s3.create_bucket(bucket: bucket_name) - rescue => e - raise e - end - - def makeBucketWrapper(bucket_name, log_output) - makeBucket(bucket_name) - rescue => e - log_output[:function] = "makeBucket(bucket_name)" - log_output[:args] = {'bucket_name': bucket_name} - raise e - end - - def removeBucket(bucket_name) - # Deletes/removes bucket, "bucket_name" on S3 client, "s3" - @@s3.bucket(bucket_name).delete - rescue => e - raise e - end - - def removeBucketWrapper(bucket_name, log_output) - removeBucket(bucket_name) - rescue => e - log_output[:function] = "removeBucket(bucket_name)" - log_output[:args] = {'bucket_name': bucket_name} - raise e - end - - def putObject(bucket_name, file) - # Creates "file" (full path) in bucket, "bucket_name", - # on S3 client, "s3" - file_name = File.basename(file) - @@s3.bucket(bucket_name).object(file_name).upload_file(file) - rescue => e - raise e - end - - def putObjectWrapper(bucket_name, file, log_output) - putObject(bucket_name, file) - rescue => e - log_output[:function] = "putObject(bucket_name, file)" - log_output[:args] = {'bucket_name': bucket_name, - 'file': file} - raise e - end - - def getObject(bucket_name, file, destination) - # Gets/Downloads file, "file", - # from bucket, "bucket_name", of S3 client, "s3" - file_name = File.basename(file) - dest = File.join(destination, file_name) - @@s3.bucket(bucket_name).object(file_name).get(response_target: dest) - rescue => e - raise e - end - - def getObjectWrapper(bucket_name, file, destination, log_output) - getObject(bucket_name, file, destination) - rescue => e - log_output[:function] = "getObject(bucket_name, file)" - log_output[:args] = {'bucket_name': bucket_name, - 'file': file, - 'destination': destination} - raise e - end - - def copyObject(source_bucket_name, target_bucket_name, source_file_name, target_file_name = '') - # Copies file, "file_name", from source bucket, - # "source_bucket_name", to target bucket, - # "target_bucket_name", on S3 client, "s3" - target_file_name = source_file_name if target_file_name.empty? - source = @@s3.bucket(source_bucket_name) - target = @@s3.bucket(target_bucket_name) - source_obj = source.object(source_file_name) - target_obj = target.object(target_file_name) - source_obj.copy_to(target_obj) - rescue => e - raise e - end - - def copyObjectWrapper(source_bucket_name, target_bucket_name, source_file_name, target_file_name = '', log_output) - copyObject(source_bucket_name, target_bucket_name, source_file_name, target_file_name) - rescue => e - log_output[:function] = 'copyObject(source_bucket_name, target_bucket_name, source_file_name, target_file_name = '')' - log_output[:args] = {'source_bucket_name': source_bucket_name, - 'target_bucket_name': target_bucket_name, - 'source_file_name': source_file_name, - 'target_file_name': target_file_name} - raise e - end - - def removeObject(bucket_name, file) - # Deletes file in bucket, - # "bucket_name", on S3 client, "s3". - # If file, "file_name" does not exist, - # it quietly returns without any error message - @@s3.bucket(bucket_name).object(file).delete - rescue => e - raise e - end - - def removeObjectWrapper(bucket_name, file_name, log_output) - removeObject(bucket_name, file_name) - rescue => e - log_output[:function] = "removeObject(bucket_name, file_name)" - log_output[:args] = {'bucket_name': bucket_name, - 'file_name': file_name} - raise e - end - - def removeObjects(bucket_name) - # Deletes all files in bucket, "bucket_name" - # on S3 client, "s3" - file_name = '' - @@s3.bucket(bucket_name).objects.each do |obj| - file_name = obj.key - obj.delete - end - rescue => e - raise "File name: '#{file_name}', #{e}" - end - - def removeObjectsWrapper(bucket_name, log_output) - removeObjects(bucket_name) - rescue => e - log_output[:function] = 'removeObjects(bucket_name)' - log_output[:args] = {'bucket_name': bucket_name} - raise e - end - - def listBuckets - # Returns an array of bucket names on S3 client, "s3" - bucket_name_list = [] - @@s3.buckets.each do |b| - bucket_name_list.push(b.name) - end - return bucket_name_list - rescue => e - raise e - end - - def listBucketsWrapper(log_output) - listBuckets - rescue => e - log_output[:function] = 'listBuckets' - log_output[:args] = {} - raise e - end - - def listObjects(bucket_name) - # Returns an array of object/file names - # in bucket, "bucket_name", on S3 client, "s3" - object_list = [] - @@s3.bucket(bucket_name).objects.each do |obj| - object_list.push(obj.key) - end - return object_list - rescue => e - raise e - end - - def listObjectsWrapper(bucket_name, log_output) - listObjects(bucket_name) - rescue => e - log_output[:function] = 'listObjects(bucket_name)' - log_output[:args] = {'bucket_name': bucket_name} - raise e - end - - def statObject(bucket_name, file_name) - return @@s3.bucket(bucket_name).object(file_name).exists? - rescue => e - raise e - end - - def statObjectWrapper(bucket_name, file_name, log_output) - statObject(bucket_name, file_name) - rescue => e - log_output[:function] = 'statObject(bucket_name, file_name)' - log_output[:args] = {'bucket_name': bucket_name, - 'file_name': file_name} - raise e - end - - def bucketExists?(bucket_name) - # Returns true if bucket, "bucket_name", exists, - # false otherwise - return @@s3.bucket(bucket_name).exists? - rescue => e - raise e - end - - def bucketExistsWrapper(bucket_name, log_output) - bucketExists?(bucket_name) - rescue => e - log_output[:function] = 'bucketExists?(bucket_name)' - log_output[:args] = {'bucket_name': bucket_name} - raise e - end - - def presignedGet(bucket_name, file_name) - # Returns download/get url - obj = @@s3.bucket(bucket_name).object(file_name) - return obj.presigned_url(:get, expires_in: 600) - rescue => e - raise e - end - - def presignedGetWrapper(bucket_name, file_name, log_output) - presignedGet(bucket_name, file_name) - rescue => e - log_output[:function] = 'presignedGet(bucket_name, file_name)' - log_output[:args] = {'bucket_name': bucket_name, - 'file_name': file_name} - raise e - end - - def presignedPut(bucket_name, file_name) - # Returns put url - obj = @@s3.bucket(bucket_name).object(file_name) - return obj.presigned_url(:put, expires_in: 600) - rescue => e - raise e - end - - def presignedPutWrapper(bucket_name, file_name, log_output) - presignedPut(bucket_name, file_name) - rescue => e - log_output[:function] = 'presignedPut(bucket_name, file_name)' - log_output[:args] = {'bucket_name': bucket_name, - 'file_name': file_name} - raise e - end - - def presignedPost(bucket_name, file_name, expires_in_sec, max_byte_size) - # Returns upload/post url - obj = @@s3.bucket(bucket_name).object(file_name) - return obj.presigned_post(expires: Time.now + expires_in_sec, - content_length_range: 1..max_byte_size) - rescue => e - raise e - end - - def presignedPostWrapper(bucket_name, file_name, expires_in_sec, max_byte_size, log_output) - presignedPost(bucket_name, file_name, expires_in_sec, max_byte_size) - rescue => e - log_output[:function] = 'presignedPost(bucket_name, file_name, expires_in_sec, max_byte_size)' - log_output[:args] = {'bucket_name': bucket_name, - 'file_name': file_name, - 'expires_in_sec': expires_in_sec, - 'max_byte_size': max_byte_size} - raise e - end - - # To be addressed. S3 API 'get_bucket_policy' does not work! - # def getBucketPolicy(bucket_name) - # # Returns bucket policy - # return @@s3.bucket(bucket_name).get_bucket_policy - # rescue => e - # raise e - # end - - # - # Test case methods - # - def listBucketsTest() - # Tests listBuckets api command by creating - # new buckets from bucket_name_list - - # get random bucket names and create list - bucket_name1 = get_random_bucket_name() - bucket_name2 = get_random_bucket_name() - bucket_name_list = [bucket_name1, bucket_name2] - # Initialize hash table, 'log_output' - log_output = initialize_log_output('listBuckets') - # Prepare arg/value hash table and set it in log_output - arg_value_hash = {} - log_output[:args].each { |x| arg_value_hash[:"#{x}"] = eval x.to_s } - log_output[:args] = arg_value_hash - - begin - start_time = Time.now - prev_total_buckets = listBucketsWrapper(log_output).length - new_buckets = bucket_name_list.length - bucket_name_list.each do |b| - makeBucketWrapper(b, log_output) - end - new_total_buckets = prev_total_buckets + new_buckets - if new_total_buckets >= prev_total_buckets + new_buckets - log_output[:status] = 'PASS' - else - log_output[:error] = 'Could not find expected number of buckets' - log_output[:status] = 'FAIL' - end - cleanUp(bucket_name_list, log_output) - rescue => log_output[:error] - log_output[:status] = 'FAIL' - end - - print_log(log_output, start_time) - end - - def makeBucketTest() - # Tests makeBucket api command. - - # get random bucket name - bucket_name = get_random_bucket_name() - # Initialize hash table, 'log_output' - log_output = initialize_log_output('makeBucket') - # Prepare arg/value hash table and set it in log_output - arg_value_hash = {} - log_output[:args].each { |x| arg_value_hash[:"#{x}"] = eval x.to_s } - log_output[:args] = arg_value_hash - - begin - start_time = Time.now - makeBucketWrapper(bucket_name, log_output) - - if bucketExistsWrapper(bucket_name, log_output) - log_output[:status] = 'PASS' - else - log_output[:error] = 'Bucket expected to be created does not exist' - log_output[:status] = 'FAIL' - end - cleanUp([bucket_name], log_output) - rescue => log_output[:error] - log_output[:status] = 'FAIL' - end - - print_log(log_output, start_time) - end - - def bucketExistsNegativeTest() - # Tests bucketExists api command. - - # get random bucket name - bucket_name = get_random_bucket_name() - # Initialize hash table, 'log_output' - log_output = initialize_log_output('bucketExists?') - # Prepare arg/value hash table and set it in log_output - arg_value_hash = {} - log_output[:args].each { |x| arg_value_hash[:"#{x}"] = eval x.to_s } - log_output[:args] = arg_value_hash - - begin - start_time = Time.now - if !bucketExistsWrapper(bucket_name, log_output) - log_output[:status] = 'PASS' - else - log_output[:error] = "Failed to return 'false' for a non-existing bucket" - log_output[:status] = 'FAIL' - end - cleanUp([bucket_name], log_output) - rescue => log_output[:error] - log_output[:status] = 'FAIL' - end - - print_log(log_output, start_time) - end - - def removeBucketTest() - # Tests removeBucket api command. - - # get a random bucket name - bucket_name = get_random_bucket_name() - # Initialize hash table, 'log_output' - log_output = initialize_log_output('removeBucket') - # Prepare arg/value hash table and set it in log_output - arg_value_hash = {} - log_output[:args].each { |x| arg_value_hash[:"#{x}"] = eval x.to_s } - log_output[:args] = arg_value_hash - - begin - start_time = Time.now - makeBucketWrapper(bucket_name, log_output) - removeBucketWrapper(bucket_name, log_output) - if !bucketExistsWrapper(bucket_name, log_output) - log_output[:status] = 'PASS' - else - log_output[:error] = 'Bucket expected to be removed still exists' - log_output[:status] = 'FAIL' - end - cleanUp([bucket_name], log_output) - rescue => log_output[:error] - log_output[:status] = 'FAIL' - end - - print_log(log_output, start_time) - end - - def putObjectTest(file) - # Tests putObject api command by uploading a file - - # get random bucket name - bucket_name = get_random_bucket_name() - # Initialize hash table, 'log_output' - log_output = initialize_log_output('putObject') - # Prepare arg/value hash table and set it in log_output - arg_value_hash = {} - log_output[:args].each { |x| arg_value_hash[:"#{x}"] = eval x.to_s } - log_output[:args] = arg_value_hash - - begin - start_time = Time.now - makeBucketWrapper(bucket_name, log_output) - putObjectWrapper(bucket_name, file, log_output) - if statObjectWrapper(bucket_name, File.basename(file), log_output) - log_output[:status] = 'PASS' - else - log_output[:error] = "Status for the created object returned 'false'" - log_output[:status] = 'FAIL' - end - cleanUp([bucket_name], log_output) - rescue => log_output[:error] - log_output[:status] = 'FAIL' - end - - print_log(log_output, start_time) - end - - def removeObjectTest(file) - # Tests removeObject api command by uploading and removing a file - - # get random bucket name - bucket_name = get_random_bucket_name() - # Initialize hash table, 'log_output' - log_output = initialize_log_output('removeObject') - # Prepare arg/value hash table and set it in log_output - arg_value_hash = {} - log_output[:args].each { |x| arg_value_hash[:"#{x}"] = eval x.to_s } - log_output[:args] = arg_value_hash - - begin - start_time = Time.now - makeBucketWrapper(bucket_name, log_output) - putObjectWrapper(bucket_name, file, log_output) - removeObjectWrapper(bucket_name, File.basename(file), log_output) - if !statObjectWrapper(bucket_name, File.basename(file), log_output) - log_output[:status] = 'PASS' - else - log_output[:error] = "Status for the removed object returned 'true'" - log_output[:status] = 'FAIL' - end - cleanUp([bucket_name], log_output) - rescue => log_output[:error] - log_output[:status] = 'FAIL' - end - - print_log(log_output, start_time) - end - - def getObjectTest(file, destination) - # Tests getObject api command - - # get random bucket name - bucket_name = get_random_bucket_name() - # Initialize hash table, 'log_output' - log_output = initialize_log_output('getObject') - # Prepare arg/value hash table and set it in log_output - arg_value_hash = {} - log_output[:args].each { |x| arg_value_hash[:"#{x}"] = eval x.to_s } - log_output[:args] = arg_value_hash - - begin - start_time = Time.now - makeBucketWrapper(bucket_name, log_output) - putObjectWrapper(bucket_name, file, log_output) - getObjectWrapper(bucket_name, file, destination, log_output) - if system("ls -l #{destination} > /dev/null") - log_output[:status] = 'PASS' - else - log_output[:error] = "Downloaded object does not exist at #{destination}" - log_output[:status] = 'FAIL' - end - cleanUp([bucket_name], log_output) - rescue => log_output[:error] - log_output[:status] = 'FAIL' - end - - print_log(log_output, start_time) - end - - def listObjectsTest(file_list) - # Tests listObjects api command - - # get random bucket name - bucket_name = get_random_bucket_name() - # Initialize hash table, 'log_output' - log_output = initialize_log_output('listObjects') - # Prepare arg/value hash table and set it in log_output - arg_value_hash = {} - log_output[:args].each { |x| arg_value_hash[:"#{x}"] = eval x.to_s } - log_output[:args] = arg_value_hash - - begin - start_time = Time.now - makeBucketWrapper(bucket_name, log_output) - # Put all objects into the bucket - file_list.each do |f| - putObjectWrapper(bucket_name, f, log_output) - end - # Total number of files uploaded - expected_no = file_list.length - # Actual number is what api returns - actual_no = listObjectsWrapper(bucket_name, log_output).length - # Compare expected and actual values - if expected_no == actual_no - log_output[:status] = 'PASS' - else - log_output[:error] = 'Expected and actual number of listed files/objects do not match!' - log_output[:status] = 'FAIL' - end - cleanUp([bucket_name], log_output) - rescue => log_output[:error] - log_output[:status] = 'FAIL' - end - - print_log(log_output, start_time) - end - - def copyObjectTest(data_dir, source_file_name, target_file_name = '') - # Tests copyObject api command - - # get random bucket names - source_bucket_name = get_random_bucket_name() - target_bucket_name = get_random_bucket_name() - # Initialize hash table, 'log_output' - log_output = initialize_log_output('copyObject') - # Prepare arg/value hash table and set it in log_output - arg_value_hash = {} - log_output[:args].each { |x| arg_value_hash[:"#{x}"] = eval x.to_s } - log_output[:args] = arg_value_hash - - begin - start_time = Time.now - target_file_name = source_file_name if target_file_name.empty? - makeBucketWrapper(source_bucket_name, log_output) - makeBucketWrapper(target_bucket_name, log_output) - putObjectWrapper(source_bucket_name, - File.join(data_dir, source_file_name), log_output) - copyObjectWrapper(source_bucket_name, target_bucket_name, - source_file_name, target_file_name, log_output) - # Check if copy worked fine - if statObjectWrapper(target_bucket_name, target_file_name, log_output) - log_output[:status] = 'PASS' - else - log_output[:error] = 'Copied file could not be found in the expected location' - log_output[:status] = 'FAIL' - end - cleanUp([source_bucket_name, target_bucket_name], log_output) - rescue => log_output[:error] - log_output[:status] = 'FAIL' - end - - print_log(log_output, start_time) - end - - def presignedGetObjectTest(data_dir, file_name) - # Tests presignedGetObject api command - - # get random bucket name - bucket_name = get_random_bucket_name() - # Initialize hash table, 'log_output' - log_output = initialize_log_output('presignedGet') - # Prepare arg/value hash table and set it in log_output - arg_value_hash = {} - log_output[:args].each { |x| arg_value_hash[:"#{x}"] = eval x.to_s } - log_output[:args] = arg_value_hash - - begin - start_time = Time.now - makeBucketWrapper(bucket_name, log_output) - file = File.join(data_dir, file_name) - # Get check sum value without the file name - cksum_orig = `cksum #{file}`.split[0..1] - putObjectWrapper(bucket_name, file, log_output) - get_url = presignedGetWrapper(bucket_name, file_name, log_output) - # Download the file using the URL - # generated by presignedGet api command - `wget -O /tmp/#{file_name}, '#{get_url}' > /dev/null 2>&1` - # Get check sum value for the downloaded file - # Split to get rid of the file name - cksum_new = `cksum /tmp/#{file_name}`.split[0..1] - - # Check if check sum values for the orig file - # and the downloaded file match - if cksum_orig == cksum_new - log_output[:status] = 'PASS' - else - log_output[:error] = 'Check sum values do NOT match' - log_output[:status] = 'FAIL' - end - cleanUp([bucket_name], log_output) - rescue => log_output[:error] - log_output[:status] = 'FAIL' - end - - print_log(log_output, start_time) - end - - def presignedPutObjectTest(data_dir, file_name) - # Tests presignedPutObject api command - - # get random bucket name - bucket_name = get_random_bucket_name() - # Initialize hash table, 'log_output' - log_output = initialize_log_output('presignedPut') - # Prepare arg/value hash table and set it in log_output - arg_value_hash = {} - log_output[:args].each { |x| arg_value_hash[:"#{x}"] = eval x.to_s } - log_output[:args] = arg_value_hash - - begin - start_time = Time.now - makeBucketWrapper(bucket_name, log_output) - file = File.join(data_dir, file_name) - - # Get check sum value and - # split to get rid of the file name - cksum_orig = `cksum #{file}`.split[0..1] - - # Generate presigned Put URL and parse it - uri = URI.parse(presignedPutWrapper(bucket_name, file_name, log_output)) - request = Net::HTTP::Put.new(uri.request_uri, 'x-amz-acl' => 'public-read') - request.body = IO.read(File.join(data_dir, file_name)) - - http = Net::HTTP.new(uri.host, uri.port) - http.use_ssl = true if ENV['ENABLE_HTTPS'] == '1' - - http.request(request) - - if statObjectWrapper(bucket_name, file_name, log_output) - getObjectWrapper(bucket_name, file_name, '/tmp', log_output) - cksum_new = `cksum /tmp/#{file_name}`.split[0..1] - # Check if check sum values of the orig file - # and the downloaded file match - if cksum_orig == cksum_new - log_output[:status] = 'PASS' - else - log_output[:error] = 'Check sum values do NOT match' - log_output[:status] = 'FAIL' - end - else - log_output[:error] = 'Expected to be created object does NOT exist' - log_output[:status] = 'FAIL' - end - cleanUp([bucket_name], log_output) - rescue => log_output[:error] - log_output[:status] = 'FAIL' - end - - print_log(log_output, start_time) - end - - def presignedPostObjectTest(data_dir, file_name, - expires_in_sec, max_byte_size) - # Tests presignedPostObject api command - - # get random bucket name - bucket_name = get_random_bucket_name() - # Initialize hash table, 'log_output' - log_output = initialize_log_output('presignedPost') - # Prepare arg/value hash table and set it in log_output - arg_value_hash = {} - log_output[:args].each { |x| arg_value_hash[:"#{x}"] = eval x.to_s } - log_output[:args] = arg_value_hash - - begin - start_time = Time.now - makeBucketWrapper(bucket_name, log_output) - file = File.join(data_dir, file_name) - - # Get check sum value and split it - # into parts to get rid of the file name - cksum_orig = `cksum #{file}`.split[0..1] - # Create the presigned POST url - post = presignedPostWrapper(bucket_name, file_name, - expires_in_sec, max_byte_size, log_output) - - # Prepare multi parts array for POST command request - file_part = Part.new name: 'file', - body: IO.read(File.join(data_dir, file_name)), - filename: file_name, - content_type: 'application/octet-stream' - parts = [file_part] - # Add POST fields into parts array - post.fields.each do |field, value| - parts.push(Part.new(field, value)) - end - boundary = "---------------------------#{rand(10_000_000_000_000_000)}" - body_parts = MultipartBody.new parts, boundary - - # Parse presigned Post URL - uri = URI.parse(post.url) - - # Create the HTTP objects - http = Net::HTTP.new(uri.host, uri.port) - http.use_ssl = true if ENV['ENABLE_HTTPS'] == '1' - request = Net::HTTP::Post.new(uri.request_uri) - request.body = body_parts.to_s - request.content_type = "multipart/form-data; boundary=#{boundary}" - # Send the request - log_output[:error] = http.request(request) - - if statObjectWrapper(bucket_name, file_name, log_output) - getObjectWrapper(bucket_name, file_name, '/tmp', log_output) - cksum_new = `cksum /tmp/#{file_name}`.split[0..1] - # Check if check sum values of the orig file - # and the downloaded file match - if cksum_orig == cksum_new - log_output[:status] = 'PASS' - # FIXME: HTTP No Content error, status code=204 is returned as error - log_output[:error] = nil - else - log_output[:error] = 'Check sum values do NOT match' - log_output[:status] = 'FAIL' - end - else - log_output[:error] = 'Expected to be created object does NOT exist' - log_output[:status] = 'FAIL' - end - cleanUp([bucket_name], log_output) - rescue => log_output[:error] - log_output[:status] = 'FAIL' - end - - print_log(log_output, start_time) - end -end - -# MAIN CODE - -# Create test Class instance and call the tests -aws = AwsSdkRubyTest.new -file_name1 = 'datafile-1-kB' -file_new_name = 'datafile-1-kB-copy' -file_name_list = ['datafile-1-kB', 'datafile-1-b', 'datafile-6-MB'] -# Add data_dir in front of each file name in file_name_list -# The location where the bucket and file -# objects are going to be created. -data_dir = ENV['MINT_DATA_DIR'] ||= 'MINT_DATA_DIR is not set' -file_list = file_name_list.map { |f| File.join(data_dir, f) } -destination = '/tmp' - -aws.listBucketsTest() -aws.listObjectsTest(file_list) -aws.makeBucketTest() -aws.bucketExistsNegativeTest() -aws.removeBucketTest() -aws.putObjectTest(File.join(data_dir, file_name1)) -aws.removeObjectTest(File.join(data_dir, file_name1)) -aws.getObjectTest(File.join(data_dir, file_name1), destination) -aws.copyObjectTest(data_dir, file_name1) -aws.copyObjectTest(data_dir, file_name1, file_new_name) -aws.presignedGetObjectTest(data_dir, file_name1) -aws.presignedPutObjectTest(data_dir, file_name1) -aws.presignedPostObjectTest(data_dir, file_name1, 60, 3*1024*1024) diff --git a/mint/run/core/aws-sdk-ruby/run.sh b/mint/run/core/aws-sdk-ruby/run.sh deleted file mode 100755 index 1e59a71c7..000000000 --- a/mint/run/core/aws-sdk-ruby/run.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -# -# Mint (C) 2017 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# handle command line arguments -if [ $# -ne 2 ]; then - echo "usage: run.sh " - exit 1 -fi - -output_log_file="$1" -error_log_file="$2" - -# run tests -chmod a+x aws-stub-tests.rb -ruby aws-stub-tests.rb 1>>"$output_log_file" 2>"$error_log_file" diff --git a/mint/run/core/awscli/README.md b/mint/run/core/awscli/README.md deleted file mode 100644 index b395f9a7c..000000000 --- a/mint/run/core/awscli/README.md +++ /dev/null @@ -1,19 +0,0 @@ -## `awscli` tests -This directory serves as the location for Mint tests using `awscli`. Top level `mint.sh` calls `run.sh` to execute tests. - -## Adding new tests -New tests is added into `test.sh` as new functions. - -## Running tests manually -- Set environment variables `MINT_DATA_DIR`, `MINT_MODE`, `SERVER_ENDPOINT`, `ACCESS_KEY`, `SECRET_KEY`, `SERVER_REGION` and `ENABLE_HTTPS` -- Call `run.sh` with output log file and error log file. for example -```bash -export MINT_DATA_DIR=~/my-mint-dir -export MINT_MODE=core -export SERVER_ENDPOINT="play.minio.io:9000" -export ACCESS_KEY="Q3AM3UQ867SPQQA43P2F" -export SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" -export ENABLE_HTTPS=1 -export SERVER_REGION=us-east-1 -./run.sh /tmp/output.log /tmp/error.log -``` diff --git a/mint/run/core/awscli/run.sh b/mint/run/core/awscli/run.sh deleted file mode 100755 index 98bd61b19..000000000 --- a/mint/run/core/awscli/run.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/bash -# -# Mint (C) 2017 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# handle command line arguments -if [ $# -ne 2 ]; then - echo "usage: run.sh " - exit 1 -fi - -output_log_file="$1" -error_log_file="$2" - -# configure awscli -aws configure set aws_access_key_id "$ACCESS_KEY" -aws configure set aws_secret_access_key "$SECRET_KEY" -aws configure set default.region "$SERVER_REGION" - -# run tests for virtual style if provided -if [ "$ENABLE_VIRTUAL_STYLE" -eq 1 ]; then - # Setup endpoint scheme - endpoint="http://$DOMAIN:$SERVER_PORT" - if [ "$ENABLE_HTTPS" -eq 1 ]; then - endpoint="https://$DOMAIN:$SERVER_PORT" - fi - dnsmasq --address="/$DOMAIN/$SERVER_IP" --user=root - echo -e "nameserver 127.0.0.1\n$(cat /etc/resolv.conf)" > /etc/resolv.conf - aws configure set default.s3.addressing_style virtual - ./test.sh "$endpoint" 1>>"$output_log_file" 2>"$error_log_file" - aws configure set default.s3.addressing_style path -fi - -endpoint="http://$SERVER_ENDPOINT" -if [ "$ENABLE_HTTPS" -eq 1 ]; then - endpoint="https://$SERVER_ENDPOINT" -fi -# run path style tests -./test.sh "$endpoint" 1>>"$output_log_file" 2>"$error_log_file" diff --git a/mint/run/core/awscli/test.sh b/mint/run/core/awscli/test.sh deleted file mode 100755 index 096f4e5d9..000000000 --- a/mint/run/core/awscli/test.sh +++ /dev/null @@ -1,1726 +0,0 @@ -#!/bin/bash -# -# Mint (C) 2017-2020 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -HASH_1_KB=$(md5sum "${MINT_DATA_DIR}/datafile-1-kB" | awk '{print $1}') -HASH_65_MB=$(md5sum "${MINT_DATA_DIR}/datafile-65-MB" | awk '{print $1}') - -_init() { - AWS="aws --endpoint-url $1" -} - -function get_time() { - date +%s%N -} - -function get_duration() { - start_time=$1 - end_time=$(get_time) - - echo $(( (end_time - start_time) / 1000000 )) -} - -function log_success() { - function=$(python -c 'import sys,json; print(json.dumps(sys.stdin.read()))' <<<"$2") - printf '{"name": "awscli", "duration": %d, "function": %s, "status": "PASS"}\n' "$1" "$function" -} - -function log_failure() { - function=$(python -c 'import sys,json; print(json.dumps(sys.stdin.read()))' <<<"$2") - err=$(echo "$3" | tr -d '\n') - printf '{"name": "awscli", "duration": %d, "function": %s, "status": "FAIL", "error": "%s"}\n' "$1" "$function" "$err" -} - -function log_alert() { - function=$(python -c 'import sys,json; print(json.dumps(sys.stdin.read()))' <<<"$2") - err=$(echo "$4" | tr -d '\n') - printf '{"name": "awscli", "duration": %d, "function": %s, "status": "FAIL", "alert": "%s", "error": "%s"}\n' "$1" "$function" "$3" "$err" -} - -function make_bucket() { - # Make bucket - bucket_name="awscli-mint-test-bucket-$RANDOM" - function="${AWS} s3api create-bucket --bucket ${bucket_name}" - - # execute the test - out=$($function 2>&1) - rv=$? - - # if command is successful print bucket_name or print error - if [ $rv -eq 0 ]; then - echo "${bucket_name}" - else - echo "${out}" - fi - - return $rv -} - -function delete_bucket() { - # Delete bucket - function="${AWS} s3 rb s3://${1} --force" - out=$($function 2>&1) - rv=$? - - # echo the output - echo "${out}" - - return $rv -} - -# Tests creating, stat and delete on a bucket. -function test_create_bucket() { - # log start time - start_time=$(get_time) - - function="make_bucket" - bucket_name=$(make_bucket) - rv=$? - # save the ref to function being tested, so it can be logged - test_function=${function} - - # if make_bucket is successful stat the bucket - if [ $rv -eq 0 ]; then - function="${AWS} s3api head-bucket --bucket ${bucket_name}" - out=$($function 2>&1) - rv=$? - else - # if make bucket failes, $bucket_name has the error output - out="${bucket_name}" - fi - - # if stat bucket is successful remove the bucket - if [ $rv -eq 0 ]; then - function="delete_bucket" - out=$(delete_bucket "${bucket_name}") - rv=$? - else - # if make bucket failes, $bucket_name has the error output - out="${bucket_name}" - fi - - if [ $rv -eq 0 ]; then - log_success "$(get_duration "$start_time")" "${test_function}" - else - # clean up and log error - ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 - log_failure "$(get_duration "$start_time")" "${function}" "${out}" - fi - - return $rv -} - -# Tests creating and deleting an object. -function test_upload_object() { - # log start time - start_time=$(get_time) - - function="make_bucket" - bucket_name=$(make_bucket) - rv=$? - - # if make bucket succeeds upload a file - if [ $rv -eq 0 ]; then - function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB" - out=$($function 2>&1) - rv=$? - else - # if make bucket fails, $bucket_name has the error output - out="${bucket_name}" - fi - - # if upload succeeds download the file - if [ $rv -eq 0 ]; then - function="${AWS} s3api get-object --bucket ${bucket_name} --key datafile-1-kB /tmp/datafile-1-kB" - # save the ref to function being tested, so it can be logged - test_function=${function} - out=$($function 2>&1) - rv=$? - # calculate the md5 hash of downloaded file - hash2=$(md5sum /tmp/datafile-1-kB | awk '{print $1}') - fi - - # if download succeeds, verify downloaded file - if [ $rv -eq 0 ]; then - if [ "$HASH_1_KB" == "$hash2" ]; then - function="delete_bucket" - out=$(delete_bucket "$bucket_name") - rv=$? - # remove download file - rm -f /tmp/datafile-1-kB - else - rv=1 - out="Checksum verification failed for uploaded object" - fi - fi - - if [ $rv -eq 0 ]; then - log_success "$(get_duration "$start_time")" "${test_function}" - else - # clean up and log error - ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 - log_failure "$(get_duration "$start_time")" "${function}" "${out}" - fi - - return $rv -} - -# Test lookup a directory prefix. -function test_lookup_object_prefix() { - # log start time - start_time=$(get_time) - - function="make_bucket" - bucket_name=$(make_bucket) - rv=$? - - # if make bucket succeeds create a directory. - if [ $rv -eq 0 ]; then - function="${AWS} s3api put-object --bucket ${bucket_name} --key prefix/directory/" - # save the ref to function being tested, so it can be logged - test_function=${function} - - out=$($function 2>&1) - - rv=$? - else - # if make_bucket fails, $bucket_name has the error output - out="${bucket_name}" - fi - - if [ $rv -eq 0 ]; then - ## Attempt an overwrite of the prefix again and should succeed as well. - function="${AWS} s3api put-object --bucket ${bucket_name} --key prefix/directory/" - # save the ref to function being tested, so it can be logged - test_function=${function} - out=$($function 2>&1) - rv=$? - fi - - # if upload succeeds lookup for the prefix. - if [ $rv -eq 0 ]; then - function="${AWS} s3api head-object --bucket ${bucket_name} --key prefix/directory/" - # save the ref to function being tested, so it can be logged - test_function=${function} - out=$($function 2>&1) - rv=$? - fi - - # if directory create succeeds, upload the object. - if [ $rv -eq 0 ]; then - function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key prefix/directory/datafile-1-kB" - # save the ref to function being tested, so it can be logged - test_function=${function} - out=$($function 2>&1) - rv=$? - fi - - # Attempt a delete on prefix shouldn't delete the directory since we have an object inside it. - if [ $rv -eq 0 ]; then - function="${AWS} s3api delete-object --bucket ${bucket_name} --key prefix/directory/" - # save the ref to function being tested, so it can be logged - test_function=${function} - out=$($function 2>&1) - rv=$? - fi - - # if upload succeeds lookup for the object should succeed. - if [ $rv -eq 0 ]; then - function="${AWS} s3api head-object --bucket ${bucket_name} --key prefix/directory/datafile-1-kB" - # save the ref to function being tested, so it can be logged - test_function=${function} - out=$($function 2>&1) - rv=$? - fi - - # delete bucket - if [ $rv -eq 0 ]; then - function="delete_bucket" - out=$(delete_bucket "$bucket_name") - rv=$? - fi - - if [ $rv -ne 0 ]; then - # clean up and log error - ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 - log_failure "$(get_duration "$start_time")" "${function}" "${out}" - else - log_success "$(get_duration "$start_time")" "${test_function}" - fi - - return $rv -} - -# Tests listing objects for both v1 and v2 API. -function test_list_objects() { - # log start time - start_time=$(get_time) - - function="make_bucket" - bucket_name=$(make_bucket) - rv=$? - - # if make bucket succeeds upload a file - if [ $rv -eq 0 ]; then - function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB" - out=$($function 2>&1) - rv=$? - else - # if make bucket fails, $bucket_name has the error output - out="${bucket_name}" - fi - - # if upload objects succeeds, list objects with existing prefix - if [ $rv -eq 0 ]; then - function="${AWS} s3api list-objects --bucket ${bucket_name} --prefix datafile-1-kB" - test_function=${function} - out=$($function) - rv=$? - key_name=$(echo "$out" | jq -r .Contents[].Key) - if [ $rv -eq 0 ] && [ "$key_name" != "datafile-1-kB" ]; then - rv=1 - # since rv is 0, command passed, but didn't return expected value. In this case set the output - out="list-objects with existing prefix failed" - fi - fi - - # if upload objects succeeds, list objects without existing prefix - if [ $rv -eq 0 ]; then - function="${AWS} s3api list-objects --bucket ${bucket_name} --prefix linux" - out=$($function) - rv=$? - key_name=$(echo "$out" | jq -r .Contents[].Key) - if [ $rv -eq 0 ] && [ "$key_name" != "" ]; then - rv=1 - out="list-objects without existing prefix failed" - fi - fi - - # if upload objects succeeds, list objectsv2 with existing prefix - if [ $rv -eq 0 ]; then - function="${AWS} s3api list-objects-v2 --bucket ${bucket_name} --prefix datafile-1-kB" - out=$($function) - rv=$? - key_name=$(echo "$out" | jq -r .Contents[].Key) - if [ $rv -eq 0 ] && [ "$key_name" != "datafile-1-kB" ]; then - rv=1 - out="list-objects-v2 with existing prefix failed" - fi - fi - - # if upload objects succeeds, list objectsv2 without existing prefix - if [ $rv -eq 0 ]; then - function="${AWS} s3api list-objects-v2 --bucket ${bucket_name} --prefix linux" - out=$($function) - rv=$? - key_name=$(echo "$out" | jq -r .Contents[].Key) - if [ $rv -eq 0 ] && [ "$key_name" != "" ]; then - rv=1 - out="list-objects-v2 without existing prefix failed" - fi - fi - - if [ $rv -eq 0 ]; then - function="delete_bucket" - out=$(delete_bucket "$bucket_name") - rv=$? - # remove download file - rm -f /tmp/datafile-1-kB - fi - - if [ $rv -eq 0 ]; then - log_success "$(get_duration "$start_time")" "${test_function}" - else - # clean up and log error - ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 - rm -f /tmp/datafile-1-kB - log_failure "$(get_duration "$start_time")" "${function}" "${out}" - fi - - return $rv -} - -# Tests multipart API with 0 byte part. -function test_multipart_upload_0byte() { - # log start time - start_time=$(get_time) - - function="make_bucket" - bucket_name=$(make_bucket) - object_name=${bucket_name}"-object" - rv=$? - - # if make bucket succeeds upload a file - if [ $rv -eq 0 ]; then - function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-0-b --bucket ${bucket_name} --key datafile-0-b" - out=$($function 2>&1) - rv=$? - else - # if make bucket fails, $bucket_name has the error output - out="${bucket_name}" - fi - - if [ $rv -eq 0 ]; then - # create multipart - function="${AWS} s3api create-multipart-upload --bucket ${bucket_name} --key ${object_name}" - test_function=${function} - out=$($function) - rv=$? - upload_id=$(echo "$out" | jq -r .UploadId) - fi - - if [ $rv -eq 0 ]; then - # Capture etag for part-number 1 - function="${AWS} s3api upload-part --bucket ${bucket_name} --key ${object_name} --body ${MINT_DATA_DIR}/datafile-0-b --upload-id ${upload_id} --part-number 1" - out=$($function) - rv=$? - etag1=$(echo "$out" | jq -r .ETag) - fi - - if [ $rv -eq 0 ]; then - # Create a multipart struct file for completing multipart transaction - echo "{ - \"Parts\": [ - { - \"ETag\": ${etag1}, - \"PartNumber\": 1 - } - ] - }" >> /tmp/multipart - fi - - if [ $rv -eq 0 ]; then - # Use saved etags to complete the multipart transaction - function="${AWS} s3api complete-multipart-upload --multipart-upload file:///tmp/multipart --bucket ${bucket_name} --key ${object_name} --upload-id ${upload_id}" - out=$($function) - rv=$? - etag=$(echo "$out" | jq -r .ETag | sed -e 's/^"//' -e 's/"$//') - if [ "${etag}" == "" ]; then - rv=1 - out="complete-multipart-upload failed" - fi - fi - - if [ $rv -eq 0 ]; then - function="${AWS} s3api get-object --bucket ${bucket_name} --key ${object_name} /tmp/datafile-0-b" - test_function=${function} - out=$($function 2>&1) - rv=$? - fi - - if [ $rv -eq 0 ]; then - ret_etag=$(echo "$out" | jq -r .ETag | sed -e 's/^"//' -e 's/"$//') - # match etag - if [ "$etag" != "$ret_etag" ]; then - rv=1 - out="Etag mismatch for multipart 0 byte object" - fi - rm -f /tmp/datafile-0-b - fi - - if [ $rv -eq 0 ]; then - function="delete_bucket" - out=$(delete_bucket "$bucket_name") - rv=$? - # remove temp file - rm -f /tmp/multipart - fi - - if [ $rv -eq 0 ]; then - log_success "$(get_duration "$start_time")" "${test_function}" - else - # clean up and log error - ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 - rm -f /tmp/multipart - log_failure "$(get_duration "$start_time")" "${function}" "${out}" - fi - - return $rv -} - -# Tests multipart API by making each individual calls. -function test_multipart_upload() { - # log start time - start_time=$(get_time) - - function="make_bucket" - bucket_name=$(make_bucket) - object_name=${bucket_name}"-object" - rv=$? - - # if make bucket succeeds upload a file - if [ $rv -eq 0 ]; then - function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB" - out=$($function 2>&1) - rv=$? - else - # if make bucket fails, $bucket_name has the error output - out="${bucket_name}" - fi - - if [ $rv -eq 0 ]; then - # create multipart - function="${AWS} s3api create-multipart-upload --bucket ${bucket_name} --key ${object_name}" - test_function=${function} - out=$($function) - rv=$? - upload_id=$(echo "$out" | jq -r .UploadId) - fi - - if [ $rv -eq 0 ]; then - # Capture etag for part-number 1 - function="${AWS} s3api upload-part --bucket ${bucket_name} --key ${object_name} --body ${MINT_DATA_DIR}/datafile-5-MB --upload-id ${upload_id} --part-number 1" - out=$($function) - rv=$? - etag1=$(echo "$out" | jq -r .ETag) - fi - - if [ $rv -eq 0 ]; then - # Capture etag for part-number 2 - function="${AWS} s3api upload-part --bucket ${bucket_name} --key ${object_name} --body ${MINT_DATA_DIR}/datafile-1-kB --upload-id ${upload_id} --part-number 2" - out=$($function) - rv=$? - etag2=$(echo "$out" | jq -r .ETag) - # Create a multipart struct file for completing multipart transaction - echo "{ - \"Parts\": [ - { - \"ETag\": ${etag1}, - \"PartNumber\": 1 - }, - { - \"ETag\": ${etag2}, - \"PartNumber\": 2 - } - ] - }" >> /tmp/multipart - fi - - if [ $rv -eq 0 ]; then - # Use saved etags to complete the multipart transaction - function="${AWS} s3api complete-multipart-upload --multipart-upload file:///tmp/multipart --bucket ${bucket_name} --key ${object_name} --upload-id ${upload_id}" - out=$($function) - rv=$? - finalETag=$(echo "$out" | jq -r .ETag | sed -e 's/^"//' -e 's/"$//') - if [ "${finalETag}" == "" ]; then - rv=1 - out="complete-multipart-upload failed" - fi - fi - - if [ $rv -eq 0 ]; then - function="delete_bucket" - out=$(delete_bucket "$bucket_name") - rv=$? - # remove temp file - rm -f /tmp/multipart - fi - - if [ $rv -eq 0 ]; then - log_success "$(get_duration "$start_time")" "${test_function}" - else - # clean up and log error - ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 - rm -f /tmp/multipart - log_failure "$(get_duration "$start_time")" "${function}" "${out}" - fi - - return $rv -} - -# List number of objects based on the maxKey -# value set. -function test_max_key_list() { - # log start time - start_time=$(get_time) - - function="make_bucket" - bucket_name=$(make_bucket) - rv=$? - - # if make bucket succeeds upload a file - if [ $rv -eq 0 ]; then - function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-b --bucket ${bucket_name} --key datafile-1-b" - out=$($function 2>&1) - rv=$? - else - # if make bucket fails, $bucket_name has the error output - out="${bucket_name}" - fi - - # copy object server side - if [ $rv -eq 0 ]; then - function="${AWS} s3api copy-object --bucket ${bucket_name} --key datafile-1-b-copy --copy-source ${bucket_name}/datafile-1-b" - out=$($function) - rv=$? - fi - - if [ $rv -eq 0 ]; then - function="${AWS} s3api list-objects-v2 --bucket ${bucket_name} --max-keys 1" - test_function=${function} - out=$($function 2>&1) - rv=$? - if [ $rv -eq 0 ]; then - out=$(echo "$out" | jq '.KeyCount') - rv=$? - fi - fi - - if [ $rv -eq 0 ]; then - function="delete_bucket" - out=$(delete_bucket "$bucket_name") - rv=$? - # The command passed, but the delete_bucket failed - out="delete_bucket for test_max_key_list failed" - fi - - if [ $rv -eq 0 ]; then - log_success "$(get_duration "$start_time")" "${test_function}" - else - # clean up and log error - ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 - log_failure "$(get_duration "$start_time")" "${function}" "${out}" - fi - - return $rv -} - -# Copy object tests for server side copy -# of the object, validates returned md5sum. -function test_copy_object() { - # log start time - start_time=$(get_time) - - function="make_bucket" - bucket_name=$(make_bucket) - rv=$? - - # if make bucket succeeds upload a file - if [ $rv -eq 0 ]; then - function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB" - out=$($function 2>&1) - rv=$? - else - # if make bucket fails, $bucket_name has the error output - out="${bucket_name}" - fi - - # copy object server side - if [ $rv -eq 0 ]; then - function="${AWS} s3api copy-object --bucket ${bucket_name} --key datafile-1-kB-copy --copy-source ${bucket_name}/datafile-1-kB" - test_function=${function} - out=$($function) - rv=$? - hash2=$(echo "$out" | jq -r .CopyObjectResult.ETag | sed -e 's/^"//' -e 's/"$//') - if [ $rv -eq 0 ] && [ "$HASH_1_KB" != "$hash2" ]; then - # Verification failed - rv=1 - out="Hash mismatch expected $HASH_1_KB, got $hash2" - fi - fi - - ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 - if [ $rv -eq 0 ]; then - log_success "$(get_duration "$start_time")" "${test_function}" - else - log_failure "$(get_duration "$start_time")" "${function}" "${out}" - fi - - return $rv -} - -# Copy object tests for server side copy -# of the object, validates returned md5sum. -# validates change in storage class as well -function test_copy_object_storage_class() { - # log start time - start_time=$(get_time) - - function="make_bucket" - bucket_name=$(make_bucket) - rv=$? - - # if make bucket succeeds upload a file - if [ $rv -eq 0 ]; then - function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB" - out=$($function 2>&1) - rv=$? - else - # if make bucket fails, $bucket_name has the error output - out="${bucket_name}" - fi - - # copy object server side - if [ $rv -eq 0 ]; then - function="${AWS} s3api copy-object --bucket ${bucket_name} --storage-class REDUCED_REDUNDANCY --key datafile-1-kB-copy --copy-source ${bucket_name}/datafile-1-kB" - test_function=${function} - out=$($function 2>&1) - rv=$? - # if this functionality is not implemented return right away. - if [ $rv -ne 0 ]; then - if echo "$out" | grep -q "NotImplemented"; then - ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 - return 0 - fi - fi - hash2=$(echo "$out" | jq -r .CopyObjectResult.ETag | sed -e 's/^"//' -e 's/"$//') - if [ $rv -eq 0 ] && [ "$HASH_1_KB" != "$hash2" ]; then - # Verification failed - rv=1 - out="Hash mismatch expected $HASH_1_KB, got $hash2" - fi - # if copy succeeds stat the object - if [ $rv -eq 0 ]; then - function="${AWS} s3api head-object --bucket ${bucket_name} --key datafile-1-kB-copy" - # save the ref to function being tested, so it can be logged - test_function=${function} - out=$($function 2>&1) - storageClass=$(echo "$out" | jq -r .StorageClass) - rv=$? - fi - # if head-object succeeds, verify metadata has storage class - if [ $rv -eq 0 ]; then - if [ "${storageClass}" == "null" ]; then - rv=1 - out="StorageClass was not applied" - elif [ "${storageClass}" == "STANDARD" ]; then - rv=1 - out="StorageClass was applied incorrectly" - fi - fi - fi - - ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 - if [ $rv -eq 0 ]; then - log_success "$(get_duration "$start_time")" "${test_function}" - else - log_failure "$(get_duration "$start_time")" "${function}" "${out}" - fi - - return $rv -} - -# Copy object tests for server side copy -# to itself by changing storage class -function test_copy_object_storage_class_same() { - # log start time - start_time=$(get_time) - - function="make_bucket" - bucket_name=$(make_bucket) - rv=$? - - # if make bucket succeeds upload a file - if [ $rv -eq 0 ]; then - function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB" - out=$($function 2>&1) - rv=$? - else - # if make bucket fails, $bucket_name has the error output - out="${bucket_name}" - fi - - # copy object server side - if [ $rv -eq 0 ]; then - function="${AWS} s3api copy-object --bucket ${bucket_name} --storage-class REDUCED_REDUNDANCY --key datafile-1-kB --copy-source ${bucket_name}/datafile-1-kB" - test_function=${function} - out=$($function 2>&1) - rv=$? - # if this functionality is not implemented return right away. - if [ $rv -ne 0 ]; then - if echo "$out" | grep -q "NotImplemented"; then - ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 - return 0 - fi - fi - hash2=$(echo "$out" | jq -r .CopyObjectResult.ETag | sed -e 's/^"//' -e 's/"$//') - if [ $rv -eq 0 ] && [ "$HASH_1_KB" != "$hash2" ]; then - # Verification failed - rv=1 - out="Hash mismatch expected $HASH_1_KB, got $hash2" - fi - # if copy succeeds stat the object - if [ $rv -eq 0 ]; then - function="${AWS} s3api head-object --bucket ${bucket_name} --key datafile-1-kB" - # save the ref to function being tested, so it can be logged - test_function=${function} - out=$($function 2>&1) - storageClass=$(echo "$out" | jq -r .StorageClass) - rv=$? - fi - # if head-object succeeds, verify metadata has storage class - if [ $rv -eq 0 ]; then - if [ "${storageClass}" == "null" ]; then - rv=1 - out="StorageClass was not applied" - elif [ "${storageClass}" == "STANDARD" ]; then - rv=1 - out="StorageClass was applied incorrectly" - fi - fi - fi - - ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 - if [ $rv -eq 0 ]; then - log_success "$(get_duration "$start_time")" "${test_function}" - else - log_failure "$(get_duration "$start_time")" "${function}" "${out}" - fi - - return $rv -} - -# Tests for presigned URL success case, presigned URL -# is correct and accessible - we calculate md5sum of -# the object and validate it against a local files md5sum. -function test_presigned_object() { - # log start time - start_time=$(get_time) - - function="make_bucket" - bucket_name=$(make_bucket) - rv=$? - - # if make bucket succeeds upload a file - if [ $rv -eq 0 ]; then - function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB" - out=$($function 2>&1) - rv=$? - else - # if make bucket fails, $bucket_name has the error output - out="${bucket_name}" - fi - - if [ $rv -eq 0 ]; then - function="${AWS} s3 presign s3://${bucket_name}/datafile-1-kB" - test_function=${function} - url=$($function) - rv=$? - curl -sS -X GET "${url}" > /tmp/datafile-1-kB - hash2=$(md5sum /tmp/datafile-1-kB | awk '{print $1}') - if [ "$HASH_1_KB" == "$hash2" ]; then - function="delete_bucket" - out=$(delete_bucket "$bucket_name") - rv=$? - # remove download file - rm -f /tmp/datafile-1-kB - else - rv=1 - out="Checksum verification failed for downloaded object" - fi - fi - - if [ $rv -eq 0 ]; then - log_success "$(get_duration "$start_time")" "${test_function}" - else - # clean up and log error - ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 - log_failure "$(get_duration "$start_time")" "${function}" "${out}" - fi - - return $rv -} - -# Tests creating and deleting an object - 10MiB -function test_upload_object_10() { - # log start time - start_time=$(get_time) - - function="make_bucket" - bucket_name=$(make_bucket) - rv=$? - - # if make bucket succeeds upload a file - if [ $rv -eq 0 ]; then - function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-10-MB --bucket ${bucket_name} --key datafile-10-MB" - out=$($function 2>&1) - rv=$? - else - # if make bucket fails, $bucket_name has the error output - out="${bucket_name}" - fi - - if [ $rv -eq 0 ]; then - log_success "$(get_duration "$start_time")" "${test_function}" - else - # clean up and log error - ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 - log_failure "$(get_duration "$start_time")" "${function}" "${out}" - fi - - return $rv -} - -# Tests multipart API by making each individual calls with 10MiB part size. -function test_multipart_upload_10() { - # log start time - start_time=$(get_time) - - function="make_bucket" - bucket_name=$(make_bucket) - object_name=${bucket_name}"-object" - rv=$? - - if [ $rv -eq 0 ]; then - # create multipart - function="${AWS} s3api create-multipart-upload --bucket ${bucket_name} --key ${object_name}" - test_function=${function} - out=$($function) - rv=$? - upload_id=$(echo "$out" | jq -r .UploadId) - fi - - if [ $rv -eq 0 ]; then - # Capture etag for part-number 1 - function="${AWS} s3api upload-part --bucket ${bucket_name} --key ${object_name} --body ${MINT_DATA_DIR}/datafile-10-MB --upload-id ${upload_id} --part-number 1" - out=$($function) - rv=$? - etag1=$(echo "$out" | jq -r .ETag) - fi - - if [ $rv -eq 0 ]; then - # Capture etag for part-number 2 - function="${AWS} s3api upload-part --bucket ${bucket_name} --key ${object_name} --body ${MINT_DATA_DIR}/datafile-10-MB --upload-id ${upload_id} --part-number 2" - out=$($function) - rv=$? - etag2=$(echo "$out" | jq -r .ETag) - # Create a multipart struct file for completing multipart transaction - echo "{ - \"Parts\": [ - { - \"ETag\": ${etag1}, - \"PartNumber\": 1 - }, - { - \"ETag\": ${etag2}, - \"PartNumber\": 2 - } - ] - }" >> /tmp/multipart - fi - - if [ $rv -eq 0 ]; then - # Use saved etags to complete the multipart transaction - function="${AWS} s3api complete-multipart-upload --multipart-upload file:///tmp/multipart --bucket ${bucket_name} --key ${object_name} --upload-id ${upload_id}" - out=$($function) - rv=$? - finalETag=$(echo "$out" | jq -r .ETag | sed -e 's/^"//' -e 's/"$//') - if [ "${finalETag}" == "" ]; then - rv=1 - out="complete-multipart-upload failed" - fi - fi - - if [ $rv -eq 0 ]; then - function="delete_bucket" - out=$(delete_bucket "$bucket_name") - rv=$? - # remove temp file - rm -f /tmp/multipart - fi - - if [ $rv -eq 0 ]; then - log_success "$(get_duration "$start_time")" "${test_function}" - else - # clean up and log error - ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 - rm -f /tmp/multipart - log_failure "$(get_duration "$start_time")" "${function}" "${out}" - fi - - return $rv -} - -# Tests `aws s3 cp` by uploading a local file. -function test_aws_s3_cp() { - file_name="${MINT_DATA_DIR}/datafile-65-MB" - - # log start time - start_time=$(get_time) - - function="make_bucket" - bucket_name=$(make_bucket) - rv=$? - - # if make bucket succeeds upload a file using cp - if [ $rv -eq 0 ]; then - function="${AWS} s3 cp $file_name s3://${bucket_name}/$(basename "$file_name")" - test_function=${function} - out=$($function 2>&1) - rv=$? - else - # if make bucket fails, $bucket_name has the error output - out="${bucket_name}" - fi - - if [ $rv -eq 0 ]; then - function="${AWS} s3 rm s3://${bucket_name}/$(basename "$file_name")" - out=$($function 2>&1) - rv=$? - fi - - if [ $rv -eq 0 ]; then - function="${AWS} s3 rb s3://${bucket_name}/" - out=$($function 2>&1) - rv=$? - fi - - if [ $rv -eq 0 ]; then - log_success "$(get_duration "$start_time")" "${test_function}" - else - # clean up and log error - ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 - log_failure "$(get_duration "$start_time")" "${function}" "${out}" - fi - - return $rv -} - -# Tests `aws s3 sync` by mirroring all the -# local content to remove bucket. -function test_aws_s3_sync() { - # log start time - start_time=$(get_time) - - function="make_bucket" - bucket_name=$(make_bucket) - rv=$? - - # if make bucket succeeds sync all the files in a directory - if [ $rv -eq 0 ]; then - function="${AWS} s3 sync --no-progress $MINT_DATA_DIR s3://${bucket_name}/" - test_function=${function} - out=$($function 2>&1) - rv=$? - else - # if make bucket fails, $bucket_name has the error output - out="${bucket_name}" - fi - - # remove files recusively - if [ $rv -eq 0 ]; then - function="${AWS} s3 rm --recursive s3://${bucket_name}/" - out=$($function 2>&1) - rv=$? - fi - - # delete bucket - if [ $rv -eq 0 ]; then - function="delete_bucket" - out=$(delete_bucket "$bucket_name") - rv=$? - fi - - if [ $rv -eq 0 ]; then - log_success "$(get_duration "$start_time")" "${test_function}" - else - # clean up and log error - ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 - log_failure "$(get_duration "$start_time")" "${function}" "${out}" - fi - - return $rv -} - -# list objects negative test - tests for following conditions. -# v1 API with max-keys=-1 and max-keys=0 -# v2 API with max-keys=-1 and max-keys=0 -function test_list_objects_error() { - # log start time - start_time=$(get_time) - - function="make_bucket" - bucket_name=$(make_bucket) - rv=$? - - # if make bucket succeeds upload a file - if [ $rv -eq 0 ]; then - function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB" - out=$($function 2>&1) - rv=$? - else - # if make bucket fails, $bucket_name has the error output - out="${bucket_name}" - fi - - if [ $rv -eq 0 ]; then - # Server replies an error for v1 with max-key=-1 - function="${AWS} s3api list-objects --bucket ${bucket_name} --prefix datafile-1-kB --max-keys=-1" - test_function=${function} - out=$($function 2>&1) - rv=$? - if [ $rv -ne 255 ]; then - rv=1 - else - rv=0 - fi - fi - - if [ $rv -eq 0 ]; then - # Server replies an error for v2 with max-keys=-1 - function="${AWS} s3api list-objects-v2 --bucket ${bucket_name} --prefix datafile-1-kB --max-keys=-1" - test_function=${function} - out=$($function 2>&1) - rv=$? - if [ $rv -ne 255 ]; then - rv=1 - else - rv=0 - fi - fi - - if [ $rv -eq 0 ]; then - # Server returns success with no keys when max-keys=0 - function="${AWS} s3api list-objects-v2 --bucket ${bucket_name} --prefix datafile-1-kB --max-keys=0" - out=$($function 2>&1) - rv=$? - if [ $rv -eq 0 ]; then - function="delete_bucket" - out=$(delete_bucket "$bucket_name") - rv=$? - fi - fi - - if [ $rv -eq 0 ]; then - log_success "$(get_duration "$start_time")" "${test_function}" - else - # clean up and log error - ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 - log_failure "$(get_duration "$start_time")" "${function}" "${out}" - fi - - return $rv -} - -# put object negative test - tests for following conditions. -# - invalid object name. -# - invalid Content-Md5 -# - invalid Content-Length -function test_put_object_error() { - # log start time - start_time=$(get_time) - - function="make_bucket" - bucket_name=$(make_bucket) - rv=$? - - # if make bucket succeeds upload an object without content-md5. - if [ $rv -eq 0 ]; then - function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB --content-md5 invalid" - test_function=${function} - out=$($function 2>&1) - rv=$? - if [ $rv -ne 255 ]; then - rv=1 - else - rv=0 - fi - fi - - # upload an object without content-length. - if [ $rv -eq 0 ]; then - function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB --content-length -1" - test_function=${function} - out=$($function 2>&1) - rv=$? - if [ $rv -ne 255 ]; then - rv=1 - else - rv=0 - fi - fi - - if [ $rv -eq 0 ]; then - function="delete_bucket" - out=$(delete_bucket "$bucket_name") - rv=$? - fi - - if [ $rv -eq 0 ]; then - log_success "$(get_duration "$start_time")" "${test_function}" - else - # clean up and log error - ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 - log_failure "$(get_duration "$start_time")" "${function}" "${out}" - fi - - return $rv -} -# tests server side encryption headers for get and put calls -function test_serverside_encryption() { - #skip server side encryption tests if HTTPS disabled. - if [ "$ENABLE_HTTPS" != "1" ]; then - return 0 - fi - # log start time - start_time=$(get_time) - - function="make_bucket" - bucket_name=$(make_bucket) - rv=$? - - # put object with server side encryption headers - if [ $rv -eq 0 ]; then - function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB --sse-customer-algorithm AES256 --sse-customer-key MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ= --sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg==" - test_function=${function} - out=$($function 2>&1) - rv=$? - fi - # now get encrypted object from server - if [ $rv -eq 0 ]; then - etag1=$(echo "$out" | jq -r .ETag) - sse_customer_key1=$(echo "$out" | jq -r .SSECustomerKeyMD5) - sse_customer_algo1=$(echo "$out" | jq -r .SSECustomerAlgorithm) - - function="${AWS} s3api get-object --bucket ${bucket_name} --key datafile-1-kB --sse-customer-algorithm AES256 --sse-customer-key MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ= --sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg== /tmp/datafile-1-kB" - test_function=${function} - out=$($function 2>&1) - rv=$? - fi - if [ $rv -eq 0 ]; then - etag2=$(echo "$out" | jq -r .ETag) - sse_customer_key2=$(echo "$out" | jq -r .SSECustomerKeyMD5) - sse_customer_algo2=$(echo "$out" | jq -r .SSECustomerAlgorithm) - hash2=$(md5sum /tmp/datafile-1-kB | awk '{print $1}') - # match downloaded object's hash to original - if [ "$HASH_1_KB" == "$hash2" ]; then - function="delete_bucket" - out=$(delete_bucket "$bucket_name") - rv=$? - # remove download file - rm -f /tmp/datafile-1-kB - else - rv=1 - out="Checksum verification failed for downloaded object" - fi - # match etag and SSE headers - if [ "$etag1" != "$etag2" ]; then - rv=1 - out="Etag mismatch for object encrypted with server side encryption" - fi - if [ "$sse_customer_algo1" != "$sse_customer_algo2" ]; then - rv=1 - out="sse customer algorithm mismatch" - fi - if [ "$sse_customer_key1" != "$sse_customer_key2" ]; then - rv=1 - out="sse customer key mismatch" - fi - fi - - if [ $rv -eq 0 ]; then - log_success "$(get_duration "$start_time")" "${test_function}" - else - # clean up and log error - ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 - log_failure "$(get_duration "$start_time")" "${function}" "${out}" - fi - - return $rv -} - -# tests server side encryption headers for multipart put -function test_serverside_encryption_multipart() { - #skip server side encryption tests if HTTPS disabled. - if [ "$ENABLE_HTTPS" != "1" ]; then - return 0 - fi - # log start time - start_time=$(get_time) - - function="make_bucket" - bucket_name=$(make_bucket) - rv=$? - - # put object with server side encryption headers - if [ $rv -eq 0 ]; then - function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-65-MB --bucket ${bucket_name} --key datafile-65-MB --sse-customer-algorithm AES256 --sse-customer-key MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ= --sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg==" - test_function=${function} - out=$($function 2>&1) - rv=$? - fi - # now get encrypted object from server - if [ $rv -eq 0 ]; then - etag1=$(echo "$out" | jq -r .ETag) - sse_customer_key1=$(echo "$out" | jq -r .SSECustomerKeyMD5) - sse_customer_algo1=$(echo "$out" | jq -r .SSECustomerAlgorithm) - - function="${AWS} s3api get-object --bucket ${bucket_name} --key datafile-65-MB --sse-customer-algorithm AES256 --sse-customer-key MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ= --sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg== /tmp/datafile-65-MB" - test_function=${function} - out=$($function 2>&1) - rv=$? - fi - if [ $rv -eq 0 ]; then - etag2=$(echo "$out" | jq -r .ETag) - sse_customer_key2=$(echo "$out" | jq -r .SSECustomerKeyMD5) - sse_customer_algo2=$(echo "$out" | jq -r .SSECustomerAlgorithm) - hash2=$(md5sum /tmp/datafile-65-MB | awk '{print $1}') - # match downloaded object's hash to original - if [ "$HASH_65_MB" == "$hash2" ]; then - function="delete_bucket" - out=$(delete_bucket "$bucket_name") - rv=$? - # remove download file - rm -f /tmp/datafile-65-MB - else - rv=1 - out="Checksum verification failed for downloaded object" - fi - # match etag and SSE headers - if [ "$etag1" != "$etag2" ]; then - rv=1 - out="Etag mismatch for object encrypted with server side encryption" - fi - if [ "$sse_customer_algo1" != "$sse_customer_algo2" ]; then - rv=1 - out="sse customer algorithm mismatch" - fi - if [ "$sse_customer_key1" != "$sse_customer_key2" ]; then - rv=1 - out="sse customer key mismatch" - fi - fi - - if [ $rv -eq 0 ]; then - log_success "$(get_duration "$start_time")" "${test_function}" - else - # clean up and log error - ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 - log_failure "$(get_duration "$start_time")" "${function}" "${out}" - fi - - return $rv -} - -# tests encrypted copy from multipart encrypted object to -# single part encrypted object. This test in particular checks if copy -# succeeds for the case where encryption overhead for individually -# encrypted parts vs encryption overhead for the original datastream -# differs. -function test_serverside_encryption_multipart_copy() { - #skip server side encryption tests if HTTPS disabled. - if [ "$ENABLE_HTTPS" != "1" ]; then - return 0 - fi - # log start time - start_time=$(get_time) - - function="make_bucket" - bucket_name=$(make_bucket) - object_name=${bucket_name}"-object" - rv=$? - - if [ $rv -eq 0 ]; then - # create multipart - function="${AWS} s3api create-multipart-upload --bucket ${bucket_name} --key ${object_name} --sse-customer-algorithm AES256 --sse-customer-key MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ= --sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg==" - out=$($function) - rv=$? - upload_id=$(echo "$out" | jq -r .UploadId) - fi - - if [ $rv -eq 0 ]; then - # Capture etag for part-number 1 - function="${AWS} s3api upload-part --bucket ${bucket_name} --key ${object_name} --body ${MINT_DATA_DIR}/datafile-5243880-b --upload-id ${upload_id} --part-number 1 --sse-customer-algorithm AES256 --sse-customer-key MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ= --sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg==" - out=$($function) - rv=$? - etag1=$(echo "$out" | jq -r .ETag) - fi - - if [ $rv -eq 0 ]; then - # Capture etag for part-number 2 - function="${AWS} s3api upload-part --bucket ${bucket_name} --key ${object_name} --body ${MINT_DATA_DIR}/datafile-5243880-b --upload-id ${upload_id} --part-number 2 --sse-customer-algorithm AES256 --sse-customer-key MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ= --sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg==" - out=$($function) - rv=$? - etag2=$(echo "$out" | jq -r .ETag) - # Create a multipart struct file for completing multipart transaction - echo "{ - \"Parts\": [ - { - \"ETag\": ${etag1}, - \"PartNumber\": 1 - }, - { - \"ETag\": ${etag2}, - \"PartNumber\": 2 - } - ] - }" >> /tmp/multipart - fi - - if [ $rv -eq 0 ]; then - # Use saved etags to complete the multipart transaction - function="${AWS} s3api complete-multipart-upload --multipart-upload file:///tmp/multipart --bucket ${bucket_name} --key ${object_name} --upload-id ${upload_id}" - out=$($function) - rv=$? - finalETag=$(echo "$out" | jq -r .ETag | sed -e 's/^"//' -e 's/"$//') - if [ "${finalETag}" == "" ]; then - rv=1 - out="complete-multipart-upload failed" - fi - fi - - # copy object server side - if [ $rv -eq 0 ]; then - function="${AWS} s3api copy-object --bucket ${bucket_name} --key ${object_name}-copy --copy-source ${bucket_name}/${object_name} --copy-source-sse-customer-algorithm AES256 --copy-source-sse-customer-key MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ= --copy-source-sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg== --sse-customer-algorithm AES256 --sse-customer-key MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ= --sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg==" - test_function=${function} - out=$($function) - rv=$? - if [ $rv -ne 255 ]; then - rv=1 - else - rv=0 - fi - fi - - if [ $rv -eq 0 ]; then - log_success "$(get_duration "$start_time")" "${test_function}" - else - # clean up and log error - ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 - rm -f /tmp/multipart - log_failure "$(get_duration "$start_time")" "${function}" "${out}" - fi - - return $rv -} -# tests server side encryption headers for range get calls -function test_serverside_encryption_get_range() { - #skip server side encryption tests if HTTPS disabled. - if [ "$ENABLE_HTTPS" != "1" ]; then - return 0 - fi - # log start time - start_time=$(get_time) - - function="make_bucket" - bucket_name=$(make_bucket) - rv=$? - # put object with server side encryption headers - if [ $rv -eq 0 ]; then - function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-10-kB --bucket ${bucket_name} --key datafile-10-kB --sse-customer-algorithm AES256 --sse-customer-key MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ= --sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg==" - test_function=${function} - out=$($function 2>&1) - rv=$? - fi - # now get encrypted object from server for range 500-999 - if [ $rv -eq 0 ]; then - etag1=$(echo "$out" | jq -r .ETag) - sse_customer_key1=$(echo "$out" | jq -r .SSECustomerKeyMD5) - sse_customer_algo1=$(echo "$out" | jq -r .SSECustomerAlgorithm) - function="${AWS} s3api get-object --bucket ${bucket_name} --key datafile-10-kB --range bytes=500-999 --sse-customer-algorithm AES256 --sse-customer-key MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ= --sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg== /tmp/datafile-10-kB" - test_function=${function} - out=$($function 2>&1) - rv=$? - fi - if [ $rv -eq 0 ]; then - cnt=$(stat -c%s /tmp/datafile-10-kB) - if [ "$cnt" -ne 500 ]; then - rv=1 - fi - fi - if [ $rv -eq 0 ]; then - log_success "$(get_duration "$start_time")" "${test_function}" - else - # clean up and log error - ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 - log_failure "$(get_duration "$start_time")" "${function}" "${out}" - fi - return $rv -} - -# tests server side encryption error for get and put calls -function test_serverside_encryption_error() { - #skip server side encryption tests if HTTPS disabled. - if [ "$ENABLE_HTTPS" != "1" ]; then - return 0 - fi - # log start time - start_time=$(get_time) - - function="make_bucket" - bucket_name=$(make_bucket) - rv=$? - - # put object with server side encryption headers with MD5Sum mismatch for sse-customer-key-md5 header - if [ $rv -eq 0 ]; then - function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB --sse-customer-algorithm AES256 --sse-customer-key MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ= --sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg" - test_function=${function} - out=$($function 2>&1) - rv=$? - fi - - if [ $rv -ne 255 ]; then - rv=1 - else - rv=0 - fi - # put object with missing server side encryption header sse-customer-algorithm - if [ $rv -eq 0 ]; then - function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB --sse-customer-key MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ= --sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg==" - test_function=${function} - out=$($function 2>&1) - rv=$? - fi - - if [ $rv -ne 255 ]; then - rv=1 - else - rv=0 - fi - - # put object with server side encryption headers successfully - if [ $rv -eq 0 ]; then - function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB --sse-customer-algorithm AES256 --sse-customer-key MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ= --sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg==" - test_function=${function} - out=$($function 2>&1) - rv=$? - fi - - # now test get on encrypted object with nonmatching sse-customer-key and sse-customer-md5 headers - if [ $rv -eq 0 ]; then - function="${AWS} s3api get-object --bucket ${bucket_name} --key datafile-1-kB --sse-customer-algorithm AES256 --sse-customer-key MzJieXRlc --sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg== /tmp/datafile-1-kB" - test_function=${function} - out=$($function 2>&1) - rv=$? - fi - if [ $rv -ne 255 ]; then - rv=1 - else - rv=0 - fi - # delete bucket - if [ $rv -eq 0 ]; then - function="delete_bucket" - out=$(delete_bucket "$bucket_name") - rv=$? - fi - if [ $rv -eq 0 ]; then - log_success "$(get_duration "$start_time")" "${test_function}" - else - # clean up and log error - ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 - log_failure "$(get_duration "$start_time")" "${function}" "${out}" - fi - - return $rv -} - -# WORM bucket tests. -function test_worm_bucket() { - # log start time - start_time=$(get_time) - - # Make bucket - bucket_name="awscli-mint-test-bucket-$RANDOM" - function="${AWS} s3api create-bucket --bucket ${bucket_name} --object-lock-enabled-for-bucket" - - # execute the test - out=$($function 2>&1) - rv=$? - - if [ $rv -ne 0 ]; then - # if this functionality is not implemented return right away. - if echo "$out" | grep -q "NotImplemented"; then - ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 - return 0 - fi - fi - - # if make bucket succeeds set object lock configuration - if [ $rv -eq 0 ]; then - function="${AWS} s3api put-object-lock-configuration --bucket ${bucket_name} --object-lock-configuration ObjectLockEnabled=Enabled" - out=$($function 2>&1) - rv=$? - if [ $rv -ne 0 ]; then - # if this functionality is not implemented return right away. - if echo "$out" | grep -q "NotImplemented"; then - ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 - return 0 - fi - fi - else - # if make bucket fails, $bucket_name has the error output - out="${bucket_name}" - fi - - # if setting object lock configuration succeeds, upload a file first time - if [ $rv -eq 0 ]; then - function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB" - out=$($function 2>&1) - rv=$? - else - # if make bucket fails, $bucket_name has the error output - out="${bucket_name}" - fi - - # second time upload of same file should fail due to WORM setting - if [ $rv -eq 0 ]; then - function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB" - out=$($function 2>&1) - rv=$? - else - out="First time object upload failed" - fi - - if [ $rv -eq 0 ]; then - log_success "$(get_duration "$start_time")" "${test_function}" - else - # cleanup is not possible due to one day validity of object lock configurataion - log_failure "$(get_duration "$start_time")" "${function}" "${out}" - fi - - return $rv -} - -# Tests creating and deleting an object with legal hold. -function test_legal_hold() { - # log start time - start_time=$(get_time) - - # Make bucket - bucket_name="awscli-mint-test-bucket-$RANDOM" - function="${AWS} s3api create-bucket --bucket ${bucket_name} --object-lock-enabled-for-bucket" - - # execute the test - out=$($function 2>&1) - rv=$? - - if [ $rv -ne 0 ]; then - # if this functionality is not implemented return right away. - if echo "$out" | grep -q "NotImplemented"; then - ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 - return 0 - fi - fi - - # if make bucket succeeds upload a file - if [ $rv -eq 0 ]; then - function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB --object-lock-legal-hold-status ON" - out=$($function 2>&1) - errcnt=$(echo "$out" | sed -n '/Bucket is missing ObjectLockConfiguration/p' | wc -l) - # skip test for gateways - if [ "$errcnt" -eq 1 ]; then - return 0 - fi - rv=$? - else - # if make bucket fails, $bucket_name has the error output - out="${bucket_name}" - fi - - # if upload succeeds stat the file - if [ $rv -eq 0 ]; then - function="${AWS} s3api head-object --bucket ${bucket_name} --key datafile-1-kB" - # save the ref to function being tested, so it can be logged - test_function=${function} - out=$($function 2>&1) - lhold=$(echo "$out" | jq -r .ObjectLockLegalHoldStatus) - rv=$? - fi - - # if head-object succeeds, verify metadata has legal hold status - if [ $rv -eq 0 ]; then - if [ "${lhold}" == "" ]; then - rv=1 - out="Legal hold was not applied" - fi - if [ "${lhold}" == "OFF" ]; then - rv=1 - out="Legal hold was not applied" - fi - fi - if [ $rv -eq 0 ]; then - function="${AWS} s3api put-object-legal-hold --bucket ${bucket_name} --key datafile-1-kB --legal-hold Status=OFF" - out=$($function 2>&1) - rv=$? - else - # if make bucket fails, $bucket_name has the error output - out="${bucket_name}" - fi - # if upload succeeds download the file - if [ $rv -eq 0 ]; then - function="${AWS} s3api get-object-legal-hold --bucket ${bucket_name} --key datafile-1-kB" - # save the ref to function being tested, so it can be logged - test_function=${function} - out=$($function 2>&1) - lhold=$(echo "$out" | jq -r .LegalHold.Status) - rv=$? - fi - - # if head-object succeeds, verify metadata has legal hold status - if [ $rv -eq 0 ]; then - if [ "${lhold}" == "" ]; then - rv=1 - out="Legal hold was not applied" - fi - if [ "${lhold}" == "ON" ]; then - rv=1 - out="Legal hold status not turned off" - fi - fi - # Attempt a delete on prefix shouldn't delete the directory since we have an object inside it. - if [ $rv -eq 0 ]; then - function="${AWS} s3api delete-object --bucket ${bucket_name} --key datafile-1-kB" - # save the ref to function being tested, so it can be logged - test_function=${function} - out=$($function 2>&1) - rv=$? - fi - if [ $rv -eq 0 ]; then - log_success "$(get_duration "$start_time")" "${test_function}" - else - # clean up and log error - ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1 - log_failure "$(get_duration "$start_time")" "${function}" "${out}" - fi - - return $rv -} - -# main handler for all the tests. -main() { - # Success tests - test_create_bucket && \ - test_upload_object && \ - test_lookup_object_prefix && \ - test_list_objects && \ - test_multipart_upload_0byte && \ - test_multipart_upload && \ - test_max_key_list && \ - test_copy_object && \ - test_copy_object_storage_class && \ - test_copy_object_storage_class_same && \ - test_presigned_object && \ - test_upload_object_10 && \ - test_multipart_upload_10 && \ - test_serverside_encryption && \ - test_serverside_encryption_get_range && \ - test_serverside_encryption_multipart && \ - test_serverside_encryption_multipart_copy && \ - # Success cli ops. - test_aws_s3_cp && \ - test_aws_s3_sync && \ - # Error tests - test_list_objects_error && \ - test_put_object_error && \ - test_serverside_encryption_error && \ - test_worm_bucket && \ - test_legal_hold - - return $? -} - -_init "$@" && main diff --git a/mint/run/core/healthcheck/healthcheck.go b/mint/run/core/healthcheck/healthcheck.go deleted file mode 100644 index d142fa67d..000000000 --- a/mint/run/core/healthcheck/healthcheck.go +++ /dev/null @@ -1,219 +0,0 @@ -/* -* -* Mint, (C) 2019 Minio, Inc. -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software - -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -* - */ - -package main - -import ( - "crypto/tls" - "encoding/json" - "fmt" - "net/http" - "net/url" - "os" - "time" - - jwtgo "github.com/dgrijalva/jwt-go" - log "github.com/sirupsen/logrus" -) - -const ( - pass = "PASS" // Indicate that a test passed - fail = "FAIL" // Indicate that a test failed - livenessPath = "/minio/health/live" - readinessPath = "/minio/health/ready" - prometheusPath = "/minio/prometheus/metrics" - timeout = time.Duration(30 * time.Second) -) - -type mintJSONFormatter struct { -} - -func (f *mintJSONFormatter) Format(entry *log.Entry) ([]byte, error) { - data := make(log.Fields, len(entry.Data)) - for k, v := range entry.Data { - switch v := v.(type) { - case error: - // Otherwise errors are ignored by `encoding/json` - // https://github.com/sirupsen/logrus/issues/137 - data[k] = v.Error() - default: - data[k] = v - } - } - - serialized, err := json.Marshal(data) - if err != nil { - return nil, fmt.Errorf("Failed to marshal fields to JSON, %w", err) - } - return append(serialized, '\n'), nil -} - -// log successful test runs -func successLogger(function string, args map[string]interface{}, startTime time.Time) *log.Entry { - // calculate the test case duration - duration := time.Since(startTime) - // log with the fields as per mint - fields := log.Fields{"name": "healthcheck", "function": function, "args": args, "duration": duration.Nanoseconds() / 1000000, "status": pass} - return log.WithFields(fields) -} - -// log failed test runs -func failureLog(function string, args map[string]interface{}, startTime time.Time, alert string, message string, err error) *log.Entry { - // calculate the test case duration - duration := time.Since(startTime) - var fields log.Fields - // log with the fields as per mint - if err != nil { - fields = log.Fields{"name": "healthcheck", "function": function, "args": args, - "duration": duration.Nanoseconds() / 1000000, "status": fail, "alert": alert, "message": message, "error": err} - } else { - fields = log.Fields{"name": "healthcheck", "function": function, "args": args, - "duration": duration.Nanoseconds() / 1000000, "status": fail, "alert": alert, "message": message} - } - return log.WithFields(fields) -} - -func testLivenessEndpoint(endpoint string) { - startTime := time.Now() - function := "testLivenessEndpoint" - - u, err := url.Parse(fmt.Sprintf("%s%s", endpoint, livenessPath)) - if err != nil { - // Could not parse URL successfully - failureLog(function, nil, startTime, "", "URL Parsing for Healthcheck Liveness handler failed", err).Fatal() - } - - tr := &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: u.Scheme == "https"}, - } - client := &http.Client{Transport: tr, Timeout: timeout} - resp, err := client.Get(u.String()) - if err != nil { - // GET request errored - failureLog(function, nil, startTime, "", "GET request failed", err).Fatal() - } - if resp.StatusCode != http.StatusOK { - // Status not 200 OK - failureLog(function, nil, startTime, "", fmt.Sprintf("GET /minio/health/live returned %s", resp.Status), err).Fatal() - } - - defer resp.Body.Close() - defer successLogger(function, nil, startTime).Info() -} - -func testReadinessEndpoint(endpoint string) { - startTime := time.Now() - function := "testReadinessEndpoint" - - u, err := url.Parse(fmt.Sprintf("%s%s", endpoint, readinessPath)) - if err != nil { - // Could not parse URL successfully - failureLog(function, nil, startTime, "", "URL Parsing for Healthcheck Readiness handler failed", err).Fatal() - } - - tr := &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: u.Scheme == "https"}, - } - client := &http.Client{Transport: tr, Timeout: timeout} - resp, err := client.Get(u.String()) - if err != nil { - // GET request errored - failureLog(function, nil, startTime, "", "GET request to Readiness endpoint failed", err).Fatal() - } - if resp.StatusCode != http.StatusOK { - // Status not 200 OK - failureLog(function, nil, startTime, "", "GET /minio/health/ready returned non OK status", err).Fatal() - } - - defer resp.Body.Close() - defer successLogger(function, nil, startTime).Info() -} - -const ( - defaultPrometheusJWTExpiry = 100 * 365 * 24 * time.Hour -) - -func testPrometheusEndpoint(endpoint string) { - startTime := time.Now() - function := "testPrometheusEndpoint" - - u, err := url.Parse(fmt.Sprintf("%s%s", endpoint, prometheusPath)) - if err != nil { - // Could not parse URL successfully - failureLog(function, nil, startTime, "", "URL Parsing for Healthcheck Prometheus handler failed", err).Fatal() - } - - jwt := jwtgo.NewWithClaims(jwtgo.SigningMethodHS512, jwtgo.StandardClaims{ - ExpiresAt: time.Now().UTC().Add(defaultPrometheusJWTExpiry).Unix(), - Subject: os.Getenv("ACCESS_KEY"), - Issuer: "prometheus", - }) - - token, err := jwt.SignedString([]byte(os.Getenv("SECRET_KEY"))) - if err != nil { - failureLog(function, nil, startTime, "", "jwt generation failed", err).Fatal() - } - - tr := &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: u.Scheme == "https"}, - } - client := &http.Client{Transport: tr, Timeout: timeout} - - req, err := http.NewRequest(http.MethodGet, u.String(), nil) - if err != nil { - failureLog(function, nil, startTime, "", "Initializing GET request to Prometheus endpoint failed", err).Fatal() - } - req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token)) - - resp, err := client.Do(req) - if err != nil { - // GET request errored - failureLog(function, nil, startTime, "", "GET request to Prometheus endpoint failed", err).Fatal() - } - - if resp.StatusCode != http.StatusOK { - // Status not 200 OK - failureLog(function, nil, startTime, "", "GET /minio/prometheus/metrics returned non OK status", err).Fatal() - } - - defer resp.Body.Close() - defer successLogger(function, nil, startTime).Info() -} - -func main() { - endpoint := os.Getenv("SERVER_ENDPOINT") - secure := os.Getenv("ENABLE_HTTPS") - endpoint = "http://" + endpoint - if secure == "1" { - endpoint = "https://" + endpoint - } - - // Output to stdout instead of the default stderr - log.SetOutput(os.Stdout) - // create custom formatter - mintFormatter := mintJSONFormatter{} - // set custom formatter - log.SetFormatter(&mintFormatter) - // log Info or above -- success cases are Info level, failures are Fatal level - log.SetLevel(log.InfoLevel) - // execute tests - testLivenessEndpoint(endpoint) - testReadinessEndpoint(endpoint) - testPrometheusEndpoint(endpoint) -} diff --git a/mint/run/core/healthcheck/run.sh b/mint/run/core/healthcheck/run.sh deleted file mode 100755 index 727c5a856..000000000 --- a/mint/run/core/healthcheck/run.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -# -# Mint (C) 2019 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# handle command line arguments -if [ $# -ne 2 ]; then - echo "usage: run.sh " - exit 1 -fi - -output_log_file="$1" -error_log_file="$2" - -# run tests -/mint/run/core/healthcheck/healthcheck 1>>"$output_log_file" 2>"$error_log_file" diff --git a/mint/run/core/mc/README.md b/mint/run/core/mc/README.md deleted file mode 100644 index 9a550e09d..000000000 --- a/mint/run/core/mc/README.md +++ /dev/null @@ -1,19 +0,0 @@ -## `mc` tests -This directory serves as the location for Mint tests using `mc`. Top level `mint.sh` calls `run.sh` to execute tests. - -## Adding new tests -New tests is added into `test.sh` as new functions. - -## Running tests manually -- Set environment variables `MINT_DATA_DIR`, `MINT_MODE`, `SERVER_ENDPOINT`, `ACCESS_KEY`, `SECRET_KEY`, `SERVER_REGION` and `ENABLE_HTTPS` -- Call `run.sh` with output log file and error log file. for example -```bash -export MINT_DATA_DIR=~/my-mint-dir -export MINT_MODE=core -export SERVER_ENDPOINT="play.minio.io:9000" -export ACCESS_KEY="Q3AM3UQ867SPQQA43P2F" -export SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" -export ENABLE_HTTPS=1 -export SERVER_REGION=us-east-1 -./run.sh /tmp/output.log /tmp/error.log -``` diff --git a/mint/run/core/mc/run.sh b/mint/run/core/mc/run.sh deleted file mode 100755 index b13c5e296..000000000 --- a/mint/run/core/mc/run.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash -# -# Minio Cloud Storage, (C) 2017 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# handle command line arguments -if [ $# -ne 2 ]; then - echo "usage: run.sh " - exit 1 -fi - -output_log_file="$1" -error_log_file="$2" - -./functional-tests.sh 1>>"$output_log_file" 2>"$error_log_file" diff --git a/mint/run/core/minio-dotnet/run.sh b/mint/run/core/minio-dotnet/run.sh deleted file mode 100755 index 473a8e141..000000000 --- a/mint/run/core/minio-dotnet/run.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -# -# Mint (C) 2017 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# handle command line arguments -if [ $# -ne 2 ]; then - echo "usage: run.sh " - exit 1 -fi - -output_log_file="$1" -error_log_file="$2" -/mint/run/core/minio-dotnet/out/Minio.Functional.Tests 1>>"$output_log_file" 2>"$error_log_file" diff --git a/mint/run/core/minio-go/README.md b/mint/run/core/minio-go/README.md deleted file mode 100644 index 6f1723b72..000000000 --- a/mint/run/core/minio-go/README.md +++ /dev/null @@ -1,19 +0,0 @@ -## `minio-go` tests -This directory serves as the location for Mint tests using `minio-go`. Top level `mint.sh` calls `run.sh` to execute tests. - -## Adding new tests -New tests are added in functional tests of minio-go. Please check https://github.com/minio/minio-go - -## Running tests manually -- Set environment variables `MINT_DATA_DIR`, `MINT_MODE`, `SERVER_ENDPOINT`, `ACCESS_KEY`, `SECRET_KEY`, `SERVER_REGION` and `ENABLE_HTTPS` -- Call `run.sh` with output log file and error log file. for example -```bash -export MINT_DATA_DIR=~/my-mint-dir -export MINT_MODE=core -export SERVER_ENDPOINT="play.minio.io:9000" -export ACCESS_KEY="Q3AM3UQ867SPQQA43P2F" -export SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" -export ENABLE_HTTPS=1 -export SERVER_REGION=us-east-1 -./run.sh /tmp/output.log /tmp/error.log -``` diff --git a/mint/run/core/minio-go/run.sh b/mint/run/core/minio-go/run.sh deleted file mode 100755 index 5a3255bc6..000000000 --- a/mint/run/core/minio-go/run.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -# -# Mint (C) 2017 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# handle command line arguments -if [ $# -ne 2 ]; then - echo "usage: run.sh " - exit 1 -fi - -output_log_file="$1" -error_log_file="$2" - -# run tests -/mint/run/core/minio-go/minio-go 1>>"$output_log_file" 2>"$error_log_file" diff --git a/mint/run/core/minio-java/README.md b/mint/run/core/minio-java/README.md deleted file mode 100644 index 8e73d7d48..000000000 --- a/mint/run/core/minio-java/README.md +++ /dev/null @@ -1,19 +0,0 @@ -## `minio-java` tests -This directory serves as the location for Mint tests using `minio-java`. Top level `mint.sh` calls `run.sh` to execute tests. - -## Adding new tests -New tests is added in functional tests of minio-java. Please check https://github.com/minio/minio-java - -## Running tests manually -- Set environment variables `MINT_DATA_DIR`, `MINT_MODE`, `SERVER_ENDPOINT`, `ACCESS_KEY`, `SECRET_KEY`, `SERVER_REGION` and `ENABLE_HTTPS` -- Call `run.sh` with output log file and error log file. for example -```bash -export MINT_DATA_DIR=~/my-mint-dir -export MINT_MODE=core -export SERVER_ENDPOINT="play.minio.io:9000" -export ACCESS_KEY="Q3AM3UQ867SPQQA43P2F" -export SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" -export ENABLE_HTTPS=1 -export SERVER_REGION=us-east-1 -./run.sh /tmp/output.log /tmp/error.log -``` diff --git a/mint/run/core/minio-java/run.sh b/mint/run/core/minio-java/run.sh deleted file mode 100755 index 6a2fc14ae..000000000 --- a/mint/run/core/minio-java/run.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash -# -# Mint (C) 2017 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# handle command line arguments -if [ $# -ne 2 ]; then - echo "usage: run.sh " - exit 1 -fi - -output_log_file="$1" -error_log_file="$2" - -# run tests -endpoint="http://$SERVER_ENDPOINT" -if [ "$ENABLE_HTTPS" -eq 1 ]; then - endpoint="https://$SERVER_ENDPOINT" -fi - -java -Xmx4096m -Xms256m -cp "/mint/run/core/minio-java/*:." FunctionalTest \ - "$endpoint" "$ACCESS_KEY" "$SECRET_KEY" "$SERVER_REGION" 1>>"$output_log_file" 2>"$error_log_file" diff --git a/mint/run/core/minio-js/README.md b/mint/run/core/minio-js/README.md deleted file mode 100644 index 1144a61f4..000000000 --- a/mint/run/core/minio-js/README.md +++ /dev/null @@ -1,19 +0,0 @@ -## `minio-js` tests -This directory serves as the location for Mint tests using `minio-js`. Top level `mint.sh` calls `run.sh` to execute tests. - -## Adding new tests -New tests is added in functional tests of minio-js. Please check https://github.com/minio/minio-js - -## Running tests manually -- Set environment variables `MINT_DATA_DIR`, `MINT_MODE`, `SERVER_ENDPOINT`, `ACCESS_KEY`, `SECRET_KEY`, `SERVER_REGION` and `ENABLE_HTTPS` -- Call `run.sh` with output log file and error log file. for example -```bash -export MINT_DATA_DIR=~/my-mint-dir -export MINT_MODE=core -export SERVER_ENDPOINT="play.minio.io:9000" -export ACCESS_KEY="Q3AM3UQ867SPQQA43P2F" -export SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" -export ENABLE_HTTPS=1 -export SERVER_REGION=us-east-1 -./run.sh /tmp/output.log /tmp/error.log -``` diff --git a/mint/run/core/minio-js/minioreporter.js b/mint/run/core/minio-js/minioreporter.js deleted file mode 100644 index 3e77934d0..000000000 --- a/mint/run/core/minio-js/minioreporter.js +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Minio Reporter for JSON formatted logging, (C) 2017 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -var mocha = require('mocha'); -module.exports = minioreporter; - -function minioreporter(runner) { - mocha.reporters.Base.call(this, runner); - var self = this; - - runner.on('pass', function (test) { - GenerateJsonEntry(test) - }); - - runner.on('fail', function (test, err) { - GenerateJsonEntry(test, err) - }); - -} - -/** - * Convert test result into a JSON object and print on the console. - * - * @api private - * @param test, err - */ - -function GenerateJsonEntry (test, err) { - var res = test.title.split("_") - var jsonEntry = {}; - - jsonEntry.name = "minio-js" - - if (res.length > 0 && res[0].length) { - jsonEntry.function = res[0] - } - - if (res.length > 1 && res[1].length) { - jsonEntry.args = res[1] - } - - jsonEntry.duration = test.duration - - if (res.length > 2 && res[2].length) { - jsonEntry.alert = res[2] - } - - if (err != null ) { - jsonEntry.status = "FAIL" - jsonEntry.error = err.stack.replace(/\n/g, " ").replace(/ +(?= )/g,'') - } else { - jsonEntry.status = "PASS" - } - - process.stdout.write(JSON.stringify(jsonEntry) + "\n") -} diff --git a/mint/run/core/minio-js/package.json b/mint/run/core/minio-js/package.json deleted file mode 100644 index 84d581cf2..000000000 --- a/mint/run/core/minio-js/package.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "name": "bin", - "version": "1.0.0", - "main": "functional_test.js", - "scripts": { - "test": "echo \"Error: no test specified\" && exit 1" - }, - "keywords": [], - "author": "", - "license": "ISC", - "description": "", - "dependencies": { - "app-module-path": "*", - "async": "*", - "block-stream2": "*", - "concat-stream": "*", - "es6-error": "*", - "json-stream": "*", - "lodash": "*", - "mime-types": "*", - "mkdirp": "*", - "moment": "*", - "source-map-support": "*", - "through2": "*", - "xml": "*", - "xml2js": "*" - }, - "devDependencies": { - "browserify": "*", - "chai": "*", - "gulp": "*", - "gulp-babel": "*", - "gulp-jscs": "*", - "jshint":"2.*", - "gulp-jshint": "*", - "gulp-mocha": "*", - "gulp-notify": "*", - "gulp-sourcemaps": "*", - "jshint-stylish": "*", - "mocha": "*", - "mocha-steps": "*", - "nock": "*", - "rewire": "*", - "superagent": "*" - }, - "scripts": { - "test": "mocha" - } -} diff --git a/mint/run/core/minio-js/run.sh b/mint/run/core/minio-js/run.sh deleted file mode 100755 index c6a0c58a9..000000000 --- a/mint/run/core/minio-js/run.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -# -# Minio Cloud Storage, (C) 2017 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# handle command line arguments -if [ $# -ne 2 ]; then - echo "usage: run.sh " - exit 1 -fi - -output_log_file="$1" -error_log_file="$2" - -# run tests -./node_modules/mocha/bin/mocha -R minioreporter -b --exit 1>>"$output_log_file" 2>"$error_log_file" diff --git a/mint/run/core/minio-py/README.md b/mint/run/core/minio-py/README.md deleted file mode 100644 index 36c65cc2f..000000000 --- a/mint/run/core/minio-py/README.md +++ /dev/null @@ -1,19 +0,0 @@ -## `minio-py` tests -This directory serves as the location for Mint tests using `minio-py`. Top level `mint.sh` calls `run.sh` to execute tests. - -## Adding new tests -New tests is added in functional tests of minio-py. Please check https://github.com/minio/minio-py - -## Running tests manually -- Set environment variables `MINT_DATA_DIR`, `MINT_MODE`, `SERVER_ENDPOINT`, `ACCESS_KEY`, `SECRET_KEY`, `SERVER_REGION` and `ENABLE_HTTPS` -- Call `run.sh` with output log file and error log file. for example -```bash -export MINT_DATA_DIR=~/my-mint-dir -export MINT_MODE=core -export SERVER_ENDPOINT="play.minio.io:9000" -export ACCESS_KEY="Q3AM3UQ867SPQQA43P2F" -export SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" -export ENABLE_HTTPS=1 -export SERVER_REGION=us-east-1 -./run.sh /tmp/output.log /tmp/error.log -``` diff --git a/mint/run/core/minio-py/run.sh b/mint/run/core/minio-py/run.sh deleted file mode 100755 index 2538783c0..000000000 --- a/mint/run/core/minio-py/run.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -# -# Mint (C) 2017 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# handle command line arguments -if [ $# -ne 2 ]; then - echo "usage: run.sh " - exit 1 -fi - -output_log_file="$1" -error_log_file="$2" - -# run tests -python "/mint/run/core/minio-py/tests.py" 1>>"$output_log_file" 2>"$error_log_file" diff --git a/mint/run/core/s3cmd/.gitignore b/mint/run/core/s3cmd/.gitignore deleted file mode 100644 index 518a1c6f3..000000000 --- a/mint/run/core/s3cmd/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -*~ -*.log diff --git a/mint/run/core/s3cmd/README.md b/mint/run/core/s3cmd/README.md deleted file mode 100644 index a961c02c2..000000000 --- a/mint/run/core/s3cmd/README.md +++ /dev/null @@ -1,19 +0,0 @@ -## `s3cmd` tests -This directory serves as the location for Mint tests using `s3cmd`. Top level `mint.sh` calls `run.sh` to execute tests. - -## Adding new tests -New tests is added into `test.sh` as new functions. - -## Running tests manually -- Set environment variables `MINT_DATA_DIR`, `MINT_MODE`, `SERVER_ENDPOINT`, `ACCESS_KEY`, `SECRET_KEY`, `SERVER_REGION` and `ENABLE_HTTPS` -- Call `run.sh` with output log file and error log file. for example -```bash -export MINT_DATA_DIR=~/my-mint-dir -export MINT_MODE=core -export SERVER_ENDPOINT="play.minio.io:9000" -export ACCESS_KEY="Q3AM3UQ867SPQQA43P2F" -export SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" -export ENABLE_HTTPS=1 -export SERVER_REGION=us-east-1 -./run.sh /tmp/output.log /tmp/error.log -``` diff --git a/mint/run/core/s3cmd/run.sh b/mint/run/core/s3cmd/run.sh deleted file mode 100755 index e6cdc3fc3..000000000 --- a/mint/run/core/s3cmd/run.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -# -# Mint (C) 2017 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# handle command line arguments -if [ $# -ne 2 ]; then - echo "usage: run.sh " - exit 1 -fi - -output_log_file="$1" -error_log_file="$2" - -# run tests -./test.sh 1>>"$output_log_file" 2>"$error_log_file" diff --git a/mint/run/core/s3cmd/test.sh b/mint/run/core/s3cmd/test.sh deleted file mode 100755 index a33d50562..000000000 --- a/mint/run/core/s3cmd/test.sh +++ /dev/null @@ -1,406 +0,0 @@ -#!/bin/bash -# -# Mint (C) 2017-2020 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - - -if [ -n "$MINT_MODE" ]; then - if [ -z "${MINT_DATA_DIR+x}" ]; then - echo "MINT_DATA_DIR not defined" - exit 1 - fi - if [ -z "${SERVER_ENDPOINT+x}" ]; then - echo "SERVER_ENDPOINT not defined" - exit 1 - fi - if [ -z "${ACCESS_KEY+x}" ]; then - echo "ACCESS_KEY not defined" - exit 1 - fi - if [ -z "${SECRET_KEY+x}" ]; then - echo "SECRET_KEY not defined" - exit 1 - fi -fi - -if [ -z "${SERVER_ENDPOINT+x}" ]; then - SERVER_ENDPOINT="play.minio.io:9000" - ACCESS_KEY="Q3AM3UQ867SPQQA43P2F" - SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" - ENABLE_HTTPS=1 - SERVER_REGION="us-east-1" -fi - -WORK_DIR="$PWD" -DATA_DIR="$MINT_DATA_DIR" -if [ -z "$MINT_MODE" ]; then - WORK_DIR="$PWD/.run-$RANDOM" - DATA_DIR="$WORK_DIR/data" -fi - -FILE_1_MB="$DATA_DIR/datafile-1-MB" -FILE_65_MB="$DATA_DIR/datafile-65-MB" -declare FILE_1_MB_MD5SUM -declare FILE_65_MB_MD5SUM - -BUCKET_NAME="s3cmd-test-bucket-$RANDOM" -S3CMD=$(which s3cmd) -declare -a S3CMD_CMD - -function get_md5sum() -{ - filename="$FILE_1_MB" - out=$(md5sum "$filename" 2>/dev/null) - rv=$? - if [ "$rv" -eq 0 ]; then - awk '{ print $1 }' <<< "$out" - fi - - return "$rv" -} - -function get_time() -{ - date +%s%N -} - -function get_duration() -{ - start_time=$1 - end_time=$(get_time) - - echo $(( (end_time - start_time) / 1000000 )) -} - -function log_success() -{ - if [ -n "$MINT_MODE" ]; then - printf '{"name": "s3cmd", "duration": "%d", "function": "%s", "status": "PASS"}\n' "$(get_duration "$1")" "$2" - fi -} - -function show() -{ - if [ -z "$MINT_MODE" ]; then - func_name="$1" - echo "Running $func_name()" - fi -} - -function fail() -{ - rv="$1" - shift - - if [ "$rv" -ne 0 ]; then - echo "$@" - fi - - return "$rv" -} - -function assert() -{ - expected_rv="$1" - shift - start_time="$1" - shift - func_name="$1" - shift - - err=$("$@" 2>&1) - rv=$? - if [ "$rv" -ne 0 ] && [ "$expected_rv" -eq 0 ]; then - if [ -n "$MINT_MODE" ]; then - err=$(printf '%s' "$err" | python -c 'import sys,json; print(json.dumps(sys.stdin.read()))') - ## err is already JSON string, no need to double quote - printf '{"name": "s3cmd", "duration": "%d", "function": "%s", "status": "FAIL", "error": %s}\n' "$(get_duration "$start_time")" "$func_name" "$err" - else - echo "s3cmd: $func_name: $err" - fi - - exit "$rv" - fi - - return 0 -} - -function assert_success() { - assert 0 "$@" -} - -function assert_failure() { - assert 1 "$@" -} - -function s3cmd_cmd() -{ - cmd=( "${S3CMD_CMD[@]}" "$@" ) - "${cmd[@]}" - rv=$? - return "$rv" -} - -function check_md5sum() -{ - expected_checksum="$1" - shift - filename="$*" - - checksum="$(get_md5sum "$filename")" - rv=$? - if [ "$rv" -ne 0 ]; then - echo "unable to get md5sum for $filename" - return "$rv" - fi - - if [ "$checksum" != "$expected_checksum" ]; then - echo "$filename: md5sum mismatch" - return 1 - fi - - return 0 -} - -function test_make_bucket() -{ - show "${FUNCNAME[0]}" - - start_time=$(get_time) - bucket_name="s3cmd-test-bucket-$RANDOM" - assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd mb "s3://${bucket_name}" - assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rb "s3://${bucket_name}" - - log_success "$start_time" "${FUNCNAME[0]}" -} - -function test_make_bucket_error() { - show "${FUNCNAME[0]}" - - start_time=$(get_time) - bucket_name="S3CMD-test%bucket%$RANDOM" - assert_failure "$start_time" "${FUNCNAME[0]}" s3cmd_cmd mb "s3://${bucket_name}" - - log_success "$start_time" "${FUNCNAME[0]}" -} - -function setup() -{ - start_time=$(get_time) - assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd mb "s3://${BUCKET_NAME}" -} - -function teardown() -{ - start_time=$(get_time) - assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rm --force --recursive "s3://${BUCKET_NAME}" - assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rb --force "s3://${BUCKET_NAME}" -} - -function test_put_object() -{ - show "${FUNCNAME[0]}" - - start_time=$(get_time) - object_name="s3cmd-test-object-$RANDOM" - assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd put "${FILE_1_MB}" "s3://${BUCKET_NAME}/${object_name}" - assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rm "s3://${BUCKET_NAME}/${object_name}" - - log_success "$start_time" "${FUNCNAME[0]}" -} - -function test_put_object_error() -{ - show "${FUNCNAME[0]}" - start_time=$(get_time) - - object_long_name=$(printf "s3cmd-test-object-%01100d" 1) - assert_failure "$start_time" "${FUNCNAME[0]}" s3cmd_cmd put "${FILE_1_MB}" "s3://${BUCKET_NAME}/${object_long_name}" - - log_success "$start_time" "${FUNCNAME[0]}" -} - -function test_put_object_multipart() -{ - show "${FUNCNAME[0]}" - - start_time=$(get_time) - object_name="s3cmd-test-object-$RANDOM" - assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd put "${FILE_65_MB}" "s3://${BUCKET_NAME}/${object_name}" - assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rm "s3://${BUCKET_NAME}/${object_name}" - - log_success "$start_time" "${FUNCNAME[0]}" -} - -function test_get_object() -{ - show "${FUNCNAME[0]}" - - start_time=$(get_time) - object_name="s3cmd-test-object-$RANDOM" - assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd put "${FILE_1_MB}" "s3://${BUCKET_NAME}/${object_name}" - assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd get "s3://${BUCKET_NAME}/${object_name}" "${object_name}.downloaded" - assert_success "$start_time" "${FUNCNAME[0]}" check_md5sum "$FILE_1_MB_MD5SUM" "${object_name}.downloaded" - assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rm "s3://${BUCKET_NAME}/${object_name}" - assert_success "$start_time" "${FUNCNAME[0]}" rm -f "${object_name}.downloaded" - - log_success "$start_time" "${FUNCNAME[0]}" -} - -function test_get_object_error() -{ - show "${FUNCNAME[0]}" - - start_time=$(get_time) - object_name="s3cmd-test-object-$RANDOM" - assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd put "${FILE_1_MB}" "s3://${BUCKET_NAME}/${object_name}" - assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rm "s3://${BUCKET_NAME}/${object_name}" - assert_failure "$start_time" "${FUNCNAME[0]}" s3cmd_cmd get "s3://${BUCKET_NAME}/${object_name}" "${object_name}.downloaded" - assert_success "$start_time" "${FUNCNAME[0]}" rm -f "${object_name}.downloaded" - - log_success "$start_time" "${FUNCNAME[0]}" -} - -function test_get_object_multipart() -{ - show "${FUNCNAME[0]}" - - start_time=$(get_time) - object_name="s3cmd-test-object-$RANDOM" - assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd put "${FILE_65_MB}" "s3://${BUCKET_NAME}/${object_name}" - assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd get "s3://${BUCKET_NAME}/${object_name}" "${object_name}.downloaded" - assert_success "$start_time" "${FUNCNAME[0]}" check_md5sum "$FILE_65_MB_MD5SUM" "${object_name}.downloaded" - assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rm "s3://${BUCKET_NAME}/${object_name}" - assert_success "$start_time" "${FUNCNAME[0]}" rm -f "${object_name}.downloaded" - - log_success "$start_time" "${FUNCNAME[0]}" -} - -function test_sync_list_objects() -{ - show "${FUNCNAME[0]}" - - start_time=$(get_time) - bucket_name="s3cmd-test-bucket-$RANDOM" - object_name="s3cmd-test-object-$RANDOM" - assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd mb "s3://${bucket_name}" - assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd sync "$DATA_DIR/" "s3://${bucket_name}" - - diff -bB <(ls "$DATA_DIR") <("${S3CMD_CMD[@]}" ls "s3://${bucket_name}" | awk '{print $4}' | sed "s/s3:*..${bucket_name}.//g") >/dev/null 2>&1 - assert_success "$start_time" "${FUNCNAME[0]}" fail $? "sync and list differs" - assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rb --force --recursive "s3://${bucket_name}" - - log_success "$start_time" "${FUNCNAME[0]}" -} - -function run_test() -{ - test_make_bucket - test_make_bucket_error - - setup - - test_put_object - test_put_object_error - test_put_object_multipart - test_get_object - test_get_object_multipart - test_sync_list_objects - - teardown -} - -function __init__() -{ - set -e - - S3CMD_CONFIG_DIR="/tmp/.s3cmd-$RANDOM" - mkdir -p $S3CMD_CONFIG_DIR - S3CMD_CONFIG_FILE="$S3CMD_CONFIG_DIR/s3cfg" - - # configure s3cmd - cat > $S3CMD_CONFIG_FILE <"$FILE_1_MB" - fi - - if [ ! -e "$FILE_65_MB" ]; then - shred -n 1 -s 65MB - >"$FILE_65_MB" - fi - - set -E - set -o pipefail - - FILE_1_MB_MD5SUM="$(get_md5sum "$FILE_1_MB")" - rv=$? - if [ $rv -ne 0 ]; then - echo "unable to get md5sum of $FILE_1_MB" - exit 1 - fi - - FILE_65_MB_MD5SUM="$(get_md5sum "$FILE_65_MB")" - rv=$? - if [ $rv -ne 0 ]; then - echo "unable to get md5sum of $FILE_65_MB" - exit 1 - fi - - set +e -} - -function main() -{ - ( run_test ) - rv=$? - - rm -fr "$S3CMD_CONFIG_FILE" - if [ -z "$MINT_MODE" ]; then - rm -fr "$WORK_DIR" "$DATA_DIR" - fi - - exit "$rv" -} - -__init__ "$@" -main "$@" diff --git a/mint/run/core/s3select/README.md b/mint/run/core/s3select/README.md deleted file mode 100644 index 73a69a22b..000000000 --- a/mint/run/core/s3select/README.md +++ /dev/null @@ -1,21 +0,0 @@ - -## `s3select` tests -This directory serves as the location for Mint tests for s3select features. Top level `mint.sh` calls `run.sh` to execute tests. - -## Adding new tests -New tests are added into `s3select/tests.py` as new functions. - -## Running tests manually -- Set environment variables `MINT_DATA_DIR`, `MINT_MODE`, `SERVER_ENDPOINT`, `ACCESS_KEY`, `SECRET_KEY`, `SERVER_REGION` and `ENABLE_HTTPS` -- Call `run.sh` with output log file and error log file. for example - -```bash -export MINT_DATA_DIR=~/my-mint-dir -export MINT_MODE=core -export SERVER_ENDPOINT="play.min.io" -export ACCESS_KEY="Q3AM3UQ867SPQQA43P2F" -export SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" -export ENABLE_HTTPS=1 -export SERVER_REGION=us-east-1 -./run.sh /tmp/output.log /tmp/error.log -``` diff --git a/mint/run/core/s3select/csv.py b/mint/run/core/s3select/csv.py deleted file mode 100644 index faee96518..000000000 --- a/mint/run/core/s3select/csv.py +++ /dev/null @@ -1,172 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# MinIO Python Library for Amazon S3 Compatible Cloud Storage, -# (C) 2015-2020 MinIO, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import io - -from minio import Minio -from minio.select.options import (SelectObjectOptions, CSVInput, - RequestProgress, InputSerialization, - OutputSerialization, CSVOutput, JsonOutput) - -from utils import * - -def test_sql_api(test_name, client, bucket_name, input_data, sql_opts, expected_output): - """ Test if the passed SQL request has the output equal to the passed execpted one""" - object_name = generate_object_name() - got_output = b'' - try: - bytes_content = io.BytesIO(input_data) - client.put_object(bucket_name, object_name, io.BytesIO(input_data), len(input_data)) - data = client.select_object_content(bucket_name, object_name, sql_opts) - # Get the records - records = io.BytesIO() - for d in data.stream(10*1024): - records.write(d.encode('utf-8')) - got_output = records.getvalue() - except Exception as select_err: - if not isinstance(expected_output, Exception): - raise ValueError('Test {} unexpectedly failed with: {}'.format(test_name, select_err)) - else: - if isinstance(expected_output, Exception): - raise ValueError('Test {}: expected an exception, got {}'.format(test_name, got_output)) - if got_output != expected_output: - raise ValueError('Test {}: data mismatch. Expected : {}, Received {}'.format(test_name, expected_output, got_output)) - finally: - client.remove_object(bucket_name, object_name) - - -def test_csv_input_custom_quote_char(client, log_output): - # Get a unique bucket_name and object_name - log_output.args['bucket_name'] = bucket_name = generate_bucket_name() - - tests = [ - # Invalid quote character, should fail - ('""', '"', b'col1,col2,col3\n', Exception()), - # UTF-8 quote character - ('ع', '"', 'عcol1ع,عcol2ع,عcol3ع\n'.encode(), b'{"_1":"col1","_2":"col2","_3":"col3"}\n'), - # Only one field is quoted - ('"', '"', b'"col1",col2,col3\n', b'{"_1":"col1","_2":"col2","_3":"col3"}\n'), - ('"', '"', b'"col1,col2,col3"\n', b'{"_1":"col1,col2,col3"}\n'), - ('\'', '"', b'"col1",col2,col3\n', b'{"_1":"\\"col1\\"","_2":"col2","_3":"col3"}\n'), - ('', '"', b'"col1",col2,col3\n', b'{"_1":"\\"col1\\"","_2":"col2","_3":"col3"}\n'), - ('', '"', b'"col1",col2,col3\n', b'{"_1":"\\"col1\\"","_2":"col2","_3":"col3"}\n'), - ('', '"', b'"col1","col2","col3"\n', b'{"_1":"\\"col1\\"","_2":"\\"col2\\"","_3":"\\"col3\\""}\n'), - ('"', '"', b'""""""\n', b'{"_1":"\\"\\""}\n'), - ('"', '"', b'A",B\n', b'{"_1":"A\\"","_2":"B"}\n'), - ('"', '"', b'A"",B\n', b'{"_1":"A\\"\\"","_2":"B"}\n'), - ('"', '\\', b'A\\B,C\n', b'{"_1":"A\\\\B","_2":"C"}\n'), - ('"', '"', b'"A""B","CD"\n', b'{"_1":"A\\"B","_2":"CD"}\n'), - ('"', '\\', b'"A\\B","CD"\n', b'{"_1":"AB","_2":"CD"}\n'), - ('"', '\\', b'"A\\,","CD"\n', b'{"_1":"A,","_2":"CD"}\n'), - ('"', '\\', b'"A\\"B","CD"\n', b'{"_1":"A\\"B","_2":"CD"}\n'), - ('"', '\\', b'"A\\""\n', b'{"_1":"A\\""}\n'), - ('"', '\\', b'"A\\"\\"B"\n', b'{"_1":"A\\"\\"B"}\n'), - ('"', '\\', b'"A\\"","\\"B"\n', b'{"_1":"A\\"","_2":"\\"B"}\n'), - ] - - client.make_bucket(bucket_name) - - try: - for idx, (quote_char, escape_char, data, expected_output) in enumerate(tests): - sql_opts = SelectObjectOptions( - expression="select * from s3object", - input_serialization=InputSerialization( - compression_type="NONE", - csv=CSVInput(FileHeaderInfo="NONE", - RecordDelimiter="\n", - FieldDelimiter=",", - QuoteCharacter=quote_char, - QuoteEscapeCharacter=escape_char, - Comments="#", - AllowQuotedRecordDelimiter="FALSE",), - ), - output_serialization=OutputSerialization( - json = JsonOutput( - RecordDelimiter="\n", - ) - ), - request_progress=RequestProgress( - enabled="False" - ) - ) - - test_sql_api(f'test_{idx}', client, bucket_name, data, sql_opts, expected_output) - finally: - client.remove_bucket(bucket_name) - - # Test passes - print(log_output.json_report()) - -def test_csv_output_custom_quote_char(client, log_output): - # Get a unique bucket_name and object_name - log_output.args['bucket_name'] = bucket_name = generate_bucket_name() - - tests = [ - # UTF-8 quote character - ("''", "''", b'col1,col2,col3\n', Exception()), - ("'", "'", b'col1,col2,col3\n', b"'col1','col2','col3'\n"), - ("", '"', b'col1,col2,col3\n', b'\x00col1\x00,\x00col2\x00,\x00col3\x00\n'), - ('"', '"', b'col1,col2,col3\n', b'"col1","col2","col3"\n'), - ('"', '"', b'col"1,col2,col3\n', b'"col""1","col2","col3"\n'), - ('"', '"', b'""""\n', b'""""\n'), - ('"', '"', b'\n', b''), - ("'", "\\", b'col1,col2,col3\n', b"'col1','col2','col3'\n"), - ("'", "\\", b'col""1,col2,col3\n', b"'col\"\"1','col2','col3'\n"), - ("'", "\\", b'col\'1,col2,col3\n', b"'col\\'1','col2','col3'\n"), - ("'", "\\", b'"col\'1","col2","col3"\n', b"'col\\'1','col2','col3'\n"), - ("'", "\\", b'col\'\n', b"'col\\''\n"), - # Two consecutive escaped quotes - ("'", "\\", b'"a"""""\n', b"'a\"\"'\n"), - ] - - client.make_bucket(bucket_name) - - try: - for idx, (quote_char, escape_char, input_data, expected_output) in enumerate(tests): - sql_opts = SelectObjectOptions( - expression="select * from s3object", - input_serialization=InputSerialization( - compression_type="NONE", - csv=CSVInput(FileHeaderInfo="NONE", - RecordDelimiter="\n", - FieldDelimiter=",", - QuoteCharacter='"', - QuoteEscapeCharacter='"', - Comments="#", - AllowQuotedRecordDelimiter="FALSE",), - ), - output_serialization=OutputSerialization( - csv=CSVOutput(QuoteFields="ALWAYS", - RecordDelimiter="\n", - FieldDelimiter=",", - QuoteCharacter=quote_char, - QuoteEscapeCharacter=escape_char,) - ), - request_progress=RequestProgress( - enabled="False" - ) - ) - - test_sql_api(f'test_{idx}', client, bucket_name, input_data, sql_opts, expected_output) - finally: - client.remove_bucket(bucket_name) - - # Test passes - print(log_output.json_report()) - - diff --git a/mint/run/core/s3select/run.sh b/mint/run/core/s3select/run.sh deleted file mode 100755 index 0574ea85f..000000000 --- a/mint/run/core/s3select/run.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -# -# Mint (C) 2020 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# handle command line arguments -if [ $# -ne 2 ]; then - echo "usage: run.sh " - exit 1 -fi - -output_log_file="$1" -error_log_file="$2" - -# run path style tests -python "./tests.py" 1>>"$output_log_file" 2>"$error_log_file" diff --git a/mint/run/core/s3select/sql_ops.py b/mint/run/core/s3select/sql_ops.py deleted file mode 100644 index 83931ad21..000000000 --- a/mint/run/core/s3select/sql_ops.py +++ /dev/null @@ -1,399 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# MinIO Python Library for Amazon S3 Compatible Cloud Storage, -# (C) 2020 MinIO, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import io -from datetime import datetime - -from minio import Minio -from minio.select.options import (SelectObjectOptions, CSVInput, JSONInput, - RequestProgress, InputSerialization, - OutputSerialization, CSVOutput, JsonOutput) - -from utils import * - -def test_sql_expressions_custom_input_output(client, input_bytes, sql_input, sql_output, tests, log_output): - bucket_name = generate_bucket_name() - object_name = generate_object_name() - - log_output.args['total_tests'] = 0 - log_output.args['total_success'] = 0 - - client.make_bucket(bucket_name) - try: - content = io.BytesIO(bytes(input_bytes, 'utf-8')) - client.put_object(bucket_name, object_name, content, len(input_bytes)) - - for idx, (test_name, select_expression, expected_output) in enumerate(tests): - if select_expression == '': - continue - try: - log_output.args['total_tests'] += 1 - options = SelectObjectOptions( - expression=select_expression, - input_serialization=sql_input, - output_serialization=sql_output, - request_progress=RequestProgress( - enabled="False" - ) - ) - - data = client.select_object_content(bucket_name, object_name, options) - - # Get the records - records = io.BytesIO() - for d in data.stream(10*1024): - records.write(d.encode('utf-8')) - got_output = records.getvalue() - - if got_output != expected_output: - if type(expected_output) == datetime: - # Attempt to parse the date which will throw an exception for any issue - datetime.strptime(got_output.decode("utf-8").strip(), '%Y-%m-%dT%H:%M:%S.%f%z') - else: - raise ValueError('Test {}: data mismatch. Expected : {}. Received: {}.'.format(idx+1, expected_output, got_output)) - - log_output.args['total_success'] += 1 - except Exception as err: - continue ## TODO, raise instead - # raise Exception(err) - finally: - client.remove_object(bucket_name, object_name) - client.remove_bucket(bucket_name) - - -def test_sql_expressions(client, input_json_bytes, tests, log_output): - input_serialization = InputSerialization( - compression_type="NONE", - json=JSONInput(Type="DOCUMENT"), - ) - - output_serialization=OutputSerialization( - csv=CSVOutput(QuoteFields="ASNEEDED") - ) - - test_sql_expressions_custom_input_output(client, input_json_bytes, - input_serialization, output_serialization, tests, log_output) - - -def test_sql_operators(client, log_output): - - json_testfile = """{"id": 1, "name": "John", "age": 3} -{"id": 2, "name": "Elliot", "age": 4} -{"id": 3, "name": "Yves", "age": 5} -{"id": 4, "name": null, "age": 0} -""" - - tests = [ - # Logical operators - ("AND", "select * from S3Object s where s.id = 1 AND s.name = 'John'", b'1,John,3\n'), - ("NOT", "select * from S3Object s where NOT s.id = 1", b'2,Elliot,4\n3,Yves,5\n4,,0\n'), - ("OR", "select * from S3Object s where s.id = 1 OR s.id = 3", b'1,John,3\n3,Yves,5\n'), - # Comparison Operators - ("<", "select * from S3Object s where s.age < 4", b'1,John,3\n4,,0\n'), - (">", "select * from S3Object s where s.age > 4", b'3,Yves,5\n'), - ("<=", "select * from S3Object s where s.age <= 4", b'1,John,3\n2,Elliot,4\n4,,0\n'), - (">=", "select * from S3Object s where s.age >= 4", b'2,Elliot,4\n3,Yves,5\n'), - ("=", "select * from S3Object s where s.age = 4", b'2,Elliot,4\n'), - ("<>", "select * from S3Object s where s.age <> 4", b'1,John,3\n3,Yves,5\n4,,0\n'), - ("!=", "select * from S3Object s where s.age != 4", b'1,John,3\n3,Yves,5\n4,,0\n'), - ("BETWEEN", "select * from S3Object s where s.age BETWEEN 4 AND 5", b'2,Elliot,4\n3,Yves,5\n'), - ("IN", "select * from S3Object s where s.age IN (3,5)", b'1,John,3\n3,Yves,5\n'), - # Pattern Matching Operators - ("LIKE_", "select * from S3Object s where s.name LIKE '_ves'", b'3,Yves,5\n'), - ("LIKE%", "select * from S3Object s where s.name LIKE 'Ell%t'", b'2,Elliot,4\n'), - # Unitary Operators - ("NULL", "select * from S3Object s where s.name IS NULL", b'4,,0\n'), - ("NOT_NULL", "select * from S3Object s where s.age IS NOT NULL", b'1,John,3\n2,Elliot,4\n3,Yves,5\n4,,0\n'), - # Math Operators - ("+", "select * from S3Object s where s.age = 1+3 ", b'2,Elliot,4\n'), - ("-", "select * from S3Object s where s.age = 5-1 ", b'2,Elliot,4\n'), - ("*", "select * from S3Object s where s.age = 2*2 ", b'2,Elliot,4\n'), - ("%", "select * from S3Object s where s.age = 10%6 ", b'2,Elliot,4\n'), - ] - - try: - test_sql_expressions(client, json_testfile, tests, log_output) - except Exception as select_err: - raise select_err - # raise ValueError('Test {} unexpectedly failed with: {}'.format(test_name, select_err)) - # pass - - # Test passes - print(log_output.json_report()) - - -def test_sql_operators_precedence(client, log_output): - - json_testfile = """{"id": 1, "name": "Eric"}""" - - tests = [ - ("-_1", "select -3*3 from S3Object", b'-9\n'), - ("*", "select 10-3*2 from S3Object", b'4\n'), - ("/", "select 13-10/5 from S3Object", b'11\n'), - ("%", "select 13-10%5 from S3Object", b'13\n'), - ("+", "select 1+1*3 from S3Object", b'4\n'), - ("-_2", "select 1-1*3 from S3Object", b'-2\n'), - ("=", "select * from S3Object as s where s.id = 13-12", b'1,Eric\n'), - ("<>", "select * from S3Object as s where s.id <> 1-1", b'1,Eric\n'), - ("NOT", "select * from S3Object where false OR NOT false", b'1,Eric\n'), - ("AND", "select * from S3Object where true AND true OR false ", b'1,Eric\n'), - ("OR", "select * from S3Object where false OR NOT false", b'1,Eric\n'), - ("IN", "select * from S3Object as s where s.id <> -1 AND s.id IN (1,2,3)", b'1,Eric\n'), - ("BETWEEN", "select * from S3Object as s where s.id <> -1 AND s.id BETWEEN -1 AND 3", b'1,Eric\n'), - ("LIKE", "select * from S3Object as s where s.id <> -1 AND s.name LIKE 'E%'", b'1,Eric\n'), - ] - - try: - test_sql_expressions(client, json_testfile, tests, log_output) - except Exception as select_err: - raise select_err - # raise ValueError('Test {} unexpectedly failed with: {}'.format(test_name, select_err)) - # pass - - # Test passes - print(log_output.json_report()) - - - -def test_sql_functions_agg_cond_conv(client, log_output): - - json_testfile = """{"id": 1, "name": "John", "age": 3} -{"id": 2, "name": "Elliot", "age": 4} -{"id": 3, "name": "Yves", "age": 5} -{"id": 4, "name": "Christine", "age": null} -{"id": 5, "name": "Eric", "age": 0} -""" - tests = [ - # Aggregate functions - ("COUNT", "select count(*) from S3Object s", b'5\n'), - ("AVG", "select avg(s.age) from S3Object s", b'3\n'), - ("MAX", "select max(s.age) from S3Object s", b'5\n'), - ("MIN", "select min(s.age) from S3Object s", b'0\n'), - ("SUM", "select sum(s.age) from S3Object s", b'12\n'), - # Conditional functions - ("COALESCE", "SELECT COALESCE(s.age, 99) FROM S3Object s", b'3\n4\n5\n99\n0\n'), - ("NULLIF", "SELECT NULLIF(s.age, 0) FROM S3Object s", b'3\n4\n5\n\n\n'), - ## Conversion functions - ("CAST", "SELECT CAST(s.age AS FLOAT) FROM S3Object s", b'3.0\n4.0\n5.0\n\n0.0\n'), - - ] - - try: - test_sql_expressions(client, json_testfile, tests, log_output) - except Exception as select_err: - raise select_err - # raise ValueError('Test {} unexpectedly failed with: {}'.format(test_name, select_err)) - # pass - - # Test passes - print(log_output.json_report()) - - -def test_sql_functions_date(client, log_output): - - json_testfile = """ -{"id": 1, "name": "John", "datez": "2017-01-02T03:04:05.006+07:30"} -""" - - tests = [ - # DATE_ADD - ("DATE_ADD_1", "select DATE_ADD(year, 5, TO_TIMESTAMP(s.datez)) from S3Object as s", b'2022-01-02T03:04:05.006+07:30\n'), - ("DATE_ADD_2", "select DATE_ADD(month, 1, TO_TIMESTAMP(s.datez)) from S3Object as s", b'2017-02-02T03:04:05.006+07:30\n'), - ("DATE_ADD_3", "select DATE_ADD(day, -1, TO_TIMESTAMP(s.datez)) from S3Object as s", b'2017-01-01T03:04:05.006+07:30\n'), - ("DATE_ADD_4", "select DATE_ADD(hour, 1, TO_TIMESTAMP(s.datez)) from S3Object as s", b'2017-01-02T04:04:05.006+07:30\n'), - ("DATE_ADD_5", "select DATE_ADD(minute, 5, TO_TIMESTAMP(s.datez)) from S3Object as s", b'2017-01-02T03:09:05.006+07:30\n'), - ("DATE_ADD_6", "select DATE_ADD(second, 5, TO_TIMESTAMP(s.datez)) from S3Object as s", b'2017-01-02T03:04:10.006+07:30\n'), - # DATE_DIFF - ("DATE_DIFF_1", "select DATE_DIFF(year, TO_TIMESTAMP(s.datez), TO_TIMESTAMP('2011-01-01T')) from S3Object as s", b'-6\n'), - ("DATE_DIFF_2", "select DATE_DIFF(month, TO_TIMESTAMP(s.datez), TO_TIMESTAMP('2011T')) from S3Object as s", b'-72\n'), - ("DATE_DIFF_3", "select DATE_DIFF(day, TO_TIMESTAMP(s.datez), TO_TIMESTAMP('2010-01-02T')) from S3Object as s", b'-2556\n'), - # EXTRACT - ("EXTRACT_1", "select EXTRACT(year FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'2017\n'), - ("EXTRACT_2", "select EXTRACT(month FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'1\n'), - ("EXTRACT_3", "select EXTRACT(hour FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'3\n'), - ("EXTRACT_4", "select EXTRACT(minute FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'4\n'), - ("EXTRACT_5", "select EXTRACT(timezone_hour FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'7\n'), - ("EXTRACT_6", "select EXTRACT(timezone_minute FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'30\n'), - # TO_STRING - ("TO_STRING_1", "select TO_STRING(TO_TIMESTAMP(s.datez), 'MMMM d, y') from S3Object as s", b'"January 2, 2017"\n'), - ("TO_STRING_2", "select TO_STRING(TO_TIMESTAMP(s.datez), 'MMM d, yyyy') from S3Object as s", b'"Jan 2, 2017"\n'), - ("TO_STRING_3", "select TO_STRING(TO_TIMESTAMP(s.datez), 'M-d-yy') from S3Object as s", b'1-2-17\n'), - ("TO_STRING_4", "select TO_STRING(TO_TIMESTAMP(s.datez), 'MM-d-y') from S3Object as s", b'01-2-2017\n'), - ("TO_STRING_5", "select TO_STRING(TO_TIMESTAMP(s.datez), 'MMMM d, y h:m a') from S3Object as s", b'"January 2, 2017 3:4 AM"\n'), - ("TO_STRING_6", "select TO_STRING(TO_TIMESTAMP(s.datez), 'y-MM-dd''T''H:m:ssX') from S3Object as s", b'2017-01-02T3:4:05+0730\n'), - ("TO_STRING_7", "select TO_STRING(TO_TIMESTAMP(s.datez), 'y-MM-dd''T''H:m:ssX') from S3Object as s", b'2017-01-02T3:4:05+0730\n'), - ("TO_STRING_8", "select TO_STRING(TO_TIMESTAMP(s.datez), 'y-MM-dd''T''H:m:ssXXXX') from S3Object as s", b'2017-01-02T3:4:05+0730\n'), - ("TO_STRING_9", "select TO_STRING(TO_TIMESTAMP(s.datez), 'y-MM-dd''T''H:m:ssXXXXX') from S3Object as s", b'2017-01-02T3:4:05+07:30\n'), - ("TO_TIMESTAMP", "select TO_TIMESTAMP(s.datez) from S3Object as s", b'2017-01-02T03:04:05.006+07:30\n'), - ("UTCNOW", "select UTCNOW() from S3Object", datetime(1,1,1)), - - ] - - try: - test_sql_expressions(client, json_testfile, tests, log_output) - except Exception as select_err: - raise select_err - # raise ValueError('Test {} unexpectedly failed with: {}'.format(test_name, select_err)) - # pass - - # Test passes - print(log_output.json_report()) - -def test_sql_functions_string(client, log_output): - - json_testfile = """ -{"id": 1, "name": "John"} -{"id": 2, "name": " \tfoobar\t "} -{"id": 3, "name": "1112211foobar22211122"} -""" - - tests = [ - # CHAR_LENGTH - ("CHAR_LENGTH", "select CHAR_LENGTH(s.name) from S3Object as s", b'4\n24\n21\n'), - ("CHARACTER_LENGTH", "select CHARACTER_LENGTH(s.name) from S3Object as s", b'4\n24\n21\n'), - # LOWER - ("LOWER", "select LOWER(s.name) from S3Object as s where s.id= 1", b'john\n'), - # SUBSTRING - ("SUBSTRING_1", "select SUBSTRING(s.name FROM 2) from S3Object as s where s.id = 1", b'ohn\n'), - ("SUBSTRING_2", "select SUBSTRING(s.name FROM 2 FOR 2) from S3Object as s where s.id = 1", b'oh\n'), - ("SUBSTRING_3", "select SUBSTRING(s.name FROM -1 FOR 2) from S3Object as s where s.id = 1", b'\n'), - # TRIM - ("TRIM_1", "select TRIM(s.name) from S3Object as s where s.id = 2", b'\tfoobar\t\n'), - ("TRIM_2", "select TRIM(LEADING FROM s.name) from S3Object as s where s.id = 2", b'\tfoobar\t \n'), - ("TRIM_3", "select TRIM(TRAILING FROM s.name) from S3Object as s where s.id = 2", b' \tfoobar\t\n'), - ("TRIM_4", "select TRIM(BOTH FROM s.name) from S3Object as s where s.id = 2", b'\tfoobar\t\n'), - ("TRIM_5", "select TRIM(BOTH '12' FROM s.name) from S3Object as s where s.id = 3", b'foobar\n'), - # UPPER - ("UPPER", "select UPPER(s.name) from S3Object as s where s.id= 1", b'JOHN\n'), - ] - - try: - test_sql_expressions(client, json_testfile, tests, log_output) - except Exception as select_err: - raise select_err - # raise ValueError('Test {} unexpectedly failed with: {}'.format(test_name, select_err)) - # pass - - # Test passes - print(log_output.json_report()) - - -def test_sql_datatypes(client, log_output): - json_testfile = """ -{"name": "John"} -""" - tests = [ - ("bool", "select CAST('true' AS BOOL) from S3Object", b'true\n'), - ("int", "select CAST('13' AS INT) from S3Object", b'13\n'), - ("integer", "select CAST('13' AS INTEGER) from S3Object", b'13\n'), - ("string", "select CAST(true AS STRING) from S3Object", b'true\n'), - ("float", "select CAST('13.3' AS FLOAT) from S3Object", b'13.3\n'), - ("decimal", "select CAST('14.3' AS FLOAT) from S3Object", b'14.3\n'), - ("numeric", "select CAST('14.3' AS FLOAT) from S3Object", b'14.3\n'), - ("timestamp", "select CAST('2007-04-05T14:30Z' AS TIMESTAMP) from S3Object", b'2007-04-05T14:30Z\n'), - ] - - try: - test_sql_expressions(client, json_testfile, tests, log_output) - except Exception as select_err: - raise select_err - # raise ValueError('Test {} unexpectedly failed with: {}'.format(test_name, select_err)) - # pass - - # Test passes - print(log_output.json_report()) - - -def test_sql_select(client, log_output): - - json_testfile = """{"id": 1, "created": "June 27", "modified": "July 6" } -{"id": 2, "Created": "June 28", "Modified": "July 7", "Cast": "Random Date" }""" - tests = [ - ("select_1", "select * from S3Object", b'1,June 27,July 6\n2,June 28,July 7,Random Date\n'), - ("select_2", "select * from S3Object s", b'1,June 27,July 6\n2,June 28,July 7,Random Date\n'), - ("select_3", "select * from S3Object as s", b'1,June 27,July 6\n2,June 28,July 7,Random Date\n'), - ("select_4", "select s.line from S3Object as s", b'\n\n'), - ("select_5", 'select s."Created" from S3Object as s', b'\nJune 28\n'), - ("select_5", 'select s."Cast" from S3Object as s', b'\nRandom Date\n'), - ("where", 'select s.created from S3Object as s', b'June 27\nJune 28\n'), - ("limit", 'select * from S3Object as s LIMIT 1', b'1,June 27,July 6\n'), - ] - - try: - test_sql_expressions(client, json_testfile, tests, log_output) - except Exception as select_err: - raise select_err - # raise ValueError('Test {} unexpectedly failed with: {}'.format(test_name, select_err)) - # pass - - # Test passes - print(log_output.json_report()) - -def test_sql_select_json(client, log_output): - json_testcontent = """{ "Rules": [ {"id": "1"}, {"expr": "y > x"}, {"id": "2", "expr": "z = DEBUG"} ]} -{ "created": "June 27", "modified": "July 6" } -""" - tests = [ - ("select_1", "SELECT id FROM S3Object[*].Rules[*].id", b'{"id":"1"}\n{}\n{"id":"2"}\n{}\n'), - ("select_2", "SELECT id FROM S3Object[*].Rules[*].id WHERE id IS NOT MISSING", b'{"id":"1"}\n{"id":"2"}\n'), - ("select_3", "SELECT d.created, d.modified FROM S3Object[*] d", b'{}\n{"created":"June 27","modified":"July 6"}\n'), - ("select_4", "SELECT _1.created, _1.modified FROM S3Object[*]", b'{}\n{"created":"June 27","modified":"July 6"}\n'), - ("select_5", "Select s.rules[1].expr from S3Object s", b'{"expr":"y > x"}\n{}\n'), - ] - - input_serialization = InputSerialization(json=JSONInput(Type="DOCUMENT")) - output_serialization = OutputSerialization(json=JsonOutput()) - try: - test_sql_expressions_custom_input_output(client, json_testcontent, - input_serialization, output_serialization, tests, log_output) - except Exception as select_err: - raise select_err - # raise ValueError('Test {} unexpectedly failed with: {}'.format(test_name, select_err)) - # pass - - # Test passes - print(log_output.json_report()) - - -def test_sql_select_csv_no_header(client, log_output): - json_testcontent = """val1,val2,val3 -val4,val5,val6 -""" - tests = [ - ("select_1", "SELECT s._2 FROM S3Object as s", b'val2\nval5\n'), - ] - - input_serialization=InputSerialization( - csv=CSVInput( - FileHeaderInfo="NONE", - AllowQuotedRecordDelimiter="FALSE", - ), - ) - - output_serialization=OutputSerialization(csv=CSVOutput()) - try: - test_sql_expressions_custom_input_output(client, json_testcontent, - input_serialization, output_serialization, tests, log_output) - except Exception as select_err: - raise select_err - # raise ValueError('Test {} unexpectedly failed with: {}'.format(test_name, select_err)) - # pass - - # Test passes - print(log_output.json_report()) - - diff --git a/mint/run/core/s3select/tests.py b/mint/run/core/s3select/tests.py deleted file mode 100644 index 3e8c26dde..000000000 --- a/mint/run/core/s3select/tests.py +++ /dev/null @@ -1,86 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# MinIO Python Library for Amazon S3 Compatible Cloud Storage, -# (C) 2015-2020 MinIO, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -from sys import exit -from minio import Minio - -from utils import LogOutput -from sql_ops import * -from csv import * - -def main(): - """ - Functional testing for S3 select. - """ - - try: - access_key = os.getenv('ACCESS_KEY', 'Q3AM3UQ867SPQQA43P2F') - secret_key = os.getenv('SECRET_KEY', - 'zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG') - server_endpoint = os.getenv('SERVER_ENDPOINT', 'play.min.io') - secure = os.getenv('ENABLE_HTTPS', '1') == '1' - if server_endpoint == 'play.min.io': - access_key = 'Q3AM3UQ867SPQQA43P2F' - secret_key = 'zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG' - secure = True - - client = Minio(server_endpoint, access_key, secret_key, secure=False) - - log_output = LogOutput(client.select_object_content, 'test_csv_input_quote_char') - test_csv_input_custom_quote_char(client, log_output) - - log_output = LogOutput(client.select_object_content, 'test_csv_output_quote_char') - test_csv_output_custom_quote_char(client, log_output) - - log_output = LogOutput(client.select_object_content, 'test_sql_operators') - test_sql_operators(client, log_output) - - log_output = LogOutput(client.select_object_content, 'test_sql_operators_precedence') - test_sql_operators_precedence(client, log_output) - - log_output = LogOutput(client.select_object_content, 'test_sql_functions_agg_cond_conv') - test_sql_functions_agg_cond_conv(client, log_output) - - log_output = LogOutput(client.select_object_content, 'test_sql_functions_date') - test_sql_functions_date(client, log_output) - - log_output = LogOutput(client.select_object_content, 'test_sql_functions_string') - test_sql_functions_string(client, log_output) - - log_output = LogOutput(client.select_object_content, 'test_sql_datatypes') - test_sql_datatypes(client, log_output) - - log_output = LogOutput(client.select_object_content, 'test_sql_select') - test_sql_select(client, log_output) - - log_output = LogOutput(client.select_object_content, 'test_sql_select_json') - test_sql_select_json(client, log_output) - - log_output = LogOutput(client.select_object_content, 'test_sql_select_csv') - test_sql_select_csv_no_header(client, log_output) - - except Exception as err: - print(log_output.json_report(err)) - exit(1) - -if __name__ == "__main__": - # Execute only if run as a script - main() - - - diff --git a/mint/run/core/s3select/utils.py b/mint/run/core/s3select/utils.py deleted file mode 100644 index 1baabf983..000000000 --- a/mint/run/core/s3select/utils.py +++ /dev/null @@ -1,106 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# MinIO Python Library for Amazon S3 Compatible Cloud Storage, -# (C) 2015-2020 MinIO, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import uuid -import inspect -import json -import time -import traceback - -class LogOutput(object): - """ - LogOutput is the class for log output. It is required standard for all - SDK tests controlled by mint. - Here are its attributes: - 'name': name of the SDK under test, e.g. 's3select' - 'function': name of the method/api under test with its signature - The following python code can be used to - pull args information of a and to - put together with the method name: - .__name__+'('+', '.join(args_list)+')' - e.g. 'remove_object(bucket_name, object_name)' - 'args': method/api arguments with their values, in - dictionary form: {'arg1': val1, 'arg2': val2, ...} - 'duration': duration of the whole test in milliseconds, - defaults to 0 - 'alert': any extra information user is needed to be alerted about, - like whether this is a Blocker/Gateway/Server related - issue, etc., defaults to None - 'message': descriptive error message, defaults to None - 'error': stack-trace/exception message(only in case of failure), - actual low level exception/error thrown by the program, - defaults to None - 'status': exit status, possible values are 'PASS', 'FAIL', 'NA', - defaults to 'PASS' - """ - - PASS = 'PASS' - FAIL = 'FAIL' - NA = 'NA' - - def __init__(self, meth, test_name): - self.__args_list = inspect.getargspec(meth).args[1:] - self.__name = 's3select:'+test_name - self.__function = meth.__name__+'('+', '.join(self.__args_list)+')' - self.__args = {} - self.__duration = 0 - self.__alert = '' - self.__message = None - self.__error = None - self.__status = self.PASS - self.__start_time = time.time() - - @property - def name(self): return self.__name - - @property - def function(self): return self.__function - - @property - def args(self): return self.__args - - @name.setter - def name(self, val): self.__name = val - - @function.setter - def function(self, val): self.__function = val - - @args.setter - def args(self, val): self.__args = val - - def json_report(self, err_msg='', alert='', status=''): - self.__args = {k: v for k, v in self.__args.items() if v and v != ''} - entry = {'name': self.__name, - 'function': self.__function, - 'args': self.__args, - 'duration': int(round((time.time() - self.__start_time)*1000)), - 'alert': str(alert), - 'message': str(err_msg), - 'error': traceback.format_exc() if err_msg and err_msg != '' else '', - 'status': status if status and status != '' else - self.FAIL if err_msg and err_msg != '' else self.PASS - } - return json.dumps({k: v for k, v in entry.items() if v and v != ''}) - - -def generate_bucket_name(): - return "s3select-test-" + str(uuid.uuid4()) - -def generate_object_name(): - return str(uuid.uuid4()) - - diff --git a/mint/run/core/security/run.sh b/mint/run/core/security/run.sh deleted file mode 100755 index 9e7c2175a..000000000 --- a/mint/run/core/security/run.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -# -# Mint (C) 2018 Minio, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# handle command line arguments -if [ $# -ne 2 ]; then - echo "usage: run.sh " - exit 1 -fi - -output_log_file="$1" -error_log_file="$2" - -# run tests -/mint/run/core/security/tls-tests 1>>"$output_log_file" 2>"$error_log_file" diff --git a/mint/run/core/security/tls-tests.go b/mint/run/core/security/tls-tests.go deleted file mode 100644 index adbd87b52..000000000 --- a/mint/run/core/security/tls-tests.go +++ /dev/null @@ -1,272 +0,0 @@ -// Mint, (C) 2018 Minio, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software - -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "crypto/tls" - "encoding/json" - "fmt" - "os" - "time" - - log "github.com/sirupsen/logrus" -) - -const testName = "TLS-tests" - -const ( - // PASS indicate that a test passed - PASS = "PASS" - // FAIL indicate that a test failed - FAIL = "FAIL" - // NA indicates that a test is not applicable - NA = "NA" -) - -func main() { - log.SetOutput(os.Stdout) - log.SetFormatter(&mintJSONFormatter{}) - log.SetLevel(log.InfoLevel) - - endpoint := os.Getenv("SERVER_ENDPOINT") - secure := os.Getenv("ENABLE_HTTPS") - if secure != "1" { - log.WithFields(log.Fields{"name:": testName, "status": NA, "message": "TLS is not enabled"}).Info() - return - } - - testTLSVersions(endpoint) - testTLSCiphers(endpoint) - testTLSEllipticCurves(endpoint) -} - -// Tests whether the endpoint accepts TLS1.0 or TLS1.1 connections - fail if so. -// Tests whether the endpoint accepts TLS1.2 connections - fail if not. -func testTLSVersions(endpoint string) { - const function = "TLSVersions" - startTime := time.Now() - - // Tests whether the endpoint accepts TLS1.0 or TLS1.1 connections - args := map[string]interface{}{ - "MinVersion": "tls.VersionTLS10", - "MaxVersion": "tls.VersionTLS11", - } - _, err := tls.Dial("tcp", endpoint, &tls.Config{ - MinVersion: tls.VersionTLS10, - MaxVersion: tls.VersionTLS11, - }) - if err == nil { - failureLog(function, args, startTime, "", "Endpoint accepts insecure connection", err).Error() - return - } - - // Tests whether the endpoint accepts TLS1.2 connections - args = map[string]interface{}{ - "MinVersion": "tls.VersionTLS12", - } - _, err = tls.Dial("tcp", endpoint, &tls.Config{ - MinVersion: tls.VersionTLS12, - }) - if err != nil { - failureLog(function, args, startTime, "", "Endpoint rejects secure connection", err).Error() - return - } - successLog(function, args, startTime) -} - -// Tests whether the endpoint accepts SSL3.0, TLS1.0 or TLS1.1 connections - fail if so. -// Tests whether the endpoint accepts TLS1.2 connections - fail if not. -func testTLSCiphers(endpoint string) { - const function = "TLSCiphers" - startTime := time.Now() - - // Tests whether the endpoint accepts insecure ciphers - args := map[string]interface{}{ - "MinVersion": "tls.VersionTLS12", - "CipherSuites": unsupportedCipherSuites, - } - _, err := tls.Dial("tcp", endpoint, &tls.Config{ - MinVersion: tls.VersionTLS12, - CipherSuites: unsupportedCipherSuites, - }) - if err == nil { - failureLog(function, args, startTime, "", "Endpoint accepts insecure cipher suites", err).Error() - return - } - - // Tests whether the endpoint accepts at least one secure cipher - args = map[string]interface{}{ - "MinVersion": "tls.VersionTLS12", - "CipherSuites": supportedCipherSuites, - } - _, err = tls.Dial("tcp", endpoint, &tls.Config{ - MinVersion: tls.VersionTLS12, - CipherSuites: supportedCipherSuites, - }) - if err != nil { - failureLog(function, args, startTime, "", "Endpoint rejects all secure cipher suites", err).Error() - return - } - - // Tests whether the endpoint accepts at least one default cipher - args = map[string]interface{}{ - "MinVersion": "tls.VersionTLS12", - "CipherSuites": nil, - } - _, err = tls.Dial("tcp", endpoint, &tls.Config{ - MinVersion: tls.VersionTLS12, - CipherSuites: nil, // default value - }) - if err != nil { - failureLog(function, args, startTime, "", "Endpoint rejects default cipher suites", err).Error() - return - } - successLog(function, args, startTime) -} - -// Tests whether the endpoint accepts the P-384 or P-521 elliptic curve - fail if so. -// Tests whether the endpoint accepts Curve25519 or P-256 - fail if not. -func testTLSEllipticCurves(endpoint string) { - const function = "TLSEllipticCurves" - startTime := time.Now() - - // Tests whether the endpoint accepts curves using non-constant time implementations. - args := map[string]interface{}{ - "CurvePreferences": unsupportedCurves, - } - _, err := tls.Dial("tcp", endpoint, &tls.Config{ - MinVersion: tls.VersionTLS12, - CurvePreferences: unsupportedCurves, - CipherSuites: supportedCipherSuites, - }) - if err == nil { - failureLog(function, args, startTime, "", "Endpoint accepts insecure elliptic curves", err).Error() - return - } - - // Tests whether the endpoint accepts curves using constant time implementations. - args = map[string]interface{}{ - "CurvePreferences": unsupportedCurves, - } - _, err = tls.Dial("tcp", endpoint, &tls.Config{ - MinVersion: tls.VersionTLS12, - CurvePreferences: supportedCurves, - CipherSuites: supportedCipherSuites, - }) - if err != nil { - failureLog(function, args, startTime, "", "Endpoint does not accept secure elliptic curves", err).Error() - return - } - successLog(function, args, startTime) -} - -func successLog(function string, args map[string]interface{}, startTime time.Time) *log.Entry { - duration := time.Since(startTime).Nanoseconds() / 1000000 - return log.WithFields(log.Fields{ - "name": testName, - "function": function, - "args": args, - "duration": duration, - "status": PASS, - }) -} - -func failureLog(function string, args map[string]interface{}, startTime time.Time, alert string, message string, err error) *log.Entry { - duration := time.Since(startTime).Nanoseconds() / 1000000 - fields := log.Fields{ - "name": testName, - "function": function, - "args": args, - "duration": duration, - "status": FAIL, - "alert": alert, - "message": message, - } - if err != nil { - fields["error"] = err - } - return log.WithFields(fields) -} - -type mintJSONFormatter struct { -} - -func (f *mintJSONFormatter) Format(entry *log.Entry) ([]byte, error) { - data := make(log.Fields, len(entry.Data)) - for k, v := range entry.Data { - switch v := v.(type) { - case error: - // Otherwise errors are ignored by `encoding/json` - // https://github.com/sirupsen/logrus/issues/137 - data[k] = v.Error() - default: - data[k] = v - } - } - - serialized, err := json.Marshal(data) - if err != nil { - return nil, fmt.Errorf("Failed to marshal fields to JSON, %w", err) - } - return append(serialized, '\n'), nil -} - -// Secure Go implementations of modern TLS ciphers -// The following ciphers are excluded because: -// - RC4 ciphers: RC4 is broken -// - 3DES ciphers: Because of the 64 bit blocksize of DES (Sweet32) -// - CBC-SHA256 ciphers: No countermeasures against Lucky13 timing attack -// - CBC-SHA ciphers: Legacy ciphers (SHA-1) and non-constant time -// implementation of CBC. -// (CBC-SHA ciphers can be enabled again if required) -// - RSA key exchange ciphers: Disabled because of dangerous PKCS1-v1.5 RSA -// padding scheme. See Bleichenbacher attacks. -var supportedCipherSuites = []uint16{ - tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, - tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305, - tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, - tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, - tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, - tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, -} - -// Supported elliptic curves: Implementations are constant-time. -var supportedCurves = []tls.CurveID{tls.X25519, tls.CurveP256} - -var unsupportedCipherSuites = []uint16{ - tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA, // Go stack contains (some) countermeasures against timing attacks (Lucky13) - tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256, // No countermeasures against timing attacks - tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA, // Go stack contains (some) countermeasures against timing attacks (Lucky13) - tls.TLS_ECDHE_ECDSA_WITH_RC4_128_SHA, // Broken cipher - tls.TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA, // Sweet32 - tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, // Go stack contains (some) countermeasures against timing attacks (Lucky13) - tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256, // No countermeasures against timing attacks - tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, // Go stack contains (some) countermeasures against timing attacks (Lucky13) - tls.TLS_ECDHE_RSA_WITH_RC4_128_SHA, // Broken cipher - - // all RSA-PKCS1-v1.5 ciphers are disabled - danger of Bleichenbacher attack variants - tls.TLS_RSA_WITH_3DES_EDE_CBC_SHA, // Sweet32 - tls.TLS_RSA_WITH_AES_128_CBC_SHA, // Go stack contains (some) countermeasures against timing attacks (Lucky13) - tls.TLS_RSA_WITH_AES_128_CBC_SHA256, // No countermeasures against timing attacks - tls.TLS_RSA_WITH_AES_256_CBC_SHA, // Go stack contains (some) countermeasures against timing attacks (Lucky13) - tls.TLS_RSA_WITH_RC4_128_SHA, // Broken cipher - - tls.TLS_RSA_WITH_AES_128_GCM_SHA256, // Disabled because of RSA-PKCS1-v1.5 - AES-GCM is considered secure. - tls.TLS_RSA_WITH_AES_256_GCM_SHA384, // Disabled because of RSA-PKCS1-v1.5 - AES-GCM is considered secure. -} - -// Unsupported elliptic curves: Implementations are not constant-time. -var unsupportedCurves = []tls.CurveID{tls.CurveP384, tls.CurveP521} diff --git a/misc/build.go b/misc/build.go new file mode 100644 index 000000000..3d82ab96f --- /dev/null +++ b/misc/build.go @@ -0,0 +1,6 @@ +package misc + +var ( + Build = "now" + Version = "dev" +)