EliChin/frostfs-s3-gw
1
0
Fork 0
forked from TrueCloudLab/frostfs-s3-gw
Code Issues Pull requests Projects Releases Packages Wiki Activity

Cleanup + Makefile + Dockerfile

Browse source
This commit is contained in:
Evgeniy Kulikov 2020-07-03 18:08:57 +03:00
parent a1334a4d1e
commit 8d97dcbf10
89 changed files with 98 additions and 9650 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

31
.gitignore vendored
View file

@ -1,24 +1,9 @@
**/*.swp
cover.out
*~
minio
!*/
site/
**/*.test
**/*.sublime-workspace
/.idea/
/Minio.iml
**/access.log
vendor/**/*.js
vendor/**/*.json
release
# IDE
.idea
.vscode
# Vendoring
vendor
# tempfiles
.DS_Store
*.syso
coverage.txt
.vscode/
*.tar.bz2
parts/
prime/
stage/
.sia_temp/
config.json

27
.golangci.yml
View file

@ -1,27 +0,0 @@
linters-settings:
golint:
min-confidence: 0
misspell:
locale: US
linters:
disable-all: true
enable:
- typecheck
- goimports
- misspell
- govet
- golint
- ineffassign
- gosimple
- deadcode
- structcheck
issues:
exclude-use-default: false
exclude:
- should have a package comment
- error strings should not be capitalized or end with punctuation or a newline
service:
golangci-lint-version: 1.20.0 # use the fixed version to not introduce new linters unexpectedly

18
.mailmap
View file

@ -1,18 +0,0 @@
# Generate CONTRIBUTORS.md: contributors.sh
# Tip for finding duplicates (besides scanning the output of CONTRIBUTORS.md for name
# duplicates that aren't also email duplicates): scan the output of:
# git log --format='%aE - %aN' | sort -uf
#
# For explanation on this file format: man git-shortlog
Anand Babu (AB) Periasamy <ab@min.io> Anand Babu (AB) Periasamy <abperiasamy@users.noreply.github.com>
Anand Babu (AB) Periasamy <ab@min.io> <ab@unlocksmith.org>
Anis Elleuch <vadmeste@gmail.com>
Frederick F. Kautz IV <fkautz@min.io> <fkautz@alumni.cmu.edu>
Harshavardhana <harsha@min.io> <harsha@harshavardhana.net>
Harshavardhana <harsha@min.io> <badger@gitter.im>
Harshavardhana <harsha@min.io>
Krishna Srinivas <krishna@min.io> <krishna.srinivas@gmail.com>
Matthew Farrellee <matt@cs.wisc.edu>
Nate Rosenblum <flander@gmail.com>

50
Dockerfile
View file

@ -1,36 +1,30 @@
FROM golang:1.13-alpine
FROM golang:1 as builder
LABEL maintainer="MinIO Inc <dev@min.io>"
COPY . /src
ENV GOPATH /go
ENV CGO_ENABLED 0
ENV GO111MODULE on
WORKDIR /src
RUN \
apk add --no-cache git && \
git clone https://github.com/minio/minio && cd minio && \
go install -v -ldflags "$(go run buildscripts/gen-ldflags.go)"
ARG VERSION=dev
# https://github.com/golang/go/wiki/Modules#how-do-i-use-vendoring-with-modules-is-vendoring-going-away
# go build -mod=vendor
# The -gcflags "all=-N -l" flag helps us get a better debug experience
RUN set -x \
&& export BUILD=$(date -u +%s%N) \
&& export REPO=$(go list -m) \
&& export LDFLAGS="-X ${REPO}/misc.Version=${VERSION} -X ${REPO}/misc.Build=${BUILD}" \
&& export GOGC=off \
&& export CGO_ENABLED=0 \
&& [ -d "./vendor" ] || go mod vendor \
&& go build -v -mod=vendor -trimpath -gcflags "all=-N -l" -ldflags "${LDFLAGS}" -o /go/bin/neofs-s3 ./main.go
# Executable image
FROM alpine:3.10
ENV MINIO_UPDATE off
ENV MINIO_ACCESS_KEY_FILE=access_key \
MINIO_SECRET_KEY_FILE=secret_key \
MINIO_KMS_MASTER_KEY_FILE=kms_master_key \
MINIO_SSE_MASTER_KEY_FILE=sse_master_key
WORKDIR /
EXPOSE 9000
COPY --from=builder /go/bin/neofs-s3 /usr/bin/neofs-s3
COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
COPY --from=0 /go/bin/minio /usr/bin/minio
COPY --from=0 /go/minio/CREDITS /third_party/
COPY --from=0 /go/minio/dockerscripts/docker-entrypoint.sh /usr/bin/
RUN \
apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' && \
echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf
ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
VOLUME ["/data"]
CMD ["minio"]
# Run delve
CMD ["/usr/bin/neofs-s3"]

41
Dockerfile.arm.release
View file

@ -1,41 +0,0 @@
FROM golang:1.13-alpine as builder
WORKDIR /home
ENV GOPATH /go
ENV CGO_ENABLED 0
ENV GO111MODULE on
RUN \
apk add --no-cache git 'curl>7.61.0' && \
git clone https://github.com/minio/minio && \
curl -L https://github.com/balena-io/qemu/releases/download/v3.0.0%2Bresin/qemu-3.0.0+resin-arm.tar.gz | tar zxvf - -C . && mv qemu-3.0.0+resin-arm/qemu-arm-static .
FROM arm32v7/alpine:3.10
LABEL maintainer="MinIO Inc <dev@min.io>"
COPY dockerscripts/docker-entrypoint.sh /usr/bin/
COPY CREDITS /third_party/
COPY --from=builder /home/qemu-arm-static /usr/bin/qemu-arm-static
ENV MINIO_UPDATE off
ENV MINIO_ACCESS_KEY_FILE=access_key \
MINIO_SECRET_KEY_FILE=secret_key \
MINIO_KMS_MASTER_KEY_FILE=kms_master_key \
MINIO_SSE_MASTER_KEY_FILE=sse_master_key
RUN \
apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' && \
echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
curl https://dl.min.io/server/minio/release/linux-arm/minio > /usr/bin/minio && \
chmod +x /usr/bin/minio && \
chmod +x /usr/bin/docker-entrypoint.sh
EXPOSE 9000
ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
VOLUME ["/data"]
CMD ["minio"]

41
Dockerfile.arm64.release
View file

@ -1,41 +0,0 @@
FROM golang:1.13-alpine as builder
WORKDIR /home
ENV GOPATH /go
ENV CGO_ENABLED 0
ENV GO111MODULE on
RUN \
apk add --no-cache git 'curl>7.61.0' && \
git clone https://github.com/minio/minio && \
curl -L https://github.com/balena-io/qemu/releases/download/v3.0.0%2Bresin/qemu-3.0.0+resin-arm.tar.gz | tar zxvf - -C . && mv qemu-3.0.0+resin-arm/qemu-arm-static .
FROM arm64v8/alpine:3.10
LABEL maintainer="MinIO Inc <dev@min.io>"
COPY dockerscripts/docker-entrypoint.sh /usr/bin/
COPY CREDITS /third_party/
COPY --from=builder /home/qemu-arm-static /usr/bin/qemu-arm-static
ENV MINIO_UPDATE off
ENV MINIO_ACCESS_KEY_FILE=access_key \
MINIO_SECRET_KEY_FILE=secret_key \
MINIO_KMS_MASTER_KEY_FILE=kms_master_key \
MINIO_SSE_MASTER_KEY_FILE=sse_master_key
RUN \
apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' && \
echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
curl https://dl.min.io/server/minio/release/linux-arm64/minio > /usr/bin/minio && \
chmod +x /usr/bin/minio && \
chmod +x /usr/bin/docker-entrypoint.sh
EXPOSE 9000
ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
VOLUME ["/data"]
CMD ["minio"]

27
Dockerfile.dev
View file

@ -1,27 +0,0 @@
FROM alpine:3.10
LABEL maintainer="MinIO Inc <dev@min.io>"
COPY dockerscripts/docker-entrypoint.sh /usr/bin/
COPY minio /usr/bin/
COPY CREDITS /third_party/
ENV MINIO_UPDATE off
ENV MINIO_ACCESS_KEY_FILE=access_key \
MINIO_SECRET_KEY_FILE=secret_key \
MINIO_KMS_MASTER_KEY_FILE=kms_master_key \
MINIO_SSE_MASTER_KEY_FILE=sse_master_key
RUN \
apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' && \
echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
chmod +x /usr/bin/minio && \
chmod +x /usr/bin/docker-entrypoint.sh
EXPOSE 9000
ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
VOLUME ["/data"]
CMD ["minio"]

13
Dockerfile.dev.browser
View file

@ -1,13 +0,0 @@
FROM ubuntu
LABEL maintainer="MinIO Inc <dev@min.io>"
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends --no-install-suggests \
git golang make npm && \
apt-get clean && rm -rf /var/lib/apt/lists/*
ENV PATH=$PATH:/root/go/bin
RUN go get github.com/go-bindata/go-bindata/go-bindata && \
go get github.com/elazarl/go-bindata-assetfs/go-bindata-assetfs

17
Dockerfile.mint
View file

@ -1,17 +0,0 @@
FROM ubuntu:18.04
ENV DEBIAN_FRONTEND noninteractive
ENV LANG C.UTF-8
ENV GOROOT /usr/local/go
ENV GOPATH /usr/local/gopath
ENV PATH $GOPATH/bin:$GOROOT/bin:$PATH
ENV MINT_ROOT_DIR /mint
COPY mint /mint
RUN apt-get --yes update && apt-get --yes upgrade && \
apt-get --yes --quiet install wget jq curl git dnsmasq && \
cd /mint && /mint/release.sh
WORKDIR /mint
ENTRYPOINT ["/mint/entrypoint.sh"]

37
Dockerfile.release
View file

@ -1,37 +0,0 @@
FROM golang:1.13-alpine
ENV GOPATH /go
ENV CGO_ENABLED 0
ENV GO111MODULE on
RUN \
apk add --no-cache git && \
git clone https://github.com/minio/minio
FROM alpine:3.10
LABEL maintainer="MinIO Inc <dev@min.io>"
COPY dockerscripts/docker-entrypoint.sh /usr/bin/
COPY CREDITS /third_party/
ENV MINIO_UPDATE off
ENV MINIO_ACCESS_KEY_FILE=access_key \
MINIO_SECRET_KEY_FILE=secret_key \
MINIO_KMS_MASTER_KEY_FILE=kms_master_key \
MINIO_SSE_MASTER_KEY_FILE=sse_master_key
RUN \
apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' && \
echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
curl https://dl.min.io/server/minio/release/linux-amd64/minio > /usr/bin/minio && \
chmod +x /usr/bin/minio && \
chmod +x /usr/bin/docker-entrypoint.sh
EXPOSE 9000
ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
VOLUME ["/data"]
CMD ["minio"]

108
Makefile
View file

@ -1,79 +1,39 @@
PWD := $(shell pwd)
GOPATH := $(shell go env GOPATH)
LDFLAGS := $(shell go run buildscripts/gen-ldflags.go)
VERSION ?= "$(shell git describe --tags 2>/dev/null | sed 's/^v//')"
BUILD_VERSION ?= "$(shell git describe --abbrev=0 --tags | sed 's/^v//')"
GOARCH := $(shell go env GOARCH)
GOOS := $(shell go env GOOS)
.PHONY: help format deps
VERSION ?= $(shell git describe --tags)
TAG ?= "minio/minio:$(VERSION)"
# Show this help prompt
help:
@echo ' Usage:'
@echo ''
@echo ' make <target>'
@echo ''
@echo ' Targets:'
@echo ''
@awk '/^#/{ comment = substr($$0,3) } comment && /^[a-zA-Z][a-zA-Z0-9_-]+ ?:/{ print " ", $$1, comment }' $(MAKEFILE_LIST) | column -t -s ':' | grep -v 'IGNORE' | sort | uniq
all: build
# Reformat code
format:
@[ ! -z `which goimports` ] || (echo "install goimports" && exit 2)
@for f in `find . -type f -name '*.go' -not -path './vendor/*' -not -name '*.pb.go' -prune`; do \
echo "⇒ Processing $$f"; \
goimports -w $$f; \
done
checks:
@echo "Checking dependencies"
@(env bash $(PWD)/buildscripts/checkdeps.sh)
# Make sure that all files added to commit
deps:
@printf "⇒ Ensure vendor: "
@go mod tidy -v && echo OK || (echo fail && exit 2)
@printf "⇒ Download requirements: "
@go mod download && echo OK || (echo fail && exit 2)
@printf "⇒ Store vendor localy: "
@go mod vendor && echo OK || (echo fail && exit 2)
getdeps:
@mkdir -p ${GOPATH}/bin
@which golangci-lint 1>/dev/null || (echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOPATH)/bin v1.27.0)
crosscompile:
@(env bash $(PWD)/buildscripts/cross-compile.sh)
verifiers: getdeps fmt lint
fmt:
@echo "Running $@ check"
@GO111MODULE=on gofmt -d cmd/
@GO111MODULE=on gofmt -d pkg/
lint:
@echo "Running $@ check"
@GO111MODULE=on ${GOPATH}/bin/golangci-lint cache clean
@GO111MODULE=on ${GOPATH}/bin/golangci-lint run --timeout=5m --config ./.golangci.yml
# Builds minio, runs the verifiers then runs the tests.
check: test
test: verifiers build
@echo "Running unit tests"
@GO111MODULE=on CGO_ENABLED=0 go test -tags kqueue ./... 1>/dev/null
test-race: verifiers build
@echo "Running unit tests under -race"
@(env bash $(PWD)/buildscripts/race.sh)
# Verify minio binary
verify:
@echo "Verifying build with race"
@GO111MODULE=on CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
@(env bash $(PWD)/buildscripts/verify-build.sh)
# Verify healing of disks with minio binary
verify-healing:
@echo "Verify healing build with race"
@GO111MODULE=on CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
@(env bash $(PWD)/buildscripts/verify-healing.sh)
# Builds minio locally.
build: checks
@echo "Building minio binary to './minio'"
@GO111MODULE=on CGO_ENABLED=0 go build -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
docker: build
@docker build -t $(TAG) . -f Dockerfile.dev
# Builds minio and installs it to $GOPATH/bin.
install: build
@echo "Installing minio binary to '$(GOPATH)/bin/minio'"
@mkdir -p $(GOPATH)/bin && cp -f $(PWD)/minio $(GOPATH)/bin/minio
@echo "Installation successful. To learn more, try \"minio --help\"."
clean:
@echo "Cleaning up all the generated files"
@find . -name '*.test' | xargs rm -fv
@find . -name '*~' | xargs rm -fv
@rm -rvf minio
@rm -rvf build
@rm -rvf release
@rm -rvf .verify*
# Build current docker image
image-build:
@echo "⇒ Build docker-image"
@docker build \
--build-arg VERSION=$(VERSION) \
-f Dockerfile \
-t nspccdev/neofs-s3-gate:$(VERSION) .

37
cmd/web-handlers.go
View file

@ -37,7 +37,6 @@ import (
"github.com/klauspost/compress/zip"
miniogopolicy "github.com/minio/minio-go/v6/pkg/policy"
"github.com/minio/minio-go/v6/pkg/s3utils"
"github.com/minio/minio/browser"
"github.com/minio/minio/cmd/config/etcd/dns"
"github.com/minio/minio/cmd/config/identity/openid"
"github.com/minio/minio/cmd/crypto"
@ -108,7 +107,7 @@ func (web *webAPIHandlers) ServerInfo(r *http.Request, args *WebGenericArgs, rep
reply.MinioPlatform = platform
reply.MinioRuntime = goruntime
reply.UIVersion = browser.UIVersion
reply.UIVersion = "browser.UIVersion"
return nil
}
@ -131,7 +130,7 @@ func (web *webAPIHandlers) StorageInfo(r *http.Request, args *WebGenericArgs, re
}
dataUsageInfo, _ := loadDataUsageFromBackend(ctx, objectAPI)
reply.Used = dataUsageInfo.ObjectsTotalSize
reply.UIVersion = browser.UIVersion
reply.UIVersion = "browser.UIVersion"
return nil
}
@ -181,7 +180,7 @@ func (web *webAPIHandlers) MakeBucket(r *http.Request, args *MakeBucketArgs, rep
return toJSONError(ctx, err)
}
reply.UIVersion = browser.UIVersion
reply.UIVersion = "browser.UIVersion"
return nil
}
return toJSONError(ctx, err)
@ -193,7 +192,7 @@ func (web *webAPIHandlers) MakeBucket(r *http.Request, args *MakeBucketArgs, rep
return toJSONError(ctx, err, args.BucketName)
}
reply.UIVersion = browser.UIVersion
reply.UIVersion = "browser.UIVersion"
return nil
}
@ -231,7 +230,7 @@ func (web *webAPIHandlers) DeleteBucket(r *http.Request, args *RemoveBucketArgs,
return toJSONError(ctx, errInvalidBucketName)
}
reply.UIVersion = browser.UIVersion
reply.UIVersion = "browser.UIVersion"
if isRemoteCallRequired(ctx, args.BucketName, objectAPI) {
sr, err := globalDNSConfig.Get(args.BucketName)
@ -351,7 +350,7 @@ func (web *webAPIHandlers) ListBuckets(r *http.Request, args *WebGenericArgs, re
}
}
reply.UIVersion = browser.UIVersion
reply.UIVersion = "browser.UIVersion"
return nil
}
@ -384,7 +383,7 @@ type WebObjectInfo struct {
// ListObjects - list objects api.
func (web *webAPIHandlers) ListObjects(r *http.Request, args *ListObjectsArgs, reply *ListObjectsRep) error {
ctx := newWebContext(r, args, "WebListObjects")
reply.UIVersion = browser.UIVersion
reply.UIVersion = "browser.UIVersion"
objectAPI := web.ObjectAPI()
if objectAPI == nil {
return toJSONError(ctx, errServerNotInitialized)
@ -622,7 +621,7 @@ func (web *webAPIHandlers) RemoveObject(r *http.Request, args *RemoveObjectArgs,
return toJSONError(ctx, errInvalidBucketName)
}
reply.UIVersion = browser.UIVersion
reply.UIVersion = "browser.UIVersion"
if isRemoteCallRequired(ctx, args.BucketName, objectAPI) {
sr, err := globalDNSConfig.Get(args.BucketName)
if err != nil {
@ -813,7 +812,7 @@ func (web *webAPIHandlers) Login(r *http.Request, args *LoginArgs, reply *LoginR
}
reply.Token = token
reply.UIVersion = browser.UIVersion
reply.UIVersion = "browser.UIVersion"
return nil
}
@ -839,7 +838,7 @@ func (web webAPIHandlers) GenerateAuth(r *http.Request, args *WebGenericArgs, re
}
reply.AccessKey = cred.AccessKey
reply.SecretKey = cred.SecretKey
reply.UIVersion = browser.UIVersion
reply.UIVersion = "browser.UIVersion"
return nil
}
@ -898,7 +897,7 @@ func (web *webAPIHandlers) SetAuth(r *http.Request, args *SetAuthArgs, reply *Se
return toJSONError(ctx, err)
}
reply.UIVersion = browser.UIVersion
reply.UIVersion = "browser.UIVersion"
return nil
}
@ -937,7 +936,7 @@ func (web *webAPIHandlers) CreateURLToken(r *http.Request, args *WebGenericArgs,
reply.Token = token
}
reply.UIVersion = browser.UIVersion
reply.UIVersion = "browser.UIVersion"
return nil
}
@ -1672,7 +1671,7 @@ func (web *webAPIHandlers) GetBucketPolicy(r *http.Request, args *GetBucketPolic
}
}
reply.UIVersion = browser.UIVersion
reply.UIVersion = "browser.UIVersion"
reply.Policy = miniogopolicy.GetPolicy(policyInfo.Statements, args.BucketName, args.Prefix)
return nil
@ -1764,7 +1763,7 @@ func (web *webAPIHandlers) ListAllBucketPolicies(r *http.Request, args *ListAllB
}
}
reply.UIVersion = browser.UIVersion
reply.UIVersion = "browser.UIVersion"
for prefix, policy := range miniogopolicy.GetPolicies(policyInfo.Statements, args.BucketName, "") {
bucketName, objectPrefix := path2BucketObject(prefix)
objectPrefix = strings.TrimSuffix(objectPrefix, "*")
@ -1789,7 +1788,7 @@ type SetBucketPolicyWebArgs struct {
func (web *webAPIHandlers) SetBucketPolicy(r *http.Request, args *SetBucketPolicyWebArgs, reply *WebGenericRep) error {
ctx := newWebContext(r, args, "WebSetBucketPolicy")
objectAPI := web.ObjectAPI()
reply.UIVersion = browser.UIVersion
reply.UIVersion = "browser.UIVersion"
if objectAPI == nil {
return toJSONError(ctx, errServerNotInitialized)
@ -1983,7 +1982,7 @@ func (web *webAPIHandlers) PresignedGet(r *http.Request, args *PresignedGetArgs,
return toJSONError(ctx, errPresignedNotAllowed)
}
reply.UIVersion = browser.UIVersion
reply.UIVersion = "browser.UIVersion"
reply.URL = presignedGet(args.HostName, args.BucketName, args.ObjectName, args.Expiry, creds, region)
return nil
}
@ -2040,7 +2039,7 @@ func (web *webAPIHandlers) GetDiscoveryDoc(r *http.Request, args *WebGenericArgs
reply.DiscoveryDoc = globalOpenIDConfig.DiscoveryDoc
reply.ClientID = globalOpenIDConfig.ClientID
}
reply.UIVersion = browser.UIVersion
reply.UIVersion = "browser.UIVersion"
return nil
}
@ -2099,7 +2098,7 @@ func (web *webAPIHandlers) LoginSTS(r *http.Request, args *LoginSTSArgs, reply *
}
reply.Token = a.Result.Credentials.SessionToken
reply.UIVersion = browser.UIVersion
reply.UIVersion = "browser.UIVersion"
return nil
}

9
cmd/web-router.go
View file

@ -17,15 +17,16 @@
package cmd
import (
"errors"
"fmt"
"net/http"
"os"
assetfs "github.com/elazarl/go-bindata-assetfs"
"github.com/gorilla/handlers"
"github.com/gorilla/mux"
jsonrpc "github.com/gorilla/rpc/v2"
"github.com/gorilla/rpc/v2/json2"
"github.com/minio/minio/browser"
)
// webAPI container for Web API.
@ -48,9 +49,9 @@ const assetPrefix = "production"
func assetFS() *assetfs.AssetFS {
return &assetfs.AssetFS{
Asset: browser.Asset,
AssetDir: browser.AssetDir,
AssetInfo: browser.AssetInfo,
Asset: func(path string) ([]byte, error) { return nil, errors.New("no ui") },
AssetDir: func(path string) ([]string, error) { return nil, errors.New("no ui") },
AssetInfo: func(path string) (os.FileInfo, error) { return nil, errors.New("no ui") },
Prefix: assetPrefix,
}
}

2
go.mod
View file

@ -11,7 +11,6 @@ require (
github.com/Shopify/sarama v1.24.1
github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect
github.com/alecthomas/participle v0.2.1
github.com/aws/aws-sdk-go v1.20.21
github.com/bcicen/jstream v0.0.0-20190220045926-16c1f8af81c2
github.com/beevik/ntp v0.2.0
github.com/cespare/xxhash/v2 v2.1.1
@ -88,7 +87,6 @@ require (
github.com/rs/cors v1.7.0
github.com/secure-io/sio-go v0.3.0
github.com/shirou/gopsutil v2.20.3-0.20200314133625-53cec6b37e6a+incompatible
github.com/sirupsen/logrus v1.5.0
github.com/smartystreets/assertions v0.0.0-20190401211740-f487f9de1cd3 // indirect
github.com/spf13/viper v1.7.0
github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94

4
go.sum
View file

@ -52,8 +52,6 @@ github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQh
github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
github.com/awalterschulze/gographviz v0.0.0-20181013152038-b2885df04310 h1:t+qxRrRtwNiUYA+Xh2jSXhoG2grnMCMKX4Fg6lx9X1U=
github.com/awalterschulze/gographviz v0.0.0-20181013152038-b2885df04310/go.mod h1:GEV5wmg4YquNw7v1kkyoX9etIk8yVmXj+AkDHuuETHs=
github.com/aws/aws-sdk-go v1.20.21 h1:22vHWL9rur+SRTYPHAXlxJMFIA9OSYsYDIAHFDhQ7Z0=
github.com/aws/aws-sdk-go v1.20.21/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/bcicen/jstream v0.0.0-20190220045926-16c1f8af81c2 h1:M+TYzBcNIRyzPRg66ndEqUMd7oWDmhvdQmaPC6EZNwM=
github.com/bcicen/jstream v0.0.0-20190220045926-16c1f8af81c2/go.mod h1:RDu/qcrnpEdJC/p8tx34+YBFqqX71lB7dOX9QE+ZC4M=
github.com/beevik/ntp v0.2.0 h1:sGsd+kAXzT0bfVfzJfce04g+dSRfrs+tbQW8lweuYgw=
@ -284,8 +282,6 @@ github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf/go.mod h
github.com/jcmturner/gofork v0.0.0-20180107083740-2aebee971930/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o=
github.com/jcmturner/gofork v0.0.0-20190328161633-dc7c13fece03 h1:FUwcHNlEqkqLjLBdCp5PRlCFijNjvcYANOZXzCfXwCM=
github.com/jcmturner/gofork v0.0.0-20190328161633-dc7c13fece03/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o=
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo=
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=

10
main.go
View file

@ -25,14 +25,14 @@
package main // import "github.com/minio/minio"
import (
"os"
minio "github.com/minio/minio/cmd"
"fmt"
"github.com/minio/minio/misc"
// Import gateway
_ "github.com/minio/minio/cmd/gateway"
// _ "github.com/minio/minio/cmd/gateway"
)
func main() {
minio.Main(os.Args)
fmt.Println(misc.Build)
fmt.Println(misc.Version)
}

17
mint/.gitignore vendored
View file

@ -1,17 +0,0 @@
*.test
*.jar
src/*
temp
__pycache__/
log/*
minio.test
bin/*
node_modules
# exception to the rule
!log/.gitkeep
!bin/.gitkeep
*.class
*~
run/core/minio-dotnet/bin/*
run/core/minio-dotnet/obj/*
run/core/minio-dotnet/out/*

119
mint/README.md
View file

@ -1,119 +0,0 @@
# Mint [![Slack](https://slack.minio.io/slack?type=svg)](https://slack.minio.io) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/mint.svg?maxAge=604800)](https://hub.docker.com/r/minio/mint/)
Mint is a testing framework for Minio object server, available as a docker image. It runs correctness, benchmarking and stress tests. Following are the SDKs/tools used in correctness tests.
- awscli
- aws-sdk-go
- aws-sdk-php
- aws-sdk-ruby
- aws-sdk-java
- mc
- minio-go
- minio-java
- minio-js
- minio-py
- minio-dotnet
- s3cmd
## Running Mint
Mint is run by `docker run` command which requires Docker to be installed. For Docker installation follow the steps [here](https://docs.docker.com/engine/installation/linux/docker-ce/ubuntu/).
To run Mint with Minio Play server as test target,
```sh
$ docker run -e SERVER_ENDPOINT=play.minio.io:9000 -e ACCESS_KEY=Q3AM3UQ867SPQQA43P2F \
-e SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG -e ENABLE_HTTPS=1 minio/mint
```
After the tests are run, output is stored in `/mint/log` directory inside the container. To get these logs, use `docker cp` command. For example
```sh
docker cp <container-id>:/mint/log /tmp/logs
```
### Mint environment variables
Below environment variables are required to be passed to the docker container. Supported environment variables:
| Environment variable | Description | Example |
|:--- |:--- |:--- |
| `SERVER_ENDPOINT` | Endpoint of Minio server in the format `HOST:PORT`; for virtual style `IP:PORT` | `play.minio.io:9000` |
| `ACCESS_KEY` | Access key of access `SERVER_ENDPOINT` | `Q3AM3UQ867SPQQA43P2F` |
| `SECRET_KEY` | Secret Key of access `SERVER_ENDPOINT` | `zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG` |
| `ENABLE_HTTPS` | (Optional) Set `1` to indicate to use HTTPS to access `SERVER_ENDPOINT`. Defaults to `0` (HTTP) | `1` |
| `MINT_MODE` | (Optional) Set mode indicating what category of tests to be run by values `core`, `full`. Defaults to `core` | `full` |
| `DOMAIN` | (Optional) Value of MINIO_DOMAIN environment variable used in Minio server | `myminio.com` |
| `ENABLE_VIRTUAL_STYLE` | (Optional) Set `1` to indicate virtual style access . Defaults to `0` (Path style) | `1` |
### Test virtual style access against Minio server
To test Minio server virtual style access with Mint, follow these steps:
- Set a domain in your Minio server using environment variable MINIO_DOMAIN. For example `export MINIO_DOMAIN=myminio.com`.
- Start Minio server.
- Execute Mint against Minio server (with `MINIO_DOMAIN` set to `myminio.com`) using this command
```sh
$ docker run -e "SERVER_ENDPOINT=192.168.86.133:9000" -e "DOMAIN=minio.com" \
-e "ACCESS_KEY=minio" -e "SECRET_KEY=minio123" -e "ENABLE_HTTPS=0" \
-e "ENABLE_VIRTUAL_STYLE=1" minio/mint
```
### Mint log format
All test logs are stored in `/mint/log/log.json` as multiple JSON document. Below is the JSON format for every entry in the log file.
| JSON field | Type | Description | Example |
|:--- |:--- |:--- |:--- |
| `name` | _string_ | Testing tool/SDK name | `"aws-sdk-php"` |
| `function` | _string_ | Test function name | `"getBucketLocation ( array $params = [] )"` |
| `args` | _object_ | (Optional) Key/Value map of arguments passed to test function | `{"Bucket":"aws-sdk-php-bucket-20341"}` |
| `duration` | _int_ | Time taken in milliseconds to run the test | `384` |
| `status` | _string_ | one of `PASS`, `FAIL` or `NA` | `"PASS"` |
| `alert` | _string_ | (Optional) Alert message indicating test failure | `"I/O error on create file"` |
| `message` | _string_ | (Optional) Any log message | `"validating checksum of downloaded object"` |
| `error` | _string_ | Detailed error message including stack trace on status `FAIL` | `"Error executing \"CompleteMultipartUpload\" on ...` |
## For Developers
### Running Mint development code
After making changes to Mint source code a local docker image can be built/run by
```sh
$ docker build -t minio/mint . -f Dockerfile.mint
$ docker run -e SERVER_ENDPOINT=play.minio.io:9000 -e ACCESS_KEY=Q3AM3UQ867SPQQA43P2F \
-e SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG \
-e ENABLE_HTTPS=1 -e MINT_MODE=full minio/mint:latest
```
### Adding tests with new tool/SDK
Below are the steps need to be followed
- Create new app directory under [build](https://github.com/minio/mint/tree/master/build) and [run/core](https://github.com/minio/mint/tree/master/run/core) directories.
- Create `install.sh` which does installation of required tool/SDK under app directory.
- Any build and install time dependencies should be added to [install-packages.list](https://github.com/minio/mint/blob/master/install-packages.list).
- Build time dependencies should be added to [remove-packages.list](https://github.com/minio/mint/blob/master/remove-packages.list) for removal to have clean Mint docker image.
- Add `run.sh` in app directory under `run/core` which execute actual tests.
#### Test data
Tests may use pre-created data set to perform various object operations on Minio server. Below data files are available under `/mint/data` directory.
| File name | Size |
|:--- |:--- |
| datafile-0-b | 0B |
| datafile-1-b | 1B |
| datafile-1-kB |1KiB |
| datafile-10-kB |10KiB |
| datafile-33-kB |33KiB |
| datafile-100-kB |100KiB |
| datafile-1-MB |1MiB |
| datafile-1.03-MB |1.03MiB |
| datafile-5-MB |5MiB |
| datafile-6-MB |6MiB |
| datafile-10-MB |10MiB |
| datafile-11-MB |11MiB |
| datafile-65-MB |65MiB |
| datafile-129-MB |129MiB |

19
mint/build/aws-sdk-go/install.sh
View file

@ -1,19 +0,0 @@
#!/bin/bash -e
#
# Mint (C) 2017 Minio, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
test_run_dir="$MINT_RUN_CORE_DIR/aws-sdk-go"
GO111MODULE=on go build -o "$test_run_dir/aws-sdk-go" "$test_run_dir/quick-tests.go"

58
mint/build/aws-sdk-java/build.xml
View file

@ -1,58 +0,0 @@
<project xmlns:ivy="antlib:org.apache.ivy.ant" name="aws-sdk-java-tests" default="run">
<property name="ivy.install.version" value="2.5.0" />
<condition property="ivy.home" value="${env.IVY_HOME}">
<isset property="env.IVY_HOME" />
</condition>
<property name="ivy.home" value="${user.home}/.ant" />
<property name="ivy.jar.dir" value="${ivy.home}/lib" />
<property name="ivy.jar.file" value="${ivy.jar.dir}/ivy.jar" />
<target name="download-ivy" unless="offline">
<mkdir dir="${ivy.jar.dir}"/>
<get src="https://repo1.maven.org/maven2/org/apache/ivy/ivy/${ivy.install.version}/ivy-${ivy.install.version}.jar"
dest="${ivy.jar.file}" usetimestamp="true"/>
</target>
<target name="init-ivy" depends="download-ivy">
<path id="ivy.lib.path">
<fileset dir="${ivy.jar.dir}" includes="*.jar"/>
</path>
<taskdef resource="org/apache/ivy/ant/antlib.xml"
uri="antlib:org.apache.ivy.ant" classpathref="ivy.lib.path"/>
</target>
<target name="resolve" description="--> retrieve dependencies with ivy">
<ivy:retrieve />
</target>
<target name="clean">
<delete dir="build"/>
</target>
<path id="aws-s3-sdk-deps">
<fileset dir="lib">
<include name="*.jar"/>
</fileset>
</path>
<target name="compile">
<mkdir dir="build/classes"/>
<javac srcdir="src" destdir="build/classes">
<classpath refid="aws-s3-sdk-deps" />
</javac>
</target>
<target name="jar">
<mkdir dir="build/jar"/>
<jar destfile="build/jar/FunctionalTests.jar" basedir="build/classes">
<archives>
<zips>
<fileset dir="lib/" includes="*.jar"/>
</zips>
</archives>
<manifest>
<attribute name="Main-Class" value="io.minio.awssdk.tests.FunctionalTests"/>
</manifest>
</jar>
</target>
</project>

30
mint/build/aws-sdk-java/install.sh
View file

@ -1,30 +0,0 @@
#!/bin/bash -e
#
# Mint (C) 2017 Minio, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
test_run_dir="$MINT_RUN_CORE_DIR/aws-sdk-java"
cd "$(dirname "$(realpath "$0")")"
ant init-ivy && \
ant resolve && \
ant compile && \
ant jar
cp build/jar/FunctionalTests.jar "$test_run_dir/"
rm -rf lib/ build/

6
mint/build/aws-sdk-java/ivy.xml
View file

@ -1,6 +0,0 @@
<ivy-module version="2.0">
<info organisation="org.apache" module="aws-sdk-java-tests"/>
<dependencies>
<dependency org="com.amazonaws" name="aws-java-sdk-s3" rev="1.11.706"/>
</dependencies>
</ivy-module>

632
mint/build/aws-sdk-java/src/FunctionalTests.java
View file

@ -1,632 +0,0 @@
/*
* Mint, (C) 2018 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
*
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.minio.awssdk.tests;
import java.io.*;
import java.security.NoSuchAlgorithmException;
import java.security.SecureRandom;
import java.security.*;
import java.util.*;
import java.nio.file.*;
import java.math.BigInteger;
import javax.crypto.KeyGenerator;
import javax.crypto.SecretKey;
import javax.crypto.spec.SecretKeySpec;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.AmazonClientException;
import com.amazonaws.AmazonServiceException;
import com.amazonaws.auth.profile.ProfileCredentialsProvider;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.model.CreateBucketRequest;
import com.amazonaws.services.s3.model.ObjectListing;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.amazonaws.services.s3.model.SSECustomerKey;
// Main Testing class
public class FunctionalTests {
private static final String PASS = "PASS";
private static final String FAILED = "FAIL";
private static final String IGNORED = "NA";
private static String accessKey;
private static String secretKey;
private static String region;
private static String endpoint;
private static boolean enableHTTPS;
private static final Random random = new Random(new SecureRandom().nextLong());
private static String bucketName = getRandomName();
private static boolean mintEnv = false;
private static String file1Kb;
private static String file1Mb;
private static String file6Mb;
private static SSECustomerKey sseKey1;
private static SSECustomerKey sseKey2;
private static SSECustomerKey sseKey3;
private static AmazonS3 s3Client;
private static S3TestUtils s3TestUtils;
public static String getRandomName() {
return "aws-java-sdk-test-" + new BigInteger(32, random).toString(32);
}
/**
* Prints a success log entry in JSON format.
*/
public static void mintSuccessLog(String function, String args, long startTime) {
if (mintEnv) {
System.out.println(
new MintLogger(function, args, System.currentTimeMillis() - startTime, PASS, null, null, null));
}
}
/**
* Prints a failure log entry in JSON format.
*/
public static void mintFailedLog(String function, String args, long startTime, String message, String error) {
if (mintEnv) {
System.out.println(new MintLogger(function, args, System.currentTimeMillis() - startTime, FAILED, null,
message, error));
}
}
/**
* Prints a ignore log entry in JSON format.
*/
public static void mintIgnoredLog(String function, String args, long startTime) {
if (mintEnv) {
System.out.println(
new MintLogger(function, args, System.currentTimeMillis() - startTime, IGNORED, null, null, null));
}
}
public static void initTests() throws IOException {
// Create encryption key.
byte[] rawKey1 = "32byteslongsecretkeymustgenerate".getBytes();
SecretKey secretKey1 = new SecretKeySpec(rawKey1, 0, rawKey1.length, "AES");
sseKey1 = new SSECustomerKey(secretKey1);
// Create new encryption key for target so it is saved using sse-c
byte[] rawKey2 = "xxbytescopysecretkeymustprovided".getBytes();
SecretKey secretKey2 = new SecretKeySpec(rawKey2, 0, rawKey2.length, "AES");
sseKey2 = new SSECustomerKey(secretKey2);
// Create new encryption key for target so it is saved using sse-c
byte[] rawKey3 = "32byteslongsecretkeymustgenerat1".getBytes();
SecretKey secretKey3 = new SecretKeySpec(rawKey3, 0, rawKey3.length, "AES");
sseKey3 = new SSECustomerKey(secretKey3);
// Create bucket
s3Client.createBucket(new CreateBucketRequest(bucketName));
}
public static void teardown() throws IOException {
// Remove all objects under the test bucket & the bucket itself
// TODO: use multi delete API instead
ObjectListing objectListing = s3Client.listObjects(bucketName);
while (true) {
for (Iterator<?> iterator = objectListing.getObjectSummaries().iterator(); iterator.hasNext();) {
S3ObjectSummary summary = (S3ObjectSummary) iterator.next();
s3Client.deleteObject(bucketName, summary.getKey());
}
// more objectListing to retrieve?
if (objectListing.isTruncated()) {
objectListing = s3Client.listNextBatchOfObjects(objectListing);
} else {
break;
}
}
;
s3Client.deleteBucket(bucketName);
}
// Test regular object upload using encryption
public static void uploadObjectEncryption_test1() throws Exception {
if (!mintEnv) {
System.out.println(
"Test: uploadObject(String bucketName, String objectName, String f, SSECustomerKey sseKey)");
}
if (!enableHTTPS) {
return;
}
long startTime = System.currentTimeMillis();
String file1KbMD5 = Utils.getFileMD5(file1Kb);
String objectName = "testobject";
try {
s3TestUtils.uploadObject(bucketName, objectName, file1Kb, sseKey1);
s3TestUtils.downloadObject(bucketName, objectName, sseKey1, file1KbMD5);
mintSuccessLog("uploadObject(String bucketName, String objectName, String f, SSECustomerKey sseKey)",
"bucketName: " + bucketName + ", objectName: " + objectName + ", String: " + file1Kb
+ ", SSECustomerKey: " + sseKey1,
startTime);
} catch (Exception e) {
mintFailedLog("uploadObject(String bucketName, String objectName, String f, SSECustomerKey sseKey)",
"bucketName: " + bucketName + ", objectName: " + objectName + ", String: " + file1Kb
+ ", SSECustomerKey: " + sseKey1,
startTime, null, e.toString() + " >>> " + Arrays.toString(e.getStackTrace()));
throw e;
}
}
// Test downloading an object with a wrong encryption key
public static void downloadObjectEncryption_test1() throws Exception {
if (!mintEnv) {
System.out.println("Test: downloadObject(String bucketName, String objectName, SSECustomerKey sseKey)");
}
if (!enableHTTPS) {
return;
}
long startTime = System.currentTimeMillis();
String file1KbMD5 = Utils.getFileMD5(file1Kb);
String objectName = "testobject";
try {
s3TestUtils.uploadObject(bucketName, "testobject", file1Kb, sseKey1);
s3TestUtils.downloadObject(bucketName, objectName, sseKey2);
Exception ex = new Exception("downloadObject did not throw an S3 Access denied exception");
mintFailedLog("downloadObject(String bucketName, String objectName, SSECustomerKey sseKey)",
"bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey2,
startTime, null, ex.toString() + " >>> " + Arrays.toString(ex.getStackTrace()));
throw ex;
} catch (Exception e) {
if (!e.getMessage().contains("Access Denied")) {
Exception ex = new Exception(
"downloadObject did not throw S3 Access denied Exception but it did throw: " + e.getMessage());
mintFailedLog("downloadObject(String bucketName, String objectName, SSECustomerKey sseKey)",
"bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey2,
startTime, null, ex.toString() + " >>> " + Arrays.toString(ex.getStackTrace()));
throw ex;
}
mintSuccessLog("downloadObject(String bucketName, String objectName, SSECustomerKey sseKey)",
"bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey2,
startTime);
}
}
// Test copying object with a new different encryption key
public static void copyObjectEncryption_test1() throws Exception {
if (!mintEnv) {
System.out.println("Test: copyObject(String bucketName, String objectName, SSECustomerKey sseKey, "
+ "String destBucketName, String dstObjectName, SSECustomerKey sseKey2, boolean replaceDirective)");
}
if (!enableHTTPS) {
return;
}
long startTime = System.currentTimeMillis();
String file1KbMD5 = Utils.getFileMD5(file1Kb);
String objectName = "testobject";
String dstObjectName = "dir/newobject";
try {
s3TestUtils.uploadObject(bucketName, objectName, file1Kb, sseKey1);
s3TestUtils.copyObject(bucketName, objectName, sseKey1, bucketName, dstObjectName, sseKey2, false);
s3TestUtils.downloadObject(bucketName, dstObjectName, sseKey2, file1KbMD5);
} catch (Exception e) {
mintFailedLog("copyObject(String bucketName, String objectName, SSECustomerKey sseKey, "
+ "String destBucketName, String dstObjectName, SSECustomerKey sseKey2, boolean replaceDirective)",
"bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1
+ "DstbucketName: " + bucketName + ", DstObjectName: " + dstObjectName
+ ", SSECustomerKey: " + sseKey2 + ", replaceDirective: " + false,
startTime, null, e.toString() + " >>> " + Arrays.toString(e.getStackTrace()));
throw e;
}
mintSuccessLog("copyObject(String bucketName, String objectName, SSECustomerKey sseKey, "
+ "String destBucketName, String dstObjectName, SSECustomerKey sseKey2, boolean replaceDirective)",
"bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1
+ "DstbucketName: " + bucketName + ", DstObjectName: " + dstObjectName + ", SSECustomerKey: "
+ sseKey2 + ", replaceDirective: " + false,
startTime);
}
// Test copying object with wrong source encryption key
public static void copyObjectEncryption_test2() throws Exception {
if (!mintEnv) {
System.out.println("Test: copyObject(String bucketName, String objectName, SSECustomerKey sseKey, "
+ "String destBucketName, String dstObjectName, SSECustomerKey sseKey2, boolean replaceDirective)");
}
if (!enableHTTPS) {
return;
}
String objectName = "testobject";
String dstObjectName = "dir/newobject";
long startTime = System.currentTimeMillis();
try {
s3TestUtils.copyObject(bucketName, objectName, sseKey3, bucketName, dstObjectName, sseKey2, false);
Exception ex = new Exception("copyObject did not throw an S3 Access denied exception");
mintFailedLog("copyObject(String bucketName, String objectName, SSECustomerKey sseKey, "
+ "String destBucketName, String dstObjectName, SSECustomerKey sseKey2, boolean replaceDirective)",
"bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey3
+ "DstbucketName: " + bucketName + ", DstObjectName: " + dstObjectName
+ ", SSECustomerKey: " + sseKey2 + ", replaceDirective: " + false,
startTime, null, ex.toString() + " >>> " + Arrays.toString(ex.getStackTrace()));
throw ex;
} catch (Exception e) {
if (!e.getMessage().contains("Access Denied")) {
Exception ex = new Exception(
"copyObject did not throw S3 Access denied Exception but it did throw: " + e.getMessage());
mintFailedLog("copyObject(String bucketName, String objectName, SSECustomerKey sseKey, "
+ "String destBucketName, String dstObjectName, SSECustomerKey sseKey2, boolean replaceDirective)",
"bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey3
+ "DstbucketName: " + bucketName + ", DstObjectName: " + dstObjectName
+ ", SSECustomerKey: " + sseKey2 + ", replaceDirective: " + false,
startTime, null, ex.toString() + " >>> " + Arrays.toString(ex.getStackTrace()));
throw ex;
}
mintSuccessLog("copyObject(String bucketName, String objectName, SSECustomerKey sseKey, "
+ "String destBucketName, String dstObjectName, SSECustomerKey sseKey2, boolean replaceDirective)",
"bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey3
+ "DstbucketName: " + bucketName + ", DstObjectName: " + dstObjectName
+ ", SSECustomerKey: " + sseKey2 + ", replaceDirective: " + false,
startTime);
}
}
// Test copying multipart object
public static void copyObjectEncryption_test3() throws Exception {
if (!mintEnv) {
System.out.println("Test: copyObject(String bucketName, String objectName, SSECustomerKey sseKey, "
+ "String destBucketName, String dstObjectName, SSECustomerKey sseKey2, boolean replaceDirective)");
}
if (!enableHTTPS) {
return;
}
long startTime = System.currentTimeMillis();
String file6MbMD5 = Utils.getFileMD5(file6Mb);
String objectName = "testobject";
String dstObjectName = "dir/newobject";
try {
s3TestUtils.uploadMultipartObject(bucketName, objectName, file6Mb, sseKey1);
s3TestUtils.copyObject(bucketName, objectName, sseKey1, bucketName, dstObjectName, sseKey2, false);
s3TestUtils.downloadObject(bucketName, dstObjectName, sseKey2, file6MbMD5);
} catch (Exception e) {
mintFailedLog("copyObject(String bucketName, String objectName, SSECustomerKey sseKey, "
+ "String destBucketName, String dstObjectName, SSECustomerKey sseKey2, boolean replaceDirective)",
"bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1
+ "DstbucketName: " + bucketName + ", DstObjectName: " + dstObjectName
+ ", SSECustomerKey: " + sseKey2 + ", replaceDirective: " + false,
startTime, null, e.toString() + " >>> " + Arrays.toString(e.getStackTrace()));
throw e;
}
mintSuccessLog("copyObject(String bucketName, String objectName, SSECustomerKey sseKey, "
+ "String destBucketName, String dstObjectName, SSECustomerKey sseKey2, boolean replaceDirective)",
"bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1
+ "DstbucketName: " + bucketName + ", DstObjectName: " + dstObjectName + ", SSECustomerKey: "
+ sseKey2 + ", replaceDirective: " + false,
startTime);
}
// Test downloading encrypted object with Get Range, 0 -> 1024
public static void downloadGetRangeEncryption_test1() throws Exception {
if (!mintEnv) {
System.out.println("Test: downloadObjectGetRange(String bucketName, String objectName, "
+ "SSECustomerKey sseKey, String expectedMD5, int start, int length)");
}
if (!enableHTTPS) {
return;
}
long startTime = System.currentTimeMillis();
String objectName = "testobject";
String range1MD5 = Utils.getFileMD5(file1Kb);
int start = 0;
int length = 1024;
try {
s3TestUtils.uploadObject(bucketName, objectName, file1Kb, sseKey1);
s3TestUtils.downloadObject(bucketName, objectName, sseKey1, range1MD5, start, length);
} catch (Exception e) {
mintFailedLog(
"downloadObjectGetRange(String bucketName, String objectName, "
+ "SSECustomerKey sseKey, String expectedMD5, int start, int length)",
"bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1
+ ", expectedMD5: " + range1MD5 + ", start: " + start + ", length: " + length,
startTime, null, e.toString() + " >>> " + Arrays.toString(e.getStackTrace()));
throw e;
}
mintSuccessLog(
"downloadObjectGetRange(String bucketName, String objectName, "
+ "SSECustomerKey sseKey, String expectedMD5, int start, int length)",
"bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1
+ ", expectedMD5: " + range1MD5 + ", start: " + start + ", length: " + length,
startTime);
}
// Test downloading encrypted object with Get Range, 0 -> 1
public static void downloadGetRangeEncryption_test2() throws Exception {
if (!mintEnv) {
System.out.println("Test: downloadObjectGetRange(String bucketName, String objectName, "
+ "SSECustomerKey sseKey, String expectedMD5, int start, int length)");
}
if (!enableHTTPS) {
return;
}
long startTime = System.currentTimeMillis();
String objectName = "testobject";
int start = 0;
int length = 1;
String range1MD5 = Utils.getFileMD5(file1Kb, start, length);
try {
s3TestUtils.uploadObject(bucketName, objectName, file1Kb, sseKey1);
s3TestUtils.downloadObject(bucketName, objectName, sseKey1, range1MD5, start, length);
} catch (Exception e) {
mintFailedLog(
"downloadObjectGetRange(String bucketName, String objectName, "
+ "SSECustomerKey sseKey, String expectedMD5, int start, int length)",
"bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1
+ ", expectedMD5: " + range1MD5 + ", start: " + start + ", length: " + length,
startTime, null, e.toString() + " >>> " + Arrays.toString(e.getStackTrace()));
throw e;
}
mintSuccessLog(
"downloadObjectGetRange(String bucketName, String objectName, "
+ "SSECustomerKey sseKey, String expectedMD5, int start, int length)",
"bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1
+ ", expectedMD5: " + range1MD5 + ", start: " + start + ", length: " + length,
startTime);
}
// Test downloading encrypted object with Get Range, 0 -> 1024-1
public static void downloadGetRangeEncryption_test3() throws Exception {
if (!mintEnv) {
System.out.println("Test: downloadObjectGetRange(String bucketName, String objectName, "
+ "SSECustomerKey sseKey, String expectedMD5, int start, int length)");
}
if (!enableHTTPS) {
return;
}
long startTime = System.currentTimeMillis();
String objectName = "testobject";
int start = 0;
int length = 1023;
String range1MD5 = Utils.getFileMD5(file1Kb, start, length);
try {
s3TestUtils.uploadObject(bucketName, objectName, file1Kb, sseKey1);
s3TestUtils.downloadObject(bucketName, objectName, sseKey1, range1MD5, start, length);
} catch (Exception e) {
mintFailedLog(
"downloadObjectGetRange(String bucketName, String objectName, "
+ "SSECustomerKey sseKey, String expectedMD5, int start, int length)",
"bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1
+ ", expectedMD5: " + range1MD5 + ", start: " + start + ", length: " + length,
startTime, null, e.toString() + " >>> " + Arrays.toString(e.getStackTrace()));
throw e;
}
mintSuccessLog(
"downloadObjectGetRange(String bucketName, String objectName, "
+ "SSECustomerKey sseKey, String expectedMD5, int start, int length)",
"bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1
+ ", expectedMD5: " + range1MD5 + ", start: " + start + ", length: " + length,
startTime);
}
// Test downloading encrypted object with Get Range, 1 -> 1024-1
public static void downloadGetRangeEncryption_test4() throws Exception {
if (!mintEnv) {
System.out.println("Test: downloadObjectGetRange(String bucketName, String objectName, "
+ "SSECustomerKey sseKey, String expectedMD5, int start, int length)");
}
if (!enableHTTPS) {
return;
}
long startTime = System.currentTimeMillis();
String objectName = "testobject";
int start = 1;
int length = 1023;
String range1MD5 = Utils.getFileMD5(file1Kb, start, length);
try {
s3TestUtils.uploadObject(bucketName, objectName, file1Kb, sseKey1);
s3TestUtils.downloadObject(bucketName, objectName, sseKey1, range1MD5, start, length);
} catch (Exception e) {
mintFailedLog(
"downloadObjectGetRange(String bucketName, String objectName, "
+ "SSECustomerKey sseKey, String expectedMD5, int start, int length)",
"bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1
+ ", expectedMD5: " + range1MD5 + ", start: " + start + ", length: " + length,
startTime, null, e.toString() + " >>> " + Arrays.toString(e.getStackTrace()));
throw e;
}
mintSuccessLog(
"downloadObjectGetRange(String bucketName, String objectName, "
+ "SSECustomerKey sseKey, String expectedMD5, int start, int length)",
"bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1
+ ", expectedMD5: " + range1MD5 + ", start: " + start + ", length: " + length,
startTime);
}
// Test downloading encrypted object with Get Range, 64*1024 -> 64*1024
public static void downloadGetRangeEncryption_test5() throws Exception {
if (!mintEnv) {
System.out.println("Test: downloadObjectGetRange(String bucketName, String objectName, "
+ "SSECustomerKey sseKey, String expectedMD5, int start, int length)");
}
if (!enableHTTPS) {
return;
}
long startTime = System.currentTimeMillis();
String objectName = "testobject";
int start = 64 * 1024;
int length = 64 * 1024;
String range1MD5 = Utils.getFileMD5(file1Mb, start, length);
try {
s3TestUtils.uploadObject(bucketName, objectName, file1Mb, sseKey1);
s3TestUtils.downloadObject(bucketName, objectName, sseKey1, range1MD5, start, length);
} catch (Exception e) {
mintFailedLog(
"downloadObjectGetRange(String bucketName, String objectName, "
+ "SSECustomerKey sseKey, String expectedMD5, int start, int length)",
"bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1
+ ", expectedMD5: " + range1MD5 + ", start: " + start + ", length: " + length,
startTime, null, e.toString() + " >>> " + Arrays.toString(e.getStackTrace()));
throw e;
}
mintSuccessLog(
"downloadObjectGetRange(String bucketName, String objectName, "
+ "SSECustomerKey sseKey, String expectedMD5, int start, int length)",
"bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1
+ ", expectedMD5: " + range1MD5 + ", start: " + start + ", length: " + length,
startTime);
}
// Test downloading encrypted object with Get Range, 64*1024 ->
// 1024*1024-64*1024
public static void downloadGetRangeEncryption_test6() throws Exception {
if (!mintEnv) {
System.out.println("Test: downloadObjectGetRange(String bucketName, String objectName, "
+ "SSECustomerKey sseKey, String expectedMD5, int start, int length)");
}
if (!enableHTTPS) {
return;
}
long startTime = System.currentTimeMillis();
String objectName = "testobject";
int start = 64 * 1024;
int length = 1024 * 1024 - 64 * 1024;
String range1MD5 = Utils.getFileMD5(file1Mb, start, length);
try {
s3TestUtils.uploadObject(bucketName, objectName, file1Mb, sseKey1);
s3TestUtils.downloadObject(bucketName, objectName, sseKey1, range1MD5, start, length);
} catch (Exception e) {
mintFailedLog(
"downloadObjectGetRange(String bucketName, String objectName, "
+ "SSECustomerKey sseKey, String expectedMD5, int start, int length)",
"bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1
+ ", expectedMD5: " + range1MD5 + ", start: " + start + ", length: " + length,
startTime, null, e.toString() + " >>> " + Arrays.toString(e.getStackTrace()));
throw e;
}
mintSuccessLog(
"downloadObjectGetRange(String bucketName, String objectName, "
+ "SSECustomerKey sseKey, String expectedMD5, int start, int length)",
"bucketName: " + bucketName + ", objectName: " + objectName + ", SSECustomerKey: " + sseKey1
+ ", expectedMD5: " + range1MD5 + ", start: " + start + ", length: " + length,
startTime);
}
// Run tests
public static void runTests() throws Exception {
uploadObjectEncryption_test1();
downloadObjectEncryption_test1();
copyObjectEncryption_test1();
copyObjectEncryption_test2();
copyObjectEncryption_test3();
downloadGetRangeEncryption_test1();
downloadGetRangeEncryption_test2();
downloadGetRangeEncryption_test3();
downloadGetRangeEncryption_test4();
downloadGetRangeEncryption_test5();
downloadGetRangeEncryption_test6();
}
public static void main(String[] args) throws Exception, IOException, NoSuchAlgorithmException {
endpoint = System.getenv("SERVER_ENDPOINT");
accessKey = System.getenv("ACCESS_KEY");
secretKey = System.getenv("SECRET_KEY");
enableHTTPS = System.getenv("ENABLE_HTTPS").equals("1");
region = "us-east-1";
if (enableHTTPS) {
endpoint = "https://" + endpoint;
} else {
endpoint = "http://" + endpoint;
}
String dataDir = System.getenv("MINT_DATA_DIR");
if (dataDir != null && !dataDir.equals("")) {
mintEnv = true;
file1Kb = Paths.get(dataDir, "datafile-1-kB").toString();
file1Mb = Paths.get(dataDir, "datafile-1-MB").toString();
file6Mb = Paths.get(dataDir, "datafile-6-MB").toString();
}
String mintMode = null;
if (mintEnv) {
mintMode = System.getenv("MINT_MODE");
}
AWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey);
AmazonS3ClientBuilder.EndpointConfiguration endpointConfiguration = new AmazonS3ClientBuilder.EndpointConfiguration(
endpoint, region);
AmazonS3ClientBuilder clientBuilder = AmazonS3ClientBuilder.standard();
clientBuilder.setCredentials(new AWSStaticCredentialsProvider(credentials));
clientBuilder.setEndpointConfiguration(endpointConfiguration);
clientBuilder.setPathStyleAccessEnabled(true);
s3Client = clientBuilder.build();
s3TestUtils = new S3TestUtils(s3Client);
try {
initTests();
FunctionalTests.runTests();
} catch (Exception e) {
e.printStackTrace();
System.exit(-1);
} finally {
teardown();
}
}
}

60
mint/build/aws-sdk-java/src/LimitedInputStream.java
View file

@ -1,60 +0,0 @@
/*
* Mint, (C) 2018 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
*
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.minio.awssdk.tests;
import java.io.*;
// LimitedInputStream wraps a regular InputStream, calling
// read() will skip some bytes as configured and will also
// return only data with configured length
class LimitedInputStream extends InputStream {
private int skip;
private int length;
private InputStream is;
LimitedInputStream(InputStream is, int skip, int length) {
this.is = is;
this.skip = skip;
this.length = length;
}
@Override
public int read() throws IOException {
int r;
while (skip > 0) {
r = is.read();
if (r < 0) {
throw new IOException("stream ended before being able to skip all bytes");
}
skip--;
}
if (length == 0) {
return -1;
}
r = is.read();
if (r < 0) {
throw new IOException("stream ended before being able to read all bytes");
}
length--;
return r;
}
}

151
mint/build/aws-sdk-java/src/MintLogger.java
View file

@ -1,151 +0,0 @@
/*
* Minio Java SDK for Amazon S3 Compatible Cloud Storage,
* (C) 2015, 2016, 2017, 2018 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.minio.awssdk.tests;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonInclude.Include;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY)
public class MintLogger {
@JsonProperty("name")
private String name;
@JsonProperty("function")
private String function;
@JsonProperty("args")
private String args;
@JsonProperty("duration")
private long duration;
@JsonProperty("status")
private String status;
@JsonProperty("alert")
private String alert;
@JsonProperty("message")
private String message;
@JsonProperty("error")
private String error;
/**
* Constructor.
**/
public MintLogger(String function,
String args,
long duration,
String status,
String alert,
String message,
String error) {
this.name = "aws-sdk-java";
this.function = function;
this.duration = duration;
this.args = args;
this.status = status;
this.alert = alert;
this.message = message;
this.error = error;
}
/**
* Return JSON Log Entry.
**/
@JsonIgnore
public String toString() {
try {
return new ObjectMapper().setSerializationInclusion(Include.NON_NULL).writeValueAsString(this);
} catch (JsonProcessingException e) {
e.printStackTrace();
}
return "";
}
/**
* Return Alert.
**/
@JsonIgnore
public String alert() {
return alert;
}
/**
* Return Error.
**/
@JsonIgnore
public String error() {
return error;
}
/**
* Return Message.
**/
@JsonIgnore
public String message() {
return message;
}
/**
* Return args.
**/
@JsonIgnore
public String args() {
return args;
}
/**
* Return status.
**/
@JsonIgnore
public String status() {
return status;
}
/**
* Return name.
**/
@JsonIgnore
public String name() {
return name;
}
/**
* Return function.
**/
@JsonIgnore
public String function() {
return function;
}
/**
* Return duration.
**/
@JsonIgnore
public long duration() {
return duration;
}
}

187
mint/build/aws-sdk-java/src/S3TestUtils.java
View file

@ -1,187 +0,0 @@
/*
* Mint, (C) 2018 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
*
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.minio.awssdk.tests;
import java.io.*;
import java.util.*;
import java.nio.channels.Channels;
import com.amazonaws.services.s3.model.GetObjectMetadataRequest;
import com.amazonaws.services.s3.model.GetObjectRequest;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.PutObjectRequest;
import com.amazonaws.services.s3.model.CopyObjectRequest;
import com.amazonaws.services.s3.model.S3Object;
import com.amazonaws.services.s3.model.S3ObjectInputStream;
import com.amazonaws.services.s3.model.SSECustomerKey;
import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest;
import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest;
import com.amazonaws.services.s3.model.InitiateMultipartUploadResult;
import com.amazonaws.services.s3.model.PartETag;
import com.amazonaws.services.s3.model.UploadPartRequest;
import com.amazonaws.services.s3.model.MetadataDirective;
import com.amazonaws.services.s3.AmazonS3;
class S3TestUtils {
private AmazonS3 s3Client;
S3TestUtils(AmazonS3 s3Client) {
this.s3Client = s3Client;
}
void uploadMultipartObject(String bucketName, String keyName,
String filePath, SSECustomerKey sseKey) throws IOException {
File file = new File(filePath);
List<PartETag> partETags = new ArrayList<PartETag>();
// Step 1: Initialize.
InitiateMultipartUploadRequest initRequest = new
InitiateMultipartUploadRequest(bucketName, keyName);
if (sseKey != null) {
initRequest.setSSECustomerKey(sseKey);
}
InitiateMultipartUploadResult initResponse =
s3Client.initiateMultipartUpload(initRequest);
long contentLength = file.length();
long partSize = 5242880; // Set part size to 5 MB.
// Step 2: Upload parts.
long filePosition = 0;
for (int i = 1; filePosition < contentLength; i++) {
// Last part can be less than 5 MB. Adjust part size.
partSize = Math.min(partSize, (contentLength - filePosition));
// Create request to upload a part.
UploadPartRequest uploadRequest = new UploadPartRequest()
.withBucketName(bucketName).withKey(keyName)
.withUploadId(initResponse.getUploadId()).withPartNumber(i)
.withFileOffset(filePosition)
.withFile(file)
.withPartSize(partSize);
if (sseKey != null) {
uploadRequest.withSSECustomerKey(sseKey);
}
// Upload part and add response to our list.
partETags.add(s3Client.uploadPart(uploadRequest).getPartETag());
filePosition += partSize;
}
// Step 3: Complete.
CompleteMultipartUploadRequest compRequest = new
CompleteMultipartUploadRequest(
bucketName,
keyName,
initResponse.getUploadId(),
partETags);
s3Client.completeMultipartUpload(compRequest);
}
void uploadObject(String bucketName, String keyName,
String filePath, SSECustomerKey sseKey) throws IOException {
File f = new File(filePath);
PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, keyName, f);
if (sseKey != null) {
putObjectRequest.withSSECustomerKey(sseKey);
}
s3Client.putObject(putObjectRequest);
}
void downloadObject(String bucketName, String keyName, SSECustomerKey sseKey)
throws Exception, IOException {
downloadObject(bucketName, keyName, sseKey, "", -1, -1);
}
void downloadObject(String bucketName, String keyName, SSECustomerKey sseKey,
String expectedMD5)
throws Exception, IOException {
downloadObject(bucketName, keyName, sseKey, expectedMD5, -1, -1);
}
void downloadObject(String bucketName, String keyName, SSECustomerKey sseKey,
String expectedMD5, int start, int length) throws Exception, IOException {
GetObjectRequest getObjectRequest = new GetObjectRequest(bucketName, keyName)
.withSSECustomerKey(sseKey);
if (start >= 0 && length >= 0) {
getObjectRequest.setRange(start, start+length-1);
}
S3Object s3Object = s3Client.getObject(getObjectRequest);
int size = 0;
int c;
S3ObjectInputStream input = s3Object.getObjectContent();
ByteArrayOutputStream output = new ByteArrayOutputStream();
String data = "";
while ((c = input.read()) != -1) {
output.write((byte) c);
size++;
}
if (length >= 0 && size != length) {
throw new Exception("downloaded object has unexpected size, expected: " + length + ", received: " + size);
}
String calculatedMD5 = Utils.getBufferMD5(output.toByteArray());
if (!expectedMD5.equals("") && !calculatedMD5.equals(expectedMD5)) {
throw new Exception("downloaded object has unexpected md5sum, expected: " + expectedMD5 + ", found: " + calculatedMD5);
}
}
void copyObject(String bucketName, String keyName, SSECustomerKey sseKey,
String targetBucketName, String targetKeyName, SSECustomerKey newSseKey,
boolean replace) {
CopyObjectRequest copyRequest = new CopyObjectRequest(bucketName, keyName, targetBucketName, targetKeyName);
if (sseKey != null) {
copyRequest.withSourceSSECustomerKey(sseKey);
}
if (newSseKey != null) {
copyRequest.withDestinationSSECustomerKey(newSseKey);
}
if (replace) {
copyRequest.withMetadataDirective(MetadataDirective.COPY);
}
s3Client.copyObject(copyRequest);
}
long retrieveObjectMetadata(String bucketName, String keyName, SSECustomerKey sseKey) {
GetObjectMetadataRequest getMetadataRequest = new GetObjectMetadataRequest(bucketName, keyName)
.withSSECustomerKey(sseKey);
ObjectMetadata objectMetadata = s3Client.getObjectMetadata(getMetadataRequest);
return objectMetadata.getContentLength();
}
}

76
mint/build/aws-sdk-java/src/Utils.java
View file

@ -1,76 +0,0 @@
/*
* Mint, (C) 2018 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
*
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.minio.awssdk.tests;
import java.io.*;
import java.nio.channels.*;
import java.security.*;
class Utils {
public static byte[] createChecksum(InputStream is, int skip, int length) throws Exception {
int numRead;
byte[] buffer = new byte[1024];
MessageDigest complete = MessageDigest.getInstance("MD5");
if (skip > -1 && length > -1) {
is = new LimitedInputStream(is, skip, length);
}
do {
numRead = is.read(buffer);
if (numRead > 0) {
complete.update(buffer, 0, numRead);
}
} while (numRead != -1);
return complete.digest();
}
public static String getInputStreamMD5(InputStream is) throws Exception {
return getInputStreamMD5(is, -1, -1);
}
public static String getInputStreamMD5(InputStream is, int start, int length) throws Exception {
byte[] b = createChecksum(is, start, length);
String result = "";
for (int i=0; i < b.length; i++) {
result += Integer.toString( ( b[i] & 0xff ) + 0x100, 16).substring( 1 );
}
return result;
}
public static String getFileMD5(String filePath) throws Exception {
return getFileMD5(filePath, -1, -1);
}
public static String getFileMD5(String filePath, int start, int length) throws Exception {
File f = new File(filePath);
InputStream is = new FileInputStream(f);
return getInputStreamMD5(is, start, length);
}
public static String getBufferMD5(byte[] data) throws Exception {
ByteArrayInputStream bis = new ByteArrayInputStream(data);
return getInputStreamMD5(bis);
}
}

20
mint/build/aws-sdk-php/install.sh
View file

@ -1,20 +0,0 @@
#!/bin/bash -e
#
# Mint (C) 2017 Minio, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
test_run_dir="$MINT_RUN_CORE_DIR/aws-sdk-php"
$WGET --output-document=- https://getcomposer.org/installer | php -- --install-dir="$test_run_dir"
php "$test_run_dir/composer.phar" --working-dir="$test_run_dir" install

18
mint/build/aws-sdk-ruby/install.sh
View file

@ -1,18 +0,0 @@
#!/bin/bash -e
#
# Mint (C) 20172-2020 Minio, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
gem install --no-rdoc --no-ri aws-sdk-resources aws-sdk multipart_body

18
mint/build/awscli/install.sh
View file

@ -1,18 +0,0 @@
#!/bin/bash -e
#
# Mint (C) 2017-2020 Minio, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
pip3 install awscli --upgrade

19
mint/build/healthcheck/install.sh
View file

@ -1,19 +0,0 @@
#!/bin/bash -e
#
# Mint (C) 2019 Minio, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
test_run_dir="$MINT_RUN_CORE_DIR/healthcheck"
GO111MODULE=on go build -o "$test_run_dir/healthcheck" "$test_run_dir/healthcheck.go"

31
mint/build/mc/install.sh
View file

@ -1,31 +0,0 @@
#!/bin/bash -e
#
# Mint (C) 2017 Minio, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
MC_VERSION=$(curl --retry 10 -Ls -o /dev/null -w "%{url_effective}" https://github.com/minio/mc/releases/latest | sed "s/https:\/\/github.com\/minio\/mc\/releases\/tag\///")
if [ -z "$MC_VERSION" ]; then
echo "unable to get mc version from github"
exit 1
fi
test_run_dir="$MINT_RUN_CORE_DIR/mc"
$WGET --output-document="${test_run_dir}/mc" "https://dl.minio.io/client/mc/release/linux-amd64/mc.${MC_VERSION}"
chmod a+x "${test_run_dir}/mc"
git clone --quiet https://github.com/minio/mc.git "$test_run_dir/mc.git"
(cd "$test_run_dir/mc.git"; git checkout --quiet "tags/${MC_VERSION}")
cp -a "${test_run_dir}/mc.git/functional-tests.sh" "$test_run_dir/"
rm -fr "$test_run_dir/mc.git"

42
mint/build/minio-dotnet/install.sh
View file

@ -1,42 +0,0 @@
#!/bin/bash
#
# Mint (C) 2017-2020 Minio, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
set -e
MINIO_DOTNET_SDK_PATH="$MINT_RUN_CORE_DIR/minio-dotnet"
MINIO_DOTNET_SDK_VERSION=$(curl --retry 10 -Ls -o /dev/null -w "%{url_effective}" https://github.com/minio/minio-dotnet/releases/latest | sed "s/https:\/\/github.com\/minio\/minio-dotnet\/releases\/tag\///")
if [ -z "$MINIO_DOTNET_SDK_VERSION" ]; then
echo "unable to get minio-dotnet version from github"
exit 1
fi
out_dir="$MINIO_DOTNET_SDK_PATH/out"
if [ -z "$out_dir" ]; then
mkdir "$out_dir"
fi
temp_dir="$MINIO_DOTNET_SDK_PATH/temp"
git clone --quiet https://github.com/minio/minio-dotnet.git "${temp_dir}/minio-dotnet.git/"
(cd "${temp_dir}/minio-dotnet.git"; git checkout --quiet "tags/${MINIO_DOTNET_SDK_VERSION}")
cp -a "${temp_dir}/minio-dotnet.git/Minio.Functional.Tests/"* "${MINIO_DOTNET_SDK_PATH}/"
rm -fr "${temp_dir}"
cd "$MINIO_DOTNET_SDK_PATH"
dotnet restore /p:Configuration=Mint
dotnet publish --runtime ubuntu.18.04-x64 --output out /p:Configuration=Mint

27
mint/build/minio-go/install.sh
View file

@ -1,27 +0,0 @@
#!/bin/bash -e
#
# Mint (C) 2017 Minio, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
MINIO_GO_VERSION=$(curl --retry 10 -Ls -o /dev/null -w "%{url_effective}" https://github.com/minio/minio-go/releases/latest | sed "s/https:\/\/github.com\/minio\/minio-go\/releases\/tag\///")
if [ -z "$MINIO_GO_VERSION" ]; then
echo "unable to get minio-go version from github"
exit 1
fi
test_run_dir="$MINT_RUN_CORE_DIR/minio-go"
(git clone https://github.com/minio/minio-go && cd minio-go && git checkout --quiet "tags/$MINIO_GO_VERSION")
GO111MODULE=on CGO_ENABLED=0 go build -o "$test_run_dir/minio-go" "minio-go/functional_tests.go"
rm -rf minio-go

30
mint/build/minio-java/install.sh
View file

@ -1,30 +0,0 @@
#!/bin/bash -e
#
# Mint (C) 2017 Minio, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
MINIO_JAVA_VERSION=$(curl --retry 10 -s "https://repo1.maven.org/maven2/io/minio/minio/maven-metadata.xml" | sed -n "/<latest>/{s/<.[^>]*>//g;p;q}" | sed "s/ *//g")
if [ -z "$MINIO_JAVA_VERSION" ]; then
echo "unable to get latest minio-java version from maven"
exit 1
fi
test_run_dir="$MINT_RUN_CORE_DIR/minio-java"
git clone --quiet https://github.com/minio/minio-java.git "$test_run_dir/minio-java.git"
(cd "$test_run_dir/minio-java.git"; git checkout --quiet "tags/${MINIO_JAVA_VERSION}")
$WGET --output-document="$test_run_dir/minio-${MINIO_JAVA_VERSION}-all.jar" "https://repo1.maven.org/maven2/io/minio/minio/${MINIO_JAVA_VERSION}/minio-${MINIO_JAVA_VERSION}-all.jar"
javac -cp "$test_run_dir/minio-${MINIO_JAVA_VERSION}-all.jar" "${test_run_dir}/minio-java.git/functional"/*.java
cp -a "${test_run_dir}/minio-java.git/functional"/*.class "$test_run_dir/"
rm -fr "$test_run_dir/minio-java.git"

28
mint/build/minio-js/install.sh
View file

@ -1,28 +0,0 @@
#!/bin/bash -e
#
# Mint (C) 2017 Minio, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
MINIO_JS_VERSION=$(curl --retry 10 -Ls -o /dev/null -w "%{url_effective}" https://github.com/minio/minio-js/releases/latest | sed "s/https:\/\/github.com\/minio\/minio-js\/releases\/tag\///")
if [ -z "$MINIO_JS_VERSION" ]; then
echo "unable to get minio-js version from github"
exit 1
fi
test_run_dir="$MINT_RUN_CORE_DIR/minio-js"
mkdir "${test_run_dir}/test"
$WGET --output-document="${test_run_dir}/test/functional-tests.js" "https://raw.githubusercontent.com/minio/minio-js/${MINIO_JS_VERSION}/src/test/functional/functional-tests.js"
npm --prefix "$test_run_dir" install --save "minio@$MINIO_JS_VERSION"
npm --prefix "$test_run_dir" install

27
mint/build/minio-py/install.sh
View file

@ -1,27 +0,0 @@
#!/bin/bash -e
#
# Mint (C) 2017-2020 Minio, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
MINIO_PY_VERSION=$(curl --retry 10 -Ls -o /dev/null -w "%{url_effective}" https://github.com/minio/minio-py/releases/latest | sed "s/https:\/\/github.com\/minio\/minio-py\/releases\/tag\///")
if [ -z "$MINIO_PY_VERSION" ]; then
echo "unable to get minio-py version from github"
exit 1
fi
test_run_dir="$MINT_RUN_CORE_DIR/minio-py"
pip3 install --user faker
pip3 install minio=="$MINIO_PY_VERSION"
$WGET --output-document="$test_run_dir/tests.py" "https://raw.githubusercontent.com/minio/minio-py/${MINIO_PY_VERSION}/tests/functional/tests.py"

19
mint/build/s3cmd/install.sh
View file

@ -1,19 +0,0 @@
#!/bin/bash -e
#
# Mint (C) 2017 Minio, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Always install the latest.
python -m pip install s3cmd

18
mint/build/s3select/install.sh
View file

@ -1,18 +0,0 @@
#!/bin/bash -e
#
# Mint (C) 2020 Minio, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
python -m pip install minio

19
mint/build/security/install.sh
View file

@ -1,19 +0,0 @@
#!/bin/bash -e
#
# Mint (C) 2018 Minio, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
test_run_dir="$MINT_RUN_CORE_DIR/security"
GO111MODULE=on go build -o "$test_run_dir/tls-tests" "$test_run_dir/tls-tests.go"

44
mint/create-data-files.sh
View file

@ -1,44 +0,0 @@
#!/bin/bash -e
#
# Mint (C) 2017 Minio, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
MINT_DATA_DIR="$MINT_ROOT_DIR/data"
declare -A data_file_map
data_file_map["datafile-0-b"]="0"
data_file_map["datafile-1-b"]="1"
data_file_map["datafile-1-kB"]="1K"
data_file_map["datafile-10-kB"]="10K"
data_file_map["datafile-33-kB"]="33K"
data_file_map["datafile-100-kB"]="100K"
data_file_map["datafile-1.03-MB"]="1056K"
data_file_map["datafile-1-MB"]="1M"
data_file_map["datafile-5-MB"]="5M"
data_file_map["datafile-5243880-b"]="5243880"
data_file_map["datafile-6-MB"]="6M"
data_file_map["datafile-10-MB"]="10M"
data_file_map["datafile-11-MB"]="11M"
data_file_map["datafile-65-MB"]="65M"
data_file_map["datafile-129-MB"]="129M"
mkdir -p "$MINT_DATA_DIR"
for filename in "${!data_file_map[@]}"; do
echo "creating $MINT_DATA_DIR/$filename"
if ! shred -n 1 -s "${data_file_map[$filename]}" - 1>"$MINT_DATA_DIR/$filename" 2>/dev/null; then
echo "unable to create data file $MINT_DATA_DIR/$filename"
exit 1
fi
done

24
mint/entrypoint.sh
View file

@ -1,24 +0,0 @@
#!/bin/bash
#
# Mint (C) 2017 Minio, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
./mint.sh "$@" &
# Get the pid to be used for kill command if required
main_pid="$!"
trap 'echo -e "\nAborting Mint..."; kill $main_pid' SIGINT SIGTERM
# use -n here to catch mint.sh exit code, notify to ci
wait -n

18
mint/install-packages.list
View file

@ -1,18 +0,0 @@
git
python3-pip
nodejs
openjdk-8-jdk
openjdk-8-jre
dirmngr
apt-transport-https
dotnet-sdk-2.1
ca-certificates-mono
nuget
libunwind8
ruby
ruby-dev
ruby-bundler
php
php-curl
php-xml
ant

190
mint/mint.sh
View file

@ -1,190 +0,0 @@
#!/bin/bash
#
# Mint (C) 2017, 2018 Minio, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
CONTAINER_ID=$(grep -o -e '[0-f]\{12,\}' /proc/1/cpuset | awk '{print substr($1, 1, 12)}')
MINT_DATA_DIR=${MINT_DATA_DIR:-/mint/data}
MINT_MODE=${MINT_MODE:-core}
SERVER_REGION=${SERVER_REGION:-us-east-1}
ENABLE_HTTPS=${ENABLE_HTTPS:-0}
ENABLE_VIRTUAL_STYLE=${ENABLE_VIRTUAL_STYLE:-0}
GO111MODULE=on
if [ -z "$SERVER_ENDPOINT" ]; then
SERVER_ENDPOINT="play.minio.io:9000"
ACCESS_KEY="Q3AM3UQ867SPQQA43P2F"
SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG"
ENABLE_HTTPS=1
fi
if [ "$ENABLE_VIRTUAL_STYLE" -eq 1 ]; then
SERVER_IP="${SERVER_ENDPOINT%%:*}"
SERVER_PORT="${SERVER_ENDPOINT/*:/}"
# Check if SERVER_IP is actually IPv4 address
octets=("${SERVER_IP//./ }")
if [ "${#octets[@]}" -ne 4 ]; then
echo "$SERVER_IP must be an IP address"
exit 1
fi
for octet in "${octets[@]}"; do
if [ "$octet" -lt 0 ] 2>/dev/null || [ "$octet" -gt 255 ] 2>/dev/null; then
echo "$SERVER_IP must be an IP address"
exit 1
fi
done
fi
ROOT_DIR="$PWD"
TESTS_DIR="$ROOT_DIR/run/core"
BASE_LOG_DIR="$ROOT_DIR/log"
LOG_FILE="log.json"
ERROR_FILE="error.log"
mkdir -p "$BASE_LOG_DIR"
function humanize_time()
{
time="$1"
days=$(( time / 60 / 60 / 24 ))
hours=$(( time / 60 / 60 % 24 ))
minutes=$(( time / 60 % 60 ))
seconds=$(( time % 60 ))
(( days > 0 )) && echo -n "$days days "
(( hours > 0 )) && echo -n "$hours hours "
(( minutes > 0 )) && echo -n "$minutes minutes "
(( days > 0 || hours > 0 || minutes > 0 )) && echo -n "and "
echo "$seconds seconds"
}
function run_test()
{
if [ ! -d "$1" ]; then
return 1
fi
start=$(date +%s)
mkdir -p "$BASE_LOG_DIR/$sdk_name"
(cd "$sdk_dir" && ./run.sh "$BASE_LOG_DIR/$LOG_FILE" "$BASE_LOG_DIR/$sdk_name/$ERROR_FILE")
rv=$?
end=$(date +%s)
duration=$(humanize_time $(( end - start )))
if [ "$rv" -eq 0 ]; then
echo "done in $duration"
else
echo "FAILED in $duration"
entry=$(tail -n 1 "$BASE_LOG_DIR/$LOG_FILE")
status=$(jq -e -r .status <<<"$entry")
jq_rv=$?
if [ "$jq_rv" -ne 0 ]; then
echo "$entry"
fi
## Show error.log when status is empty or not "FAIL".
## This may happen when test run failed without providing logs.
if [ "$jq_rv" -ne 0 ] || [ -z "$status" ] || ([ "$status" != "FAIL" ] && [ "$status" != "fail" ]); then
cat "$BASE_LOG_DIR/$sdk_name/$ERROR_FILE"
else
jq . <<<"$entry"
fi
fi
return $rv
}
function trust_s3_endpoint_tls_cert()
{
# Download the public certificate from the server
openssl s_client -showcerts -connect "$SERVER_ENDPOINT" </dev/null 2>/dev/null | \
openssl x509 -outform PEM -out /usr/local/share/ca-certificates/s3_server_cert.crt || \
exit 1
# Load the certificate in the system
update-ca-certificates --fresh >/dev/null
# Ask different SDKs/tools to load system certificates
export REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt
export NODE_EXTRA_CA_CERTS=/etc/ssl/certs/ca-certificates.crt
export SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt
}
function main()
{
export MINT_DATA_DIR
export MINT_MODE
export SERVER_ENDPOINT
export SERVER_IP
export SERVER_PORT
export ACCESS_KEY
export SECRET_KEY
export ENABLE_HTTPS
export SERVER_REGION
export ENABLE_VIRTUAL_STYLE
export GO111MODULE
echo "Running with"
echo "SERVER_ENDPOINT: $SERVER_ENDPOINT"
echo "ACCESS_KEY: $ACCESS_KEY"
echo "SECRET_KEY: ***REDACTED***"
echo "ENABLE_HTTPS: $ENABLE_HTTPS"
echo "SERVER_REGION: $SERVER_REGION"
echo "MINT_DATA_DIR: $MINT_DATA_DIR"
echo "MINT_MODE: $MINT_MODE"
echo "ENABLE_VIRTUAL_STYLE: $ENABLE_VIRTUAL_STYLE"
echo
echo "To get logs, run 'docker cp ${CONTAINER_ID}:/mint/log /tmp/mint-logs'"
echo
[ "$ENABLE_HTTPS" == "1" ] && trust_s3_endpoint_tls_cert
declare -a run_list
sdks=( "$@" )
if [ "$#" -eq 0 ]; then
sdks=( $(ls "$TESTS_DIR") )
fi
for sdk in "${sdks[@]}"; do
run_list=( "${run_list[@]}" "$TESTS_DIR/$sdk" )
done
count="${#run_list[@]}"
i=0
for sdk_dir in "${run_list[@]}"; do
sdk_name=$(basename "$sdk_dir")
(( i++ ))
if [ ! -d "$sdk_dir" ]; then
echo "Test $sdk_name not found. Exiting Mint."
exit 1
fi
echo -n "($i/$count) Running $sdk_name tests ... "
if ! run_test "$sdk_dir"; then
(( i-- ))
fi
done
## Report when all tests in run_list are run
if [ $i -eq "$count" ]; then
echo -e "\nAll tests ran successfully"
else
echo -e "\nExecuted $i out of $count tests successfully."
exit 1
fi
}
main "$@"

28
mint/postinstall.sh
View file

@ -1,28 +0,0 @@
#!/bin/bash -e
#
# Mint (C) 2017 Minio, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
export APT="apt --quiet --yes"
# remove all packages listed in remove-packages.list
xargs --arg-file="${MINT_ROOT_DIR}/remove-packages.list" apt --quiet --yes purge
${APT} autoremove
# remove unwanted files
rm -fr "$GOROOT" "$GOPATH/src" /var/lib/apt/lists/*
# flush to disk
sync

56
mint/preinstall.sh
View file

@ -1,56 +0,0 @@
#!/bin/bash -e
#
# Mint (C) 2017-2020 Minio, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
export APT="apt --quiet --yes"
export WGET="wget --quiet --no-check-certificate"
# install nodejs source list
if ! $WGET --output-document=- https://deb.nodesource.com/setup_13.x | bash -; then
echo "unable to set nodejs repository"
exit 1
fi
$APT install apt-transport-https
if ! $WGET --output-document=packages-microsoft-prod.deb https://packages.microsoft.com/config/ubuntu/18.04/packages-microsoft-prod.deb | bash -; then
echo "unable to download dotnet packages"
exit 1
fi
dpkg -i packages-microsoft-prod.deb
rm -f packages-microsoft-prod.deb
$APT update
$APT install gnupg ca-certificates
# download and install golang
GO_VERSION="1.13.10"
GO_INSTALL_PATH="/usr/local"
download_url="https://storage.googleapis.com/golang/go${GO_VERSION}.linux-amd64.tar.gz"
if ! $WGET --output-document=- "$download_url" | tar -C "${GO_INSTALL_PATH}" -zxf -; then
echo "unable to install go$GO_VERSION"
exit 1
fi
xargs --arg-file="${MINT_ROOT_DIR}/install-packages.list" apt --quiet --yes install
nuget update -self
# set python 3.6 as default
update-alternatives --install /usr/bin/python python /usr/bin/python3.6 1
sync

32
mint/release.sh
View file

@ -1,32 +0,0 @@
#!/bin/bash -e
#
# Minio Cloud Storage, (C) 2017 Minio, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
export MINT_ROOT_DIR=${MINT_ROOT_DIR:-/mint}
export MINT_RUN_CORE_DIR="$MINT_ROOT_DIR/run/core"
export MINT_RUN_SECURITY_DIR="$MINT_ROOT_DIR/run/security"
export WGET="wget --quiet --no-check-certificate"
"${MINT_ROOT_DIR}"/create-data-files.sh
"${MINT_ROOT_DIR}"/preinstall.sh
# install mint app packages
for pkg in "$MINT_ROOT_DIR/build"/*/install.sh; do
echo "Running $pkg"
$pkg
done
"${MINT_ROOT_DIR}"/postinstall.sh

9
mint/remove-packages.list
View file

@ -1,9 +0,0 @@
wget
git
python3-pip
ruby-dev
ruby-bundler
openjdk-8-jdk
ant
dotnet
nuget

1066
mint/run/core/aws-sdk-go/quick-tests.go
View file

File diff suppressed because it is too large Load diff

28
mint/run/core/aws-sdk-go/run.sh
View file

@ -1,28 +0,0 @@
#!/bin/bash
#
# Mint (C) 2017 Minio, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# handle command line arguments
if [ $# -ne 2 ]; then
echo "usage: run.sh <OUTPUT-LOG-FILE> <ERROR-LOG-FILE>"
exit 1
fi
output_log_file="$1"
error_log_file="$2"
# run tests
/mint/run/core/aws-sdk-go/aws-sdk-go 1>>"$output_log_file" 2>"$error_log_file"

30
mint/run/core/aws-sdk-java/run.sh
View file

@ -1,30 +0,0 @@
#!/bin/bash
#
# Mint (C) 2018 Minio, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# handle command line arguments
if [ $# -ne 2 ]; then
echo "usage: run.sh <OUTPUT-LOG-FILE> <ERROR-LOG-FILE>"
exit 1
fi
output_log_file="$1"
error_log_file="$2"
# run tests
cd /mint/run/core/aws-sdk-java/ || exit 1
java -jar FunctionalTests.jar 1>>"$output_log_file" 2>"$error_log_file"

19
mint/run/core/aws-sdk-php/README.md
View file

@ -1,19 +0,0 @@
## `aws-sdk-php` tests
This directory serves as the location for Mint tests using `aws-sdk-php`. Top level `mint.sh` calls `run.sh` to execute tests.
## Adding new tests
New tests is added into `quick-tests.php` as new functions.
## Running tests manually
- Set environment variables `MINT_DATA_DIR`, `MINT_MODE`, `SERVER_ENDPOINT`, `ACCESS_KEY`, `SECRET_KEY`, `SERVER_REGION` and `ENABLE_HTTPS`
- Call `run.sh` with output log file and error log file. for example
```bash
export MINT_DATA_DIR=~/my-mint-dir
export MINT_MODE=core
export SERVER_ENDPOINT="play.minio.io:9000"
export ACCESS_KEY="Q3AM3UQ867SPQQA43P2F"
export SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG"
export ENABLE_HTTPS=1
export SERVER_REGION=us-east-1
./run.sh /tmp/output.log /tmp/error.log
```

6
mint/run/core/aws-sdk-php/composer.json
View file

@ -1,6 +0,0 @@
{
"require": {
"aws/aws-sdk-php": "^3.30",
"guzzlehttp/psr7": "^1.4"
}
}

1124
mint/run/core/aws-sdk-php/quick-tests.php
View file

File diff suppressed because it is too large Load diff

28
mint/run/core/aws-sdk-php/run.sh
View file

@ -1,28 +0,0 @@
#!/bin/bash
#
# Mint, (C) 2017 Minio, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# handle command line arguments
if [ $# -ne 2 ]; then
echo "usage: run.sh <OUTPUT-LOG-FILE> <ERROR-LOG-FILE>"
exit 1
fi
output_log_file="$1"
error_log_file="$2"
# run tests
php ./quick-tests.php 1>>"$output_log_file" 2>"$error_log_file"

19
mint/run/core/aws-sdk-ruby/README.md
View file

@ -1,19 +0,0 @@
## `aws-sdk-ruby` tests
This directory serves as the location for Mint tests using `aws-sdk-ruby`. Top level `mint.sh` calls `run.sh` to execute tests.
## Adding new tests
New tests is added into `aws-stub-test.rb` as new functions.
## Running tests manually
- Set environment variables `MINT_DATA_DIR`, `MINT_MODE`, `SERVER_ENDPOINT`, `ACCESS_KEY`, `SECRET_KEY`, `SERVER_REGION` and `ENABLE_HTTPS`
- Call `run.sh` with output log file and error log file. for example
```bash
export MINT_DATA_DIR=~/my-mint-dir
export MINT_MODE=core
export SERVER_ENDPOINT="play.minio.io:9000"
export ACCESS_KEY="Q3AM3UQ867SPQQA43P2F"
export SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG"
export ENABLE_HTTPS=1
export SERVER_REGION=us-east-1
./run.sh /tmp/output.log /tmp/error.log
```

855
mint/run/core/aws-sdk-ruby/aws-stub-tests.rb
View file

@ -1,855 +0,0 @@
#!/usr/bin/env ruby
#
# Mint (C) 2017 Minio, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
require 'aws-sdk'
require 'securerandom'
require 'net/http'
require 'multipart_body'
class AwsSdkRubyTest
# Set variables necessary to create an s3 client instance.
# Get them from the environment variables
# Region information, eg. "us-east-1"
region = ENV['SERVER_REGION'] ||= 'SERVER_REGION is not set'
# Minio server, eg. "play.minio.io:9000"
endpoint = ENV['SERVER_ENDPOINT'] ||= 'SERVER_ENDPOINT is not set'
access_key_id = ENV['ACCESS_KEY'] ||= 'ACCESS_KEY is not set'
secret_access_key = ENV['SECRET_KEY'] ||= 'SECRET_KEY is not set'
enable_https = ENV['ENABLE_HTTPS']
endpoint = enable_https == '1' ? 'https://' + endpoint : 'http://' + endpoint
# Create s3 client instances, "s3Resource" and "s3Client"
@@s3 = Aws::S3::Resource.new(region: region,
endpoint: endpoint,
access_key_id: access_key_id,
secret_access_key: secret_access_key,
force_path_style: true)
def initialize_log_output(meth, alert = nil)
# Initialize and return log content in log_output hash table
# Collect args in args_arr
args_arr = method(meth).parameters.flatten.map(&:to_s)
.reject { |x| x == 'req' || x == 'opt' }
# Create and return log output content
{ name: 'aws-sdk-ruby',
function: "#{meth}(#{args_arr.join(',')})", # method name and arguments
args: args_arr, # array of arg names. This'll be replaced with a
# a arg/value pairs insdie the caller method
duration: 0, # test runtime duration in seconds
alert: alert,
message: nil,
error: nil }
end
def get_random_bucket_name()
bucket_name = "aws-sdk-ruby-bucket-"+SecureRandom.hex(6)
return bucket_name
end
def calculate_duration(t2, t1)
# Durations are in miliseconds, with precision of 2 decimal places
((t2 - t1) * 1000).round(2)
end
def print_log(log_output, start_time)
# Calculate duration in miliseconds
log_output[:duration] = calculate_duration(Time.now, start_time)
# Get rid of the log_output fields if nil
puts log_output.delete_if{|k, value| value == nil}.to_json
# Exit at the first failure
exit 1 if log_output[:status] == 'FAIL'
end
def cleanUp(buckets, log_output)
# Removes objects and bucket if bucket exists
bucket_name = ''
buckets.each do |b|
bucket_name = b
if bucketExistsWrapper(b, log_output)
removeObjectsWrapper(b, log_output)
removeBucketWrapper(b, log_output)
end
end
rescue => e
raise "Failed to clean-up bucket '#{bucket_name}', #{e}"
end
#
# API commands/methods
#
def makeBucket(bucket_name)
# Creates a bucket, "bucket_name"
# on S3 client , "s3".
# Returns bucket_name if already exists
@@s3.bucket(bucket_name).exists? ? @@s3.bucket(bucket_name) : @@s3.create_bucket(bucket: bucket_name)
rescue => e
raise e
end
def makeBucketWrapper(bucket_name, log_output)
makeBucket(bucket_name)
rescue => e
log_output[:function] = "makeBucket(bucket_name)"
log_output[:args] = {'bucket_name': bucket_name}
raise e
end
def removeBucket(bucket_name)
# Deletes/removes bucket, "bucket_name" on S3 client, "s3"
@@s3.bucket(bucket_name).delete
rescue => e
raise e
end
def removeBucketWrapper(bucket_name, log_output)
removeBucket(bucket_name)
rescue => e
log_output[:function] = "removeBucket(bucket_name)"
log_output[:args] = {'bucket_name': bucket_name}
raise e
end
def putObject(bucket_name, file)
# Creates "file" (full path) in bucket, "bucket_name",
# on S3 client, "s3"
file_name = File.basename(file)
@@s3.bucket(bucket_name).object(file_name).upload_file(file)
rescue => e
raise e
end
def putObjectWrapper(bucket_name, file, log_output)
putObject(bucket_name, file)
rescue => e
log_output[:function] = "putObject(bucket_name, file)"
log_output[:args] = {'bucket_name': bucket_name,
'file': file}
raise e
end
def getObject(bucket_name, file, destination)
# Gets/Downloads file, "file",
# from bucket, "bucket_name", of S3 client, "s3"
file_name = File.basename(file)
dest = File.join(destination, file_name)
@@s3.bucket(bucket_name).object(file_name).get(response_target: dest)
rescue => e
raise e
end
def getObjectWrapper(bucket_name, file, destination, log_output)
getObject(bucket_name, file, destination)
rescue => e
log_output[:function] = "getObject(bucket_name, file)"
log_output[:args] = {'bucket_name': bucket_name,
'file': file,
'destination': destination}
raise e
end
def copyObject(source_bucket_name, target_bucket_name, source_file_name, target_file_name = '')
# Copies file, "file_name", from source bucket,
# "source_bucket_name", to target bucket,
# "target_bucket_name", on S3 client, "s3"
target_file_name = source_file_name if target_file_name.empty?
source = @@s3.bucket(source_bucket_name)
target = @@s3.bucket(target_bucket_name)
source_obj = source.object(source_file_name)
target_obj = target.object(target_file_name)
source_obj.copy_to(target_obj)
rescue => e
raise e
end
def copyObjectWrapper(source_bucket_name, target_bucket_name, source_file_name, target_file_name = '', log_output)
copyObject(source_bucket_name, target_bucket_name, source_file_name, target_file_name)
rescue => e
log_output[:function] = 'copyObject(source_bucket_name, target_bucket_name, source_file_name, target_file_name = '')'
log_output[:args] = {'source_bucket_name': source_bucket_name,
'target_bucket_name': target_bucket_name,
'source_file_name': source_file_name,
'target_file_name': target_file_name}
raise e
end
def removeObject(bucket_name, file)
# Deletes file in bucket,
# "bucket_name", on S3 client, "s3".
# If file, "file_name" does not exist,
# it quietly returns without any error message
@@s3.bucket(bucket_name).object(file).delete
rescue => e
raise e
end
def removeObjectWrapper(bucket_name, file_name, log_output)
removeObject(bucket_name, file_name)
rescue => e
log_output[:function] = "removeObject(bucket_name, file_name)"
log_output[:args] = {'bucket_name': bucket_name,
'file_name': file_name}
raise e
end
def removeObjects(bucket_name)
# Deletes all files in bucket, "bucket_name"
# on S3 client, "s3"
file_name = ''
@@s3.bucket(bucket_name).objects.each do |obj|
file_name = obj.key
obj.delete
end
rescue => e
raise "File name: '#{file_name}', #{e}"
end
def removeObjectsWrapper(bucket_name, log_output)
removeObjects(bucket_name)
rescue => e
log_output[:function] = 'removeObjects(bucket_name)'
log_output[:args] = {'bucket_name': bucket_name}
raise e
end
def listBuckets
# Returns an array of bucket names on S3 client, "s3"
bucket_name_list = []
@@s3.buckets.each do |b|
bucket_name_list.push(b.name)
end
return bucket_name_list
rescue => e
raise e
end
def listBucketsWrapper(log_output)
listBuckets
rescue => e
log_output[:function] = 'listBuckets'
log_output[:args] = {}
raise e
end
def listObjects(bucket_name)
# Returns an array of object/file names
# in bucket, "bucket_name", on S3 client, "s3"
object_list = []
@@s3.bucket(bucket_name).objects.each do |obj|
object_list.push(obj.key)
end
return object_list
rescue => e
raise e
end
def listObjectsWrapper(bucket_name, log_output)
listObjects(bucket_name)
rescue => e
log_output[:function] = 'listObjects(bucket_name)'
log_output[:args] = {'bucket_name': bucket_name}
raise e
end
def statObject(bucket_name, file_name)
return @@s3.bucket(bucket_name).object(file_name).exists?
rescue => e
raise e
end
def statObjectWrapper(bucket_name, file_name, log_output)
statObject(bucket_name, file_name)
rescue => e
log_output[:function] = 'statObject(bucket_name, file_name)'
log_output[:args] = {'bucket_name': bucket_name,
'file_name': file_name}
raise e
end
def bucketExists?(bucket_name)
# Returns true if bucket, "bucket_name", exists,
# false otherwise
return @@s3.bucket(bucket_name).exists?
rescue => e
raise e
end
def bucketExistsWrapper(bucket_name, log_output)
bucketExists?(bucket_name)
rescue => e
log_output[:function] = 'bucketExists?(bucket_name)'
log_output[:args] = {'bucket_name': bucket_name}
raise e
end
def presignedGet(bucket_name, file_name)
# Returns download/get url
obj = @@s3.bucket(bucket_name).object(file_name)
return obj.presigned_url(:get, expires_in: 600)
rescue => e
raise e
end
def presignedGetWrapper(bucket_name, file_name, log_output)
presignedGet(bucket_name, file_name)
rescue => e
log_output[:function] = 'presignedGet(bucket_name, file_name)'
log_output[:args] = {'bucket_name': bucket_name,
'file_name': file_name}
raise e
end
def presignedPut(bucket_name, file_name)
# Returns put url
obj = @@s3.bucket(bucket_name).object(file_name)
return obj.presigned_url(:put, expires_in: 600)
rescue => e
raise e
end
def presignedPutWrapper(bucket_name, file_name, log_output)
presignedPut(bucket_name, file_name)
rescue => e
log_output[:function] = 'presignedPut(bucket_name, file_name)'
log_output[:args] = {'bucket_name': bucket_name,
'file_name': file_name}
raise e
end
def presignedPost(bucket_name, file_name, expires_in_sec, max_byte_size)
# Returns upload/post url
obj = @@s3.bucket(bucket_name).object(file_name)
return obj.presigned_post(expires: Time.now + expires_in_sec,
content_length_range: 1..max_byte_size)
rescue => e
raise e
end
def presignedPostWrapper(bucket_name, file_name, expires_in_sec, max_byte_size, log_output)
presignedPost(bucket_name, file_name, expires_in_sec, max_byte_size)
rescue => e
log_output[:function] = 'presignedPost(bucket_name, file_name, expires_in_sec, max_byte_size)'
log_output[:args] = {'bucket_name': bucket_name,
'file_name': file_name,
'expires_in_sec': expires_in_sec,
'max_byte_size': max_byte_size}
raise e
end
# To be addressed. S3 API 'get_bucket_policy' does not work!
# def getBucketPolicy(bucket_name)
# # Returns bucket policy
# return @@s3.bucket(bucket_name).get_bucket_policy
# rescue => e
# raise e
# end
#
# Test case methods
#
def listBucketsTest()
# Tests listBuckets api command by creating
# new buckets from bucket_name_list
# get random bucket names and create list
bucket_name1 = get_random_bucket_name()
bucket_name2 = get_random_bucket_name()
bucket_name_list = [bucket_name1, bucket_name2]
# Initialize hash table, 'log_output'
log_output = initialize_log_output('listBuckets')
# Prepare arg/value hash table and set it in log_output
arg_value_hash = {}
log_output[:args].each { |x| arg_value_hash[:"#{x}"] = eval x.to_s }
log_output[:args] = arg_value_hash
begin
start_time = Time.now
prev_total_buckets = listBucketsWrapper(log_output).length
new_buckets = bucket_name_list.length
bucket_name_list.each do |b|
makeBucketWrapper(b, log_output)
end
new_total_buckets = prev_total_buckets + new_buckets
if new_total_buckets >= prev_total_buckets + new_buckets
log_output[:status] = 'PASS'
else
log_output[:error] = 'Could not find expected number of buckets'
log_output[:status] = 'FAIL'
end
cleanUp(bucket_name_list, log_output)
rescue => log_output[:error]
log_output[:status] = 'FAIL'
end
print_log(log_output, start_time)
end
def makeBucketTest()
# Tests makeBucket api command.
# get random bucket name
bucket_name = get_random_bucket_name()
# Initialize hash table, 'log_output'
log_output = initialize_log_output('makeBucket')
# Prepare arg/value hash table and set it in log_output
arg_value_hash = {}
log_output[:args].each { |x| arg_value_hash[:"#{x}"] = eval x.to_s }
log_output[:args] = arg_value_hash
begin
start_time = Time.now
makeBucketWrapper(bucket_name, log_output)
if bucketExistsWrapper(bucket_name, log_output)
log_output[:status] = 'PASS'
else
log_output[:error] = 'Bucket expected to be created does not exist'
log_output[:status] = 'FAIL'
end
cleanUp([bucket_name], log_output)
rescue => log_output[:error]
log_output[:status] = 'FAIL'
end
print_log(log_output, start_time)
end
def bucketExistsNegativeTest()
# Tests bucketExists api command.
# get random bucket name
bucket_name = get_random_bucket_name()
# Initialize hash table, 'log_output'
log_output = initialize_log_output('bucketExists?')
# Prepare arg/value hash table and set it in log_output
arg_value_hash = {}
log_output[:args].each { |x| arg_value_hash[:"#{x}"] = eval x.to_s }
log_output[:args] = arg_value_hash
begin
start_time = Time.now
if !bucketExistsWrapper(bucket_name, log_output)
log_output[:status] = 'PASS'
else
log_output[:error] = "Failed to return 'false' for a non-existing bucket"
log_output[:status] = 'FAIL'
end
cleanUp([bucket_name], log_output)
rescue => log_output[:error]
log_output[:status] = 'FAIL'
end
print_log(log_output, start_time)
end
def removeBucketTest()
# Tests removeBucket api command.
# get a random bucket name
bucket_name = get_random_bucket_name()
# Initialize hash table, 'log_output'
log_output = initialize_log_output('removeBucket')
# Prepare arg/value hash table and set it in log_output
arg_value_hash = {}
log_output[:args].each { |x| arg_value_hash[:"#{x}"] = eval x.to_s }
log_output[:args] = arg_value_hash
begin
start_time = Time.now
makeBucketWrapper(bucket_name, log_output)
removeBucketWrapper(bucket_name, log_output)
if !bucketExistsWrapper(bucket_name, log_output)
log_output[:status] = 'PASS'
else
log_output[:error] = 'Bucket expected to be removed still exists'
log_output[:status] = 'FAIL'
end
cleanUp([bucket_name], log_output)
rescue => log_output[:error]
log_output[:status] = 'FAIL'
end
print_log(log_output, start_time)
end
def putObjectTest(file)
# Tests putObject api command by uploading a file
# get random bucket name
bucket_name = get_random_bucket_name()
# Initialize hash table, 'log_output'
log_output = initialize_log_output('putObject')
# Prepare arg/value hash table and set it in log_output
arg_value_hash = {}
log_output[:args].each { |x| arg_value_hash[:"#{x}"] = eval x.to_s }
log_output[:args] = arg_value_hash
begin
start_time = Time.now
makeBucketWrapper(bucket_name, log_output)
putObjectWrapper(bucket_name, file, log_output)
if statObjectWrapper(bucket_name, File.basename(file), log_output)
log_output[:status] = 'PASS'
else
log_output[:error] = "Status for the created object returned 'false'"
log_output[:status] = 'FAIL'
end
cleanUp([bucket_name], log_output)
rescue => log_output[:error]
log_output[:status] = 'FAIL'
end
print_log(log_output, start_time)
end
def removeObjectTest(file)
# Tests removeObject api command by uploading and removing a file
# get random bucket name
bucket_name = get_random_bucket_name()
# Initialize hash table, 'log_output'
log_output = initialize_log_output('removeObject')
# Prepare arg/value hash table and set it in log_output
arg_value_hash = {}
log_output[:args].each { |x| arg_value_hash[:"#{x}"] = eval x.to_s }
log_output[:args] = arg_value_hash
begin
start_time = Time.now
makeBucketWrapper(bucket_name, log_output)
putObjectWrapper(bucket_name, file, log_output)
removeObjectWrapper(bucket_name, File.basename(file), log_output)
if !statObjectWrapper(bucket_name, File.basename(file), log_output)
log_output[:status] = 'PASS'
else
log_output[:error] = "Status for the removed object returned 'true'"
log_output[:status] = 'FAIL'
end
cleanUp([bucket_name], log_output)
rescue => log_output[:error]
log_output[:status] = 'FAIL'
end
print_log(log_output, start_time)
end
def getObjectTest(file, destination)
# Tests getObject api command
# get random bucket name
bucket_name = get_random_bucket_name()
# Initialize hash table, 'log_output'
log_output = initialize_log_output('getObject')
# Prepare arg/value hash table and set it in log_output
arg_value_hash = {}
log_output[:args].each { |x| arg_value_hash[:"#{x}"] = eval x.to_s }
log_output[:args] = arg_value_hash
begin
start_time = Time.now
makeBucketWrapper(bucket_name, log_output)
putObjectWrapper(bucket_name, file, log_output)
getObjectWrapper(bucket_name, file, destination, log_output)
if system("ls -l #{destination} > /dev/null")
log_output[:status] = 'PASS'
else
log_output[:error] = "Downloaded object does not exist at #{destination}"
log_output[:status] = 'FAIL'
end
cleanUp([bucket_name], log_output)
rescue => log_output[:error]
log_output[:status] = 'FAIL'
end
print_log(log_output, start_time)
end
def listObjectsTest(file_list)
# Tests listObjects api command
# get random bucket name
bucket_name = get_random_bucket_name()
# Initialize hash table, 'log_output'
log_output = initialize_log_output('listObjects')
# Prepare arg/value hash table and set it in log_output
arg_value_hash = {}
log_output[:args].each { |x| arg_value_hash[:"#{x}"] = eval x.to_s }
log_output[:args] = arg_value_hash
begin
start_time = Time.now
makeBucketWrapper(bucket_name, log_output)
# Put all objects into the bucket
file_list.each do |f|
putObjectWrapper(bucket_name, f, log_output)
end
# Total number of files uploaded
expected_no = file_list.length
# Actual number is what api returns
actual_no = listObjectsWrapper(bucket_name, log_output).length
# Compare expected and actual values
if expected_no == actual_no
log_output[:status] = 'PASS'
else
log_output[:error] = 'Expected and actual number of listed files/objects do not match!'
log_output[:status] = 'FAIL'
end
cleanUp([bucket_name], log_output)
rescue => log_output[:error]
log_output[:status] = 'FAIL'
end
print_log(log_output, start_time)
end
def copyObjectTest(data_dir, source_file_name, target_file_name = '')
# Tests copyObject api command
# get random bucket names
source_bucket_name = get_random_bucket_name()
target_bucket_name = get_random_bucket_name()
# Initialize hash table, 'log_output'
log_output = initialize_log_output('copyObject')
# Prepare arg/value hash table and set it in log_output
arg_value_hash = {}
log_output[:args].each { |x| arg_value_hash[:"#{x}"] = eval x.to_s }
log_output[:args] = arg_value_hash
begin
start_time = Time.now
target_file_name = source_file_name if target_file_name.empty?
makeBucketWrapper(source_bucket_name, log_output)
makeBucketWrapper(target_bucket_name, log_output)
putObjectWrapper(source_bucket_name,
File.join(data_dir, source_file_name), log_output)
copyObjectWrapper(source_bucket_name, target_bucket_name,
source_file_name, target_file_name, log_output)
# Check if copy worked fine
if statObjectWrapper(target_bucket_name, target_file_name, log_output)
log_output[:status] = 'PASS'
else
log_output[:error] = 'Copied file could not be found in the expected location'
log_output[:status] = 'FAIL'
end
cleanUp([source_bucket_name, target_bucket_name], log_output)
rescue => log_output[:error]
log_output[:status] = 'FAIL'
end
print_log(log_output, start_time)
end
def presignedGetObjectTest(data_dir, file_name)
# Tests presignedGetObject api command
# get random bucket name
bucket_name = get_random_bucket_name()
# Initialize hash table, 'log_output'
log_output = initialize_log_output('presignedGet')
# Prepare arg/value hash table and set it in log_output
arg_value_hash = {}
log_output[:args].each { |x| arg_value_hash[:"#{x}"] = eval x.to_s }
log_output[:args] = arg_value_hash
begin
start_time = Time.now
makeBucketWrapper(bucket_name, log_output)
file = File.join(data_dir, file_name)
# Get check sum value without the file name
cksum_orig = `cksum #{file}`.split[0..1]
putObjectWrapper(bucket_name, file, log_output)
get_url = presignedGetWrapper(bucket_name, file_name, log_output)
# Download the file using the URL
# generated by presignedGet api command
`wget -O /tmp/#{file_name}, '#{get_url}' > /dev/null 2>&1`
# Get check sum value for the downloaded file
# Split to get rid of the file name
cksum_new = `cksum /tmp/#{file_name}`.split[0..1]
# Check if check sum values for the orig file
# and the downloaded file match
if cksum_orig == cksum_new
log_output[:status] = 'PASS'
else
log_output[:error] = 'Check sum values do NOT match'
log_output[:status] = 'FAIL'
end
cleanUp([bucket_name], log_output)
rescue => log_output[:error]
log_output[:status] = 'FAIL'
end
print_log(log_output, start_time)
end
def presignedPutObjectTest(data_dir, file_name)
# Tests presignedPutObject api command
# get random bucket name
bucket_name = get_random_bucket_name()
# Initialize hash table, 'log_output'
log_output = initialize_log_output('presignedPut')
# Prepare arg/value hash table and set it in log_output
arg_value_hash = {}
log_output[:args].each { |x| arg_value_hash[:"#{x}"] = eval x.to_s }
log_output[:args] = arg_value_hash
begin
start_time = Time.now
makeBucketWrapper(bucket_name, log_output)
file = File.join(data_dir, file_name)
# Get check sum value and
# split to get rid of the file name
cksum_orig = `cksum #{file}`.split[0..1]
# Generate presigned Put URL and parse it
uri = URI.parse(presignedPutWrapper(bucket_name, file_name, log_output))
request = Net::HTTP::Put.new(uri.request_uri, 'x-amz-acl' => 'public-read')
request.body = IO.read(File.join(data_dir, file_name))
http = Net::HTTP.new(uri.host, uri.port)
http.use_ssl = true if ENV['ENABLE_HTTPS'] == '1'
http.request(request)
if statObjectWrapper(bucket_name, file_name, log_output)
getObjectWrapper(bucket_name, file_name, '/tmp', log_output)
cksum_new = `cksum /tmp/#{file_name}`.split[0..1]
# Check if check sum values of the orig file
# and the downloaded file match
if cksum_orig == cksum_new
log_output[:status] = 'PASS'
else
log_output[:error] = 'Check sum values do NOT match'
log_output[:status] = 'FAIL'
end
else
log_output[:error] = 'Expected to be created object does NOT exist'
log_output[:status] = 'FAIL'
end
cleanUp([bucket_name], log_output)
rescue => log_output[:error]
log_output[:status] = 'FAIL'
end
print_log(log_output, start_time)
end
def presignedPostObjectTest(data_dir, file_name,
expires_in_sec, max_byte_size)
# Tests presignedPostObject api command
# get random bucket name
bucket_name = get_random_bucket_name()
# Initialize hash table, 'log_output'
log_output = initialize_log_output('presignedPost')
# Prepare arg/value hash table and set it in log_output
arg_value_hash = {}
log_output[:args].each { |x| arg_value_hash[:"#{x}"] = eval x.to_s }
log_output[:args] = arg_value_hash
begin
start_time = Time.now
makeBucketWrapper(bucket_name, log_output)
file = File.join(data_dir, file_name)
# Get check sum value and split it
# into parts to get rid of the file name
cksum_orig = `cksum #{file}`.split[0..1]
# Create the presigned POST url
post = presignedPostWrapper(bucket_name, file_name,
expires_in_sec, max_byte_size, log_output)
# Prepare multi parts array for POST command request
file_part = Part.new name: 'file',
body: IO.read(File.join(data_dir, file_name)),
filename: file_name,
content_type: 'application/octet-stream'
parts = [file_part]
# Add POST fields into parts array
post.fields.each do |field, value|
parts.push(Part.new(field, value))
end
boundary = "---------------------------#{rand(10_000_000_000_000_000)}"
body_parts = MultipartBody.new parts, boundary
# Parse presigned Post URL
uri = URI.parse(post.url)
# Create the HTTP objects
http = Net::HTTP.new(uri.host, uri.port)
http.use_ssl = true if ENV['ENABLE_HTTPS'] == '1'
request = Net::HTTP::Post.new(uri.request_uri)
request.body = body_parts.to_s
request.content_type = "multipart/form-data; boundary=#{boundary}"
# Send the request
log_output[:error] = http.request(request)
if statObjectWrapper(bucket_name, file_name, log_output)
getObjectWrapper(bucket_name, file_name, '/tmp', log_output)
cksum_new = `cksum /tmp/#{file_name}`.split[0..1]
# Check if check sum values of the orig file
# and the downloaded file match
if cksum_orig == cksum_new
log_output[:status] = 'PASS'
# FIXME: HTTP No Content error, status code=204 is returned as error
log_output[:error] = nil
else
log_output[:error] = 'Check sum values do NOT match'
log_output[:status] = 'FAIL'
end
else
log_output[:error] = 'Expected to be created object does NOT exist'
log_output[:status] = 'FAIL'
end
cleanUp([bucket_name], log_output)
rescue => log_output[:error]
log_output[:status] = 'FAIL'
end
print_log(log_output, start_time)
end
end
# MAIN CODE
# Create test Class instance and call the tests
aws = AwsSdkRubyTest.new
file_name1 = 'datafile-1-kB'
file_new_name = 'datafile-1-kB-copy'
file_name_list = ['datafile-1-kB', 'datafile-1-b', 'datafile-6-MB']
# Add data_dir in front of each file name in file_name_list
# The location where the bucket and file
# objects are going to be created.
data_dir = ENV['MINT_DATA_DIR'] ||= 'MINT_DATA_DIR is not set'
file_list = file_name_list.map { |f| File.join(data_dir, f) }
destination = '/tmp'
aws.listBucketsTest()
aws.listObjectsTest(file_list)
aws.makeBucketTest()
aws.bucketExistsNegativeTest()
aws.removeBucketTest()
aws.putObjectTest(File.join(data_dir, file_name1))
aws.removeObjectTest(File.join(data_dir, file_name1))
aws.getObjectTest(File.join(data_dir, file_name1), destination)
aws.copyObjectTest(data_dir, file_name1)
aws.copyObjectTest(data_dir, file_name1, file_new_name)
aws.presignedGetObjectTest(data_dir, file_name1)
aws.presignedPutObjectTest(data_dir, file_name1)
aws.presignedPostObjectTest(data_dir, file_name1, 60, 3*1024*1024)

29
mint/run/core/aws-sdk-ruby/run.sh
View file

@ -1,29 +0,0 @@
#!/bin/bash
#
# Mint (C) 2017 Minio, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# handle command line arguments
if [ $# -ne 2 ]; then
echo "usage: run.sh <OUTPUT-LOG-FILE> <ERROR-LOG-FILE>"
exit 1
fi
output_log_file="$1"
error_log_file="$2"
# run tests
chmod a+x aws-stub-tests.rb
ruby aws-stub-tests.rb 1>>"$output_log_file" 2>"$error_log_file"

19
mint/run/core/awscli/README.md
View file

@ -1,19 +0,0 @@
## `awscli` tests
This directory serves as the location for Mint tests using `awscli`. Top level `mint.sh` calls `run.sh` to execute tests.
## Adding new tests
New tests is added into `test.sh` as new functions.
## Running tests manually
- Set environment variables `MINT_DATA_DIR`, `MINT_MODE`, `SERVER_ENDPOINT`, `ACCESS_KEY`, `SECRET_KEY`, `SERVER_REGION` and `ENABLE_HTTPS`
- Call `run.sh` with output log file and error log file. for example
```bash
export MINT_DATA_DIR=~/my-mint-dir
export MINT_MODE=core
export SERVER_ENDPOINT="play.minio.io:9000"
export ACCESS_KEY="Q3AM3UQ867SPQQA43P2F"
export SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG"
export ENABLE_HTTPS=1
export SERVER_REGION=us-east-1
./run.sh /tmp/output.log /tmp/error.log
```

51
mint/run/core/awscli/run.sh
View file

@ -1,51 +0,0 @@
#!/bin/bash
#
# Mint (C) 2017 Minio, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# handle command line arguments
if [ $# -ne 2 ]; then
echo "usage: run.sh <OUTPUT-LOG-FILE> <ERROR-LOG-FILE>"
exit 1
fi
output_log_file="$1"
error_log_file="$2"
# configure awscli
aws configure set aws_access_key_id "$ACCESS_KEY"
aws configure set aws_secret_access_key "$SECRET_KEY"
aws configure set default.region "$SERVER_REGION"
# run tests for virtual style if provided
if [ "$ENABLE_VIRTUAL_STYLE" -eq 1 ]; then
# Setup endpoint scheme
endpoint="http://$DOMAIN:$SERVER_PORT"
if [ "$ENABLE_HTTPS" -eq 1 ]; then
endpoint="https://$DOMAIN:$SERVER_PORT"
fi
dnsmasq --address="/$DOMAIN/$SERVER_IP" --user=root
echo -e "nameserver 127.0.0.1\n$(cat /etc/resolv.conf)" > /etc/resolv.conf
aws configure set default.s3.addressing_style virtual
./test.sh "$endpoint" 1>>"$output_log_file" 2>"$error_log_file"
aws configure set default.s3.addressing_style path
fi
endpoint="http://$SERVER_ENDPOINT"
if [ "$ENABLE_HTTPS" -eq 1 ]; then
endpoint="https://$SERVER_ENDPOINT"
fi
# run path style tests
./test.sh "$endpoint" 1>>"$output_log_file" 2>"$error_log_file"

1726
mint/run/core/awscli/test.sh
View file

File diff suppressed because it is too large Load diff

219
mint/run/core/healthcheck/healthcheck.go
View file

@ -1,219 +0,0 @@
/*
*
* Mint, (C) 2019 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package main
import (
"crypto/tls"
"encoding/json"
"fmt"
"net/http"
"net/url"
"os"
"time"
jwtgo "github.com/dgrijalva/jwt-go"
log "github.com/sirupsen/logrus"
)
const (
pass = "PASS" // Indicate that a test passed
fail = "FAIL" // Indicate that a test failed
livenessPath = "/minio/health/live"
readinessPath = "/minio/health/ready"
prometheusPath = "/minio/prometheus/metrics"
timeout = time.Duration(30 * time.Second)
)
type mintJSONFormatter struct {
}
func (f *mintJSONFormatter) Format(entry *log.Entry) ([]byte, error) {
data := make(log.Fields, len(entry.Data))
for k, v := range entry.Data {
switch v := v.(type) {
case error:
// Otherwise errors are ignored by `encoding/json`
// https://github.com/sirupsen/logrus/issues/137
data[k] = v.Error()
default:
data[k] = v
}
}
serialized, err := json.Marshal(data)
if err != nil {
return nil, fmt.Errorf("Failed to marshal fields to JSON, %w", err)
}
return append(serialized, '\n'), nil
}
// log successful test runs
func successLogger(function string, args map[string]interface{}, startTime time.Time) *log.Entry {
// calculate the test case duration
duration := time.Since(startTime)
// log with the fields as per mint
fields := log.Fields{"name": "healthcheck", "function": function, "args": args, "duration": duration.Nanoseconds() / 1000000, "status": pass}
return log.WithFields(fields)
}
// log failed test runs
func failureLog(function string, args map[string]interface{}, startTime time.Time, alert string, message string, err error) *log.Entry {
// calculate the test case duration
duration := time.Since(startTime)
var fields log.Fields
// log with the fields as per mint
if err != nil {
fields = log.Fields{"name": "healthcheck", "function": function, "args": args,
"duration": duration.Nanoseconds() / 1000000, "status": fail, "alert": alert, "message": message, "error": err}
} else {
fields = log.Fields{"name": "healthcheck", "function": function, "args": args,
"duration": duration.Nanoseconds() / 1000000, "status": fail, "alert": alert, "message": message}
}
return log.WithFields(fields)
}
func testLivenessEndpoint(endpoint string) {
startTime := time.Now()
function := "testLivenessEndpoint"
u, err := url.Parse(fmt.Sprintf("%s%s", endpoint, livenessPath))
if err != nil {
// Could not parse URL successfully
failureLog(function, nil, startTime, "", "URL Parsing for Healthcheck Liveness handler failed", err).Fatal()
}
tr := &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: u.Scheme == "https"},
}
client := &http.Client{Transport: tr, Timeout: timeout}
resp, err := client.Get(u.String())
if err != nil {
// GET request errored
failureLog(function, nil, startTime, "", "GET request failed", err).Fatal()
}
if resp.StatusCode != http.StatusOK {
// Status not 200 OK
failureLog(function, nil, startTime, "", fmt.Sprintf("GET /minio/health/live returned %s", resp.Status), err).Fatal()
}
defer resp.Body.Close()
defer successLogger(function, nil, startTime).Info()
}
func testReadinessEndpoint(endpoint string) {
startTime := time.Now()
function := "testReadinessEndpoint"
u, err := url.Parse(fmt.Sprintf("%s%s", endpoint, readinessPath))
if err != nil {
// Could not parse URL successfully
failureLog(function, nil, startTime, "", "URL Parsing for Healthcheck Readiness handler failed", err).Fatal()
}
tr := &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: u.Scheme == "https"},
}
client := &http.Client{Transport: tr, Timeout: timeout}
resp, err := client.Get(u.String())
if err != nil {
// GET request errored
failureLog(function, nil, startTime, "", "GET request to Readiness endpoint failed", err).Fatal()
}
if resp.StatusCode != http.StatusOK {
// Status not 200 OK
failureLog(function, nil, startTime, "", "GET /minio/health/ready returned non OK status", err).Fatal()
}
defer resp.Body.Close()
defer successLogger(function, nil, startTime).Info()
}
const (
defaultPrometheusJWTExpiry = 100 * 365 * 24 * time.Hour
)
func testPrometheusEndpoint(endpoint string) {
startTime := time.Now()
function := "testPrometheusEndpoint"
u, err := url.Parse(fmt.Sprintf("%s%s", endpoint, prometheusPath))
if err != nil {
// Could not parse URL successfully
failureLog(function, nil, startTime, "", "URL Parsing for Healthcheck Prometheus handler failed", err).Fatal()
}
jwt := jwtgo.NewWithClaims(jwtgo.SigningMethodHS512, jwtgo.StandardClaims{
ExpiresAt: time.Now().UTC().Add(defaultPrometheusJWTExpiry).Unix(),
Subject: os.Getenv("ACCESS_KEY"),
Issuer: "prometheus",
})
token, err := jwt.SignedString([]byte(os.Getenv("SECRET_KEY")))
if err != nil {
failureLog(function, nil, startTime, "", "jwt generation failed", err).Fatal()
}
tr := &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: u.Scheme == "https"},
}
client := &http.Client{Transport: tr, Timeout: timeout}
req, err := http.NewRequest(http.MethodGet, u.String(), nil)
if err != nil {
failureLog(function, nil, startTime, "", "Initializing GET request to Prometheus endpoint failed", err).Fatal()
}
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token))
resp, err := client.Do(req)
if err != nil {
// GET request errored
failureLog(function, nil, startTime, "", "GET request to Prometheus endpoint failed", err).Fatal()
}
if resp.StatusCode != http.StatusOK {
// Status not 200 OK
failureLog(function, nil, startTime, "", "GET /minio/prometheus/metrics returned non OK status", err).Fatal()
}
defer resp.Body.Close()
defer successLogger(function, nil, startTime).Info()
}
func main() {
endpoint := os.Getenv("SERVER_ENDPOINT")
secure := os.Getenv("ENABLE_HTTPS")
endpoint = "http://" + endpoint
if secure == "1" {
endpoint = "https://" + endpoint
}
// Output to stdout instead of the default stderr
log.SetOutput(os.Stdout)
// create custom formatter
mintFormatter := mintJSONFormatter{}
// set custom formatter
log.SetFormatter(&mintFormatter)
// log Info or above -- success cases are Info level, failures are Fatal level
log.SetLevel(log.InfoLevel)
// execute tests
testLivenessEndpoint(endpoint)
testReadinessEndpoint(endpoint)
testPrometheusEndpoint(endpoint)
}

28
mint/run/core/healthcheck/run.sh
View file

@ -1,28 +0,0 @@
#!/bin/bash
#
# Mint (C) 2019 Minio, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# handle command line arguments
if [ $# -ne 2 ]; then
echo "usage: run.sh <OUTPUT-LOG-FILE> <ERROR-LOG-FILE>"
exit 1
fi
output_log_file="$1"
error_log_file="$2"
# run tests
/mint/run/core/healthcheck/healthcheck 1>>"$output_log_file" 2>"$error_log_file"

19
mint/run/core/mc/README.md
View file

@ -1,19 +0,0 @@
## `mc` tests
This directory serves as the location for Mint tests using `mc`. Top level `mint.sh` calls `run.sh` to execute tests.
## Adding new tests
New tests is added into `test.sh` as new functions.
## Running tests manually
- Set environment variables `MINT_DATA_DIR`, `MINT_MODE`, `SERVER_ENDPOINT`, `ACCESS_KEY`, `SECRET_KEY`, `SERVER_REGION` and `ENABLE_HTTPS`
- Call `run.sh` with output log file and error log file. for example
```bash
export MINT_DATA_DIR=~/my-mint-dir
export MINT_MODE=core
export SERVER_ENDPOINT="play.minio.io:9000"
export ACCESS_KEY="Q3AM3UQ867SPQQA43P2F"
export SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG"
export ENABLE_HTTPS=1
export SERVER_REGION=us-east-1
./run.sh /tmp/output.log /tmp/error.log
```

27
mint/run/core/mc/run.sh
View file

@ -1,27 +0,0 @@
#!/bin/bash
#
# Minio Cloud Storage, (C) 2017 Minio, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# handle command line arguments
if [ $# -ne 2 ]; then
echo "usage: run.sh <OUTPUT-LOG-FILE> <ERROR-LOG-FILE>"
exit 1
fi
output_log_file="$1"
error_log_file="$2"
./functional-tests.sh 1>>"$output_log_file" 2>"$error_log_file"

26
mint/run/core/minio-dotnet/run.sh
View file

@ -1,26 +0,0 @@
#!/bin/bash
#
# Mint (C) 2017 Minio, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# handle command line arguments
if [ $# -ne 2 ]; then
echo "usage: run.sh <OUTPUT-LOG-FILE> <ERROR-LOG-FILE>"
exit 1
fi
output_log_file="$1"
error_log_file="$2"
/mint/run/core/minio-dotnet/out/Minio.Functional.Tests 1>>"$output_log_file" 2>"$error_log_file"

19
mint/run/core/minio-go/README.md
View file

@ -1,19 +0,0 @@
## `minio-go` tests
This directory serves as the location for Mint tests using `minio-go`. Top level `mint.sh` calls `run.sh` to execute tests.
## Adding new tests
New tests are added in functional tests of minio-go. Please check https://github.com/minio/minio-go
## Running tests manually
- Set environment variables `MINT_DATA_DIR`, `MINT_MODE`, `SERVER_ENDPOINT`, `ACCESS_KEY`, `SECRET_KEY`, `SERVER_REGION` and `ENABLE_HTTPS`
- Call `run.sh` with output log file and error log file. for example
```bash
export MINT_DATA_DIR=~/my-mint-dir
export MINT_MODE=core
export SERVER_ENDPOINT="play.minio.io:9000"
export ACCESS_KEY="Q3AM3UQ867SPQQA43P2F"
export SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG"
export ENABLE_HTTPS=1
export SERVER_REGION=us-east-1
./run.sh /tmp/output.log /tmp/error.log
```

28
mint/run/core/minio-go/run.sh
View file

@ -1,28 +0,0 @@
#!/bin/bash
#
# Mint (C) 2017 Minio, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# handle command line arguments
if [ $# -ne 2 ]; then
echo "usage: run.sh <OUTPUT-LOG-FILE> <ERROR-LOG-FILE>"
exit 1
fi
output_log_file="$1"
error_log_file="$2"
# run tests
/mint/run/core/minio-go/minio-go 1>>"$output_log_file" 2>"$error_log_file"

19
mint/run/core/minio-java/README.md
View file

@ -1,19 +0,0 @@
## `minio-java` tests
This directory serves as the location for Mint tests using `minio-java`. Top level `mint.sh` calls `run.sh` to execute tests.
## Adding new tests
New tests is added in functional tests of minio-java. Please check https://github.com/minio/minio-java
## Running tests manually
- Set environment variables `MINT_DATA_DIR`, `MINT_MODE`, `SERVER_ENDPOINT`, `ACCESS_KEY`, `SECRET_KEY`, `SERVER_REGION` and `ENABLE_HTTPS`
- Call `run.sh` with output log file and error log file. for example
```bash
export MINT_DATA_DIR=~/my-mint-dir
export MINT_MODE=core
export SERVER_ENDPOINT="play.minio.io:9000"
export ACCESS_KEY="Q3AM3UQ867SPQQA43P2F"
export SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG"
export ENABLE_HTTPS=1
export SERVER_REGION=us-east-1
./run.sh /tmp/output.log /tmp/error.log
```

34
mint/run/core/minio-java/run.sh
View file

@ -1,34 +0,0 @@
#!/bin/bash
#
# Mint (C) 2017 Minio, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# handle command line arguments
if [ $# -ne 2 ]; then
echo "usage: run.sh <OUTPUT-LOG-FILE> <ERROR-LOG-FILE>"
exit 1
fi
output_log_file="$1"
error_log_file="$2"
# run tests
endpoint="http://$SERVER_ENDPOINT"
if [ "$ENABLE_HTTPS" -eq 1 ]; then
endpoint="https://$SERVER_ENDPOINT"
fi
java -Xmx4096m -Xms256m -cp "/mint/run/core/minio-java/*:." FunctionalTest \
"$endpoint" "$ACCESS_KEY" "$SECRET_KEY" "$SERVER_REGION" 1>>"$output_log_file" 2>"$error_log_file"

19
mint/run/core/minio-js/README.md
View file

@ -1,19 +0,0 @@
## `minio-js` tests
This directory serves as the location for Mint tests using `minio-js`. Top level `mint.sh` calls `run.sh` to execute tests.
## Adding new tests
New tests is added in functional tests of minio-js. Please check https://github.com/minio/minio-js
## Running tests manually
- Set environment variables `MINT_DATA_DIR`, `MINT_MODE`, `SERVER_ENDPOINT`, `ACCESS_KEY`, `SECRET_KEY`, `SERVER_REGION` and `ENABLE_HTTPS`
- Call `run.sh` with output log file and error log file. for example
```bash
export MINT_DATA_DIR=~/my-mint-dir
export MINT_MODE=core
export SERVER_ENDPOINT="play.minio.io:9000"
export ACCESS_KEY="Q3AM3UQ867SPQQA43P2F"
export SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG"
export ENABLE_HTTPS=1
export SERVER_REGION=us-east-1
./run.sh /tmp/output.log /tmp/error.log
```

69
mint/run/core/minio-js/minioreporter.js
View file

@ -1,69 +0,0 @@
/*
* Minio Reporter for JSON formatted logging, (C) 2017 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
var mocha = require('mocha');
module.exports = minioreporter;
function minioreporter(runner) {
mocha.reporters.Base.call(this, runner);
var self = this;
runner.on('pass', function (test) {
GenerateJsonEntry(test)
});
runner.on('fail', function (test, err) {
GenerateJsonEntry(test, err)
});
}
/**
* Convert test result into a JSON object and print on the console.
*
* @api private
* @param test, err
*/
function GenerateJsonEntry (test, err) {
var res = test.title.split("_")
var jsonEntry = {};
jsonEntry.name = "minio-js"
if (res.length > 0 && res[0].length) {
jsonEntry.function = res[0]
}
if (res.length > 1 && res[1].length) {
jsonEntry.args = res[1]
}
jsonEntry.duration = test.duration
if (res.length > 2 && res[2].length) {
jsonEntry.alert = res[2]
}
if (err != null ) {
jsonEntry.status = "FAIL"
jsonEntry.error = err.stack.replace(/\n/g, " ").replace(/ +(?= )/g,'')
} else {
jsonEntry.status = "PASS"
}
process.stdout.write(JSON.stringify(jsonEntry) + "\n")
}

49
mint/run/core/minio-js/package.json
View file

@ -1,49 +0,0 @@
{
"name": "bin",
"version": "1.0.0",
"main": "functional_test.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
},
"keywords": [],
"author": "",
"license": "ISC",
"description": "",
"dependencies": {
"app-module-path": "*",
"async": "*",
"block-stream2": "*",
"concat-stream": "*",
"es6-error": "*",
"json-stream": "*",
"lodash": "*",
"mime-types": "*",
"mkdirp": "*",
"moment": "*",
"source-map-support": "*",
"through2": "*",
"xml": "*",
"xml2js": "*"
},
"devDependencies": {
"browserify": "*",
"chai": "*",
"gulp": "*",
"gulp-babel": "*",
"gulp-jscs": "*",
"jshint":"2.*",
"gulp-jshint": "*",
"gulp-mocha": "*",
"gulp-notify": "*",
"gulp-sourcemaps": "*",
"jshint-stylish": "*",
"mocha": "*",
"mocha-steps": "*",
"nock": "*",
"rewire": "*",
"superagent": "*"
},
"scripts": {
"test": "mocha"
}
}

28
mint/run/core/minio-js/run.sh
View file

@ -1,28 +0,0 @@
#!/bin/bash
#
# Minio Cloud Storage, (C) 2017 Minio, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# handle command line arguments
if [ $# -ne 2 ]; then
echo "usage: run.sh <OUTPUT-LOG-FILE> <ERROR-LOG-FILE>"
exit 1
fi
output_log_file="$1"
error_log_file="$2"
# run tests
./node_modules/mocha/bin/mocha -R minioreporter -b --exit 1>>"$output_log_file" 2>"$error_log_file"

19
mint/run/core/minio-py/README.md
View file

@ -1,19 +0,0 @@
## `minio-py` tests
This directory serves as the location for Mint tests using `minio-py`. Top level `mint.sh` calls `run.sh` to execute tests.
## Adding new tests
New tests is added in functional tests of minio-py. Please check https://github.com/minio/minio-py
## Running tests manually
- Set environment variables `MINT_DATA_DIR`, `MINT_MODE`, `SERVER_ENDPOINT`, `ACCESS_KEY`, `SECRET_KEY`, `SERVER_REGION` and `ENABLE_HTTPS`
- Call `run.sh` with output log file and error log file. for example
```bash
export MINT_DATA_DIR=~/my-mint-dir
export MINT_MODE=core
export SERVER_ENDPOINT="play.minio.io:9000"
export ACCESS_KEY="Q3AM3UQ867SPQQA43P2F"
export SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG"
export ENABLE_HTTPS=1
export SERVER_REGION=us-east-1
./run.sh /tmp/output.log /tmp/error.log
```

28
mint/run/core/minio-py/run.sh
View file

@ -1,28 +0,0 @@
#!/bin/bash
#
# Mint (C) 2017 Minio, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# handle command line arguments
if [ $# -ne 2 ]; then
echo "usage: run.sh <OUTPUT-LOG-FILE> <ERROR-LOG-FILE>"
exit 1
fi
output_log_file="$1"
error_log_file="$2"
# run tests
python "/mint/run/core/minio-py/tests.py" 1>>"$output_log_file" 2>"$error_log_file"

2
mint/run/core/s3cmd/.gitignore vendored
View file

@ -1,2 +0,0 @@
*~
*.log

19
mint/run/core/s3cmd/README.md
View file

@ -1,19 +0,0 @@
## `s3cmd` tests
This directory serves as the location for Mint tests using `s3cmd`. Top level `mint.sh` calls `run.sh` to execute tests.
## Adding new tests
New tests is added into `test.sh` as new functions.
## Running tests manually
- Set environment variables `MINT_DATA_DIR`, `MINT_MODE`, `SERVER_ENDPOINT`, `ACCESS_KEY`, `SECRET_KEY`, `SERVER_REGION` and `ENABLE_HTTPS`
- Call `run.sh` with output log file and error log file. for example
```bash
export MINT_DATA_DIR=~/my-mint-dir
export MINT_MODE=core
export SERVER_ENDPOINT="play.minio.io:9000"
export ACCESS_KEY="Q3AM3UQ867SPQQA43P2F"
export SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG"
export ENABLE_HTTPS=1
export SERVER_REGION=us-east-1
./run.sh /tmp/output.log /tmp/error.log
```

28
mint/run/core/s3cmd/run.sh
View file

@ -1,28 +0,0 @@
#!/bin/bash
#
# Mint (C) 2017 Minio, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# handle command line arguments
if [ $# -ne 2 ]; then
echo "usage: run.sh <OUTPUT-LOG-FILE> <ERROR-LOG-FILE>"
exit 1
fi
output_log_file="$1"
error_log_file="$2"
# run tests
./test.sh 1>>"$output_log_file" 2>"$error_log_file"

406
mint/run/core/s3cmd/test.sh
View file

@ -1,406 +0,0 @@
#!/bin/bash
#
# Mint (C) 2017-2020 Minio, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
if [ -n "$MINT_MODE" ]; then
if [ -z "${MINT_DATA_DIR+x}" ]; then
echo "MINT_DATA_DIR not defined"
exit 1
fi
if [ -z "${SERVER_ENDPOINT+x}" ]; then
echo "SERVER_ENDPOINT not defined"
exit 1
fi
if [ -z "${ACCESS_KEY+x}" ]; then
echo "ACCESS_KEY not defined"
exit 1
fi
if [ -z "${SECRET_KEY+x}" ]; then
echo "SECRET_KEY not defined"
exit 1
fi
fi
if [ -z "${SERVER_ENDPOINT+x}" ]; then
SERVER_ENDPOINT="play.minio.io:9000"
ACCESS_KEY="Q3AM3UQ867SPQQA43P2F"
SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG"
ENABLE_HTTPS=1
SERVER_REGION="us-east-1"
fi
WORK_DIR="$PWD"
DATA_DIR="$MINT_DATA_DIR"
if [ -z "$MINT_MODE" ]; then
WORK_DIR="$PWD/.run-$RANDOM"
DATA_DIR="$WORK_DIR/data"
fi
FILE_1_MB="$DATA_DIR/datafile-1-MB"
FILE_65_MB="$DATA_DIR/datafile-65-MB"
declare FILE_1_MB_MD5SUM
declare FILE_65_MB_MD5SUM
BUCKET_NAME="s3cmd-test-bucket-$RANDOM"
S3CMD=$(which s3cmd)
declare -a S3CMD_CMD
function get_md5sum()
{
filename="$FILE_1_MB"
out=$(md5sum "$filename" 2>/dev/null)
rv=$?
if [ "$rv" -eq 0 ]; then
awk '{ print $1 }' <<< "$out"
fi
return "$rv"
}
function get_time()
{
date +%s%N
}
function get_duration()
{
start_time=$1
end_time=$(get_time)
echo $(( (end_time - start_time) / 1000000 ))
}
function log_success()
{
if [ -n "$MINT_MODE" ]; then
printf '{"name": "s3cmd", "duration": "%d", "function": "%s", "status": "PASS"}\n' "$(get_duration "$1")" "$2"
fi
}
function show()
{
if [ -z "$MINT_MODE" ]; then
func_name="$1"
echo "Running $func_name()"
fi
}
function fail()
{
rv="$1"
shift
if [ "$rv" -ne 0 ]; then
echo "$@"
fi
return "$rv"
}
function assert()
{
expected_rv="$1"
shift
start_time="$1"
shift
func_name="$1"
shift
err=$("$@" 2>&1)
rv=$?
if [ "$rv" -ne 0 ] && [ "$expected_rv" -eq 0 ]; then
if [ -n "$MINT_MODE" ]; then
err=$(printf '%s' "$err" | python -c 'import sys,json; print(json.dumps(sys.stdin.read()))')
## err is already JSON string, no need to double quote
printf '{"name": "s3cmd", "duration": "%d", "function": "%s", "status": "FAIL", "error": %s}\n' "$(get_duration "$start_time")" "$func_name" "$err"
else
echo "s3cmd: $func_name: $err"
fi
exit "$rv"
fi
return 0
}
function assert_success() {
assert 0 "$@"
}
function assert_failure() {
assert 1 "$@"
}
function s3cmd_cmd()
{
cmd=( "${S3CMD_CMD[@]}" "$@" )
"${cmd[@]}"
rv=$?
return "$rv"
}
function check_md5sum()
{
expected_checksum="$1"
shift
filename="$*"
checksum="$(get_md5sum "$filename")"
rv=$?
if [ "$rv" -ne 0 ]; then
echo "unable to get md5sum for $filename"
return "$rv"
fi
if [ "$checksum" != "$expected_checksum" ]; then
echo "$filename: md5sum mismatch"
return 1
fi
return 0
}
function test_make_bucket()
{
show "${FUNCNAME[0]}"
start_time=$(get_time)
bucket_name="s3cmd-test-bucket-$RANDOM"
assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd mb "s3://${bucket_name}"
assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rb "s3://${bucket_name}"
log_success "$start_time" "${FUNCNAME[0]}"
}
function test_make_bucket_error() {
show "${FUNCNAME[0]}"
start_time=$(get_time)
bucket_name="S3CMD-test%bucket%$RANDOM"
assert_failure "$start_time" "${FUNCNAME[0]}" s3cmd_cmd mb "s3://${bucket_name}"
log_success "$start_time" "${FUNCNAME[0]}"
}
function setup()
{
start_time=$(get_time)
assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd mb "s3://${BUCKET_NAME}"
}
function teardown()
{
start_time=$(get_time)
assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rm --force --recursive "s3://${BUCKET_NAME}"
assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rb --force "s3://${BUCKET_NAME}"
}
function test_put_object()
{
show "${FUNCNAME[0]}"
start_time=$(get_time)
object_name="s3cmd-test-object-$RANDOM"
assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd put "${FILE_1_MB}" "s3://${BUCKET_NAME}/${object_name}"
assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rm "s3://${BUCKET_NAME}/${object_name}"
log_success "$start_time" "${FUNCNAME[0]}"
}
function test_put_object_error()
{
show "${FUNCNAME[0]}"
start_time=$(get_time)
object_long_name=$(printf "s3cmd-test-object-%01100d" 1)
assert_failure "$start_time" "${FUNCNAME[0]}" s3cmd_cmd put "${FILE_1_MB}" "s3://${BUCKET_NAME}/${object_long_name}"
log_success "$start_time" "${FUNCNAME[0]}"
}
function test_put_object_multipart()
{
show "${FUNCNAME[0]}"
start_time=$(get_time)
object_name="s3cmd-test-object-$RANDOM"
assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd put "${FILE_65_MB}" "s3://${BUCKET_NAME}/${object_name}"
assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rm "s3://${BUCKET_NAME}/${object_name}"
log_success "$start_time" "${FUNCNAME[0]}"
}
function test_get_object()
{
show "${FUNCNAME[0]}"
start_time=$(get_time)
object_name="s3cmd-test-object-$RANDOM"
assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd put "${FILE_1_MB}" "s3://${BUCKET_NAME}/${object_name}"
assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd get "s3://${BUCKET_NAME}/${object_name}" "${object_name}.downloaded"
assert_success "$start_time" "${FUNCNAME[0]}" check_md5sum "$FILE_1_MB_MD5SUM" "${object_name}.downloaded"
assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rm "s3://${BUCKET_NAME}/${object_name}"
assert_success "$start_time" "${FUNCNAME[0]}" rm -f "${object_name}.downloaded"
log_success "$start_time" "${FUNCNAME[0]}"
}
function test_get_object_error()
{
show "${FUNCNAME[0]}"
start_time=$(get_time)
object_name="s3cmd-test-object-$RANDOM"
assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd put "${FILE_1_MB}" "s3://${BUCKET_NAME}/${object_name}"
assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rm "s3://${BUCKET_NAME}/${object_name}"
assert_failure "$start_time" "${FUNCNAME[0]}" s3cmd_cmd get "s3://${BUCKET_NAME}/${object_name}" "${object_name}.downloaded"
assert_success "$start_time" "${FUNCNAME[0]}" rm -f "${object_name}.downloaded"
log_success "$start_time" "${FUNCNAME[0]}"
}
function test_get_object_multipart()
{
show "${FUNCNAME[0]}"
start_time=$(get_time)
object_name="s3cmd-test-object-$RANDOM"
assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd put "${FILE_65_MB}" "s3://${BUCKET_NAME}/${object_name}"
assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd get "s3://${BUCKET_NAME}/${object_name}" "${object_name}.downloaded"
assert_success "$start_time" "${FUNCNAME[0]}" check_md5sum "$FILE_65_MB_MD5SUM" "${object_name}.downloaded"
assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rm "s3://${BUCKET_NAME}/${object_name}"
assert_success "$start_time" "${FUNCNAME[0]}" rm -f "${object_name}.downloaded"
log_success "$start_time" "${FUNCNAME[0]}"
}
function test_sync_list_objects()
{
show "${FUNCNAME[0]}"
start_time=$(get_time)
bucket_name="s3cmd-test-bucket-$RANDOM"
object_name="s3cmd-test-object-$RANDOM"
assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd mb "s3://${bucket_name}"
assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd sync "$DATA_DIR/" "s3://${bucket_name}"
diff -bB <(ls "$DATA_DIR") <("${S3CMD_CMD[@]}" ls "s3://${bucket_name}" | awk '{print $4}' | sed "s/s3:*..${bucket_name}.//g") >/dev/null 2>&1
assert_success "$start_time" "${FUNCNAME[0]}" fail $? "sync and list differs"
assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rb --force --recursive "s3://${bucket_name}"
log_success "$start_time" "${FUNCNAME[0]}"
}
function run_test()
{
test_make_bucket
test_make_bucket_error
setup
test_put_object
test_put_object_error
test_put_object_multipart
test_get_object
test_get_object_multipart
test_sync_list_objects
teardown
}
function __init__()
{
set -e
S3CMD_CONFIG_DIR="/tmp/.s3cmd-$RANDOM"
mkdir -p $S3CMD_CONFIG_DIR
S3CMD_CONFIG_FILE="$S3CMD_CONFIG_DIR/s3cfg"
# configure s3cmd
cat > $S3CMD_CONFIG_FILE <<EOF
signature_v2 = False
host_base = $SERVER_ENDPOINT
host_bucket = $SERVER_ENDPOINT
bucket_location = $SERVER_REGION
use_https = $ENABLE_HTTPS
access_key = $ACCESS_KEY
secret_key = $SECRET_KEY
EOF
# For Mint, setup is already done. For others, setup the environment
if [ -z "$MINT_MODE" ]; then
mkdir -p "$WORK_DIR"
mkdir -p "$DATA_DIR"
# If s3cmd executable binary is not available in current directory, use it in the path.
if [ ! -x "$S3CMD" ]; then
echo "'s3cmd' executable binary not found in current directory and in path"
exit 1
fi
fi
if [ ! -x "$S3CMD" ]; then
echo "$S3CMD executable binary not found"
exit 1
fi
S3CMD_CMD=( "${S3CMD}" --config "$S3CMD_CONFIG_FILE" )
if [ ! -e "$FILE_1_MB" ]; then
shred -n 1 -s 1MB - >"$FILE_1_MB"
fi
if [ ! -e "$FILE_65_MB" ]; then
shred -n 1 -s 65MB - >"$FILE_65_MB"
fi
set -E
set -o pipefail
FILE_1_MB_MD5SUM="$(get_md5sum "$FILE_1_MB")"
rv=$?
if [ $rv -ne 0 ]; then
echo "unable to get md5sum of $FILE_1_MB"
exit 1
fi
FILE_65_MB_MD5SUM="$(get_md5sum "$FILE_65_MB")"
rv=$?
if [ $rv -ne 0 ]; then
echo "unable to get md5sum of $FILE_65_MB"
exit 1
fi
set +e
}
function main()
{
( run_test )
rv=$?
rm -fr "$S3CMD_CONFIG_FILE"
if [ -z "$MINT_MODE" ]; then
rm -fr "$WORK_DIR" "$DATA_DIR"
fi
exit "$rv"
}
__init__ "$@"
main "$@"

21
mint/run/core/s3select/README.md
View file

@ -1,21 +0,0 @@
## `s3select` tests
This directory serves as the location for Mint tests for s3select features. Top level `mint.sh` calls `run.sh` to execute tests.
## Adding new tests
New tests are added into `s3select/tests.py` as new functions.
## Running tests manually
- Set environment variables `MINT_DATA_DIR`, `MINT_MODE`, `SERVER_ENDPOINT`, `ACCESS_KEY`, `SECRET_KEY`, `SERVER_REGION` and `ENABLE_HTTPS`
- Call `run.sh` with output log file and error log file. for example
```bash
export MINT_DATA_DIR=~/my-mint-dir
export MINT_MODE=core
export SERVER_ENDPOINT="play.min.io"
export ACCESS_KEY="Q3AM3UQ867SPQQA43P2F"
export SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG"
export ENABLE_HTTPS=1
export SERVER_REGION=us-east-1
./run.sh /tmp/output.log /tmp/error.log
```

172
mint/run/core/s3select/csv.py
View file

@ -1,172 +0,0 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# MinIO Python Library for Amazon S3 Compatible Cloud Storage,
# (C) 2015-2020 MinIO, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import io
from minio import Minio
from minio.select.options import (SelectObjectOptions, CSVInput,
RequestProgress, InputSerialization,
OutputSerialization, CSVOutput, JsonOutput)
from utils import *
def test_sql_api(test_name, client, bucket_name, input_data, sql_opts, expected_output):
""" Test if the passed SQL request has the output equal to the passed execpted one"""
object_name = generate_object_name()
got_output = b''
try:
bytes_content = io.BytesIO(input_data)
client.put_object(bucket_name, object_name, io.BytesIO(input_data), len(input_data))
data = client.select_object_content(bucket_name, object_name, sql_opts)
# Get the records
records = io.BytesIO()
for d in data.stream(10*1024):
records.write(d.encode('utf-8'))
got_output = records.getvalue()
except Exception as select_err:
if not isinstance(expected_output, Exception):
raise ValueError('Test {} unexpectedly failed with: {}'.format(test_name, select_err))
else:
if isinstance(expected_output, Exception):
raise ValueError('Test {}: expected an exception, got {}'.format(test_name, got_output))
if got_output != expected_output:
raise ValueError('Test {}: data mismatch. Expected : {}, Received {}'.format(test_name, expected_output, got_output))
finally:
client.remove_object(bucket_name, object_name)
def test_csv_input_custom_quote_char(client, log_output):
# Get a unique bucket_name and object_name
log_output.args['bucket_name'] = bucket_name = generate_bucket_name()
tests = [
# Invalid quote character, should fail
('""', '"', b'col1,col2,col3\n', Exception()),
# UTF-8 quote character
('ع', '"', 'عcol1ع,عcol2ع,عcol3ع\n'.encode(), b'{"_1":"col1","_2":"col2","_3":"col3"}\n'),
# Only one field is quoted
('"', '"', b'"col1",col2,col3\n', b'{"_1":"col1","_2":"col2","_3":"col3"}\n'),
('"', '"', b'"col1,col2,col3"\n', b'{"_1":"col1,col2,col3"}\n'),
('\'', '"', b'"col1",col2,col3\n', b'{"_1":"\\"col1\\"","_2":"col2","_3":"col3"}\n'),
('', '"', b'"col1",col2,col3\n', b'{"_1":"\\"col1\\"","_2":"col2","_3":"col3"}\n'),
('', '"', b'"col1",col2,col3\n', b'{"_1":"\\"col1\\"","_2":"col2","_3":"col3"}\n'),
('', '"', b'"col1","col2","col3"\n', b'{"_1":"\\"col1\\"","_2":"\\"col2\\"","_3":"\\"col3\\""}\n'),
('"', '"', b'""""""\n', b'{"_1":"\\"\\""}\n'),
('"', '"', b'A",B\n', b'{"_1":"A\\"","_2":"B"}\n'),
('"', '"', b'A"",B\n', b'{"_1":"A\\"\\"","_2":"B"}\n'),
('"', '\\', b'A\\B,C\n', b'{"_1":"A\\\\B","_2":"C"}\n'),
('"', '"', b'"A""B","CD"\n', b'{"_1":"A\\"B","_2":"CD"}\n'),
('"', '\\', b'"A\\B","CD"\n', b'{"_1":"AB","_2":"CD"}\n'),
('"', '\\', b'"A\\,","CD"\n', b'{"_1":"A,","_2":"CD"}\n'),
('"', '\\', b'"A\\"B","CD"\n', b'{"_1":"A\\"B","_2":"CD"}\n'),
('"', '\\', b'"A\\""\n', b'{"_1":"A\\""}\n'),
('"', '\\', b'"A\\"\\"B"\n', b'{"_1":"A\\"\\"B"}\n'),
('"', '\\', b'"A\\"","\\"B"\n', b'{"_1":"A\\"","_2":"\\"B"}\n'),
]
client.make_bucket(bucket_name)
try:
for idx, (quote_char, escape_char, data, expected_output) in enumerate(tests):
sql_opts = SelectObjectOptions(
expression="select * from s3object",
input_serialization=InputSerialization(
compression_type="NONE",
csv=CSVInput(FileHeaderInfo="NONE",
RecordDelimiter="\n",
FieldDelimiter=",",
QuoteCharacter=quote_char,
QuoteEscapeCharacter=escape_char,
Comments="#",
AllowQuotedRecordDelimiter="FALSE",),
),
output_serialization=OutputSerialization(
json = JsonOutput(
RecordDelimiter="\n",
)
),
request_progress=RequestProgress(
enabled="False"
)
)
test_sql_api(f'test_{idx}', client, bucket_name, data, sql_opts, expected_output)
finally:
client.remove_bucket(bucket_name)
# Test passes
print(log_output.json_report())
def test_csv_output_custom_quote_char(client, log_output):
# Get a unique bucket_name and object_name
log_output.args['bucket_name'] = bucket_name = generate_bucket_name()
tests = [
# UTF-8 quote character
("''", "''", b'col1,col2,col3\n', Exception()),
("'", "'", b'col1,col2,col3\n', b"'col1','col2','col3'\n"),
("", '"', b'col1,col2,col3\n', b'\x00col1\x00,\x00col2\x00,\x00col3\x00\n'),
('"', '"', b'col1,col2,col3\n', b'"col1","col2","col3"\n'),
('"', '"', b'col"1,col2,col3\n', b'"col""1","col2","col3"\n'),
('"', '"', b'""""\n', b'""""\n'),
('"', '"', b'\n', b''),
("'", "\\", b'col1,col2,col3\n', b"'col1','col2','col3'\n"),
("'", "\\", b'col""1,col2,col3\n', b"'col\"\"1','col2','col3'\n"),
("'", "\\", b'col\'1,col2,col3\n', b"'col\\'1','col2','col3'\n"),
("'", "\\", b'"col\'1","col2","col3"\n', b"'col\\'1','col2','col3'\n"),
("'", "\\", b'col\'\n', b"'col\\''\n"),
# Two consecutive escaped quotes
("'", "\\", b'"a"""""\n', b"'a\"\"'\n"),
]
client.make_bucket(bucket_name)
try:
for idx, (quote_char, escape_char, input_data, expected_output) in enumerate(tests):
sql_opts = SelectObjectOptions(
expression="select * from s3object",
input_serialization=InputSerialization(
compression_type="NONE",
csv=CSVInput(FileHeaderInfo="NONE",
RecordDelimiter="\n",
FieldDelimiter=",",
QuoteCharacter='"',
QuoteEscapeCharacter='"',
Comments="#",
AllowQuotedRecordDelimiter="FALSE",),
),
output_serialization=OutputSerialization(
csv=CSVOutput(QuoteFields="ALWAYS",
RecordDelimiter="\n",
FieldDelimiter=",",
QuoteCharacter=quote_char,
QuoteEscapeCharacter=escape_char,)
),
request_progress=RequestProgress(
enabled="False"
)
)
test_sql_api(f'test_{idx}', client, bucket_name, input_data, sql_opts, expected_output)
finally:
client.remove_bucket(bucket_name)
# Test passes
print(log_output.json_report())

28
mint/run/core/s3select/run.sh
View file

@ -1,28 +0,0 @@
#!/bin/bash
#
# Mint (C) 2020 Minio, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# handle command line arguments
if [ $# -ne 2 ]; then
echo "usage: run.sh <OUTPUT-LOG-FILE> <ERROR-LOG-FILE>"
exit 1
fi
output_log_file="$1"
error_log_file="$2"
# run path style tests
python "./tests.py" 1>>"$output_log_file" 2>"$error_log_file"

399
mint/run/core/s3select/sql_ops.py
View file

@ -1,399 +0,0 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# MinIO Python Library for Amazon S3 Compatible Cloud Storage,
# (C) 2020 MinIO, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import io
from datetime import datetime
from minio import Minio
from minio.select.options import (SelectObjectOptions, CSVInput, JSONInput,
RequestProgress, InputSerialization,
OutputSerialization, CSVOutput, JsonOutput)
from utils import *
def test_sql_expressions_custom_input_output(client, input_bytes, sql_input, sql_output, tests, log_output):
bucket_name = generate_bucket_name()
object_name = generate_object_name()
log_output.args['total_tests'] = 0
log_output.args['total_success'] = 0
client.make_bucket(bucket_name)
try:
content = io.BytesIO(bytes(input_bytes, 'utf-8'))
client.put_object(bucket_name, object_name, content, len(input_bytes))
for idx, (test_name, select_expression, expected_output) in enumerate(tests):
if select_expression == '':
continue
try:
log_output.args['total_tests'] += 1
options = SelectObjectOptions(
expression=select_expression,
input_serialization=sql_input,
output_serialization=sql_output,
request_progress=RequestProgress(
enabled="False"
)
)
data = client.select_object_content(bucket_name, object_name, options)
# Get the records
records = io.BytesIO()
for d in data.stream(10*1024):
records.write(d.encode('utf-8'))
got_output = records.getvalue()
if got_output != expected_output:
if type(expected_output) == datetime:
# Attempt to parse the date which will throw an exception for any issue
datetime.strptime(got_output.decode("utf-8").strip(), '%Y-%m-%dT%H:%M:%S.%f%z')
else:
raise ValueError('Test {}: data mismatch. Expected : {}. Received: {}.'.format(idx+1, expected_output, got_output))
log_output.args['total_success'] += 1
except Exception as err:
continue ## TODO, raise instead
# raise Exception(err)
finally:
client.remove_object(bucket_name, object_name)
client.remove_bucket(bucket_name)
def test_sql_expressions(client, input_json_bytes, tests, log_output):
input_serialization = InputSerialization(
compression_type="NONE",
json=JSONInput(Type="DOCUMENT"),
)
output_serialization=OutputSerialization(
csv=CSVOutput(QuoteFields="ASNEEDED")
)
test_sql_expressions_custom_input_output(client, input_json_bytes,
input_serialization, output_serialization, tests, log_output)
def test_sql_operators(client, log_output):
json_testfile = """{"id": 1, "name": "John", "age": 3}
{"id": 2, "name": "Elliot", "age": 4}
{"id": 3, "name": "Yves", "age": 5}
{"id": 4, "name": null, "age": 0}
"""
tests = [
# Logical operators
("AND", "select * from S3Object s where s.id = 1 AND s.name = 'John'", b'1,John,3\n'),
("NOT", "select * from S3Object s where NOT s.id = 1", b'2,Elliot,4\n3,Yves,5\n4,,0\n'),
("OR", "select * from S3Object s where s.id = 1 OR s.id = 3", b'1,John,3\n3,Yves,5\n'),
# Comparison Operators
("<", "select * from S3Object s where s.age < 4", b'1,John,3\n4,,0\n'),
(">", "select * from S3Object s where s.age > 4", b'3,Yves,5\n'),
("<=", "select * from S3Object s where s.age <= 4", b'1,John,3\n2,Elliot,4\n4,,0\n'),
(">=", "select * from S3Object s where s.age >= 4", b'2,Elliot,4\n3,Yves,5\n'),
("=", "select * from S3Object s where s.age = 4", b'2,Elliot,4\n'),
("<>", "select * from S3Object s where s.age <> 4", b'1,John,3\n3,Yves,5\n4,,0\n'),
("!=", "select * from S3Object s where s.age != 4", b'1,John,3\n3,Yves,5\n4,,0\n'),
("BETWEEN", "select * from S3Object s where s.age BETWEEN 4 AND 5", b'2,Elliot,4\n3,Yves,5\n'),
("IN", "select * from S3Object s where s.age IN (3,5)", b'1,John,3\n3,Yves,5\n'),
# Pattern Matching Operators
("LIKE_", "select * from S3Object s where s.name LIKE '_ves'", b'3,Yves,5\n'),
("LIKE%", "select * from S3Object s where s.name LIKE 'Ell%t'", b'2,Elliot,4\n'),
# Unitary Operators
("NULL", "select * from S3Object s where s.name IS NULL", b'4,,0\n'),
("NOT_NULL", "select * from S3Object s where s.age IS NOT NULL", b'1,John,3\n2,Elliot,4\n3,Yves,5\n4,,0\n'),
# Math Operators
("+", "select * from S3Object s where s.age = 1+3 ", b'2,Elliot,4\n'),
("-", "select * from S3Object s where s.age = 5-1 ", b'2,Elliot,4\n'),
("*", "select * from S3Object s where s.age = 2*2 ", b'2,Elliot,4\n'),
("%", "select * from S3Object s where s.age = 10%6 ", b'2,Elliot,4\n'),
]
try:
test_sql_expressions(client, json_testfile, tests, log_output)
except Exception as select_err:
raise select_err
# raise ValueError('Test {} unexpectedly failed with: {}'.format(test_name, select_err))
# pass
# Test passes
print(log_output.json_report())
def test_sql_operators_precedence(client, log_output):
json_testfile = """{"id": 1, "name": "Eric"}"""
tests = [
("-_1", "select -3*3 from S3Object", b'-9\n'),
("*", "select 10-3*2 from S3Object", b'4\n'),
("/", "select 13-10/5 from S3Object", b'11\n'),
("%", "select 13-10%5 from S3Object", b'13\n'),
("+", "select 1+1*3 from S3Object", b'4\n'),
("-_2", "select 1-1*3 from S3Object", b'-2\n'),
("=", "select * from S3Object as s where s.id = 13-12", b'1,Eric\n'),
("<>", "select * from S3Object as s where s.id <> 1-1", b'1,Eric\n'),
("NOT", "select * from S3Object where false OR NOT false", b'1,Eric\n'),
("AND", "select * from S3Object where true AND true OR false ", b'1,Eric\n'),
("OR", "select * from S3Object where false OR NOT false", b'1,Eric\n'),
("IN", "select * from S3Object as s where s.id <> -1 AND s.id IN (1,2,3)", b'1,Eric\n'),
("BETWEEN", "select * from S3Object as s where s.id <> -1 AND s.id BETWEEN -1 AND 3", b'1,Eric\n'),
("LIKE", "select * from S3Object as s where s.id <> -1 AND s.name LIKE 'E%'", b'1,Eric\n'),
]
try:
test_sql_expressions(client, json_testfile, tests, log_output)
except Exception as select_err:
raise select_err
# raise ValueError('Test {} unexpectedly failed with: {}'.format(test_name, select_err))
# pass
# Test passes
print(log_output.json_report())
def test_sql_functions_agg_cond_conv(client, log_output):
json_testfile = """{"id": 1, "name": "John", "age": 3}
{"id": 2, "name": "Elliot", "age": 4}
{"id": 3, "name": "Yves", "age": 5}
{"id": 4, "name": "Christine", "age": null}
{"id": 5, "name": "Eric", "age": 0}
"""
tests = [
# Aggregate functions
("COUNT", "select count(*) from S3Object s", b'5\n'),
("AVG", "select avg(s.age) from S3Object s", b'3\n'),
("MAX", "select max(s.age) from S3Object s", b'5\n'),
("MIN", "select min(s.age) from S3Object s", b'0\n'),
("SUM", "select sum(s.age) from S3Object s", b'12\n'),
# Conditional functions
("COALESCE", "SELECT COALESCE(s.age, 99) FROM S3Object s", b'3\n4\n5\n99\n0\n'),
("NULLIF", "SELECT NULLIF(s.age, 0) FROM S3Object s", b'3\n4\n5\n\n\n'),
## Conversion functions
("CAST", "SELECT CAST(s.age AS FLOAT) FROM S3Object s", b'3.0\n4.0\n5.0\n\n0.0\n'),
]
try:
test_sql_expressions(client, json_testfile, tests, log_output)
except Exception as select_err:
raise select_err
# raise ValueError('Test {} unexpectedly failed with: {}'.format(test_name, select_err))
# pass
# Test passes
print(log_output.json_report())
def test_sql_functions_date(client, log_output):
json_testfile = """
{"id": 1, "name": "John", "datez": "2017-01-02T03:04:05.006+07:30"}
"""
tests = [
# DATE_ADD
("DATE_ADD_1", "select DATE_ADD(year, 5, TO_TIMESTAMP(s.datez)) from S3Object as s", b'2022-01-02T03:04:05.006+07:30\n'),
("DATE_ADD_2", "select DATE_ADD(month, 1, TO_TIMESTAMP(s.datez)) from S3Object as s", b'2017-02-02T03:04:05.006+07:30\n'),
("DATE_ADD_3", "select DATE_ADD(day, -1, TO_TIMESTAMP(s.datez)) from S3Object as s", b'2017-01-01T03:04:05.006+07:30\n'),
("DATE_ADD_4", "select DATE_ADD(hour, 1, TO_TIMESTAMP(s.datez)) from S3Object as s", b'2017-01-02T04:04:05.006+07:30\n'),
("DATE_ADD_5", "select DATE_ADD(minute, 5, TO_TIMESTAMP(s.datez)) from S3Object as s", b'2017-01-02T03:09:05.006+07:30\n'),
("DATE_ADD_6", "select DATE_ADD(second, 5, TO_TIMESTAMP(s.datez)) from S3Object as s", b'2017-01-02T03:04:10.006+07:30\n'),
# DATE_DIFF
("DATE_DIFF_1", "select DATE_DIFF(year, TO_TIMESTAMP(s.datez), TO_TIMESTAMP('2011-01-01T')) from S3Object as s", b'-6\n'),
("DATE_DIFF_2", "select DATE_DIFF(month, TO_TIMESTAMP(s.datez), TO_TIMESTAMP('2011T')) from S3Object as s", b'-72\n'),
("DATE_DIFF_3", "select DATE_DIFF(day, TO_TIMESTAMP(s.datez), TO_TIMESTAMP('2010-01-02T')) from S3Object as s", b'-2556\n'),
# EXTRACT
("EXTRACT_1", "select EXTRACT(year FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'2017\n'),
("EXTRACT_2", "select EXTRACT(month FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'1\n'),
("EXTRACT_3", "select EXTRACT(hour FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'3\n'),
("EXTRACT_4", "select EXTRACT(minute FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'4\n'),
("EXTRACT_5", "select EXTRACT(timezone_hour FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'7\n'),
("EXTRACT_6", "select EXTRACT(timezone_minute FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'30\n'),
# TO_STRING
("TO_STRING_1", "select TO_STRING(TO_TIMESTAMP(s.datez), 'MMMM d, y') from S3Object as s", b'"January 2, 2017"\n'),
("TO_STRING_2", "select TO_STRING(TO_TIMESTAMP(s.datez), 'MMM d, yyyy') from S3Object as s", b'"Jan 2, 2017"\n'),
("TO_STRING_3", "select TO_STRING(TO_TIMESTAMP(s.datez), 'M-d-yy') from S3Object as s", b'1-2-17\n'),
("TO_STRING_4", "select TO_STRING(TO_TIMESTAMP(s.datez), 'MM-d-y') from S3Object as s", b'01-2-2017\n'),
("TO_STRING_5", "select TO_STRING(TO_TIMESTAMP(s.datez), 'MMMM d, y h:m a') from S3Object as s", b'"January 2, 2017 3:4 AM"\n'),
("TO_STRING_6", "select TO_STRING(TO_TIMESTAMP(s.datez), 'y-MM-dd''T''H:m:ssX') from S3Object as s", b'2017-01-02T3:4:05+0730\n'),
("TO_STRING_7", "select TO_STRING(TO_TIMESTAMP(s.datez), 'y-MM-dd''T''H:m:ssX') from S3Object as s", b'2017-01-02T3:4:05+0730\n'),
("TO_STRING_8", "select TO_STRING(TO_TIMESTAMP(s.datez), 'y-MM-dd''T''H:m:ssXXXX') from S3Object as s", b'2017-01-02T3:4:05+0730\n'),
("TO_STRING_9", "select TO_STRING(TO_TIMESTAMP(s.datez), 'y-MM-dd''T''H:m:ssXXXXX') from S3Object as s", b'2017-01-02T3:4:05+07:30\n'),
("TO_TIMESTAMP", "select TO_TIMESTAMP(s.datez) from S3Object as s", b'2017-01-02T03:04:05.006+07:30\n'),
("UTCNOW", "select UTCNOW() from S3Object", datetime(1,1,1)),
]
try:
test_sql_expressions(client, json_testfile, tests, log_output)
except Exception as select_err:
raise select_err
# raise ValueError('Test {} unexpectedly failed with: {}'.format(test_name, select_err))
# pass
# Test passes
print(log_output.json_report())
def test_sql_functions_string(client, log_output):
json_testfile = """
{"id": 1, "name": "John"}
{"id": 2, "name": " \tfoobar\t "}
{"id": 3, "name": "1112211foobar22211122"}
"""
tests = [
# CHAR_LENGTH
("CHAR_LENGTH", "select CHAR_LENGTH(s.name) from S3Object as s", b'4\n24\n21\n'),
("CHARACTER_LENGTH", "select CHARACTER_LENGTH(s.name) from S3Object as s", b'4\n24\n21\n'),
# LOWER
("LOWER", "select LOWER(s.name) from S3Object as s where s.id= 1", b'john\n'),
# SUBSTRING
("SUBSTRING_1", "select SUBSTRING(s.name FROM 2) from S3Object as s where s.id = 1", b'ohn\n'),
("SUBSTRING_2", "select SUBSTRING(s.name FROM 2 FOR 2) from S3Object as s where s.id = 1", b'oh\n'),
("SUBSTRING_3", "select SUBSTRING(s.name FROM -1 FOR 2) from S3Object as s where s.id = 1", b'\n'),
# TRIM
("TRIM_1", "select TRIM(s.name) from S3Object as s where s.id = 2", b'\tfoobar\t\n'),
("TRIM_2", "select TRIM(LEADING FROM s.name) from S3Object as s where s.id = 2", b'\tfoobar\t \n'),
("TRIM_3", "select TRIM(TRAILING FROM s.name) from S3Object as s where s.id = 2", b' \tfoobar\t\n'),
("TRIM_4", "select TRIM(BOTH FROM s.name) from S3Object as s where s.id = 2", b'\tfoobar\t\n'),
("TRIM_5", "select TRIM(BOTH '12' FROM s.name) from S3Object as s where s.id = 3", b'foobar\n'),
# UPPER
("UPPER", "select UPPER(s.name) from S3Object as s where s.id= 1", b'JOHN\n'),
]
try:
test_sql_expressions(client, json_testfile, tests, log_output)
except Exception as select_err:
raise select_err
# raise ValueError('Test {} unexpectedly failed with: {}'.format(test_name, select_err))
# pass
# Test passes
print(log_output.json_report())
def test_sql_datatypes(client, log_output):
json_testfile = """
{"name": "John"}
"""
tests = [
("bool", "select CAST('true' AS BOOL) from S3Object", b'true\n'),
("int", "select CAST('13' AS INT) from S3Object", b'13\n'),
("integer", "select CAST('13' AS INTEGER) from S3Object", b'13\n'),
("string", "select CAST(true AS STRING) from S3Object", b'true\n'),
("float", "select CAST('13.3' AS FLOAT) from S3Object", b'13.3\n'),
("decimal", "select CAST('14.3' AS FLOAT) from S3Object", b'14.3\n'),
("numeric", "select CAST('14.3' AS FLOAT) from S3Object", b'14.3\n'),
("timestamp", "select CAST('2007-04-05T14:30Z' AS TIMESTAMP) from S3Object", b'2007-04-05T14:30Z\n'),
]
try:
test_sql_expressions(client, json_testfile, tests, log_output)
except Exception as select_err:
raise select_err
# raise ValueError('Test {} unexpectedly failed with: {}'.format(test_name, select_err))
# pass
# Test passes
print(log_output.json_report())
def test_sql_select(client, log_output):
json_testfile = """{"id": 1, "created": "June 27", "modified": "July 6" }
{"id": 2, "Created": "June 28", "Modified": "July 7", "Cast": "Random Date" }"""
tests = [
("select_1", "select * from S3Object", b'1,June 27,July 6\n2,June 28,July 7,Random Date\n'),
("select_2", "select * from S3Object s", b'1,June 27,July 6\n2,June 28,July 7,Random Date\n'),
("select_3", "select * from S3Object as s", b'1,June 27,July 6\n2,June 28,July 7,Random Date\n'),
("select_4", "select s.line from S3Object as s", b'\n\n'),
("select_5", 'select s."Created" from S3Object as s', b'\nJune 28\n'),
("select_5", 'select s."Cast" from S3Object as s', b'\nRandom Date\n'),
("where", 'select s.created from S3Object as s', b'June 27\nJune 28\n'),
("limit", 'select * from S3Object as s LIMIT 1', b'1,June 27,July 6\n'),
]
try:
test_sql_expressions(client, json_testfile, tests, log_output)
except Exception as select_err:
raise select_err
# raise ValueError('Test {} unexpectedly failed with: {}'.format(test_name, select_err))
# pass
# Test passes
print(log_output.json_report())
def test_sql_select_json(client, log_output):
json_testcontent = """{ "Rules": [ {"id": "1"}, {"expr": "y > x"}, {"id": "2", "expr": "z = DEBUG"} ]}
{ "created": "June 27", "modified": "July 6" }
"""
tests = [
("select_1", "SELECT id FROM S3Object[*].Rules[*].id", b'{"id":"1"}\n{}\n{"id":"2"}\n{}\n'),
("select_2", "SELECT id FROM S3Object[*].Rules[*].id WHERE id IS NOT MISSING", b'{"id":"1"}\n{"id":"2"}\n'),
("select_3", "SELECT d.created, d.modified FROM S3Object[*] d", b'{}\n{"created":"June 27","modified":"July 6"}\n'),
("select_4", "SELECT _1.created, _1.modified FROM S3Object[*]", b'{}\n{"created":"June 27","modified":"July 6"}\n'),
("select_5", "Select s.rules[1].expr from S3Object s", b'{"expr":"y > x"}\n{}\n'),
]
input_serialization = InputSerialization(json=JSONInput(Type="DOCUMENT"))
output_serialization = OutputSerialization(json=JsonOutput())
try:
test_sql_expressions_custom_input_output(client, json_testcontent,
input_serialization, output_serialization, tests, log_output)
except Exception as select_err:
raise select_err
# raise ValueError('Test {} unexpectedly failed with: {}'.format(test_name, select_err))
# pass
# Test passes
print(log_output.json_report())
def test_sql_select_csv_no_header(client, log_output):
json_testcontent = """val1,val2,val3
val4,val5,val6
"""
tests = [
("select_1", "SELECT s._2 FROM S3Object as s", b'val2\nval5\n'),
]
input_serialization=InputSerialization(
csv=CSVInput(
FileHeaderInfo="NONE",
AllowQuotedRecordDelimiter="FALSE",
),
)
output_serialization=OutputSerialization(csv=CSVOutput())
try:
test_sql_expressions_custom_input_output(client, json_testcontent,
input_serialization, output_serialization, tests, log_output)
except Exception as select_err:
raise select_err
# raise ValueError('Test {} unexpectedly failed with: {}'.format(test_name, select_err))
# pass
# Test passes
print(log_output.json_report())

86
mint/run/core/s3select/tests.py
View file

@ -1,86 +0,0 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# MinIO Python Library for Amazon S3 Compatible Cloud Storage,
# (C) 2015-2020 MinIO, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
from sys import exit
from minio import Minio
from utils import LogOutput
from sql_ops import *
from csv import *
def main():
"""
Functional testing for S3 select.
"""
try:
access_key = os.getenv('ACCESS_KEY', 'Q3AM3UQ867SPQQA43P2F')
secret_key = os.getenv('SECRET_KEY',
'zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG')
server_endpoint = os.getenv('SERVER_ENDPOINT', 'play.min.io')
secure = os.getenv('ENABLE_HTTPS', '1') == '1'
if server_endpoint == 'play.min.io':
access_key = 'Q3AM3UQ867SPQQA43P2F'
secret_key = 'zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG'
secure = True
client = Minio(server_endpoint, access_key, secret_key, secure=False)
log_output = LogOutput(client.select_object_content, 'test_csv_input_quote_char')
test_csv_input_custom_quote_char(client, log_output)
log_output = LogOutput(client.select_object_content, 'test_csv_output_quote_char')
test_csv_output_custom_quote_char(client, log_output)
log_output = LogOutput(client.select_object_content, 'test_sql_operators')
test_sql_operators(client, log_output)
log_output = LogOutput(client.select_object_content, 'test_sql_operators_precedence')
test_sql_operators_precedence(client, log_output)
log_output = LogOutput(client.select_object_content, 'test_sql_functions_agg_cond_conv')
test_sql_functions_agg_cond_conv(client, log_output)
log_output = LogOutput(client.select_object_content, 'test_sql_functions_date')
test_sql_functions_date(client, log_output)
log_output = LogOutput(client.select_object_content, 'test_sql_functions_string')
test_sql_functions_string(client, log_output)
log_output = LogOutput(client.select_object_content, 'test_sql_datatypes')
test_sql_datatypes(client, log_output)
log_output = LogOutput(client.select_object_content, 'test_sql_select')
test_sql_select(client, log_output)
log_output = LogOutput(client.select_object_content, 'test_sql_select_json')
test_sql_select_json(client, log_output)
log_output = LogOutput(client.select_object_content, 'test_sql_select_csv')
test_sql_select_csv_no_header(client, log_output)
except Exception as err:
print(log_output.json_report(err))
exit(1)
if __name__ == "__main__":
# Execute only if run as a script
main()

106
mint/run/core/s3select/utils.py
View file

@ -1,106 +0,0 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# MinIO Python Library for Amazon S3 Compatible Cloud Storage,
# (C) 2015-2020 MinIO, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import uuid
import inspect
import json
import time
import traceback
class LogOutput(object):
"""
LogOutput is the class for log output. It is required standard for all
SDK tests controlled by mint.
Here are its attributes:
'name': name of the SDK under test, e.g. 's3select'
'function': name of the method/api under test with its signature
The following python code can be used to
pull args information of a <method> and to
put together with the method name:
<method>.__name__+'('+', '.join(args_list)+')'
e.g. 'remove_object(bucket_name, object_name)'
'args': method/api arguments with their values, in
dictionary form: {'arg1': val1, 'arg2': val2, ...}
'duration': duration of the whole test in milliseconds,
defaults to 0
'alert': any extra information user is needed to be alerted about,
like whether this is a Blocker/Gateway/Server related
issue, etc., defaults to None
'message': descriptive error message, defaults to None
'error': stack-trace/exception message(only in case of failure),
actual low level exception/error thrown by the program,
defaults to None
'status': exit status, possible values are 'PASS', 'FAIL', 'NA',
defaults to 'PASS'
"""
PASS = 'PASS'
FAIL = 'FAIL'
NA = 'NA'
def __init__(self, meth, test_name):
self.__args_list = inspect.getargspec(meth).args[1:]
self.__name = 's3select:'+test_name
self.__function = meth.__name__+'('+', '.join(self.__args_list)+')'
self.__args = {}
self.__duration = 0
self.__alert = ''
self.__message = None
self.__error = None
self.__status = self.PASS
self.__start_time = time.time()
@property
def name(self): return self.__name
@property
def function(self): return self.__function
@property
def args(self): return self.__args
@name.setter
def name(self, val): self.__name = val
@function.setter
def function(self, val): self.__function = val
@args.setter
def args(self, val): self.__args = val
def json_report(self, err_msg='', alert='', status=''):
self.__args = {k: v for k, v in self.__args.items() if v and v != ''}
entry = {'name': self.__name,
'function': self.__function,
'args': self.__args,
'duration': int(round((time.time() - self.__start_time)*1000)),
'alert': str(alert),
'message': str(err_msg),
'error': traceback.format_exc() if err_msg and err_msg != '' else '',
'status': status if status and status != '' else
self.FAIL if err_msg and err_msg != '' else self.PASS
}
return json.dumps({k: v for k, v in entry.items() if v and v != ''})
def generate_bucket_name():
return "s3select-test-" + str(uuid.uuid4())
def generate_object_name():
return str(uuid.uuid4())

28
mint/run/core/security/run.sh
View file

@ -1,28 +0,0 @@
#!/bin/bash
#
# Mint (C) 2018 Minio, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# handle command line arguments
if [ $# -ne 2 ]; then
echo "usage: run.sh <OUTPUT-LOG-FILE> <ERROR-LOG-FILE>"
exit 1
fi
output_log_file="$1"
error_log_file="$2"
# run tests
/mint/run/core/security/tls-tests 1>>"$output_log_file" 2>"$error_log_file"

272
mint/run/core/security/tls-tests.go
View file

@ -1,272 +0,0 @@
// Mint, (C) 2018 Minio, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"crypto/tls"
"encoding/json"
"fmt"
"os"
"time"
log "github.com/sirupsen/logrus"
)
const testName = "TLS-tests"
const (
// PASS indicate that a test passed
PASS = "PASS"
// FAIL indicate that a test failed
FAIL = "FAIL"
// NA indicates that a test is not applicable
NA = "NA"
)
func main() {
log.SetOutput(os.Stdout)
log.SetFormatter(&mintJSONFormatter{})
log.SetLevel(log.InfoLevel)
endpoint := os.Getenv("SERVER_ENDPOINT")
secure := os.Getenv("ENABLE_HTTPS")
if secure != "1" {
log.WithFields(log.Fields{"name:": testName, "status": NA, "message": "TLS is not enabled"}).Info()
return
}
testTLSVersions(endpoint)
testTLSCiphers(endpoint)
testTLSEllipticCurves(endpoint)
}
// Tests whether the endpoint accepts TLS1.0 or TLS1.1 connections - fail if so.
// Tests whether the endpoint accepts TLS1.2 connections - fail if not.
func testTLSVersions(endpoint string) {
const function = "TLSVersions"
startTime := time.Now()
// Tests whether the endpoint accepts TLS1.0 or TLS1.1 connections
args := map[string]interface{}{
"MinVersion": "tls.VersionTLS10",
"MaxVersion": "tls.VersionTLS11",
}
_, err := tls.Dial("tcp", endpoint, &tls.Config{
MinVersion: tls.VersionTLS10,
MaxVersion: tls.VersionTLS11,
})
if err == nil {
failureLog(function, args, startTime, "", "Endpoint accepts insecure connection", err).Error()
return
}
// Tests whether the endpoint accepts TLS1.2 connections
args = map[string]interface{}{
"MinVersion": "tls.VersionTLS12",
}
_, err = tls.Dial("tcp", endpoint, &tls.Config{
MinVersion: tls.VersionTLS12,
})
if err != nil {
failureLog(function, args, startTime, "", "Endpoint rejects secure connection", err).Error()
return
}
successLog(function, args, startTime)
}
// Tests whether the endpoint accepts SSL3.0, TLS1.0 or TLS1.1 connections - fail if so.
// Tests whether the endpoint accepts TLS1.2 connections - fail if not.
func testTLSCiphers(endpoint string) {
const function = "TLSCiphers"
startTime := time.Now()
// Tests whether the endpoint accepts insecure ciphers
args := map[string]interface{}{
"MinVersion": "tls.VersionTLS12",
"CipherSuites": unsupportedCipherSuites,
}
_, err := tls.Dial("tcp", endpoint, &tls.Config{
MinVersion: tls.VersionTLS12,
CipherSuites: unsupportedCipherSuites,
})
if err == nil {
failureLog(function, args, startTime, "", "Endpoint accepts insecure cipher suites", err).Error()
return
}
// Tests whether the endpoint accepts at least one secure cipher
args = map[string]interface{}{
"MinVersion": "tls.VersionTLS12",
"CipherSuites": supportedCipherSuites,
}
_, err = tls.Dial("tcp", endpoint, &tls.Config{
MinVersion: tls.VersionTLS12,
CipherSuites: supportedCipherSuites,
})
if err != nil {
failureLog(function, args, startTime, "", "Endpoint rejects all secure cipher suites", err).Error()
return
}
// Tests whether the endpoint accepts at least one default cipher
args = map[string]interface{}{
"MinVersion": "tls.VersionTLS12",
"CipherSuites": nil,
}
_, err = tls.Dial("tcp", endpoint, &tls.Config{
MinVersion: tls.VersionTLS12,
CipherSuites: nil, // default value
})
if err != nil {
failureLog(function, args, startTime, "", "Endpoint rejects default cipher suites", err).Error()
return
}
successLog(function, args, startTime)
}
// Tests whether the endpoint accepts the P-384 or P-521 elliptic curve - fail if so.
// Tests whether the endpoint accepts Curve25519 or P-256 - fail if not.
func testTLSEllipticCurves(endpoint string) {
const function = "TLSEllipticCurves"
startTime := time.Now()
// Tests whether the endpoint accepts curves using non-constant time implementations.
args := map[string]interface{}{
"CurvePreferences": unsupportedCurves,
}
_, err := tls.Dial("tcp", endpoint, &tls.Config{
MinVersion: tls.VersionTLS12,
CurvePreferences: unsupportedCurves,
CipherSuites: supportedCipherSuites,
})
if err == nil {
failureLog(function, args, startTime, "", "Endpoint accepts insecure elliptic curves", err).Error()
return
}
// Tests whether the endpoint accepts curves using constant time implementations.
args = map[string]interface{}{
"CurvePreferences": unsupportedCurves,
}
_, err = tls.Dial("tcp", endpoint, &tls.Config{
MinVersion: tls.VersionTLS12,
CurvePreferences: supportedCurves,
CipherSuites: supportedCipherSuites,
})
if err != nil {
failureLog(function, args, startTime, "", "Endpoint does not accept secure elliptic curves", err).Error()
return
}
successLog(function, args, startTime)
}
func successLog(function string, args map[string]interface{}, startTime time.Time) *log.Entry {
duration := time.Since(startTime).Nanoseconds() / 1000000
return log.WithFields(log.Fields{
"name": testName,
"function": function,
"args": args,
"duration": duration,
"status": PASS,
})
}
func failureLog(function string, args map[string]interface{}, startTime time.Time, alert string, message string, err error) *log.Entry {
duration := time.Since(startTime).Nanoseconds() / 1000000
fields := log.Fields{
"name": testName,
"function": function,
"args": args,
"duration": duration,
"status": FAIL,
"alert": alert,
"message": message,
}
if err != nil {
fields["error"] = err
}
return log.WithFields(fields)
}
type mintJSONFormatter struct {
}
func (f *mintJSONFormatter) Format(entry *log.Entry) ([]byte, error) {
data := make(log.Fields, len(entry.Data))
for k, v := range entry.Data {
switch v := v.(type) {
case error:
// Otherwise errors are ignored by `encoding/json`
// https://github.com/sirupsen/logrus/issues/137
data[k] = v.Error()
default:
data[k] = v
}
}
serialized, err := json.Marshal(data)
if err != nil {
return nil, fmt.Errorf("Failed to marshal fields to JSON, %w", err)
}
return append(serialized, '\n'), nil
}
// Secure Go implementations of modern TLS ciphers
// The following ciphers are excluded because:
// - RC4 ciphers: RC4 is broken
// - 3DES ciphers: Because of the 64 bit blocksize of DES (Sweet32)
// - CBC-SHA256 ciphers: No countermeasures against Lucky13 timing attack
// - CBC-SHA ciphers: Legacy ciphers (SHA-1) and non-constant time
// implementation of CBC.
// (CBC-SHA ciphers can be enabled again if required)
// - RSA key exchange ciphers: Disabled because of dangerous PKCS1-v1.5 RSA
// padding scheme. See Bleichenbacher attacks.
var supportedCipherSuites = []uint16{
tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,
tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,
tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
}
// Supported elliptic curves: Implementations are constant-time.
var supportedCurves = []tls.CurveID{tls.X25519, tls.CurveP256}
var unsupportedCipherSuites = []uint16{
tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA, // Go stack contains (some) countermeasures against timing attacks (Lucky13)
tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256, // No countermeasures against timing attacks
tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA, // Go stack contains (some) countermeasures against timing attacks (Lucky13)
tls.TLS_ECDHE_ECDSA_WITH_RC4_128_SHA, // Broken cipher
tls.TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA, // Sweet32
tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, // Go stack contains (some) countermeasures against timing attacks (Lucky13)
tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256, // No countermeasures against timing attacks
tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, // Go stack contains (some) countermeasures against timing attacks (Lucky13)
tls.TLS_ECDHE_RSA_WITH_RC4_128_SHA, // Broken cipher
// all RSA-PKCS1-v1.5 ciphers are disabled - danger of Bleichenbacher attack variants
tls.TLS_RSA_WITH_3DES_EDE_CBC_SHA, // Sweet32
tls.TLS_RSA_WITH_AES_128_CBC_SHA, // Go stack contains (some) countermeasures against timing attacks (Lucky13)
tls.TLS_RSA_WITH_AES_128_CBC_SHA256, // No countermeasures against timing attacks
tls.TLS_RSA_WITH_AES_256_CBC_SHA, // Go stack contains (some) countermeasures against timing attacks (Lucky13)
tls.TLS_RSA_WITH_RC4_128_SHA, // Broken cipher
tls.TLS_RSA_WITH_AES_128_GCM_SHA256, // Disabled because of RSA-PKCS1-v1.5 - AES-GCM is considered secure.
tls.TLS_RSA_WITH_AES_256_GCM_SHA384, // Disabled because of RSA-PKCS1-v1.5 - AES-GCM is considered secure.
}
// Unsupported elliptic curves: Implementations are not constant-time.
var unsupportedCurves = []tls.CurveID{tls.CurveP384, tls.CurveP521}

6
misc/build.go Normal file
View file

@ -0,0 +1,6 @@
package misc
var (
Build = "now"
Version = "dev"
)