From 8af01b89a151d17fab6a8ee700081a921948b9f2 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Thu, 13 May 2021 22:06:02 +0300 Subject: [PATCH 1/9] Makefile: rework based on neofs-http-gate Non-Docker builds by default, no vendoring, more useful targets. --- Makefile | 129 +++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 96 insertions(+), 33 deletions(-) diff --git a/Makefile b/Makefile index ded7221d1..5310c17b1 100644 --- a/Makefile +++ b/Makefile @@ -1,43 +1,106 @@ --include .env --include help.mk - -HUB_IMAGE=nspccdev/neofs +#!/usr/bin/make -f +REPO ?= $(shell go list -m) VERSION ?= "$(shell git describe --tags 2>/dev/null || git rev-parse --short HEAD | sed 's/^v//')" -BUILD_VERSION ?= "$(shell git describe --abbrev=0 --tags | sed 's/^v//')" -.PHONY: format deps image publish +BIN_NAME=neofs-s3-gw +HUB_IMAGE="nspccdev/$(BIN_NAME)" +HUB_TAG ?= "$(shell echo ${VERSION} | sed 's/^v//')" +BINDIR = bin +BIN = "$(BINDIR)/$(BIN_NAME)" -# Show current version -version: - @echo $(BUILD_VERSION) +.PHONY: help all dep clean format test cover lint docker/lint image-push image dirty-image + +# Make all binaries +all: $(BIN) + +$(BIN): $(BINDIR) dep + @echo "⇒ Build $@" + CGO_ENABLED=0 \ + go build -v -trimpath \ + -ldflags "-X main.Version=$(VERSION)" \ + -o $@ ./cmd/gate + +$(BINDIR): + @echo "⇒ Ensure dir: $@" + @mkdir -p $@ + +# Pull go dependencies +dep: + @printf "⇒ Download requirements: " + @CGO_ENABLED=0 \ + go mod download && echo OK + @printf "⇒ Tidy requirements: " + @CGO_ENABLED=0 \ + go mod tidy -v && echo OK + +# Run tests +test: + @go test ./... -cover + +# Run tests with race detection and produce coverage output +cover: + @go test -v -race ./... -coverprofile=coverage.txt -covermode=atomic + @go tool cover -html=coverage.txt -o coverage.html # Reformat code format: - @[ ! -z `which goimports` ] || (echo "install goimports" && exit 2) - @for f in `find . -type f -name '*.go' -not -path './vendor/*' -not -name '*.pb.go' -prune`; do \ - echo "⇒ Processing $$f"; \ - goimports -w $$f; \ - done + @echo "⇒ Processing gofmt check" + @gofmt -s -w ./ + @echo "⇒ Processing goimports check" + @goimports -w ./ -# Check and ensure dependencies -deps: - @printf "⇒ Ensure vendor: " - @go mod tidy -v && echo OK || (echo fail && exit 2) - @printf "⇒ Download requirements: " - @go mod download && echo OK || (echo fail && exit 2) - @printf "⇒ Store vendor localy: " - @go mod vendor && echo OK || (echo fail && exit 2) - -# Build current docker image -image: deps - @echo "⇒ Build docker-image" +# Build clean Docker image +image: + @echo "⇒ Build NeoFS S3 Gateway docker image " @docker build \ - --build-arg VERSION=$(BUILD_VERSION) \ - -f Dockerfile \ - -t $(HUB_IMAGE)-s3-gate:$(BUILD_VERSION) . + --build-arg REPO=$(REPO) \ + --build-arg VERSION=$(VERSION) \ + --rm \ + -f Dockerfile \ + -t $(HUB_IMAGE):$(HUB_TAG) . -# Publish docker image -publish: - @echo "${B}${G}⇒ publish docker image ${R}" - @docker push $(HUB_IMAGE)-s3-gate:$(VERSION) \ No newline at end of file +# Push Docker image to the hub +image-push: + @echo "⇒ Publish image" + @docker push $(HUB_IMAGE):$(HUB_TAG) + +# Build dirty Docker image +dirty-image: + @echo "⇒ Build NeoFS S3 Gateway dirty docker image " + @docker build \ + --build-arg REPO=$(REPO) \ + --build-arg VERSION=$(VERSION) \ + --rm \ + -f Dockerfile.dirty \ + -t $(HUB_IMAGE)-dirty:$(HUB_TAG) . + +# Run linters +lint: + @golangci-lint --timeout=5m run + +# Run linters in Docker +docker/lint: + docker run --rm -it \ + -v `pwd`:/src \ + -u `stat -c "%u:%g" .` \ + --env HOME=/src \ + golangci/golangci-lint:v1.40 bash -c 'cd /src/ && make lint' + +# Show current version +version: + @echo $(VERSION) + +# Show this help prompt +help: + @echo ' Usage:' + @echo '' + @echo ' make ' + @echo '' + @echo ' Targets:' + @echo '' + @awk '/^#/{ comment = substr($$0,3) } comment && /^[a-zA-Z][a-zA-Z0-9_-]+ ?:/{ print " ", $$1, comment }' $(MAKEFILE_LIST) | column -t -s ':' | grep -v 'IGNORE' | sort -u + +# Clean up +clean: + rm -rf $(BINDIR) From 4577077981ad0d26201c69e0f4464bd15669b1fb Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Thu, 13 May 2021 22:08:20 +0300 Subject: [PATCH 2/9] gitignore: more ignores --- .gitignore | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.gitignore b/.gitignore index e2f53be22..201350129 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,11 @@ vendor # tempfiles .DS_Store +*~ + +# binary +bin/ + +# coverage +coverage.txt +coverage.html From e15159443b28667233796ccd8960842a8bc8bad0 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Thu, 13 May 2021 22:08:33 +0300 Subject: [PATCH 3/9] Dockerfile: reuse make, drop upx upx is nice, but not necessary for this and duplicating Makefile is just not good. Note that in absence of vendoring building this can be problematic at the moment because of private repository dependency, it'll be solved in future. --- Dockerfile | 24 ++++-------------------- 1 file changed, 4 insertions(+), 20 deletions(-) diff --git a/Dockerfile b/Dockerfile index 94847f35e..1f8d308e7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,36 +4,20 @@ WORKDIR /src RUN set -x \ && apt update \ - && apt install -y upx-ucl + && apt install -y make COPY . /src ARG VERSION=dev -# https://github.com/golang/go/wiki/Modules#how-do-i-use-vendoring-with-modules-is-vendoring-going-away -# go build -mod=vendor -# The -gcflags "all=-N -l" flag helps us get a better debug experience -RUN set -x \ - && export BUILD=$(date -u +%s%N) \ - && export REPO=$(go list -m) \ - && export LDFLAGS="-X ${REPO}/misc.Version=${VERSION} -X ${REPO}/misc.Build=${BUILD}" \ - && export GOGC=off \ - && export CGO_ENABLED=0 \ - && [ -d "./vendor" ] || go mod vendor \ - && go build \ - -v \ - -mod=vendor \ - -trimpath \ - -ldflags "${LDFLAGS}" \ - -o /go/bin/neofs-s3 ./cmd/gate \ - && upx -3 /go/bin/neofs-s3 +RUN make # Executable image FROM scratch WORKDIR / -COPY --from=builder /go/bin/neofs-s3 /bin/neofs-s3 +COPY --from=builder /src/bin/neofs-s3-gw /bin/neofs-s3-gw COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ -ENTRYPOINT ["/bin/neofs-s3"] +ENTRYPOINT ["/bin/neofs-s3-gw"] From 7ce5b3392f3471b8c2ce15f880b4e4e92035f12e Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Thu, 13 May 2021 22:11:55 +0300 Subject: [PATCH 4/9] cmd/gate: fix gosimple suggestion cmd/gate/app-settings.go:108:36 gosimple S1019: should use make(map[string]float64) instead --- cmd/gate/app-settings.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/gate/app-settings.go b/cmd/gate/app-settings.go index 2024fd59f..1bc642fd2 100644 --- a/cmd/gate/app-settings.go +++ b/cmd/gate/app-settings.go @@ -105,7 +105,7 @@ var ignore = map[string]struct{}{ func (empty) Read([]byte) (int, error) { return 0, io.EOF } func fetchPeers(l *zap.Logger, v *viper.Viper) map[string]float64 { - peers := make(map[string]float64, 0) + peers := make(map[string]float64) for i := 0; ; i++ { From b851889934d8f21fa81e6c81e766ebf91119751c Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Thu, 13 May 2021 22:13:09 +0300 Subject: [PATCH 5/9] *: fix whitespace errors --- api/handler/response.go | 1 - api/user-auth.go | 1 - cmd/gate/app-settings.go | 1 - 3 files changed, 3 deletions(-) diff --git a/api/handler/response.go b/api/handler/response.go index 13610c6da..c44e71b14 100644 --- a/api/handler/response.go +++ b/api/handler/response.go @@ -127,7 +127,6 @@ type CopyObjectResponse struct { // MarshalXML - StringMap marshals into XML. func (s StringMap) MarshalXML(e *xml.Encoder, start xml.StartElement) error { - tokens := []xml.Token{start} for key, value := range s { diff --git a/api/user-auth.go b/api/user-auth.go index 5c514ef3f..ceaa6b9eb 100644 --- a/api/user-auth.go +++ b/api/user-auth.go @@ -21,7 +21,6 @@ func AttachUserAuth(router *mux.Router, center auth.Center, log *zap.Logger) { h.ServeHTTP(w, r.WithContext( sdk.SetBearerToken(r.Context(), token))) - }) }) } diff --git a/cmd/gate/app-settings.go b/cmd/gate/app-settings.go index 1bc642fd2..6662fb866 100644 --- a/cmd/gate/app-settings.go +++ b/cmd/gate/app-settings.go @@ -108,7 +108,6 @@ func fetchPeers(l *zap.Logger, v *viper.Viper) map[string]float64 { peers := make(map[string]float64) for i := 0; ; i++ { - key := cfgPeers + "." + strconv.Itoa(i) + "." address := v.GetString(key + "address") weight := v.GetFloat64(key + "weight") From a0ecb8ff5240fb99753f73d37712db06e3d6c8d5 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Thu, 13 May 2021 22:15:21 +0300 Subject: [PATCH 6/9] handler: suppress exhaustive linter --- api/handler/head.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/handler/head.go b/api/handler/head.go index fdbb00bdd..c805ad97b 100644 --- a/api/handler/head.go +++ b/api/handler/head.go @@ -109,7 +109,7 @@ func (h *handler) HeadBucketHandler(w http.ResponseWriter, r *http.Request) { code := http.StatusBadRequest if st, ok := status.FromError(err); ok && st != nil { - switch st.Code() { + switch st.Code() { //nolint:exhaustive // we have default value set above case codes.NotFound: code = http.StatusNotFound case codes.PermissionDenied: From 70a70bfa2ca8426d6cfd8d26ae79970c1880b583 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Thu, 13 May 2021 22:25:32 +0300 Subject: [PATCH 7/9] *: fix all godot errors --- api/errors.go | 34 +++++++++++++++++----------------- api/handler/delete.go | 2 +- api/handler/response.go | 12 ++++++------ api/headers.go | 2 +- api/layer/layer.go | 2 +- api/metrics/api.go | 10 +++++----- api/reqinfo.go | 14 +++++++------- api/response.go | 12 ++++++------ cmd/gate/app-settings.go | 24 ++++++++++++------------ 9 files changed, 56 insertions(+), 56 deletions(-) diff --git a/api/errors.go b/api/errors.go index a0e6e39b7..6fb52b4ef 100644 --- a/api/errors.go +++ b/api/errors.go @@ -118,7 +118,7 @@ const ( ErrInvalidTagDirective // Add new error codes here. - // SSE-S3 related API errors + // SSE-S3 related API errors. ErrInvalidEncryptionMethod // Server-Side-Encryption (with Customer provided key) related API errors. @@ -157,8 +157,8 @@ const ( // Add new extended error codes here. // MinIO extended errors. - // ErrReadQuorum - // ErrWriteQuorum + // ErrReadQuorum + // ErrWriteQuorum ErrParentIsObject ErrStorageFull ErrRequestBodyParse @@ -170,7 +170,7 @@ const ( ErrOperationTimedOut ErrOperationMaxedOut ErrInvalidRequest - // MinIO storage class error codes + // MinIO storage class error codes. ErrInvalidStorageClass ErrBackendDown // Add new extended error codes here. @@ -192,7 +192,7 @@ const ( ErrAdminCredentialsMismatch ErrInsecureClientRequest ErrObjectTampered - // Bucket Quota error codes + // Bucket Quota error codes. ErrAdminBucketQuotaExceeded ErrAdminNoSuchQuotaConfiguration ErrAdminBucketQuotaDisabled @@ -205,7 +205,7 @@ const ( ErrHealOverlappingPaths ErrIncorrectContinuationToken - // S3 Select Errors + // S3 Select Errors. ErrEmptyRequestBody ErrUnsupportedFunction ErrInvalidExpressionType @@ -1625,7 +1625,7 @@ func GetAPIError(code ErrorCode) Error { } // getErrorResponse gets in standard error and resource value and -// provides a encodable populated response values +// provides a encodable populated response values. func getAPIErrorResponse(ctx context.Context, err error, resource, requestID, hostID string) ErrorResponse { code := "BadRequest" desc := err.Error() @@ -1803,21 +1803,21 @@ func (e BucketLifecycleNotFound) Error() string { return "No bucket lifecycle configuration found for bucket : " + e.Bucket } -// BucketSSEConfigNotFound - no bucket encryption found +// BucketSSEConfigNotFound - no bucket encryption found. type BucketSSEConfigNotFound GenericError func (e BucketSSEConfigNotFound) Error() string { return "No bucket encryption configuration found for bucket: " + e.Bucket } -// BucketTaggingNotFound - no bucket tags found +// BucketTaggingNotFound - no bucket tags found. type BucketTaggingNotFound GenericError func (e BucketTaggingNotFound) Error() string { return "No bucket tags found for bucket: " + e.Bucket } -// BucketObjectLockConfigNotFound - no bucket object lock config found +// BucketObjectLockConfigNotFound - no bucket object lock config found. type BucketObjectLockConfigNotFound GenericError func (e BucketObjectLockConfigNotFound) Error() string { @@ -1874,7 +1874,7 @@ func (e ObjectNamePrefixAsSlash) Error() string { return "Object name contains forward slash as pefix: " + e.Bucket + "#" + e.Object } -// AllAccessDisabled All access to this object has been disabled +// AllAccessDisabled All access to this object has been disabled. type AllAccessDisabled GenericError // Error returns string an error formatted as the given text. @@ -1945,7 +1945,7 @@ func (e InvalidUploadID) Error() string { return "Invalid upload id " + e.UploadID } -// InvalidPart One or more of the specified parts could not be found +// InvalidPart One or more of the specified parts could not be found. type InvalidPart struct { PartNumber int ExpETag string @@ -1975,21 +1975,21 @@ func (e PartTooBig) Error() string { return "Part size bigger than the allowed limit" } -// InvalidETag error returned when the etag has changed on disk +// InvalidETag error returned when the etag has changed on disk. type InvalidETag struct{} func (e InvalidETag) Error() string { return "etag of the object has changed" } -// NotImplemented If a feature is not implemented +// NotImplemented If a feature is not implemented. type NotImplemented struct{} func (e NotImplemented) Error() string { return "Not Implemented" } -// UnsupportedMetadata - unsupported metadata +// UnsupportedMetadata - unsupported metadata. type UnsupportedMetadata struct{} func (e UnsupportedMetadata) Error() string { @@ -2003,14 +2003,14 @@ func (e BackendDown) Error() string { return "Backend down" } -// PreConditionFailed - Check if copy precondition failed +// PreConditionFailed - Check if copy precondition failed. type PreConditionFailed struct{} func (e PreConditionFailed) Error() string { return "At least one of the pre-conditions you specified did not hold" } -// DeleteError - returns when cant remove object +// DeleteError - returns when cant remove object. type DeleteError struct { Err error Object string diff --git a/api/handler/delete.go b/api/handler/delete.go index e99d47315..72e2da228 100644 --- a/api/handler/delete.go +++ b/api/handler/delete.go @@ -68,7 +68,7 @@ func (h *handler) DeleteObjectHandler(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusNoContent) } -// DeleteMultipleObjectsHandler : +// DeleteMultipleObjectsHandler handles multiple delete requests. func (h *handler) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Request) { var ( req = mux.Vars(r) diff --git a/api/handler/response.go b/api/handler/response.go index c44e71b14..cccdd23d3 100644 --- a/api/handler/response.go +++ b/api/handler/response.go @@ -2,7 +2,7 @@ package handler import "encoding/xml" -// ListBucketsResponse - format for list buckets response +// ListBucketsResponse - format for list buckets response. type ListBucketsResponse struct { XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListAllMyBucketsResult" json:"-"` @@ -45,13 +45,13 @@ type ListObjectsV2Response struct { EncodingType string `xml:"EncodingType,omitempty"` } -// Bucket container for bucket metadata +// Bucket container for bucket metadata. type Bucket struct { Name string CreationDate string // time string of format "2006-01-02T15:04:05.000Z" } -// Owner - bucket owner/principal +// Owner - bucket owner/principal. type Owner struct { ID string DisplayName string @@ -87,12 +87,12 @@ type ListObjectsResponse struct { EncodingType string `xml:"EncodingType,omitempty"` } -// CommonPrefix container for prefix response in ListObjectsResponse +// CommonPrefix container for prefix response in ListObjectsResponse. type CommonPrefix struct { Prefix string } -// Object container for object metadata +// Object container for object metadata. type Object struct { Key string LastModified string // time string of format "2006-01-02T15:04:05.000Z" @@ -118,7 +118,7 @@ type LocationResponse struct { Location string `xml:",chardata"` } -// CopyObjectResponse container returns ETag and LastModified of the successfully copied object +// CopyObjectResponse container returns ETag and LastModified of the successfully copied object. type CopyObjectResponse struct { XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CopyObjectResult" json:"-"` LastModified string // time string of format "2006-01-02T15:04:05.000Z" diff --git a/api/headers.go b/api/headers.go index 225e24f28..085e3f773 100644 --- a/api/headers.go +++ b/api/headers.go @@ -1,6 +1,6 @@ package api -// Standard S3 HTTP response constants +// Standard S3 HTTP response constants. const ( LastModified = "Last-Modified" Date = "Date" diff --git a/api/layer/layer.go b/api/layer/layer.go index 2f88eae22..927c02797 100644 --- a/api/layer/layer.go +++ b/api/layer/layer.go @@ -103,7 +103,7 @@ func (n *layer) Owner(ctx context.Context) *owner.ID { return n.cli.Owner() } -// Get NeoFS Object by refs.Address (should be used by auth.Center) +// Get NeoFS Object by refs.Address (should be used by auth.Center). func (n *layer) Get(ctx context.Context, address *object.Address) (*object.Object, error) { return n.cli.Object().Get(ctx, address) } diff --git a/api/metrics/api.go b/api/metrics/api.go index d72297a6b..56b406389 100644 --- a/api/metrics/api.go +++ b/api/metrics/api.go @@ -20,7 +20,7 @@ type ( } // HTTPStats holds statistics information about - // HTTP requests made by all clients + // HTTP requests made by all clients. HTTPStats struct { currentS3Requests HTTPAPIStats totalS3Requests HTTPAPIStats @@ -63,8 +63,8 @@ var ( ) ) -// collects http metrics for NeoFS S3 Gate in Prometheus specific format -// and sends to given channel +// Collects HTTP metrics for NeoFS S3 Gate in Prometheus specific format +// and sends to given channel. func collectHTTPMetrics(ch chan<- prometheus.Metric) { for api, value := range httpStatsMetric.currentS3Requests.Load() { ch <- prometheus.MustNewConstMetric( @@ -176,7 +176,7 @@ func (st *HTTPStats) getOutputBytes() uint64 { return atomic.LoadUint64(&st.totalOutputBytes) } -// Update statistics from http request and response data +// Update statistics from http request and response data. func (st *HTTPStats) updateStats(api string, w http.ResponseWriter, r *http.Request, durationSecs float64) { var code int @@ -200,7 +200,7 @@ func (st *HTTPStats) updateStats(api string, w http.ResponseWriter, r *http.Requ } } -// WriteHeader - writes http status code +// WriteHeader - writes http status code. func (w *responseWrapper) WriteHeader(code int) { w.Do(func() { w.statusCode = code diff --git a/api/reqinfo.go b/api/reqinfo.go index 606ada4e5..374a13083 100644 --- a/api/reqinfo.go +++ b/api/reqinfo.go @@ -13,7 +13,7 @@ import ( ) type ( - // KeyVal - appended to ReqInfo.Tags + // KeyVal - appended to ReqInfo.Tags. KeyVal struct { Key string Val string @@ -40,7 +40,7 @@ type ( } ) -// Key used for Get/SetReqInfo +// Key used for Get/SetReqInfo. type contextKeyType string const ctxRequestInfo = contextKeyType("NeoFS-S3-Gate") @@ -54,7 +54,7 @@ var ( var ( // RFC7239 defines a new "Forwarded: " header designed to replace the // existing use of X-Forwarded-* headers. - // e.g. Forwarded: for=192.0.2.60;proto=https;by=203.0.113.43 + // e.g. Forwarded: for=192.0.2.60;proto=https;by=203.0.113.43. forwarded = http.CanonicalHeaderKey("Forwarded") // Allows for a sub-match of the first value after 'for=' to the next // comma, semi-colon or space. The match is case-insensitive. @@ -125,7 +125,7 @@ func prepareContext(w http.ResponseWriter, r *http.Request) context.Context { })) } -// NewReqInfo : +// NewReqInfo returns new ReqInfo based on parameters. func NewReqInfo(w http.ResponseWriter, r *http.Request, req ObjectRequest) *ReqInfo { return &ReqInfo{ API: req.Method, @@ -138,7 +138,7 @@ func NewReqInfo(w http.ResponseWriter, r *http.Request, req ObjectRequest) *ReqI } } -// AppendTags - appends key/val to ReqInfo.tags +// AppendTags - appends key/val to ReqInfo.tags. func (r *ReqInfo) AppendTags(key string, val string) *ReqInfo { if r == nil { return nil @@ -149,7 +149,7 @@ func (r *ReqInfo) AppendTags(key string, val string) *ReqInfo { return r } -// SetTags - sets key/val to ReqInfo.tags +// SetTags - sets key/val to ReqInfo.tags. func (r *ReqInfo) SetTags(key string, val string) *ReqInfo { if r == nil { return nil @@ -172,7 +172,7 @@ func (r *ReqInfo) SetTags(key string, val string) *ReqInfo { return r } -// GetTags - returns the user defined tags +// GetTags - returns the user defined tags. func (r *ReqInfo) GetTags() []KeyVal { if r == nil { return nil diff --git a/api/response.go b/api/response.go index 6640d4cb2..872eeac1f 100644 --- a/api/response.go +++ b/api/response.go @@ -14,7 +14,7 @@ import ( ) type ( - // ErrorResponse - error response format + // ErrorResponse - error response format. ErrorResponse struct { XMLName xml.Name `xml:"Error" json:"-"` Code string @@ -32,11 +32,11 @@ type ( // Captures the server string returned in response header. Server string `xml:"-" json:"-"` - // Underlying HTTP status code for the returned error + // Underlying HTTP status code for the returned error. StatusCode int `xml:"-" json:"-"` } - // APIError structure + // APIError structure. Error struct { Code string Description string @@ -117,7 +117,7 @@ var s3ErrorResponseMap = map[string]string{ // Add new API errors here. } -// WriteErrorResponse writes error headers +// WriteErrorResponse writes error headers. func WriteErrorResponse(ctx context.Context, w http.ResponseWriter, err error, reqURL *url.URL) { code := http.StatusBadRequest @@ -141,7 +141,7 @@ func WriteErrorResponse(ctx context.Context, w http.ResponseWriter, err error, r WriteResponse(w, code, encodedErrorResponse, MimeXML) } -// If none of the http routes match respond with appropriate errors +// If none of the http routes match respond with appropriate errors. func errorResponseHandler(w http.ResponseWriter, r *http.Request) { desc := fmt.Sprintf("Unknown API request at %s", r.URL.Path) WriteErrorResponse(r.Context(), w, Error{ @@ -151,7 +151,7 @@ func errorResponseHandler(w http.ResponseWriter, r *http.Request) { }, r.URL) } -// Write http common headers +// Write http common headers. func setCommonHeaders(w http.ResponseWriter) { w.Header().Set(hdrServerInfo, "NeoFS-S3-Gate/"+misc.Version) w.Header().Set(hdrAcceptRanges, "bytes") diff --git a/cmd/gate/app-settings.go b/cmd/gate/app-settings.go index 6662fb866..5c9b705ed 100644 --- a/cmd/gate/app-settings.go +++ b/cmd/gate/app-settings.go @@ -34,8 +34,8 @@ const ( defaultMaxClientsDeadline = time.Second * 30 ) -const ( // settings - // Logger: +const ( // Settings. + // Logger. cfgLoggerLevel = "logger.level" cfgLoggerFormat = "logger.format" cfgLoggerTraceLevel = "logger.trace_level" @@ -44,47 +44,47 @@ const ( // settings cfgLoggerSamplingInitial = "logger.sampling.initial" cfgLoggerSamplingThereafter = "logger.sampling.thereafter" - // KeepAlive + // KeepAlive. cfgKeepaliveTime = "keepalive.time" cfgKeepaliveTimeout = "keepalive.timeout" cfgKeepalivePermitWithoutStream = "keepalive.permit_without_stream" - // Keys + // Keys. cfgNeoFSPrivateKey = "neofs-key" cfgGateAuthPrivateKey = "auth-key" - // HTTPS/TLS + // HTTPS/TLS. cfgTLSKeyFile = "tls.key_file" cfgTLSCertFile = "tls.cert_file" - // Timeouts + // Timeouts. cfgConnectionTTL = "con_ttl" cfgConnectTimeout = "connect_timeout" cfgRequestTimeout = "request_timeout" cfgRebalanceTimer = "rebalance_timer" - // MaxClients + // MaxClients. cfgMaxClientsCount = "max_clients_count" cfgMaxClientsDeadline = "max_clients_deadline" - // gRPC + // gRPC. cfgGRPCVerbose = "verbose" - // Metrics / Profiler / Web + // Metrics / Profiler / Web. cfgEnableMetrics = "metrics" cfgEnableProfiler = "pprof" cfgListenAddress = "listen_address" cfgListenDomains = "listen_domains" - // Peers + // Peers. cfgPeers = "peers" - // Application + // Application. cfgApplicationName = "app.name" cfgApplicationVersion = "app.version" cfgApplicationBuildTime = "app.build_time" - // command line args + // Command line args. cmdHelp = "help" cmdVersion = "version" ) From 5eb863dc2208c91034a5932c512de55c4e391fe9 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Thu, 13 May 2021 23:25:31 +0300 Subject: [PATCH 8/9] *: fix golint warnings about comments to exported things. --- api/auth/center.go | 2 ++ api/handler/api.go | 2 ++ api/handler/list.go | 7 +++++++ api/layer/container.go | 2 ++ api/layer/layer.go | 12 ++++++++++-- api/layer/util.go | 5 +++++ api/max-clients.go | 4 ++++ api/metrics/api.go | 1 + api/reqinfo.go | 1 + api/response.go | 5 ++++- api/router.go | 4 ++++ api/user-auth.go | 1 + cmd/gate/app-healthy.go | 1 + cmd/gate/app.go | 4 ++++ misc/build.go | 6 +++++- 15 files changed, 53 insertions(+), 4 deletions(-) diff --git a/api/auth/center.go b/api/auth/center.go index 59a36f811..2aca7b6cc 100644 --- a/api/auth/center.go +++ b/api/auth/center.go @@ -24,6 +24,7 @@ import ( var authorizationFieldRegexp = regexp.MustCompile(`AWS4-HMAC-SHA256 Credential=(?P[^/]+)/(?P[^/]+)/(?P[^/]+)/(?P[^/]*)/(?P[^/]+)/aws4_request,\s*SignedHeaders=(?P.+),\s*Signature=(?P.+)`) type ( + // Center is a user authentication interface. Center interface { Authenticate(request *http.Request) (*token.BearerToken, error) } @@ -33,6 +34,7 @@ type ( cli bearer.Credentials } + // Params stores node connection parameters. Params struct { Client sdk.Client Logger *zap.Logger diff --git a/api/handler/api.go b/api/handler/api.go index bdea9147c..293a27c8d 100644 --- a/api/handler/api.go +++ b/api/handler/api.go @@ -14,6 +14,7 @@ type ( obj layer.Client } + // Params holds logger and client. Params struct { Log *zap.Logger Obj layer.Client @@ -24,6 +25,7 @@ const notSupported = "Not supported by NeoFS S3 Gate: " var _ api.Handler = (*handler)(nil) +// New creates new api.Handler using given logger and client. func New(log *zap.Logger, obj layer.Client) (api.Handler, error) { switch { case obj == nil: diff --git a/api/handler/list.go b/api/handler/list.go index 972ebf007..452f0f7de 100644 --- a/api/handler/list.go +++ b/api/handler/list.go @@ -22,12 +22,14 @@ type listObjectsArgs struct { Version string } +// VersioningConfiguration contains VersioningConfiguration XML representation. type VersioningConfiguration struct { XMLName xml.Name `xml:"VersioningConfiguration"` Text string `xml:",chardata"` Xmlns string `xml:"xmlns,attr"` } +// ListMultipartUploadsResult contains ListMultipartUploadsResult XML representation. type ListMultipartUploadsResult struct { XMLName xml.Name `xml:"ListMultipartUploadsResult"` Text string `xml:",chardata"` @@ -36,6 +38,7 @@ type ListMultipartUploadsResult struct { var maxObjectList = 10000 // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse. +// ListBucketsHandler handles bucket listing requests. func (h *handler) ListBucketsHandler(w http.ResponseWriter, r *http.Request) { var ( err error @@ -134,6 +137,7 @@ func (h *handler) listObjects(w http.ResponseWriter, r *http.Request) (*listObje return arg, list, nil } +// ListObjectsV1Handler handles objects listing requests for API version 1. func (h *handler) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) { var rid = api.GetRequestID(r.Context()) if arg, list, err := h.listObjects(w, r); err != nil { @@ -193,6 +197,7 @@ func encodeV1(arg *listObjectsArgs, list *layer.ListObjectsInfo) *ListObjectsRes return res } +// ListObjectsV2Handler handles objects listing requests for API version 2. func (h *handler) ListObjectsV2Handler(w http.ResponseWriter, r *http.Request) { var rid = api.GetRequestID(r.Context()) if arg, list, err := h.listObjects(w, r); err != nil { @@ -278,6 +283,7 @@ func parseListObjectArgs(r *http.Request) (*listObjectsArgs, error) { return &res, nil } +// GetBucketVersioningHandler implements bucket versioning getter handler. func (h *handler) GetBucketVersioningHandler(w http.ResponseWriter, r *http.Request) { var ( rid = api.GetRequestID(r.Context()) @@ -299,6 +305,7 @@ func (h *handler) GetBucketVersioningHandler(w http.ResponseWriter, r *http.Requ } } +// ListMultipartUploadsHandler implements multipart uploads listing handler. func (h *handler) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Request) { var ( rid = api.GetRequestID(r.Context()) diff --git a/api/layer/container.go b/api/layer/container.go index bdfe3e1b2..5138a4d0f 100644 --- a/api/layer/container.go +++ b/api/layer/container.go @@ -12,6 +12,7 @@ import ( ) type ( + // BucketInfo stores basic bucket data. BucketInfo struct { Name string CID *container.ID @@ -19,6 +20,7 @@ type ( Created time.Time } + // ListObjectsParams represents object listing request parameters. ListObjectsParams struct { Bucket string Prefix string diff --git a/api/layer/layer.go b/api/layer/layer.go index 927c02797..46b1144e4 100644 --- a/api/layer/layer.go +++ b/api/layer/layer.go @@ -25,6 +25,7 @@ type ( log *zap.Logger } + // Params stores basic API parameters. Params struct { Pool pool.Client Logger *zap.Logger @@ -32,6 +33,7 @@ type ( Credential neofs.Credentials } + // GetObjectParams stores object get request parameters. GetObjectParams struct { Bucket string Object string @@ -40,6 +42,7 @@ type ( Writer io.Writer } + // PutObjectParams stores object put request parameters. PutObjectParams struct { Bucket string Object string @@ -48,6 +51,7 @@ type ( Header map[string]string } + // CopyObjectParams stores object copy request parameters. CopyObjectParams struct { SrcBucket string DstBucket string @@ -56,10 +60,12 @@ type ( Header map[string]string } + // NeoFS provides basic NeoFS interface. NeoFS interface { Get(ctx context.Context, address *object.Address) (*object.Object, error) } + // Client provides S3 API client interface. Client interface { NeoFS @@ -81,11 +87,13 @@ type ( ) var ( - ErrObjectExists = errors.New("object exists") + // ErrObjectExists is returned on attempts to create already existing object. + ErrObjectExists = errors.New("object exists") + // ErrObjectNotExists is returned on attempts to work with non-existing object. ErrObjectNotExists = errors.New("object not exists") ) -// NewGatewayLayer creates instance of layer. It checks credentials +// NewLayer creates instance of layer. It checks credentials // and establishes gRPC connection with node. func NewLayer(log *zap.Logger, cli sdk.Client) Client { return &layer{ diff --git a/api/layer/util.go b/api/layer/util.go index 014eed896..966560955 100644 --- a/api/layer/util.go +++ b/api/layer/util.go @@ -12,6 +12,7 @@ import ( ) type ( + // ObjectInfo holds S3 object data. ObjectInfo struct { id *object.ID isDir bool @@ -52,6 +53,7 @@ type ( const ( rootSeparator = "root://" + // PathSeparator is a path components separator string. PathSeparator = string(os.PathSeparator) ) @@ -140,11 +142,14 @@ func nameFromObject(o *object.Object) (string, string) { return NameFromString(name) } +// NameFromString splits name into base file name and directory path. func NameFromString(name string) (string, string) { ind := strings.LastIndex(name, PathSeparator) return name[ind+1:], name[:ind+1] } +// ID returns object ID from ObjectInfo. func (o *ObjectInfo) ID() *object.ID { return o.id } +// IsDir allows to check if object is a directory. func (o *ObjectInfo) IsDir() bool { return o.isDir } diff --git a/api/max-clients.go b/api/max-clients.go index 93902f044..f2eb25708 100644 --- a/api/max-clients.go +++ b/api/max-clients.go @@ -6,6 +6,7 @@ import ( ) type ( + // MaxClients provides HTTP handler wrapper with client limit. MaxClients interface { Handle(http.HandlerFunc) http.HandlerFunc } @@ -18,6 +19,8 @@ type ( const defaultRequestDeadline = time.Second * 30 +// NewMaxClientsMiddleware returns MaxClients interface with handler wrapper based on +// provided count and timeout limits. func NewMaxClientsMiddleware(count int, timeout time.Duration) MaxClients { if timeout <= 0 { timeout = defaultRequestDeadline @@ -29,6 +32,7 @@ func NewMaxClientsMiddleware(count int, timeout time.Duration) MaxClients { } } +// Handler wraps HTTP handler function with logic limiting access to it. func (m *maxClients) Handle(f http.HandlerFunc) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { if m.pool == nil { diff --git a/api/metrics/api.go b/api/metrics/api.go index 56b406389..503cb171c 100644 --- a/api/metrics/api.go +++ b/api/metrics/api.go @@ -103,6 +103,7 @@ func collectHTTPMetrics(ch chan<- prometheus.Metric) { } } +// APIStats wraps http handler for api with basic statistics collection. func APIStats(api string, f http.HandlerFunc) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { httpStatsMetric.currentS3Requests.Inc(api) diff --git a/api/reqinfo.go b/api/reqinfo.go index 374a13083..c6594e1f2 100644 --- a/api/reqinfo.go +++ b/api/reqinfo.go @@ -33,6 +33,7 @@ type ( tags []KeyVal // Any additional info not accommodated by above fields } + // ObjectRequest represents object request data. ObjectRequest struct { Bucket string Object string diff --git a/api/response.go b/api/response.go index 872eeac1f..33b34109c 100644 --- a/api/response.go +++ b/api/response.go @@ -36,7 +36,7 @@ type ( StatusCode int `xml:"-" json:"-"` } - // APIError structure. + // Error structure represents API error. Error struct { Code string Description string @@ -168,6 +168,7 @@ func removeSensitiveHeaders(h http.Header) { h.Del(hdrSSECopyKey) } +// WriteResponse writes given statusCode and response into w (with mType header if set). func WriteResponse(w http.ResponseWriter, statusCode int, response []byte, mType mimeType) { setCommonHeaders(w) if mType != MimeNone { @@ -214,6 +215,8 @@ func EncodeToResponse(w http.ResponseWriter, response interface{}) error { // WriteResponse(w, http.StatusOK, response, MimeXML) // } +// WriteSuccessResponseHeadersOnly writes HTTP (200) OK response with no data +// to the client. func WriteSuccessResponseHeadersOnly(w http.ResponseWriter) { WriteResponse(w, http.StatusOK, nil, MimeNone) } diff --git a/api/router.go b/api/router.go index 6291a4442..8ae7b65b3 100644 --- a/api/router.go +++ b/api/router.go @@ -14,6 +14,7 @@ import ( ) type ( + // Handler is an S3 API handler interface. Handler interface { HeadObjectHandler(http.ResponseWriter, *http.Request) CopyObjectPartHandler(http.ResponseWriter, *http.Request) @@ -158,6 +159,7 @@ func logErrorResponse(l *zap.Logger) mux.MiddlewareFunc { } } +// GetRequestID returns request ID from response writer or context. func GetRequestID(v interface{}) string { switch t := v.(type) { case context.Context: @@ -169,6 +171,8 @@ func GetRequestID(v interface{}) string { } } +// Attach adds S3 API handlers from h to r for domains with m client limit using +// center authentication and log logger. func Attach(r *mux.Router, domains []string, m MaxClients, h Handler, center auth.Center, log *zap.Logger) { api := r.PathPrefix(SlashSeparator).Subrouter() diff --git a/api/user-auth.go b/api/user-auth.go index ceaa6b9eb..4baaffb6d 100644 --- a/api/user-auth.go +++ b/api/user-auth.go @@ -9,6 +9,7 @@ import ( "go.uber.org/zap" ) +// AttachUserAuth adds user authentication via center to router using log for logging. func AttachUserAuth(router *mux.Router, center auth.Center, log *zap.Logger) { router.Use(func(h http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/cmd/gate/app-healthy.go b/cmd/gate/app-healthy.go index 576fe4db6..ca487a93e 100644 --- a/cmd/gate/app-healthy.go +++ b/cmd/gate/app-healthy.go @@ -7,6 +7,7 @@ import ( "github.com/gorilla/mux" ) +// Healthy is a health check interface. type Healthy interface { Status() error } diff --git a/cmd/gate/app.go b/cmd/gate/app.go index 955eb6613..474bd4ada 100644 --- a/cmd/gate/app.go +++ b/cmd/gate/app.go @@ -22,6 +22,7 @@ import ( ) type ( + // App is the main application structure. App struct { cli pool.Client ctr auth.Center @@ -179,6 +180,7 @@ func newApp(ctx context.Context, l *zap.Logger, v *viper.Viper) *App { } } +// Wait waits for application to finish. func (a *App) Wait() { a.log.Info("application started") @@ -192,6 +194,7 @@ func (a *App) Wait() { a.log.Info("application finished") } +// Server runs HTTP server to handle S3 API requests. func (a *App) Server(ctx context.Context) { var ( err error @@ -256,6 +259,7 @@ func (a *App) Server(ctx context.Context) { close(a.webDone) } +// Worker runs client worker. func (a *App) Worker(ctx context.Context) { a.cli.Worker(ctx) a.log.Info("stopping worker") diff --git a/misc/build.go b/misc/build.go index c014aacad..c954d3554 100644 --- a/misc/build.go +++ b/misc/build.go @@ -1,12 +1,16 @@ package misc const ( + // ApplicationName is gateway name. ApplicationName = "neofs-s3-gate" + // Prefix is configuration environment variables prefix. Prefix = "S3_GW" ) var ( - Build = "now" + // Build holds build timestamp. + Build = "now" + // Version contains application version. Version = "dev" ) From 8e67a374d281bf29fab04b805274a8b9229ddefc Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Thu, 13 May 2021 23:26:05 +0300 Subject: [PATCH 9/9] golangci: add configuration The same one as used by HTTP gate. --- .golangci.yml | 59 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 .golangci.yml diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 000000000..79702d945 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,59 @@ +# This file contains all available configuration options +# with their default values. + +# options for analysis running +run: + # timeout for analysis, e.g. 30s, 5m, default is 1m + timeout: 5m + + # include test files or not, default is true + tests: true + +# output configuration options +output: + # colored-line-number|line-number|json|tab|checkstyle|code-climate, default is "colored-line-number" + format: tab + +# all available settings of specific linters +linters-settings: + exhaustive: + # indicates that switch statements are to be considered exhaustive if a + # 'default' case is present, even if all enum members aren't listed in the + # switch + default-signifies-exhaustive: true + govet: + # report about shadowed variables + check-shadowing: false + +linters: + enable: + # mandatory linters + - govet + - golint + + # some default golangci-lint linters + - deadcode + - errcheck + - gosimple + - ineffassign + - staticcheck + - structcheck + - typecheck + - unused + - varcheck + + # extra linters + - exhaustive + - godot + - gofmt + - whitespace + - goimports + disable-all: true + fast: false + +issues: + include: + - EXC0002 # should have a comment + - EXC0003 # test/Test ... consider calling this + - EXC0004 # govet + - EXC0005 # C-style breaks