diff --git a/.gitignore b/.gitignore index e2f53be..2013501 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,11 @@ vendor # tempfiles .DS_Store +*~ + +# binary +bin/ + +# coverage +coverage.txt +coverage.html diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..79702d9 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,59 @@ +# This file contains all available configuration options +# with their default values. + +# options for analysis running +run: + # timeout for analysis, e.g. 30s, 5m, default is 1m + timeout: 5m + + # include test files or not, default is true + tests: true + +# output configuration options +output: + # colored-line-number|line-number|json|tab|checkstyle|code-climate, default is "colored-line-number" + format: tab + +# all available settings of specific linters +linters-settings: + exhaustive: + # indicates that switch statements are to be considered exhaustive if a + # 'default' case is present, even if all enum members aren't listed in the + # switch + default-signifies-exhaustive: true + govet: + # report about shadowed variables + check-shadowing: false + +linters: + enable: + # mandatory linters + - govet + - golint + + # some default golangci-lint linters + - deadcode + - errcheck + - gosimple + - ineffassign + - staticcheck + - structcheck + - typecheck + - unused + - varcheck + + # extra linters + - exhaustive + - godot + - gofmt + - whitespace + - goimports + disable-all: true + fast: false + +issues: + include: + - EXC0002 # should have a comment + - EXC0003 # test/Test ... consider calling this + - EXC0004 # govet + - EXC0005 # C-style breaks diff --git a/Dockerfile b/Dockerfile index 94847f3..1f8d308 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,36 +4,20 @@ WORKDIR /src RUN set -x \ && apt update \ - && apt install -y upx-ucl + && apt install -y make COPY . /src ARG VERSION=dev -# https://github.com/golang/go/wiki/Modules#how-do-i-use-vendoring-with-modules-is-vendoring-going-away -# go build -mod=vendor -# The -gcflags "all=-N -l" flag helps us get a better debug experience -RUN set -x \ - && export BUILD=$(date -u +%s%N) \ - && export REPO=$(go list -m) \ - && export LDFLAGS="-X ${REPO}/misc.Version=${VERSION} -X ${REPO}/misc.Build=${BUILD}" \ - && export GOGC=off \ - && export CGO_ENABLED=0 \ - && [ -d "./vendor" ] || go mod vendor \ - && go build \ - -v \ - -mod=vendor \ - -trimpath \ - -ldflags "${LDFLAGS}" \ - -o /go/bin/neofs-s3 ./cmd/gate \ - && upx -3 /go/bin/neofs-s3 +RUN make # Executable image FROM scratch WORKDIR / -COPY --from=builder /go/bin/neofs-s3 /bin/neofs-s3 +COPY --from=builder /src/bin/neofs-s3-gw /bin/neofs-s3-gw COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ -ENTRYPOINT ["/bin/neofs-s3"] +ENTRYPOINT ["/bin/neofs-s3-gw"] diff --git a/Makefile b/Makefile index ded7221..5310c17 100644 --- a/Makefile +++ b/Makefile @@ -1,43 +1,106 @@ --include .env --include help.mk - -HUB_IMAGE=nspccdev/neofs +#!/usr/bin/make -f +REPO ?= $(shell go list -m) VERSION ?= "$(shell git describe --tags 2>/dev/null || git rev-parse --short HEAD | sed 's/^v//')" -BUILD_VERSION ?= "$(shell git describe --abbrev=0 --tags | sed 's/^v//')" -.PHONY: format deps image publish +BIN_NAME=neofs-s3-gw +HUB_IMAGE="nspccdev/$(BIN_NAME)" +HUB_TAG ?= "$(shell echo ${VERSION} | sed 's/^v//')" +BINDIR = bin +BIN = "$(BINDIR)/$(BIN_NAME)" -# Show current version -version: - @echo $(BUILD_VERSION) +.PHONY: help all dep clean format test cover lint docker/lint image-push image dirty-image + +# Make all binaries +all: $(BIN) + +$(BIN): $(BINDIR) dep + @echo "⇒ Build $@" + CGO_ENABLED=0 \ + go build -v -trimpath \ + -ldflags "-X main.Version=$(VERSION)" \ + -o $@ ./cmd/gate + +$(BINDIR): + @echo "⇒ Ensure dir: $@" + @mkdir -p $@ + +# Pull go dependencies +dep: + @printf "⇒ Download requirements: " + @CGO_ENABLED=0 \ + go mod download && echo OK + @printf "⇒ Tidy requirements: " + @CGO_ENABLED=0 \ + go mod tidy -v && echo OK + +# Run tests +test: + @go test ./... -cover + +# Run tests with race detection and produce coverage output +cover: + @go test -v -race ./... -coverprofile=coverage.txt -covermode=atomic + @go tool cover -html=coverage.txt -o coverage.html # Reformat code format: - @[ ! -z `which goimports` ] || (echo "install goimports" && exit 2) - @for f in `find . -type f -name '*.go' -not -path './vendor/*' -not -name '*.pb.go' -prune`; do \ - echo "⇒ Processing $$f"; \ - goimports -w $$f; \ - done + @echo "⇒ Processing gofmt check" + @gofmt -s -w ./ + @echo "⇒ Processing goimports check" + @goimports -w ./ -# Check and ensure dependencies -deps: - @printf "⇒ Ensure vendor: " - @go mod tidy -v && echo OK || (echo fail && exit 2) - @printf "⇒ Download requirements: " - @go mod download && echo OK || (echo fail && exit 2) - @printf "⇒ Store vendor localy: " - @go mod vendor && echo OK || (echo fail && exit 2) - -# Build current docker image -image: deps - @echo "⇒ Build docker-image" +# Build clean Docker image +image: + @echo "⇒ Build NeoFS S3 Gateway docker image " @docker build \ - --build-arg VERSION=$(BUILD_VERSION) \ - -f Dockerfile \ - -t $(HUB_IMAGE)-s3-gate:$(BUILD_VERSION) . + --build-arg REPO=$(REPO) \ + --build-arg VERSION=$(VERSION) \ + --rm \ + -f Dockerfile \ + -t $(HUB_IMAGE):$(HUB_TAG) . -# Publish docker image -publish: - @echo "${B}${G}⇒ publish docker image ${R}" - @docker push $(HUB_IMAGE)-s3-gate:$(VERSION) \ No newline at end of file +# Push Docker image to the hub +image-push: + @echo "⇒ Publish image" + @docker push $(HUB_IMAGE):$(HUB_TAG) + +# Build dirty Docker image +dirty-image: + @echo "⇒ Build NeoFS S3 Gateway dirty docker image " + @docker build \ + --build-arg REPO=$(REPO) \ + --build-arg VERSION=$(VERSION) \ + --rm \ + -f Dockerfile.dirty \ + -t $(HUB_IMAGE)-dirty:$(HUB_TAG) . + +# Run linters +lint: + @golangci-lint --timeout=5m run + +# Run linters in Docker +docker/lint: + docker run --rm -it \ + -v `pwd`:/src \ + -u `stat -c "%u:%g" .` \ + --env HOME=/src \ + golangci/golangci-lint:v1.40 bash -c 'cd /src/ && make lint' + +# Show current version +version: + @echo $(VERSION) + +# Show this help prompt +help: + @echo ' Usage:' + @echo '' + @echo ' make ' + @echo '' + @echo ' Targets:' + @echo '' + @awk '/^#/{ comment = substr($$0,3) } comment && /^[a-zA-Z][a-zA-Z0-9_-]+ ?:/{ print " ", $$1, comment }' $(MAKEFILE_LIST) | column -t -s ':' | grep -v 'IGNORE' | sort -u + +# Clean up +clean: + rm -rf $(BINDIR) diff --git a/api/auth/center.go b/api/auth/center.go index 59a36f8..2aca7b6 100644 --- a/api/auth/center.go +++ b/api/auth/center.go @@ -24,6 +24,7 @@ import ( var authorizationFieldRegexp = regexp.MustCompile(`AWS4-HMAC-SHA256 Credential=(?P[^/]+)/(?P[^/]+)/(?P[^/]+)/(?P[^/]*)/(?P[^/]+)/aws4_request,\s*SignedHeaders=(?P.+),\s*Signature=(?P.+)`) type ( + // Center is a user authentication interface. Center interface { Authenticate(request *http.Request) (*token.BearerToken, error) } @@ -33,6 +34,7 @@ type ( cli bearer.Credentials } + // Params stores node connection parameters. Params struct { Client sdk.Client Logger *zap.Logger diff --git a/api/errors.go b/api/errors.go index a0e6e39..6fb52b4 100644 --- a/api/errors.go +++ b/api/errors.go @@ -118,7 +118,7 @@ const ( ErrInvalidTagDirective // Add new error codes here. - // SSE-S3 related API errors + // SSE-S3 related API errors. ErrInvalidEncryptionMethod // Server-Side-Encryption (with Customer provided key) related API errors. @@ -157,8 +157,8 @@ const ( // Add new extended error codes here. // MinIO extended errors. - // ErrReadQuorum - // ErrWriteQuorum + // ErrReadQuorum + // ErrWriteQuorum ErrParentIsObject ErrStorageFull ErrRequestBodyParse @@ -170,7 +170,7 @@ const ( ErrOperationTimedOut ErrOperationMaxedOut ErrInvalidRequest - // MinIO storage class error codes + // MinIO storage class error codes. ErrInvalidStorageClass ErrBackendDown // Add new extended error codes here. @@ -192,7 +192,7 @@ const ( ErrAdminCredentialsMismatch ErrInsecureClientRequest ErrObjectTampered - // Bucket Quota error codes + // Bucket Quota error codes. ErrAdminBucketQuotaExceeded ErrAdminNoSuchQuotaConfiguration ErrAdminBucketQuotaDisabled @@ -205,7 +205,7 @@ const ( ErrHealOverlappingPaths ErrIncorrectContinuationToken - // S3 Select Errors + // S3 Select Errors. ErrEmptyRequestBody ErrUnsupportedFunction ErrInvalidExpressionType @@ -1625,7 +1625,7 @@ func GetAPIError(code ErrorCode) Error { } // getErrorResponse gets in standard error and resource value and -// provides a encodable populated response values +// provides a encodable populated response values. func getAPIErrorResponse(ctx context.Context, err error, resource, requestID, hostID string) ErrorResponse { code := "BadRequest" desc := err.Error() @@ -1803,21 +1803,21 @@ func (e BucketLifecycleNotFound) Error() string { return "No bucket lifecycle configuration found for bucket : " + e.Bucket } -// BucketSSEConfigNotFound - no bucket encryption found +// BucketSSEConfigNotFound - no bucket encryption found. type BucketSSEConfigNotFound GenericError func (e BucketSSEConfigNotFound) Error() string { return "No bucket encryption configuration found for bucket: " + e.Bucket } -// BucketTaggingNotFound - no bucket tags found +// BucketTaggingNotFound - no bucket tags found. type BucketTaggingNotFound GenericError func (e BucketTaggingNotFound) Error() string { return "No bucket tags found for bucket: " + e.Bucket } -// BucketObjectLockConfigNotFound - no bucket object lock config found +// BucketObjectLockConfigNotFound - no bucket object lock config found. type BucketObjectLockConfigNotFound GenericError func (e BucketObjectLockConfigNotFound) Error() string { @@ -1874,7 +1874,7 @@ func (e ObjectNamePrefixAsSlash) Error() string { return "Object name contains forward slash as pefix: " + e.Bucket + "#" + e.Object } -// AllAccessDisabled All access to this object has been disabled +// AllAccessDisabled All access to this object has been disabled. type AllAccessDisabled GenericError // Error returns string an error formatted as the given text. @@ -1945,7 +1945,7 @@ func (e InvalidUploadID) Error() string { return "Invalid upload id " + e.UploadID } -// InvalidPart One or more of the specified parts could not be found +// InvalidPart One or more of the specified parts could not be found. type InvalidPart struct { PartNumber int ExpETag string @@ -1975,21 +1975,21 @@ func (e PartTooBig) Error() string { return "Part size bigger than the allowed limit" } -// InvalidETag error returned when the etag has changed on disk +// InvalidETag error returned when the etag has changed on disk. type InvalidETag struct{} func (e InvalidETag) Error() string { return "etag of the object has changed" } -// NotImplemented If a feature is not implemented +// NotImplemented If a feature is not implemented. type NotImplemented struct{} func (e NotImplemented) Error() string { return "Not Implemented" } -// UnsupportedMetadata - unsupported metadata +// UnsupportedMetadata - unsupported metadata. type UnsupportedMetadata struct{} func (e UnsupportedMetadata) Error() string { @@ -2003,14 +2003,14 @@ func (e BackendDown) Error() string { return "Backend down" } -// PreConditionFailed - Check if copy precondition failed +// PreConditionFailed - Check if copy precondition failed. type PreConditionFailed struct{} func (e PreConditionFailed) Error() string { return "At least one of the pre-conditions you specified did not hold" } -// DeleteError - returns when cant remove object +// DeleteError - returns when cant remove object. type DeleteError struct { Err error Object string diff --git a/api/handler/api.go b/api/handler/api.go index bdea914..293a27c 100644 --- a/api/handler/api.go +++ b/api/handler/api.go @@ -14,6 +14,7 @@ type ( obj layer.Client } + // Params holds logger and client. Params struct { Log *zap.Logger Obj layer.Client @@ -24,6 +25,7 @@ const notSupported = "Not supported by NeoFS S3 Gate: " var _ api.Handler = (*handler)(nil) +// New creates new api.Handler using given logger and client. func New(log *zap.Logger, obj layer.Client) (api.Handler, error) { switch { case obj == nil: diff --git a/api/handler/delete.go b/api/handler/delete.go index e99d473..72e2da2 100644 --- a/api/handler/delete.go +++ b/api/handler/delete.go @@ -68,7 +68,7 @@ func (h *handler) DeleteObjectHandler(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusNoContent) } -// DeleteMultipleObjectsHandler : +// DeleteMultipleObjectsHandler handles multiple delete requests. func (h *handler) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Request) { var ( req = mux.Vars(r) diff --git a/api/handler/head.go b/api/handler/head.go index fdbb00b..c805ad9 100644 --- a/api/handler/head.go +++ b/api/handler/head.go @@ -109,7 +109,7 @@ func (h *handler) HeadBucketHandler(w http.ResponseWriter, r *http.Request) { code := http.StatusBadRequest if st, ok := status.FromError(err); ok && st != nil { - switch st.Code() { + switch st.Code() { //nolint:exhaustive // we have default value set above case codes.NotFound: code = http.StatusNotFound case codes.PermissionDenied: diff --git a/api/handler/list.go b/api/handler/list.go index 972ebf0..452f0f7 100644 --- a/api/handler/list.go +++ b/api/handler/list.go @@ -22,12 +22,14 @@ type listObjectsArgs struct { Version string } +// VersioningConfiguration contains VersioningConfiguration XML representation. type VersioningConfiguration struct { XMLName xml.Name `xml:"VersioningConfiguration"` Text string `xml:",chardata"` Xmlns string `xml:"xmlns,attr"` } +// ListMultipartUploadsResult contains ListMultipartUploadsResult XML representation. type ListMultipartUploadsResult struct { XMLName xml.Name `xml:"ListMultipartUploadsResult"` Text string `xml:",chardata"` @@ -36,6 +38,7 @@ type ListMultipartUploadsResult struct { var maxObjectList = 10000 // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse. +// ListBucketsHandler handles bucket listing requests. func (h *handler) ListBucketsHandler(w http.ResponseWriter, r *http.Request) { var ( err error @@ -134,6 +137,7 @@ func (h *handler) listObjects(w http.ResponseWriter, r *http.Request) (*listObje return arg, list, nil } +// ListObjectsV1Handler handles objects listing requests for API version 1. func (h *handler) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) { var rid = api.GetRequestID(r.Context()) if arg, list, err := h.listObjects(w, r); err != nil { @@ -193,6 +197,7 @@ func encodeV1(arg *listObjectsArgs, list *layer.ListObjectsInfo) *ListObjectsRes return res } +// ListObjectsV2Handler handles objects listing requests for API version 2. func (h *handler) ListObjectsV2Handler(w http.ResponseWriter, r *http.Request) { var rid = api.GetRequestID(r.Context()) if arg, list, err := h.listObjects(w, r); err != nil { @@ -278,6 +283,7 @@ func parseListObjectArgs(r *http.Request) (*listObjectsArgs, error) { return &res, nil } +// GetBucketVersioningHandler implements bucket versioning getter handler. func (h *handler) GetBucketVersioningHandler(w http.ResponseWriter, r *http.Request) { var ( rid = api.GetRequestID(r.Context()) @@ -299,6 +305,7 @@ func (h *handler) GetBucketVersioningHandler(w http.ResponseWriter, r *http.Requ } } +// ListMultipartUploadsHandler implements multipart uploads listing handler. func (h *handler) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Request) { var ( rid = api.GetRequestID(r.Context()) diff --git a/api/handler/response.go b/api/handler/response.go index 13610c6..cccdd23 100644 --- a/api/handler/response.go +++ b/api/handler/response.go @@ -2,7 +2,7 @@ package handler import "encoding/xml" -// ListBucketsResponse - format for list buckets response +// ListBucketsResponse - format for list buckets response. type ListBucketsResponse struct { XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListAllMyBucketsResult" json:"-"` @@ -45,13 +45,13 @@ type ListObjectsV2Response struct { EncodingType string `xml:"EncodingType,omitempty"` } -// Bucket container for bucket metadata +// Bucket container for bucket metadata. type Bucket struct { Name string CreationDate string // time string of format "2006-01-02T15:04:05.000Z" } -// Owner - bucket owner/principal +// Owner - bucket owner/principal. type Owner struct { ID string DisplayName string @@ -87,12 +87,12 @@ type ListObjectsResponse struct { EncodingType string `xml:"EncodingType,omitempty"` } -// CommonPrefix container for prefix response in ListObjectsResponse +// CommonPrefix container for prefix response in ListObjectsResponse. type CommonPrefix struct { Prefix string } -// Object container for object metadata +// Object container for object metadata. type Object struct { Key string LastModified string // time string of format "2006-01-02T15:04:05.000Z" @@ -118,7 +118,7 @@ type LocationResponse struct { Location string `xml:",chardata"` } -// CopyObjectResponse container returns ETag and LastModified of the successfully copied object +// CopyObjectResponse container returns ETag and LastModified of the successfully copied object. type CopyObjectResponse struct { XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CopyObjectResult" json:"-"` LastModified string // time string of format "2006-01-02T15:04:05.000Z" @@ -127,7 +127,6 @@ type CopyObjectResponse struct { // MarshalXML - StringMap marshals into XML. func (s StringMap) MarshalXML(e *xml.Encoder, start xml.StartElement) error { - tokens := []xml.Token{start} for key, value := range s { diff --git a/api/headers.go b/api/headers.go index 225e24f..085e3f7 100644 --- a/api/headers.go +++ b/api/headers.go @@ -1,6 +1,6 @@ package api -// Standard S3 HTTP response constants +// Standard S3 HTTP response constants. const ( LastModified = "Last-Modified" Date = "Date" diff --git a/api/layer/container.go b/api/layer/container.go index bdfe3e1..5138a4d 100644 --- a/api/layer/container.go +++ b/api/layer/container.go @@ -12,6 +12,7 @@ import ( ) type ( + // BucketInfo stores basic bucket data. BucketInfo struct { Name string CID *container.ID @@ -19,6 +20,7 @@ type ( Created time.Time } + // ListObjectsParams represents object listing request parameters. ListObjectsParams struct { Bucket string Prefix string diff --git a/api/layer/layer.go b/api/layer/layer.go index 2f88eae..46b1144 100644 --- a/api/layer/layer.go +++ b/api/layer/layer.go @@ -25,6 +25,7 @@ type ( log *zap.Logger } + // Params stores basic API parameters. Params struct { Pool pool.Client Logger *zap.Logger @@ -32,6 +33,7 @@ type ( Credential neofs.Credentials } + // GetObjectParams stores object get request parameters. GetObjectParams struct { Bucket string Object string @@ -40,6 +42,7 @@ type ( Writer io.Writer } + // PutObjectParams stores object put request parameters. PutObjectParams struct { Bucket string Object string @@ -48,6 +51,7 @@ type ( Header map[string]string } + // CopyObjectParams stores object copy request parameters. CopyObjectParams struct { SrcBucket string DstBucket string @@ -56,10 +60,12 @@ type ( Header map[string]string } + // NeoFS provides basic NeoFS interface. NeoFS interface { Get(ctx context.Context, address *object.Address) (*object.Object, error) } + // Client provides S3 API client interface. Client interface { NeoFS @@ -81,11 +87,13 @@ type ( ) var ( - ErrObjectExists = errors.New("object exists") + // ErrObjectExists is returned on attempts to create already existing object. + ErrObjectExists = errors.New("object exists") + // ErrObjectNotExists is returned on attempts to work with non-existing object. ErrObjectNotExists = errors.New("object not exists") ) -// NewGatewayLayer creates instance of layer. It checks credentials +// NewLayer creates instance of layer. It checks credentials // and establishes gRPC connection with node. func NewLayer(log *zap.Logger, cli sdk.Client) Client { return &layer{ @@ -103,7 +111,7 @@ func (n *layer) Owner(ctx context.Context) *owner.ID { return n.cli.Owner() } -// Get NeoFS Object by refs.Address (should be used by auth.Center) +// Get NeoFS Object by refs.Address (should be used by auth.Center). func (n *layer) Get(ctx context.Context, address *object.Address) (*object.Object, error) { return n.cli.Object().Get(ctx, address) } diff --git a/api/layer/util.go b/api/layer/util.go index 014eed8..9665609 100644 --- a/api/layer/util.go +++ b/api/layer/util.go @@ -12,6 +12,7 @@ import ( ) type ( + // ObjectInfo holds S3 object data. ObjectInfo struct { id *object.ID isDir bool @@ -52,6 +53,7 @@ type ( const ( rootSeparator = "root://" + // PathSeparator is a path components separator string. PathSeparator = string(os.PathSeparator) ) @@ -140,11 +142,14 @@ func nameFromObject(o *object.Object) (string, string) { return NameFromString(name) } +// NameFromString splits name into base file name and directory path. func NameFromString(name string) (string, string) { ind := strings.LastIndex(name, PathSeparator) return name[ind+1:], name[:ind+1] } +// ID returns object ID from ObjectInfo. func (o *ObjectInfo) ID() *object.ID { return o.id } +// IsDir allows to check if object is a directory. func (o *ObjectInfo) IsDir() bool { return o.isDir } diff --git a/api/max-clients.go b/api/max-clients.go index 93902f0..f2eb257 100644 --- a/api/max-clients.go +++ b/api/max-clients.go @@ -6,6 +6,7 @@ import ( ) type ( + // MaxClients provides HTTP handler wrapper with client limit. MaxClients interface { Handle(http.HandlerFunc) http.HandlerFunc } @@ -18,6 +19,8 @@ type ( const defaultRequestDeadline = time.Second * 30 +// NewMaxClientsMiddleware returns MaxClients interface with handler wrapper based on +// provided count and timeout limits. func NewMaxClientsMiddleware(count int, timeout time.Duration) MaxClients { if timeout <= 0 { timeout = defaultRequestDeadline @@ -29,6 +32,7 @@ func NewMaxClientsMiddleware(count int, timeout time.Duration) MaxClients { } } +// Handler wraps HTTP handler function with logic limiting access to it. func (m *maxClients) Handle(f http.HandlerFunc) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { if m.pool == nil { diff --git a/api/metrics/api.go b/api/metrics/api.go index d72297a..503cb17 100644 --- a/api/metrics/api.go +++ b/api/metrics/api.go @@ -20,7 +20,7 @@ type ( } // HTTPStats holds statistics information about - // HTTP requests made by all clients + // HTTP requests made by all clients. HTTPStats struct { currentS3Requests HTTPAPIStats totalS3Requests HTTPAPIStats @@ -63,8 +63,8 @@ var ( ) ) -// collects http metrics for NeoFS S3 Gate in Prometheus specific format -// and sends to given channel +// Collects HTTP metrics for NeoFS S3 Gate in Prometheus specific format +// and sends to given channel. func collectHTTPMetrics(ch chan<- prometheus.Metric) { for api, value := range httpStatsMetric.currentS3Requests.Load() { ch <- prometheus.MustNewConstMetric( @@ -103,6 +103,7 @@ func collectHTTPMetrics(ch chan<- prometheus.Metric) { } } +// APIStats wraps http handler for api with basic statistics collection. func APIStats(api string, f http.HandlerFunc) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { httpStatsMetric.currentS3Requests.Inc(api) @@ -176,7 +177,7 @@ func (st *HTTPStats) getOutputBytes() uint64 { return atomic.LoadUint64(&st.totalOutputBytes) } -// Update statistics from http request and response data +// Update statistics from http request and response data. func (st *HTTPStats) updateStats(api string, w http.ResponseWriter, r *http.Request, durationSecs float64) { var code int @@ -200,7 +201,7 @@ func (st *HTTPStats) updateStats(api string, w http.ResponseWriter, r *http.Requ } } -// WriteHeader - writes http status code +// WriteHeader - writes http status code. func (w *responseWrapper) WriteHeader(code int) { w.Do(func() { w.statusCode = code diff --git a/api/reqinfo.go b/api/reqinfo.go index 606ada4..c6594e1 100644 --- a/api/reqinfo.go +++ b/api/reqinfo.go @@ -13,7 +13,7 @@ import ( ) type ( - // KeyVal - appended to ReqInfo.Tags + // KeyVal - appended to ReqInfo.Tags. KeyVal struct { Key string Val string @@ -33,6 +33,7 @@ type ( tags []KeyVal // Any additional info not accommodated by above fields } + // ObjectRequest represents object request data. ObjectRequest struct { Bucket string Object string @@ -40,7 +41,7 @@ type ( } ) -// Key used for Get/SetReqInfo +// Key used for Get/SetReqInfo. type contextKeyType string const ctxRequestInfo = contextKeyType("NeoFS-S3-Gate") @@ -54,7 +55,7 @@ var ( var ( // RFC7239 defines a new "Forwarded: " header designed to replace the // existing use of X-Forwarded-* headers. - // e.g. Forwarded: for=192.0.2.60;proto=https;by=203.0.113.43 + // e.g. Forwarded: for=192.0.2.60;proto=https;by=203.0.113.43. forwarded = http.CanonicalHeaderKey("Forwarded") // Allows for a sub-match of the first value after 'for=' to the next // comma, semi-colon or space. The match is case-insensitive. @@ -125,7 +126,7 @@ func prepareContext(w http.ResponseWriter, r *http.Request) context.Context { })) } -// NewReqInfo : +// NewReqInfo returns new ReqInfo based on parameters. func NewReqInfo(w http.ResponseWriter, r *http.Request, req ObjectRequest) *ReqInfo { return &ReqInfo{ API: req.Method, @@ -138,7 +139,7 @@ func NewReqInfo(w http.ResponseWriter, r *http.Request, req ObjectRequest) *ReqI } } -// AppendTags - appends key/val to ReqInfo.tags +// AppendTags - appends key/val to ReqInfo.tags. func (r *ReqInfo) AppendTags(key string, val string) *ReqInfo { if r == nil { return nil @@ -149,7 +150,7 @@ func (r *ReqInfo) AppendTags(key string, val string) *ReqInfo { return r } -// SetTags - sets key/val to ReqInfo.tags +// SetTags - sets key/val to ReqInfo.tags. func (r *ReqInfo) SetTags(key string, val string) *ReqInfo { if r == nil { return nil @@ -172,7 +173,7 @@ func (r *ReqInfo) SetTags(key string, val string) *ReqInfo { return r } -// GetTags - returns the user defined tags +// GetTags - returns the user defined tags. func (r *ReqInfo) GetTags() []KeyVal { if r == nil { return nil diff --git a/api/response.go b/api/response.go index 6640d4c..33b3410 100644 --- a/api/response.go +++ b/api/response.go @@ -14,7 +14,7 @@ import ( ) type ( - // ErrorResponse - error response format + // ErrorResponse - error response format. ErrorResponse struct { XMLName xml.Name `xml:"Error" json:"-"` Code string @@ -32,11 +32,11 @@ type ( // Captures the server string returned in response header. Server string `xml:"-" json:"-"` - // Underlying HTTP status code for the returned error + // Underlying HTTP status code for the returned error. StatusCode int `xml:"-" json:"-"` } - // APIError structure + // Error structure represents API error. Error struct { Code string Description string @@ -117,7 +117,7 @@ var s3ErrorResponseMap = map[string]string{ // Add new API errors here. } -// WriteErrorResponse writes error headers +// WriteErrorResponse writes error headers. func WriteErrorResponse(ctx context.Context, w http.ResponseWriter, err error, reqURL *url.URL) { code := http.StatusBadRequest @@ -141,7 +141,7 @@ func WriteErrorResponse(ctx context.Context, w http.ResponseWriter, err error, r WriteResponse(w, code, encodedErrorResponse, MimeXML) } -// If none of the http routes match respond with appropriate errors +// If none of the http routes match respond with appropriate errors. func errorResponseHandler(w http.ResponseWriter, r *http.Request) { desc := fmt.Sprintf("Unknown API request at %s", r.URL.Path) WriteErrorResponse(r.Context(), w, Error{ @@ -151,7 +151,7 @@ func errorResponseHandler(w http.ResponseWriter, r *http.Request) { }, r.URL) } -// Write http common headers +// Write http common headers. func setCommonHeaders(w http.ResponseWriter) { w.Header().Set(hdrServerInfo, "NeoFS-S3-Gate/"+misc.Version) w.Header().Set(hdrAcceptRanges, "bytes") @@ -168,6 +168,7 @@ func removeSensitiveHeaders(h http.Header) { h.Del(hdrSSECopyKey) } +// WriteResponse writes given statusCode and response into w (with mType header if set). func WriteResponse(w http.ResponseWriter, statusCode int, response []byte, mType mimeType) { setCommonHeaders(w) if mType != MimeNone { @@ -214,6 +215,8 @@ func EncodeToResponse(w http.ResponseWriter, response interface{}) error { // WriteResponse(w, http.StatusOK, response, MimeXML) // } +// WriteSuccessResponseHeadersOnly writes HTTP (200) OK response with no data +// to the client. func WriteSuccessResponseHeadersOnly(w http.ResponseWriter) { WriteResponse(w, http.StatusOK, nil, MimeNone) } diff --git a/api/router.go b/api/router.go index 6291a44..8ae7b65 100644 --- a/api/router.go +++ b/api/router.go @@ -14,6 +14,7 @@ import ( ) type ( + // Handler is an S3 API handler interface. Handler interface { HeadObjectHandler(http.ResponseWriter, *http.Request) CopyObjectPartHandler(http.ResponseWriter, *http.Request) @@ -158,6 +159,7 @@ func logErrorResponse(l *zap.Logger) mux.MiddlewareFunc { } } +// GetRequestID returns request ID from response writer or context. func GetRequestID(v interface{}) string { switch t := v.(type) { case context.Context: @@ -169,6 +171,8 @@ func GetRequestID(v interface{}) string { } } +// Attach adds S3 API handlers from h to r for domains with m client limit using +// center authentication and log logger. func Attach(r *mux.Router, domains []string, m MaxClients, h Handler, center auth.Center, log *zap.Logger) { api := r.PathPrefix(SlashSeparator).Subrouter() diff --git a/api/user-auth.go b/api/user-auth.go index 5c514ef..4baaffb 100644 --- a/api/user-auth.go +++ b/api/user-auth.go @@ -9,6 +9,7 @@ import ( "go.uber.org/zap" ) +// AttachUserAuth adds user authentication via center to router using log for logging. func AttachUserAuth(router *mux.Router, center auth.Center, log *zap.Logger) { router.Use(func(h http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -21,7 +22,6 @@ func AttachUserAuth(router *mux.Router, center auth.Center, log *zap.Logger) { h.ServeHTTP(w, r.WithContext( sdk.SetBearerToken(r.Context(), token))) - }) }) } diff --git a/cmd/gate/app-healthy.go b/cmd/gate/app-healthy.go index 576fe4d..ca487a9 100644 --- a/cmd/gate/app-healthy.go +++ b/cmd/gate/app-healthy.go @@ -7,6 +7,7 @@ import ( "github.com/gorilla/mux" ) +// Healthy is a health check interface. type Healthy interface { Status() error } diff --git a/cmd/gate/app-settings.go b/cmd/gate/app-settings.go index 2024fd5..5c9b705 100644 --- a/cmd/gate/app-settings.go +++ b/cmd/gate/app-settings.go @@ -34,8 +34,8 @@ const ( defaultMaxClientsDeadline = time.Second * 30 ) -const ( // settings - // Logger: +const ( // Settings. + // Logger. cfgLoggerLevel = "logger.level" cfgLoggerFormat = "logger.format" cfgLoggerTraceLevel = "logger.trace_level" @@ -44,47 +44,47 @@ const ( // settings cfgLoggerSamplingInitial = "logger.sampling.initial" cfgLoggerSamplingThereafter = "logger.sampling.thereafter" - // KeepAlive + // KeepAlive. cfgKeepaliveTime = "keepalive.time" cfgKeepaliveTimeout = "keepalive.timeout" cfgKeepalivePermitWithoutStream = "keepalive.permit_without_stream" - // Keys + // Keys. cfgNeoFSPrivateKey = "neofs-key" cfgGateAuthPrivateKey = "auth-key" - // HTTPS/TLS + // HTTPS/TLS. cfgTLSKeyFile = "tls.key_file" cfgTLSCertFile = "tls.cert_file" - // Timeouts + // Timeouts. cfgConnectionTTL = "con_ttl" cfgConnectTimeout = "connect_timeout" cfgRequestTimeout = "request_timeout" cfgRebalanceTimer = "rebalance_timer" - // MaxClients + // MaxClients. cfgMaxClientsCount = "max_clients_count" cfgMaxClientsDeadline = "max_clients_deadline" - // gRPC + // gRPC. cfgGRPCVerbose = "verbose" - // Metrics / Profiler / Web + // Metrics / Profiler / Web. cfgEnableMetrics = "metrics" cfgEnableProfiler = "pprof" cfgListenAddress = "listen_address" cfgListenDomains = "listen_domains" - // Peers + // Peers. cfgPeers = "peers" - // Application + // Application. cfgApplicationName = "app.name" cfgApplicationVersion = "app.version" cfgApplicationBuildTime = "app.build_time" - // command line args + // Command line args. cmdHelp = "help" cmdVersion = "version" ) @@ -105,10 +105,9 @@ var ignore = map[string]struct{}{ func (empty) Read([]byte) (int, error) { return 0, io.EOF } func fetchPeers(l *zap.Logger, v *viper.Viper) map[string]float64 { - peers := make(map[string]float64, 0) + peers := make(map[string]float64) for i := 0; ; i++ { - key := cfgPeers + "." + strconv.Itoa(i) + "." address := v.GetString(key + "address") weight := v.GetFloat64(key + "weight") diff --git a/cmd/gate/app.go b/cmd/gate/app.go index 955eb66..474bd4a 100644 --- a/cmd/gate/app.go +++ b/cmd/gate/app.go @@ -22,6 +22,7 @@ import ( ) type ( + // App is the main application structure. App struct { cli pool.Client ctr auth.Center @@ -179,6 +180,7 @@ func newApp(ctx context.Context, l *zap.Logger, v *viper.Viper) *App { } } +// Wait waits for application to finish. func (a *App) Wait() { a.log.Info("application started") @@ -192,6 +194,7 @@ func (a *App) Wait() { a.log.Info("application finished") } +// Server runs HTTP server to handle S3 API requests. func (a *App) Server(ctx context.Context) { var ( err error @@ -256,6 +259,7 @@ func (a *App) Server(ctx context.Context) { close(a.webDone) } +// Worker runs client worker. func (a *App) Worker(ctx context.Context) { a.cli.Worker(ctx) a.log.Info("stopping worker") diff --git a/misc/build.go b/misc/build.go index c014aac..c954d35 100644 --- a/misc/build.go +++ b/misc/build.go @@ -1,12 +1,16 @@ package misc const ( + // ApplicationName is gateway name. ApplicationName = "neofs-s3-gate" + // Prefix is configuration environment variables prefix. Prefix = "S3_GW" ) var ( - Build = "now" + // Build holds build timestamp. + Build = "now" + // Version contains application version. Version = "dev" )