Merge pull request #45 from roman-khimov/build-fixes

Build fixes
This commit is contained in:
Roman Khimov 2021-05-14 10:25:26 +03:00 committed by GitHub
commit 2b967db7c0
25 changed files with 277 additions and 117 deletions

8
.gitignore vendored
View file

@ -7,3 +7,11 @@ vendor
# tempfiles # tempfiles
.DS_Store .DS_Store
*~
# binary
bin/
# coverage
coverage.txt
coverage.html

59
.golangci.yml Normal file
View file

@ -0,0 +1,59 @@
# This file contains all available configuration options
# with their default values.
# options for analysis running
run:
# timeout for analysis, e.g. 30s, 5m, default is 1m
timeout: 5m
# include test files or not, default is true
tests: true
# output configuration options
output:
# colored-line-number|line-number|json|tab|checkstyle|code-climate, default is "colored-line-number"
format: tab
# all available settings of specific linters
linters-settings:
exhaustive:
# indicates that switch statements are to be considered exhaustive if a
# 'default' case is present, even if all enum members aren't listed in the
# switch
default-signifies-exhaustive: true
govet:
# report about shadowed variables
check-shadowing: false
linters:
enable:
# mandatory linters
- govet
- golint
# some default golangci-lint linters
- deadcode
- errcheck
- gosimple
- ineffassign
- staticcheck
- structcheck
- typecheck
- unused
- varcheck
# extra linters
- exhaustive
- godot
- gofmt
- whitespace
- goimports
disable-all: true
fast: false
issues:
include:
- EXC0002 # should have a comment
- EXC0003 # test/Test ... consider calling this
- EXC0004 # govet
- EXC0005 # C-style breaks

View file

@ -4,36 +4,20 @@ WORKDIR /src
RUN set -x \ RUN set -x \
&& apt update \ && apt update \
&& apt install -y upx-ucl && apt install -y make
COPY . /src COPY . /src
ARG VERSION=dev ARG VERSION=dev
# https://github.com/golang/go/wiki/Modules#how-do-i-use-vendoring-with-modules-is-vendoring-going-away RUN make
# go build -mod=vendor
# The -gcflags "all=-N -l" flag helps us get a better debug experience
RUN set -x \
&& export BUILD=$(date -u +%s%N) \
&& export REPO=$(go list -m) \
&& export LDFLAGS="-X ${REPO}/misc.Version=${VERSION} -X ${REPO}/misc.Build=${BUILD}" \
&& export GOGC=off \
&& export CGO_ENABLED=0 \
&& [ -d "./vendor" ] || go mod vendor \
&& go build \
-v \
-mod=vendor \
-trimpath \
-ldflags "${LDFLAGS}" \
-o /go/bin/neofs-s3 ./cmd/gate \
&& upx -3 /go/bin/neofs-s3
# Executable image # Executable image
FROM scratch FROM scratch
WORKDIR / WORKDIR /
COPY --from=builder /go/bin/neofs-s3 /bin/neofs-s3 COPY --from=builder /src/bin/neofs-s3-gw /bin/neofs-s3-gw
COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
ENTRYPOINT ["/bin/neofs-s3"] ENTRYPOINT ["/bin/neofs-s3-gw"]

129
Makefile
View file

@ -1,43 +1,106 @@
-include .env #!/usr/bin/make -f
-include help.mk
HUB_IMAGE=nspccdev/neofs
REPO ?= $(shell go list -m)
VERSION ?= "$(shell git describe --tags 2>/dev/null || git rev-parse --short HEAD | sed 's/^v//')" VERSION ?= "$(shell git describe --tags 2>/dev/null || git rev-parse --short HEAD | sed 's/^v//')"
BUILD_VERSION ?= "$(shell git describe --abbrev=0 --tags | sed 's/^v//')"
.PHONY: format deps image publish BIN_NAME=neofs-s3-gw
HUB_IMAGE="nspccdev/$(BIN_NAME)"
HUB_TAG ?= "$(shell echo ${VERSION} | sed 's/^v//')"
BINDIR = bin
BIN = "$(BINDIR)/$(BIN_NAME)"
# Show current version .PHONY: help all dep clean format test cover lint docker/lint image-push image dirty-image
version:
@echo $(BUILD_VERSION) # Make all binaries
all: $(BIN)
$(BIN): $(BINDIR) dep
@echo "⇒ Build $@"
CGO_ENABLED=0 \
go build -v -trimpath \
-ldflags "-X main.Version=$(VERSION)" \
-o $@ ./cmd/gate
$(BINDIR):
@echo "⇒ Ensure dir: $@"
@mkdir -p $@
# Pull go dependencies
dep:
@printf "⇒ Download requirements: "
@CGO_ENABLED=0 \
go mod download && echo OK
@printf "⇒ Tidy requirements: "
@CGO_ENABLED=0 \
go mod tidy -v && echo OK
# Run tests
test:
@go test ./... -cover
# Run tests with race detection and produce coverage output
cover:
@go test -v -race ./... -coverprofile=coverage.txt -covermode=atomic
@go tool cover -html=coverage.txt -o coverage.html
# Reformat code # Reformat code
format: format:
@[ ! -z `which goimports` ] || (echo "install goimports" && exit 2) @echo "⇒ Processing gofmt check"
@for f in `find . -type f -name '*.go' -not -path './vendor/*' -not -name '*.pb.go' -prune`; do \ @gofmt -s -w ./
echo "⇒ Processing $$f"; \ @echo "⇒ Processing goimports check"
goimports -w $$f; \ @goimports -w ./
done
# Check and ensure dependencies # Build clean Docker image
deps: image:
@printf "⇒ Ensure vendor: " @echo "⇒ Build NeoFS S3 Gateway docker image "
@go mod tidy -v && echo OK || (echo fail && exit 2)
@printf "⇒ Download requirements: "
@go mod download && echo OK || (echo fail && exit 2)
@printf "⇒ Store vendor localy: "
@go mod vendor && echo OK || (echo fail && exit 2)
# Build current docker image
image: deps
@echo "⇒ Build docker-image"
@docker build \ @docker build \
--build-arg VERSION=$(BUILD_VERSION) \ --build-arg REPO=$(REPO) \
-f Dockerfile \ --build-arg VERSION=$(VERSION) \
-t $(HUB_IMAGE)-s3-gate:$(BUILD_VERSION) . --rm \
-f Dockerfile \
-t $(HUB_IMAGE):$(HUB_TAG) .
# Publish docker image # Push Docker image to the hub
publish: image-push:
@echo "${B}${G}⇒ publish docker image ${R}" @echo "⇒ Publish image"
@docker push $(HUB_IMAGE)-s3-gate:$(VERSION) @docker push $(HUB_IMAGE):$(HUB_TAG)
# Build dirty Docker image
dirty-image:
@echo "⇒ Build NeoFS S3 Gateway dirty docker image "
@docker build \
--build-arg REPO=$(REPO) \
--build-arg VERSION=$(VERSION) \
--rm \
-f Dockerfile.dirty \
-t $(HUB_IMAGE)-dirty:$(HUB_TAG) .
# Run linters
lint:
@golangci-lint --timeout=5m run
# Run linters in Docker
docker/lint:
docker run --rm -it \
-v `pwd`:/src \
-u `stat -c "%u:%g" .` \
--env HOME=/src \
golangci/golangci-lint:v1.40 bash -c 'cd /src/ && make lint'
# Show current version
version:
@echo $(VERSION)
# Show this help prompt
help:
@echo ' Usage:'
@echo ''
@echo ' make <target>'
@echo ''
@echo ' Targets:'
@echo ''
@awk '/^#/{ comment = substr($$0,3) } comment && /^[a-zA-Z][a-zA-Z0-9_-]+ ?:/{ print " ", $$1, comment }' $(MAKEFILE_LIST) | column -t -s ':' | grep -v 'IGNORE' | sort -u
# Clean up
clean:
rm -rf $(BINDIR)

View file

@ -24,6 +24,7 @@ import (
var authorizationFieldRegexp = regexp.MustCompile(`AWS4-HMAC-SHA256 Credential=(?P<access_key_id_cid>[^/]+)/(?P<access_key_id_oid>[^/]+)/(?P<date>[^/]+)/(?P<region>[^/]*)/(?P<service>[^/]+)/aws4_request,\s*SignedHeaders=(?P<signed_header_fields>.+),\s*Signature=(?P<v4_signature>.+)`) var authorizationFieldRegexp = regexp.MustCompile(`AWS4-HMAC-SHA256 Credential=(?P<access_key_id_cid>[^/]+)/(?P<access_key_id_oid>[^/]+)/(?P<date>[^/]+)/(?P<region>[^/]*)/(?P<service>[^/]+)/aws4_request,\s*SignedHeaders=(?P<signed_header_fields>.+),\s*Signature=(?P<v4_signature>.+)`)
type ( type (
// Center is a user authentication interface.
Center interface { Center interface {
Authenticate(request *http.Request) (*token.BearerToken, error) Authenticate(request *http.Request) (*token.BearerToken, error)
} }
@ -33,6 +34,7 @@ type (
cli bearer.Credentials cli bearer.Credentials
} }
// Params stores node connection parameters.
Params struct { Params struct {
Client sdk.Client Client sdk.Client
Logger *zap.Logger Logger *zap.Logger

View file

@ -118,7 +118,7 @@ const (
ErrInvalidTagDirective ErrInvalidTagDirective
// Add new error codes here. // Add new error codes here.
// SSE-S3 related API errors // SSE-S3 related API errors.
ErrInvalidEncryptionMethod ErrInvalidEncryptionMethod
// Server-Side-Encryption (with Customer provided key) related API errors. // Server-Side-Encryption (with Customer provided key) related API errors.
@ -157,8 +157,8 @@ const (
// Add new extended error codes here. // Add new extended error codes here.
// MinIO extended errors. // MinIO extended errors.
// ErrReadQuorum // ErrReadQuorum
// ErrWriteQuorum // ErrWriteQuorum
ErrParentIsObject ErrParentIsObject
ErrStorageFull ErrStorageFull
ErrRequestBodyParse ErrRequestBodyParse
@ -170,7 +170,7 @@ const (
ErrOperationTimedOut ErrOperationTimedOut
ErrOperationMaxedOut ErrOperationMaxedOut
ErrInvalidRequest ErrInvalidRequest
// MinIO storage class error codes // MinIO storage class error codes.
ErrInvalidStorageClass ErrInvalidStorageClass
ErrBackendDown ErrBackendDown
// Add new extended error codes here. // Add new extended error codes here.
@ -192,7 +192,7 @@ const (
ErrAdminCredentialsMismatch ErrAdminCredentialsMismatch
ErrInsecureClientRequest ErrInsecureClientRequest
ErrObjectTampered ErrObjectTampered
// Bucket Quota error codes // Bucket Quota error codes.
ErrAdminBucketQuotaExceeded ErrAdminBucketQuotaExceeded
ErrAdminNoSuchQuotaConfiguration ErrAdminNoSuchQuotaConfiguration
ErrAdminBucketQuotaDisabled ErrAdminBucketQuotaDisabled
@ -205,7 +205,7 @@ const (
ErrHealOverlappingPaths ErrHealOverlappingPaths
ErrIncorrectContinuationToken ErrIncorrectContinuationToken
// S3 Select Errors // S3 Select Errors.
ErrEmptyRequestBody ErrEmptyRequestBody
ErrUnsupportedFunction ErrUnsupportedFunction
ErrInvalidExpressionType ErrInvalidExpressionType
@ -1625,7 +1625,7 @@ func GetAPIError(code ErrorCode) Error {
} }
// getErrorResponse gets in standard error and resource value and // getErrorResponse gets in standard error and resource value and
// provides a encodable populated response values // provides a encodable populated response values.
func getAPIErrorResponse(ctx context.Context, err error, resource, requestID, hostID string) ErrorResponse { func getAPIErrorResponse(ctx context.Context, err error, resource, requestID, hostID string) ErrorResponse {
code := "BadRequest" code := "BadRequest"
desc := err.Error() desc := err.Error()
@ -1803,21 +1803,21 @@ func (e BucketLifecycleNotFound) Error() string {
return "No bucket lifecycle configuration found for bucket : " + e.Bucket return "No bucket lifecycle configuration found for bucket : " + e.Bucket
} }
// BucketSSEConfigNotFound - no bucket encryption found // BucketSSEConfigNotFound - no bucket encryption found.
type BucketSSEConfigNotFound GenericError type BucketSSEConfigNotFound GenericError
func (e BucketSSEConfigNotFound) Error() string { func (e BucketSSEConfigNotFound) Error() string {
return "No bucket encryption configuration found for bucket: " + e.Bucket return "No bucket encryption configuration found for bucket: " + e.Bucket
} }
// BucketTaggingNotFound - no bucket tags found // BucketTaggingNotFound - no bucket tags found.
type BucketTaggingNotFound GenericError type BucketTaggingNotFound GenericError
func (e BucketTaggingNotFound) Error() string { func (e BucketTaggingNotFound) Error() string {
return "No bucket tags found for bucket: " + e.Bucket return "No bucket tags found for bucket: " + e.Bucket
} }
// BucketObjectLockConfigNotFound - no bucket object lock config found // BucketObjectLockConfigNotFound - no bucket object lock config found.
type BucketObjectLockConfigNotFound GenericError type BucketObjectLockConfigNotFound GenericError
func (e BucketObjectLockConfigNotFound) Error() string { func (e BucketObjectLockConfigNotFound) Error() string {
@ -1874,7 +1874,7 @@ func (e ObjectNamePrefixAsSlash) Error() string {
return "Object name contains forward slash as pefix: " + e.Bucket + "#" + e.Object return "Object name contains forward slash as pefix: " + e.Bucket + "#" + e.Object
} }
// AllAccessDisabled All access to this object has been disabled // AllAccessDisabled All access to this object has been disabled.
type AllAccessDisabled GenericError type AllAccessDisabled GenericError
// Error returns string an error formatted as the given text. // Error returns string an error formatted as the given text.
@ -1945,7 +1945,7 @@ func (e InvalidUploadID) Error() string {
return "Invalid upload id " + e.UploadID return "Invalid upload id " + e.UploadID
} }
// InvalidPart One or more of the specified parts could not be found // InvalidPart One or more of the specified parts could not be found.
type InvalidPart struct { type InvalidPart struct {
PartNumber int PartNumber int
ExpETag string ExpETag string
@ -1975,21 +1975,21 @@ func (e PartTooBig) Error() string {
return "Part size bigger than the allowed limit" return "Part size bigger than the allowed limit"
} }
// InvalidETag error returned when the etag has changed on disk // InvalidETag error returned when the etag has changed on disk.
type InvalidETag struct{} type InvalidETag struct{}
func (e InvalidETag) Error() string { func (e InvalidETag) Error() string {
return "etag of the object has changed" return "etag of the object has changed"
} }
// NotImplemented If a feature is not implemented // NotImplemented If a feature is not implemented.
type NotImplemented struct{} type NotImplemented struct{}
func (e NotImplemented) Error() string { func (e NotImplemented) Error() string {
return "Not Implemented" return "Not Implemented"
} }
// UnsupportedMetadata - unsupported metadata // UnsupportedMetadata - unsupported metadata.
type UnsupportedMetadata struct{} type UnsupportedMetadata struct{}
func (e UnsupportedMetadata) Error() string { func (e UnsupportedMetadata) Error() string {
@ -2003,14 +2003,14 @@ func (e BackendDown) Error() string {
return "Backend down" return "Backend down"
} }
// PreConditionFailed - Check if copy precondition failed // PreConditionFailed - Check if copy precondition failed.
type PreConditionFailed struct{} type PreConditionFailed struct{}
func (e PreConditionFailed) Error() string { func (e PreConditionFailed) Error() string {
return "At least one of the pre-conditions you specified did not hold" return "At least one of the pre-conditions you specified did not hold"
} }
// DeleteError - returns when cant remove object // DeleteError - returns when cant remove object.
type DeleteError struct { type DeleteError struct {
Err error Err error
Object string Object string

View file

@ -14,6 +14,7 @@ type (
obj layer.Client obj layer.Client
} }
// Params holds logger and client.
Params struct { Params struct {
Log *zap.Logger Log *zap.Logger
Obj layer.Client Obj layer.Client
@ -24,6 +25,7 @@ const notSupported = "Not supported by NeoFS S3 Gate: "
var _ api.Handler = (*handler)(nil) var _ api.Handler = (*handler)(nil)
// New creates new api.Handler using given logger and client.
func New(log *zap.Logger, obj layer.Client) (api.Handler, error) { func New(log *zap.Logger, obj layer.Client) (api.Handler, error) {
switch { switch {
case obj == nil: case obj == nil:

View file

@ -68,7 +68,7 @@ func (h *handler) DeleteObjectHandler(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusNoContent) w.WriteHeader(http.StatusNoContent)
} }
// DeleteMultipleObjectsHandler : // DeleteMultipleObjectsHandler handles multiple delete requests.
func (h *handler) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Request) { func (h *handler) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Request) {
var ( var (
req = mux.Vars(r) req = mux.Vars(r)

View file

@ -109,7 +109,7 @@ func (h *handler) HeadBucketHandler(w http.ResponseWriter, r *http.Request) {
code := http.StatusBadRequest code := http.StatusBadRequest
if st, ok := status.FromError(err); ok && st != nil { if st, ok := status.FromError(err); ok && st != nil {
switch st.Code() { switch st.Code() { //nolint:exhaustive // we have default value set above
case codes.NotFound: case codes.NotFound:
code = http.StatusNotFound code = http.StatusNotFound
case codes.PermissionDenied: case codes.PermissionDenied:

View file

@ -22,12 +22,14 @@ type listObjectsArgs struct {
Version string Version string
} }
// VersioningConfiguration contains VersioningConfiguration XML representation.
type VersioningConfiguration struct { type VersioningConfiguration struct {
XMLName xml.Name `xml:"VersioningConfiguration"` XMLName xml.Name `xml:"VersioningConfiguration"`
Text string `xml:",chardata"` Text string `xml:",chardata"`
Xmlns string `xml:"xmlns,attr"` Xmlns string `xml:"xmlns,attr"`
} }
// ListMultipartUploadsResult contains ListMultipartUploadsResult XML representation.
type ListMultipartUploadsResult struct { type ListMultipartUploadsResult struct {
XMLName xml.Name `xml:"ListMultipartUploadsResult"` XMLName xml.Name `xml:"ListMultipartUploadsResult"`
Text string `xml:",chardata"` Text string `xml:",chardata"`
@ -36,6 +38,7 @@ type ListMultipartUploadsResult struct {
var maxObjectList = 10000 // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse. var maxObjectList = 10000 // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse.
// ListBucketsHandler handles bucket listing requests.
func (h *handler) ListBucketsHandler(w http.ResponseWriter, r *http.Request) { func (h *handler) ListBucketsHandler(w http.ResponseWriter, r *http.Request) {
var ( var (
err error err error
@ -134,6 +137,7 @@ func (h *handler) listObjects(w http.ResponseWriter, r *http.Request) (*listObje
return arg, list, nil return arg, list, nil
} }
// ListObjectsV1Handler handles objects listing requests for API version 1.
func (h *handler) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) { func (h *handler) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) {
var rid = api.GetRequestID(r.Context()) var rid = api.GetRequestID(r.Context())
if arg, list, err := h.listObjects(w, r); err != nil { if arg, list, err := h.listObjects(w, r); err != nil {
@ -193,6 +197,7 @@ func encodeV1(arg *listObjectsArgs, list *layer.ListObjectsInfo) *ListObjectsRes
return res return res
} }
// ListObjectsV2Handler handles objects listing requests for API version 2.
func (h *handler) ListObjectsV2Handler(w http.ResponseWriter, r *http.Request) { func (h *handler) ListObjectsV2Handler(w http.ResponseWriter, r *http.Request) {
var rid = api.GetRequestID(r.Context()) var rid = api.GetRequestID(r.Context())
if arg, list, err := h.listObjects(w, r); err != nil { if arg, list, err := h.listObjects(w, r); err != nil {
@ -278,6 +283,7 @@ func parseListObjectArgs(r *http.Request) (*listObjectsArgs, error) {
return &res, nil return &res, nil
} }
// GetBucketVersioningHandler implements bucket versioning getter handler.
func (h *handler) GetBucketVersioningHandler(w http.ResponseWriter, r *http.Request) { func (h *handler) GetBucketVersioningHandler(w http.ResponseWriter, r *http.Request) {
var ( var (
rid = api.GetRequestID(r.Context()) rid = api.GetRequestID(r.Context())
@ -299,6 +305,7 @@ func (h *handler) GetBucketVersioningHandler(w http.ResponseWriter, r *http.Requ
} }
} }
// ListMultipartUploadsHandler implements multipart uploads listing handler.
func (h *handler) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Request) { func (h *handler) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Request) {
var ( var (
rid = api.GetRequestID(r.Context()) rid = api.GetRequestID(r.Context())

View file

@ -2,7 +2,7 @@ package handler
import "encoding/xml" import "encoding/xml"
// ListBucketsResponse - format for list buckets response // ListBucketsResponse - format for list buckets response.
type ListBucketsResponse struct { type ListBucketsResponse struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListAllMyBucketsResult" json:"-"` XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListAllMyBucketsResult" json:"-"`
@ -45,13 +45,13 @@ type ListObjectsV2Response struct {
EncodingType string `xml:"EncodingType,omitempty"` EncodingType string `xml:"EncodingType,omitempty"`
} }
// Bucket container for bucket metadata // Bucket container for bucket metadata.
type Bucket struct { type Bucket struct {
Name string Name string
CreationDate string // time string of format "2006-01-02T15:04:05.000Z" CreationDate string // time string of format "2006-01-02T15:04:05.000Z"
} }
// Owner - bucket owner/principal // Owner - bucket owner/principal.
type Owner struct { type Owner struct {
ID string ID string
DisplayName string DisplayName string
@ -87,12 +87,12 @@ type ListObjectsResponse struct {
EncodingType string `xml:"EncodingType,omitempty"` EncodingType string `xml:"EncodingType,omitempty"`
} }
// CommonPrefix container for prefix response in ListObjectsResponse // CommonPrefix container for prefix response in ListObjectsResponse.
type CommonPrefix struct { type CommonPrefix struct {
Prefix string Prefix string
} }
// Object container for object metadata // Object container for object metadata.
type Object struct { type Object struct {
Key string Key string
LastModified string // time string of format "2006-01-02T15:04:05.000Z" LastModified string // time string of format "2006-01-02T15:04:05.000Z"
@ -118,7 +118,7 @@ type LocationResponse struct {
Location string `xml:",chardata"` Location string `xml:",chardata"`
} }
// CopyObjectResponse container returns ETag and LastModified of the successfully copied object // CopyObjectResponse container returns ETag and LastModified of the successfully copied object.
type CopyObjectResponse struct { type CopyObjectResponse struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CopyObjectResult" json:"-"` XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CopyObjectResult" json:"-"`
LastModified string // time string of format "2006-01-02T15:04:05.000Z" LastModified string // time string of format "2006-01-02T15:04:05.000Z"
@ -127,7 +127,6 @@ type CopyObjectResponse struct {
// MarshalXML - StringMap marshals into XML. // MarshalXML - StringMap marshals into XML.
func (s StringMap) MarshalXML(e *xml.Encoder, start xml.StartElement) error { func (s StringMap) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
tokens := []xml.Token{start} tokens := []xml.Token{start}
for key, value := range s { for key, value := range s {

View file

@ -1,6 +1,6 @@
package api package api
// Standard S3 HTTP response constants // Standard S3 HTTP response constants.
const ( const (
LastModified = "Last-Modified" LastModified = "Last-Modified"
Date = "Date" Date = "Date"

View file

@ -12,6 +12,7 @@ import (
) )
type ( type (
// BucketInfo stores basic bucket data.
BucketInfo struct { BucketInfo struct {
Name string Name string
CID *container.ID CID *container.ID
@ -19,6 +20,7 @@ type (
Created time.Time Created time.Time
} }
// ListObjectsParams represents object listing request parameters.
ListObjectsParams struct { ListObjectsParams struct {
Bucket string Bucket string
Prefix string Prefix string

View file

@ -25,6 +25,7 @@ type (
log *zap.Logger log *zap.Logger
} }
// Params stores basic API parameters.
Params struct { Params struct {
Pool pool.Client Pool pool.Client
Logger *zap.Logger Logger *zap.Logger
@ -32,6 +33,7 @@ type (
Credential neofs.Credentials Credential neofs.Credentials
} }
// GetObjectParams stores object get request parameters.
GetObjectParams struct { GetObjectParams struct {
Bucket string Bucket string
Object string Object string
@ -40,6 +42,7 @@ type (
Writer io.Writer Writer io.Writer
} }
// PutObjectParams stores object put request parameters.
PutObjectParams struct { PutObjectParams struct {
Bucket string Bucket string
Object string Object string
@ -48,6 +51,7 @@ type (
Header map[string]string Header map[string]string
} }
// CopyObjectParams stores object copy request parameters.
CopyObjectParams struct { CopyObjectParams struct {
SrcBucket string SrcBucket string
DstBucket string DstBucket string
@ -56,10 +60,12 @@ type (
Header map[string]string Header map[string]string
} }
// NeoFS provides basic NeoFS interface.
NeoFS interface { NeoFS interface {
Get(ctx context.Context, address *object.Address) (*object.Object, error) Get(ctx context.Context, address *object.Address) (*object.Object, error)
} }
// Client provides S3 API client interface.
Client interface { Client interface {
NeoFS NeoFS
@ -81,11 +87,13 @@ type (
) )
var ( var (
ErrObjectExists = errors.New("object exists") // ErrObjectExists is returned on attempts to create already existing object.
ErrObjectExists = errors.New("object exists")
// ErrObjectNotExists is returned on attempts to work with non-existing object.
ErrObjectNotExists = errors.New("object not exists") ErrObjectNotExists = errors.New("object not exists")
) )
// NewGatewayLayer creates instance of layer. It checks credentials // NewLayer creates instance of layer. It checks credentials
// and establishes gRPC connection with node. // and establishes gRPC connection with node.
func NewLayer(log *zap.Logger, cli sdk.Client) Client { func NewLayer(log *zap.Logger, cli sdk.Client) Client {
return &layer{ return &layer{
@ -103,7 +111,7 @@ func (n *layer) Owner(ctx context.Context) *owner.ID {
return n.cli.Owner() return n.cli.Owner()
} }
// Get NeoFS Object by refs.Address (should be used by auth.Center) // Get NeoFS Object by refs.Address (should be used by auth.Center).
func (n *layer) Get(ctx context.Context, address *object.Address) (*object.Object, error) { func (n *layer) Get(ctx context.Context, address *object.Address) (*object.Object, error) {
return n.cli.Object().Get(ctx, address) return n.cli.Object().Get(ctx, address)
} }

View file

@ -12,6 +12,7 @@ import (
) )
type ( type (
// ObjectInfo holds S3 object data.
ObjectInfo struct { ObjectInfo struct {
id *object.ID id *object.ID
isDir bool isDir bool
@ -52,6 +53,7 @@ type (
const ( const (
rootSeparator = "root://" rootSeparator = "root://"
// PathSeparator is a path components separator string.
PathSeparator = string(os.PathSeparator) PathSeparator = string(os.PathSeparator)
) )
@ -140,11 +142,14 @@ func nameFromObject(o *object.Object) (string, string) {
return NameFromString(name) return NameFromString(name)
} }
// NameFromString splits name into base file name and directory path.
func NameFromString(name string) (string, string) { func NameFromString(name string) (string, string) {
ind := strings.LastIndex(name, PathSeparator) ind := strings.LastIndex(name, PathSeparator)
return name[ind+1:], name[:ind+1] return name[ind+1:], name[:ind+1]
} }
// ID returns object ID from ObjectInfo.
func (o *ObjectInfo) ID() *object.ID { return o.id } func (o *ObjectInfo) ID() *object.ID { return o.id }
// IsDir allows to check if object is a directory.
func (o *ObjectInfo) IsDir() bool { return o.isDir } func (o *ObjectInfo) IsDir() bool { return o.isDir }

View file

@ -6,6 +6,7 @@ import (
) )
type ( type (
// MaxClients provides HTTP handler wrapper with client limit.
MaxClients interface { MaxClients interface {
Handle(http.HandlerFunc) http.HandlerFunc Handle(http.HandlerFunc) http.HandlerFunc
} }
@ -18,6 +19,8 @@ type (
const defaultRequestDeadline = time.Second * 30 const defaultRequestDeadline = time.Second * 30
// NewMaxClientsMiddleware returns MaxClients interface with handler wrapper based on
// provided count and timeout limits.
func NewMaxClientsMiddleware(count int, timeout time.Duration) MaxClients { func NewMaxClientsMiddleware(count int, timeout time.Duration) MaxClients {
if timeout <= 0 { if timeout <= 0 {
timeout = defaultRequestDeadline timeout = defaultRequestDeadline
@ -29,6 +32,7 @@ func NewMaxClientsMiddleware(count int, timeout time.Duration) MaxClients {
} }
} }
// Handler wraps HTTP handler function with logic limiting access to it.
func (m *maxClients) Handle(f http.HandlerFunc) http.HandlerFunc { func (m *maxClients) Handle(f http.HandlerFunc) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) { return func(w http.ResponseWriter, r *http.Request) {
if m.pool == nil { if m.pool == nil {

View file

@ -20,7 +20,7 @@ type (
} }
// HTTPStats holds statistics information about // HTTPStats holds statistics information about
// HTTP requests made by all clients // HTTP requests made by all clients.
HTTPStats struct { HTTPStats struct {
currentS3Requests HTTPAPIStats currentS3Requests HTTPAPIStats
totalS3Requests HTTPAPIStats totalS3Requests HTTPAPIStats
@ -63,8 +63,8 @@ var (
) )
) )
// collects http metrics for NeoFS S3 Gate in Prometheus specific format // Collects HTTP metrics for NeoFS S3 Gate in Prometheus specific format
// and sends to given channel // and sends to given channel.
func collectHTTPMetrics(ch chan<- prometheus.Metric) { func collectHTTPMetrics(ch chan<- prometheus.Metric) {
for api, value := range httpStatsMetric.currentS3Requests.Load() { for api, value := range httpStatsMetric.currentS3Requests.Load() {
ch <- prometheus.MustNewConstMetric( ch <- prometheus.MustNewConstMetric(
@ -103,6 +103,7 @@ func collectHTTPMetrics(ch chan<- prometheus.Metric) {
} }
} }
// APIStats wraps http handler for api with basic statistics collection.
func APIStats(api string, f http.HandlerFunc) http.HandlerFunc { func APIStats(api string, f http.HandlerFunc) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) { return func(w http.ResponseWriter, r *http.Request) {
httpStatsMetric.currentS3Requests.Inc(api) httpStatsMetric.currentS3Requests.Inc(api)
@ -176,7 +177,7 @@ func (st *HTTPStats) getOutputBytes() uint64 {
return atomic.LoadUint64(&st.totalOutputBytes) return atomic.LoadUint64(&st.totalOutputBytes)
} }
// Update statistics from http request and response data // Update statistics from http request and response data.
func (st *HTTPStats) updateStats(api string, w http.ResponseWriter, r *http.Request, durationSecs float64) { func (st *HTTPStats) updateStats(api string, w http.ResponseWriter, r *http.Request, durationSecs float64) {
var code int var code int
@ -200,7 +201,7 @@ func (st *HTTPStats) updateStats(api string, w http.ResponseWriter, r *http.Requ
} }
} }
// WriteHeader - writes http status code // WriteHeader - writes http status code.
func (w *responseWrapper) WriteHeader(code int) { func (w *responseWrapper) WriteHeader(code int) {
w.Do(func() { w.Do(func() {
w.statusCode = code w.statusCode = code

View file

@ -13,7 +13,7 @@ import (
) )
type ( type (
// KeyVal - appended to ReqInfo.Tags // KeyVal - appended to ReqInfo.Tags.
KeyVal struct { KeyVal struct {
Key string Key string
Val string Val string
@ -33,6 +33,7 @@ type (
tags []KeyVal // Any additional info not accommodated by above fields tags []KeyVal // Any additional info not accommodated by above fields
} }
// ObjectRequest represents object request data.
ObjectRequest struct { ObjectRequest struct {
Bucket string Bucket string
Object string Object string
@ -40,7 +41,7 @@ type (
} }
) )
// Key used for Get/SetReqInfo // Key used for Get/SetReqInfo.
type contextKeyType string type contextKeyType string
const ctxRequestInfo = contextKeyType("NeoFS-S3-Gate") const ctxRequestInfo = contextKeyType("NeoFS-S3-Gate")
@ -54,7 +55,7 @@ var (
var ( var (
// RFC7239 defines a new "Forwarded: " header designed to replace the // RFC7239 defines a new "Forwarded: " header designed to replace the
// existing use of X-Forwarded-* headers. // existing use of X-Forwarded-* headers.
// e.g. Forwarded: for=192.0.2.60;proto=https;by=203.0.113.43 // e.g. Forwarded: for=192.0.2.60;proto=https;by=203.0.113.43.
forwarded = http.CanonicalHeaderKey("Forwarded") forwarded = http.CanonicalHeaderKey("Forwarded")
// Allows for a sub-match of the first value after 'for=' to the next // Allows for a sub-match of the first value after 'for=' to the next
// comma, semi-colon or space. The match is case-insensitive. // comma, semi-colon or space. The match is case-insensitive.
@ -125,7 +126,7 @@ func prepareContext(w http.ResponseWriter, r *http.Request) context.Context {
})) }))
} }
// NewReqInfo : // NewReqInfo returns new ReqInfo based on parameters.
func NewReqInfo(w http.ResponseWriter, r *http.Request, req ObjectRequest) *ReqInfo { func NewReqInfo(w http.ResponseWriter, r *http.Request, req ObjectRequest) *ReqInfo {
return &ReqInfo{ return &ReqInfo{
API: req.Method, API: req.Method,
@ -138,7 +139,7 @@ func NewReqInfo(w http.ResponseWriter, r *http.Request, req ObjectRequest) *ReqI
} }
} }
// AppendTags - appends key/val to ReqInfo.tags // AppendTags - appends key/val to ReqInfo.tags.
func (r *ReqInfo) AppendTags(key string, val string) *ReqInfo { func (r *ReqInfo) AppendTags(key string, val string) *ReqInfo {
if r == nil { if r == nil {
return nil return nil
@ -149,7 +150,7 @@ func (r *ReqInfo) AppendTags(key string, val string) *ReqInfo {
return r return r
} }
// SetTags - sets key/val to ReqInfo.tags // SetTags - sets key/val to ReqInfo.tags.
func (r *ReqInfo) SetTags(key string, val string) *ReqInfo { func (r *ReqInfo) SetTags(key string, val string) *ReqInfo {
if r == nil { if r == nil {
return nil return nil
@ -172,7 +173,7 @@ func (r *ReqInfo) SetTags(key string, val string) *ReqInfo {
return r return r
} }
// GetTags - returns the user defined tags // GetTags - returns the user defined tags.
func (r *ReqInfo) GetTags() []KeyVal { func (r *ReqInfo) GetTags() []KeyVal {
if r == nil { if r == nil {
return nil return nil

View file

@ -14,7 +14,7 @@ import (
) )
type ( type (
// ErrorResponse - error response format // ErrorResponse - error response format.
ErrorResponse struct { ErrorResponse struct {
XMLName xml.Name `xml:"Error" json:"-"` XMLName xml.Name `xml:"Error" json:"-"`
Code string Code string
@ -32,11 +32,11 @@ type (
// Captures the server string returned in response header. // Captures the server string returned in response header.
Server string `xml:"-" json:"-"` Server string `xml:"-" json:"-"`
// Underlying HTTP status code for the returned error // Underlying HTTP status code for the returned error.
StatusCode int `xml:"-" json:"-"` StatusCode int `xml:"-" json:"-"`
} }
// APIError structure // Error structure represents API error.
Error struct { Error struct {
Code string Code string
Description string Description string
@ -117,7 +117,7 @@ var s3ErrorResponseMap = map[string]string{
// Add new API errors here. // Add new API errors here.
} }
// WriteErrorResponse writes error headers // WriteErrorResponse writes error headers.
func WriteErrorResponse(ctx context.Context, w http.ResponseWriter, err error, reqURL *url.URL) { func WriteErrorResponse(ctx context.Context, w http.ResponseWriter, err error, reqURL *url.URL) {
code := http.StatusBadRequest code := http.StatusBadRequest
@ -141,7 +141,7 @@ func WriteErrorResponse(ctx context.Context, w http.ResponseWriter, err error, r
WriteResponse(w, code, encodedErrorResponse, MimeXML) WriteResponse(w, code, encodedErrorResponse, MimeXML)
} }
// If none of the http routes match respond with appropriate errors // If none of the http routes match respond with appropriate errors.
func errorResponseHandler(w http.ResponseWriter, r *http.Request) { func errorResponseHandler(w http.ResponseWriter, r *http.Request) {
desc := fmt.Sprintf("Unknown API request at %s", r.URL.Path) desc := fmt.Sprintf("Unknown API request at %s", r.URL.Path)
WriteErrorResponse(r.Context(), w, Error{ WriteErrorResponse(r.Context(), w, Error{
@ -151,7 +151,7 @@ func errorResponseHandler(w http.ResponseWriter, r *http.Request) {
}, r.URL) }, r.URL)
} }
// Write http common headers // Write http common headers.
func setCommonHeaders(w http.ResponseWriter) { func setCommonHeaders(w http.ResponseWriter) {
w.Header().Set(hdrServerInfo, "NeoFS-S3-Gate/"+misc.Version) w.Header().Set(hdrServerInfo, "NeoFS-S3-Gate/"+misc.Version)
w.Header().Set(hdrAcceptRanges, "bytes") w.Header().Set(hdrAcceptRanges, "bytes")
@ -168,6 +168,7 @@ func removeSensitiveHeaders(h http.Header) {
h.Del(hdrSSECopyKey) h.Del(hdrSSECopyKey)
} }
// WriteResponse writes given statusCode and response into w (with mType header if set).
func WriteResponse(w http.ResponseWriter, statusCode int, response []byte, mType mimeType) { func WriteResponse(w http.ResponseWriter, statusCode int, response []byte, mType mimeType) {
setCommonHeaders(w) setCommonHeaders(w)
if mType != MimeNone { if mType != MimeNone {
@ -214,6 +215,8 @@ func EncodeToResponse(w http.ResponseWriter, response interface{}) error {
// WriteResponse(w, http.StatusOK, response, MimeXML) // WriteResponse(w, http.StatusOK, response, MimeXML)
// } // }
// WriteSuccessResponseHeadersOnly writes HTTP (200) OK response with no data
// to the client.
func WriteSuccessResponseHeadersOnly(w http.ResponseWriter) { func WriteSuccessResponseHeadersOnly(w http.ResponseWriter) {
WriteResponse(w, http.StatusOK, nil, MimeNone) WriteResponse(w, http.StatusOK, nil, MimeNone)
} }

View file

@ -14,6 +14,7 @@ import (
) )
type ( type (
// Handler is an S3 API handler interface.
Handler interface { Handler interface {
HeadObjectHandler(http.ResponseWriter, *http.Request) HeadObjectHandler(http.ResponseWriter, *http.Request)
CopyObjectPartHandler(http.ResponseWriter, *http.Request) CopyObjectPartHandler(http.ResponseWriter, *http.Request)
@ -158,6 +159,7 @@ func logErrorResponse(l *zap.Logger) mux.MiddlewareFunc {
} }
} }
// GetRequestID returns request ID from response writer or context.
func GetRequestID(v interface{}) string { func GetRequestID(v interface{}) string {
switch t := v.(type) { switch t := v.(type) {
case context.Context: case context.Context:
@ -169,6 +171,8 @@ func GetRequestID(v interface{}) string {
} }
} }
// Attach adds S3 API handlers from h to r for domains with m client limit using
// center authentication and log logger.
func Attach(r *mux.Router, domains []string, m MaxClients, h Handler, center auth.Center, log *zap.Logger) { func Attach(r *mux.Router, domains []string, m MaxClients, h Handler, center auth.Center, log *zap.Logger) {
api := r.PathPrefix(SlashSeparator).Subrouter() api := r.PathPrefix(SlashSeparator).Subrouter()

View file

@ -9,6 +9,7 @@ import (
"go.uber.org/zap" "go.uber.org/zap"
) )
// AttachUserAuth adds user authentication via center to router using log for logging.
func AttachUserAuth(router *mux.Router, center auth.Center, log *zap.Logger) { func AttachUserAuth(router *mux.Router, center auth.Center, log *zap.Logger) {
router.Use(func(h http.Handler) http.Handler { router.Use(func(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@ -21,7 +22,6 @@ func AttachUserAuth(router *mux.Router, center auth.Center, log *zap.Logger) {
h.ServeHTTP(w, r.WithContext( h.ServeHTTP(w, r.WithContext(
sdk.SetBearerToken(r.Context(), token))) sdk.SetBearerToken(r.Context(), token)))
}) })
}) })
} }

View file

@ -7,6 +7,7 @@ import (
"github.com/gorilla/mux" "github.com/gorilla/mux"
) )
// Healthy is a health check interface.
type Healthy interface { type Healthy interface {
Status() error Status() error
} }

View file

@ -34,8 +34,8 @@ const (
defaultMaxClientsDeadline = time.Second * 30 defaultMaxClientsDeadline = time.Second * 30
) )
const ( // settings const ( // Settings.
// Logger: // Logger.
cfgLoggerLevel = "logger.level" cfgLoggerLevel = "logger.level"
cfgLoggerFormat = "logger.format" cfgLoggerFormat = "logger.format"
cfgLoggerTraceLevel = "logger.trace_level" cfgLoggerTraceLevel = "logger.trace_level"
@ -44,47 +44,47 @@ const ( // settings
cfgLoggerSamplingInitial = "logger.sampling.initial" cfgLoggerSamplingInitial = "logger.sampling.initial"
cfgLoggerSamplingThereafter = "logger.sampling.thereafter" cfgLoggerSamplingThereafter = "logger.sampling.thereafter"
// KeepAlive // KeepAlive.
cfgKeepaliveTime = "keepalive.time" cfgKeepaliveTime = "keepalive.time"
cfgKeepaliveTimeout = "keepalive.timeout" cfgKeepaliveTimeout = "keepalive.timeout"
cfgKeepalivePermitWithoutStream = "keepalive.permit_without_stream" cfgKeepalivePermitWithoutStream = "keepalive.permit_without_stream"
// Keys // Keys.
cfgNeoFSPrivateKey = "neofs-key" cfgNeoFSPrivateKey = "neofs-key"
cfgGateAuthPrivateKey = "auth-key" cfgGateAuthPrivateKey = "auth-key"
// HTTPS/TLS // HTTPS/TLS.
cfgTLSKeyFile = "tls.key_file" cfgTLSKeyFile = "tls.key_file"
cfgTLSCertFile = "tls.cert_file" cfgTLSCertFile = "tls.cert_file"
// Timeouts // Timeouts.
cfgConnectionTTL = "con_ttl" cfgConnectionTTL = "con_ttl"
cfgConnectTimeout = "connect_timeout" cfgConnectTimeout = "connect_timeout"
cfgRequestTimeout = "request_timeout" cfgRequestTimeout = "request_timeout"
cfgRebalanceTimer = "rebalance_timer" cfgRebalanceTimer = "rebalance_timer"
// MaxClients // MaxClients.
cfgMaxClientsCount = "max_clients_count" cfgMaxClientsCount = "max_clients_count"
cfgMaxClientsDeadline = "max_clients_deadline" cfgMaxClientsDeadline = "max_clients_deadline"
// gRPC // gRPC.
cfgGRPCVerbose = "verbose" cfgGRPCVerbose = "verbose"
// Metrics / Profiler / Web // Metrics / Profiler / Web.
cfgEnableMetrics = "metrics" cfgEnableMetrics = "metrics"
cfgEnableProfiler = "pprof" cfgEnableProfiler = "pprof"
cfgListenAddress = "listen_address" cfgListenAddress = "listen_address"
cfgListenDomains = "listen_domains" cfgListenDomains = "listen_domains"
// Peers // Peers.
cfgPeers = "peers" cfgPeers = "peers"
// Application // Application.
cfgApplicationName = "app.name" cfgApplicationName = "app.name"
cfgApplicationVersion = "app.version" cfgApplicationVersion = "app.version"
cfgApplicationBuildTime = "app.build_time" cfgApplicationBuildTime = "app.build_time"
// command line args // Command line args.
cmdHelp = "help" cmdHelp = "help"
cmdVersion = "version" cmdVersion = "version"
) )
@ -105,10 +105,9 @@ var ignore = map[string]struct{}{
func (empty) Read([]byte) (int, error) { return 0, io.EOF } func (empty) Read([]byte) (int, error) { return 0, io.EOF }
func fetchPeers(l *zap.Logger, v *viper.Viper) map[string]float64 { func fetchPeers(l *zap.Logger, v *viper.Viper) map[string]float64 {
peers := make(map[string]float64, 0) peers := make(map[string]float64)
for i := 0; ; i++ { for i := 0; ; i++ {
key := cfgPeers + "." + strconv.Itoa(i) + "." key := cfgPeers + "." + strconv.Itoa(i) + "."
address := v.GetString(key + "address") address := v.GetString(key + "address")
weight := v.GetFloat64(key + "weight") weight := v.GetFloat64(key + "weight")

View file

@ -22,6 +22,7 @@ import (
) )
type ( type (
// App is the main application structure.
App struct { App struct {
cli pool.Client cli pool.Client
ctr auth.Center ctr auth.Center
@ -179,6 +180,7 @@ func newApp(ctx context.Context, l *zap.Logger, v *viper.Viper) *App {
} }
} }
// Wait waits for application to finish.
func (a *App) Wait() { func (a *App) Wait() {
a.log.Info("application started") a.log.Info("application started")
@ -192,6 +194,7 @@ func (a *App) Wait() {
a.log.Info("application finished") a.log.Info("application finished")
} }
// Server runs HTTP server to handle S3 API requests.
func (a *App) Server(ctx context.Context) { func (a *App) Server(ctx context.Context) {
var ( var (
err error err error
@ -256,6 +259,7 @@ func (a *App) Server(ctx context.Context) {
close(a.webDone) close(a.webDone)
} }
// Worker runs client worker.
func (a *App) Worker(ctx context.Context) { func (a *App) Worker(ctx context.Context) {
a.cli.Worker(ctx) a.cli.Worker(ctx)
a.log.Info("stopping worker") a.log.Info("stopping worker")

View file

@ -1,12 +1,16 @@
package misc package misc
const ( const (
// ApplicationName is gateway name.
ApplicationName = "neofs-s3-gate" ApplicationName = "neofs-s3-gate"
// Prefix is configuration environment variables prefix.
Prefix = "S3_GW" Prefix = "S3_GW"
) )
var ( var (
Build = "now" // Build holds build timestamp.
Build = "now"
// Version contains application version.
Version = "dev" Version = "dev"
) )