Merge pull request #45 from roman-khimov/build-fixes

Build fixes
This commit is contained in:
Roman Khimov 2021-05-14 10:25:26 +03:00 committed by GitHub
commit 2b967db7c0
25 changed files with 277 additions and 117 deletions

8
.gitignore vendored
View file

@ -7,3 +7,11 @@ vendor
# tempfiles
.DS_Store
*~
# binary
bin/
# coverage
coverage.txt
coverage.html

59
.golangci.yml Normal file
View file

@ -0,0 +1,59 @@
# This file contains all available configuration options
# with their default values.
# options for analysis running
run:
# timeout for analysis, e.g. 30s, 5m, default is 1m
timeout: 5m
# include test files or not, default is true
tests: true
# output configuration options
output:
# colored-line-number|line-number|json|tab|checkstyle|code-climate, default is "colored-line-number"
format: tab
# all available settings of specific linters
linters-settings:
exhaustive:
# indicates that switch statements are to be considered exhaustive if a
# 'default' case is present, even if all enum members aren't listed in the
# switch
default-signifies-exhaustive: true
govet:
# report about shadowed variables
check-shadowing: false
linters:
enable:
# mandatory linters
- govet
- golint
# some default golangci-lint linters
- deadcode
- errcheck
- gosimple
- ineffassign
- staticcheck
- structcheck
- typecheck
- unused
- varcheck
# extra linters
- exhaustive
- godot
- gofmt
- whitespace
- goimports
disable-all: true
fast: false
issues:
include:
- EXC0002 # should have a comment
- EXC0003 # test/Test ... consider calling this
- EXC0004 # govet
- EXC0005 # C-style breaks

View file

@ -4,36 +4,20 @@ WORKDIR /src
RUN set -x \
&& apt update \
&& apt install -y upx-ucl
&& apt install -y make
COPY . /src
ARG VERSION=dev
# https://github.com/golang/go/wiki/Modules#how-do-i-use-vendoring-with-modules-is-vendoring-going-away
# go build -mod=vendor
# The -gcflags "all=-N -l" flag helps us get a better debug experience
RUN set -x \
&& export BUILD=$(date -u +%s%N) \
&& export REPO=$(go list -m) \
&& export LDFLAGS="-X ${REPO}/misc.Version=${VERSION} -X ${REPO}/misc.Build=${BUILD}" \
&& export GOGC=off \
&& export CGO_ENABLED=0 \
&& [ -d "./vendor" ] || go mod vendor \
&& go build \
-v \
-mod=vendor \
-trimpath \
-ldflags "${LDFLAGS}" \
-o /go/bin/neofs-s3 ./cmd/gate \
&& upx -3 /go/bin/neofs-s3
RUN make
# Executable image
FROM scratch
WORKDIR /
COPY --from=builder /go/bin/neofs-s3 /bin/neofs-s3
COPY --from=builder /src/bin/neofs-s3-gw /bin/neofs-s3-gw
COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
ENTRYPOINT ["/bin/neofs-s3"]
ENTRYPOINT ["/bin/neofs-s3-gw"]

129
Makefile
View file

@ -1,43 +1,106 @@
-include .env
-include help.mk
HUB_IMAGE=nspccdev/neofs
#!/usr/bin/make -f
REPO ?= $(shell go list -m)
VERSION ?= "$(shell git describe --tags 2>/dev/null || git rev-parse --short HEAD | sed 's/^v//')"
BUILD_VERSION ?= "$(shell git describe --abbrev=0 --tags | sed 's/^v//')"
.PHONY: format deps image publish
BIN_NAME=neofs-s3-gw
HUB_IMAGE="nspccdev/$(BIN_NAME)"
HUB_TAG ?= "$(shell echo ${VERSION} | sed 's/^v//')"
BINDIR = bin
BIN = "$(BINDIR)/$(BIN_NAME)"
# Show current version
version:
@echo $(BUILD_VERSION)
.PHONY: help all dep clean format test cover lint docker/lint image-push image dirty-image
# Make all binaries
all: $(BIN)
$(BIN): $(BINDIR) dep
@echo "⇒ Build $@"
CGO_ENABLED=0 \
go build -v -trimpath \
-ldflags "-X main.Version=$(VERSION)" \
-o $@ ./cmd/gate
$(BINDIR):
@echo "⇒ Ensure dir: $@"
@mkdir -p $@
# Pull go dependencies
dep:
@printf "⇒ Download requirements: "
@CGO_ENABLED=0 \
go mod download && echo OK
@printf "⇒ Tidy requirements: "
@CGO_ENABLED=0 \
go mod tidy -v && echo OK
# Run tests
test:
@go test ./... -cover
# Run tests with race detection and produce coverage output
cover:
@go test -v -race ./... -coverprofile=coverage.txt -covermode=atomic
@go tool cover -html=coverage.txt -o coverage.html
# Reformat code
format:
@[ ! -z `which goimports` ] || (echo "install goimports" && exit 2)
@for f in `find . -type f -name '*.go' -not -path './vendor/*' -not -name '*.pb.go' -prune`; do \
echo "⇒ Processing $$f"; \
goimports -w $$f; \
done
@echo "⇒ Processing gofmt check"
@gofmt -s -w ./
@echo "⇒ Processing goimports check"
@goimports -w ./
# Check and ensure dependencies
deps:
@printf "⇒ Ensure vendor: "
@go mod tidy -v && echo OK || (echo fail && exit 2)
@printf "⇒ Download requirements: "
@go mod download && echo OK || (echo fail && exit 2)
@printf "⇒ Store vendor localy: "
@go mod vendor && echo OK || (echo fail && exit 2)
# Build current docker image
image: deps
@echo "⇒ Build docker-image"
# Build clean Docker image
image:
@echo "⇒ Build NeoFS S3 Gateway docker image "
@docker build \
--build-arg VERSION=$(BUILD_VERSION) \
-f Dockerfile \
-t $(HUB_IMAGE)-s3-gate:$(BUILD_VERSION) .
--build-arg REPO=$(REPO) \
--build-arg VERSION=$(VERSION) \
--rm \
-f Dockerfile \
-t $(HUB_IMAGE):$(HUB_TAG) .
# Publish docker image
publish:
@echo "${B}${G}⇒ publish docker image ${R}"
@docker push $(HUB_IMAGE)-s3-gate:$(VERSION)
# Push Docker image to the hub
image-push:
@echo "⇒ Publish image"
@docker push $(HUB_IMAGE):$(HUB_TAG)
# Build dirty Docker image
dirty-image:
@echo "⇒ Build NeoFS S3 Gateway dirty docker image "
@docker build \
--build-arg REPO=$(REPO) \
--build-arg VERSION=$(VERSION) \
--rm \
-f Dockerfile.dirty \
-t $(HUB_IMAGE)-dirty:$(HUB_TAG) .
# Run linters
lint:
@golangci-lint --timeout=5m run
# Run linters in Docker
docker/lint:
docker run --rm -it \
-v `pwd`:/src \
-u `stat -c "%u:%g" .` \
--env HOME=/src \
golangci/golangci-lint:v1.40 bash -c 'cd /src/ && make lint'
# Show current version
version:
@echo $(VERSION)
# Show this help prompt
help:
@echo ' Usage:'
@echo ''
@echo ' make <target>'
@echo ''
@echo ' Targets:'
@echo ''
@awk '/^#/{ comment = substr($$0,3) } comment && /^[a-zA-Z][a-zA-Z0-9_-]+ ?:/{ print " ", $$1, comment }' $(MAKEFILE_LIST) | column -t -s ':' | grep -v 'IGNORE' | sort -u
# Clean up
clean:
rm -rf $(BINDIR)

View file

@ -24,6 +24,7 @@ import (
var authorizationFieldRegexp = regexp.MustCompile(`AWS4-HMAC-SHA256 Credential=(?P<access_key_id_cid>[^/]+)/(?P<access_key_id_oid>[^/]+)/(?P<date>[^/]+)/(?P<region>[^/]*)/(?P<service>[^/]+)/aws4_request,\s*SignedHeaders=(?P<signed_header_fields>.+),\s*Signature=(?P<v4_signature>.+)`)
type (
// Center is a user authentication interface.
Center interface {
Authenticate(request *http.Request) (*token.BearerToken, error)
}
@ -33,6 +34,7 @@ type (
cli bearer.Credentials
}
// Params stores node connection parameters.
Params struct {
Client sdk.Client
Logger *zap.Logger

View file

@ -118,7 +118,7 @@ const (
ErrInvalidTagDirective
// Add new error codes here.
// SSE-S3 related API errors
// SSE-S3 related API errors.
ErrInvalidEncryptionMethod
// Server-Side-Encryption (with Customer provided key) related API errors.
@ -157,8 +157,8 @@ const (
// Add new extended error codes here.
// MinIO extended errors.
// ErrReadQuorum
// ErrWriteQuorum
// ErrReadQuorum
// ErrWriteQuorum
ErrParentIsObject
ErrStorageFull
ErrRequestBodyParse
@ -170,7 +170,7 @@ const (
ErrOperationTimedOut
ErrOperationMaxedOut
ErrInvalidRequest
// MinIO storage class error codes
// MinIO storage class error codes.
ErrInvalidStorageClass
ErrBackendDown
// Add new extended error codes here.
@ -192,7 +192,7 @@ const (
ErrAdminCredentialsMismatch
ErrInsecureClientRequest
ErrObjectTampered
// Bucket Quota error codes
// Bucket Quota error codes.
ErrAdminBucketQuotaExceeded
ErrAdminNoSuchQuotaConfiguration
ErrAdminBucketQuotaDisabled
@ -205,7 +205,7 @@ const (
ErrHealOverlappingPaths
ErrIncorrectContinuationToken
// S3 Select Errors
// S3 Select Errors.
ErrEmptyRequestBody
ErrUnsupportedFunction
ErrInvalidExpressionType
@ -1625,7 +1625,7 @@ func GetAPIError(code ErrorCode) Error {
}
// getErrorResponse gets in standard error and resource value and
// provides a encodable populated response values
// provides a encodable populated response values.
func getAPIErrorResponse(ctx context.Context, err error, resource, requestID, hostID string) ErrorResponse {
code := "BadRequest"
desc := err.Error()
@ -1803,21 +1803,21 @@ func (e BucketLifecycleNotFound) Error() string {
return "No bucket lifecycle configuration found for bucket : " + e.Bucket
}
// BucketSSEConfigNotFound - no bucket encryption found
// BucketSSEConfigNotFound - no bucket encryption found.
type BucketSSEConfigNotFound GenericError
func (e BucketSSEConfigNotFound) Error() string {
return "No bucket encryption configuration found for bucket: " + e.Bucket
}
// BucketTaggingNotFound - no bucket tags found
// BucketTaggingNotFound - no bucket tags found.
type BucketTaggingNotFound GenericError
func (e BucketTaggingNotFound) Error() string {
return "No bucket tags found for bucket: " + e.Bucket
}
// BucketObjectLockConfigNotFound - no bucket object lock config found
// BucketObjectLockConfigNotFound - no bucket object lock config found.
type BucketObjectLockConfigNotFound GenericError
func (e BucketObjectLockConfigNotFound) Error() string {
@ -1874,7 +1874,7 @@ func (e ObjectNamePrefixAsSlash) Error() string {
return "Object name contains forward slash as pefix: " + e.Bucket + "#" + e.Object
}
// AllAccessDisabled All access to this object has been disabled
// AllAccessDisabled All access to this object has been disabled.
type AllAccessDisabled GenericError
// Error returns string an error formatted as the given text.
@ -1945,7 +1945,7 @@ func (e InvalidUploadID) Error() string {
return "Invalid upload id " + e.UploadID
}
// InvalidPart One or more of the specified parts could not be found
// InvalidPart One or more of the specified parts could not be found.
type InvalidPart struct {
PartNumber int
ExpETag string
@ -1975,21 +1975,21 @@ func (e PartTooBig) Error() string {
return "Part size bigger than the allowed limit"
}
// InvalidETag error returned when the etag has changed on disk
// InvalidETag error returned when the etag has changed on disk.
type InvalidETag struct{}
func (e InvalidETag) Error() string {
return "etag of the object has changed"
}
// NotImplemented If a feature is not implemented
// NotImplemented If a feature is not implemented.
type NotImplemented struct{}
func (e NotImplemented) Error() string {
return "Not Implemented"
}
// UnsupportedMetadata - unsupported metadata
// UnsupportedMetadata - unsupported metadata.
type UnsupportedMetadata struct{}
func (e UnsupportedMetadata) Error() string {
@ -2003,14 +2003,14 @@ func (e BackendDown) Error() string {
return "Backend down"
}
// PreConditionFailed - Check if copy precondition failed
// PreConditionFailed - Check if copy precondition failed.
type PreConditionFailed struct{}
func (e PreConditionFailed) Error() string {
return "At least one of the pre-conditions you specified did not hold"
}
// DeleteError - returns when cant remove object
// DeleteError - returns when cant remove object.
type DeleteError struct {
Err error
Object string

View file

@ -14,6 +14,7 @@ type (
obj layer.Client
}
// Params holds logger and client.
Params struct {
Log *zap.Logger
Obj layer.Client
@ -24,6 +25,7 @@ const notSupported = "Not supported by NeoFS S3 Gate: "
var _ api.Handler = (*handler)(nil)
// New creates new api.Handler using given logger and client.
func New(log *zap.Logger, obj layer.Client) (api.Handler, error) {
switch {
case obj == nil:

View file

@ -68,7 +68,7 @@ func (h *handler) DeleteObjectHandler(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusNoContent)
}
// DeleteMultipleObjectsHandler :
// DeleteMultipleObjectsHandler handles multiple delete requests.
func (h *handler) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Request) {
var (
req = mux.Vars(r)

View file

@ -109,7 +109,7 @@ func (h *handler) HeadBucketHandler(w http.ResponseWriter, r *http.Request) {
code := http.StatusBadRequest
if st, ok := status.FromError(err); ok && st != nil {
switch st.Code() {
switch st.Code() { //nolint:exhaustive // we have default value set above
case codes.NotFound:
code = http.StatusNotFound
case codes.PermissionDenied:

View file

@ -22,12 +22,14 @@ type listObjectsArgs struct {
Version string
}
// VersioningConfiguration contains VersioningConfiguration XML representation.
type VersioningConfiguration struct {
XMLName xml.Name `xml:"VersioningConfiguration"`
Text string `xml:",chardata"`
Xmlns string `xml:"xmlns,attr"`
}
// ListMultipartUploadsResult contains ListMultipartUploadsResult XML representation.
type ListMultipartUploadsResult struct {
XMLName xml.Name `xml:"ListMultipartUploadsResult"`
Text string `xml:",chardata"`
@ -36,6 +38,7 @@ type ListMultipartUploadsResult struct {
var maxObjectList = 10000 // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse.
// ListBucketsHandler handles bucket listing requests.
func (h *handler) ListBucketsHandler(w http.ResponseWriter, r *http.Request) {
var (
err error
@ -134,6 +137,7 @@ func (h *handler) listObjects(w http.ResponseWriter, r *http.Request) (*listObje
return arg, list, nil
}
// ListObjectsV1Handler handles objects listing requests for API version 1.
func (h *handler) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) {
var rid = api.GetRequestID(r.Context())
if arg, list, err := h.listObjects(w, r); err != nil {
@ -193,6 +197,7 @@ func encodeV1(arg *listObjectsArgs, list *layer.ListObjectsInfo) *ListObjectsRes
return res
}
// ListObjectsV2Handler handles objects listing requests for API version 2.
func (h *handler) ListObjectsV2Handler(w http.ResponseWriter, r *http.Request) {
var rid = api.GetRequestID(r.Context())
if arg, list, err := h.listObjects(w, r); err != nil {
@ -278,6 +283,7 @@ func parseListObjectArgs(r *http.Request) (*listObjectsArgs, error) {
return &res, nil
}
// GetBucketVersioningHandler implements bucket versioning getter handler.
func (h *handler) GetBucketVersioningHandler(w http.ResponseWriter, r *http.Request) {
var (
rid = api.GetRequestID(r.Context())
@ -299,6 +305,7 @@ func (h *handler) GetBucketVersioningHandler(w http.ResponseWriter, r *http.Requ
}
}
// ListMultipartUploadsHandler implements multipart uploads listing handler.
func (h *handler) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Request) {
var (
rid = api.GetRequestID(r.Context())

View file

@ -2,7 +2,7 @@ package handler
import "encoding/xml"
// ListBucketsResponse - format for list buckets response
// ListBucketsResponse - format for list buckets response.
type ListBucketsResponse struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListAllMyBucketsResult" json:"-"`
@ -45,13 +45,13 @@ type ListObjectsV2Response struct {
EncodingType string `xml:"EncodingType,omitempty"`
}
// Bucket container for bucket metadata
// Bucket container for bucket metadata.
type Bucket struct {
Name string
CreationDate string // time string of format "2006-01-02T15:04:05.000Z"
}
// Owner - bucket owner/principal
// Owner - bucket owner/principal.
type Owner struct {
ID string
DisplayName string
@ -87,12 +87,12 @@ type ListObjectsResponse struct {
EncodingType string `xml:"EncodingType,omitempty"`
}
// CommonPrefix container for prefix response in ListObjectsResponse
// CommonPrefix container for prefix response in ListObjectsResponse.
type CommonPrefix struct {
Prefix string
}
// Object container for object metadata
// Object container for object metadata.
type Object struct {
Key string
LastModified string // time string of format "2006-01-02T15:04:05.000Z"
@ -118,7 +118,7 @@ type LocationResponse struct {
Location string `xml:",chardata"`
}
// CopyObjectResponse container returns ETag and LastModified of the successfully copied object
// CopyObjectResponse container returns ETag and LastModified of the successfully copied object.
type CopyObjectResponse struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CopyObjectResult" json:"-"`
LastModified string // time string of format "2006-01-02T15:04:05.000Z"
@ -127,7 +127,6 @@ type CopyObjectResponse struct {
// MarshalXML - StringMap marshals into XML.
func (s StringMap) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
tokens := []xml.Token{start}
for key, value := range s {

View file

@ -1,6 +1,6 @@
package api
// Standard S3 HTTP response constants
// Standard S3 HTTP response constants.
const (
LastModified = "Last-Modified"
Date = "Date"

View file

@ -12,6 +12,7 @@ import (
)
type (
// BucketInfo stores basic bucket data.
BucketInfo struct {
Name string
CID *container.ID
@ -19,6 +20,7 @@ type (
Created time.Time
}
// ListObjectsParams represents object listing request parameters.
ListObjectsParams struct {
Bucket string
Prefix string

View file

@ -25,6 +25,7 @@ type (
log *zap.Logger
}
// Params stores basic API parameters.
Params struct {
Pool pool.Client
Logger *zap.Logger
@ -32,6 +33,7 @@ type (
Credential neofs.Credentials
}
// GetObjectParams stores object get request parameters.
GetObjectParams struct {
Bucket string
Object string
@ -40,6 +42,7 @@ type (
Writer io.Writer
}
// PutObjectParams stores object put request parameters.
PutObjectParams struct {
Bucket string
Object string
@ -48,6 +51,7 @@ type (
Header map[string]string
}
// CopyObjectParams stores object copy request parameters.
CopyObjectParams struct {
SrcBucket string
DstBucket string
@ -56,10 +60,12 @@ type (
Header map[string]string
}
// NeoFS provides basic NeoFS interface.
NeoFS interface {
Get(ctx context.Context, address *object.Address) (*object.Object, error)
}
// Client provides S3 API client interface.
Client interface {
NeoFS
@ -81,11 +87,13 @@ type (
)
var (
ErrObjectExists = errors.New("object exists")
// ErrObjectExists is returned on attempts to create already existing object.
ErrObjectExists = errors.New("object exists")
// ErrObjectNotExists is returned on attempts to work with non-existing object.
ErrObjectNotExists = errors.New("object not exists")
)
// NewGatewayLayer creates instance of layer. It checks credentials
// NewLayer creates instance of layer. It checks credentials
// and establishes gRPC connection with node.
func NewLayer(log *zap.Logger, cli sdk.Client) Client {
return &layer{
@ -103,7 +111,7 @@ func (n *layer) Owner(ctx context.Context) *owner.ID {
return n.cli.Owner()
}
// Get NeoFS Object by refs.Address (should be used by auth.Center)
// Get NeoFS Object by refs.Address (should be used by auth.Center).
func (n *layer) Get(ctx context.Context, address *object.Address) (*object.Object, error) {
return n.cli.Object().Get(ctx, address)
}

View file

@ -12,6 +12,7 @@ import (
)
type (
// ObjectInfo holds S3 object data.
ObjectInfo struct {
id *object.ID
isDir bool
@ -52,6 +53,7 @@ type (
const (
rootSeparator = "root://"
// PathSeparator is a path components separator string.
PathSeparator = string(os.PathSeparator)
)
@ -140,11 +142,14 @@ func nameFromObject(o *object.Object) (string, string) {
return NameFromString(name)
}
// NameFromString splits name into base file name and directory path.
func NameFromString(name string) (string, string) {
ind := strings.LastIndex(name, PathSeparator)
return name[ind+1:], name[:ind+1]
}
// ID returns object ID from ObjectInfo.
func (o *ObjectInfo) ID() *object.ID { return o.id }
// IsDir allows to check if object is a directory.
func (o *ObjectInfo) IsDir() bool { return o.isDir }

View file

@ -6,6 +6,7 @@ import (
)
type (
// MaxClients provides HTTP handler wrapper with client limit.
MaxClients interface {
Handle(http.HandlerFunc) http.HandlerFunc
}
@ -18,6 +19,8 @@ type (
const defaultRequestDeadline = time.Second * 30
// NewMaxClientsMiddleware returns MaxClients interface with handler wrapper based on
// provided count and timeout limits.
func NewMaxClientsMiddleware(count int, timeout time.Duration) MaxClients {
if timeout <= 0 {
timeout = defaultRequestDeadline
@ -29,6 +32,7 @@ func NewMaxClientsMiddleware(count int, timeout time.Duration) MaxClients {
}
}
// Handler wraps HTTP handler function with logic limiting access to it.
func (m *maxClients) Handle(f http.HandlerFunc) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
if m.pool == nil {

View file

@ -20,7 +20,7 @@ type (
}
// HTTPStats holds statistics information about
// HTTP requests made by all clients
// HTTP requests made by all clients.
HTTPStats struct {
currentS3Requests HTTPAPIStats
totalS3Requests HTTPAPIStats
@ -63,8 +63,8 @@ var (
)
)
// collects http metrics for NeoFS S3 Gate in Prometheus specific format
// and sends to given channel
// Collects HTTP metrics for NeoFS S3 Gate in Prometheus specific format
// and sends to given channel.
func collectHTTPMetrics(ch chan<- prometheus.Metric) {
for api, value := range httpStatsMetric.currentS3Requests.Load() {
ch <- prometheus.MustNewConstMetric(
@ -103,6 +103,7 @@ func collectHTTPMetrics(ch chan<- prometheus.Metric) {
}
}
// APIStats wraps http handler for api with basic statistics collection.
func APIStats(api string, f http.HandlerFunc) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
httpStatsMetric.currentS3Requests.Inc(api)
@ -176,7 +177,7 @@ func (st *HTTPStats) getOutputBytes() uint64 {
return atomic.LoadUint64(&st.totalOutputBytes)
}
// Update statistics from http request and response data
// Update statistics from http request and response data.
func (st *HTTPStats) updateStats(api string, w http.ResponseWriter, r *http.Request, durationSecs float64) {
var code int
@ -200,7 +201,7 @@ func (st *HTTPStats) updateStats(api string, w http.ResponseWriter, r *http.Requ
}
}
// WriteHeader - writes http status code
// WriteHeader - writes http status code.
func (w *responseWrapper) WriteHeader(code int) {
w.Do(func() {
w.statusCode = code

View file

@ -13,7 +13,7 @@ import (
)
type (
// KeyVal - appended to ReqInfo.Tags
// KeyVal - appended to ReqInfo.Tags.
KeyVal struct {
Key string
Val string
@ -33,6 +33,7 @@ type (
tags []KeyVal // Any additional info not accommodated by above fields
}
// ObjectRequest represents object request data.
ObjectRequest struct {
Bucket string
Object string
@ -40,7 +41,7 @@ type (
}
)
// Key used for Get/SetReqInfo
// Key used for Get/SetReqInfo.
type contextKeyType string
const ctxRequestInfo = contextKeyType("NeoFS-S3-Gate")
@ -54,7 +55,7 @@ var (
var (
// RFC7239 defines a new "Forwarded: " header designed to replace the
// existing use of X-Forwarded-* headers.
// e.g. Forwarded: for=192.0.2.60;proto=https;by=203.0.113.43
// e.g. Forwarded: for=192.0.2.60;proto=https;by=203.0.113.43.
forwarded = http.CanonicalHeaderKey("Forwarded")
// Allows for a sub-match of the first value after 'for=' to the next
// comma, semi-colon or space. The match is case-insensitive.
@ -125,7 +126,7 @@ func prepareContext(w http.ResponseWriter, r *http.Request) context.Context {
}))
}
// NewReqInfo :
// NewReqInfo returns new ReqInfo based on parameters.
func NewReqInfo(w http.ResponseWriter, r *http.Request, req ObjectRequest) *ReqInfo {
return &ReqInfo{
API: req.Method,
@ -138,7 +139,7 @@ func NewReqInfo(w http.ResponseWriter, r *http.Request, req ObjectRequest) *ReqI
}
}
// AppendTags - appends key/val to ReqInfo.tags
// AppendTags - appends key/val to ReqInfo.tags.
func (r *ReqInfo) AppendTags(key string, val string) *ReqInfo {
if r == nil {
return nil
@ -149,7 +150,7 @@ func (r *ReqInfo) AppendTags(key string, val string) *ReqInfo {
return r
}
// SetTags - sets key/val to ReqInfo.tags
// SetTags - sets key/val to ReqInfo.tags.
func (r *ReqInfo) SetTags(key string, val string) *ReqInfo {
if r == nil {
return nil
@ -172,7 +173,7 @@ func (r *ReqInfo) SetTags(key string, val string) *ReqInfo {
return r
}
// GetTags - returns the user defined tags
// GetTags - returns the user defined tags.
func (r *ReqInfo) GetTags() []KeyVal {
if r == nil {
return nil

View file

@ -14,7 +14,7 @@ import (
)
type (
// ErrorResponse - error response format
// ErrorResponse - error response format.
ErrorResponse struct {
XMLName xml.Name `xml:"Error" json:"-"`
Code string
@ -32,11 +32,11 @@ type (
// Captures the server string returned in response header.
Server string `xml:"-" json:"-"`
// Underlying HTTP status code for the returned error
// Underlying HTTP status code for the returned error.
StatusCode int `xml:"-" json:"-"`
}
// APIError structure
// Error structure represents API error.
Error struct {
Code string
Description string
@ -117,7 +117,7 @@ var s3ErrorResponseMap = map[string]string{
// Add new API errors here.
}
// WriteErrorResponse writes error headers
// WriteErrorResponse writes error headers.
func WriteErrorResponse(ctx context.Context, w http.ResponseWriter, err error, reqURL *url.URL) {
code := http.StatusBadRequest
@ -141,7 +141,7 @@ func WriteErrorResponse(ctx context.Context, w http.ResponseWriter, err error, r
WriteResponse(w, code, encodedErrorResponse, MimeXML)
}
// If none of the http routes match respond with appropriate errors
// If none of the http routes match respond with appropriate errors.
func errorResponseHandler(w http.ResponseWriter, r *http.Request) {
desc := fmt.Sprintf("Unknown API request at %s", r.URL.Path)
WriteErrorResponse(r.Context(), w, Error{
@ -151,7 +151,7 @@ func errorResponseHandler(w http.ResponseWriter, r *http.Request) {
}, r.URL)
}
// Write http common headers
// Write http common headers.
func setCommonHeaders(w http.ResponseWriter) {
w.Header().Set(hdrServerInfo, "NeoFS-S3-Gate/"+misc.Version)
w.Header().Set(hdrAcceptRanges, "bytes")
@ -168,6 +168,7 @@ func removeSensitiveHeaders(h http.Header) {
h.Del(hdrSSECopyKey)
}
// WriteResponse writes given statusCode and response into w (with mType header if set).
func WriteResponse(w http.ResponseWriter, statusCode int, response []byte, mType mimeType) {
setCommonHeaders(w)
if mType != MimeNone {
@ -214,6 +215,8 @@ func EncodeToResponse(w http.ResponseWriter, response interface{}) error {
// WriteResponse(w, http.StatusOK, response, MimeXML)
// }
// WriteSuccessResponseHeadersOnly writes HTTP (200) OK response with no data
// to the client.
func WriteSuccessResponseHeadersOnly(w http.ResponseWriter) {
WriteResponse(w, http.StatusOK, nil, MimeNone)
}

View file

@ -14,6 +14,7 @@ import (
)
type (
// Handler is an S3 API handler interface.
Handler interface {
HeadObjectHandler(http.ResponseWriter, *http.Request)
CopyObjectPartHandler(http.ResponseWriter, *http.Request)
@ -158,6 +159,7 @@ func logErrorResponse(l *zap.Logger) mux.MiddlewareFunc {
}
}
// GetRequestID returns request ID from response writer or context.
func GetRequestID(v interface{}) string {
switch t := v.(type) {
case context.Context:
@ -169,6 +171,8 @@ func GetRequestID(v interface{}) string {
}
}
// Attach adds S3 API handlers from h to r for domains with m client limit using
// center authentication and log logger.
func Attach(r *mux.Router, domains []string, m MaxClients, h Handler, center auth.Center, log *zap.Logger) {
api := r.PathPrefix(SlashSeparator).Subrouter()

View file

@ -9,6 +9,7 @@ import (
"go.uber.org/zap"
)
// AttachUserAuth adds user authentication via center to router using log for logging.
func AttachUserAuth(router *mux.Router, center auth.Center, log *zap.Logger) {
router.Use(func(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@ -21,7 +22,6 @@ func AttachUserAuth(router *mux.Router, center auth.Center, log *zap.Logger) {
h.ServeHTTP(w, r.WithContext(
sdk.SetBearerToken(r.Context(), token)))
})
})
}

View file

@ -7,6 +7,7 @@ import (
"github.com/gorilla/mux"
)
// Healthy is a health check interface.
type Healthy interface {
Status() error
}

View file

@ -34,8 +34,8 @@ const (
defaultMaxClientsDeadline = time.Second * 30
)
const ( // settings
// Logger:
const ( // Settings.
// Logger.
cfgLoggerLevel = "logger.level"
cfgLoggerFormat = "logger.format"
cfgLoggerTraceLevel = "logger.trace_level"
@ -44,47 +44,47 @@ const ( // settings
cfgLoggerSamplingInitial = "logger.sampling.initial"
cfgLoggerSamplingThereafter = "logger.sampling.thereafter"
// KeepAlive
// KeepAlive.
cfgKeepaliveTime = "keepalive.time"
cfgKeepaliveTimeout = "keepalive.timeout"
cfgKeepalivePermitWithoutStream = "keepalive.permit_without_stream"
// Keys
// Keys.
cfgNeoFSPrivateKey = "neofs-key"
cfgGateAuthPrivateKey = "auth-key"
// HTTPS/TLS
// HTTPS/TLS.
cfgTLSKeyFile = "tls.key_file"
cfgTLSCertFile = "tls.cert_file"
// Timeouts
// Timeouts.
cfgConnectionTTL = "con_ttl"
cfgConnectTimeout = "connect_timeout"
cfgRequestTimeout = "request_timeout"
cfgRebalanceTimer = "rebalance_timer"
// MaxClients
// MaxClients.
cfgMaxClientsCount = "max_clients_count"
cfgMaxClientsDeadline = "max_clients_deadline"
// gRPC
// gRPC.
cfgGRPCVerbose = "verbose"
// Metrics / Profiler / Web
// Metrics / Profiler / Web.
cfgEnableMetrics = "metrics"
cfgEnableProfiler = "pprof"
cfgListenAddress = "listen_address"
cfgListenDomains = "listen_domains"
// Peers
// Peers.
cfgPeers = "peers"
// Application
// Application.
cfgApplicationName = "app.name"
cfgApplicationVersion = "app.version"
cfgApplicationBuildTime = "app.build_time"
// command line args
// Command line args.
cmdHelp = "help"
cmdVersion = "version"
)
@ -105,10 +105,9 @@ var ignore = map[string]struct{}{
func (empty) Read([]byte) (int, error) { return 0, io.EOF }
func fetchPeers(l *zap.Logger, v *viper.Viper) map[string]float64 {
peers := make(map[string]float64, 0)
peers := make(map[string]float64)
for i := 0; ; i++ {
key := cfgPeers + "." + strconv.Itoa(i) + "."
address := v.GetString(key + "address")
weight := v.GetFloat64(key + "weight")

View file

@ -22,6 +22,7 @@ import (
)
type (
// App is the main application structure.
App struct {
cli pool.Client
ctr auth.Center
@ -179,6 +180,7 @@ func newApp(ctx context.Context, l *zap.Logger, v *viper.Viper) *App {
}
}
// Wait waits for application to finish.
func (a *App) Wait() {
a.log.Info("application started")
@ -192,6 +194,7 @@ func (a *App) Wait() {
a.log.Info("application finished")
}
// Server runs HTTP server to handle S3 API requests.
func (a *App) Server(ctx context.Context) {
var (
err error
@ -256,6 +259,7 @@ func (a *App) Server(ctx context.Context) {
close(a.webDone)
}
// Worker runs client worker.
func (a *App) Worker(ctx context.Context) {
a.cli.Worker(ctx)
a.log.Info("stopping worker")

View file

@ -1,12 +1,16 @@
package misc
const (
// ApplicationName is gateway name.
ApplicationName = "neofs-s3-gate"
// Prefix is configuration environment variables prefix.
Prefix = "S3_GW"
)
var (
Build = "now"
// Build holds build timestamp.
Build = "now"
// Version contains application version.
Version = "dev"
)