forked from TrueCloudLab/frostfs-s3-gw
commit
2b967db7c0
25 changed files with 277 additions and 117 deletions
8
.gitignore
vendored
8
.gitignore
vendored
|
@ -7,3 +7,11 @@ vendor
|
||||||
|
|
||||||
# tempfiles
|
# tempfiles
|
||||||
.DS_Store
|
.DS_Store
|
||||||
|
*~
|
||||||
|
|
||||||
|
# binary
|
||||||
|
bin/
|
||||||
|
|
||||||
|
# coverage
|
||||||
|
coverage.txt
|
||||||
|
coverage.html
|
||||||
|
|
59
.golangci.yml
Normal file
59
.golangci.yml
Normal file
|
@ -0,0 +1,59 @@
|
||||||
|
# This file contains all available configuration options
|
||||||
|
# with their default values.
|
||||||
|
|
||||||
|
# options for analysis running
|
||||||
|
run:
|
||||||
|
# timeout for analysis, e.g. 30s, 5m, default is 1m
|
||||||
|
timeout: 5m
|
||||||
|
|
||||||
|
# include test files or not, default is true
|
||||||
|
tests: true
|
||||||
|
|
||||||
|
# output configuration options
|
||||||
|
output:
|
||||||
|
# colored-line-number|line-number|json|tab|checkstyle|code-climate, default is "colored-line-number"
|
||||||
|
format: tab
|
||||||
|
|
||||||
|
# all available settings of specific linters
|
||||||
|
linters-settings:
|
||||||
|
exhaustive:
|
||||||
|
# indicates that switch statements are to be considered exhaustive if a
|
||||||
|
# 'default' case is present, even if all enum members aren't listed in the
|
||||||
|
# switch
|
||||||
|
default-signifies-exhaustive: true
|
||||||
|
govet:
|
||||||
|
# report about shadowed variables
|
||||||
|
check-shadowing: false
|
||||||
|
|
||||||
|
linters:
|
||||||
|
enable:
|
||||||
|
# mandatory linters
|
||||||
|
- govet
|
||||||
|
- golint
|
||||||
|
|
||||||
|
# some default golangci-lint linters
|
||||||
|
- deadcode
|
||||||
|
- errcheck
|
||||||
|
- gosimple
|
||||||
|
- ineffassign
|
||||||
|
- staticcheck
|
||||||
|
- structcheck
|
||||||
|
- typecheck
|
||||||
|
- unused
|
||||||
|
- varcheck
|
||||||
|
|
||||||
|
# extra linters
|
||||||
|
- exhaustive
|
||||||
|
- godot
|
||||||
|
- gofmt
|
||||||
|
- whitespace
|
||||||
|
- goimports
|
||||||
|
disable-all: true
|
||||||
|
fast: false
|
||||||
|
|
||||||
|
issues:
|
||||||
|
include:
|
||||||
|
- EXC0002 # should have a comment
|
||||||
|
- EXC0003 # test/Test ... consider calling this
|
||||||
|
- EXC0004 # govet
|
||||||
|
- EXC0005 # C-style breaks
|
24
Dockerfile
24
Dockerfile
|
@ -4,36 +4,20 @@ WORKDIR /src
|
||||||
|
|
||||||
RUN set -x \
|
RUN set -x \
|
||||||
&& apt update \
|
&& apt update \
|
||||||
&& apt install -y upx-ucl
|
&& apt install -y make
|
||||||
|
|
||||||
COPY . /src
|
COPY . /src
|
||||||
|
|
||||||
ARG VERSION=dev
|
ARG VERSION=dev
|
||||||
|
|
||||||
# https://github.com/golang/go/wiki/Modules#how-do-i-use-vendoring-with-modules-is-vendoring-going-away
|
RUN make
|
||||||
# go build -mod=vendor
|
|
||||||
# The -gcflags "all=-N -l" flag helps us get a better debug experience
|
|
||||||
RUN set -x \
|
|
||||||
&& export BUILD=$(date -u +%s%N) \
|
|
||||||
&& export REPO=$(go list -m) \
|
|
||||||
&& export LDFLAGS="-X ${REPO}/misc.Version=${VERSION} -X ${REPO}/misc.Build=${BUILD}" \
|
|
||||||
&& export GOGC=off \
|
|
||||||
&& export CGO_ENABLED=0 \
|
|
||||||
&& [ -d "./vendor" ] || go mod vendor \
|
|
||||||
&& go build \
|
|
||||||
-v \
|
|
||||||
-mod=vendor \
|
|
||||||
-trimpath \
|
|
||||||
-ldflags "${LDFLAGS}" \
|
|
||||||
-o /go/bin/neofs-s3 ./cmd/gate \
|
|
||||||
&& upx -3 /go/bin/neofs-s3
|
|
||||||
|
|
||||||
# Executable image
|
# Executable image
|
||||||
FROM scratch
|
FROM scratch
|
||||||
|
|
||||||
WORKDIR /
|
WORKDIR /
|
||||||
|
|
||||||
COPY --from=builder /go/bin/neofs-s3 /bin/neofs-s3
|
COPY --from=builder /src/bin/neofs-s3-gw /bin/neofs-s3-gw
|
||||||
COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
|
COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
|
||||||
|
|
||||||
ENTRYPOINT ["/bin/neofs-s3"]
|
ENTRYPOINT ["/bin/neofs-s3-gw"]
|
||||||
|
|
127
Makefile
127
Makefile
|
@ -1,43 +1,106 @@
|
||||||
-include .env
|
#!/usr/bin/make -f
|
||||||
-include help.mk
|
|
||||||
|
|
||||||
HUB_IMAGE=nspccdev/neofs
|
|
||||||
|
|
||||||
|
REPO ?= $(shell go list -m)
|
||||||
VERSION ?= "$(shell git describe --tags 2>/dev/null || git rev-parse --short HEAD | sed 's/^v//')"
|
VERSION ?= "$(shell git describe --tags 2>/dev/null || git rev-parse --short HEAD | sed 's/^v//')"
|
||||||
BUILD_VERSION ?= "$(shell git describe --abbrev=0 --tags | sed 's/^v//')"
|
|
||||||
|
|
||||||
.PHONY: format deps image publish
|
BIN_NAME=neofs-s3-gw
|
||||||
|
HUB_IMAGE="nspccdev/$(BIN_NAME)"
|
||||||
|
HUB_TAG ?= "$(shell echo ${VERSION} | sed 's/^v//')"
|
||||||
|
BINDIR = bin
|
||||||
|
BIN = "$(BINDIR)/$(BIN_NAME)"
|
||||||
|
|
||||||
# Show current version
|
.PHONY: help all dep clean format test cover lint docker/lint image-push image dirty-image
|
||||||
version:
|
|
||||||
@echo $(BUILD_VERSION)
|
# Make all binaries
|
||||||
|
all: $(BIN)
|
||||||
|
|
||||||
|
$(BIN): $(BINDIR) dep
|
||||||
|
@echo "⇒ Build $@"
|
||||||
|
CGO_ENABLED=0 \
|
||||||
|
go build -v -trimpath \
|
||||||
|
-ldflags "-X main.Version=$(VERSION)" \
|
||||||
|
-o $@ ./cmd/gate
|
||||||
|
|
||||||
|
$(BINDIR):
|
||||||
|
@echo "⇒ Ensure dir: $@"
|
||||||
|
@mkdir -p $@
|
||||||
|
|
||||||
|
# Pull go dependencies
|
||||||
|
dep:
|
||||||
|
@printf "⇒ Download requirements: "
|
||||||
|
@CGO_ENABLED=0 \
|
||||||
|
go mod download && echo OK
|
||||||
|
@printf "⇒ Tidy requirements: "
|
||||||
|
@CGO_ENABLED=0 \
|
||||||
|
go mod tidy -v && echo OK
|
||||||
|
|
||||||
|
# Run tests
|
||||||
|
test:
|
||||||
|
@go test ./... -cover
|
||||||
|
|
||||||
|
# Run tests with race detection and produce coverage output
|
||||||
|
cover:
|
||||||
|
@go test -v -race ./... -coverprofile=coverage.txt -covermode=atomic
|
||||||
|
@go tool cover -html=coverage.txt -o coverage.html
|
||||||
|
|
||||||
# Reformat code
|
# Reformat code
|
||||||
format:
|
format:
|
||||||
@[ ! -z `which goimports` ] || (echo "install goimports" && exit 2)
|
@echo "⇒ Processing gofmt check"
|
||||||
@for f in `find . -type f -name '*.go' -not -path './vendor/*' -not -name '*.pb.go' -prune`; do \
|
@gofmt -s -w ./
|
||||||
echo "⇒ Processing $$f"; \
|
@echo "⇒ Processing goimports check"
|
||||||
goimports -w $$f; \
|
@goimports -w ./
|
||||||
done
|
|
||||||
|
|
||||||
# Check and ensure dependencies
|
# Build clean Docker image
|
||||||
deps:
|
image:
|
||||||
@printf "⇒ Ensure vendor: "
|
@echo "⇒ Build NeoFS S3 Gateway docker image "
|
||||||
@go mod tidy -v && echo OK || (echo fail && exit 2)
|
|
||||||
@printf "⇒ Download requirements: "
|
|
||||||
@go mod download && echo OK || (echo fail && exit 2)
|
|
||||||
@printf "⇒ Store vendor localy: "
|
|
||||||
@go mod vendor && echo OK || (echo fail && exit 2)
|
|
||||||
|
|
||||||
# Build current docker image
|
|
||||||
image: deps
|
|
||||||
@echo "⇒ Build docker-image"
|
|
||||||
@docker build \
|
@docker build \
|
||||||
--build-arg VERSION=$(BUILD_VERSION) \
|
--build-arg REPO=$(REPO) \
|
||||||
|
--build-arg VERSION=$(VERSION) \
|
||||||
|
--rm \
|
||||||
-f Dockerfile \
|
-f Dockerfile \
|
||||||
-t $(HUB_IMAGE)-s3-gate:$(BUILD_VERSION) .
|
-t $(HUB_IMAGE):$(HUB_TAG) .
|
||||||
|
|
||||||
# Publish docker image
|
# Push Docker image to the hub
|
||||||
publish:
|
image-push:
|
||||||
@echo "${B}${G}⇒ publish docker image ${R}"
|
@echo "⇒ Publish image"
|
||||||
@docker push $(HUB_IMAGE)-s3-gate:$(VERSION)
|
@docker push $(HUB_IMAGE):$(HUB_TAG)
|
||||||
|
|
||||||
|
# Build dirty Docker image
|
||||||
|
dirty-image:
|
||||||
|
@echo "⇒ Build NeoFS S3 Gateway dirty docker image "
|
||||||
|
@docker build \
|
||||||
|
--build-arg REPO=$(REPO) \
|
||||||
|
--build-arg VERSION=$(VERSION) \
|
||||||
|
--rm \
|
||||||
|
-f Dockerfile.dirty \
|
||||||
|
-t $(HUB_IMAGE)-dirty:$(HUB_TAG) .
|
||||||
|
|
||||||
|
# Run linters
|
||||||
|
lint:
|
||||||
|
@golangci-lint --timeout=5m run
|
||||||
|
|
||||||
|
# Run linters in Docker
|
||||||
|
docker/lint:
|
||||||
|
docker run --rm -it \
|
||||||
|
-v `pwd`:/src \
|
||||||
|
-u `stat -c "%u:%g" .` \
|
||||||
|
--env HOME=/src \
|
||||||
|
golangci/golangci-lint:v1.40 bash -c 'cd /src/ && make lint'
|
||||||
|
|
||||||
|
# Show current version
|
||||||
|
version:
|
||||||
|
@echo $(VERSION)
|
||||||
|
|
||||||
|
# Show this help prompt
|
||||||
|
help:
|
||||||
|
@echo ' Usage:'
|
||||||
|
@echo ''
|
||||||
|
@echo ' make <target>'
|
||||||
|
@echo ''
|
||||||
|
@echo ' Targets:'
|
||||||
|
@echo ''
|
||||||
|
@awk '/^#/{ comment = substr($$0,3) } comment && /^[a-zA-Z][a-zA-Z0-9_-]+ ?:/{ print " ", $$1, comment }' $(MAKEFILE_LIST) | column -t -s ':' | grep -v 'IGNORE' | sort -u
|
||||||
|
|
||||||
|
# Clean up
|
||||||
|
clean:
|
||||||
|
rm -rf $(BINDIR)
|
||||||
|
|
|
@ -24,6 +24,7 @@ import (
|
||||||
var authorizationFieldRegexp = regexp.MustCompile(`AWS4-HMAC-SHA256 Credential=(?P<access_key_id_cid>[^/]+)/(?P<access_key_id_oid>[^/]+)/(?P<date>[^/]+)/(?P<region>[^/]*)/(?P<service>[^/]+)/aws4_request,\s*SignedHeaders=(?P<signed_header_fields>.+),\s*Signature=(?P<v4_signature>.+)`)
|
var authorizationFieldRegexp = regexp.MustCompile(`AWS4-HMAC-SHA256 Credential=(?P<access_key_id_cid>[^/]+)/(?P<access_key_id_oid>[^/]+)/(?P<date>[^/]+)/(?P<region>[^/]*)/(?P<service>[^/]+)/aws4_request,\s*SignedHeaders=(?P<signed_header_fields>.+),\s*Signature=(?P<v4_signature>.+)`)
|
||||||
|
|
||||||
type (
|
type (
|
||||||
|
// Center is a user authentication interface.
|
||||||
Center interface {
|
Center interface {
|
||||||
Authenticate(request *http.Request) (*token.BearerToken, error)
|
Authenticate(request *http.Request) (*token.BearerToken, error)
|
||||||
}
|
}
|
||||||
|
@ -33,6 +34,7 @@ type (
|
||||||
cli bearer.Credentials
|
cli bearer.Credentials
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Params stores node connection parameters.
|
||||||
Params struct {
|
Params struct {
|
||||||
Client sdk.Client
|
Client sdk.Client
|
||||||
Logger *zap.Logger
|
Logger *zap.Logger
|
||||||
|
|
|
@ -118,7 +118,7 @@ const (
|
||||||
ErrInvalidTagDirective
|
ErrInvalidTagDirective
|
||||||
// Add new error codes here.
|
// Add new error codes here.
|
||||||
|
|
||||||
// SSE-S3 related API errors
|
// SSE-S3 related API errors.
|
||||||
ErrInvalidEncryptionMethod
|
ErrInvalidEncryptionMethod
|
||||||
|
|
||||||
// Server-Side-Encryption (with Customer provided key) related API errors.
|
// Server-Side-Encryption (with Customer provided key) related API errors.
|
||||||
|
@ -170,7 +170,7 @@ const (
|
||||||
ErrOperationTimedOut
|
ErrOperationTimedOut
|
||||||
ErrOperationMaxedOut
|
ErrOperationMaxedOut
|
||||||
ErrInvalidRequest
|
ErrInvalidRequest
|
||||||
// MinIO storage class error codes
|
// MinIO storage class error codes.
|
||||||
ErrInvalidStorageClass
|
ErrInvalidStorageClass
|
||||||
ErrBackendDown
|
ErrBackendDown
|
||||||
// Add new extended error codes here.
|
// Add new extended error codes here.
|
||||||
|
@ -192,7 +192,7 @@ const (
|
||||||
ErrAdminCredentialsMismatch
|
ErrAdminCredentialsMismatch
|
||||||
ErrInsecureClientRequest
|
ErrInsecureClientRequest
|
||||||
ErrObjectTampered
|
ErrObjectTampered
|
||||||
// Bucket Quota error codes
|
// Bucket Quota error codes.
|
||||||
ErrAdminBucketQuotaExceeded
|
ErrAdminBucketQuotaExceeded
|
||||||
ErrAdminNoSuchQuotaConfiguration
|
ErrAdminNoSuchQuotaConfiguration
|
||||||
ErrAdminBucketQuotaDisabled
|
ErrAdminBucketQuotaDisabled
|
||||||
|
@ -205,7 +205,7 @@ const (
|
||||||
ErrHealOverlappingPaths
|
ErrHealOverlappingPaths
|
||||||
ErrIncorrectContinuationToken
|
ErrIncorrectContinuationToken
|
||||||
|
|
||||||
// S3 Select Errors
|
// S3 Select Errors.
|
||||||
ErrEmptyRequestBody
|
ErrEmptyRequestBody
|
||||||
ErrUnsupportedFunction
|
ErrUnsupportedFunction
|
||||||
ErrInvalidExpressionType
|
ErrInvalidExpressionType
|
||||||
|
@ -1625,7 +1625,7 @@ func GetAPIError(code ErrorCode) Error {
|
||||||
}
|
}
|
||||||
|
|
||||||
// getErrorResponse gets in standard error and resource value and
|
// getErrorResponse gets in standard error and resource value and
|
||||||
// provides a encodable populated response values
|
// provides a encodable populated response values.
|
||||||
func getAPIErrorResponse(ctx context.Context, err error, resource, requestID, hostID string) ErrorResponse {
|
func getAPIErrorResponse(ctx context.Context, err error, resource, requestID, hostID string) ErrorResponse {
|
||||||
code := "BadRequest"
|
code := "BadRequest"
|
||||||
desc := err.Error()
|
desc := err.Error()
|
||||||
|
@ -1803,21 +1803,21 @@ func (e BucketLifecycleNotFound) Error() string {
|
||||||
return "No bucket lifecycle configuration found for bucket : " + e.Bucket
|
return "No bucket lifecycle configuration found for bucket : " + e.Bucket
|
||||||
}
|
}
|
||||||
|
|
||||||
// BucketSSEConfigNotFound - no bucket encryption found
|
// BucketSSEConfigNotFound - no bucket encryption found.
|
||||||
type BucketSSEConfigNotFound GenericError
|
type BucketSSEConfigNotFound GenericError
|
||||||
|
|
||||||
func (e BucketSSEConfigNotFound) Error() string {
|
func (e BucketSSEConfigNotFound) Error() string {
|
||||||
return "No bucket encryption configuration found for bucket: " + e.Bucket
|
return "No bucket encryption configuration found for bucket: " + e.Bucket
|
||||||
}
|
}
|
||||||
|
|
||||||
// BucketTaggingNotFound - no bucket tags found
|
// BucketTaggingNotFound - no bucket tags found.
|
||||||
type BucketTaggingNotFound GenericError
|
type BucketTaggingNotFound GenericError
|
||||||
|
|
||||||
func (e BucketTaggingNotFound) Error() string {
|
func (e BucketTaggingNotFound) Error() string {
|
||||||
return "No bucket tags found for bucket: " + e.Bucket
|
return "No bucket tags found for bucket: " + e.Bucket
|
||||||
}
|
}
|
||||||
|
|
||||||
// BucketObjectLockConfigNotFound - no bucket object lock config found
|
// BucketObjectLockConfigNotFound - no bucket object lock config found.
|
||||||
type BucketObjectLockConfigNotFound GenericError
|
type BucketObjectLockConfigNotFound GenericError
|
||||||
|
|
||||||
func (e BucketObjectLockConfigNotFound) Error() string {
|
func (e BucketObjectLockConfigNotFound) Error() string {
|
||||||
|
@ -1874,7 +1874,7 @@ func (e ObjectNamePrefixAsSlash) Error() string {
|
||||||
return "Object name contains forward slash as pefix: " + e.Bucket + "#" + e.Object
|
return "Object name contains forward slash as pefix: " + e.Bucket + "#" + e.Object
|
||||||
}
|
}
|
||||||
|
|
||||||
// AllAccessDisabled All access to this object has been disabled
|
// AllAccessDisabled All access to this object has been disabled.
|
||||||
type AllAccessDisabled GenericError
|
type AllAccessDisabled GenericError
|
||||||
|
|
||||||
// Error returns string an error formatted as the given text.
|
// Error returns string an error formatted as the given text.
|
||||||
|
@ -1945,7 +1945,7 @@ func (e InvalidUploadID) Error() string {
|
||||||
return "Invalid upload id " + e.UploadID
|
return "Invalid upload id " + e.UploadID
|
||||||
}
|
}
|
||||||
|
|
||||||
// InvalidPart One or more of the specified parts could not be found
|
// InvalidPart One or more of the specified parts could not be found.
|
||||||
type InvalidPart struct {
|
type InvalidPart struct {
|
||||||
PartNumber int
|
PartNumber int
|
||||||
ExpETag string
|
ExpETag string
|
||||||
|
@ -1975,21 +1975,21 @@ func (e PartTooBig) Error() string {
|
||||||
return "Part size bigger than the allowed limit"
|
return "Part size bigger than the allowed limit"
|
||||||
}
|
}
|
||||||
|
|
||||||
// InvalidETag error returned when the etag has changed on disk
|
// InvalidETag error returned when the etag has changed on disk.
|
||||||
type InvalidETag struct{}
|
type InvalidETag struct{}
|
||||||
|
|
||||||
func (e InvalidETag) Error() string {
|
func (e InvalidETag) Error() string {
|
||||||
return "etag of the object has changed"
|
return "etag of the object has changed"
|
||||||
}
|
}
|
||||||
|
|
||||||
// NotImplemented If a feature is not implemented
|
// NotImplemented If a feature is not implemented.
|
||||||
type NotImplemented struct{}
|
type NotImplemented struct{}
|
||||||
|
|
||||||
func (e NotImplemented) Error() string {
|
func (e NotImplemented) Error() string {
|
||||||
return "Not Implemented"
|
return "Not Implemented"
|
||||||
}
|
}
|
||||||
|
|
||||||
// UnsupportedMetadata - unsupported metadata
|
// UnsupportedMetadata - unsupported metadata.
|
||||||
type UnsupportedMetadata struct{}
|
type UnsupportedMetadata struct{}
|
||||||
|
|
||||||
func (e UnsupportedMetadata) Error() string {
|
func (e UnsupportedMetadata) Error() string {
|
||||||
|
@ -2003,14 +2003,14 @@ func (e BackendDown) Error() string {
|
||||||
return "Backend down"
|
return "Backend down"
|
||||||
}
|
}
|
||||||
|
|
||||||
// PreConditionFailed - Check if copy precondition failed
|
// PreConditionFailed - Check if copy precondition failed.
|
||||||
type PreConditionFailed struct{}
|
type PreConditionFailed struct{}
|
||||||
|
|
||||||
func (e PreConditionFailed) Error() string {
|
func (e PreConditionFailed) Error() string {
|
||||||
return "At least one of the pre-conditions you specified did not hold"
|
return "At least one of the pre-conditions you specified did not hold"
|
||||||
}
|
}
|
||||||
|
|
||||||
// DeleteError - returns when cant remove object
|
// DeleteError - returns when cant remove object.
|
||||||
type DeleteError struct {
|
type DeleteError struct {
|
||||||
Err error
|
Err error
|
||||||
Object string
|
Object string
|
||||||
|
|
|
@ -14,6 +14,7 @@ type (
|
||||||
obj layer.Client
|
obj layer.Client
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Params holds logger and client.
|
||||||
Params struct {
|
Params struct {
|
||||||
Log *zap.Logger
|
Log *zap.Logger
|
||||||
Obj layer.Client
|
Obj layer.Client
|
||||||
|
@ -24,6 +25,7 @@ const notSupported = "Not supported by NeoFS S3 Gate: "
|
||||||
|
|
||||||
var _ api.Handler = (*handler)(nil)
|
var _ api.Handler = (*handler)(nil)
|
||||||
|
|
||||||
|
// New creates new api.Handler using given logger and client.
|
||||||
func New(log *zap.Logger, obj layer.Client) (api.Handler, error) {
|
func New(log *zap.Logger, obj layer.Client) (api.Handler, error) {
|
||||||
switch {
|
switch {
|
||||||
case obj == nil:
|
case obj == nil:
|
||||||
|
|
|
@ -68,7 +68,7 @@ func (h *handler) DeleteObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
w.WriteHeader(http.StatusNoContent)
|
w.WriteHeader(http.StatusNoContent)
|
||||||
}
|
}
|
||||||
|
|
||||||
// DeleteMultipleObjectsHandler :
|
// DeleteMultipleObjectsHandler handles multiple delete requests.
|
||||||
func (h *handler) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
var (
|
var (
|
||||||
req = mux.Vars(r)
|
req = mux.Vars(r)
|
||||||
|
|
|
@ -109,7 +109,7 @@ func (h *handler) HeadBucketHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
|
|
||||||
code := http.StatusBadRequest
|
code := http.StatusBadRequest
|
||||||
if st, ok := status.FromError(err); ok && st != nil {
|
if st, ok := status.FromError(err); ok && st != nil {
|
||||||
switch st.Code() {
|
switch st.Code() { //nolint:exhaustive // we have default value set above
|
||||||
case codes.NotFound:
|
case codes.NotFound:
|
||||||
code = http.StatusNotFound
|
code = http.StatusNotFound
|
||||||
case codes.PermissionDenied:
|
case codes.PermissionDenied:
|
||||||
|
|
|
@ -22,12 +22,14 @@ type listObjectsArgs struct {
|
||||||
Version string
|
Version string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// VersioningConfiguration contains VersioningConfiguration XML representation.
|
||||||
type VersioningConfiguration struct {
|
type VersioningConfiguration struct {
|
||||||
XMLName xml.Name `xml:"VersioningConfiguration"`
|
XMLName xml.Name `xml:"VersioningConfiguration"`
|
||||||
Text string `xml:",chardata"`
|
Text string `xml:",chardata"`
|
||||||
Xmlns string `xml:"xmlns,attr"`
|
Xmlns string `xml:"xmlns,attr"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ListMultipartUploadsResult contains ListMultipartUploadsResult XML representation.
|
||||||
type ListMultipartUploadsResult struct {
|
type ListMultipartUploadsResult struct {
|
||||||
XMLName xml.Name `xml:"ListMultipartUploadsResult"`
|
XMLName xml.Name `xml:"ListMultipartUploadsResult"`
|
||||||
Text string `xml:",chardata"`
|
Text string `xml:",chardata"`
|
||||||
|
@ -36,6 +38,7 @@ type ListMultipartUploadsResult struct {
|
||||||
|
|
||||||
var maxObjectList = 10000 // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse.
|
var maxObjectList = 10000 // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse.
|
||||||
|
|
||||||
|
// ListBucketsHandler handles bucket listing requests.
|
||||||
func (h *handler) ListBucketsHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) ListBucketsHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
var (
|
var (
|
||||||
err error
|
err error
|
||||||
|
@ -134,6 +137,7 @@ func (h *handler) listObjects(w http.ResponseWriter, r *http.Request) (*listObje
|
||||||
return arg, list, nil
|
return arg, list, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ListObjectsV1Handler handles objects listing requests for API version 1.
|
||||||
func (h *handler) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) {
|
||||||
var rid = api.GetRequestID(r.Context())
|
var rid = api.GetRequestID(r.Context())
|
||||||
if arg, list, err := h.listObjects(w, r); err != nil {
|
if arg, list, err := h.listObjects(w, r); err != nil {
|
||||||
|
@ -193,6 +197,7 @@ func encodeV1(arg *listObjectsArgs, list *layer.ListObjectsInfo) *ListObjectsRes
|
||||||
return res
|
return res
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ListObjectsV2Handler handles objects listing requests for API version 2.
|
||||||
func (h *handler) ListObjectsV2Handler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) ListObjectsV2Handler(w http.ResponseWriter, r *http.Request) {
|
||||||
var rid = api.GetRequestID(r.Context())
|
var rid = api.GetRequestID(r.Context())
|
||||||
if arg, list, err := h.listObjects(w, r); err != nil {
|
if arg, list, err := h.listObjects(w, r); err != nil {
|
||||||
|
@ -278,6 +283,7 @@ func parseListObjectArgs(r *http.Request) (*listObjectsArgs, error) {
|
||||||
return &res, nil
|
return &res, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// GetBucketVersioningHandler implements bucket versioning getter handler.
|
||||||
func (h *handler) GetBucketVersioningHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) GetBucketVersioningHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
var (
|
var (
|
||||||
rid = api.GetRequestID(r.Context())
|
rid = api.GetRequestID(r.Context())
|
||||||
|
@ -299,6 +305,7 @@ func (h *handler) GetBucketVersioningHandler(w http.ResponseWriter, r *http.Requ
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ListMultipartUploadsHandler implements multipart uploads listing handler.
|
||||||
func (h *handler) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
var (
|
var (
|
||||||
rid = api.GetRequestID(r.Context())
|
rid = api.GetRequestID(r.Context())
|
||||||
|
|
|
@ -2,7 +2,7 @@ package handler
|
||||||
|
|
||||||
import "encoding/xml"
|
import "encoding/xml"
|
||||||
|
|
||||||
// ListBucketsResponse - format for list buckets response
|
// ListBucketsResponse - format for list buckets response.
|
||||||
type ListBucketsResponse struct {
|
type ListBucketsResponse struct {
|
||||||
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListAllMyBucketsResult" json:"-"`
|
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListAllMyBucketsResult" json:"-"`
|
||||||
|
|
||||||
|
@ -45,13 +45,13 @@ type ListObjectsV2Response struct {
|
||||||
EncodingType string `xml:"EncodingType,omitempty"`
|
EncodingType string `xml:"EncodingType,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// Bucket container for bucket metadata
|
// Bucket container for bucket metadata.
|
||||||
type Bucket struct {
|
type Bucket struct {
|
||||||
Name string
|
Name string
|
||||||
CreationDate string // time string of format "2006-01-02T15:04:05.000Z"
|
CreationDate string // time string of format "2006-01-02T15:04:05.000Z"
|
||||||
}
|
}
|
||||||
|
|
||||||
// Owner - bucket owner/principal
|
// Owner - bucket owner/principal.
|
||||||
type Owner struct {
|
type Owner struct {
|
||||||
ID string
|
ID string
|
||||||
DisplayName string
|
DisplayName string
|
||||||
|
@ -87,12 +87,12 @@ type ListObjectsResponse struct {
|
||||||
EncodingType string `xml:"EncodingType,omitempty"`
|
EncodingType string `xml:"EncodingType,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// CommonPrefix container for prefix response in ListObjectsResponse
|
// CommonPrefix container for prefix response in ListObjectsResponse.
|
||||||
type CommonPrefix struct {
|
type CommonPrefix struct {
|
||||||
Prefix string
|
Prefix string
|
||||||
}
|
}
|
||||||
|
|
||||||
// Object container for object metadata
|
// Object container for object metadata.
|
||||||
type Object struct {
|
type Object struct {
|
||||||
Key string
|
Key string
|
||||||
LastModified string // time string of format "2006-01-02T15:04:05.000Z"
|
LastModified string // time string of format "2006-01-02T15:04:05.000Z"
|
||||||
|
@ -118,7 +118,7 @@ type LocationResponse struct {
|
||||||
Location string `xml:",chardata"`
|
Location string `xml:",chardata"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// CopyObjectResponse container returns ETag and LastModified of the successfully copied object
|
// CopyObjectResponse container returns ETag and LastModified of the successfully copied object.
|
||||||
type CopyObjectResponse struct {
|
type CopyObjectResponse struct {
|
||||||
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CopyObjectResult" json:"-"`
|
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CopyObjectResult" json:"-"`
|
||||||
LastModified string // time string of format "2006-01-02T15:04:05.000Z"
|
LastModified string // time string of format "2006-01-02T15:04:05.000Z"
|
||||||
|
@ -127,7 +127,6 @@ type CopyObjectResponse struct {
|
||||||
|
|
||||||
// MarshalXML - StringMap marshals into XML.
|
// MarshalXML - StringMap marshals into XML.
|
||||||
func (s StringMap) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
|
func (s StringMap) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
|
||||||
|
|
||||||
tokens := []xml.Token{start}
|
tokens := []xml.Token{start}
|
||||||
|
|
||||||
for key, value := range s {
|
for key, value := range s {
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
package api
|
package api
|
||||||
|
|
||||||
// Standard S3 HTTP response constants
|
// Standard S3 HTTP response constants.
|
||||||
const (
|
const (
|
||||||
LastModified = "Last-Modified"
|
LastModified = "Last-Modified"
|
||||||
Date = "Date"
|
Date = "Date"
|
||||||
|
|
|
@ -12,6 +12,7 @@ import (
|
||||||
)
|
)
|
||||||
|
|
||||||
type (
|
type (
|
||||||
|
// BucketInfo stores basic bucket data.
|
||||||
BucketInfo struct {
|
BucketInfo struct {
|
||||||
Name string
|
Name string
|
||||||
CID *container.ID
|
CID *container.ID
|
||||||
|
@ -19,6 +20,7 @@ type (
|
||||||
Created time.Time
|
Created time.Time
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ListObjectsParams represents object listing request parameters.
|
||||||
ListObjectsParams struct {
|
ListObjectsParams struct {
|
||||||
Bucket string
|
Bucket string
|
||||||
Prefix string
|
Prefix string
|
||||||
|
|
|
@ -25,6 +25,7 @@ type (
|
||||||
log *zap.Logger
|
log *zap.Logger
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Params stores basic API parameters.
|
||||||
Params struct {
|
Params struct {
|
||||||
Pool pool.Client
|
Pool pool.Client
|
||||||
Logger *zap.Logger
|
Logger *zap.Logger
|
||||||
|
@ -32,6 +33,7 @@ type (
|
||||||
Credential neofs.Credentials
|
Credential neofs.Credentials
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// GetObjectParams stores object get request parameters.
|
||||||
GetObjectParams struct {
|
GetObjectParams struct {
|
||||||
Bucket string
|
Bucket string
|
||||||
Object string
|
Object string
|
||||||
|
@ -40,6 +42,7 @@ type (
|
||||||
Writer io.Writer
|
Writer io.Writer
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// PutObjectParams stores object put request parameters.
|
||||||
PutObjectParams struct {
|
PutObjectParams struct {
|
||||||
Bucket string
|
Bucket string
|
||||||
Object string
|
Object string
|
||||||
|
@ -48,6 +51,7 @@ type (
|
||||||
Header map[string]string
|
Header map[string]string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// CopyObjectParams stores object copy request parameters.
|
||||||
CopyObjectParams struct {
|
CopyObjectParams struct {
|
||||||
SrcBucket string
|
SrcBucket string
|
||||||
DstBucket string
|
DstBucket string
|
||||||
|
@ -56,10 +60,12 @@ type (
|
||||||
Header map[string]string
|
Header map[string]string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// NeoFS provides basic NeoFS interface.
|
||||||
NeoFS interface {
|
NeoFS interface {
|
||||||
Get(ctx context.Context, address *object.Address) (*object.Object, error)
|
Get(ctx context.Context, address *object.Address) (*object.Object, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Client provides S3 API client interface.
|
||||||
Client interface {
|
Client interface {
|
||||||
NeoFS
|
NeoFS
|
||||||
|
|
||||||
|
@ -81,11 +87,13 @@ type (
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
|
// ErrObjectExists is returned on attempts to create already existing object.
|
||||||
ErrObjectExists = errors.New("object exists")
|
ErrObjectExists = errors.New("object exists")
|
||||||
|
// ErrObjectNotExists is returned on attempts to work with non-existing object.
|
||||||
ErrObjectNotExists = errors.New("object not exists")
|
ErrObjectNotExists = errors.New("object not exists")
|
||||||
)
|
)
|
||||||
|
|
||||||
// NewGatewayLayer creates instance of layer. It checks credentials
|
// NewLayer creates instance of layer. It checks credentials
|
||||||
// and establishes gRPC connection with node.
|
// and establishes gRPC connection with node.
|
||||||
func NewLayer(log *zap.Logger, cli sdk.Client) Client {
|
func NewLayer(log *zap.Logger, cli sdk.Client) Client {
|
||||||
return &layer{
|
return &layer{
|
||||||
|
@ -103,7 +111,7 @@ func (n *layer) Owner(ctx context.Context) *owner.ID {
|
||||||
return n.cli.Owner()
|
return n.cli.Owner()
|
||||||
}
|
}
|
||||||
|
|
||||||
// Get NeoFS Object by refs.Address (should be used by auth.Center)
|
// Get NeoFS Object by refs.Address (should be used by auth.Center).
|
||||||
func (n *layer) Get(ctx context.Context, address *object.Address) (*object.Object, error) {
|
func (n *layer) Get(ctx context.Context, address *object.Address) (*object.Object, error) {
|
||||||
return n.cli.Object().Get(ctx, address)
|
return n.cli.Object().Get(ctx, address)
|
||||||
}
|
}
|
||||||
|
|
|
@ -12,6 +12,7 @@ import (
|
||||||
)
|
)
|
||||||
|
|
||||||
type (
|
type (
|
||||||
|
// ObjectInfo holds S3 object data.
|
||||||
ObjectInfo struct {
|
ObjectInfo struct {
|
||||||
id *object.ID
|
id *object.ID
|
||||||
isDir bool
|
isDir bool
|
||||||
|
@ -52,6 +53,7 @@ type (
|
||||||
|
|
||||||
const (
|
const (
|
||||||
rootSeparator = "root://"
|
rootSeparator = "root://"
|
||||||
|
// PathSeparator is a path components separator string.
|
||||||
PathSeparator = string(os.PathSeparator)
|
PathSeparator = string(os.PathSeparator)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -140,11 +142,14 @@ func nameFromObject(o *object.Object) (string, string) {
|
||||||
return NameFromString(name)
|
return NameFromString(name)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// NameFromString splits name into base file name and directory path.
|
||||||
func NameFromString(name string) (string, string) {
|
func NameFromString(name string) (string, string) {
|
||||||
ind := strings.LastIndex(name, PathSeparator)
|
ind := strings.LastIndex(name, PathSeparator)
|
||||||
return name[ind+1:], name[:ind+1]
|
return name[ind+1:], name[:ind+1]
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ID returns object ID from ObjectInfo.
|
||||||
func (o *ObjectInfo) ID() *object.ID { return o.id }
|
func (o *ObjectInfo) ID() *object.ID { return o.id }
|
||||||
|
|
||||||
|
// IsDir allows to check if object is a directory.
|
||||||
func (o *ObjectInfo) IsDir() bool { return o.isDir }
|
func (o *ObjectInfo) IsDir() bool { return o.isDir }
|
||||||
|
|
|
@ -6,6 +6,7 @@ import (
|
||||||
)
|
)
|
||||||
|
|
||||||
type (
|
type (
|
||||||
|
// MaxClients provides HTTP handler wrapper with client limit.
|
||||||
MaxClients interface {
|
MaxClients interface {
|
||||||
Handle(http.HandlerFunc) http.HandlerFunc
|
Handle(http.HandlerFunc) http.HandlerFunc
|
||||||
}
|
}
|
||||||
|
@ -18,6 +19,8 @@ type (
|
||||||
|
|
||||||
const defaultRequestDeadline = time.Second * 30
|
const defaultRequestDeadline = time.Second * 30
|
||||||
|
|
||||||
|
// NewMaxClientsMiddleware returns MaxClients interface with handler wrapper based on
|
||||||
|
// provided count and timeout limits.
|
||||||
func NewMaxClientsMiddleware(count int, timeout time.Duration) MaxClients {
|
func NewMaxClientsMiddleware(count int, timeout time.Duration) MaxClients {
|
||||||
if timeout <= 0 {
|
if timeout <= 0 {
|
||||||
timeout = defaultRequestDeadline
|
timeout = defaultRequestDeadline
|
||||||
|
@ -29,6 +32,7 @@ func NewMaxClientsMiddleware(count int, timeout time.Duration) MaxClients {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Handler wraps HTTP handler function with logic limiting access to it.
|
||||||
func (m *maxClients) Handle(f http.HandlerFunc) http.HandlerFunc {
|
func (m *maxClients) Handle(f http.HandlerFunc) http.HandlerFunc {
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
if m.pool == nil {
|
if m.pool == nil {
|
||||||
|
|
|
@ -20,7 +20,7 @@ type (
|
||||||
}
|
}
|
||||||
|
|
||||||
// HTTPStats holds statistics information about
|
// HTTPStats holds statistics information about
|
||||||
// HTTP requests made by all clients
|
// HTTP requests made by all clients.
|
||||||
HTTPStats struct {
|
HTTPStats struct {
|
||||||
currentS3Requests HTTPAPIStats
|
currentS3Requests HTTPAPIStats
|
||||||
totalS3Requests HTTPAPIStats
|
totalS3Requests HTTPAPIStats
|
||||||
|
@ -63,8 +63,8 @@ var (
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
// collects http metrics for NeoFS S3 Gate in Prometheus specific format
|
// Collects HTTP metrics for NeoFS S3 Gate in Prometheus specific format
|
||||||
// and sends to given channel
|
// and sends to given channel.
|
||||||
func collectHTTPMetrics(ch chan<- prometheus.Metric) {
|
func collectHTTPMetrics(ch chan<- prometheus.Metric) {
|
||||||
for api, value := range httpStatsMetric.currentS3Requests.Load() {
|
for api, value := range httpStatsMetric.currentS3Requests.Load() {
|
||||||
ch <- prometheus.MustNewConstMetric(
|
ch <- prometheus.MustNewConstMetric(
|
||||||
|
@ -103,6 +103,7 @@ func collectHTTPMetrics(ch chan<- prometheus.Metric) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// APIStats wraps http handler for api with basic statistics collection.
|
||||||
func APIStats(api string, f http.HandlerFunc) http.HandlerFunc {
|
func APIStats(api string, f http.HandlerFunc) http.HandlerFunc {
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
httpStatsMetric.currentS3Requests.Inc(api)
|
httpStatsMetric.currentS3Requests.Inc(api)
|
||||||
|
@ -176,7 +177,7 @@ func (st *HTTPStats) getOutputBytes() uint64 {
|
||||||
return atomic.LoadUint64(&st.totalOutputBytes)
|
return atomic.LoadUint64(&st.totalOutputBytes)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Update statistics from http request and response data
|
// Update statistics from http request and response data.
|
||||||
func (st *HTTPStats) updateStats(api string, w http.ResponseWriter, r *http.Request, durationSecs float64) {
|
func (st *HTTPStats) updateStats(api string, w http.ResponseWriter, r *http.Request, durationSecs float64) {
|
||||||
var code int
|
var code int
|
||||||
|
|
||||||
|
@ -200,7 +201,7 @@ func (st *HTTPStats) updateStats(api string, w http.ResponseWriter, r *http.Requ
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// WriteHeader - writes http status code
|
// WriteHeader - writes http status code.
|
||||||
func (w *responseWrapper) WriteHeader(code int) {
|
func (w *responseWrapper) WriteHeader(code int) {
|
||||||
w.Do(func() {
|
w.Do(func() {
|
||||||
w.statusCode = code
|
w.statusCode = code
|
||||||
|
|
|
@ -13,7 +13,7 @@ import (
|
||||||
)
|
)
|
||||||
|
|
||||||
type (
|
type (
|
||||||
// KeyVal - appended to ReqInfo.Tags
|
// KeyVal - appended to ReqInfo.Tags.
|
||||||
KeyVal struct {
|
KeyVal struct {
|
||||||
Key string
|
Key string
|
||||||
Val string
|
Val string
|
||||||
|
@ -33,6 +33,7 @@ type (
|
||||||
tags []KeyVal // Any additional info not accommodated by above fields
|
tags []KeyVal // Any additional info not accommodated by above fields
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ObjectRequest represents object request data.
|
||||||
ObjectRequest struct {
|
ObjectRequest struct {
|
||||||
Bucket string
|
Bucket string
|
||||||
Object string
|
Object string
|
||||||
|
@ -40,7 +41,7 @@ type (
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
// Key used for Get/SetReqInfo
|
// Key used for Get/SetReqInfo.
|
||||||
type contextKeyType string
|
type contextKeyType string
|
||||||
|
|
||||||
const ctxRequestInfo = contextKeyType("NeoFS-S3-Gate")
|
const ctxRequestInfo = contextKeyType("NeoFS-S3-Gate")
|
||||||
|
@ -54,7 +55,7 @@ var (
|
||||||
var (
|
var (
|
||||||
// RFC7239 defines a new "Forwarded: " header designed to replace the
|
// RFC7239 defines a new "Forwarded: " header designed to replace the
|
||||||
// existing use of X-Forwarded-* headers.
|
// existing use of X-Forwarded-* headers.
|
||||||
// e.g. Forwarded: for=192.0.2.60;proto=https;by=203.0.113.43
|
// e.g. Forwarded: for=192.0.2.60;proto=https;by=203.0.113.43.
|
||||||
forwarded = http.CanonicalHeaderKey("Forwarded")
|
forwarded = http.CanonicalHeaderKey("Forwarded")
|
||||||
// Allows for a sub-match of the first value after 'for=' to the next
|
// Allows for a sub-match of the first value after 'for=' to the next
|
||||||
// comma, semi-colon or space. The match is case-insensitive.
|
// comma, semi-colon or space. The match is case-insensitive.
|
||||||
|
@ -125,7 +126,7 @@ func prepareContext(w http.ResponseWriter, r *http.Request) context.Context {
|
||||||
}))
|
}))
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewReqInfo :
|
// NewReqInfo returns new ReqInfo based on parameters.
|
||||||
func NewReqInfo(w http.ResponseWriter, r *http.Request, req ObjectRequest) *ReqInfo {
|
func NewReqInfo(w http.ResponseWriter, r *http.Request, req ObjectRequest) *ReqInfo {
|
||||||
return &ReqInfo{
|
return &ReqInfo{
|
||||||
API: req.Method,
|
API: req.Method,
|
||||||
|
@ -138,7 +139,7 @@ func NewReqInfo(w http.ResponseWriter, r *http.Request, req ObjectRequest) *ReqI
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// AppendTags - appends key/val to ReqInfo.tags
|
// AppendTags - appends key/val to ReqInfo.tags.
|
||||||
func (r *ReqInfo) AppendTags(key string, val string) *ReqInfo {
|
func (r *ReqInfo) AppendTags(key string, val string) *ReqInfo {
|
||||||
if r == nil {
|
if r == nil {
|
||||||
return nil
|
return nil
|
||||||
|
@ -149,7 +150,7 @@ func (r *ReqInfo) AppendTags(key string, val string) *ReqInfo {
|
||||||
return r
|
return r
|
||||||
}
|
}
|
||||||
|
|
||||||
// SetTags - sets key/val to ReqInfo.tags
|
// SetTags - sets key/val to ReqInfo.tags.
|
||||||
func (r *ReqInfo) SetTags(key string, val string) *ReqInfo {
|
func (r *ReqInfo) SetTags(key string, val string) *ReqInfo {
|
||||||
if r == nil {
|
if r == nil {
|
||||||
return nil
|
return nil
|
||||||
|
@ -172,7 +173,7 @@ func (r *ReqInfo) SetTags(key string, val string) *ReqInfo {
|
||||||
return r
|
return r
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetTags - returns the user defined tags
|
// GetTags - returns the user defined tags.
|
||||||
func (r *ReqInfo) GetTags() []KeyVal {
|
func (r *ReqInfo) GetTags() []KeyVal {
|
||||||
if r == nil {
|
if r == nil {
|
||||||
return nil
|
return nil
|
||||||
|
|
|
@ -14,7 +14,7 @@ import (
|
||||||
)
|
)
|
||||||
|
|
||||||
type (
|
type (
|
||||||
// ErrorResponse - error response format
|
// ErrorResponse - error response format.
|
||||||
ErrorResponse struct {
|
ErrorResponse struct {
|
||||||
XMLName xml.Name `xml:"Error" json:"-"`
|
XMLName xml.Name `xml:"Error" json:"-"`
|
||||||
Code string
|
Code string
|
||||||
|
@ -32,11 +32,11 @@ type (
|
||||||
// Captures the server string returned in response header.
|
// Captures the server string returned in response header.
|
||||||
Server string `xml:"-" json:"-"`
|
Server string `xml:"-" json:"-"`
|
||||||
|
|
||||||
// Underlying HTTP status code for the returned error
|
// Underlying HTTP status code for the returned error.
|
||||||
StatusCode int `xml:"-" json:"-"`
|
StatusCode int `xml:"-" json:"-"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// APIError structure
|
// Error structure represents API error.
|
||||||
Error struct {
|
Error struct {
|
||||||
Code string
|
Code string
|
||||||
Description string
|
Description string
|
||||||
|
@ -117,7 +117,7 @@ var s3ErrorResponseMap = map[string]string{
|
||||||
// Add new API errors here.
|
// Add new API errors here.
|
||||||
}
|
}
|
||||||
|
|
||||||
// WriteErrorResponse writes error headers
|
// WriteErrorResponse writes error headers.
|
||||||
func WriteErrorResponse(ctx context.Context, w http.ResponseWriter, err error, reqURL *url.URL) {
|
func WriteErrorResponse(ctx context.Context, w http.ResponseWriter, err error, reqURL *url.URL) {
|
||||||
code := http.StatusBadRequest
|
code := http.StatusBadRequest
|
||||||
|
|
||||||
|
@ -141,7 +141,7 @@ func WriteErrorResponse(ctx context.Context, w http.ResponseWriter, err error, r
|
||||||
WriteResponse(w, code, encodedErrorResponse, MimeXML)
|
WriteResponse(w, code, encodedErrorResponse, MimeXML)
|
||||||
}
|
}
|
||||||
|
|
||||||
// If none of the http routes match respond with appropriate errors
|
// If none of the http routes match respond with appropriate errors.
|
||||||
func errorResponseHandler(w http.ResponseWriter, r *http.Request) {
|
func errorResponseHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
desc := fmt.Sprintf("Unknown API request at %s", r.URL.Path)
|
desc := fmt.Sprintf("Unknown API request at %s", r.URL.Path)
|
||||||
WriteErrorResponse(r.Context(), w, Error{
|
WriteErrorResponse(r.Context(), w, Error{
|
||||||
|
@ -151,7 +151,7 @@ func errorResponseHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
}, r.URL)
|
}, r.URL)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Write http common headers
|
// Write http common headers.
|
||||||
func setCommonHeaders(w http.ResponseWriter) {
|
func setCommonHeaders(w http.ResponseWriter) {
|
||||||
w.Header().Set(hdrServerInfo, "NeoFS-S3-Gate/"+misc.Version)
|
w.Header().Set(hdrServerInfo, "NeoFS-S3-Gate/"+misc.Version)
|
||||||
w.Header().Set(hdrAcceptRanges, "bytes")
|
w.Header().Set(hdrAcceptRanges, "bytes")
|
||||||
|
@ -168,6 +168,7 @@ func removeSensitiveHeaders(h http.Header) {
|
||||||
h.Del(hdrSSECopyKey)
|
h.Del(hdrSSECopyKey)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// WriteResponse writes given statusCode and response into w (with mType header if set).
|
||||||
func WriteResponse(w http.ResponseWriter, statusCode int, response []byte, mType mimeType) {
|
func WriteResponse(w http.ResponseWriter, statusCode int, response []byte, mType mimeType) {
|
||||||
setCommonHeaders(w)
|
setCommonHeaders(w)
|
||||||
if mType != MimeNone {
|
if mType != MimeNone {
|
||||||
|
@ -214,6 +215,8 @@ func EncodeToResponse(w http.ResponseWriter, response interface{}) error {
|
||||||
// WriteResponse(w, http.StatusOK, response, MimeXML)
|
// WriteResponse(w, http.StatusOK, response, MimeXML)
|
||||||
// }
|
// }
|
||||||
|
|
||||||
|
// WriteSuccessResponseHeadersOnly writes HTTP (200) OK response with no data
|
||||||
|
// to the client.
|
||||||
func WriteSuccessResponseHeadersOnly(w http.ResponseWriter) {
|
func WriteSuccessResponseHeadersOnly(w http.ResponseWriter) {
|
||||||
WriteResponse(w, http.StatusOK, nil, MimeNone)
|
WriteResponse(w, http.StatusOK, nil, MimeNone)
|
||||||
}
|
}
|
||||||
|
|
|
@ -14,6 +14,7 @@ import (
|
||||||
)
|
)
|
||||||
|
|
||||||
type (
|
type (
|
||||||
|
// Handler is an S3 API handler interface.
|
||||||
Handler interface {
|
Handler interface {
|
||||||
HeadObjectHandler(http.ResponseWriter, *http.Request)
|
HeadObjectHandler(http.ResponseWriter, *http.Request)
|
||||||
CopyObjectPartHandler(http.ResponseWriter, *http.Request)
|
CopyObjectPartHandler(http.ResponseWriter, *http.Request)
|
||||||
|
@ -158,6 +159,7 @@ func logErrorResponse(l *zap.Logger) mux.MiddlewareFunc {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// GetRequestID returns request ID from response writer or context.
|
||||||
func GetRequestID(v interface{}) string {
|
func GetRequestID(v interface{}) string {
|
||||||
switch t := v.(type) {
|
switch t := v.(type) {
|
||||||
case context.Context:
|
case context.Context:
|
||||||
|
@ -169,6 +171,8 @@ func GetRequestID(v interface{}) string {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Attach adds S3 API handlers from h to r for domains with m client limit using
|
||||||
|
// center authentication and log logger.
|
||||||
func Attach(r *mux.Router, domains []string, m MaxClients, h Handler, center auth.Center, log *zap.Logger) {
|
func Attach(r *mux.Router, domains []string, m MaxClients, h Handler, center auth.Center, log *zap.Logger) {
|
||||||
api := r.PathPrefix(SlashSeparator).Subrouter()
|
api := r.PathPrefix(SlashSeparator).Subrouter()
|
||||||
|
|
||||||
|
|
|
@ -9,6 +9,7 @@ import (
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// AttachUserAuth adds user authentication via center to router using log for logging.
|
||||||
func AttachUserAuth(router *mux.Router, center auth.Center, log *zap.Logger) {
|
func AttachUserAuth(router *mux.Router, center auth.Center, log *zap.Logger) {
|
||||||
router.Use(func(h http.Handler) http.Handler {
|
router.Use(func(h http.Handler) http.Handler {
|
||||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
@ -21,7 +22,6 @@ func AttachUserAuth(router *mux.Router, center auth.Center, log *zap.Logger) {
|
||||||
|
|
||||||
h.ServeHTTP(w, r.WithContext(
|
h.ServeHTTP(w, r.WithContext(
|
||||||
sdk.SetBearerToken(r.Context(), token)))
|
sdk.SetBearerToken(r.Context(), token)))
|
||||||
|
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
|
@ -7,6 +7,7 @@ import (
|
||||||
"github.com/gorilla/mux"
|
"github.com/gorilla/mux"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// Healthy is a health check interface.
|
||||||
type Healthy interface {
|
type Healthy interface {
|
||||||
Status() error
|
Status() error
|
||||||
}
|
}
|
||||||
|
|
|
@ -34,8 +34,8 @@ const (
|
||||||
defaultMaxClientsDeadline = time.Second * 30
|
defaultMaxClientsDeadline = time.Second * 30
|
||||||
)
|
)
|
||||||
|
|
||||||
const ( // settings
|
const ( // Settings.
|
||||||
// Logger:
|
// Logger.
|
||||||
cfgLoggerLevel = "logger.level"
|
cfgLoggerLevel = "logger.level"
|
||||||
cfgLoggerFormat = "logger.format"
|
cfgLoggerFormat = "logger.format"
|
||||||
cfgLoggerTraceLevel = "logger.trace_level"
|
cfgLoggerTraceLevel = "logger.trace_level"
|
||||||
|
@ -44,47 +44,47 @@ const ( // settings
|
||||||
cfgLoggerSamplingInitial = "logger.sampling.initial"
|
cfgLoggerSamplingInitial = "logger.sampling.initial"
|
||||||
cfgLoggerSamplingThereafter = "logger.sampling.thereafter"
|
cfgLoggerSamplingThereafter = "logger.sampling.thereafter"
|
||||||
|
|
||||||
// KeepAlive
|
// KeepAlive.
|
||||||
cfgKeepaliveTime = "keepalive.time"
|
cfgKeepaliveTime = "keepalive.time"
|
||||||
cfgKeepaliveTimeout = "keepalive.timeout"
|
cfgKeepaliveTimeout = "keepalive.timeout"
|
||||||
cfgKeepalivePermitWithoutStream = "keepalive.permit_without_stream"
|
cfgKeepalivePermitWithoutStream = "keepalive.permit_without_stream"
|
||||||
|
|
||||||
// Keys
|
// Keys.
|
||||||
cfgNeoFSPrivateKey = "neofs-key"
|
cfgNeoFSPrivateKey = "neofs-key"
|
||||||
cfgGateAuthPrivateKey = "auth-key"
|
cfgGateAuthPrivateKey = "auth-key"
|
||||||
|
|
||||||
// HTTPS/TLS
|
// HTTPS/TLS.
|
||||||
cfgTLSKeyFile = "tls.key_file"
|
cfgTLSKeyFile = "tls.key_file"
|
||||||
cfgTLSCertFile = "tls.cert_file"
|
cfgTLSCertFile = "tls.cert_file"
|
||||||
|
|
||||||
// Timeouts
|
// Timeouts.
|
||||||
cfgConnectionTTL = "con_ttl"
|
cfgConnectionTTL = "con_ttl"
|
||||||
cfgConnectTimeout = "connect_timeout"
|
cfgConnectTimeout = "connect_timeout"
|
||||||
cfgRequestTimeout = "request_timeout"
|
cfgRequestTimeout = "request_timeout"
|
||||||
cfgRebalanceTimer = "rebalance_timer"
|
cfgRebalanceTimer = "rebalance_timer"
|
||||||
|
|
||||||
// MaxClients
|
// MaxClients.
|
||||||
cfgMaxClientsCount = "max_clients_count"
|
cfgMaxClientsCount = "max_clients_count"
|
||||||
cfgMaxClientsDeadline = "max_clients_deadline"
|
cfgMaxClientsDeadline = "max_clients_deadline"
|
||||||
|
|
||||||
// gRPC
|
// gRPC.
|
||||||
cfgGRPCVerbose = "verbose"
|
cfgGRPCVerbose = "verbose"
|
||||||
|
|
||||||
// Metrics / Profiler / Web
|
// Metrics / Profiler / Web.
|
||||||
cfgEnableMetrics = "metrics"
|
cfgEnableMetrics = "metrics"
|
||||||
cfgEnableProfiler = "pprof"
|
cfgEnableProfiler = "pprof"
|
||||||
cfgListenAddress = "listen_address"
|
cfgListenAddress = "listen_address"
|
||||||
cfgListenDomains = "listen_domains"
|
cfgListenDomains = "listen_domains"
|
||||||
|
|
||||||
// Peers
|
// Peers.
|
||||||
cfgPeers = "peers"
|
cfgPeers = "peers"
|
||||||
|
|
||||||
// Application
|
// Application.
|
||||||
cfgApplicationName = "app.name"
|
cfgApplicationName = "app.name"
|
||||||
cfgApplicationVersion = "app.version"
|
cfgApplicationVersion = "app.version"
|
||||||
cfgApplicationBuildTime = "app.build_time"
|
cfgApplicationBuildTime = "app.build_time"
|
||||||
|
|
||||||
// command line args
|
// Command line args.
|
||||||
cmdHelp = "help"
|
cmdHelp = "help"
|
||||||
cmdVersion = "version"
|
cmdVersion = "version"
|
||||||
)
|
)
|
||||||
|
@ -105,10 +105,9 @@ var ignore = map[string]struct{}{
|
||||||
func (empty) Read([]byte) (int, error) { return 0, io.EOF }
|
func (empty) Read([]byte) (int, error) { return 0, io.EOF }
|
||||||
|
|
||||||
func fetchPeers(l *zap.Logger, v *viper.Viper) map[string]float64 {
|
func fetchPeers(l *zap.Logger, v *viper.Viper) map[string]float64 {
|
||||||
peers := make(map[string]float64, 0)
|
peers := make(map[string]float64)
|
||||||
|
|
||||||
for i := 0; ; i++ {
|
for i := 0; ; i++ {
|
||||||
|
|
||||||
key := cfgPeers + "." + strconv.Itoa(i) + "."
|
key := cfgPeers + "." + strconv.Itoa(i) + "."
|
||||||
address := v.GetString(key + "address")
|
address := v.GetString(key + "address")
|
||||||
weight := v.GetFloat64(key + "weight")
|
weight := v.GetFloat64(key + "weight")
|
||||||
|
|
|
@ -22,6 +22,7 @@ import (
|
||||||
)
|
)
|
||||||
|
|
||||||
type (
|
type (
|
||||||
|
// App is the main application structure.
|
||||||
App struct {
|
App struct {
|
||||||
cli pool.Client
|
cli pool.Client
|
||||||
ctr auth.Center
|
ctr auth.Center
|
||||||
|
@ -179,6 +180,7 @@ func newApp(ctx context.Context, l *zap.Logger, v *viper.Viper) *App {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Wait waits for application to finish.
|
||||||
func (a *App) Wait() {
|
func (a *App) Wait() {
|
||||||
a.log.Info("application started")
|
a.log.Info("application started")
|
||||||
|
|
||||||
|
@ -192,6 +194,7 @@ func (a *App) Wait() {
|
||||||
a.log.Info("application finished")
|
a.log.Info("application finished")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Server runs HTTP server to handle S3 API requests.
|
||||||
func (a *App) Server(ctx context.Context) {
|
func (a *App) Server(ctx context.Context) {
|
||||||
var (
|
var (
|
||||||
err error
|
err error
|
||||||
|
@ -256,6 +259,7 @@ func (a *App) Server(ctx context.Context) {
|
||||||
close(a.webDone)
|
close(a.webDone)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Worker runs client worker.
|
||||||
func (a *App) Worker(ctx context.Context) {
|
func (a *App) Worker(ctx context.Context) {
|
||||||
a.cli.Worker(ctx)
|
a.cli.Worker(ctx)
|
||||||
a.log.Info("stopping worker")
|
a.log.Info("stopping worker")
|
||||||
|
|
|
@ -1,12 +1,16 @@
|
||||||
package misc
|
package misc
|
||||||
|
|
||||||
const (
|
const (
|
||||||
|
// ApplicationName is gateway name.
|
||||||
ApplicationName = "neofs-s3-gate"
|
ApplicationName = "neofs-s3-gate"
|
||||||
|
|
||||||
|
// Prefix is configuration environment variables prefix.
|
||||||
Prefix = "S3_GW"
|
Prefix = "S3_GW"
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
|
// Build holds build timestamp.
|
||||||
Build = "now"
|
Build = "now"
|
||||||
|
// Version contains application version.
|
||||||
Version = "dev"
|
Version = "dev"
|
||||||
)
|
)
|
||||||
|
|
Loading…
Reference in a new issue