forked from TrueCloudLab/frostfs-s3-gw
Roman Khimov
GitHub
commit
2b967db7c0
25 changed files with 277 additions and 117 deletions
8
.gitignore
vendored
8
.gitignore
vendored
|
|
@ -7,3 +7,11 @@ vendor
|
|||
|
||||
# tempfiles
|
||||
.DS_Store
|
||||
*~
|
||||
|
||||
# binary
|
||||
bin/
|
||||
|
||||
# coverage
|
||||
coverage.txt
|
||||
coverage.html
|
||||
|
|
|
|||
59
.golangci.yml
Normal file
59
.golangci.yml
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
# This file contains all available configuration options
|
||||
# with their default values.
|
||||
|
||||
# options for analysis running
|
||||
run:
|
||||
# timeout for analysis, e.g. 30s, 5m, default is 1m
|
||||
timeout: 5m
|
||||
|
||||
# include test files or not, default is true
|
||||
tests: true
|
||||
|
||||
# output configuration options
|
||||
output:
|
||||
# colored-line-number|line-number|json|tab|checkstyle|code-climate, default is "colored-line-number"
|
||||
format: tab
|
||||
|
||||
# all available settings of specific linters
|
||||
linters-settings:
|
||||
exhaustive:
|
||||
# indicates that switch statements are to be considered exhaustive if a
|
||||
# 'default' case is present, even if all enum members aren't listed in the
|
||||
# switch
|
||||
default-signifies-exhaustive: true
|
||||
govet:
|
||||
# report about shadowed variables
|
||||
check-shadowing: false
|
||||
|
||||
linters:
|
||||
enable:
|
||||
# mandatory linters
|
||||
- govet
|
||||
- golint
|
||||
|
||||
# some default golangci-lint linters
|
||||
- deadcode
|
||||
- errcheck
|
||||
- gosimple
|
||||
- ineffassign
|
||||
- staticcheck
|
||||
- structcheck
|
||||
- typecheck
|
||||
- unused
|
||||
- varcheck
|
||||
|
||||
# extra linters
|
||||
- exhaustive
|
||||
- godot
|
||||
- gofmt
|
||||
- whitespace
|
||||
- goimports
|
||||
disable-all: true
|
||||
fast: false
|
||||
|
||||
issues:
|
||||
include:
|
||||
- EXC0002 # should have a comment
|
||||
- EXC0003 # test/Test ... consider calling this
|
||||
- EXC0004 # govet
|
||||
- EXC0005 # C-style breaks
|
||||
24
Dockerfile
24
Dockerfile
|
|
@ -4,36 +4,20 @@ WORKDIR /src
|
|||
|
||||
RUN set -x \
|
||||
&& apt update \
|
||||
&& apt install -y upx-ucl
|
||||
&& apt install -y make
|
||||
|
||||
COPY . /src
|
||||
|
||||
ARG VERSION=dev
|
||||
|
||||
# https://github.com/golang/go/wiki/Modules#how-do-i-use-vendoring-with-modules-is-vendoring-going-away
|
||||
# go build -mod=vendor
|
||||
# The -gcflags "all=-N -l" flag helps us get a better debug experience
|
||||
RUN set -x \
|
||||
&& export BUILD=$(date -u +%s%N) \
|
||||
&& export REPO=$(go list -m) \
|
||||
&& export LDFLAGS="-X ${REPO}/misc.Version=${VERSION} -X ${REPO}/misc.Build=${BUILD}" \
|
||||
&& export GOGC=off \
|
||||
&& export CGO_ENABLED=0 \
|
||||
&& [ -d "./vendor" ] || go mod vendor \
|
||||
&& go build \
|
||||
-v \
|
||||
-mod=vendor \
|
||||
-trimpath \
|
||||
-ldflags "${LDFLAGS}" \
|
||||
-o /go/bin/neofs-s3 ./cmd/gate \
|
||||
&& upx -3 /go/bin/neofs-s3
|
||||
RUN make
|
||||
|
||||
# Executable image
|
||||
FROM scratch
|
||||
|
||||
WORKDIR /
|
||||
|
||||
COPY --from=builder /go/bin/neofs-s3 /bin/neofs-s3
|
||||
COPY --from=builder /src/bin/neofs-s3-gw /bin/neofs-s3-gw
|
||||
COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
|
||||
|
||||
ENTRYPOINT ["/bin/neofs-s3"]
|
||||
ENTRYPOINT ["/bin/neofs-s3-gw"]
|
||||
|
|
|
|||
127
Makefile
127
Makefile
|
|
@ -1,43 +1,106 @@
|
|||
-include .env
|
||||
-include help.mk
|
||||
|
||||
HUB_IMAGE=nspccdev/neofs
|
||||
#!/usr/bin/make -f
|
||||
|
||||
REPO ?= $(shell go list -m)
|
||||
VERSION ?= "$(shell git describe --tags 2>/dev/null || git rev-parse --short HEAD | sed 's/^v//')"
|
||||
BUILD_VERSION ?= "$(shell git describe --abbrev=0 --tags | sed 's/^v//')"
|
||||
|
||||
.PHONY: format deps image publish
|
||||
BIN_NAME=neofs-s3-gw
|
||||
HUB_IMAGE="nspccdev/$(BIN_NAME)"
|
||||
HUB_TAG ?= "$(shell echo ${VERSION} | sed 's/^v//')"
|
||||
BINDIR = bin
|
||||
BIN = "$(BINDIR)/$(BIN_NAME)"
|
||||
|
||||
# Show current version
|
||||
version:
|
||||
@echo $(BUILD_VERSION)
|
||||
.PHONY: help all dep clean format test cover lint docker/lint image-push image dirty-image
|
||||
|
||||
# Make all binaries
|
||||
all: $(BIN)
|
||||
|
||||
$(BIN): $(BINDIR) dep
|
||||
@echo "⇒ Build $@"
|
||||
CGO_ENABLED=0 \
|
||||
go build -v -trimpath \
|
||||
-ldflags "-X main.Version=$(VERSION)" \
|
||||
-o $@ ./cmd/gate
|
||||
|
||||
$(BINDIR):
|
||||
@echo "⇒ Ensure dir: $@"
|
||||
@mkdir -p $@
|
||||
|
||||
# Pull go dependencies
|
||||
dep:
|
||||
@printf "⇒ Download requirements: "
|
||||
@CGO_ENABLED=0 \
|
||||
go mod download && echo OK
|
||||
@printf "⇒ Tidy requirements: "
|
||||
@CGO_ENABLED=0 \
|
||||
go mod tidy -v && echo OK
|
||||
|
||||
# Run tests
|
||||
test:
|
||||
@go test ./... -cover
|
||||
|
||||
# Run tests with race detection and produce coverage output
|
||||
cover:
|
||||
@go test -v -race ./... -coverprofile=coverage.txt -covermode=atomic
|
||||
@go tool cover -html=coverage.txt -o coverage.html
|
||||
|
||||
# Reformat code
|
||||
format:
|
||||
@[ ! -z `which goimports` ] || (echo "install goimports" && exit 2)
|
||||
@for f in `find . -type f -name '*.go' -not -path './vendor/*' -not -name '*.pb.go' -prune`; do \
|
||||
echo "⇒ Processing $$f"; \
|
||||
goimports -w $$f; \
|
||||
done
|
||||
@echo "⇒ Processing gofmt check"
|
||||
@gofmt -s -w ./
|
||||
@echo "⇒ Processing goimports check"
|
||||
@goimports -w ./
|
||||
|
||||
# Check and ensure dependencies
|
||||
deps:
|
||||
@printf "⇒ Ensure vendor: "
|
||||
@go mod tidy -v && echo OK || (echo fail && exit 2)
|
||||
@printf "⇒ Download requirements: "
|
||||
@go mod download && echo OK || (echo fail && exit 2)
|
||||
@printf "⇒ Store vendor localy: "
|
||||
@go mod vendor && echo OK || (echo fail && exit 2)
|
||||
|
||||
# Build current docker image
|
||||
image: deps
|
||||
@echo "⇒ Build docker-image"
|
||||
# Build clean Docker image
|
||||
image:
|
||||
@echo "⇒ Build NeoFS S3 Gateway docker image "
|
||||
@docker build \
|
||||
--build-arg VERSION=$(BUILD_VERSION) \
|
||||
--build-arg REPO=$(REPO) \
|
||||
--build-arg VERSION=$(VERSION) \
|
||||
--rm \
|
||||
-f Dockerfile \
|
||||
-t $(HUB_IMAGE)-s3-gate:$(BUILD_VERSION) .
|
||||
-t $(HUB_IMAGE):$(HUB_TAG) .
|
||||
|
||||
# Publish docker image
|
||||
publish:
|
||||
@echo "${B}${G}⇒ publish docker image ${R}"
|
||||
@docker push $(HUB_IMAGE)-s3-gate:$(VERSION)
|
||||
# Push Docker image to the hub
|
||||
image-push:
|
||||
@echo "⇒ Publish image"
|
||||
@docker push $(HUB_IMAGE):$(HUB_TAG)
|
||||
|
||||
# Build dirty Docker image
|
||||
dirty-image:
|
||||
@echo "⇒ Build NeoFS S3 Gateway dirty docker image "
|
||||
@docker build \
|
||||
--build-arg REPO=$(REPO) \
|
||||
--build-arg VERSION=$(VERSION) \
|
||||
--rm \
|
||||
-f Dockerfile.dirty \
|
||||
-t $(HUB_IMAGE)-dirty:$(HUB_TAG) .
|
||||
|
||||
# Run linters
|
||||
lint:
|
||||
@golangci-lint --timeout=5m run
|
||||
|
||||
# Run linters in Docker
|
||||
docker/lint:
|
||||
docker run --rm -it \
|
||||
-v `pwd`:/src \
|
||||
-u `stat -c "%u:%g" .` \
|
||||
--env HOME=/src \
|
||||
golangci/golangci-lint:v1.40 bash -c 'cd /src/ && make lint'
|
||||
|
||||
# Show current version
|
||||
version:
|
||||
@echo $(VERSION)
|
||||
|
||||
# Show this help prompt
|
||||
help:
|
||||
@echo ' Usage:'
|
||||
@echo ''
|
||||
@echo ' make <target>'
|
||||
@echo ''
|
||||
@echo ' Targets:'
|
||||
@echo ''
|
||||
@awk '/^#/{ comment = substr($$0,3) } comment && /^[a-zA-Z][a-zA-Z0-9_-]+ ?:/{ print " ", $$1, comment }' $(MAKEFILE_LIST) | column -t -s ':' | grep -v 'IGNORE' | sort -u
|
||||
|
||||
# Clean up
|
||||
clean:
|
||||
rm -rf $(BINDIR)
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ import (
|
|||
var authorizationFieldRegexp = regexp.MustCompile(`AWS4-HMAC-SHA256 Credential=(?P<access_key_id_cid>[^/]+)/(?P<access_key_id_oid>[^/]+)/(?P<date>[^/]+)/(?P<region>[^/]*)/(?P<service>[^/]+)/aws4_request,\s*SignedHeaders=(?P<signed_header_fields>.+),\s*Signature=(?P<v4_signature>.+)`)
|
||||
|
||||
type (
|
||||
// Center is a user authentication interface.
|
||||
Center interface {
|
||||
Authenticate(request *http.Request) (*token.BearerToken, error)
|
||||
}
|
||||
|
|
@ -33,6 +34,7 @@ type (
|
|||
cli bearer.Credentials
|
||||
}
|
||||
|
||||
// Params stores node connection parameters.
|
||||
Params struct {
|
||||
Client sdk.Client
|
||||
Logger *zap.Logger
|
||||
|
|
|
|||
|
|
@ -118,7 +118,7 @@ const (
|
|||
ErrInvalidTagDirective
|
||||
// Add new error codes here.
|
||||
|
||||
// SSE-S3 related API errors
|
||||
// SSE-S3 related API errors.
|
||||
ErrInvalidEncryptionMethod
|
||||
|
||||
// Server-Side-Encryption (with Customer provided key) related API errors.
|
||||
|
|
@ -170,7 +170,7 @@ const (
|
|||
ErrOperationTimedOut
|
||||
ErrOperationMaxedOut
|
||||
ErrInvalidRequest
|
||||
// MinIO storage class error codes
|
||||
// MinIO storage class error codes.
|
||||
ErrInvalidStorageClass
|
||||
ErrBackendDown
|
||||
// Add new extended error codes here.
|
||||
|
|
@ -192,7 +192,7 @@ const (
|
|||
ErrAdminCredentialsMismatch
|
||||
ErrInsecureClientRequest
|
||||
ErrObjectTampered
|
||||
// Bucket Quota error codes
|
||||
// Bucket Quota error codes.
|
||||
ErrAdminBucketQuotaExceeded
|
||||
ErrAdminNoSuchQuotaConfiguration
|
||||
ErrAdminBucketQuotaDisabled
|
||||
|
|
@ -205,7 +205,7 @@ const (
|
|||
ErrHealOverlappingPaths
|
||||
ErrIncorrectContinuationToken
|
||||
|
||||
// S3 Select Errors
|
||||
// S3 Select Errors.
|
||||
ErrEmptyRequestBody
|
||||
ErrUnsupportedFunction
|
||||
ErrInvalidExpressionType
|
||||
|
|
@ -1625,7 +1625,7 @@ func GetAPIError(code ErrorCode) Error {
|
|||
}
|
||||
|
||||
// getErrorResponse gets in standard error and resource value and
|
||||
// provides a encodable populated response values
|
||||
// provides a encodable populated response values.
|
||||
func getAPIErrorResponse(ctx context.Context, err error, resource, requestID, hostID string) ErrorResponse {
|
||||
code := "BadRequest"
|
||||
desc := err.Error()
|
||||
|
|
@ -1803,21 +1803,21 @@ func (e BucketLifecycleNotFound) Error() string {
|
|||
return "No bucket lifecycle configuration found for bucket : " + e.Bucket
|
||||
}
|
||||
|
||||
// BucketSSEConfigNotFound - no bucket encryption found
|
||||
// BucketSSEConfigNotFound - no bucket encryption found.
|
||||
type BucketSSEConfigNotFound GenericError
|
||||
|
||||
func (e BucketSSEConfigNotFound) Error() string {
|
||||
return "No bucket encryption configuration found for bucket: " + e.Bucket
|
||||
}
|
||||
|
||||
// BucketTaggingNotFound - no bucket tags found
|
||||
// BucketTaggingNotFound - no bucket tags found.
|
||||
type BucketTaggingNotFound GenericError
|
||||
|
||||
func (e BucketTaggingNotFound) Error() string {
|
||||
return "No bucket tags found for bucket: " + e.Bucket
|
||||
}
|
||||
|
||||
// BucketObjectLockConfigNotFound - no bucket object lock config found
|
||||
// BucketObjectLockConfigNotFound - no bucket object lock config found.
|
||||
type BucketObjectLockConfigNotFound GenericError
|
||||
|
||||
func (e BucketObjectLockConfigNotFound) Error() string {
|
||||
|
|
@ -1874,7 +1874,7 @@ func (e ObjectNamePrefixAsSlash) Error() string {
|
|||
return "Object name contains forward slash as pefix: " + e.Bucket + "#" + e.Object
|
||||
}
|
||||
|
||||
// AllAccessDisabled All access to this object has been disabled
|
||||
// AllAccessDisabled All access to this object has been disabled.
|
||||
type AllAccessDisabled GenericError
|
||||
|
||||
// Error returns string an error formatted as the given text.
|
||||
|
|
@ -1945,7 +1945,7 @@ func (e InvalidUploadID) Error() string {
|
|||
return "Invalid upload id " + e.UploadID
|
||||
}
|
||||
|
||||
// InvalidPart One or more of the specified parts could not be found
|
||||
// InvalidPart One or more of the specified parts could not be found.
|
||||
type InvalidPart struct {
|
||||
PartNumber int
|
||||
ExpETag string
|
||||
|
|
@ -1975,21 +1975,21 @@ func (e PartTooBig) Error() string {
|
|||
return "Part size bigger than the allowed limit"
|
||||
}
|
||||
|
||||
// InvalidETag error returned when the etag has changed on disk
|
||||
// InvalidETag error returned when the etag has changed on disk.
|
||||
type InvalidETag struct{}
|
||||
|
||||
func (e InvalidETag) Error() string {
|
||||
return "etag of the object has changed"
|
||||
}
|
||||
|
||||
// NotImplemented If a feature is not implemented
|
||||
// NotImplemented If a feature is not implemented.
|
||||
type NotImplemented struct{}
|
||||
|
||||
func (e NotImplemented) Error() string {
|
||||
return "Not Implemented"
|
||||
}
|
||||
|
||||
// UnsupportedMetadata - unsupported metadata
|
||||
// UnsupportedMetadata - unsupported metadata.
|
||||
type UnsupportedMetadata struct{}
|
||||
|
||||
func (e UnsupportedMetadata) Error() string {
|
||||
|
|
@ -2003,14 +2003,14 @@ func (e BackendDown) Error() string {
|
|||
return "Backend down"
|
||||
}
|
||||
|
||||
// PreConditionFailed - Check if copy precondition failed
|
||||
// PreConditionFailed - Check if copy precondition failed.
|
||||
type PreConditionFailed struct{}
|
||||
|
||||
func (e PreConditionFailed) Error() string {
|
||||
return "At least one of the pre-conditions you specified did not hold"
|
||||
}
|
||||
|
||||
// DeleteError - returns when cant remove object
|
||||
// DeleteError - returns when cant remove object.
|
||||
type DeleteError struct {
|
||||
Err error
|
||||
Object string
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ type (
|
|||
obj layer.Client
|
||||
}
|
||||
|
||||
// Params holds logger and client.
|
||||
Params struct {
|
||||
Log *zap.Logger
|
||||
Obj layer.Client
|
||||
|
|
@ -24,6 +25,7 @@ const notSupported = "Not supported by NeoFS S3 Gate: "
|
|||
|
||||
var _ api.Handler = (*handler)(nil)
|
||||
|
||||
// New creates new api.Handler using given logger and client.
|
||||
func New(log *zap.Logger, obj layer.Client) (api.Handler, error) {
|
||||
switch {
|
||||
case obj == nil:
|
||||
|
|
|
|||
|
|
@ -68,7 +68,7 @@ func (h *handler) DeleteObjectHandler(w http.ResponseWriter, r *http.Request) {
|
|||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
// DeleteMultipleObjectsHandler :
|
||||
// DeleteMultipleObjectsHandler handles multiple delete requests.
|
||||
func (h *handler) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Request) {
|
||||
var (
|
||||
req = mux.Vars(r)
|
||||
|
|
|
|||
|
|
@ -109,7 +109,7 @@ func (h *handler) HeadBucketHandler(w http.ResponseWriter, r *http.Request) {
|
|||
|
||||
code := http.StatusBadRequest
|
||||
if st, ok := status.FromError(err); ok && st != nil {
|
||||
switch st.Code() {
|
||||
switch st.Code() { //nolint:exhaustive // we have default value set above
|
||||
case codes.NotFound:
|
||||
code = http.StatusNotFound
|
||||
case codes.PermissionDenied:
|
||||
|
|
|
|||
|
|
@ -22,12 +22,14 @@ type listObjectsArgs struct {
|
|||
Version string
|
||||
}
|
||||
|
||||
// VersioningConfiguration contains VersioningConfiguration XML representation.
|
||||
type VersioningConfiguration struct {
|
||||
XMLName xml.Name `xml:"VersioningConfiguration"`
|
||||
Text string `xml:",chardata"`
|
||||
Xmlns string `xml:"xmlns,attr"`
|
||||
}
|
||||
|
||||
// ListMultipartUploadsResult contains ListMultipartUploadsResult XML representation.
|
||||
type ListMultipartUploadsResult struct {
|
||||
XMLName xml.Name `xml:"ListMultipartUploadsResult"`
|
||||
Text string `xml:",chardata"`
|
||||
|
|
@ -36,6 +38,7 @@ type ListMultipartUploadsResult struct {
|
|||
|
||||
var maxObjectList = 10000 // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse.
|
||||
|
||||
// ListBucketsHandler handles bucket listing requests.
|
||||
func (h *handler) ListBucketsHandler(w http.ResponseWriter, r *http.Request) {
|
||||
var (
|
||||
err error
|
||||
|
|
@ -134,6 +137,7 @@ func (h *handler) listObjects(w http.ResponseWriter, r *http.Request) (*listObje
|
|||
return arg, list, nil
|
||||
}
|
||||
|
||||
// ListObjectsV1Handler handles objects listing requests for API version 1.
|
||||
func (h *handler) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) {
|
||||
var rid = api.GetRequestID(r.Context())
|
||||
if arg, list, err := h.listObjects(w, r); err != nil {
|
||||
|
|
@ -193,6 +197,7 @@ func encodeV1(arg *listObjectsArgs, list *layer.ListObjectsInfo) *ListObjectsRes
|
|||
return res
|
||||
}
|
||||
|
||||
// ListObjectsV2Handler handles objects listing requests for API version 2.
|
||||
func (h *handler) ListObjectsV2Handler(w http.ResponseWriter, r *http.Request) {
|
||||
var rid = api.GetRequestID(r.Context())
|
||||
if arg, list, err := h.listObjects(w, r); err != nil {
|
||||
|
|
@ -278,6 +283,7 @@ func parseListObjectArgs(r *http.Request) (*listObjectsArgs, error) {
|
|||
return &res, nil
|
||||
}
|
||||
|
||||
// GetBucketVersioningHandler implements bucket versioning getter handler.
|
||||
func (h *handler) GetBucketVersioningHandler(w http.ResponseWriter, r *http.Request) {
|
||||
var (
|
||||
rid = api.GetRequestID(r.Context())
|
||||
|
|
@ -299,6 +305,7 @@ func (h *handler) GetBucketVersioningHandler(w http.ResponseWriter, r *http.Requ
|
|||
}
|
||||
}
|
||||
|
||||
// ListMultipartUploadsHandler implements multipart uploads listing handler.
|
||||
func (h *handler) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Request) {
|
||||
var (
|
||||
rid = api.GetRequestID(r.Context())
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ package handler
|
|||
|
||||
import "encoding/xml"
|
||||
|
||||
// ListBucketsResponse - format for list buckets response
|
||||
// ListBucketsResponse - format for list buckets response.
|
||||
type ListBucketsResponse struct {
|
||||
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListAllMyBucketsResult" json:"-"`
|
||||
|
||||
|
|
@ -45,13 +45,13 @@ type ListObjectsV2Response struct {
|
|||
EncodingType string `xml:"EncodingType,omitempty"`
|
||||
}
|
||||
|
||||
// Bucket container for bucket metadata
|
||||
// Bucket container for bucket metadata.
|
||||
type Bucket struct {
|
||||
Name string
|
||||
CreationDate string // time string of format "2006-01-02T15:04:05.000Z"
|
||||
}
|
||||
|
||||
// Owner - bucket owner/principal
|
||||
// Owner - bucket owner/principal.
|
||||
type Owner struct {
|
||||
ID string
|
||||
DisplayName string
|
||||
|
|
@ -87,12 +87,12 @@ type ListObjectsResponse struct {
|
|||
EncodingType string `xml:"EncodingType,omitempty"`
|
||||
}
|
||||
|
||||
// CommonPrefix container for prefix response in ListObjectsResponse
|
||||
// CommonPrefix container for prefix response in ListObjectsResponse.
|
||||
type CommonPrefix struct {
|
||||
Prefix string
|
||||
}
|
||||
|
||||
// Object container for object metadata
|
||||
// Object container for object metadata.
|
||||
type Object struct {
|
||||
Key string
|
||||
LastModified string // time string of format "2006-01-02T15:04:05.000Z"
|
||||
|
|
@ -118,7 +118,7 @@ type LocationResponse struct {
|
|||
Location string `xml:",chardata"`
|
||||
}
|
||||
|
||||
// CopyObjectResponse container returns ETag and LastModified of the successfully copied object
|
||||
// CopyObjectResponse container returns ETag and LastModified of the successfully copied object.
|
||||
type CopyObjectResponse struct {
|
||||
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CopyObjectResult" json:"-"`
|
||||
LastModified string // time string of format "2006-01-02T15:04:05.000Z"
|
||||
|
|
@ -127,7 +127,6 @@ type CopyObjectResponse struct {
|
|||
|
||||
// MarshalXML - StringMap marshals into XML.
|
||||
func (s StringMap) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
|
||||
|
||||
tokens := []xml.Token{start}
|
||||
|
||||
for key, value := range s {
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
package api
|
||||
|
||||
// Standard S3 HTTP response constants
|
||||
// Standard S3 HTTP response constants.
|
||||
const (
|
||||
LastModified = "Last-Modified"
|
||||
Date = "Date"
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ import (
|
|||
)
|
||||
|
||||
type (
|
||||
// BucketInfo stores basic bucket data.
|
||||
BucketInfo struct {
|
||||
Name string
|
||||
CID *container.ID
|
||||
|
|
@ -19,6 +20,7 @@ type (
|
|||
Created time.Time
|
||||
}
|
||||
|
||||
// ListObjectsParams represents object listing request parameters.
|
||||
ListObjectsParams struct {
|
||||
Bucket string
|
||||
Prefix string
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ type (
|
|||
log *zap.Logger
|
||||
}
|
||||
|
||||
// Params stores basic API parameters.
|
||||
Params struct {
|
||||
Pool pool.Client
|
||||
Logger *zap.Logger
|
||||
|
|
@ -32,6 +33,7 @@ type (
|
|||
Credential neofs.Credentials
|
||||
}
|
||||
|
||||
// GetObjectParams stores object get request parameters.
|
||||
GetObjectParams struct {
|
||||
Bucket string
|
||||
Object string
|
||||
|
|
@ -40,6 +42,7 @@ type (
|
|||
Writer io.Writer
|
||||
}
|
||||
|
||||
// PutObjectParams stores object put request parameters.
|
||||
PutObjectParams struct {
|
||||
Bucket string
|
||||
Object string
|
||||
|
|
@ -48,6 +51,7 @@ type (
|
|||
Header map[string]string
|
||||
}
|
||||
|
||||
// CopyObjectParams stores object copy request parameters.
|
||||
CopyObjectParams struct {
|
||||
SrcBucket string
|
||||
DstBucket string
|
||||
|
|
@ -56,10 +60,12 @@ type (
|
|||
Header map[string]string
|
||||
}
|
||||
|
||||
// NeoFS provides basic NeoFS interface.
|
||||
NeoFS interface {
|
||||
Get(ctx context.Context, address *object.Address) (*object.Object, error)
|
||||
}
|
||||
|
||||
// Client provides S3 API client interface.
|
||||
Client interface {
|
||||
NeoFS
|
||||
|
||||
|
|
@ -81,11 +87,13 @@ type (
|
|||
)
|
||||
|
||||
var (
|
||||
// ErrObjectExists is returned on attempts to create already existing object.
|
||||
ErrObjectExists = errors.New("object exists")
|
||||
// ErrObjectNotExists is returned on attempts to work with non-existing object.
|
||||
ErrObjectNotExists = errors.New("object not exists")
|
||||
)
|
||||
|
||||
// NewGatewayLayer creates instance of layer. It checks credentials
|
||||
// NewLayer creates instance of layer. It checks credentials
|
||||
// and establishes gRPC connection with node.
|
||||
func NewLayer(log *zap.Logger, cli sdk.Client) Client {
|
||||
return &layer{
|
||||
|
|
@ -103,7 +111,7 @@ func (n *layer) Owner(ctx context.Context) *owner.ID {
|
|||
return n.cli.Owner()
|
||||
}
|
||||
|
||||
// Get NeoFS Object by refs.Address (should be used by auth.Center)
|
||||
// Get NeoFS Object by refs.Address (should be used by auth.Center).
|
||||
func (n *layer) Get(ctx context.Context, address *object.Address) (*object.Object, error) {
|
||||
return n.cli.Object().Get(ctx, address)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ import (
|
|||
)
|
||||
|
||||
type (
|
||||
// ObjectInfo holds S3 object data.
|
||||
ObjectInfo struct {
|
||||
id *object.ID
|
||||
isDir bool
|
||||
|
|
@ -52,6 +53,7 @@ type (
|
|||
|
||||
const (
|
||||
rootSeparator = "root://"
|
||||
// PathSeparator is a path components separator string.
|
||||
PathSeparator = string(os.PathSeparator)
|
||||
)
|
||||
|
||||
|
|
@ -140,11 +142,14 @@ func nameFromObject(o *object.Object) (string, string) {
|
|||
return NameFromString(name)
|
||||
}
|
||||
|
||||
// NameFromString splits name into base file name and directory path.
|
||||
func NameFromString(name string) (string, string) {
|
||||
ind := strings.LastIndex(name, PathSeparator)
|
||||
return name[ind+1:], name[:ind+1]
|
||||
}
|
||||
|
||||
// ID returns object ID from ObjectInfo.
|
||||
func (o *ObjectInfo) ID() *object.ID { return o.id }
|
||||
|
||||
// IsDir allows to check if object is a directory.
|
||||
func (o *ObjectInfo) IsDir() bool { return o.isDir }
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import (
|
|||
)
|
||||
|
||||
type (
|
||||
// MaxClients provides HTTP handler wrapper with client limit.
|
||||
MaxClients interface {
|
||||
Handle(http.HandlerFunc) http.HandlerFunc
|
||||
}
|
||||
|
|
@ -18,6 +19,8 @@ type (
|
|||
|
||||
const defaultRequestDeadline = time.Second * 30
|
||||
|
||||
// NewMaxClientsMiddleware returns MaxClients interface with handler wrapper based on
|
||||
// provided count and timeout limits.
|
||||
func NewMaxClientsMiddleware(count int, timeout time.Duration) MaxClients {
|
||||
if timeout <= 0 {
|
||||
timeout = defaultRequestDeadline
|
||||
|
|
@ -29,6 +32,7 @@ func NewMaxClientsMiddleware(count int, timeout time.Duration) MaxClients {
|
|||
}
|
||||
}
|
||||
|
||||
// Handler wraps HTTP handler function with logic limiting access to it.
|
||||
func (m *maxClients) Handle(f http.HandlerFunc) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
if m.pool == nil {
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ type (
|
|||
}
|
||||
|
||||
// HTTPStats holds statistics information about
|
||||
// HTTP requests made by all clients
|
||||
// HTTP requests made by all clients.
|
||||
HTTPStats struct {
|
||||
currentS3Requests HTTPAPIStats
|
||||
totalS3Requests HTTPAPIStats
|
||||
|
|
@ -63,8 +63,8 @@ var (
|
|||
)
|
||||
)
|
||||
|
||||
// collects http metrics for NeoFS S3 Gate in Prometheus specific format
|
||||
// and sends to given channel
|
||||
// Collects HTTP metrics for NeoFS S3 Gate in Prometheus specific format
|
||||
// and sends to given channel.
|
||||
func collectHTTPMetrics(ch chan<- prometheus.Metric) {
|
||||
for api, value := range httpStatsMetric.currentS3Requests.Load() {
|
||||
ch <- prometheus.MustNewConstMetric(
|
||||
|
|
@ -103,6 +103,7 @@ func collectHTTPMetrics(ch chan<- prometheus.Metric) {
|
|||
}
|
||||
}
|
||||
|
||||
// APIStats wraps http handler for api with basic statistics collection.
|
||||
func APIStats(api string, f http.HandlerFunc) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
httpStatsMetric.currentS3Requests.Inc(api)
|
||||
|
|
@ -176,7 +177,7 @@ func (st *HTTPStats) getOutputBytes() uint64 {
|
|||
return atomic.LoadUint64(&st.totalOutputBytes)
|
||||
}
|
||||
|
||||
// Update statistics from http request and response data
|
||||
// Update statistics from http request and response data.
|
||||
func (st *HTTPStats) updateStats(api string, w http.ResponseWriter, r *http.Request, durationSecs float64) {
|
||||
var code int
|
||||
|
||||
|
|
@ -200,7 +201,7 @@ func (st *HTTPStats) updateStats(api string, w http.ResponseWriter, r *http.Requ
|
|||
}
|
||||
}
|
||||
|
||||
// WriteHeader - writes http status code
|
||||
// WriteHeader - writes http status code.
|
||||
func (w *responseWrapper) WriteHeader(code int) {
|
||||
w.Do(func() {
|
||||
w.statusCode = code
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ import (
|
|||
)
|
||||
|
||||
type (
|
||||
// KeyVal - appended to ReqInfo.Tags
|
||||
// KeyVal - appended to ReqInfo.Tags.
|
||||
KeyVal struct {
|
||||
Key string
|
||||
Val string
|
||||
|
|
@ -33,6 +33,7 @@ type (
|
|||
tags []KeyVal // Any additional info not accommodated by above fields
|
||||
}
|
||||
|
||||
// ObjectRequest represents object request data.
|
||||
ObjectRequest struct {
|
||||
Bucket string
|
||||
Object string
|
||||
|
|
@ -40,7 +41,7 @@ type (
|
|||
}
|
||||
)
|
||||
|
||||
// Key used for Get/SetReqInfo
|
||||
// Key used for Get/SetReqInfo.
|
||||
type contextKeyType string
|
||||
|
||||
const ctxRequestInfo = contextKeyType("NeoFS-S3-Gate")
|
||||
|
|
@ -54,7 +55,7 @@ var (
|
|||
var (
|
||||
// RFC7239 defines a new "Forwarded: " header designed to replace the
|
||||
// existing use of X-Forwarded-* headers.
|
||||
// e.g. Forwarded: for=192.0.2.60;proto=https;by=203.0.113.43
|
||||
// e.g. Forwarded: for=192.0.2.60;proto=https;by=203.0.113.43.
|
||||
forwarded = http.CanonicalHeaderKey("Forwarded")
|
||||
// Allows for a sub-match of the first value after 'for=' to the next
|
||||
// comma, semi-colon or space. The match is case-insensitive.
|
||||
|
|
@ -125,7 +126,7 @@ func prepareContext(w http.ResponseWriter, r *http.Request) context.Context {
|
|||
}))
|
||||
}
|
||||
|
||||
// NewReqInfo :
|
||||
// NewReqInfo returns new ReqInfo based on parameters.
|
||||
func NewReqInfo(w http.ResponseWriter, r *http.Request, req ObjectRequest) *ReqInfo {
|
||||
return &ReqInfo{
|
||||
API: req.Method,
|
||||
|
|
@ -138,7 +139,7 @@ func NewReqInfo(w http.ResponseWriter, r *http.Request, req ObjectRequest) *ReqI
|
|||
}
|
||||
}
|
||||
|
||||
// AppendTags - appends key/val to ReqInfo.tags
|
||||
// AppendTags - appends key/val to ReqInfo.tags.
|
||||
func (r *ReqInfo) AppendTags(key string, val string) *ReqInfo {
|
||||
if r == nil {
|
||||
return nil
|
||||
|
|
@ -149,7 +150,7 @@ func (r *ReqInfo) AppendTags(key string, val string) *ReqInfo {
|
|||
return r
|
||||
}
|
||||
|
||||
// SetTags - sets key/val to ReqInfo.tags
|
||||
// SetTags - sets key/val to ReqInfo.tags.
|
||||
func (r *ReqInfo) SetTags(key string, val string) *ReqInfo {
|
||||
if r == nil {
|
||||
return nil
|
||||
|
|
@ -172,7 +173,7 @@ func (r *ReqInfo) SetTags(key string, val string) *ReqInfo {
|
|||
return r
|
||||
}
|
||||
|
||||
// GetTags - returns the user defined tags
|
||||
// GetTags - returns the user defined tags.
|
||||
func (r *ReqInfo) GetTags() []KeyVal {
|
||||
if r == nil {
|
||||
return nil
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ import (
|
|||
)
|
||||
|
||||
type (
|
||||
// ErrorResponse - error response format
|
||||
// ErrorResponse - error response format.
|
||||
ErrorResponse struct {
|
||||
XMLName xml.Name `xml:"Error" json:"-"`
|
||||
Code string
|
||||
|
|
@ -32,11 +32,11 @@ type (
|
|||
// Captures the server string returned in response header.
|
||||
Server string `xml:"-" json:"-"`
|
||||
|
||||
// Underlying HTTP status code for the returned error
|
||||
// Underlying HTTP status code for the returned error.
|
||||
StatusCode int `xml:"-" json:"-"`
|
||||
}
|
||||
|
||||
// APIError structure
|
||||
// Error structure represents API error.
|
||||
Error struct {
|
||||
Code string
|
||||
Description string
|
||||
|
|
@ -117,7 +117,7 @@ var s3ErrorResponseMap = map[string]string{
|
|||
// Add new API errors here.
|
||||
}
|
||||
|
||||
// WriteErrorResponse writes error headers
|
||||
// WriteErrorResponse writes error headers.
|
||||
func WriteErrorResponse(ctx context.Context, w http.ResponseWriter, err error, reqURL *url.URL) {
|
||||
code := http.StatusBadRequest
|
||||
|
||||
|
|
@ -141,7 +141,7 @@ func WriteErrorResponse(ctx context.Context, w http.ResponseWriter, err error, r
|
|||
WriteResponse(w, code, encodedErrorResponse, MimeXML)
|
||||
}
|
||||
|
||||
// If none of the http routes match respond with appropriate errors
|
||||
// If none of the http routes match respond with appropriate errors.
|
||||
func errorResponseHandler(w http.ResponseWriter, r *http.Request) {
|
||||
desc := fmt.Sprintf("Unknown API request at %s", r.URL.Path)
|
||||
WriteErrorResponse(r.Context(), w, Error{
|
||||
|
|
@ -151,7 +151,7 @@ func errorResponseHandler(w http.ResponseWriter, r *http.Request) {
|
|||
}, r.URL)
|
||||
}
|
||||
|
||||
// Write http common headers
|
||||
// Write http common headers.
|
||||
func setCommonHeaders(w http.ResponseWriter) {
|
||||
w.Header().Set(hdrServerInfo, "NeoFS-S3-Gate/"+misc.Version)
|
||||
w.Header().Set(hdrAcceptRanges, "bytes")
|
||||
|
|
@ -168,6 +168,7 @@ func removeSensitiveHeaders(h http.Header) {
|
|||
h.Del(hdrSSECopyKey)
|
||||
}
|
||||
|
||||
// WriteResponse writes given statusCode and response into w (with mType header if set).
|
||||
func WriteResponse(w http.ResponseWriter, statusCode int, response []byte, mType mimeType) {
|
||||
setCommonHeaders(w)
|
||||
if mType != MimeNone {
|
||||
|
|
@ -214,6 +215,8 @@ func EncodeToResponse(w http.ResponseWriter, response interface{}) error {
|
|||
// WriteResponse(w, http.StatusOK, response, MimeXML)
|
||||
// }
|
||||
|
||||
// WriteSuccessResponseHeadersOnly writes HTTP (200) OK response with no data
|
||||
// to the client.
|
||||
func WriteSuccessResponseHeadersOnly(w http.ResponseWriter) {
|
||||
WriteResponse(w, http.StatusOK, nil, MimeNone)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ import (
|
|||
)
|
||||
|
||||
type (
|
||||
// Handler is an S3 API handler interface.
|
||||
Handler interface {
|
||||
HeadObjectHandler(http.ResponseWriter, *http.Request)
|
||||
CopyObjectPartHandler(http.ResponseWriter, *http.Request)
|
||||
|
|
@ -158,6 +159,7 @@ func logErrorResponse(l *zap.Logger) mux.MiddlewareFunc {
|
|||
}
|
||||
}
|
||||
|
||||
// GetRequestID returns request ID from response writer or context.
|
||||
func GetRequestID(v interface{}) string {
|
||||
switch t := v.(type) {
|
||||
case context.Context:
|
||||
|
|
@ -169,6 +171,8 @@ func GetRequestID(v interface{}) string {
|
|||
}
|
||||
}
|
||||
|
||||
// Attach adds S3 API handlers from h to r for domains with m client limit using
|
||||
// center authentication and log logger.
|
||||
func Attach(r *mux.Router, domains []string, m MaxClients, h Handler, center auth.Center, log *zap.Logger) {
|
||||
api := r.PathPrefix(SlashSeparator).Subrouter()
|
||||
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import (
|
|||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// AttachUserAuth adds user authentication via center to router using log for logging.
|
||||
func AttachUserAuth(router *mux.Router, center auth.Center, log *zap.Logger) {
|
||||
router.Use(func(h http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
|
|
@ -21,7 +22,6 @@ func AttachUserAuth(router *mux.Router, center auth.Center, log *zap.Logger) {
|
|||
|
||||
h.ServeHTTP(w, r.WithContext(
|
||||
sdk.SetBearerToken(r.Context(), token)))
|
||||
|
||||
})
|
||||
})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import (
|
|||
"github.com/gorilla/mux"
|
||||
)
|
||||
|
||||
// Healthy is a health check interface.
|
||||
type Healthy interface {
|
||||
Status() error
|
||||
}
|
||||
|
|
|
|||
|
|
@ -34,8 +34,8 @@ const (
|
|||
defaultMaxClientsDeadline = time.Second * 30
|
||||
)
|
||||
|
||||
const ( // settings
|
||||
// Logger:
|
||||
const ( // Settings.
|
||||
// Logger.
|
||||
cfgLoggerLevel = "logger.level"
|
||||
cfgLoggerFormat = "logger.format"
|
||||
cfgLoggerTraceLevel = "logger.trace_level"
|
||||
|
|
@ -44,47 +44,47 @@ const ( // settings
|
|||
cfgLoggerSamplingInitial = "logger.sampling.initial"
|
||||
cfgLoggerSamplingThereafter = "logger.sampling.thereafter"
|
||||
|
||||
// KeepAlive
|
||||
// KeepAlive.
|
||||
cfgKeepaliveTime = "keepalive.time"
|
||||
cfgKeepaliveTimeout = "keepalive.timeout"
|
||||
cfgKeepalivePermitWithoutStream = "keepalive.permit_without_stream"
|
||||
|
||||
// Keys
|
||||
// Keys.
|
||||
cfgNeoFSPrivateKey = "neofs-key"
|
||||
cfgGateAuthPrivateKey = "auth-key"
|
||||
|
||||
// HTTPS/TLS
|
||||
// HTTPS/TLS.
|
||||
cfgTLSKeyFile = "tls.key_file"
|
||||
cfgTLSCertFile = "tls.cert_file"
|
||||
|
||||
// Timeouts
|
||||
// Timeouts.
|
||||
cfgConnectionTTL = "con_ttl"
|
||||
cfgConnectTimeout = "connect_timeout"
|
||||
cfgRequestTimeout = "request_timeout"
|
||||
cfgRebalanceTimer = "rebalance_timer"
|
||||
|
||||
// MaxClients
|
||||
// MaxClients.
|
||||
cfgMaxClientsCount = "max_clients_count"
|
||||
cfgMaxClientsDeadline = "max_clients_deadline"
|
||||
|
||||
// gRPC
|
||||
// gRPC.
|
||||
cfgGRPCVerbose = "verbose"
|
||||
|
||||
// Metrics / Profiler / Web
|
||||
// Metrics / Profiler / Web.
|
||||
cfgEnableMetrics = "metrics"
|
||||
cfgEnableProfiler = "pprof"
|
||||
cfgListenAddress = "listen_address"
|
||||
cfgListenDomains = "listen_domains"
|
||||
|
||||
// Peers
|
||||
// Peers.
|
||||
cfgPeers = "peers"
|
||||
|
||||
// Application
|
||||
// Application.
|
||||
cfgApplicationName = "app.name"
|
||||
cfgApplicationVersion = "app.version"
|
||||
cfgApplicationBuildTime = "app.build_time"
|
||||
|
||||
// command line args
|
||||
// Command line args.
|
||||
cmdHelp = "help"
|
||||
cmdVersion = "version"
|
||||
)
|
||||
|
|
@ -105,10 +105,9 @@ var ignore = map[string]struct{}{
|
|||
func (empty) Read([]byte) (int, error) { return 0, io.EOF }
|
||||
|
||||
func fetchPeers(l *zap.Logger, v *viper.Viper) map[string]float64 {
|
||||
peers := make(map[string]float64, 0)
|
||||
peers := make(map[string]float64)
|
||||
|
||||
for i := 0; ; i++ {
|
||||
|
||||
key := cfgPeers + "." + strconv.Itoa(i) + "."
|
||||
address := v.GetString(key + "address")
|
||||
weight := v.GetFloat64(key + "weight")
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ import (
|
|||
)
|
||||
|
||||
type (
|
||||
// App is the main application structure.
|
||||
App struct {
|
||||
cli pool.Client
|
||||
ctr auth.Center
|
||||
|
|
@ -179,6 +180,7 @@ func newApp(ctx context.Context, l *zap.Logger, v *viper.Viper) *App {
|
|||
}
|
||||
}
|
||||
|
||||
// Wait waits for application to finish.
|
||||
func (a *App) Wait() {
|
||||
a.log.Info("application started")
|
||||
|
||||
|
|
@ -192,6 +194,7 @@ func (a *App) Wait() {
|
|||
a.log.Info("application finished")
|
||||
}
|
||||
|
||||
// Server runs HTTP server to handle S3 API requests.
|
||||
func (a *App) Server(ctx context.Context) {
|
||||
var (
|
||||
err error
|
||||
|
|
@ -256,6 +259,7 @@ func (a *App) Server(ctx context.Context) {
|
|||
close(a.webDone)
|
||||
}
|
||||
|
||||
// Worker runs client worker.
|
||||
func (a *App) Worker(ctx context.Context) {
|
||||
a.cli.Worker(ctx)
|
||||
a.log.Info("stopping worker")
|
||||
|
|
|
|||
|
|
@ -1,12 +1,16 @@
|
|||
package misc
|
||||
|
||||
const (
|
||||
// ApplicationName is gateway name.
|
||||
ApplicationName = "neofs-s3-gate"
|
||||
|
||||
// Prefix is configuration environment variables prefix.
|
||||
Prefix = "S3_GW"
|
||||
)
|
||||
|
||||
var (
|
||||
// Build holds build timestamp.
|
||||
Build = "now"
|
||||
// Version contains application version.
|
||||
Version = "dev"
|
||||
)
|
||||
|
|
|
|||
Loading…
Reference in a new issue