forked from TrueCloudLab/frostfs-s3-gw
Compare commits
44 commits
KirillovDe
...
master
Author | SHA1 | Date | |
---|---|---|---|
fb90c0f52c | |||
f2f90e260e | |||
a0937126cb | |||
655889a1a2 | |||
ef556bd8ac | |||
5104683f68 | |||
8151753eeb | |||
2282c32822 | |||
43685e03d9 | |||
cf18158da4 | |||
5c62010331 | |||
0af06c3bd9 | |||
680c0dbe3d | |||
596381c382 | |||
64e7356acc | |||
32bf915502 | |||
813aa2f173 | |||
6eb7966800 | |||
2dcb3c283d | |||
740acadd37 | |||
3ab77c8990 | |||
d00163aadc | |||
aadefd98b6 | |||
8ac630ee71 | |||
|
744b52322d | ||
8d9d1f9235 | |||
787d1a347a | |||
9f823bd65a | |||
9dcacc230e | |||
430f1e734f | |||
1ce8b8a30d | |||
b35f146cec | |||
5ee4bf80ae | |||
f9f52ce8e0 | |||
e278ab9362 | |||
c5570e661d | |||
fc5c09c084 | |||
86e881694d | |||
361d1d3881 | |||
9ad7982807 | |||
a0d5b18184 | |||
533b12d8bb | |||
cafe079072 | |||
9473335234 |
131 changed files with 2438 additions and 2071 deletions
|
@ -1,7 +1,7 @@
|
||||||
FROM golang:1.19 as builder
|
FROM golang:1.19 as builder
|
||||||
|
|
||||||
ARG BUILD=now
|
ARG BUILD=now
|
||||||
ARG REPO=github.com/TrueCloudLab/frostfs-s3-gw
|
ARG REPO=git.frostfs.info/TrueCloudLab/frostfs-s3-gw
|
||||||
ARG VERSION=dev
|
ARG VERSION=dev
|
||||||
|
|
||||||
WORKDIR /src
|
WORKDIR /src
|
||||||
|
|
45
.github/ISSUE_TEMPLATE/bug_report.md
vendored
Normal file
45
.github/ISSUE_TEMPLATE/bug_report.md
vendored
Normal file
|
@ -0,0 +1,45 @@
|
||||||
|
---
|
||||||
|
name: Bug report
|
||||||
|
about: Create a report to help us improve
|
||||||
|
title: ''
|
||||||
|
labels: community, triage, bug
|
||||||
|
assignees: ''
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
<!--- Provide a general summary of the issue in the Title above -->
|
||||||
|
|
||||||
|
## Expected Behavior
|
||||||
|
<!--- If you're describing a bug, tell us what should happen -->
|
||||||
|
<!--- If you're suggesting a change/improvement, tell us how it should work -->
|
||||||
|
|
||||||
|
## Current Behavior
|
||||||
|
<!--- If describing a bug, tell us what happens instead of the expected behavior -->
|
||||||
|
<!--- If suggesting a change/improvement, explain the difference from current behavior -->
|
||||||
|
|
||||||
|
## Possible Solution
|
||||||
|
<!--- Not obligatory -->
|
||||||
|
<!--- If no reason/fix/additions for the bug can be suggested, -->
|
||||||
|
<!--- uncomment the following phrase: -->
|
||||||
|
|
||||||
|
<!--- No fix can be suggested by a QA engineer. Further solutions shall be up to developers. -->
|
||||||
|
|
||||||
|
## Steps to Reproduce (for bugs)
|
||||||
|
<!--- Provide a link to a live example, or an unambiguous set of steps to -->
|
||||||
|
<!--- reproduce this bug. -->
|
||||||
|
|
||||||
|
1.
|
||||||
|
|
||||||
|
## Context
|
||||||
|
<!--- How has this issue affected you? What are you trying to accomplish? -->
|
||||||
|
<!--- Providing context helps us come up with a solution that is most useful in the real world -->
|
||||||
|
|
||||||
|
## Regression
|
||||||
|
<!-- Is this issue a regression? (Yes / No) -->
|
||||||
|
<!-- If Yes, optionally please include version or commit id or PR# that caused this regression, if you have these details. -->
|
||||||
|
|
||||||
|
## Your Environment
|
||||||
|
<!--- Include as many relevant details about the environment you experienced the bug in -->
|
||||||
|
* Version used:
|
||||||
|
* Server setup and configuration:
|
||||||
|
* Operating System and version (`uname -a`):
|
1
.github/ISSUE_TEMPLATE/config.yml
vendored
Normal file
1
.github/ISSUE_TEMPLATE/config.yml
vendored
Normal file
|
@ -0,0 +1 @@
|
||||||
|
blank_issues_enabled: false
|
20
.github/ISSUE_TEMPLATE/feature_request.md
vendored
Normal file
20
.github/ISSUE_TEMPLATE/feature_request.md
vendored
Normal file
|
@ -0,0 +1,20 @@
|
||||||
|
---
|
||||||
|
name: Feature request
|
||||||
|
about: Suggest an idea for this project
|
||||||
|
title: ''
|
||||||
|
labels: community, triage
|
||||||
|
assignees: ''
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Is your feature request related to a problem? Please describe.
|
||||||
|
<!--- A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] -->
|
||||||
|
|
||||||
|
## Describe the solution you'd like
|
||||||
|
<!--- A clear and concise description of what you want to happen. -->
|
||||||
|
|
||||||
|
## Describe alternatives you've considered
|
||||||
|
<!--- A clear and concise description of any alternative solutions or features you've considered. -->
|
||||||
|
|
||||||
|
## Additional context
|
||||||
|
<!--- Add any other context or screenshots about the feature request here. -->
|
2
.github/workflows/tests.yml
vendored
2
.github/workflows/tests.yml
vendored
|
@ -68,7 +68,7 @@ jobs:
|
||||||
runs-on: ubuntu-20.04
|
runs-on: ubuntu-20.04
|
||||||
strategy:
|
strategy:
|
||||||
matrix:
|
matrix:
|
||||||
go_versions: [ '1.17', '1.18.x', '1.19.x' ]
|
go_versions: [ '1.18.x', '1.19.x' ]
|
||||||
fail-fast: false
|
fail-fast: false
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v2
|
- uses: actions/checkout@v2
|
||||||
|
|
9
.gitignore
vendored
9
.gitignore
vendored
|
@ -21,4 +21,11 @@ coverage.txt
|
||||||
coverage.html
|
coverage.html
|
||||||
|
|
||||||
# debhelpers
|
# debhelpers
|
||||||
**/.debhelper
|
**/*debhelper*
|
||||||
|
|
||||||
|
# debian package build files
|
||||||
|
debian/files
|
||||||
|
debian/*.log
|
||||||
|
debian/*.substvars
|
||||||
|
debian/frostfs-s3-gw/
|
||||||
|
|
||||||
|
|
11
.gitlint
Normal file
11
.gitlint
Normal file
|
@ -0,0 +1,11 @@
|
||||||
|
[general]
|
||||||
|
fail-without-commits=True
|
||||||
|
regex-style-search=True
|
||||||
|
contrib=CC1
|
||||||
|
|
||||||
|
[title-match-regex]
|
||||||
|
regex=^\[\#[0-9Xx]+\]\s
|
||||||
|
|
||||||
|
[ignore-by-title]
|
||||||
|
regex=^Release(.*)
|
||||||
|
ignore=title-match-regex
|
45
.pre-commit-config.yaml
Normal file
45
.pre-commit-config.yaml
Normal file
|
@ -0,0 +1,45 @@
|
||||||
|
ci:
|
||||||
|
autofix_prs: false
|
||||||
|
|
||||||
|
repos:
|
||||||
|
- repo: https://github.com/jorisroovers/gitlint
|
||||||
|
rev: v0.19.1
|
||||||
|
hooks:
|
||||||
|
- id: gitlint
|
||||||
|
stages: [commit-msg]
|
||||||
|
- id: gitlint-ci
|
||||||
|
|
||||||
|
- repo: https://github.com/pre-commit/pre-commit-hooks
|
||||||
|
rev: v4.4.0
|
||||||
|
hooks:
|
||||||
|
- id: check-added-large-files
|
||||||
|
- id: check-case-conflict
|
||||||
|
- id: check-executables-have-shebangs
|
||||||
|
- id: check-shebang-scripts-are-executable
|
||||||
|
- id: check-merge-conflict
|
||||||
|
- id: check-json
|
||||||
|
- id: check-xml
|
||||||
|
- id: check-yaml
|
||||||
|
- id: trailing-whitespace
|
||||||
|
args: [--markdown-linebreak-ext=md]
|
||||||
|
- id: end-of-file-fixer
|
||||||
|
exclude: ".key$"
|
||||||
|
|
||||||
|
- repo: https://github.com/shellcheck-py/shellcheck-py
|
||||||
|
rev: v0.9.0.2
|
||||||
|
hooks:
|
||||||
|
- id: shellcheck
|
||||||
|
|
||||||
|
- repo: https://github.com/golangci/golangci-lint
|
||||||
|
rev: v1.51.2
|
||||||
|
hooks:
|
||||||
|
- id: golangci-lint
|
||||||
|
|
||||||
|
- repo: local
|
||||||
|
hooks:
|
||||||
|
- id: go-unit-tests
|
||||||
|
name: go unit tests
|
||||||
|
entry: make test
|
||||||
|
pass_filenames: false
|
||||||
|
types: [go]
|
||||||
|
language: system
|
24
CHANGELOG.md
24
CHANGELOG.md
|
@ -4,6 +4,30 @@ This document outlines major changes between releases.
|
||||||
|
|
||||||
## [Unreleased]
|
## [Unreleased]
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
- Get empty bucket CORS from frostfs (TrueCloudLab#36)
|
||||||
|
- Don't count pool error on client abort (#35)
|
||||||
|
|
||||||
|
### Added
|
||||||
|
- Return container name in `head-bucket` response (TrueCloudLab#18)
|
||||||
|
- Billing metrics (TrueCloudLab#5)
|
||||||
|
- Multiple configs support (TrueCloudLab#21)
|
||||||
|
- Bucket name resolving policy (TrueCloudLab#25)
|
||||||
|
- Support string `Action` and `Resource` fields in `bucketPolicy.Statement` (TrueCloudLab#32)
|
||||||
|
- Add new `kludge.use_default_xmlns_for_complete_multipart` config param (TrueCloudLab#40)
|
||||||
|
|
||||||
|
### Changed
|
||||||
|
- Update neo-go to v0.101.0 (#14)
|
||||||
|
- Update viper to v1.15.0 (#14)
|
||||||
|
- Using multiple servers require only one healthy (TrueCloudLab#12)
|
||||||
|
- Update go version to go1.18 (TrueCloudLab#16)
|
||||||
|
- Return error on invalid LocationConstraint (TrueCloudLab#23)
|
||||||
|
- Place billing metrics to separate url path (TrueCloudLab#26)
|
||||||
|
- Add generated deb builder files to .gitignore, and fix typo (TrueCloudLab#28)
|
||||||
|
- Limit number of objects to delete at one time (TrueCloudLab#37)
|
||||||
|
- CompleteMultipartUpload handler now sends whitespace characters to keep alive client's connection (#60)
|
||||||
|
- Support new system attributes (#64)
|
||||||
|
|
||||||
## [0.26.0] - 2022-12-28
|
## [0.26.0] - 2022-12-28
|
||||||
|
|
||||||
### Added
|
### Added
|
||||||
|
|
14
Makefile
Normal file → Executable file
14
Makefile
Normal file → Executable file
|
@ -16,14 +16,14 @@ REPO_BASENAME = $(shell basename `go list -m`)
|
||||||
HUB_IMAGE ?= "truecloudlab/$(REPO_BASENAME)"
|
HUB_IMAGE ?= "truecloudlab/$(REPO_BASENAME)"
|
||||||
HUB_TAG ?= "$(shell echo ${VERSION} | sed 's/^v//')"
|
HUB_TAG ?= "$(shell echo ${VERSION} | sed 's/^v//')"
|
||||||
|
|
||||||
.PHONY: all $(BINS) $(BINDIR) dep docker/ test cover format image image-push dirty-image lint docker/lint version clean protoc
|
.PHONY: all $(BINS) $(BINDIR) dep docker/ test cover format image image-push dirty-image lint docker/lint pre-commit unpre-commit version clean protoc
|
||||||
|
|
||||||
# .deb package versioning
|
# .deb package versioning
|
||||||
OS_RELEASE = $(shell lsb_release -cs)
|
OS_RELEASE = $(shell lsb_release -cs)
|
||||||
PKG_VERSION ?= $(shell echo $(VERSION) | sed "s/^v//" | \
|
PKG_VERSION ?= $(shell echo $(VERSION) | sed "s/^v//" | \
|
||||||
sed -E "s/(.*)-(g[a-fA-F0-9]{6,8})(.*)/\1\3~\2/" | \
|
sed -E "s/(.*)-(g[a-fA-F0-9]{6,8})(.*)/\1\3~\2/" | \
|
||||||
sed "s/-/~/")-${OS_RELEASE}
|
sed "s/-/~/")-${OS_RELEASE}
|
||||||
.PHONY: debpackage debclean
|
.PHONY: debpackage debclean
|
||||||
|
|
||||||
# Make all binaries
|
# Make all binaries
|
||||||
all: $(BINS)
|
all: $(BINS)
|
||||||
|
@ -115,6 +115,14 @@ docker/lint:
|
||||||
--env HOME=/src \
|
--env HOME=/src \
|
||||||
golangci/golangci-lint:v$(LINT_VERSION) bash -c 'cd /src/ && make lint'
|
golangci/golangci-lint:v$(LINT_VERSION) bash -c 'cd /src/ && make lint'
|
||||||
|
|
||||||
|
# Activate pre-commit hooks
|
||||||
|
pre-commit:
|
||||||
|
pre-commit install -t pre-commit -t commit-msg
|
||||||
|
|
||||||
|
# Deactivate pre-commit hooks
|
||||||
|
unpre-commit:
|
||||||
|
pre-commit uninstall -t pre-commit -t commit-msg
|
||||||
|
|
||||||
# Show current version
|
# Show current version
|
||||||
version:
|
version:
|
||||||
@echo $(VERSION)
|
@echo $(VERSION)
|
||||||
|
@ -143,6 +151,6 @@ debpackage:
|
||||||
dpkg-buildpackage --no-sign -b
|
dpkg-buildpackage --no-sign -b
|
||||||
|
|
||||||
debclean:
|
debclean:
|
||||||
dh clean
|
dh clean
|
||||||
|
|
||||||
include help.mk
|
include help.mk
|
||||||
|
|
|
@ -6,9 +6,7 @@
|
||||||
</p>
|
</p>
|
||||||
|
|
||||||
---
|
---
|
||||||
[![Report](https://goreportcard.com/badge/github.com/TrueCloudLab/frostfs-s3-gw)](https://goreportcard.com/report/github.com/TrueCloudLab/frostfs-s3-gw)
|
[![Report](https://goreportcard.com/badge/git.frostfs.info/TrueCloudLab/frostfs-s3-gw)](https://goreportcard.com/report/git.frostfs.info/TrueCloudLab/frostfs-s3-gw)
|
||||||
![GitHub release (latest SemVer)](https://img.shields.io/github/v/release/TrueCloudLab/frostfs-s3-gw?sort=semver)
|
|
||||||
![License](https://img.shields.io/github/license/TrueCloudLab/frostfs-s3-gw.svg?style=popout)
|
|
||||||
|
|
||||||
# FrostFS S3 Gateway
|
# FrostFS S3 Gateway
|
||||||
|
|
||||||
|
@ -16,7 +14,7 @@ FrostFS S3 gateway provides API compatible with Amazon S3 cloud storage service.
|
||||||
|
|
||||||
## Installation
|
## Installation
|
||||||
|
|
||||||
```go get -u github.com/TrueCloudLab/frostfs-s3-gw```
|
```go get -u git.frostfs.info/TrueCloudLab/frostfs-s3-gw```
|
||||||
|
|
||||||
Or you can call `make` to build it from the cloned repository (the binary will
|
Or you can call `make` to build it from the cloned repository (the binary will
|
||||||
end up in `bin/frostfs-s3-gw` with authmate helper in `bin/frostfs-s3-authmate`).
|
end up in `bin/frostfs-s3-gw` with authmate helper in `bin/frostfs-s3-authmate`).
|
||||||
|
|
|
@ -14,12 +14,12 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
v4 "github.com/TrueCloudLab/frostfs-s3-gw/api/auth/signer/v4"
|
v4 "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth/signer/v4"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/cache"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/cache"
|
||||||
apiErrors "github.com/TrueCloudLab/frostfs-s3-gw/api/errors"
|
apiErrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/creds/tokens"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/tokens"
|
||||||
oid "github.com/TrueCloudLab/frostfs-sdk-go/object/id"
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
"github.com/aws/aws-sdk-go/aws/credentials"
|
"github.com/aws/aws-sdk-go/aws/credentials"
|
||||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||||
)
|
)
|
||||||
|
|
|
@ -5,7 +5,7 @@ import (
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
2
api/cache/access_control.go
vendored
2
api/cache/access_control.go
vendored
|
@ -4,7 +4,7 @@ import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/user"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||||
"github.com/bluele/gcache"
|
"github.com/bluele/gcache"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
4
api/cache/accessbox.go
vendored
4
api/cache/accessbox.go
vendored
|
@ -4,8 +4,8 @@ import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
||||||
oid "github.com/TrueCloudLab/frostfs-sdk-go/object/id"
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
"github.com/bluele/gcache"
|
"github.com/bluele/gcache"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
2
api/cache/buckets.go
vendored
2
api/cache/buckets.go
vendored
|
@ -4,7 +4,7 @@ import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"github.com/bluele/gcache"
|
"github.com/bluele/gcache"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
8
api/cache/cache_test.go
vendored
8
api/cache/cache_test.go
vendored
|
@ -3,10 +3,10 @@ package cache
|
||||||
import (
|
import (
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
||||||
cidtest "github.com/TrueCloudLab/frostfs-sdk-go/container/id/test"
|
cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
|
||||||
oidtest "github.com/TrueCloudLab/frostfs-sdk-go/object/id/test"
|
oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
"go.uber.org/zap/zaptest/observer"
|
"go.uber.org/zap/zaptest/observer"
|
||||||
|
|
2
api/cache/names.go
vendored
2
api/cache/names.go
vendored
|
@ -4,7 +4,7 @@ import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
oid "github.com/TrueCloudLab/frostfs-sdk-go/object/id"
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
"github.com/bluele/gcache"
|
"github.com/bluele/gcache"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
4
api/cache/objects.go
vendored
4
api/cache/objects.go
vendored
|
@ -4,8 +4,8 @@ import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
oid "github.com/TrueCloudLab/frostfs-sdk-go/object/id"
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
"github.com/bluele/gcache"
|
"github.com/bluele/gcache"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
6
api/cache/objects_test.go
vendored
6
api/cache/objects_test.go
vendored
|
@ -4,9 +4,9 @@ import (
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
oid "github.com/TrueCloudLab/frostfs-sdk-go/object/id"
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
objecttest "github.com/TrueCloudLab/frostfs-sdk-go/object/test"
|
objecttest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/test"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
4
api/cache/objectslist.go
vendored
4
api/cache/objectslist.go
vendored
|
@ -6,8 +6,8 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
cid "github.com/TrueCloudLab/frostfs-sdk-go/container/id"
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||||
"github.com/bluele/gcache"
|
"github.com/bluele/gcache"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
6
api/cache/objectslist_test.go
vendored
6
api/cache/objectslist_test.go
vendored
|
@ -4,9 +4,9 @@ import (
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
cidtest "github.com/TrueCloudLab/frostfs-sdk-go/container/id/test"
|
cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
|
||||||
oidtest "github.com/TrueCloudLab/frostfs-sdk-go/object/id/test"
|
oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
2
api/cache/system.go
vendored
2
api/cache/system.go
vendored
|
@ -4,7 +4,7 @@ import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"github.com/bluele/gcache"
|
"github.com/bluele/gcache"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
|
@ -4,9 +4,9 @@ import (
|
||||||
"encoding/xml"
|
"encoding/xml"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
cid "github.com/TrueCloudLab/frostfs-sdk-go/container/id"
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||||
oid "github.com/TrueCloudLab/frostfs-sdk-go/object/id"
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/user"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
|
@ -22,7 +22,8 @@ const (
|
||||||
type (
|
type (
|
||||||
// BucketInfo stores basic bucket data.
|
// BucketInfo stores basic bucket data.
|
||||||
BucketInfo struct {
|
BucketInfo struct {
|
||||||
Name string
|
Name string // container name from system attribute
|
||||||
|
Zone string // container zone from system attribute
|
||||||
CID cid.ID
|
CID cid.ID
|
||||||
Owner user.ID
|
Owner user.ID
|
||||||
Created time.Time
|
Created time.Time
|
||||||
|
@ -59,13 +60,6 @@ type (
|
||||||
BucketSettings struct {
|
BucketSettings struct {
|
||||||
Versioning string `json:"versioning"`
|
Versioning string `json:"versioning"`
|
||||||
LockConfiguration *ObjectLockConfiguration `json:"lock_configuration"`
|
LockConfiguration *ObjectLockConfiguration `json:"lock_configuration"`
|
||||||
LifecycleConfig *LifecycleConfig `json:"lifecycle_configuration"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// LifecycleConfig stores lifecycle config old and current settings.
|
|
||||||
LifecycleConfig struct {
|
|
||||||
OldConfigurationID string `json:"old_id"`
|
|
||||||
CurrentConfiguration *LifecycleConfiguration
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// CORSConfiguration stores CORS configuration of a request.
|
// CORSConfiguration stores CORS configuration of a request.
|
||||||
|
@ -131,6 +125,3 @@ func (b BucketSettings) VersioningEnabled() bool {
|
||||||
func (b BucketSettings) VersioningSuspended() bool {
|
func (b BucketSettings) VersioningSuspended() bool {
|
||||||
return b.Versioning == VersioningSuspended
|
return b.Versioning == VersioningSuspended
|
||||||
}
|
}
|
||||||
|
|
||||||
// ExpirationObject returns name of system object for expiration tick object.
|
|
||||||
func (o *ObjectInfo) ExpirationObject() string { return ".expiration." + o.Name }
|
|
||||||
|
|
|
@ -1,171 +0,0 @@
|
||||||
package data
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/xml"
|
|
||||||
"strings"
|
|
||||||
)
|
|
||||||
|
|
||||||
type (
|
|
||||||
LifecycleConfiguration struct {
|
|
||||||
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ LifecycleConfiguration" json:"-"`
|
|
||||||
Rules []Rule `xml:"Rule" json:"Rule"`
|
|
||||||
}
|
|
||||||
|
|
||||||
Rule struct {
|
|
||||||
AbortIncompleteMultipartUpload *AbortIncompleteMultipartUpload `xml:"AbortIncompleteMultipartUpload" json:"AbortIncompleteMultipartUpload"`
|
|
||||||
Expiration *Expiration `xml:"Expiration" json:"Expiration"`
|
|
||||||
Filter *LifecycleRuleFilter `xml:"Filter" json:"Filter"`
|
|
||||||
ID string `xml:"ID" json:"ID"`
|
|
||||||
NoncurrentVersionExpiration *NoncurrentVersionExpiration `xml:"NoncurrentVersionExpiration" json:"NoncurrentVersionExpiration"`
|
|
||||||
NoncurrentVersionTransitions []NoncurrentVersionTransition `xml:"NoncurrentVersionTransition" json:"NoncurrentVersionTransition"`
|
|
||||||
Prefix *string `xml:"Prefix" json:"Prefix"`
|
|
||||||
Status string `xml:"Status" json:"Status"`
|
|
||||||
Transitions []Transition `xml:"Transition" json:"Transition"`
|
|
||||||
}
|
|
||||||
|
|
||||||
AbortIncompleteMultipartUpload struct {
|
|
||||||
DaysAfterInitiation int64 `xml:"DaysAfterInitiation"`
|
|
||||||
}
|
|
||||||
|
|
||||||
Expiration struct {
|
|
||||||
Date *string `xml:"Date" json:"Date"`
|
|
||||||
Days *int64 `xml:"Days" json:"Days"`
|
|
||||||
ExpiredObjectDeleteMarker bool `xml:"ExpiredObjectDeleteMarker" json:"ExpiredObjectDeleteMarker"`
|
|
||||||
}
|
|
||||||
|
|
||||||
LifecycleRuleFilter struct {
|
|
||||||
And *LifecycleRuleAndOperator `xml:"And" json:"And"`
|
|
||||||
ObjectSizeGreaterThan *int64 `xml:"ObjectSizeGreaterThan" json:"ObjectSizeGreaterThan"`
|
|
||||||
ObjectSizeLessThan *int64 `xml:"ObjectSizeLessThan" json:"ObjectSizeLessThan"`
|
|
||||||
Prefix *string `xml:"Prefix" json:"Prefix"`
|
|
||||||
Tag *Tag `xml:"Tag" json:"Tag"`
|
|
||||||
}
|
|
||||||
|
|
||||||
LifecycleRuleAndOperator struct {
|
|
||||||
ObjectSizeGreaterThan *int64 `xml:"ObjectSizeGreaterThan" json:"ObjectSizeGreaterThan"`
|
|
||||||
ObjectSizeLessThan *int64 `xml:"ObjectSizeLessThan" json:"ObjectSizeLessThan"`
|
|
||||||
Prefix *string `xml:"Prefix" json:"Prefix"`
|
|
||||||
Tags []Tag `xml:"Tags" json:"Tags"`
|
|
||||||
}
|
|
||||||
|
|
||||||
Tag struct {
|
|
||||||
Key string `xml:"Key" json:"Key"`
|
|
||||||
Value string `xml:"Value" json:"Value"`
|
|
||||||
}
|
|
||||||
|
|
||||||
NoncurrentVersionExpiration struct {
|
|
||||||
NewerNoncurrentVersions *int64 `xml:"NewerNoncurrentVersions" json:"NewerNoncurrentVersions"`
|
|
||||||
NoncurrentDays *int64 `xml:"NoncurrentDays" json:"NoncurrentDays"`
|
|
||||||
}
|
|
||||||
|
|
||||||
NoncurrentVersionTransition struct {
|
|
||||||
NewerNoncurrentVersions *int64 `xml:"NewerNoncurrentVersions" json:"NewerNoncurrentVersions"`
|
|
||||||
NoncurrentDays *int64 `xml:"NoncurrentDays" json:"NoncurrentDays"`
|
|
||||||
StorageClass string `xml:"StorageClass" json:"StorageClass"`
|
|
||||||
}
|
|
||||||
|
|
||||||
Transition struct {
|
|
||||||
Date *string `xml:"Date" json:"Date"`
|
|
||||||
Days *int64 `xml:"Days" json:"Days"`
|
|
||||||
StorageClass string `xml:"StorageClass" json:"StorageClass"`
|
|
||||||
}
|
|
||||||
|
|
||||||
ExpirationObject struct {
|
|
||||||
Expiration *Expiration
|
|
||||||
RuleID string
|
|
||||||
LifecycleConfigID string
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
func (r *Rule) RealPrefix() string {
|
|
||||||
if r.Filter == nil {
|
|
||||||
if r.Prefix != nil {
|
|
||||||
return *r.Prefix
|
|
||||||
}
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
|
|
||||||
if r.Filter.And == nil {
|
|
||||||
if r.Filter.Prefix != nil {
|
|
||||||
return *r.Filter.Prefix
|
|
||||||
}
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
|
|
||||||
if r.Filter.And.Prefix != nil {
|
|
||||||
return *r.Filter.And.Prefix
|
|
||||||
}
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *Rule) NeedTags() bool {
|
|
||||||
if r.Filter == nil {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
if r.Filter.And == nil {
|
|
||||||
return r.Filter.Tag != nil
|
|
||||||
}
|
|
||||||
|
|
||||||
return len(r.Filter.And.Tags) != 0
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *Rule) MatchObject(obj *ObjectInfo, tags map[string]string) bool {
|
|
||||||
if r.Filter == nil {
|
|
||||||
if r.Prefix != nil {
|
|
||||||
return strings.HasPrefix(obj.Name, *r.Prefix)
|
|
||||||
}
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
if r.Filter.And == nil {
|
|
||||||
if r.Filter.Prefix != nil && !strings.HasPrefix(obj.Name, *r.Filter.Prefix) {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
if r.Filter.Tag != nil {
|
|
||||||
if tags == nil {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
if tagVal := tags[r.Filter.Tag.Key]; tagVal != r.Filter.Tag.Value {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if r.Filter.ObjectSizeLessThan != nil && *r.Filter.ObjectSizeLessThan > 0 && obj.Size >= *r.Filter.ObjectSizeLessThan {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
if r.Filter.ObjectSizeGreaterThan != nil && obj.Size <= *r.Filter.ObjectSizeGreaterThan {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
if r.Filter.And.Prefix != nil && !strings.HasPrefix(obj.Name, *r.Filter.And.Prefix) {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(r.Filter.And.Tags) != 0 {
|
|
||||||
if tags == nil {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tag := range r.Filter.And.Tags {
|
|
||||||
if tagVal := tags[tag.Key]; tagVal != tag.Value {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if r.Filter.And.ObjectSizeLessThan != nil && obj.Size >= *r.Filter.And.ObjectSizeLessThan {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
if r.Filter.And.ObjectSizeGreaterThan != nil && obj.Size <= *r.Filter.And.ObjectSizeGreaterThan {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
return true
|
|
||||||
}
|
|
|
@ -4,9 +4,9 @@ import (
|
||||||
"strconv"
|
"strconv"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
cid "github.com/TrueCloudLab/frostfs-sdk-go/container/id"
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||||
oid "github.com/TrueCloudLab/frostfs-sdk-go/object/id"
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/user"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
|
|
|
@ -90,6 +90,7 @@ const (
|
||||||
ErrMissingFields
|
ErrMissingFields
|
||||||
ErrMissingCredTag
|
ErrMissingCredTag
|
||||||
ErrCredMalformed
|
ErrCredMalformed
|
||||||
|
ErrInvalidLocationConstraint
|
||||||
ErrInvalidRegion
|
ErrInvalidRegion
|
||||||
ErrInvalidServiceS3
|
ErrInvalidServiceS3
|
||||||
ErrInvalidServiceSTS
|
ErrInvalidServiceSTS
|
||||||
|
@ -680,6 +681,12 @@ var errorCodes = errorCodeMap{
|
||||||
Description: "Error parsing the X-Amz-Credential parameter; the region is wrong;",
|
Description: "Error parsing the X-Amz-Credential parameter; the region is wrong;",
|
||||||
HTTPStatusCode: http.StatusBadRequest,
|
HTTPStatusCode: http.StatusBadRequest,
|
||||||
},
|
},
|
||||||
|
ErrInvalidLocationConstraint: {
|
||||||
|
ErrCode: ErrInvalidLocationConstraint,
|
||||||
|
Code: "InvalidLocationConstraint",
|
||||||
|
Description: "The specified location (Region) constraint is not valid.",
|
||||||
|
HTTPStatusCode: http.StatusBadRequest,
|
||||||
|
},
|
||||||
ErrInvalidRegion: {
|
ErrInvalidRegion: {
|
||||||
ErrCode: ErrInvalidRegion,
|
ErrCode: ErrInvalidRegion,
|
||||||
Code: "InvalidRegion",
|
Code: "InvalidRegion",
|
||||||
|
|
|
@ -14,15 +14,15 @@ import (
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
v2acl "github.com/TrueCloudLab/frostfs-api-go/v2/acl"
|
v2acl "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/acl"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/layer"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/eacl"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/object"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||||
oid "github.com/TrueCloudLab/frostfs-sdk-go/object/id"
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/session"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
||||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
@ -158,6 +158,90 @@ func (s ServiceRecord) ToEACLRecord() *eacl.Record {
|
||||||
return serviceRecord
|
return serviceRecord
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var (
|
||||||
|
errInvalidStatement = stderrors.New("invalid statement")
|
||||||
|
errInvalidPrincipal = stderrors.New("invalid principal")
|
||||||
|
)
|
||||||
|
|
||||||
|
func (s *statement) UnmarshalJSON(data []byte) error {
|
||||||
|
var statementMap map[string]interface{}
|
||||||
|
if err := json.Unmarshal(data, &statementMap); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
sidField, ok := statementMap["Sid"]
|
||||||
|
if ok {
|
||||||
|
if s.Sid, ok = sidField.(string); !ok {
|
||||||
|
return errInvalidStatement
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
effectField, ok := statementMap["Effect"]
|
||||||
|
if ok {
|
||||||
|
if s.Effect, ok = effectField.(string); !ok {
|
||||||
|
return errInvalidStatement
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
principalField, ok := statementMap["Principal"]
|
||||||
|
if ok {
|
||||||
|
principalMap, ok := principalField.(map[string]interface{})
|
||||||
|
if !ok {
|
||||||
|
return errInvalidPrincipal
|
||||||
|
}
|
||||||
|
|
||||||
|
awsField, ok := principalMap["AWS"]
|
||||||
|
if ok {
|
||||||
|
if s.Principal.AWS, ok = awsField.(string); !ok {
|
||||||
|
return fmt.Errorf("%w: 'AWS' field must be string", errInvalidPrincipal)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
canonicalUserField, ok := principalMap["CanonicalUser"]
|
||||||
|
if ok {
|
||||||
|
if s.Principal.CanonicalUser, ok = canonicalUserField.(string); !ok {
|
||||||
|
return errInvalidPrincipal
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
actionField, ok := statementMap["Action"]
|
||||||
|
if ok {
|
||||||
|
switch actionField := actionField.(type) {
|
||||||
|
case []interface{}:
|
||||||
|
s.Action = make([]string, len(actionField))
|
||||||
|
for i, action := range actionField {
|
||||||
|
if s.Action[i], ok = action.(string); !ok {
|
||||||
|
return errInvalidStatement
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case string:
|
||||||
|
s.Action = []string{actionField}
|
||||||
|
default:
|
||||||
|
return errInvalidStatement
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
resourceField, ok := statementMap["Resource"]
|
||||||
|
if ok {
|
||||||
|
switch resourceField := resourceField.(type) {
|
||||||
|
case []interface{}:
|
||||||
|
s.Resource = make([]string, len(resourceField))
|
||||||
|
for i, action := range resourceField {
|
||||||
|
if s.Resource[i], ok = action.(string); !ok {
|
||||||
|
return errInvalidStatement
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case string:
|
||||||
|
s.Resource = []string{resourceField}
|
||||||
|
default:
|
||||||
|
return errInvalidStatement
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func (h *handler) GetBucketACLHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) GetBucketACLHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
reqInfo := api.GetReqInfo(r.Context())
|
reqInfo := api.GetReqInfo(r.Context())
|
||||||
|
|
||||||
|
|
|
@ -13,14 +13,14 @@ import (
|
||||||
"net/http"
|
"net/http"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/bearer"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/eacl"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/object"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||||
oid "github.com/TrueCloudLab/frostfs-sdk-go/object/id"
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/session"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
||||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
@ -1352,6 +1352,85 @@ func TestBucketPolicy(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestBucketPolicyUnmarshal(t *testing.T) {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
policy string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "action/resource array",
|
||||||
|
policy: `
|
||||||
|
{
|
||||||
|
"Version": "2012-10-17",
|
||||||
|
"Statement": [{
|
||||||
|
"Principal": {
|
||||||
|
"AWS": "arn:aws:iam::111122223333:role/JohnDoe"
|
||||||
|
},
|
||||||
|
"Effect": "Allow",
|
||||||
|
"Action": [
|
||||||
|
"s3:GetObject",
|
||||||
|
"s3:GetObjectVersion"
|
||||||
|
],
|
||||||
|
"Resource": [
|
||||||
|
"arn:aws:s3:::DOC-EXAMPLE-BUCKET/*",
|
||||||
|
"arn:aws:s3:::DOC-EXAMPLE-BUCKET2/*"
|
||||||
|
]
|
||||||
|
}]
|
||||||
|
}
|
||||||
|
`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "action/resource string",
|
||||||
|
policy: `
|
||||||
|
{
|
||||||
|
"Version": "2012-10-17",
|
||||||
|
"Statement": [{
|
||||||
|
"Principal": {
|
||||||
|
"AWS": "arn:aws:iam::111122223333:role/JohnDoe"
|
||||||
|
},
|
||||||
|
"Effect": "Deny",
|
||||||
|
"Action": "s3:GetObject",
|
||||||
|
"Resource": "arn:aws:s3:::DOC-EXAMPLE-BUCKET/*"
|
||||||
|
}]
|
||||||
|
}
|
||||||
|
`,
|
||||||
|
},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
bktPolicy := &bucketPolicy{}
|
||||||
|
err := json.Unmarshal([]byte(tc.policy), bktPolicy)
|
||||||
|
require.NoError(t, err)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPutBucketPolicy(t *testing.T) {
|
||||||
|
bktPolicy := `
|
||||||
|
{
|
||||||
|
"Version": "2012-10-17",
|
||||||
|
"Statement": [{
|
||||||
|
"Principal": {
|
||||||
|
"AWS": "*"
|
||||||
|
},
|
||||||
|
"Effect": "Deny",
|
||||||
|
"Action": "s3:GetObject",
|
||||||
|
"Resource": "arn:aws:s3:::bucket-for-policy/*"
|
||||||
|
}]
|
||||||
|
}
|
||||||
|
`
|
||||||
|
hc := prepareHandlerContext(t)
|
||||||
|
bktName := "bucket-for-policy"
|
||||||
|
|
||||||
|
box, _ := createAccessBox(t)
|
||||||
|
createBucket(t, hc, bktName, box)
|
||||||
|
|
||||||
|
w, r := prepareTestPayloadRequest(hc, bktName, "", bytes.NewReader([]byte(bktPolicy)))
|
||||||
|
ctx := context.WithValue(r.Context(), api.BoxData, box)
|
||||||
|
r = r.WithContext(ctx)
|
||||||
|
hc.Handler().PutBucketPolicyHandler(w, r)
|
||||||
|
assertStatus(hc.t, w, http.StatusOK)
|
||||||
|
}
|
||||||
|
|
||||||
func getBucketPolicy(hc *handlerContext, bktName string) *bucketPolicy {
|
func getBucketPolicy(hc *handlerContext, bktName string) *bucketPolicy {
|
||||||
w, r := prepareTestRequest(hc, bktName, "", nil)
|
w, r := prepareTestRequest(hc, bktName, "", nil)
|
||||||
hc.Handler().GetBucketPolicyHandler(w, r)
|
hc.Handler().GetBucketPolicyHandler(w, r)
|
||||||
|
|
|
@ -1,12 +1,14 @@
|
||||||
package handler
|
package handler
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"encoding/xml"
|
||||||
"errors"
|
"errors"
|
||||||
|
"io"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/layer"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/netmap"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -25,16 +27,24 @@ type (
|
||||||
|
|
||||||
// Config contains data which handler needs to keep.
|
// Config contains data which handler needs to keep.
|
||||||
Config struct {
|
Config struct {
|
||||||
Policy PlacementPolicy
|
Policy PlacementPolicy
|
||||||
DefaultMaxAge int
|
XMLDecoder XMLDecoderProvider
|
||||||
NotificatorEnabled bool
|
DefaultMaxAge int
|
||||||
CopiesNumber uint32
|
NotificatorEnabled bool
|
||||||
|
CopiesNumber uint32
|
||||||
|
ResolveZoneList []string
|
||||||
|
IsResolveListAllow bool // True if ResolveZoneList contains allowed zones
|
||||||
|
CompleteMultipartKeepalive time.Duration
|
||||||
}
|
}
|
||||||
|
|
||||||
PlacementPolicy interface {
|
PlacementPolicy interface {
|
||||||
Default() netmap.PlacementPolicy
|
Default() netmap.PlacementPolicy
|
||||||
Get(string) (netmap.PlacementPolicy, bool)
|
Get(string) (netmap.PlacementPolicy, bool)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
XMLDecoderProvider interface {
|
||||||
|
NewCompleteMultipartDecoder(io.Reader) *xml.Decoder
|
||||||
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
|
|
|
@ -6,10 +6,10 @@ import (
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/layer"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
@ -4,7 +4,7 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
@ -6,12 +6,12 @@ import (
|
||||||
"regexp"
|
"regexp"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/auth"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/layer"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/session"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
@ -6,7 +6,7 @@ import (
|
||||||
"net/url"
|
"net/url"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
@ -5,9 +5,9 @@ import (
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/layer"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
41
api/handler/cors_test.go
Normal file
41
api/handler/cors_test.go
Normal file
|
@ -0,0 +1,41 @@
|
||||||
|
package handler
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"net/http"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestCORSOriginWildcard(t *testing.T) {
|
||||||
|
body := `
|
||||||
|
<CORSConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||||
|
<CORSRule>
|
||||||
|
<AllowedMethod>GET</AllowedMethod>
|
||||||
|
<AllowedOrigin>*</AllowedOrigin>
|
||||||
|
</CORSRule>
|
||||||
|
</CORSConfiguration>
|
||||||
|
`
|
||||||
|
hc := prepareHandlerContext(t)
|
||||||
|
|
||||||
|
bktName := "bucket-for-cors"
|
||||||
|
box, _ := createAccessBox(t)
|
||||||
|
w, r := prepareTestRequest(hc, bktName, "", nil)
|
||||||
|
ctx := context.WithValue(r.Context(), api.BoxData, box)
|
||||||
|
r = r.WithContext(ctx)
|
||||||
|
r.Header.Add(api.AmzACL, "public-read")
|
||||||
|
hc.Handler().CreateBucketHandler(w, r)
|
||||||
|
assertStatus(t, w, http.StatusOK)
|
||||||
|
|
||||||
|
w, r = prepareTestPayloadRequest(hc, bktName, "", strings.NewReader(body))
|
||||||
|
ctx = context.WithValue(r.Context(), api.BoxData, box)
|
||||||
|
r = r.WithContext(ctx)
|
||||||
|
hc.Handler().PutBucketCorsHandler(w, r)
|
||||||
|
assertStatus(t, w, http.StatusOK)
|
||||||
|
|
||||||
|
w, r = prepareTestPayloadRequest(hc, bktName, "", nil)
|
||||||
|
hc.Handler().GetBucketCorsHandler(w, r)
|
||||||
|
assertStatus(t, w, http.StatusOK)
|
||||||
|
}
|
|
@ -6,17 +6,20 @@ import (
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/layer"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
apistatus "github.com/TrueCloudLab/frostfs-sdk-go/client/status"
|
apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
|
||||||
oid "github.com/TrueCloudLab/frostfs-sdk-go/object/id"
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/session"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
"go.uber.org/zap/zapcore"
|
"go.uber.org/zap/zapcore"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// limitation of AWS https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html
|
||||||
|
const maxObjectsToDelete = 1000
|
||||||
|
|
||||||
// DeleteObjectsRequest -- xml carrying the object key names which should be deleted.
|
// DeleteObjectsRequest -- xml carrying the object key names which should be deleted.
|
||||||
type DeleteObjectsRequest struct {
|
type DeleteObjectsRequest struct {
|
||||||
// Element to enable quiet mode for the request
|
// Element to enable quiet mode for the request
|
||||||
|
@ -176,6 +179,11 @@ func (h *handler) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Re
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if len(requested.Objects) == 0 || len(requested.Objects) > maxObjectsToDelete {
|
||||||
|
h.logAndSendError(w, "number of objects to delete must be greater than 0 and less or equal to 1000", reqInfo, errors.GetAPIError(errors.ErrMalformedXML))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
removed := make(map[string]*layer.VersionedObject)
|
removed := make(map[string]*layer.VersionedObject)
|
||||||
toRemove := make([]*layer.VersionedObject, 0, len(requested.Objects))
|
toRemove := make([]*layer.VersionedObject, 0, len(requested.Objects))
|
||||||
for _, obj := range requested.Objects {
|
for _, obj := range requested.Objects {
|
||||||
|
|
|
@ -6,8 +6,8 @@ import (
|
||||||
"net/url"
|
"net/url"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
@ -12,8 +12,8 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/layer"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
@ -8,10 +8,10 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/layer"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -195,11 +195,6 @@ func (h *handler) GetObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if err = h.setExpirationHeader(r.Context(), bktInfo, info, w.Header()); err != nil {
|
|
||||||
h.logAndSendError(w, "could not get expiration info", reqInfo, err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
bktSettings, err := h.obj.GetBucketSettings(r.Context(), bktInfo)
|
bktSettings, err := h.obj.GetBucketSettings(r.Context(), bktInfo)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.logAndSendError(w, "could not get bucket settings", reqInfo, err)
|
h.logAndSendError(w, "could not get bucket settings", reqInfo, err)
|
||||||
|
|
|
@ -8,9 +8,9 @@ import (
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/layer"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
@ -13,15 +13,15 @@ import (
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/layer"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/resolver"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/resolver"
|
||||||
cid "github.com/TrueCloudLab/frostfs-sdk-go/container/id"
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/netmap"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/object"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||||
oid "github.com/TrueCloudLab/frostfs-sdk-go/object/id"
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/user"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
|
@ -63,6 +63,12 @@ func (p *placementPolicyMock) Get(string) (netmap.PlacementPolicy, bool) {
|
||||||
return netmap.PlacementPolicy{}, false
|
return netmap.PlacementPolicy{}, false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type xmlDecoderProviderMock struct{}
|
||||||
|
|
||||||
|
func (p *xmlDecoderProviderMock) NewCompleteMultipartDecoder(r io.Reader) *xml.Decoder {
|
||||||
|
return xml.NewDecoder(r)
|
||||||
|
}
|
||||||
|
|
||||||
func prepareHandlerContext(t *testing.T) *handlerContext {
|
func prepareHandlerContext(t *testing.T) *handlerContext {
|
||||||
key, err := keys.NewPrivateKey()
|
key, err := keys.NewPrivateKey()
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
@ -93,7 +99,8 @@ func prepareHandlerContext(t *testing.T) *handlerContext {
|
||||||
log: l,
|
log: l,
|
||||||
obj: layer.NewLayer(l, tp, layerCfg),
|
obj: layer.NewLayer(l, tp, layerCfg),
|
||||||
cfg: &Config{
|
cfg: &Config{
|
||||||
Policy: &placementPolicyMock{defaultPolicy: pp},
|
Policy: &placementPolicyMock{defaultPolicy: pp},
|
||||||
|
XMLDecoder: &xmlDecoderProviderMock{},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -2,16 +2,12 @@ package handler
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/url"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/layer"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -107,11 +103,6 @@ func (h *handler) HeadObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if err = h.setExpirationHeader(r.Context(), bktInfo, info, w.Header()); err != nil {
|
|
||||||
h.logAndSendError(w, "could not get expiration info", reqInfo, err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
bktSettings, err := h.obj.GetBucketSettings(r.Context(), bktInfo)
|
bktSettings, err := h.obj.GetBucketSettings(r.Context(), bktInfo)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.logAndSendError(w, "could not get bucket settings", reqInfo, err)
|
h.logAndSendError(w, "could not get bucket settings", reqInfo, err)
|
||||||
|
@ -133,6 +124,12 @@ func (h *handler) HeadBucketHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
|
|
||||||
w.Header().Set(api.ContainerID, bktInfo.CID.EncodeToString())
|
w.Header().Set(api.ContainerID, bktInfo.CID.EncodeToString())
|
||||||
w.Header().Set(api.AmzBucketRegion, bktInfo.LocationConstraint)
|
w.Header().Set(api.AmzBucketRegion, bktInfo.LocationConstraint)
|
||||||
|
|
||||||
|
if isAvailableToResolve(bktInfo.Zone, h.cfg.ResolveZoneList, h.cfg.IsResolveListAllow) {
|
||||||
|
w.Header().Set(api.ContainerName, bktInfo.Name)
|
||||||
|
w.Header().Set(api.ContainerZone, bktInfo.Zone)
|
||||||
|
}
|
||||||
|
|
||||||
api.WriteResponse(w, http.StatusOK, nil, api.MimeNone)
|
api.WriteResponse(w, http.StatusOK, nil, api.MimeNone)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -167,23 +164,24 @@ func writeLockHeaders(h http.Header, legalHold *data.LegalHold, retention *data.
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *handler) setExpirationHeader(ctx context.Context, bktInfo *data.BucketInfo, objInfo *data.ObjectInfo, header http.Header) error {
|
func isAvailableToResolve(zone string, list []string, isAllowList bool) bool {
|
||||||
var expirationObjInfo data.ObjectInfo
|
// empty zone means container doesn't have proper system name,
|
||||||
|
// so we don't have to resolve it
|
||||||
// todo get expiration object info
|
if len(zone) == 0 {
|
||||||
|
return false
|
||||||
ruleID := expirationObjInfo.Headers[layer.AttributeExpireRuleID]
|
|
||||||
|
|
||||||
expDate, err := time.Parse(time.RFC3339, expirationObjInfo.Headers[layer.AttributeExpireDate])
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("couldn't parse ivalid expiration time: %w", err)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
writeExpirationHeader(header, ruleID, expDate)
|
var zoneInList bool
|
||||||
return nil
|
for _, t := range list {
|
||||||
}
|
if t == zone {
|
||||||
|
zoneInList = true
|
||||||
func writeExpirationHeader(h http.Header, ruleID string, expDate time.Time) {
|
break
|
||||||
header := "expiry-date=\"%s\", rule-id=\"%s\""
|
}
|
||||||
h.Set(api.AmzExpiration, fmt.Sprintf(header, expDate.UTC().Format(http.TimeFormat), url.QueryEscape(ruleID)))
|
}
|
||||||
|
// InList | IsAllowList | Result
|
||||||
|
// 0 0 1
|
||||||
|
// 0 1 0
|
||||||
|
// 1 0 0
|
||||||
|
// 1 1 1
|
||||||
|
return zoneInList == isAllowList
|
||||||
}
|
}
|
||||||
|
|
|
@ -6,10 +6,10 @@ import (
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/bearer"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/eacl"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
|
||||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
@ -86,6 +86,26 @@ func TestInvalidAccessThroughCache(t *testing.T) {
|
||||||
assertStatus(t, w, http.StatusForbidden)
|
assertStatus(t, w, http.StatusForbidden)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestIsAvailableToResolve(t *testing.T) {
|
||||||
|
list := []string{"container", "s3"}
|
||||||
|
|
||||||
|
for i, testCase := range [...]struct {
|
||||||
|
isAllowList bool
|
||||||
|
list []string
|
||||||
|
zone string
|
||||||
|
expected bool
|
||||||
|
}{
|
||||||
|
{isAllowList: true, list: list, zone: "container", expected: true},
|
||||||
|
{isAllowList: true, list: list, zone: "sftp", expected: false},
|
||||||
|
{isAllowList: false, list: list, zone: "s3", expected: false},
|
||||||
|
{isAllowList: false, list: list, zone: "system", expected: true},
|
||||||
|
{isAllowList: true, list: list, zone: "", expected: false},
|
||||||
|
} {
|
||||||
|
result := isAvailableToResolve(testCase.zone, testCase.list, testCase.isAllowList)
|
||||||
|
require.Equal(t, testCase.expected, result, "case %d", i+1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func newTestAccessBox(t *testing.T, key *keys.PrivateKey) *accessbox.Box {
|
func newTestAccessBox(t *testing.T, key *keys.PrivateKey) *accessbox.Box {
|
||||||
var err error
|
var err error
|
||||||
if key == nil {
|
if key == nil {
|
||||||
|
|
|
@ -3,7 +3,7 @@ package handler
|
||||||
import (
|
import (
|
||||||
"net/http"
|
"net/http"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
)
|
)
|
||||||
|
|
||||||
func (h *handler) GetBucketLocationHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) GetBucketLocationHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
|
|
|
@ -1,168 +0,0 @@
|
||||||
package handler
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/xml"
|
|
||||||
"fmt"
|
|
||||||
"net/http"
|
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api"
|
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
|
||||||
apiErrors "github.com/TrueCloudLab/frostfs-s3-gw/api/errors"
|
|
||||||
)
|
|
||||||
|
|
||||||
func (h *handler) PutBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
|
|
||||||
reqInfo := api.GetReqInfo(r.Context())
|
|
||||||
|
|
||||||
bktInfo, err := h.obj.GetBucketInfo(r.Context(), reqInfo.BucketName)
|
|
||||||
if err != nil {
|
|
||||||
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if err = checkOwner(bktInfo, r.Header.Get(api.AmzExpectedBucketOwner)); err != nil {
|
|
||||||
h.logAndSendError(w, "expected owner doesn't match", reqInfo, err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
lifecycleConf := &data.LifecycleConfiguration{}
|
|
||||||
if err = xml.NewDecoder(r.Body).Decode(lifecycleConf); err != nil {
|
|
||||||
h.logAndSendError(w, "couldn't parse lifecycle configuration", reqInfo, err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if err = checkLifecycleConfiguration(lifecycleConf); err != nil {
|
|
||||||
h.logAndSendError(w, "invalid lifecycle configuration", reqInfo, err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if err = h.updateLifecycleConfiguration(r.Context(), bktInfo, lifecycleConf); err != nil {
|
|
||||||
h.logAndSendError(w, "couldn't put bucket settings", reqInfo, err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (h *handler) GetBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
|
|
||||||
reqInfo := api.GetReqInfo(r.Context())
|
|
||||||
|
|
||||||
bktInfo, err := h.obj.GetBucketInfo(r.Context(), reqInfo.BucketName)
|
|
||||||
if err != nil {
|
|
||||||
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if err = checkOwner(bktInfo, r.Header.Get(api.AmzExpectedBucketOwner)); err != nil {
|
|
||||||
h.logAndSendError(w, "expected owner doesn't match", reqInfo, err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
settings, err := h.obj.GetBucketSettings(r.Context(), bktInfo)
|
|
||||||
if err != nil {
|
|
||||||
h.logAndSendError(w, "couldn't get bucket settings", reqInfo, err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if settings.LifecycleConfig == nil || settings.LifecycleConfig.CurrentConfiguration == nil {
|
|
||||||
h.logAndSendError(w, "lifecycle configuration doesn't exist", reqInfo,
|
|
||||||
apiErrors.GetAPIError(apiErrors.ErrNoSuchLifecycleConfiguration))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if err = api.EncodeToResponse(w, settings.LifecycleConfig.CurrentConfiguration); err != nil {
|
|
||||||
h.logAndSendError(w, "something went wrong", reqInfo, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (h *handler) DeleteBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
|
|
||||||
reqInfo := api.GetReqInfo(r.Context())
|
|
||||||
|
|
||||||
bktInfo, err := h.obj.GetBucketInfo(r.Context(), reqInfo.BucketName)
|
|
||||||
if err != nil {
|
|
||||||
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if err = checkOwner(bktInfo, r.Header.Get(api.AmzExpectedBucketOwner)); err != nil {
|
|
||||||
h.logAndSendError(w, "expected owner doesn't match", reqInfo, err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if err = h.updateLifecycleConfiguration(r.Context(), bktInfo, nil); err != nil {
|
|
||||||
h.logAndSendError(w, "couldn't put bucket settings", reqInfo, err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
w.WriteHeader(http.StatusNoContent)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (h *handler) updateLifecycleConfiguration(ctx context.Context, bktInfo *data.BucketInfo, lifecycleConf *data.LifecycleConfiguration) error {
|
|
||||||
// todo consider run as separate goroutine
|
|
||||||
if err := h.obj.ScheduleLifecycle(ctx, bktInfo, lifecycleConf); err != nil {
|
|
||||||
return fmt.Errorf("couldn't apply lifecycle: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func checkLifecycleConfiguration(conf *data.LifecycleConfiguration) error {
|
|
||||||
err := apiErrors.GetAPIError(apiErrors.ErrMalformedXML)
|
|
||||||
|
|
||||||
if len(conf.Rules) == 0 {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if len(conf.Rules) > 1000 {
|
|
||||||
return fmt.Errorf("you cannot have more than 1000 rules")
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, rule := range conf.Rules {
|
|
||||||
if rule.Status != enabledValue && rule.Status != disabledValue {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if rule.Prefix != nil && rule.Filter != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
if rule.Filter != nil {
|
|
||||||
if rule.Filter.ObjectSizeGreaterThan != nil && *rule.Filter.ObjectSizeGreaterThan < 0 ||
|
|
||||||
rule.Filter.ObjectSizeLessThan != nil && *rule.Filter.ObjectSizeLessThan < 0 {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
if !filterContainsOneOption(rule.Filter) {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if !ruleHasAction(rule) {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// currently only expiration action is supported
|
|
||||||
if rule.Expiration == nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if rule.Expiration.Days != nil && rule.Expiration.Date != nil ||
|
|
||||||
rule.Expiration.Days == nil && rule.Expiration.Date == nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func filterContainsOneOption(filter *data.LifecycleRuleFilter) bool {
|
|
||||||
exactlyOneOption := 0
|
|
||||||
if filter.Prefix != nil {
|
|
||||||
exactlyOneOption++
|
|
||||||
}
|
|
||||||
if filter.And != nil {
|
|
||||||
exactlyOneOption++
|
|
||||||
}
|
|
||||||
if filter.Tag != nil {
|
|
||||||
exactlyOneOption++
|
|
||||||
}
|
|
||||||
|
|
||||||
return exactlyOneOption == 1
|
|
||||||
}
|
|
||||||
|
|
||||||
func ruleHasAction(rule data.Rule) bool {
|
|
||||||
return rule.AbortIncompleteMultipartUpload != nil || rule.Expiration != nil ||
|
|
||||||
rule.NoncurrentVersionExpiration != nil || len(rule.Transitions) != 0 ||
|
|
||||||
len(rule.NoncurrentVersionTransitions) != 0
|
|
||||||
}
|
|
|
@ -1,152 +0,0 @@
|
||||||
package handler
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/xml"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"strconv"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
|
||||||
apiErrors "github.com/TrueCloudLab/frostfs-s3-gw/api/errors"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestCheckLifecycleConfiguration(t *testing.T) {
|
|
||||||
numRules := 1001
|
|
||||||
rules := make([]data.Rule, numRules)
|
|
||||||
for i := 0; i < numRules; i++ {
|
|
||||||
rules[i] = data.Rule{ID: strconv.Itoa(i), Status: disabledValue}
|
|
||||||
}
|
|
||||||
|
|
||||||
prefix := "prefix"
|
|
||||||
invalidSize := int64(-1)
|
|
||||||
days := int64(1)
|
|
||||||
|
|
||||||
for _, tc := range []struct {
|
|
||||||
name string
|
|
||||||
configuration *data.LifecycleConfiguration
|
|
||||||
noError bool
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "basic",
|
|
||||||
configuration: &data.LifecycleConfiguration{Rules: []data.Rule{{
|
|
||||||
ID: "Some ID",
|
|
||||||
Status: "Disabled",
|
|
||||||
Expiration: &data.Expiration{Days: &days},
|
|
||||||
}}},
|
|
||||||
noError: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "invalid status",
|
|
||||||
configuration: &data.LifecycleConfiguration{Rules: []data.Rule{{
|
|
||||||
ID: "Some ID",
|
|
||||||
Status: "",
|
|
||||||
}}},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "zero rules",
|
|
||||||
configuration: &data.LifecycleConfiguration{},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "more than max rules",
|
|
||||||
configuration: &data.LifecycleConfiguration{Rules: rules},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "invalid empty filter",
|
|
||||||
configuration: &data.LifecycleConfiguration{Rules: []data.Rule{{
|
|
||||||
Status: enabledValue,
|
|
||||||
Filter: &data.LifecycleRuleFilter{},
|
|
||||||
}}},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "invalid filter not exactly one option",
|
|
||||||
configuration: &data.LifecycleConfiguration{Rules: []data.Rule{{
|
|
||||||
Status: enabledValue,
|
|
||||||
Filter: &data.LifecycleRuleFilter{
|
|
||||||
Prefix: &prefix,
|
|
||||||
Tag: &data.Tag{},
|
|
||||||
},
|
|
||||||
}}},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "invalid filter greater obj size",
|
|
||||||
configuration: &data.LifecycleConfiguration{Rules: []data.Rule{{
|
|
||||||
Status: enabledValue,
|
|
||||||
Filter: &data.LifecycleRuleFilter{
|
|
||||||
ObjectSizeGreaterThan: &invalidSize,
|
|
||||||
},
|
|
||||||
}}},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "invalid filter less obj size",
|
|
||||||
configuration: &data.LifecycleConfiguration{Rules: []data.Rule{{
|
|
||||||
Status: enabledValue,
|
|
||||||
Filter: &data.LifecycleRuleFilter{
|
|
||||||
ObjectSizeLessThan: &invalidSize,
|
|
||||||
},
|
|
||||||
}}},
|
|
||||||
},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
err := checkLifecycleConfiguration(tc.configuration)
|
|
||||||
if tc.noError {
|
|
||||||
require.NoError(t, err)
|
|
||||||
} else {
|
|
||||||
require.Error(t, err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBucketLifecycleConfiguration(t *testing.T) {
|
|
||||||
hc := prepareHandlerContext(t)
|
|
||||||
|
|
||||||
bktName := "bucket-for-lifecycle"
|
|
||||||
createTestBucket(hc, bktName)
|
|
||||||
|
|
||||||
w, r := prepareTestRequest(hc, bktName, "", nil)
|
|
||||||
hc.Handler().GetBucketLifecycleHandler(w, r)
|
|
||||||
assertS3Error(t, w, apiErrors.GetAPIError(apiErrors.ErrNoSuchLifecycleConfiguration))
|
|
||||||
|
|
||||||
days := int64(1)
|
|
||||||
lifecycleConf := &data.LifecycleConfiguration{
|
|
||||||
XMLName: xmlName("LifecycleConfiguration"),
|
|
||||||
Rules: []data.Rule{
|
|
||||||
{
|
|
||||||
Expiration: &data.Expiration{Days: &days},
|
|
||||||
ID: "Test",
|
|
||||||
Status: "Disabled",
|
|
||||||
},
|
|
||||||
}}
|
|
||||||
w, r = prepareTestRequest(hc, bktName, "", lifecycleConf)
|
|
||||||
hc.Handler().PutBucketLifecycleHandler(w, r)
|
|
||||||
require.Equal(t, http.StatusOK, w.Code)
|
|
||||||
|
|
||||||
w, r = prepareTestRequest(hc, bktName, "", nil)
|
|
||||||
hc.Handler().GetBucketLifecycleHandler(w, r)
|
|
||||||
assertXMLEqual(t, w, lifecycleConf, &data.LifecycleConfiguration{})
|
|
||||||
|
|
||||||
w, r = prepareTestRequest(hc, bktName, "", lifecycleConf)
|
|
||||||
hc.Handler().DeleteBucketLifecycleHandler(w, r)
|
|
||||||
require.Equal(t, http.StatusNoContent, w.Code)
|
|
||||||
|
|
||||||
// make sure deleting is idempotent operation
|
|
||||||
w, r = prepareTestRequest(hc, bktName, "", lifecycleConf)
|
|
||||||
hc.Handler().DeleteBucketLifecycleHandler(w, r)
|
|
||||||
require.Equal(t, http.StatusNoContent, w.Code)
|
|
||||||
}
|
|
||||||
|
|
||||||
func assertXMLEqual(t *testing.T, w *httptest.ResponseRecorder, expected, actual interface{}) {
|
|
||||||
err := xml.NewDecoder(w.Result().Body).Decode(actual)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Equal(t, expected, actual)
|
|
||||||
require.Equal(t, http.StatusOK, w.Code)
|
|
||||||
}
|
|
||||||
|
|
||||||
func xmlName(local string) xml.Name {
|
|
||||||
return xml.Name{
|
|
||||||
Space: "http://s3.amazonaws.com/doc/2006-03-01/",
|
|
||||||
Local: local,
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -4,8 +4,8 @@ import (
|
||||||
"net/http"
|
"net/http"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/user"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||||
)
|
)
|
||||||
|
|
||||||
const maxObjectList = 1000 // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse.
|
const maxObjectList = 1000 // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse.
|
||||||
|
|
|
@ -8,10 +8,10 @@ import (
|
||||||
"strconv"
|
"strconv"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
apiErrors "github.com/TrueCloudLab/frostfs-s3-gw/api/errors"
|
apiErrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/layer"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
|
@ -19,7 +19,6 @@ const (
|
||||||
yearDuration = 365 * dayDuration
|
yearDuration = 365 * dayDuration
|
||||||
|
|
||||||
enabledValue = "Enabled"
|
enabledValue = "Enabled"
|
||||||
disabledValue = "Disabled"
|
|
||||||
governanceMode = "GOVERNANCE"
|
governanceMode = "GOVERNANCE"
|
||||||
complianceMode = "COMPLIANCE"
|
complianceMode = "COMPLIANCE"
|
||||||
legalHoldOn = "ON"
|
legalHoldOn = "ON"
|
||||||
|
|
|
@ -10,9 +10,9 @@ import (
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
apiErrors "github.com/TrueCloudLab/frostfs-s3-gw/api/errors"
|
apiErrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
@ -2,16 +2,17 @@ package handler
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/xml"
|
"encoding/xml"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/url"
|
"net/url"
|
||||||
"strconv"
|
"strconv"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/layer"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/session"
|
|
||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
@ -372,8 +373,6 @@ func (h *handler) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.
|
||||||
}
|
}
|
||||||
|
|
||||||
var (
|
var (
|
||||||
sessionTokenSetEACL *session.Container
|
|
||||||
|
|
||||||
uploadID = r.URL.Query().Get(uploadIDHeaderName)
|
uploadID = r.URL.Query().Get(uploadIDHeaderName)
|
||||||
uploadInfo = &layer.UploadInfoParams{
|
uploadInfo = &layer.UploadInfoParams{
|
||||||
UploadID: uploadID,
|
UploadID: uploadID,
|
||||||
|
@ -384,7 +383,7 @@ func (h *handler) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.
|
||||||
)
|
)
|
||||||
|
|
||||||
reqBody := new(CompleteMultipartUpload)
|
reqBody := new(CompleteMultipartUpload)
|
||||||
if err = xml.NewDecoder(r.Body).Decode(reqBody); err != nil {
|
if err = h.cfg.XMLDecoder.NewCompleteMultipartDecoder(r.Body).Decode(reqBody); err != nil {
|
||||||
h.logAndSendError(w, "could not read complete multipart upload xml", reqInfo,
|
h.logAndSendError(w, "could not read complete multipart upload xml", reqInfo,
|
||||||
errors.GetAPIError(errors.ErrMalformedXML), additional...)
|
errors.GetAPIError(errors.ErrMalformedXML), additional...)
|
||||||
return
|
return
|
||||||
|
@ -399,10 +398,51 @@ func (h *handler) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.
|
||||||
Parts: reqBody.Parts,
|
Parts: reqBody.Parts,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Next operations might take some time, so we want to keep client's
|
||||||
|
// connection alive. To do so, gateway sends periodic white spaces
|
||||||
|
// back to the client the same way as Amazon S3 service does.
|
||||||
|
stopPeriodicResponseWriter := periodicXMLWriter(w, h.cfg.CompleteMultipartKeepalive)
|
||||||
|
|
||||||
|
// Start complete multipart upload which may take some time to fetch object
|
||||||
|
// and re-upload it part by part.
|
||||||
|
objInfo, err := h.completeMultipartUpload(r, c, bktInfo, reqInfo)
|
||||||
|
|
||||||
|
// Stop periodic writer as complete multipart upload is finished
|
||||||
|
// successfully or not.
|
||||||
|
headerIsWritten := stopPeriodicResponseWriter()
|
||||||
|
|
||||||
|
responseWriter := api.EncodeToResponse
|
||||||
|
errLogger := h.logAndSendError
|
||||||
|
// Do not send XML and HTTP headers if periodic writer was invoked at this point.
|
||||||
|
if headerIsWritten {
|
||||||
|
responseWriter = api.EncodeToResponseNoHeader
|
||||||
|
errLogger = h.logAndSendErrorNoHeader
|
||||||
|
}
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
errLogger(w, "complete multipart error", reqInfo, err, additional...)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
response := CompleteMultipartUploadResponse{
|
||||||
|
Bucket: objInfo.Bucket,
|
||||||
|
ETag: objInfo.HashSum,
|
||||||
|
Key: objInfo.Name,
|
||||||
|
}
|
||||||
|
|
||||||
|
// Here we previously set api.AmzVersionID header for versioned bucket.
|
||||||
|
// It is not possible after #60, because of periodic white
|
||||||
|
// space XML writer to keep connection with the client.
|
||||||
|
|
||||||
|
if err = responseWriter(w, response); err != nil {
|
||||||
|
errLogger(w, "something went wrong", reqInfo, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *handler) completeMultipartUpload(r *http.Request, c *layer.CompleteMultipartParams, bktInfo *data.BucketInfo, reqInfo *api.ReqInfo) (*data.ObjectInfo, error) {
|
||||||
uploadData, extendedObjInfo, err := h.obj.CompleteMultipartUpload(r.Context(), c)
|
uploadData, extendedObjInfo, err := h.obj.CompleteMultipartUpload(r.Context(), c)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.logAndSendError(w, "could not complete multipart upload", reqInfo, err, additional...)
|
return nil, fmt.Errorf("could not complete multipart upload: %w", err)
|
||||||
return
|
|
||||||
}
|
}
|
||||||
objInfo := extendedObjInfo.ObjectInfo
|
objInfo := extendedObjInfo.ObjectInfo
|
||||||
|
|
||||||
|
@ -417,21 +457,22 @@ func (h *handler) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.
|
||||||
NodeVersion: extendedObjInfo.NodeVersion,
|
NodeVersion: extendedObjInfo.NodeVersion,
|
||||||
}
|
}
|
||||||
if _, err = h.obj.PutObjectTagging(r.Context(), tagPrm); err != nil {
|
if _, err = h.obj.PutObjectTagging(r.Context(), tagPrm); err != nil {
|
||||||
h.logAndSendError(w, "could not put tagging file of completed multipart upload", reqInfo, err, additional...)
|
return nil, fmt.Errorf("could not put tagging file of completed multipart upload: %w", err)
|
||||||
return
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(uploadData.ACLHeaders) != 0 {
|
if len(uploadData.ACLHeaders) != 0 {
|
||||||
|
sessionTokenSetEACL, err := getSessionTokenSetEACL(r.Context())
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("couldn't get eacl token: %w", err)
|
||||||
|
}
|
||||||
key, err := h.bearerTokenIssuerKey(r.Context())
|
key, err := h.bearerTokenIssuerKey(r.Context())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.logAndSendError(w, "couldn't get gate key", reqInfo, err)
|
return nil, fmt.Errorf("couldn't get gate key: %w", err)
|
||||||
return
|
|
||||||
}
|
}
|
||||||
acl, err := parseACLHeaders(r.Header, key)
|
acl, err := parseACLHeaders(r.Header, key)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.logAndSendError(w, "could not parse acl", reqInfo, err)
|
return nil, fmt.Errorf("could not parse acl: %w", err)
|
||||||
return
|
|
||||||
}
|
}
|
||||||
|
|
||||||
resInfo := &resourceInfo{
|
resInfo := &resourceInfo{
|
||||||
|
@ -440,12 +481,10 @@ func (h *handler) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.
|
||||||
}
|
}
|
||||||
astObject, err := aclToAst(acl, resInfo)
|
astObject, err := aclToAst(acl, resInfo)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.logAndSendError(w, "could not translate acl of completed multipart upload to ast", reqInfo, err, additional...)
|
return nil, fmt.Errorf("could not translate acl of completed multipart upload to ast: %w", err)
|
||||||
return
|
|
||||||
}
|
}
|
||||||
if _, err = h.updateBucketACL(r, astObject, bktInfo, sessionTokenSetEACL); err != nil {
|
if _, err = h.updateBucketACL(r, astObject, bktInfo, sessionTokenSetEACL); err != nil {
|
||||||
h.logAndSendError(w, "could not update bucket acl while completing multipart upload", reqInfo, err, additional...)
|
return nil, fmt.Errorf("could not update bucket acl while completing multipart upload: %w", err)
|
||||||
return
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -459,24 +498,7 @@ func (h *handler) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.
|
||||||
h.log.Error("couldn't send notification: %w", zap.Error(err))
|
h.log.Error("couldn't send notification: %w", zap.Error(err))
|
||||||
}
|
}
|
||||||
|
|
||||||
bktSettings, err := h.obj.GetBucketSettings(r.Context(), bktInfo)
|
return objInfo, nil
|
||||||
if err != nil {
|
|
||||||
h.logAndSendError(w, "could not get bucket settings", reqInfo, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
response := CompleteMultipartUploadResponse{
|
|
||||||
Bucket: objInfo.Bucket,
|
|
||||||
ETag: objInfo.HashSum,
|
|
||||||
Key: objInfo.Name,
|
|
||||||
}
|
|
||||||
|
|
||||||
if bktSettings.VersioningEnabled() {
|
|
||||||
w.Header().Set(api.AmzVersionID, objInfo.VersionID())
|
|
||||||
}
|
|
||||||
|
|
||||||
if err = api.EncodeToResponse(w, response); err != nil {
|
|
||||||
h.logAndSendError(w, "something went wrong", reqInfo, err)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *handler) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
|
@ -681,3 +703,62 @@ func encodeListPartsToResponse(info *layer.ListPartsInfo, params *layer.ListPart
|
||||||
Parts: info.Parts,
|
Parts: info.Parts,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// periodicXMLWriter creates go routine to write xml header and whitespaces
|
||||||
|
// over time to avoid connection drop from the client. To work properly,
|
||||||
|
// pass `http.ResponseWriter` with implemented `http.Flusher` interface.
|
||||||
|
// Returns stop function which returns boolean if writer has been used
|
||||||
|
// during goroutine execution. To disable writer, pass 0 duration value.
|
||||||
|
func periodicXMLWriter(w io.Writer, dur time.Duration) (stop func() bool) {
|
||||||
|
if dur == 0 { // 0 duration disables periodic writer
|
||||||
|
return func() bool { return false }
|
||||||
|
}
|
||||||
|
|
||||||
|
whitespaceChar := []byte(" ")
|
||||||
|
closer := make(chan struct{})
|
||||||
|
done := make(chan struct{})
|
||||||
|
headerWritten := false
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
defer close(done)
|
||||||
|
|
||||||
|
tick := time.NewTicker(dur)
|
||||||
|
defer tick.Stop()
|
||||||
|
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-tick.C:
|
||||||
|
if !headerWritten {
|
||||||
|
_, err := w.Write([]byte(xml.Header))
|
||||||
|
headerWritten = err == nil
|
||||||
|
}
|
||||||
|
_, err := w.Write(whitespaceChar)
|
||||||
|
if err != nil {
|
||||||
|
return // is there anything we can do better than ignore error?
|
||||||
|
}
|
||||||
|
if buffered, ok := w.(http.Flusher); ok {
|
||||||
|
buffered.Flush()
|
||||||
|
}
|
||||||
|
case <-closer:
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
stop = func() bool {
|
||||||
|
close(closer)
|
||||||
|
<-done // wait for goroutine to stop
|
||||||
|
return headerWritten
|
||||||
|
}
|
||||||
|
|
||||||
|
return stop
|
||||||
|
}
|
||||||
|
|
||||||
|
// periodicWriterErrorSender returns handler function to send error. If header is
|
||||||
|
// alreay written by periodic XML writer, do not send HTTP and XML headers.
|
||||||
|
func (h *handler) periodicWriterErrorSender(headerWritten bool) func(http.ResponseWriter, string, *api.ReqInfo, error, ...zap.Field) {
|
||||||
|
if headerWritten {
|
||||||
|
return h.logAndSendErrorNoHeader
|
||||||
|
}
|
||||||
|
return h.logAndSendError
|
||||||
|
}
|
||||||
|
|
48
api/handler/multipart_upload_test.go
Normal file
48
api/handler/multipart_upload_test.go
Normal file
|
@ -0,0 +1,48 @@
|
||||||
|
package handler
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/xml"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestPeriodicWriter(t *testing.T) {
|
||||||
|
const dur = 100 * time.Millisecond
|
||||||
|
const whitespaces = 8
|
||||||
|
expected := []byte(xml.Header)
|
||||||
|
for i := 0; i < whitespaces; i++ {
|
||||||
|
expected = append(expected, []byte(" ")...)
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Run("writes data", func(t *testing.T) {
|
||||||
|
buf := bytes.NewBuffer(nil)
|
||||||
|
stop := periodicXMLWriter(buf, dur)
|
||||||
|
|
||||||
|
// N number of whitespaces + half durations to guarantee at least N writes in buffer
|
||||||
|
time.Sleep(whitespaces*dur + dur/2)
|
||||||
|
require.True(t, stop())
|
||||||
|
require.Equal(t, expected, buf.Bytes())
|
||||||
|
|
||||||
|
t.Run("no additional data after stop", func(t *testing.T) {
|
||||||
|
time.Sleep(2 * dur)
|
||||||
|
require.Equal(t, expected, buf.Bytes())
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("does not write data", func(t *testing.T) {
|
||||||
|
buf := bytes.NewBuffer(nil)
|
||||||
|
stop := periodicXMLWriter(buf, dur)
|
||||||
|
time.Sleep(dur / 2)
|
||||||
|
require.False(t, stop())
|
||||||
|
require.Empty(t, buf.Bytes())
|
||||||
|
|
||||||
|
t.Run("disabled", func(t *testing.T) {
|
||||||
|
stop = periodicXMLWriter(buf, 0)
|
||||||
|
require.False(t, stop())
|
||||||
|
require.Empty(t, buf.Bytes())
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
|
@ -3,14 +3,18 @@ package handler
|
||||||
import (
|
import (
|
||||||
"net/http"
|
"net/http"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
)
|
)
|
||||||
|
|
||||||
func (h *handler) DeleteBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) DeleteBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
h.logAndSendError(w, "not supported", api.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotSupported))
|
h.logAndSendError(w, "not supported", api.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotSupported))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (h *handler) DeleteBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
|
h.logAndSendError(w, "not supported", api.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotSupported))
|
||||||
|
}
|
||||||
|
|
||||||
func (h *handler) DeleteBucketEncryptionHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) DeleteBucketEncryptionHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
h.logAndSendError(w, "not supported", api.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotSupported))
|
h.logAndSendError(w, "not supported", api.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotSupported))
|
||||||
}
|
}
|
||||||
|
|
|
@ -8,11 +8,11 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/layer"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/bearer"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
@ -3,8 +3,8 @@ package handler
|
||||||
import (
|
import (
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
@ -6,11 +6,11 @@ import (
|
||||||
"strconv"
|
"strconv"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/layer"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
oid "github.com/TrueCloudLab/frostfs-sdk-go/object/id"
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
)
|
)
|
||||||
|
|
||||||
// ListObjectsV1Handler handles objects listing requests for API version 1.
|
// ListObjectsV1Handler handles objects listing requests for API version 1.
|
||||||
|
|
|
@ -6,7 +6,7 @@ import (
|
||||||
"strconv"
|
"strconv"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
@ -16,15 +16,15 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/auth"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/layer"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/eacl"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/session"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -720,7 +720,10 @@ func (h *handler) CreateBucketHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
h.setPolicy(p, createParams.LocationConstraint, policies)
|
if err = h.setPolicy(p, createParams.LocationConstraint, policies); err != nil {
|
||||||
|
h.logAndSendError(w, "couldn't set placement policy", reqInfo, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
p.ObjectLockEnabled = isLockEnabled(r.Header)
|
p.ObjectLockEnabled = isLockEnabled(r.Header)
|
||||||
|
|
||||||
|
@ -748,25 +751,27 @@ func (h *handler) CreateBucketHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
api.WriteSuccessResponseHeadersOnly(w)
|
api.WriteSuccessResponseHeadersOnly(w)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h handler) setPolicy(prm *layer.CreateBucketParams, locationConstraint string, userPolicies []*accessbox.ContainerPolicy) {
|
func (h handler) setPolicy(prm *layer.CreateBucketParams, locationConstraint string, userPolicies []*accessbox.ContainerPolicy) error {
|
||||||
prm.Policy = h.cfg.Policy.Default()
|
prm.Policy = h.cfg.Policy.Default()
|
||||||
|
prm.LocationConstraint = locationConstraint
|
||||||
|
|
||||||
if locationConstraint == "" {
|
if locationConstraint == "" {
|
||||||
return
|
return nil
|
||||||
}
|
|
||||||
|
|
||||||
if policy, ok := h.cfg.Policy.Get(locationConstraint); ok {
|
|
||||||
prm.Policy = policy
|
|
||||||
prm.LocationConstraint = locationConstraint
|
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, placementPolicy := range userPolicies {
|
for _, placementPolicy := range userPolicies {
|
||||||
if placementPolicy.LocationConstraint == locationConstraint {
|
if placementPolicy.LocationConstraint == locationConstraint {
|
||||||
prm.Policy = placementPolicy.Policy
|
prm.Policy = placementPolicy.Policy
|
||||||
prm.LocationConstraint = locationConstraint
|
return nil
|
||||||
return
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if policy, ok := h.cfg.Policy.Get(locationConstraint); ok {
|
||||||
|
prm.Policy = policy
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return errors.GetAPIError(errors.ErrInvalidLocationConstraint)
|
||||||
}
|
}
|
||||||
|
|
||||||
func isLockEnabled(header http.Header) bool {
|
func isLockEnabled(header http.Header) bool {
|
||||||
|
|
|
@ -8,8 +8,8 @@ import (
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/layer"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
@ -8,10 +8,10 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
"unicode"
|
"unicode"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/layer"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
@ -3,14 +3,18 @@ package handler
|
||||||
import (
|
import (
|
||||||
"net/http"
|
"net/http"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
)
|
)
|
||||||
|
|
||||||
func (h *handler) SelectObjectContentHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) SelectObjectContentHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
h.logAndSendError(w, "not implemented", api.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
|
h.logAndSendError(w, "not implemented", api.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (h *handler) GetBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
|
h.logAndSendError(w, "not implemented", api.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
|
||||||
|
}
|
||||||
|
|
||||||
func (h *handler) GetBucketEncryptionHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) GetBucketEncryptionHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
h.logAndSendError(w, "not implemented", api.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
|
h.logAndSendError(w, "not implemented", api.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
|
||||||
}
|
}
|
||||||
|
@ -47,6 +51,10 @@ func (h *handler) ListObjectsV2MHandler(w http.ResponseWriter, r *http.Request)
|
||||||
h.logAndSendError(w, "not implemented", api.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
|
h.logAndSendError(w, "not implemented", api.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (h *handler) PutBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
|
h.logAndSendError(w, "not implemented", api.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
|
||||||
|
}
|
||||||
|
|
||||||
func (h *handler) PutBucketEncryptionHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) PutBucketEncryptionHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
h.logAndSendError(w, "not implemented", api.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
|
h.logAndSendError(w, "not implemented", api.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
|
||||||
}
|
}
|
||||||
|
|
|
@ -7,11 +7,11 @@ import (
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/layer"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/session"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -29,6 +29,19 @@ func (h *handler) logAndSendError(w http.ResponseWriter, logText string, reqInfo
|
||||||
h.log.Error("call method", fields...)
|
h.log.Error("call method", fields...)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (h *handler) logAndSendErrorNoHeader(w http.ResponseWriter, logText string, reqInfo *api.ReqInfo, err error, additional ...zap.Field) {
|
||||||
|
api.WriteErrorResponseNoHeader(w, reqInfo, transformToS3Error(err))
|
||||||
|
fields := []zap.Field{
|
||||||
|
zap.String("request_id", reqInfo.RequestID),
|
||||||
|
zap.String("method", reqInfo.API),
|
||||||
|
zap.String("bucket", reqInfo.BucketName),
|
||||||
|
zap.String("object", reqInfo.ObjectName),
|
||||||
|
zap.String("description", logText),
|
||||||
|
zap.Error(err)}
|
||||||
|
fields = append(fields, additional...)
|
||||||
|
h.log.Error("call method", fields...)
|
||||||
|
}
|
||||||
|
|
||||||
func transformToS3Error(err error) error {
|
func transformToS3Error(err error) error {
|
||||||
if _, ok := err.(errors.Error); ok {
|
if _, ok := err.(errors.Error); ok {
|
||||||
return err
|
return err
|
||||||
|
@ -42,6 +55,10 @@ func transformToS3Error(err error) error {
|
||||||
return errors.GetAPIError(errors.ErrInternalError)
|
return errors.GetAPIError(errors.ErrInternalError)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (h *handler) ResolveBucket(ctx context.Context, bucket string) (*data.BucketInfo, error) {
|
||||||
|
return h.obj.GetBucketInfo(ctx, bucket)
|
||||||
|
}
|
||||||
|
|
||||||
func (h *handler) getBucketAndCheckOwner(r *http.Request, bucket string, header ...string) (*data.BucketInfo, error) {
|
func (h *handler) getBucketAndCheckOwner(r *http.Request, bucket string, header ...string) (*data.BucketInfo, error) {
|
||||||
bktInfo, err := h.obj.GetBucketInfo(r.Context(), bucket)
|
bktInfo, err := h.obj.GetBucketInfo(r.Context(), bucket)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
|
@ -4,10 +4,10 @@ import (
|
||||||
"encoding/xml"
|
"encoding/xml"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/layer"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
)
|
)
|
||||||
|
|
||||||
func (h *handler) PutBucketVersioningHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) PutBucketVersioningHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
|
|
|
@ -57,13 +57,14 @@ const (
|
||||||
AmzObjectAttributes = "X-Amz-Object-Attributes"
|
AmzObjectAttributes = "X-Amz-Object-Attributes"
|
||||||
AmzMaxParts = "X-Amz-Max-Parts"
|
AmzMaxParts = "X-Amz-Max-Parts"
|
||||||
AmzPartNumberMarker = "X-Amz-Part-Number-Marker"
|
AmzPartNumberMarker = "X-Amz-Part-Number-Marker"
|
||||||
AmzExpiration = "X-Amz-Expiration"
|
|
||||||
|
|
||||||
AmzServerSideEncryptionCustomerAlgorithm = "x-amz-server-side-encryption-customer-algorithm"
|
AmzServerSideEncryptionCustomerAlgorithm = "x-amz-server-side-encryption-customer-algorithm"
|
||||||
AmzServerSideEncryptionCustomerKey = "x-amz-server-side-encryption-customer-key"
|
AmzServerSideEncryptionCustomerKey = "x-amz-server-side-encryption-customer-key"
|
||||||
AmzServerSideEncryptionCustomerKeyMD5 = "x-amz-server-side-encryption-customer-key-MD5"
|
AmzServerSideEncryptionCustomerKeyMD5 = "x-amz-server-side-encryption-customer-key-MD5"
|
||||||
|
|
||||||
ContainerID = "X-Container-Id"
|
ContainerID = "X-Container-Id"
|
||||||
|
ContainerName = "X-Container-Name"
|
||||||
|
ContainerZone = "X-Container-Zone"
|
||||||
|
|
||||||
AccessControlAllowOrigin = "Access-Control-Allow-Origin"
|
AccessControlAllowOrigin = "Access-Control-Allow-Origin"
|
||||||
AccessControlAllowMethods = "Access-Control-Allow-Methods"
|
AccessControlAllowMethods = "Access-Control-Allow-Methods"
|
||||||
|
|
|
@ -1,11 +1,11 @@
|
||||||
package layer
|
package layer
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/cache"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/cache"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
cid "github.com/TrueCloudLab/frostfs-sdk-go/container/id"
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||||
oid "github.com/TrueCloudLab/frostfs-sdk-go/object/id"
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/user"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
@ -4,8 +4,8 @@ import (
|
||||||
"context"
|
"context"
|
||||||
errorsStd "errors"
|
errorsStd "errors"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
)
|
)
|
||||||
|
|
||||||
func (n *layer) GetObjectTaggingAndLock(ctx context.Context, objVersion *ObjectVersion, nodeVersion *data.NodeVersion) (map[string]string, *data.LockInfo, error) {
|
func (n *layer) GetObjectTaggingAndLock(ctx context.Context, objVersion *ObjectVersion, nodeVersion *data.NodeVersion) (map[string]string, *data.LockInfo, error) {
|
||||||
|
|
|
@ -5,14 +5,15 @@ import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"strconv"
|
"strconv"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api"
|
v2container "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/client"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/container"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
|
||||||
cid "github.com/TrueCloudLab/frostfs-sdk-go/container/id"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/eacl"
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/session"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -56,6 +57,7 @@ func (n *layer) containerInfo(ctx context.Context, idCnr cid.ID) (*data.BucketIn
|
||||||
info.Owner = cnr.Owner()
|
info.Owner = cnr.Owner()
|
||||||
if domain := container.ReadDomain(cnr); domain.Name() != "" {
|
if domain := container.ReadDomain(cnr); domain.Name() != "" {
|
||||||
info.Name = domain.Name()
|
info.Name = domain.Name()
|
||||||
|
info.Zone = domain.Zone()
|
||||||
}
|
}
|
||||||
info.Created = container.CreatedAt(cnr)
|
info.Created = container.CreatedAt(cnr)
|
||||||
info.LocationConstraint = cnr.Attribute(attributeLocationConstraint)
|
info.LocationConstraint = cnr.Attribute(attributeLocationConstraint)
|
||||||
|
@ -114,6 +116,7 @@ func (n *layer) createContainer(ctx context.Context, p *CreateBucketParams) (*da
|
||||||
}
|
}
|
||||||
bktInfo := &data.BucketInfo{
|
bktInfo := &data.BucketInfo{
|
||||||
Name: p.Name,
|
Name: p.Name,
|
||||||
|
Zone: v2container.SysAttributeZoneDefault,
|
||||||
Owner: ownerID,
|
Owner: ownerID,
|
||||||
Created: TimeNow(ctx),
|
Created: TimeNow(ctx),
|
||||||
LocationConstraint: p.LocationConstraint,
|
LocationConstraint: p.LocationConstraint,
|
||||||
|
|
|
@ -8,8 +8,8 @@ import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -39,7 +39,7 @@ func (n *layer) PutBucketCORS(ctx context.Context, p *PutCORSParams) error {
|
||||||
prm := PrmObjectCreate{
|
prm := PrmObjectCreate{
|
||||||
Container: p.BktInfo.CID,
|
Container: p.BktInfo.CID,
|
||||||
Creator: p.BktInfo.Owner,
|
Creator: p.BktInfo.Owner,
|
||||||
Payload: p.Reader,
|
Payload: &buf,
|
||||||
Filepath: p.BktInfo.CORSObjectName(),
|
Filepath: p.BktInfo.CORSObjectName(),
|
||||||
CreationTime: TimeNow(ctx),
|
CreationTime: TimeNow(ctx),
|
||||||
CopiesNumber: p.CopiesNumber,
|
CopiesNumber: p.CopiesNumber,
|
||||||
|
|
|
@ -7,16 +7,16 @@ import (
|
||||||
"io"
|
"io"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/bearer"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/container"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/container/acl"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl"
|
||||||
cid "github.com/TrueCloudLab/frostfs-sdk-go/container/id"
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/eacl"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/netmap"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/object"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||||
oid "github.com/TrueCloudLab/frostfs-sdk-go/object/id"
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/session"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/user"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||||
)
|
)
|
||||||
|
|
||||||
// PrmContainerCreate groups parameters of FrostFS.CreateContainer operation.
|
// PrmContainerCreate groups parameters of FrostFS.CreateContainer operation.
|
||||||
|
|
|
@ -11,18 +11,18 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/bearer"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
||||||
cid "github.com/TrueCloudLab/frostfs-sdk-go/container/id"
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/eacl"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/netmap"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
|
||||||
oid "github.com/TrueCloudLab/frostfs-sdk-go/object/id"
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/session"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/user"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||||
"github.com/nats-io/nats.go"
|
"github.com/nats-io/nats.go"
|
||||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
|
@ -191,8 +191,6 @@ type (
|
||||||
GetBucketSettings(ctx context.Context, bktInfo *data.BucketInfo) (*data.BucketSettings, error)
|
GetBucketSettings(ctx context.Context, bktInfo *data.BucketInfo) (*data.BucketSettings, error)
|
||||||
PutBucketSettings(ctx context.Context, p *PutSettingsParams) error
|
PutBucketSettings(ctx context.Context, p *PutSettingsParams) error
|
||||||
|
|
||||||
ScheduleLifecycle(ctx context.Context, bktInfo *data.BucketInfo, new *data.LifecycleConfiguration) error
|
|
||||||
|
|
||||||
PutBucketCORS(ctx context.Context, p *PutCORSParams) error
|
PutBucketCORS(ctx context.Context, p *PutCORSParams) error
|
||||||
GetBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) (*data.CORSConfiguration, error)
|
GetBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) (*data.CORSConfiguration, error)
|
||||||
DeleteBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) error
|
DeleteBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) error
|
||||||
|
@ -290,9 +288,7 @@ func (n *layer) Initialize(ctx context.Context, c EventListener) error {
|
||||||
return fmt.Errorf("already initialized")
|
return fmt.Errorf("already initialized")
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := c.Subscribe(ctx, ExpireTopic, MsgHandlerFunc(n.handleExpireTick)); err != nil {
|
// todo add notification handlers (e.g. for lifecycles)
|
||||||
return fmt.Errorf("couldn't initialize layer: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
c.Listen(ctx)
|
c.Listen(ctx)
|
||||||
|
|
||||||
|
|
|
@ -1,211 +0,0 @@
|
||||||
package layer
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"crypto/sha256"
|
|
||||||
"encoding/hex"
|
|
||||||
"encoding/xml"
|
|
||||||
"fmt"
|
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
|
||||||
oid "github.com/TrueCloudLab/frostfs-sdk-go/object/id"
|
|
||||||
"github.com/nats-io/nats.go"
|
|
||||||
"go.uber.org/zap"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
AttributeExpirationEpoch = "__NEOFS__EXPIRATION_EPOCH"
|
|
||||||
AttributeSysTickEpoch = "__NEOFS__TICK_EPOCH"
|
|
||||||
AttributeSysTickTopic = "__NEOFS__TICK_TOPIC"
|
|
||||||
AttributeParentObject = ".s3-expire-parent-object"
|
|
||||||
AttributeParentBucket = ".s3-expire-parent-bucket"
|
|
||||||
AttributeExpireDate = ".s3-expire-date"
|
|
||||||
AttributeExpireRuleID = ".s3-expire-rule-id"
|
|
||||||
AttributeLifecycleConfigID = ".s3-lifecycle-config"
|
|
||||||
ExpireTopic = "expire"
|
|
||||||
)
|
|
||||||
|
|
||||||
func (n *layer) handleExpireTick(ctx context.Context, msg *nats.Msg) error {
|
|
||||||
var addr oid.Address
|
|
||||||
if err := addr.DecodeString(string(msg.Data)); err != nil {
|
|
||||||
return fmt.Errorf("invalid msg, address expected: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
n.log.Debug("handling expiration tick", zap.String("address", string(msg.Data)))
|
|
||||||
|
|
||||||
// and make sure having right access
|
|
||||||
|
|
||||||
//todo redo
|
|
||||||
bktInfo := &data.BucketInfo{CID: addr.Container()}
|
|
||||||
|
|
||||||
obj, err := n.objectHead(ctx, bktInfo, addr.Object())
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("couldn't head expiration object: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
header := userHeaders(obj.Attributes())
|
|
||||||
objName := header[AttributeParentObject]
|
|
||||||
bktName := header[AttributeParentBucket]
|
|
||||||
if objName == "" || bktName == "" {
|
|
||||||
return fmt.Errorf("couldn't know bucket/object to expire")
|
|
||||||
}
|
|
||||||
|
|
||||||
p := &DeleteObjectParams{
|
|
||||||
BktInfo: bktInfo,
|
|
||||||
Objects: []*VersionedObject{{Name: objName}},
|
|
||||||
}
|
|
||||||
|
|
||||||
res := n.DeleteObjects(ctx, p)
|
|
||||||
if res[0].Error != nil {
|
|
||||||
return fmt.Errorf("couldn't delete expired object: %w", res[0].Error)
|
|
||||||
}
|
|
||||||
|
|
||||||
return n.objectDelete(ctx, bktInfo, addr.Object())
|
|
||||||
}
|
|
||||||
|
|
||||||
func (n *layer) ScheduleLifecycle(ctx context.Context, bktInfo *data.BucketInfo, newConf *data.LifecycleConfiguration) error {
|
|
||||||
if newConf == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
lifecycleID, err := computeLifecycleID(newConf)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("couldn't compute lifecycle id: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// We want to be able to revert partly applied lifecycle if something goes wrong.
|
|
||||||
if err = n.updateLifecycle(ctx, bktInfo, &data.LifecycleConfig{
|
|
||||||
OldConfigurationID: lifecycleID,
|
|
||||||
}); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
if err = n.applyLifecycle(ctx, bktInfo, lifecycleID, newConf); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
return n.updateLifecycle(ctx, bktInfo, &data.LifecycleConfig{
|
|
||||||
OldConfigurationID: lifecycleID,
|
|
||||||
CurrentConfiguration: newConf,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func (n *layer) updateLifecycle(ctx context.Context, bktInfo *data.BucketInfo, lifecycleConfig *data.LifecycleConfig) error {
|
|
||||||
settings, err := n.GetBucketSettings(ctx, bktInfo)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("couldn't get bucket settings: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
settings.LifecycleConfig = lifecycleConfig
|
|
||||||
sp := &PutSettingsParams{
|
|
||||||
BktInfo: bktInfo,
|
|
||||||
Settings: settings,
|
|
||||||
}
|
|
||||||
|
|
||||||
if err = n.PutBucketSettings(ctx, sp); err != nil {
|
|
||||||
return fmt.Errorf("couldn't put bucket settings: %w", err)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (n *layer) applyLifecycle(ctx context.Context, bktInfo *data.BucketInfo, lifecycleID string, conf *data.LifecycleConfiguration) error {
|
|
||||||
for _, rule := range conf.Rules {
|
|
||||||
if rule.Status == "Disabled" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
listParam := allObjectParams{
|
|
||||||
Bucket: bktInfo,
|
|
||||||
Prefix: rule.RealPrefix(),
|
|
||||||
}
|
|
||||||
|
|
||||||
objects, _, err := n.getLatestObjectsVersions(ctx, listParam)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
if err = n.applyLifecycleToObjects(ctx, bktInfo, lifecycleID, rule, objects); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (n *layer) applyLifecycleToObjects(ctx context.Context, bktInfo *data.BucketInfo, lifecycleID string, rule data.Rule, objects []*data.ObjectInfo) error {
|
|
||||||
var tags []map[string]string
|
|
||||||
var err error
|
|
||||||
if rule.NeedTags() {
|
|
||||||
tags = make([]map[string]string, len(objects))
|
|
||||||
p := &ObjectVersion{
|
|
||||||
BktInfo: bktInfo,
|
|
||||||
}
|
|
||||||
for i, obj := range objects {
|
|
||||||
p.ObjectName = obj.Name
|
|
||||||
p.VersionID = obj.VersionID()
|
|
||||||
if _, tags[i], err = n.GetObjectTagging(ctx, p); err != nil {
|
|
||||||
return fmt.Errorf("couldn't get object tags: %w", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
for i, obj := range objects {
|
|
||||||
var objTags map[string]string
|
|
||||||
if len(tags) != 0 {
|
|
||||||
objTags = tags[i]
|
|
||||||
}
|
|
||||||
if !rule.MatchObject(obj, objTags) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
expObj := &data.ExpirationObject{
|
|
||||||
Expiration: rule.Expiration,
|
|
||||||
RuleID: rule.ID,
|
|
||||||
LifecycleConfigID: lifecycleID,
|
|
||||||
}
|
|
||||||
|
|
||||||
if _, err = n.putExpirationObject(ctx, bktInfo, obj, expObj); err != nil {
|
|
||||||
return fmt.Errorf("couldn't put expiration object: %w", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (n *layer) putLifecycleObjects(ctx context.Context, bktInfo *data.BucketInfo, obj *data.ObjectInfo, lifecycle *data.LifecycleConfig) error {
|
|
||||||
if lifecycle == nil || lifecycle.CurrentConfiguration == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, rule := range lifecycle.CurrentConfiguration.Rules {
|
|
||||||
if rule.Status == "Disabled" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
// at this time lifecycle.OldConfigurationID is the same as lifecycle.CurrentConfiguration id
|
|
||||||
if err := n.applyLifecycleToObjects(ctx, bktInfo, lifecycle.OldConfigurationID, rule, []*data.ObjectInfo{obj}); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func computeLifecycleID(conf *data.LifecycleConfiguration) (string, error) {
|
|
||||||
raw, err := xml.Marshal(conf)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("couldn't marshall new lifecycle configuration: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
sha := sha256.New()
|
|
||||||
sha.Write(raw)
|
|
||||||
sum := sha.Sum(nil)
|
|
||||||
|
|
||||||
id := hex.EncodeToString(sum)
|
|
||||||
|
|
||||||
if id == "" {
|
|
||||||
return "", fmt.Errorf("computed id is empty")
|
|
||||||
}
|
|
||||||
|
|
||||||
return id, nil
|
|
||||||
}
|
|
|
@ -1,159 +0,0 @@
|
||||||
package layer
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestComputeLifecycleID(t *testing.T) {
|
|
||||||
conf := &data.LifecycleConfiguration{Rules: []data.Rule{
|
|
||||||
{
|
|
||||||
ID: "id",
|
|
||||||
Status: "Enabled",
|
|
||||||
},
|
|
||||||
}}
|
|
||||||
|
|
||||||
id, err := computeLifecycleID(conf)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Equal(t, "51ff619dc848622287764fc7c4aec06b7c1a5936c25b8eee48a0dbcb4eeac9f4", id)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRuleMatchObject(t *testing.T) {
|
|
||||||
prefix, suffix := "prefix", "suffix"
|
|
||||||
objSizeMin, objSizeMax := int64(512), int64(1024)
|
|
||||||
|
|
||||||
for _, tc := range []struct {
|
|
||||||
name string
|
|
||||||
rule data.Rule
|
|
||||||
obj *data.ObjectInfo
|
|
||||||
tags map[string]string
|
|
||||||
expected bool
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "basic match",
|
|
||||||
rule: data.Rule{Prefix: &prefix},
|
|
||||||
obj: &data.ObjectInfo{Name: prefix + suffix},
|
|
||||||
expected: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "basic no match",
|
|
||||||
rule: data.Rule{Prefix: &prefix},
|
|
||||||
obj: &data.ObjectInfo{Name: suffix + prefix},
|
|
||||||
expected: false,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "filter and sizes",
|
|
||||||
rule: data.Rule{Filter: &data.LifecycleRuleFilter{
|
|
||||||
And: &data.LifecycleRuleAndOperator{
|
|
||||||
ObjectSizeGreaterThan: &objSizeMin,
|
|
||||||
ObjectSizeLessThan: &objSizeMax,
|
|
||||||
},
|
|
||||||
}},
|
|
||||||
obj: &data.ObjectInfo{Name: suffix, Size: 768},
|
|
||||||
expected: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "filter prefix",
|
|
||||||
rule: data.Rule{Filter: &data.LifecycleRuleFilter{
|
|
||||||
Prefix: &prefix,
|
|
||||||
}},
|
|
||||||
obj: &data.ObjectInfo{Name: prefix + suffix},
|
|
||||||
expected: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "filter prefix no match",
|
|
||||||
rule: data.Rule{Filter: &data.LifecycleRuleFilter{
|
|
||||||
Prefix: &prefix,
|
|
||||||
}},
|
|
||||||
obj: &data.ObjectInfo{Name: suffix},
|
|
||||||
expected: false,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "filter tags",
|
|
||||||
rule: data.Rule{Filter: &data.LifecycleRuleFilter{
|
|
||||||
Tag: &data.Tag{
|
|
||||||
Key: "key",
|
|
||||||
Value: "val",
|
|
||||||
},
|
|
||||||
}},
|
|
||||||
tags: map[string]string{"key": "val"},
|
|
||||||
obj: &data.ObjectInfo{},
|
|
||||||
expected: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "filter and tags no match",
|
|
||||||
rule: data.Rule{Filter: &data.LifecycleRuleFilter{
|
|
||||||
And: &data.LifecycleRuleAndOperator{
|
|
||||||
Tags: []data.Tag{{
|
|
||||||
Key: "key",
|
|
||||||
Value: "val",
|
|
||||||
}},
|
|
||||||
},
|
|
||||||
}},
|
|
||||||
tags: map[string]string{"key": "val2"},
|
|
||||||
obj: &data.ObjectInfo{},
|
|
||||||
expected: false,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "filter size no match",
|
|
||||||
rule: data.Rule{Filter: &data.LifecycleRuleFilter{
|
|
||||||
ObjectSizeGreaterThan: &objSizeMax,
|
|
||||||
}},
|
|
||||||
obj: &data.ObjectInfo{Size: objSizeMin},
|
|
||||||
expected: false,
|
|
||||||
},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
require.Equal(t, tc.expected, tc.rule.MatchObject(tc.obj, tc.tags))
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestScheduleLifecycle(t *testing.T) {
|
|
||||||
tc := prepareContext(t)
|
|
||||||
|
|
||||||
obj1 := tc.putObject([]byte("content"))
|
|
||||||
|
|
||||||
date := "2022-03-14T09:59:03Z"
|
|
||||||
date2 := "2022-03-15T09:59:03Z"
|
|
||||||
prefix := "prefix"
|
|
||||||
tc.obj = prefix
|
|
||||||
obj2 := tc.putObject([]byte("content2"))
|
|
||||||
|
|
||||||
conf := &data.LifecycleConfiguration{
|
|
||||||
Rules: []data.Rule{{
|
|
||||||
Filter: &data.LifecycleRuleFilter{
|
|
||||||
Prefix: &prefix,
|
|
||||||
},
|
|
||||||
Expiration: &data.Expiration{
|
|
||||||
Date: &date,
|
|
||||||
}},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
err := tc.layer.ScheduleLifecycle(tc.ctx, tc.bktInfo, conf)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
expObj1, _ := tc.getObject(obj1.ExpirationObject(), "", false)
|
|
||||||
require.Nil(t, expObj1)
|
|
||||||
expObj2, _ := tc.getObject(obj2.ExpirationObject(), "", false)
|
|
||||||
require.NotNil(t, expObj2)
|
|
||||||
assertExpirationObject(t, expObj2, date)
|
|
||||||
|
|
||||||
conf.Rules[0].Expiration.Date = &date2
|
|
||||||
err = tc.layer.ScheduleLifecycle(tc.ctx, tc.bktInfo, conf)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
expObj2, _ = tc.getObject(obj2.ExpirationObject(), "", false)
|
|
||||||
require.NotNil(t, expObj2)
|
|
||||||
assertExpirationObject(t, expObj2, date2)
|
|
||||||
}
|
|
||||||
|
|
||||||
func assertExpirationObject(t *testing.T, expObjInfo *data.ObjectInfo, date string) {
|
|
||||||
require.Equal(t, expObjInfo.Headers[AttributeExpireDate], date)
|
|
||||||
require.Contains(t, expObjInfo.Headers, AttributeSysTickEpoch)
|
|
||||||
require.Contains(t, expObjInfo.Headers, AttributeSysTickTopic)
|
|
||||||
require.Contains(t, expObjInfo.Headers, AttributeLifecycleConfigID)
|
|
||||||
}
|
|
|
@ -4,7 +4,8 @@ import (
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -43,10 +44,10 @@ func TestObjectLockAttributes(t *testing.T) {
|
||||||
|
|
||||||
expEpoch := false
|
expEpoch := false
|
||||||
for _, attr := range lockObj.Attributes() {
|
for _, attr := range lockObj.Attributes() {
|
||||||
if attr.Key() == AttributeExpirationEpoch {
|
if attr.Key() == object.SysAttributeExpEpoch {
|
||||||
expEpoch = true
|
expEpoch = true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
require.Truef(t, expEpoch, "system header __NEOFS__EXPIRATION_EPOCH presence")
|
require.Truef(t, expEpoch, "system header __SYSTEM__EXPIRATION_EPOCH presence")
|
||||||
}
|
}
|
||||||
|
|
|
@ -11,12 +11,12 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
|
||||||
oid "github.com/TrueCloudLab/frostfs-sdk-go/object/id"
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/user"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||||
"github.com/minio/sio"
|
"github.com/minio/sio"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
|
@ -10,18 +10,18 @@ import (
|
||||||
"io"
|
"io"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
objectv2 "github.com/TrueCloudLab/frostfs-api-go/v2/object"
|
objectv2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/bearer"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/checksum"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/container"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
|
||||||
cid "github.com/TrueCloudLab/frostfs-sdk-go/container/id"
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/eacl"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/object"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||||
oid "github.com/TrueCloudLab/frostfs-sdk-go/object/id"
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/session"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/user"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||||
)
|
)
|
||||||
|
|
||||||
type TestFrostFS struct {
|
type TestFrostFS struct {
|
||||||
|
@ -144,7 +144,7 @@ func (t *TestFrostFS) ReadObject(ctx context.Context, prm PrmObjectRead) (*Objec
|
||||||
|
|
||||||
if obj, ok := t.objects[sAddr]; ok {
|
if obj, ok := t.objects[sAddr]; ok {
|
||||||
owner := getOwner(ctx)
|
owner := getOwner(ctx)
|
||||||
if !obj.OwnerID().Equals(owner) {
|
if !obj.OwnerID().Equals(owner) && !t.isPublicRead(prm.Container) {
|
||||||
return nil, ErrAccessDenied
|
return nil, ErrAccessDenied
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -282,6 +282,25 @@ func (t *TestFrostFS) ContainerEACL(_ context.Context, cnrID cid.ID) (*eacl.Tabl
|
||||||
return table, nil
|
return table, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (t *TestFrostFS) isPublicRead(cnrID cid.ID) bool {
|
||||||
|
table, ok := t.eaclTables[cnrID.EncodeToString()]
|
||||||
|
if !ok {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, rec := range table.Records() {
|
||||||
|
if rec.Operation() == eacl.OperationGet && len(rec.Filters()) == 0 {
|
||||||
|
for _, trgt := range rec.Targets() {
|
||||||
|
if trgt.Role() == eacl.RoleOthers {
|
||||||
|
return rec.Action() == eacl.ActionAllow
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
func getOwner(ctx context.Context) user.ID {
|
func getOwner(ctx context.Context) user.ID {
|
||||||
if bd, ok := ctx.Value(api.BoxData).(*accessbox.Box); ok && bd != nil && bd.Gate != nil && bd.Gate.BearerToken != nil {
|
if bd, ok := ctx.Value(api.BoxData).(*accessbox.Box); ok && bd != nil && bd.Gate != nil && bd.Gate.BearerToken != nil {
|
||||||
return bearer.ResolveIssuer(*bd.Gate.BearerToken)
|
return bearer.ResolveIssuer(*bd.Gate.BearerToken)
|
||||||
|
|
|
@ -7,8 +7,8 @@ import (
|
||||||
errorsStd "errors"
|
errorsStd "errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
@ -14,14 +14,14 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/cache"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/cache"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
apiErrors "github.com/TrueCloudLab/frostfs-s3-gw/api/errors"
|
apiErrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/client"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
|
||||||
cid "github.com/TrueCloudLab/frostfs-sdk-go/container/id"
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/object"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||||
oid "github.com/TrueCloudLab/frostfs-sdk-go/object/id"
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
"github.com/minio/sio"
|
"github.com/minio/sio"
|
||||||
"github.com/panjf2000/ants/v2"
|
"github.com/panjf2000/ants/v2"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
|
@ -298,11 +298,6 @@ func (n *layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Extend
|
||||||
NodeVersion: newVersion,
|
NodeVersion: newVersion,
|
||||||
}
|
}
|
||||||
|
|
||||||
// todo filling api.AmzExpiration header
|
|
||||||
if err = n.putLifecycleObjects(ctx, p.BktInfo, objInfo, bktSettings.LifecycleConfig); err != nil {
|
|
||||||
return nil, fmt.Errorf("couldn't put expiration system objects: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
n.cache.PutObjectWithName(owner, extendedObjInfo)
|
n.cache.PutObjectWithName(owner, extendedObjInfo)
|
||||||
|
|
||||||
return extendedObjInfo, nil
|
return extendedObjInfo, nil
|
||||||
|
|
|
@ -9,9 +9,10 @@ import (
|
||||||
"strconv"
|
"strconv"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
oid "github.com/TrueCloudLab/frostfs-sdk-go/object/id"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
|
@ -129,54 +130,6 @@ func (n *layer) putLockObject(ctx context.Context, bktInfo *data.BucketInfo, obj
|
||||||
return id, err
|
return id, err
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *layer) putExpirationObject(ctx context.Context, bktInfo *data.BucketInfo, objInfo *data.ObjectInfo, expObj *data.ExpirationObject) (oid.ID, error) {
|
|
||||||
prm := PrmObjectCreate{
|
|
||||||
Container: bktInfo.CID,
|
|
||||||
Creator: bktInfo.Owner,
|
|
||||||
Filepath: objInfo.ExpirationObject(),
|
|
||||||
}
|
|
||||||
|
|
||||||
var (
|
|
||||||
err error
|
|
||||||
exp uint64
|
|
||||||
expTime time.Time
|
|
||||||
)
|
|
||||||
|
|
||||||
if expObj.Expiration.Days != nil {
|
|
||||||
expTime = objInfo.Created.Add(time.Duration(*expObj.Expiration.Days) * 24 * time.Hour).UTC()
|
|
||||||
// emulate rounding the resulting time to the next day midnight UTC
|
|
||||||
toMidnight := 24 - expTime.UTC().Hour()
|
|
||||||
expTime = expTime.Add(time.Duration(toMidnight) * time.Hour)
|
|
||||||
} else {
|
|
||||||
expTime, err = time.Parse(time.RFC3339, *expObj.Expiration.Date)
|
|
||||||
if err != nil {
|
|
||||||
return oid.ID{}, fmt.Errorf("couldn't parse expiration date '%s': %w", *expObj.Expiration.Date, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
now := TimeNow(ctx)
|
|
||||||
if expTime.After(now) {
|
|
||||||
_, exp, err = n.frostFS.TimeToEpoch(ctx, now, expTime)
|
|
||||||
if err != nil {
|
|
||||||
return oid.ID{}, fmt.Errorf("couldn't compute expiration epoch: %w", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
prm.Attributes = [][2]string{
|
|
||||||
{AttributeExpirationEpoch, strconv.FormatUint(exp+4, 10)},
|
|
||||||
{AttributeSysTickEpoch, strconv.FormatUint(exp, 10)},
|
|
||||||
{AttributeSysTickTopic, ExpireTopic},
|
|
||||||
{AttributeParentObject, objInfo.Name},
|
|
||||||
{AttributeParentBucket, bktInfo.Name},
|
|
||||||
{AttributeExpireDate, expTime.Format(time.RFC3339)},
|
|
||||||
{AttributeExpireRuleID, expObj.RuleID},
|
|
||||||
{AttributeLifecycleConfigID, expObj.LifecycleConfigID},
|
|
||||||
}
|
|
||||||
|
|
||||||
id, _, err := n.objectPutAndHash(ctx, prm, bktInfo)
|
|
||||||
return id, err
|
|
||||||
}
|
|
||||||
|
|
||||||
func (n *layer) GetLockInfo(ctx context.Context, objVersion *ObjectVersion) (*data.LockInfo, error) {
|
func (n *layer) GetLockInfo(ctx context.Context, objVersion *ObjectVersion) (*data.LockInfo, error) {
|
||||||
owner := n.Owner(ctx)
|
owner := n.Owner(ctx)
|
||||||
if lockInfo := n.cache.GetLockInfo(owner, lockObjectKey(objVersion)); lockInfo != nil {
|
if lockInfo := n.cache.GetLockInfo(owner, lockObjectKey(objVersion)); lockInfo != nil {
|
||||||
|
@ -285,7 +238,7 @@ func (n *layer) attributesFromLock(ctx context.Context, lock *data.ObjectLock) (
|
||||||
}
|
}
|
||||||
|
|
||||||
if lock.LegalHold != nil && lock.LegalHold.Enabled {
|
if lock.LegalHold != nil && lock.LegalHold.Enabled {
|
||||||
// todo: (@KirillovDenis) reconsider this when FrostFS will support Legal Hold https://github.com/TrueCloudLab/frostfs-contract/issues/2
|
// todo: (@KirillovDenis) reconsider this when FrostFS will support Legal Hold https://git.frostfs.info/TrueCloudLab/frostfs-contract/issues/2
|
||||||
// Currently lock object must have an expiration epoch.
|
// Currently lock object must have an expiration epoch.
|
||||||
// Besides we need to override retention expiration epoch since legal hold cannot be deleted yet.
|
// Besides we need to override retention expiration epoch since legal hold cannot be deleted yet.
|
||||||
expEpoch = math.MaxUint64
|
expEpoch = math.MaxUint64
|
||||||
|
@ -293,7 +246,7 @@ func (n *layer) attributesFromLock(ctx context.Context, lock *data.ObjectLock) (
|
||||||
|
|
||||||
if expEpoch != 0 {
|
if expEpoch != 0 {
|
||||||
result = append(result, [2]string{
|
result = append(result, [2]string{
|
||||||
AttributeExpirationEpoch, strconv.FormatUint(expEpoch, 10),
|
object.SysAttributeExpEpoch, strconv.FormatUint(expEpoch, 10),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -4,12 +4,12 @@ import (
|
||||||
"context"
|
"context"
|
||||||
errorsStd "errors"
|
errorsStd "errors"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
cid "github.com/TrueCloudLab/frostfs-sdk-go/container/id"
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||||
oid "github.com/TrueCloudLab/frostfs-sdk-go/object/id"
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/user"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
@ -6,8 +6,8 @@ import (
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
oid "github.com/TrueCloudLab/frostfs-sdk-go/object/id"
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
)
|
)
|
||||||
|
|
||||||
type TreeServiceMock struct {
|
type TreeServiceMock struct {
|
||||||
|
@ -109,11 +109,32 @@ func (t *TreeServiceMock) PutNotificationConfigurationNode(ctx context.Context,
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *TreeServiceMock) GetBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) (oid.ID, error) {
|
func (t *TreeServiceMock) GetBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) (oid.ID, error) {
|
||||||
panic("implement me")
|
systemMap, ok := t.system[bktInfo.CID.EncodeToString()]
|
||||||
|
if !ok {
|
||||||
|
return oid.ID{}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
node, ok := systemMap["cors"]
|
||||||
|
if !ok {
|
||||||
|
return oid.ID{}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return node.OID, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *TreeServiceMock) PutBucketCORS(ctx context.Context, bktInfo *data.BucketInfo, objID oid.ID) (oid.ID, error) {
|
func (t *TreeServiceMock) PutBucketCORS(ctx context.Context, bktInfo *data.BucketInfo, objID oid.ID) (oid.ID, error) {
|
||||||
panic("implement me")
|
systemMap, ok := t.system[bktInfo.CID.EncodeToString()]
|
||||||
|
if !ok {
|
||||||
|
systemMap = make(map[string]*data.BaseNodeVersion)
|
||||||
|
}
|
||||||
|
|
||||||
|
systemMap["cors"] = &data.BaseNodeVersion{
|
||||||
|
OID: objID,
|
||||||
|
}
|
||||||
|
|
||||||
|
t.system[bktInfo.CID.EncodeToString()] = systemMap
|
||||||
|
|
||||||
|
return oid.ID{}, ErrNoNodeToRemove
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *TreeServiceMock) DeleteBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) (oid.ID, error) {
|
func (t *TreeServiceMock) DeleteBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) (oid.ID, error) {
|
||||||
|
|
|
@ -4,8 +4,8 @@ import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
oid "github.com/TrueCloudLab/frostfs-sdk-go/object/id"
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
)
|
)
|
||||||
|
|
||||||
// TreeService provide interface to interact with tree service using s3 data models.
|
// TreeService provide interface to interact with tree service using s3 data models.
|
||||||
|
|
|
@ -9,11 +9,11 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/object"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||||
)
|
)
|
||||||
|
|
||||||
type (
|
type (
|
||||||
|
|
|
@ -6,11 +6,11 @@ import (
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/checksum"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum"
|
||||||
cid "github.com/TrueCloudLab/frostfs-sdk-go/container/id"
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||||
oid "github.com/TrueCloudLab/frostfs-sdk-go/object/id"
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/user"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
@ -4,7 +4,7 @@ import (
|
||||||
"context"
|
"context"
|
||||||
"sort"
|
"sort"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
)
|
)
|
||||||
|
|
||||||
func (n *layer) ListObjectVersions(ctx context.Context, p *ListObjectVersionsParams) (*ListObjectVersionsInfo, error) {
|
func (n *layer) ListObjectVersions(ctx context.Context, p *ListObjectVersionsParams) (*ListObjectVersionsInfo, error) {
|
||||||
|
|
|
@ -5,13 +5,13 @@ import (
|
||||||
"context"
|
"context"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
||||||
bearertest "github.com/TrueCloudLab/frostfs-sdk-go/bearer/test"
|
bearertest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer/test"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/object"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||||
oid "github.com/TrueCloudLab/frostfs-sdk-go/object/id"
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/user"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
|
|
|
@ -4,7 +4,7 @@ import (
|
||||||
"net/http"
|
"net/http"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
)
|
)
|
||||||
|
|
||||||
type (
|
type (
|
||||||
|
|
|
@ -1,6 +1,7 @@
|
||||||
package metrics
|
package api
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
"strings"
|
"strings"
|
||||||
|
@ -8,9 +9,68 @@ import (
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
||||||
"github.com/prometheus/client_golang/prometheus"
|
"github.com/prometheus/client_golang/prometheus"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
type RequestType int
|
||||||
|
|
||||||
|
const (
|
||||||
|
UNKNOWNRequest RequestType = iota
|
||||||
|
HEADRequest RequestType = iota
|
||||||
|
PUTRequest RequestType = iota
|
||||||
|
LISTRequest RequestType = iota
|
||||||
|
GETRequest RequestType = iota
|
||||||
|
DELETERequest RequestType = iota
|
||||||
|
)
|
||||||
|
|
||||||
|
func (t RequestType) String() string {
|
||||||
|
switch t {
|
||||||
|
case 1:
|
||||||
|
return "HEAD"
|
||||||
|
case 2:
|
||||||
|
return "PUT"
|
||||||
|
case 3:
|
||||||
|
return "LIST"
|
||||||
|
case 4:
|
||||||
|
return "GET"
|
||||||
|
case 5:
|
||||||
|
return "DELETE"
|
||||||
|
default:
|
||||||
|
return "Unknown"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func RequestTypeFromAPI(api string) RequestType {
|
||||||
|
switch api {
|
||||||
|
case "Options", "HeadObject", "HeadBucket":
|
||||||
|
return HEADRequest
|
||||||
|
case "CreateMultipartUpload", "UploadPartCopy", "UploadPart", "CompleteMultipartUpload",
|
||||||
|
"PutObjectACL", "PutObjectTagging", "CopyObject", "PutObjectRetention", "PutObjectLegalHold",
|
||||||
|
"PutObject", "PutBucketCors", "PutBucketACL", "PutBucketLifecycle", "PutBucketEncryption",
|
||||||
|
"PutBucketPolicy", "PutBucketObjectLockConfig", "PutBucketTagging", "PutBucketVersioning",
|
||||||
|
"PutBucketNotification", "CreateBucket", "PostObject":
|
||||||
|
return PUTRequest
|
||||||
|
case "ListObjectParts", "ListMultipartUploads", "ListObjectsV2M", "ListObjectsV2", "ListBucketVersions",
|
||||||
|
"ListObjectsV1", "ListBuckets":
|
||||||
|
return LISTRequest
|
||||||
|
case "GetObjectACL", "GetObjectTagging", "SelectObjectContent", "GetObjectRetention", "getobjectlegalhold",
|
||||||
|
"GetObjectAttributes", "GetObject", "GetBucketLocation", "GetBucketPolicy",
|
||||||
|
"GetBucketLifecycle", "GetBucketEncryption", "GetBucketCors", "GetBucketACL",
|
||||||
|
"GetBucketWebsite", "GetBucketAccelerate", "GetBucketRequestPayment", "GetBucketLogging",
|
||||||
|
"GetBucketReplication", "GetBucketTagging", "GetBucketObjectLockConfig",
|
||||||
|
"GetBucketVersioning", "GetBucketNotification", "ListenBucketNotification":
|
||||||
|
return GETRequest
|
||||||
|
case "AbortMultipartUpload", "DeleteObjectTagging", "DeleteObject", "DeleteBucketCors",
|
||||||
|
"DeleteBucketWebsite", "DeleteBucketTagging", "DeleteMultipleObjects", "DeleteBucketPolicy",
|
||||||
|
"DeleteBucketLifecycle", "DeleteBucketEncryption", "DeleteBucket":
|
||||||
|
return DELETERequest
|
||||||
|
default:
|
||||||
|
return UNKNOWNRequest
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
type (
|
type (
|
||||||
// HTTPAPIStats holds statistics information about
|
// HTTPAPIStats holds statistics information about
|
||||||
// the API given in the requests.
|
// the API given in the requests.
|
||||||
|
@ -19,6 +79,10 @@ type (
|
||||||
sync.RWMutex
|
sync.RWMutex
|
||||||
}
|
}
|
||||||
|
|
||||||
|
UsersStat interface {
|
||||||
|
Update(user, bucket, cnrID string, reqType RequestType, in, out uint64)
|
||||||
|
}
|
||||||
|
|
||||||
// HTTPStats holds statistics information about
|
// HTTPStats holds statistics information about
|
||||||
// HTTP requests made by all clients.
|
// HTTP requests made by all clients.
|
||||||
HTTPStats struct {
|
HTTPStats struct {
|
||||||
|
@ -103,11 +167,16 @@ func collectHTTPMetrics(ch chan<- prometheus.Metric) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// APIStats wraps http handler for api with basic statistics collection.
|
// CIDResolveFunc is a func to resolve CID in Stats handler.
|
||||||
func APIStats(api string, f http.HandlerFunc) http.HandlerFunc {
|
type CIDResolveFunc func(ctx context.Context, reqInfo *ReqInfo) (cnrID string)
|
||||||
|
|
||||||
|
// Stats is a handler that update metrics.
|
||||||
|
func Stats(f http.HandlerFunc, resolveCID CIDResolveFunc, usersStat UsersStat) http.HandlerFunc {
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
httpStatsMetric.currentS3Requests.Inc(api)
|
reqInfo := GetReqInfo(r.Context())
|
||||||
defer httpStatsMetric.currentS3Requests.Dec(api)
|
|
||||||
|
httpStatsMetric.currentS3Requests.Inc(reqInfo.API)
|
||||||
|
defer httpStatsMetric.currentS3Requests.Dec(reqInfo.API)
|
||||||
|
|
||||||
in := &readCounter{ReadCloser: r.Body}
|
in := &readCounter{ReadCloser: r.Body}
|
||||||
out := &writeCounter{ResponseWriter: w}
|
out := &writeCounter{ResponseWriter: w}
|
||||||
|
@ -119,20 +188,45 @@ func APIStats(api string, f http.HandlerFunc) http.HandlerFunc {
|
||||||
startTime: time.Now(),
|
startTime: time.Now(),
|
||||||
}
|
}
|
||||||
|
|
||||||
f.ServeHTTP(statsWriter, r)
|
f(statsWriter, r)
|
||||||
|
|
||||||
// Time duration in secs since the call started.
|
// Time duration in secs since the call started.
|
||||||
// We don't need to do nanosecond precision here
|
// We don't need to do nanosecond precision here
|
||||||
// simply for the fact that it is not human readable.
|
// simply for the fact that it is not human-readable.
|
||||||
durationSecs := time.Since(statsWriter.startTime).Seconds()
|
durationSecs := time.Since(statsWriter.startTime).Seconds()
|
||||||
|
|
||||||
httpStatsMetric.updateStats(api, statsWriter, r, durationSecs)
|
user := resolveUser(r.Context())
|
||||||
|
cnrID := resolveCID(r.Context(), reqInfo)
|
||||||
|
usersStat.Update(user, reqInfo.BucketName, cnrID, RequestTypeFromAPI(reqInfo.API), in.countBytes, out.countBytes)
|
||||||
|
|
||||||
|
code := statsWriter.statusCode
|
||||||
|
// A successful request has a 2xx response code
|
||||||
|
successReq := code >= http.StatusOK && code < http.StatusMultipleChoices
|
||||||
|
if !strings.HasSuffix(r.URL.Path, systemPath) {
|
||||||
|
httpStatsMetric.totalS3Requests.Inc(reqInfo.API)
|
||||||
|
if !successReq && code != 0 {
|
||||||
|
httpStatsMetric.totalS3Errors.Inc(reqInfo.API)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if r.Method == http.MethodGet {
|
||||||
|
// Increment the prometheus http request response histogram with appropriate label
|
||||||
|
httpRequestsDuration.With(prometheus.Labels{"api": reqInfo.API}).Observe(durationSecs)
|
||||||
|
}
|
||||||
|
|
||||||
atomic.AddUint64(&httpStatsMetric.totalInputBytes, in.countBytes)
|
atomic.AddUint64(&httpStatsMetric.totalInputBytes, in.countBytes)
|
||||||
atomic.AddUint64(&httpStatsMetric.totalOutputBytes, out.countBytes)
|
atomic.AddUint64(&httpStatsMetric.totalOutputBytes, out.countBytes)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func resolveUser(ctx context.Context) string {
|
||||||
|
user := "anon"
|
||||||
|
if bd, ok := ctx.Value(BoxData).(*accessbox.Box); ok && bd != nil && bd.Gate != nil && bd.Gate.BearerToken != nil {
|
||||||
|
user = bearer.ResolveIssuer(*bd.Gate.BearerToken).String()
|
||||||
|
}
|
||||||
|
return user
|
||||||
|
}
|
||||||
|
|
||||||
// Inc increments the api stats counter.
|
// Inc increments the api stats counter.
|
||||||
func (stats *HTTPAPIStats) Inc(api string) {
|
func (stats *HTTPAPIStats) Inc(api string) {
|
||||||
if stats == nil {
|
if stats == nil {
|
||||||
|
@ -177,30 +271,6 @@ func (st *HTTPStats) getOutputBytes() uint64 {
|
||||||
return atomic.LoadUint64(&st.totalOutputBytes)
|
return atomic.LoadUint64(&st.totalOutputBytes)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Update statistics from http request and response data.
|
|
||||||
func (st *HTTPStats) updateStats(api string, w http.ResponseWriter, r *http.Request, durationSecs float64) {
|
|
||||||
var code int
|
|
||||||
|
|
||||||
if res, ok := w.(*responseWrapper); ok {
|
|
||||||
code = res.statusCode
|
|
||||||
}
|
|
||||||
|
|
||||||
// A successful request has a 2xx response code
|
|
||||||
successReq := code >= http.StatusOK && code < http.StatusMultipleChoices
|
|
||||||
|
|
||||||
if !strings.HasSuffix(r.URL.Path, systemPath) {
|
|
||||||
st.totalS3Requests.Inc(api)
|
|
||||||
if !successReq && code != 0 {
|
|
||||||
st.totalS3Errors.Inc(api)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if r.Method == http.MethodGet {
|
|
||||||
// Increment the prometheus http request response histogram with appropriate label
|
|
||||||
httpRequestsDuration.With(prometheus.Labels{"api": api}).Observe(durationSecs)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// WriteHeader -- writes http status code.
|
// WriteHeader -- writes http status code.
|
||||||
func (w *responseWrapper) WriteHeader(code int) {
|
func (w *responseWrapper) WriteHeader(code int) {
|
||||||
w.Do(func() {
|
w.Do(func() {
|
||||||
|
@ -216,6 +286,12 @@ func (w *responseWrapper) Flush() {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (w *writeCounter) Flush() {
|
||||||
|
if f, ok := w.ResponseWriter.(http.Flusher); ok {
|
||||||
|
f.Flush()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func (w *writeCounter) Write(p []byte) (int, error) {
|
func (w *writeCounter) Write(p []byte) (int, error) {
|
||||||
n, err := w.ResponseWriter.Write(p)
|
n, err := w.ResponseWriter.Write(p)
|
||||||
atomic.AddUint64(&w.countBytes, uint64(n))
|
atomic.AddUint64(&w.countBytes, uint64(n))
|
|
@ -1,7 +1,7 @@
|
||||||
package metrics
|
package api
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/internal/version"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/version"
|
||||||
"github.com/prometheus/client_golang/prometheus"
|
"github.com/prometheus/client_golang/prometheus"
|
||||||
)
|
)
|
||||||
|
|
|
@ -7,8 +7,8 @@ import (
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/handler"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/handler"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/layer"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
"github.com/nats-io/nats.go"
|
"github.com/nats-io/nats.go"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
|
@ -6,9 +6,9 @@ import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"sync"
|
"sync"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/container"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
|
||||||
cid "github.com/TrueCloudLab/frostfs-sdk-go/container/id"
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/ns"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/ns"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
|
|
|
@ -7,8 +7,8 @@ import (
|
||||||
"net/http"
|
"net/http"
|
||||||
"strconv"
|
"strconv"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/internal/version"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/version"
|
||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -133,6 +133,13 @@ func WriteErrorResponse(w http.ResponseWriter, reqInfo *ReqInfo, err error) int
|
||||||
return code
|
return code
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// WriteErrorResponseNoHeader writes XML encoded error to the response body.
|
||||||
|
func WriteErrorResponseNoHeader(w http.ResponseWriter, reqInfo *ReqInfo, err error) {
|
||||||
|
errorResponse := getAPIErrorResponse(reqInfo, err)
|
||||||
|
encodedErrorResponse := EncodeResponse(errorResponse)
|
||||||
|
WriteResponseBody(w, encodedErrorResponse)
|
||||||
|
}
|
||||||
|
|
||||||
// If none of the http routes match respond with appropriate errors.
|
// If none of the http routes match respond with appropriate errors.
|
||||||
func errorResponseHandler(w http.ResponseWriter, r *http.Request) {
|
func errorResponseHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
desc := fmt.Sprintf("Unknown API request at %s", r.URL.Path)
|
desc := fmt.Sprintf("Unknown API request at %s", r.URL.Path)
|
||||||
|
@ -172,6 +179,11 @@ func WriteResponse(w http.ResponseWriter, statusCode int, response []byte, mType
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
WriteResponseBody(w, response)
|
||||||
|
}
|
||||||
|
|
||||||
|
// WriteResponseBody writes response into w.
|
||||||
|
func WriteResponseBody(w http.ResponseWriter, response []byte) {
|
||||||
_, _ = w.Write(response)
|
_, _ = w.Write(response)
|
||||||
if flusher, ok := w.(http.Flusher); ok {
|
if flusher, ok := w.(http.Flusher); ok {
|
||||||
flusher.Flush()
|
flusher.Flush()
|
||||||
|
@ -188,13 +200,30 @@ func EncodeResponse(response interface{}) []byte {
|
||||||
return bytesBuffer.Bytes()
|
return bytesBuffer.Bytes()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// EncodeResponseNoHeader encodes response without setting xml.Header.
|
||||||
|
// Should be used with periodicXMLWriter which sends xml.Header to the client
|
||||||
|
// with whitespaces to keep connection alive.
|
||||||
|
func EncodeResponseNoHeader(response interface{}) []byte {
|
||||||
|
var bytesBuffer bytes.Buffer
|
||||||
|
_ = xml.NewEncoder(&bytesBuffer).Encode(response)
|
||||||
|
return bytesBuffer.Bytes()
|
||||||
|
}
|
||||||
|
|
||||||
// EncodeToResponse encodes the response into ResponseWriter.
|
// EncodeToResponse encodes the response into ResponseWriter.
|
||||||
func EncodeToResponse(w http.ResponseWriter, response interface{}) error {
|
func EncodeToResponse(w http.ResponseWriter, response interface{}) error {
|
||||||
w.WriteHeader(http.StatusOK)
|
w.WriteHeader(http.StatusOK)
|
||||||
|
|
||||||
if _, err := w.Write(xmlHeader); err != nil {
|
if _, err := w.Write(xmlHeader); err != nil {
|
||||||
return fmt.Errorf("write headers: %w", err)
|
return fmt.Errorf("write headers: %w", err)
|
||||||
} else if err = xml.NewEncoder(w).Encode(response); err != nil {
|
}
|
||||||
|
|
||||||
|
return EncodeToResponseNoHeader(w, response)
|
||||||
|
}
|
||||||
|
|
||||||
|
// EncodeToResponseNoHeader encodes the response into ResponseWriter without
|
||||||
|
// header status.
|
||||||
|
func EncodeToResponseNoHeader(w http.ResponseWriter, response interface{}) error {
|
||||||
|
if err := xml.NewEncoder(w).Encode(response); err != nil {
|
||||||
return fmt.Errorf("encode xml response: %w", err)
|
return fmt.Errorf("encode xml response: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
280
api/router.go
280
api/router.go
|
@ -5,8 +5,8 @@ import (
|
||||||
"net/http"
|
"net/http"
|
||||||
"sync"
|
"sync"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/auth"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/metrics"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
"github.com/gorilla/mux"
|
"github.com/gorilla/mux"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
|
@ -82,6 +82,8 @@ type (
|
||||||
AbortMultipartUploadHandler(http.ResponseWriter, *http.Request)
|
AbortMultipartUploadHandler(http.ResponseWriter, *http.Request)
|
||||||
ListPartsHandler(w http.ResponseWriter, r *http.Request)
|
ListPartsHandler(w http.ResponseWriter, r *http.Request)
|
||||||
ListMultipartUploadsHandler(http.ResponseWriter, *http.Request)
|
ListMultipartUploadsHandler(http.ResponseWriter, *http.Request)
|
||||||
|
|
||||||
|
ResolveBucket(ctx context.Context, bucket string) (*data.BucketInfo, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
// mimeType represents various MIME types used in API responses.
|
// mimeType represents various MIME types used in API responses.
|
||||||
|
@ -106,7 +108,7 @@ const (
|
||||||
MimeXML mimeType = "application/xml"
|
MimeXML mimeType = "application/xml"
|
||||||
)
|
)
|
||||||
|
|
||||||
var _ = logErrorResponse
|
var _ = logSuccessResponse
|
||||||
|
|
||||||
func (lrw *logResponseWriter) WriteHeader(code int) {
|
func (lrw *logResponseWriter) WriteHeader(code int) {
|
||||||
lrw.Do(func() {
|
lrw.Do(func() {
|
||||||
|
@ -115,6 +117,12 @@ func (lrw *logResponseWriter) WriteHeader(code int) {
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (lrw *logResponseWriter) Flush() {
|
||||||
|
if f, ok := lrw.ResponseWriter.(http.Flusher); ok {
|
||||||
|
f.Flush()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func setRequestID(h http.Handler) http.Handler {
|
func setRequestID(h http.Handler) http.Handler {
|
||||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
// generate random UUIDv4
|
// generate random UUIDv4
|
||||||
|
@ -145,7 +153,37 @@ func appendCORS(handler Handler) mux.MiddlewareFunc {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func logErrorResponse(l *zap.Logger) mux.MiddlewareFunc {
|
// BucketResolveFunc is a func to resolve bucket info by name.
|
||||||
|
type BucketResolveFunc func(ctx context.Context, bucket string) (*data.BucketInfo, error)
|
||||||
|
|
||||||
|
// metricsMiddleware wraps http handler for api with basic statistics collection.
|
||||||
|
func metricsMiddleware(log *zap.Logger, resolveBucket BucketResolveFunc, usersStat UsersStat) mux.MiddlewareFunc {
|
||||||
|
return func(h http.Handler) http.Handler {
|
||||||
|
return Stats(h.ServeHTTP, resolveCID(log, resolveBucket), usersStat)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// resolveCID forms CIDResolveFunc using BucketResolveFunc.
|
||||||
|
func resolveCID(log *zap.Logger, resolveBucket BucketResolveFunc) CIDResolveFunc {
|
||||||
|
return func(ctx context.Context, reqInfo *ReqInfo) (cnrID string) {
|
||||||
|
if reqInfo.BucketName == "" || reqInfo.API == "CreateBucket" || reqInfo.API == "" {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
bktInfo, err := resolveBucket(ctx, reqInfo.BucketName)
|
||||||
|
if err != nil {
|
||||||
|
log.Debug("failed to resolve CID",
|
||||||
|
zap.String("request_id", reqInfo.RequestID), zap.String("method", reqInfo.API),
|
||||||
|
zap.String("bucket", reqInfo.BucketName), zap.String("object", reqInfo.ObjectName),
|
||||||
|
zap.Error(err))
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
return bktInfo.CID.EncodeToString()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func logSuccessResponse(l *zap.Logger) mux.MiddlewareFunc {
|
||||||
return func(h http.Handler) http.Handler {
|
return func(h http.Handler) http.Handler {
|
||||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
lw := &logResponseWriter{ResponseWriter: w}
|
lw := &logResponseWriter{ResponseWriter: w}
|
||||||
|
@ -183,21 +221,49 @@ func GetRequestID(v interface{}) string {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func setErrorAPI(apiName string, h http.Handler) http.Handler {
|
||||||
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
ctx := SetReqInfo(r.Context(), &ReqInfo{API: apiName})
|
||||||
|
h.ServeHTTP(w, r.WithContext(ctx))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// attachErrorHandler set NotFoundHandler and MethodNotAllowedHandler for mux.Router.
|
||||||
|
func attachErrorHandler(api *mux.Router, log *zap.Logger, h Handler, center auth.Center, usersStat UsersStat) {
|
||||||
|
middlewares := []mux.MiddlewareFunc{
|
||||||
|
AuthMiddleware(log, center),
|
||||||
|
metricsMiddleware(log, h.ResolveBucket, usersStat),
|
||||||
|
}
|
||||||
|
|
||||||
|
var errorHandler http.Handler = http.HandlerFunc(errorResponseHandler)
|
||||||
|
for i := len(middlewares) - 1; i >= 0; i-- {
|
||||||
|
errorHandler = middlewares[i](errorHandler)
|
||||||
|
}
|
||||||
|
|
||||||
|
// If none of the routes match, add default error handler routes
|
||||||
|
api.NotFoundHandler = setErrorAPI("NotFound", errorHandler)
|
||||||
|
api.MethodNotAllowedHandler = setErrorAPI("MethodNotAllowed", errorHandler)
|
||||||
|
}
|
||||||
|
|
||||||
// Attach adds S3 API handlers from h to r for domains with m client limit using
|
// Attach adds S3 API handlers from h to r for domains with m client limit using
|
||||||
// center authentication and log logger.
|
// center authentication and log logger.
|
||||||
func Attach(r *mux.Router, domains []string, m MaxClients, h Handler, center auth.Center, log *zap.Logger) {
|
func Attach(r *mux.Router, domains []string, m MaxClients, h Handler, center auth.Center, log *zap.Logger, usersStat UsersStat) {
|
||||||
api := r.PathPrefix(SlashSeparator).Subrouter()
|
api := r.PathPrefix(SlashSeparator).Subrouter()
|
||||||
|
|
||||||
api.Use(
|
api.Use(
|
||||||
// -- prepare request
|
// -- prepare request
|
||||||
setRequestID,
|
setRequestID,
|
||||||
|
|
||||||
|
// Attach user authentication for all S3 routes.
|
||||||
|
AuthMiddleware(log, center),
|
||||||
|
|
||||||
|
metricsMiddleware(log, h.ResolveBucket, usersStat),
|
||||||
|
|
||||||
// -- logging error requests
|
// -- logging error requests
|
||||||
logErrorResponse(log),
|
logSuccessResponse(log),
|
||||||
)
|
)
|
||||||
|
|
||||||
// Attach user authentication for all S3 routes.
|
attachErrorHandler(api, log, h, center, usersStat)
|
||||||
AttachUserAuth(api, center, log)
|
|
||||||
|
|
||||||
buckets := make([]*mux.Router, 0, len(domains)+1)
|
buckets := make([]*mux.Router, 0, len(domains)+1)
|
||||||
buckets = append(buckets, api.PathPrefix("/{bucket}").Subrouter())
|
buckets = append(buckets, api.PathPrefix("/{bucket}").Subrouter())
|
||||||
|
@ -213,277 +279,327 @@ func Attach(r *mux.Router, domains []string, m MaxClients, h Handler, center aut
|
||||||
// -- append CORS headers to a response for
|
// -- append CORS headers to a response for
|
||||||
appendCORS(h),
|
appendCORS(h),
|
||||||
)
|
)
|
||||||
bucket.Methods(http.MethodOptions).HandlerFunc(m.Handle(metrics.APIStats("preflight", h.Preflight))).Name("Options")
|
bucket.Methods(http.MethodOptions).HandlerFunc(
|
||||||
|
m.Handle(h.Preflight)).
|
||||||
|
Name("Options")
|
||||||
bucket.Methods(http.MethodHead).Path("/{object:.+}").HandlerFunc(
|
bucket.Methods(http.MethodHead).Path("/{object:.+}").HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("headobject", h.HeadObjectHandler))).Name("HeadObject")
|
m.Handle(h.HeadObjectHandler)).
|
||||||
|
Name("HeadObject")
|
||||||
// CopyObjectPart
|
// CopyObjectPart
|
||||||
bucket.Methods(http.MethodPut).Path("/{object:.+}").Headers(hdrAmzCopySource, "").HandlerFunc(m.Handle(metrics.APIStats("uploadpartcopy", h.UploadPartCopy))).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}").
|
bucket.Methods(http.MethodPut).Path("/{object:.+}").Headers(hdrAmzCopySource, "").HandlerFunc(
|
||||||
|
m.Handle(h.UploadPartCopy)).
|
||||||
|
Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}").
|
||||||
Name("UploadPartCopy")
|
Name("UploadPartCopy")
|
||||||
// PutObjectPart
|
// PutObjectPart
|
||||||
bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
|
bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("uploadpart", h.UploadPartHandler))).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}").
|
m.Handle(h.UploadPartHandler)).
|
||||||
|
Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}").
|
||||||
Name("UploadPart")
|
Name("UploadPart")
|
||||||
// ListParts
|
// ListParts
|
||||||
bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
|
bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("listobjectparts", h.ListPartsHandler))).Queries("uploadId", "{uploadId:.*}").
|
m.Handle(h.ListPartsHandler)).
|
||||||
|
Queries("uploadId", "{uploadId:.*}").
|
||||||
Name("ListObjectParts")
|
Name("ListObjectParts")
|
||||||
// CompleteMultipartUpload
|
// CompleteMultipartUpload
|
||||||
bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(
|
bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("completemutipartupload", h.CompleteMultipartUploadHandler))).Queries("uploadId", "{uploadId:.*}").
|
m.Handle(h.CompleteMultipartUploadHandler)).
|
||||||
|
Queries("uploadId", "{uploadId:.*}").
|
||||||
Name("CompleteMultipartUpload")
|
Name("CompleteMultipartUpload")
|
||||||
// CreateMultipartUpload
|
// CreateMultipartUpload
|
||||||
bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(
|
bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("createmultipartupload", h.CreateMultipartUploadHandler))).Queries("uploads", "").
|
m.Handle(h.CreateMultipartUploadHandler)).
|
||||||
|
Queries("uploads", "").
|
||||||
Name("CreateMultipartUpload")
|
Name("CreateMultipartUpload")
|
||||||
// AbortMultipartUpload
|
// AbortMultipartUpload
|
||||||
bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(
|
bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("abortmultipartupload", h.AbortMultipartUploadHandler))).Queries("uploadId", "{uploadId:.*}").
|
m.Handle(h.AbortMultipartUploadHandler)).
|
||||||
|
Queries("uploadId", "{uploadId:.*}").
|
||||||
Name("AbortMultipartUpload")
|
Name("AbortMultipartUpload")
|
||||||
// ListMultipartUploads
|
// ListMultipartUploads
|
||||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("listmultipartuploads", h.ListMultipartUploadsHandler))).Queries("uploads", "").
|
m.Handle(h.ListMultipartUploadsHandler)).
|
||||||
|
Queries("uploads", "").
|
||||||
Name("ListMultipartUploads")
|
Name("ListMultipartUploads")
|
||||||
// GetObjectACL -- this is a dummy call.
|
// GetObjectACL -- this is a dummy call.
|
||||||
bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
|
bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("getobjectacl", h.GetObjectACLHandler))).Queries("acl", "").
|
m.Handle(h.GetObjectACLHandler)).
|
||||||
|
Queries("acl", "").
|
||||||
Name("GetObjectACL")
|
Name("GetObjectACL")
|
||||||
// PutObjectACL -- this is a dummy call.
|
// PutObjectACL -- this is a dummy call.
|
||||||
bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
|
bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("putobjectacl", h.PutObjectACLHandler))).Queries("acl", "").
|
m.Handle(h.PutObjectACLHandler)).
|
||||||
|
Queries("acl", "").
|
||||||
Name("PutObjectACL")
|
Name("PutObjectACL")
|
||||||
// GetObjectTagging
|
// GetObjectTagging
|
||||||
bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
|
bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("getobjecttagging", h.GetObjectTaggingHandler))).Queries("tagging", "").
|
m.Handle(h.GetObjectTaggingHandler)).
|
||||||
|
Queries("tagging", "").
|
||||||
Name("GetObjectTagging")
|
Name("GetObjectTagging")
|
||||||
// PutObjectTagging
|
// PutObjectTagging
|
||||||
bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
|
bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("putobjecttagging", h.PutObjectTaggingHandler))).Queries("tagging", "").
|
m.Handle(h.PutObjectTaggingHandler)).
|
||||||
|
Queries("tagging", "").
|
||||||
Name("PutObjectTagging")
|
Name("PutObjectTagging")
|
||||||
// DeleteObjectTagging
|
// DeleteObjectTagging
|
||||||
bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(
|
bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("deleteobjecttagging", h.DeleteObjectTaggingHandler))).Queries("tagging", "").
|
m.Handle(h.DeleteObjectTaggingHandler)).
|
||||||
|
Queries("tagging", "").
|
||||||
Name("DeleteObjectTagging")
|
Name("DeleteObjectTagging")
|
||||||
// SelectObjectContent
|
// SelectObjectContent
|
||||||
bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(
|
bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("selectobjectcontent", h.SelectObjectContentHandler))).Queries("select", "").Queries("select-type", "2").
|
m.Handle(h.SelectObjectContentHandler)).
|
||||||
|
Queries("select", "").Queries("select-type", "2").
|
||||||
Name("SelectObjectContent")
|
Name("SelectObjectContent")
|
||||||
// GetObjectRetention
|
// GetObjectRetention
|
||||||
bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
|
bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("getobjectretention", h.GetObjectRetentionHandler))).Queries("retention", "").
|
m.Handle(h.GetObjectRetentionHandler)).
|
||||||
|
Queries("retention", "").
|
||||||
Name("GetObjectRetention")
|
Name("GetObjectRetention")
|
||||||
// GetObjectLegalHold
|
// GetObjectLegalHold
|
||||||
bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
|
bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("getobjectlegalhold", h.GetObjectLegalHoldHandler))).Queries("legal-hold", "").
|
m.Handle(h.GetObjectLegalHoldHandler)).
|
||||||
|
Queries("legal-hold", "").
|
||||||
Name("GetObjectLegalHold")
|
Name("GetObjectLegalHold")
|
||||||
// GetObjectAttributes
|
// GetObjectAttributes
|
||||||
bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
|
bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("getobjectattributes", h.GetObjectAttributesHandler))).Queries("attributes", "").
|
m.Handle(h.GetObjectAttributesHandler)).
|
||||||
|
Queries("attributes", "").
|
||||||
Name("GetObjectAttributes")
|
Name("GetObjectAttributes")
|
||||||
// GetObject
|
// GetObject
|
||||||
bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
|
bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("getobject", h.GetObjectHandler))).
|
m.Handle(h.GetObjectHandler)).
|
||||||
Name("GetObject")
|
Name("GetObject")
|
||||||
// CopyObject
|
// CopyObject
|
||||||
bucket.Methods(http.MethodPut).Path("/{object:.+}").Headers(hdrAmzCopySource, "").HandlerFunc(m.Handle(metrics.APIStats("copyobject", h.CopyObjectHandler))).
|
bucket.Methods(http.MethodPut).Path("/{object:.+}").Headers(hdrAmzCopySource, "").HandlerFunc(
|
||||||
|
m.Handle(h.CopyObjectHandler)).
|
||||||
Name("CopyObject")
|
Name("CopyObject")
|
||||||
// PutObjectRetention
|
// PutObjectRetention
|
||||||
bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
|
bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("putobjectretention", h.PutObjectRetentionHandler))).Queries("retention", "").
|
m.Handle(h.PutObjectRetentionHandler)).
|
||||||
|
Queries("retention", "").
|
||||||
Name("PutObjectRetention")
|
Name("PutObjectRetention")
|
||||||
// PutObjectLegalHold
|
// PutObjectLegalHold
|
||||||
bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
|
bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("putobjectlegalhold", h.PutObjectLegalHoldHandler))).Queries("legal-hold", "").
|
m.Handle(h.PutObjectLegalHoldHandler)).
|
||||||
|
Queries("legal-hold", "").
|
||||||
Name("PutObjectLegalHold")
|
Name("PutObjectLegalHold")
|
||||||
|
|
||||||
// PutObject
|
// PutObject
|
||||||
bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
|
bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("putobject", h.PutObjectHandler))).
|
m.Handle(h.PutObjectHandler)).
|
||||||
Name("PutObject")
|
Name("PutObject")
|
||||||
// DeleteObject
|
// DeleteObject
|
||||||
bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(
|
bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("deleteobject", h.DeleteObjectHandler))).
|
m.Handle(h.DeleteObjectHandler)).
|
||||||
Name("DeleteObject")
|
Name("DeleteObject")
|
||||||
|
|
||||||
// Bucket operations
|
// Bucket operations
|
||||||
// GetBucketLocation
|
// GetBucketLocation
|
||||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("getbucketlocation", h.GetBucketLocationHandler))).Queries("location", "").
|
m.Handle(h.GetBucketLocationHandler)).
|
||||||
|
Queries("location", "").
|
||||||
Name("GetBucketLocation")
|
Name("GetBucketLocation")
|
||||||
// GetBucketPolicy
|
// GetBucketPolicy
|
||||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("getbucketpolicy", h.GetBucketPolicyHandler))).Queries("policy", "").
|
m.Handle(h.GetBucketPolicyHandler)).
|
||||||
|
Queries("policy", "").
|
||||||
Name("GetBucketPolicy")
|
Name("GetBucketPolicy")
|
||||||
// GetBucketLifecycle
|
// GetBucketLifecycle
|
||||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("getbucketlifecycle", h.GetBucketLifecycleHandler))).Queries("lifecycle", "").
|
m.Handle(h.GetBucketLifecycleHandler)).
|
||||||
|
Queries("lifecycle", "").
|
||||||
Name("GetBucketLifecycle")
|
Name("GetBucketLifecycle")
|
||||||
// GetBucketEncryption
|
// GetBucketEncryption
|
||||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("getbucketencryption", h.GetBucketEncryptionHandler))).Queries("encryption", "").
|
m.Handle(h.GetBucketEncryptionHandler)).
|
||||||
|
Queries("encryption", "").
|
||||||
Name("GetBucketEncryption")
|
Name("GetBucketEncryption")
|
||||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("getbucketcors", h.GetBucketCorsHandler))).Queries("cors", "").
|
m.Handle(h.GetBucketCorsHandler)).
|
||||||
|
Queries("cors", "").
|
||||||
Name("GetBucketCors")
|
Name("GetBucketCors")
|
||||||
bucket.Methods(http.MethodPut).HandlerFunc(
|
bucket.Methods(http.MethodPut).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("putbucketcors", h.PutBucketCorsHandler))).Queries("cors", "").
|
m.Handle(h.PutBucketCorsHandler)).
|
||||||
|
Queries("cors", "").
|
||||||
Name("PutBucketCors")
|
Name("PutBucketCors")
|
||||||
bucket.Methods(http.MethodDelete).HandlerFunc(
|
bucket.Methods(http.MethodDelete).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("deletebucketcors", h.DeleteBucketCorsHandler))).Queries("cors", "").
|
m.Handle(h.DeleteBucketCorsHandler)).
|
||||||
|
Queries("cors", "").
|
||||||
Name("DeleteBucketCors")
|
Name("DeleteBucketCors")
|
||||||
// Dummy Bucket Calls
|
// Dummy Bucket Calls
|
||||||
// GetBucketACL -- this is a dummy call.
|
// GetBucketACL -- this is a dummy call.
|
||||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("getbucketacl", h.GetBucketACLHandler))).Queries("acl", "").
|
m.Handle(h.GetBucketACLHandler)).
|
||||||
|
Queries("acl", "").
|
||||||
Name("GetBucketACL")
|
Name("GetBucketACL")
|
||||||
// PutBucketACL -- this is a dummy call.
|
// PutBucketACL -- this is a dummy call.
|
||||||
bucket.Methods(http.MethodPut).HandlerFunc(
|
bucket.Methods(http.MethodPut).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("putbucketacl", h.PutBucketACLHandler))).Queries("acl", "").
|
m.Handle(h.PutBucketACLHandler)).
|
||||||
|
Queries("acl", "").
|
||||||
Name("PutBucketACL")
|
Name("PutBucketACL")
|
||||||
// GetBucketWebsiteHandler -- this is a dummy call.
|
// GetBucketWebsiteHandler -- this is a dummy call.
|
||||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("getbucketwebsite", h.GetBucketWebsiteHandler))).Queries("website", "").
|
m.Handle(h.GetBucketWebsiteHandler)).
|
||||||
|
Queries("website", "").
|
||||||
Name("GetBucketWebsite")
|
Name("GetBucketWebsite")
|
||||||
// GetBucketAccelerateHandler -- this is a dummy call.
|
// GetBucketAccelerateHandler -- this is a dummy call.
|
||||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("getbucketaccelerate", h.GetBucketAccelerateHandler))).Queries("accelerate", "").
|
m.Handle(h.GetBucketAccelerateHandler)).
|
||||||
|
Queries("accelerate", "").
|
||||||
Name("GetBucketAccelerate")
|
Name("GetBucketAccelerate")
|
||||||
// GetBucketRequestPaymentHandler -- this is a dummy call.
|
// GetBucketRequestPaymentHandler -- this is a dummy call.
|
||||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("getbucketrequestpayment", h.GetBucketRequestPaymentHandler))).Queries("requestPayment", "").
|
m.Handle(h.GetBucketRequestPaymentHandler)).
|
||||||
|
Queries("requestPayment", "").
|
||||||
Name("GetBucketRequestPayment")
|
Name("GetBucketRequestPayment")
|
||||||
// GetBucketLoggingHandler -- this is a dummy call.
|
// GetBucketLoggingHandler -- this is a dummy call.
|
||||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("getbucketlogging", h.GetBucketLoggingHandler))).Queries("logging", "").
|
m.Handle(h.GetBucketLoggingHandler)).
|
||||||
|
Queries("logging", "").
|
||||||
Name("GetBucketLogging")
|
Name("GetBucketLogging")
|
||||||
// GetBucketLifecycleHandler -- this is a dummy call.
|
|
||||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
|
||||||
m.Handle(metrics.APIStats("getbucketlifecycle", h.GetBucketLifecycleHandler))).Queries("lifecycle", "").
|
|
||||||
Name("GetBucketLifecycle")
|
|
||||||
// GetBucketReplicationHandler -- this is a dummy call.
|
// GetBucketReplicationHandler -- this is a dummy call.
|
||||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("getbucketreplication", h.GetBucketReplicationHandler))).Queries("replication", "").
|
m.Handle(h.GetBucketReplicationHandler)).
|
||||||
|
Queries("replication", "").
|
||||||
Name("GetBucketReplication")
|
Name("GetBucketReplication")
|
||||||
// GetBucketTaggingHandler
|
// GetBucketTaggingHandler
|
||||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("getbuckettagging", h.GetBucketTaggingHandler))).Queries("tagging", "").
|
m.Handle(h.GetBucketTaggingHandler)).
|
||||||
|
Queries("tagging", "").
|
||||||
Name("GetBucketTagging")
|
Name("GetBucketTagging")
|
||||||
// DeleteBucketWebsiteHandler
|
// DeleteBucketWebsiteHandler
|
||||||
bucket.Methods(http.MethodDelete).HandlerFunc(
|
bucket.Methods(http.MethodDelete).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("deletebucketwebsite", h.DeleteBucketWebsiteHandler))).Queries("website", "").
|
m.Handle(h.DeleteBucketWebsiteHandler)).
|
||||||
|
Queries("website", "").
|
||||||
Name("DeleteBucketWebsite")
|
Name("DeleteBucketWebsite")
|
||||||
// DeleteBucketTaggingHandler
|
// DeleteBucketTaggingHandler
|
||||||
bucket.Methods(http.MethodDelete).HandlerFunc(
|
bucket.Methods(http.MethodDelete).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("deletebuckettagging", h.DeleteBucketTaggingHandler))).Queries("tagging", "").
|
m.Handle(h.DeleteBucketTaggingHandler)).
|
||||||
|
Queries("tagging", "").
|
||||||
Name("DeleteBucketTagging")
|
Name("DeleteBucketTagging")
|
||||||
|
|
||||||
// GetBucketObjectLockConfig
|
// GetBucketObjectLockConfig
|
||||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("getbucketobjectlockconfiguration", h.GetBucketObjectLockConfigHandler))).Queries("object-lock", "").
|
m.Handle(h.GetBucketObjectLockConfigHandler)).
|
||||||
|
Queries("object-lock", "").
|
||||||
Name("GetBucketObjectLockConfig")
|
Name("GetBucketObjectLockConfig")
|
||||||
// GetBucketVersioning
|
// GetBucketVersioning
|
||||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("getbucketversioning", h.GetBucketVersioningHandler))).Queries("versioning", "").
|
m.Handle(h.GetBucketVersioningHandler)).
|
||||||
|
Queries("versioning", "").
|
||||||
Name("GetBucketVersioning")
|
Name("GetBucketVersioning")
|
||||||
// GetBucketNotification
|
// GetBucketNotification
|
||||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("getbucketnotification", h.GetBucketNotificationHandler))).Queries("notification", "").
|
m.Handle(h.GetBucketNotificationHandler)).
|
||||||
|
Queries("notification", "").
|
||||||
Name("GetBucketNotification")
|
Name("GetBucketNotification")
|
||||||
// ListenBucketNotification
|
// ListenBucketNotification
|
||||||
bucket.Methods(http.MethodGet).HandlerFunc(metrics.APIStats("listenbucketnotification", h.ListenBucketNotificationHandler)).Queries("events", "{events:.*}").
|
bucket.Methods(http.MethodGet).HandlerFunc(h.ListenBucketNotificationHandler).
|
||||||
|
Queries("events", "{events:.*}").
|
||||||
Name("ListenBucketNotification")
|
Name("ListenBucketNotification")
|
||||||
// ListObjectsV2M
|
// ListObjectsV2M
|
||||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("listobjectsv2M", h.ListObjectsV2MHandler))).Queries("list-type", "2", "metadata", "true").
|
m.Handle(h.ListObjectsV2MHandler)).
|
||||||
|
Queries("list-type", "2", "metadata", "true").
|
||||||
Name("ListObjectsV2M")
|
Name("ListObjectsV2M")
|
||||||
// ListObjectsV2
|
// ListObjectsV2
|
||||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("listobjectsv2", h.ListObjectsV2Handler))).Queries("list-type", "2").
|
m.Handle(h.ListObjectsV2Handler)).
|
||||||
|
Queries("list-type", "2").
|
||||||
Name("ListObjectsV2")
|
Name("ListObjectsV2")
|
||||||
// ListBucketVersions
|
// ListBucketVersions
|
||||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("listbucketversions", h.ListBucketObjectVersionsHandler))).Queries("versions", "").
|
m.Handle(h.ListBucketObjectVersionsHandler)).
|
||||||
|
Queries("versions", "").
|
||||||
Name("ListBucketVersions")
|
Name("ListBucketVersions")
|
||||||
// ListObjectsV1 (Legacy)
|
// ListObjectsV1 (Legacy)
|
||||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("listobjectsv1", h.ListObjectsV1Handler))).
|
m.Handle(h.ListObjectsV1Handler)).
|
||||||
Name("ListObjectsV1")
|
Name("ListObjectsV1")
|
||||||
// PutBucketLifecycle
|
// PutBucketLifecycle
|
||||||
bucket.Methods(http.MethodPut).HandlerFunc(
|
bucket.Methods(http.MethodPut).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("putbucketlifecycle", h.PutBucketLifecycleHandler))).Queries("lifecycle", "").
|
m.Handle(h.PutBucketLifecycleHandler)).
|
||||||
|
Queries("lifecycle", "").
|
||||||
Name("PutBucketLifecycle")
|
Name("PutBucketLifecycle")
|
||||||
// PutBucketEncryption
|
// PutBucketEncryption
|
||||||
bucket.Methods(http.MethodPut).HandlerFunc(
|
bucket.Methods(http.MethodPut).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("putbucketencryption", h.PutBucketEncryptionHandler))).Queries("encryption", "").
|
m.Handle(h.PutBucketEncryptionHandler)).
|
||||||
|
Queries("encryption", "").
|
||||||
Name("PutBucketEncryption")
|
Name("PutBucketEncryption")
|
||||||
|
|
||||||
// PutBucketPolicy
|
// PutBucketPolicy
|
||||||
bucket.Methods(http.MethodPut).HandlerFunc(
|
bucket.Methods(http.MethodPut).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("putbucketpolicy", h.PutBucketPolicyHandler))).Queries("policy", "").
|
m.Handle(h.PutBucketPolicyHandler)).
|
||||||
|
Queries("policy", "").
|
||||||
Name("PutBucketPolicy")
|
Name("PutBucketPolicy")
|
||||||
|
|
||||||
// PutBucketObjectLockConfig
|
// PutBucketObjectLockConfig
|
||||||
bucket.Methods(http.MethodPut).HandlerFunc(
|
bucket.Methods(http.MethodPut).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("putbucketobjectlockconfig", h.PutBucketObjectLockConfigHandler))).Queries("object-lock", "").
|
m.Handle(h.PutBucketObjectLockConfigHandler)).
|
||||||
|
Queries("object-lock", "").
|
||||||
Name("PutBucketObjectLockConfig")
|
Name("PutBucketObjectLockConfig")
|
||||||
// PutBucketTaggingHandler
|
// PutBucketTaggingHandler
|
||||||
bucket.Methods(http.MethodPut).HandlerFunc(
|
bucket.Methods(http.MethodPut).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("putbuckettagging", h.PutBucketTaggingHandler))).Queries("tagging", "").
|
m.Handle(h.PutBucketTaggingHandler)).
|
||||||
|
Queries("tagging", "").
|
||||||
Name("PutBucketTagging")
|
Name("PutBucketTagging")
|
||||||
// PutBucketVersioning
|
// PutBucketVersioning
|
||||||
bucket.Methods(http.MethodPut).HandlerFunc(
|
bucket.Methods(http.MethodPut).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("putbucketversioning", h.PutBucketVersioningHandler))).Queries("versioning", "").
|
m.Handle(h.PutBucketVersioningHandler)).
|
||||||
|
Queries("versioning", "").
|
||||||
Name("PutBucketVersioning")
|
Name("PutBucketVersioning")
|
||||||
// PutBucketNotification
|
// PutBucketNotification
|
||||||
bucket.Methods(http.MethodPut).HandlerFunc(
|
bucket.Methods(http.MethodPut).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("putbucketnotification", h.PutBucketNotificationHandler))).Queries("notification", "").
|
m.Handle(h.PutBucketNotificationHandler)).
|
||||||
|
Queries("notification", "").
|
||||||
Name("PutBucketNotification")
|
Name("PutBucketNotification")
|
||||||
// CreateBucket
|
// CreateBucket
|
||||||
bucket.Methods(http.MethodPut).HandlerFunc(
|
bucket.Methods(http.MethodPut).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("createbucket", h.CreateBucketHandler))).
|
m.Handle(h.CreateBucketHandler)).
|
||||||
Name("CreateBucket")
|
Name("CreateBucket")
|
||||||
// HeadBucket
|
// HeadBucket
|
||||||
bucket.Methods(http.MethodHead).HandlerFunc(
|
bucket.Methods(http.MethodHead).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("headbucket", h.HeadBucketHandler))).
|
m.Handle(h.HeadBucketHandler)).
|
||||||
Name("HeadBucket")
|
Name("HeadBucket")
|
||||||
// PostPolicy
|
// PostPolicy
|
||||||
bucket.Methods(http.MethodPost).HeadersRegexp(hdrContentType, "multipart/form-data*").HandlerFunc(
|
bucket.Methods(http.MethodPost).HeadersRegexp(hdrContentType, "multipart/form-data*").HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("postobject", h.PostObject))).
|
m.Handle(h.PostObject)).
|
||||||
Name("PostObject")
|
Name("PostObject")
|
||||||
// DeleteMultipleObjects
|
// DeleteMultipleObjects
|
||||||
bucket.Methods(http.MethodPost).HandlerFunc(
|
bucket.Methods(http.MethodPost).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("deletemultipleobjects", h.DeleteMultipleObjectsHandler))).Queries("delete", "").
|
m.Handle(h.DeleteMultipleObjectsHandler)).
|
||||||
|
Queries("delete", "").
|
||||||
Name("DeleteMultipleObjects")
|
Name("DeleteMultipleObjects")
|
||||||
// DeleteBucketPolicy
|
// DeleteBucketPolicy
|
||||||
bucket.Methods(http.MethodDelete).HandlerFunc(
|
bucket.Methods(http.MethodDelete).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("deletebucketpolicy", h.DeleteBucketPolicyHandler))).Queries("policy", "").
|
m.Handle(h.DeleteBucketPolicyHandler)).
|
||||||
|
Queries("policy", "").
|
||||||
Name("DeleteBucketPolicy")
|
Name("DeleteBucketPolicy")
|
||||||
// DeleteBucketLifecycle
|
// DeleteBucketLifecycle
|
||||||
bucket.Methods(http.MethodDelete).HandlerFunc(
|
bucket.Methods(http.MethodDelete).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("deletebucketlifecycle", h.DeleteBucketLifecycleHandler))).Queries("lifecycle", "").
|
m.Handle(h.DeleteBucketLifecycleHandler)).
|
||||||
|
Queries("lifecycle", "").
|
||||||
Name("DeleteBucketLifecycle")
|
Name("DeleteBucketLifecycle")
|
||||||
// DeleteBucketEncryption
|
// DeleteBucketEncryption
|
||||||
bucket.Methods(http.MethodDelete).HandlerFunc(
|
bucket.Methods(http.MethodDelete).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("deletebucketencryption", h.DeleteBucketEncryptionHandler))).Queries("encryption", "").
|
m.Handle(h.DeleteBucketEncryptionHandler)).
|
||||||
|
Queries("encryption", "").
|
||||||
Name("DeleteBucketEncryption")
|
Name("DeleteBucketEncryption")
|
||||||
// DeleteBucket
|
// DeleteBucket
|
||||||
bucket.Methods(http.MethodDelete).HandlerFunc(
|
bucket.Methods(http.MethodDelete).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("deletebucket", h.DeleteBucketHandler))).
|
m.Handle(h.DeleteBucketHandler)).
|
||||||
Name("DeleteBucket")
|
Name("DeleteBucket")
|
||||||
}
|
}
|
||||||
// Root operation
|
// Root operation
|
||||||
|
|
||||||
// ListBuckets
|
// ListBuckets
|
||||||
api.Methods(http.MethodGet).Path(SlashSeparator).HandlerFunc(
|
api.Methods(http.MethodGet).Path(SlashSeparator).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("listbuckets", h.ListBucketsHandler))).
|
m.Handle(h.ListBucketsHandler)).
|
||||||
Name("ListBuckets")
|
Name("ListBuckets")
|
||||||
|
|
||||||
// S3 browser with signature v4 adds '//' for ListBuckets request, so rather
|
// S3 browser with signature v4 adds '//' for ListBuckets request, so rather
|
||||||
// than failing with UnknownAPIRequest we simply handle it for now.
|
// than failing with UnknownAPIRequest we simply handle it for now.
|
||||||
api.Methods(http.MethodGet).Path(SlashSeparator + SlashSeparator).HandlerFunc(
|
api.Methods(http.MethodGet).Path(SlashSeparator + SlashSeparator).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("listbuckets", h.ListBucketsHandler))).
|
m.Handle(h.ListBucketsHandler)).
|
||||||
Name("ListBuckets")
|
Name("ListBuckets")
|
||||||
|
|
||||||
// If none of the routes match, add default error handler routes
|
|
||||||
api.NotFoundHandler = metrics.APIStats("notfound", errorResponseHandler)
|
|
||||||
api.MethodNotAllowedHandler = metrics.APIStats("methodnotallowed", errorResponseHandler)
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -4,8 +4,8 @@ import (
|
||||||
"context"
|
"context"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/auth"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"github.com/gorilla/mux"
|
"github.com/gorilla/mux"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
@ -19,9 +19,9 @@ var BoxData = KeyWrapper("__context_box_key")
|
||||||
// ClientTime is an ID used to store client time.Time in a context.
|
// ClientTime is an ID used to store client time.Time in a context.
|
||||||
var ClientTime = KeyWrapper("__context_client_time")
|
var ClientTime = KeyWrapper("__context_client_time")
|
||||||
|
|
||||||
// AttachUserAuth adds user authentication via center to router using log for logging.
|
// AuthMiddleware adds user authentication via center to router using log for logging.
|
||||||
func AttachUserAuth(router *mux.Router, center auth.Center, log *zap.Logger) {
|
func AuthMiddleware(log *zap.Logger, center auth.Center) mux.MiddlewareFunc {
|
||||||
router.Use(func(h http.Handler) http.Handler {
|
return func(h http.Handler) http.Handler {
|
||||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
var ctx context.Context
|
var ctx context.Context
|
||||||
box, err := center.Authenticate(r)
|
box, err := center.Authenticate(r)
|
||||||
|
@ -46,5 +46,5 @@ func AttachUserAuth(router *mux.Router, center auth.Center, log *zap.Logger) {
|
||||||
|
|
||||||
h.ServeHTTP(w, r.WithContext(ctx))
|
h.ServeHTTP(w, r.WithContext(ctx))
|
||||||
})
|
})
|
||||||
})
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -11,17 +11,17 @@ import (
|
||||||
"os"
|
"os"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/cache"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/cache"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/creds/tokens"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/tokens"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/bearer"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
||||||
cid "github.com/TrueCloudLab/frostfs-sdk-go/container/id"
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||||
frostfsecdsa "github.com/TrueCloudLab/frostfs-sdk-go/crypto/ecdsa"
|
frostfsecdsa "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto/ecdsa"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/eacl"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/netmap"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
|
||||||
oid "github.com/TrueCloudLab/frostfs-sdk-go/object/id"
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/session"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/user"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
|
|
|
@ -4,9 +4,9 @@ import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
|
||||||
apisession "github.com/TrueCloudLab/frostfs-api-go/v2/session"
|
apisession "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session"
|
||||||
cid "github.com/TrueCloudLab/frostfs-sdk-go/container/id"
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/session"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
||||||
)
|
)
|
||||||
|
|
||||||
type (
|
type (
|
||||||
|
|
|
@ -3,7 +3,7 @@ package authmate
|
||||||
import (
|
import (
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/session"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
@ -13,13 +13,13 @@ import (
|
||||||
"syscall"
|
"syscall"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/authmate"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/authmate"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/internal/frostfs"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/internal/version"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/version"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/internal/wallet"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/wallet"
|
||||||
cid "github.com/TrueCloudLab/frostfs-sdk-go/container/id"
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/pool"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
|
||||||
"github.com/aws/aws-sdk-go/aws"
|
"github.com/aws/aws-sdk-go/aws"
|
||||||
"github.com/aws/aws-sdk-go/aws/credentials"
|
"github.com/aws/aws-sdk-go/aws/credentials"
|
||||||
"github.com/aws/aws-sdk-go/aws/session"
|
"github.com/aws/aws-sdk-go/aws/session"
|
||||||
|
|
164
cmd/s3-gw/app.go
164
cmd/s3-gw/app.go
|
@ -13,18 +13,20 @@ import (
|
||||||
"syscall"
|
"syscall"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/auth"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/cache"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/cache"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/handler"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/handler"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/layer"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/notifications"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/notifications"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/resolver"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/resolver"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/internal/frostfs"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/internal/version"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/version"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/internal/wallet"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/wallet"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/netmap"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/xml"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/pool"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/metrics"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
|
||||||
"github.com/gorilla/mux"
|
"github.com/gorilla/mux"
|
||||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||||
"github.com/spf13/viper"
|
"github.com/spf13/viper"
|
||||||
|
@ -45,7 +47,7 @@ type (
|
||||||
|
|
||||||
servers []Server
|
servers []Server
|
||||||
|
|
||||||
metrics *appMetrics
|
metrics *metrics.AppMetrics
|
||||||
bucketResolver *resolver.BucketResolver
|
bucketResolver *resolver.BucketResolver
|
||||||
services []*Service
|
services []*Service
|
||||||
settings *appSettings
|
settings *appSettings
|
||||||
|
@ -56,8 +58,9 @@ type (
|
||||||
}
|
}
|
||||||
|
|
||||||
appSettings struct {
|
appSettings struct {
|
||||||
logLevel zap.AtomicLevel
|
logLevel zap.AtomicLevel
|
||||||
policies *placementPolicy
|
policies *placementPolicy
|
||||||
|
xmlDecoder *xml.DecoderProvider
|
||||||
}
|
}
|
||||||
|
|
||||||
Logger struct {
|
Logger struct {
|
||||||
|
@ -65,18 +68,6 @@ type (
|
||||||
lvl zap.AtomicLevel
|
lvl zap.AtomicLevel
|
||||||
}
|
}
|
||||||
|
|
||||||
appMetrics struct {
|
|
||||||
logger *zap.Logger
|
|
||||||
provider GateMetricsCollector
|
|
||||||
mu sync.RWMutex
|
|
||||||
enabled bool
|
|
||||||
}
|
|
||||||
|
|
||||||
GateMetricsCollector interface {
|
|
||||||
SetHealth(int32)
|
|
||||||
Unregister()
|
|
||||||
}
|
|
||||||
|
|
||||||
placementPolicy struct {
|
placementPolicy struct {
|
||||||
mu sync.RWMutex
|
mu sync.RWMutex
|
||||||
defaultPolicy netmap.PlacementPolicy
|
defaultPolicy netmap.PlacementPolicy
|
||||||
|
@ -163,8 +154,9 @@ func newAppSettings(log *Logger, v *viper.Viper) *appSettings {
|
||||||
}
|
}
|
||||||
|
|
||||||
return &appSettings{
|
return &appSettings{
|
||||||
logLevel: log.lvl,
|
logLevel: log.lvl,
|
||||||
policies: policies,
|
policies: policies,
|
||||||
|
xmlDecoder: xml.NewDecoderProvider(v.GetBool(cfgKludgeUseDefaultXMLNSForCompleteMultipartUpload)),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -183,8 +175,7 @@ func (a *App) initAPI(ctx context.Context) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (a *App) initMetrics() {
|
func (a *App) initMetrics() {
|
||||||
gateMetricsProvider := newGateMetrics(frostfs.NewPoolStatistic(a.pool))
|
a.metrics = metrics.NewAppMetrics(a.log, frostfs.NewPoolStatistic(a.pool), a.cfg.GetBool(cfgPrometheusEnabled))
|
||||||
a.metrics = newAppMetrics(a.log, gateMetricsProvider, a.cfg.GetBool(cfgPrometheusEnabled))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (a *App) initResolver() {
|
func (a *App) initResolver() {
|
||||||
|
@ -273,6 +264,7 @@ func getPool(ctx context.Context, logger *zap.Logger, cfg *viper.Viper) (*pool.P
|
||||||
errorThreshold = defaultPoolErrorThreshold
|
errorThreshold = defaultPoolErrorThreshold
|
||||||
}
|
}
|
||||||
prm.SetErrorThreshold(errorThreshold)
|
prm.SetErrorThreshold(errorThreshold)
|
||||||
|
prm.SetLogger(logger)
|
||||||
|
|
||||||
p, err := pool.NewPool(prm)
|
p, err := pool.NewPool(prm)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -343,47 +335,6 @@ func (p *placementPolicy) update(defaultPolicy string, regionPolicyFilepath stri
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func newAppMetrics(logger *zap.Logger, provider GateMetricsCollector, enabled bool) *appMetrics {
|
|
||||||
if !enabled {
|
|
||||||
logger.Warn("metrics are disabled")
|
|
||||||
}
|
|
||||||
return &appMetrics{
|
|
||||||
logger: logger,
|
|
||||||
provider: provider,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *appMetrics) SetEnabled(enabled bool) {
|
|
||||||
if !enabled {
|
|
||||||
m.logger.Warn("metrics are disabled")
|
|
||||||
}
|
|
||||||
|
|
||||||
m.mu.Lock()
|
|
||||||
m.enabled = enabled
|
|
||||||
m.mu.Unlock()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *appMetrics) SetHealth(status int32) {
|
|
||||||
m.mu.RLock()
|
|
||||||
if !m.enabled {
|
|
||||||
m.mu.RUnlock()
|
|
||||||
return
|
|
||||||
}
|
|
||||||
m.mu.RUnlock()
|
|
||||||
|
|
||||||
m.provider.SetHealth(status)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *appMetrics) Shutdown() {
|
|
||||||
m.mu.Lock()
|
|
||||||
if m.enabled {
|
|
||||||
m.provider.SetHealth(0)
|
|
||||||
m.enabled = false
|
|
||||||
}
|
|
||||||
m.provider.Unregister()
|
|
||||||
m.mu.Unlock()
|
|
||||||
}
|
|
||||||
|
|
||||||
func remove(list []string, element string) []string {
|
func remove(list []string, element string) []string {
|
||||||
for i, item := range list {
|
for i, item := range list {
|
||||||
if item == element {
|
if item == element {
|
||||||
|
@ -421,7 +372,7 @@ func (a *App) Serve(ctx context.Context) {
|
||||||
domains := a.cfg.GetStringSlice(cfgListenDomains)
|
domains := a.cfg.GetStringSlice(cfgListenDomains)
|
||||||
a.log.Info("fetch domains, prepare to use API", zap.Strings("domains", domains))
|
a.log.Info("fetch domains, prepare to use API", zap.Strings("domains", domains))
|
||||||
router := mux.NewRouter().SkipClean(true).UseEncodedPath()
|
router := mux.NewRouter().SkipClean(true).UseEncodedPath()
|
||||||
api.Attach(router, domains, a.maxClients, a.api, a.ctr, a.log)
|
api.Attach(router, domains, a.maxClients, a.api, a.ctr, a.log, a.metrics)
|
||||||
|
|
||||||
// Use mux.Router as http.Handler
|
// Use mux.Router as http.Handler
|
||||||
srv := new(http.Server)
|
srv := new(http.Server)
|
||||||
|
@ -471,11 +422,11 @@ func shutdownContext() (context.Context, context.CancelFunc) {
|
||||||
func (a *App) configReload() {
|
func (a *App) configReload() {
|
||||||
a.log.Info("SIGHUP config reload started")
|
a.log.Info("SIGHUP config reload started")
|
||||||
|
|
||||||
if !a.cfg.IsSet(cmdConfig) {
|
if !a.cfg.IsSet(cmdConfig) && !a.cfg.IsSet(cmdConfigDir) {
|
||||||
a.log.Warn("failed to reload config because it's missed")
|
a.log.Warn("failed to reload config because it's missed")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if err := readConfig(a.cfg); err != nil {
|
if err := readInConfig(a.cfg); err != nil {
|
||||||
a.log.Warn("failed to reload config", zap.Error(err))
|
a.log.Warn("failed to reload config", zap.Error(err))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
@ -509,6 +460,8 @@ func (a *App) updateSettings() {
|
||||||
if err := a.settings.policies.update(getDefaultPolicyValue(a.cfg), a.cfg.GetString(cfgPolicyRegionMapFile)); err != nil {
|
if err := a.settings.policies.update(getDefaultPolicyValue(a.cfg), a.cfg.GetString(cfgPolicyRegionMapFile)); err != nil {
|
||||||
a.log.Warn("policies won't be updated", zap.Error(err))
|
a.log.Warn("policies won't be updated", zap.Error(err))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
a.settings.xmlDecoder.UseDefaultNamespaceForCompleteMultipart(a.cfg.GetBool(cfgKludgeUseDefaultXMLNSForCompleteMultipartUpload))
|
||||||
}
|
}
|
||||||
|
|
||||||
func (a *App) startServices() {
|
func (a *App) startServices() {
|
||||||
|
@ -518,7 +471,7 @@ func (a *App) startServices() {
|
||||||
a.services = append(a.services, pprofService)
|
a.services = append(a.services, pprofService)
|
||||||
go pprofService.Start()
|
go pprofService.Start()
|
||||||
|
|
||||||
prometheusService := NewPrometheusService(a.cfg, a.log)
|
prometheusService := NewPrometheusService(a.cfg, a.log, a.metrics.Handler())
|
||||||
a.services = append(a.services, prometheusService)
|
a.services = append(a.services, prometheusService)
|
||||||
go prometheusService.Start()
|
go prometheusService.Start()
|
||||||
}
|
}
|
||||||
|
@ -526,37 +479,61 @@ func (a *App) startServices() {
|
||||||
func (a *App) initServers(ctx context.Context) {
|
func (a *App) initServers(ctx context.Context) {
|
||||||
serversInfo := fetchServers(a.cfg)
|
serversInfo := fetchServers(a.cfg)
|
||||||
|
|
||||||
a.servers = make([]Server, len(serversInfo))
|
a.servers = make([]Server, 0, len(serversInfo))
|
||||||
for i, serverInfo := range serversInfo {
|
for _, serverInfo := range serversInfo {
|
||||||
a.log.Info("added server",
|
fields := []zap.Field{
|
||||||
zap.String("address", serverInfo.Address), zap.Bool("tls enabled", serverInfo.TLS.Enabled),
|
zap.String("address", serverInfo.Address), zap.Bool("tls enabled", serverInfo.TLS.Enabled),
|
||||||
zap.String("tls cert", serverInfo.TLS.CertFile), zap.String("tls key", serverInfo.TLS.KeyFile))
|
zap.String("tls cert", serverInfo.TLS.CertFile), zap.String("tls key", serverInfo.TLS.KeyFile),
|
||||||
a.servers[i] = newServer(ctx, serverInfo, a.log)
|
}
|
||||||
|
srv, err := newServer(ctx, serverInfo)
|
||||||
|
if err != nil {
|
||||||
|
a.log.Warn("failed to add server", append(fields, zap.Error(err))...)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
a.servers = append(a.servers, srv)
|
||||||
|
a.log.Info("add server", fields...)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(a.servers) == 0 {
|
||||||
|
a.log.Fatal("no healthy servers")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (a *App) updateServers() error {
|
func (a *App) updateServers() error {
|
||||||
serversInfo := fetchServers(a.cfg)
|
serversInfo := fetchServers(a.cfg)
|
||||||
|
|
||||||
if len(serversInfo) != len(a.servers) {
|
var found bool
|
||||||
return fmt.Errorf("invalid servers configuration: amount mismatch: old '%d', new '%d", len(a.servers), len(serversInfo))
|
for _, serverInfo := range serversInfo {
|
||||||
}
|
index := a.serverIndex(serverInfo.Address)
|
||||||
|
if index == -1 {
|
||||||
for i, serverInfo := range serversInfo {
|
continue
|
||||||
if serverInfo.Address != a.servers[i].Address() {
|
|
||||||
return fmt.Errorf("invalid servers configuration: addresses mismatch: old '%s', new '%s", a.servers[i].Address(), serverInfo.Address)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if serverInfo.TLS.Enabled {
|
if serverInfo.TLS.Enabled {
|
||||||
if err := a.servers[i].UpdateCert(serverInfo.TLS.CertFile, serverInfo.TLS.KeyFile); err != nil {
|
if err := a.servers[index].UpdateCert(serverInfo.TLS.CertFile, serverInfo.TLS.KeyFile); err != nil {
|
||||||
return fmt.Errorf("failed to update tls certs: %w", err)
|
return fmt.Errorf("failed to update tls certs: %w", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
found = true
|
||||||
|
}
|
||||||
|
|
||||||
|
if !found {
|
||||||
|
return fmt.Errorf("invalid servers configuration: no known server found")
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (a *App) serverIndex(address string) int {
|
||||||
|
for i := range a.servers {
|
||||||
|
if a.servers[i].Address() == address {
|
||||||
|
return i
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return -1
|
||||||
|
}
|
||||||
|
|
||||||
func (a *App) stopServices() {
|
func (a *App) stopServices() {
|
||||||
ctx, cancel := shutdownContext()
|
ctx, cancel := shutdownContext()
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
@ -653,6 +630,7 @@ func (a *App) initHandler() {
|
||||||
DefaultMaxAge: handler.DefaultMaxAge,
|
DefaultMaxAge: handler.DefaultMaxAge,
|
||||||
NotificatorEnabled: a.cfg.GetBool(cfgEnableNATS),
|
NotificatorEnabled: a.cfg.GetBool(cfgEnableNATS),
|
||||||
CopiesNumber: handler.DefaultCopiesNumber,
|
CopiesNumber: handler.DefaultCopiesNumber,
|
||||||
|
XMLDecoder: a.settings.xmlDecoder,
|
||||||
}
|
}
|
||||||
|
|
||||||
if a.cfg.IsSet(cfgDefaultMaxAge) {
|
if a.cfg.IsSet(cfgDefaultMaxAge) {
|
||||||
|
@ -670,6 +648,14 @@ func (a *App) initHandler() {
|
||||||
cfg.CopiesNumber = val
|
cfg.CopiesNumber = val
|
||||||
}
|
}
|
||||||
|
|
||||||
|
cfg.ResolveZoneList = a.cfg.GetStringSlice(cfgResolveBucketAllow)
|
||||||
|
cfg.IsResolveListAllow = len(cfg.ResolveZoneList) > 0
|
||||||
|
if !cfg.IsResolveListAllow {
|
||||||
|
cfg.ResolveZoneList = a.cfg.GetStringSlice(cfgResolveBucketDeny)
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg.CompleteMultipartKeepalive = a.cfg.GetDuration(cfgKludgeCompleteMultipartUploadKeepalive)
|
||||||
|
|
||||||
var err error
|
var err error
|
||||||
a.api, err = handler.New(a.log, a.obj, a.nc, cfg)
|
a.api, err = handler.New(a.log, a.obj, a.nc, cfg)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
|
@ -3,227 +3,12 @@ package main
|
||||||
import (
|
import (
|
||||||
"net/http"
|
"net/http"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/pool"
|
|
||||||
"github.com/prometheus/client_golang/prometheus"
|
|
||||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
|
||||||
"github.com/spf13/viper"
|
"github.com/spf13/viper"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
|
||||||
namespace = "frostfs_s3_gw"
|
|
||||||
stateSubsystem = "state"
|
|
||||||
poolSubsystem = "pool"
|
|
||||||
|
|
||||||
methodGetBalance = "get_balance"
|
|
||||||
methodPutContainer = "put_container"
|
|
||||||
methodGetContainer = "get_container"
|
|
||||||
methodListContainer = "list_container"
|
|
||||||
methodDeleteContainer = "delete_container"
|
|
||||||
methodGetContainerEacl = "get_container_eacl"
|
|
||||||
methodSetContainerEacl = "set_container_eacl"
|
|
||||||
methodEndpointInfo = "endpoint_info"
|
|
||||||
methodNetworkInfo = "network_info"
|
|
||||||
methodPutObject = "put_object"
|
|
||||||
methodDeleteObject = "delete_object"
|
|
||||||
methodGetObject = "get_object"
|
|
||||||
methodHeadObject = "head_object"
|
|
||||||
methodRangeObject = "range_object"
|
|
||||||
methodCreateSession = "create_session"
|
|
||||||
)
|
|
||||||
|
|
||||||
type StatisticScraper interface {
|
|
||||||
Statistic() pool.Statistic
|
|
||||||
}
|
|
||||||
|
|
||||||
type GateMetrics struct {
|
|
||||||
stateMetrics
|
|
||||||
poolMetricsCollector
|
|
||||||
}
|
|
||||||
|
|
||||||
type stateMetrics struct {
|
|
||||||
healthCheck prometheus.Gauge
|
|
||||||
}
|
|
||||||
|
|
||||||
type poolMetricsCollector struct {
|
|
||||||
poolStatScraper StatisticScraper
|
|
||||||
overallErrors prometheus.Gauge
|
|
||||||
overallNodeErrors *prometheus.GaugeVec
|
|
||||||
overallNodeRequests *prometheus.GaugeVec
|
|
||||||
currentErrors *prometheus.GaugeVec
|
|
||||||
requestDuration *prometheus.GaugeVec
|
|
||||||
}
|
|
||||||
|
|
||||||
func newGateMetrics(scraper StatisticScraper) *GateMetrics {
|
|
||||||
stateMetric := newStateMetrics()
|
|
||||||
stateMetric.register()
|
|
||||||
|
|
||||||
poolMetric := newPoolMetricsCollector(scraper)
|
|
||||||
poolMetric.register()
|
|
||||||
|
|
||||||
return &GateMetrics{
|
|
||||||
stateMetrics: *stateMetric,
|
|
||||||
poolMetricsCollector: *poolMetric,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (g *GateMetrics) Unregister() {
|
|
||||||
g.stateMetrics.unregister()
|
|
||||||
prometheus.Unregister(&g.poolMetricsCollector)
|
|
||||||
}
|
|
||||||
|
|
||||||
func newStateMetrics() *stateMetrics {
|
|
||||||
return &stateMetrics{
|
|
||||||
healthCheck: prometheus.NewGauge(prometheus.GaugeOpts{
|
|
||||||
Namespace: namespace,
|
|
||||||
Subsystem: stateSubsystem,
|
|
||||||
Name: "health",
|
|
||||||
Help: "Current S3 gateway state",
|
|
||||||
}),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m stateMetrics) register() {
|
|
||||||
prometheus.MustRegister(m.healthCheck)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m stateMetrics) unregister() {
|
|
||||||
prometheus.Unregister(m.healthCheck)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m stateMetrics) SetHealth(s int32) {
|
|
||||||
m.healthCheck.Set(float64(s))
|
|
||||||
}
|
|
||||||
|
|
||||||
func newPoolMetricsCollector(scraper StatisticScraper) *poolMetricsCollector {
|
|
||||||
overallErrors := prometheus.NewGauge(
|
|
||||||
prometheus.GaugeOpts{
|
|
||||||
Namespace: namespace,
|
|
||||||
Subsystem: poolSubsystem,
|
|
||||||
Name: "overall_errors",
|
|
||||||
Help: "Total number of errors in pool",
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
overallNodeErrors := prometheus.NewGaugeVec(
|
|
||||||
prometheus.GaugeOpts{
|
|
||||||
Namespace: namespace,
|
|
||||||
Subsystem: poolSubsystem,
|
|
||||||
Name: "overall_node_errors",
|
|
||||||
Help: "Total number of errors for connection in pool",
|
|
||||||
},
|
|
||||||
[]string{
|
|
||||||
"node",
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
overallNodeRequests := prometheus.NewGaugeVec(
|
|
||||||
prometheus.GaugeOpts{
|
|
||||||
Namespace: namespace,
|
|
||||||
Subsystem: poolSubsystem,
|
|
||||||
Name: "overall_node_requests",
|
|
||||||
Help: "Total number of requests to specific node in pool",
|
|
||||||
},
|
|
||||||
[]string{
|
|
||||||
"node",
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
currentErrors := prometheus.NewGaugeVec(
|
|
||||||
prometheus.GaugeOpts{
|
|
||||||
Namespace: namespace,
|
|
||||||
Subsystem: poolSubsystem,
|
|
||||||
Name: "current_errors",
|
|
||||||
Help: "Number of errors on current connections that will be reset after the threshold",
|
|
||||||
},
|
|
||||||
[]string{
|
|
||||||
"node",
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
requestsDuration := prometheus.NewGaugeVec(
|
|
||||||
prometheus.GaugeOpts{
|
|
||||||
Namespace: namespace,
|
|
||||||
Subsystem: poolSubsystem,
|
|
||||||
Name: "avg_request_duration",
|
|
||||||
Help: "Average request duration (in milliseconds) for specific method on node in pool",
|
|
||||||
},
|
|
||||||
[]string{
|
|
||||||
"node",
|
|
||||||
"method",
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
return &poolMetricsCollector{
|
|
||||||
poolStatScraper: scraper,
|
|
||||||
overallErrors: overallErrors,
|
|
||||||
overallNodeErrors: overallNodeErrors,
|
|
||||||
overallNodeRequests: overallNodeRequests,
|
|
||||||
currentErrors: currentErrors,
|
|
||||||
requestDuration: requestsDuration,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *poolMetricsCollector) Collect(ch chan<- prometheus.Metric) {
|
|
||||||
m.updateStatistic()
|
|
||||||
m.overallErrors.Collect(ch)
|
|
||||||
m.overallNodeErrors.Collect(ch)
|
|
||||||
m.overallNodeRequests.Collect(ch)
|
|
||||||
m.currentErrors.Collect(ch)
|
|
||||||
m.requestDuration.Collect(ch)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *poolMetricsCollector) Describe(descs chan<- *prometheus.Desc) {
|
|
||||||
m.overallErrors.Describe(descs)
|
|
||||||
m.overallNodeErrors.Describe(descs)
|
|
||||||
m.overallNodeRequests.Describe(descs)
|
|
||||||
m.currentErrors.Describe(descs)
|
|
||||||
m.requestDuration.Describe(descs)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *poolMetricsCollector) register() {
|
|
||||||
prometheus.MustRegister(m)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *poolMetricsCollector) updateStatistic() {
|
|
||||||
stat := m.poolStatScraper.Statistic()
|
|
||||||
|
|
||||||
m.overallNodeErrors.Reset()
|
|
||||||
m.overallNodeRequests.Reset()
|
|
||||||
m.currentErrors.Reset()
|
|
||||||
m.requestDuration.Reset()
|
|
||||||
|
|
||||||
for _, node := range stat.Nodes() {
|
|
||||||
m.overallNodeErrors.WithLabelValues(node.Address()).Set(float64(node.OverallErrors()))
|
|
||||||
m.overallNodeRequests.WithLabelValues(node.Address()).Set(float64(node.Requests()))
|
|
||||||
|
|
||||||
m.currentErrors.WithLabelValues(node.Address()).Set(float64(node.CurrentErrors()))
|
|
||||||
m.updateRequestsDuration(node)
|
|
||||||
}
|
|
||||||
|
|
||||||
m.overallErrors.Set(float64(stat.OverallErrors()))
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *poolMetricsCollector) updateRequestsDuration(node pool.NodeStatistic) {
|
|
||||||
m.requestDuration.WithLabelValues(node.Address(), methodGetBalance).Set(float64(node.AverageGetBalance().Milliseconds()))
|
|
||||||
m.requestDuration.WithLabelValues(node.Address(), methodPutContainer).Set(float64(node.AveragePutContainer().Milliseconds()))
|
|
||||||
m.requestDuration.WithLabelValues(node.Address(), methodGetContainer).Set(float64(node.AverageGetContainer().Milliseconds()))
|
|
||||||
m.requestDuration.WithLabelValues(node.Address(), methodListContainer).Set(float64(node.AverageListContainer().Milliseconds()))
|
|
||||||
m.requestDuration.WithLabelValues(node.Address(), methodDeleteContainer).Set(float64(node.AverageDeleteContainer().Milliseconds()))
|
|
||||||
m.requestDuration.WithLabelValues(node.Address(), methodGetContainerEacl).Set(float64(node.AverageGetContainerEACL().Milliseconds()))
|
|
||||||
m.requestDuration.WithLabelValues(node.Address(), methodSetContainerEacl).Set(float64(node.AverageSetContainerEACL().Milliseconds()))
|
|
||||||
m.requestDuration.WithLabelValues(node.Address(), methodEndpointInfo).Set(float64(node.AverageEndpointInfo().Milliseconds()))
|
|
||||||
m.requestDuration.WithLabelValues(node.Address(), methodNetworkInfo).Set(float64(node.AverageNetworkInfo().Milliseconds()))
|
|
||||||
m.requestDuration.WithLabelValues(node.Address(), methodPutObject).Set(float64(node.AveragePutObject().Milliseconds()))
|
|
||||||
m.requestDuration.WithLabelValues(node.Address(), methodDeleteObject).Set(float64(node.AverageDeleteObject().Milliseconds()))
|
|
||||||
m.requestDuration.WithLabelValues(node.Address(), methodGetObject).Set(float64(node.AverageGetObject().Milliseconds()))
|
|
||||||
m.requestDuration.WithLabelValues(node.Address(), methodHeadObject).Set(float64(node.AverageHeadObject().Milliseconds()))
|
|
||||||
m.requestDuration.WithLabelValues(node.Address(), methodRangeObject).Set(float64(node.AverageRangeObject().Milliseconds()))
|
|
||||||
m.requestDuration.WithLabelValues(node.Address(), methodCreateSession).Set(float64(node.AverageCreateSession().Milliseconds()))
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewPrometheusService creates a new service for gathering prometheus metrics.
|
// NewPrometheusService creates a new service for gathering prometheus metrics.
|
||||||
func NewPrometheusService(v *viper.Viper, log *zap.Logger) *Service {
|
func NewPrometheusService(v *viper.Viper, log *zap.Logger, handler http.Handler) *Service {
|
||||||
if log == nil {
|
if log == nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
@ -231,7 +16,7 @@ func NewPrometheusService(v *viper.Viper, log *zap.Logger) *Service {
|
||||||
return &Service{
|
return &Service{
|
||||||
Server: &http.Server{
|
Server: &http.Server{
|
||||||
Addr: v.GetString(cfgPrometheusAddress),
|
Addr: v.GetString(cfgPrometheusAddress),
|
||||||
Handler: promhttp.Handler(),
|
Handler: handler,
|
||||||
},
|
},
|
||||||
enabled: v.GetBool(cfgPrometheusEnabled),
|
enabled: v.GetBool(cfgPrometheusEnabled),
|
||||||
serviceType: "Prometheus",
|
serviceType: "Prometheus",
|
||||||
|
|
|
@ -3,15 +3,16 @@ package main
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
|
"path"
|
||||||
"runtime"
|
"runtime"
|
||||||
"sort"
|
"sort"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/api/resolver"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/resolver"
|
||||||
"github.com/TrueCloudLab/frostfs-s3-gw/internal/version"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/version"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/pool"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
|
||||||
"github.com/spf13/pflag"
|
"github.com/spf13/pflag"
|
||||||
"github.com/spf13/viper"
|
"github.com/spf13/viper"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
|
@ -112,12 +113,17 @@ const ( // Settings.
|
||||||
// Application.
|
// Application.
|
||||||
cfgApplicationBuildTime = "app.build_time"
|
cfgApplicationBuildTime = "app.build_time"
|
||||||
|
|
||||||
|
// Kludge.
|
||||||
|
cfgKludgeUseDefaultXMLNSForCompleteMultipartUpload = "kludge.use_default_xmlns_for_complete_multipart"
|
||||||
|
cfgKludgeCompleteMultipartUploadKeepalive = "kludge.complete_multipart_keepalive"
|
||||||
|
|
||||||
// Command line args.
|
// Command line args.
|
||||||
cmdHelp = "help"
|
cmdHelp = "help"
|
||||||
cmdVersion = "version"
|
cmdVersion = "version"
|
||||||
cmdConfig = "config"
|
cmdConfig = "config"
|
||||||
cmdPProf = "pprof"
|
cmdConfigDir = "config-dir"
|
||||||
cmdMetrics = "metrics"
|
cmdPProf = "pprof"
|
||||||
|
cmdMetrics = "metrics"
|
||||||
|
|
||||||
cmdListenAddress = "listen_address"
|
cmdListenAddress = "listen_address"
|
||||||
|
|
||||||
|
@ -128,6 +134,10 @@ const ( // Settings.
|
||||||
// List of allowed AccessKeyID prefixes.
|
// List of allowed AccessKeyID prefixes.
|
||||||
cfgAllowedAccessKeyIDPrefixes = "allowed_access_key_id_prefixes"
|
cfgAllowedAccessKeyIDPrefixes = "allowed_access_key_id_prefixes"
|
||||||
|
|
||||||
|
// Bucket resolving options.
|
||||||
|
cfgResolveBucketAllow = "resolve_bucket.allow"
|
||||||
|
cfgResolveBucketDeny = "resolve_bucket.deny"
|
||||||
|
|
||||||
// envPrefix is an environment variables prefix used for configuration.
|
// envPrefix is an environment variables prefix used for configuration.
|
||||||
envPrefix = "S3_GW"
|
envPrefix = "S3_GW"
|
||||||
)
|
)
|
||||||
|
@ -214,7 +224,8 @@ func newSettings() *viper.Viper {
|
||||||
|
|
||||||
flags.StringP(cmdWallet, "w", "", `path to the wallet`)
|
flags.StringP(cmdWallet, "w", "", `path to the wallet`)
|
||||||
flags.String(cmdAddress, "", `address of wallet account`)
|
flags.String(cmdAddress, "", `address of wallet account`)
|
||||||
flags.String(cmdConfig, "", "config path")
|
flags.StringArray(cmdConfig, nil, "config paths")
|
||||||
|
flags.String(cmdConfigDir, "", "config dir path")
|
||||||
|
|
||||||
flags.Duration(cfgHealthcheckTimeout, defaultHealthcheckTimeout, "set timeout to check node health during rebalance")
|
flags.Duration(cfgHealthcheckTimeout, defaultHealthcheckTimeout, "set timeout to check node health during rebalance")
|
||||||
flags.Duration(cfgConnectTimeout, defaultConnectTimeout, "set timeout to connect to FrostFS nodes")
|
flags.Duration(cfgConnectTimeout, defaultConnectTimeout, "set timeout to connect to FrostFS nodes")
|
||||||
|
@ -246,6 +257,10 @@ func newSettings() *viper.Viper {
|
||||||
v.SetDefault(cfgPProfAddress, "localhost:8085")
|
v.SetDefault(cfgPProfAddress, "localhost:8085")
|
||||||
v.SetDefault(cfgPrometheusAddress, "localhost:8086")
|
v.SetDefault(cfgPrometheusAddress, "localhost:8086")
|
||||||
|
|
||||||
|
// kludge
|
||||||
|
v.SetDefault(cfgKludgeUseDefaultXMLNSForCompleteMultipartUpload, false)
|
||||||
|
v.SetDefault(cfgKludgeCompleteMultipartUploadKeepalive, 10*time.Second)
|
||||||
|
|
||||||
// Bind flags
|
// Bind flags
|
||||||
if err := bindFlags(v, flags); err != nil {
|
if err := bindFlags(v, flags); err != nil {
|
||||||
panic(fmt.Errorf("bind flags: %w", err))
|
panic(fmt.Errorf("bind flags: %w", err))
|
||||||
|
@ -313,10 +328,8 @@ func newSettings() *viper.Viper {
|
||||||
os.Exit(0)
|
os.Exit(0)
|
||||||
}
|
}
|
||||||
|
|
||||||
if v.IsSet(cmdConfig) {
|
if err := readInConfig(v); err != nil {
|
||||||
if err := readConfig(v); err != nil {
|
panic(err)
|
||||||
panic(err)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return v
|
return v
|
||||||
|
@ -332,6 +345,9 @@ func bindFlags(v *viper.Viper, flags *pflag.FlagSet) error {
|
||||||
if err := v.BindPFlag(cmdConfig, flags.Lookup(cmdConfig)); err != nil {
|
if err := v.BindPFlag(cmdConfig, flags.Lookup(cmdConfig)); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
if err := v.BindPFlag(cmdConfigDir, flags.Lookup(cmdConfigDir)); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
if err := v.BindPFlag(cfgWalletPath, flags.Lookup(cmdWallet)); err != nil {
|
if err := v.BindPFlag(cfgWalletPath, flags.Lookup(cmdWallet)); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
@ -370,17 +386,72 @@ func bindFlags(v *viper.Viper, flags *pflag.FlagSet) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func readConfig(v *viper.Viper) error {
|
func readInConfig(v *viper.Viper) error {
|
||||||
cfgFileName := v.GetString(cmdConfig)
|
if v.IsSet(cmdConfig) {
|
||||||
cfgFile, err := os.Open(cfgFileName)
|
if err := readConfig(v); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if v.IsSet(cmdConfigDir) {
|
||||||
|
if err := readConfigDir(v); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func readConfigDir(v *viper.Viper) error {
|
||||||
|
cfgSubConfigDir := v.GetString(cmdConfigDir)
|
||||||
|
entries, err := os.ReadDir(cfgSubConfigDir)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if err = v.ReadConfig(cfgFile); err != nil {
|
|
||||||
|
for _, entry := range entries {
|
||||||
|
if entry.IsDir() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
ext := path.Ext(entry.Name())
|
||||||
|
if ext != ".yaml" && ext != ".yml" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if err = mergeConfig(v, path.Join(cfgSubConfigDir, entry.Name())); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func readConfig(v *viper.Viper) error {
|
||||||
|
for _, fileName := range v.GetStringSlice(cmdConfig) {
|
||||||
|
if err := mergeConfig(v, fileName); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func mergeConfig(v *viper.Viper, fileName string) error {
|
||||||
|
cfgFile, err := os.Open(fileName)
|
||||||
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
return cfgFile.Close()
|
defer func() {
|
||||||
|
if errClose := cfgFile.Close(); errClose != nil {
|
||||||
|
panic(errClose)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
if err = v.MergeConfig(cfgFile); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// newLogger constructs a Logger instance for the current application.
|
// newLogger constructs a Logger instance for the current application.
|
||||||
|
|
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue